diff --git a/.gitignore b/.gitignore
index 9d7f06a3a8648cd33f8496a7c0921eb6811c5f04..22c1c2a58466d13424047fcba0f493355df6bcf4 100644
--- a/.gitignore
+++ b/.gitignore
@@ -10,6 +10,7 @@ env_configs
.vscode/
.env
.idea/
+builds/
config/
run_EnvSuitPipeline.py
custom_plotting.py
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 3e792092277c9aa7ba0bf09643f4af6ffee68bb0..29c01b3a12b20c0173df6584abc40138d2c3beb4 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -59,10 +59,10 @@ test:
- python -m coverage html -d $CI_PROJECT_DIR/coverage
- cd $CI_PROJECT_DIR/tests/integration
- python3 -m coverage run -m unittest test_deposition.py
- - python3 -m unittest test_deposition.py
- - python3 -m unittest test_env_suit.py
- - python3 -m unittest test_survey.py
- - python3 -m unittest test_advisory.py
+ - python3 -m coverage run -m unittest test_deposition.py
+ - python3 -m coverage run -m unittest test_env_suit.py
+ - python3 -m coverage run -m unittest test_survey.py
+ - python3 -m coverage run -m unittest test_advisory.py
- python -m coverage report
- python -m coverage html -d $CI_PROJECT_DIR/coverage
- ls
diff --git a/EWSMemoryHandler.py b/EWSMemoryHandler.py
new file mode 100644
index 0000000000000000000000000000000000000000..cd305e27ee78e321d4ce2ee807150e2bc110fa05
--- /dev/null
+++ b/EWSMemoryHandler.py
@@ -0,0 +1,33 @@
+from logging import LogRecord, Handler, ERROR
+from logging.handlers import MemoryHandler
+from typing import Optional
+
+
+class EWSMemoryHandler(MemoryHandler):
+
+ def __init__(self, capacity: int, flushLevel: int = ..., target: Optional[Handler] = ..., flushOnClose: bool = ...) -> None:
+ super().__init__(capacity, flushLevel, target, flushOnClose)
+ self.target = target
+
+ def flush(self) -> None:
+ self.acquire()
+ try:
+ if self.target:
+ merged_messages = ""
+ for record in self.buffer:
+ # self.target.handle(record)
+ # print(record.message)
+ merged_messages += record.message + "\n"
+
+ if merged_messages is not "":
+ print(f"BEGIN:{merged_messages}:END")
+ merged_record = LogRecord("merged_record", ERROR, "pathname", 0, merged_messages, None, )
+ self.target.handle(merged_record)
+
+ self.buffer = []
+ finally:
+ self.release()
+
+
+ def shouldFlush(self, record: LogRecord) -> bool:
+ return False
diff --git a/configs/config_SouthAsia_fc_lbdev.json b/configs/config_SouthAsia_fc_lbdev.json
new file mode 100644
index 0000000000000000000000000000000000000000..aefb28419d6f132110f3ac0eaad3d85829b64e55
--- /dev/null
+++ b/configs/config_SouthAsia_fc_lbdev.json
@@ -0,0 +1,18 @@
+{
+ "RegionName" : "SouthAsia",
+ "SubRegionNames" : ["SouthAsia"],
+ "StartTime" : "?",
+ "StartString" : "?",
+ "WorkspacePathout" : "/media/scratch/lb584_scratch/projects/ews/ews_coordinator/",
+ "WorkspacePath" : "not_set",
+ "ServerPath" : "not_set",
+ "ServerName" : "no_upload",
+ "ServerKey" : "not_set",
+ "Survey" : {
+ "ProcessPreJob" : "process_pre_job_survey",
+ "ProcessInJob" : "do_nothing",
+ "ProcessEWSPlotting" : "do_nothing",
+ "ServerCredentialsFile" : "do_nothing",
+ "ServerPathExtra" : "do_nothing"
+ }
+}
\ No newline at end of file
diff --git a/configs/docker/build/conda-env-py3EWS-withbuilds.yml b/configs/docker/build/conda-env-py3EWS-withbuilds.yml
new file mode 100644
index 0000000000000000000000000000000000000000..91e247f14dab77470c02844f0fb6891b58748a7e
--- /dev/null
+++ b/configs/docker/build/conda-env-py3EWS-withbuilds.yml
@@ -0,0 +1,185 @@
+name: /storage/app/EWS/envs/conda/py3EWS
+channels:
+ - conda-forge
+ - defaults
+dependencies:
+ - _libgcc_mutex=0.1=main
+ - _openmp_mutex=4.5=1_gnu
+ - affine=2.3.1=pyhd8ed1ab_0
+ - antlr-python-runtime=4.7.2=py39hf3d152e_1003
+ - attrs=21.4.0=pyhd8ed1ab_0
+ - boost-cpp=1.74.0=h312852a_4
+ - branca=0.5.0=pyhd8ed1ab_0
+ - brotlipy=0.7.0=py39h27cfd23_1003
+ - bzip2=1.0.8=h7f98852_4
+ - c-ares=1.17.1=h7f98852_1
+ - ca-certificates=2022.5.18=ha878542_0
+ - cairo=1.16.0=h6cf1ce9_1008
+ - cartopy=0.20.0=py39h2103e0b_0
+ - certifi=2022.5.18=py39hf3d152e_0
+ - cf-units=3.0.1=py39hce5d2b2_2
+ - cffi=1.15.0=py39hd667e15_1
+ - cfitsio=3.470=hb418390_7
+ - cftime=1.6.0=py39hd257fcd_1
+ - charset-normalizer=2.0.4=pyhd3eb1b0_0
+ - click=7.1.2=pyh9f0ad1d_0
+ - click-plugins=1.1.1=py_0
+ - cligj=0.7.2=pyhd8ed1ab_1
+ - cloudpickle=2.0.0=pyhd8ed1ab_0
+ - conda=4.12.0=py39hf3d152e_0
+ - conda-content-trust=0.1.1=pyhd3eb1b0_0
+ - conda-package-handling=1.7.3=py39h27cfd23_1
+ - contextily=1.0.1=py_0
+ - cryptography=36.0.0=py39h9ce1e76_0
+ - curl=7.78.0=hea6ffbf_0
+ - cycler=0.11.0=pyhd8ed1ab_0
+ - dask-core=2022.5.0=pyhd8ed1ab_0
+ - expat=2.4.1=h9c3ff4c_0
+ - fiona=1.8.20=py39h427c1bf_1
+ - folium=0.12.1.post1=pyhd8ed1ab_1
+ - fontconfig=2.13.1=hba837de_1005
+ - freetype=2.10.4=h0708190_1
+ - freexl=1.0.6=h7f98852_0
+ - fsspec=2022.3.0=pyhd8ed1ab_0
+ - gdal=3.3.1=py39h409cc32_1
+ - geographiclib=1.52=pyhd8ed1ab_0
+ - geopandas=0.10.2=pyhd8ed1ab_1
+ - geopandas-base=0.10.2=pyha770c72_1
+ - geopy=2.2.0=pyhd8ed1ab_0
+ - geos=3.9.1=h9c3ff4c_2
+ - geotiff=1.6.0=h4f31c25_6
+ - gettext=0.19.8.1=h0b5b191_1005
+ - giflib=5.2.1=h36c2ea0_2
+ - hdf4=4.2.15=h10796ff_3
+ - hdf5=1.10.6=nompi_h6a2412b_1114
+ - icu=68.1=h58526e2_0
+ - idna=3.3=pyhd3eb1b0_0
+ - iris=3.2.1=pyhd8ed1ab_0
+ - jbig=2.1=h7f98852_2003
+ - jinja2=3.1.2=pyhd8ed1ab_0
+ - joblib=1.1.0=pyhd8ed1ab_0
+ - jpeg=9d=h36c2ea0_0
+ - json-c=0.15=h98cffda_0
+ - kealib=1.4.14=hcc255d8_2
+ - kiwisolver=1.3.1=py39h1a9c180_1
+ - krb5=1.19.2=hcc1bbae_0
+ - lcms2=2.12=hddcbb42_0
+ - ld_impl_linux-64=2.35.1=h7274673_9
+ - lerc=2.2.1=h9c3ff4c_0
+ - libblas=3.9.0=11_linux64_openblas
+ - libcblas=3.9.0=11_linux64_openblas
+ - libcurl=7.78.0=h2574ce0_0
+ - libdap4=3.20.6=hd7c4107_2
+ - libdeflate=1.7=h7f98852_5
+ - libedit=3.1.20191231=he28a2e2_2
+ - libev=4.33=h516909a_1
+ - libffi=3.3=he6710b0_2
+ - libgcc-ng=11.2.0=h1234567_0
+ - libgdal=3.3.1=h8f005ca_1
+ - libgfortran-ng=12.1.0=h69a702a_16
+ - libgfortran5=12.1.0=hdcd56e2_16
+ - libglib=2.68.3=h3e27bee_0
+ - libgomp=11.2.0=h1234567_0
+ - libiconv=1.16=h516909a_0
+ - libkml=1.3.0=h238a007_1014
+ - liblapack=3.9.0=11_linux64_openblas
+ - libnetcdf=4.8.0=nompi_hcd642e3_103
+ - libnghttp2=1.43.0=h812cca2_0
+ - libopenblas=0.3.17=pthreads_h8fe5266_1
+ - libpng=1.6.37=h21135ba_2
+ - libpq=13.3=hd57d9b9_0
+ - librttopo=1.1.0=h1185371_6
+ - libspatialindex=1.9.3=h9c3ff4c_4
+ - libspatialite=5.0.1=h8694cbe_5
+ - libssh2=1.9.0=ha56f1ee_6
+ - libstdcxx-ng=12.1.0=ha89aaad_16
+ - libtiff=4.3.0=hf544144_1
+ - libuuid=2.32.1=h7f98852_1000
+ - libwebp-base=1.2.0=h7f98852_2
+ - libxcb=1.13=h7f98852_1003
+ - libxml2=2.9.12=h72842e0_0
+ - libxslt=1.1.33=h15afd5d_2
+ - libzip=1.8.0=h4de3113_0
+ - locket=1.0.0=pyhd8ed1ab_0
+ - lxml=4.6.3=py39h107f48f_0
+ - lz4-c=1.9.3=h9c3ff4c_1
+ - mapclassify=2.4.3=pyhd8ed1ab_0
+ - markupsafe=2.0.1=py39h3811e60_0
+ - matplotlib-base=3.4.2=py39h2fa2bec_0
+ - mercantile=1.2.1=pyhd8ed1ab_0
+ - munch=2.5.0=py_0
+ - ncurses=6.3=h7f8727e_2
+ - netcdf4=1.5.7=nompi_py39hc6dca20_100
+ - networkx=2.8.1=pyhd8ed1ab_0
+ - numpy=1.21.1=py39hdbf815f_0
+ - olefile=0.46=pyh9f0ad1d_1
+ - openjpeg=2.4.0=hb52868f_1
+ - openssl=1.1.1o=h166bdaf_0
+ - packaging=21.3=pyhd8ed1ab_0
+ - pandas=1.3.1=py39hde0f152_0
+ - partd=1.2.0=pyhd8ed1ab_0
+ - pcre=8.45=h9c3ff4c_0
+ - pillow=8.3.1=py39ha612740_0
+ - pip=21.2.4=py39h06a4308_0
+ - pixman=0.40.0=h36c2ea0_0
+ - poppler=21.03.0=h93df280_0
+ - poppler-data=0.4.11=hd8ed1ab_0
+ - postgresql=13.3=h2510834_0
+ - proj=8.0.1=h277dcde_0
+ - pthread-stubs=0.4=h36c2ea0_1001
+ - pycosat=0.6.3=py39h27cfd23_0
+ - pycparser=2.21=pyhd3eb1b0_0
+ - pyopenssl=21.0.0=pyhd3eb1b0_1
+ - pyparsing=3.0.9=pyhd8ed1ab_0
+ - pyproj=3.1.0=py39ha9a7ae0_3
+ - pyshp=2.3.0=pyhd8ed1ab_0
+ - pysocks=1.7.1=py39h06a4308_0
+ - python=3.9.7=h12debd9_1
+ - python-dateutil=2.8.2=pyhd8ed1ab_0
+ - python-docx=0.8.11=pyhd8ed1ab_0
+ - python-xxhash=3.0.0=py39hb9d737c_1
+ - python_abi=3.9=2_cp39
+ - pytz=2022.1=pyhd8ed1ab_0
+ - pyyaml=6.0=py39hb9d737c_4
+ - rasterio=1.2.6=py39hbc4e497_2
+ - readline=8.1.2=h7f8727e_1
+ - requests=2.27.1=pyhd3eb1b0_0
+ - rtree=1.0.0=py39hb102c33_1
+ - ruamel_yaml=0.15.100=py39h27cfd23_0
+ - scikit-learn=0.24.2=py39h4dfa638_0
+ - scipy=1.7.0=py39hee8e79c_1
+ - setuptools=58.0.4=py39h06a4308_0
+ - shapely=1.7.1=py39ha61afbd_5
+ - six=1.16.0=pyhd3eb1b0_0
+ - snuggs=1.4.7=py_0
+ - sqlite=3.37.0=hc218d9a_0
+ - threadpoolctl=3.1.0=pyh8a188c0_0
+ - tiledb=2.3.2=he87e0bf_0
+ - tk=8.6.11=h1ccaba5_0
+ - toolz=0.11.2=pyhd8ed1ab_0
+ - tornado=6.1=py39h3811e60_1
+ - tqdm=4.62.3=pyhd3eb1b0_1
+ - tzcode=2021a=h7f98852_2
+ - tzdata=2021e=hda174b7_0
+ - udunits2=2.2.27.27=hc3e0081_3
+ - urllib3=1.26.7=pyhd3eb1b0_0
+ - wheel=0.37.1=pyhd3eb1b0_0
+ - xerces-c=3.2.3=h9d8b166_2
+ - xorg-kbproto=1.0.7=h7f98852_1002
+ - xorg-libice=1.0.10=h7f98852_0
+ - xorg-libsm=1.2.3=hd9c2040_1000
+ - xorg-libx11=1.7.2=h7f98852_0
+ - xorg-libxau=1.0.9=h7f98852_0
+ - xorg-libxdmcp=1.1.3=h7f98852_0
+ - xorg-libxext=1.3.4=h7f98852_1
+ - xorg-libxrender=0.9.10=h7f98852_1003
+ - xorg-renderproto=0.11.1=h7f98852_1002
+ - xorg-xextproto=7.3.0=h7f98852_1002
+ - xorg-xproto=7.0.31=h7f98852_1007
+ - xxhash=0.8.0=h7f98852_3
+ - xyzservices=2022.4.0=pyhd8ed1ab_0
+ - xz=5.2.5=h7b6447c_0
+ - yaml=0.2.5=h7b6447c_0
+ - zlib=1.2.11=h7f8727e_4
+ - zstd=1.5.0=ha95c52a_0
+prefix: /storage/app/EWS/envs/conda/py3EWS
diff --git a/configs/docker/run/launchDocker.sh b/configs/docker/run/launchDocker.sh
new file mode 100755
index 0000000000000000000000000000000000000000..839257344d83c057701da760833998553ce9c231
--- /dev/null
+++ b/configs/docker/run/launchDocker.sh
@@ -0,0 +1 @@
+sudo docker run -it -v "/media/scratch/lb584_scratch/projects/ews_aws/ews_3/code:/storage/app/EWS_prod/code" -v "/media/scratch/lb584_scratch/projects/ews_aws/ews_3/regions:/storage/app/EWS_prod/regions" -v "/media/scratch/lb584_scratch/projects/ews_aws/ews_3/envs/credentials:/storage/app/EWS_prod/envs/credentials" -w "/storage/app/EWS_prod/code" ews_coordinator
diff --git a/coordinator/ProcessorDeposition.py b/coordinator/ProcessorDeposition.py
index cd4860510198a00cb472bbd114578841d9f0b9bf..07bcd0020cf65ebe97ae7ac2bada46f45cc1e8e9 100644
--- a/coordinator/ProcessorDeposition.py
+++ b/coordinator/ProcessorDeposition.py
@@ -37,11 +37,12 @@ def process_in_job_dep(jobPath,status,config,component):
# TODO: perform ssh file transfer in python instead of subprocess
server_name: str = config['ServerName']
- machine_seperator: str = ":"
- if not server_name:
- machine_seperator = ""
+ if server_name == "":
+ cmd_scp = ["scp", f"{file_path}/{file_name}.tar.gz", jobPath]
+ else:
+ cmd_scp = ["scp", "-i", config['ServerKey'], "-o", "StrictHostKeyChecking=no",
+ f"{server_name}:{file_path}/{file_name}.tar.gz", jobPath]
- cmd_scp = ["scp", "-i", config['ServerKey'], "-o", "StrictHostKeyChecking=no", f"{server_name}{machine_seperator}{file_path}/{file_name}.tar.gz", jobPath]
description_short = 'dep scp'
description_long = 'scp from server to job directory'
subprocess_and_log(cmd_scp, description_short, description_long)
diff --git a/coordinator/ProcessorEnvironment.py b/coordinator/ProcessorEnvironment.py
index e39386e84fe821ffd739080663f3e86c27aa57e7..5e96960a473253f498c320afa92f71be47dbf916 100644
--- a/coordinator/ProcessorEnvironment.py
+++ b/coordinator/ProcessorEnvironment.py
@@ -40,11 +40,12 @@ def process_in_job_env2_0(jobPath,status,config,component):
# TODO: perform ssh file transfer in python instead of subprocess
server_name: str = config['ServerName']
- machine_seperator: str = ":"
- if not server_name:
- machine_seperator = ""
+ if server_name == "":
+ cmd_scp: list = ["scp", f"{file_path}/{file_name}.tar.gz", jobPath]
+ else:
+ cmd_scp: list = ["scp", "-i", config['ServerKey'], "-o", "StrictHostKeyChecking=no",
+ f"{config['ServerName']}:{file_path}/{file_name}.tar.gz", jobPath]
- cmd_scp: list = ["scp","-i",config['ServerKey'],"-o","StrictHostKeyChecking=no",f"{config['ServerName']}{machine_seperator}{file_path}/{file_name}.tar.gz", jobPath]
description_short = 'env2 scp'
description_long = 'Copying file from remote server to job directory'
# lawrence comment in/out
diff --git a/coordinator/ProcessorServer.py b/coordinator/ProcessorServer.py
index 1ef1cebb1dacf0e70463fc9ae9d95e1050d12c02..d8aad9beb9c856fa595d36d30633ce1b12f66879 100644
--- a/coordinator/ProcessorServer.py
+++ b/coordinator/ProcessorServer.py
@@ -37,28 +37,26 @@ def process_pre_job_server_download(input_args: dict):
file_path = Template(config[component]['ServerPathTemplate']).substitute(**config)
file_name = Template(config[component]['InputFileTemplate']).substitute(**config)
- file_path_full = f"{file_path}/{file_name}.tar.gz"
- logger.info(f"Checking for existence of {file_path_full}")
+ logger.info(f"Checking for existence of {file_path}/{file_name}.tar.gz")
timenow = datetime.datetime.now(tz=datetime.timezone.utc).time()
- # test whether the file exists (if not, returns error code 1)
- # and test whether the tar file is complete (if not, error code is 2)
- cmd_ssh = [
- "ssh",
- "-i",
- config['ServerKey'],
- "-o",
- "StrictHostKeyChecking=no",
- config['ServerName'],
- f"test -f {file_path_full} && tar -tzf {file_path_full} >/dev/null"]
+ server_name: str = config['ServerName']
+ full_file_path = f"{file_path}/{file_name}.tar.gz"
+ if server_name == "":
+ cmd_check_file = [f"test -f {full_file_path} && tar -tzf {full_file_path} > /dev/null"]
+ run_in_shell: bool = True
+ else:
+ cmd_check_file = ["ssh", "-i", config['ServerKey'], "-o", "StrictHostKeyChecking=no", server_name,
+ f"test -f {full_file_path} && tar -tzf {full_file_path} >/dev/null"]
+ run_in_shell: bool = False
description_short = 'subprocess_ssh'
- description_long = f"Checking for existence of {file_path_full}"
+ description_long = f"Checking for existence of {file_path}/{file_name}.tar.gz"
status = subprocess_and_log(cmd_ssh,description_short,description_long,check=False)
- if status.returncode in [1,2]:
+ if status.returncode > 0:
# a time check in UTC. If it's late, raise warning, if very late, raise error
@@ -95,15 +93,6 @@ def process_pre_job_server_download(input_args: dict):
elif status.returncode == 0:
logger.info(f"Data is available for config {i+1} of {len(config_paths)}, calculation shall proceed")
- # silence other return codes
- # there is a known issue from tests that ssh to a blank server will
- # raise error code 255. This is kept silent while we come up with a tidy
- # solution. This doesn't affect production runs as the error would be
- # picked up by the later scp commands.
- #else:
- # logger.error(f"Unexpected return code from ssh command: {status.returncode}")
- # endScript(premature=False)
-
return True
def upload(config,FilesToSend,component):
@@ -132,15 +121,26 @@ def upload(config,FilesToSend,component):
logger.debug("Making path directory on remote server if it doesn't already exist")
- ssh_cmd = ["ssh","-i",config['ServerKey'],"-o","StrictHostKeyChecking=no",config['ServerName'], f"mkdir -p {OutputServerPath}"]
+ server_key = config['ServerKey']
+ if server_key == "":
+ ssh_cmd = ["ssh", f"mkdir -p {OutputServerPath}"]
+ run_in_shell: bool = True
+ else:
+ ssh_cmd = ["ssh", "-i", server_key, "-o", "StrictHostKeyChecking=no", config['ServerName'],
+ f"mkdir -p {OutputServerPath}"]
+ run_in_shell: bool = False
description_short = 'upload ssh'
description_long = 'make remote directory'
- subprocess_and_log(ssh_cmd, description_short, description_long)
+ subprocess_and_log(ssh_cmd, description_short, description_long, shell=run_in_shell)
logger.debug('Sending file(s) to remote server')
- scp_cmd = ["scp","-ri",config['ServerKey'],"-o","StrictHostKeyChecking=no",*FilesToSend, f"{config['ServerName']}:{OutputServerPath}"]
+ if server_key == "":
+ scp_cmd = ["scp", *FilesToSend, {OutputServerPath}]
+ else:
+ scp_cmd = ["scp", "-ri", server_key, "-o", "StrictHostKeyChecking=no", *FilesToSend,
+ f"{config['ServerName']}:{OutputServerPath}"]
description_short = 'upload scp'
description_long = 'scp files to remote directory'
diff --git a/coordinator/ProcessorUtils.py b/coordinator/ProcessorUtils.py
index 8ddfa4b0e974e3e9f88cc4b247c61c6b1f6e614d..1b7d01700ffe00b9d2befaf26796af556ff61f85 100644
--- a/coordinator/ProcessorUtils.py
+++ b/coordinator/ProcessorUtils.py
@@ -125,7 +125,7 @@ def get_only_existing_globs(file_globs,inplace=True):
globs_out += [fg]
return globs_out
-def subprocess_and_log(cmd,description_short,description_long,check=True,log_type='error',**kwargs):
+def subprocess_and_log(cmd,description_short,description_long,check=True,log_type='error', shell: bool = False, **kwargs):
'''Run a shell command (described by a comma separated list) and send stdout
and stderr to logfile, and raise any exception.
@@ -140,6 +140,7 @@ def subprocess_and_log(cmd,description_short,description_long,check=True,log_typ
process = subprocess.run(
cmd,
check=check,
+ shell=shell,
stdout = subprocess.PIPE,
stderr = subprocess.STDOUT,
**kwargs)
diff --git a/test.py b/test.py
new file mode 100644
index 0000000000000000000000000000000000000000..4eea79c6df740898a2ff564c0ac6a1cef093f739
--- /dev/null
+++ b/test.py
@@ -0,0 +1,4 @@
+from test2 import function
+
+if __name__ == '__main__':
+ function()
\ No newline at end of file
diff --git a/test2.py b/test2.py
new file mode 100644
index 0000000000000000000000000000000000000000..dec137b1379ab91c5a0b50e9ebe1d3d019c569bc
--- /dev/null
+++ b/test2.py
@@ -0,0 +1,23 @@
+import iris
+from iris.cube import CubeList
+
+
+# cube_wildcard = "/media/scratch/lb584_scratch/projects/ews_local_prod/regions/EastAfrica/workspace/ENVIRONMENT_2.0_20220730/NAME_Met_as_netcdf/*.nc"
+# cubes: CubeList = iris.load(cube_wildcard)
+# all_timepoints_present: bool = True
+# for cube in cubes:
+# coord = cube.coord("time")
+# timepoints = coord.shape[0]
+# if timepoints != 57:
+# all_timepoints_present = False
+# break
+# i = 0
+
+def function():
+ print("doing function thing")
+ if __name__ == '__main__':
+ print("doing main thing")
+
+
+if __name__ == '__main__':
+ function()
diff --git a/tests/integration/depo_asserts.py b/tests/integration/depo_asserts.py
new file mode 100644
index 0000000000000000000000000000000000000000..29ec5358ca0787d208f97bd852c7653f586a9de1
--- /dev/null
+++ b/tests/integration/depo_asserts.py
@@ -0,0 +1,61 @@
+import os
+import unittest
+
+from integration.integration_test_utils import IntegrationTestUtils
+from integration.test_deposition import TestDeposition
+
+
+class DepoAsserts(TestDeposition):
+
+
+ def test_standard_run_input_status_success(self):
+ status_file_path = os.path.join(TestDeposition.TEST_OUT_PATH, TestDeposition.TEST_JOB_DIR, "STATUS_SUCCESS")
+ success_file_exists: bool = os.path.isfile(status_file_path)
+ self.assertTrue(success_file_exists)
+
+
+ def test_standard_run_input_all_regions_ran(self):
+ """
+ working on the assumption that if there are images for each region, it must have run through
+ (at least past the region iteration)
+ """
+
+ east_africa_image_path = os.path.join(TestDeposition.TEST_OUT_PATH, TestDeposition.TEST_JOB_DIR,
+ "plotting", "eastafrica", "images", "Weekly",
+ "deposition_eastafrica_leaf_rust_total_202210010000_202210080000_map.png")
+ ethiopia_image_path = os.path.join(TestDeposition.TEST_OUT_PATH, TestDeposition.TEST_JOB_DIR,
+ "plotting", "ethiopia", "images", "Weekly",
+ "deposition_ethiopia_leaf_rust_total_202210010000_202210080000_map.png")
+
+ ea_file_exists: bool = os.path.isfile(east_africa_image_path)
+ eth_file_exists: bool = os.path.isfile(ethiopia_image_path)
+ self.assertTrue(ea_file_exists)
+ self.assertTrue(eth_file_exists)
+
+
+ def test_standard_run_all_input_csvs_produced(self):
+ east_africa_csv_path = os.path.join(TestDeposition.TEST_OUT_PATH, TestDeposition.TEST_JOB_DIR,
+ "plotting", "eastafrica", "input_csvs", "*.csv")
+ ethiopia_csv_path = os.path.join(TestDeposition.TEST_OUT_PATH, TestDeposition.TEST_JOB_DIR,
+ "plotting", "ethiopia", "input_csvs", "*.csv")
+
+ ea_csv_count: int = IntegrationTestUtils.count_files_in_wildcard(east_africa_csv_path)
+ eth_csv_count: int = IntegrationTestUtils.count_files_in_wildcard(ethiopia_csv_path)
+ self.assertEqual(9, ea_csv_count)
+ self.assertEqual(27, eth_csv_count)
+
+
+ def test_standard_run_all_images_produced(self):
+ east_africa_image_path = os.path.join(TestDeposition.TEST_OUT_PATH, TestDeposition.TEST_JOB_DIR,
+ "plotting", "eastafrica", "images", "Weekly", "*.png")
+ ethiopia_image_path = os.path.join(TestDeposition.TEST_OUT_PATH, TestDeposition.TEST_JOB_DIR,
+ "plotting", "ethiopia", "images", "Weekly", "*.png")
+
+ ea_csv_count: int = IntegrationTestUtils.count_files_in_wildcard(east_africa_image_path)
+ eth_csv_count: int = IntegrationTestUtils.count_files_in_wildcard(ethiopia_image_path)
+ self.assertEqual(3, ea_csv_count)
+ self.assertEqual(6, eth_csv_count)
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/tests/integration/sub_test.py b/tests/integration/sub_test.py
new file mode 100644
index 0000000000000000000000000000000000000000..5dfdd4a2c5728ebee98d8be3aebd6e281de66651
--- /dev/null
+++ b/tests/integration/sub_test.py
@@ -0,0 +1,23 @@
+import unittest
+
+from integration.test_test import MyTestCase
+
+
+class SubTest(unittest.TestCase):
+
+ def __init__(self, methodName: str = ...) -> None:
+ super().__init__(methodName)
+ print("RUNNING SETUP")
+
+ def test_1(self):
+ self.assertEqual(True, True) # add assertion here
+
+ def test_2(self):
+ self.assertEqual(True, True) # add assertion here
+
+ def test_3(self):
+ self.assertEqual(True, True) # add assertion here
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/tests/integration/test_advisory.py b/tests/integration/test_advisory.py
index cfa8d7240f0e3f48e17cbf664ebf6e23e1c628ac..a5bde542faf57542efa0d0d7517ad645105dbdaf 100644
--- a/tests/integration/test_advisory.py
+++ b/tests/integration/test_advisory.py
@@ -1,4 +1,5 @@
import copy
+import glob
import os
import unittest
@@ -67,6 +68,29 @@ class TestAdvisory(unittest.TestCase):
self.assertTrue(ea_file_exists)
self.assertTrue(eth_file_exists)
+ def test_standard_run_input_images_produced(self):
+
+ east_africa_image_wildcard = os.path.join(TestAdvisory.TEST_OUT_PATH, TestAdvisory.TEST_JOB_DIR,
+ "images",
+ "*eastafrica*.png")
+ ethiopia_image_wildcard = os.path.join(TestAdvisory.TEST_OUT_PATH, TestAdvisory.TEST_JOB_DIR,
+ "images",
+ "*ethiopia*.png")
+
+ africa_image_count: int = len(glob.glob(east_africa_image_wildcard))
+ ethiopia_image_count: int = len(glob.glob(ethiopia_image_wildcard))
+ self.assertEqual(3, africa_image_count)
+ self.assertEqual(3, ethiopia_image_count)
+
+ def test_standard_run_input_shapefiles_produced(self):
+
+ east_africa_image_wildcard = os.path.join(TestAdvisory.TEST_OUT_PATH, TestAdvisory.TEST_JOB_DIR,
+ "images",
+ "*eastafrica*.shp")
+
+ africa_image_count: int = len(glob.glob(east_africa_image_wildcard))
+ self.assertEqual(3, africa_image_count)
+
if __name__ == '__main__':
unittest.main()
diff --git a/tests/integration/test_env_suit.py b/tests/integration/test_env_suit.py
index b51487f95d455b64ae5ba871c828f6089ca7956c..5b6e1ad9673bafe72b3d7dbf42d222e97b2128f1 100644
--- a/tests/integration/test_env_suit.py
+++ b/tests/integration/test_env_suit.py
@@ -20,7 +20,7 @@ class TestEnvSuit(unittest.TestCase):
@staticmethod
def write_temp_run_config_file():
nowstring: str = IntegrationTestUtils.get_now_string()
- prefix: str = "temp_depo_" + nowstring
+ prefix: str = "temp_env_" + nowstring
# prefix: str = ""
default_config = IntegrationTestUtils.DEFAULT_CONFIG_FILE_PATH
diff --git a/tests/integration/test_epi.py b/tests/integration/test_epi.py
new file mode 100644
index 0000000000000000000000000000000000000000..18d406d9c5879288bd4f2cc864b5a10743efac48
--- /dev/null
+++ b/tests/integration/test_epi.py
@@ -0,0 +1,72 @@
+import copy
+import os
+import unittest
+
+from integration.integration_test_utils import IntegrationTestUtils
+
+
+class TestEpi(unittest.TestCase):
+
+ TEST_OUT_PATH: str = "not_set"
+ TEST_START_DATE: str = '20221001'
+ TEST_JOB_DIR: str = "SUMMARY_" + TEST_START_DATE
+
+ @classmethod
+ def setUpClass(cls) -> None:
+ TestEpi.write_temp_run_config_files()
+ TestEpi.unpack_dependencies()
+ TestEpi.run_advisory_pipeline()
+
+
+ @staticmethod
+ def write_temp_run_config_files():
+ nowstring: str = IntegrationTestUtils.get_now_string()
+ prefix: str = "temp_epi_" + nowstring
+ # prefix: str = ""
+
+ default_config = IntegrationTestUtils.DEFAULT_CONFIG_FILE_PATH
+ default_config_dict: dict = IntegrationTestUtils.load_json_file(default_config)
+ run_dict: dict = copy.deepcopy(default_config_dict)
+ TestEpi.TEST_OUT_PATH = run_dict['WorkspacePathout'] + prefix + os.sep
+ run_dict['WorkspacePathout'] = TestEpi.TEST_OUT_PATH
+ run_dict['WorkspacePath'] = TestEpi.TEST_OUT_PATH
+ run_dict['ServerName'] = '' # nothing, as local machine
+
+ IntegrationTestUtils.write_json_file(run_dict, IntegrationTestUtils.TEMP_CONFIG_FILE_PATH)
+
+
+ @staticmethod
+ def unpack_dependencies():
+ IntegrationTestUtils.unpack_zip(IntegrationTestUtils.EXAMPLE_SURVEY_FILE_PATH, TestEpi.TEST_OUT_PATH)
+ IntegrationTestUtils.unpack_zip(IntegrationTestUtils.EXAMPLE_DEPO_FILE_PATH, TestEpi.TEST_OUT_PATH)
+ IntegrationTestUtils.unpack_zip(IntegrationTestUtils.EXAMPLE_ENV_SUIT_FILE_PATH, TestEpi.TEST_OUT_PATH)
+
+
+ @staticmethod
+ def run_advisory_pipeline():
+ component = 'Epidemiology'
+ IntegrationTestUtils.run_pipeline(component, TestEpi.TEST_START_DATE)
+
+
+ def test_standard_run_input_status_success(self):
+ status_file_path = os.path.join(TestEpi.TEST_OUT_PATH, TestEpi.TEST_JOB_DIR, "STATUS_SUCCESS")
+ success_file_exists: bool = os.path.isfile(status_file_path)
+ self.assertTrue(success_file_exists)
+
+ def test_standard_run_input_all_docs_produced(self):
+
+ east_africa_image_path = os.path.join(TestEpi.TEST_OUT_PATH, TestEpi.TEST_JOB_DIR,
+ "tight-layout",
+ "wheat_rust_advisory_template_EastAfrica_20221001.docx")
+ ethiopia_image_path = os.path.join(TestEpi.TEST_OUT_PATH, TestEpi.TEST_JOB_DIR,
+ "tight-layout",
+ "wheat_rust_advisory_template_Ethiopia_20221001.docx")
+
+ ea_file_exists: bool = os.path.isfile(east_africa_image_path)
+ eth_file_exists: bool = os.path.isfile(ethiopia_image_path)
+ self.assertTrue(ea_file_exists)
+ self.assertTrue(eth_file_exists)
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/tests/integration/test_test.py b/tests/integration/test_test.py
new file mode 100644
index 0000000000000000000000000000000000000000..286267bf64390bd292f4f540ade818074a563471
--- /dev/null
+++ b/tests/integration/test_test.py
@@ -0,0 +1,8 @@
+import unittest
+
+
+class MyTestCase(unittest.TestCase):
+
+ def setUp(self) -> None:
+ print("running setup")
+
diff --git a/tests/test_data/test_deployment/regions/EastAfrica/resources/coordinator/configs/.gitignore b/tests/test_data/test_deployment/regions/EastAfrica/resources/coordinator/configs/.gitignore
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/tests/test_data/test_deployment/regions/EastAfrica/resources/coordinator/configs/config_EastAfrica_fc_live.json b/tests/test_data/test_deployment/regions/EastAfrica/resources/coordinator/configs/config_EastAfrica_fc_live.json
index 12cf38253f1968c179f5332c4da2632cf472a7dd..366f6a606a11305b4470b38fdf63036a0195802c 100644
--- a/tests/test_data/test_deployment/regions/EastAfrica/resources/coordinator/configs/config_EastAfrica_fc_live.json
+++ b/tests/test_data/test_deployment/regions/EastAfrica/resources/coordinator/configs/config_EastAfrica_fc_live.json
@@ -8,7 +8,7 @@
"ResourcesPath" : "../test_data/test_deployment/regions/EastAfrica/resources/",
"ServerPath" : "/storage/moved/Ethiopia/",
"ServerName" : "ewsmanager@willow.csx.cam.ac.uk",
- "ServerKey" : "../test_data/test_deployment/regions/EastAfrica/resources/coordinator/configs/ssh_key_willow",
+ "ServerKey" : "not used, set to empty string in tests",
"Survey" : {
"ProcessPreJob" : "process_pre_job_survey",
"ProcessInJob" : "process_in_job_survey",
@@ -148,7 +148,7 @@
},
"Epidemiology" : {
"DiseaseNames" : ["StemRust"],
- "CalculationSpanDays" : ["20220501",5],
+ "CalculationSpanDays" : [0,1],
"TimeStep_hours": "3",
"ProcessPreJob" : "process_pre_job_epi",
"ProcessInJob" : "process_in_job_epi",
@@ -220,7 +220,7 @@
"ProcessPreJob" : "query_past_successes",
"ProcessInJob" : "process_in_job_advisory",
"ProcessEWSPlotting" : "do_nothing",
- "seasonStartString" : "20220501",
+ "seasonStartString" : "20210901",
"Environment" : {
"SuccessFileTemplate" : "${WorkspacePath}ENVIRONMENT_2.0_${StartString}/STATUS_SUCCESS",
"DataPathTemplate" : "${WorkspacePath}/ENVIRONMENT_2.0_${dateString}/plotting/${SubRegionNameLower}/input_csvs/",
@@ -270,7 +270,7 @@
}
},
"Surveys" : {
- "variety_names_tidy_fn" : "/media/scratch/lb584_scratch/projects/ews_local_prod/regions/EastAfrica/resources/advisory_builder/configs/config_EastAfrica_variety_names_tidy.json",
+ "variety_names_tidy_fn" : "../test_data/test_deployment/regions/EastAfrica/resources/advisory_builder/configs/config_EastAfrica_variety_names_tidy.json",
"CountryColumnName" : "surveyor_infromation-country",
"EastAfrica" : {
"ShapeFilenameAdmin0" : "../test_data/test_deployment/regions/EastAfrica/resources/advisory_builder/assets/EthKen_admin0.shp",
diff --git a/wind_precip_20220730.png b/wind_precip_20220730.png
new file mode 100644
index 0000000000000000000000000000000000000000..dad499b3eefbde14494b9049b6574a87d9dd9774
Binary files /dev/null and b/wind_precip_20220730.png differ