diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 2ef78b126c8728ce4f7d2e1bf23241d5241ac6f5..648fb6603ef5eba3f5173eda781d371f04456cf7 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -4,6 +4,7 @@ variables:
# WHEN CALLED FROM A MULTI-PIPELINE TRIGGER, VARIABLES WITH SHARED NAMES WILL BE OVERWRITTEN
OUTPUT_DIR: "$CI_PROJECT_DIR/outputs/"
+ LOGS_DIR: "$CI_PROJECT_DIR/logs/"
# OUTPUT_DIR: "/storage/app/EWS_prod/regions/EastAfrica/workspace/"
TEST_WORKSPACE_DIR: "/storage/app/EWS_prod/regions/EastAfrica/test_workspace/"
FULL_FAT_RUN_DATE:
@@ -19,14 +20,6 @@ variables:
CUSTOM_RUN_DATE: "NOT_SET"
- FULL_FAT_COPY_OUTPUT:
- value: "false"
- options:
- - "true"
- - "false"
- description: "Select \"true\" to make a copy of the full-fat integration test output. Outputs will be copied to the
- production machine at /storage/app/EWS_prod/regions/EastAfrica/test_workspace"
-
cache:
paths:
- .cache/pip
@@ -40,6 +33,8 @@ stages:
.before_script:
before_script:
+ - BRANCH="lb584_27_11_23"
+ - echo "branch - " $BRANCH
- PACKAGES_DIR="$CI_PROJECT_DIR/packages/"
- echo "project dir - " $CI_PROJECT_DIR
- echo "output dir - " $OUTPUT_DIR
@@ -47,21 +42,20 @@ stages:
- echo "pip dir - " $PIP_CACHE_DIR
- echo "full-fat run date - " $FULL_FAT_RUN_DATE
- echo "custom full-fat run date - " $CUSTOM_RUN_DATE
- - echo "full-fat copy output - " $FULL_FAT_COPY_OUTPUT
- - source /storage/app/miniconda3/bin/activate /storage/app/EWS_prod/envs/conda/py3EWS/
+ - source /storage/app/EWS_prod/envs/venv/ews_prod/bin/activate
- pip install coverage
- python3 -V # Print out python version for debugging
- mkdir -p $PACKAGES_DIR
- mkdir -p $OUTPUT_DIR
- - git clone "https://nouser:$GROUP_WHEAT_ACCESS_VAR@gitlab.developers.cam.ac.uk/gilligan-epid/wheat-rusts/ews-source-generation.git" $PACKAGES_DIR/source_gen
- - git clone "https://nouser:$GROUP_WHEAT_ACCESS_VAR@gitlab.developers.cam.ac.uk/gilligan-epid/wheat-rusts/ews_plotting.git" $PACKAGES_DIR/plotting
- - git clone "https://nouser:$GROUP_WHEAT_ACCESS_VAR@gitlab.developers.cam.ac.uk/gilligan-epid/wheat-rusts/flagdir.git" $PACKAGES_DIR/flagdir
- - git clone "https://nouser:$GROUP_WHEAT_ACCESS_VAR@gitlab.developers.cam.ac.uk/gilligan-epid/wheat-rusts/ews-epimodel.git" $PACKAGES_DIR/epimodel
- - git clone "https://nouser:$GROUP_WHEAT_ACCESS_VAR@gitlab.developers.cam.ac.uk/gilligan-epid/wheat-rusts/ews-advisory-builder.git" $PACKAGES_DIR/advisory_builder
- - git clone "https://nouser:$GROUP_WHEAT_ACCESS_VAR@gitlab.developers.cam.ac.uk/gilligan-epid/wheat-rusts/ews_environmental_suitability_v2.git" $PACKAGES_DIR/environmental_suitability
- - git clone "https://nouser:$GROUP_WHEAT_ACCESS_VAR@gitlab.developers.cam.ac.uk/gilligan-epid/wheat-rusts/ews-postprocessing.git" $PACKAGES_DIR/post_processing
- - git clone "https://nouser:$GROUP_WHEAT_ACCESS_VAR@gitlab.developers.cam.ac.uk/gilligan-epid/wheat-rusts/ews-met-processing.git" $PACKAGES_DIR/met_processing
+ - git clone -b $BRANCH "https://nouser:$GROUP_WHEAT_ACCESS_VAR@gitlab.developers.cam.ac.uk/gilligan-epid/wheat-rusts/ews-source-generation.git" $PACKAGES_DIR/source_gen
+ - git clone -b $BRANCH "https://nouser:$GROUP_WHEAT_ACCESS_VAR@gitlab.developers.cam.ac.uk/gilligan-epid/wheat-rusts/ews_plotting.git" $PACKAGES_DIR/plotting
+ - git clone -b main "https://nouser:$GROUP_WHEAT_ACCESS_VAR@gitlab.developers.cam.ac.uk/gilligan-epid/wheat-rusts/flagdir.git" $PACKAGES_DIR/flagdir
+ - git clone -b $BRANCH "https://nouser:$GROUP_WHEAT_ACCESS_VAR@gitlab.developers.cam.ac.uk/gilligan-epid/wheat-rusts/ews-epimodel.git" $PACKAGES_DIR/epimodel
+ - git clone -b $BRANCH "https://nouser:$GROUP_WHEAT_ACCESS_VAR@gitlab.developers.cam.ac.uk/gilligan-epid/wheat-rusts/ews-advisory-builder.git" $PACKAGES_DIR/advisory_builder
+ - git clone -b main "https://nouser:$GROUP_WHEAT_ACCESS_VAR@gitlab.developers.cam.ac.uk/gilligan-epid/wheat-rusts/ews_environmental_suitability_v2.git" $PACKAGES_DIR/environmental_suitability
+ - git clone -b $BRANCH "https://nouser:$GROUP_WHEAT_ACCESS_VAR@gitlab.developers.cam.ac.uk/gilligan-epid/wheat-rusts/ews-postprocessing.git" $PACKAGES_DIR/post_processing
+ - git clone -b $BRANCH "https://nouser:$GROUP_WHEAT_ACCESS_VAR@gitlab.developers.cam.ac.uk/gilligan-epid/wheat-rusts/ews-met-processing.git" $PACKAGES_DIR/met_processing
- flagdir=$PACKAGES_DIR/flagdir
- epimodel=$PACKAGES_DIR/epimodel
- advisory=$PACKAGES_DIR/advisory_builder
@@ -81,18 +75,13 @@ half_fat_tests:
when: always
stage: half-fat
extends: .before_script
- image: lb584/ews_coordinator:latest
+ image: lb584/ews_coordinator_pip:gitlab
script:
- cd $CI_PROJECT_DIR/tests/integration/partial/
- python3 -m coverage run run_test_suite.py $CI_PROJECT_DIR/test_reports
-# - python3 -m coverage run -m unittest test_advisory.py
-# - python3 -m coverage run -m unittest test_deposition.py
-# - python3 -m coverage run -m unittest test_env_suit.py
-# - python3 -m coverage run -m unittest test_survey.py
- python3 -m coverage report
- python3 -m coverage html -d $CI_PROJECT_DIR/coverage
-# - ls
artifacts:
when: always
paths:
@@ -104,7 +93,7 @@ half_fat_epi_tests:
when: manual
stage: half-fat
extends: .before_script
- image: lb584/ews_coordinator:latest
+ image: lb584/ews_coordinator_pip:gitlab
script:
- cd $CI_PROJECT_DIR/tests/integration/partial/
@@ -122,40 +111,29 @@ full_fat_depo:
when: manual
stage: full-fat
extends: .before_script
- image: lb584/ews_coordinator:latest
+ image: lb584/ews_coordinator_pip:gitlab
tags: ["ewas-production"]
script:
- >
cd $CI_PROJECT_DIR/tests/integration/full/
-
python3 full_test_deposition.py
--config /storage/app/EWS_prod/regions/EastAfrica/resources/coordinator/configs/config_EastAfrica_fc_live.json
- --outdir $OUTPUT_DIR
+ --outdir ${TEST_WORKSPACE_DIR}/integration/full/
--email_cred /storage/app/EWS_prod/envs/credentials/Cred_gmail.json
--test_report_dir $CI_PROJECT_DIR/test_reports
--run_date_type $FULL_FAT_RUN_DATE
--custom_run_date $CUSTOM_RUN_DATE
- after_script:
- - >
- if [[ "$FULL_FAT_COPY_OUTPUT" == "true" ]];
- then
- date=$(date '+%Y-%m-%d-%H%M');
- mkdir ${TEST_WORKSPACE_DIR}/integration/full/${date};
- cp -r $OUTPUT_DIR/* ${TEST_WORKSPACE_DIR}/integration/full/${date};
- fi
-
artifacts:
paths:
- - $CI_PROJECT_DIR/coverage
- - $CI_PROJECT_DIR/test_reports
+ - $LOGS_DIR
expire_in: 10 days
full_fat_env_suit:
when: manual
stage: full-fat
extends: .before_script
- image: lb584/ews_coordinator:latest
+ image: lb584/ews_coordinator_pip:gitlab
tags: ["ewas-production"]
script:
- >
@@ -163,65 +141,42 @@ full_fat_env_suit:
python3 full_test_env_suit.py
--config /storage/app/EWS_prod/regions/EastAfrica/resources/coordinator/configs/config_EastAfrica_fc_live.json
- --outdir $OUTPUT_DIR
+ --outdir ${TEST_WORKSPACE_DIR}/integration/full/
--email_cred /storage/app/EWS_prod/envs/credentials/Cred_gmail.json
--run_date_type $FULL_FAT_RUN_DATE
--custom_run_date $CUSTOM_RUN_DATE
- after_script:
- - >
- if [[ "$FULL_FAT_COPY_OUTPUT" == "true" ]];
- then
- date=$(date '+%Y-%m-%d-%H%M');
- mkdir ${TEST_WORKSPACE_DIR}/integration/full/${date};
- cp -r $OUTPUT_DIR/* ${TEST_WORKSPACE_DIR}/integration/full/${date};
- fi
-
artifacts:
- when: on_failure
paths:
- - $OUTPUT_DIR/*/log.txt
- - $OUTPUT_DIR/*.txt
+ - $LOGS_DIR
expire_in: 10 days
full_fat_survey:
when: manual
stage: full-fat
extends: .before_script
- image: lb584/ews_coordinator:latest
+ image: lb584/ews_coordinator_pip:gitlab
tags: ["ewas-production"]
script:
- >
- cd $CI_PROJECT_DIR/tests/integration/full/
-
+ cd $CI_PROJECT_DIR/tests/integration/full/;
python3 full_test_survey.py
--config /storage/app/EWS_prod/regions/EastAfrica/resources/coordinator/configs/config_EastAfrica_fc_live.json
- --outdir $OUTPUT_DIR
+ --outdir ${TEST_WORKSPACE_DIR}/integration/full/
--email_cred /storage/app/EWS_prod/envs/credentials/Cred_gmail.json
--run_date_type $FULL_FAT_RUN_DATE
--custom_run_date $CUSTOM_RUN_DATE
- after_script:
- - >
- if [[ "$FULL_FAT_COPY_OUTPUT" == "true" ]];
- then
- date=$(date '+%Y-%m-%d-%H%M');
- mkdir ${TEST_WORKSPACE_DIR}/integration/full/${date};
- cp -r $OUTPUT_DIR/* ${TEST_WORKSPACE_DIR}/integration/full/${date};
- fi
-
artifacts:
- when: on_failure
paths:
- - $OUTPUT_DIR/*/log.txt
- - $OUTPUT_DIR/*.txt
+ - $LOGS_DIR
expire_in: 10 days
full_fat_advisory:
when: manual
stage: full-fat
extends: .before_script
- image: lb584/ews_coordinator:latest
+ image: lb584/ews_coordinator_pip:gitlab
tags: ["ewas-production"]
script:
- >
@@ -229,32 +184,21 @@ full_fat_advisory:
python3 full_test_advisory.py
--config /storage/app/EWS_prod/regions/EastAfrica/resources/coordinator/configs/config_EastAfrica_fc_live.json
- --outdir $OUTPUT_DIR
+ --outdir ${TEST_WORKSPACE_DIR}/integration/full/
--email_cred /storage/app/EWS_prod/envs/credentials/Cred_gmail.json
--run_date_type $FULL_FAT_RUN_DATE
--custom_run_date $CUSTOM_RUN_DATE
- after_script:
- - >
- if [[ "$FULL_FAT_COPY_OUTPUT" == "true" ]];
- then
- date=$(date '+%Y-%m-%d-%H%M');
- mkdir ${TEST_WORKSPACE_DIR}/integration/full/${date};
- cp -r $OUTPUT_DIR/* ${TEST_WORKSPACE_DIR}/integration/full/${date};
- fi
-
artifacts:
- when: on_failure
paths:
- - $OUTPUT_DIR/*/log.txt
- - $OUTPUT_DIR/*.txt
+ - $LOGS_DIR
expire_in: 10 days
full_fat_epi:
when: manual
stage: full-fat
extends: .before_script
- image: lb584/ews_coordinator:latest
+ image: lb584/ews_coordinator_pip:gitlab
tags: ["ewas-production"]
script:
- >
@@ -262,32 +206,21 @@ full_fat_epi:
python3 full_test_epi.py
--config /storage/app/EWS_prod/regions/EastAfrica/resources/coordinator/configs/config_EastAfrica_fc_live.json
- --outdir $OUTPUT_DIR
+ --outdir ${TEST_WORKSPACE_DIR}/integration/full/
--email_cred /storage/app/EWS_prod/envs/credentials/Cred_gmail.json
--run_date_type $FULL_FAT_RUN_DATE
--custom_run_date $CUSTOM_RUN_DATE
- after_script:
- - >
- if [[ "$FULL_FAT_COPY_OUTPUT" == "true" ]];
- then
- date=$(date '+%Y-%m-%d-%H%M');
- mkdir ${TEST_WORKSPACE_DIR}/integration/full/${date};
- cp -r $OUTPUT_DIR/* ${TEST_WORKSPACE_DIR}/integration/full/${date};
- fi
-
artifacts:
- when: on_failure
paths:
- - $OUTPUT_DIR/*/log.txt
- - $OUTPUT_DIR/*.txt
+ - $LOGS_DIR
expire_in: 10 days
pages:
- when: always
+ when: manual
stage: docs
extends: .before_script
- image: lb584/ews_coordinator:latest
+ image: lb584/ews_coordinator_pip:gitlab
script:
- export PATH=$PATH:/home/ewsmanager/.local/bin
- pip install sphinx
diff --git a/configs/conda/conda-env-py3EWS-nobuilds.yml b/configs/conda/conda-env-py3EWS-nobuilds.yml
index e17e6e116531c2e020e7e06d444747de82db9091..ef10bacf169540e4494ea4d4f1bcfa46c2bbe450 100644
--- a/configs/conda/conda-env-py3EWS-nobuilds.yml
+++ b/configs/conda/conda-env-py3EWS-nobuilds.yml
@@ -111,12 +111,12 @@ dependencies:
- ncurses=6.3
- netcdf4=1.5.7
- networkx=2.8.1
- - numpy=1.21.1
+ - numpy=1.22
- olefile=0.46
- openjpeg=2.4.0
- openssl=1.1.1o
- packaging=21.3
- - pandas=1.3.1
+ - pandas=1.4.2
- partd=1.2.0
- pcre=8.45
- pillow=8.3.1
@@ -183,7 +183,165 @@ dependencies:
- zlib=1.2.11
- zstd=1.5.0
- pip:
+ - aiohttp==3.8.5
+ - aiosignal==1.3.1
+ - alembic==1.11.3
+ - anyio==3.7.1
+ - apache-airflow==2.7.0
+ - apache-airflow-providers-cncf-kubernetes==7.4.2
+ - apache-airflow-providers-common-sql==1.7.0
+ - apache-airflow-providers-ftp==3.5.0
+ - apache-airflow-providers-http==4.5.0
+ - apache-airflow-providers-imap==3.3.0
+ - apache-airflow-providers-sqlite==3.4.3
+ - apispec==6.3.0
+ - argcomplete==3.1.1
+ - asgiref==3.7.2
+ - astroid==2.15.5
+ - async-timeout==4.0.3
+ - attrs==23.1.0
+ - backoff==1.10.0
+ - blinker==1.6.2
+ - cachelib==0.9.0
+ - cachetools==5.3.1
+ - cattrs==23.1.2
+ - cffi==1.15.1
+ - charset-normalizer==3.2.0
+ - click==8.1.7
+ - clickclick==20.10.2
+ - colorama==0.4.6
+ - colorlog==4.8.0
+ - configupdater==3.1.1
+ - connexion==2.14.2
- coverage==6.5.0
+ - cron-descriptor==1.4.0
+ - croniter==1.4.1
+ - cryptography==41.0.3
+ - deprecated==1.2.14
+ - dill==0.3.1.1
+ - dnspython==2.4.2
+ - docutils
+ - email-validator==1.3.1
+ - exceptiongroup==1.1.3
+ - flask==2.2.5
+ - flask-appbuilder==4.3.3
+ - flask-babel==2.0.0
+ - flask-caching==2.0.2
+ - flask-jwt-extended==4.5.2
+ - flask-limiter==3.3.1
+ - flask-login==0.6.2
+ - flask-session==0.5.0
+ - flask-sqlalchemy==2.5.1
+ - flask-wtf==1.1.1
+ - frozenlist==1.4.0
+ - google-auth==2.22.0
+ - google-re2==1.1
+ - googleapis-common-protos==1.60.0
+ - greenlet==2.0.2
+ - grpcio==1.57.0
+ - gunicorn==21.2.0
+ - h11==0.14.0
- html-testrunner==1.2.1
+ - httpcore==0.16.3
+ - httpx==0.23.3
+ - idna==3.4
+ - importlib-metadata==6.8.0
+ - importlib-resources==6.0.1
+ - inflection==0.5.1
+ - isort==5.12.0
+ - itsdangerous==2.1.2
+ - jsonschema==4.19.0
+ - jsonschema-specifications==2023.7.1
+ - kubernetes==23.6.0
+ - kubernetes-asyncio==24.2.3
+ - lazy-object-proxy==1.9.0
+ - limits==3.5.0
+ - linkify-it-py==2.0.2
+ - lockfile==0.12.2
+ - mako==1.2.4
+ - markdown==3.4.4
+ - markdown-it-py==3.0.0
+ - markupsafe==2.1.3
+ - marshmallow==3.20.1
+ - marshmallow-oneofschema==3.0.1
+ - marshmallow-sqlalchemy==0.26.1
+ - mccabe==0.7.0
+ - mdit-py-plugins==0.4.0
+ - mdurl==0.1.2
+ - multidict==6.0.4
+ - numpy==1.22
+ - oauthlib==3.2.2
+ - opentelemetry-api==1.15.0
+ - opentelemetry-exporter-otlp==1.15.0
+ - opentelemetry-exporter-otlp-proto-grpc==1.15.0
+ - opentelemetry-exporter-otlp-proto-http==1.15.0
+ - opentelemetry-proto==1.15.0
+ - opentelemetry-sdk==1.15.0
+ - opentelemetry-semantic-conventions==0.36b0
+ - ordered-set==4.1.0
+ - packaging==23.1
+ - pandas==1.4.2
+ - pathspec==0.11.2
+ - pbr==5.11.1
+ - pendulum==2.1.2
+ - platformdirs==3.5.1
+ - pluggy==1.2.0
+ - prison==0.2.1
+ - protobuf==4.21.12
+ - psutil==5.9.5
+ - pyasn1==0.5.0
+ - pyasn1-modules==0.3.0
+ - pydantic==1.10.12
+ - pygments==2.16.1
+ - pyjwt==2.8.0
+ - pylint==2.17.4
+ - pyqt5-sip
+ - pyqtchart==5.12
+ - pyqtwebengine==5.12.1
+ - python-daemon==3.0.1
+ - python-nvd3==0.15.0
+ - python-slugify==8.0.1
+ - pytz==2023.3
+ - pytzdata==2020.1
+ - pyyaml==6.0.1
+ - referencing==0.30.2
+ - requests==2.31.0
+ - requests-oauthlib==1.3.1
+ - requests-toolbelt==1.0.0
+ - rfc3339-validator==0.1.4
+ - rfc3986==1.5.0
+ - rich==13.5.2
+ - rich-argparse==1.2.0
+ - rioxarray==0.15.0
+ - rpds-py==0.9.2
+ - rsa==4.9
+ - setproctitle==1.3.2
+ - setuptools==68.1.2
+ - sniffio==1.3.0
+ - sphinx==6.2.1
+ - sphinx-autodoc-typehints==1.23.0
+ - sphinx-rtd-theme==1.2.1
+ - sphinxcontrib-jquery==4.1
+ - sqlalchemy==1.4.49
+ - sqlalchemy-jsonfield==1.0.1.post0
+ - sqlalchemy-utils==0.41.1
+ - sqlparse==0.4.4
+ - tabulate==0.9.0
+ - tenacity==8.2.3
+ - termcolor==2.3.0
- testresources==2.0.1
+ - text-unidecode==1.3
+ - tomli==2.0.1
+ - tomlkit==0.11.8
+ - typing-extensions==4.7.1
+ - uc-micro-py==1.0.2
+ - unicodecsv==0.14.1
+ - urllib3==1.26.16
+ - websocket-client==1.6.2
+ - werkzeug==2.2.3
+ - wrapt==1.15.0
+ - wtforms==3.0.1
+ - xarray==2023.11.0
+ - yarl==1.9.2
+ - zipp==3.16.2
prefix: /storage/app/EWS/envs/conda/py3EWS
diff --git a/configs/conda/conda-env-py3EWS-withbuilds.yml b/configs/conda/conda-env-py3EWS-withbuilds.yml
index f682bd2ea419fd476d37bca7e22df65c8882f233..79c75487829f8bf15f93faa8b9cf0fcf43aad79c 100644
--- a/configs/conda/conda-env-py3EWS-withbuilds.yml
+++ b/configs/conda/conda-env-py3EWS-withbuilds.yml
@@ -186,4 +186,5 @@ dependencies:
- coverage==6.5.0
- html-testrunner==1.2.1
- testresources==2.0.1
+ - rioxarray==0.15.0
prefix: /storage/app/EWS/envs/conda/py3EWS
diff --git a/configs/docker/build/Dockerfile b/configs/docker/build/Dockerfile
index 34e948fc702670458f8a1f679edbd40b74727a71..39d9b28da32047e30f572e5211c93a5f417aa73d 100644
--- a/configs/docker/build/Dockerfile
+++ b/configs/docker/build/Dockerfile
@@ -1,39 +1,16 @@
# syntax=docker/dockerfile:1
-FROM ubuntu:18.04 as base_image
+FROM ubuntu:24.04 as base_image
-#run mkdir as root
-RUN mkdir -p /storage/app/EWS_prod/regions/EastAfrica/workspace
-RUN mkdir -p /storage/app/EWS_prod/regions/SouthAsia/workspace
-
-#create dirs
-WORKDIR /home
-#RUN mkdir /home/ewsmanager/code
-
-#copy resources in
-COPY ./conda-env-py3EWS-withbuilds.yml .
+SHELL ["/bin/bash", "-c"]
#install system packages
-RUN apt-get update;apt-get install --assume-yes wget unzip emacs gdal-bin libgdal-dev build-essential gfortran zlib1g zlib1g-dev libbz2-dev apt-utils liblzma-dev libpcre++-dev libcurl4-openssl-dev default-jre lib32readline7 lib32readline6-dev openssh-server libfribidi-dev libfontconfig1-dev git nano dos2unix;
+RUN apt-get update;apt-get install --assume-yes python-dev-is-python3 python3-dev python3.11-venv wget unzip emacs gdal-bin libgdal-dev build-essential gfortran zlib1g zlib1g-dev libbz2-dev apt-utils liblzma-dev libcurl4-openssl-dev default-jre openssh-server libfribidi-dev libfontconfig1-dev libspatialindex-dev git dos2unix nano rsync;
RUN echo "alias e='emacs -nw $1'" >> ~/.bashrc;
RUN alias ll="ls -ltr"
#sets the timezone to London
RUN DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt-get -y install tzdata
-# Install miniconda
-ENV CONDA_DIR /storage/app/miniconda3
-RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O ~/miniconda.sh \
- && /bin/bash ~/miniconda.sh -b -p $CONDA_DIR \
- && rm -v ~/miniconda.sh;
-# Put conda in path so we can use conda activate
-ENV PATH=$CONDA_DIR/bin:$PATH
-
-#build the conda env(s)
-RUN conda env create --file conda-env-py3EWS-withbuilds.yml --prefix /storage/app/EWS_prod/envs/conda/py3EWS
-
-#tidy
-RUN rm -v conda-env-py3EWS-withbuilds.yml;
-
#these usernames and ids match the production server (IaaS), override when building a local image if needed
ARG GNAME=ews
ARG UNAME=ewsmanager
@@ -41,8 +18,32 @@ ARG UID=1006
ARG GID=1008
RUN groupadd -g $GID $GNAME
RUN useradd -m -u $UID -g $GID -o -s /bin/bash $UNAME
+
+#run mkdir as root
+RUN mkdir -p /storage/app/EWS_prod/regions/EastAfrica/workspace
+RUN mkdir -p /storage/app/EWS_prod/regions/SouthAsia/workspace
+
+RUN chown -R $UNAME /storage
+
+#swith to the user before installing stuff
USER $UNAME
+
+#create dirs
+WORKDIR /home
+#RUN mkdir /home/ewsmanager/code
+
+#copy resources in
+#COPY ./conda-env-py3EWS-nobuilds.yml .
+COPY ./requirements.txt .
+
+RUN echo "alias e='emacs -nw $1'" >> ~/.bashrc;
+RUN alias ll="ls -ltr"
+
+#build the venv
+RUN python3 -m venv /storage/app/EWS_prod/envs/venv/ews_prod
+RUN source /storage/app/EWS_prod/envs/venv/ews_prod/bin/activate;pip install -r requirements.txt
+
RUN echo $PYTHONPATH
WORKDIR /home/ewsmanager
diff --git a/configs/docker/build/build.sh b/configs/docker/build/build.sh
index 2600b98ef58b8daadb898498487b8261458351f6..a9ecb85cf0bcb9b1dce0441a64de425469ad8a9d 100644
--- a/configs/docker/build/build.sh
+++ b/configs/docker/build/build.sh
@@ -1,4 +1,4 @@
-cp ../../conda/conda-env-py3EWS-withbuilds.yml .;
+cp ../../conda/conda-env-py3EWS-nobuilds.yml .;
# note that if you want to build an image that runs with your credentials, override the following variables when
# building the image, e.g. for user 'lb584' with GID and UID 1001:
# sudo docker build -t ews_coordinator --build-arg GNAME=lb584 --build-arg UNAME=lb584
@@ -7,4 +7,4 @@ cp ../../conda/conda-env-py3EWS-withbuilds.yml .;
# that are mounted. If the default command is run, the user credentials are set to be those of the Cam group server
sudo docker build -t ews_coordinator -f Dockerfile .;
-rm -v conda-env-py3EWS-withbuilds.yml;
+rm -v conda-env-py3EWS-nobuilds.yml;
diff --git a/configs/docker/build/requirements.txt b/configs/docker/build/requirements.txt
new file mode 100644
index 0000000000000000000000000000000000000000..23caa527ed36034823368a38627d84b3f8d750e0
--- /dev/null
+++ b/configs/docker/build/requirements.txt
@@ -0,0 +1,8 @@
+contextily==1.5.0
+geopandas==0.14.2
+HTMLTestRunner-rv==1.1.2
+python-docx==1.1.0
+rioxarray==0.15.0
+Rtree==1.1.0
+scitools-iris==3.7.0
+coverage==7.4.0
\ No newline at end of file
diff --git a/configs/docker/run/launchDockerRunner_lb584_Local.sh b/configs/docker/run/launchDockerRunner_lb584_Local.sh
index 60017f6444e45b395191ab54bf45f0b7b8d9dad8..f84542bd822d51738c5c02ae799877b17b47c07a 100755
--- a/configs/docker/run/launchDockerRunner_lb584_Local.sh
+++ b/configs/docker/run/launchDockerRunner_lb584_Local.sh
@@ -1,9 +1,9 @@
#!/bin/bash
set -e
-docker run -it -v "/media/scratch/lb584_scratch/projects/ews_aws/ews_3/code:/storage/app/EWS_prod/code" \
- -v "/media/scratch/lb584_scratch/projects/ews_aws/ews_3/regions:/storage/app/EWS_prod/regions" \
- -v "/media/scratch/lb584_scratch/projects/ews_aws/ews_3/envs/credentials:/storage/app/EWS_prod/envs/credentials" \
+docker run -it -v "/media/scratch/lb584_scratch/projects/ews_instructions/code:/storage/app/EWS_prod/code" \
+ -v "/media/scratch/lb584_scratch/projects/ews_instructionsregions:/storage/app/EWS_prod/regions" \
+ -v "/media/scratch/lb584_scratch/projects/ews_instructions/envs/credentials:/storage/app/EWS_prod/envs/credentials" \
-w "/storage/app/EWS_prod/code" \
--name ews_runner \
ews_coordinator
diff --git a/coordinator/EnvSuitPipeline.py b/coordinator/EnvSuitPipeline.py
index daf5e3a3a90c653b11dd5a2bf6c68378ddc00d4a..eb830f801b5345ce457f29b75562b9b06a42bf41 100644
--- a/coordinator/EnvSuitPipeline.py
+++ b/coordinator/EnvSuitPipeline.py
@@ -261,13 +261,13 @@ def run_pipeline(pipeline_config, region, dateString, extracted = False, prevent
# run_merger(config, sys_config, processor_name)
- resultFile = envSuitPath + 'RIE.csv'
+ resultFile = envSuitPath + 'RIE.nc'
strain_outPath = os.path.join(region_outPath,strain)
- strain_outFile = strain_outPath + '/RIE_value.csv'
+ strain_outFile = strain_outPath + '/RIE_value.nc'
# Check results dimension
- result = pd.read_csv(resultFile)
- result_dims = result.shape
+ # result = pd.read_csv(resultFile)
+ # result_dims = result.shape
"""
read in the input_spatial_points.csv file and get the number of spatial points - this is used for
@@ -278,14 +278,21 @@ def run_pipeline(pipeline_config, region, dateString, extracted = False, prevent
spatial_points = pd.read_csv(region_spatial_points_file)
spatial_dim = spatial_points.shape[0]
- if ((result_dims[0] != spatial_dim) or (result_dims[1] != (temporal_dim + 4))): # + 4 required because there are extra columns in the result file
- logger.error(f"Result dimension {result_dims} does not match with the expected: ({spatial_dim}, {temporal_dim + 4})")
- raise IndexError
+ # if ((result_dims[0] != spatial_dim) or (result_dims[1] != (temporal_dim + 4))): # + 4 required because there are extra columns in the result file
+ # logger.error(f"Result dimension {result_dims} does not match with the expected: ({spatial_dim}, {temporal_dim + 4})")
+ # raise IndexError
if not os.path.exists(strain_outPath):
os.makedirs(strain_outPath)
shutil.copy(resultFile,strain_outFile)
+
+ # todo - Add a flag to this part of the code to enable/disable csv writing as an option
+ # resultCSVFile = envSuitPath + 'RIE.csv'
+ # if os.path.isfile(resultCSVFile):
+ # strain_outFile = strain_outPath + '/RIE_value.csv'
+ # shutil.copy(resultCSVFile,strain_outFile)
+
logger.info(f"{strain} result successfully created and moved to {strain_outPath}/")
logger.info('SUCCESSFULLY FINISHED')
diff --git a/coordinator/ProcessorEnvironment.py b/coordinator/ProcessorEnvironment.py
index b2f38ec29863eb59e32bcc1a5e8a789f2aa5e2d3..4d1a6468929cdda345365ac9a3c6ecfea03647eb 100644
--- a/coordinator/ProcessorEnvironment.py
+++ b/coordinator/ProcessorEnvironment.py
@@ -139,8 +139,8 @@ class ProcessorEnvironment(Processor):
for region in subregions:
output_dir = f"{jobPath}/plotting/{region.lower()}"
- csv_template_dir = input_dir + "/{DISEASE_DIR}/RIE_value.csv"
-
+ rie_template_dir = input_dir + "/{DISEASE_DIR}/RIE_value.nc"
+
Path(output_dir).mkdir(parents=True, exist_ok=True)
sys_config = config['Environment']['EWS-Plotting']['SysConfig']
@@ -158,16 +158,16 @@ class ProcessorEnvironment(Processor):
run_params_file_arg = run_config,
es_output_dir_arg = output_dir,
issue_date_arg = config['StartString'],
- disease_csv_template_arg = csv_template_dir)
+ disease_nc_template_arg = rie_template_dir)
env_suit_processor.run_params.FILTER_FOR_COUNTRY = (filter_for_country.upper() == "TRUE")
# Include further diseases in plotting. In this case the irrigated suitabilite for the rusts.
# TODO: move this part out into a config
extra_diseases = [
- EnvSuitDiseaseInfo("Stem rust temp-only", "stem_rust_temponly", config['StartString'], "StemRust_TempOnly", csv_template_dir),
- EnvSuitDiseaseInfo("Leaf rust temp-only", "leaf_rust_temponly", config['StartString'], "LeafRust_TempOnly", csv_template_dir),
- EnvSuitDiseaseInfo("Stripe rust temp-only", "stripe_temponly", config['StartString'], "StripeRust_TempOnly", csv_template_dir),
+ EnvSuitDiseaseInfo("Stem rust temp-only", "stem_rust_temponly", config['StartString'], "StemRust_TempOnly", rie_template_dir),
+ EnvSuitDiseaseInfo("Leaf rust temp-only", "leaf_rust_temponly", config['StartString'], "LeafRust_TempOnly", rie_template_dir),
+ EnvSuitDiseaseInfo("Stripe rust temp-only", "stripe_temponly", config['StartString'], "StripeRust_TempOnly", rie_template_dir),
]
env_suit_processor.add_diseases(diseases=extra_diseases)
diff --git a/coordinator/ProcessorEpidemiology.py b/coordinator/ProcessorEpidemiology.py
index f71031da52d0dd548b6a5e5b6cb237162a523a03..2714e2a4ccca821990ee50a6523e75e7ad879896 100644
--- a/coordinator/ProcessorEpidemiology.py
+++ b/coordinator/ProcessorEpidemiology.py
@@ -590,7 +590,7 @@ class ProcessorEpidemiology(Processor):
# load the full epi results
df_full = read_csv(outfile,header=[0],index_col=[0,1])
- column_date_fmt = f"X{config_epi['StartTimeShort']}_X%Y%m%d%H%M"
+ column_date_fmt = f"{config_epi['StartTimeShort']}_%Y%m%d%H%M"
df_full_dates = to_datetime(df_full.columns.astype('str'),format=column_date_fmt)
unit_description = ''
diff --git a/docs/_source/installation.rst b/docs/_source/installation.rst
index 31e90b89c5ec5c5f8a97c277fe39d2390f1c8330..528689e7d89c588cd4e5fd9c752ac173d5e1de53 100644
--- a/docs/_source/installation.rst
+++ b/docs/_source/installation.rst
@@ -41,12 +41,21 @@ In this example, we are using the Linux user “ewsmanager†and group “ews
mkdir ./logs;
mkdir -p ./envs/credentials
+.. code-block:: bash
+
+ mkdir -p ./code regions/EastAfrica/workspace regions/SouthAsia/workspace regions/EastAfrica/resources regions/SouthAsia/resources met_inputs/metofficeupload/upload/Ethiopia/toMO/ met_inputs/metofficeupload/upload/Ethiopia/fromMO/daily_name/ web_data/Ethiopia met_inputs/metofficeupload/upload/SouthAsia/toMO/ met_inputs/metofficeupload/upload/SouthAsia/fromMO/daily_name/ web_data/SouthAsia ./logs ./envs/credentials
.. code-block:: bash
- chmod -R g+sw <install_location>
+ mkdir -p .\code regions\EastAfrica\workspace regions\SouthAsia\workspace regions\EastAfrica\resources regions\SouthAsia\resources met_inputs\metofficeupload\upload\Ethiopia\toMO\ met_inputs\metofficeupload\upload\Ethiopia\fromMO\daily_name\ web_data\Ethiopia met_inputs\metofficeupload\upload\SouthAsia\toMO\ met_inputs\metofficeupload\upload\SouthAsia\fromMO\daily_name\ web_data\SouthAsia .\logs .\envs\credentials
+
+shell scripts to perform the above operations:
+:download:`Download .cmd file <../_static/assets/windows_create_dirs.cmd>`
+:download:`Download .sh file <../_static/assets/linux_create_dirs.sh>`
-(make all files written here default to the current user group, and group-writable
+chmod -R g+sw <install_location>
+
+make all files written here default to the current user group, and group-writable. This is not necessary if you are depoying in a single-user environment, such as a personal computer
2: Clone code
-------------
@@ -56,7 +65,7 @@ cd into <install_location>/code
.. code-block:: bash
- git clone git@gitlab.developers.cam.ac.uk:gilligan-epid/wheat-rusts/ews-coordinator.git coordinator;git clone git@gitlab.developers.cam.ac.uk:gilligan-epid/wheat-rusts/ews-environmental-suitability.git environmental_suitability;git clone git@gitlab.developers.cam.ac.uk:gilligan-epid/wheat-rusts/ews-advisory-builder.git advisory_builder;git clone git@gitlab.developers.cam.ac.uk:gilligan-epid/wheat-rusts/ews-plotting.git plotting;git clone git@gitlab.developers.cam.ac.uk:gilligan-epid/wheat-rusts/ews-epidemiology.git epimodel;git clone git@gitlab.developers.cam.ac.uk:gilligan-epid/wheat-rusts/ews-source-calculation.git source_gen;git clone git@gitlab.developers.cam.ac.uk:gilligan-epid/wheat-rusts/ews-coordinator.git coordinator;git clone git@gitlab.developers.cam.ac.uk:gilligan-epid/wheat-rusts/ews-job-flagging.git flagdir;git clone git@gitlab.developers.cam.ac.uk:gilligan-epid/wheat-rusts/ews-met-processing.git met_extractor_v2;git clone git@gitlab.developers.cam.ac.uk:gilligan-epid/wheat-rusts/ews-postprocessing.git post_processing;
+ git clone git@gitlab.developers.cam.ac.uk:gilligan-epid/wheat-rusts/ews-coordinator.git coordinator;git clone git@gitlab.developers.cam.ac.uk:gilligan-epid/wheat-rusts/ews-environmental-suitability.git environmental_suitability;git clone git@gitlab.developers.cam.ac.uk:gilligan-epid/wheat-rusts/ews-advisory-builder.git advisory_builder;git clone git@gitlab.developers.cam.ac.uk:gilligan-epid/wheat-rusts/ews-plotting.git plotting;git clone git@gitlab.developers.cam.ac.uk:gilligan-epid/wheat-rusts/ews-epidemiology.git epimodel;git clone git@gitlab.developers.cam.ac.uk:gilligan-epid/wheat-rusts/ews-source-calculation.git source_gen;git clone git@gitlab.developers.cam.ac.uk:gilligan-epid/wheat-rusts/ews-job-flagging.git flagdir;git clone git@gitlab.developers.cam.ac.uk:gilligan-epid/wheat-rusts/ews-met-processing.git met_extractor_v2;git clone git@gitlab.developers.cam.ac.uk:gilligan-epid/wheat-rusts/ews-postprocessing.git post_processing;
or generate this string in bash:
@@ -68,7 +77,6 @@ or generate this string in bash:
ews-plotting,plotting \
ews-epidemiology,epimodel \
ews-source-calculation,source_gen \
- ews-coordinator,coordinator \
ews-job-flagging,flagdir \
ews-met-processing,met_extractor_v2 \
ews-postprocessing,post_processing;
@@ -78,6 +86,11 @@ or generate this string in bash:
printf "%s" "git clone git@gitlab.developers.cam.ac.uk:gilligan-epid/wheat-rusts/${repo}.git ${folder};"
done
+shell scripts to perform the above operations:
+:download:`Download .cmd file <../_static/assets/windows_clone_git.cmd>`
+:download:`Download .sh file <../_static/assets/linux_clone_git.sh>`
+
+
3: Copy the Resources needed for the various chart areas:
---------------------------------------------------------
.. admonition:: Configuration files from Cambridge required
@@ -85,13 +98,22 @@ or generate this string in bash:
Due to the size, project resources are not included in the git repository. You will need to ask the Cambridge team
for the latest version of the resources archive.
+.. admonition:: Add resources for one region at a time
+
+ There is a seperate resources archive for each geographic region. Currently there are two regions, EastAfrica and
+ SouthAsia. The instructions below denote the different regions with <EastAfrica/SouthAsia>. We suggest you start
+ with EastAfrica and get that working before setting up other regions.
+
Once you have received an archive of ``<EastAfrica/SouthAsia>_resources_<latest_date>.tar.gz`` from Cambridge, unpack
into ``<install_location>/regions/<EastAfrica/SouthAsia>/resources``. This will place all the necessary JSON
configuration and other assets required for the pipeline to run.
+e.g.:
+
.. code-block:: bash
- tar -zxvf resources_<latest_date>.tar.gz
+ cd <install_location>/regions/EastAfrica/resources
+ tar -zxvf eastafrica_resources_<latest_date>.tar.gz
Edit the coordinator run script to fix the root paths **(not necessary if running in our docker image)**:
@@ -138,6 +160,9 @@ following credentials**:
Where the various variables are all LINUX user credentials (type the ‘id’ command to see all these values)
+**if you are running the ews for use on your personal computer, you can exclude the --build-arg arguments from the build
+step, note that all files written by the pipeline will be owned by the root user, rather than your user account.**
+
Once you have edited the build.sh script, run it (or run the commands within it manually). **if you are running on
Windows and run these commands manually, make sure you run the "cp ../../conda/conda-env-py3EWS-withbuilds.yml ."
equivalent to place the conda dependencies in the same dir before running docker build.
@@ -146,6 +171,7 @@ equivalent to place the conda dependencies in the same dir before running docker
bash build.sh
+**building docker can take a while, typically 5-10 mins**
Make a copy of the docker launch template file and edit the source dir to match that of your installation.
@@ -180,8 +206,8 @@ when attached to the running docker container:
.. code-block:: bash
- cd /storage/app/EWS_prod/code/coordonator/tests/integration/
- ./run_tests.sh
+ cd /storage/app/EWS_prod/code/coordinator/tests/integration/partial/
+ ./run_test_suite.sh
If successful, you will see lots of system out passing by, with a message like "Ran 14 tests in 0:01:13 OK" at the end.
diff --git a/docs/_static/assets/linux_clone_git.sh b/docs/_static/assets/linux_clone_git.sh
new file mode 100644
index 0000000000000000000000000000000000000000..1a6c4d2fe22287c5ca66581812d02eb39043560d
--- /dev/null
+++ b/docs/_static/assets/linux_clone_git.sh
@@ -0,0 +1,10 @@
+#!/bin/bash
+git clone git@gitlab.developers.cam.ac.uk:gilligan-epid/wheat-rusts/ews-coordinator.git coordinator;
+git clone git@gitlab.developers.cam.ac.uk:gilligan-epid/wheat-rusts/ews-environmental-suitability.git environmental_suitability;
+git clone git@gitlab.developers.cam.ac.uk:gilligan-epid/wheat-rusts/ews-advisory-builder.git advisory_builder;
+git clone git@gitlab.developers.cam.ac.uk:gilligan-epid/wheat-rusts/ews-plotting.git plotting;
+git clone git@gitlab.developers.cam.ac.uk:gilligan-epid/wheat-rusts/ews-epidemiology.git epimodel;
+git clone git@gitlab.developers.cam.ac.uk:gilligan-epid/wheat-rusts/ews-source-calculation.git source_gen;
+git clone git@gitlab.developers.cam.ac.uk:gilligan-epid/wheat-rusts/ews-job-flagging.git flagdir;
+git clone git@gitlab.developers.cam.ac.uk:gilligan-epid/wheat-rusts/ews-met-processing.git met_extractor_v2;
+git clone git@gitlab.developers.cam.ac.uk:gilligan-epid/wheat-rusts/ews-postprocessing.git post_processing;
\ No newline at end of file
diff --git a/docs/_static/assets/linux_create_dirs.sh b/docs/_static/assets/linux_create_dirs.sh
new file mode 100644
index 0000000000000000000000000000000000000000..fea167b1ca19f0703fecd90fe7f0d5bde490c974
--- /dev/null
+++ b/docs/_static/assets/linux_create_dirs.sh
@@ -0,0 +1,14 @@
+#!/bin/bash
+mkdir ./code;
+mkdir -p regions/EastAfrica/workspace;
+mkdir -p regions/SouthAsia/workspace;
+mkdir -p regions/EastAfrica/resources;
+mkdir -p regions/SouthAsia/resources;
+mkdir -p met_inputs/metofficeupload/upload/Ethiopia/toMO/;
+mkdir -p met_inputs/metofficeupload/upload/Ethiopia/fromMO/daily_name/;
+mkdir -p web_data/Ethiopia;
+mkdir -p met_inputs/metofficeupload/upload/SouthAsia/toMO/;
+mkdir -p met_inputs/metofficeupload/upload/SouthAsia/fromMO/daily_name/;
+mkdir -p web_data/SouthAsia;
+mkdir ./logs;
+mkdir -p ./envs/credentials;
\ No newline at end of file
diff --git a/docs/_static/assets/windows_clone_git.cmd b/docs/_static/assets/windows_clone_git.cmd
new file mode 100644
index 0000000000000000000000000000000000000000..e1dbd4ffdac9f54fd81a32872e4aa27b0376aca3
--- /dev/null
+++ b/docs/_static/assets/windows_clone_git.cmd
@@ -0,0 +1,10 @@
+@ECHO OFF
+git clone git@gitlab.developers.cam.ac.uk:gilligan-epid/wheat-rusts/ews-coordinator.git coordinator
+git clone git@gitlab.developers.cam.ac.uk:gilligan-epid/wheat-rusts/ews-environmental-suitability.git environmental_suitability
+git clone git@gitlab.developers.cam.ac.uk:gilligan-epid/wheat-rusts/ews-advisory-builder.git advisory_builder
+git clone git@gitlab.developers.cam.ac.uk:gilligan-epid/wheat-rusts/ews-plotting.git plotting
+git clone git@gitlab.developers.cam.ac.uk:gilligan-epid/wheat-rusts/ews-epidemiology.git epimodel
+git clone git@gitlab.developers.cam.ac.uk:gilligan-epid/wheat-rusts/ews-source-calculation.git source_gen
+git clone git@gitlab.developers.cam.ac.uk:gilligan-epid/wheat-rusts/ews-job-flagging.git flagdir
+git clone git@gitlab.developers.cam.ac.uk:gilligan-epid/wheat-rusts/ews-met-processing.git met_extractor_v2
+git clone git@gitlab.developers.cam.ac.uk:gilligan-epid/wheat-rusts/ews-postprocessing.git post_processing
\ No newline at end of file
diff --git a/docs/_static/assets/windows_create_dirs.cmd b/docs/_static/assets/windows_create_dirs.cmd
new file mode 100644
index 0000000000000000000000000000000000000000..78039402c415a3cedcdd68f833612e3cd36c27e9
--- /dev/null
+++ b/docs/_static/assets/windows_create_dirs.cmd
@@ -0,0 +1,2 @@
+@ECHO OFF
+mkdir .\code regions\EastAfrica\workspace regions\SouthAsia\workspace regions\EastAfrica\resources regions\SouthAsia\resources met_inputs\metofficeupload\upload\Ethiopia\toMO\ met_inputs\metofficeupload\upload\Ethiopia\fromMO\daily_name\ web_data\Ethiopia met_inputs\metofficeupload\upload\SouthAsia\toMO\ met_inputs\metofficeupload\upload\SouthAsia\fromMO\daily_name\ web_data\SouthAsia .\logs .\envs\credentials
\ No newline at end of file
diff --git a/docs/api.rst b/docs/api.rst
index 92c6af18c4618c7a5de60e983a8304f22acc9f17..3d57073301b307ffbeb4f08da253caf2ac081c6f 100644
--- a/docs/api.rst
+++ b/docs/api.rst
@@ -6,7 +6,7 @@ API
:template: custom-module-template.rst
:recursive:
- coordinator
+.. coordinator
plotting
ews_postprocessing
met_processing
diff --git a/scripts/run_utils.sh b/scripts/run_utils.sh
index 895d09f2af5bbc9f301b82c66c1ce5ff94ba23b3..727bdf191dee15266cf84a28648fcbf0f21cce86 100755
--- a/scripts/run_utils.sh
+++ b/scripts/run_utils.sh
@@ -26,13 +26,13 @@ function setup_conda_env() {
# activate conda environment of python modules so they can be imported
#TODO: Move conda_env from bin to envs
- conda_env=${envs}/conda/py3EWS
- source /storage/app/miniconda3/bin/activate ${conda_env}
+ venv=${envs}/conda/venv;
+ source ${venv}/bin/activate;
}
function teardown_conda_env() {
# deactivate conda environment
- source /storage/app/miniconda3/bin/deactivate ${conda_env}
+ deactivate;
}
diff --git a/tests/integration/full/convertDos2unix.sh b/tests/integration/full/convertDos2unix.sh
new file mode 100644
index 0000000000000000000000000000000000000000..908021444a56bf7e5b9c2f9e69fe7623067ff2f3
--- /dev/null
+++ b/tests/integration/full/convertDos2unix.sh
@@ -0,0 +1,2 @@
+# run this script within the ews docker image if you have checked out the code on a windows machine, then run the new .sh script to launch the tests
+dos2unix run_tests.sh > windows_run_tests.sh
\ No newline at end of file
diff --git a/tests/integration/full/full_test_advisory.py b/tests/integration/full/full_test_advisory.py
index 4e7e2e2015d840e3317666699230ab1391b8102c..16c9335d3b41bf914fbf93fe9d58dcbf1ba71f59 100644
--- a/tests/integration/full/full_test_advisory.py
+++ b/tests/integration/full/full_test_advisory.py
@@ -22,7 +22,8 @@ class FullTestAdvisory(BaseAdvisoryTestSuite.AdvisoryTestSuite):
self.set_expected_values()
- if IntegrationTestUtils.TEST_OUT_PATH is None or not os.path.isdir(IntegrationTestUtils.TEST_OUT_PATH):
+ if IntegrationTestUtils.TEST_OUT_PATH is None \
+ or not os.listdir(IntegrationTestUtils.TEST_OUT_PATH):
# if True:
FullTestAdvisory.write_temp_run_config_files()
FullTestAdvisory.run_dependent_pipelines()
diff --git a/tests/integration/full/full_test_deposition.py b/tests/integration/full/full_test_deposition.py
index 8e7f947e099cd9f1847fc414249ffd3493d366bd..87ff3eeeb285c744250b0bc0109be3e47be7dfdd 100644
--- a/tests/integration/full/full_test_deposition.py
+++ b/tests/integration/full/full_test_deposition.py
@@ -22,7 +22,7 @@ class FullTestDeposition(BaseDepoTestSuite.DepoTestSuite):
path = IntegrationTestUtils.TEST_OUT_PATH
if path is None \
- or not os.path.isdir(IntegrationTestUtils.TEST_OUT_PATH):
+ or not os.listdir(IntegrationTestUtils.TEST_OUT_PATH):
FullTestDeposition.write_temp_run_config_file()
FullTestDeposition.run_depo_pipeline()
else:
diff --git a/tests/integration/full/full_test_env_suit.py b/tests/integration/full/full_test_env_suit.py
index d6a9f390e44fb159e777dafd4d55c63e51bf3591..619636474bd268c362db0c1dcf31b13403d252b4 100644
--- a/tests/integration/full/full_test_env_suit.py
+++ b/tests/integration/full/full_test_env_suit.py
@@ -19,8 +19,8 @@ class FullTestEnvSuit(BaseEnvSuitTestSuite.EnvSuitTestSuite):
self.set_expected_values()
- if IntegrationTestUtils.TEST_OUT_PATH is None or not os.path.isdir(
- IntegrationTestUtils.TEST_OUT_PATH):
+ if IntegrationTestUtils.TEST_OUT_PATH is None \
+ or not os.listdir(IntegrationTestUtils.TEST_OUT_PATH):
FullTestEnvSuit.write_temp_run_config_file()
FullTestEnvSuit.run_env_pipeline()
else:
@@ -40,7 +40,7 @@ class FullTestEnvSuit(BaseEnvSuitTestSuite.EnvSuitTestSuite):
run_dict['Environment']['EWS-Plotting']['Ethiopia']['FilterForCountry'] = "True"
run_dict['Environment']['EWS-Plotting']['Kenya']['FilterForCountry'] = "True"
- os.makedirs(IntegrationTestUtils.TEST_OUT_PATH)
+ os.makedirs(IntegrationTestUtils.TEST_OUT_PATH, exist_ok = True)
IntegrationTestUtils.TEMP_CONFIG_FILE_NAME = IntegrationTestUtils.TEST_OUT_PATH + "temp_config.json"
IntegrationTestUtils.write_json_file(run_dict, IntegrationTestUtils.TEMP_CONFIG_FILE_NAME)
diff --git a/tests/integration/full/full_test_epi.py b/tests/integration/full/full_test_epi.py
index 33fb8edf30b8998f7448b0662553e3bef945b438..9389f26db1bb9e1afbc47f9036805e5e12f1a3d7 100644
--- a/tests/integration/full/full_test_epi.py
+++ b/tests/integration/full/full_test_epi.py
@@ -20,7 +20,8 @@ class FullTestEpi(BaseEpiTestSuite.EpiTestSuite):
self.set_expected_values()
- if IntegrationTestUtils.TEST_OUT_PATH is None or not os.path.isdir(IntegrationTestUtils.TEST_OUT_PATH):
+ if IntegrationTestUtils.TEST_OUT_PATH is None \
+ or not os.listdir(IntegrationTestUtils.TEST_OUT_PATH):
# if True:
FullTestEpi.write_temp_run_config_files()
FullTestEpi.run_dependent_pipelines()
diff --git a/tests/integration/full/full_test_survey.py b/tests/integration/full/full_test_survey.py
index f38bcd7fe1bdaf9ca61b10cef778a82e5702cda3..157d6cc607c84c18218efc4a479c62fb0531bbec 100644
--- a/tests/integration/full/full_test_survey.py
+++ b/tests/integration/full/full_test_survey.py
@@ -18,8 +18,8 @@ class FullTestSurvey(BaseSurveyTestSuite.SurveyTestSuite):
self.set_expected_values()
- if IntegrationTestUtils.TEST_OUT_PATH is None or not os.path.isdir(
- IntegrationTestUtils.TEST_OUT_PATH):
+ if IntegrationTestUtils.TEST_OUT_PATH is None \
+ or not os.listdir(IntegrationTestUtils.TEST_OUT_PATH):
FullTestSurvey.write_temp_run_config_file()
FullTestSurvey.run_survey_pipeline()
else:
@@ -35,7 +35,7 @@ class FullTestSurvey(BaseSurveyTestSuite.SurveyTestSuite):
run_dict['WorkspacePath'] = IntegrationTestUtils.TEST_OUT_PATH
run_dict['Survey']['SkipServerDownload'] = False
- os.makedirs(IntegrationTestUtils.TEST_OUT_PATH)
+ os.makedirs(IntegrationTestUtils.TEST_OUT_PATH, exist_ok = True)
IntegrationTestUtils.TEMP_CONFIG_FILE_NAME = IntegrationTestUtils.TEST_OUT_PATH + "temp_config.json"
IntegrationTestUtils.write_json_file(run_dict, IntegrationTestUtils.TEMP_CONFIG_FILE_NAME)
diff --git a/tests/integration/partial/convertDos2unix.sh b/tests/integration/partial/convertDos2unix.sh
new file mode 100644
index 0000000000000000000000000000000000000000..908021444a56bf7e5b9c2f9e69fe7623067ff2f3
--- /dev/null
+++ b/tests/integration/partial/convertDos2unix.sh
@@ -0,0 +1,2 @@
+# run this script within the ews docker image if you have checked out the code on a windows machine, then run the new .sh script to launch the tests
+dos2unix run_tests.sh > windows_run_tests.sh
\ No newline at end of file
diff --git a/tests/integration/partial/integration_test_utils.py b/tests/integration/partial/integration_test_utils.py
index 9903274e81c3eecb127359666ecc018a3c5dc606..c52a98a3bfefa7fb723d61356085b2a3c4533d35 100644
--- a/tests/integration/partial/integration_test_utils.py
+++ b/tests/integration/partial/integration_test_utils.py
@@ -3,15 +3,14 @@ import datetime
import glob
import json
import os
-from importlib import reload
+
from typing import List, Type
from unittest import TestSuite, TestLoader, TestCase, TestResult
from zipfile import ZipFile
-from HtmlTestRunner import HTMLTestRunner
+from HTMLTestRunner import HTMLTestRunner
from Processor import Processor
-from ProcessorDeposition import ProcessorDeposition
class IntegrationTestUtils:
@@ -102,7 +101,12 @@ class IntegrationTestUtils:
if _test_report_dir is None:
_test_report_dir = IntegrationTestUtils.TEST_JOB_DIR
- runner = HTMLTestRunner(output = _test_report_dir, combine_reports = True)
+ """
+ the HTMLTestRunner will create the directory if it does not exist, we are putting the output into the
+ TEST_OUT_PATH, which is the top-level test directory for this run, not the TEST_JOB_DIR, which is the dir
+ for the pipeline being tested
+ """
+ runner = HTMLTestRunner(output = IntegrationTestUtils.TEST_OUT_PATH)
result: TestResult = runner.run(tests)
return result.wasSuccessful()
@@ -264,6 +268,6 @@ class IntegrationTestUtils:
result = custom_run_date
else:
print("default")
- result = "20230126"
+ result = "20231127"
return result
diff --git a/tests/integration/partial/run_test_suite.py b/tests/integration/partial/run_test_suite.py
index 4ba4814959194eeb37343d3574104f57e2fcef9b..cde1c81d2b68ccfc3a4b57d3c61824ad472ed079 100644
--- a/tests/integration/partial/run_test_suite.py
+++ b/tests/integration/partial/run_test_suite.py
@@ -1,7 +1,7 @@
import sys
-from unittest import TestLoader, TestSuite, TestResult
+from unittest import TestLoader, TestSuite, TestResult, TextTestRunner
-from HtmlTestRunner import HTMLTestRunner
+from HTMLTestRunner import HTMLTestRunner
from integration.partial.test_advisory import TestAdvisory
from integration.partial.test_deposition import TestDeposition
@@ -21,7 +21,7 @@ def build_test_runner() -> TestSuite:
if __name__ == '__main__':
_output_dir = sys.argv.pop()
_tests = build_test_runner()
- _runner = HTMLTestRunner(output=_output_dir, combine_reports = True)
+ _runner = HTMLTestRunner(output=_output_dir)
_result: TestResult = _runner.run(_tests)
if not _result.wasSuccessful():
sys.exit(1)
diff --git a/tests/integration/partial/test_env_suit.py b/tests/integration/partial/test_env_suit.py
index 11169a6601406b1bb4c8a193dd9f2ce72097f47f..843556a46f285d9c7ea66d71ed99b1a94bf6be38 100644
--- a/tests/integration/partial/test_env_suit.py
+++ b/tests/integration/partial/test_env_suit.py
@@ -43,7 +43,7 @@ class TestEnvSuit(BaseEnvSuitTestSuite.EnvSuitTestSuite):
def write_temp_run_config_file():
nowstring: str = IntegrationTestUtils.get_now_string()
prefix: str = "temp_env_" + nowstring
- #prefix: str = "temp_env"
+ # prefix: str = "temp_env"
default_config = IntegrationTestUtils.DEFAULT_CONFIG_FILE_PATH
default_config_dict: dict = IntegrationTestUtils.load_json_file(default_config)
diff --git a/tests/integration/test_suites/depo_test_suite.py b/tests/integration/test_suites/depo_test_suite.py
index 15f88ea5994fbe13901254faff5e141781ceabca..6065f7d50615d47d88b2b539ea4303155cbb39b3 100644
--- a/tests/integration/test_suites/depo_test_suite.py
+++ b/tests/integration/test_suites/depo_test_suite.py
@@ -61,10 +61,10 @@ class BaseDepoTestSuite:
def test_standard_run_all_input_csvs_produced(self):
east_africa_csv_path = os.path.join(IntegrationTestUtils.TEST_OUT_PATH,
IntegrationTestUtils.TEST_JOB_DIR,
- "plotting", "eastafrica", "input_csvs", "*.csv")
+ "plotting", "eastafrica", "input_nc", "*.nc")
ethiopia_csv_path = os.path.join(IntegrationTestUtils.TEST_OUT_PATH,
IntegrationTestUtils.TEST_JOB_DIR,
- "plotting", "ethiopia", "input_csvs", "*.csv")
+ "plotting", "ethiopia", "input_nc", "*.nc")
ea_csv_count: int = IntegrationTestUtils.count_files_in_wildcard(east_africa_csv_path)
eth_csv_count: int = IntegrationTestUtils.count_files_in_wildcard(ethiopia_csv_path)
diff --git a/tests/integration/test_suites/env_suit_test_suite.py b/tests/integration/test_suites/env_suit_test_suite.py
index e21de97763de795e9a427431c2e3233cff53ec08..8981a64b001e022a6bbf5e2e62b6ac8e85d40238 100644
--- a/tests/integration/test_suites/env_suit_test_suite.py
+++ b/tests/integration/test_suites/env_suit_test_suite.py
@@ -59,13 +59,13 @@ class BaseEnvSuitTestSuite:
self.assertTrue(eth_file_exists)
- def test_standard_run_all_input_csvs_produced(self):
+ def test_standard_run_all_input_ncs_produced(self):
east_africa_csv_path = os.path.join(IntegrationTestUtils.TEST_OUT_PATH,
IntegrationTestUtils.TEST_JOB_DIR,
- "plotting", "eastafrica", "input_csvs", "*.csv")
+ "plotting", "eastafrica", "input_nc", "*.nc")
ethiopia_csv_path = os.path.join(IntegrationTestUtils.TEST_OUT_PATH,
IntegrationTestUtils.TEST_JOB_DIR,
- "plotting", "ethiopia", "input_csvs", "*.csv")
+ "plotting", "ethiopia", "input_nc", "*.nc")
ea_csv_count: int = IntegrationTestUtils.count_files_in_wildcard(east_africa_csv_path)
eth_csv_count: int = IntegrationTestUtils.count_files_in_wildcard(ethiopia_csv_path)
diff --git a/tests/integration/test_suites/epi_test_suite.py b/tests/integration/test_suites/epi_test_suite.py
index ab09140ae099953d40926a97bdaa69d56fd267fa..c2b08dd7799d620c13bc562c4b4c2a67f2a3ebb0 100644
--- a/tests/integration/test_suites/epi_test_suite.py
+++ b/tests/integration/test_suites/epi_test_suite.py
@@ -67,10 +67,10 @@ class BaseEpiTestSuite:
input_csvs_root = os.path.join(IntegrationTestUtils.TEST_OUT_PATH,
IntegrationTestUtils.TEST_JOB_DIR,
"plotting",
- "input_csvs",
+ "input_nc",
"*stem*")
- self.check_wildcard_exists_and_not_empty(input_csvs_root, ".csv")
+ self.check_wildcard_exists_and_not_empty(input_csvs_root, ".nc")
images_root = os.path.join(IntegrationTestUtils.TEST_OUT_PATH,
IntegrationTestUtils.TEST_JOB_DIR,
diff --git a/tests/integration/test_suites/survey_test_suite.py b/tests/integration/test_suites/survey_test_suite.py
index 83ef237a7212493e6867c819dac1cefcacd14a9a..4406f032007494e70f62e9839794590164d3dd31 100644
--- a/tests/integration/test_suites/survey_test_suite.py
+++ b/tests/integration/test_suites/survey_test_suite.py
@@ -1,4 +1,5 @@
import abc
+import glob
import os
import unittest
@@ -25,8 +26,8 @@ class BaseSurveyTestSuite:
def test_standard_run_input_status_success(self):
status_file_path = os.path.join(IntegrationTestUtils.TEST_OUT_PATH,
- IntegrationTestUtils.TEST_JOB_DIR, "STATUS_SUCCESS")
- success_file_exists: bool = os.path.isfile(status_file_path)
+ IntegrationTestUtils.TEST_JOB_DIR, "STATUS_SUCCESS*")
+ success_file_exists: bool = len(glob.glob(status_file_path)) > 0
self.assertTrue(success_file_exists)
diff --git a/tests/test_data/test_deployment/regions/EastAfrica/resources/coordinator/assets/example_depo_run.zip b/tests/test_data/test_deployment/regions/EastAfrica/resources/coordinator/assets/example_depo_run.zip
index e436dbad88a91ef82fef697ead280399df9d6666..cbb8985701391643a754100a1652659959f4332d 100644
Binary files a/tests/test_data/test_deployment/regions/EastAfrica/resources/coordinator/assets/example_depo_run.zip and b/tests/test_data/test_deployment/regions/EastAfrica/resources/coordinator/assets/example_depo_run.zip differ
diff --git a/tests/test_data/test_deployment/regions/EastAfrica/resources/coordinator/assets/example_env_suit_run.zip b/tests/test_data/test_deployment/regions/EastAfrica/resources/coordinator/assets/example_env_suit_run.zip
index 6f44bffff4fe87608a28131d0f60e2e555667d8b..b1ed4e3561b7290902a0f6a7552da5751bbb7bd6 100644
Binary files a/tests/test_data/test_deployment/regions/EastAfrica/resources/coordinator/assets/example_env_suit_run.zip and b/tests/test_data/test_deployment/regions/EastAfrica/resources/coordinator/assets/example_env_suit_run.zip differ
diff --git a/tests/test_data/test_deployment/regions/EastAfrica/resources/coordinator/configs/config_EastAfrica_fc_live.json b/tests/test_data/test_deployment/regions/EastAfrica/resources/coordinator/configs/config_EastAfrica_fc_live.json
index 55e02761a1bdd6a5b51c82063f8c7ea4bff01e53..105636292c30151f5804e9520f345746764c46c3 100644
--- a/tests/test_data/test_deployment/regions/EastAfrica/resources/coordinator/configs/config_EastAfrica_fc_live.json
+++ b/tests/test_data/test_deployment/regions/EastAfrica/resources/coordinator/configs/config_EastAfrica_fc_live.json
@@ -182,7 +182,7 @@
"PathTemplate" : "${WorkspacePath}ENVIRONMENT_2.0_${DateString}/processed/${RegionName}/${DiseaseName}/",
"SuccessFileTemplate" : "${WorkspacePath}ENVIRONMENT_2.0_${StartString}/STATUS_SUCCESS",
"FileListerFunction" : "list_onefile_operational",
- "FileNameTemplate" : "RIE_value.csv",
+ "FileNameTemplate" : "RIE_value.nc",
"FileNamePrepared" : "?"
},
"Epi" : [
@@ -292,7 +292,7 @@
"stem": "${depPlotPath}/infection_${SubRegionNameLower}_stem_seasonincforecast_*0300_*0000_map.png"
},
"Aggregation" : {
- "raster_path_template" : "${depDataPath}/${SubRegionNameLower}_${disease_name_snakecase}_masked.csv",
+ "raster_path_template" : "${depDataPath}/${SubRegionNameLower}_${disease_name_snakecase}_masked.nc",
"raster_name_template" : "predicted_infected_area_${disease_name_camelcase}",
"shape_fn_entry" : "ShapeFilenameAdmin2",
"shape_coln" : "ADM2_EN",