adding some new docker config/images for running on IaaS

configs/docker/build/Dockerfile

 # syntax=docker/dockerfile:1
-FROM ubuntu:18.04
+FROM ubuntu:18.04 as base_image
 
 #create dirs
 WORKDIR /home
@@ -8,7 +8,7 @@ RUN mkdir -p /storage/app/EWS_prod/regions/EastAfrica/workspace
 RUN mkdir -p /storage/app/EWS_prod/regions/SouthAsia/workspace
 
 #copy resources in
-COPY ./installRPackages.txt /home
+#COPY ./installRPackages.txt /home
 COPY ./conda-env-py3EWS-withbuilds.yml .
 
 #install system packages
@@ -37,4 +37,7 @@ ENV PATH=$CONDA_DIR/bin:$PATH
 RUN conda env create --file conda-env-py3EWS-withbuilds.yml --prefix /storage/app/EWS_prod/envs/conda/py3EWS
 
 #tidy
-RUN rm -v ~/miniconda.sh conda-env-py3EWS-withbuilds.yml;
+RUN rm -v ~/miniconda.sh conda-env-py3EWS-withbuilds.yml;
\ No newline at end of file
+
+FROM base_image as runner
+CMD ["echo", "hello world"]
\ No newline at end of file

configs/docker/build/RunnerDockerfile

+FROM ews_coordinator:latest
+
+ENTRYPOINT ["/storage/app/EWS_prod/code/coordinator/run_Processor.sh"]
+#CMD ["-p" "-c", "--noupload", "-s"]
\ No newline at end of file

configs/docker/build/build.sh

 cp ../../../conda-env-py3EWS-withbuilds.yml .;
-sudo docker build -t ews_coordinator .;
+sudo docker build --target base_image -t ews_coordinator .;
 rm -v conda-env-py3EWS-withbuilds.yml;

configs/docker/build/conda-env-py3EWS-withbuilds.yml

-name: /storage/app/EWS/envs/conda/py3EWS
-channels:
-  - conda-forge
-  - defaults
-dependencies:
-  - _libgcc_mutex=0.1=main
-  - _openmp_mutex=4.5=1_gnu
-  - affine=2.3.1=pyhd8ed1ab_0
-  - antlr-python-runtime=4.7.2=py39hf3d152e_1003
-  - attrs=21.4.0=pyhd8ed1ab_0
-  - boost-cpp=1.74.0=h312852a_4
-  - branca=0.5.0=pyhd8ed1ab_0
-  - brotlipy=0.7.0=py39h27cfd23_1003
-  - bzip2=1.0.8=h7f98852_4
-  - c-ares=1.17.1=h7f98852_1
-  - ca-certificates=2022.5.18=ha878542_0
-  - cairo=1.16.0=h6cf1ce9_1008
-  - cartopy=0.20.0=py39h2103e0b_0
-  - certifi=2022.5.18=py39hf3d152e_0
-  - cf-units=3.0.1=py39hce5d2b2_2
-  - cffi=1.15.0=py39hd667e15_1
-  - cfitsio=3.470=hb418390_7
-  - cftime=1.6.0=py39hd257fcd_1
-  - charset-normalizer=2.0.4=pyhd3eb1b0_0
-  - click=7.1.2=pyh9f0ad1d_0
-  - click-plugins=1.1.1=py_0
-  - cligj=0.7.2=pyhd8ed1ab_1
-  - cloudpickle=2.0.0=pyhd8ed1ab_0
-  - conda=4.12.0=py39hf3d152e_0
-  - conda-content-trust=0.1.1=pyhd3eb1b0_0
-  - conda-package-handling=1.7.3=py39h27cfd23_1
-  - contextily=1.0.1=py_0
-  - cryptography=36.0.0=py39h9ce1e76_0
-  - curl=7.78.0=hea6ffbf_0
-  - cycler=0.11.0=pyhd8ed1ab_0
-  - dask-core=2022.5.0=pyhd8ed1ab_0
-  - expat=2.4.1=h9c3ff4c_0
-  - fiona=1.8.20=py39h427c1bf_1
-  - folium=0.12.1.post1=pyhd8ed1ab_1
-  - fontconfig=2.13.1=hba837de_1005
-  - freetype=2.10.4=h0708190_1
-  - freexl=1.0.6=h7f98852_0
-  - fsspec=2022.3.0=pyhd8ed1ab_0
-  - gdal=3.3.1=py39h409cc32_1
-  - geographiclib=1.52=pyhd8ed1ab_0
-  - geopandas=0.10.2=pyhd8ed1ab_1
-  - geopandas-base=0.10.2=pyha770c72_1
-  - geopy=2.2.0=pyhd8ed1ab_0
-  - geos=3.9.1=h9c3ff4c_2
-  - geotiff=1.6.0=h4f31c25_6
-  - gettext=0.19.8.1=h0b5b191_1005
-  - giflib=5.2.1=h36c2ea0_2
-  - hdf4=4.2.15=h10796ff_3
-  - hdf5=1.10.6=nompi_h6a2412b_1114
-  - icu=68.1=h58526e2_0
-  - idna=3.3=pyhd3eb1b0_0
-  - iris=3.2.1=pyhd8ed1ab_0
-  - jbig=2.1=h7f98852_2003
-  - jinja2=3.1.2=pyhd8ed1ab_0
-  - joblib=1.1.0=pyhd8ed1ab_0
-  - jpeg=9d=h36c2ea0_0
-  - json-c=0.15=h98cffda_0
-  - kealib=1.4.14=hcc255d8_2
-  - kiwisolver=1.3.1=py39h1a9c180_1
-  - krb5=1.19.2=hcc1bbae_0
-  - lcms2=2.12=hddcbb42_0
-  - ld_impl_linux-64=2.35.1=h7274673_9
-  - lerc=2.2.1=h9c3ff4c_0
-  - libblas=3.9.0=11_linux64_openblas
-  - libcblas=3.9.0=11_linux64_openblas
-  - libcurl=7.78.0=h2574ce0_0
-  - libdap4=3.20.6=hd7c4107_2
-  - libdeflate=1.7=h7f98852_5
-  - libedit=3.1.20191231=he28a2e2_2
-  - libev=4.33=h516909a_1
-  - libffi=3.3=he6710b0_2
-  - libgcc-ng=11.2.0=h1234567_0
-  - libgdal=3.3.1=h8f005ca_1
-  - libgfortran-ng=12.1.0=h69a702a_16
-  - libgfortran5=12.1.0=hdcd56e2_16
-  - libglib=2.68.3=h3e27bee_0
-  - libgomp=11.2.0=h1234567_0
-  - libiconv=1.16=h516909a_0
-  - libkml=1.3.0=h238a007_1014
-  - liblapack=3.9.0=11_linux64_openblas
-  - libnetcdf=4.8.0=nompi_hcd642e3_103
-  - libnghttp2=1.43.0=h812cca2_0
-  - libopenblas=0.3.17=pthreads_h8fe5266_1
-  - libpng=1.6.37=h21135ba_2
-  - libpq=13.3=hd57d9b9_0
-  - librttopo=1.1.0=h1185371_6
-  - libspatialindex=1.9.3=h9c3ff4c_4
-  - libspatialite=5.0.1=h8694cbe_5
-  - libssh2=1.9.0=ha56f1ee_6
-  - libstdcxx-ng=12.1.0=ha89aaad_16
-  - libtiff=4.3.0=hf544144_1
-  - libuuid=2.32.1=h7f98852_1000
-  - libwebp-base=1.2.0=h7f98852_2
-  - libxcb=1.13=h7f98852_1003
-  - libxml2=2.9.12=h72842e0_0
-  - libxslt=1.1.33=h15afd5d_2
-  - libzip=1.8.0=h4de3113_0
-  - locket=1.0.0=pyhd8ed1ab_0
-  - lxml=4.6.3=py39h107f48f_0
-  - lz4-c=1.9.3=h9c3ff4c_1
-  - mapclassify=2.4.3=pyhd8ed1ab_0
-  - markupsafe=2.0.1=py39h3811e60_0
-  - matplotlib-base=3.4.2=py39h2fa2bec_0
-  - mercantile=1.2.1=pyhd8ed1ab_0
-  - munch=2.5.0=py_0
-  - ncurses=6.3=h7f8727e_2
-  - netcdf4=1.5.7=nompi_py39hc6dca20_100
-  - networkx=2.8.1=pyhd8ed1ab_0
-  - numpy=1.21.1=py39hdbf815f_0
-  - olefile=0.46=pyh9f0ad1d_1
-  - openjpeg=2.4.0=hb52868f_1
-  - openssl=1.1.1o=h166bdaf_0
-  - packaging=21.3=pyhd8ed1ab_0
-  - pandas=1.3.1=py39hde0f152_0
-  - partd=1.2.0=pyhd8ed1ab_0
-  - pcre=8.45=h9c3ff4c_0
-  - pillow=8.3.1=py39ha612740_0
-  - pip=21.2.4=py39h06a4308_0
-  - pixman=0.40.0=h36c2ea0_0
-  - poppler=21.03.0=h93df280_0
-  - poppler-data=0.4.11=hd8ed1ab_0
-  - postgresql=13.3=h2510834_0
-  - proj=8.0.1=h277dcde_0
-  - pthread-stubs=0.4=h36c2ea0_1001
-  - pycosat=0.6.3=py39h27cfd23_0
-  - pycparser=2.21=pyhd3eb1b0_0
-  - pyopenssl=21.0.0=pyhd3eb1b0_1
-  - pyparsing=3.0.9=pyhd8ed1ab_0
-  - pyproj=3.1.0=py39ha9a7ae0_3
-  - pyshp=2.3.0=pyhd8ed1ab_0
-  - pysocks=1.7.1=py39h06a4308_0
-  - python=3.9.7=h12debd9_1
-  - python-dateutil=2.8.2=pyhd8ed1ab_0
-  - python-docx=0.8.11=pyhd8ed1ab_0
-  - python-xxhash=3.0.0=py39hb9d737c_1
-  - python_abi=3.9=2_cp39
-  - pytz=2022.1=pyhd8ed1ab_0
-  - pyyaml=6.0=py39hb9d737c_4
-  - rasterio=1.2.6=py39hbc4e497_2
-  - readline=8.1.2=h7f8727e_1
-  - requests=2.27.1=pyhd3eb1b0_0
-  - rtree=1.0.0=py39hb102c33_1
-  - ruamel_yaml=0.15.100=py39h27cfd23_0
-  - scikit-learn=0.24.2=py39h4dfa638_0
-  - scipy=1.7.0=py39hee8e79c_1
-  - setuptools=58.0.4=py39h06a4308_0
-  - shapely=1.7.1=py39ha61afbd_5
-  - six=1.16.0=pyhd3eb1b0_0
-  - snuggs=1.4.7=py_0
-  - sqlite=3.37.0=hc218d9a_0
-  - threadpoolctl=3.1.0=pyh8a188c0_0
-  - tiledb=2.3.2=he87e0bf_0
-  - tk=8.6.11=h1ccaba5_0
-  - toolz=0.11.2=pyhd8ed1ab_0
-  - tornado=6.1=py39h3811e60_1
-  - tqdm=4.62.3=pyhd3eb1b0_1
-  - tzcode=2021a=h7f98852_2
-  - tzdata=2021e=hda174b7_0
-  - udunits2=2.2.27.27=hc3e0081_3
-  - urllib3=1.26.7=pyhd3eb1b0_0
-  - wheel=0.37.1=pyhd3eb1b0_0
-  - xerces-c=3.2.3=h9d8b166_2
-  - xorg-kbproto=1.0.7=h7f98852_1002
-  - xorg-libice=1.0.10=h7f98852_0
-  - xorg-libsm=1.2.3=hd9c2040_1000
-  - xorg-libx11=1.7.2=h7f98852_0
-  - xorg-libxau=1.0.9=h7f98852_0
-  - xorg-libxdmcp=1.1.3=h7f98852_0
-  - xorg-libxext=1.3.4=h7f98852_1
-  - xorg-libxrender=0.9.10=h7f98852_1003
-  - xorg-renderproto=0.11.1=h7f98852_1002
-  - xorg-xextproto=7.3.0=h7f98852_1002
-  - xorg-xproto=7.0.31=h7f98852_1007
-  - xxhash=0.8.0=h7f98852_3
-  - xyzservices=2022.4.0=pyhd8ed1ab_0
-  - xz=5.2.5=h7b6447c_0
-  - yaml=0.2.5=h7b6447c_0
-  - zlib=1.2.11=h7f8727e_4
-  - zstd=1.5.0=ha95c52a_0
-prefix: /storage/app/EWS/envs/conda/py3EWS

configs/docker/run/launchDockerRunnerIaas.sh

+#!/bin/bash
+set -e
+
+upload='--noupload' # default
+rundate=$(date '+%Y%m%d') # default today
+
+SHORT=p:,c:,s:,h
+LONG=component:,config:,rundate::,help
+OPTS=$(getopt -a -n weather --options $SHORT --longoptions $LONG -- "$@")
+
+eval set -- "$OPTS"
+
+while :
+do
+  case "$1" in
+    -p | --component )
+      component="$2"
+      shift 2
+      ;;
+    -c | --config )
+      config="$2"
+      shift 2
+      ;;
+    -s | --rundate )
+      rundate="$2"
+      shift 2
+      ;;
+    -h | --help)
+      "This is a weather script"
+      exit 2
+      ;;
+    --)
+      shift;
+      break
+      ;;
+    *)
+      echo "Unexpected option: $1"
+      ;;
+  esac
+done
+
+printf "component: %s\nconfig: %s\nrundate: %s\nupload: %s\n" $component $config $rundate $upload
+
+function run_depo_plotting() {
+  docker run -it -v "/media/scratch/lb584_scratch/projects/ews_aws/ews_3/code:/storage/app/EWS_prod/code" \
+    -v "/media/scratch/lb584_scratch/projects/ews_aws/ews_3/regions:/storage/app/EWS_prod/regions" \
+    -v "/media/scratch/lb584_scratch/projects/ews_aws/ews_3/envs/credentials:/storage/app/EWS_prod/envs/credentials" \
+    -w "/storage/app/EWS_prod/code" \
+    ews_coordinator_runner \
+    -p $component \
+    -c $config \
+    -s $rundate \
+    $upload
+}
+
+run_depo_plotting