From c479144ae0849ed7e4fca05a96708e738fac0fef Mon Sep 17 00:00:00 2001
From: lb584 <lb584@cam.ac.uk>
Date: Fri, 16 Dec 2022 10:51:36 +0000
Subject: [PATCH] adding some new docker config/images for running on IaaS
---
configs/docker/build/Dockerfile | 9 +-
configs/docker/build/RunnerDockerfile | 4 +
configs/docker/build/build.sh | 2 +-
.../build/conda-env-py3EWS-withbuilds.yml | 185 ------------------
configs/docker/run/launchDockerRunnerIaas.sh | 56 ++++++
5 files changed, 67 insertions(+), 189 deletions(-)
create mode 100644 configs/docker/build/RunnerDockerfile
delete mode 100644 configs/docker/build/conda-env-py3EWS-withbuilds.yml
create mode 100755 configs/docker/run/launchDockerRunnerIaas.sh
diff --git a/configs/docker/build/Dockerfile b/configs/docker/build/Dockerfile
index effc049..afe9953 100644
--- a/configs/docker/build/Dockerfile
+++ b/configs/docker/build/Dockerfile
@@ -1,5 +1,5 @@
# syntax=docker/dockerfile:1
-FROM ubuntu:18.04
+FROM ubuntu:18.04 as base_image
#create dirs
WORKDIR /home
@@ -8,7 +8,7 @@ RUN mkdir -p /storage/app/EWS_prod/regions/EastAfrica/workspace
RUN mkdir -p /storage/app/EWS_prod/regions/SouthAsia/workspace
#copy resources in
-COPY ./installRPackages.txt /home
+#COPY ./installRPackages.txt /home
COPY ./conda-env-py3EWS-withbuilds.yml .
#install system packages
@@ -37,4 +37,7 @@ ENV PATH=$CONDA_DIR/bin:$PATH
RUN conda env create --file conda-env-py3EWS-withbuilds.yml --prefix /storage/app/EWS_prod/envs/conda/py3EWS
#tidy
-RUN rm -v ~/miniconda.sh conda-env-py3EWS-withbuilds.yml;
\ No newline at end of file
+RUN rm -v ~/miniconda.sh conda-env-py3EWS-withbuilds.yml;
+
+FROM base_image as runner
+CMD ["echo", "hello world"]
\ No newline at end of file
diff --git a/configs/docker/build/RunnerDockerfile b/configs/docker/build/RunnerDockerfile
new file mode 100644
index 0000000..171dd3c
--- /dev/null
+++ b/configs/docker/build/RunnerDockerfile
@@ -0,0 +1,4 @@
+FROM ews_coordinator:latest
+
+ENTRYPOINT ["/storage/app/EWS_prod/code/coordinator/run_Processor.sh"]
+#CMD ["-p" "-c", "--noupload", "-s"]
\ No newline at end of file
diff --git a/configs/docker/build/build.sh b/configs/docker/build/build.sh
index 8c7e590..0d8837e 100644
--- a/configs/docker/build/build.sh
+++ b/configs/docker/build/build.sh
@@ -1,3 +1,3 @@
cp ../../../conda-env-py3EWS-withbuilds.yml .;
-sudo docker build -t ews_coordinator .;
+sudo docker build --target base_image -t ews_coordinator .;
rm -v conda-env-py3EWS-withbuilds.yml;
diff --git a/configs/docker/build/conda-env-py3EWS-withbuilds.yml b/configs/docker/build/conda-env-py3EWS-withbuilds.yml
deleted file mode 100644
index 91e247f..0000000
--- a/configs/docker/build/conda-env-py3EWS-withbuilds.yml
+++ /dev/null
@@ -1,185 +0,0 @@
-name: /storage/app/EWS/envs/conda/py3EWS
-channels:
- - conda-forge
- - defaults
-dependencies:
- - _libgcc_mutex=0.1=main
- - _openmp_mutex=4.5=1_gnu
- - affine=2.3.1=pyhd8ed1ab_0
- - antlr-python-runtime=4.7.2=py39hf3d152e_1003
- - attrs=21.4.0=pyhd8ed1ab_0
- - boost-cpp=1.74.0=h312852a_4
- - branca=0.5.0=pyhd8ed1ab_0
- - brotlipy=0.7.0=py39h27cfd23_1003
- - bzip2=1.0.8=h7f98852_4
- - c-ares=1.17.1=h7f98852_1
- - ca-certificates=2022.5.18=ha878542_0
- - cairo=1.16.0=h6cf1ce9_1008
- - cartopy=0.20.0=py39h2103e0b_0
- - certifi=2022.5.18=py39hf3d152e_0
- - cf-units=3.0.1=py39hce5d2b2_2
- - cffi=1.15.0=py39hd667e15_1
- - cfitsio=3.470=hb418390_7
- - cftime=1.6.0=py39hd257fcd_1
- - charset-normalizer=2.0.4=pyhd3eb1b0_0
- - click=7.1.2=pyh9f0ad1d_0
- - click-plugins=1.1.1=py_0
- - cligj=0.7.2=pyhd8ed1ab_1
- - cloudpickle=2.0.0=pyhd8ed1ab_0
- - conda=4.12.0=py39hf3d152e_0
- - conda-content-trust=0.1.1=pyhd3eb1b0_0
- - conda-package-handling=1.7.3=py39h27cfd23_1
- - contextily=1.0.1=py_0
- - cryptography=36.0.0=py39h9ce1e76_0
- - curl=7.78.0=hea6ffbf_0
- - cycler=0.11.0=pyhd8ed1ab_0
- - dask-core=2022.5.0=pyhd8ed1ab_0
- - expat=2.4.1=h9c3ff4c_0
- - fiona=1.8.20=py39h427c1bf_1
- - folium=0.12.1.post1=pyhd8ed1ab_1
- - fontconfig=2.13.1=hba837de_1005
- - freetype=2.10.4=h0708190_1
- - freexl=1.0.6=h7f98852_0
- - fsspec=2022.3.0=pyhd8ed1ab_0
- - gdal=3.3.1=py39h409cc32_1
- - geographiclib=1.52=pyhd8ed1ab_0
- - geopandas=0.10.2=pyhd8ed1ab_1
- - geopandas-base=0.10.2=pyha770c72_1
- - geopy=2.2.0=pyhd8ed1ab_0
- - geos=3.9.1=h9c3ff4c_2
- - geotiff=1.6.0=h4f31c25_6
- - gettext=0.19.8.1=h0b5b191_1005
- - giflib=5.2.1=h36c2ea0_2
- - hdf4=4.2.15=h10796ff_3
- - hdf5=1.10.6=nompi_h6a2412b_1114
- - icu=68.1=h58526e2_0
- - idna=3.3=pyhd3eb1b0_0
- - iris=3.2.1=pyhd8ed1ab_0
- - jbig=2.1=h7f98852_2003
- - jinja2=3.1.2=pyhd8ed1ab_0
- - joblib=1.1.0=pyhd8ed1ab_0
- - jpeg=9d=h36c2ea0_0
- - json-c=0.15=h98cffda_0
- - kealib=1.4.14=hcc255d8_2
- - kiwisolver=1.3.1=py39h1a9c180_1
- - krb5=1.19.2=hcc1bbae_0
- - lcms2=2.12=hddcbb42_0
- - ld_impl_linux-64=2.35.1=h7274673_9
- - lerc=2.2.1=h9c3ff4c_0
- - libblas=3.9.0=11_linux64_openblas
- - libcblas=3.9.0=11_linux64_openblas
- - libcurl=7.78.0=h2574ce0_0
- - libdap4=3.20.6=hd7c4107_2
- - libdeflate=1.7=h7f98852_5
- - libedit=3.1.20191231=he28a2e2_2
- - libev=4.33=h516909a_1
- - libffi=3.3=he6710b0_2
- - libgcc-ng=11.2.0=h1234567_0
- - libgdal=3.3.1=h8f005ca_1
- - libgfortran-ng=12.1.0=h69a702a_16
- - libgfortran5=12.1.0=hdcd56e2_16
- - libglib=2.68.3=h3e27bee_0
- - libgomp=11.2.0=h1234567_0
- - libiconv=1.16=h516909a_0
- - libkml=1.3.0=h238a007_1014
- - liblapack=3.9.0=11_linux64_openblas
- - libnetcdf=4.8.0=nompi_hcd642e3_103
- - libnghttp2=1.43.0=h812cca2_0
- - libopenblas=0.3.17=pthreads_h8fe5266_1
- - libpng=1.6.37=h21135ba_2
- - libpq=13.3=hd57d9b9_0
- - librttopo=1.1.0=h1185371_6
- - libspatialindex=1.9.3=h9c3ff4c_4
- - libspatialite=5.0.1=h8694cbe_5
- - libssh2=1.9.0=ha56f1ee_6
- - libstdcxx-ng=12.1.0=ha89aaad_16
- - libtiff=4.3.0=hf544144_1
- - libuuid=2.32.1=h7f98852_1000
- - libwebp-base=1.2.0=h7f98852_2
- - libxcb=1.13=h7f98852_1003
- - libxml2=2.9.12=h72842e0_0
- - libxslt=1.1.33=h15afd5d_2
- - libzip=1.8.0=h4de3113_0
- - locket=1.0.0=pyhd8ed1ab_0
- - lxml=4.6.3=py39h107f48f_0
- - lz4-c=1.9.3=h9c3ff4c_1
- - mapclassify=2.4.3=pyhd8ed1ab_0
- - markupsafe=2.0.1=py39h3811e60_0
- - matplotlib-base=3.4.2=py39h2fa2bec_0
- - mercantile=1.2.1=pyhd8ed1ab_0
- - munch=2.5.0=py_0
- - ncurses=6.3=h7f8727e_2
- - netcdf4=1.5.7=nompi_py39hc6dca20_100
- - networkx=2.8.1=pyhd8ed1ab_0
- - numpy=1.21.1=py39hdbf815f_0
- - olefile=0.46=pyh9f0ad1d_1
- - openjpeg=2.4.0=hb52868f_1
- - openssl=1.1.1o=h166bdaf_0
- - packaging=21.3=pyhd8ed1ab_0
- - pandas=1.3.1=py39hde0f152_0
- - partd=1.2.0=pyhd8ed1ab_0
- - pcre=8.45=h9c3ff4c_0
- - pillow=8.3.1=py39ha612740_0
- - pip=21.2.4=py39h06a4308_0
- - pixman=0.40.0=h36c2ea0_0
- - poppler=21.03.0=h93df280_0
- - poppler-data=0.4.11=hd8ed1ab_0
- - postgresql=13.3=h2510834_0
- - proj=8.0.1=h277dcde_0
- - pthread-stubs=0.4=h36c2ea0_1001
- - pycosat=0.6.3=py39h27cfd23_0
- - pycparser=2.21=pyhd3eb1b0_0
- - pyopenssl=21.0.0=pyhd3eb1b0_1
- - pyparsing=3.0.9=pyhd8ed1ab_0
- - pyproj=3.1.0=py39ha9a7ae0_3
- - pyshp=2.3.0=pyhd8ed1ab_0
- - pysocks=1.7.1=py39h06a4308_0
- - python=3.9.7=h12debd9_1
- - python-dateutil=2.8.2=pyhd8ed1ab_0
- - python-docx=0.8.11=pyhd8ed1ab_0
- - python-xxhash=3.0.0=py39hb9d737c_1
- - python_abi=3.9=2_cp39
- - pytz=2022.1=pyhd8ed1ab_0
- - pyyaml=6.0=py39hb9d737c_4
- - rasterio=1.2.6=py39hbc4e497_2
- - readline=8.1.2=h7f8727e_1
- - requests=2.27.1=pyhd3eb1b0_0
- - rtree=1.0.0=py39hb102c33_1
- - ruamel_yaml=0.15.100=py39h27cfd23_0
- - scikit-learn=0.24.2=py39h4dfa638_0
- - scipy=1.7.0=py39hee8e79c_1
- - setuptools=58.0.4=py39h06a4308_0
- - shapely=1.7.1=py39ha61afbd_5
- - six=1.16.0=pyhd3eb1b0_0
- - snuggs=1.4.7=py_0
- - sqlite=3.37.0=hc218d9a_0
- - threadpoolctl=3.1.0=pyh8a188c0_0
- - tiledb=2.3.2=he87e0bf_0
- - tk=8.6.11=h1ccaba5_0
- - toolz=0.11.2=pyhd8ed1ab_0
- - tornado=6.1=py39h3811e60_1
- - tqdm=4.62.3=pyhd3eb1b0_1
- - tzcode=2021a=h7f98852_2
- - tzdata=2021e=hda174b7_0
- - udunits2=2.2.27.27=hc3e0081_3
- - urllib3=1.26.7=pyhd3eb1b0_0
- - wheel=0.37.1=pyhd3eb1b0_0
- - xerces-c=3.2.3=h9d8b166_2
- - xorg-kbproto=1.0.7=h7f98852_1002
- - xorg-libice=1.0.10=h7f98852_0
- - xorg-libsm=1.2.3=hd9c2040_1000
- - xorg-libx11=1.7.2=h7f98852_0
- - xorg-libxau=1.0.9=h7f98852_0
- - xorg-libxdmcp=1.1.3=h7f98852_0
- - xorg-libxext=1.3.4=h7f98852_1
- - xorg-libxrender=0.9.10=h7f98852_1003
- - xorg-renderproto=0.11.1=h7f98852_1002
- - xorg-xextproto=7.3.0=h7f98852_1002
- - xorg-xproto=7.0.31=h7f98852_1007
- - xxhash=0.8.0=h7f98852_3
- - xyzservices=2022.4.0=pyhd8ed1ab_0
- - xz=5.2.5=h7b6447c_0
- - yaml=0.2.5=h7b6447c_0
- - zlib=1.2.11=h7f8727e_4
- - zstd=1.5.0=ha95c52a_0
-prefix: /storage/app/EWS/envs/conda/py3EWS
diff --git a/configs/docker/run/launchDockerRunnerIaas.sh b/configs/docker/run/launchDockerRunnerIaas.sh
new file mode 100755
index 0000000..63d8ad9
--- /dev/null
+++ b/configs/docker/run/launchDockerRunnerIaas.sh
@@ -0,0 +1,56 @@
+#!/bin/bash
+set -e
+
+upload='--noupload' # default
+rundate=$(date '+%Y%m%d') # default today
+
+SHORT=p:,c:,s:,h
+LONG=component:,config:,rundate::,help
+OPTS=$(getopt -a -n weather --options $SHORT --longoptions $LONG -- "$@")
+
+eval set -- "$OPTS"
+
+while :
+do
+ case "$1" in
+ -p | --component )
+ component="$2"
+ shift 2
+ ;;
+ -c | --config )
+ config="$2"
+ shift 2
+ ;;
+ -s | --rundate )
+ rundate="$2"
+ shift 2
+ ;;
+ -h | --help)
+ "This is a weather script"
+ exit 2
+ ;;
+ --)
+ shift;
+ break
+ ;;
+ *)
+ echo "Unexpected option: $1"
+ ;;
+ esac
+done
+
+printf "component: %s\nconfig: %s\nrundate: %s\nupload: %s\n" $component $config $rundate $upload
+
+function run_depo_plotting() {
+ docker run -it -v "/media/scratch/lb584_scratch/projects/ews_aws/ews_3/code:/storage/app/EWS_prod/code" \
+ -v "/media/scratch/lb584_scratch/projects/ews_aws/ews_3/regions:/storage/app/EWS_prod/regions" \
+ -v "/media/scratch/lb584_scratch/projects/ews_aws/ews_3/envs/credentials:/storage/app/EWS_prod/envs/credentials" \
+ -w "/storage/app/EWS_prod/code" \
+ ews_coordinator_runner \
+ -p $component \
+ -c $config \
+ -s $rundate \
+ $upload
+}
+
+run_depo_plotting
--
GitLab