changes to desktop configs

configs/docker/build/Dockerfile

@@ -16,7 +16,7 @@ ARG GNAME=ews
 ARG UNAME=ewsmanager
 ARG UID=1006
 ARG GID=1008
-RUN groupadd -g $GID $GNAME
+#RUN groupadd -g $GID $GNAME
 RUN useradd -m -u $UID -g $GID -o -s /bin/bash $UNAME
 
 #run mkdir as root

configs/docker/build/Dockerfile~

+# syntax=docker/dockerfile:1
+FROM ubuntu:24.04 as base_image
+
+SHELL ["/bin/bash", "-c"]
+
+#install system packages
+#RUN apt-get update;apt-get install --assume-yes python3.11 python3.11-venv wget unzip emacs gdal-bin libgdal-dev build-essential gfortran zlib1g zlib1g-dev libbz2-dev apt-utils liblzma-dev libcurl4-openssl-dev default-jre openssh-server libfribidi-dev libfontconfig1-dev libspatialindex-dev git dos2unix nano rsync;
+RUN echo "alias e='emacs -nw $1'" >> ~/.bashrc;
+RUN alias ll="ls -ltr"
+
+#sets the timezone to London
+#RUN DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt-get -y install tzdata
+
+#these usernames and ids match the production server (IaaS), override when building a local image if needed
+ARG GNAME=ews
+ARG UNAME=ewsmanager
+ARG UID=1006
+ARG GID=1008
+#RUN groupadd -g $GID $GNAME
+#RUN useradd -m -u $UID -g $GID -o -s /bin/bash $UNAME
+
+#run mkdir as root
+RUN mkdir -p /storage/app/EWS_prod/regions/EastAfrica/workspace
+RUN mkdir -p /storage/app/EWS_prod/regions/SouthAsia/workspace
+
+#RUN chown -R $UNAME /storage
+
+#swith to the user before installing stuff
+#USER $UNAME
+
+WORKDIR /home
+
+#copy resources in
+COPY ./requirements.txt .
+
+#build the venv
+#RUN python3.11 -m venv /storage/app/EWS_prod/envs/venv/ews_prod
+#RUN source /storage/app/EWS_prod/envs/venv/ews_prod/bin/activate;pip install -r requirements.txt
+
+RUN echo $PYTHONPATH
+
+WORKDIR /home/ewsmanager

configs/docker/build/conda-env-py3EWS-nobuilds.yml

+name: /storage/app/EWS/envs/conda/py3EWS
+channels:
+  - conda-forge
+  - defaults
+dependencies:
+  - _libgcc_mutex=0.1
+  - _openmp_mutex=4.5
+  - affine=2.3.1
+  - antlr-python-runtime=4.7.2
+  - attrs=21.4.0
+  - boost-cpp=1.74.0
+  - branca=0.5.0
+  - brotlipy=0.7.0
+  - bzip2=1.0.8
+  - c-ares=1.17.1
+  - ca-certificates=2022.5.18
+  - cairo=1.16.0
+  - cartopy=0.20.0
+  - certifi=2022.5.18
+  - cf-units=3.0.1
+  - cffi=1.15.0
+  - cfitsio=3.470
+  - cftime=1.6.0
+  - charset-normalizer=2.0.4
+  - click=7.1.2
+  - click-plugins=1.1.1
+  - cligj=0.7.2
+  - cloudpickle=2.0.0
+  - conda=4.12.0
+  - conda-content-trust=0.1.1
+  - conda-package-handling=1.7.3
+  - contextily=1.0.1
+  - cryptography=36.0.0
+  - curl=7.78.0
+  - cycler=0.11.0
+  - dask-core=2022.5.0
+  - expat=2.4.1
+  - fiona=1.8.20
+  - folium=0.12.1.post1
+  - fontconfig=2.13.1
+  - freetype=2.10.4
+  - freexl=1.0.6
+  - fsspec=2022.3.0
+  - gdal=3.3.1
+  - geographiclib=1.52
+  - geopandas=0.10.2
+  - geopandas-base=0.10.2
+  - geopy=2.2.0
+  - geos=3.9.1
+  - geotiff=1.6.0
+  - gettext=0.19.8.1
+  - giflib=5.2.1
+  - hdf4=4.2.15
+  - hdf5=1.10.6
+  - icu=68.1
+  - idna=3.3
+  - iris=3.2.1
+  - jbig=2.1
+  - jinja2=3.1.2
+  - joblib=1.1.0
+  - jpeg=9d
+  - json-c=0.15
+  - kealib=1.4.14
+  - kiwisolver=1.3.1
+  - krb5=1.19.2
+  - lcms2=2.12
+  - ld_impl_linux-64=2.35.1
+  - lerc=2.2.1
+  - libblas=3.9.0
+  - libcblas=3.9.0
+  - libcurl=7.78.0
+  - libdap4=3.20.6
+  - libdeflate=1.7
+  - libedit=3.1.20191231
+  - libev=4.33
+  - libffi=3.3
+  - libgcc-ng=11.2.0
+  - libgdal=3.3.1
+  - libgfortran-ng=12.1.0
+  - libgfortran5=12.1.0
+  - libglib=2.68.3
+  - libgomp=11.2.0
+  - libiconv=1.16
+  - libkml=1.3.0
+  - liblapack=3.9.0
+  - libnetcdf=4.8.0
+  - libnghttp2=1.43.0
+  - libopenblas=0.3.17
+  - libpng=1.6.37
+  - libpq=13.3
+  - librttopo=1.1.0
+  - libspatialindex=1.9.3
+  - libspatialite=5.0.1
+  - libssh2=1.9.0
+  - libstdcxx-ng=12.1.0
+  - libtiff=4.3.0
+  - libuuid=2.32.1
+  - libwebp-base=1.2.0
+  - libxcb=1.13
+  - libxml2=2.9.12
+  - libxslt=1.1.33
+  - libzip=1.8.0
+  - locket=1.0.0
+  - lxml=4.6.3
+  - lz4-c=1.9.3
+  - mapclassify=2.4.3
+  - markupsafe=2.0.1
+  - matplotlib-base=3.4.2
+  - mercantile=1.2.1
+  - munch=2.5.0
+  - ncurses=6.3
+  - netcdf4=1.5.7
+  - networkx=2.8.1
+  - numpy=1.22
+  - olefile=0.46
+  - openjpeg=2.4.0
+  - openssl=1.1.1o
+  - packaging=21.3
+  - pandas=1.4.2
+  - partd=1.2.0
+  - pcre=8.45
+  - pillow=8.3.1
+  - pip=21.2.4
+  - pixman=0.40.0
+  - poppler=21.03.0
+  - poppler-data=0.4.11
+  - postgresql=13.3
+  - proj=8.0.1
+  - pthread-stubs=0.4
+  - pycosat=0.6.3
+  - pycparser=2.21
+  - pyopenssl=21.0.0
+  - pyparsing=3.0.9
+  - pyproj=3.1.0
+  - pyshp=2.3.0
+  - pysocks=1.7.1
+  - python=3.9.7
+  - python-dateutil=2.8.2
+  - python-docx=0.8.11
+  - python-xxhash=3.0.0
+  - python_abi=3.9
+  - pytz=2022.1
+  - pyyaml=6.0
+  - rasterio=1.2.6
+  - readline=8.1.2
+  - requests=2.27.1
+  - rtree=1.0.0
+  - ruamel_yaml=0.15.100
+  - scikit-learn=0.24.2
+  - scipy=1.7.0
+  - setuptools=58.0.4
+  - shapely=1.7.1
+  - six=1.16.0
+  - snuggs=1.4.7
+  - sqlite=3.37.0
+  - threadpoolctl=3.1.0
+  - tiledb=2.3.2
+  - tk=8.6.11
+  - toolz=0.11.2
+  - tornado=6.1
+  - tqdm=4.62.3
+  - tzcode=2021a
+  - tzdata=2021e
+  - udunits2=2.2.27.27
+  - urllib3=1.26.7
+  - wheel=0.37.1
+  - xerces-c=3.2.3
+  - xorg-kbproto=1.0.7
+  - xorg-libice=1.0.10
+  - xorg-libsm=1.2.3
+  - xorg-libx11=1.7.2
+  - xorg-libxau=1.0.9
+  - xorg-libxdmcp=1.1.3
+  - xorg-libxext=1.3.4
+  - xorg-libxrender=0.9.10
+  - xorg-renderproto=0.11.1
+  - xorg-xextproto=7.3.0
+  - xorg-xproto=7.0.31
+  - xxhash=0.8.0
+  - xyzservices=2022.4.0
+  - xz=5.2.5
+  - yaml=0.2.5
+  - zlib=1.2.11
+  - zstd=1.5.0
+  - pip:
+      - aiohttp==3.8.5
+      - aiosignal==1.3.1
+      - alembic==1.11.3
+      - anyio==3.7.1
+      - apache-airflow==2.7.0
+      - apache-airflow-providers-cncf-kubernetes==7.4.2
+      - apache-airflow-providers-common-sql==1.7.0
+      - apache-airflow-providers-ftp==3.5.0
+      - apache-airflow-providers-http==4.5.0
+      - apache-airflow-providers-imap==3.3.0
+      - apache-airflow-providers-sqlite==3.4.3
+      - apispec==6.3.0
+      - argcomplete==3.1.1
+      - asgiref==3.7.2
+      - astroid==2.15.5
+      - async-timeout==4.0.3
+      - attrs==23.1.0
+      - backoff==1.10.0
+      - blinker==1.6.2
+      - cachelib==0.9.0
+      - cachetools==5.3.1
+      - cattrs==23.1.2
+      - cffi==1.15.1
+      - charset-normalizer==3.2.0
+      - click==8.1.7
+      - clickclick==20.10.2
+      - colorama==0.4.6
+      - colorlog==4.8.0
+      - configupdater==3.1.1
+      - connexion==2.14.2
+      - coverage==6.5.0
+      - cron-descriptor==1.4.0
+      - croniter==1.4.1
+      - cryptography==41.0.3
+      - deprecated==1.2.14
+      - dill==0.3.1.1
+      - dnspython==2.4.2
+      - docutils
+      - email-validator==1.3.1
+      - exceptiongroup==1.1.3
+      - flask==2.2.5
+      - flask-appbuilder==4.3.3
+      - flask-babel==2.0.0
+      - flask-caching==2.0.2
+      - flask-jwt-extended==4.5.2
+      - flask-limiter==3.3.1
+      - flask-login==0.6.2
+      - flask-session==0.5.0
+      - flask-sqlalchemy==2.5.1
+      - flask-wtf==1.1.1
+      - frozenlist==1.4.0
+      - google-auth==2.22.0
+      - google-re2==1.1
+      - googleapis-common-protos==1.60.0
+      - greenlet==2.0.2
+      - grpcio==1.57.0
+      - gunicorn==21.2.0
+      - h11==0.14.0
+      - html-testrunner==1.2.1
+      - httpcore==0.16.3
+      - httpx==0.23.3
+      - idna==3.4
+      - importlib-metadata==6.8.0
+      - importlib-resources==6.0.1
+      - inflection==0.5.1
+      - isort==5.12.0
+      - itsdangerous==2.1.2
+      - jsonschema==4.19.0
+      - jsonschema-specifications==2023.7.1
+      - kubernetes==23.6.0
+      - kubernetes-asyncio==24.2.3
+      - lazy-object-proxy==1.9.0
+      - limits==3.5.0
+      - linkify-it-py==2.0.2
+      - lockfile==0.12.2
+      - mako==1.2.4
+      - markdown==3.4.4
+      - markdown-it-py==3.0.0
+      - markupsafe==2.1.3
+      - marshmallow==3.20.1
+      - marshmallow-oneofschema==3.0.1
+      - marshmallow-sqlalchemy==0.26.1
+      - mccabe==0.7.0
+      - mdit-py-plugins==0.4.0
+      - mdurl==0.1.2
+      - multidict==6.0.4
+      - numpy==1.22
+      - oauthlib==3.2.2
+      - opentelemetry-api==1.15.0
+      - opentelemetry-exporter-otlp==1.15.0
+      - opentelemetry-exporter-otlp-proto-grpc==1.15.0
+      - opentelemetry-exporter-otlp-proto-http==1.15.0
+      - opentelemetry-proto==1.15.0
+      - opentelemetry-sdk==1.15.0
+      - opentelemetry-semantic-conventions==0.36b0
+      - ordered-set==4.1.0
+      - packaging==23.1
+      - pandas==1.4.2
+      - pathspec==0.11.2
+      - pbr==5.11.1
+      - pendulum==2.1.2
+      - platformdirs==3.5.1
+      - pluggy==1.2.0
+      - prison==0.2.1
+      - protobuf==4.21.12
+      - psutil==5.9.5
+      - pyasn1==0.5.0
+      - pyasn1-modules==0.3.0
+      - pydantic==1.10.12
+      - pygments==2.16.1
+      - pyjwt==2.8.0
+      - pylint==2.17.4
+      - pyqt5-sip
+      - pyqtchart==5.12
+      - pyqtwebengine==5.12.1
+      - python-daemon==3.0.1
+      - python-nvd3==0.15.0
+      - python-slugify==8.0.1
+      - pytz==2023.3
+      - pytzdata==2020.1
+      - pyyaml==6.0.1
+      - referencing==0.30.2
+      - requests==2.31.0
+      - requests-oauthlib==1.3.1
+      - requests-toolbelt==1.0.0
+      - rfc3339-validator==0.1.4
+      - rfc3986==1.5.0
+      - rich==13.5.2
+      - rich-argparse==1.2.0
+      - rioxarray==0.15.0
+      - rpds-py==0.9.2
+      - rsa==4.9
+      - setproctitle==1.3.2
+      - setuptools==68.1.2
+      - sniffio==1.3.0
+      - sphinx==6.2.1
+      - sphinx-autodoc-typehints==1.23.0
+      - sphinx-rtd-theme==1.2.1
+      - sphinxcontrib-jquery==4.1
+      - sqlalchemy==1.4.49
+      - sqlalchemy-jsonfield==1.0.1.post0
+      - sqlalchemy-utils==0.41.1
+      - sqlparse==0.4.4
+      - tabulate==0.9.0
+      - tenacity==8.2.3
+      - termcolor==2.3.0
+      - testresources==2.0.1
+      - text-unidecode==1.3
+      - tomli==2.0.1
+      - tomlkit==0.11.8
+      - typing-extensions==4.7.1
+      - uc-micro-py==1.0.2
+      - unicodecsv==0.14.1
+      - urllib3==1.26.16
+      - websocket-client==1.6.2
+      - werkzeug==2.2.3
+      - wrapt==1.15.0
+      - wtforms==3.0.1
+      - xarray==2023.11.0
+      - yarl==1.9.2
+      - zipp==3.16.2
+prefix: /storage/app/EWS/envs/conda/py3EWS

configs/docker/run/launchDocker_desktop.sh

+docker run -it -v "/media/scratch/lb584_scratch/projects/ews_aws/code:/storage/app/EWS_prod/code" -v "/media/scratch/lb584_scratch/projects/ews_aws/regions:/storage/app/EWS_prod/regions" -v "/media/scratch/lb584_scratch/projects/ews_aws/envs/credentials:/storage/app/EWS_prod/envs/credentials" -v "/media/scratch/lb584_scratch/projects/ews_aws/sftp:/storage/sftp" -v "/media/scratch/lb584_scratch/projects/ews_aws/webdata:/storage/moved/" -w "/storage/app/EWS_prod/code/"  --name ews_runner ews_coordinator

configs/docker/run/runDockerDesktop.sh

+#!/bin/bash
+set -e
+
+upload=''
+islive=''
+clearup=''
+component='not_set'
+config='/media/scratch/lb584_scratch/projects/ews_aws/resources/coordinator/configs/config_EastAfrica_fc_live.json' # default
+rundate=$(date '+%Y%m%d') # default today
+
+
+SHORT=p:c:s:h
+LONG=component:,config:,rundate:,noupload::,islive::,clearup::,help
+OPTS=$(getopt -a --options $SHORT --longoptions $LONG -- "$@")
+echo $OPTS
+eval set -- "$OPTS"
+
+while :
+do
+  case "$1" in
+    -p | --component )
+      component="$2"
+      shift 2
+      ;;
+    -c | --config )
+      config="$2"
+      shift 2
+      ;;
+    -s | --rundate )
+      rundate="$2"
+      shift 2
+      ;;
+    --noupload )
+      upload="--noupload"
+      shift 2;
+      ;;
+    --islive )
+      islive="--islive"
+      shift 2;
+      ;;
+    --clearup )
+      clearup="--clearup"
+      shift 2;
+      ;;
+    -h | --help)
+      "This is a launch script, example usage: runDockerEWS.sh -p Deposition -c -c /<path>/config_EastAfrica_fc_live.json -s 20220808"
+      exit 2
+      ;;
+    --)
+      shift;
+      break
+      ;;
+    *)
+      echo "Unexpected option: $1"
+      ;;
+  esac
+done
+
+
+
+function run_coordinator() {
+    printf "component: %s\nconfig: %s\nrundate: %s\n" "$component" "$config" "$rundate"
+    printf "optional args:\n%s\n" "$upload $islive $clearup"
+    docker container ls;
+    docker exec \
+    ews_runner \
+    /storage/app/EWS_prod/code/coordinator/scripts/run_Processor.sh \
+    -p "$component" \
+    -c "$config" \
+    -s "$rundate" \
+    $upload \
+    $islive \
+    $clearup;
+    exit $?;
+}
+
+run_coordinator
+exit $?

configs/docker/run/runDockerDesktop.sh~

+#!/bin/bash
+set -e
+
+upload=''
+islive=''
+clearup=''
+component='not_set'
+config='/media/scratch/lb584_scratch/projects/ews_aws/resources/coordinator/configs/config_EastAfrica_fc_live.json' # default
+rundate=$(date '+%Y%m%d') # default today
+
+
+SHORT=p:c:s:h
+LONG=component:,config:,rundate:,noupload::,islive::,clearup::,help
+OPTS=$(getopt -a --options $SHORT --longoptions $LONG -- "$@")
+echo $OPTS
+eval set -- "$OPTS"
+
+while :
+do
+  case "$1" in
+    -p | --component )
+      component="$2"
+      shift 2
+      ;;
+    -c | --config )
+      config="$2"
+      shift 2
+      ;;
+    -s | --rundate )
+      rundate="$2"
+      shift 2
+      ;;
+    --noupload )
+      upload="--noupload"
+      shift 2;
+      ;;
+    --islive )
+      islive="--islive"
+      shift 2;
+      ;;
+    --clearup )
+      clearup="--clearup"
+      shift 2;
+      ;;
+    -h | --help)
+      "This is a launch script, example usage: runDockerEWS.sh -p Deposition -c -c /<path>/config_EastAfrica_fc_live.json -s 20220808"
+      exit 2
+      ;;
+    --)
+      shift;
+      break
+      ;;
+    *)
+      echo "Unexpected option: $1"
+      ;;
+  esac
+done
+
+
+
+function run_coordinator() {
+    printf "component: %s\nconfig: %s\nrundate: %s\n" "$component" "$config" "$rundate"
+    printf "optional args:\n%s\n" "$upload $islive $clearup"
+    docker container ls;
+    docker exec \
+    ews_runner \
+    sh -c /storage/app/EWS_prod/code/coordinator/scripts/run_Processor.sh \
+    -p "$component" \
+    -c "$config" \
+    -s "$rundate" \
+    $upload \
+    $islive \
+    $clearup;
+    exit $?;
+}
+
+run_coordinator
+exit $?