minor edits to system out etc

.gitlab-ci.yml

@@ -7,7 +7,6 @@ variables:
 
   OUTPUT_DIR: "$CI_PROJECT_DIR/outputs/"
   LOGS_DIR: "$CI_PROJECT_DIR/logs/"
-#  OUTPUT_DIR: "/storage/app/EWS_prod/regions/EastAfrica/workspace/"
   TEST_WORKSPACE_DIR: "/storage/app/EWS_prod/regions/EastAfrica/test_workspace/"
   FULL_FAT_RUN_DATE:
     value: "default"

docs/_source/iaas_setup.rst

@@ -535,45 +535,88 @@ cd airflow
 set the airflow home location
 export AIRFLOW_HOME=<path>/airflow
 
-python3 -m venv airflow-env
-source airflow-env/bin/activate
+python3 -m venv airflow-env;
+source airflow-env/bin/activate;
 
-pip install "apache-airflow[celery]==2.7.1" --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-2.7.1/constraints-3.8.txt"
+pip install wheel;
+pip install apache-airflow[celery]==2.7.1 --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-2.7.1/constraints-3.8.txt"
+
+note that there are two dashes before constraint above, not a single long dash
 
 pip check
 
-install docker operator
+**install the docker operator**
 
 pip install apache-airflow-providers-docker
 
-Setup mysql database:
+**install mysql (need root privileges)**
+
+https://www.digitalocean.com/community/tutorials/how-to-install-mysql-on-ubuntu-20-04
+
+sudo apt update;
+sudo apt install mysql-server;
+sudo systemctl start mysql.service;
+
+follow the additional instructions to work around the mysql_secure_installation issue
+
+sudo mysql;
+ALTER USER 'root'@'localhost' IDENTIFIED WITH mysql_native_password BY '<put_root_password_here>';
+exit;
+
+The password above has been set to the same as the ewsmanager password. You can now run the security manager script:
+
+sudo mysql_secure_installation;
+
+<choose sensible security options at your discretion>
+
+**Setup mysql database:**
 
 mysql -u root -p
 
 CREATE DATABASE airflow_db CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
-CREATE USER 'airflow_user' IDENTIFIED BY 'airflow_pass';
+
+CREATE USER 'airflow_user'@'localhost' IDENTIFIED BY 'generate_a_non_secret_password_for_here';
+
 GRANT ALL PRIVILEGES ON airflow_db.* TO 'airflow_user';
+
 exit;
 
 
-set database defaults (sqllite)
+**set database defaults (sqllite)**
+
 airflow db migrate
 
 edit the resulting ${AIRFLOW_HOME}/airflow.cfg file on line 435 to the mysyl database
-sql_alchemy_conn = mysql+mysqldb://airflow_user:airflow_pass@localhost:3306/airflow_db
+sql_alchemy_conn = mysql+mysqldb://airflow_user:<password_from_above>@localhost:3306/airflow_db
+
+**install the mysql provider**
+
+sudo apt install gcc;
+sudo apt-get install python3-dev;
+pip install apache-airflow-providers-mysql
+
+the above may fail if you dont specify the mysql headers first as in this post:
 
-install the mysql provider
-pip3 install apache-airflow-providers-mysql
+https://stackoverflow.com/questions/76875507/can-not-install-apache-airflow-providers-mysql-pkg-config-error
+
+sudo apt install libmysqlclient-dev;
+export MYSQLCLIENT_CFLAGS="$(mysql_config --cflags)";
+export MYSQLCLIENT_LDFLAGS="$(mysql_config --libs)";
+
+**migrate the db again to use mysql this time**
 
-migrate the db again to use mysql this time
 airflow db migrate
 
 possible sql mode error:
+
 https://stackoverflow.com/questions/36882149/error-1067-42000-invalid-default-value-for-created-at
+
 create an admin user for the webserver
 
 airflow users  create --role Admin --username admin --email lb584@cam.ac.uk --firstname admin --lastname admin --password admin
 
+(you can set the password to something more secure when logged in)
+
 run the scheduler and the webserver
 
 airflow scheduler;
@@ -581,6 +624,6 @@ airflow webserver;
 
 can run as demons with the -D flag
 
-connect to the webserver (note you will only be able to connect on localhost by default)
+connect to the webserver http://<server_ip>:8090
 
 

docs/api.rst

@@ -6,7 +6,7 @@ API
    :template: custom-module-template.rst
    :recursive:
 
-..    coordinator
+    coordinator
     plotting
     ews_postprocessing
     met_processing

scripts/run_Deposition_Processor.sh

@@ -7,6 +7,8 @@ source "${proc_path}"/run_utils.sh
 
 setup_conda_env
 
+echo "Running ProcessorDeposition.py with path: "
+pwd;
 python "${proc_path}"/../coordinator/ProcessorDeposition.py "$@"
 exit_code=$?