From c1105a99d176a693d141aab697b581c99f6b782f Mon Sep 17 00:00:00 2001
From: lb584 <lb584@cam.ac.uk>
Date: Tue, 13 Feb 2024 10:41:19 +0000
Subject: [PATCH] minor edits to system out etc
---
.gitlab-ci.yml | 1 -
docs/_source/iaas_setup.rst | 67 +++++++++++++++++++++++------
docs/api.rst | 2 +-
scripts/run_Deposition_Processor.sh | 2 +
4 files changed, 58 insertions(+), 14 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index acee70e..9a40ef8 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -7,7 +7,6 @@ variables:
OUTPUT_DIR: "$CI_PROJECT_DIR/outputs/"
LOGS_DIR: "$CI_PROJECT_DIR/logs/"
-# OUTPUT_DIR: "/storage/app/EWS_prod/regions/EastAfrica/workspace/"
TEST_WORKSPACE_DIR: "/storage/app/EWS_prod/regions/EastAfrica/test_workspace/"
FULL_FAT_RUN_DATE:
value: "default"
diff --git a/docs/_source/iaas_setup.rst b/docs/_source/iaas_setup.rst
index 23524f4..fa90fcb 100644
--- a/docs/_source/iaas_setup.rst
+++ b/docs/_source/iaas_setup.rst
@@ -535,45 +535,88 @@ cd airflow
set the airflow home location
export AIRFLOW_HOME=<path>/airflow
-python3 -m venv airflow-env
-source airflow-env/bin/activate
+python3 -m venv airflow-env;
+source airflow-env/bin/activate;
-pip install "apache-airflow[celery]==2.7.1" --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-2.7.1/constraints-3.8.txt"
+pip install wheel;
+pip install apache-airflow[celery]==2.7.1 --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-2.7.1/constraints-3.8.txt"
+
+note that there are two dashes before constraint above, not a single long dash
pip check
-install docker operator
+**install the docker operator**
pip install apache-airflow-providers-docker
-Setup mysql database:
+**install mysql (need root privileges)**
+
+https://www.digitalocean.com/community/tutorials/how-to-install-mysql-on-ubuntu-20-04
+
+sudo apt update;
+sudo apt install mysql-server;
+sudo systemctl start mysql.service;
+
+follow the additional instructions to work around the mysql_secure_installation issue
+
+sudo mysql;
+ALTER USER 'root'@'localhost' IDENTIFIED WITH mysql_native_password BY '<put_root_password_here>';
+exit;
+
+The password above has been set to the same as the ewsmanager password. You can now run the security manager script:
+
+sudo mysql_secure_installation;
+
+<choose sensible security options at your discretion>
+
+**Setup mysql database:**
mysql -u root -p
CREATE DATABASE airflow_db CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
-CREATE USER 'airflow_user' IDENTIFIED BY 'airflow_pass';
+
+CREATE USER 'airflow_user'@'localhost' IDENTIFIED BY 'generate_a_non_secret_password_for_here';
+
GRANT ALL PRIVILEGES ON airflow_db.* TO 'airflow_user';
+
exit;
-set database defaults (sqllite)
+**set database defaults (sqllite)**
+
airflow db migrate
edit the resulting ${AIRFLOW_HOME}/airflow.cfg file on line 435 to the mysyl database
-sql_alchemy_conn = mysql+mysqldb://airflow_user:airflow_pass@localhost:3306/airflow_db
+sql_alchemy_conn = mysql+mysqldb://airflow_user:<password_from_above>@localhost:3306/airflow_db
+
+**install the mysql provider**
+
+sudo apt install gcc;
+sudo apt-get install python3-dev;
+pip install apache-airflow-providers-mysql
+
+the above may fail if you dont specify the mysql headers first as in this post:
-install the mysql provider
-pip3 install apache-airflow-providers-mysql
+https://stackoverflow.com/questions/76875507/can-not-install-apache-airflow-providers-mysql-pkg-config-error
+
+sudo apt install libmysqlclient-dev;
+export MYSQLCLIENT_CFLAGS="$(mysql_config --cflags)";
+export MYSQLCLIENT_LDFLAGS="$(mysql_config --libs)";
+
+**migrate the db again to use mysql this time**
-migrate the db again to use mysql this time
airflow db migrate
possible sql mode error:
+
https://stackoverflow.com/questions/36882149/error-1067-42000-invalid-default-value-for-created-at
+
create an admin user for the webserver
airflow users create --role Admin --username admin --email lb584@cam.ac.uk --firstname admin --lastname admin --password admin
+(you can set the password to something more secure when logged in)
+
run the scheduler and the webserver
airflow scheduler;
@@ -581,6 +624,6 @@ airflow webserver;
can run as demons with the -D flag
-connect to the webserver (note you will only be able to connect on localhost by default)
+connect to the webserver http://<server_ip>:8090
diff --git a/docs/api.rst b/docs/api.rst
index 3d57073..92c6af1 100644
--- a/docs/api.rst
+++ b/docs/api.rst
@@ -6,7 +6,7 @@ API
:template: custom-module-template.rst
:recursive:
-.. coordinator
+ coordinator
plotting
ews_postprocessing
met_processing
diff --git a/scripts/run_Deposition_Processor.sh b/scripts/run_Deposition_Processor.sh
index 29205bc..bccfa0e 100755
--- a/scripts/run_Deposition_Processor.sh
+++ b/scripts/run_Deposition_Processor.sh
@@ -7,6 +7,8 @@ source "${proc_path}"/run_utils.sh
setup_conda_env
+echo "Running ProcessorDeposition.py with path: "
+pwd;
python "${proc_path}"/../coordinator/ProcessorDeposition.py "$@"
exit_code=$?
--
GitLab