From 360ebe105b6865ff5e76c0b4e869f0584e6105ab Mon Sep 17 00:00:00 2001
From: lb584 <lb584@cam.ac.uk>
Date: Wed, 7 Feb 2024 12:25:52 +0000
Subject: [PATCH] adding airflow installation
---
docs/_source/iaas_setup.rst | 90 ++++++++++++++++++++++++++++++++++++-
1 file changed, 89 insertions(+), 1 deletion(-)
diff --git a/docs/_source/iaas_setup.rst b/docs/_source/iaas_setup.rst
index 65e13b7..23524f4 100644
--- a/docs/_source/iaas_setup.rst
+++ b/docs/_source/iaas_setup.rst
@@ -138,13 +138,14 @@ mkdir -p /storage/sftp/
chown -R root:root /storage/sftp/
chmod -R 755 /storage/sftp/
+###REPEAT THIS PROCESS FOR THE PLANTVILAGE USER####
adduser --no-create-home --shell /usr/sbin/nologin metofficeupload # create the user and an associated group with the same name
adduser metofficeupload sftponly # add to existing sftponly group to allow (only) the sftp subsystem to be used under the chroot
mkdir -p /storage/sftp/metofficeupload/upload
chown root:root /storage/sftp/metofficeupload
-chmod -R 755 /storage/sftp/metofficeupload
chown metofficeupload:metofficeupload /storage/sftp/metofficeupload/upload
+chmod -R 755 /storage/sftp/metofficeupload
mkdir -p /storage/sftp/metofficeupload/upload/Ethiopia/fromMO/daily_name
mkdir -p /storage/sftp/metofficeupload/upload/Ethiopia/toMO/
@@ -153,6 +154,8 @@ sftp metofficeupload@10.136.11.3
add the ewsmanager user to metofficeupload group so it can manage files written by this user
sudo usermod -aG metofficeupload ewsmanager
+###END REPEAT THIS PROCESS FOR THE PLANTVILAGE USER####
+
Deploy and test the EWS code
----------------------------
@@ -496,3 +499,88 @@ check_interval = 0
shm_size = 0
[runners.cache]
+Airflow installation
+====================
+
+Install mysql:
+
+https://www.digitalocean.com/community/tutorials/how-to-install-mysql-on-ubuntu-20-04
+
+sudo apt install mysql-server
+
+sudo systemctl start mysql.service
+
+hack to allow root access via sudo on ubuntu (in the digital ocean docs)
+
+sudo mysql
+ALTER USER 'root'@'localhost' IDENTIFIED WITH mysql_native_password BY 'password_here';
+exit
+
+sudo mysql_secure_installation
+(no opinion on what options are selected, except disabling root access from remote host (yes))
+
+finish the hack for ubuntu root access:
+mysql -u root -p
+ALTER USER 'root'@'localhost' IDENTIFIED WITH auth_socket;
+exit
+</end ubuntu hack>
+
+Install Airflow
+---------------
+
+https://airflow.apache.org/docs/apache-airflow/stable/installation/installing-from-pypi.html
+
+mkdir <path>/airflow
+cd airflow
+set the airflow home location
+export AIRFLOW_HOME=<path>/airflow
+
+python3 -m venv airflow-env
+source airflow-env/bin/activate
+
+pip install "apache-airflow[celery]==2.7.1" --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-2.7.1/constraints-3.8.txt"
+
+pip check
+
+install docker operator
+
+pip install apache-airflow-providers-docker
+
+Setup mysql database:
+
+mysql -u root -p
+
+CREATE DATABASE airflow_db CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
+CREATE USER 'airflow_user' IDENTIFIED BY 'airflow_pass';
+GRANT ALL PRIVILEGES ON airflow_db.* TO 'airflow_user';
+exit;
+
+
+set database defaults (sqllite)
+airflow db migrate
+
+edit the resulting ${AIRFLOW_HOME}/airflow.cfg file on line 435 to the mysyl database
+sql_alchemy_conn = mysql+mysqldb://airflow_user:airflow_pass@localhost:3306/airflow_db
+
+install the mysql provider
+pip3 install apache-airflow-providers-mysql
+
+migrate the db again to use mysql this time
+airflow db migrate
+
+possible sql mode error:
+https://stackoverflow.com/questions/36882149/error-1067-42000-invalid-default-value-for-created-at
+create an admin user for the webserver
+
+airflow users create --role Admin --username admin --email lb584@cam.ac.uk --firstname admin --lastname admin --password admin
+
+run the scheduler and the webserver
+
+airflow scheduler;
+airflow webserver;
+
+can run as demons with the -D flag
+
+connect to the webserver (note you will only be able to connect on localhost by default)
+
+
--
GitLab