FAQ | This is a LIVE service | Changelog

Skip to content
Snippets Groups Projects
Commit f3a8e530 authored by J.W. Smith's avatar J.W. Smith
Browse files

feat: New component - Media scraper

Can be called with command line flag '-p Scraper'.
To use, the config file requires the following minimal entry (with
URL for South Asia):
    "Scraper" : {
	    "ProcessPreJob" : "do_nothing_return_true",
        "ProcessInJob" : "process_in_job_media_scraper",
        "ProcessEWSPlotting" : "do_nothing",
        "seasonStartString" : "20220101",
        "URL" : "https://arrcc-viewer.onrender.com/assets/sample_data/data.zip"
    },
parent f02c682d
No related branches found
No related tags found
No related merge requests found
......@@ -5,6 +5,7 @@ are handled by Processor.py .'''
import logging
import os
from typing import List
# submodules of this project
# All of the process_* functions are callable from config files for the three
......@@ -24,6 +25,9 @@ from ProcessorEpidemiology import (
process_in_job_epi,
process_EWS_plotting_epi,
)
from ProcessorScraper import (
process_in_job_media_scraper,
)
from ProcessorServer import (
process_pre_job_server_download,
upload
......@@ -56,10 +60,18 @@ script_path = os.path.dirname(__file__)+'/'
coordinator_path = script_path
def do_nothing(*args, **kwargs):
def do_nothing(*args, **kwargs) -> List:
'''Dummy function'''
logger.info('Called do_nothing(). Nothing to do here')
pass
return []
def do_nothing_return_true(*args, **kwargs) -> bool:
'''Dummy function'''
logger.info('Called do_nothing_return_true(). Nothing to do here')
pass
return True
\ No newline at end of file
......@@ -19,6 +19,7 @@ short_name = {
'Environment' : 'ENVIRONMENT_2.0',
'Epidemiology' : 'EPI',
'Survey' : 'SURVEYDATA',
'Scraper' : 'SCRAPER',
}
disease_latin_name_dict = {
......
#!/bin/bash
# activate conda environment of python modules so they can be imported
source /storage/app/miniconda3/bin/activate /storage/app/EWS/General/EWS-python/py3EWSepi
python /storage/app/EWS/General/EWS-Coordinator/GatherScrapedMediaReports.py "$@"
# deactivate conda environment
source /storage/app/miniconda3/bin/deactivate /storage/app/EWS/General/EWS-python/py3EWSepi
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment