fixing tests

ews/coordinator/processor_deposition.py

@@ -56,12 +56,7 @@ class ProcessorDeposition(ProcessorBase):
         logger.info('Copying file from remote server to job directory')
 
         # TODO: perform ssh file transfer in python instead of subprocess
-        server_name: str = config['ServerName']
-        if server_name == "":
-            cmd_scp = ["scp", f"{file_path}/{file_name}.tar.gz", jobPath]
-        else:
-            cmd_scp = ["scp", "-i", config['ServerKey'], "-o", "StrictHostKeyChecking=no",
-                       f"{server_name}:{file_path}/{file_name}.tar.gz", jobPath]
+        cmd_scp = ["scp", f"{file_path}/{file_name}.tar.gz", jobPath]
 
         description_short = 'dep scp'
         description_long = 'scp from server to job directory'

ews/coordinator/utils/processor_utils.py

@@ -3,6 +3,7 @@
 
 import datetime
 import glob
+import inspect
 import json
 import logging
 import logging.config

tests/integration/full/full_test_advisory.py

@@ -8,6 +8,9 @@ from ews.coordinator.processor_environment import ProcessorEnvironment
 from ews.coordinator.processor_surveys import ProcessorSurveys
 from integration.partial.integration_test_utils import IntegrationTestUtils
 from integration.test_suites.advisory_test_suite import BaseAdvisoryTestSuite
+from integration.test_suites.depo_test_suite import BaseDepoTestSuite
+from integration.test_suites.env_suit_test_suite import BaseEnvSuitTestSuite
+from integration.test_suites.survey_test_suite import BaseSurveyTestSuite
 
 
 class FullTestAdvisory(BaseAdvisoryTestSuite.AdvisoryTestSuite):
@@ -62,18 +65,21 @@ class FullTestAdvisory(BaseAdvisoryTestSuite.AdvisoryTestSuite):
     @staticmethod
     def run_dependent_pipelines():
         depo_processor = ProcessorDeposition()
-        IntegrationTestUtils.run_external_pipeline("Deposition",
+        IntegrationTestUtils.run_external_pipeline(BaseDepoTestSuite.DepoTestSuite.DEPO_COMPONENT_NAME,
+                                                   BaseDepoTestSuite.DepoTestSuite.DEPO_PROCESSOR_DIR,
                                                    IntegrationTestUtils.TEST_START_DATE,
                                                    depo_processor)
 
         env_processor = ProcessorEnvironment()
-        IntegrationTestUtils.run_external_pipeline("Environment",
+        IntegrationTestUtils.run_external_pipeline(BaseEnvSuitTestSuite.EnvSuitTestSuite.ENV_COMPONENT_NAME,
+                                                   BaseEnvSuitTestSuite.EnvSuitTestSuite.ENV_PROCESSOR_DIR,
                                                    IntegrationTestUtils.TEST_START_DATE,
                                                    env_processor)
 
         survey_processor = ProcessorSurveys()
         previous_day_string: str = IntegrationTestUtils.get_day_before_as_string(IntegrationTestUtils.TEST_START_DATE)
-        IntegrationTestUtils.run_external_pipeline("Survey",
+        IntegrationTestUtils.run_external_pipeline(BaseSurveyTestSuite.SurveyTestSuite.SURVEY_COMPONENT_NAME,
+                                                   BaseSurveyTestSuite.SurveyTestSuite.SURVEY_PROCESSOR_DIR,
                                                    previous_day_string,
                                                    survey_processor)
         pass
@@ -82,6 +88,7 @@ class FullTestAdvisory(BaseAdvisoryTestSuite.AdvisoryTestSuite):
     def run_advisory_pipeline():
         advisory_processor = ProcessorAdvisory()
         IntegrationTestUtils.run_external_pipeline(BaseAdvisoryTestSuite.AdvisoryTestSuite.ADVISORY_COMPONENT_NAME,
+                                                   BaseAdvisoryTestSuite.AdvisoryTestSuite.ADVISORY_PROCESSOR_DIR,
                                                    IntegrationTestUtils.TEST_START_DATE,
                                                    advisory_processor)
 

tests/integration/full/full_test_deposition.py

@@ -48,6 +48,7 @@ class FullTestDeposition(BaseDepoTestSuite.DepoTestSuite):
     def run_depo_pipeline():
         depo_processor = ProcessorDeposition()
         IntegrationTestUtils.run_external_pipeline(BaseDepoTestSuite.DepoTestSuite.DEPO_COMPONENT_NAME,
+                                                   BaseDepoTestSuite.DepoTestSuite.DEPO_PROCESSOR_DIR,
                                                    IntegrationTestUtils.TEST_START_DATE,
                                                    depo_processor)
 

tests/integration/full/full_test_env_suit.py

@@ -49,9 +49,11 @@ class FullTestEnvSuit(BaseEnvSuitTestSuite.EnvSuitTestSuite):
     def run_env_pipeline():
         env_processor = ProcessorEnvironment()
         IntegrationTestUtils.run_external_pipeline(BaseEnvSuitTestSuite.EnvSuitTestSuite.ENV_COMPONENT_NAME,
+                                                   BaseEnvSuitTestSuite.EnvSuitTestSuite.ENV_PROCESSOR_DIR,
                                                    IntegrationTestUtils.TEST_START_DATE,
                                                    env_processor)
 
+
 if __name__ == '__main__':
     _success: bool = IntegrationTestUtils.run_full_integration_test_pipeline(FullTestEnvSuit,
                                                                              test_prefix = "env",

tests/integration/full/full_test_epi.py

@@ -6,6 +6,8 @@ from ews.coordinator.processor_deposition import ProcessorDeposition
 from ews.coordinator.processor_environment import ProcessorEnvironment
 from ews.coordinator.processor_epidemiology import ProcessorEpidemiology
 from integration.partial.integration_test_utils import IntegrationTestUtils
+from integration.test_suites.depo_test_suite import BaseDepoTestSuite
+from integration.test_suites.env_suit_test_suite import BaseEnvSuitTestSuite
 from integration.test_suites.epi_test_suite import BaseEpiTestSuite
 
 
@@ -69,11 +71,14 @@ class FullTestEpi(BaseEpiTestSuite.EpiTestSuite):
     @staticmethod
     def run_dependent_pipelines():
         depo_processor = ProcessorDeposition()
-        IntegrationTestUtils.run_external_pipeline("Deposition",
+        IntegrationTestUtils.run_external_pipeline(BaseDepoTestSuite.DepoTestSuite.DEPO_COMPONENT_NAME,
+                                                   BaseDepoTestSuite.DepoTestSuite.DEPO_PROCESSOR_DIR,
                                                    IntegrationTestUtils.TEST_START_DATE,
                                                    depo_processor)
+
         env_processor = ProcessorEnvironment()
-        IntegrationTestUtils.run_external_pipeline("Environment",
+        IntegrationTestUtils.run_external_pipeline(BaseEnvSuitTestSuite.EnvSuitTestSuite.ENV_COMPONENT_NAME,
+                                                   BaseEnvSuitTestSuite.EnvSuitTestSuite.ENV_PROCESSOR_DIR,
                                                    IntegrationTestUtils.TEST_START_DATE,
                                                    env_processor)
 
@@ -83,6 +88,7 @@ class FullTestEpi(BaseEpiTestSuite.EpiTestSuite):
     def run_epi_pipeline():
         epi_processor = ProcessorEpidemiology()
         IntegrationTestUtils.run_external_pipeline(BaseEpiTestSuite.EpiTestSuite.EPI_COMPONENT_NAME,
+                                                   BaseEpiTestSuite.EpiTestSuite.EPI_PROCESSOR_DIR,
                                                    IntegrationTestUtils.TEST_START_DATE,
                                                    epi_processor)
 

tests/integration/full/full_test_survey.py

@@ -43,6 +43,7 @@ class FullTestSurvey(BaseSurveyTestSuite.SurveyTestSuite):
     def run_survey_pipeline():
         survey_processor = ProcessorSurveys()
         IntegrationTestUtils.run_external_pipeline(BaseSurveyTestSuite.SurveyTestSuite.SURVEY_COMPONENT_NAME,
+                                                    BaseSurveyTestSuite.SurveyTestSuite.SURVEY_PROCESSOR_DIR,
                                                    IntegrationTestUtils.TEST_START_DATE,
                                                    survey_processor)
 

tests/integration/partial/integration_test_utils.py

@@ -218,6 +218,7 @@ class IntegrationTestUtils:
 
     @staticmethod
     def run_external_pipeline(component: str,
+                              short_name: str,
                               start_date: str,
                               processor: ProcessorBase,
                               **kwargs):
@@ -229,6 +230,7 @@ class IntegrationTestUtils:
         args_dict['noupload'] = True
         args_dict['start_date'] = start_date
         args_dict['component'] = component
+        args_dict['short_name'] = short_name
         args_dict['config_paths'] = [IntegrationTestUtils.TEMP_CONFIG_FILE_NAME]
         args_dict['log_level'] = 'info'
         args_dict['clearup'] = True