From 11f14044ad4c215460999b7e7423101ef0e008a8 Mon Sep 17 00:00:00 2001
From: thomaswoehlke <thomas.woehlke@gmail.com>
Date: Wed, 17 Feb 2021 18:41:52 +0100
Subject: [PATCH] ### 0.0.17 Release * Fixed #123 split RkiBundeslaenderService
into two Services, one for bundeslaender and one for landkreise * Fixed #128
add fields from csv to RkiLandkreiseImport * Fixed #139 refactor
RkiBundeslaenderServiceDownload to new method scheme introduced 07.02.2021 *
Fixed #140 move WhoImport to RKI in: rk_service_import.py * Fixed #125
implement RkiLandkreise * Fixed #126 implement RkiBundeslaenderImport
### 0.0.18 Release
* Fixed #39 SQLalchemy instead of SQL: AllModelClasses.remove_all()
* Fixed #40 SQLalchemy instead of SQL: EcdcImport.get_date_rep()
* Fixed #41 SQLalchemy instead of SQL: EcdcImport.get_countries_of_continent()
* Fixed #107 SQLalchemy instead of SQL in: EcdcImport.get_countries_of_continent
* Fixed #109 SQLalchemy instead of SQL in: EcdcImport.get_date_rep
* Fixed #110 SQLalchemy instead of SQL in: EcdcImport.get_continent
---
src/covid19/blueprints/admin/admin_views.py | 41 +++++++++++++------
src/covid19/blueprints/ecdc/ecdc_service.py | 24 ++++-------
.../blueprints/ecdc/ecdc_service_import.py | 2 +-
.../rki_bundeslaender/rki_service.py | 5 ++-
.../blueprints/rki_landkreise/rki_service.py | 3 ++
.../rki_vaccination_service.py | 6 ++-
src/covid19/blueprints/who/who_service.py | 4 ++
7 files changed, 54 insertions(+), 31 deletions(-)
diff --git a/src/covid19/blueprints/admin/admin_views.py b/src/covid19/blueprints/admin/admin_views.py
index b5188c88..258d88f5 100644
--- a/src/covid19/blueprints/admin/admin_views.py
+++ b/src/covid19/blueprints/admin/admin_views.py
@@ -26,7 +26,7 @@ def task_admin_alive_message(self):
logger = get_task_logger(__name__)
self.update_state(state=states.STARTED)
logger.info("------------------------------------------------------------")
- logger.info(" Received: task_admin_alive_message [OK] ")
+ logger.info(" task_admin_alive_message [received] ")
logger.info("------------------------------------------------------------")
self.update_state(state=states.SUCCESS)
result = "OK (task_admin_alive_message)"
@@ -38,17 +38,38 @@ def task_admin_database_drop_create(self):
logger = get_task_logger(__name__)
self.update_state(state=states.STARTED)
logger.info("------------------------------------------------------------")
- logger.info(" Received: task_admin_database_drop_create [OK] ")
+ logger.info(" task_admin_database_drop_create [start] ")
logger.info("------------------------------------------------------------")
who_service.task_database_drop_create()
ecdc_service.task_database_drop_create()
rki_vaccination_service.task_database_drop_create()
admin_service.task_database_drop_create()
+ logger.info("------------------------------------------------------------")
+ logger.info(" task_admin_database_drop_create [done] ")
+ logger.info("------------------------------------------------------------")
self.update_state(state=states.SUCCESS)
result = "OK (task_admin_database_drop_create)"
return result
+@celery.task(bind=True)
+def task_admin_import_all_files(self):
+ logger = get_task_logger(__name__)
+ self.update_state(state=states.STARTED)
+ logger.info("------------------------------------------------------------")
+ logger.info(" task_admin_import_all_files [start] ")
+ logger.info("------------------------------------------------------------")
+ who_service.task_import_all_files()
+ ecdc_service.task_import_all_files()
+ rki_vaccination_service.task_import_all_files()
+ self.update_state(state=states.SUCCESS)
+ logger.info("------------------------------------------------------------")
+ logger.info(" task_admin_import_all_files [done] ")
+ logger.info("------------------------------------------------------------")
+ result = "OK (task_admin_import_all_files)"
+ return result
+
+
@app_admin.route('/')
def url_admin_index():
page_info = ApplicationPage('Admin', "Covid19 Admin")
@@ -143,15 +164,9 @@ def url_admin_download_all_files():
@app_admin.route('/import/all')
def url_admin_import_all_files():
- who_service.download_all_files()
- flash("who_service.download_all_files Done")
- ecdc_service.download_all_files()
- flash("who_service.download_all_files Done")
- rki_vaccination_service.download_all_files()
- flash("who_service.download_all_files Done")
- rki_service_bundeslaender.download_all_files()
- flash("who_service.download_all_files Done")
- rki_service_landkreise.download_all_files()
- flash("who_service.download_all_files Done")
- app.logger.info("url_admin_download_all_files [done]")
+ app.logger.info("url_admin_import_all_files [start]")
+ flash("task_admin_import_all_files async started")
+ task_admin_import_all_files.apply_async()
+ app.logger.info("task_admin_import_all_files async started")
+ app.logger.info("url_admin_import_all_files [done]")
return redirect(url_for('app_admin.url_admin_tasks'))
diff --git a/src/covid19/blueprints/ecdc/ecdc_service.py b/src/covid19/blueprints/ecdc/ecdc_service.py
index 90d0f636..3b422506 100644
--- a/src/covid19/blueprints/ecdc/ecdc_service.py
+++ b/src/covid19/blueprints/ecdc/ecdc_service.py
@@ -24,7 +24,7 @@ class EcdcService:
return self
def task_database_drop_create(self):
- self.service_import.import_datafile_to_db()
+ self.service_import.import_file()
self.service_update.update_db_short()
return self
@@ -33,41 +33,35 @@ class EcdcService:
return self
def run_import_only(self):
- self.service_import.import_datafile_to_db()
+ self.service_import.import_file()
return self
- # TODO: #112 implement EcdcService.run_update_dimension_tables_only
- # TODO: #111 refactor to new method scheme itroduced 07.02.2021
def run_update_dimension_tables_only(self):
self.service_update.update_dimension_tables_only()
return self
- # TODO: #113 implement EcdcService.run_update_fact_table_incremental_only
- # TODO: #111 refactor to new method scheme itroduced 07.02.2021
def run_update_fact_table_incremental_only(self):
self.service_update.update_fact_table_incremental_only()
return self
- # TODO: #114 implement EcdcService.run_update_fact_table_initial_only
- # TODO: #111 refactor to new method scheme itroduced 07.02.2021
def run_update_fact_table_initial_only(self):
self.service_update.update_fact_table_initial_only()
return self
- # TODO: #115 implement EcdcService.run_update_star_schema_incremental
- # TODO: #111 refactor to new method scheme itroduced 07.02.2021
def run_update_star_schema_incremental(self):
- self.service_import.import_datafile_to_db()
+ self.service_import.import_file()
self.service_update.update_star_schema_incremental()
return self
- # TODO: #116 implement EcdcService.run_update_star_schema_initial
- # TODO: #111 refactor to new method scheme itroduced 07.02.2021
def run_update_star_schema_initial(self):
- self.service_import.import_datafile_to_db()
+ self.service_import.import_file()
self.service_update.update_star_schema_initial()
return self
def download_all_files(self):
self.service_download.download_file()
- return self
\ No newline at end of file
+ return self
+
+ def task_import_all_files(self):
+ self.service_import.import_file()
+ return self
diff --git a/src/covid19/blueprints/ecdc/ecdc_service_import.py b/src/covid19/blueprints/ecdc/ecdc_service_import.py
index c07f06dd..0f354c5b 100644
--- a/src/covid19/blueprints/ecdc/ecdc_service_import.py
+++ b/src/covid19/blueprints/ecdc/ecdc_service_import.py
@@ -17,7 +17,7 @@ class EcdcServiceImport:
app.logger.debug("------------------------------------------------------------")
app.logger.debug(" ECDC Service Import [ready] ")
- def import_datafile_to_db(self):
+ def import_file(self):
src_cvsfile_name = self.cfg.data_path+os.sep+self.cfg.cvsfile_name
app.logger.info(" import ECDC [begin]")
app.logger.info("------------------------------------------------------------")
diff --git a/src/covid19/blueprints/rki_bundeslaender/rki_service.py b/src/covid19/blueprints/rki_bundeslaender/rki_service.py
index 3a7b9e9f..b441534f 100644
--- a/src/covid19/blueprints/rki_bundeslaender/rki_service.py
+++ b/src/covid19/blueprints/rki_bundeslaender/rki_service.py
@@ -111,4 +111,7 @@ class RkiBundeslaenderService:
def download_all_files(self):
self.service_download.download_file()
- return self
\ No newline at end of file
+ return self
+
+ def task_import_all_files(self):
+ return self
diff --git a/src/covid19/blueprints/rki_landkreise/rki_service.py b/src/covid19/blueprints/rki_landkreise/rki_service.py
index 4a1558be..225a5d6f 100644
--- a/src/covid19/blueprints/rki_landkreise/rki_service.py
+++ b/src/covid19/blueprints/rki_landkreise/rki_service.py
@@ -112,3 +112,6 @@ class RkiLandkreiseService:
self.service_download.download_file()
return self
+ def task_import_all_files(self):
+ return self
+
diff --git a/src/covid19/blueprints/rki_vaccination/rki_vaccination_service.py b/src/covid19/blueprints/rki_vaccination/rki_vaccination_service.py
index d0f28f43..5d387662 100644
--- a/src/covid19/blueprints/rki_vaccination/rki_vaccination_service.py
+++ b/src/covid19/blueprints/rki_vaccination/rki_vaccination_service.py
@@ -61,4 +61,8 @@ class RkiVaccinationService:
def download_all_files(self):
self.service_download.download_file()
- return self
\ No newline at end of file
+ return self
+
+ def task_import_all_files(self):
+ self.service_import.import_file()
+ return self
diff --git a/src/covid19/blueprints/who/who_service.py b/src/covid19/blueprints/who/who_service.py
index a43381d8..45d931d5 100644
--- a/src/covid19/blueprints/who/who_service.py
+++ b/src/covid19/blueprints/who/who_service.py
@@ -106,3 +106,7 @@ class WhoService:
self.service_download.download_file()
return self
+ def task_import_all_files(self):
+ self.service_import.import_file()
+ return self
+
--
GitLab