From 29c4c47b155ee40f414b0147549c1739573dfdec Mon Sep 17 00:00:00 2001
From: thomaswoehlke <thomas.woehlke@gmail.com>
Date: Fri, 19 Feb 2021 21:50:16 +0100
Subject: [PATCH] ### 0.0.18 Release
---
.../rki_landkreise/rki_landkreise_model.py | 65 +++++----
.../rki_landkreise_model_import.py | 1 -
.../rki_landkreise/rki_landkreise_service.py | 9 +-
.../rki_landkreise_service_download.py | 3 +-
.../rki_landkreise_service_import.py | 2 +-
.../rki_landkreise_service_update.py | 127 ++----------------
6 files changed, 45 insertions(+), 162 deletions(-)
diff --git a/src/covid19/blueprints/rki_landkreise/rki_landkreise_model.py b/src/covid19/blueprints/rki_landkreise/rki_landkreise_model.py
index 202b8607..fc81cdcb 100644
--- a/src/covid19/blueprints/rki_landkreise/rki_landkreise_model.py
+++ b/src/covid19/blueprints/rki_landkreise/rki_landkreise_model.py
@@ -9,54 +9,53 @@ class RkiLandkreise(db.Model):
__tablename__ = 'rki_landkreise'
id = db.Column(db.Integer, primary_key=True)
-
- OBJECTID = db.Column(db.String(255), nullable=False)
- ADE = db.Column(db.String(255), nullable=False)
- GF = db.Column(db.String(255), nullable=False)
- BSG = db.Column(db.String(255), nullable=False)
- RS = db.Column(db.String(255), nullable=False)
- AGS = db.Column(db.String(255), nullable=False)
- SDV_RS = db.Column(db.String(255), nullable=False)
- GEN = db.Column(db.String(255), nullable=False)
- BEZ = db.Column(db.String(255), nullable=False)
- IBZ = db.Column(db.String(255), nullable=False)
- BEM = db.Column(db.String(255), nullable=False)
- NBD = db.Column(db.String(255), nullable=False)
- SN_L = db.Column(db.String(255), nullable=False)
- SN_R = db.Column(db.String(255), nullable=False)
- SN_K = db.Column(db.String(255), nullable=False)
- SN_V1 = db.Column(db.String(255), nullable=False)
- SN_V2 = db.Column(db.String(255), nullable=False)
- SN_G = db.Column(db.String(255), nullable=False)
- FK_S3 = db.Column(db.String(255), nullable=False)
- NUTS = db.Column(db.String(255), nullable=False)
- RS_0 = db.Column(db.String(255), nullable=False)
- AGS_0 = db.Column(db.String(255), nullable=False)
- WSK = db.Column(db.String(255), nullable=False)
- EWZ = db.Column(db.String(255), nullable=False)
- KFL = db.Column(db.String(255), nullable=False)
- DEBKG_ID = db.Column(db.String(255), nullable=False)
+ objectid = db.Column(db.String(255), nullable=False)
+ ade = db.Column(db.String(255), nullable=False)
+ gf = db.Column(db.String(255), nullable=False)
+ bsg = db.Column(db.String(255), nullable=False)
+ rs = db.Column(db.String(255), nullable=False)
+ ags = db.Column(db.String(255), nullable=False)
+ sdv_rs = db.Column(db.String(255), nullable=False)
+ gen = db.Column(db.String(255), nullable=False)
+ bez = db.Column(db.String(255), nullable=False)
+ ibz = db.Column(db.String(255), nullable=False)
+ bem = db.Column(db.String(255), nullable=False)
+ nbd = db.Column(db.String(255), nullable=False)
+ sn_l = db.Column(db.String(255), nullable=False)
+ sn_r = db.Column(db.String(255), nullable=False)
+ sn_k = db.Column(db.String(255), nullable=False)
+ sn_v1 = db.Column(db.String(255), nullable=False)
+ sn_v2 = db.Column(db.String(255), nullable=False)
+ sn_g = db.Column(db.String(255), nullable=False)
+ fk_s3 = db.Column(db.String(255), nullable=False)
+ nuts = db.Column(db.String(255), nullable=False)
+ rs_0 = db.Column(db.String(255), nullable=False)
+ ags_0 = db.Column(db.String(255), nullable=False)
+ wsk = db.Column(db.String(255), nullable=False)
+ ewz = db.Column(db.String(255), nullable=False)
+ kfl = db.Column(db.String(255), nullable=False)
+ debkg_id = db.Column(db.String(255), nullable=False)
death_rate = db.Column(db.String(255), nullable=False)
cases = db.Column(db.String(255), nullable=False)
deaths = db.Column(db.String(255), nullable=False)
cases_per_100k = db.Column(db.String(255), nullable=False)
cases_per_population = db.Column(db.String(255), nullable=False)
- BL = db.Column(db.String(255), nullable=False)
- BL_ID = db.Column(db.String(255), nullable=False)
+ bl = db.Column(db.String(255), nullable=False)
+ bl_id = db.Column(db.String(255), nullable=False)
county = db.Column(db.String(255), nullable=False)
last_update = db.Column(db.String(255), nullable=False)
cases7_per_100k = db.Column(db.String(255), nullable=False)
recovered = db.Column(db.String(255), nullable=False)
- EWZ_BL = db.Column(db.String(255), nullable=False)
+ ewz_bl = db.Column(db.String(255), nullable=False)
cases7_bl_per_100k = db.Column(db.String(255), nullable=False)
cases7_bl = db.Column(db.String(255), nullable=False)
death7_bl = db.Column(db.String(255), nullable=False)
cases7_lk = db.Column(db.String(255), nullable=False)
death7_lk = db.Column(db.String(255), nullable=False)
cases7_per_100k_txt = db.Column(db.String(255), nullable=False)
- AdmUnitId = db.Column(db.String(255), nullable=False)
- SHAPE_Length = db.Column(db.String(255), nullable=False)
- SHAPE_Area = db.Column(db.String(255), nullable=False)
+ adm_unit_id = db.Column(db.String(255), nullable=False)
+ shape_length = db.Column(db.String(255), nullable=False)
+ shape_area = db.Column(db.String(255), nullable=False)
@classmethod
def remove_all(cls):
diff --git a/src/covid19/blueprints/rki_landkreise/rki_landkreise_model_import.py b/src/covid19/blueprints/rki_landkreise/rki_landkreise_model_import.py
index 3566afb7..f0590162 100644
--- a/src/covid19/blueprints/rki_landkreise/rki_landkreise_model_import.py
+++ b/src/covid19/blueprints/rki_landkreise/rki_landkreise_model_import.py
@@ -7,7 +7,6 @@ class RkiLandkreiseImport(db.Model):
__tablename__ = 'rki_landkreise_import'
id = db.Column(db.Integer, primary_key=True)
-
OBJECTID = db.Column(db.String(255), nullable=False)
ADE = db.Column(db.String(255), nullable=False)
GF = db.Column(db.String(255), nullable=False)
diff --git a/src/covid19/blueprints/rki_landkreise/rki_landkreise_service.py b/src/covid19/blueprints/rki_landkreise/rki_landkreise_service.py
index e72c0be7..5291ca97 100644
--- a/src/covid19/blueprints/rki_landkreise/rki_landkreise_service.py
+++ b/src/covid19/blueprints/rki_landkreise/rki_landkreise_service.py
@@ -27,9 +27,7 @@ class RkiLandkreiseService:
def task_database_drop_create(self):
self.service_import.import_file()
- self.service_update.update_dimension_tables_only()
- self.service_update.update_fact_table_incremental_only()
- # TODO: #133 implement RkiBundeslaenderService.task_database_drop_create
+ self.service_update.update_star_schema_initial()
return self
def run_download_only(self):
@@ -42,29 +40,24 @@ class RkiLandkreiseService:
def run_update_dimension_tables_only(self):
self.service_update.update_dimension_tables_only()
- # TODO: #134 implement RkiBundeslaenderService.run_update_dimension_tables_only
return self
def run_update_fact_table_incremental_only(self):
self.service_update.update_fact_table_incremental_only()
- # TODO: #135 implement RkiBundeslaenderService.run_update_fact_table_incremental_only
return self
def run_update_fact_table_initial_only(self):
self.service_update.update_fact_table_initial_only()
- # TODO: #136 implement RkiBundeslaenderService.run_update_fact_table_initial_only
return self
def run_update_star_schema_incremental(self):
self.service_import.import_file()
self.service_update.update_star_schema_incremental()
- # TODO: #137 implement RkiBundeslaenderService.run_update_star_schema_incremental
return self
def run_update_star_schema_initial(self):
self.service_import.import_file()
self.service_update.update_star_schema_initial()
- # TODO: #138 implement RkiBundeslaenderService.run_update_star_schema_initial
return self
def download_all_files(self):
diff --git a/src/covid19/blueprints/rki_landkreise/rki_landkreise_service_download.py b/src/covid19/blueprints/rki_landkreise/rki_landkreise_service_download.py
index 515559bd..939b3e22 100644
--- a/src/covid19/blueprints/rki_landkreise/rki_landkreise_service_download.py
+++ b/src/covid19/blueprints/rki_landkreise/rki_landkreise_service_download.py
@@ -3,7 +3,7 @@ import wget
from flask import flash
from database import app
-from covid19.blueprints.rki_landkreise.rki_service_config import RkiLandkreiseServiceConfig
+from covid19.blueprints.rki_landkreise.rki_landkreise_service_config import RkiLandkreiseServiceConfig
class RkiLandkreiseServiceDownload:
@@ -22,7 +22,6 @@ class RkiLandkreiseServiceDownload:
app.logger.info(" download FILE: "+self.cfg.cvsfile_name+" from "+self.cfg.url_src)
app.logger.info("------------------------------------------------------------")
try:
- os.makedirs(self.cfg.data_path, exist_ok=True)
if os.path.isfile(self.cfg.cvsfile_path):
os.remove(self.cfg.cvsfile_path)
data_file = wget.download(self.cfg.url_src, self.cfg.cvsfile_path)
diff --git a/src/covid19/blueprints/rki_landkreise/rki_landkreise_service_import.py b/src/covid19/blueprints/rki_landkreise/rki_landkreise_service_import.py
index 282b2b8c..5ebf1e82 100644
--- a/src/covid19/blueprints/rki_landkreise/rki_landkreise_service_import.py
+++ b/src/covid19/blueprints/rki_landkreise/rki_landkreise_service_import.py
@@ -30,6 +30,7 @@ class RkiLandkreiseServiceImport:
file_reader = csv.DictReader(csv_file, delimiter=',', quotechar='"')
k = 0
for row in file_reader:
+ k += 1
o = RkiLandkreiseImport(
OBJECTID=row['OBJECTID'],
ADE=row['ADE'],
@@ -83,7 +84,6 @@ class RkiLandkreiseServiceImport:
if (k % 2000) == 0:
db.session.commit()
app.logger.info(" import import_file ... " + str(k) + " rows")
- k = k + 1
db.session.commit()
app.logger.info(" import import_file ... " + str(k) + " rows total")
except KeyError as error:
diff --git a/src/covid19/blueprints/rki_landkreise/rki_landkreise_service_update.py b/src/covid19/blueprints/rki_landkreise/rki_landkreise_service_update.py
index 80a12eac..70bfd0ad 100644
--- a/src/covid19/blueprints/rki_landkreise/rki_landkreise_service_update.py
+++ b/src/covid19/blueprints/rki_landkreise/rki_landkreise_service_update.py
@@ -16,8 +16,7 @@ class RkiLandkreiseServiceUpdate:
app.logger.debug("------------------------------------------------------------")
app.logger.debug(" RKI Service Update [ready]")
- # TODO: #147 refactor RkiBundeslaenderServiceUpdate.__update_who_date_reported
- def __update_who_date_reported(self):
+ def __update_date_reported(self):
app.logger.info(" update who_date_reported [begin]")
app.logger.info("------------------------------------------------------------")
i = 0
@@ -37,108 +36,7 @@ class RkiLandkreiseServiceUpdate:
app.logger.info("------------------------------------------------------------")
return self
- # TODO: #148 refactor RkiBundeslaenderServiceUpdate.__update_who_region
- def __update_who_region(self):
- app.logger.info(" update who_region [begin]")
- app.logger.info("------------------------------------------------------------")
- i = 0
- for i_who_region, in db.session.query(RkiLandkreiseImport.who_region).distinct():
- c = db.session.query(RkiRegion).filter(RkiRegion.region == i_who_region).count()
- if c == 0:
- o = RkiRegion(region=i_who_region)
- db.session.add(o)
- app.logger.info(i_who_region +" added NEW ")
- else:
- app.logger.info(i_who_region +" not added ")
- if i % 10 == 0:
- db.session.commit()
- i += 1
- db.session.commit()
- app.logger.info("")
- app.logger.info(" update who_region [done]")
- app.logger.info("------------------------------------------------------------")
- return self
-
- # TODO: #149 refactor RkiBundeslaenderServiceUpdate.__update_who_country
- def __update_who_country(self):
- app.logger.info(" update who_country [begin]")
- app.logger.info("------------------------------------------------------------")
- sql_text = """
- select distinct
- who_global_data_import.country_code,
- who_global_data_import.country,
- who_global_data_import.who_region
- from who_global_data_import
- """
- result = db.session.execute(sql_text).fetchall()
- for result_item in result:
- i_country_code = result_item.country_code
- i_country = result_item.country
- i_who_region = result_item.who_region
- output = i_country_code + " " + i_country + " " + i_who_region
- my_region = RkiRegion.find_by_region(i_who_region)
- my_country = RkiCountry.find_by_country_code_and_country_and_who_region_id(
- i_country_code, i_country, my_region
- )
- if my_country is None:
- o = RkiCountry(
- country=i_country,
- country_code=i_country_code,
- region=my_region)
- db.session.add(o)
- db.session.commit()
- my_country = RkiCountry.find_by_country_code_and_country_and_who_region_id(
- i_country_code, i_country, my_region
- )
- output += " added NEW "
- else:
- output += " not added "
- output += i_country_code + " id=" + str(my_country.id)
- app.logger.info(output)
- db.session.commit()
- app.logger.info("")
- app.logger.info(" update who_country [done]")
- app.logger.info("------------------------------------------------------------")
- return self
-
- # TODO: #150 refactor RkiBundeslaenderServiceUpdate.__update_who_global_data
- def __update_who_global_data(self):
- app.logger.info(" update WHO [begin]")
- app.logger.info("------------------------------------------------------------")
- dates_reported = RkiDateReported.get_all_as_dict()
- countries = RkiCountry.get_all_as_dict()
- #
- #
- i = 0
- result = RkiLandkreiseImport.get_all()
- for result_item in result:
- my_country = countries[result_item.country_code]
- my_date_reported = dates_reported[result_item.date_reported]
- result_who_global_data = RkiLandkreise.find_one_or_none_by_date_and_country(
- my_date_reported,
- my_country)
- if result_who_global_data is None:
- o = RkiLandkreise(
- cases_new=int(result_item.new_cases),
- cases_cumulative=int(result_item.cumulative_cases),
- deaths_new=int(result_item.new_deaths),
- deaths_cumulative=int(result_item.cumulative_deaths),
- date_reported=my_date_reported,
- country=my_country
- )
- db.session.add(o)
- if i % 2000 == 0:
- app.logger.info(" update WHO ... "+str(i)+" rows")
- db.session.commit()
- i += 1
- db.session.commit()
- app.logger.info(" update RKI : "+str(i)+" total rows")
- app.logger.info(" update RKI [done]")
- app.logger.info("------------------------------------------------------------")
- return self
-
- # TODO: #151 refactor RkiBundeslaenderServiceUpdate.__update_who_global_data_short
- def __update_who_global_data_short(self):
+ def __update_data_incremental(self):
app.logger.info(" update RKI short [begin]")
app.logger.info("------------------------------------------------------------")
new_dates_reported_from_import = RkiLandkreiseImport.get_new_dates_as_array()
@@ -146,14 +44,12 @@ class RkiLandkreiseServiceUpdate:
for my_date_reported in new_dates_reported_from_import:
my_date = RkiDateReported.find_by_date_reported(my_date_reported)
for result_item in RkiLandkreiseImport.get_for_one_day(my_date_reported):
- my_country = RkiCountry.find_by_country_code(result_item.country_code)
o = RkiLandkreise(
cases_new=int(result_item.new_cases),
cases_cumulative=int(result_item.cumulative_cases),
deaths_new=int(result_item.new_deaths),
deaths_cumulative=int(result_item.cumulative_deaths),
date_reported=my_date,
- country=my_country
)
db.session.add(o)
result_item.row_imported = True
@@ -168,8 +64,7 @@ class RkiLandkreiseServiceUpdate:
app.logger.info("------------------------------------------------------------")
return self
- # TODO: #152 refactor RkiBundeslaenderServiceUpdate.__update_who_global_data_initial
- def __update_who_global_data_initial(self):
+ def __update_data_initial(self):
app.logger.info(" update RKI initial [begin]")
app.logger.info("------------------------------------------------------------")
RkiLandkreise.remove_all()
@@ -178,14 +73,12 @@ class RkiLandkreiseServiceUpdate:
for my_date_reported in new_dates_reported_from_import:
my_date = RkiDateReported.find_by_date_reported(my_date_reported)
for result_item in RkiLandkreiseImport.get_for_one_day(my_date_reported):
- my_country = RkiCountry.find_by_country_code(result_item.country_code)
o = RkiLandkreise(
cases_new=int(result_item.new_cases),
cases_cumulative=int(result_item.cumulative_cases),
deaths_new=int(result_item.new_deaths),
deaths_cumulative=int(result_item.cumulative_deaths),
- date_reported=my_date,
- country=my_country
+ date_reported=my_date
)
db.session.add(o)
result_item.row_imported = True
@@ -204,8 +97,8 @@ class RkiLandkreiseServiceUpdate:
def update_db(self):
app.logger.info(" update db [begin]")
app.logger.info("------------------------------------------------------------")
- self.__update_who_date_reported()
- self.__update_who_region()
+ self.__update_date_reported()
+ self.__update_region()
self.__update_who_country()
self.__update_who_global_data()
app.logger.info(" update db [done]")
@@ -216,8 +109,8 @@ class RkiLandkreiseServiceUpdate:
def update_db_short(self):
app.logger.info(" update db short [begin]")
app.logger.info("------------------------------------------------------------")
- self.__update_who_date_reported()
- self.__update_who_region()
+ self.__update_date_reported()
+ self.__update_region()
self.__update_who_country()
self.__update_who_global_data_short()
app.logger.info(" update db short [done]")
@@ -228,8 +121,8 @@ class RkiLandkreiseServiceUpdate:
def update_db_initial(self):
app.logger.info(" update db initial [begin]")
app.logger.info("------------------------------------------------------------")
- self.__update_who_date_reported()
- self.__update_who_region()
+ self.__update_date_reported()
+ self.__update_region()
self.__update_who_country()
self.__update_who_global_data_initial()
app.logger.info(" update db initial [done]")
--
GitLab