diff --git a/data/covid19data.sql b/data/covid19data.sql
index c8974967d6feffc9e8ae597ec0d77d4d4e7935eb..a385efcf36fcb800c28ef6e98bda0b3bc3982393 100644
Binary files a/data/covid19data.sql and b/data/covid19data.sql differ
diff --git a/src/covid19/blueprints/application/application_model.py b/src/covid19/blueprints/application/application_model.py
index 12c7817e3fa65ab0aebdce7ab76fc9f94ce762c6..13a96bc94fbbf52959f2509210f06870cba565ca 100644
--- a/src/covid19/blueprints/application/application_model.py
+++ b/src/covid19/blueprints/application/application_model.py
@@ -1,5 +1,5 @@
from sqlalchemy import and_, func
-from datetime import date
+from datetime import date, datetime, time
from database import db, ITEMS_PER_PAGE
from sqlalchemy.orm import joinedload
@@ -230,7 +230,7 @@ class RkiDateReported(ApplicationDateReported):
)
id = db.Column(db.Integer, primary_key=True)
- date_reported = db.Column(db.String(255), nullable=False, unique=True)
+ date_reported = db.Column(db.String(255), nullable=False)
year_week = db.Column(db.String(255), nullable=False)
datum = db.Column(db.Date, nullable=False, unique=True)
year = db.Column(db.Integer, nullable=False)
@@ -238,10 +238,17 @@ class RkiDateReported(ApplicationDateReported):
day_of_month = db.Column(db.Integer, nullable=False)
day_of_week = db.Column(db.Integer, nullable=False)
week_of_year = db.Column(db.Integer, nullable=False)
+ time_of_date_str = db.Column(db.String(255), nullable=False)
+ time_of_date = db.Column(db.Time, nullable=False)
+ aktualisierung = db.Column(db.String(255), nullable=False, unique=True)
@classmethod
- def create_new_object_factory(cls, my_date_rep: str):
- my_datum = date.fromisoformat(my_date_rep)
+ def create_new_object_factory(cls, aktualisierung: str):
+ aktualisierung_datetime = datetime.fromtimestamp(aktualisierung)
+ my_datum = aktualisierung_datetime.date()
+ my_time_of_date = aktualisierung_datetime.time()
+ my_date_reported_str = my_datum.isoformat()
+ my_time_of_date_str = my_time_of_date.isoformat(timespec='seconds')
(my_iso_year, week_number, weekday) = my_datum.isocalendar()
my_year_week = "" + str(my_iso_year)
if week_number < 10:
@@ -250,98 +257,21 @@ class RkiDateReported(ApplicationDateReported):
my_year_week += "-"
my_year_week += str(week_number)
return RkiDateReported(
- date_reported=my_date_rep,
+ date_reported=my_date_reported_str,
datum=my_datum,
year=my_datum.year,
month=my_datum.month,
day_of_month=my_datum.day,
day_of_week=weekday,
week_of_year=week_number,
- year_week=my_year_week
+ year_week=my_year_week,
+ time_of_date_str=my_time_of_date_str,
+ time_of_date=my_time_of_date,
+ aktualisierung=aktualisierung
)
-
-class RkiRegion(ApplicationRegion):
- __tablename__ = 'rki_region'
- __mapper_args__ = {'concrete': True}
- __table_args__ = (
- db.UniqueConstraint('region', name="uix_rki_region"),
- )
-
- id = db.Column(db.Integer, primary_key=True)
- region = db.Column(db.String(255), nullable=False, unique=True)
-
-
-class RkiCountry(db.Model):
- __tablename__ = 'rki_country'
-
- id = db.Column(db.Integer, primary_key=True)
- country_code = db.Column(db.String(255), unique=True, nullable=False)
- country = db.Column(db.String(255), unique=True, nullable=False)
- region_id = db.Column(db.Integer, db.ForeignKey('rki_region.id'), nullable=False)
- region = db.relationship(
- 'RkiRegion',
- lazy='subquery',
- order_by='RkiRegion.region')
-
- @classmethod
- def remove_all(cls):
- for one in cls.get_all():
- db.session.delete(one)
- db.session.commit()
- return None
-
- @classmethod
- def get_all_as_page(cls, page: int):
- return db.session.query(cls)\
- .order_by(cls.country)\
- .paginate(page, per_page=ITEMS_PER_PAGE)
-
- @classmethod
- def get_all(cls):
- return db.session.query(cls)\
- .order_by(cls.country)\
- .all()
-
- @classmethod
- def get_all_as_dict(cls):
- countries = {}
- for my_country in cls.get_all():
- countries[my_country.country_code] = my_country
- return countries
-
- @classmethod
- def get_by_id(cls, other_id: int):
- return db.session.query(cls)\
- .filter(cls.id == other_id)\
- .one()
-
- @classmethod
- def get_germany(cls):
- return db.session.query(cls)\
- .filter(cls.country_code == 'DE')\
- .one()
-
- @classmethod
- def find_by_country_code_and_country_and_who_region_id(cls,
- i_country_code: str, i_country: str, my_region: RkiRegion):
- return db.session.query(cls).filter(
- and_(
- cls.country_code == i_country_code,
- cls.country == i_country,
- cls.region_id == my_region.id
- )
- ).one_or_none()
-
@classmethod
- def find_by_country_code(cls, i_country_code: str):
- return db.session.query(cls).filter(
- cls.country_code == i_country_code
- ).one_or_none()
-
- @classmethod
- def get_who_countries_for_region(cls, region: str, page: int):
- return db.session.query(cls).filter(
- cls.region == region
- ).order_by(cls.country)\
- .paginate(page, per_page=ITEMS_PER_PAGE)
+ def find_by_aktualisierung(cls, aktualisierung_from_import: str):
+ return db.session.query(cls) \
+ .filter(cls.aktualisierung == aktualisierung_from_import) \
+ .one_or_none()
diff --git a/src/covid19/blueprints/rki_bundeslaender/rki_model.py b/src/covid19/blueprints/rki_bundeslaender/rki_model.py
index f1ec0ddb01c8edf4376f08ed7d1982bdb80b3ff3..2c1eec6c295556ab3060008a791382727eebe4da 100644
--- a/src/covid19/blueprints/rki_bundeslaender/rki_model.py
+++ b/src/covid19/blueprints/rki_bundeslaender/rki_model.py
@@ -1,38 +1,29 @@
-from sqlalchemy import and_
-from datetime import date
-from sqlalchemy.orm import joinedload
-
from database import db, ITEMS_PER_PAGE
-# OBJECTID_1,LAN_ew_AGS,LAN_ew_GEN,LAN_ew_BEZ,LAN_ew_EWZ,OBJECTID,Fallzahl,Aktualisierung,AGS_TXT,GlobalID,faelle_100000_EW,Death,cases7_bl_per_100k,cases7_bl,death7_bl,cases7_bl_per_100k_txt,AdmUnitId,SHAPE_Length,SHAPE_Area
-
-# TODO: #123 split RkiBundeslaenderService into two Services, one for bundeslaender and one for landkreise
-# TODO: #124 rename RkiBundeslaender to RkiBundeslaender
class RkiBundeslaender(db.Model):
__tablename__ = 'rki_bundeslaender'
id = db.Column(db.Integer, primary_key=True)
-
- OBJECTID_1 = db.Column(db.String(255), nullable=False)
- LAN_ew_AGS = db.Column(db.String(255), nullable=False)
- LAN_ew_GEN = db.Column(db.String(255), nullable=False)
- LAN_ew_BEZ = db.Column(db.String(255), nullable=False)
- LAN_ew_EWZ = db.Column(db.String(255), nullable=False)
- OBJECTID = db.Column(db.String(255), nullable=False)
- Fallzahl = db.Column(db.String(255), nullable=False)
- Aktualisierung = db.Column(db.String(255), nullable=False)
- AGS_TXT = db.Column(db.String(255), nullable=False)
- GlobalID = db.Column(db.String(255), nullable=False)
- faelle_100000_EW = db.Column(db.String(255), nullable=False)
- Death = db.Column(db.String(255), nullable=False)
- cases7_bl_per_100k = db.Column(db.String(255), nullable=False)
- cases7_bl = db.Column(db.String(255), nullable=False)
- death7_bl = db.Column(db.String(255), nullable=False)
+ object_id_1 = db.Column(db.Integer, nullable=False)
+ lan_ew_ags = db.Column(db.Integer, nullable=False)
+ lan_ew_gen = db.Column(db.String(255), nullable=False)
+ lan_ew_bez = db.Column(db.String(255), nullable=False)
+ lan_ew_ewz = db.Column(db.Integer, nullable=False)
+ object_id = db.Column(db.Integer, nullable=False)
+ fallzahl = db.Column(db.Integer, nullable=False)
+ aktualisierung = db.Column(db.String(255), nullable=False)
+ ags_txt = db.Column(db.Integer, nullable=False)
+ global_id = db.Column(db.String(255), nullable=False)
+ faelle_100000_ew = db.Column(db.Float, nullable=False)
+ death = db.Column(db.Integer, nullable=False)
+ cases7_bl_per_100k = db.Column(db.Float, nullable=False)
+ cases7_bl = db.Column(db.Integer, nullable=False)
+ death7_bl = db.Column(db.Integer, nullable=False)
cases7_bl_per_100k_txt = db.Column(db.String(255), nullable=False)
- AdmUnitId = db.Column(db.String(255), nullable=False)
- SHAPE_Length = db.Column(db.String(255), nullable=False)
- SHAPE_Area = db.Column(db.String(255), nullable=False)
+ adm_unit_id = db.Column(db.Integer, nullable=False)
+ shape_length = db.Column(db.Float, nullable=False)
+ shape_area = db.Column(db.Float, nullable=False)
@classmethod
def remove_all(cls):
diff --git a/src/covid19/blueprints/rki_bundeslaender/rki_model_import.py b/src/covid19/blueprints/rki_bundeslaender/rki_model_import.py
index f747f2b48b8000e35ba435964b99370169c2ea44..4b9bd790e3aed70bad3bafe959d76156266a3ccd 100644
--- a/src/covid19/blueprints/rki_bundeslaender/rki_model_import.py
+++ b/src/covid19/blueprints/rki_bundeslaender/rki_model_import.py
@@ -1,15 +1,11 @@
from database import db, ITEMS_PER_PAGE
-# OBJECTID_1,LAN_ew_AGS,LAN_ew_GEN,LAN_ew_BEZ,LAN_ew_EWZ,OBJECTID,Fallzahl,Aktualisierung,AGS_TXT,GlobalID,faelle_100000_EW,Death,cases7_bl_per_100k,cases7_bl,death7_bl,cases7_bl_per_100k_txt,AdmUnitId,SHAPE_Length,SHAPE_Area
-
-# TODO: #123 split RkiBundeslaenderService into two Services, one for bundeslaender and one for landkreise
# TODO: #126 implement RkiBundeslaenderImport
class RkiBundeslaenderImport(db.Model):
__tablename__ = 'rki_bundeslsaender_import'
id = db.Column(db.Integer, primary_key=True)
-
OBJECTID_1 = db.Column(db.String(255), nullable=False)
LAN_ew_AGS = db.Column(db.String(255), nullable=False)
LAN_ew_GEN = db.Column(db.String(255), nullable=False)
@@ -38,7 +34,7 @@ class RkiBundeslaenderImport(db.Model):
return None
@classmethod
- def get_all_as_page(cls, page):
+ def get_all_as_page(cls, page: int):
return db.session.query(cls).paginate(page, per_page=ITEMS_PER_PAGE)
@classmethod
@@ -46,10 +42,17 @@ class RkiBundeslaenderImport(db.Model):
return db.session.query(cls).all()
@classmethod
- def get_by_id(cls, other_id):
+ def get_by_id(cls, other_id: int):
return db.session.query(cls).filter(cls.id == other_id).one()
@classmethod
- def get_dates_reported(self):
- # TODO: #127 implement RkiBundeslaenderImport.get_dates_reported
- return self
+ def get_aktualisierungen_as_array(cls):
+ return []
+
+ @classmethod
+ def find_by_aktualisierung(cls, aktualisierung_from_import: str):
+ return []
+
+ @classmethod
+ def get_new_aktualisierungen_as_array(cls):
+ return []
diff --git a/src/covid19/blueprints/rki_bundeslaender/rki_service.py b/src/covid19/blueprints/rki_bundeslaender/rki_service.py
index b5f8d7e486ff32d4a3302dd7bcdd9fd7ddeb8624..896496f1cb397c2c6c2f3e188b7aff3692d2a8e5 100644
--- a/src/covid19/blueprints/rki_bundeslaender/rki_service.py
+++ b/src/covid19/blueprints/rki_bundeslaender/rki_service.py
@@ -26,8 +26,8 @@ class RkiBundeslaenderService:
return self
def task_database_drop_create(self):
- # TODO: #132 refactor RkiBundeslaenderService to new method scheme introduced 07.02.2021
- # TODO: #133 implement RkiBundeslaenderService.task_database_drop_create
+ self.service_import.import_file()
+ self.service_update.update_star_schema_initial()
return self
def run_download_only(self):
@@ -40,74 +40,27 @@ class RkiBundeslaenderService:
return self
def run_update_dimension_tables_only(self):
- # TODO: #132 refactor RkiBundeslaenderService to new method scheme introduced 07.02.2021
- # TODO: #134 implement RkiBundeslaenderService.run_update_dimension_tables_only
+ self.service_update.update_dimension_tables_only()
return self
def run_update_fact_table_incremental_only(self):
- # TODO: #132 refactor RkiBundeslaenderService to new method scheme introduced 07.02.2021
- # TODO: #135 implement RkiBundeslaenderService.run_update_fact_table_incremental_only
+ self.service_update.update_fact_table_incremental_only()
return self
def run_update_fact_table_initial_only(self):
- # TODO: #132 refactor RkiBundeslaenderService to new method scheme introduced 07.02.2021
- # TODO: #136 implement RkiBundeslaenderService.run_update_fact_table_initial_only
+ self.service_update.update_fact_table_initial_only()
return self
def run_update_star_schema_incremental(self):
- # TODO: #132 refactor RkiBundeslaenderService to new method scheme introduced 07.02.2021
- # TODO: #137 implement RkiBundeslaenderService.run_update_star_schema_incremental
+ self.service_download.download_file()
+ self.service_import.import_file()
+ self.service_update.update_star_schema_incremental()
return self
def run_update_star_schema_initial(self):
- # TODO: #132 refactor RkiBundeslaenderService to new method scheme introduced 07.02.2021
- # TODO: #138 implement RkiBundeslaenderService.run_update_star_schema_initial
- return self
-
- # TODO: #132 refactor RkiBundeslaenderService to new method scheme introduced 07.02.2021
- def run_download(self):
- app.logger.info(" run update [begin]")
- app.logger.info("------------------------------------------------------------")
- success = self.service_download.download_file()
- app.logger.info("")
- app.logger.info(" run update [done]")
- app.logger.info("------------------------------------------------------------")
- return success
-
- # TODO: #132 refactor RkiBundeslaenderService to new method scheme introduced 07.02.2021
- def run_update(self, import_file=True):
- app.logger.info(" run update [begin]")
- app.logger.info("------------------------------------------------------------")
- if import_file:
- self.service_import.import_file()
- self.service_update.update_db()
- app.logger.info("")
- app.logger.info(" run update [done]")
- app.logger.info("------------------------------------------------------------")
- return self
-
- # TODO: #132 refactor RkiBundeslaenderService to new method scheme introduced 07.02.2021
- def run_update_short(self, import_file=True):
- app.logger.info(" run update short [begin]")
- app.logger.info("------------------------------------------------------------")
- if import_file:
- self.service_import.import_file()
- self.service_update.update_db_short()
- app.logger.info("")
- app.logger.info(" run update short [done]")
- app.logger.info("------------------------------------------------------------")
- return self
-
- # TODO: #132 refactor RkiBundeslaenderService to new method scheme introduced 07.02.2021
- def run_update_initial(self, import_file=True):
- app.logger.info(" run update initial [begin]")
- app.logger.info("------------------------------------------------------------")
- if import_file:
- self.service_import.import_file()
- self.service_update.update_db_initial()
- app.logger.info("")
- app.logger.info(" run update initial [done]")
- app.logger.info("------------------------------------------------------------")
+ self.service_download.download_file()
+ self.service_import.import_file()
+ self.service_update.update_star_schema_initial()
return self
def download_all_files(self):
diff --git a/src/covid19/blueprints/rki_bundeslaender/rki_service_download.py b/src/covid19/blueprints/rki_bundeslaender/rki_service_download.py
index 921e1ac8b868bd7b4f28f1fe19b94c821156b1e6..da8a384c77ffb857a6c454c600ab572aa5489723 100644
--- a/src/covid19/blueprints/rki_bundeslaender/rki_service_download.py
+++ b/src/covid19/blueprints/rki_bundeslaender/rki_service_download.py
@@ -22,11 +22,10 @@ class RkiBundeslaenderServiceDownload:
app.logger.info(" download FILE: "+self.cfg.cvsfile_path+" from "+self.cfg.url_src)
app.logger.info("------------------------------------------------------------")
try:
- os.makedirs(self.cfg.data_path, exist_ok=True)
if os.path.isfile(self.cfg.cvsfile_path):
os.remove(self.cfg.cvsfile_path)
data_file = wget.download(self.cfg.url_src, self.cfg.cvsfile_path)
- app.logger.info(" " + data_file + " ")
+ app.logger.info("download success: " + data_file + " ")
except RuntimeError as runtimeError:
app.logger.error("############################################################")
app.logger.error(" " + runtimeError + " ")
diff --git a/src/covid19/blueprints/rki_bundeslaender/rki_service_import.py b/src/covid19/blueprints/rki_bundeslaender/rki_service_import.py
index c9ea1c9b6c21a22c98341e3e79b0270c6771837c..7f8a6e93c680a49e5cc9ca9d934f7017e48a18e1 100644
--- a/src/covid19/blueprints/rki_bundeslaender/rki_service_import.py
+++ b/src/covid19/blueprints/rki_bundeslaender/rki_service_import.py
@@ -28,6 +28,7 @@ class RkiBundeslaenderServiceImport:
file_reader = csv.DictReader(csv_file, delimiter=',', quotechar='"')
k = 0
for row in file_reader:
+ k = k + 1
o = RkiBundeslaenderImport(
OBJECTID_1=row['OBJECTID_1'],
LAN_ew_AGS=row['LAN_ew_AGS'],
@@ -52,24 +53,23 @@ class RkiBundeslaenderServiceImport:
db.session.add(o)
if (k % 2000) == 0:
db.session.commit()
- app.logger.info(" import RKI ... " + str(k) + " rows")
- k = k + 1
+ app.logger.info(" import RKI Bundeslaender ... " + str(k) + " rows")
db.session.commit()
- app.logger.info(" import RKI ... " + str(k) + " rows total")
+ app.logger.info(" import RKI Bundeslaender ... " + str(k) + " rows total")
except KeyError as error:
app.logger.warning("WARN: import RKI [begin]")
app.logger.warning(":::"+str(error)+":::")
for item_key, item_value in row.items():
app.logger.warning(item_key+" : "+item_value)
- app.logger.warning("WARN: import RKI [end]")
+ app.logger.warning("WARN: import RKI Bundeslaender [end]")
except (Exception, psycopg2.DatabaseError) as error:
- app.logger.warning("WARN: import RKI [begin]")
+ app.logger.warning("WARN: import RKI Bundeslaender [begin]")
app.logger.warning(error)
- app.logger.warning("WARN: import RKI [end]")
+ app.logger.warning("WARN: import RKI Bundeslaender [end]")
finally:
app.logger.info("")
app.logger.info("------------------------------------------------------------")
app.logger.info(" imported into TABLE: "+self.cfg.tablename+" from "+self.cfg.cvsfile_path)
app.logger.info("------------------------------------------------------------")
- app.logger.info(" import RKI [done]")
+ app.logger.info(" import RKI Bundeslaender [done]")
return self
diff --git a/src/covid19/blueprints/rki_bundeslaender/rki_service_update.py b/src/covid19/blueprints/rki_bundeslaender/rki_service_update.py
index 990f7709a9770c1d83a1e6b67f4f95464ab0cc18..be6f25eded3d2137acc550fcda1b65e1647ec768 100644
--- a/src/covid19/blueprints/rki_bundeslaender/rki_service_update.py
+++ b/src/covid19/blueprints/rki_bundeslaender/rki_service_update.py
@@ -1,6 +1,6 @@
from database import db, app
-from covid19.blueprints.application.application_model import RkiRegion, RkiDateReported, RkiCountry
+from covid19.blueprints.application.application_model import RkiDateReported
from covid19.blueprints.rki_bundeslaender.rki_model import RkiBundeslaender
from covid19.blueprints.rki_bundeslaender.rki_model_import import RkiBundeslaenderImport
from covid19.blueprints.rki_bundeslaender.rki_service_config import RkiBundeslaenderServiceConfig
@@ -16,143 +16,56 @@ class RkiBundeslaenderServiceUpdate:
app.logger.debug("------------------------------------------------------------")
app.logger.debug(" RKI Service Update [ready]")
- def __update_who_date_reported(self):
- app.logger.info(" update who_date_reported [begin]")
+ def __update_date_reported(self):
+ app.logger.info(" update RkiDateReported [begin]")
app.logger.info("------------------------------------------------------------")
i = 0
- for i_date_reported, in RkiBundeslaenderImport.get_dates_reported():
- c = RkiDateReported.find_by_date_reported(i_date_reported)
- if c is None:
- o = RkiDateReported.create_new_object_factory(my_date_rep=i_date_reported)
- db.session.add(o)
- app.logger.info(" update who_date_reported "+i_date_reported+" added NEW")
- if i % 10 == 0:
- app.logger.info(" update who_date_reported "+i_date_reported+" not added")
- db.session.commit()
- i += 1
- db.session.commit()
- app.logger.info("")
- app.logger.info(" update who_date_reported [done]")
- app.logger.info("------------------------------------------------------------")
- return self
-
- # TODO: #148 refactor RkiBundeslaenderServiceUpdate.__update_who_region
- def __update_who_region(self):
- app.logger.info(" update who_region [begin]")
- app.logger.info("------------------------------------------------------------")
- i = 0
- for i_who_region, in db.session.query(RkiBundeslaenderImport.who_region).distinct():
- c = db.session.query(RkiRegion).filter(RkiRegion.region == i_who_region).count()
- if c == 0:
- o = RkiRegion(region=i_who_region)
- db.session.add(o)
- app.logger.info(i_who_region +" added NEW ")
- else:
- app.logger.info(i_who_region +" not added ")
- if i % 10 == 0:
- db.session.commit()
+ for aktualisierung in RkiBundeslaenderImport.get_aktualisierungen_as_array():
i += 1
- db.session.commit()
- app.logger.info("")
- app.logger.info(" update who_region [done]")
- app.logger.info("------------------------------------------------------------")
- return self
-
- # TODO: #149 refactor RkiBundeslaenderServiceUpdate.__update_who_country
- def __update_who_country(self):
- app.logger.info(" update who_country [begin]")
- app.logger.info("------------------------------------------------------------")
- sql_text = """
- select distinct
- who_global_data_import.country_code,
- who_global_data_import.country,
- who_global_data_import.who_region
- from who_global_data_import
- """
- result = db.session.execute(sql_text).fetchall()
- for result_item in result:
- i_country_code = result_item.country_code
- i_country = result_item.country
- i_who_region = result_item.who_region
- output = i_country_code + " " + i_country + " " + i_who_region
- my_region = RkiRegion.find_by_region(i_who_region)
- my_country = RkiCountry.find_by_country_code_and_country_and_who_region_id(
- i_country_code, i_country, my_region
- )
- if my_country is None:
- o = RkiCountry(
- country=i_country,
- country_code=i_country_code,
- region=my_region)
+ output = " [ " + str(i) + " ] " + aktualisierung
+ c = RkiDateReported.find_by_date_reported(aktualisierung)
+ if c is None:
+ o = RkiDateReported.create_new_object_factory(aktualisierung=aktualisierung)
db.session.add(o)
db.session.commit()
- my_country = RkiCountry.find_by_country_code_and_country_and_who_region_id(
- i_country_code, i_country, my_region
- )
- output += " added NEW "
+ output += " added"
else:
- output += " not added "
- output += i_country_code + " id=" + str(my_country.id)
+ output += " NOT added " + str(c.id)
app.logger.info(output)
db.session.commit()
app.logger.info("")
- app.logger.info(" update who_country [done]")
+ app.logger.info(" update RkiDateReported [done]")
app.logger.info("------------------------------------------------------------")
return self
- # TODO: #150 refactor RkiBundeslaenderServiceUpdate.__update_who_global_data
- def __update_who_global_data(self):
- app.logger.info(" update WHO [begin]")
+ def __update_data_incremental(self):
+ app.logger.info(" update RkiBundeslaender short [begin]")
app.logger.info("------------------------------------------------------------")
- dates_reported = RkiDateReported.get_all_as_dict()
- countries = RkiCountry.get_all_as_dict()
- #
- #
+ aktualisierungen_from_import = RkiBundeslaenderImport.get_aktualisierungen_as_array()
i = 0
- result = RkiBundeslaenderImport.get_all()
- for result_item in result:
- my_country = countries[result_item.country_code]
- my_date_reported = dates_reported[result_item.date_reported]
- result_who_global_data = RkiBundeslaender.find_one_or_none_by_date_and_country(
- my_date_reported,
- my_country)
- if result_who_global_data is None:
+ for aktualisierung_from_import in aktualisierungen_from_import:
+ my_date = RkiDateReported.find_by_aktualisierung(aktualisierung_from_import)
+ for result_item in RkiBundeslaenderImport.find_by_aktualisierung(aktualisierung_from_import):
o = RkiBundeslaender(
- cases_new=int(result_item.new_cases),
- cases_cumulative=int(result_item.cumulative_cases),
- deaths_new=int(result_item.new_deaths),
- deaths_cumulative=int(result_item.cumulative_deaths),
- date_reported=my_date_reported,
- country=my_country
- )
- db.session.add(o)
- if i % 2000 == 0:
- app.logger.info(" update WHO ... "+str(i)+" rows")
- db.session.commit()
- i += 1
- db.session.commit()
- app.logger.info(" update RKI : "+str(i)+" total rows")
- app.logger.info(" update RKI [done]")
- app.logger.info("------------------------------------------------------------")
- return self
-
- # TODO: #151 refactor RkiBundeslaenderServiceUpdate.__update_who_global_data_short
- def __update_who_global_data_short(self):
- app.logger.info(" update RKI short [begin]")
- app.logger.info("------------------------------------------------------------")
- new_dates_reported_from_import = RkiBundeslaenderImport.get_new_dates_as_array()
- i = 0
- for my_date_reported in new_dates_reported_from_import:
- my_date = RkiDateReported.find_by_date_reported(my_date_reported)
- for result_item in RkiBundeslaenderImport.get_for_one_day(my_date_reported):
- my_country = RkiCountry.find_by_country_code(result_item.country_code)
- o = RkiBundeslaender(
- cases_new=int(result_item.new_cases),
- cases_cumulative=int(result_item.cumulative_cases),
- deaths_new=int(result_item.new_deaths),
- deaths_cumulative=int(result_item.cumulative_deaths),
- date_reported=my_date,
- country=my_country
+ object_id_1=int(result_item.OBJECTID_1),
+ lan_ew_ags=int(result_item.LAN_ew_AGS),
+ lan_ew_gen=result_item.LAN_ew_GEN,
+ lan_ew_bez=result_item.LAN_ew_BEZ,
+ lan_ew_ewz=int(result_item.LAN_ew_EWZ),
+ object_id=int(result_item.OBJECTID),
+ fallzahl=int(result_item.Fallzahl),
+ aktualisierung=result_item.Aktualisierung,
+ ags_txt=int(result_item.AGS_TXT),
+ global_id=result_item.GlobalID, # uuid?
+ faelle_100000_ew=float(result_item.faelle_100000_EW),
+ death=int(result_item.Death),
+ cases7_bl_per_100k=int(result_item.cases7_bl_per_100k),
+ cases7_bl=int(result_item.cases7_bl),
+ death7_bl=int(result_item.death7_bl),
+ cases7_bl_per_100k_txt=result_item.cases7_bl_per_100k_txt,
+ adm_unit_id=int(result_item.AdmUnitId),
+ shape_length=float(result_item.SHAPE_Length),
+ shape_area=float(result_item.SHAPE_Area),
)
db.session.add(o)
result_item.row_imported = True
@@ -162,33 +75,42 @@ class RkiBundeslaenderServiceUpdate:
app.logger.info(" update WHO short ... "+str(i)+" rows")
db.session.commit()
db.session.commit()
- app.logger.info(" update RKI short : "+str(i)+" total rows")
- app.logger.info(" update RKI short [done]")
+ app.logger.info(" update RkiBundeslaender short : "+str(i)+" total rows")
+ app.logger.info(" update RkiBundeslaender short [done]")
app.logger.info("------------------------------------------------------------")
return self
- # TODO: #152 refactor RkiBundeslaenderServiceUpdate.__update_who_global_data_initial
- def __update_who_global_data_initial(self):
+ def __update_data_initial(self):
app.logger.info(" update RKI initial [begin]")
app.logger.info("------------------------------------------------------------")
RkiBundeslaender.remove_all()
- new_dates_reported_from_import = RkiBundeslaenderImport.get_new_dates_as_array()
+ aktualisierungen_from_import = RkiBundeslaenderImport.get_new_aktualisierungen_as_array()
i = 0
- for my_date_reported in new_dates_reported_from_import:
- my_date = RkiDateReported.find_by_date_reported(my_date_reported)
- for result_item in RkiBundeslaenderImport.get_for_one_day(my_date_reported):
- my_country = RkiCountry.find_by_country_code(result_item.country_code)
+ for aktualisierung_from_import in aktualisierungen_from_import:
+ my_date = RkiDateReported.find_by_aktualisierung(aktualisierung_from_import)
+ for result_item in RkiBundeslaenderImport.find_by_aktualisierung(aktualisierung_from_import):
o = RkiBundeslaender(
- cases_new=int(result_item.new_cases),
- cases_cumulative=int(result_item.cumulative_cases),
- deaths_new=int(result_item.new_deaths),
- deaths_cumulative=int(result_item.cumulative_deaths),
- date_reported=my_date,
- country=my_country
+ object_id_1=int(result_item.OBJECTID_1),
+ lan_ew_ags=int(result_item.LAN_ew_AGS),
+ lan_ew_gen=result_item.LAN_ew_GEN,
+ lan_ew_bez=result_item.LAN_ew_BEZ,
+ lan_ew_ewz=int(result_item.LAN_ew_EWZ),
+ object_id=int(result_item.OBJECTID),
+ fallzahl=int(result_item.Fallzahl),
+ aktualisierung=result_item.Aktualisierung,
+ ags_txt=int(result_item.AGS_TXT),
+ global_id=result_item.GlobalID, # uuid?
+ faelle_100000_ew=float(result_item.faelle_100000_EW),
+ death=int(result_item.Death),
+ cases7_bl_per_100k=int(result_item.cases7_bl_per_100k),
+ cases7_bl=int(result_item.cases7_bl),
+ death7_bl=int(result_item.death7_bl),
+ cases7_bl_per_100k_txt=result_item.cases7_bl_per_100k_txt,
+ adm_unit_id=int(result_item.AdmUnitId),
+ shape_length=float(result_item.SHAPE_Length),
+ shape_area=float(result_item.SHAPE_Area),
)
db.session.add(o)
- result_item.row_imported = True
- db.session.add(result_item)
i += 1
if i % 500 == 0:
app.logger.info(" update WHO initial ... "+str(i)+" rows")
@@ -199,64 +121,25 @@ class RkiBundeslaenderServiceUpdate:
app.logger.info("------------------------------------------------------------")
return self
- # TODO: #153 refactor RkiBundeslaenderServiceUpdate.update_db
- def update_db(self):
- app.logger.info(" update db [begin]")
- app.logger.info("------------------------------------------------------------")
- self.__update_who_date_reported()
- self.__update_who_region()
- self.__update_who_country()
- self.__update_who_global_data()
- app.logger.info(" update db [done]")
- app.logger.info("------------------------------------------------------------")
- return self
-
- # TODO: #154 refactor RkiBundeslaenderServiceUpdate.update_db_short
- def update_db_short(self):
- app.logger.info(" update db short [begin]")
- app.logger.info("------------------------------------------------------------")
- self.__update_who_date_reported()
- self.__update_who_region()
- self.__update_who_country()
- self.__update_who_global_data_short()
- app.logger.info(" update db short [done]")
- app.logger.info("------------------------------------------------------------")
- return self
-
- # TODO: #155 refactor RkiBundeslaenderServiceUpdate.update_db_initial
- def update_db_initial(self):
- app.logger.info(" update db initial [begin]")
- app.logger.info("------------------------------------------------------------")
- self.__update_who_date_reported()
- self.__update_who_region()
- self.__update_who_country()
- self.__update_who_global_data_initial()
- app.logger.info(" update db initial [done]")
- app.logger.info("------------------------------------------------------------")
- return self
-
def update_dimension_tables_only(self):
- #TODO: #123 split RkiBundeslaenderService into two Services, one for bundeslaender and one for landkreise
- #TODO: #141 implement RkiBundeslaenderServiceUpdate.update_dimension_tables_only
+ self.__update_date_reported()
return self
def update_fact_table_incremental_only(self):
- #TODO: #123 split RkiBundeslaenderService into two Services, one for bundeslaender and one for landkreise
- #TODO: #142 implement RkiBundeslaenderServiceUpdate.update_fact_table_incremental_only
+ self.__update_data_incremental()
return self
def update_fact_table_initial_only(self):
- #TODO: #123 split RkiBundeslaenderService into two Services, one for bundeslaender and one for landkreise
- #TODO: #143 implement RkiBundeslaenderServiceUpdate.update_fact_table_initial_only
+ self.__update_data_initial()
return self
def update_star_schema_incremental(self):
- #TODO: #123 split RkiBundeslaenderService into two Services, one for bundeslaender and one for landkreise
- #TODO: #144 implement RkiBundeslaenderServiceUpdate.update_star_schema_incremental
+ self.__update_date_reported()
+ self.__update_data_incremental()
return self
def update_star_schema_initial(self):
- #TODO: #123 split RkiBundeslaenderService into two Services, one for bundeslaender and one for landkreise
- #TODO: #145 implement RkiBundeslaenderServiceUpdate.update_star_schema_initial
+ self.__update_date_reported()
+ self.__update_data_initial()
return self
diff --git a/src/covid19/blueprints/who/who_service_download.py b/src/covid19/blueprints/who/who_service_download.py
index cb646d145802458f6be0c064c09ed4d0f2e943f8..50d225668fabc5977b56c027553f835743f8f7ef 100644
--- a/src/covid19/blueprints/who/who_service_download.py
+++ b/src/covid19/blueprints/who/who_service_download.py
@@ -21,7 +21,6 @@ class WhoServiceDownload:
app.logger.info(" download FILE: "+self.cfg.cvsfile_name+" from "+self.cfg.url_src)
app.logger.info("------------------------------------------------------------")
try:
- os.makedirs(self.cfg.data_path, exist_ok=True)
if os.path.isfile(self.cfg.cvsfile_path):
os.remove(self.cfg.cvsfile_path)
data_file = wget.download(self.cfg.url_src, self.cfg.cvsfile_path)
diff --git a/src/covid19/blueprints/who/who_service_update.py b/src/covid19/blueprints/who/who_service_update.py
index 7cfc02b07ab5cf1ddc3eda4c1e338da8425fadef..6e5d80e41fd2b65bd0618aeb93ab38dcfea4583a 100644
--- a/src/covid19/blueprints/who/who_service_update.py
+++ b/src/covid19/blueprints/who/who_service_update.py
@@ -7,15 +7,15 @@ from covid19.blueprints.who.who_service_download import WhoServiceConfig
class WhoServiceUpdate:
def __init__(self, database, config: WhoServiceConfig):
app.logger.debug("------------------------------------------------------------")
- app.logger.debug(" WHO Service Update [init]")
+ app.logger.debug(" WhoServiceUpdate [init]")
app.logger.debug("------------------------------------------------------------")
self.__database = database
self.cfg = config
app.logger.debug("------------------------------------------------------------")
- app.logger.debug(" WHO Service Update [ready]")
+ app.logger.debug(" WhoServiceUpdate [ready]")
- def __update_who_date_reported(self):
- app.logger.info(" __update_who_date_reported [begin]")
+ def __update_date_reported(self):
+ app.logger.info(" WhoServiceUpdate.__update_date_reported [begin]")
app.logger.info("------------------------------------------------------------")
i = 0
for i_date_reported, in WhoImport.get_dates_reported():
@@ -31,12 +31,12 @@ class WhoServiceUpdate:
output += " NOT added "+str(c.id)
app.logger.info(output)
app.logger.info("")
- app.logger.info(" __update_who_date_reported [done]")
+ app.logger.info(" WhoServiceUpdate.__update_date_reported [done]")
app.logger.info("------------------------------------------------------------")
return self
- def __update_who_region(self):
- app.logger.info(" __update_who_region [begin]")
+ def __update_region(self):
+ app.logger.info(" WhoServiceUpdate.__update_who_region [begin]")
app.logger.info("------------------------------------------------------------")
i = 0
for i_who_region, in WhoImport.get_regions():
@@ -52,12 +52,12 @@ class WhoServiceUpdate:
output += " NOT added ( " + str(c.id) + " ) "
app.logger.info(output)
app.logger.info("")
- app.logger.info(" __update_who_region [done]")
+ app.logger.info(" WhoServiceUpdate.__update_who_region [done]")
app.logger.info("------------------------------------------------------------")
return self
- def __update_who_country(self):
- app.logger.info(" __update_who_country [begin]")
+ def __update_country(self):
+ app.logger.info(" WhoServiceUpdate.__update_who_country [begin]")
app.logger.info("------------------------------------------------------------")
result = WhoImport.countries()
i = 0
@@ -85,12 +85,12 @@ class WhoServiceUpdate:
app.logger.info(output)
db.session.commit()
app.logger.info("")
- app.logger.info(" __update_who_country [done]")
+ app.logger.info(" WhoServiceUpdate.__update_who_country [done]")
app.logger.info("------------------------------------------------------------")
return self
- def __update_fact_table_incremental(self):
- app.logger.info(" __update_fact_tables_incremental [begin]")
+ def __update_data_incremental(self):
+ app.logger.info(" WhoServiceUpdate.__update_fact_tables_incremental [begin]")
app.logger.info("------------------------------------------------------------")
new_dates_reported_from_import = WhoImport.get_new_dates_as_array()
i = 0
@@ -123,12 +123,12 @@ class WhoServiceUpdate:
db.session.commit()
app.logger.info(" update WHO incremental ... " + str(i) + " rows [" + str(my_date) + "] (" + str(k) + ")")
app.logger.info(" update WHO incremental : "+str(i)+" rows total")
- app.logger.info(" __update_fact_tables_incremental [done]")
+ app.logger.info(" WhoServiceUpdate.__update_fact_tables_incremental [done]")
app.logger.info("------------------------------------------------------------")
return self
- def __update_fact_table_initial(self):
- app.logger.info(" __update_fact_table_initial [begin]")
+ def __update_data_initial(self):
+ app.logger.info(" WhoServiceUpdate.__update_fact_table_initial [begin]")
app.logger.info("------------------------------------------------------------")
WhoData.remove_all()
new_dates_reported_from_import = WhoImport.get_new_dates_as_array()
@@ -158,14 +158,14 @@ class WhoServiceUpdate:
db.session.commit()
db.session.commit()
app.logger.info(" update WHO initial : "+str(i)+" total rows")
- app.logger.info(" __update_fact_table_initial [done]")
+ app.logger.info(" WhoServiceUpdate.__update_fact_table_initial [done]")
app.logger.info("------------------------------------------------------------")
return self
def __update_dimension_tables(self):
- self.__update_who_date_reported()
- self.__update_who_region()
- self.__update_who_country()
+ self.__update_date_reported()
+ self.__update_region()
+ self.__update_country()
return self
def update_dimension_tables_only(self):
@@ -179,7 +179,7 @@ class WhoServiceUpdate:
def update_fact_table_incremental_only(self):
app.logger.info(" update_fact_tables_incremental_only [begin]")
app.logger.info("------------------------------------------------------------")
- self.__update_fact_table_incremental()
+ self.__update_data_incremental()
app.logger.info(" update_fact_tables_incremental_only [done]")
app.logger.info("------------------------------------------------------------")
return self
@@ -187,7 +187,7 @@ class WhoServiceUpdate:
def update_fact_table_initial_only(self):
app.logger.info(" update_fact_tables_initial_only [begin]")
app.logger.info("------------------------------------------------------------")
- self.__update_fact_table_initial()
+ self.__update_data_initial()
app.logger.info(" update_fact_tables_initial_only [done]")
app.logger.info("------------------------------------------------------------")
return self
@@ -196,7 +196,7 @@ class WhoServiceUpdate:
app.logger.info(" update_star_schema_incremental [begin]")
app.logger.info("------------------------------------------------------------")
self.__update_dimension_tables()
- self.__update_fact_table_incremental()
+ self.__update_data_incremental()
app.logger.info(" update_star_schema_incremental [done]")
app.logger.info("------------------------------------------------------------")
return self
@@ -205,7 +205,7 @@ class WhoServiceUpdate:
app.logger.info(" update_star_schema_initial [begin]")
app.logger.info("------------------------------------------------------------")
self.__update_dimension_tables()
- self.__update_fact_table_initial()
+ self.__update_data_initial()
app.logger.info(" update_star_schema_initial [done]")
app.logger.info("------------------------------------------------------------")
return self