from database import db, app
from flask_covid19.blueprints.app_all.all_config import BlueprintConfig
from flask_covid19.blueprints.app_web.web_model_factory import BlueprintDateReportedFactory
from flask_covid19.blueprints.data_rki.rki_model import RkiData, RkiMeldedatum, RkiDatenstand, RkiRefDatum, RkiAltersgruppe, RkiBundesland, RkiLandkreis
from flask_covid19.blueprints.data_rki.rki_model_import import RkiImport
class RkiServiceUpdateBase:
def __init__(self, database, config: BlueprintConfig):
app.logger.debug("------------------------------------------------------------")
app.logger.debug(" RKI Service Update [init]")
app.logger.debug("------------------------------------------------------------")
self.__database = database
self.cfg = config
app.logger.debug("------------------------------------------------------------")
app.logger.debug(" RKI Service Update [ready]")
class RkiServiceUpdateFull(RkiServiceUpdateBase):
def __full_update_date_datenstand(self):
app.logger.info(" RkiServiceUpdateFull.__full_update_date_datenstand [begin]")
app.logger.info("------------------------------------------------------------")
RkiDatenstand.remove_all()
i = 0
output_lines = []
for datum_of_import in RkiImport.get_date_datenstand_of_all_import():
i += 1
o = BlueprintDateReportedFactory.create_new_object_for_rki_date_datenstand(my_date_rep=datum_of_import)
db.session.add(o)
output = " [ " + str(i) + " ] full update RKI date_datenstand ... " + str(o)
output_lines.append(output)
db.session.commit()
for output in output_lines:
app.logger.info(output)
app.logger.info("")
app.logger.info(" RkiServiceUpdateFull.__full_update_date_datenstand [done]")
app.logger.info("------------------------------------------------------------")
return self
def __full_update_date_ref_datum(self):
app.logger.info(" RkiServiceUpdateFull.__full_update_date_ref_datum [begin]")
app.logger.info("------------------------------------------------------------")
RkiRefDatum.remove_all()
i = 0
output_lines = []
for datum_of_import in RkiImport.get_date_ref_datum_of_all_import():
i += 1
o = BlueprintDateReportedFactory.create_new_object_for_rki_ref_datum(my_date_rep=datum_of_import)
db.session.add(o)
output = " [ " + str(i) + " ] full update RKI date_ref_datum ... " + str(o)
output_lines.append(output)
db.session.commit()
for output in output_lines:
app.logger.info(output)
app.logger.info("")
app.logger.info(" RkiServiceUpdateFull.__full_update_date_ref_datum [done]")
app.logger.info("------------------------------------------------------------")
return self
def __full_update_date_reported(self):
app.logger.info(" RkiServiceUpdateFull.__full_update_date_reported [begin]")
app.logger.info("------------------------------------------------------------")
RkiMeldedatum.remove_all()
i = 0
output_lines = []
for datum_of_import in RkiImport.get_datum_of_all_import():
i += 1
o = BlueprintDateReportedFactory.create_new_object_for_rki(my_date_rep=datum_of_import)
db.session.add(o)
output = " [ " + str(i) + " ] full update RKI meldedatum ... " + str(o)
output_lines.append(output)
db.session.commit()
for output in output_lines:
app.logger.info(output)
app.logger.info("")
app.logger.info(" RkiServiceUpdateFull.__full_update_date_reported [done]")
app.logger.info("------------------------------------------------------------")
return self
def __full_update_altersgruppe(self):
app.logger.info(" RkiServiceUpdateFull.__full_update_altersgruppe [begin]")
app.logger.info("------------------------------------------------------------")
RkiAltersgruppe.remove_all()
app.logger.info("")
i = 0
output_lines = []
for altersgruppe_of_import in RkiImport.get_altersgruppe_list():
i += 1
my_altersgruppe = altersgruppe_of_import[0]
o = RkiAltersgruppe(
altersgruppe=my_altersgruppe,
processed_update=False,
processed_full_update=False,
)
db.session.add(o)
output = " [ " + str(i) + " ] full update RKI altersgruppe ... " + str(o)
output_lines.append(output)
db.session.commit()
for output in output_lines:
app.logger.info(output)
app.logger.info(" RkiServiceUpdateFull.__full_update_altersgruppe [done]")
app.logger.info("------------------------------------------------------------")
return self
def __full_update_bundesland(self):
app.logger.info(" RkiServiceUpdateFull.__full_update_bundesland [begin]")
app.logger.info("------------------------------------------------------------")
RkiBundesland.remove_all()
app.logger.info("")
i = 0
output_lines = []
for bundesland_of_import in RkiImport.get_bundesland_list():
i += 1
b1 = bundesland_of_import[0]
b2 = bundesland_of_import[1]
o = RkiBundesland(
id_bundesland=b2,
location_group=b1,
processed_update=False,
processed_full_update=False,
)
db.session.add(o)
output = " [ " + str(i) + " ] full update RKI bundesland ... " + str(o)
output_lines.append(output)
db.session.commit()
for output in output_lines:
app.logger.info(output)
app.logger.info(" RkiServiceUpdateFull.__full_update_bundesland [done]")
app.logger.info("------------------------------------------------------------")
return self
def __full_update_landkreis(self):
RkiLandkreis.remove_all()
self.__full_update_bundesland()
app.logger.info(" RkiServiceUpdateFull.__full_update_landkreis [begin]")
app.logger.info("------------------------------------------------------------")
i = 0
output_lines = []
for bundesland in RkiBundesland.find_all():
for landkreis_from_import in RkiImport.get_landkreis_for_bundesland(bundesland=bundesland.location_group):
i += 1
# app.logger.info("landkreis_from_import: "+str(landkreis_from_import))
my_location_tmp = landkreis_from_import[0].split(" ")
my_id_landkreis = landkreis_from_import[1]
my_location_type = my_location_tmp[0]
my_location = my_location_tmp[1]
o = RkiLandkreis(
location=my_location,
id_landkreis=my_id_landkreis,
location_type=my_location_type,
location_group=bundesland,
processed_update=False,
processed_full_update=True,
)
db.session.add(o)
output = " [ " + str(i) + " ] full update RKI landkreis ... " + str(o)
output_lines.append(output)
db.session.commit()
for output in output_lines:
app.logger.info(output)
app.logger.info("------------------------------------------------------------")
app.logger.info("")
app.logger.info(" RkiServiceUpdateFull.__full_update_landkreis [done]")
app.logger.info("------------------------------------------------------------")
return self
def __full_update_data(self):
app.logger.info(" RkiServiceUpdateFull.__full_update_data [begin]")
app.logger.info("------------------------------------------------------------")
RkiData.remove_all()
i = 0
last_rki_data = None
dict_locations = RkiLandkreis.find_all_as_dict()
dict_altersgruppen = RkiAltersgruppe.find_all_as_dict()
dict_ref_datum = RkiRefDatum.find_all_as_dict()
dict_datenstand = RkiDatenstand.find_all_as_dict()
# for l_key in locations.keys():
# app.logger.info(" location: " + str(l_key) + " -> " + str(locations[l_key]))
# app.logger.info("------------------------------------------------------------")
for my_meldedatum in RkiMeldedatum.find_all():
d = my_meldedatum.datum
# app.logger.info(" my_meldedatum: " + str(my_meldedatum) + " " + d.isoformat())
# app.logger.info("------------------------------------------------------------")
list_imports = RkiImport.find_by_datum(my_datum=d)
# if l_imports is None:
# app.logger.info("list_imports is None ")
# else:
# nr = len(list_imports)
# app.logger.info("len(list_imports): " + str(nr))
# app.logger.info("------------------------------------------------------------")
for o_import in list_imports:
# app.logger.info("o_import.landkreis " + o_import.landkreis)
key_location = o_import.landkreis.split(" ")[1]
key_altersgruppen = o_import.altersgruppe
key_datenstand = o_import.datenstand
key_ref_datum = o_import.ref_datum
my_altersgruppe = dict_altersgruppen[key_altersgruppen]
my_landkreis = dict_locations[key_location]
my_datenstand = dict_datenstand[key_datenstand]
my_ref_datum = dict_ref_datum[key_ref_datum]
# app.logger.info(str(landkreis))
o = RkiData(
date_reported=my_meldedatum,
datenstand=my_datenstand,
location=my_landkreis,
ref_datum=my_ref_datum,
fid=o_import.fid,
geschlecht=o_import.geschlecht,
anzahl_fall=o_import.anzahl_fall,
anzahl_todesfall=o_import.anzahl_todesfall,
neuer_fall=o_import.neuer_fall,
neuer_todesfall=o_import.neuer_todesfall,
neu_genesen=o_import.neu_genesen,
anzahl_genesen=o_import.anzahl_genesen,
ist_erkrankungsbeginn=o_import.ist_erkrankungsbeginn,
altersgruppe2=o_import.altersgruppe2,
processed_update=False,
processed_full_update=True,
)
if last_rki_data == o:
last_rki_data.altersgruppe.append(my_altersgruppe)
else:
o.altersgruppen.append(my_altersgruppe)
db.session.add(o)
last_rki_data = o
i += 1
if i % 2000 == 0:
app.logger.info(" full update RKI data ... "+str(i)+" rows")
db.session.commit()
db.session.commit()
app.logger.info(" full update RKI data ... "+str(i)+" total rows")
app.logger.info("")
app.logger.info(" RkiServiceUpdateFull.__full_update_data [done]")
app.logger.info("------------------------------------------------------------")
return self
def full_update_dimension_tables(self):
RkiData.remove_all()
self.__full_update_date_reported()
self.__full_update_date_datenstand()
self.__full_update_date_ref_datum()
self.__full_update_altersgruppe()
self.__full_update_landkreis()
return self
def full_update_fact_table(self):
RkiData.remove_all()
self.__full_update_data()
return self
def full_update_star_schema(self):
self.full_update_dimension_tables()
self.full_update_fact_table()
return self
class RkiServiceUpdate(RkiServiceUpdateBase):
def __update_date_reported(self):
app.logger.info(" RkiServiceUpdate.__update_date_reported [begin]")
app.logger.info("------------------------------------------------------------")
i = 0
for aktualisierung in RkiImport.get_aktualisierungen_as_array():
i += 1
output = " [ " + str(i) + " ] " + aktualisierung
c = RkiMeldedatum.find_by_date_reported(aktualisierung)
if c is None:
o = RkiMeldedatum.create_new_object_factory(aktualisierung=aktualisierung)
db.session.add(o)
db.session.commit()
output += " added"
else:
output += " NOT added " + str(c.id)
app.logger.info(output)
db.session.commit()
app.logger.info("")
app.logger.info(" RkiServiceUpdate.__update_date_reported [done]")
app.logger.info("------------------------------------------------------------")
return self
def __update_data(self):
app.logger.info(" RkiServiceUpdate.__update_data [begin]")
app.logger.info("------------------------------------------------------------")
aktualisierungen_from_import = RkiImport.get_aktualisierungen_as_array()
i = 0
for aktualisierung_from_import in aktualisierungen_from_import:
my_date = RkiMeldedatum.find_by_aktualisierung(aktualisierung_from_import)
for result_item in RkiImport.find_by_aktualisierung(aktualisierung_from_import):
o = RkiData(
object_id_1=int(result_item.OBJECTID_1),
lan_ew_ags=int(result_item.LAN_ew_AGS),
lan_ew_gen=result_item.LAN_ew_GEN,
lan_ew_bez=result_item.LAN_ew_BEZ,
lan_ew_ewz=int(result_item.LAN_ew_EWZ),
object_id=int(result_item.OBJECTID),
fallzahl=int(result_item.Fallzahl),
aktualisierung=result_item.Aktualisierung,
ags_txt=int(result_item.AGS_TXT),
global_id=result_item.GlobalID, # uuid?
faelle_100000_ew=float(result_item.faelle_100000_EW),
death=int(result_item.Death),
cases7_bl_per_100k=int(result_item.cases7_bl_per_100k),
cases7_bl=int(result_item.cases7_bl),
death7_bl=int(result_item.death7_bl),
cases7_bl_per_100k_txt=result_item.cases7_bl_per_100k_txt,
adm_unit_id=int(result_item.AdmUnitId),
shape_length=float(result_item.SHAPE_Length),
shape_area=float(result_item.SHAPE_Area),
)
db.session.add(o)
i += 1
if i % 500 == 0:
app.logger.info(" update RkiBundeslaender short ... "+str(i)+" rows")
db.session.commit()
db.session.commit()
app.logger.info(" update RkiBundeslaender short : "+str(i)+" total rows")
app.logger.info(" RkiServiceUpdate.__update_data [done]")
app.logger.info("------------------------------------------------------------")
return self
def update_dimension_tables(self):
self.__update_date_reported()
return self
def update_fact_table(self):
self.__update_data()
return self
def update_star_schema(self):
self.__update_date_reported()
self.__update_data()
return self