From 6890b341c4101f4b1472274011cfb08e41be7291 Mon Sep 17 00:00:00 2001 From: thomaswoehlke <thomas.woehlke@gmail.com> Date: Fri, 19 Feb 2021 23:12:50 +0100 Subject: [PATCH] ### 0.0.18 Release --- ...OVID19__2021-02-18__landkreise____tmp.csv} | 24 ++- .../rki_bundeslaender_service_update.py | 4 +- .../rki_landkreise/rki_landkreise_model.py | 60 +++--- .../rki_landkreise_model_import.py | 12 ++ .../rki_landkreise_service_update.py | 171 +++++++++--------- 5 files changed, 151 insertions(+), 120 deletions(-) rename data/{RKI_COVID19__2021-02-18__landkreise.csv => RKI_COVID19__2021-02-18__landkreise____tmp.csv} (99%) diff --git a/data/RKI_COVID19__2021-02-18__landkreise.csv b/data/RKI_COVID19__2021-02-18__landkreise____tmp.csv similarity index 99% rename from data/RKI_COVID19__2021-02-18__landkreise.csv rename to data/RKI_COVID19__2021-02-18__landkreise____tmp.csv index e788bc81..bd60c3f8 100644 --- a/data/RKI_COVID19__2021-02-18__landkreise.csv +++ b/data/RKI_COVID19__2021-02-18__landkreise____tmp.csv @@ -1,5 +1,25 @@ -OBJECTID,ADE,GF,BSG,RS,AGS,SDV_RS,GEN,BEZ,IBZ,BEM,NBD,SN_L,SN_R,SN_K,SN_V1,SN_V2,SN_G,FK_S3,NUTS,RS_0,AGS_0,WSK,EWZ,KFL,DEBKG_ID,death_rate,cases,deaths,cases_per_100k,cases_per_population,BL,BL_ID,county,last_update,cases7_per_100k,recovered,EWZ_BL,cases7_bl_per_100k,cases7_bl,death7_bl,cases7_lk,death7_lk,cases7_per_100k_txt,AdmUnitId,SHAPE_Length,SHAPE_Area -1,4,4,1,01001,01001,010010000000,Flensburg,Kreisfreie Stadt,40,--,ja,01,0,01,00,00,000,R,DEF01,010010000000,01001000,2008/01/01 00:00:00.000,90164,56.73,DEBKGDL20000002R,2.01005025125628,1393,28,1544.96251275454,1.54496251275454,Schleswig-Holstein,1,SK Flensburg,"18.02.2021, 00:00 Uhr",185.218047114148,,2903773,52.3801275099672,1521,3,167,0,"185,2",1001,0.524721434982726,0.00687272234946988 +OBJECTID,ADE,GF,BSG,RS,AGS,SDV_RS,GEN,BEZ,IBZ, +1,4,4,1,01001,01001,010010000000,Flensburg,Kreisfreie Stadt,40, + +BEM,NBD,SN_L,SN_R,SN_K,SN_V1,SN_V2,SN_G,FK_S3,NUTS, +--,ja,01,0,01,00,00,000,R,DEF01,010010000000,01001000, + +RS_0,AGS_0,WSK,EWZ,KFL,DEBKG_ID,death_rate,cases,deaths,cases_per_100k, +2008/01/01 00:00:00.000,90164,56.73,DEBKGDL20000002R,2.01005025125628,1393,28,1544.96251275454, + +cases_per_population,BL,BL_ID,county,last_update, +1.54496251275454,Schleswig-Holstein,1,SK Flensburg,"18.02.2021, 00:00 Uhr", + +cases7_per_100k,recovered,EWZ_BL,cases7_bl_per_100k, +185.218047114148,,2903773,52.3801275099672, + +cases7_bl,death7_bl,cases7_lk,death7_lk,cases7_per_100k_txt,AdmUnitId, +1521,3,167,0,"185,2",1001, + +SHAPE_Length,SHAPE_Area +0.524721434982726,0.00687272234946988 + + 2,4,4,1,01002,01002,010020000000,Kiel,Kreisfreie Stadt,40,--,ja,01,0,02,00,00,000,R,DEF02,010020000000,01002000,2006/01/01 00:00:00.000,246794,118.65,DEBKGDL20000E43X,2.43751928417155,3241,79,1313.24100261757,1.31324100261757,Schleswig-Holstein,1,SK Kiel,"18.02.2021, 00:00 Uhr",37.2780537614367,,2903773,52.3801275099672,1521,3,92,0,"37,3",1002,1.2746835038485,0.0155068431388598 3,4,4,1,01003,01003,010030000000,Lübeck,Kreisfreie Stadt,40,--,ja,01,0,03,00,00,000,R,DEF03,010030000000,01003000,2006/02/01 00:00:00.000,216530,214.19,DEBKGDL20000DYM9,1.68674698795181,4150,70,1916.59354361982,1.91659354361982,Schleswig-Holstein,1,SK Lübeck,"18.02.2021, 00:00 Uhr",69.2744654320418,,2903773,52.3801275099672,1521,3,150,1,"69,3",1003,1.83453358761886,0.0289275157001577 4,4,4,1,01004,01004,010040000000,Neumünster,Kreisfreie Stadt,40,--,ja,01,0,04,00,00,000,R,DEF04,010040000000,01004000,1970/04/26 00:00:00.000,80196,71.66,DEBKGDL20000E4KX,1.85701021355617,1077,20,1342.95974861589,1.34295974861589,Schleswig-Holstein,1,SK Neumünster,"18.02.2021, 00:00 Uhr",27.43278966532,,2903773,52.3801275099672,1521,3,22,0,"27,4",1004,0.663261867357153,0.00980836750335333 diff --git a/src/covid19/blueprints/rki_bundeslaender/rki_bundeslaender_service_update.py b/src/covid19/blueprints/rki_bundeslaender/rki_bundeslaender_service_update.py index eb5781cd..01bfc549 100644 --- a/src/covid19/blueprints/rki_bundeslaender/rki_bundeslaender_service_update.py +++ b/src/covid19/blueprints/rki_bundeslaender/rki_bundeslaender_service_update.py @@ -68,11 +68,9 @@ class RkiBundeslaenderServiceUpdate: shape_area=float(result_item.SHAPE_Area), ) db.session.add(o) - result_item.row_imported = True - db.session.add(result_item) i += 1 if i % 500 == 0: - app.logger.info(" update WHO short ... "+str(i)+" rows") + app.logger.info(" update RkiBundeslaender short ... "+str(i)+" rows") db.session.commit() db.session.commit() app.logger.info(" update RkiBundeslaender short : "+str(i)+" total rows") diff --git a/src/covid19/blueprints/rki_landkreise/rki_landkreise_model.py b/src/covid19/blueprints/rki_landkreise/rki_landkreise_model.py index fc81cdcb..8c2d1016 100644 --- a/src/covid19/blueprints/rki_landkreise/rki_landkreise_model.py +++ b/src/covid19/blueprints/rki_landkreise/rki_landkreise_model.py @@ -9,53 +9,59 @@ class RkiLandkreise(db.Model): __tablename__ = 'rki_landkreise' id = db.Column(db.Integer, primary_key=True) - objectid = db.Column(db.String(255), nullable=False) - ade = db.Column(db.String(255), nullable=False) - gf = db.Column(db.String(255), nullable=False) - bsg = db.Column(db.String(255), nullable=False) + + objectid = db.Column(db.Integer, nullable=False) + ade = db.Column(db.Integer, nullable=False) + gf = db.Column(db.Integer, nullable=False) + bsg = db.Column(db.Integer, nullable=False) rs = db.Column(db.String(255), nullable=False) ags = db.Column(db.String(255), nullable=False) sdv_rs = db.Column(db.String(255), nullable=False) gen = db.Column(db.String(255), nullable=False) bez = db.Column(db.String(255), nullable=False) - ibz = db.Column(db.String(255), nullable=False) + ibz = db.Column(db.Integer, nullable=False) + bem = db.Column(db.String(255), nullable=False) nbd = db.Column(db.String(255), nullable=False) sn_l = db.Column(db.String(255), nullable=False) - sn_r = db.Column(db.String(255), nullable=False) + sn_r = db.Column(db.Integer, nullable=False) sn_k = db.Column(db.String(255), nullable=False) sn_v1 = db.Column(db.String(255), nullable=False) sn_v2 = db.Column(db.String(255), nullable=False) sn_g = db.Column(db.String(255), nullable=False) fk_s3 = db.Column(db.String(255), nullable=False) nuts = db.Column(db.String(255), nullable=False) + rs_0 = db.Column(db.String(255), nullable=False) - ags_0 = db.Column(db.String(255), nullable=False) - wsk = db.Column(db.String(255), nullable=False) - ewz = db.Column(db.String(255), nullable=False) - kfl = db.Column(db.String(255), nullable=False) + ags_0 = db.Column(db.Integer, nullable=False) + wsk = db.Column(db.Float, nullable=False) + ewz = db.Column(db.Integer, nullable=False) + kfl = db.Column(db.Float, nullable=False) debkg_id = db.Column(db.String(255), nullable=False) - death_rate = db.Column(db.String(255), nullable=False) - cases = db.Column(db.String(255), nullable=False) - deaths = db.Column(db.String(255), nullable=False) - cases_per_100k = db.Column(db.String(255), nullable=False) - cases_per_population = db.Column(db.String(255), nullable=False) + death_rate = db.Column(db.Float, nullable=False) + cases = db.Column(db.Integer, nullable=False) + deaths = db.Column(db.Integer, nullable=False) + cases_per_100k = db.Column(db.Float, nullable=False) + + cases_per_population = db.Column(db.Float, nullable=False) bl = db.Column(db.String(255), nullable=False) - bl_id = db.Column(db.String(255), nullable=False) + bl_id = db.Column(db.Integer, nullable=False) county = db.Column(db.String(255), nullable=False) last_update = db.Column(db.String(255), nullable=False) - cases7_per_100k = db.Column(db.String(255), nullable=False) - recovered = db.Column(db.String(255), nullable=False) - ewz_bl = db.Column(db.String(255), nullable=False) - cases7_bl_per_100k = db.Column(db.String(255), nullable=False) - cases7_bl = db.Column(db.String(255), nullable=False) - death7_bl = db.Column(db.String(255), nullable=False) - cases7_lk = db.Column(db.String(255), nullable=False) - death7_lk = db.Column(db.String(255), nullable=False) + + cases7_per_100k = db.Column(db.Float, nullable=False) + recovered = db.Column(db.Integer, nullable=False) + ewz_bl = db.Column(db.Integer, nullable=False) + cases7_bl_per_100k = db.Column(db.Float, nullable=False) + + cases7_bl = db.Column(db.Integer, nullable=False) + death7_bl = db.Column(db.Integer, nullable=False) + cases7_lk = db.Column(db.Integer, nullable=False) + death7_lk = db.Column(db.Integer, nullable=False) cases7_per_100k_txt = db.Column(db.String(255), nullable=False) - adm_unit_id = db.Column(db.String(255), nullable=False) - shape_length = db.Column(db.String(255), nullable=False) - shape_area = db.Column(db.String(255), nullable=False) + adm_unit_id = db.Column(db.Integer, nullable=False) + shape_length = db.Column(db.Float, nullable=False) + shape_area = db.Column(db.Float, nullable=False) @classmethod def remove_all(cls): diff --git a/src/covid19/blueprints/rki_landkreise/rki_landkreise_model_import.py b/src/covid19/blueprints/rki_landkreise/rki_landkreise_model_import.py index 0e8924a3..16f27caf 100644 --- a/src/covid19/blueprints/rki_landkreise/rki_landkreise_model_import.py +++ b/src/covid19/blueprints/rki_landkreise/rki_landkreise_model_import.py @@ -77,3 +77,15 @@ class RkiLandkreiseImport(db.Model): @classmethod def find_by_id(cls, other_id): return db.session.query(cls).filter(cls.id == other_id).one_or_none() + + @classmethod + def find_by_last_update(cls, last_update_from_import: str): + return [] + + @classmethod + def get_last_updates(cls): + return [] + + @classmethod + def get_new_last_update_as_array(cls): + return [] diff --git a/src/covid19/blueprints/rki_landkreise/rki_landkreise_service_update.py b/src/covid19/blueprints/rki_landkreise/rki_landkreise_service_update.py index fb3d9b99..85a7eb0a 100644 --- a/src/covid19/blueprints/rki_landkreise/rki_landkreise_service_update.py +++ b/src/covid19/blueprints/rki_landkreise/rki_landkreise_service_update.py @@ -17,188 +17,183 @@ class RkiLandkreiseServiceUpdate: app.logger.debug(" RKI Service Update [ready]") def __update_date_reported(self): - app.logger.info(" update who_date_reported [begin]") + app.logger.info(" update RkiLandkreiseServiceUpdate [begin]") app.logger.info("------------------------------------------------------------") i = 0 - for i_date_reported, in RkiLandkreiseImport.get_dates_reported(): - c = RkiDateReported.find_by_date_reported(i_date_reported) + for last_update in RkiLandkreiseImport.get_last_updates(): + i += 1 + output = " [ " + str(i) + " ] " + last_update + c = RkiDateReported.find_by_date_reported(last_update) if c is None: - o = RkiDateReported.create_new_object_factory(my_date_rep=i_date_reported) + o = RkiDateReported.create_new_object_factory(aktualisierung=last_update) db.session.add(o) - app.logger.info(" update who_date_reported "+i_date_reported+" added NEW") - if i % 10 == 0: - app.logger.info(" update who_date_reported "+i_date_reported+" not added") db.session.commit() - i += 1 - db.session.commit() + output += " added" + else: + output += " NOT added " + str(c.id) + app.logger.info(output) app.logger.info("") - app.logger.info(" update who_date_reported [done]") + app.logger.info(" update RkiLandkreiseServiceUpdate [done]") app.logger.info("------------------------------------------------------------") return self def __update_data_incremental(self): - app.logger.info(" update RKI short [begin]") + app.logger.info(" update RkiLandkreiseServiceUpdate short [begin]") app.logger.info("------------------------------------------------------------") - new_dates_reported_from_import = RkiLandkreiseImport.get_new_dates_as_array() + new_dates_reported_from_import = RkiLandkreiseImport.get_new_last_update_as_array() i = 0 for my_date_reported in new_dates_reported_from_import: my_date = RkiDateReported.find_by_date_reported(my_date_reported) - for result_item in RkiLandkreiseImport.get_for_one_day(my_date_reported): + for result_item in RkiLandkreiseImport.find_by_last_update(my_date_reported): o = RkiLandkreise( objectid=int(result_item.objectid), ade=int(result_item.ade), gf=int(result_item.gf), bsg=int(result_item.bsg), - rs=int(result_item.rs), - ags=int(result_item.ags), - sdv_rs=int(result_item.sdv_rs), - gen=int(result_item.gen), - bez=int(result_item.bez), + rs=result_item.rs, + ags=result_item.ags, + sdv_rs=result_item.sdv_rs, + gen=result_item.gen, + bez=result_item.bez, ibz=int(result_item.ibz), - bem=int(result_item.bem), - nbd=int(result_item.nbd), - sn_l=int(result_item.sn_l), + bem=result_item.bem, + nbd=result_item.nbd, + sn_l=result_item.sn_l, sn_r=int(result_item.sn_r), - sn_k=int(result_item.sn_k), - sn_v1=int(result_item.sn_v1), - sn_v2=int(result_item.sn_v2), - sn_g=int(result_item.sn_g), - fk_s3=int(result_item.fk_s3), - nuts=int(result_item.nuts), - rs_0=int(result_item.rs_0), + sn_k=result_item.sn_k, + sn_v1=result_item.sn_v1, + sn_v2=result_item.sn_v2, + sn_g=result_item.sn_g, + fk_s3=result_item.fk_s3, + nuts=result_item.nuts, + rs_0=result_item.rs_0, ags_0=int(result_item.ags_0), wsk=int(result_item.wsk), ewz=int(result_item.ewz), kfl=int(result_item.kfl), - debkg_id=int(result_item.debkg_id), + debkg_id=result_item.debkg_id, death_rate=int(result_item.death_rate), cases=int(result_item.cases), deaths=int(result_item.deaths), cases_per_100k=int(result_item.cases_per_100k), - cases_per_population=int(result_item.cases_per_population), - bl=int(result_item.bl), - bl_id=int(result_item.bl_id), - county=int(result_item.county), - last_update=int(result_item.last_update), + cases_per_population=float(result_item.cases_per_population), + bl=result_item.bl, + bl_id=result_item.bl_id, + county=result_item.county, + last_update=result_item.last_update, cases7_per_100k=int(result_item.cases7_per_100k), recovered=int(result_item.recovered), ewz_bl=int(result_item.ewz_bl), - cases7_bl_per_100k=int(result_item.cases7_bl_per_100k), + cases7_bl_per_100k=float(result_item.cases7_bl_per_100k), cases7_bl=int(result_item.cases7_bl), death7_bl=int(result_item.death7_bl), cases7_lk=int(result_item.cases7_lk), death7_lk=int(result_item.death7_lk), - cases7_per_100k_txt=int(result_item.cases7_per_100k_txt), + cases7_per_100k_txt=result_item.cases7_per_100k_txt, adm_unit_id=int(result_item.adm_unit_id), - shape_length=int(result_item.shape_length), - shape_area=int(result_item.shape_area), + shape_length=float(result_item.shape_length), + shape_area=float(result_item.shape_area), ) db.session.add(o) - result_item.row_imported = True - db.session.add(result_item) i += 1 if i % 500 == 0: - app.logger.info(" update WHO short ... "+str(i)+" rows") + app.logger.info(" update RkiLandkreiseServiceUpdate short ... "+str(i)+" rows") db.session.commit() db.session.commit() - app.logger.info(" update RKI short : "+str(i)+" total rows") - app.logger.info(" update RKI short [done]") + app.logger.info(" update RkiLandkreiseServiceUpdate short : "+str(i)+" total rows") + app.logger.info(" update RkiLandkreiseServiceUpdate short [done]") app.logger.info("------------------------------------------------------------") return self def __update_data_initial(self): - app.logger.info(" update RKI initial [begin]") + app.logger.info(" update RkiLandkreiseServiceUpdate initial [begin]") app.logger.info("------------------------------------------------------------") RkiLandkreise.remove_all() - new_dates_reported_from_import = RkiLandkreiseImport.get_new_dates_as_array() + last_updates_from_import = RkiLandkreiseImport.get_last_updates() i = 0 - for my_date_reported in new_dates_reported_from_import: - my_date = RkiDateReported.find_by_date_reported(my_date_reported) - for result_item in RkiLandkreiseImport.get_for_one_day(my_date_reported): + for last_update_from_import in last_updates_from_import: + my_date = RkiDateReported.find_by_date_reported(last_update_from_import) + for result_item in RkiLandkreiseImport.find_by_last_update(last_update_from_import): o = RkiLandkreise( objectid=int(result_item.objectid), ade=int(result_item.ade), gf=int(result_item.gf), bsg=int(result_item.bsg), - rs=int(result_item.rs), - ags=int(result_item.ags), - sdv_rs=int(result_item.sdv_rs), - gen=int(result_item.gen), - bez=int(result_item.bez), + rs=result_item.rs, + ags=result_item.ags, + sdv_rs=result_item.sdv_rs, + gen=result_item.gen, + bez=result_item.bez, ibz=int(result_item.ibz), - bem=int(result_item.bem), - nbd=int(result_item.nbd), - sn_l=int(result_item.sn_l), + bem=result_item.bem, + nbd=result_item.nbd, + sn_l=result_item.sn_l, sn_r=int(result_item.sn_r), - sn_k=int(result_item.sn_k), - sn_v1=int(result_item.sn_v1), - sn_v2=int(result_item.sn_v2), - sn_g=int(result_item.sn_g), - fk_s3=int(result_item.fk_s3), - nuts=int(result_item.nuts), - rs_0=int(result_item.rs_0), + sn_k=result_item.sn_k, + sn_v1=result_item.sn_v1, + sn_v2=result_item.sn_v2, + sn_g=result_item.sn_g, + fk_s3=result_item.fk_s3, + nuts=result_item.nuts, + rs_0=result_item.rs_0, ags_0=int(result_item.ags_0), wsk=int(result_item.wsk), ewz=int(result_item.ewz), kfl=int(result_item.kfl), - debkg_id=int(result_item.debkg_id), + debkg_id=result_item.debkg_id, death_rate=int(result_item.death_rate), cases=int(result_item.cases), deaths=int(result_item.deaths), cases_per_100k=int(result_item.cases_per_100k), - cases_per_population=int(result_item.cases_per_population), - bl=int(result_item.bl), - bl_id=int(result_item.bl_id), - county=int(result_item.county), - last_update=int(result_item.last_update), + cases_per_population=float(result_item.cases_per_population), + bl=result_item.bl, + bl_id=result_item.bl_id, + county=result_item.county, + last_update=result_item.last_update, cases7_per_100k=int(result_item.cases7_per_100k), recovered=int(result_item.recovered), ewz_bl=int(result_item.ewz_bl), - cases7_bl_per_100k=int(result_item.cases7_bl_per_100k), + cases7_bl_per_100k=float(result_item.cases7_bl_per_100k), cases7_bl=int(result_item.cases7_bl), death7_bl=int(result_item.death7_bl), cases7_lk=int(result_item.cases7_lk), death7_lk=int(result_item.death7_lk), - cases7_per_100k_txt=int(result_item.cases7_per_100k_txt), + cases7_per_100k_txt=result_item.cases7_per_100k_txt, adm_unit_id=int(result_item.adm_unit_id), - shape_length=int(result_item.shape_length), - shape_area=int(result_item.shape_area), + shape_length=float(result_item.shape_length), + shape_area=float(result_item.shape_area), ) db.session.add(o) - result_item.row_imported = True - db.session.add(result_item) i += 1 if i % 500 == 0: - app.logger.info(" update WHO initial ... "+str(i)+" rows") + app.logger.info(" update RkiLandkreiseServiceUpdate initial ... "+str(i)+" rows") db.session.commit() db.session.commit() - app.logger.info(" update WHO initial : "+str(i)+" total rows") - app.logger.info(" update WHO initial [done]") + app.logger.info(" update RkiLandkreiseServiceUpdate initial : "+str(i)+" total rows") + app.logger.info(" update RkiLandkreiseServiceUpdate initial [done]") app.logger.info("------------------------------------------------------------") return self def update_dimension_tables_only(self): - #TODO: #123 split RkiBundeslaenderService into two Services, one for bundeslaender and one for landkreise - #TODO: #141 implement RkiBundeslaenderServiceUpdate.update_dimension_tables_only + self.__update_date_reported() return self def update_fact_table_incremental_only(self): - #TODO: #123 split RkiBundeslaenderService into two Services, one for bundeslaender and one for landkreise - #TODO: #142 implement RkiBundeslaenderServiceUpdate.update_fact_table_incremental_only + self.__update_data_incremental() return self def update_fact_table_initial_only(self): - #TODO: #123 split RkiBundeslaenderService into two Services, one for bundeslaender and one for landkreise - #TODO: #143 implement RkiBundeslaenderServiceUpdate.update_fact_table_initial_only + self.__update_data_initial() return self def update_star_schema_incremental(self): - #TODO: #123 split RkiBundeslaenderService into two Services, one for bundeslaender and one for landkreise - #TODO: #144 implement RkiBundeslaenderServiceUpdate.update_star_schema_incremental + self.__update_date_reported() + self.__update_data_incremental() return self def update_star_schema_initial(self): - #TODO: #123 split RkiBundeslaenderService into two Services, one for bundeslaender and one for landkreise - #TODO: #145 implement RkiBundeslaenderServiceUpdate.update_star_schema_initial + self.__update_date_reported() + self.__update_data_initial() return self + -- GitLab