From 38ca7aedaf27f65db8f93f3bf01a36ff69af636c Mon Sep 17 00:00:00 2001
From: thomaswoehlke <thomas.woehlke@gmail.com>
Date: Tue, 19 Jan 2021 13:52:04 +0100
Subject: [PATCH] ### 0.0.8 Release * Fixed #13 /who/imported/ * Fixed #14
/europe/imported/ * Fixed #15 /who/update: Download * Fixed #16 /who/update:
Import File to DB * Issue #17 /who/update: Update DB * Issue #3 ORM: 3NF for
ecdc_europa_data_import * Issue #4 data update for 3NF
ecdc_europa_data_import
---
org/woehlke/covid19/europe/europe_model.py | 15 ++--
org/woehlke/covid19/europe/europe_service.py | 68 -------------------
.../covid19/europe/europe_service_update.py | 11 +++
3 files changed, 20 insertions(+), 74 deletions(-)
diff --git a/org/woehlke/covid19/europe/europe_model.py b/org/woehlke/covid19/europe/europe_model.py
index ffc93156..ddd018ba 100644
--- a/org/woehlke/covid19/europe/europe_model.py
+++ b/org/woehlke/covid19/europe/europe_model.py
@@ -34,15 +34,18 @@ class EuropeDataImportTable(db.Model):
def get_by_id(cls, other_id):
return db.session.query(cls).filter(cls.id == other_id).one()
+ @classmethod
+ def get_date_rep(cls):
+ sql = "select distinct date_rep, year_week from europe_data_import order by year_week desc"
+ return db.session.execute(sql).all()
+
class EuropeDateReported(db.Model):
__tablename__ = 'europe_date_reported'
id = db.Column(db.Integer, primary_key=True)
date_rep = db.Column(db.String(255), nullable=False)
- day = db.Column(db.String(255), nullable=False)
- month = db.Column(db.String(255), nullable=False)
- year = db.Column(db.String(255), nullable=False)
+ year_week = db.Column(db.String(255), nullable=False)
@classmethod
def remove_all(cls):
@@ -123,9 +126,9 @@ class EuropeData(db.Model):
__tablename__ = 'europe_data'
id = db.Column(db.Integer, primary_key=True)
- deaths = db.Column(db.String(255), nullable=False)
- cases = db.Column(db.String(255), nullable=False)
- cases_cumulative_14days_per_100000 = db.Column(db.String(255), nullable=False)
+ deaths_weekly = db.Column(db.String(255), nullable=False)
+ cases_weekly = db.Column(db.String(255), nullable=False)
+ notification_rate_per_100000_population_14days = db.Column(db.String(255), nullable=False)
europe_country_id = db.Column(db.Integer, db.ForeignKey('europe_country.id'), nullable=False)
europe_country = db.relationship('EuropeCountry', lazy='joined')
diff --git a/org/woehlke/covid19/europe/europe_service.py b/org/woehlke/covid19/europe/europe_service.py
index 607f44e0..12ac0783 100644
--- a/org/woehlke/covid19/europe/europe_service.py
+++ b/org/woehlke/covid19/europe/europe_service.py
@@ -28,74 +28,6 @@ class EuropeService:
app.logger.info("------------------------------------------------------------")
app.logger.info(" Europe Service [ready] ")
- def __download(self):
- app.logger.info("------------------------------------------------------------")
- app.logger.info(" download Europa [begin]")
- app.logger.info("------------------------------------------------------------")
- app.logger.info(" FILE: "+self.__src_europa_cvsfile_name+" <- "+self.__url_src_data)
- app.logger.info("------------------------------------------------------------")
- os.makedirs('data', exist_ok=True)
- app.logger.info("------------------------------------------------------------")
- try:
- data_file = wget.download(self.__url_src_data, self.__src_europa_cvsfile_name)
- #os.remove(self.__src_europa_cvsfile_name)
- #os.renames(data_file, self.__src_europa_cvsfile_name)
- app.logger.info("------------------------------------------------------------")
- except Exception as error:
- app.logger.warning(error)
- app.logger.warning("------------------------------------------------------------")
- finally:
- app.logger.info(" download Europa [done]")
- return self
-
- def __import(self):
- app.logger.info(" import Europa [begin]")
- app.logger.info("------------------------------------------------------------")
- app.logger.info(" FILE: "+self.__src_europa_cvsfile_name)
- app.logger.info(" TABLE: europe_data_import")
- app.logger.info("------------------------------------------------------------")
- try:
- EuropeDataImportTable.remove_all()
- with open(self.__src_europa_cvsfile_name, newline='') as csv_file:
- file_reader = csv.DictReader(csv_file, delimiter=',', quotechar='"')
- k = 0
- for row in file_reader:
- o = EuropeDataImportTable(
- date_rep=row['dateRep'],
- day=row['day'],
- month=row['month'],
- year=row['year'],
- cases=row['cases'],
- deaths=row['deaths'],
- countries_and_territories=row['countriesAndTerritories'],
- geo_id=row['geoId'],
- country_territory_code=row['countryterritoryCode'],
- pop_data_2019=row['popData2019'],
- continent_exp=row['continentExp'],
- cumulative_number_for_14_days_of_covid19_cases_per_100000
- =row['Cumulative_number_for_14_days_of_COVID-19_cases_per_100000']
- )
- db.session.add(o)
- if (k % 1000) == 0:
- db.session.commit()
- app.logger.info(" import Europa ... " + str(k) + " rows")
- k = k + 1
- db.session.commit()
- except KeyError as error:
- app.logger.warning("KeyError: import Europa [begin]")
- app.logger.warning(":::"+str(error)+":::")
- for item_key, item_value in row.items():
- app.logger.warning(item_key+" : "+item_value)
- app.logger.warning("KeyError: import Europa [end]")
- except (Exception, psycopg2.DatabaseError) as error:
- app.logger.warning("WARN: import Europa [begin]")
- app.logger.warning(error)
- app.logger.warning("WARN: import Europa [end]")
- finally:
- app.logger.info("------------------------------------------------------------")
- app.logger.info(" import Europa [done]")
- return self
-
def download(self):
app.logger.info(" download [begin]")
app.logger.info("------------------------------------------------------------")
diff --git a/org/woehlke/covid19/europe/europe_service_update.py b/org/woehlke/covid19/europe/europe_service_update.py
index 9074065e..bda421df 100644
--- a/org/woehlke/covid19/europe/europe_service_update.py
+++ b/org/woehlke/covid19/europe/europe_service_update.py
@@ -1,6 +1,7 @@
import os
import psycopg2
from database import db, app
+from org.woehlke.covid19.europe.europe_model import EuropeDataImportTable
class EuropeServiceUpdate:
@@ -17,11 +18,21 @@ class EuropeServiceUpdate:
app.logger.info("------------------------------------------------------------")
app.logger.info(" Europe Service Update [ready] ")
+ def __update_date_reported(self):
+ EuropeDataImportTable.remove_all()
+ result = EuropeDataImportTable.get_date_rep()
+ for result_item in result.items:
+ my_date_rep = result_item['date_rep']
+ year_week = result_item['year_week']
+ app.logger.info("| "+my_date_rep+" | "+year_week+" |")
+ return self
+
def update_db(self):
app.logger.info(" update_db [begin]")
app.logger.info("------------------------------------------------------------")
app.logger.info(" ... ")
app.logger.info(" ... TBD ... ")
+ self.__update_date_reported()
app.logger.info(" ... ")
app.logger.info(" update_db [done]")
app.logger.info("------------------------------------------------------------")
--
GitLab