From c1a670f8d39dffbdf18705e2d01e2939a68ce7fb Mon Sep 17 00:00:00 2001
From: thomaswoehlke <thomas.woehlke@gmail.com>
Date: Mon, 25 Jan 2021 12:48:11 +0100
Subject: [PATCH] work
---
app.py | 11 +-
.../covid19/vaccination/vaccination_model.py | 31 ---
.../vaccination/vaccination_service.py | 31 +--
.../vaccination/vaccination_service_import.py | 6 -
.../vaccination/vaccination_service_update.py | 238 ------------------
...nt_vaccination_germany_timeline_table.html | 28 +++
.../vaccination_germany_timeline.html | 62 +++++
.../vaccination/vaccination_imported.html | 88 -------
8 files changed, 98 insertions(+), 397 deletions(-)
delete mode 100644 org/woehlke/covid19/vaccination/vaccination_service_update.py
create mode 100644 templates/vaccination/fragment_vaccination_germany_timeline_table.html
create mode 100644 templates/vaccination/vaccination_germany_timeline.html
delete mode 100644 templates/vaccination/vaccination_imported.html
diff --git a/app.py b/app.py
index d3c6928a..00d65d52 100644
--- a/app.py
+++ b/app.py
@@ -13,7 +13,7 @@ from org.woehlke.covid19.who.who_service import WhoService
from org.woehlke.covid19.europe.europe_service import EuropeService
from org.woehlke.covid19.vaccination.vaccination_service import VaccinationService
from org.woehlke.covid19.admin.admin_service import AdminService
-from org.woehlke.covid19.vaccination.vaccination_model import VaccinationDataImportTable
+from org.woehlke.covid19.vaccination.vaccination_model import VaccinationGermanyTimeline
from celery.utils.log import get_task_logger
@@ -645,6 +645,7 @@ def url_vaccination_tasks():
@app.route('/vaccination/update/initial')
def url_vaccination_update_data():
vaccination_service.run_download()
+ flash("vaccination_service.run_download done")
task_vaccination_update_initial.apply_async()
flash("vaccination_service.run_update started")
return redirect(url_for('url_vaccination_tasks'))
@@ -652,11 +653,11 @@ def url_vaccination_update_data():
@app.route('/vaccination/imported/page/<int:page>')
@app.route('/vaccination/imported')
-def url_vaccination_data_imported(page=1):
- page_info = ApplicationPage('Vaccination', "Last Import")
- page_data = VaccinationDataImportTable.get_all_as_page(page)
+def url_vaccination_germany_timeline(page=1):
+ page_info = ApplicationPage('Vaccination', "Germany Timeline")
+ page_data = VaccinationGermanyTimeline.get_all_as_page(page)
return render_template(
- 'vaccination/vaccination_imported.html',
+ 'vaccination/vaccination_germany_timeline.html',
page_data=page_data,
page_info=page_info)
diff --git a/org/woehlke/covid19/vaccination/vaccination_model.py b/org/woehlke/covid19/vaccination/vaccination_model.py
index a64ddd2a..007fae52 100644
--- a/org/woehlke/covid19/vaccination/vaccination_model.py
+++ b/org/woehlke/covid19/vaccination/vaccination_model.py
@@ -1,7 +1,4 @@
-from flask_sqlalchemy import Pagination
-from sqlalchemy import and_, func
from database import db, ITEMS_PER_PAGE
-from sqlalchemy.orm import joinedload, raiseload
class VaccinationGermanyTimeline(db.Model):
@@ -49,31 +46,3 @@ class VaccinationGermanyTimeline(db.Model):
def get_by_id(cls, other_id):
return db.session.query(cls).filter(cls.id == other_id).one()
- @classmethod
- def get_new_dates_as_array(cls):
- sql_query = """
- select
- date_reported
- from
- vaccination_data_import
- where
- date_reported
- not in (
- select
- distinct
- vaccination_date_reported.date_reported
- from
- vaccination_global_data
- left join
- vaccination_date_reported
- on
- vaccination_global_data.date_reported_id=vaccination_date_reported.id
- )
- group by
- vaccination_data_import.date_reported
- order by date_reported desc
- """
- new_dates = []
- for item in db.session.execute(sql_query):
- new_dates.append(item['date_reported'])
- return new_dates
diff --git a/org/woehlke/covid19/vaccination/vaccination_service.py b/org/woehlke/covid19/vaccination/vaccination_service.py
index c764d16e..6f6c0bcf 100644
--- a/org/woehlke/covid19/vaccination/vaccination_service.py
+++ b/org/woehlke/covid19/vaccination/vaccination_service.py
@@ -2,7 +2,6 @@ import os
from database import app
from org.woehlke.covid19.vaccination.vaccination_service_download import VaccinationServiceDownload
from org.woehlke.covid19.vaccination.vaccination_service_import import VaccinationServiceImport
-from org.woehlke.covid19.vaccination.vaccination_service_update import VaccinationServiceUpdate
vaccination_service = None
@@ -20,7 +19,6 @@ class VaccinationService:
self.__url_src_data = "https://impfdashboard.de/static/data/germany_vaccinations_timeseries_v2.tsv"
self.vaccination_service_download = VaccinationServiceDownload(database)
self.vaccination_service_import = VaccinationServiceImport(database)
- self.vaccination_service_update = VaccinationServiceUpdate(database)
app.logger.info("------------------------------------------------------------")
app.logger.info(" Vaccination Service [ready]")
@@ -33,35 +31,10 @@ class VaccinationService:
app.logger.info("------------------------------------------------------------")
return success
- def run_update(self, import_file=True):
- app.logger.info(" run update [begin]")
- app.logger.info("------------------------------------------------------------")
- if import_file:
- self.vaccination_service_import.import_file()
- self.vaccination_service_update.update_db()
- app.logger.info("")
- app.logger.info(" run update [done]")
- app.logger.info("------------------------------------------------------------")
- return self
-
- def run_update_short(self, import_file=True):
- app.logger.info(" run update short [begin]")
- app.logger.info("------------------------------------------------------------")
- app.logger.info(" ...")
- #if import_file:
- # self.vaccination_service_import.import_file()
- #self.vaccination_service_update.update_db_short()
- app.logger.info("")
- app.logger.info(" run update short [done]")
- app.logger.info("------------------------------------------------------------")
- return self
-
- def run_update_initial(self, import_file=True):
+ def run_update_initial(self):
app.logger.info(" run update initial [begin]")
app.logger.info("------------------------------------------------------------")
- if import_file:
- self.vaccination_service_import.import_file()
- #self.vaccination_service_update.update_db_initial()
+ self.vaccination_service_import.import_file()
app.logger.info("")
app.logger.info(" run update initial [done]")
app.logger.info("------------------------------------------------------------")
diff --git a/org/woehlke/covid19/vaccination/vaccination_service_import.py b/org/woehlke/covid19/vaccination/vaccination_service_import.py
index 1ec838a0..50d49485 100644
--- a/org/woehlke/covid19/vaccination/vaccination_service_import.py
+++ b/org/woehlke/covid19/vaccination/vaccination_service_import.py
@@ -1,5 +1,4 @@
import os
-import sys
import csv
import psycopg2
from database import db, app
@@ -29,11 +28,6 @@ class VaccinationServiceImport:
app.logger.info(" FILE: "+self.__src_cvsfile_name)
app.logger.info(" TABLE: who_global_data_import")
app.logger.info("------------------------------------------------------------")
- row = None
- if sys.platform == 'linux':
- keyDate_reported ='\ufeffDate_reported'
- else:
- keyDate_reported = 'Date_reported'
try:
VaccinationGermanyTimeline.remove_all()
k = 0
diff --git a/org/woehlke/covid19/vaccination/vaccination_service_update.py b/org/woehlke/covid19/vaccination/vaccination_service_update.py
deleted file mode 100644
index 749fb51f..00000000
--- a/org/woehlke/covid19/vaccination/vaccination_service_update.py
+++ /dev/null
@@ -1,238 +0,0 @@
-import os
-from database import db, app
-from org.woehlke.covid19.vaccination.vaccination_model import VaccinationDataImportTable, VaccinationDateReported
-from org.woehlke.covid19.vaccination.vaccination_model import VaccinationCountry, VaccinationRegion, VaccinationData
-
-
-vaccination_service_update = None
-
-
-class VaccinationServiceUpdate:
- def __init__(self, database):
- app.logger.info("------------------------------------------------------------")
- app.logger.info(" Vaccination Service Update [init]")
- app.logger.info("------------------------------------------------------------")
- self.__database = database
- self.limit_nr = 20
- self.__cvsfile_name = "germany_vaccinations_timeseries_v2.tsv"
- self.__src_cvsfile_name = "data"+os.sep+self.__cvsfile_name
- self.__src_cvsfile_tmp_name = "data"+os.sep+"tmp_"+self.__cvsfile_name
- self.__url_src_data = "https://impfdashboard.de/static/data/germany_vaccinations_timeseries_v2.tsv"
- app.logger.info("------------------------------------------------------------")
- app.logger.info(" Vaccination Service Update [ready]")
-
- def __update_who_date_reported(self):
- app.logger.info(" update who_date_reported [begin]")
- app.logger.info("------------------------------------------------------------")
- i = 0
- for i_date_reported, in VaccinationDataImportTable.get_dates_reported():
- c = VaccinationDateReported.find_by_date_reported(i_date_reported)
- if c is None:
- o = VaccinationDateReported(date_reported=i_date_reported)
- db.session.add(o)
- app.logger.info(" update who_date_reported "+i_date_reported+" added NEW")
- if i % 10 == 0:
- app.logger.info(" update who_date_reported "+i_date_reported+" not added")
- db.session.commit()
- i += 1
- db.session.commit()
- app.logger.info("")
- app.logger.info(" update who_date_reported [done]")
- app.logger.info("------------------------------------------------------------")
- return self
-
- def __update_who_region(self):
- app.logger.info(" update who_region [begin]")
- app.logger.info("------------------------------------------------------------")
- i = 0
- for i_who_region, in db.session.query(VaccinationDataImportTable.who_region).distinct():
- c = db.session.query(VaccinationRegion).filter(VaccinationRegion.region == i_who_region).count()
- if c == 0:
- o = VaccinationRegion(region=i_who_region)
- db.session.add(o)
- app.logger.info(i_who_region +" added NEW ")
- else:
- app.logger.info(i_who_region +" not added ")
- if i % 10 == 0:
- db.session.commit()
- i += 1
- db.session.commit()
- app.logger.info("")
- app.logger.info(" update who_region [done]")
- app.logger.info("------------------------------------------------------------")
- return self
-
- def __update_who_country(self):
- app.logger.info(" update who_country [begin]")
- app.logger.info("------------------------------------------------------------")
- sql_text = """
- select distinct
- who_global_data_import.country_code,
- who_global_data_import.country,
- who_global_data_import.who_region
- from who_global_data_import
- """
- result = db.session.execute(sql_text).fetchall()
- for result_item in result:
- i_country_code = result_item.country_code
- i_country = result_item.country
- i_who_region = result_item.who_region
- output = i_country_code + " " + i_country + " " + i_who_region
- my_region = VaccinationRegion.find_by_region(i_who_region)
- my_country = VaccinationCountry.find_by_country_code_and_country_and_who_region_id(
- i_country_code, i_country, my_region
- )
- if my_country is None:
- o = VaccinationCountry(
- country=i_country,
- country_code=i_country_code,
- region=my_region)
- db.session.add(o)
- db.session.commit()
- my_country = VaccinationCountry.find_by_country_code_and_country_and_who_region_id(
- i_country_code, i_country, my_region
- )
- output += " added NEW "
- else:
- output += " not added "
- output += i_country_code + " id=" + str(my_country.id)
- app.logger.info(output)
- db.session.commit()
- app.logger.info("")
- app.logger.info(" update who_country [done]")
- app.logger.info("------------------------------------------------------------")
- return self
-
- def __update_who_global_data(self):
- app.logger.info(" update Vaccination [begin]")
- app.logger.info("------------------------------------------------------------")
- dates_reported = VaccinationDateReported.get_all_as_dict()
- regions = VaccinationRegion.get_all_as_dict()
- countries = VaccinationCountry.get_all_as_dict()
- #
- #
- i = 0
- result = VaccinationDataImportTable.get_all()
- for result_item in result:
- my_country = countries[result_item.country_code]
- my_date_reported = dates_reported[result_item.date_reported]
- result_who_global_data = VaccinationData.find_one_or_none_by_date_and_country(
- my_date_reported,
- my_country)
- if result_who_global_data is None:
- o = VaccinationData(
- cases_new=int(result_item.new_cases),
- cases_cumulative=int(result_item.cumulative_cases),
- deaths_new=int(result_item.new_deaths),
- deaths_cumulative=int(result_item.cumulative_deaths),
- date_reported=my_date_reported,
- country=my_country
- )
- db.session.add(o)
- if i % 2000 == 0:
- app.logger.info(" update Vaccination ... "+str(i)+" rows")
- db.session.commit()
- i += 1
- db.session.commit()
- app.logger.info(" update Vaccination : "+str(i)+" total rows")
- app.logger.info(" update Vaccination [done]")
- app.logger.info("------------------------------------------------------------")
- return self
-
- def __update_who_global_data_short(self):
- app.logger.info(" update Vaccination short [begin]")
- app.logger.info("------------------------------------------------------------")
- new_dates_reported_from_import = VaccinationDataImportTable.get_new_dates_as_array()
- i = 0
- for my_date_reported in new_dates_reported_from_import:
- my_date = VaccinationDateReported.find_by_date_reported(my_date_reported)
- for result_item in VaccinationDataImportTable.get_for_one_day(my_date_reported):
- my_country = VaccinationCountry.find_by_country_code(result_item.country_code)
- o = VaccinationData(
- cases_new=int(result_item.new_cases),
- cases_cumulative=int(result_item.cumulative_cases),
- deaths_new=int(result_item.new_deaths),
- deaths_cumulative=int(result_item.cumulative_deaths),
- date_reported=my_date,
- country=my_country
- )
- db.session.add(o)
- result_item.row_imported = True
- db.session.add(result_item)
- i += 1
- if i % 500 == 0:
- app.logger.info(" update Vaccination short ... "+str(i)+" rows")
- db.session.commit()
- db.session.commit()
- app.logger.info(" update Vaccination short : "+str(i)+" total rows")
- app.logger.info(" update Vaccination short [done]")
- app.logger.info("------------------------------------------------------------")
- return self
-
- def __update_who_global_data_initial(self):
- app.logger.info(" update Vaccination initial [begin]")
- app.logger.info("------------------------------------------------------------")
- VaccinationData.remove_all()
- new_dates_reported_from_import = VaccinationDataImportTable.get_new_dates_as_array()
- i = 0
- for my_date_reported in new_dates_reported_from_import:
- my_date = VaccinationDateReported.find_by_date_reported(my_date_reported)
- for result_item in VaccinationDataImportTable.get_for_one_day(my_date_reported):
- my_country = VaccinationCountry.find_by_country_code(result_item.country_code)
- o = VaccinationData(
- cases_new=int(result_item.new_cases),
- cases_cumulative=int(result_item.cumulative_cases),
- deaths_new=int(result_item.new_deaths),
- deaths_cumulative=int(result_item.cumulative_deaths),
- date_reported=my_date,
- country=my_country
- )
- db.session.add(o)
- result_item.row_imported = True
- db.session.add(result_item)
- i += 1
- if i % 500 == 0:
- app.logger.info(" update Vaccination initial ... "+str(i)+" rows")
- db.session.commit()
- db.session.commit()
- app.logger.info(" update Vaccination initial : "+str(i)+" total rows")
- app.logger.info(" update Vaccination initial [done]")
- app.logger.info("------------------------------------------------------------")
- return self
-
- def update_db(self):
- app.logger.info(" update db [begin]")
- app.logger.info("------------------------------------------------------------")
- self.__update_who_date_reported()
- self.__update_who_region()
- self.__update_who_country()
- self.__update_who_global_data()
- app.logger.info(" update db [done]")
- app.logger.info("------------------------------------------------------------")
- return self
-
- def update_db_short(self):
- app.logger.info(" update db short [begin]")
- app.logger.info("------------------------------------------------------------")
- self.__update_who_date_reported()
- self.__update_who_region()
- self.__update_who_country()
- self.__update_who_global_data_short()
- app.logger.info(" update db short [done]")
- app.logger.info("------------------------------------------------------------")
- return self
-
- def update_db_initial(self):
- app.logger.info(" update db initial [begin]")
- app.logger.info("------------------------------------------------------------")
- self.__update_who_date_reported()
- self.__update_who_region()
- self.__update_who_country()
- self.__update_who_global_data_initial()
- app.logger.info(" update db initial [done]")
- app.logger.info("------------------------------------------------------------")
- return self
-
- def update_who_country(self):
- self.__update_who_country()
- return self
diff --git a/templates/vaccination/fragment_vaccination_germany_timeline_table.html b/templates/vaccination/fragment_vaccination_germany_timeline_table.html
new file mode 100644
index 00000000..655215a3
--- /dev/null
+++ b/templates/vaccination/fragment_vaccination_germany_timeline_table.html
@@ -0,0 +1,28 @@
+ <table class="table table-hover table-striped table-dark">
+ <thead>
+ <tr>
+ <th scope="col">date_reported</th>
+ <th scope="col">country_code</th>
+ <th scope="col">country</th>
+ <th scope="col">who_region</th>
+ <th scope="col">new_cases</th>
+ <th scope="col">cumulative_cases</th>
+ <th scope="col">new_deaths</th>
+ <th scope="col">cumulative_deaths</th>
+ </tr>
+ </thead>
+ <tbody>
+ {% for vaccination_germany_timeline in page_data.items %}
+ <tr>
+ <td>{{ vaccination_germany_timeline.date_reported }}</td>
+ <td>{{ vaccination_germany_timeline.country_code }}</td>
+ <td>{{ vaccination_germany_timeline.country }}</td>
+ <td>{{ vaccination_germany_timeline.who_region }}</td>
+ <td>{{ vaccination_germany_timeline.new_cases }}</td>
+ <td>{{ vaccination_germany_timeline.cumulative_cases }}</td>
+ <td>{{ vaccination_germany_timeline.new_deaths }}</td>
+ <td>{{ vaccination_germany_timeline.cumulative_deaths }}</td>
+ </tr>
+ {% endfor %}
+ </tbody>
+ </table>
\ No newline at end of file
diff --git a/templates/vaccination/vaccination_germany_timeline.html b/templates/vaccination/vaccination_germany_timeline.html
new file mode 100644
index 00000000..b10cd69e
--- /dev/null
+++ b/templates/vaccination/vaccination_germany_timeline.html
@@ -0,0 +1,62 @@
+{% extends 'page_layout.html' %}
+
+{% include 'fragment_pagination.html' %}
+
+{% block navigation_breadcrumb %}
+
+{% endblock %}
+
+
+{% block navigation_navtabs %}
+ {% include 'vaccination/fragment_navtabs_vaccination.html' %}
+{% endblock %}
+
+
+{% block main_container %}
+
+ {% if page_data.pages > 1 %}
+ <!-- previous page -->
+ <ul class="pagination">
+ {% if page_data.has_prev %}
+ <li class="page-item">
+ <a class="page-link" href="{{ url_for('url_vaccination_germany_timeline', page=page_data.prev_num) }}">Previous</a>
+ </li>
+ {% endif %}
+ <!-- all page numbers -->
+ {% for page_num in page_data.iter_pages() %}
+ {% if page_num %}
+ {% if page_num != page_data.page %}
+ <li class="page-item">
+ <a class="page-link" href="{{ url_for('url_vaccination_germany_timeline', page=page_num) }}">{{ page_num }}</a>
+ </li>
+ {% else %}
+ <li class="page-item active">
+ <a class="page-link" href="#">{{ page_num }}</a>
+ </li>
+ {% endif %}
+ {% else %}
+ <li class="page-item">
+ <span class="ellipsis page-link my-page-item-ellipsis-page-link">…</span>
+ </li>
+ {% endif %}
+ {% endfor %}
+ <!-- next page -->
+ {% if page_data.has_next %}
+ <li class="page-item">
+ <a class="page-link" href="{{ url_for('url_vaccination_germany_timeline', page=page_data.next_num) }}">Next</a>
+ </li>
+ {% endif %}
+ </ul>
+ {% endif %}
+
+ {% include 'vaccination/fragment_vaccination_germany_timeline_table.html' %}
+
+{% endblock %}
+
+
+{% block footer_container %}
+
+{% endblock %}
+
+
+
diff --git a/templates/vaccination/vaccination_imported.html b/templates/vaccination/vaccination_imported.html
deleted file mode 100644
index 6714d23e..00000000
--- a/templates/vaccination/vaccination_imported.html
+++ /dev/null
@@ -1,88 +0,0 @@
-{% extends 'page_layout.html' %}
-
-{% include 'fragment_pagination.html' %}
-
-{% block navigation_breadcrumb %}
-
-{% endblock %}
-
-
-{% block navigation_navtabs %}
- {% include 'vaccination/fragment_navtabs_vaccination.html' %}
-{% endblock %}
-
-
-{% block main_container %}
-
- {% if page_data.pages > 1 %}
- <!-- previous page -->
- <ul class="pagination">
- {% if page_data.has_prev %}
- <li class="page-item">
- <a class="page-link" href="{{ url_for('url_rki_imported', page=page_data.prev_num) }}">Previous</a>
- </li>
- {% endif %}
- <!-- all page numbers -->
- {% for page_num in page_data.iter_pages() %}
- {% if page_num %}
- {% if page_num != page_data.page %}
- <li class="page-item">
- <a class="page-link" href="{{ url_for('url_rki_imported', page=page_num) }}">{{ page_num }}</a>
- </li>
- {% else %}
- <li class="page-item active">
- <a class="page-link" href="#">{{ page_num }}</a>
- </li>
- {% endif %}
- {% else %}
- <li class="page-item">
- <span class="ellipsis page-link my-page-item-ellipsis-page-link">…</span>
- </li>
- {% endif %}
- {% endfor %}
- <!-- next page -->
- {% if page_data.has_next %}
- <li class="page-item">
- <a class="page-link" href="{{ url_for('url_rki_imported', page=page_data.next_num) }}">Next</a>
- </li>
- {% endif %}
- </ul>
- {% endif %}
-
- <table class="table table-hover table-striped">
- <thead>
- <tr>
- <th scope="col">date_reported</th>
- <th scope="col">country_code</th>
- <th scope="col">country</th>
- <th scope="col">who_region</th>
- <th scope="col">new_cases</th>
- <th scope="col">cumulative_cases</th>
- <th scope="col">new_deaths</th>
- <th scope="col">cumulative_deaths</th>
- </tr>
- </thead>
- <tbody>
- {% for who_global_data_import in page_data.items %}
- <tr>
- <td>{{ who_global_data_import.date_reported }}</td>
- <td>{{ who_global_data_import.country_code }}</td>
- <td>{{ who_global_data_import.country }}</td>
- <td>{{ who_global_data_import.who_region }}</td>
- <td>{{ who_global_data_import.new_cases }}</td>
- <td>{{ who_global_data_import.cumulative_cases }}</td>
- <td>{{ who_global_data_import.new_deaths }}</td>
- <td>{{ who_global_data_import.cumulative_deaths }}</td>
- </tr>
- {% endfor %}
- </tbody>
- </table>
-{% endblock %}
-
-
-{% block footer_container %}
-
-{% endblock %}
-
-
-
--
GitLab