From c303684329075ee3561afdf6aa98d5be8d4befdb Mon Sep 17 00:00:00 2001 From: thomaswoehlke <thomas.woehlke@gmail.com> Date: Mon, 10 May 2021 15:35:30 +0200 Subject: [PATCH] working on: 0.0.34 Release --- .../uml/ecdc_domain_model_star_schmema.txt | 350 +++++++++++++ .../uml/owid_domain_model_star_schema.txt | 481 ++++++++++++++++++ .../rki_cases_domain_model_star_schema.txt | 352 +++++++++++++ ...i_vaccination_domain_model_star_schema.txt | 253 +++++++++ 4 files changed, 1436 insertions(+) create mode 100644 docs/blueprints/ecdc/uml/ecdc_domain_model_star_schmema.txt create mode 100644 docs/blueprints/owid/uml/owid_domain_model_star_schema.txt create mode 100644 docs/blueprints/rki_cases/uml/rki_cases_domain_model_star_schema.txt create mode 100644 docs/blueprints/rki_vaccination/uml/rki_vaccination_domain_model_star_schema.txt diff --git a/docs/blueprints/ecdc/uml/ecdc_domain_model_star_schmema.txt b/docs/blueprints/ecdc/uml/ecdc_domain_model_star_schmema.txt new file mode 100644 index 00000000..6ed013f5 --- /dev/null +++ b/docs/blueprints/ecdc/uml/ecdc_domain_model_star_schmema.txt @@ -0,0 +1,350 @@ +@startuml + +package covid19.blueprints.all { + + entity ApplicationDateReported { + id: Integer + date_reported: String + year_week: String + datum: Date + year: Integer + month: Integer + day_of_month: Integer + day_of_week: Integer + week_of_year: Integer + get_name_for_weekday() + {classifier} get_names_for_weekday() + {classifier} get_datum_parts(my_date_rep: String) + {classifier} get_datum(my_year: Integer, my_month: Integer, my_day: Integer) + {classifier} get_datum_as_str(my_year: Integer, my_month: Integer, my_day: Integer) + {classifier} my_year_week(my_iso_year: Integer, week_number: Integer) + {classifier} create_new_object_factory(my_date_rep: String) + } + entity ApplicationDateReported { + id: Integer + date_reported: String + year_week: String + datum: Date + year: Integer + month: Integer + day_of_month: Integer + day_of_week: Integer + week_of_year: Integer + get_name_for_weekday() + {classifier} get_names_for_weekday() + {classifier} get_datum_parts(my_date_rep: String) + {classifier} get_datum(my_year: Integer, my_month: Integer, my_day: Integer) + {classifier} get_datum_as_str(my_year: Integer, my_month: Integer, my_day: Integer) + {classifier} my_year_week(my_iso_year: Integer, week_number: Integer) + {classifier} create_new_object_factory(my_date_rep: String) + } + entity ApplicationRegion { + id: Integer + continent: String + {classifier} remove_all() + {classifier} get_all_as_page(page: Integer) + {classifier} get_all() + {classifier} get_all_as_dict() + {classifier} get_by_id(other_id: Integer) + {classifier} find_by_id(other_id: Integer) + } + class ApplicationServiceDownload << (S,red) Service >> { + -log_error() + -log_success() + -prepare_download() + -download_with_wget() + -download_with_subprocess_and_os_native_wget() + download_file() + } + class ApplicationServiceConfig << (T,#FF7700) Transient >> { + limit_nr: Integer + data_path: String + slug: String + category: String + sub_category: String + tablename: String + cvsfile_name: String + url_src: String + cvsfile_path: String + msg_job: String + msg_ok: String + msg_error: String + create_config_for_who() + create_config_for_rki_vaccination() + create_config_for_owid() + create_config_for_ecdc() + create_config_for_rki_bundeslaender() + create_config_for_rki_landkreise() + } + + interface AllImportFull { + {classifier} download_file() + {classifier} import_file() + {classifier} full_update_dimension_tables() + {classifier} full_update_fact_table() + {classifier} full_update_star_schema() + } + interface AllImportIncrement { + {classifier} download_file() + {classifier} import_file() + {classifier} update_dimension_tables() + {classifier} update_fact_table() + {classifier} update_star_schema() + } + interface DataImport { + datum_reported: Date + location: String + location_group: String + {classifier} download_file() + {classifier} import_file() + {classifier} full_update_dimension_tables() + {classifier} full_update_fact_table() + {classifier} full_update_star_schema() + {classifier} update_dimension_tables() + {classifier} update_fact_table() + {classifier} update_star_schema() + } + interface AllFlat { + id: Integer + datum_reported: Date + year: Integer + month: Integer + day_of_month: Integer + day_of_year: Integer + day_of_week: Integer + week_of_year: Integer + year_week: String + day_of_week_str: String + month_str: String + location: String + location_group: String + {classifier} import_file_flat() + {classifier} get_years(): Integer[] + {classifier} get_days(): Date[] + {classifier} get_days_for_year(year: Integer): Date[] + {classifier} get_location_groups(): String[] + {classifier} get_locations_for_location_group(location_group: String): String[] + {classifier} get_data_for_location(location: String): AllFlat[] + {classifier} get_data_for_day(datum_reported: Date): AllFlat[] + } +} + +package covid19.blueprints.ecdc { + + entity EcdcImport { + id: Integer + date_rep: String + date_rep_day: String + date_rep_month: String + date_rep_year: String + cases: String + deaths: String + pop_data_2019: String + countries_and_territories: String + geo_id: String + country_territory_code: String + continent_exp: String + cumulative_number_for_14_days_of_covid19_cases_per_100000: String + {classifier} download_file() + {classifier} import_file() + {classifier} full_update_dimension_tables() + {classifier} full_update_fact_table() + {classifier} full_update_star_schema() + {classifier} update_dimension_tables() + {classifier} update_fact_table() + {classifier} update_star_schema() + } + + entity EcdcFlat { + id: Integer + datum_reported: Date + year: Integer + month: Integer + day_of_month: Integer + day_of_year: Integer + day_of_week: Integer + week_of_year: Integer + year_week: String + day_of_week_str: String + month_str: String + location: String + location_group: String + date_rep: String + date_rep_day: String + date_rep_month: String + date_rep_year: String + cases: String + deaths: String + pop_data_2019: String + countries_and_territories: String + geo_id: String + country_territory_code: String + continent_exp: String + cumulative_number_for_14_days_of_covid19_cases_per_100000: String + {classifier} import_file_flat() + {classifier} get_years(): Integer[] + {classifier} get_days(): Date[] + {classifier} get_days_for_year(year: Integer): Date[] + {classifier} get_location_groups(): String[] + {classifier} get_locations_for_location_group(location_group: String): String[] + {classifier} get_data_for_location(location: String): EcdcFlat[] + {classifier} get_data_for_day(datum_reported: Date): EcdcFlat[] + } + + entity EcdcDateReported { + id: Integer + dateiname: String + parent_dir: String + title: String + path_relative: String + get_absolute_path() + get_relative_path() + {classifier} remove_all() + {classifier} get_all_as_page(page: Integer) + {classifier} get_all() + {classifier} get_all_as_dict() + {classifier} get_by_id(other_id: Integer) + {classifier} find_by_id(other_id: Integer) + {classifier} get_kurse_all() + } + entity EcdcContinent { + id: Integer + dateiname: String + parent_dir: String + get_absolute_path() + get_relative_path() + {classifier} remove_all() + {classifier} get_all_as_page(page: Integer) + {classifier} get_all() + {classifier} get_all_as_dict() + {classifier} get_by_id(other_id: Integer) + {classifier} find_by_id(other_id: Integer) + {classifier} get_workflow_for_id(workflow_id: Integer) + } + entity EcdcCountry { + id: Integer + dateiname: String + parent_dir: String + title: String + lektion_nr: Integer + verified: Boolean + get_absolute_path() + get_relative_path() + {classifier} remove_all() + {classifier} get_all_as_page(page: Integer) + {classifier} get_all() + {classifier} get_all_as_dict() + {classifier} get_by_id(other_id: Integer) + {classifier} find_by_id(other_id: Integer) + {classifier} get_lektionen_for_workflow(workflow: Workflow) + {classifier} get_lektion_for_id(lektion_id: Integer) + {classifier} get_folien_for_lektion_id(lektion_id: Integer) + {classifier} get_all_verified() + } + entity EcdcData { + id: Integer + dateiname: String + parent_dir: String + title: String + lektion_nr: Integer + foliensatz_nr_in_lektion: Integer + get_absolute_path() + get_relative_path() + {classifier} get_foliensaetze_for_lektion(lektion: Lektion) + {classifier} remove_all() + {classifier} get_all_as_page(page: Integer) + {classifier} get_all() + {classifier} get_all_as_dict() + {classifier} get_by_id(other_id: Integer) + {classifier} find_by_id(other_id: Integer) + } + class EcdcService << (S,red) Service >> { + pretask_database_drop_create() + task_database_drop_create() + run_download_only() + run_import_only() + run_update_dimension_tables_only() + run_update_fact_table_incremental_only() + run_update_fact_table_initial_only() + run_update_star_schema_incremental() + run_update_star_schema_initial() + download_all_files() + task_import_all_files() + update_dimension_tables_only() + update_fact_table_initial_only() + update_fact_table_incremental_only() + } + class EcdcServiceImport << (S,red) Service >> { + import_file() + } + class EcdcServiceUpdate << (S,red) Service >> { + -update_date_reported() + -update_continent() + -update_country() + -get_continent_from_import(ecdc_import: EcdcImport): + -get_country_from_import(ecdc_import: EcdcImport): + -get_date_reported_from_import() + -update_data_initial() + update_dimension_tables_only() + update_fact_table_initial_only() + update_star_schema_incremental() + update_star_schema_initial() + } + class ecdc_views_frontend << (B,orchid) Boundary >> { + url_ecdc_info() + url_ecdc_tasks() + url_ecdc_data_imported() + url_ecdc_date_reported_all() + url_ecdc_date_reported_one_notification_rate() + url_ecdc_date_reported_one_deaths_weekly() + url_ecdc_date_reported_one_cases_weekly() + url_ecdc_continent_all() + url_ecdc_continent_one() + url_ecdc_country_all() + url_ecdc_country_one() + url_ecdc_country_germany() + } + class ecdc_views_for_tasks << (B,orchid) Boundary >> { + task_ecdc_download_only() + task_who_import_only() + task_who_update_dimension_tables_only() + task_who_update_fact_table_initial_only() + task_who_update_fact_table_incremental_only() + task_who_update_star_schema_initial() + task_who_update_star_schema_incremental() + } + class ecdc_celery_tasks << (B,orchid) Boundary >> { + task_ecdc_download_only() + task_ecdc_import_only() + task_ecdc_update_dimension_tables_only() + task_ecdc_update_fact_table_initial_only() + task_ecdc_update_fact_table_incremental_only() + task_ecdc_update_star_schema_initial() + task_ecdc_update_star_schema_incremental() + } +} +ApplicationServiceDownload : SQLAlchemy database +EcdcServiceImport : SQLAlchemy database +EcdcServiceUpdate : SQLAlchemy database +EcdcService : SQLAlchemy database +ApplicationDateReported <|-- EcdcDateReported +ApplicationRegion <|-- EcdcContinent +EcdcCountry "1" *--> "*" EcdcContinent: continent +EcdcData "1" *--> "*" EcdcCountry: country +EcdcData "1" *--> "*" EcdcDateReported: date_reported +EcdcService "*" *--> "1" ApplicationServiceConfig: cfg +EcdcServiceImport "*" --> "1" ApplicationServiceConfig: cfg +EcdcServiceUpdate "*" --> "1" ApplicationServiceConfig: cfg +EcdcService "1" *--> "1" EcdcServiceImport: ecdc_service_import +EcdcService "1" *--> "1" EcdcServiceUpdate: ecdc_service_update +EcdcService "1" *--> "1" ApplicationServiceDownload: service_download +ecdc_views_frontend "1" --> "1" EcdcService: ecdc_service +ecdc_views_for_tasks "1" --> "1" EcdcService: ecdc_service +ecdc_celery_tasks "1" --> "1" EcdcService: ecdc_service +AllImportFull <|-- DataImport +AllImportIncrement <|-- DataImport +DataImport <|-- AllFlat +DataImport <|-- EcdcImport +AllFlat <|-- EcdcFlat +EcdcImport <|-- EcdcFlat +@enduml diff --git a/docs/blueprints/owid/uml/owid_domain_model_star_schema.txt b/docs/blueprints/owid/uml/owid_domain_model_star_schema.txt new file mode 100644 index 00000000..17b21165 --- /dev/null +++ b/docs/blueprints/owid/uml/owid_domain_model_star_schema.txt @@ -0,0 +1,481 @@ +@startuml + +package covid19.blueprints.all { + + entity ApplicationDateReported { + id: Integer + date_reported: String + year_week: String + datum: Date + year: Integer + month: Integer + day_of_month: Integer + day_of_week: Integer + week_of_year: Integer + get_name_for_weekday() + {classifier} get_names_for_weekday() + {classifier} get_datum_parts(my_date_rep: String) + {classifier} get_datum(my_year: Integer, my_month: Integer, my_day: Integer) + {classifier} get_datum_as_str(my_year: Integer, my_month: Integer, my_day: Integer) + {classifier} my_year_week(my_iso_year: Integer, week_number: Integer) + {classifier} create_new_object_factory(my_date_rep: String) + } + entity ApplicationRegion { + id: Integer + continent: String + {classifier} remove_all() + {classifier} get_all_as_page(page: Integer) + {classifier} get_all() + {classifier} get_all_as_dict() + {classifier} get_by_id(other_id: Integer) + {classifier} find_by_id(other_id: Integer) + } + class ApplicationServiceDownload << (S,red) Service >> { + -log_error() + -log_success() + -prepare_download() + -download_with_wget() + -download_with_subprocess_and_os_native_wget() + download_file() + } + class ApplicationServiceConfig << (T,#FF7700) Transient >> { + limit_nr: Integer + data_path: String + slug: String + category: String + sub_category: String + tablename: String + cvsfile_name: String + url_src: String + cvsfile_path: String + msg_job: String + msg_ok: String + msg_error: String + create_config_for_who() + create_config_for_rki_vaccination() + create_config_for_owid() + create_config_for_ecdc() + create_config_for_rki_bundeslaender() + create_config_for_rki_landkreise() + } + + interface AllImportFull { + {classifier} download_file() + {classifier} import_file() + {classifier} full_update_dimension_tables() + {classifier} full_update_fact_table() + {classifier} full_update_star_schema() + } + + interface AllImportIncrement { + {classifier} download_file() + {classifier} import_file() + {classifier} update_dimension_tables() + {classifier} update_fact_table() + {classifier} update_star_schema() + } + + interface DataImport { + datum_reported: Date + location: String + location_group: String + {classifier} download_file() + {classifier} import_file() + {classifier} full_update_dimension_tables() + {classifier} full_update_fact_table() + {classifier} full_update_star_schema() + {classifier} update_dimension_tables() + {classifier} update_fact_table() + {classifier} update_star_schema() + } + + interface AllFlat { + id: Integer + datum_reported: Date + year: Integer + month: Integer + day_of_month: Integer + day_of_year: Integer + day_of_week: Integer + week_of_year: Integer + year_week: String + day_of_week_str: String + month_str: String + location: String + location_group: String + {classifier} import_file_flat() + {classifier} get_years(): Integer[] + {classifier} get_days(): Date[] + {classifier} get_days_for_year(year: Integer): Date[] + {classifier} get_location_groups(): String[] + {classifier} get_locations_for_location_group(location_group: String): String[] + {classifier} get_data_for_location(location: String): AllFlat[] + {classifier} get_data_for_day(datum_reported: Date): AllFlat[] + } +} + +package covid19.blueprints.owid { + entity OwidImport { + id: Integer + iso_code: String + continent: String + location: String + date: String + total_cases: String + new_cases: String + new_cases_smoothed: String + total_deaths: String + new_deaths: String + new_deaths_smoothed: String + total_cases_per_million: String + new_cases_per_million: String + new_cases_smoothed_per_million: String + total_deaths_per_million: String + new_deaths_per_million: String + new_deaths_smoothed_per_million: String + reproduction_rate: String + icu_patients: String + icu_patients_per_million: String + hosp_patients: String + hosp_patients_per_million: String + weekly_icu_admissions: String + weekly_icu_admissions_per_million: String + weekly_hosp_admissions: String + weekly_hosp_admissions_per_million: String + new_tests: String + total_tests: String + total_tests_per_thousand: String + new_tests_per_thousand: String + new_tests_smoothed: String + new_tests_smoothed_per_thousand: String + positive_rate: String + tests_per_case: String + tests_units: String + total_vaccinations: String + people_vaccinated: String + people_fully_vaccinated: String + new_vaccinations: String + new_vaccinations_smoothed: String + total_vaccinations_per_hundred: String + people_vaccinated_per_hundred: String + people_fully_vaccinated_per_hundred: String + new_vaccinations_smoothed_per_million: String + stringency_index: String + population: String + population_density: String + median_age: String + aged_65_older: String + aged_70_older: String + gdp_per_capita: String + extreme_poverty: String + cardiovasc_death_rate: String + diabetes_prevalence: String + female_smokers: String + male_smokers: String + handwashing_facilities: String + hospital_beds_per_thousand: String + life_expectancy: String + human_development_index: String + {classifier} remove_all() + {classifier} get_all_as_page(page: Integer) + {classifier} get_all() + {classifier} get_by_id(other_id: Integer) + {classifier} find_by_id(other_id: Integer) + {classifier} get_dates() + {classifier} get_for_one_day(day: String) + {classifier} get_dates_reported_as_array() + {classifier} get_new_dates_reported_as_array() + {classifier} get_continents(page: Integer) + {classifier} get_all_continents() + } + + entity OwidFlat { + id: Integer + datum_reported: Date + year: Integer + month: Integer + day_of_month: Integer + day_of_year: Integer + day_of_week: Integer + week_of_year: Integer + year_week: String + day_of_week_str: String + month_str: String + location: String + location_group: String + iso_code: String + continent: String + location: String + date: String + total_cases: String + new_cases: String + new_cases_smoothed: String + total_deaths: String + new_deaths: String + new_deaths_smoothed: String + total_cases_per_million: String + new_cases_per_million: String + new_cases_smoothed_per_million: String + total_deaths_per_million: String + new_deaths_per_million: String + new_deaths_smoothed_per_million: String + reproduction_rate: String + icu_patients: String + icu_patients_per_million: String + hosp_patients: String + hosp_patients_per_million: String + weekly_icu_admissions: String + weekly_icu_admissions_per_million: String + weekly_hosp_admissions: String + weekly_hosp_admissions_per_million: String + new_tests: String + total_tests: String + total_tests_per_thousand: String + new_tests_per_thousand: String + new_tests_smoothed: String + new_tests_smoothed_per_thousand: String + positive_rate: String + tests_per_case: String + tests_units: String + total_vaccinations: String + people_vaccinated: String + people_fully_vaccinated: String + new_vaccinations: String + new_vaccinations_smoothed: String + total_vaccinations_per_hundred: String + people_vaccinated_per_hundred: String + people_fully_vaccinated_per_hundred: String + new_vaccinations_smoothed_per_million: String + stringency_index: String + population: String + population_density: String + median_age: String + aged_65_older: String + aged_70_older: String + gdp_per_capita: String + extreme_poverty: String + cardiovasc_death_rate: String + diabetes_prevalence: String + female_smokers: String + male_smokers: String + handwashing_facilities: String + hospital_beds_per_thousand: String + life_expectancy: String + human_development_index: String + {classifier} import_file_flat() + {classifier} get_years(): Integer[] + {classifier} get_days(): Date[] + {classifier} get_days_for_year(year: Integer): Date[] + {classifier} get_location_groups(): String[] + {classifier} get_locations_for_location_group(location_group: String): String[] + {classifier} get_data_for_location(location: String): OwidFlat[] + {classifier} get_data_for_day(datum_reported: Date): OwidFlat[] + } + entity OwidDateReported { + id: Integer + date_reported: String + year_week: String + datum: Date + year: Integer + month: Integer + day_of_month: Integer + day_of_week: Integer + week_of_year: Integer + {classifier} remove_all() + {classifier} get_all_as_page(page: Integer) + {classifier} get_all() + {classifier} get_all_as_dict() + {classifier} get_by_id(other_id: Integer) + {classifier} find_by_id(other_id: Integer) + } + entity OwidContinent { + id: Integer + continent: String + {classifier} remove_all() + {classifier} get_all_as_page(page: Integer) + {classifier} get_all() + {classifier} get_all_as_dict() + {classifier} get_by_id(other_id: Integer) + {classifier} find_by_id(other_id: Integer) + } + entity OwidCountry { + id: Integer + iso_code: String + location: String + stringency_index: String + population: String + population_density: String + median_age: String + aged_65_older: String + aged_70_older: String + gdp_per_capita: String + extreme_poverty: String + cardiovasc_death_rate: String + diabetes_prevalence: String + female_smokers: String + male_smokers: String + handwashing_facilities: String + hospital_beds_per_thousand: String + life_expectancy: String + human_development_index: String + {classifier} remove_all() + {classifier} get_all_as_page(page: Integer) + {classifier} get_all() + {classifier} get_all_as_dict() + {classifier} get_by_id(other_id: Integer) + {classifier} find_by_id(other_id: Integer) + } + entity OwidData { + id: Integer + total_cases: String + new_cases: String + new_cases_smoothed: String + total_deaths: String + new_deaths: String + new_deaths_smoothed: String + total_cases_per_million: String + new_cases_per_million: String + new_cases_smoothed_per_million: String + total_deaths_per_million: String + new_deaths_per_million: String + new_deaths_smoothed_per_million: String + reproduction_rate: String + icu_patients: String + icu_patients_per_million: String + hosp_patients: String + hosp_patients_per_million: String + weekly_icu_admissions: String + weekly_icu_admissions_per_million: String + weekly_hosp_admissions: String + weekly_hosp_admissions_per_million: String + new_tests: String + total_tests: String + total_tests_per_thousand: String + new_tests_per_thousand: String + new_tests_smoothed: String + new_tests_smoothed_per_thousand: String + positive_rate: String + tests_per_case: String + tests_units: String + total_vaccinations: String + people_vaccinated: String + people_fully_vaccinated: String + new_vaccinations: String + new_vaccinations_smoothed: String + total_vaccinations_per_hundred: String + people_vaccinated_per_hundred: String + people_fully_vaccinated_per_hundred: String + new_vaccinations_smoothed_per_million: String + stringency_index: String + {classifier} remove_all() + {classifier} get_all_as_page(page: Integer) + {classifier} get_all() + {classifier} get_all_as_dict() + {classifier} get_by_id(other_id: Integer) + {classifier} find_by_id(other_id: Integer) + } + class OwidServiceImport << (S,red) Service >> { + import_file() + } + class OwidServiceUpdateBase << (S,red) Service >> { + } + class OwidServiceUpdateFull << (S,red) Service >> { + -full_update_date_reported() + -full_update_continent() + -full_update_country() + -full_update_fact_table() + -full_update_dimension_tables() + full_update_dimension_tables() + full_update_fact_table() + full_update_star_schema() + } + class OwidServiceUpdate << (S,red) Service >> { + -update_date_reported() + -full_update_continent() + -full_update_country() + -update_fact_table() + -update_dimension_tables() + update_dimension_tables() + update_fact_table() + update_star_schema() + -get_new_dates_reported_from_import() + } + class OwidService << (S,red) Service >> { + pretask_database_drop_create() + task_database_drop_create() + run_download_only() + run_import_only() + run_update_dimension_tables_only() + run_update_fact_table_incremental_only() + run_update_fact_table_initial_only() + run_update_star_schema_incremental() + run_update_star_schema_initial() + download_all_files() + task_import_all_files() + update_dimension_tables_only() + update_fact_table_initial_only() + update_fact_table_incremental_only() + } + class owid_views_frontend << (B,orchid) Boundary >> { + url_owid_info() + url_owid_tasks() + url_owid_test() + url_owid_imported() + url_owid_date_reported_all() + url_owid_date_reported() + url_owid_date_reported_cases_new() + url_owid_date_reported_cases_cumulative() + url_owid_date_reported_deaths_new() + url_owid_date_reported_deaths_cumulative() + } + class owid_views_for_tasks << (B,orchid) Boundary >> { + url_task_owid_download_only() + url_task_owid_import_only() + url_task_owid_update_dimension_tables_only() + url_task_owid_update_fact_table_incremental_only() + url_task_owid_update_fact_table_initial_only() + url_task_owid_update_star_schema_initial() + url_task_owid_update_star_schema_incremental() + } + class owid_celery_tasks << (B,orchid) Boundary >> { + task_owid_download_only() + task_owid_import_only() + task_owid_update_dimension_tables_only() + task_owid_update_fact_table_incremental_only() + task_owid_update_fact_table_initial_only() + task_owid_update_star_schema_incremental() + task_owid_update_star_schema_initial() + } + +} +OwidServiceImport : SQLAlchemy database +OwidServiceUpdateBase : SQLAlchemy database +OwidServiceUpdateFull : SQLAlchemy database +OwidServiceUpdate : SQLAlchemy database +OwidService : SQLAlchemy database +ApplicationServiceDownload : SQLAlchemy database +OwidServiceUpdateFull <|-- OwidServiceUpdateBase +OwidServiceUpdate <|-- OwidServiceUpdateBase +ApplicationDateReported <|-- OwidDateReported +ApplicationRegion <|-- OwidContinent +OwidCountry "1" *--> "*" OwidContinent : region +OwidData "1" *--> "*" OwidCountry : country +OwidData "1" *--> "*" OwidDateReported : date_reported +OwidService "*" *--> "1" ApplicationServiceConfig : cfg +OwidServiceImport "*" --> "1" ApplicationServiceConfig : cfg +OwidServiceUpdate "*" --> "1" ApplicationServiceConfig : cfg +ApplicationServiceDownload "*" --> "1" ApplicationServiceConfig : cfg +OwidService "1" *--> "1" OwidServiceImport : owid_service_import +OwidService "1" *--> "1" OwidServiceUpdate : owid_service_update +OwidService "1" *--> "1" OwidServiceUpdateFull : owid_service_update +OwidService "1" *--> "1" ApplicationServiceDownload : service_download +owid_views_frontend "1" --> "1" OwidService : owid_service +owid_views_for_tasks "1" --> "1" OwidService : owid_service +owid_celery_tasks "1" --> "1" OwidService : owid_service +AllImportFull <|-- DataImport +AllImportIncrement <|-- DataImport +DataImport <|-- AllFlat +DataImport <|-- OwidImport +AllFlat <|-- OwidFlat +OwidImport <|-- OwidFlat +@enduml diff --git a/docs/blueprints/rki_cases/uml/rki_cases_domain_model_star_schema.txt b/docs/blueprints/rki_cases/uml/rki_cases_domain_model_star_schema.txt new file mode 100644 index 00000000..e2717e6b --- /dev/null +++ b/docs/blueprints/rki_cases/uml/rki_cases_domain_model_star_schema.txt @@ -0,0 +1,352 @@ +@startuml +package covid19.blueprints.all { + + interface ApplicationDateReported { + id: Integer + date_reported: String + year_week: String + datum: Date + year: Integer + month: Integer + day_of_month: Integer + day_of_week: Integer + week_of_year: Integer + get_name_for_weekday() + {classifier} get_names_for_weekday() + {classifier} get_datum_parts(my_date_rep: String) + {classifier} get_datum(my_year: Integer, my_month: Integer, my_day: Integer) + {classifier} get_datum_as_str(my_year: Integer, my_month: Integer, my_day: Integer) + {classifier} my_year_week(my_iso_year: Integer, week_number: Integer) + {classifier} create_new_object_factory(my_date_rep: String) + } + interface ApplicationRegion { + id: Integer + continent: String + {classifier} remove_all() + {classifier} get_all_as_page(page: Integer) + {classifier} get_all() + {classifier} get_all_as_dict() + {classifier} get_by_id(other_id: Integer) + {classifier} find_by_id(other_id: Integer) + } + class ApplicationServiceDownload << (S,red) Service >> { + -log_error() + -log_success() + -prepare_download() + -download_with_wget() + -download_with_subprocess_and_os_native_wget() + download_file() + } + class ApplicationServiceConfig << (T,#FF7700) Transient >> { + limit_nr: Integer + data_path: String + slug: String + category: String + sub_category: String + tablename: String + cvsfile_name: String + url_src: String + cvsfile_path: String + msg_job: String + msg_ok: String + msg_error: String + create_config_for_who() + create_config_for_rki_vaccination() + create_config_for_owid() + create_config_for_ecdc() + create_config_for_rki_bundeslaender() + create_config_for_rki_landkreise() + } + + interface AllImportFull { + {classifier} download_file() + {classifier} import_file() + {classifier} full_update_dimension_tables() + {classifier} full_update_fact_table() + {classifier} full_update_star_schema() + } + + interface AllImportIncrement { + {classifier} download_file() + {classifier} import_file() + {classifier} update_dimension_tables() + {classifier} update_fact_table() + {classifier} update_star_schema() + } + + interface DataImport { + datum_reported: Date + location: String + location_group: String + {classifier} download_file() + {classifier} import_file() + {classifier} full_update_dimension_tables() + {classifier} full_update_fact_table() + {classifier} full_update_star_schema() + {classifier} update_dimension_tables() + {classifier} update_fact_table() + {classifier} update_star_schema() + } + + interface AllFlat { + id: Integer + datum_reported: Date + year: Integer + month: Integer + day_of_month: Integer + day_of_year: Integer + day_of_week: Integer + week_of_year: Integer + year_week: String + day_of_week_str: String + month_str: String + location: String + location_group: String + {classifier} import_file_flat() + {classifier} get_years(): Integer[] + {classifier} get_days(): Date[] + {classifier} get_days_for_year(year: Integer): Date[] + {classifier} get_location_groups(): String[] + {classifier} get_locations_for_location_group(location_group: String): String[] + {classifier} get_data_for_location(location: String): AllFlat[] + {classifier} get_data_for_day(datum_reported: Date): AllFlat[] + } +} +package covid19.blueprints.rki_cases { + + entity RkiImport { + id: Integer + fid: String + id_bundesland: String + bundesland: String + landkreis: String + altersgruppe: String + geschlecht: String + anzahl_fall: String + anzahl_todesfall: String + meldedatum: String + id_landkreis: String + datenstand: String + neuer_fall: String + neuer_todesfall: String + ref_datum: String + neu_genesen: String + anzahl_genesen: String + ist_erkrankungsbeginn: String + altersgruppe2: String + {classifier} download_file() + {classifier} import_file() + {classifier} full_update_dimension_tables() + {classifier} full_update_fact_table() + {classifier} full_update_star_schema() + {classifier} update_dimension_tables() + {classifier} update_fact_table() + {classifier} update_star_schema() + } + + entity RkiFlat { + id: Integer + datum_reported: Date + year: Integer + month: Integer + day_of_month: Integer + day_of_year: Integer + day_of_week: Integer + week_of_year: Integer + year_week: String + day_of_week_str: String + month_str: String + location: String + location_group: String + fid: String + id_bundesland: String + bundesland: String + landkreis: String + altersgruppe: String + geschlecht: String + anzahl_fall: String + anzahl_todesfall: String + meldedatum: String + id_landkreis: String + datenstand: String + neuer_fall: String + neuer_todesfall: String + ref_datum: String + neu_genesen: String + anzahl_genesen: String + ist_erkrankungsbeginn: String + altersgruppe2: String + {classifier} import_file_flat() + {classifier} get_years(): Integer[] + {classifier} get_days(): Date[] + {classifier} get_days_for_year(year: Integer): Date[] + {classifier} get_location_groups(): String[] + {classifier} get_locations_for_location_group(location_group: String): String[] + {classifier} get_data_for_location(location: String): RkiCasesFlat[] + {classifier} get_data_for_day(datum_reported: Date): RkiCasesFlat[] + } + + entity RkiMeldedatum { + id: Integer + date_reported: String + year_week: String + datum: Date + year: Integer + month: Integer + day_of_month: Integer + day_of_week: Integer + week_of_year: Integer + {classifier} remove_all() + {classifier} get_all_as_page(page: Integer) + {classifier} get_all() + {classifier} get_all_as_dict() + {classifier} get_by_id(other_id: Integer) + {classifier} find_by_id(other_id: Integer) + } + entity RkiBundesland { + id: Integer + id_bundesland: String + region: String + {classifier} remove_all() + {classifier} get_all_as_page(page: Integer) + {classifier} get_all() + {classifier} get_all_as_dict() + {classifier} get_by_id(other_id: Integer) + {classifier} find_by_id(other_id: Integer) + } + + entity RkiLandkreis { + id: Integer + id_landkreis: String + landkreis: String + {classifier} remove_all() + {classifier} get_all_as_page(page: Integer) + {classifier} get_all() + {classifier} get_all_as_dict() + {classifier} get_by_id(other_id: Integer) + {classifier} find_by_id(other_id: Integer) + } + entity RkiData { + id: Integer + fid: String + id_bundesland: String + bundesland: String + landkreis: String + altersgruppe: String + geschlecht: String + anzahl_fall: String + anzahl_todesfall: String + meldedatum: String + id_landkreis: String + datenstand: String + neuer_fall: String + neuer_todesfall: String + ref_datum: String + neu_genesen: String + anzahl_genesen: String + ist_erkrankungsbeginn: String + altersgruppe2: String + {classifier} remove_all() + {classifier} get_all_as_page(page: Integer) + {classifier} get_all() + {classifier} get_all_as_dict() + {classifier} get_by_id(other_id: Integer) + {classifier} find_by_id(other_id: Integer) + } + class RkiServiceImport << (S,red) Service >> { + import_file() + } + class RkiServiceUpdateBase << (S,red) Service >> { + } + class RkiServiceUpdateFull << (S,red) Service >> { + -full_update_date_reported() + -full_update_continent() + -full_update_country() + -full_update_fact_table() + -full_update_dimension_tables() + full_update_dimension_tables() + full_update_fact_table() + full_update_star_schema() + } + class RkiServiceUpdate << (S,red) Service >> { + -update_date_reported() + -full_update_continent() + -full_update_country() + -update_fact_table() + -update_dimension_tables() + update_dimension_tables() + update_fact_table() + update_star_schema() + -get_new_dates_reported_from_import() + } + class RkiService << (S,red) Service >> { + pretask_database_drop_create() + task_database_drop_create() + run_download_only() + run_import_only() + run_update_dimension_tables_only() + run_update_fact_table_incremental_only() + run_update_fact_table_initial_only() + run_update_star_schema_incremental() + run_update_star_schema_initial() + download_all_files() + task_import_all_files() + update_dimension_tables_only() + update_fact_table_initial_only() + update_fact_table_incremental_only() + } + class rki_cases_views_frontend << (B,orchid) Boundary >> { + url_get_import() + url_get_flat() + url_get_years() + url_get_days() + url_get_days_for_year() + url_get_location_groups() + url_get_locations_for_location_group(location_group: str) + url_get_data_for_location() + url_get_data_for_day() + url_page_info() + url_page_tasks() + } + class rki_cases_celery_tasks << (B,orchid) Boundary >> { + task_download_file() + task_import_file() + task_import_file_flat() + task_full_update_dimension_tables() + task_full_update_fact_table() + task_full_update_star_schema() + task_update_dimension_tables() + task_update_fact_table() + task_update_star_schema() + } +} +RkiServiceImport : SQLAlchemy database +RkiServiceUpdateBase : SQLAlchemy database +RkiServiceUpdateFull : SQLAlchemy database +RkiServiceUpdate : SQLAlchemy database +RkiService : SQLAlchemy database +ApplicationServiceDownload : SQLAlchemy database +RkiServiceUpdateBase <|-- RkiServiceUpdateFull +RkiServiceUpdateBase <|-- RkiServiceUpdate +ApplicationDateReported <|-- RkiMeldedatum +ApplicationRegion <|-- RkiBundesland +RkiLandkreis "1" *--> "*" RkiBundesland : bundesland +RkiData "1" *--> "*" RkiLandkreis : landkreis +RkiData "1" *--> "*" RkiMeldedatum : meldedatum +RkiService "*" *--> "1" ApplicationServiceConfig : cfg +RkiServiceImport "*" --> "1" ApplicationServiceConfig : cfg +RkiServiceUpdate "*" --> "1" ApplicationServiceConfig : cfg +ApplicationServiceDownload "*" --> "1" ApplicationServiceConfig : cfg +RkiService "1" *--> "1" RkiServiceImport : rki_service_import +RkiService "1" *--> "1" RkiServiceUpdate : rki_service_update +RkiService "1" *--> "1" RkiServiceUpdateFull : rki_service_update +RkiService "1" *--> "1" ApplicationServiceDownload : service_download +rki_cases_views_frontend "1" --> "1" RkiService : rki_service +rki_cases_celery_tasks "1" --> "1" RkiService : rki_service +AllImportFull <|-- DataImport +AllImportIncrement <|-- DataImport +DataImport <|-- AllFlat +DataImport <|-- RkiImport +AllFlat <|-- RkiFlat +RkiImport <|-- RkiFlat +@enduml diff --git a/docs/blueprints/rki_vaccination/uml/rki_vaccination_domain_model_star_schema.txt b/docs/blueprints/rki_vaccination/uml/rki_vaccination_domain_model_star_schema.txt new file mode 100644 index 00000000..476ae285 --- /dev/null +++ b/docs/blueprints/rki_vaccination/uml/rki_vaccination_domain_model_star_schema.txt @@ -0,0 +1,253 @@ +@startuml +package covid19.blueprints.all { + + entity ApplicationDateReported { + id: Integer + date_reported: String + year_week: String + datum: Date + year: Integer + month: Integer + day_of_year: Integer + day_of_month: Integer + day_of_week: Integer + week_of_year: Integer + get_name_for_weekday() + {classifier} get_names_for_weekday() + {classifier} get_datum_parts(my_date_rep: String) + {classifier} get_datum(my_year: Integer, my_month: Integer, my_day: Integer) + {classifier} get_datum_as_str(my_year: Integer, my_month: Integer, my_day: Integer) + {classifier} my_year_week(my_iso_year: Integer, week_number: Integer) + {classifier} create_new_object_factory(my_date_rep: String) + } + class ApplicationServiceDownload << (S,red) Service >> { + -log_error() + -log_success() + -prepare_download() + -download_with_wget() + -download_with_subprocess_and_os_native_wget() + download_file() + } + class ApplicationServiceConfig << (T,#FF7700) Transient >> { + limit_nr: Integer + data_path: String + slug: String + category: String + sub_category: String + tablename: String + cvsfile_name: String + url_src: String + cvsfile_path: String + msg_job: String + msg_ok: String + msg_error: String + create_config_for_who() + create_config_for_rki_vaccination() + create_config_for_owid() + create_config_for_ecdc() + create_config_for_rki_bundeslaender() + create_config_for_rki_landkreise() + } +} +package covid19.blueprints.rki_vVaccination { + entity RkiVaccinationDateReported { + id: Integer + date_reported: String + year_week: String + datum: Date + year: Integer + month: Integer + day_of_month: Integer + day_of_week: Integer + week_of_year: Integer + get_name_for_weekday() + {classifier} get_names_for_weekday() + {classifier} get_datum_parts(my_date_rep: String) + {classifier} get_datum(my_year: Integer, my_month: Integer, my_day: Integer) + {classifier} get_datum_as_str(my_year: Integer, my_month: Integer, my_day: Integer) + {classifier} my_year_week(my_iso_year: Integer, week_number: Integer) + {classifier} create_new_object_factory(my_date_rep: String) + } + entity RkiVaccinationData { + id: Integer + dosen_kumulativ: Integer + dosen_differenz_zum_vortag: Integer + dosen_biontech_kumulativ: Integer + dosen_moderna_kumulativ: Integer + personen_erst_kumulativ: Integer + personen_voll_kumulativ: Integer + impf_quote_erst: Float + impf_quote_voll: Float + indikation_alter_dosen: Integer + indikation_beruf_dosen: Integer + indikation_medizinisch_dosen: Integer + indikation_pflegeheim_dosen: Integer + indikation_alter_erst: Integer + indikation_beruf_erst: Integer + indikation_medizinisch_erst: Integer + indikation_pflegeheim_erst: Integer + indikation_alter_voll: Integer + indikation_beruf_voll: Integer + indikation_medizinisch_voll: Integer + indikation_pflegeheim_voll: Integer + get_absolute_path() + get_relative_path() + {classifier} remove_all() + {classifier} get_all_as_page(page: Integer) + {classifier} get_all() + {classifier} get_all_as_dict() + {classifier} get_by_id(other_id: Integer) + {classifier} find_by_id(other_id: Integer) + {classifier} get_kurse_all() + } + entity RkiVaccinationImport { + datum: String + dosen_kumulativ: Integer + dosen_differenz_zum_vortag: Integer + dosen_biontech_kumulativ: Integer + dosen_moderna_kumulativ: Integer + personen_erst_kumulativ: Integer + personen_voll_kumulativ: Integer + impf_quote_erst: Float + impf_quote_voll: Float + indikation_alter_dosen: Integer + indikation_beruf_dosen: Integer + indikation_medizinisch_dosen: Integer + indikation_pflegeheim_dosen: Integer + indikation_alter_erst: Integer + indikation_beruf_erst: Integer + indikation_medizinisch_erst: Integer + indikation_pflegeheim_erst: Integer + indikation_alter_voll: Integer + indikation_beruf_voll: Integer + indikation_medizinisch_voll: Integer + indikation_pflegeheim_voll: Integer + + {classifier} remove_all() + {classifier} get_all_as_page(page: Integer) + {classifier} get_all() + {classifier} get_by_id(other_id: Integer) + {classifier} find_by_id(other_id: Integer) + {classifier} find_by_datum(other_datum: String) + {classifier} get_date_rep() + {classifier} get_date_reported_as_array() + {classifier} get_daterep_missing_in_vaccination_data() + } + entity RkiVaccinationFlat { + id: Integer + datum_reported: Date + year: Integer + month: Integer + day_of_month: Integer + day_of_year: Integer + day_of_week: Integer + week_of_year: Integer + year_week: String + day_of_week_str: String + month_str: String + location: String + location_group: String + dosen_kumulativ: Integer + dosen_differenz_zum_vortag: Integer + dosen_biontech_kumulativ: Integer + dosen_moderna_kumulativ: Integer + personen_erst_kumulativ: Integer + personen_voll_kumulativ: Integer + impf_quote_erst: Float + impf_quote_voll: Float + indikation_alter_dosen: Integer + indikation_beruf_dosen: Integer + indikation_medizinisch_dosen: Integer + indikation_pflegeheim_dosen: Integer + indikation_alter_erst: Integer + indikation_beruf_erst: Integer + indikation_medizinisch_erst: Integer + indikation_pflegeheim_erst: Integer + indikation_alter_voll: Integer + indikation_beruf_voll: Integer + indikation_medizinisch_voll: Integer + indikation_pflegeheim_voll: Integer + date_reported: String + {classifier} import_file_flat() + {classifier} get_years(): Integer[] + {classifier} get_days(): Date[] + {classifier} get_days_for_year(year: Integer): Date[] + {classifier} get_location_groups(): String[] + {classifier} get_locations_for_location_group(location_group: String): String[] + {classifier} get_data_for_location(location: String): RkiVaccinationFlat[] + {classifier} get_data_for_day(datum_reported: Date): RkiVaccinationFlat[] + } + class RkiVaccinationService << (S,red) Service >> { + pretask_database_drop_create() + task_database_drop_create() + run_download_only() + run_import_only() + run_update_dimension_tables_only() + run_update_fact_table_incremental_only() + run_update_fact_table_initial_only() + run_update_star_schema_incremental() + run_update_star_schema_initial() + download_all_files() + task_import_all_files() + update_dimension_tables_only() + update_fact_table_initial_only() + update_fact_table_incremental_only() + } + class RkiVaccinationServiceImport << (S,red) Service >> { + import_file() + } + class RkiVaccinationServiceUpdate << (S,red) Service >> { + -update_date_reported() + -update_region() + -update_country() + -update_data_incremental() + -update_data_initial() + -update_dimension_tables() + update_dimension_tables_only() + update_fact_table_incremental_only() + update_fact_table_initial_only() + update_star_schema_incremental() + update_star_schema_initial() + } + class rki_vaccination_views_frontend << (B,orchid) Boundary >> { + url_vaccination_info() + url_vaccination_tasks() + url_vaccination_imported() + url_vaccination_data() + } + class rki_vaccination_views_for_tasks << (B,orchid) Boundary >> { + task_who_download_only() + task_who_import_only() + task_who_update_dimension_tables_only() + task_who_update_fact_table_initial_only() + task_who_update_fact_table_incremental_only() + task_who_update_star_schema_initial() + task_who_update_star_schema_incremental() + } + class rki_vaccination_celery_tasks << (B,orchid) Boundary >> { + url_vaccination_task_download_only() + url_vaccination_task_import_only() + url_vaccination_task_update_dimensiontables_only() + url_vaccination_task_update_facttable_incremental_only() + url_vaccination_task_update_facttable_initial_only() + url_vaccination_task_update_starschema_initial() + url_vaccination_task_update_starschema_incremental() + } +} +RkiVaccinationServiceImport : SQLAlchemy database +RkiVaccinationServiceUpdate : SQLAlchemy database +RkiVaccinationService : SQLAlchemy database +ApplicationServiceDownload : SQLAlchemy database +ApplicationDateReported <|-- RkiVaccinationDateReported +RkiVaccinationData "1" *--> "*" RkiVaccinationDateReported : date_reported +RkiVaccinationService "*" *--> "1" ApplicationServiceConfig : cfg +RkiVaccinationServiceImport "*" --> "1" ApplicationServiceConfig : cfg +RkiVaccinationServiceUpdate "*" --> "1" ApplicationServiceConfig : cfg +ApplicationServiceDownload "*" --> "1" ApplicationServiceConfig : cfg +RkiVaccinationService "1" *--> "1" RkiVaccinationServiceImport : rki_vaccination_service_import +RkiVaccinationService "1" *--> "1" RkiVaccinationServiceUpdate : rki_vaccination_service_update +RkiVaccinationService "1" *--> "1" ApplicationServiceDownload : service_download +rki_vaccination_views_frontend "1" --> "1" RkiVaccinationService : rki_vaccination_service +rki_vaccination_views_for_tasks "1" --> "1" RkiVaccinationService : rki_vaccination_service +rki_vaccination_celery_tasks "1" --> "1" RkiVaccinationService : rki_vaccination_service +@enduml -- GitLab