From c4b50e5b3b85fc9d969a10d44e23ee2793afbc31 Mon Sep 17 00:00:00 2001 From: nesnoj Date: Fri, 24 Aug 2018 11:45:08 +0200 Subject: [PATCH 01/84] add new docs page for data references --- documentation/doc/data_references.rst | 12 ++++++++++++ documentation/doc/index.rst | 1 + 2 files changed, 13 insertions(+) create mode 100644 documentation/doc/data_references.rst diff --git a/documentation/doc/data_references.rst b/documentation/doc/data_references.rst new file mode 100644 index 00000000..b6547f0c --- /dev/null +++ b/documentation/doc/data_references.rst @@ -0,0 +1,12 @@ +=============== +Data References +=============== + +Example: + +.. [Amme2017] J. Amme, G. Pleßmann, J. Bühler, L. Hülk, E. Kötter, P. Schwaegerl: + *The eGo grid model: An open-source and open-data based synthetic medium-voltage + grid model for distribution power supply systems*. Journal of Physics Conference + Series 977(1):012007, 2018, `doi:10.1088/1742-6596/977/1/012007 + `_ + diff --git a/documentation/doc/index.rst b/documentation/doc/index.rst index 07a13ab9..50266a38 100644 --- a/documentation/doc/index.rst +++ b/documentation/doc/index.rst @@ -22,6 +22,7 @@ Welcome to Data processing documentation of open_eGo scenario whatsnew + data_references From 8d5ed8dd2bef23846e4d5854822b011f851e1863 Mon Sep 17 00:00:00 2001 From: nesnoj Date: Fri, 24 Aug 2018 11:51:58 +0200 Subject: [PATCH 02/84] Revert "add new docs page for data references" This reverts commit c4b50e5b3b85fc9d969a10d44e23ee2793afbc31. --- documentation/doc/data_references.rst | 12 ------------ documentation/doc/index.rst | 1 - 2 files changed, 13 deletions(-) delete mode 100644 documentation/doc/data_references.rst diff --git a/documentation/doc/data_references.rst b/documentation/doc/data_references.rst deleted file mode 100644 index b6547f0c..00000000 --- a/documentation/doc/data_references.rst +++ /dev/null @@ -1,12 +0,0 @@ -=============== -Data References -=============== - -Example: - -.. [Amme2017] J. Amme, G. Pleßmann, J. Bühler, L. Hülk, E. Kötter, P. Schwaegerl: - *The eGo grid model: An open-source and open-data based synthetic medium-voltage - grid model for distribution power supply systems*. Journal of Physics Conference - Series 977(1):012007, 2018, `doi:10.1088/1742-6596/977/1/012007 - `_ - diff --git a/documentation/doc/index.rst b/documentation/doc/index.rst index 50266a38..07a13ab9 100644 --- a/documentation/doc/index.rst +++ b/documentation/doc/index.rst @@ -22,7 +22,6 @@ Welcome to Data processing documentation of open_eGo scenario whatsnew - data_references From 1db2e7cfe716f0dbb5d8be206e8e6a5c392819e2 Mon Sep 17 00:00:00 2001 From: wolfbunke Date: Thu, 20 Sep 2018 09:12:35 +0200 Subject: [PATCH 03/84] update scenario importer of renpass --- .../renpass_gis/scenario_import/Readme.rst | 26 +++++++++++++++++++ .../renpass_gis/scenario_import/db.py | 3 ++- .../scenario_import/requirements.txt | 16 ++++++++++++ 3 files changed, 44 insertions(+), 1 deletion(-) create mode 100644 preprocessing/python_scripts/renpass_gis/scenario_import/Readme.rst create mode 100644 preprocessing/python_scripts/renpass_gis/scenario_import/requirements.txt diff --git a/preprocessing/python_scripts/renpass_gis/scenario_import/Readme.rst b/preprocessing/python_scripts/renpass_gis/scenario_import/Readme.rst new file mode 100644 index 00000000..65110cd9 --- /dev/null +++ b/preprocessing/python_scripts/renpass_gis/scenario_import/Readme.rst @@ -0,0 +1,26 @@ +============================= +Scenario import of renpassG!S +============================= + + + +Installation +============ + +.. code-block:: + $ virtualenv renpass_importer --clear -p python3.5 + $ source bin/activate + $ pip3 install -r requirements.txt + + + +How to import scnario csv files to the oedb +=========================================== + + +.. code-block:: + python3 scenario_import.py "/PATH/status_quo.csv" "/PATH/status_quo_seq.csv" "/PATH/results/scenario_status_quo_-dd-dd_results_complete.csv" -n "status quo" -c "scenario of sensitivity" + + + + diff --git a/preprocessing/python_scripts/renpass_gis/scenario_import/db.py b/preprocessing/python_scripts/renpass_gis/scenario_import/db.py index 3eb2b3da..a5b41654 100644 --- a/preprocessing/python_scripts/renpass_gis/scenario_import/db.py +++ b/preprocessing/python_scripts/renpass_gis/scenario_import/db.py @@ -14,6 +14,7 @@ from sqlalchemy.ext.automap import automap_base import configparser as cp import os.path as path +import oedialect # read configuration file FILENAME = 'config.ini' @@ -29,7 +30,7 @@ password=cfg.get(section, 'password'), host=cfg.get(section, 'host'), port=cfg.get(section, 'port'), - db=cfg.get(section, 'db'))) + db=cfg.get(section, 'database'))) print("Connected to database.") diff --git a/preprocessing/python_scripts/renpass_gis/scenario_import/requirements.txt b/preprocessing/python_scripts/renpass_gis/scenario_import/requirements.txt new file mode 100644 index 00000000..805d6242 --- /dev/null +++ b/preprocessing/python_scripts/renpass_gis/scenario_import/requirements.txt @@ -0,0 +1,16 @@ +certifi==2018.8.24 +chardet==3.0.4 +configparser==3.5.0 +docopt==0.6.2 +idna==2.7 +numpy==1.15.1 +oedialect==0.0.2.dev0 +pandas==0.20.1 +pathlib==1.0.1 +psycopg2==2.7.5 +python-dateutil==2.7.3 +pytz==2018.5 +requests==2.19.1 +six==1.11.0 +SQLAlchemy==1.2.0 +urllib3==1.23 From 37e746c85ff8dc7cd19df76ba9f3753346dcbd51 Mon Sep 17 00:00:00 2001 From: WolfBunke Date: Thu, 20 Sep 2018 09:18:48 +0200 Subject: [PATCH 04/84] Update Readme.rst --- .../python_scripts/renpass_gis/scenario_import/Readme.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/preprocessing/python_scripts/renpass_gis/scenario_import/Readme.rst b/preprocessing/python_scripts/renpass_gis/scenario_import/Readme.rst index 65110cd9..9a6be09d 100644 --- a/preprocessing/python_scripts/renpass_gis/scenario_import/Readme.rst +++ b/preprocessing/python_scripts/renpass_gis/scenario_import/Readme.rst @@ -8,6 +8,7 @@ Installation ============ .. code-block:: + $ virtualenv renpass_importer --clear -p python3.5 $ source bin/activate $ pip3 install -r requirements.txt @@ -17,9 +18,9 @@ Installation How to import scnario csv files to the oedb =========================================== - .. code-block:: - python3 scenario_import.py "/PATH/status_quo.csv" "/PATH/status_quo_seq.csv" "/PATH/results/scenario_status_quo_-dd-dd_results_complete.csv" -n "status quo" -c "scenario of sensitivity" + + $ python3 scenario_import.py "/PATH/status_quo.csv" "/PATH/status_quo_seq.csv" "/PATH/results/scenario_status_quo_-dd-dd_results_complete.csv" -n "status quo" -c "scenario of sensitivity" From 24ce75c9b03602edd9b2e02a7457972506338063 Mon Sep 17 00:00:00 2001 From: IlkaCu Date: Thu, 20 Feb 2020 10:00:03 +0100 Subject: [PATCH 05/84] update bpmn section substation --- .../ego_dp_bpmn_section_substation.graphml | 451 ++++++------------ 1 file changed, 137 insertions(+), 314 deletions(-) diff --git a/documentation/bpmn/ego_dp_bpmn_section_substation.graphml b/documentation/bpmn/ego_dp_bpmn_section_substation.graphml index 3888de0e..ad1e2b71 100644 --- a/documentation/bpmn/ego_dp_bpmn_section_substation.graphml +++ b/documentation/bpmn/ego_dp_bpmn_section_substation.graphml @@ -1,6 +1,6 @@ - + @@ -13,14 +13,14 @@ - + - + - eGo DataProcessing - substation + eGo DataProcessing - substation @@ -31,22 +31,22 @@ + + + - - - - - + + @@ -78,7 +78,7 @@ - ego_dp_substation_hvmv_voronoi.sql + ego_dp_substation_hvmv_voronoi.sql @@ -96,20 +96,14 @@ - model_draft.ego_grid_hvmv_substation_dummy -model_draft.ego_grid_hvmv_substation_voronoi - - - - - - + model_draft.ego_grid_hvmv_substation_dummy +model_draft.ego_grid_hvmv_substation_voronoi + - @@ -120,13 +114,7 @@ model_draft.ego_grid_hvmv_substation_voronoi - 6 min - - - - - - + 6 min @@ -142,19 +130,14 @@ model_draft.ego_grid_hvmv_substation_voronoi - model_draft.ego_grid_hvmv_substation - - - - - - + model_draft.ego_grid_hvmv_substation +model_draft.ego_boundaries_bkg_vg250_6_gem_clean + - @@ -165,19 +148,13 @@ model_draft.ego_grid_hvmv_substation_voronoi - model_draft.ego_grid_hvmv_substation - - - - - - + model_draft.ego_grid_hvmv_substation + - @@ -188,7 +165,7 @@ model_draft.ego_grid_hvmv_substation_voronoi - + ego_dp_substation_hvmv.sql @@ -208,7 +185,7 @@ ego_dp_substation_hvmv.sql - + ego_dp_substation_ehv.sql @@ -228,13 +205,7 @@ ego_dp_substation_ehv.sql - 32 min - - - - - - + 32 min @@ -250,13 +221,7 @@ ego_dp_substation_ehv.sql - 3 min - - - - - - + 3 min @@ -272,23 +237,18 @@ ego_dp_substation_ehv.sql - openstreetmap.osm_deu_ways + openstreetmap.osm_deu_ways openstreetmap.osm_deu_polygon +openstreetmap.osm_deu_line openstreetmap.osm_deu_nodes -openstreetmap.osm_deu_rels -political_boundary.bkg_vg250_1_sta_union_mview - - - - - - +openstreetmap.osm_deu_point +boundaries.bkg_vg250_1_sta_union_mview + - @@ -299,23 +259,19 @@ political_boundary.bkg_vg250_1_sta_union_mview - openstreetmap.osm_deu_ways + openstreetmap.osm_deu_ways openstreetmap.osm_deu_polygon openstreetmap.osm_deu_nodes +openstreetmap.osm_deu_line +openstreetmap.osm_deu_point openstreetmap.osm_deu_rels -political_bkg_vg250_1_sta_union_mview - - - - - - +boundaries.bkg_vg250_1_sta_union_mview + - @@ -326,19 +282,13 @@ political_bkg_vg250_1_sta_union_mview - model_draft.ego_grid_ehv_substation - - - - - - + model_draft.ego_grid_ehv_substation + - @@ -358,8 +308,8 @@ political_bkg_vg250_1_sta_union_mview - + @@ -373,7 +323,7 @@ political_bkg_vg250_1_sta_union_mview - + ego_dp_substation_otg.sql @@ -393,13 +343,7 @@ ego_dp_substation_otg.sql - 1 sec - - - - - - + 1 sec @@ -415,21 +359,15 @@ ego_dp_substation_otg.sql - grid.otg_ehvhv_bus_data + grid.otg_ehvhv_bus_data model_draft.ego_grid_ehv_substation -model_draft.ego_grid_hvmv_substation - - - - - - +model_draft.ego_grid_hvmv_substation + - @@ -440,20 +378,14 @@ model_draft.ego_grid_hvmv_substation - model_draft.ego_grid_ehv_substation -model_draft.ego_grid_hvmv_substation - - - - - - + model_draft.ego_grid_ehv_substation +model_draft.ego_grid_hvmv_substation + - @@ -464,14 +396,8 @@ model_draft.ego_grid_hvmv_substation - Open Database License (ODbL) v1.0 -© OpenStreetMap contributors - - - - - - + Open Database License (ODbL) v1.0 +© OpenStreetMap contributors @@ -496,8 +422,8 @@ model_draft.ego_grid_hvmv_substation - + @@ -511,19 +437,13 @@ model_draft.ego_grid_hvmv_substation - openstreetmap.osm_deu - - - - - - + openstreetmap.osm_deu + - @@ -534,14 +454,8 @@ model_draft.ego_grid_hvmv_substation - Geodatenzugangsgesetz (GeoZG) -© GeoBasis-DE / BKG 2016 (Daten verändert) - - - - - - + Geodatenzugangsgesetz (GeoZG) +© GeoBasis-DE / BKG 2016 (Daten verändert) @@ -557,19 +471,13 @@ model_draft.ego_grid_hvmv_substation - political_boundary.bkg_vg250 - - - - - - + boundaries.bkg_vg250 + - @@ -580,14 +488,8 @@ model_draft.ego_grid_hvmv_substation - Open Database License (ODbL) v1.0 -© OpenStreetMap contributors - - - - - - + Open Database License (ODbL) v1.0 +© OpenStreetMap contributors @@ -603,14 +505,8 @@ model_draft.ego_grid_hvmv_substation - Open Database License (ODbL) v1.0 -© OpenStreetMap contributors - - - - - - + Open Database License (ODbL) v1.0 +© OpenStreetMap contributors @@ -626,14 +522,8 @@ model_draft.ego_grid_hvmv_substation - Open Database License (ODbL) v1.0 -© OpenStreetMap contributors - - - - - - + Open Database License (ODbL) v1.0 +© OpenStreetMap contributors @@ -649,14 +539,8 @@ model_draft.ego_grid_hvmv_substation - Open Database License (ODbL) v1.0 -© OpenStreetMap contributors - - - - - - + Open Database License (ODbL) v1.0 +© OpenStreetMap contributors @@ -672,14 +556,8 @@ model_draft.ego_grid_hvmv_substation - Open Database License (ODbL) v1.0 -© OpenStreetMap contributors - - - - - - + Open Database License (ODbL) v1.0 +© OpenStreetMap contributors @@ -695,14 +573,8 @@ model_draft.ego_grid_hvmv_substation - Geodatenzugangsgesetz (GeoZG) -© GeoBasis-DE / BKG 2016 (Daten verändert) - - - - - - + Geodatenzugangsgesetz (GeoZG) +© GeoBasis-DE / BKG 2016 (Daten verändert) @@ -718,14 +590,8 @@ model_draft.ego_grid_hvmv_substation - Geodatenzugangsgesetz (GeoZG) -© GeoBasis-DE / BKG 2016 (Daten verändert) - - - - - - + Geodatenzugangsgesetz (GeoZG) +© GeoBasis-DE / BKG 2016 (Daten verändert) @@ -741,14 +607,8 @@ model_draft.ego_grid_hvmv_substation - Geodatenzugangsgesetz (GeoZG) -© GeoBasis-DE / BKG 2016 (Daten verändert) - - - - - - + Geodatenzugangsgesetz (GeoZG) +© GeoBasis-DE / BKG 2016 (Daten verändert) @@ -764,14 +624,8 @@ model_draft.ego_grid_hvmv_substation - Geodatenzugangsgesetz (GeoZG) -© GeoBasis-DE / BKG 2016 (Daten verändert) - - - - - - + Geodatenzugangsgesetz (GeoZG) +© GeoBasis-DE / BKG 2016 (Daten verändert) @@ -787,20 +641,14 @@ model_draft.ego_grid_hvmv_substation - model_draft.ego_grid_hvmv_substation -grid.ego_hvmv_substation - - - - - - + model_draft.ego_grid_hvmv_substation +grid.ego_hvmv_substation + - @@ -811,14 +659,8 @@ grid.ego_hvmv_substation - Open Database License (ODbL) v1.0 -© OpenStreetMap contributors - - - - - - + Open Database License (ODbL) v1.0 +© OpenStreetMap contributors @@ -834,14 +676,8 @@ grid.ego_hvmv_substation - Geodatenzugangsgesetz (GeoZG) -© GeoBasis-DE / BKG 2016 (Daten verändert) - - - - - - + Geodatenzugangsgesetz (GeoZG) +© GeoBasis-DE / BKG 2016 (Daten verändert) @@ -857,7 +693,7 @@ grid.ego_hvmv_substation - ego_dp_substation_ehv_voronoi.sql + ego_dp_substation_ehv_voronoi.sql @@ -876,19 +712,13 @@ grid.ego_hvmv_substation - model_draft.ego_grid_ehv_substation_voronoi - - - - - - + model_draft.ego_grid_ehv_substation_voronoi + - @@ -899,13 +729,7 @@ grid.ego_hvmv_substation - ? - - - - - - + ? @@ -921,19 +745,14 @@ grid.ego_hvmv_substation - model_draft.ego_grid_ehv_substation - - - - - - + model_draft.ego_grid_hvmv_substation_dummy +model_draft.ego_grid_ehv_substation + - @@ -944,20 +763,14 @@ grid.ego_hvmv_substation - model_draft.ego_grid_ehv_substation -grid.ego_ehv_substation - - - - - - + model_draft.ego_grid_ehv_substation +grid.ego_ehv_substation + - @@ -968,14 +781,8 @@ grid.ego_ehv_substation - Open Database License (ODbL) v1.0 -© OpenStreetMap contributors - - - - - - + Open Database License (ODbL) v1.0 +© OpenStreetMap contributors @@ -991,14 +798,8 @@ grid.ego_ehv_substation - Geodatenzugangsgesetz (GeoZG) -© GeoBasis-DE / BKG 2016 (Daten verändert) - - - - - - + Geodatenzugangsgesetz (GeoZG) +© GeoBasis-DE / BKG 2016 (Daten verändert) @@ -1014,14 +815,8 @@ grid.ego_ehv_substation - Open Database License (ODbL) v1.0 -© OpenStreetMap contributors - - - - - - + Open Database License (ODbL) v1.0 +© OpenStreetMap contributors @@ -1037,14 +832,8 @@ grid.ego_ehv_substation - Geodatenzugangsgesetz (GeoZG) -© GeoBasis-DE / BKG 2016 (Daten verändert) - - - - - - + Geodatenzugangsgesetz (GeoZG) +© GeoBasis-DE / BKG 2016 (Daten verändert) @@ -1060,13 +849,25 @@ grid.ego_ehv_substation - grid.otg_ehvhv_bus_data - - - - - - + grid.otg_ehvhv_bus_data + + + + + + + + + + + + + + + + + preprocessing +model_draft.ego_boundaries_bkg_vg250_6_gem_clean @@ -1463,6 +1264,28 @@ grid.ego_ehv_substation + + + + + + + + + + + + + + + + + + + + + + From faabd4b31d830b4a9d30b28672a303bed2b80bd4 Mon Sep 17 00:00:00 2001 From: IlkaCu Date: Thu, 20 Feb 2020 11:13:42 +0100 Subject: [PATCH 06/84] update bpmn section griddistrict --- .../ego_dp_bpmn_section_griddistrict.graphml | 330 ++++++++++-------- 1 file changed, 185 insertions(+), 145 deletions(-) diff --git a/documentation/bpmn/ego_dp_bpmn_section_griddistrict.graphml b/documentation/bpmn/ego_dp_bpmn_section_griddistrict.graphml index 72c3f0f9..bfac1dff 100644 --- a/documentation/bpmn/ego_dp_bpmn_section_griddistrict.graphml +++ b/documentation/bpmn/ego_dp_bpmn_section_griddistrict.graphml @@ -1,6 +1,6 @@ - + @@ -13,15 +13,15 @@ - + - + - eGo DataProcessing - griddistrict - + eGo DataProcessing - griddistrict + @@ -53,18 +53,18 @@ - + - + - + @@ -78,7 +78,7 @@ - ego_dp_mv_griddistrict.sql + ego_dp_mv_griddistrict.sql @@ -96,18 +96,12 @@ - ego_political_boundary_hvmv_subst_per_gem_1_mview -ego_political_boundary_hvmv_subst_per_gem_2_mview -ego_political_boundary_hvmv_subst_per_gem_3_mview -ego_grid_hvmv_substation_voronoi_mview -ego_grid_mv_griddistrict -ego_grid_mv_griddistrict_dump - - - - - - + model_draft.ego_boundaries_hvmv_subst_per_gem_1_mview +model_draft.ego_boundaries_hvmv_subst_per_gem_2_mview +model_draft.ego_boundaries_hvmv_subst_per_gem_3_mview +model_draft.ego_grid_hvmv_substation_voronoi_mview +model_draft.ego_grid_mv_griddistrict +model_draft.ego_grid_mv_griddistrict_dump @@ -124,13 +118,7 @@ ego_grid_mv_griddistrict_dump - 2 min - - - - - - + 2 min @@ -146,15 +134,6 @@ ego_grid_mv_griddistrict_dump - model_draft.vg250_6_gem_clean_mview -model_draft.ego_grid_hvmv_substation -model_draft.ego_grid_hvmv_substation_voronoi - - - - - - @@ -192,7 +171,7 @@ model_draft.ego_grid_hvmv_substation_voronoi - + @@ -219,14 +198,8 @@ model_draft.ego_grid_hvmv_substation_voronoi - Geodatenzugangsgesetz (GeoZG) -© GeoBasis-DE / BKG 2016 (Daten verändert) - - - - - - + Geodatenzugangsgesetz (GeoZG) +© GeoBasis-DE / BKG 2016 (Daten verändert) @@ -242,13 +215,7 @@ model_draft.ego_grid_hvmv_substation_voronoi - political_boundary.bkg_vg250 - - - - - - + model_draft.ego_boundaries_bkg_vg250_6_gem_clean @@ -265,14 +232,8 @@ model_draft.ego_grid_hvmv_substation_voronoi - Geodatenzugangsgesetz (GeoZG) -© GeoBasis-DE / BKG 2016 (Daten verändert) - - - - - - + Geodatenzugangsgesetz (GeoZG) +© GeoBasis-DE / BKG 2016 (Daten verändert) @@ -288,14 +249,8 @@ model_draft.ego_grid_hvmv_substation_voronoi - Open Database License (ODbL) v1.0 -© OpenStreetMap contributors - - - - - - + Open Database License (ODbL) v1.0 +© OpenStreetMap contributors @@ -308,17 +263,11 @@ model_draft.ego_grid_hvmv_substation_voronoi - + - model_draft.ego_grid_mv_griddistrict -grid.ego_dp_mv_griddistrict - - - - - - + model_draft.ego_grid_hvmv_substation +model_draft.ego_grid_hvmv_substation_voronoi @@ -332,17 +281,11 @@ grid.ego_dp_mv_griddistrict - + - Open Database License (ODbL) v1.0 -© OpenStreetMap contributors - - - - - - + Open Database License (ODbL) v1.0 +© OpenStreetMap contributors @@ -355,17 +298,11 @@ grid.ego_dp_mv_griddistrict - + - Geodatenzugangsgesetz (GeoZG) -© GeoBasis-DE / BKG 2016 (Daten verändert) - - - - - - + Geodatenzugangsgesetz (GeoZG) +© GeoBasis-DE / BKG 2016 (Daten verändert) @@ -376,71 +313,121 @@ grid.ego_dp_mv_griddistrict + + + + + + ego_dp_substation_id_to_generator.sql + + + + + + + + + + + + - - + + - model_draft.ego_grid_hvmv_substation -grid.ego_hvmv_substation + + - + - + - + - - + + - Open Database License (ODbL) v1.0 -© OpenStreetMap contributors - - - - - - + model_draft.ego_dp_supply_conv_powerplant +model_draft.ego_dp_supply_res_powerplant - + + - + - - + + - Geodatenzugangsgesetz (GeoZG) -© GeoBasis-DE / BKG 2016 (Daten verändert) - - - - - - + model_draft.ego_dp_supply_conv_powerplant +model_draft.ego_dp_supply_res_powerplant +model_draft.ego_grid_hvmv_substation +model_draft.ego_grid_mv_griddistrict +model_draft.ego_grid_ehv_substation_voronoi - + + + + + + + + + + + + + preprocessing +ego_dp_structure_boundaries_vg250.sql + + + + + + + + + + + + + + + + + + preprocessing +ego_dp_preprocessing_conv_powerplant.sql +ego_dp_preprocessing_res_powerplant.sql + + + + + + + @@ -495,19 +482,19 @@ grid.ego_hvmv_substation - + - - - + + + - + - + @@ -519,28 +506,81 @@ grid.ego_hvmv_substation - + - + - + - - - - + + - + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - + + + + + + + + + + + + + + + + + + + + + + + + From a07821bdd4ddb4a336101d088bcd0b611685b002 Mon Sep 17 00:00:00 2001 From: IlkaCu Date: Thu, 20 Feb 2020 14:40:07 +0100 Subject: [PATCH 07/84] Update bpmn docu --- .../ego_dp_bpmn_section_griddistrict.graphml | 55 +- .../bpmn/ego_dp_bpmn_section_loadarea.graphml | 637 ++++++------------ .../ego_dp_bpmn_section_substation.graphml | 172 +++-- 3 files changed, 354 insertions(+), 510 deletions(-) diff --git a/documentation/bpmn/ego_dp_bpmn_section_griddistrict.graphml b/documentation/bpmn/ego_dp_bpmn_section_griddistrict.graphml index bfac1dff..d84302f4 100644 --- a/documentation/bpmn/ego_dp_bpmn_section_griddistrict.graphml +++ b/documentation/bpmn/ego_dp_bpmn_section_griddistrict.graphml @@ -17,11 +17,11 @@ - + - eGo DataProcessing - griddistrict - + eGo DataProcessing - griddistrict + @@ -64,7 +64,7 @@ - + @@ -96,11 +96,10 @@ - model_draft.ego_boundaries_hvmv_subst_per_gem_1_mview + model_draft.ego_boundaries_hvmv_subst_per_gem_1_mview model_draft.ego_boundaries_hvmv_subst_per_gem_2_mview model_draft.ego_boundaries_hvmv_subst_per_gem_3_mview model_draft.ego_grid_hvmv_substation_voronoi_mview -model_draft.ego_grid_mv_griddistrict model_draft.ego_grid_mv_griddistrict_dump @@ -134,6 +133,7 @@ model_draft.ego_grid_mv_griddistrict_dump + @@ -346,10 +346,10 @@ model_draft.ego_grid_hvmv_substation_voronoi + - @@ -361,13 +361,14 @@ model_draft.ego_grid_hvmv_substation_voronoi model_draft.ego_dp_supply_conv_powerplant -model_draft.ego_dp_supply_res_powerplant +model_draft.ego_dp_supply_res_powerplant + + - @@ -385,10 +386,10 @@ model_draft.ego_grid_mv_griddistrict model_draft.ego_grid_ehv_substation_voronoi + - @@ -432,6 +433,24 @@ ego_dp_preprocessing_res_powerplant.sql + + + + + + + model_draft.ego_grid_mv_griddistrict +grid.ego_mv_griddistrict + + + + + + + + + + @@ -529,7 +548,6 @@ ego_dp_preprocessing_res_powerplant.sql - @@ -540,7 +558,6 @@ ego_dp_preprocessing_res_powerplant.sql - @@ -551,7 +568,6 @@ ego_dp_preprocessing_res_powerplant.sql - @@ -562,7 +578,6 @@ ego_dp_preprocessing_res_powerplant.sql - @@ -573,7 +588,6 @@ ego_dp_preprocessing_res_powerplant.sql - @@ -583,6 +597,17 @@ ego_dp_preprocessing_res_powerplant.sql + + + + + + + + + + + diff --git a/documentation/bpmn/ego_dp_bpmn_section_loadarea.graphml b/documentation/bpmn/ego_dp_bpmn_section_loadarea.graphml index e31e6455..b5066d26 100644 --- a/documentation/bpmn/ego_dp_bpmn_section_loadarea.graphml +++ b/documentation/bpmn/ego_dp_bpmn_section_loadarea.graphml @@ -1,6 +1,6 @@ - + @@ -13,15 +13,15 @@ - + - + - eGo DataProcessing - substation - + eGo DataProcessing - loadarea + @@ -53,18 +53,18 @@ - + - + - + @@ -78,7 +78,7 @@ - ego_dp_loadarea_landuse.sql + ego_dp_loadarea_landuse.sql @@ -96,23 +96,17 @@ - openstreetmap.osm_deu_polygon_urban -osm_deu_polygon_urban_sector_1_residential_mview -osm_deu_polygon_urban_sector_2_retail_mview -osm_deu_polygon_urban_sector_3_industrial_mview -osm_deu_polygon_urban_sector_4_agricultural_mview - - - - - - + openstreetmap.osm_deu_polygon_urban +.osm_deu_polygon_urban_sector_1_residential_mview +.osm_deu_polygon_urban_sector_2_retail_mview +.osm_deu_polygon_urban_sector_3_industrial_mview +.osm_deu_polygon_urban_sector_4_agricultural_mview + - @@ -123,21 +117,15 @@ osm_deu_polygon_urban_sector_4_agricultural_mview - model_draft.ego_demand_la_zensus + model_draft.ego_demand_la_zensus model_draft.ego_demand_la_zensus_cluster -model_draft.zensus_population_per_load_area_stats_mview - - - - - - +model_draft.zensus_population_per_load_area_stats_mview + - @@ -148,7 +136,7 @@ model_draft.zensus_population_per_load_area_stats_mview - ego_dp_loadarea_loadmelt.sql + ego_dp_loadarea_loadmelt.sql @@ -166,21 +154,15 @@ model_draft.zensus_population_per_load_area_stats_mview - model_draft.ego_demand_load_collect + model_draft.ego_demand_load_collect model_draft.ego_demand_load_collect_buffer100_mview -model_draft.ego_demand_load_melt - - - - - - +model_draft.ego_demand_load_melt + - @@ -191,7 +173,7 @@ model_draft.ego_demand_load_melt - ego_dp_loadarea_loadcut_griddistrict.sql + ego_dp_loadarea_loadcut_griddistrict.sql @@ -209,26 +191,20 @@ model_draft.ego_demand_load_melt - model_draft.ego_osm_sector_per_griddistrict_1_residential + model_draft.ego_osm_sector_per_griddistrict_1_residential model_draft.ego_osm_sector_per_griddistrict_2_retail model_draft.ego_osm_sector_per_griddistrict_3_industrial model_draft.ego_osm_sector_per_griddistrict_4_agricultural model_draft.ego_demand_loadarea_centre_mview model_draft.ego_demand_loadarea_smaller100m2_mview model_draft.ego_demand_loadarea_error_noags_mview -model_draft.ego_demand_loadarea - - - - - - +model_draft.ego_demand_loadarea + - @@ -239,13 +215,7 @@ model_draft.ego_demand_loadarea - 20 sec - - - - - - + 20 sec @@ -261,13 +231,7 @@ model_draft.ego_demand_loadarea - 30 min - - - - - - + 30 min @@ -283,13 +247,7 @@ model_draft.ego_demand_loadarea - 20 min - - - - - - + 20 min @@ -305,13 +263,7 @@ model_draft.ego_demand_loadarea - 30 min - - - - - - + 30 min @@ -327,20 +279,14 @@ model_draft.ego_demand_loadarea - openstreetmap.osm_deu_polygon -political_bkg_vg250_1_sta_union_mview - - - - - - + openstreetmap.osm_deu_polygon +boundaries.bkg_vg250_1_sta_union_mview + - @@ -351,20 +297,14 @@ political_bkg_vg250_1_sta_union_mview - social.destatis_zensus_population_per_ha_mview -model_draft.ego_demand_la_osm - - - - - - + social.destatis_zensus_population_per_ha_mview +model_draft.ego_demand_la_osm + - @@ -375,7 +315,7 @@ model_draft.ego_demand_la_osm - ego_dp_loadarea_industry_consumer.sql + ego_dp_loadarea_industry_consumer.sql @@ -393,20 +333,14 @@ model_draft.ego_demand_la_osm - model_draft.ego_landuse_industry -model_draft.ego_demand_hv_largescaleconsumer - - - - - - + model_draft.ego_landuse_industry +model_draft.ego_demand_hv_largescaleconsumer + - @@ -417,25 +351,19 @@ model_draft.ego_demand_hv_largescaleconsumer - demand.ego_demand_federalstate + demand.ego_demand_federalstate economic.destatis_gva_per_districts political_boundary.bkg_vg250_4_krs_mview -supply.ego_conv_powerplant +model_draft.ego_dp_supply_conv_powerplant model_draft.ego_grid_ehv_substation_voronoi model_draft.ego_grid_hvmv_substation - - - - - - - + + - @@ -446,13 +374,7 @@ model_draft.ego_grid_hvmv_substation - 10 sec - - - - - - + 10 sec @@ -468,7 +390,7 @@ model_draft.ego_grid_hvmv_substation - ego_dp_loadarea_loads.sql + ego_dp_loadarea_loads.sql @@ -486,19 +408,13 @@ model_draft.ego_grid_hvmv_substation - model_draft.ego_demand_la_osm - - - - - - + model_draft.ego_demand_la_osm + - @@ -509,13 +425,7 @@ model_draft.ego_grid_hvmv_substation - 45 min - - - - - - + 45 min @@ -531,20 +441,14 @@ model_draft.ego_grid_hvmv_substation - model_draft.ego_demand_hv_largescaleconsumer -openstreetmap.osm_deu_polygon_urban - - - - - - + model_draft.ego_demand_hv_largescaleconsumer +openstreetmap.osm_deu_polygon_urban + - @@ -555,20 +459,14 @@ openstreetmap.osm_deu_polygon_urban - model_draft.ego_deu_loads_osm -model_draft.ego_social_zensus_load_cluster - - - - - - + model_draft.ego_deu_loads_osm +model_draft.ego_social_zensus_load_cluster + - @@ -579,7 +477,7 @@ model_draft.ego_social_zensus_load_cluster - ego_dp_loadarea_consumption.sql + ego_dp_loadarea_consumption.sql @@ -597,20 +495,14 @@ model_draft.ego_social_zensus_load_cluster - model_draft.ego_demand_loadarea -demand.ego_loadarea - - - - - - + model_draft.ego_demand_loadarea +demand.ego_loadarea + - @@ -621,13 +513,7 @@ demand.ego_loadarea - 16 sec - - - - - - + 16 sec @@ -643,21 +529,15 @@ demand.ego_loadarea - model_draft.ego_demand_per_district + model_draft.ego_demand_per_district model_draft.ego_demand_loadarea -demand.ego_demand_federalstate - - - - - - +demand.ego_demand_federalstate + - @@ -668,7 +548,7 @@ demand.ego_demand_federalstate - model_draft.ego_demand_load_melt + model_draft.ego_demand_load_melt model_draft.ego_grid_mv_griddistrict social.destatis_zensus_population_per_ha_mview economic.ioer_urban_share_industrial_centroid @@ -676,19 +556,13 @@ openstreetmap.osm_deu_polygon_urban_sector_1_residential_mview openstreetmap.osm_deu_polygon_urban_sector_2_retail_mview openstreetmap.osm_deu_polygon_urban_sector_3_industrial_mview openstreetmap.osm_deu_polygon_urban_sector_4_agricultural_mview -model_draft.ego_political_boundary_bkg_vg250_6_gem_clean - - - - - - +model_draft.ego_political_boundary_bkg_vg250_6_gem_clean + - @@ -699,14 +573,8 @@ model_draft.ego_political_boundary_bkg_vg250_6_gem_clean - Datenlizenz Deutschland – Namensnennung – Version 2.0 -Statistisches Bundesamt, Wiesbaden, Genesis-Online, 2016; Datenlizenz by-2-0 - - - - - - + Datenlizenz Deutschland – Namensnennung – Version 2.0 +Statistisches Bundesamt, Wiesbaden, Genesis-Online, 2016; Datenlizenz by-2-0 @@ -722,7 +590,7 @@ Statistisches Bundesamt, Wiesbaden, Genesis-Online, 2016; Datenlizenz by-2-0 - ego_dp_loadarea_census.sql + ego_dp_loadarea_census.sql @@ -749,8 +617,8 @@ Statistisches Bundesamt, Wiesbaden, Genesis-Online, 2016; Datenlizenz by-2-0 - + @@ -761,17 +629,11 @@ Statistisches Bundesamt, Wiesbaden, Genesis-Online, 2016; Datenlizenz by-2-0 - + - Open Database License (ODbL) v1.0 -© OpenStreetMap contributors - - - - - - + Open Database License (ODbL) v1.0 +© OpenStreetMap contributors @@ -784,16 +646,16 @@ Statistisches Bundesamt, Wiesbaden, Genesis-Online, 2016; Datenlizenz by-2-0 - + - openstreetmap.osm_deu + openstreetmap.osm_deu + - @@ -801,16 +663,16 @@ Statistisches Bundesamt, Wiesbaden, Genesis-Online, 2016; Datenlizenz by-2-0 - + - economic.destatis_gva_per_districts + economic.destatis_gva_per_districts + - @@ -818,17 +680,11 @@ Statistisches Bundesamt, Wiesbaden, Genesis-Online, 2016; Datenlizenz by-2-0 - + - Geodatenzugangsgesetz (GeoZG) -© GeoBasis-DE / BKG 2016 (Daten verändert) - - - - - - + Geodatenzugangsgesetz (GeoZG) +© GeoBasis-DE / BKG 2016 (Daten verändert) @@ -841,16 +697,16 @@ Statistisches Bundesamt, Wiesbaden, Genesis-Online, 2016; Datenlizenz by-2-0 - + - political_boundary.bkg_vg250 + boundaries.bkg_vg250 + - @@ -870,8 +726,8 @@ Statistisches Bundesamt, Wiesbaden, Genesis-Online, 2016; Datenlizenz by-2-0 - + @@ -885,13 +741,13 @@ Statistisches Bundesamt, Wiesbaden, Genesis-Online, 2016; Datenlizenz by-2-0 - social.destatis_zensus_population_per_ha + social.destatis_zensus_population_per_ha + - @@ -902,14 +758,8 @@ Statistisches Bundesamt, Wiesbaden, Genesis-Online, 2016; Datenlizenz by-2-0 - Open Database License (ODbL) v1.0 -© OpenStreetMap contributors - - - - - - + Open Database License (ODbL) v1.0 +© OpenStreetMap contributors @@ -925,14 +775,8 @@ Statistisches Bundesamt, Wiesbaden, Genesis-Online, 2016; Datenlizenz by-2-0 - Geodatenzugangsgesetz (GeoZG) -© GeoBasis-DE / BKG 2016 (Daten verändert) - - - - - - + Geodatenzugangsgesetz (GeoZG) +© GeoBasis-DE / BKG 2016 (Daten verändert) @@ -948,14 +792,8 @@ Statistisches Bundesamt, Wiesbaden, Genesis-Online, 2016; Datenlizenz by-2-0 - Open Database License (ODbL) v1.0 -© OpenStreetMap contributors - - - - - - + Open Database License (ODbL) v1.0 +© OpenStreetMap contributors @@ -971,14 +809,8 @@ Statistisches Bundesamt, Wiesbaden, Genesis-Online, 2016; Datenlizenz by-2-0 - Geodatenzugangsgesetz (GeoZG) -© GeoBasis-DE / BKG 2016 (Daten verändert) - - - - - - + Geodatenzugangsgesetz (GeoZG) +© GeoBasis-DE / BKG 2016 (Daten verändert) @@ -994,14 +826,8 @@ Statistisches Bundesamt, Wiesbaden, Genesis-Online, 2016; Datenlizenz by-2-0 - Open Database License (ODbL) v1.0 -© OpenStreetMap contributors - - - - - - + Open Database License (ODbL) v1.0 +© OpenStreetMap contributors @@ -1017,14 +843,8 @@ Statistisches Bundesamt, Wiesbaden, Genesis-Online, 2016; Datenlizenz by-2-0 - Geodatenzugangsgesetz (GeoZG) -© GeoBasis-DE / BKG 2016 (Daten verändert) - - - - - - + Geodatenzugangsgesetz (GeoZG) +© GeoBasis-DE / BKG 2016 (Daten verändert) @@ -1040,14 +860,8 @@ Statistisches Bundesamt, Wiesbaden, Genesis-Online, 2016; Datenlizenz by-2-0 - Datenlizenz Deutschland – Namensnennung – Version 2.0 -Statistisches Bundesamt, Wiesbaden, Genesis-Online, 2016; Datenlizenz by-2-0 - - - - - - + Datenlizenz Deutschland – Namensnennung – Version 2.0 +Statistisches Bundesamt, Wiesbaden, Genesis-Online, 2016; Datenlizenz by-2-0 @@ -1060,14 +874,14 @@ Statistisches Bundesamt, Wiesbaden, Genesis-Online, 2016; Datenlizenz by-2-0 - - + + - supply.ego_conv_powerplant + model_draft.ego_dp_supply_conv_powerplant - + @@ -1077,17 +891,17 @@ Statistisches Bundesamt, Wiesbaden, Genesis-Online, 2016; Datenlizenz by-2-0 - + - model_draft.ego_grid_ehv_substation -grid.ego_ehv_substation + model_draft.ego_grid_ehv_substation_voronoi + + - @@ -1095,104 +909,46 @@ grid.ego_ehv_substation - - - - Open Database License (ODbL) v1.0 -© OpenStreetMap contributors - - - - - - - - - - - - - - - - - - - - - - Geodatenzugangsgesetz (GeoZG) -© GeoBasis-DE / BKG 2016 (Daten verändert) - - - - - - - - - - - - - - - - - - - + - demand.ego_demand_federalstate + demand.ego_demand_federalstate + - - + - model_draft.ego_grid_hvmv_substation + model_draft.ego_grid_hvmv_substation model_draft.ego_grid_mv_griddistrict -model_draft.ego_demand_loadarea - - - - - - +model_draft.ego_demand_loadarea + - - + - 5 min - - - - - - + 5 min @@ -1202,38 +958,32 @@ model_draft.ego_demand_loadarea - + - model_draft.ego_data_processing_results + model_draft.ego_data_processing_results model_draft.ego_data_processing_results_mvgd -boundaries.bkg_vg250_statistics_mview - - - - - - +boundaries.bkg_vg250_statistics_mview + - - + - ego_dp_loadarea_statistic.sql + ego_dp_loadarea_statistic.sql @@ -1245,13 +995,13 @@ boundaries.bkg_vg250_statistics_mview - + - ego_dp_loadarea_loadcut_voronoi.sql + ego_dp_loadarea_loadcut_voronoi.sql @@ -1263,13 +1013,13 @@ boundaries.bkg_vg250_statistics_mview - + - ego_dp_loadarea_peakload.py + ego_dp_loadarea_peakload.py @@ -1281,66 +1031,48 @@ boundaries.bkg_vg250_statistics_mview - + - model_draft.ego_demand_loadarea -scenario.ego_slp_parameters - - - - - - + model_draft.ego_demand_loadarea +scenario.ego_slp_parameters + - - + - model_draft.ego_demand_loadarea - - - - - - + model_draft.ego_demand_loadarea + - - + - 1 h - - - - - - + 1 h @@ -1350,13 +1082,13 @@ scenario.ego_slp_parameters - + - ego_dp_loadarea_griddistrict_results.sql + ego_dp_loadarea_griddistrict_results.sql @@ -1368,20 +1100,32 @@ scenario.ego_slp_parameters - + - Open Database License (ODbL) v1.0 -© OpenStreetMap contributors - - - - - - + Open Database License (ODbL) v1.0 +© OpenStreetMap contributors + + + + + + + + + + + + + + + + Vervielfältigung und Verbreitung, auch +auszugsweise, mit Quellenangabe gestattet. +© Statistisches Landesamt Baden-Württemberg @@ -1873,7 +1617,7 @@ scenario.ego_slp_parameters - + @@ -1885,7 +1629,7 @@ scenario.ego_slp_parameters - + @@ -1897,7 +1641,7 @@ scenario.ego_slp_parameters - + @@ -1909,7 +1653,7 @@ scenario.ego_slp_parameters - + @@ -1921,7 +1665,7 @@ scenario.ego_slp_parameters - + @@ -1933,7 +1677,7 @@ scenario.ego_slp_parameters - + @@ -1943,7 +1687,7 @@ scenario.ego_slp_parameters - + @@ -1953,7 +1697,7 @@ scenario.ego_slp_parameters - + @@ -1963,7 +1707,7 @@ scenario.ego_slp_parameters - + @@ -1975,7 +1719,7 @@ scenario.ego_slp_parameters - + @@ -1987,7 +1731,7 @@ scenario.ego_slp_parameters - + @@ -1999,7 +1743,7 @@ scenario.ego_slp_parameters - + @@ -2011,7 +1755,7 @@ scenario.ego_slp_parameters - + @@ -2023,6 +1767,39 @@ scenario.ego_slp_parameters + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/documentation/bpmn/ego_dp_bpmn_section_substation.graphml b/documentation/bpmn/ego_dp_bpmn_section_substation.graphml index ad1e2b71..7e515859 100644 --- a/documentation/bpmn/ego_dp_bpmn_section_substation.graphml +++ b/documentation/bpmn/ego_dp_bpmn_section_substation.graphml @@ -17,11 +17,11 @@ - + - eGo DataProcessing - substation - + eGo DataProcessing - substation + @@ -53,7 +53,7 @@ - + @@ -64,7 +64,7 @@ - + @@ -97,7 +97,8 @@ model_draft.ego_grid_hvmv_substation_dummy -model_draft.ego_grid_hvmv_substation_voronoi +model_draft.ego_grid_hvmv_substation_voronoi + @@ -245,10 +246,10 @@ openstreetmap.osm_deu_point boundaries.bkg_vg250_1_sta_union_mview - + @@ -440,10 +441,10 @@ model_draft.ego_grid_hvmv_substationopenstreetmap.osm_deu - + @@ -474,32 +475,15 @@ model_draft.ego_grid_hvmv_substationboundaries.bkg_vg250 - + - - - - - - Open Database License (ODbL) v1.0 -© OpenStreetMap contributors - - - - - - - - - - @@ -516,7 +500,7 @@ model_draft.ego_grid_hvmv_substation - + @@ -533,7 +517,7 @@ model_draft.ego_grid_hvmv_substation - + @@ -550,7 +534,7 @@ model_draft.ego_grid_hvmv_substation - + @@ -567,7 +551,7 @@ model_draft.ego_grid_hvmv_substation - + @@ -584,7 +568,7 @@ model_draft.ego_grid_hvmv_substation - + @@ -601,7 +585,7 @@ model_draft.ego_grid_hvmv_substation - + @@ -618,7 +602,7 @@ model_draft.ego_grid_hvmv_substation - + @@ -635,7 +619,7 @@ model_draft.ego_grid_hvmv_substation - + @@ -653,7 +637,7 @@ grid.ego_hvmv_substation - + @@ -670,7 +654,7 @@ grid.ego_hvmv_substation - + @@ -687,7 +671,7 @@ grid.ego_hvmv_substation - + @@ -706,13 +690,14 @@ grid.ego_hvmv_substation - + - model_draft.ego_grid_ehv_substation_voronoi + model_draft.ego_grid_ehv_substation_voronoi + @@ -723,7 +708,7 @@ grid.ego_hvmv_substation - + @@ -739,7 +724,7 @@ grid.ego_hvmv_substation - + @@ -757,7 +742,7 @@ model_draft.ego_grid_ehv_substation - + @@ -775,7 +760,7 @@ grid.ego_ehv_substation - + @@ -792,7 +777,7 @@ grid.ego_ehv_substation - + @@ -809,7 +794,7 @@ grid.ego_ehv_substation - + @@ -826,7 +811,7 @@ grid.ego_ehv_substation - + @@ -843,7 +828,7 @@ grid.ego_ehv_substation - + @@ -860,20 +845,56 @@ grid.ego_ehv_substation + + + + + + + preprocessing +ego_dp_structure_boundaries_vg250.sql + + + + + + + + + + + + + + + + + osmTGmod + + + + + + + + + + + + - - + + - preprocessing -model_draft.ego_boundaries_bkg_vg250_6_gem_clean + Open Database License (ODbL) v1.0 +© OpenStreetMap contributors - - + @@ -904,7 +925,7 @@ model_draft.ego_boundaries_bkg_vg250_6_gem_clean - + @@ -916,7 +937,7 @@ model_draft.ego_boundaries_bkg_vg250_6_gem_clean - + @@ -1072,7 +1093,7 @@ model_draft.ego_boundaries_bkg_vg250_6_gem_clean - + @@ -1144,7 +1165,7 @@ model_draft.ego_boundaries_bkg_vg250_6_gem_clean - + @@ -1204,7 +1225,7 @@ model_draft.ego_boundaries_bkg_vg250_6_gem_clean - + @@ -1216,7 +1237,7 @@ model_draft.ego_boundaries_bkg_vg250_6_gem_clean - + @@ -1228,7 +1249,7 @@ model_draft.ego_boundaries_bkg_vg250_6_gem_clean - + @@ -1240,7 +1261,7 @@ model_draft.ego_boundaries_bkg_vg250_6_gem_clean - + @@ -1252,7 +1273,7 @@ model_draft.ego_boundaries_bkg_vg250_6_gem_clean - + @@ -1264,8 +1285,7 @@ model_draft.ego_boundaries_bkg_vg250_6_gem_clean - - + @@ -1275,7 +1295,29 @@ model_draft.ego_boundaries_bkg_vg250_6_gem_clean - + + + + + + + + + + + + + + + + + + + + + + + From 52a171e3fd576853c2e652aa0f1bdc9f5132bc98 Mon Sep 17 00:00:00 2001 From: IlkaCu Date: Wed, 26 Feb 2020 10:58:49 +0100 Subject: [PATCH 08/84] update bpmn docu --- .../ego_dp_bpmn_section_griddistrict.graphml | 117 +- .../ego_dp_bpmn_section_powerflow.graphml | 2228 +++++++++++++++++ .../ego_dp_bpmn_section_substation.graphml | 314 +-- .../bpmn/ego_dp_bpmn_sections_input.graphml | 795 ++++++ 4 files changed, 3160 insertions(+), 294 deletions(-) create mode 100644 documentation/bpmn/ego_dp_bpmn_section_powerflow.graphml create mode 100644 documentation/bpmn/ego_dp_bpmn_sections_input.graphml diff --git a/documentation/bpmn/ego_dp_bpmn_section_griddistrict.graphml b/documentation/bpmn/ego_dp_bpmn_section_griddistrict.graphml index d84302f4..e3054dad 100644 --- a/documentation/bpmn/ego_dp_bpmn_section_griddistrict.graphml +++ b/documentation/bpmn/ego_dp_bpmn_section_griddistrict.graphml @@ -17,11 +17,11 @@ - + - eGo DataProcessing - griddistrict - + eGo DataProcessing - griddistrict + @@ -53,18 +53,18 @@ - + - + - + @@ -96,7 +96,8 @@ - model_draft.ego_boundaries_hvmv_subst_per_gem_1_mview + model_draft.ego_grid_mv_griddistrict +model_draft.ego_boundaries_hvmv_subst_per_gem_1_mview model_draft.ego_boundaries_hvmv_subst_per_gem_2_mview model_draft.ego_boundaries_hvmv_subst_per_gem_3_mview model_draft.ego_grid_hvmv_substation_voronoi_mview @@ -117,7 +118,7 @@ model_draft.ego_grid_mv_griddistrict_dump - 2 min + ? @@ -134,6 +135,9 @@ model_draft.ego_grid_mv_griddistrict_dump + model_draft.ego_boundaries_bkg_vg250_6_gem_clean +model_draft.ego_grid_hvmv_substation +model_draft.ego_grid_hvmv_substation_voronoi @@ -215,7 +219,7 @@ model_draft.ego_grid_mv_griddistrict_dump - model_draft.ego_boundaries_bkg_vg250_6_gem_clean + bkg_vg250 @@ -266,8 +270,7 @@ model_draft.ego_grid_mv_griddistrict_dump - model_draft.ego_grid_hvmv_substation -model_draft.ego_grid_hvmv_substation_voronoi + DP substation @@ -336,12 +339,25 @@ model_draft.ego_grid_hvmv_substation_voronoi - + model_draft.ego_dp_supply_conv_powerplant +model_draft.ego_dp_supply_res_powerplant +model_draft.ego_grid_hvmv_substation +model_draft.ego_grid_mv_griddistrict +model_draft.ego_grid_ehv_substation_voronoi + - + + + + + + + + + @@ -358,7 +374,7 @@ model_draft.ego_grid_hvmv_substation_voronoi - + model_draft.ego_dp_supply_conv_powerplant model_draft.ego_dp_supply_res_powerplant @@ -379,11 +395,7 @@ model_draft.ego_dp_supply_res_powerplant - model_draft.ego_dp_supply_conv_powerplant -model_draft.ego_dp_supply_res_powerplant -model_draft.ego_grid_hvmv_substation -model_draft.ego_grid_mv_griddistrict -model_draft.ego_grid_ehv_substation_voronoi + supply_powerplant @@ -395,58 +407,17 @@ model_draft.ego_grid_ehv_substation_voronoi - - - - - - preprocessing -ego_dp_structure_boundaries_vg250.sql - - - - - - - - - - - - - - - - - - preprocessing -ego_dp_preprocessing_conv_powerplant.sql -ego_dp_preprocessing_res_powerplant.sql - - - - - - - - - - - - - - + + - model_draft.ego_grid_mv_griddistrict -grid.ego_mv_griddistrict + ? - - + @@ -577,32 +548,34 @@ grid.ego_mv_griddistrict - + + - - + + - + + - - + + - + - + diff --git a/documentation/bpmn/ego_dp_bpmn_section_powerflow.graphml b/documentation/bpmn/ego_dp_bpmn_section_powerflow.graphml new file mode 100644 index 00000000..821753fd --- /dev/null +++ b/documentation/bpmn/ego_dp_bpmn_section_powerflow.graphml @@ -0,0 +1,2228 @@ + + + + + + + + + + + + + + + + + + + + + + + eGo DataProcessing - powerflow + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ego_dp_powerflow_fix_ehv_subnetworks.sql + + + + + + + + + + + + + + + + + + model_draft.ego_grid_pf_hv_scenario_settings +model_draft.ego_grid_pf_hv_source +model_draft.ego_grid_pf_hv_bus +model_draft.ego_grid_pf_hv_busmap +model_draft.ego_grid_pf_hv_generator +model_draft.ego_grid_pf_hv_line +model_draft.ego_grid_pf_hv_load +model_draft.ego_grid_pf_hv_storage +model_draft.ego_grid_pf_hv_temp_resolution +model_draft.ego_grid_pf_hv_transformer +model_draft.ego_grid_pf_hv_bus_v_mag_set +model_draft.ego_grid_pf_hv_generator_pq_set +model_draft.ego_grid_pf_hv_load_pq_set +model_draft.ego_grid_pf_hv_storage_pq_set +model_draft.ego_grid_pf_hv_results_bus +model_draft.ego_grid_pf_hv_results_bus_t +model_draft.ego_grid_pf_hv_results_generator +model_draft.ego_grid_pf_hv_results_generator_t +model_draft.ego_grid_pf_hv_results_line +model_draft.ego_grid_pf_hv_results_line_t +model_draft.ego_grid_pf_hv_results_load +model_draft.ego_grid_pf_hv_results_load_t +model_draft.ego_grid_pf_hv_results_meta +model_draft.ego_grid_pf_hv_results_storage +model_draft.ego_grid_pf_hv_results_storage_t +model_draft.ego_grid_pf_hv_results_transformer +model_draft.ego_grid_pf_hv_results_transformer_t +model_draft.ego_grid_pf_hv_link + + + + + + + + + + + + + + + + + ego_dp_powerflow_hv_setup.sql + + + + + + + + + + + + + + + + + + ego_dp_powerflow_osmtgmod_to_pypsa.sql + + + + + + + + + + + + + + + + + + grid.otg_ehvhv_bus_data +grid.otg_ehvhv_branch_data +grid.otg_ehvhv_branch_data +model_draft.ego_grid_pf_hv_bus +model_draft.ego_grid_pf_hv_line +model_draft.ego_grid_pf_hv_transformer + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ego_dp_powerflow_electrical_neighbour.sql + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ego_dp_powerflow_grid_future_scenarios.sql + + + + + + + + + + + + + + + + + + + model_draft.ego_grid_pf_hv_bus +model_draft.ego_grid_pf_hv_line +model_draft.ego_grid_pf_hv_transformer + + + + + + + + + + + + + + + + + ego_dp_powerflow_assignment_otgid.sql + + + + + + + + + + + + + + + + + + + ego_dp_powerflow_assignment_unid.sql + + + + + + + + + + + + + + + + + + + ego_dp_powerflow_create_pp_mviews.sql + + + + + + + + + + + + + + + + + + + ego_dp_powerflow_assignment_generator.sql + + + + + + + + + + + + + + + + + + + ego_dp_powerflow_assignment_load.sql + + + + + + + + + + + + + + + + + + + ego_dp_powerflow_assignment_storage.sql + + + + + + + + + + + + + + + + + + + ego_dp_powerflow_timeseries_generator.sql + + + + + + + + + + + + + + + + + + + ego_dp_powerflow_griddistrict_demand.py + + + + + + + + + + + + + + + + + + + ego_dp_powerflow_timeseries_demand.sql + + + + + + + + + + + + + + + + + + + ego_dp_powerflow_lopf_data.sql + + + + + + + + + + + + + + + + + + + ego_dp_datacheck.sql + + + + + + + + + + + + + + + + + + + model_draft.ego_grid_hv_electrical_neighbours_bus +model_draft.ego_grid_hv_electrical_neighbours_line +model_draft.ego_grid_hv_electrical_neighbours_transformer +model_draft.ego_grid_hv_electrical_neighbours_link + + + + + + + + + + + + + + + + + model_draft.ego_grid_pp_entsoe_bus +model_draft.ego_grid_pp_entsoe_line +model_draft.ego_grid_pf_hv_line +grid.otg_ehvhv_branch_data +grid.otg_ehvhv_bus_data + + + + + + + + + + + + + + + + + + + model_draft.ego_grid_pf_hv_bus +model_draft.ego_grid_pf_hv_line +model_draft.ego_grid_pf_hv_transformer +grid.otg_ehvhv_bus_data + + + + + + + + + + + + + + + + + + model_draft.ego_grid_pf_hv_bus +model_draft.ego_grid_pf_hv_line +model_draft.ego_grid_pf_hv_transformer + + + + + + + + + + + + + + + + + model_draft.ego_grid_pf_hv_bus +model_draft.ego_grid_pf_hv_line +model_draft.ego_grid_pf_hv_link +model_draft.ego_grid_pf_hv_transformer + + + + + + + + + + + + + + + + + + model_draft.ego_grid_pf_hv_bus +model_draft.ego_grid_pf_hv_line +model_draft.ego_grid_pf_hv_link +model_draft.ego_grid_pf_hv_transformer + + + + + + + + + + + + + + + + + + model_draft.ego_dp_supply_res_powerplant +model_draft.ego_dp_supply_conv_powerplant +model_draft.ego_grid_hvmv_substation +model_draft.ego_grid_ehv_substation +model_draft.ego_grid_pf_hv_bus +climate.cosmoclmgrid + + + + + + + + + + + + + + + + + + + + + + + + + + model_draft.ego_dp_supply_res_powerplant +model_draft.ego_dp_supply_conv_powerplant + + + + + + + + + + + + + + + + + cosmoclmgrid + + + + + + + + + + + + + + + + + DP griddistrict + + + + + + + + + + + + + + + + + DP substation + + + + + + + + + + + + + + + + + model_draft.ego_dp_supply_res_powerplant +model_draft.ego_dp_supply_conv_powerplant + + + + + + + + + + + + + + + + + model_draft.ego_supply_generator + + + + + + + + + + + + + + + + + + + + + + + + + model_draft.ego_dp_supply_res_powerplant +model_draft.ego_dp_supply_conv_powerplant + + + + + + + + + + + + + + + + + model_draft.ego_supply_conv_powerplant_sq_mview +model_draft.ego_supply_conv_powerplant_nep2035_mview +model_draft.ego_supply_conv_powerplant_ego100_mview +model_draft.ego_supply_res_powerplant_sq_mview +model_draft.ego_supply_res_powerplant_nep2035_mview +model_draft.ego_supply_res_powerplant_ego100_mview + + + + + + + + + + + + + + + + + model_draft.ego_dp_supply_res_powerplant +model_draft.ego_dp_supply_conv_powerplant + + + + + + + + + + + + + + + + + model_draft.ego_supply_conv_powerplant_sq_mview +model_draft.ego_supply_res_powerplant_sq_mview +model_draft.ego_supply_conv_powerplant_nep2035_mview +model_draft.ego_supply_res_powerplant_nep2035_mview +model_draft.ego_supply_conv_powerplant_ego100_mview +model_draft.ego_supply_res_powerplant_ego100_mview +model_draft.ego_grid_pf_hv_source +model_draft.ego_power_class +model_draft.ego_grid_pf_hv_generator + + + + + + + + + + + + + + + + + + + + + + + + + + model_draft.ego_grid_pf_hv_generator + + + + + + + + + + + + + + + + + model_draft.ego_supply_pf_generator_single +model_draft.ego_supply_aggr_weather + + + + + + + + + + + + + + + + + + model_draft.ego_demand_loadarea +model_draft.ego_demand_hv_largescaleconsumer +model_draft.ego_grid_pf_hv_load + + + + + + + + + + + + + + + + + + + + + + + + + + model_draft.ego_grid_pf_hv_load + + + + + + + + + + + + + + + + + model_draft.ego_demand_loads +model_draft.ego_demand_pf_load_single + + + + + + + + + + + + + + + + + model_draft.ego_supply_conv_powerplant_sq_mview +model_draft.ego_grid_pf_hv_source +model_draft.ego_grid_pf_hv_storage +model_draft.ego_grid_hv_electrical_neighbours_bus +calc_renpass_gis.renpass_gis_storage + + + + + + + + + + + + + + + + + + + + + + + + + model_draft.ego_grid_pf_hv_storage + + + + + + + + + + + + + + + + + model_draft.ego_demand_pf_storage_single + + + + + + + + + + + + + + + + + model_draft.ego_supply_pf_generator_single +model_draft.ego_grid_pf_hv_generator +model_draft.ego_renewable_feedin +model_draft.ego_grid_hv_electrical_neighbours_bus +calc_renpass_gis.renpass_gis_linear_transformer +calc_renpass_gis.renpass_gis_source +calc_renpass_gis.renpass_gis_results + + + + + + + + + + + + + + + + + + + + + + + + + model_draft.ego_grid_pf_hv_generator_pq_set +model_draft.ego_grid_pf_hv_generator +model_draft.ego_grid_pf_hv_load + + + + + + + + + + + + + + + + + + calc_renpass_gis.pf_pp_by_source_aggr_id +calc_renpass_gis.pp_feedin_by_pf_source +model_draft.ren_feedin_by_gen_id +calc_renpass_gis.translate_to_pf +model_draft.ren_feedin_foreign +model_draft.offshore_feedin_foreign + + + + + + + + + + + + + + + + + renpassGIS + + + + + + + + + + + + + + + + + Preprocessing +ego_setup_renewable_feedin.sql + + + + + + + + + + + + + + + + + DP loadarea + + + + + + + + + + + + + + + + + + + + + + + + + model_draft.ego_grid_pf_hv_load_pq_set + + + + + + + + + + + + + + + + + + + model_draft.ego_grid_hvmv_substation +model_draft.ego_demand_hvmv_demand +model_draft.ego_grid_hv_electrical_neighbours_bus +calc_renpass_gis.renpass_gis_results +model_draft.ego_grid_pf_hv_load +model_draft.ego_grid_pf_hv_load_pq_set + + + + + + + + + + + + + + + + + + + + + + + + + model_draft.ego_grid_pf_hv_generator +model_draft.ego_grid_pf_hv_generator_pq_set +model_draft.ego_grid_pf_hv_storage + + + + + + + + + + + + + + + + + model_draft.ego_grid_pf_hv_generator +model_draft.ego_grid_pf_hv_generator_pq_set +model_draft.ego_grid_pf_hv_storage +model_draft.ego_grid_hvmv_substation +model_draft.ego_storage_h2_areas_de + + + + + + + + + + + + + + + + + storage h2 areas + + + + + + + + + + + + + + + + + model_draft_ego_grid_pf_hv_data_check + + + + + + + + + + + + + + + + + model_draft.ego_grid_pf_hv_line +model_draft.ego_grid_pf_hv_bus +model_draft.ego_grid_pf_hv_transformer +model_draft.ego_grid_pf_hv_load +model_draft.ego_grid_pf_hv_load_pq_set +model_draft.ego_grid_hvmv_substation +model_draft.ego_grid_pf_hv_generator +model_draft.ego_supply_pf_generator_single +model_draft.ego_grid_pf_hv_generator_pq_set +model_draft.ego_dp_supply_res_powerplant +model_draft.ego_grid_mv_griddistrict + + + + + + + + + + + + + + + + + osmTGmod /otg + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/documentation/bpmn/ego_dp_bpmn_section_substation.graphml b/documentation/bpmn/ego_dp_bpmn_section_substation.graphml index 7e515859..3d4ae096 100644 --- a/documentation/bpmn/ego_dp_bpmn_section_substation.graphml +++ b/documentation/bpmn/ego_dp_bpmn_section_substation.graphml @@ -17,11 +17,11 @@ - + - eGo DataProcessing - substation - + eGo DataProcessing - substation + @@ -31,40 +31,40 @@ - - - - + + + + + - - + - + - + @@ -93,11 +93,12 @@ - + - model_draft.ego_grid_hvmv_substation_dummy + model_draft.ego_grid_hvmv_substation_dummy model_draft.ego_grid_hvmv_substation_voronoi + @@ -115,7 +116,7 @@ model_draft.ego_grid_hvmv_substation_voronoi - 6 min + ? @@ -146,7 +147,7 @@ model_draft.ego_boundaries_bkg_vg250_6_gem_clean - + model_draft.ego_grid_hvmv_substation @@ -206,7 +207,7 @@ ego_dp_substation_ehv.sql - 32 min + min? @@ -222,7 +223,7 @@ ego_dp_substation_ehv.sql - 3 min + min? @@ -280,7 +281,7 @@ boundaries.bkg_vg250_1_sta_union_mview - + model_draft.ego_grid_ehv_substation @@ -344,7 +345,7 @@ ego_dp_substation_otg.sql - 1 sec + ? @@ -377,7 +378,7 @@ model_draft.ego_grid_hvmv_substation - + model_draft.ego_grid_ehv_substation model_draft.ego_grid_hvmv_substation @@ -438,13 +439,13 @@ model_draft.ego_grid_hvmv_substation - openstreetmap.osm_deu + openstreetmap + - @@ -472,13 +473,13 @@ model_draft.ego_grid_hvmv_substation - boundaries.bkg_vg250 + bkg_vg250 + - @@ -620,58 +621,6 @@ model_draft.ego_grid_hvmv_substation - - - - - - model_draft.ego_grid_hvmv_substation -grid.ego_hvmv_substation - - - - - - - - - - - - - - - - - Open Database License (ODbL) v1.0 -© OpenStreetMap contributors - - - - - - - - - - - - - - - - Geodatenzugangsgesetz (GeoZG) -© GeoBasis-DE / BKG 2016 (Daten verändert) - - - - - - - - - - @@ -690,13 +639,14 @@ grid.ego_hvmv_substation - + - + - model_draft.ego_grid_ehv_substation_voronoi + model_draft.ego_grid_ehv_substation_voronoi + @@ -708,7 +658,7 @@ grid.ego_hvmv_substation - + @@ -724,7 +674,7 @@ grid.ego_hvmv_substation - + @@ -742,28 +692,10 @@ model_draft.ego_grid_ehv_substation - - - - - - - model_draft.ego_grid_ehv_substation -grid.ego_ehv_substation - - - - - - - - - - - + - + Open Database License (ODbL) v1.0 @@ -777,10 +709,10 @@ grid.ego_ehv_substation - + - + Geodatenzugangsgesetz (GeoZG) @@ -794,31 +726,31 @@ grid.ego_ehv_substation - + - - + + - Open Database License (ODbL) v1.0 -© OpenStreetMap contributors + osmTGmod /otg + - + - + - + - Geodatenzugangsgesetz (GeoZG) -© GeoBasis-DE / BKG 2016 (Daten verändert) + Open Database License (ODbL) v1.0 +© OpenStreetMap contributors @@ -828,73 +760,36 @@ grid.ego_ehv_substation - + - - + + - grid.otg_ehvhv_bus_data + model_draft.ego_grid_hvmv_substation - + - - - - - - - preprocessing -ego_dp_structure_boundaries_vg250.sql - - - - - - - - - - - - - - - - - - osmTGmod - - - - - - - - - - - - + - - + + - Open Database License (ODbL) v1.0 -© OpenStreetMap contributors + model_draft.ego_grid_ehv_substation - + + @@ -926,18 +821,6 @@ ego_dp_structure_boundaries_vg250.sql - - - - - - - - - - - - @@ -949,7 +832,7 @@ ego_dp_structure_boundaries_vg250.sql - + @@ -961,7 +844,7 @@ ego_dp_structure_boundaries_vg250.sql - + @@ -973,7 +856,7 @@ ego_dp_structure_boundaries_vg250.sql - + @@ -985,7 +868,7 @@ ego_dp_structure_boundaries_vg250.sql - + @@ -997,7 +880,7 @@ ego_dp_structure_boundaries_vg250.sql - + @@ -1009,7 +892,7 @@ ego_dp_structure_boundaries_vg250.sql - + @@ -1021,7 +904,7 @@ ego_dp_structure_boundaries_vg250.sql - + @@ -1033,7 +916,7 @@ ego_dp_structure_boundaries_vg250.sql - + @@ -1045,7 +928,7 @@ ego_dp_structure_boundaries_vg250.sql - + @@ -1057,7 +940,7 @@ ego_dp_structure_boundaries_vg250.sql - + @@ -1069,7 +952,7 @@ ego_dp_structure_boundaries_vg250.sql - + @@ -1081,7 +964,7 @@ ego_dp_structure_boundaries_vg250.sql - + @@ -1093,7 +976,7 @@ ego_dp_structure_boundaries_vg250.sql - + @@ -1105,7 +988,7 @@ ego_dp_structure_boundaries_vg250.sql - + @@ -1117,7 +1000,7 @@ ego_dp_structure_boundaries_vg250.sql - + @@ -1129,7 +1012,7 @@ ego_dp_structure_boundaries_vg250.sql - + @@ -1141,7 +1024,7 @@ ego_dp_structure_boundaries_vg250.sql - + @@ -1153,7 +1036,7 @@ ego_dp_structure_boundaries_vg250.sql - + @@ -1165,7 +1048,7 @@ ego_dp_structure_boundaries_vg250.sql - + @@ -1177,7 +1060,7 @@ ego_dp_structure_boundaries_vg250.sql - + @@ -1189,7 +1072,7 @@ ego_dp_structure_boundaries_vg250.sql - + @@ -1201,7 +1084,7 @@ ego_dp_structure_boundaries_vg250.sql - + @@ -1213,7 +1096,7 @@ ego_dp_structure_boundaries_vg250.sql - + @@ -1225,7 +1108,7 @@ ego_dp_structure_boundaries_vg250.sql - + @@ -1237,7 +1120,7 @@ ego_dp_structure_boundaries_vg250.sql - + @@ -1249,19 +1132,7 @@ ego_dp_structure_boundaries_vg250.sql - - - - - - - - - - - - - + @@ -1273,7 +1144,7 @@ ego_dp_structure_boundaries_vg250.sql - + @@ -1285,7 +1156,7 @@ ego_dp_structure_boundaries_vg250.sql - + @@ -1295,34 +1166,33 @@ ego_dp_structure_boundaries_vg250.sql - - + - - + + - + - - + + - + - + diff --git a/documentation/bpmn/ego_dp_bpmn_sections_input.graphml b/documentation/bpmn/ego_dp_bpmn_sections_input.graphml new file mode 100644 index 00000000..d59e13e1 --- /dev/null +++ b/documentation/bpmn/ego_dp_bpmn_sections_input.graphml @@ -0,0 +1,795 @@ + + + + + + + + + + + + + + + + + + + + + + + eGo DataProcessing + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + INPUT DATA +CHECK + + + + + + + + + + + + + + + + + + GRIDDISTRICT + + + + + + + + + + + + + + + + + + SUBSTATION + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + POWERFLOW + + + + + + + + + + + + + + + + + + LOADAREA + + + + + + + + + + + + + + + + + + DP substation + + + + + + + + + + + + + + + + + LOWVOLTAGE + + + + + + + + + + + + + + + + + + REA + + + + + + + + + + + + + + + + + + VERSIONING + + + + + + + + + + + + + + + + + + POST- +PROCESSING + + + + + + + + + + + + + + + + + + openstreetmap +vg250 +osmTGmod + + + + + + + + + + + + + + + + + vg250 +conv_powerplant +res_powerplant + + + + + + + + + + + + + + + + + openstreetmap +vg250 +VGR / gva +zensus +conv_powerplant +demand_federalstate + + + + + + + + + + + + + + + + + + DP substation + + + + + + + + + + + + + + + + + slp_parameter + + + + + + + + + + + + + + + + + DP loadarea +DP griddistrict + + + + + + + + + + + + + + + + + vg250 +wind_potential_area +res_powerplant + + + + + + + + + + + + + + + + + DP loadarea +DP griddistrict + + + + + + + + + + + + + + + + + osmTGmod +cosmoclmgrid +renpassGIS +renewable_feedin +storage h2 areas + + + + + + + + + + + + + + + + + DP griddistrict +DP loadarea +DP substation + + + + + + + + + + + + + + + + + bnetza_bbplg +bnetza_enlag +renpassGIS + + + + + + + + + + + + + + + + + DP powerflow + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + From a376e6a69ebfeb2d9c7dce979deea5f8f3cfeff6 Mon Sep 17 00:00:00 2001 From: IlkaCu Date: Tue, 3 Mar 2020 14:13:40 +0100 Subject: [PATCH 09/84] update bpmn for loadarea section --- .../bpmn/ego_dp_bpmn_section_loadarea.graphml | 1069 ++++++++--------- 1 file changed, 532 insertions(+), 537 deletions(-) diff --git a/documentation/bpmn/ego_dp_bpmn_section_loadarea.graphml b/documentation/bpmn/ego_dp_bpmn_section_loadarea.graphml index b5066d26..a852c905 100644 --- a/documentation/bpmn/ego_dp_bpmn_section_loadarea.graphml +++ b/documentation/bpmn/ego_dp_bpmn_section_loadarea.graphml @@ -17,11 +17,11 @@ - + - eGo DataProcessing - loadarea - + eGo DataProcessing - loadarea + @@ -53,7 +53,7 @@ - + @@ -64,7 +64,7 @@ - + @@ -75,7 +75,7 @@ - + ego_dp_loadarea_landuse.sql @@ -93,14 +93,18 @@ - + - openstreetmap.osm_deu_polygon_urban -.osm_deu_polygon_urban_sector_1_residential_mview -.osm_deu_polygon_urban_sector_2_retail_mview -.osm_deu_polygon_urban_sector_3_industrial_mview -.osm_deu_polygon_urban_sector_4_agricultural_mview + openstreetmap.osm_deu_polygon_urban_sector_1_residential_mview +openstreetmap.osm_deu_polygon_urban_sector_2_retail_mview +openstreetmap.osm_deu_polygon_urban_sector_3_industrial_mview +openstreetmap.osm_deu_polygon_urban_sector_4_agricultural_mview +openstreetmap.osm_deu_polygon_urban +openstreetmap.osm_deu_polygon_urban_error_geom_vg250_mview +openstreetmap.osm_deu_polygon_urban_vg250_cut_mview +openstreetmap.osm_deu_polygon_urban_vg250_clean_cut_multi_mview +openstreetmap.osm_deu_polygon_urban_vg250_clean_cut_mview @@ -114,12 +118,12 @@ - + - model_draft.ego_demand_la_zensus + model_draft.ego_demand_la_zensus model_draft.ego_demand_la_zensus_cluster -model_draft.zensus_population_per_load_area_stats_mview +model_draft.ego_society_zensus_per_la_mview @@ -133,7 +137,7 @@ model_draft.zensus_population_per_load_area_stats_mview - + ego_dp_loadarea_loadmelt.sql @@ -151,7 +155,7 @@ model_draft.zensus_population_per_load_area_stats_mview - + model_draft.ego_demand_load_collect @@ -170,7 +174,7 @@ model_draft.ego_demand_load_melt - + ego_dp_loadarea_loadcut_griddistrict.sql @@ -188,17 +192,17 @@ model_draft.ego_demand_load_melt - + - model_draft.ego_osm_sector_per_griddistrict_1_residential + model_draft.ego_osm_sector_per_griddistrict_1_residential model_draft.ego_osm_sector_per_griddistrict_2_retail model_draft.ego_osm_sector_per_griddistrict_3_industrial model_draft.ego_osm_sector_per_griddistrict_4_agricultural -model_draft.ego_demand_loadarea_centre_mview model_draft.ego_demand_loadarea_smaller100m2_mview model_draft.ego_demand_loadarea_error_noags_mview -model_draft.ego_demand_loadarea +model_draft.ego_demand_loadarea +openstreetmap.osm_deu_polygon_urban_sector_3_industrial_nolargescale_mview @@ -212,71 +216,7 @@ model_draft.ego_demand_loadarea - - - - 20 sec - - - - - - - - - - - - - - - - 30 min - - - - - - - - - - - - - - - - 20 min - - - - - - - - - - - - - - - - 30 min - - - - - - - - - - - - - + openstreetmap.osm_deu_polygon @@ -291,13 +231,14 @@ boundaries.bkg_vg250_1_sta_union_mview - + - + - social.destatis_zensus_population_per_ha_mview + model_draft.destatis_zensus_population_per_ha_invg_mview +society.destatis_zensus_population_per_ha_mview model_draft.ego_demand_la_osm @@ -309,10 +250,10 @@ model_draft.ego_demand_la_osm - + - + ego_dp_loadarea_industry_consumer.sql @@ -327,14 +268,16 @@ model_draft.ego_demand_la_osm - + - + - model_draft.ego_landuse_industry -model_draft.ego_demand_hv_largescaleconsumer + model_draft.ego_landuse_industry +model_draft.ego_demand_hv_largescaleconsumer +model_draft.ego_demand_per_district +model_draft.ego_demand_per_gva @@ -345,18 +288,19 @@ model_draft.ego_demand_hv_largescaleconsumer - + - + - demand.ego_demand_federalstate + demand.ego_demand_federalstate economic.destatis_gva_per_districts political_boundary.bkg_vg250_4_krs_mview model_draft.ego_dp_supply_conv_powerplant model_draft.ego_grid_ehv_substation_voronoi model_draft.ego_grid_hvmv_substation +openstreetmap.osm_deu_polygon_urban_sector_3_industrial_mview @@ -368,26 +312,10 @@ model_draft.ego_grid_hvmv_substation - - - - - - - 10 sec - - - - - - - - - - + - + ego_dp_loadarea_loads.sql @@ -402,13 +330,15 @@ model_draft.ego_grid_hvmv_substation - + - + - model_draft.ego_demand_la_osm + model_draft.ego_demand_la_osm +model_draft.osm_deu_polygon_urban_buffer100_mview + @@ -419,26 +349,10 @@ model_draft.ego_grid_hvmv_substation - - - - - - - 45 min - - - - - - - - - - + - + model_draft.ego_demand_hv_largescaleconsumer @@ -453,10 +367,10 @@ openstreetmap.osm_deu_polygon_urban - + - + model_draft.ego_deu_loads_osm @@ -471,10 +385,10 @@ model_draft.ego_social_zensus_load_cluster - + - + ego_dp_loadarea_consumption.sql @@ -489,14 +403,14 @@ model_draft.ego_social_zensus_load_cluster - + - - + + - model_draft.ego_demand_loadarea -demand.ego_loadarea + model_draft.ego_demand_loadarea +model_draft.ego_demand_per_district @@ -507,31 +421,16 @@ demand.ego_loadarea - - - - - - - 16 sec - - - - - - - - - - + - + - model_draft.ego_demand_per_district + model_draft.ego_demand_per_district model_draft.ego_demand_loadarea -demand.ego_demand_federalstate +demand.ego_demand_federalstate +model_draft.ego_grid_hvmv_substation @@ -542,21 +441,22 @@ demand.ego_demand_federalstate + - + model_draft.ego_demand_load_melt model_draft.ego_grid_mv_griddistrict -social.destatis_zensus_population_per_ha_mview -economic.ioer_urban_share_industrial_centroid +society.destatis_zensus_population_per_ha_mview openstreetmap.osm_deu_polygon_urban_sector_1_residential_mview openstreetmap.osm_deu_polygon_urban_sector_2_retail_mview -openstreetmap.osm_deu_polygon_urban_sector_3_industrial_mview openstreetmap.osm_deu_polygon_urban_sector_4_agricultural_mview -model_draft.ego_political_boundary_bkg_vg250_6_gem_clean +openstreetmap.osm_deu_polygon_urban +model_draft.ego_demand_hv_largescaleconsumer +model_draft.ego_boundaries_bkg_vg250_6_gem_clean + @@ -567,10 +467,10 @@ model_draft.ego_political_boundary_bkg_vg250_6_gem_clean - + - + Datenlizenz Deutschland – Namensnennung – Version 2.0 @@ -584,10 +484,10 @@ Statistisches Bundesamt, Wiesbaden, Genesis-Online, 2016; Datenlizenz by-2-0 - + - + ego_dp_loadarea_census.sql @@ -602,10 +502,10 @@ Statistisches Bundesamt, Wiesbaden, Genesis-Online, 2016; Datenlizenz by-2-0 - + - + @@ -626,10 +526,10 @@ Statistisches Bundesamt, Wiesbaden, Genesis-Online, 2016; Datenlizenz by-2-0 - + - + Open Database License (ODbL) v1.0 @@ -643,13 +543,13 @@ Statistisches Bundesamt, Wiesbaden, Genesis-Online, 2016; Datenlizenz by-2-0 - + - + - openstreetmap.osm_deu + openstreetmap @@ -660,13 +560,13 @@ Statistisches Bundesamt, Wiesbaden, Genesis-Online, 2016; Datenlizenz by-2-0 - + - + - economic.destatis_gva_per_districts + destatis gva @@ -677,10 +577,10 @@ Statistisches Bundesamt, Wiesbaden, Genesis-Online, 2016; Datenlizenz by-2-0 - + - + Geodatenzugangsgesetz (GeoZG) @@ -694,13 +594,13 @@ Statistisches Bundesamt, Wiesbaden, Genesis-Online, 2016; Datenlizenz by-2-0 - + - + - boundaries.bkg_vg250 + bkg_vg250 @@ -711,10 +611,10 @@ Statistisches Bundesamt, Wiesbaden, Genesis-Online, 2016; Datenlizenz by-2-0 - + - + @@ -735,13 +635,13 @@ Statistisches Bundesamt, Wiesbaden, Genesis-Online, 2016; Datenlizenz by-2-0 - + - + - social.destatis_zensus_population_per_ha + zensus @@ -752,153 +652,169 @@ Statistisches Bundesamt, Wiesbaden, Genesis-Online, 2016; Datenlizenz by-2-0 - + - - + + - Open Database License (ODbL) v1.0 -© OpenStreetMap contributors + DP griddistrict + - - + + - + - - + + - Geodatenzugangsgesetz (GeoZG) -© GeoBasis-DE / BKG 2016 (Daten verändert) + DP substation + - + - + - - + + - Open Database License (ODbL) v1.0 -© OpenStreetMap contributors + demand federalstate + - + - + - - + + - Geodatenzugangsgesetz (GeoZG) -© GeoBasis-DE / BKG 2016 (Daten verändert) + model_draft.ego_grid_hvmv_substation +model_draft.ego_grid_mv_griddistrict +model_draft.ego_grid_mv_griddistrict_type1 +model_draft.ego_grid_mv_griddistrict_type2 +model_draft.ego_grid_mv_griddistrict_type3 +model_draft.ego_demand_loadarea +model_draft.ego_boundaries_bkg_vg250_6_gem_clean +model_draft.ego_boundaries_hvmv_subst_per_gem_1_mview +model_draft.ego_boundaries_hvmv_subst_per_gem_3_mview +model_draft.ego_grid_hvmv_substation_voronoi_cut +boundaries.bkg_vg250_6_gem +boundaries.bkg_vg250_1_sta_mview + - + - + - - + + - Open Database License (ODbL) v1.0 -© OpenStreetMap contributors + boundaries.bkg_vg250_6_gem_pts +boundaries.bkg_vg250_statistics_mview + + - + - + - - - - - Geodatenzugangsgesetz (GeoZG) -© GeoBasis-DE / BKG 2016 (Daten verändert) + + + + + ego_dp_loadarea_statistic.sql + - + + - + - - - - - Datenlizenz Deutschland – Namensnennung – Version 2.0 -Statistisches Bundesamt, Wiesbaden, Genesis-Online, 2016; Datenlizenz by-2-0 + + + + + ego_dp_loadarea_loadcut_voronoi.sql + - + + - + - - - - - model_draft.ego_dp_supply_conv_powerplant + + + + + ego_dp_loadarea_peakload.py + - - - + + + - + - - + + - model_draft.ego_grid_ehv_substation_voronoi - + model_draft.ego_demand_loadarea +scenario.ego_slp_parameters - + @@ -906,16 +822,16 @@ Statistisches Bundesamt, Wiesbaden, Genesis-Online, 2016; Datenlizenz by-2-0 - + - - + + - demand.ego_demand_federalstate + model_draft.ego_demand_loadarea - + @@ -923,32 +839,33 @@ Statistisches Bundesamt, Wiesbaden, Genesis-Online, 2016; Datenlizenz by-2-0 - + - - - - - model_draft.ego_grid_hvmv_substation -model_draft.ego_grid_mv_griddistrict -model_draft.ego_demand_loadarea + + + + + ego_dp_loadarea_griddistrict_results.sql - + - + + - + - - + + - 5 min + Vervielfältigung und Verbreitung, auch +auszugsweise, mit Quellenangabe gestattet. +© Statistisches Landesamt Baden-Württemberg @@ -958,15 +875,14 @@ model_draft.ego_demand_loadarea + - + - model_draft.ego_data_processing_results -model_draft.ego_data_processing_results_mvgd -boundaries.bkg_vg250_statistics_mview + model_draft.ego_demand_loadarea_voi +model_draft.ego_demand_loadarea_voi_smaller100m2_mview @@ -977,68 +893,16 @@ boundaries.bkg_vg250_statistics_mview - - - - - - - ego_dp_loadarea_statistic.sql - - - - - - - - - - - - - - - - - - ego_dp_loadarea_loadcut_voronoi.sql - - - - - - - - - - - - - - - - - - ego_dp_loadarea_peakload.py - - - - - - - - - - - - + - + - model_draft.ego_demand_loadarea -scenario.ego_slp_parameters + model_draft.ego_demand_load_melt +model_draft.ego_grid_hvmv_substation_voronoi +model_draft.ego_boundaries_bkg_vg250_6_gem_clean +model_draft.ego_grid_mv_griddistrict @@ -1049,95 +913,88 @@ scenario.ego_slp_parameters - + - - + + - model_draft.ego_demand_loadarea + BDEW slp - + - + - - + + - 1 h - - - - - - - - - - - - - - - - ego_dp_loadarea_griddistrict_results.sql + model_draft.ego_grid_mv_griddistrict + - - - + + - + - - + + - Open Database License (ODbL) v1.0 -© OpenStreetMap contributors + model_draft.ego_grid_mv_griddistrict +boundaries.bkg_vg250_6_gem +model_draft.ego_grid_mv_griddistrict_type1 +model_draft.ego_grid_mv_griddistrict_type2 +model_draft.ego_grid_mv_griddistrict_type3 +society.destatis_zensus_population_per_ha_mview +model_draft.ego_boundaries_hvmv_subst_per_gem_1_mview +model_draft.ego_boundaries_hvmv_subst_per_gem_3_mview +model_draft.ego_grid_hvmv_substation_voronoi_cut +model_draft.ego_demand_loadarea - + + - + - - + + - Vervielfältigung und Verbreitung, auch -auszugsweise, mit Quellenangabe gestattet. -© Statistisches Landesamt Baden-Württemberg + model_draft.ego_grid_mv_griddistrict + - + + - + @@ -1149,7 +1006,7 @@ auszugsweise, mit Quellenangabe gestattet. - + @@ -1161,55 +1018,7 @@ auszugsweise, mit Quellenangabe gestattet. - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + @@ -1221,31 +1030,7 @@ auszugsweise, mit Quellenangabe gestattet. - - - - - - - - - - - - - - - - - - - - - - - - - + @@ -1257,7 +1042,7 @@ auszugsweise, mit Quellenangabe gestattet. - + @@ -1269,7 +1054,7 @@ auszugsweise, mit Quellenangabe gestattet. - + @@ -1281,7 +1066,7 @@ auszugsweise, mit Quellenangabe gestattet. - + @@ -1293,7 +1078,7 @@ auszugsweise, mit Quellenangabe gestattet. - + @@ -1305,7 +1090,7 @@ auszugsweise, mit Quellenangabe gestattet. - + @@ -1317,7 +1102,7 @@ auszugsweise, mit Quellenangabe gestattet. - + @@ -1329,7 +1114,7 @@ auszugsweise, mit Quellenangabe gestattet. - + @@ -1341,7 +1126,7 @@ auszugsweise, mit Quellenangabe gestattet. - + @@ -1353,7 +1138,7 @@ auszugsweise, mit Quellenangabe gestattet. - + @@ -1365,7 +1150,7 @@ auszugsweise, mit Quellenangabe gestattet. - + @@ -1377,7 +1162,7 @@ auszugsweise, mit Quellenangabe gestattet. - + @@ -1389,7 +1174,7 @@ auszugsweise, mit Quellenangabe gestattet. - + @@ -1401,7 +1186,7 @@ auszugsweise, mit Quellenangabe gestattet. - + @@ -1413,19 +1198,7 @@ auszugsweise, mit Quellenangabe gestattet. - - - - - - - - - - - - - + @@ -1437,7 +1210,7 @@ auszugsweise, mit Quellenangabe gestattet. - + @@ -1449,7 +1222,7 @@ auszugsweise, mit Quellenangabe gestattet. - + @@ -1461,7 +1234,7 @@ auszugsweise, mit Quellenangabe gestattet. - + @@ -1473,7 +1246,7 @@ auszugsweise, mit Quellenangabe gestattet. - + @@ -1485,7 +1258,7 @@ auszugsweise, mit Quellenangabe gestattet. - + @@ -1497,7 +1270,7 @@ auszugsweise, mit Quellenangabe gestattet. - + @@ -1509,7 +1282,7 @@ auszugsweise, mit Quellenangabe gestattet. - + @@ -1521,7 +1294,7 @@ auszugsweise, mit Quellenangabe gestattet. - + @@ -1533,7 +1306,7 @@ auszugsweise, mit Quellenangabe gestattet. - + @@ -1545,7 +1318,7 @@ auszugsweise, mit Quellenangabe gestattet. - + @@ -1557,7 +1330,7 @@ auszugsweise, mit Quellenangabe gestattet. - + @@ -1569,7 +1342,7 @@ auszugsweise, mit Quellenangabe gestattet. - + @@ -1581,7 +1354,7 @@ auszugsweise, mit Quellenangabe gestattet. - + @@ -1593,7 +1366,7 @@ auszugsweise, mit Quellenangabe gestattet. - + @@ -1605,7 +1378,7 @@ auszugsweise, mit Quellenangabe gestattet. - + @@ -1617,7 +1390,7 @@ auszugsweise, mit Quellenangabe gestattet. - + @@ -1629,19 +1402,7 @@ auszugsweise, mit Quellenangabe gestattet. - - - - - - - - - - - - - + @@ -1653,7 +1414,7 @@ auszugsweise, mit Quellenangabe gestattet. - + @@ -1665,7 +1426,7 @@ auszugsweise, mit Quellenangabe gestattet. - + @@ -1677,37 +1438,27 @@ auszugsweise, mit Quellenangabe gestattet. - + - + - + - - - - - - - - - - - + - + @@ -1719,7 +1470,7 @@ auszugsweise, mit Quellenangabe gestattet. - + @@ -1731,7 +1482,7 @@ auszugsweise, mit Quellenangabe gestattet. - + @@ -1743,7 +1494,7 @@ auszugsweise, mit Quellenangabe gestattet. - + @@ -1755,7 +1506,7 @@ auszugsweise, mit Quellenangabe gestattet. - + @@ -1767,7 +1518,251 @@ auszugsweise, mit Quellenangabe gestattet. - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -1778,7 +1773,7 @@ auszugsweise, mit Quellenangabe gestattet. - + @@ -1789,7 +1784,7 @@ auszugsweise, mit Quellenangabe gestattet. - + From 6ce032dc7587581de71367d76ffc33a5454a53ef Mon Sep 17 00:00:00 2001 From: IlkaCu Date: Mon, 16 Mar 2020 09:18:11 +0100 Subject: [PATCH 10/84] update BPMN docu --- .../ego_dp_bpmn_section_griddistrict.graphml | 137 +-- .../ego_dp_bpmn_section_lowvoltage.graphml | 852 ++++++++---------- .../ego_dp_bpmn_section_substation.graphml | 436 ++------- .../bpmn/ego_dp_bpmn_sections_input.graphml | 162 ++-- 4 files changed, 542 insertions(+), 1045 deletions(-) diff --git a/documentation/bpmn/ego_dp_bpmn_section_griddistrict.graphml b/documentation/bpmn/ego_dp_bpmn_section_griddistrict.graphml index e3054dad..e4c0a044 100644 --- a/documentation/bpmn/ego_dp_bpmn_section_griddistrict.graphml +++ b/documentation/bpmn/ego_dp_bpmn_section_griddistrict.graphml @@ -113,22 +113,6 @@ model_draft.ego_grid_mv_griddistrict_dump - - - - - - ? - - - - - - - - - - @@ -148,7 +132,7 @@ model_draft.ego_grid_hvmv_substation_voronoi - + @@ -172,7 +156,7 @@ model_draft.ego_grid_hvmv_substation_voronoi - + @@ -196,7 +180,7 @@ model_draft.ego_grid_hvmv_substation_voronoi - + @@ -213,7 +197,7 @@ model_draft.ego_grid_hvmv_substation_voronoi - + @@ -230,41 +214,7 @@ model_draft.ego_grid_hvmv_substation_voronoi - - - - - - - Geodatenzugangsgesetz (GeoZG) -© GeoBasis-DE / BKG 2016 (Daten verändert) - - - - - - - - - - - - - - - - Open Database License (ODbL) v1.0 -© OpenStreetMap contributors - - - - - - - - - - + @@ -281,7 +231,7 @@ model_draft.ego_grid_hvmv_substation_voronoi - + @@ -298,7 +248,7 @@ model_draft.ego_grid_hvmv_substation_voronoi - + @@ -315,7 +265,7 @@ model_draft.ego_grid_hvmv_substation_voronoi - + @@ -333,7 +283,7 @@ model_draft.ego_grid_hvmv_substation_voronoi - + @@ -370,7 +320,7 @@ model_draft.ego_grid_ehv_substation_voronoi - + @@ -389,7 +339,7 @@ model_draft.ego_dp_supply_res_powerplant - + @@ -406,37 +356,9 @@ model_draft.ego_dp_supply_res_powerplant - - - - - - - ? - - - - - - - - - - - - - - - - - - - - - - + @@ -448,7 +370,7 @@ model_draft.ego_dp_supply_res_powerplant - + @@ -460,7 +382,7 @@ model_draft.ego_dp_supply_res_powerplant - + @@ -472,7 +394,7 @@ model_draft.ego_dp_supply_res_powerplant - + @@ -484,7 +406,7 @@ model_draft.ego_dp_supply_res_powerplant - + @@ -496,7 +418,7 @@ model_draft.ego_dp_supply_res_powerplant - + @@ -506,7 +428,7 @@ model_draft.ego_dp_supply_res_powerplant - + @@ -518,7 +440,7 @@ model_draft.ego_dp_supply_res_powerplant - + @@ -528,7 +450,7 @@ model_draft.ego_dp_supply_res_powerplant - + @@ -538,7 +460,7 @@ model_draft.ego_dp_supply_res_powerplant - + @@ -548,19 +470,7 @@ model_draft.ego_dp_supply_res_powerplant - - - - - - - - - - - - - + @@ -570,8 +480,7 @@ model_draft.ego_dp_supply_res_powerplant - - + diff --git a/documentation/bpmn/ego_dp_bpmn_section_lowvoltage.graphml b/documentation/bpmn/ego_dp_bpmn_section_lowvoltage.graphml index 19a0afcf..1622457e 100644 --- a/documentation/bpmn/ego_dp_bpmn_section_lowvoltage.graphml +++ b/documentation/bpmn/ego_dp_bpmn_section_lowvoltage.graphml @@ -1,6 +1,6 @@ - + @@ -13,15 +13,15 @@ - + - + - eGo DataProcessing - lowvoltage - + eGo DataProcessing - lowvoltage + @@ -31,29 +31,29 @@ + + + - - - - - + + - + @@ -64,7 +64,7 @@ - + @@ -74,186 +74,192 @@ - - + + - - ego_dp_lv_substation.sql + + + + + + + + + - + - - + - - - - - 9 min + + + + + + - + + - + - - - - - ego_dp_lv_substation_voronoi.sql + + + + + model_draft.ego_lattice_360m_lv +model_draft.ego_grid_mvlv_substation +model_draft.ego_grid_lv_loadarea_rest - + - - + + - - + + - 2 h - - - - - - + model_draft.ego_demand_loadarea +model_draft.ego_grid_mv_griddistrict + - + - - - - - - - - - - - - + + + + + model_draft.ego_grid_mvlv_substation_voronoi - + - + - - - - - - - - - - - - + + + + + model_draft.ego_grid_lv_griddistrict_cut +model_draft.ego_grid_lv_griddistrict_cut_1subst +model_draft.ego_grid_lv_griddistrict_cut_0subst +model_draft.ego_grid_lv_griddistrict_cut_xsubst +model_draft.ego_grid_lv_griddistrict_cut_nn +model_draft.ego_grid_lv_griddistrict_cut_nn_collect + - + - + + - - + + - Open Data Commons Open Database License 1.0 - - - - - - + model_draft.ego_grid_lv_griddistrict_cut_nn_collect +model_draft.ego_grid_mvlv_substation +society.destatis_zensus_population_per_ha_mview +model_draft.ego_osm_sector_per_griddistrict_1_residential +model_draft.ego_osm_sector_per_griddistrict_2_retail +model_draft.ego_osm_sector_per_griddistrict_3_industrial +model_draft.ego_osm_sector_per_griddistrict_4_agricultural + + + - + + - - - - - ego_dp_lv_loadcut.sql + + + + + model_draft.ego_grid_lv_griddistrict +model_draft.ego_demand_loadarea +scenario.ego_slp_parameters + - + - - + + - - - - - ego_dp_lv_griddistrict.sql + + + + + model_draft.ego_grid_hvmv_substation_dummy +model_draft.ego_grid_mvlv_substation - + - - + @@ -261,10 +267,10 @@ - - + + - ego_dp_lv_consumption_peakload.sql + ego_dp_lv_substation.sql @@ -278,115 +284,92 @@ - - - - - model_draft.ego_grid_mvlv_substation - - - - - - + + + + + ego_dp_lv_substation_voronoi.sql + - - + + - - - - - - model_draft.ego_demand_loadarea -model_draft.ego_grid_mv_griddistrict - - - - - - + + + + + ego_dp_lv_loadcut.sql + - - + + - - - - - Open Data Commons Open Database License 1.0 - - - - - - + + + + + ego_dp_lv_griddistrict.sql + - + + - - - - - model_draft.ego_grid_lv_griddistrict - - - - - - + + + + + ego_dp_lv_consumption_peakload.sql + - - + + + - - + + - Open Data Commons Open Database License 1.0 - - - - - - + model_draft.ego_demand_loadarea +model_draft.ego_grid_mvlv_substation +model_draft.ego_grid_mvlv_substation_voronoi + - + @@ -394,21 +377,21 @@ model_draft.ego_grid_mv_griddistrict - - + + - 20 min - - - - - - + model_draft.ego_grid_lv_griddistrict +model_draft.ego_osm_sector_per_lvgd_1_residential +model_draft.ego_osm_sector_per_lvgd_2_retail +model_draft.ego_osm_sector_per_lvgd_3_industrial +model_draft.ego_osm_sector_per_lvgd_4_agricultural + - + + @@ -416,21 +399,16 @@ model_draft.ego_grid_mv_griddistrict - - + + - 40 min - - - - - - + model_draft.ego_grid_lv_griddistrict - + + @@ -438,21 +416,16 @@ model_draft.ego_grid_mv_griddistrict - - + + - 5 min - - - - - - + zensus - + + @@ -460,16 +433,10 @@ model_draft.ego_grid_mv_griddistrict - - + + - model_draft.ego_grid_mvlv_substation_voronoi - - - - - - + DP loadarea @@ -483,16 +450,10 @@ model_draft.ego_grid_mv_griddistrict - - + + - model_draft.ego_grid_lv_griddistrict_cut_nn_collect - - - - - - + DP griddistrict @@ -504,25 +465,18 @@ model_draft.ego_grid_mv_griddistrict - - + - model_draft.ego_grid_mvlv_substation - - - - - - + DP substation - + @@ -530,39 +484,10 @@ model_draft.ego_grid_mv_griddistrict - - - - Open Data Commons Open Database License 1.0 - - - - - - - - - - - - - - - - - - - - - + + - scenario.ego_slp_parameters - - - - - - + BDEW SLP @@ -573,337 +498,324 @@ model_draft.ego_grid_mv_griddistrict - - - - - - - Open Data Commons Open Database License 1.0 - - - - - - - - - - - - - - - - + - + - - - - - - + + + + - + - + - + - - - - + + - + - + - - - - - - + + + + - + - + - + - - - - + + - + - + - - - - + + - + - + - - - - + + - + - + - - - - - - + + + + - + - + - - - - - - + + + + - + - + - + - - - - - - - + + - + - + - + - - - - - - - + + - + - + - - - - - - + + + + - + - + - - - - - - + + + + - + + - + - - - - - - + + + + - + + - + - + - - - - - - - + + - + + - + - + - - - - + + - + + - + - + - - - - + + - + + - + - + - - - - + + - + + - + - + - - - - - - - + + - + + - + - + - - - - - - - + + - + + - + - + - - - - - - - + + - + + - + - + - - - - + + - + + - + - + - - - - - - - + + - + + - + - + - - - - + + - + + - + - + + + + + + + + + + + + - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/documentation/bpmn/ego_dp_bpmn_section_substation.graphml b/documentation/bpmn/ego_dp_bpmn_section_substation.graphml index 3d4ae096..0d86ab5e 100644 --- a/documentation/bpmn/ego_dp_bpmn_section_substation.graphml +++ b/documentation/bpmn/ego_dp_bpmn_section_substation.graphml @@ -17,7 +17,7 @@ - + eGo DataProcessing - substation @@ -31,35 +31,35 @@ + + + - - - - - + + - + - + @@ -111,22 +111,6 @@ model_draft.ego_grid_hvmv_substation_voronoi - - - - - - ? - - - - - - - - - - @@ -144,7 +128,7 @@ model_draft.ego_boundaries_bkg_vg250_6_gem_clean - + @@ -161,7 +145,7 @@ model_draft.ego_boundaries_bkg_vg250_6_gem_clean - + @@ -181,7 +165,7 @@ ego_dp_substation_hvmv.sql - + @@ -201,39 +185,7 @@ ego_dp_substation_ehv.sql - - - - - - - min? - - - - - - - - - - - - - - - - min? - - - - - - - - - - + @@ -247,15 +199,15 @@ openstreetmap.osm_deu_point boundaries.bkg_vg250_1_sta_union_mview + - - + @@ -278,7 +230,7 @@ boundaries.bkg_vg250_1_sta_union_mview - + @@ -295,7 +247,7 @@ boundaries.bkg_vg250_1_sta_union_mview - + @@ -319,7 +271,7 @@ boundaries.bkg_vg250_1_sta_union_mview - + @@ -339,23 +291,7 @@ ego_dp_substation_otg.sql - - - - - - - ? - - - - - - - - - - + @@ -374,7 +310,7 @@ model_draft.ego_grid_hvmv_substation - + @@ -392,7 +328,7 @@ model_draft.ego_grid_hvmv_substation - + @@ -409,7 +345,7 @@ model_draft.ego_grid_hvmv_substation - + @@ -433,7 +369,7 @@ model_draft.ego_grid_hvmv_substation - + @@ -450,7 +386,7 @@ model_draft.ego_grid_hvmv_substation - + @@ -467,7 +403,7 @@ model_draft.ego_grid_hvmv_substation - + @@ -484,143 +420,7 @@ model_draft.ego_grid_hvmv_substation - - - - - - - Open Database License (ODbL) v1.0 -© OpenStreetMap contributors - - - - - - - - - - - - - - - - Open Database License (ODbL) v1.0 -© OpenStreetMap contributors - - - - - - - - - - - - - - - - Open Database License (ODbL) v1.0 -© OpenStreetMap contributors - - - - - - - - - - - - - - - - Open Database License (ODbL) v1.0 -© OpenStreetMap contributors - - - - - - - - - - - - - - - - Geodatenzugangsgesetz (GeoZG) -© GeoBasis-DE / BKG 2016 (Daten verändert) - - - - - - - - - - - - - - - - Geodatenzugangsgesetz (GeoZG) -© GeoBasis-DE / BKG 2016 (Daten verändert) - - - - - - - - - - - - - - - - Geodatenzugangsgesetz (GeoZG) -© GeoBasis-DE / BKG 2016 (Daten verändert) - - - - - - - - - - - - - - - - Geodatenzugangsgesetz (GeoZG) -© GeoBasis-DE / BKG 2016 (Daten verändert) - - - - - - - - - - + @@ -639,7 +439,7 @@ model_draft.ego_grid_hvmv_substation - + @@ -658,23 +458,7 @@ model_draft.ego_grid_hvmv_substation - - - - - - - ? - - - - - - - - - - + @@ -692,41 +476,7 @@ model_draft.ego_grid_ehv_substation - - - - - - - Open Database License (ODbL) v1.0 -© OpenStreetMap contributors - - - - - - - - - - - - - - - - Geodatenzugangsgesetz (GeoZG) -© GeoBasis-DE / BKG 2016 (Daten verändert) - - - - - - - - - - + @@ -743,7 +493,7 @@ model_draft.ego_grid_ehv_substation - + @@ -760,7 +510,7 @@ model_draft.ego_grid_ehv_substation - + @@ -769,15 +519,15 @@ model_draft.ego_grid_ehv_substationmodel_draft.ego_grid_hvmv_substation + - - + @@ -786,29 +536,17 @@ model_draft.ego_grid_ehv_substationmodel_draft.ego_grid_ehv_substation + - - - - - - - - - - - - - - + @@ -820,7 +558,7 @@ model_draft.ego_grid_ehv_substation - + @@ -832,7 +570,7 @@ model_draft.ego_grid_ehv_substation - + @@ -844,7 +582,7 @@ model_draft.ego_grid_ehv_substation - + @@ -856,7 +594,7 @@ model_draft.ego_grid_ehv_substation - + @@ -868,19 +606,7 @@ model_draft.ego_grid_ehv_substation - - - - - - - - - - - - - + @@ -892,19 +618,7 @@ model_draft.ego_grid_ehv_substation - - - - - - - - - - - - - + @@ -916,7 +630,7 @@ model_draft.ego_grid_ehv_substation - + @@ -928,7 +642,7 @@ model_draft.ego_grid_ehv_substation - + @@ -940,7 +654,7 @@ model_draft.ego_grid_ehv_substation - + @@ -952,7 +666,7 @@ model_draft.ego_grid_ehv_substation - + @@ -964,7 +678,7 @@ model_draft.ego_grid_ehv_substation - + @@ -976,7 +690,7 @@ model_draft.ego_grid_ehv_substation - + @@ -988,19 +702,7 @@ model_draft.ego_grid_ehv_substation - - - - - - - - - - - - - + @@ -1012,7 +714,7 @@ model_draft.ego_grid_ehv_substation - + @@ -1024,7 +726,7 @@ model_draft.ego_grid_ehv_substation - + @@ -1036,7 +738,7 @@ model_draft.ego_grid_ehv_substation - + @@ -1048,7 +750,7 @@ model_draft.ego_grid_ehv_substation - + @@ -1060,7 +762,7 @@ model_draft.ego_grid_ehv_substation - + @@ -1072,7 +774,7 @@ model_draft.ego_grid_ehv_substation - + @@ -1084,7 +786,7 @@ model_draft.ego_grid_ehv_substation - + @@ -1096,7 +798,7 @@ model_draft.ego_grid_ehv_substation - + @@ -1108,7 +810,7 @@ model_draft.ego_grid_ehv_substation - + @@ -1120,7 +822,7 @@ model_draft.ego_grid_ehv_substation - + @@ -1132,19 +834,7 @@ model_draft.ego_grid_ehv_substation - - - - - - - - - - - - - + @@ -1156,7 +846,7 @@ model_draft.ego_grid_ehv_substation - + @@ -1166,7 +856,7 @@ model_draft.ego_grid_ehv_substation - + @@ -1176,8 +866,7 @@ model_draft.ego_grid_ehv_substation - - + @@ -1187,8 +876,7 @@ model_draft.ego_grid_ehv_substation - - + diff --git a/documentation/bpmn/ego_dp_bpmn_sections_input.graphml b/documentation/bpmn/ego_dp_bpmn_sections_input.graphml index d59e13e1..614ec84e 100644 --- a/documentation/bpmn/ego_dp_bpmn_sections_input.graphml +++ b/documentation/bpmn/ego_dp_bpmn_sections_input.graphml @@ -212,29 +212,12 @@ CHECK - - - - - - DP substation - - - - - - - - - - - - LOWVOLTAGE + LOWVOLTAGE @@ -246,7 +229,7 @@ CHECK - + @@ -264,7 +247,7 @@ CHECK - + @@ -282,7 +265,7 @@ CHECK - + @@ -301,7 +284,7 @@ PROCESSING - + @@ -312,15 +295,15 @@ vg250 osmTGmod + - - + @@ -339,7 +322,7 @@ res_powerplant - + @@ -362,24 +345,7 @@ demand_federalstate - - - - - - - DP substation - - - - - - - - - - - + @@ -396,14 +362,15 @@ demand_federalstate - + - DP loadarea -DP griddistrict + DP loadarea +DP griddistrict +DP substation @@ -414,7 +381,7 @@ DP griddistrict - + @@ -433,7 +400,7 @@ res_powerplant - + @@ -451,7 +418,7 @@ DP griddistrict - + @@ -464,15 +431,15 @@ renewable_feedin storage h2 areas + - - + @@ -483,15 +450,15 @@ DP loadarea DP substation + - - + @@ -502,21 +469,55 @@ bnetza_enlag renpassGIS + - - + DP powerflow + + + + + + + + + + + + + + + + + DP substation + + + + + + + + + + + + + + + + + DP substation @@ -577,7 +578,7 @@ renpassGIS - + @@ -589,7 +590,7 @@ renpassGIS - + @@ -601,7 +602,7 @@ renpassGIS - + @@ -613,7 +614,7 @@ renpassGIS - + @@ -625,7 +626,7 @@ renpassGIS - + @@ -635,7 +636,7 @@ renpassGIS - + @@ -645,8 +646,7 @@ renpassGIS - - + @@ -656,8 +656,7 @@ renpassGIS - - + @@ -667,8 +666,7 @@ renpassGIS - - + @@ -678,8 +676,7 @@ renpassGIS - - + @@ -689,8 +686,7 @@ renpassGIS - - + @@ -700,8 +696,7 @@ renpassGIS - - + @@ -711,8 +706,7 @@ renpassGIS - - + @@ -722,8 +716,7 @@ renpassGIS - - + @@ -733,8 +726,7 @@ renpassGIS - - + @@ -744,8 +736,7 @@ renpassGIS - - + @@ -755,8 +746,7 @@ renpassGIS - - + @@ -766,8 +756,7 @@ renpassGIS - - + @@ -777,8 +766,7 @@ renpassGIS - - + From 3230046a83db5d41c839a895f54ce225754911f9 Mon Sep 17 00:00:00 2001 From: IlkaCu Date: Mon, 16 Mar 2020 09:54:41 +0100 Subject: [PATCH 11/84] add BPMN for REA and posprocessing --- .../bpmn/ego_dp_bpmn_section_REA.graphml | 1778 +++++++++++++++++ ...ego_dp_bpmn_section_postprocessing.graphml | 483 +++++ 2 files changed, 2261 insertions(+) create mode 100644 documentation/bpmn/ego_dp_bpmn_section_REA.graphml create mode 100644 documentation/bpmn/ego_dp_bpmn_section_postprocessing.graphml diff --git a/documentation/bpmn/ego_dp_bpmn_section_REA.graphml b/documentation/bpmn/ego_dp_bpmn_section_REA.graphml new file mode 100644 index 00000000..c001cb38 --- /dev/null +++ b/documentation/bpmn/ego_dp_bpmn_section_REA.graphml @@ -0,0 +1,1778 @@ + + + + + + + + + + + + + + + + + + + + + + + eGo DataProcessing - REA + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + model_draft.ego_lattice_500m + + + + + + + + + + + + + + + + + + rea/ego_dp_lattice_500m.sql + + + + + + + + + + + + + + + + + + boundaries.bkg_vg250_1_sta_union_mview + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + vernetzen + + + + + + + + + + + + + + + + + Geodatenzugangsgesetz (GeoZG) +© GeoBasis-DE / BKG 2016 (Daten verändert) + + + + + + + + + + + + + + + + bkg_vg250 + + + + + + + + + + + + + + + + + model_draft.ego_lattice_500m + + + + + + + + + + + + + + + + + rea/ego_dp_lattice_50m.sql + + + + + + + + + + + + + + + + + + boundaries.bkg_vg250_1_sta_union_mview + + + + + + + + + + + + + + + + + model_draft.ego_lattice_50m + + + + + + + + + + + + + + + + + + rea/ego_dp_rea_wpa_per_mvgd.sql + + + + + + + + + + + + + + + + + + supply.vernetzen_wind_potential_area +model_draft.ego_grid_mv_griddistrict + + + + + + + + + + + + + + + + + model_draft.ego_supply_wpa_per_mvgd + + + + + + + + + + + + + + + + + rea/ego_dp_rea_lattice_per_area_500m.sql + + + + + + + + + + + + + + + + + + model_draft.ego_lattice_500m +model_draft.ego_grid_mv_griddistrict +model_draft.ego_supply_wpa_per_mvgd +model_draft.ego_demand_loadarea + + + + + + + + + + + + + + + + + + model_draft.ego_lattice_500m_wpa_mview +model_draft.ego_lattice_500m_la_mview +model_draft.ego_lattice_500m_x_mview +model_draft.ego_lattice_500m_out_mview + + + + + + + + + + + + + + + + + rea/ego_dp_rea_lattice_per_area_50m.sql + + + + + + + + + + + + + + + + + + model_draft.ego_lattice_50m +model_draft.ego_grid_mv_griddistrict +model_draft.ego_demand_loadarea + + + + + + + + + + + + + + + + + + model_draft.ego_lattice_50m_la_mview + + + + + + + + + + + + + + + + + + model_draft.ego_lattice_50m + + + + + + + + + + + + + + + + + rea/ego_dp_rea_setup.sql + + + + + + + + + + + + + + + + + + model_draft.ego_dp_supply_res_powerplant +model_draft.ego_osm_sector_per_griddistrict_4_agricultural +model_draft.ego_grid_mv_griddistrict +model_draft.ego_grid_hvmv_substation + + + + + + + + + + + + + + + + + model_draft.ego_dp_supply_res_powerplant_out_mview +model_draft.ego_dp_supply_res_powerplant_out_nn + + + + + + + + + + + + + + + + + + model_draft.ego_dp_supply_res_powerplant +model_draft.ego_osm_sector_per_griddistrict_4_agricultural + + + + + + + + + + + + + + + + + rea/ego_dp_rea_m1.sql + + + + + + + + + + + + + + + + + + model_draft.ego_supply_rea_m1_1_a_mview +model_draft.ego_supply_rea_m1_1_dea_temp +model_draft.ego_supply_rea_m1_1_osm_temp +model_draft.ego_supply_rea_m1_1_jnt_temp +model_draft.ego_supply_rea_m1_1_mview +model_draft.ego_supply_rea_m1_1_rest_mview +model_draft.ego_supply_rea_m1_2_a_mview +model_draft.ego_supply_rea_m1_2_dea_temp +model_draft.ego_supply_rea_m1_2_osm_temp +model_draft.ego_supply_rea_m1_2_jnt_temp +model_draft.ego_supply_rea_m1_2_mview +model_draft.ego_supply_rea_m1_2_rest_mview + + + + + + + + + + + + + + + + + + model_draft.ego_dp_supply_res_powerplant +model_draft.ego_osm_sector_per_griddistrict_4_agricultural + + + + + + + + + + + + + + + + + + model_draft.ego_dp_supply_res_powerplant + + + + + + + + + + + + + + + + + rea/ego_dp_rea_m2.sql + + + + + + + + + + + + + + + + + + model_draft.ego_supply_rea_m2_a_mview +model_draft.ego_supply_rea_m2_windfarm +model_draft.ego_supply_rea_m2_farm_temp +model_draft.ego_supply_rea_m2_wpa_temp +model_draft.ego_supply_rea_m2_jnt_temp +model_draft.ego_supply_rea_m2_mview +model_draft.ego_supply_rea_m2_rest_mview + + + + + + + + + + + + + + + + + + model_draft.ego_dp_supply_res_powerplant +model_draft.ego_grid_mv_griddistrict +model_draft.ego_supply_wpa_per_mvgd + + + + + + + + + + + + + + + + + model_draft.ego_dp_supply_res_powerplant + + + + + + + + + + + + + + + + + rea/ego_dp_rea_m3.sql + + + + + + + + + + + + + + + + + + model_draft.ego_supply_rea_m3_a_mview +model_draft.ego_supply_rea_m3_dea_temp +model_draft.ego_supply_rea_m3_grid_wpa_temp +model_draft.ego_supply_rea_m3_jnt_temp +model_draft.ego_supply_rea_m3_mview +model_draft.ego_supply_rea_m3_rest_mview + + + + + + + + + + + + + + + + + + + + model_draft.ego_dp_supply_res_powerplant +model_draft.ego_lattice_500m_wpa_mview + + + + + + + + + + + + + + + + + + model_draft.ego_dp_supply_res_powerplant + + + + + + + + + + + + + + + + + rea/ego_dp_rea_m4.sql + + + + + + + + + + + + + + + + + + model_draft.ego_supply_rea_m4_a_mview +model_draft.ego_supply_rea_m4_dea_temp +model_draft.ego_supply_rea_m4_grid_wpa_temp +model_draft.ego_supply_rea_m4_jnt_temp +model_draft.ego_supply_rea_m4_mview +model_draft.ego_supply_rea_m4_rest_mview + + + + + + + + + + + + + + + + + + + + model_draft.ego_dp_supply_res_powerplant +model_draft.ego_lattice_500m_wpa_mview + + + + + + + + + + + + + + + + + + model_draft.ego_dp_supply_res_powerplant + + + + + + + + + + + + + + + + + rea/ego_dp_rea_m5.sql + + + + + + + + + + + + + + + + + + model_draft.ego_supply_rea_m5_a_mview +model_draft.ego_supply_rea_m5_dea_temp +model_draft.ego_supply_rea_m5_grid_la_temp +model_draft.ego_supply_rea_m5_jnt_temp +model_draft.ego_supply_rea_m5_rest_mview +model_draft.ego_supply_rea_m5_rest_2_mview +model_draft.ego_supply_rea_m5_mview + + + + + + + + + + + + + + + + + + + model_draft.ego_dp_supply_res_powerplant +model_draft.ego_demand_loadarea +model_draft.ego_grid_lv_griddistrict + + + + + + + + + + + + + + + + + + model_draft.ego_dp_supply_res_powerplant + + + + + + + + + + + + + + + + + rea/ego_dp_rea_results.sql + + + + + + + + + + + + + + + + + + model_draft.ego_supply_rea_per_gentype_and_voltlevel +model_draft.ego_supply_rea_per_loadarea +model_draft.ego_supply_rea_per_method + + + + + + + + + + + + + + + + + + model_draft.ego_grid_mv_griddistrict +model_draft.ego_dp_supply_res_powerplant +model_draft.ego_demand_loadarea + + + + + + + + + + + + + + + + + model_draft.ego_grid_mv_griddistrict +model_draft.ego_dp_supply_res_powerplant + + + + + + + + + + + + + + + + + DP griddistrict + + + + + + + + + + + + + + + + + DP loadarea + + + + + + + + + + + + + + + + + DP substation + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/documentation/bpmn/ego_dp_bpmn_section_postprocessing.graphml b/documentation/bpmn/ego_dp_bpmn_section_postprocessing.graphml new file mode 100644 index 00000000..4abb7506 --- /dev/null +++ b/documentation/bpmn/ego_dp_bpmn_section_postprocessing.graphml @@ -0,0 +1,483 @@ + + + + + + + + + + + + + + + + + + + + + + + eGo DataProcessing - postprocessing + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ego_pp_nep2035_grid_variations.sql + + + + + + + + + + + + + + + + + + model_draft.ego_grid_pf_hv_extension_temp_resolution +model_draft.ego_grid_pf_hv_extension_source +model_draft.ego_grid_pf_hv_extension_storage +model_draft.ego_grid_pf_hv_extension_storage_pq_set +model_draft.ego_grid_pf_hv_extension_bus +model_draft.ego_grid_pf_hv_extension_line +model_draft.ego_grid_pf_hv_extension_link +model_draft.ego_grid_pf_hv_extension_transformer +model_draft.ego_grid_pf_hv_extension_load +model_draft.ego_grid_pf_hv_extension_load_pq_set +model_draft.ego_grid_pf_hv_extension_generator +model_draft.ego_grid_pf_hv_extension_generator_pq_set +calc_renpass_gis.extension_translate_to_pf + + + + + + + + + + + + + + + + + + + model_draft.ego_grid_pf_hv_temp_resolution +model_draft.ego_grid_pf_hv_source +model_draft.ego_grid_pf_hv_bus +model_draft.ego_grid_pf_hv_line +model_draft.ego_grid_pf_hv_transformer +model_draft.ego_grid_pf_hv_link +model_draft.scn_nep2035_b2_line +grid.bnetza_vorhabenpunkte_bbplg +grid.bnetza_vorhabenpunkte_enlag +model_draft.opsd_hourly_timeseries +calc_renpass_gis.renpass_gis_linear_transformer +calc_renpass_gis.renpass_gis_source +grid.otg_ehvhv_bus_data +model_draft.ego_grid_line_expansion_costs + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + NEP 2035 B2 + + + + + + + + + + + + + + + + + DP powerflow + + + + + + + + + + + + + + + + + BBPlG + + + + + + + + + + + + + + + + + EnLAG + + + + + + + + + + + + + + + + + OPSD + + + + + + + + + + + + + + + + + renpassGIS + + + + + + + + + + + + + + + + + osmTGmod + + + + + + + + + + + + + + + + + expansion costs +NEP/dena + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + From a079ffeab8a3e507d266a51554fa0f54a42b3d76 Mon Sep 17 00:00:00 2001 From: IlkaCu Date: Mon, 16 Mar 2020 13:38:43 +0100 Subject: [PATCH 12/84] Update BPMN powerflow and DP overview --- .../ego_dp_bpmn_section_powerflow.graphml | 97 ++++- .../bpmn/ego_dp_bpmn_sections.graphml | 400 ++++-------------- 2 files changed, 157 insertions(+), 340 deletions(-) diff --git a/documentation/bpmn/ego_dp_bpmn_section_powerflow.graphml b/documentation/bpmn/ego_dp_bpmn_section_powerflow.graphml index 821753fd..00cc2f62 100644 --- a/documentation/bpmn/ego_dp_bpmn_section_powerflow.graphml +++ b/documentation/bpmn/ego_dp_bpmn_section_powerflow.graphml @@ -1327,6 +1327,41 @@ model_draft.ego_grid_mv_griddistrict + + + + + + + model_draft.ego_demand_hvmv_demand + + + + + + + + + + + + + + + + + model_draft.ego_demand_hv_largescaleconsumer +model_draft.ego_demand_loadarea + + + + + + + + + + @@ -2007,8 +2042,8 @@ model_draft.ego_grid_mv_griddistrict - - + + @@ -2134,18 +2169,16 @@ model_draft.ego_grid_mv_griddistrict - - - + + - @@ -2156,18 +2189,16 @@ model_draft.ego_grid_mv_griddistrict - - - + + - @@ -2178,7 +2209,6 @@ model_draft.ego_grid_mv_griddistrict - @@ -2189,7 +2219,6 @@ model_draft.ego_grid_mv_griddistrict - @@ -2200,7 +2229,6 @@ model_draft.ego_grid_mv_griddistrict - @@ -2211,6 +2239,16 @@ model_draft.ego_grid_mv_griddistrict + + + + + + + + + + @@ -2221,6 +2259,39 @@ model_draft.ego_grid_mv_griddistrict + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/documentation/bpmn/ego_dp_bpmn_sections.graphml b/documentation/bpmn/ego_dp_bpmn_sections.graphml index 96caf4a3..c4b152da 100644 --- a/documentation/bpmn/ego_dp_bpmn_sections.graphml +++ b/documentation/bpmn/ego_dp_bpmn_sections.graphml @@ -1,6 +1,6 @@ - + @@ -13,15 +13,15 @@ - + - + - eGo DataProcessing - + eGo DataProcessing + @@ -53,18 +53,18 @@ - + - + - + @@ -73,42 +73,13 @@ - - - - - - demand.ego_loadarea - - - - - - - - - - - - - - - - - - + - INPUT DATA -CHECK - - - - - - + INPUT DATA +CHECK @@ -120,19 +91,13 @@ CHECK - + - + - GRIDDISTRICT - - - - - - + GRIDDISTRICT @@ -144,19 +109,13 @@ CHECK - + - + - SUBSTATION - - - - - - + SUBSTATION @@ -168,34 +127,10 @@ CHECK - - - - - - - grid.ego_hvmv_substation -grid.ego_ehv_substation - - - - - - - - - - - - - - - - - + - + @@ -216,33 +151,10 @@ grid.ego_ehv_substation - - - - - - - grid.ego_mv_griddistrict - - - - - - - - - - - - - - - - - + - + @@ -263,19 +175,13 @@ grid.ego_ehv_substation - + - + - POWERFLOW - - - - - - + POWERFLOW @@ -287,19 +193,13 @@ grid.ego_ehv_substation - + - + - LOADAREA - - - - - - + LOADAREA @@ -311,89 +211,31 @@ grid.ego_ehv_substation - - - - - - - model_draft.ego_scenario_input - - - - - - - - - - - - - - - - - - - - - - - ? - - - - - - - - - - - - - - - - - + - - - - - grid.ego_dp_mvlv_substation -grid.ego_dp_lv_griddistrict - - - - - - + + + + + LOWVOLTAGE + - - + + - + - + - LOWVOLTAGE - - - - - - + REA @@ -405,43 +247,32 @@ grid.ego_dp_lv_griddistrict - + - - - - - grid.ego_dp_mvlv_substation -grid.ego_dp_lv_griddistrict - - - - - - + + + + + VERSIONING + - - + + - + - + - REA - - - - - - + POST- +PROCESSING @@ -455,22 +286,7 @@ grid.ego_dp_lv_griddistrict - - - - - - - - - - - - - - - - + @@ -482,7 +298,7 @@ grid.ego_dp_lv_griddistrict - + @@ -494,7 +310,7 @@ grid.ego_dp_lv_griddistrict - + @@ -506,7 +322,7 @@ grid.ego_dp_lv_griddistrict - + @@ -518,7 +334,7 @@ grid.ego_dp_lv_griddistrict - + @@ -530,7 +346,7 @@ grid.ego_dp_lv_griddistrict - + @@ -542,82 +358,19 @@ grid.ego_dp_lv_griddistrict - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + - - - - - - - - - - - - - - - - - - - - + + - + - + @@ -629,31 +382,24 @@ grid.ego_dp_lv_griddistrict - + - + - - - - - - - - - + + + + - + - + - - - - + + From f9eed9be3c37f91b4b35c75c4216f700b7ddaa5b Mon Sep 17 00:00:00 2001 From: IlkaCu Date: Mon, 16 Mar 2020 13:41:39 +0100 Subject: [PATCH 13/84] remove ego_dp_bpmn to archive --- documentation/bpmn/{ => archiv}/ego_dp_bpmn.graphml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename documentation/bpmn/{ => archiv}/ego_dp_bpmn.graphml (100%) diff --git a/documentation/bpmn/ego_dp_bpmn.graphml b/documentation/bpmn/archiv/ego_dp_bpmn.graphml similarity index 100% rename from documentation/bpmn/ego_dp_bpmn.graphml rename to documentation/bpmn/archiv/ego_dp_bpmn.graphml From e8ced4dff2a59cdd0d86b7e74b94d59b4954d84b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guido=20Ple=C3=9Fmann?= Date: Wed, 18 Mar 2020 21:46:54 +0100 Subject: [PATCH 14/84] Add BPMN for zensus data import and preprocessing --- .../ego_dp-pre_bpmn_section_zensus.graphml | 467 ++++++++++++++++++ 1 file changed, 467 insertions(+) create mode 100644 documentation/bpmn/ego_dp-pre_bpmn_section_zensus.graphml diff --git a/documentation/bpmn/ego_dp-pre_bpmn_section_zensus.graphml b/documentation/bpmn/ego_dp-pre_bpmn_section_zensus.graphml new file mode 100644 index 00000000..546f115c --- /dev/null +++ b/documentation/bpmn/ego_dp-pre_bpmn_section_zensus.graphml @@ -0,0 +1,467 @@ + + + + + + + + + + + + + + + + + + + + + + + eGo Pre-Processing - zensus + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ego_pp_destatis_zensus_import.sql + + + + + + + + + + + + + + + + + + society.destatis_zensus_population_per_ha + + + + + + + + + + + + + + + + + ? + + + + + + + + + + + + + + + + + orig_destatis.zensus_population_per_ha +orig_destatis.zensus_population_per_ha_grid + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ego_pp_destatis_zensus_insidevg250.sql + + + + + + + + + + + + + + + + + + society.destatis_zensus_population_per_ha + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + society.destatis_zensus_population_per_ha_mview +model_draft.destatis_zensus_population_per_ha_inside +model_draft.destatis_zensus_population_per_ha_invg_mview +model_draft.destatis_zensus_population_per_ha_outvg_mview + + + + + + + + + + + + + + + + + ? + + + + + + + + + + + + + + + + orig_destatis.zensus_population_per_ha + + + + + + + + + + + + + + + + + Datenlizenz Deutschland – Namensnennung – Version 2.0 +Statistisches Bundesamt, Wiesbaden, Genesis-Online, 2016; Datenlizenz by-2-0 + + + + + + + + + + + + + + + + Datenlizenz Deutschland – Namensnennung – Version 2.0 +Statistisches Bundesamt, Wiesbaden, Genesis-Online, 2016; Datenlizenz by-2-0 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + From 106b5a78913b6f70eb74704ffe558919a84a53c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guido=20Ple=C3=9Fmann?= Date: Thu, 19 Mar 2020 09:51:22 +0100 Subject: [PATCH 15/84] Add BPMN for creation of SLP parameters table --- ...dp-pre_bpmn_section_slp_parameters.graphml | 273 ++++++++++++++++++ 1 file changed, 273 insertions(+) create mode 100644 documentation/bpmn/ego_dp-pre_bpmn_section_slp_parameters.graphml diff --git a/documentation/bpmn/ego_dp-pre_bpmn_section_slp_parameters.graphml b/documentation/bpmn/ego_dp-pre_bpmn_section_slp_parameters.graphml new file mode 100644 index 00000000..362b8fc6 --- /dev/null +++ b/documentation/bpmn/ego_dp-pre_bpmn_section_slp_parameters.graphml @@ -0,0 +1,273 @@ + + + + + + + + + + + + + + + + + + + + + + + eGo Pre-Processing - slp_parameters + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ego_pre_slp_parameters.sql + + + + + + + + + + + + + + + + + + scenario.ego_slp_parameters + + + + + + + + + + + + + + + + + ? + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Open Database License (ODbL) v1.0 +© OpenStreetMap contributors + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + From f2ebd8892e1ab5922271bcda32e5f8247403cee6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guido=20Ple=C3=9Fmann?= Date: Thu, 19 Mar 2020 10:04:41 +0100 Subject: [PATCH 16/84] Add dreaft of BPMN for cosmmoclmgrid --- ...o_dp-pre_bpmn_section_cosmoclmgrid.graphml | 272 ++++++++++++++++++ 1 file changed, 272 insertions(+) create mode 100644 documentation/bpmn/ego_dp-pre_bpmn_section_cosmoclmgrid.graphml diff --git a/documentation/bpmn/ego_dp-pre_bpmn_section_cosmoclmgrid.graphml b/documentation/bpmn/ego_dp-pre_bpmn_section_cosmoclmgrid.graphml new file mode 100644 index 00000000..e5a7a324 --- /dev/null +++ b/documentation/bpmn/ego_dp-pre_bpmn_section_cosmoclmgrid.graphml @@ -0,0 +1,272 @@ + + + + + + + + + + + + + + + + + + + + + + + eGo Pre-Processing - cosmoclmgrid + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + <manual import> + + + + + + + + + + + + + + + + + + climate.cosmoclmgrid + + + + + + + + + + + + + + + + + ? + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + no license + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + From cc5f1515fd2e4ed29065743ee82e0d3701e67329 Mon Sep 17 00:00:00 2001 From: IlkaCu Date: Thu, 19 Mar 2020 14:14:26 +0100 Subject: [PATCH 17/84] finalize BPMN for section REA --- .../bpmn/ego_dp_bpmn_section_REA.graphml | 399 +++++++++++------- 1 file changed, 248 insertions(+), 151 deletions(-) diff --git a/documentation/bpmn/ego_dp_bpmn_section_REA.graphml b/documentation/bpmn/ego_dp_bpmn_section_REA.graphml index c001cb38..0a94dba3 100644 --- a/documentation/bpmn/ego_dp_bpmn_section_REA.graphml +++ b/documentation/bpmn/ego_dp_bpmn_section_REA.graphml @@ -31,22 +31,22 @@ + + + - - - - - + + @@ -378,10 +378,10 @@ model_draft.ego_demand_loadarea + - @@ -398,10 +398,10 @@ model_draft.ego_lattice_500m_x_mview model_draft.ego_lattice_500m_out_mview + - @@ -436,10 +436,10 @@ model_draft.ego_demand_loadarea + - @@ -454,10 +454,10 @@ model_draft.ego_demand_loadarea + - @@ -471,10 +471,10 @@ model_draft.ego_demand_loadarea model_draft.ego_lattice_50m + - @@ -509,10 +509,10 @@ model_draft.ego_grid_mv_griddistrict model_draft.ego_grid_hvmv_substation + - @@ -528,10 +528,10 @@ model_draft.ego_dp_supply_res_powerplant_out_nn + - @@ -546,10 +546,10 @@ model_draft.ego_dp_supply_res_powerplant_out_nn model_draft.ego_osm_sector_per_griddistrict_4_agricultural + - @@ -593,10 +593,10 @@ model_draft.ego_supply_rea_m1_2_rest_mview + - @@ -612,10 +612,10 @@ model_draft.ego_osm_sector_per_griddistrict_4_agricultural + - @@ -629,10 +629,10 @@ model_draft.ego_osm_sector_per_griddistrict_4_agricultural model_draft.ego_dp_supply_res_powerplant + - @@ -671,10 +671,10 @@ model_draft.ego_supply_rea_m2_rest_mview + - @@ -690,10 +690,10 @@ model_draft.ego_grid_mv_griddistrict model_draft.ego_supply_wpa_per_mvgd + - @@ -707,10 +707,10 @@ model_draft.ego_supply_wpa_per_mvgdmodel_draft.ego_dp_supply_res_powerplant + - @@ -750,10 +750,10 @@ model_draft.ego_supply_rea_m3_rest_mview + - @@ -769,10 +769,10 @@ model_draft.ego_lattice_500m_wpa_mview + - @@ -786,10 +786,10 @@ model_draft.ego_lattice_500m_wpa_mview model_draft.ego_dp_supply_res_powerplant + - @@ -829,10 +829,10 @@ model_draft.ego_supply_rea_m4_rest_mview + - @@ -848,10 +848,10 @@ model_draft.ego_lattice_500m_wpa_mview + - @@ -865,10 +865,10 @@ model_draft.ego_lattice_500m_wpa_mview model_draft.ego_dp_supply_res_powerplant + - @@ -908,10 +908,10 @@ model_draft.ego_supply_rea_m5_mview + - @@ -928,10 +928,10 @@ model_draft.ego_grid_lv_griddistrict + - @@ -945,10 +945,10 @@ model_draft.ego_grid_lv_griddistrict model_draft.ego_dp_supply_res_powerplant + - @@ -983,10 +983,10 @@ model_draft.ego_supply_rea_per_method + - @@ -1002,10 +1002,10 @@ model_draft.ego_dp_supply_res_powerplant model_draft.ego_demand_loadarea + - @@ -1020,10 +1020,10 @@ model_draft.ego_demand_loadarea + - @@ -1037,10 +1037,10 @@ model_draft.ego_dp_supply_res_powerplantDP griddistrict + - @@ -1054,10 +1054,10 @@ model_draft.ego_dp_supply_res_powerplantDP loadarea + - @@ -1071,10 +1071,27 @@ model_draft.ego_dp_supply_res_powerplantDP substation + - + + + + + + + + + + + powerplants + + + + + + @@ -1193,14 +1210,13 @@ model_draft.ego_dp_supply_res_powerplant - - + + - @@ -1211,7 +1227,6 @@ model_draft.ego_dp_supply_res_powerplant - @@ -1222,7 +1237,6 @@ model_draft.ego_dp_supply_res_powerplant - @@ -1233,7 +1247,6 @@ model_draft.ego_dp_supply_res_powerplant - @@ -1244,7 +1257,6 @@ model_draft.ego_dp_supply_res_powerplant - @@ -1255,7 +1267,6 @@ model_draft.ego_dp_supply_res_powerplant - @@ -1266,7 +1277,6 @@ model_draft.ego_dp_supply_res_powerplant - @@ -1277,7 +1287,6 @@ model_draft.ego_dp_supply_res_powerplant - @@ -1288,7 +1297,6 @@ model_draft.ego_dp_supply_res_powerplant - @@ -1299,315 +1307,286 @@ model_draft.ego_dp_supply_res_powerplant - - - + + - - - + + - - - + + - - - + + - - - + + - - - + + - - - + + - - - + + - - - + + - - - + + - - - + + - - - + + - - - + + - - - + + - - - + + - - - + + - - - + + - - - + + - - - + + - - - + + - - - + + - - - + + - - - + + - - - + + - - - + + - - - + + - - - + + - - - + + - @@ -1618,18 +1597,16 @@ model_draft.ego_dp_supply_res_powerplant - - - + + - @@ -1640,7 +1617,6 @@ model_draft.ego_dp_supply_res_powerplant - @@ -1651,7 +1627,6 @@ model_draft.ego_dp_supply_res_powerplant - @@ -1662,7 +1637,6 @@ model_draft.ego_dp_supply_res_powerplant - @@ -1673,7 +1647,6 @@ model_draft.ego_dp_supply_res_powerplant - @@ -1684,7 +1657,6 @@ model_draft.ego_dp_supply_res_powerplant - @@ -1695,7 +1667,6 @@ model_draft.ego_dp_supply_res_powerplant - @@ -1706,7 +1677,6 @@ model_draft.ego_dp_supply_res_powerplant - @@ -1717,7 +1687,6 @@ model_draft.ego_dp_supply_res_powerplant - @@ -1728,7 +1697,6 @@ model_draft.ego_dp_supply_res_powerplant - @@ -1739,34 +1707,163 @@ model_draft.ego_dp_supply_res_powerplant - - - + + - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + From a24169698041e4d1d33505c15ca7423bce06ec1b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guido=20Ple=C3=9Fmann?= Date: Thu, 19 Mar 2020 15:43:25 +0100 Subject: [PATCH 18/84] Draft BPMN --- .../ego_dp-pre_bpmn_section_supply.graphml | 975 ++++++++++++++++++ 1 file changed, 975 insertions(+) create mode 100644 documentation/bpmn/ego_dp-pre_bpmn_section_supply.graphml diff --git a/documentation/bpmn/ego_dp-pre_bpmn_section_supply.graphml b/documentation/bpmn/ego_dp-pre_bpmn_section_supply.graphml new file mode 100644 index 00000000..e8c7a548 --- /dev/null +++ b/documentation/bpmn/ego_dp-pre_bpmn_section_supply.graphml @@ -0,0 +1,975 @@ + + + + + + + + + + + + + + + + + + + + + + + eGo Pre-Processing - supply + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + preprocessing/python_scripts/ +renpass_gis/simple_feedin/simple_feedin.py + + + + + + + + + + + + + + + + + + model_draft.ego_renewable_feedin + + + + + + + + + + + + + + + + + ? + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Open Database License (ODbL) v1.0 +© OpenStreetMap contributors + + + + + + + + + + + + + + + + ego_dp_preprocessing_res_powerplant.sql + + + + + + + + + + + + + + + + + + model_draft.ego_dp_supply_res_powerplant + + + + + + + + + + + + + + + + + Open Database License (ODbL) v1.0 +© OpenStreetMap contributors + + + + + + + + + + + + + + + + model_draft.ego_supply_res_powerplant + + + + + + + + + + + + + + + + + ego_dp_preprocessing_conv_powerplant.sql + + + + + + + + + + + + + + + + + + model_draft.ego_dp_supply_conv_powerplant +model_draft.ego_supply_conv_nep2035_temp + + + + + + + + + + + + + + + + + Open Database License (ODbL) v1.0 +© OpenStreetMap contributors + + + + + + + + + + + + + + + + model_draft.ego_supply_conv_powerplant + + + + + + + + + + + + + + + + + ego_dp_res_rea_by_scenario.sql + + + + + + + + + + + + + + + + + + model_draft.ego_dp_supply_res_powerplant + + + + + + + + + + + + + + + + + + Open Database License (ODbL) v1.0 +© OpenStreetMap contributors + + + + + + + + + + + + + + + + model_draft.ego_dp_supply_res_powerplant +boundaries.bkg_vg250_2_lan_nuts_view +model_draft.ego_supply_scenario_capacities +boundaries.bkg_vg250_6_gem_rs_mview + + + + + + + + + + + + + + + + + ego_dp_preprocessing_res_powerplant_2035.sql + + + + + + + + + + + + + + + + + + model_draft.ego_dp_supply_res_powerplant + + + + + + + + + + + + + + + + + + Open Database License (ODbL) v1.0 +© OpenStreetMap contributors + + + + + + + + + + + + + + + + model_draft.ego_dp_supply_res_powerplant + + + + + + + + + + + + + + + + + + ego_dp_preprocessing_conv_powerplant_2035.sql + + + + + + + + + + + + + + + + + + model_draft.ego_dp_supply_conv_powerplant_2035 + + + + + + + + + + + + + + + + + + Open Database License (ODbL) v1.0 +© OpenStreetMap contributors + + + + + + + + + + + + + + + + model_draft.ego_dp_supply_conv_powerplant + + + + + + + + + + + + + + + + + + ego_dp_conv_by_scenario.sql + + + + + + + + + + + + + + + + + + model_draft.ego_dp_supply_conv_powerplant + + + + + + + + + + + + + + + + + + Open Database License (ODbL) v1.0 +© OpenStreetMap contributors + + + + + + + + + + + + + + + + model_draft.ego_dp_supply_conv_powerplant + + + + + + + + + + + + + + + + + + ego_dp_supply_add_coastdat_gid.sql + + + + + + + + + + + + + + + + + + model_draft.ego_dp_supply_conv_powerplant +model_draft.ego_dp_supply_res_powerplant + + + + + + + + + + + + + + + + + + Open Database License (ODbL) v1.0 +© OpenStreetMap contributors + + + + + + + + + + + + + + + + model_draft.ego_dp_supply_conv_powerplant +model_draft.ego_dp_supply_res_powerplant +coastdat.cosmoclmgrid + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + From 03cb6d615babbe9a548107ecaead8b5324ad51bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guido=20Ple=C3=9Fmann?= Date: Sat, 21 Mar 2020 20:20:16 +0100 Subject: [PATCH 19/84] Add draft of BPMN for openstreetmap --- ..._dp-pre_bpmn_section_openstreetmap.graphml | 625 ++++++++++++++++++ 1 file changed, 625 insertions(+) create mode 100644 documentation/bpmn/ego_dp-pre_bpmn_section_openstreetmap.graphml diff --git a/documentation/bpmn/ego_dp-pre_bpmn_section_openstreetmap.graphml b/documentation/bpmn/ego_dp-pre_bpmn_section_openstreetmap.graphml new file mode 100644 index 00000000..4376dd84 --- /dev/null +++ b/documentation/bpmn/ego_dp-pre_bpmn_section_openstreetmap.graphml @@ -0,0 +1,625 @@ + + + + + + + + + + + + + + + + + + + + + + + eGo DataProcessing - openstreetmap + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ego_pp_osm_deu_import.sql + + + + + + + + + + + + + + + + + + openstreetmap.osm_deu_line +openstreetmap.osm_deu_nodes +openstreetmap.osm_deu_point +openstreetmap.osm_deu_polygon +openstreetmap.osm_deu_rels +openstreetmap.osm_deu_roads +openstreetmap.osm_deu_ways + + + + + + + + + + + + + + + + + ? + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + openstreetmap (osm2pgsql) + + + + + + + + + + + + + + + + + Open Database License (ODbL) v1.0 +© OpenStreetMap contributors + + + + + + + + + + + + + + + + Open Database License (ODbL) v1.0 +© OpenStreetMap contributors + + + + + + + + + + + + + + + + ego_pp_osm_line_street_mview.sql + + + + + + + + + + + + + + + + + + openstreetmap.osm_deu_line_street_mview + + + + + + + + + + + + + + + + + ? + + + + + + + + + + + + + + + + openstreetmap.osm_deu_line + + + + + + + + + + + + + + + + + Open Database License (ODbL) v1.0 +© OpenStreetMap contributors + + + + + + + + + + + + + + + + ego_pp_osm_polygon_building_mview.sql + + + + + + + + + + + + + + + + + + openstreetmap.osm_deu_polygon_building_mview + + + + + + + + + + + + + + + + + ? + + + + + + + + + + + + + + + + openstreetmap.osm_deu_polygon + + + + + + + + + + + + + + + + + Open Database License (ODbL) v1.0 +© OpenStreetMap contributors + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + public.osm_deu_line +public.osm_deu_nodes +public.osm_deu_point +public.osm_deu_polygon +public.osm_deu_rels +public.osm_deu_roads +public.osm_deu_ways + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + From ab9ccc796c546f55a07a270590798683ce512a64 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guido=20Ple=C3=9Fmann?= Date: Sat, 21 Mar 2020 20:57:08 +0100 Subject: [PATCH 20/84] Add docker compose file --- Dockerfile.postgis | 4 ++++ README.rst | 17 +++++++++++++++++ docker-compose.yml | 18 ++++++++++++++++++ utilities/entrypoints/create_extensions.sql | 2 ++ 4 files changed, 41 insertions(+) create mode 100644 Dockerfile.postgis create mode 100644 docker-compose.yml create mode 100644 utilities/entrypoints/create_extensions.sql diff --git a/Dockerfile.postgis b/Dockerfile.postgis new file mode 100644 index 00000000..65381fb9 --- /dev/null +++ b/Dockerfile.postgis @@ -0,0 +1,4 @@ +FROM postgres:11 + +RUN apt-get update +RUN apt-get install -y postgresql-11-postgis-2.5 diff --git a/README.rst b/README.rst index 2740d6be..78bc76c6 100644 --- a/README.rst +++ b/README.rst @@ -51,3 +51,20 @@ Run ~$ ego_data_processing ``` +Local database with docker +========================== + +You need to have `docker-compose` installed. + +### Spin up database with docker + +``` +docker-compose up -d --build +``` + +### Database credentials + +| database | dp | +| port | 54321 | +| user | oeuser | +| password | egon | \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 00000000..2b856ed3 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,18 @@ +version: '3' +services: + data_processing_db: + image: postgres:11-postgis + container_name: data_processing + restart: unless-stopped + build: + context: . + dockerfile: Dockerfile.postgis + ports: + - "127.0.0.1:54321:5432" + environment: + POSTGRES_DB: dp + POSTGRES_USER: oeuser + POSTGRES_PASSWORD: egon + volumes: + - $HOME/docker/volumes/postgres:/var/lib/postgresql/data + - ./utilities/entrypoints:/docker-entrypoint-initdb.d/ diff --git a/utilities/entrypoints/create_extensions.sql b/utilities/entrypoints/create_extensions.sql new file mode 100644 index 00000000..78b98ad4 --- /dev/null +++ b/utilities/entrypoints/create_extensions.sql @@ -0,0 +1,2 @@ +create extension hstore; +CREATE EXTENSION postgis; \ No newline at end of file From ad7768d475e4a6c44024d0c8dd078691b1cbab18 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guido=20Ple=C3=9Fmann?= Date: Sun, 22 Mar 2020 11:30:39 +0100 Subject: [PATCH 21/84] Rename script to make executed first --- .../{create_extensions.sql => 00_create_extensions.sql} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename utilities/entrypoints/{create_extensions.sql => 00_create_extensions.sql} (100%) diff --git a/utilities/entrypoints/create_extensions.sql b/utilities/entrypoints/00_create_extensions.sql similarity index 100% rename from utilities/entrypoints/create_extensions.sql rename to utilities/entrypoints/00_create_extensions.sql From 098b9d1236aad6d1a1a95a9d936999f2f3a09484 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guido=20Ple=C3=9Fmann?= Date: Sun, 22 Mar 2020 11:46:30 +0100 Subject: [PATCH 22/84] Start to revise pre-processing * Put all required scripts in one workflow * Include data import in pre-procesing * Use a .yml file to specify scripts * Allow bash scripts * First examplary script: OSM import --- README.rst | 12 +- dataprocessing/tools/osm2pgsql/osm2oedb.txt | 1 - preprocessing/eGo_import_and_preprocessing.py | 111 ++++++++++++++++++ preprocessing/eGo_preprocessing.py | 91 -------------- preprocessing/import.yml | 3 + .../openstreetmap}/default.style | 0 .../openstreetmap}/empty.style | 0 .../openstreetmap}/oedb.style | 0 preprocessing/openstreetmap/osm2oedb.sh | 2 + setup.py | 5 +- 10 files changed, 130 insertions(+), 95 deletions(-) delete mode 100644 dataprocessing/tools/osm2pgsql/osm2oedb.txt create mode 100644 preprocessing/eGo_import_and_preprocessing.py delete mode 100644 preprocessing/eGo_preprocessing.py create mode 100644 preprocessing/import.yml rename {dataprocessing/tools/osm2pgsql => preprocessing/openstreetmap}/default.style (100%) rename {dataprocessing/tools/osm2pgsql => preprocessing/openstreetmap}/empty.style (100%) rename {dataprocessing/tools/osm2pgsql => preprocessing/openstreetmap}/oedb.style (100%) create mode 100644 preprocessing/openstreetmap/osm2oedb.sh diff --git a/README.rst b/README.rst index 78bc76c6..e7fc61de 100644 --- a/README.rst +++ b/README.rst @@ -67,4 +67,14 @@ docker-compose up -d --build | database | dp | | port | 54321 | | user | oeuser | -| password | egon | \ No newline at end of file +| password | egon | + + +Usage +===== + + +Pre-processing +************** + +Execute `eGo_import_and_preprocessing.py` for starting pre-processing from scratch doing data import and a pre-processing of imported tables. \ No newline at end of file diff --git a/dataprocessing/tools/osm2pgsql/osm2oedb.txt b/dataprocessing/tools/osm2pgsql/osm2oedb.txt deleted file mode 100644 index bc5ecaac..00000000 --- a/dataprocessing/tools/osm2pgsql/osm2oedb.txt +++ /dev/null @@ -1 +0,0 @@ -osm2pgsql --create --slim --hstore-all --number-processes 2 --cache 5120 -H localhost -P 5432 -d oedb -U -W -p osm_deu -S oedb.style germany-161001.osm.pbf \ No newline at end of file diff --git a/preprocessing/eGo_import_and_preprocessing.py b/preprocessing/eGo_import_and_preprocessing.py new file mode 100644 index 00000000..5342cc2d --- /dev/null +++ b/preprocessing/eGo_import_and_preprocessing.py @@ -0,0 +1,111 @@ +""" +eGo PreProcessing (eGoPP) + +This script opens an oedb database connection and executes different parts of eGo. +Reads python and SQL scripts and gives logging infos during the execution. +Also see corresponding BPML diagram. + +This file is part of project "open_eGo DataProcessing" (https://github.com/openego/data_processing/). +It's copyrighted by the contributors recorded in the version control history: +openego/data_processing/preprocessing/eGo_PreProcessing.py + +SPDX-License-Identifier: AGPL-3.0-or-later +""" + +__copyright__ = "Reiner Lemoine Institut" +__license__ = "GNU Affero General Public License Version 3 (AGPL-3.0)" +__url__ = "https://www.gnu.org/licenses/agpl-3.0.en.html" +__author__ = "gplssm, Ludee" + +import logging +import time +import os +import subprocess +from dataprocessing.tools import io +from egoio.tools import db +from sqlalchemy import create_engine +import yaml + + +SCENARIOLOG = True + + +def preprocessing(): + # Configure logging + logger = logging.getLogger('EEEE') + logger.setLevel(logging.INFO) + ch = logging.StreamHandler() + ch.setLevel(logging.INFO) + formatter = logging.Formatter('%(asctime)s %(message)s', + datefmt='%Y-%m-%d %I:%M:%S') + ch.setFormatter(formatter) + logger.addHandler(ch) + + # get current time and inform about start + total_time = time.time() + logger.info('ego preprocessing started...') + + # list of sql- and python-snippets that process the data in correct order + script_dir = os.path.abspath( + os.path.join(os.path.dirname(__file__))) + + snippets = [ + 'ego_pre_voltage_level.sql', + 'ego_pre_slp_parameters.sql' + ] + datasets = yaml.load(open("import.yml"))#, Loader=Loader) + + # get database connection + conn_oedb = db.connection(readonly=True).connect() + engine_local = create_engine('postgresql+psycopg2://oeuser:egon@localhost:54321/dp') + conn = engine_local.connect() + + # iterate over list of sql- and python-snippets and execute them + for key, dataset in datasets.items(): + for script in dataset: + + # timing and logging + snippet_time = time.time() + logger.info("Execute '{}' ...".format(script["script"])) + if script["language"] == 'SQL': + if SCENARIOLOG is True: + snippet_str = open(os.path.join(script_dir, key, script["script"])).read() + elif SCENARIOLOG is False: + snippet_str = "".join( + [l for l in open(os.path.join(script_dir, key, script["script"])).readlines() + if not l.startswith("SELECT scenario_log") and not l.startswith("SELECT ego_scenario_log")]) + + # execute desired sql snippet + conn.execution_options(autocommit=True).execute(snippet_str) + elif script["language"] == 'python': + filename = os.path.join(script_dir, snippet) + script_str = open(filename, "rb").read() + + # execute desired sql snippet + exec(compile(script_str, filename, 'exec')) + elif script["language"] == 'bash': + filename = os.path.join(script_dir, key, script["script"]) + script_str = open(filename, "rb").read() + + # execute desired bash script + rc = subprocess.call(script_str, + cwd= os.path.join(script_dir, key), + shell=True) + else: + raise NameError('{} is neither a python nor a sql script (at least it ' + 'has not the right extension). Please add an extension ' + 'to the script name (.py or .sql)'.format(script["script"])) + + # inform the user + logger.info('...successfully done in {:.2f} seconds.'.format( + time.time() - snippet_time)) + + # close database connection + conn.close() + + logger.info('eGo PreProcessing script successfully executed in {:.2f} seconds'.format( + time.time() - total_time)) + + +if __name__ == '__main__': + preprocessing() diff --git a/preprocessing/eGo_preprocessing.py b/preprocessing/eGo_preprocessing.py deleted file mode 100644 index 000ff365..00000000 --- a/preprocessing/eGo_preprocessing.py +++ /dev/null @@ -1,91 +0,0 @@ -""" -eGo PreProcessing (eGoPP) - -This script opens an oedb database connection and executes different parts of eGo. -Reads python and SQL scripts and gives logging infos during the execution. -Also see corresponding BPML diagram. - -This file is part of project "open_eGo DataProcessing" (https://github.com/openego/data_processing/). -It's copyrighted by the contributors recorded in the version control history: -openego/data_processing/preprocessing/eGo_PreProcessing.py - -SPDX-License-Identifier: AGPL-3.0-or-later -""" - -__copyright__ = "Reiner Lemoine Institut" -__license__ = "GNU Affero General Public License Version 3 (AGPL-3.0)" -__url__ "https://www.gnu.org/licenses/agpl-3.0.en.html" -__author__ = "gplssm, Ludee" - -import pandas as pd -import logging -import time -import os -import codecs -from tools import io - -def preprocessing(): - # Configure logging - logger = logging.getLogger('EEEE') - logger.setLevel(logging.INFO) - ch = logging.StreamHandler() - ch.setLevel(logging.INFO) - formatter = logging.Formatter('%(asctime)s %(message)s', - datefmt='%Y-%m-%d %I:%M:%S') - ch.setFormatter(formatter) - logger.addHandler(ch) - - # get current time and inform about start - total_time = time.time() - logger.info('ego preprocessing started...') - - # list of sql- and python-snippets that process the data in correct order - snippet_dir = os.path.abspath( - os.path.join(os.path.dirname(__file__), - 'preprocessing')) - script_dir = os.path.abspath( - os.path.join(os.path.dirname(__file__), - 'python_scripts')) - - snippets = [ - 'ego_pre_voltage_level.sql', - 'ego_pre_slp_parameters.sql' - ] - - # get database connection - conn = io.oedb_session(section='oedb') - - # iterate over list of sql- and python-snippets and execute them - for snippet in snippets: - # timing and logging - snippet_time = time.time() - logger.info("Execute '{}' ...".format(snippet)) - if os.path.splitext(snippet)[1] == '.sql': - snippet_str = open(os.path.join(snippet_dir, snippet)).read() - - # execute desired sql snippet - conn.execution_options(autocommit=True).execute(snippet_str) - elif os.path.splitext(snippet)[1] == '.py': - filename = os.path.join(script_dir, snippet) - script_str = open(filename, "rb").read() - - # execute desired sql snippet - exec(compile(script_str, filename, 'exec')) - else: - raise NameError('{} is neither a python nor a sql script (at least it ' - 'has not the right extension). Please add an extension ' - 'to the script name (.py or .sql)'.format(snippet)) - - # inform the user - logger.info('...successfully done in {:.2f} seconds.'.format( - time.time() - snippet_time)) - - # close database connection - conn.close() - - logger.info('eGo PreProcessing script successfully executed in {:.2f} seconds'.format( - time.time() - total_time)) - - -if __name__ == '__main__': - preprocessing() diff --git a/preprocessing/import.yml b/preprocessing/import.yml new file mode 100644 index 00000000..874fbacd --- /dev/null +++ b/preprocessing/import.yml @@ -0,0 +1,3 @@ +openstreetmap: + - script: osm2oedb.sh + language: bash diff --git a/dataprocessing/tools/osm2pgsql/default.style b/preprocessing/openstreetmap/default.style similarity index 100% rename from dataprocessing/tools/osm2pgsql/default.style rename to preprocessing/openstreetmap/default.style diff --git a/dataprocessing/tools/osm2pgsql/empty.style b/preprocessing/openstreetmap/empty.style similarity index 100% rename from dataprocessing/tools/osm2pgsql/empty.style rename to preprocessing/openstreetmap/empty.style diff --git a/dataprocessing/tools/osm2pgsql/oedb.style b/preprocessing/openstreetmap/oedb.style similarity index 100% rename from dataprocessing/tools/osm2pgsql/oedb.style rename to preprocessing/openstreetmap/oedb.style diff --git a/preprocessing/openstreetmap/osm2oedb.sh b/preprocessing/openstreetmap/osm2oedb.sh new file mode 100644 index 00000000..5768ed6d --- /dev/null +++ b/preprocessing/openstreetmap/osm2oedb.sh @@ -0,0 +1,2 @@ +export PGPASSWORD=egon +osm2pgsql --create --slim --hstore-all --number-processes 2 --cache 4096 -H localhost -P 54321 -d dp -U oeuser -p osm_deu -S oedb.style bremen-latest.osm.pbf \ No newline at end of file diff --git a/setup.py b/setup.py index 089d6c5d..c2b3d559 100644 --- a/setup.py +++ b/setup.py @@ -18,8 +18,9 @@ 'workalendar', 'oemof.db', 'demandlib', - 'ego.io >=0.0.1rc4, <= 0.0.2', - 'geoalchemy2' + 'egoio >= 0.4.7', + 'geoalchemy2', + "pyyaml" ], extras_require={ 'docs': [ From 8adbb423c174271e59d72ef5eb5966f649cece68 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guido=20Ple=C3=9Fmann?= Date: Sun, 22 Mar 2020 12:39:42 +0100 Subject: [PATCH 23/84] Add "scenario log" preparation scripts --- preprocessing/import.yml | 7 ++ .../scenario_log/ego_pp_scenario.sql | 5 +- .../scenario_log/ego_pp_scenario_log.sql | 4 +- .../scenario_log/function_scenario_log.sql | 67 +++++++++---------- 4 files changed, 40 insertions(+), 43 deletions(-) diff --git a/preprocessing/import.yml b/preprocessing/import.yml index 874fbacd..3fe1296b 100644 --- a/preprocessing/import.yml +++ b/preprocessing/import.yml @@ -1,3 +1,10 @@ +scenario_log: + - script: 01_function_scenario_log.sql + language: SQL + - script: 02_ego_pp_scenario.sql + language: SQL + - script: 03_ego_pp_scenario_log.sql + language: SQL openstreetmap: - script: osm2oedb.sh language: bash diff --git a/preprocessing/scenario_log/ego_pp_scenario.sql b/preprocessing/scenario_log/ego_pp_scenario.sql index 000b97c8..ccc300bc 100644 --- a/preprocessing/scenario_log/ego_pp_scenario.sql +++ b/preprocessing/scenario_log/ego_pp_scenario.sql @@ -18,11 +18,12 @@ __author__ = "Ludee" -- scenario list +CREATE SCHEMA IF NOT EXISTS model_draft; DROP TABLE IF EXISTS model_draft.ego_scenario CASCADE; CREATE TABLE model_draft.ego_scenario ( id serial, model text, - version text, + version text UNIQUE, version_name text, "release" boolean, "comment" text, @@ -99,5 +100,3 @@ COMMENT ON TABLE model_draft.ego_scenario IS '{ {"name": "timestamp", "description": "timestamp (Berlin)", "unit": "none" } ] } ], "metadata_version": "1.3"}'; --- scenario log (project,version,io,schema_name,table_name,script_name,comment) -SELECT scenario_log('eGo_PP','PP1','setup','model_draft','ego_scenario','ego_pp_scenario.sql','Update scenario list'); diff --git a/preprocessing/scenario_log/ego_pp_scenario_log.sql b/preprocessing/scenario_log/ego_pp_scenario_log.sql index 327924b6..8711ff5d 100644 --- a/preprocessing/scenario_log/ego_pp_scenario_log.sql +++ b/preprocessing/scenario_log/ego_pp_scenario_log.sql @@ -24,7 +24,6 @@ GROUP BY version ORDER BY version; */ -/* -- scenario log table DROP TABLE IF EXISTS model_draft.scenario_log CASCADE; CREATE TABLE model_draft.scenario_log ( @@ -36,7 +35,7 @@ CREATE TABLE model_draft.scenario_log ( table_name text, script_name text, entries integer, - status text, + comment text, user_name text, "timestamp" timestamp, meta_data text, @@ -49,7 +48,6 @@ ALTER TABLE model_draft.scenario_log -- access rights ALTER TABLE model_draft.scenario_log OWNER TO oeuser; -*/ -- metadata COMMENT ON TABLE model_draft.scenario_log IS '{ diff --git a/preprocessing/scenario_log/function_scenario_log.sql b/preprocessing/scenario_log/function_scenario_log.sql index 515a2996..0fe3ed70 100644 --- a/preprocessing/scenario_log/function_scenario_log.sql +++ b/preprocessing/scenario_log/function_scenario_log.sql @@ -1,4 +1,4 @@ -/* +/* Scenario Log Function (Multiproject version) Function to add an entry to the Scenario Log table: @@ -33,45 +33,38 @@ __author__ = "Ludwig Hülk" * SPDX-License-Identifier: AGPL-3.0-or-later */ - --- Scenario Log function -DROP FUNCTION IF EXISTS scenario_log(text,text,text,text,text,text,text); +DROP FUNCTION IF EXISTS scenario_log(text,text,text,text,text,text,text); CREATE OR REPLACE FUNCTION scenario_log(text,text,text,text,text,text,text) - RETURNS text AS - $$ - DECLARE - _project ALIAS FOR $1; - _version ALIAS FOR $2; - _io ALIAS FOR $3; - _schema_name ALIAS FOR $4; - _table_name ALIAS FOR $5; - _script_name ALIAS FOR $6; - _comment ALIAS FOR $7; - BEGIN - EXECUTE 'INSERT INTO model_draft.scenario_log ' || - '(project,version,io,schema_name,table_name,script_name,entries,comment,user_name,timestamp,meta_data) - SELECT ' || quote_literal(_project) || ',' || - quote_literal(_version) || ',' || - quote_literal(_io) || ',' || - quote_literal(_schema_name) || ',' || - quote_literal(_table_name) || ',' || - quote_literal(_script_name) || ',' || - 'COUNT(*),' || - quote_literal(_comment) || ',' || - 'session_user,' || - E'NOW() AT TIME ZONE \'Europe/Berlin\' ,' || - E'obj_description(\'' || _schema_name || '.' || _table_name || E'\' ::regclass) ::json ' || - 'FROM ' || _schema_name || '.' || _table_name || ';' ; - RETURN 'inserted'; - END; - $$ LANGUAGE plpgsql; + RETURNS text AS + $$ + DECLARE + _project ALIAS FOR $1; + _version ALIAS FOR $2; + _io ALIAS FOR $3; + _schema_name ALIAS FOR $4; + _table_name ALIAS FOR $5; + _script_name ALIAS FOR $6; + _comment ALIAS FOR $7; + BEGIN + EXECUTE 'INSERT INTO model_draft.scenario_log ' || + '(project,version,io,schema_name,table_name,script_name,entries,comment,user_name,timestamp,meta_data) + SELECT ' || quote_literal(_project) || ',' || + quote_literal(_version) || ',' || + quote_literal(_io) || ',' || + quote_literal(_schema_name) || ',' || + quote_literal(_table_name) || ',' || + quote_literal(_script_name) || ',' || + 'COUNT(*),' || + quote_literal(_comment) || ',' || + 'session_user,' || + E'NOW() AT TIME ZONE \'Europe/Berlin\' ,' || + E'obj_description(\'' || _schema_name || '.' || _table_name || E'\' ::regclass) ::json ' || + 'FROM ' || _schema_name || '.' || _table_name || ';' ; + RETURN 'inserted'; + END; + $$ LANGUAGE plpgsql; -- access rights ALTER FUNCTION scenario_log(text,text,text,text,text,text,text) OWNER TO oeuser; GRANT EXECUTE ON FUNCTION scenario_log(text,text,text,text,text,text,text) TO oeuser; --- scenario log (project,version,io,schema_name,table_name,script_name,comment) -SELECT scenario_log('eGo_PP','PP1','setup','model_draft','scenario_log','function_scenario_log.sql','Function testing'); - --- select latest entry -SELECT * FROM model_draft.scenario_log ORDER BY timestamp DESC LIMIT 1; From c7a213ce42df8be0416ed23693c1a98869df2c43 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guido=20Ple=C3=9Fmann?= Date: Sun, 22 Mar 2020 13:29:00 +0100 Subject: [PATCH 24/84] Add OSM import data processing script --- preprocessing/import.yml | 2 + .../openstreetmap/ego_pp_osm_deu_import.sql | 93 +++++++------------ 2 files changed, 38 insertions(+), 57 deletions(-) rename preprocessing/{sql_snippets => }/openstreetmap/ego_pp_osm_deu_import.sql (56%) diff --git a/preprocessing/import.yml b/preprocessing/import.yml index 3fe1296b..036e78b0 100644 --- a/preprocessing/import.yml +++ b/preprocessing/import.yml @@ -8,3 +8,5 @@ scenario_log: openstreetmap: - script: osm2oedb.sh language: bash + - script: ego_pp_osm_deu_import.sql + language: SQL diff --git a/preprocessing/sql_snippets/openstreetmap/ego_pp_osm_deu_import.sql b/preprocessing/openstreetmap/ego_pp_osm_deu_import.sql similarity index 56% rename from preprocessing/sql_snippets/openstreetmap/ego_pp_osm_deu_import.sql rename to preprocessing/openstreetmap/ego_pp_osm_deu_import.sql index 268bfee8..0c450bec 100644 --- a/preprocessing/sql_snippets/openstreetmap/ego_pp_osm_deu_import.sql +++ b/preprocessing/openstreetmap/ego_pp_osm_deu_import.sql @@ -1,4 +1,4 @@ -/* +/* Preprocess OpenStreetMap tables after import in public schema with osm2pgsql Basic operations: 0. grant @@ -15,37 +15,16 @@ __url__ = "https://github.com/openego/data_processing/blob/master/LICENSE" __author__ = "Ludee" */ +CREATE SCHEMA IF NOT EXISTS openstreetmap; -- ego scenario log (version,io,schema_name,table_name,script_name,comment) -SELECT ego_scenario_log('v0.2.10','input','public','osm_deu_line','ego_pp_osm_deu_import.sql','verification'); -SELECT ego_scenario_log('v0.2.10','input','public','osm_deu_nodes','ego_pp_osm_deu_import.sql','verification'); -SELECT ego_scenario_log('v0.2.10','input','public','osm_deu_point','ego_pp_osm_deu_import.sql','verification'); -SELECT ego_scenario_log('v0.2.10','input','public','osm_deu_polygon','ego_pp_osm_deu_import.sql','verification'); -SELECT ego_scenario_log('v0.2.10','input','public','osm_deu_rels','ego_pp_osm_deu_import.sql','verification'); -SELECT ego_scenario_log('v0.2.10','input','public','osm_deu_roads','ego_pp_osm_deu_import.sql','verification'); -SELECT ego_scenario_log('v0.2.10','input','public','osm_deu_ways','ego_pp_osm_deu_import.sql','verification'); +SELECT scenario_log('eGo_PP','PP1','input','public','osm_deu_line','ego_pp_osm_deu_import.sql','verification'); +SELECT scenario_log('eGo_PP','PP1','input','public','osm_deu_point','ego_pp_osm_deu_import.sql','verification'); +SELECT scenario_log('eGo_PP','PP1','input','public','osm_deu_polygon','ego_pp_osm_deu_import.sql','verification'); +SELECT scenario_log('eGo_PP','PP1','input','public','osm_deu_rels','ego_pp_osm_deu_import.sql','verification'); +SELECT scenario_log('eGo_PP','PP1','input','public','osm_deu_roads','ego_pp_osm_deu_import.sql','verification'); +SELECT scenario_log('eGo_PP','PP1','input','public','osm_deu_ways','ego_pp_osm_deu_import.sql','verification'); -/* --- Tests for the loops --- Select all tables with OSM in public -SELECT * -FROM information_schema.tables -WHERE table_schema = 'public' - AND table_name LIKE 'osm_deu_%'; - --- Select all tables with OSM in public -SELECT * -FROM pg_tables -WHERE schemaname='public' - AND tablename LIKE 'osm_deu_%'; - --- Select all indexes with OSM in public -SELECT * -FROM pg_indexes -WHERE schemaname='public' - AND tablename LIKE 'osm_deu_%' - AND indexname LIKE '%_pkey'; -*/ -- 0. grant oeuser DO @@ -53,14 +32,13 @@ $$ DECLARE row record; BEGIN - FOR row IN SELECT tablename FROM pg_tables WHERE schemaname = 'public' AND tablename LIKE 'osm_deu_%' + FOR row IN SELECT tablename FROM pg_tables WHERE schemaname = 'public' AND tablename LIKE 'osm_deu_%%' LOOP EXECUTE 'ALTER TABLE public.' || quote_ident(row.tablename) || ' OWNER TO oeuser;'; END LOOP; END; $$; - -- 1. remove wrong index -- 1.1 remove all wrong 'pkey' (_line, _point, _polygon, _roads) DO @@ -70,11 +48,11 @@ DECLARE BEGIN FOR row IN SELECT indexname FROM pg_indexes WHERE (schemaname='public' - AND indexname LIKE '%_pkey') - AND (tablename LIKE '%_line' - OR tablename LIKE '%_point' - OR tablename LIKE '%_polygon' - OR tablename LIKE '%_roads') + AND indexname LIKE '%%_pkey') + AND (tablename LIKE '%%_line' + OR tablename LIKE '%%_point' + OR tablename LIKE '%%_polygon' + OR tablename LIKE '%%_roads') LOOP EXECUTE 'DROP INDEX public.' || quote_ident(row.indexname) || ';'; END LOOP; @@ -89,13 +67,14 @@ DECLARE BEGIN FOR row IN SELECT indexname FROM pg_indexes WHERE (schemaname='public' - AND indexname LIKE '%_index') + AND indexname LIKE '%%_index') LOOP EXECUTE 'DROP INDEX public.' || quote_ident(row.indexname) || ';'; END LOOP; END; $$; + -- 2. add column 'gid' serial -- 3. add primary keys -- 4. rename geo-column 'way' to 'geom' @@ -106,10 +85,10 @@ DECLARE BEGIN FOR row IN SELECT tablename FROM pg_tables WHERE schemaname='public' - AND (tablename LIKE '%_line' - OR tablename LIKE '%_point' - OR tablename LIKE '%_polygon' - OR tablename LIKE '%_roads') + AND (tablename LIKE '%%_line' + OR tablename LIKE '%%_point' + OR tablename LIKE '%%_polygon' + OR tablename LIKE '%%_roads') LOOP EXECUTE 'ALTER TABLE public.' || quote_ident(row.tablename) || ' ADD gid SERIAL;'; EXECUTE 'ALTER TABLE public.' || quote_ident(row.tablename) || ' ADD PRIMARY KEY (gid);'; @@ -147,10 +126,10 @@ DECLARE BEGIN FOR row IN SELECT tablename FROM pg_tables WHERE schemaname='public' - AND (tablename LIKE '%_line' - OR tablename LIKE '%_point' - OR tablename LIKE '%_polygon' - OR tablename LIKE '%_roads') + AND (tablename LIKE '%%_line' + OR tablename LIKE '%%_point' + OR tablename LIKE '%%_polygon' + OR tablename LIKE '%%_roads') LOOP EXECUTE 'CREATE INDEX ' || quote_ident(row.tablename) || '_geom_idx ON public.' || quote_ident(row.tablename) || ' USING gist (geom);'; END LOOP; @@ -165,10 +144,10 @@ DECLARE BEGIN FOR row IN SELECT tablename FROM pg_tables WHERE schemaname='public' - AND (tablename LIKE '%_line' - OR tablename LIKE '%_point' - OR tablename LIKE '%_polygon' - OR tablename LIKE '%_roads') + AND (tablename LIKE '%%_line' + OR tablename LIKE '%%_point' + OR tablename LIKE '%%_polygon' + OR tablename LIKE '%%_roads') LOOP EXECUTE 'CREATE INDEX ' || quote_ident(row.tablename) || '_tags_idx ON public.' || quote_ident(row.tablename) || ' USING GIN (tags);'; END LOOP; @@ -183,7 +162,7 @@ $$ DECLARE row record; BEGIN - FOR row IN SELECT tablename FROM pg_tables WHERE schemaname = 'public' AND tablename LIKE 'osm_deu_%' + FOR row IN SELECT tablename FROM pg_tables WHERE schemaname = 'public' AND tablename LIKE 'osm_deu_%%' LOOP EXECUTE 'ALTER TABLE public.' || quote_ident(row.tablename) || ' SET SCHEMA openstreetmap;'; END LOOP; @@ -191,10 +170,10 @@ END; $$; -- ego scenario log (version,io,schema_name,table_name,script_name,comment) -SELECT ego_scenario_log('v0.2.10','preprocessing','openstreetmap','osm_deu_line','ego_pp_osm_deu_import.sql','setup osm tables'); -SELECT ego_scenario_log('v0.2.10','preprocessing','openstreetmap','osm_deu_nodes','ego_pp_osm_deu_import.sql','setup osm tables'); -SELECT ego_scenario_log('v0.2.10','preprocessing','openstreetmap','osm_deu_point','ego_pp_osm_deu_import.sql','setup osm tables'); -SELECT ego_scenario_log('v0.2.10','preprocessing','openstreetmap','osm_deu_polygon','ego_pp_osm_deu_import.sql','setup osm tables'); -SELECT ego_scenario_log('v0.2.10','preprocessing','openstreetmap','osm_deu_rels','ego_pp_osm_deu_import.sql','setup osm tables'); -SELECT ego_scenario_log('v0.2.10','preprocessing','openstreetmap','osm_deu_roads','ego_pp_osm_deu_import.sql','setup osm tables'); -SELECT ego_scenario_log('v0.2.10','preprocessing','openstreetmap','osm_deu_ways','ego_pp_osm_deu_import.sql','setup osm tables'); +SELECT scenario_log('eGo_PP','PP1','preprocessing','openstreetmap','osm_deu_line','ego_pp_osm_deu_import.sql','setup osm tables'); +SELECT scenario_log('eGo_PP','PP1','preprocessing','openstreetmap','osm_deu_nodes','ego_pp_osm_deu_import.sql','setup osm tables'); +SELECT scenario_log('eGo_PP','PP1','preprocessing','openstreetmap','osm_deu_point','ego_pp_osm_deu_import.sql','setup osm tables'); +SELECT scenario_log('eGo_PP','PP1','preprocessing','openstreetmap','osm_deu_polygon','ego_pp_osm_deu_import.sql','setup osm tables'); +SELECT scenario_log('eGo_PP','PP1','preprocessing','openstreetmap','osm_deu_rels','ego_pp_osm_deu_import.sql','setup osm tables'); +SELECT scenario_log('eGo_PP','PP1','preprocessing','openstreetmap','osm_deu_roads','ego_pp_osm_deu_import.sql','setup osm tables'); +SELECT scenario_log('eGo_PP','PP1','preprocessing','openstreetmap','osm_deu_ways','ego_pp_osm_deu_import.sql','setup osm tables'); From cae5d0f9f64ae0b751c1a51fdd288c2f0fa6b537 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guido=20Ple=C3=9Fmann?= Date: Sun, 22 Mar 2020 13:44:52 +0100 Subject: [PATCH 25/84] Add scripts creating mviews from OSM --- preprocessing/import.yml | 4 ++++ .../openstreetmap/ego_pp_osm_line_street_mview.sql | 4 ++-- .../openstreetmap/ego_pp_osm_polygon_building_mview.sql | 6 +++--- 3 files changed, 9 insertions(+), 5 deletions(-) rename preprocessing/{sql_snippets => }/openstreetmap/ego_pp_osm_line_street_mview.sql (93%) rename preprocessing/{sql_snippets => }/openstreetmap/ego_pp_osm_polygon_building_mview.sql (92%) diff --git a/preprocessing/import.yml b/preprocessing/import.yml index 036e78b0..2fad5067 100644 --- a/preprocessing/import.yml +++ b/preprocessing/import.yml @@ -10,3 +10,7 @@ openstreetmap: language: bash - script: ego_pp_osm_deu_import.sql language: SQL + - script: ego_pp_osm_line_street_mview.sql + language: SQL + - script: ego_pp_osm_polygon_building_mview.sql + language: SQL \ No newline at end of file diff --git a/preprocessing/sql_snippets/openstreetmap/ego_pp_osm_line_street_mview.sql b/preprocessing/openstreetmap/ego_pp_osm_line_street_mview.sql similarity index 93% rename from preprocessing/sql_snippets/openstreetmap/ego_pp_osm_line_street_mview.sql rename to preprocessing/openstreetmap/ego_pp_osm_line_street_mview.sql index daf8d37d..d1a9d7df 100644 --- a/preprocessing/sql_snippets/openstreetmap/ego_pp_osm_line_street_mview.sql +++ b/preprocessing/openstreetmap/ego_pp_osm_line_street_mview.sql @@ -9,7 +9,7 @@ __author__ = "jong42, Ludee" -- ego scenario log (version,io,schema_name,table_name,script_name,comment) -SELECT ego_scenario_log('v0.2.10','input','openstreetmap','osm_deu_line','ego_pp_osm_line_street_mview.sql',''); +SELECT scenario_log('eGo_PP','PP1','input','openstreetmap','osm_deu_line','ego_pp_osm_line_street_mview.sql','setup osm tables'); -- street DROP MATERIALIZED VIEW IF EXISTS openstreetmap.osm_deu_line_street_mview; @@ -84,4 +84,4 @@ COMMENT ON MATERIALIZED VIEW openstreetmap.osm_deu_line_street_mview IS '{ "metadata_version": "1.3"}'; -- ego scenario log (version,io,schema_name,table_name,script_name,comment) -SELECT ego_scenario_log('v0.2.10','output','openstreetmap','osm_deu_line_street_mview','ego_pp_osm_line_street_mview.sql','Streets'); +SELECT scenario_log('eGo_PP','PP1','output','openstreetmap','osm_deu_line_street_mview','ego_pp_osm_line_street_mview.sql','setup osm tables'); diff --git a/preprocessing/sql_snippets/openstreetmap/ego_pp_osm_polygon_building_mview.sql b/preprocessing/openstreetmap/ego_pp_osm_polygon_building_mview.sql similarity index 92% rename from preprocessing/sql_snippets/openstreetmap/ego_pp_osm_polygon_building_mview.sql rename to preprocessing/openstreetmap/ego_pp_osm_polygon_building_mview.sql index 70dd1dbe..1697e5e9 100644 --- a/preprocessing/sql_snippets/openstreetmap/ego_pp_osm_polygon_building_mview.sql +++ b/preprocessing/openstreetmap/ego_pp_osm_polygon_building_mview.sql @@ -1,4 +1,4 @@ -/* +/* Extracted OSM buildings from polygon __copyright__ = "Reiner Lemoine Institut" @@ -9,7 +9,7 @@ __author__ = "Ludee" -- ego scenario log (version,io,schema_name,table_name,script_name,comment) -SELECT ego_scenario_log('v0.2.10','input','openstreetmap','osm_deu_polygon','ego_pp_osm_polygon_building_mview.sql',''); +SELECT scenario_log('eGo_PP','PP1','input','openstreetmap','osm_deu_polygon','ego_pp_osm_polygon_building_mview.sql','setup osm tables'); -- building DROP MATERIALIZED VIEW IF EXISTS openstreetmap.osm_deu_polygon_building_mview; @@ -78,4 +78,4 @@ COMMENT ON MATERIALIZED VIEW openstreetmap.osm_deu_polygon_building_mview IS '{ "metadata_version": "1.3"}'; -- ego scenario log (version,io,schema_name,table_name,script_name,comment) -SELECT ego_scenario_log('v0.2.10','output','openstreetmap','osm_deu_polygon_building_mview','ego_pp_osm_polygon_building_mview.sql',''); +SELECT scenario_log('eGo_PP','PP1','output','openstreetmap','osm_deu_polygon_building_mview','ego_pp_osm_polygon_building_mview.sql','setup osm tables'); From 7f9c8a4c2533db9db257fe42ad4f7d12fbe41689 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guido=20Ple=C3=9Fmann?= Date: Sun, 22 Mar 2020 13:45:10 +0100 Subject: [PATCH 26/84] Fix typo in filename --- preprocessing/import.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/preprocessing/import.yml b/preprocessing/import.yml index 2fad5067..752793ba 100644 --- a/preprocessing/import.yml +++ b/preprocessing/import.yml @@ -1,9 +1,9 @@ scenario_log: - - script: 01_function_scenario_log.sql + - script: function_scenario_log.sql language: SQL - - script: 02_ego_pp_scenario.sql + - script: ego_pp_scenario.sql language: SQL - - script: 03_ego_pp_scenario_log.sql + - script: ego_pp_scenario_log.sql language: SQL openstreetmap: - script: osm2oedb.sh From 229346c00b3eea4cfaae5ac78bbeb9946693e9be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guido=20Ple=C3=9Fmann?= Date: Sun, 22 Mar 2020 15:01:00 +0100 Subject: [PATCH 27/84] Inlcude meta data writing in actual script --- .../openstreetmap/ego_pp_osm_deu_import.sql | 295 +++++++++++++++++ .../openstreetmap/ego_pp_osm_deu_metadata.sql | 302 ------------------ .../openstreetmap/osm_deu/osm_deu_nodes.sql | 63 ---- .../openstreetmap/osm_deu/osm_deu_point.sql | 63 ---- .../openstreetmap/osm_deu/osm_deu_polygon.sql | 63 ---- .../openstreetmap/osm_deu/osm_deu_rels.sql | 63 ---- .../openstreetmap/osm_deu/osm_deu_roads.sql | 63 ---- .../openstreetmap/osm_deu/osm_deu_ways.sql | 63 ---- 8 files changed, 295 insertions(+), 680 deletions(-) delete mode 100644 preprocessing/sql_snippets/openstreetmap/ego_pp_osm_deu_metadata.sql delete mode 100644 preprocessing/sql_snippets/openstreetmap/osm_deu/osm_deu_nodes.sql delete mode 100644 preprocessing/sql_snippets/openstreetmap/osm_deu/osm_deu_point.sql delete mode 100644 preprocessing/sql_snippets/openstreetmap/osm_deu/osm_deu_polygon.sql delete mode 100644 preprocessing/sql_snippets/openstreetmap/osm_deu/osm_deu_rels.sql delete mode 100644 preprocessing/sql_snippets/openstreetmap/osm_deu/osm_deu_roads.sql delete mode 100644 preprocessing/sql_snippets/openstreetmap/osm_deu/osm_deu_ways.sql diff --git a/preprocessing/openstreetmap/ego_pp_osm_deu_import.sql b/preprocessing/openstreetmap/ego_pp_osm_deu_import.sql index 0c450bec..f59ea8d6 100644 --- a/preprocessing/openstreetmap/ego_pp_osm_deu_import.sql +++ b/preprocessing/openstreetmap/ego_pp_osm_deu_import.sql @@ -177,3 +177,298 @@ SELECT scenario_log('eGo_PP','PP1','preprocessing','openstreetmap','osm_deu_poly SELECT scenario_log('eGo_PP','PP1','preprocessing','openstreetmap','osm_deu_rels','ego_pp_osm_deu_import.sql','setup osm tables'); SELECT scenario_log('eGo_PP','PP1','preprocessing','openstreetmap','osm_deu_roads','ego_pp_osm_deu_import.sql','setup osm tables'); SELECT scenario_log('eGo_PP','PP1','preprocessing','openstreetmap','osm_deu_ways','ego_pp_osm_deu_import.sql','setup osm tables'); + + + +-- metadata +COMMENT ON TABLE openstreetmap.osm_deu_line IS '{ + "title": "OpenStreetMap (OSM) - Germany - Line", + "description": "OpenStreetMap is a free, editable map of the whole world that is being built by volunteers largely from scratch and released with an open-content license.", + "language": [ "eng", "ger" ], + "spatial": + {"location": "none", + "extent": "Germany", + "resolution": "vector"}, + "temporal": + {"reference_date": "2016-10-01", + "start": "none", + "end": "none", + "resolution": "none"}, + "sources": [ + {"name": "Geofabrik - Download - OpenStreetMap Data Extracts", + "description": "", + "url": "http://download.geofabrik.de/europe/germany.html", + "license": "Open Data Commons Open Database License 1.0 (ODbL-1.0)", + "copyright": "© OpenStreetMap contributors"} ], + "license": + {"id": "ODbL-1.0", + "name": "Open Data Commons Open Database License 1.0", + "version": "1.0", + "url": "http://www.openstreetmap.org/copyright/en", + "instruction": "You are free: To Share, To Create, To Adapt; As long as you: Attribute, Share-Alike, Keep open!", + "copyright": "© OpenStreetMap contributors"}, + "contributors": [ + {"name": "Martin Glauer", "email": "", "date": "2016-10-10", "comment": "Create table with osm2pgsql"}, + {"name": "Ludee", "email": "", "date": "2016-10-11", "comment": "Execute setup"}, + {"name": "Ludee", "email": "", "date": "2017-06-30", "comment": "Update metadata to v1.3"} ], + "resources": [ + {"name": "openstreetmap.osm_deu_line", + "format": "PostgreSQL", + "fields": [ + {"name": "gid", "description": "Unique identifier", "unit": "none" }, + {"name": "osm_id", "description": "OSM identifier", "unit": "none" }, + {"name": "tags", "description": "A tag consists of two items, a key and a value. Tags describe specific features of map elements (nodes, ways, or relations) or changesets. Both items are free format text fields, but often represent numeric or other structured items. Conventions are agreed on the meaning and use of tags, which are captured on this wiki.", "url": "http://wiki.openstreetmap.org/wiki/Tags", "unit": "none" }, + {"name": "geom", "description": "Geometry", "unit": "" } ] } ], + "metadata_version": "1.3"}'; + +COMMENT ON TABLE openstreetmap.osm_deu_nodes IS '{ + "title": "OpenStreetMap (OSM) - Germany - Nodes", + "description": "OpenStreetMap is a free, editable map of the whole world that is being built by volunteers largely from scratch and released with an open-content license.", + "language": [ "eng", "ger" ], + "spatial": + {"location": "none", + "extent": "Germany", + "resolution": "none"}, + "temporal": + {"reference_date": "2016-10-01", + "start": "none", + "end": "none", + "resolution": "none"}, + "sources": [ + {"name": "Geofabrik - Download - OpenStreetMap Data Extracts", + "description": "", + "url": "http://download.geofabrik.de/europe/germany.html", + "license": "Open Data Commons Open Database License 1.0 (ODbL-1.0)", + "copyright": "© OpenStreetMap contributors"} ], + "license": + {"id": "ODbL-1.0", + "name": "Open Data Commons Open Database License 1.0", + "version": "1.0", + "url": "http://www.openstreetmap.org/copyright/en", + "instruction": "You are free: To Share, To Create, To Adapt; As long as you: Attribute, Share-Alike, Keep open!", + "copyright": "© OpenStreetMap contributors"}, + "contributors": [ + {"name": "Martin Glauer", "email": "", "date": "2016-10-10", "comment": "Create table with osm2pgsql"}, + {"name": "Ludee", "email": "", "date": "2016-10-11", "comment": "Execute setup"}, + {"name": "Ludee", "email": "", "date": "2017-06-30", "comment": "Update metadata to v1.3"} ], + "resources": [ + {"name": "openstreetmap.osm_deu_nodes", + "format": "PostgreSQL", + "fields": [ + {"name": "id", "description": "Unique identifier", "unit": "none" }, + {"name": "lat", "description": "Latitude", "unit": "none" }, + {"name": "lon", "description": "Longitutde", "unit": "none" }, + {"name": "tags", "description": "Tags", "unit": "" } ] } ], + "metadata_version": "1.3"}'; + +COMMENT ON TABLE openstreetmap.osm_deu_point IS '{ + "title": "OpenStreetMap (OSM) - Germany - Point", + "description": "OpenStreetMap is a free, editable map of the whole world that is being built by volunteers largely from scratch and released with an open-content license.", + "language": [ "eng", "ger" ], + "spatial": + {"location": "none", + "extent": "Germany", + "resolution": "none"}, + "temporal": + {"reference_date": "2016-10-01", + "start": "none", + "end": "none", + "resolution": "none"}, + "sources": [ + {"name": "Geofabrik - Download - OpenStreetMap Data Extracts", + "description": "", + "url": "http://download.geofabrik.de/europe/germany.html", + "license": "Open Data Commons Open Database License 1.0 (ODbL-1.0)", + "copyright": "© OpenStreetMap contributors"} ], + "license": + {"id": "ODbL-1.0", + "name": "Open Data Commons Open Database License 1.0", + "version": "1.0", + "url": "http://www.openstreetmap.org/copyright/en", + "instruction": "You are free: To Share, To Create, To Adapt; As long as you: Attribute, Share-Alike, Keep open!", + "copyright": "© OpenStreetMap contributors"}, + "contributors": [ + {"name": "Martin Glauer", "email": "", "date": "2016-10-10", "comment": "Create table with osm2pgsql"}, + {"name": "Ludee", "email": "", "date": "2016-10-11", "comment": "Execute setup"}, + {"name": "Ludee", "email": "", "date": "2017-06-30", "comment": "Update metadata to v1.3"} ], + "resources": [ + {"name": "openstreetmap.osm_deu_point", + "format": "PostgreSQL", + "fields": [ + {"name": "gid", "description": "Unique identifier", "unit": "none" }, + {"name": "osm_id", "description": "OSM identifier", "unit": "none" }, + {"name": "tags", "description": "A tag consists of two items, a key and a value. Tags describe specific features of map elements (nodes, ways, or relations) or changesets. Both items are free format text fields, but often represent numeric or other structured items. Conventions are agreed on the meaning and use of tags, which are captured on this wiki.", "url": "http://wiki.openstreetmap.org/wiki/Tags", "unit": "none" }, + {"name": "geom", "description": "Geometry", "unit": "" } ] } ], + "metadata_version": "1.3"}'; + +COMMENT ON TABLE openstreetmap.osm_deu_polygon IS '{ + "title": "OpenStreetMap (OSM) - Germany - Polygon", + "description": "OpenStreetMap is a free, editable map of the whole world that is being built by volunteers largely from scratch and released with an open-content license.", + "language": [ "eng", "ger" ], + "spatial": + {"location": "none", + "extent": "Germany", + "resolution": "none"}, + "temporal": + {"reference_date": "2016-10-01", + "start": "none", + "end": "none", + "resolution": "none"}, + "sources": [ + {"name": "Geofabrik - Download - OpenStreetMap Data Extracts", + "description": "", + "url": "http://download.geofabrik.de/europe/germany.html", + "license": "Open Data Commons Open Database License 1.0 (ODbL-1.0)", + "copyright": "© OpenStreetMap contributors"} ], + "license": + {"id": "ODbL-1.0", + "name": "Open Data Commons Open Database License 1.0", + "version": "1.0", + "url": "http://www.openstreetmap.org/copyright/en", + "instruction": "You are free: To Share, To Create, To Adapt; As long as you: Attribute, Share-Alike, Keep open!", + "copyright": "© OpenStreetMap contributors"}, + "contributors": [ + {"name": "Martin Glauer", "email": "", "date": "2016-10-10", "comment": "Create table with osm2pgsql"}, + {"name": "Ludee", "email": "", "date": "2016-10-11", "comment": "Execute setup"}, + {"name": "Ludee", "email": "", "date": "2017-06-30", "comment": "Update metadata to v1.3"} ], + "resources": [ + {"name": "openstreetmap.osm_deu_polygon", + "format": "PostgreSQL", + "fields": [ + {"name": "gid", "description": "Unique identifier", "unit": "none" }, + {"name": "osm_id", "description": "OSM identifier", "unit": "none" }, + {"name": "tags", "description": "A tag consists of two items, a key and a value. Tags describe specific features of map elements (nodes, ways, or relations) or changesets. Both items are free format text fields, but often represent numeric or other structured items. Conventions are agreed on the meaning and use of tags, which are captured on this wiki.", "url": "http://wiki.openstreetmap.org/wiki/Tags", "unit": "none" }, + {"name": "geom", "description": "Geometry", "unit": "" } ] } ], + "metadata_version": "1.3"}'; + +COMMENT ON TABLE openstreetmap.osm_deu_rels IS '{ + "title": "OpenStreetMap (OSM) - Germany - Rels", + "description": "OpenStreetMap is a free, editable map of the whole world that is being built by volunteers largely from scratch and released with an open-content license.", + "language": [ "eng", "ger" ], + "spatial": + {"location": "none", + "extent": "Germany", + "resolution": "none"}, + "temporal": + {"reference_date": "2016-10-01", + "start": "none", + "end": "none", + "resolution": "none"}, + "sources": [ + {"name": "Geofabrik - Download - OpenStreetMap Data Extracts", + "description": "", + "url": "http://download.geofabrik.de/europe/germany.html", + "license": "Open Data Commons Open Database License 1.0 (ODbL-1.0)", + "copyright": "© OpenStreetMap contributors"} ], + "license": + {"id": "ODbL-1.0", + "name": "Open Data Commons Open Database License 1.0", + "version": "1.0", + "url": "http://www.openstreetmap.org/copyright/en", + "instruction": "You are free: To Share, To Create, To Adapt; As long as you: Attribute, Share-Alike, Keep open!", + "copyright": "© OpenStreetMap contributors"}, + "contributors": [ + {"name": "Martin Glauer", "email": "", "date": "2016-10-10", "comment": "Create table with osm2pgsql"}, + {"name": "Ludee", "email": "", "date": "2016-10-11", "comment": "Execute setup"}, + {"name": "Ludee", "email": "", "date": "2017-06-30", "comment": "Update metadata to v1.3"} ], + "resources": [ + {"name": "openstreetmap.osm_deu_rels", + "format": "PostgreSQL", + "fields": [ + {"name": "id", "description": "Unique identifier", "unit": "none" }, + {"name": "way_off", "description": "", "unit": "none" }, + {"name": "rel_off", "description": "", "unit": "none" }, + {"name": "parts", "description": "", "unit": "none" }, + {"name": "members", "description": "", "unit": "none" }, + {"name": "tags", "description": "", "unit": "none" }, + {"name": "pending", "description": "", "unit": "" } ] } ], + "metadata_version": "1.3"}'; + +COMMENT ON TABLE openstreetmap.osm_deu_roads IS '{ + "title": "OpenStreetMap (OSM) - Germany - Roads", + "description": "OpenStreetMap is a free, editable map of the whole world that is being built by volunteers largely from scratch and released with an open-content license.", + "language": [ "eng", "ger" ], + "spatial": + {"location": "none", + "extent": "Germany", + "resolution": "none"}, + "temporal": + {"reference_date": "2016-10-01", + "start": "none", + "end": "none", + "resolution": "none"}, + "sources": [ + {"name": "Geofabrik - Download - OpenStreetMap Data Extracts", + "description": "", + "url": "http://download.geofabrik.de/europe/germany.html", + "license": "Open Data Commons Open Database License 1.0 (ODbL-1.0)", + "copyright": "© OpenStreetMap contributors"} ], + "license": + {"id": "ODbL-1.0", + "name": "Open Data Commons Open Database License 1.0", + "version": "1.0", + "url": "http://www.openstreetmap.org/copyright/en", + "instruction": "You are free: To Share, To Create, To Adapt; As long as you: Attribute, Share-Alike, Keep open!", + "copyright": "© OpenStreetMap contributors"}, + "contributors": [ + {"name": "Martin Glauer", "email": "", "date": "2016-10-10", "comment": "Create table with osm2pgsql"}, + {"name": "Ludee", "email": "", "date": "2016-10-11", "comment": "Execute setup"}, + {"name": "Ludee", "email": "", "date": "2017-06-30", "comment": "Update metadata to v1.3"} ], + "resources": [ + {"name": "openstreetmap.osm_deu_roads", + "format": "PostgreSQL", + "fields": [ + {"name": "gid", "description": "Unique identifier", "unit": "none" }, + {"name": "osm_id", "description": "OSM identifier", "unit": "none" }, + {"name": "tags", "description": "A tag consists of two items, a key and a value. Tags describe specific features of map elements (nodes, ways, or relations) or changesets. Both items are free format text fields, but often represent numeric or other structured items. Conventions are agreed on the meaning and use of tags, which are captured on this wiki.", "url": "http://wiki.openstreetmap.org/wiki/Tags", "unit": "none" }, + {"name": "geom", "description": "Geometry", "unit": "" } ] } ], + "metadata_version": "1.3"}'; + +COMMENT ON TABLE openstreetmap.osm_deu_ways IS '{ + "title": "OpenStreetMap (OSM) - Germany - Ways", + "description": "OpenStreetMap is a free, editable map of the whole world that is being built by volunteers largely from scratch and released with an open-content license.", + "language": [ "eng", "ger" ], + "spatial": + {"location": "none", + "extent": "Germany", + "resolution": "none"}, + "temporal": + {"reference_date": "2016-10-01", + "start": "none", + "end": "none", + "resolution": "none"}, + "sources": [ + {"name": "Geofabrik - Download - OpenStreetMap Data Extracts", + "description": "", + "url": "http://download.geofabrik.de/europe/germany.html", + "license": "Open Data Commons Open Database License 1.0 (ODbL-1.0)", + "copyright": "© OpenStreetMap contributors"} ], + "license": + {"id": "ODbL-1.0", + "name": "Open Data Commons Open Database License 1.0", + "version": "1.0", + "url": "http://www.openstreetmap.org/copyright/en", + "instruction": "You are free: To Share, To Create, To Adapt; As long as you: Attribute, Share-Alike, Keep open!", + "copyright": "© OpenStreetMap contributors"}, + "contributors": [ + {"name": "Martin Glauer", "email": "", "date": "2016-10-10", "comment": "Create table with osm2pgsql"}, + {"name": "Ludee", "email": "", "date": "2016-10-11", "comment": "Execute setup"}, + {"name": "Ludee", "email": "", "date": "2017-06-30", "comment": "Update metadata to v1.3"} ], + "resources": [ + {"name": "openstreetmap.osm_deu_ways", + "format": "PostgreSQL", + "fields": [ + {"name": "id", "description": "Unique identifier", "unit": "none" }, + {"name": "nodes", "description": "", "unit": "none" }, + {"name": "tags", "description": "A tag consists of two items, a key and a value. Tags describe specific features of map elements (nodes, ways, or relations) or changesets. Both items are free format text fields, but often represent numeric or other structured items. Conventions are agreed on the meaning and use of tags, which are captured on this wiki.", "url": "http://wiki.openstreetmap.org/wiki/Tags", "unit": "none" }, + {"name": "pending", "description": "", "unit": "" } ] } ], + "metadata_version": "1.3"}'; + +-- ego scenario log (version,io,schema_name,table_name,script_name,comment) +SELECT scenario_log('eGo_PP','PP1','preprocessing','openstreetmap','osm_deu_line','ego_pp_osm_deu_metadata.sql','metadata'); +SELECT scenario_log('eGo_PP','PP1','preprocessing','openstreetmap','osm_deu_nodes','ego_pp_osm_deu_metadata.sql','metadata'); +SELECT scenario_log('eGo_PP','PP1','preprocessing','openstreetmap','osm_deu_point','ego_pp_osm_deu_metadata.sql','metadata'); +SELECT scenario_log('eGo_PP','PP1','preprocessing','openstreetmap','osm_deu_polygon','ego_pp_osm_deu_metadata.sql','metadata'); +SELECT scenario_log('eGo_PP','PP1','preprocessing','openstreetmap','osm_deu_rels','ego_pp_osm_deu_metadata.sql','metadata'); +SELECT scenario_log('eGo_PP','PP1','preprocessing','openstreetmap','osm_deu_roads','ego_pp_osm_deu_metadata.sql','metadata'); +SELECT scenario_log('eGo_PP','PP1','preprocessing','openstreetmap','osm_deu_ways','ego_pp_osm_deu_metadata.sql','metadata'); \ No newline at end of file diff --git a/preprocessing/sql_snippets/openstreetmap/ego_pp_osm_deu_metadata.sql b/preprocessing/sql_snippets/openstreetmap/ego_pp_osm_deu_metadata.sql deleted file mode 100644 index 0332f8bc..00000000 --- a/preprocessing/sql_snippets/openstreetmap/ego_pp_osm_deu_metadata.sql +++ /dev/null @@ -1,302 +0,0 @@ -/* -Metadata for OpenStreetMap tables - -__copyright__ = "Reiner Lemoine Institut" -__license__ = "GNU Affero General Public License Version 3 (AGPL-3.0)" -__url__ = "https://github.com/openego/data_processing/blob/master/LICENSE" -__author__ = "Ludee" -*/ - - --- metadata -COMMENT ON TABLE openstreetmap.osm_deu_line IS '{ - "title": "OpenStreetMap (OSM) - Germany - Line", - "description": "OpenStreetMap is a free, editable map of the whole world that is being built by volunteers largely from scratch and released with an open-content license.", - "language": [ "eng", "ger" ], - "spatial": - {"location": "none", - "extent": "Germany", - "resolution": "vector"}, - "temporal": - {"reference_date": "2016-10-01", - "start": "none", - "end": "none", - "resolution": "none"}, - "sources": [ - {"name": "Geofabrik - Download - OpenStreetMap Data Extracts", - "description": "", - "url": "http://download.geofabrik.de/europe/germany.html", - "license": "Open Data Commons Open Database License 1.0 (ODbL-1.0)", - "copyright": "© OpenStreetMap contributors"} ], - "license": - {"id": "ODbL-1.0", - "name": "Open Data Commons Open Database License 1.0", - "version": "1.0", - "url": "http://www.openstreetmap.org/copyright/en", - "instruction": "You are free: To Share, To Create, To Adapt; As long as you: Attribute, Share-Alike, Keep open!", - "copyright": "© OpenStreetMap contributors"}, - "contributors": [ - {"name": "Martin Glauer", "email": "", "date": "2016-10-10", "comment": "Create table with osm2pgsql"}, - {"name": "Ludee", "email": "", "date": "2016-10-11", "comment": "Execute setup"}, - {"name": "Ludee", "email": "", "date": "2017-06-30", "comment": "Update metadata to v1.3"} ], - "resources": [ - {"name": "openstreetmap.osm_deu_line", - "format": "PostgreSQL", - "fields": [ - {"name": "gid", "description": "Unique identifier", "unit": "none" }, - {"name": "osm_id", "description": "OSM identifier", "unit": "none" }, - {"name": "tags", "description": "A tag consists of two items, a key and a value. Tags describe specific features of map elements (nodes, ways, or relations) or changesets. Both items are free format text fields, but often represent numeric or other structured items. Conventions are agreed on the meaning and use of tags, which are captured on this wiki.", "url": "http://wiki.openstreetmap.org/wiki/Tags", "unit": "none" }, - {"name": "geom", "description": "Geometry", "unit": "" } ] } ], - "metadata_version": "1.3"}'; - -COMMENT ON TABLE openstreetmap.osm_deu_nodes IS '{ - "title": "OpenStreetMap (OSM) - Germany - Nodes", - "description": "OpenStreetMap is a free, editable map of the whole world that is being built by volunteers largely from scratch and released with an open-content license.", - "language": [ "eng", "ger" ], - "spatial": - {"location": "none", - "extent": "Germany", - "resolution": "none"}, - "temporal": - {"reference_date": "2016-10-01", - "start": "none", - "end": "none", - "resolution": "none"}, - "sources": [ - {"name": "Geofabrik - Download - OpenStreetMap Data Extracts", - "description": "", - "url": "http://download.geofabrik.de/europe/germany.html", - "license": "Open Data Commons Open Database License 1.0 (ODbL-1.0)", - "copyright": "© OpenStreetMap contributors"} ], - "license": - {"id": "ODbL-1.0", - "name": "Open Data Commons Open Database License 1.0", - "version": "1.0", - "url": "http://www.openstreetmap.org/copyright/en", - "instruction": "You are free: To Share, To Create, To Adapt; As long as you: Attribute, Share-Alike, Keep open!", - "copyright": "© OpenStreetMap contributors"}, - "contributors": [ - {"name": "Martin Glauer", "email": "", "date": "2016-10-10", "comment": "Create table with osm2pgsql"}, - {"name": "Ludee", "email": "", "date": "2016-10-11", "comment": "Execute setup"}, - {"name": "Ludee", "email": "", "date": "2017-06-30", "comment": "Update metadata to v1.3"} ], - "resources": [ - {"name": "openstreetmap.osm_deu_nodes", - "format": "PostgreSQL", - "fields": [ - {"name": "id", "description": "Unique identifier", "unit": "none" }, - {"name": "lat", "description": "Latitude", "unit": "none" }, - {"name": "lon", "description": "Longitutde", "unit": "none" }, - {"name": "tags", "description": "Tags", "unit": "" } ] } ], - "metadata_version": "1.3"}'; - -COMMENT ON TABLE openstreetmap.osm_deu_point IS '{ - "title": "OpenStreetMap (OSM) - Germany - Point", - "description": "OpenStreetMap is a free, editable map of the whole world that is being built by volunteers largely from scratch and released with an open-content license.", - "language": [ "eng", "ger" ], - "spatial": - {"location": "none", - "extent": "Germany", - "resolution": "none"}, - "temporal": - {"reference_date": "2016-10-01", - "start": "none", - "end": "none", - "resolution": "none"}, - "sources": [ - {"name": "Geofabrik - Download - OpenStreetMap Data Extracts", - "description": "", - "url": "http://download.geofabrik.de/europe/germany.html", - "license": "Open Data Commons Open Database License 1.0 (ODbL-1.0)", - "copyright": "© OpenStreetMap contributors"} ], - "license": - {"id": "ODbL-1.0", - "name": "Open Data Commons Open Database License 1.0", - "version": "1.0", - "url": "http://www.openstreetmap.org/copyright/en", - "instruction": "You are free: To Share, To Create, To Adapt; As long as you: Attribute, Share-Alike, Keep open!", - "copyright": "© OpenStreetMap contributors"}, - "contributors": [ - {"name": "Martin Glauer", "email": "", "date": "2016-10-10", "comment": "Create table with osm2pgsql"}, - {"name": "Ludee", "email": "", "date": "2016-10-11", "comment": "Execute setup"}, - {"name": "Ludee", "email": "", "date": "2017-06-30", "comment": "Update metadata to v1.3"} ], - "resources": [ - {"name": "openstreetmap.osm_deu_point", - "format": "PostgreSQL", - "fields": [ - {"name": "gid", "description": "Unique identifier", "unit": "none" }, - {"name": "osm_id", "description": "OSM identifier", "unit": "none" }, - {"name": "tags", "description": "A tag consists of two items, a key and a value. Tags describe specific features of map elements (nodes, ways, or relations) or changesets. Both items are free format text fields, but often represent numeric or other structured items. Conventions are agreed on the meaning and use of tags, which are captured on this wiki.", "url": "http://wiki.openstreetmap.org/wiki/Tags", "unit": "none" }, - {"name": "geom", "description": "Geometry", "unit": "" } ] } ], - "metadata_version": "1.3"}'; - -COMMENT ON TABLE openstreetmap.osm_deu_polygon IS '{ - "title": "OpenStreetMap (OSM) - Germany - Polygon", - "description": "OpenStreetMap is a free, editable map of the whole world that is being built by volunteers largely from scratch and released with an open-content license.", - "language": [ "eng", "ger" ], - "spatial": - {"location": "none", - "extent": "Germany", - "resolution": "none"}, - "temporal": - {"reference_date": "2016-10-01", - "start": "none", - "end": "none", - "resolution": "none"}, - "sources": [ - {"name": "Geofabrik - Download - OpenStreetMap Data Extracts", - "description": "", - "url": "http://download.geofabrik.de/europe/germany.html", - "license": "Open Data Commons Open Database License 1.0 (ODbL-1.0)", - "copyright": "© OpenStreetMap contributors"} ], - "license": - {"id": "ODbL-1.0", - "name": "Open Data Commons Open Database License 1.0", - "version": "1.0", - "url": "http://www.openstreetmap.org/copyright/en", - "instruction": "You are free: To Share, To Create, To Adapt; As long as you: Attribute, Share-Alike, Keep open!", - "copyright": "© OpenStreetMap contributors"}, - "contributors": [ - {"name": "Martin Glauer", "email": "", "date": "2016-10-10", "comment": "Create table with osm2pgsql"}, - {"name": "Ludee", "email": "", "date": "2016-10-11", "comment": "Execute setup"}, - {"name": "Ludee", "email": "", "date": "2017-06-30", "comment": "Update metadata to v1.3"} ], - "resources": [ - {"name": "openstreetmap.osm_deu_polygon", - "format": "PostgreSQL", - "fields": [ - {"name": "gid", "description": "Unique identifier", "unit": "none" }, - {"name": "osm_id", "description": "OSM identifier", "unit": "none" }, - {"name": "tags", "description": "A tag consists of two items, a key and a value. Tags describe specific features of map elements (nodes, ways, or relations) or changesets. Both items are free format text fields, but often represent numeric or other structured items. Conventions are agreed on the meaning and use of tags, which are captured on this wiki.", "url": "http://wiki.openstreetmap.org/wiki/Tags", "unit": "none" }, - {"name": "geom", "description": "Geometry", "unit": "" } ] } ], - "metadata_version": "1.3"}'; - -COMMENT ON TABLE openstreetmap.osm_deu_rels IS '{ - "title": "OpenStreetMap (OSM) - Germany - Rels", - "description": "OpenStreetMap is a free, editable map of the whole world that is being built by volunteers largely from scratch and released with an open-content license.", - "language": [ "eng", "ger" ], - "spatial": - {"location": "none", - "extent": "Germany", - "resolution": "none"}, - "temporal": - {"reference_date": "2016-10-01", - "start": "none", - "end": "none", - "resolution": "none"}, - "sources": [ - {"name": "Geofabrik - Download - OpenStreetMap Data Extracts", - "description": "", - "url": "http://download.geofabrik.de/europe/germany.html", - "license": "Open Data Commons Open Database License 1.0 (ODbL-1.0)", - "copyright": "© OpenStreetMap contributors"} ], - "license": - {"id": "ODbL-1.0", - "name": "Open Data Commons Open Database License 1.0", - "version": "1.0", - "url": "http://www.openstreetmap.org/copyright/en", - "instruction": "You are free: To Share, To Create, To Adapt; As long as you: Attribute, Share-Alike, Keep open!", - "copyright": "© OpenStreetMap contributors"}, - "contributors": [ - {"name": "Martin Glauer", "email": "", "date": "2016-10-10", "comment": "Create table with osm2pgsql"}, - {"name": "Ludee", "email": "", "date": "2016-10-11", "comment": "Execute setup"}, - {"name": "Ludee", "email": "", "date": "2017-06-30", "comment": "Update metadata to v1.3"} ], - "resources": [ - {"name": "openstreetmap.osm_deu_rels", - "format": "PostgreSQL", - "fields": [ - {"name": "id", "description": "Unique identifier", "unit": "none" }, - {"name": "way_off", "description": "", "unit": "none" }, - {"name": "rel_off", "description": "", "unit": "none" }, - {"name": "parts", "description": "", "unit": "none" }, - {"name": "members", "description": "", "unit": "none" }, - {"name": "tags", "description": "", "unit": "none" }, - {"name": "pending", "description": "", "unit": "" } ] } ], - "metadata_version": "1.3"}'; - -COMMENT ON TABLE openstreetmap.osm_deu_roads IS '{ - "title": "OpenStreetMap (OSM) - Germany - Roads", - "description": "OpenStreetMap is a free, editable map of the whole world that is being built by volunteers largely from scratch and released with an open-content license.", - "language": [ "eng", "ger" ], - "spatial": - {"location": "none", - "extent": "Germany", - "resolution": "none"}, - "temporal": - {"reference_date": "2016-10-01", - "start": "none", - "end": "none", - "resolution": "none"}, - "sources": [ - {"name": "Geofabrik - Download - OpenStreetMap Data Extracts", - "description": "", - "url": "http://download.geofabrik.de/europe/germany.html", - "license": "Open Data Commons Open Database License 1.0 (ODbL-1.0)", - "copyright": "© OpenStreetMap contributors"} ], - "license": - {"id": "ODbL-1.0", - "name": "Open Data Commons Open Database License 1.0", - "version": "1.0", - "url": "http://www.openstreetmap.org/copyright/en", - "instruction": "You are free: To Share, To Create, To Adapt; As long as you: Attribute, Share-Alike, Keep open!", - "copyright": "© OpenStreetMap contributors"}, - "contributors": [ - {"name": "Martin Glauer", "email": "", "date": "2016-10-10", "comment": "Create table with osm2pgsql"}, - {"name": "Ludee", "email": "", "date": "2016-10-11", "comment": "Execute setup"}, - {"name": "Ludee", "email": "", "date": "2017-06-30", "comment": "Update metadata to v1.3"} ], - "resources": [ - {"name": "openstreetmap.osm_deu_roads", - "format": "PostgreSQL", - "fields": [ - {"name": "gid", "description": "Unique identifier", "unit": "none" }, - {"name": "osm_id", "description": "OSM identifier", "unit": "none" }, - {"name": "tags", "description": "A tag consists of two items, a key and a value. Tags describe specific features of map elements (nodes, ways, or relations) or changesets. Both items are free format text fields, but often represent numeric or other structured items. Conventions are agreed on the meaning and use of tags, which are captured on this wiki.", "url": "http://wiki.openstreetmap.org/wiki/Tags", "unit": "none" }, - {"name": "geom", "description": "Geometry", "unit": "" } ] } ], - "metadata_version": "1.3"}'; - -COMMENT ON TABLE openstreetmap.osm_deu_ways IS '{ - "title": "OpenStreetMap (OSM) - Germany - Ways", - "description": "OpenStreetMap is a free, editable map of the whole world that is being built by volunteers largely from scratch and released with an open-content license.", - "language": [ "eng", "ger" ], - "spatial": - {"location": "none", - "extent": "Germany", - "resolution": "none"}, - "temporal": - {"reference_date": "2016-10-01", - "start": "none", - "end": "none", - "resolution": "none"}, - "sources": [ - {"name": "Geofabrik - Download - OpenStreetMap Data Extracts", - "description": "", - "url": "http://download.geofabrik.de/europe/germany.html", - "license": "Open Data Commons Open Database License 1.0 (ODbL-1.0)", - "copyright": "© OpenStreetMap contributors"} ], - "license": - {"id": "ODbL-1.0", - "name": "Open Data Commons Open Database License 1.0", - "version": "1.0", - "url": "http://www.openstreetmap.org/copyright/en", - "instruction": "You are free: To Share, To Create, To Adapt; As long as you: Attribute, Share-Alike, Keep open!", - "copyright": "© OpenStreetMap contributors"}, - "contributors": [ - {"name": "Martin Glauer", "email": "", "date": "2016-10-10", "comment": "Create table with osm2pgsql"}, - {"name": "Ludee", "email": "", "date": "2016-10-11", "comment": "Execute setup"}, - {"name": "Ludee", "email": "", "date": "2017-06-30", "comment": "Update metadata to v1.3"} ], - "resources": [ - {"name": "openstreetmap.osm_deu_ways", - "format": "PostgreSQL", - "fields": [ - {"name": "id", "description": "Unique identifier", "unit": "none" }, - {"name": "nodes", "description": "", "unit": "none" }, - {"name": "tags", "description": "A tag consists of two items, a key and a value. Tags describe specific features of map elements (nodes, ways, or relations) or changesets. Both items are free format text fields, but often represent numeric or other structured items. Conventions are agreed on the meaning and use of tags, which are captured on this wiki.", "url": "http://wiki.openstreetmap.org/wiki/Tags", "unit": "none" }, - {"name": "pending", "description": "", "unit": "" } ] } ], - "metadata_version": "1.3"}'; - --- ego scenario log (version,io,schema_name,table_name,script_name,comment) -SELECT ego_scenario_log('v0.2.10','preprocessing','openstreetmap','osm_deu_line','ego_pp_osm_deu_metadata.sql','metadata'); -SELECT ego_scenario_log('v0.2.10','preprocessing','openstreetmap','osm_deu_nodes','ego_pp_osm_deu_metadata.sql','metadata'); -SELECT ego_scenario_log('v0.2.10','preprocessing','openstreetmap','osm_deu_point','ego_pp_osm_deu_metadata.sql','metadata'); -SELECT ego_scenario_log('v0.2.10','preprocessing','openstreetmap','osm_deu_polygon','ego_pp_osm_deu_metadata.sql','metadata'); -SELECT ego_scenario_log('v0.2.10','preprocessing','openstreetmap','osm_deu_rels','ego_pp_osm_deu_metadata.sql','metadata'); -SELECT ego_scenario_log('v0.2.10','preprocessing','openstreetmap','osm_deu_roads','ego_pp_osm_deu_metadata.sql','metadata'); -SELECT ego_scenario_log('v0.2.10','preprocessing','openstreetmap','osm_deu_ways','ego_pp_osm_deu_metadata.sql','metadata'); diff --git a/preprocessing/sql_snippets/openstreetmap/osm_deu/osm_deu_nodes.sql b/preprocessing/sql_snippets/openstreetmap/osm_deu/osm_deu_nodes.sql deleted file mode 100644 index fcf7a619..00000000 --- a/preprocessing/sql_snippets/openstreetmap/osm_deu/osm_deu_nodes.sql +++ /dev/null @@ -1,63 +0,0 @@ -/* -A description of the module (short but could be more than one line). -Modules names should have short, all-lowercase names. -The module name may have underscores if this improves readability. - -__copyright__ = "Copyright Reiner Lemoine Institut gGmbH" -__license__ = "GNU Affero General Public License Version 3 (AGPL-3.0)" -__url__ = "https://github.com/openego/data_processing/blob/master/LICENSE" -__author__ = "KilianZimmerer" -__contains__ = "url" -*/ - --- metadata -COMMENT ON TABLE openstreetmap.osm_deu_nodes IS '{ - "title": "OpenStreetMap - Germany", - "description": "OSM Datensatz Deutschland", - "language": [ "" ], - "spatial": { - "location": "", - "extend": "Germany", - "resolution": ""}, - "temporal": { - "reference_date": "2016-10-01", - "start": "", - "end": "", - "resolution": ""}, - "sources": [ - {"name": "Geofabrik - Download - OpenStreetMap Data Extracts", - "description": "", - "url": "http://download.geofabrik.de/europe/germany.html#", - "license": "", - "copyright": ""}], - "license": { - "id": "", - "name": "Open Data Commons Open Database Lizenz (ODbL)", - "version": "", - "url": "", - "instruction": "Wir verlangen die Verwendung des Hinweises OpenStreetMap-Mitwirkende. Du musst auch klarstellen, dass die Daten unter der Open-Database-Lizenz verf\u00fcgbar sind, und, sofern du unsere Kartenkacheln verwendest, dass die Kartografie gem\u00e4\u00df CC BY-SA lizenziert ist. Du kannst dies tun, indem du auf www.openstreetmap.org/copyright verlinkst. Ersatzweise, und als Erfordernis, falls du OSM in Datenform weitergibst, kannst du die Lizenz(en) direkt verlinken und benennen. In Medien, in denen keine Links m\u00f6glich sind (z.B. gedruckten Werken), empfehlen wir dir, deine Leser direkt auf openstreetmap.org zu verweisen (m\u00f6glicherweise mit dem Erweitern von OpenStreetMap zur vollen Adresse), auf opendatacommons.org, und, sofern zutreffend, auf creativecommons.org. Der Hinweis sollte f\u00fcr eine durchsuchbare elektronische Karte in der Ecke der Karte stehen.", - "copyright": ""}, - "contributors": [ - {"name": "Martin Glauer", - "email": " ", - "date": "2016-10-10", - "comment": "Created table with osm2pgsql"}, - {"name": "Ludwig H\u00fclk", - "email": "ludwig.huelk@rl-institut.de", - "date": "2016-10-11", - "comment": "Executed setup"}, - {"name": "Kilian Zimmerer", - "email": "", - "date": "2017-10-17", - "comment": "Update metadata to v1.3"}], - "resources": [ - {"name": "openstreetmap.osm_deu_nodes", - "format": "PostgreSQL", - "fields": [ - {"name": "osm_id", - "description": "OSM ID", - "unit": " "}, - {"name": "oedb.style", - "description": "Keys defined in this file", - "unit": " "}]}], - "metadata_version": "1.3"}'; diff --git a/preprocessing/sql_snippets/openstreetmap/osm_deu/osm_deu_point.sql b/preprocessing/sql_snippets/openstreetmap/osm_deu/osm_deu_point.sql deleted file mode 100644 index 01b8364a..00000000 --- a/preprocessing/sql_snippets/openstreetmap/osm_deu/osm_deu_point.sql +++ /dev/null @@ -1,63 +0,0 @@ -/* -A description of the module (short but could be more than one line). -Modules names should have short, all-lowercase names. -The module name may have underscores if this improves readability. - -__copyright__ = "Copyright Reiner Lemoine Institut gGmbH" -__license__ = "GNU Affero General Public License Version 3 (AGPL-3.0)" -__url__ = "https://github.com/openego/data_processing/blob/master/LICENSE" -__author__ = "KilianZimmerer" -__contains__ = "url" -*/ - --- metadata -COMMENT ON TABLE openstreetmap.osm_deu_point IS '{ - "title": "OpenStreetMap - Germany", - "description": "OSM Datensatz Deutschland", - "language": [ "" ], - "spatial": { - "location": "", - "extend": "Germany", - "resolution": ""}, - "temporal": { - "reference_date": "2016-10-01", - "start": "", - "end": "", - "resolution": ""}, - "sources": [ - {"name": "Geofabrik - Download - OpenStreetMap Data Extracts", - "description": "", - "url": "http://download.geofabrik.de/europe/germany.html#", - "license": "", - "copyright": ""}], - "license": { - "id": "", - "name": "Open Data Commons Open Database Lizenz (ODbL)", - "version": "", - "url": "", - "instruction": "Wir verlangen die Verwendung des Hinweises OpenStreetMap-Mitwirkende. Du musst auch klarstellen, dass die Daten unter der Open-Database-Lizenz verf\u00fcgbar sind, und, sofern du unsere Kartenkacheln verwendest, dass die Kartografie gem\u00e4\u00df CC BY-SA lizenziert ist. Du kannst dies tun, indem du auf www.openstreetmap.org/copyright verlinkst. Ersatzweise, und als Erfordernis, falls du OSM in Datenform weitergibst, kannst du die Lizenz(en) direkt verlinken und benennen. In Medien, in denen keine Links m\u00f6glich sind (z.B. gedruckten Werken), empfehlen wir dir, deine Leser direkt auf openstreetmap.org zu verweisen (m\u00f6glicherweise mit dem Erweitern von OpenStreetMap zur vollen Adresse), auf opendatacommons.org, und, sofern zutreffend, auf creativecommons.org. Der Hinweis sollte f\u00fcr eine durchsuchbare elektronische Karte in der Ecke der Karte stehen.", - "copyright": ""}, - "contributors": [ - {"name": "Martin Glauer", - "email": " ", - "date": "2016-10-10", - "comment": "Created table with osm2pgsql"}, - {"name": "Ludwig H\u00fclk", - "email": "ludwig.huelk@rl-institut.de", - "date": "2016-10-11", - "comment": "Executed setup"}, - {"name": "Kilian Zimmerer", - "email": "", - "date": "2017-10-17", - "comment": "Update metadata to v1.3"}], - "resources": [ - {"name": "openstreetmap.osm_deu_point", - "format": "PostgreSQL", - "fields": [ - {"name": "osm_id", - "description": "OSM ID", - "unit": " "}, - {"name": "oedb.style", - "description": "Keys defined in this file", - "unit": " "}]}], - "metadata_version": "1.3"}'; diff --git a/preprocessing/sql_snippets/openstreetmap/osm_deu/osm_deu_polygon.sql b/preprocessing/sql_snippets/openstreetmap/osm_deu/osm_deu_polygon.sql deleted file mode 100644 index cf40ba5b..00000000 --- a/preprocessing/sql_snippets/openstreetmap/osm_deu/osm_deu_polygon.sql +++ /dev/null @@ -1,63 +0,0 @@ -/* -A description of the module (short but could be more than one line). -Modules names should have short, all-lowercase names. -The module name may have underscores if this improves readability. - -__copyright__ = "Copyright Reiner Lemoine Institut gGmbH" -__license__ = "GNU Affero General Public License Version 3 (AGPL-3.0)" -__url__ = "https://github.com/openego/data_processing/blob/master/LICENSE" -__author__ = "KilianZimmerer" -__contains__ = "url" -*/ - --- metadata -COMMENT ON TABLE openstreetmap.osm_deu_polygon IS '{ - "title": "OpenStreetMap - Germany", - "description": "OSM Datensatz Deutschland", - "language": [ "" ], - "spatial": { - "location": "", - "extend": "Germany", - "resolution": ""}, - "temporal": { - "reference_date": "2016-10-01", - "start": "", - "end": "", - "resolution": ""}, - "sources": [ - {"name": "Geofabrik - Download - OpenStreetMap Data Extracts", - "description": "", - "url": "http://download.geofabrik.de/europe/germany.html#", - "license": "", - "copyright": ""}], - "license": { - "id": "", - "name": "Open Data Commons Open Database Lizenz (ODbL)", - "version": "", - "url": "", - "instruction": "Wir verlangen die Verwendung des Hinweises OpenStreetMap-Mitwirkende. Du musst auch klarstellen, dass die Daten unter der Open-Database-Lizenz verf\u00fcgbar sind, und, sofern du unsere Kartenkacheln verwendest, dass die Kartografie gem\u00e4\u00df CC BY-SA lizenziert ist. Du kannst dies tun, indem du auf www.openstreetmap.org/copyright verlinkst. Ersatzweise, und als Erfordernis, falls du OSM in Datenform weitergibst, kannst du die Lizenz(en) direkt verlinken und benennen. In Medien, in denen keine Links m\u00f6glich sind (z.B. gedruckten Werken), empfehlen wir dir, deine Leser direkt auf openstreetmap.org zu verweisen (m\u00f6glicherweise mit dem Erweitern von OpenStreetMap zur vollen Adresse), auf opendatacommons.org, und, sofern zutreffend, auf creativecommons.org. Der Hinweis sollte f\u00fcr eine durchsuchbare elektronische Karte in der Ecke der Karte stehen.", - "copyright": ""}, - "contributors": [ - {"name": "Martin Glauer", - "email": " ", - "date": "2016-10-10", - "comment": "Created table with osm2pgsql"}, - {"name": "Ludwig H\u00fclk", - "email": "ludwig.huelk@rl-institut.de", - "date": "2016-10-11", - "comment": "Executed setup"}, - {"name": "Kilian Zimmerer", - "email": "", - "date": "2017-10-17", - "comment": "Update metadata to v1.3"}], - "resources": [ - {"name": "openstreetmap.osm_deu_polygon", - "format": "PostgreSQL", - "fields": [ - {"name": "osm_id", - "description": "OSM ID", - "unit": " "}, - {"name": "oedb.style", - "description": "Keys defined in this file", - "unit": " "}]}], - "metadata_version": "1.3"}'; diff --git a/preprocessing/sql_snippets/openstreetmap/osm_deu/osm_deu_rels.sql b/preprocessing/sql_snippets/openstreetmap/osm_deu/osm_deu_rels.sql deleted file mode 100644 index d9d76176..00000000 --- a/preprocessing/sql_snippets/openstreetmap/osm_deu/osm_deu_rels.sql +++ /dev/null @@ -1,63 +0,0 @@ -/* -A description of the module (short but could be more than one line). -Modules names should have short, all-lowercase names. -The module name may have underscores if this improves readability. - -__copyright__ = "Copyright Reiner Lemoine Institut gGmbH" -__license__ = "GNU Affero General Public License Version 3 (AGPL-3.0)" -__url__ = "https://github.com/openego/data_processing/blob/master/LICENSE" -__author__ = "KilianZimmerer" -__contains__ = "url" -*/ - --- metadata -COMMENT ON TABLE openstreetmap.osm_deu_rels IS '{ - "title": "OpenStreetMap - Germany", - "description": "OSM Datensatz Deutschland", - "language": [ "" ], - "spatial": { - "location": "", - "extend": "Germany", - "resolution": ""}, - "temporal": { - "reference_date": "2016-10-01", - "start": "", - "end": "", - "resolution": ""}, - "sources": [ - {"name": "Geofabrik - Download - OpenStreetMap Data Extracts", - "description": "", - "url": "http://download.geofabrik.de/europe/germany.html#", - "license": "", - "copyright": ""}], - "license": { - "id": "", - "name": "Open Data Commons Open Database Lizenz (ODbL)", - "version": "", - "url": "", - "instruction": "Wir verlangen die Verwendung des Hinweises OpenStreetMap-Mitwirkende. Du musst auch klarstellen, dass die Daten unter der Open-Database-Lizenz verf\u00fcgbar sind, und, sofern du unsere Kartenkacheln verwendest, dass die Kartografie gem\u00e4\u00df CC BY-SA lizenziert ist. Du kannst dies tun, indem du auf www.openstreetmap.org/copyright verlinkst. Ersatzweise, und als Erfordernis, falls du OSM in Datenform weitergibst, kannst du die Lizenz(en) direkt verlinken und benennen. In Medien, in denen keine Links m\u00f6glich sind (z.B. gedruckten Werken), empfehlen wir dir, deine Leser direkt auf openstreetmap.org zu verweisen (m\u00f6glicherweise mit dem Erweitern von OpenStreetMap zur vollen Adresse), auf opendatacommons.org, und, sofern zutreffend, auf creativecommons.org. Der Hinweis sollte f\u00fcr eine durchsuchbare elektronische Karte in der Ecke der Karte stehen.", - "copyright": ""}, - "contributors": [ - {"name": "Martin Glauer", - "email": " ", - "date": "2016-10-10", - "comment": "Created table with osm2pgsql"}, - {"name": "Ludwig H\u00fclk", - "email": "ludwig.huelk@rl-institut.de", - "date": "2016-10-11", - "comment": "Executed setup"}, - {"name": "Kilian Zimmerer", - "email": "", - "date": "2017-10-17", - "comment": "Update metadata to v1.3"}], - "resources": [ - {"name": "openstreetmap.osm_deu_rels", - "format": "PostgreSQL", - "fields": [ - {"name": "osm_id", - "description": "OSM ID", - "unit": " "}, - {"name": "oedb.style", - "description": "Keys defined in this file", - "unit": " "}]}], - "metadata_version": "1.3"}'; diff --git a/preprocessing/sql_snippets/openstreetmap/osm_deu/osm_deu_roads.sql b/preprocessing/sql_snippets/openstreetmap/osm_deu/osm_deu_roads.sql deleted file mode 100644 index de7245da..00000000 --- a/preprocessing/sql_snippets/openstreetmap/osm_deu/osm_deu_roads.sql +++ /dev/null @@ -1,63 +0,0 @@ -/* -A description of the module (short but could be more than one line). -Modules names should have short, all-lowercase names. -The module name may have underscores if this improves readability. - -__copyright__ = "Copyright Reiner Lemoine Institut gGmbH" -__license__ = "GNU Affero General Public License Version 3 (AGPL-3.0)" -__url__ = "https://github.com/openego/data_processing/blob/master/LICENSE" -__author__ = "KilianZimmerer" -__contains__ = "url" -*/ - --- metadata -COMMENT ON TABLE openstreetmap.osm_deu_roads IS '{ - "title": "OpenStreetMap - Germany", - "description": "OSM Datensatz Deutschland", - "language": [ "" ], - "spatial": { - "location": "", - "extend": "Germany", - "resolution": ""}, - "temporal": { - "reference_date": "2016-10-01", - "start": "", - "end": "", - "resolution": ""}, - "sources": [ - {"name": "Geofabrik - Download - OpenStreetMap Data Extracts", - "description": "", - "url": "http://download.geofabrik.de/europe/germany.html#", - "license": "", - "copyright": ""}], - "license": { - "id": "", - "name": "Open Data Commons Open Database Lizenz (ODbL)", - "version": "", - "url": "", - "instruction": "Wir verlangen die Verwendung des Hinweises OpenStreetMap-Mitwirkende. Du musst auch klarstellen, dass die Daten unter der Open-Database-Lizenz verf\u00fcgbar sind, und, sofern du unsere Kartenkacheln verwendest, dass die Kartografie gem\u00e4\u00df CC BY-SA lizenziert ist. Du kannst dies tun, indem du auf www.openstreetmap.org/copyright verlinkst. Ersatzweise, und als Erfordernis, falls du OSM in Datenform weitergibst, kannst du die Lizenz(en) direkt verlinken und benennen. In Medien, in denen keine Links m\u00f6glich sind (z.B. gedruckten Werken), empfehlen wir dir, deine Leser direkt auf openstreetmap.org zu verweisen (m\u00f6glicherweise mit dem Erweitern von OpenStreetMap zur vollen Adresse), auf opendatacommons.org, und, sofern zutreffend, auf creativecommons.org. Der Hinweis sollte f\u00fcr eine durchsuchbare elektronische Karte in der Ecke der Karte stehen.", - "copyright": ""}, - "contributors": [ - {"name": "Martin Glauer", - "email": " ", - "date": "2016-10-10", - "comment": "Created table with osm2pgsql"}, - {"name": "Ludwig H\u00fclk", - "email": "ludwig.huelk@rl-institut.de", - "date": "2016-10-11", - "comment": "Executed setup"}, - {"name": "Kilian Zimmerer", - "email": "", - "date": "2017-10-17", - "comment": "Update metadata to v1.3"}], - "resources": [ - {"name": "openstreetmap.osm_deu_roads", - "format": "PostgreSQL", - "fields": [ - {"name": "osm_id", - "description": "OSM ID", - "unit": " "}, - {"name": "oedb.style", - "description": "Keys defined in this file", - "unit": " "}]}], - "metadata_version": "1.3"}'; diff --git a/preprocessing/sql_snippets/openstreetmap/osm_deu/osm_deu_ways.sql b/preprocessing/sql_snippets/openstreetmap/osm_deu/osm_deu_ways.sql deleted file mode 100644 index 446152fd..00000000 --- a/preprocessing/sql_snippets/openstreetmap/osm_deu/osm_deu_ways.sql +++ /dev/null @@ -1,63 +0,0 @@ -/* -A description of the module (short but could be more than one line). -Modules names should have short, all-lowercase names. -The module name may have underscores if this improves readability. - -__copyright__ = "Copyright Reiner Lemoine Institut gGmbH" -__license__ = "GNU Affero General Public License Version 3 (AGPL-3.0)" -__url__ = "https://github.com/openego/data_processing/blob/master/LICENSE" -__author__ = "KilianZimmerer" -__contains__ = "url" -*/ - --- metadata -COMMENT ON TABLE openstreetmap.osm_deu_ways IS '{ - "title": "OpenStreetMap - Germany", - "description": "OSM Datensatz Deutschland", - "language": [ "" ], - "spatial": { - "location": "", - "extend": "Germany", - "resolution": ""}, - "temporal": { - "reference_date": "2016-10-01", - "start": "", - "end": "", - "resolution": ""}, - "sources": [ - {"name": "Geofabrik - Download - OpenStreetMap Data Extracts", - "description": "", - "url": "http://download.geofabrik.de/europe/germany.html#", - "license": "", - "copyright": ""}], - "license": { - "id": "", - "name": "Open Data Commons Open Database Lizenz (ODbL)", - "version": "", - "url": "", - "instruction": "Wir verlangen die Verwendung des Hinweises OpenStreetMap-Mitwirkende. Du musst auch klarstellen, dass die Daten unter der Open-Database-Lizenz verf\u00fcgbar sind, und, sofern du unsere Kartenkacheln verwendest, dass die Kartografie gem\u00e4\u00df CC BY-SA lizenziert ist. Du kannst dies tun, indem du auf www.openstreetmap.org/copyright verlinkst. Ersatzweise, und als Erfordernis, falls du OSM in Datenform weitergibst, kannst du die Lizenz(en) direkt verlinken und benennen. In Medien, in denen keine Links m\u00f6glich sind (z.B. gedruckten Werken), empfehlen wir dir, deine Leser direkt auf openstreetmap.org zu verweisen (m\u00f6glicherweise mit dem Erweitern von OpenStreetMap zur vollen Adresse), auf opendatacommons.org, und, sofern zutreffend, auf creativecommons.org. Der Hinweis sollte f\u00fcr eine durchsuchbare elektronische Karte in der Ecke der Karte stehen.", - "copyright": ""}, - "contributors": [ - {"name": "Martin Glauer", - "email": " ", - "date": "2016-10-10", - "comment": "Created table with osm2pgsql"}, - {"name": "Ludwig H\u00fclk", - "email": "ludwig.huelk@rl-institut.de", - "date": "2016-10-11", - "comment": "Executed setup"}, - {"name": "Kilian Zimmerer", - "email": "", - "date": "2017-10-17", - "comment": "Update metadata to v1.3"}], - "resources": [ - {"name": "openstreetmap.osm_deu_ways", - "format": "PostgreSQL", - "fields": [ - {"name": "osm_id", - "description": "OSM ID", - "unit": " "}, - {"name": "oedb.style", - "description": "Keys defined in this file", - "unit": " "}]}], - "metadata_version": "1.3"}'; From 7ddeebe5ad2c706299cb74c9fb14629af0111fc6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guido=20Ple=C3=9Fmann?= Date: Sun, 22 Mar 2020 15:01:32 +0100 Subject: [PATCH 28/84] Add automatic file download in to local cached data directory --- preprocessing/eGo_import_and_preprocessing.py | 54 ++++++++++++++----- preprocessing/import.yml | 33 +++++++----- preprocessing/openstreetmap/osm2oedb.sh | 2 +- 3 files changed, 61 insertions(+), 28 deletions(-) diff --git a/preprocessing/eGo_import_and_preprocessing.py b/preprocessing/eGo_import_and_preprocessing.py index 5342cc2d..461eda6b 100644 --- a/preprocessing/eGo_import_and_preprocessing.py +++ b/preprocessing/eGo_import_and_preprocessing.py @@ -25,26 +25,51 @@ from egoio.tools import db from sqlalchemy import create_engine import yaml - +from urllib.request import urlretrieve + +# Configure logging +logger = logging.getLogger('EEEE') +logger.setLevel(logging.INFO) +ch = logging.StreamHandler() +ch.setLevel(logging.INFO) +formatter = logging.Formatter('%(asctime)s %(message)s', + datefmt='%Y-%m-%d %I:%M:%S') +ch.setFormatter(formatter) +logger.addHandler(ch) SCENARIOLOG = True +DOWNLOADDIR = os.path.join(os.path.expanduser("~"), ".egon-pre-processing-cached/") + + +def create_data_dir(dir=DOWNLOADDIR): + """ + Parameters + ---------- + dir: str + Define alternative dir for chached data + """ + + os.makedirs(dir, exist_ok=True) + logger.info('Cached data directory {} created (or already existed).'.format(dir)) + + +def download_data(url, filename): + + file = os.path.join(DOWNLOADDIR, filename) + + if not os.path.isfile(file): + urlretrieve(url, file) def preprocessing(): - # Configure logging - logger = logging.getLogger('EEEE') - logger.setLevel(logging.INFO) - ch = logging.StreamHandler() - ch.setLevel(logging.INFO) - formatter = logging.Formatter('%(asctime)s %(message)s', - datefmt='%Y-%m-%d %I:%M:%S') - ch.setFormatter(formatter) - logger.addHandler(ch) # get current time and inform about start total_time = time.time() logger.info('ego preprocessing started...') + # create directory for cached downloaded data + create_data_dir() + # list of sql- and python-snippets that process the data in correct order script_dir = os.path.abspath( os.path.join(os.path.dirname(__file__))) @@ -60,9 +85,12 @@ def preprocessing(): engine_local = create_engine('postgresql+psycopg2://oeuser:egon@localhost:54321/dp') conn = engine_local.connect() - # iterate over list of sql- and python-snippets and execute them + # iterate over data sets for key, dataset in datasets.items(): - for script in dataset: + for download in dataset.get("required_data", []): + download_data(download["url"], download["filename"]) + + for script in dataset.get("scripts", []): # timing and logging snippet_time = time.time() @@ -78,7 +106,7 @@ def preprocessing(): # execute desired sql snippet conn.execution_options(autocommit=True).execute(snippet_str) elif script["language"] == 'python': - filename = os.path.join(script_dir, snippet) + filename = os.path.join(script_dir, key, script["script"]) script_str = open(filename, "rb").read() # execute desired sql snippet diff --git a/preprocessing/import.yml b/preprocessing/import.yml index 752793ba..cba7f223 100644 --- a/preprocessing/import.yml +++ b/preprocessing/import.yml @@ -1,16 +1,21 @@ scenario_log: - - script: function_scenario_log.sql - language: SQL - - script: ego_pp_scenario.sql - language: SQL - - script: ego_pp_scenario_log.sql - language: SQL + scripts: + - script: function_scenario_log.sql + language: SQL + - script: ego_pp_scenario.sql + language: SQL + - script: ego_pp_scenario_log.sql + language: SQL openstreetmap: - - script: osm2oedb.sh - language: bash - - script: ego_pp_osm_deu_import.sql - language: SQL - - script: ego_pp_osm_line_street_mview.sql - language: SQL - - script: ego_pp_osm_polygon_building_mview.sql - language: SQL \ No newline at end of file + required_data: + - url: https://download.geofabrik.de/europe/germany/bremen-latest.osm.pbf + filename: bremen-latest.osm.pbf + scripts: + - script: osm2oedb.sh + language: bash + - script: ego_pp_osm_deu_import.sql + language: SQL + - script: ego_pp_osm_line_street_mview.sql + language: SQL + - script: ego_pp_osm_polygon_building_mview.sql + language: SQL \ No newline at end of file diff --git a/preprocessing/openstreetmap/osm2oedb.sh b/preprocessing/openstreetmap/osm2oedb.sh index 5768ed6d..02caf755 100644 --- a/preprocessing/openstreetmap/osm2oedb.sh +++ b/preprocessing/openstreetmap/osm2oedb.sh @@ -1,2 +1,2 @@ export PGPASSWORD=egon -osm2pgsql --create --slim --hstore-all --number-processes 2 --cache 4096 -H localhost -P 54321 -d dp -U oeuser -p osm_deu -S oedb.style bremen-latest.osm.pbf \ No newline at end of file +osm2pgsql --create --slim --hstore-all --number-processes 4 --cache 4096 -H localhost -P 54321 -d dp -U oeuser -p osm_deu -S oedb.style ~/.egon-pre-processing-cached/bremen-latest.osm.pbf \ No newline at end of file From 08bbb32a35625d5972268433f7ccc0adf9ef44b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guido=20Ple=C3=9Fmann?= Date: Sun, 22 Mar 2020 20:15:24 +0100 Subject: [PATCH 29/84] Add VG250 import --- preprocessing/eGo_import_and_preprocessing.py | 28 ++++++++--- preprocessing/import.yml | 50 ++++++++++++------- preprocessing/vg250/__init__.py | 0 .../vg250/create_boundaries_schema.sql | 1 + .../ego_pp_vg250_import.sql | 28 +++++------ .../ego_pp_vg250_metadata.sql | 12 ++--- .../ego_pp_vg250_mview.sql | 2 +- preprocessing/vg250/import_vg250.py | 27 ++++++++++ 8 files changed, 104 insertions(+), 44 deletions(-) create mode 100644 preprocessing/vg250/__init__.py create mode 100644 preprocessing/vg250/create_boundaries_schema.sql rename preprocessing/{sql_snippets/boundaries => vg250}/ego_pp_vg250_import.sql (89%) rename preprocessing/{sql_snippets/boundaries => vg250}/ego_pp_vg250_metadata.sql (98%) rename preprocessing/{sql_snippets/boundaries => vg250}/ego_pp_vg250_mview.sql (92%) create mode 100644 preprocessing/vg250/import_vg250.py diff --git a/preprocessing/eGo_import_and_preprocessing.py b/preprocessing/eGo_import_and_preprocessing.py index 461eda6b..5f475fdd 100644 --- a/preprocessing/eGo_import_and_preprocessing.py +++ b/preprocessing/eGo_import_and_preprocessing.py @@ -26,6 +26,7 @@ from sqlalchemy import create_engine import yaml from urllib.request import urlretrieve +import importlib # Configure logging logger = logging.getLogger('EEEE') @@ -40,6 +41,14 @@ SCENARIOLOG = True DOWNLOADDIR = os.path.join(os.path.expanduser("~"), ".egon-pre-processing-cached/") +db_info = { + "host": "localhost", + "port": 54321, + "user": "oeuser", + "password": "egon", + "database": "dp" +} + def create_data_dir(dir=DOWNLOADDIR): """ @@ -82,7 +91,8 @@ def preprocessing(): # get database connection conn_oedb = db.connection(readonly=True).connect() - engine_local = create_engine('postgresql+psycopg2://oeuser:egon@localhost:54321/dp') + # engine_local = create_engine('postgresql+psycopg2://oeuser:egon@localhost:54321/dp') + engine_local = create_engine('postgresql+psycopg2://{user}:{password}@{host}:{port}/{database}'.format(**db_info)) conn = engine_local.connect() # iterate over data sets @@ -106,11 +116,17 @@ def preprocessing(): # execute desired sql snippet conn.execution_options(autocommit=True).execute(snippet_str) elif script["language"] == 'python': - filename = os.path.join(script_dir, key, script["script"]) - script_str = open(filename, "rb").read() - - # execute desired sql snippet - exec(compile(script_str, filename, 'exec')) + if len(script["script"].split("::")) > 1: + module, func = script["script"].split("::") + mod = importlib.import_module(module) + func_to_call = getattr(mod, func) + func_to_call(script["filename"], DOWNLOADDIR, db_info=db_info) + else: + filename = os.path.join(script_dir, key, script["script"]) + script_str = open(filename, "rb").read() + + # execute desired sql snippet + exec(compile(script_str, filename, 'exec')) elif script["language"] == 'bash': filename = os.path.join(script_dir, key, script["script"]) script_str = open(filename, "rb").read() diff --git a/preprocessing/import.yml b/preprocessing/import.yml index cba7f223..7678ebc1 100644 --- a/preprocessing/import.yml +++ b/preprocessing/import.yml @@ -1,21 +1,37 @@ -scenario_log: - scripts: - - script: function_scenario_log.sql - language: SQL - - script: ego_pp_scenario.sql - language: SQL - - script: ego_pp_scenario_log.sql - language: SQL -openstreetmap: +#scenario_log: +# scripts: +# - script: function_scenario_log.sql +# language: SQL +# - script: ego_pp_scenario.sql +# language: SQL +# - script: ego_pp_scenario_log.sql +# language: SQL +#openstreetmap: +# required_data: +# - url: https://download.geofabrik.de/europe/germany/bremen-latest.osm.pbf +# filename: bremen-latest.osm.pbf +# scripts: +# - script: osm2oedb.sh +# language: bash +# - script: ego_pp_osm_deu_import.sql +# language: SQL +# - script: ego_pp_osm_line_street_mview.sql +# language: SQL +# - script: ego_pp_osm_polygon_building_mview.sql +# language: SQL +vg250: required_data: - - url: https://download.geofabrik.de/europe/germany/bremen-latest.osm.pbf - filename: bremen-latest.osm.pbf + - url: https://daten.gdz.bkg.bund.de/produkte/vg/vg250_ebenen_0101/aktuell/vg250_01-01.gk3.shape.ebenen.zip + filename: vg250_01-01.gk3.shape.ebenen.zip scripts: - - script: osm2oedb.sh - language: bash - - script: ego_pp_osm_deu_import.sql + - script: create_boundaries_schema.sql + language: SQL +# - script: preprocessing.vg250.import_vg250::import_vg250 +# language: python +# filename: vg250_01-01.gk3.shape.ebenen.zip +# - script: ego_pp_vg250_import.sql +# language: SQL + - script: ego_pp_vg250_metadata.sql language: SQL - - script: ego_pp_osm_line_street_mview.sql + - script: ego_pp_vg250_mview.sql language: SQL - - script: ego_pp_osm_polygon_building_mview.sql - language: SQL \ No newline at end of file diff --git a/preprocessing/vg250/__init__.py b/preprocessing/vg250/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/preprocessing/vg250/create_boundaries_schema.sql b/preprocessing/vg250/create_boundaries_schema.sql new file mode 100644 index 00000000..38a91000 --- /dev/null +++ b/preprocessing/vg250/create_boundaries_schema.sql @@ -0,0 +1 @@ +CREATE SCHEMA IF NOT EXISTS boundaries; \ No newline at end of file diff --git a/preprocessing/sql_snippets/boundaries/ego_pp_vg250_import.sql b/preprocessing/vg250/ego_pp_vg250_import.sql similarity index 89% rename from preprocessing/sql_snippets/boundaries/ego_pp_vg250_import.sql rename to preprocessing/vg250/ego_pp_vg250_import.sql index d0c4edc0..12e6fd5a 100644 --- a/preprocessing/sql_snippets/boundaries/ego_pp_vg250_import.sql +++ b/preprocessing/vg250/ego_pp_vg250_import.sql @@ -1,4 +1,4 @@ -/* +/* Import vg250 tables __copyright__ = "Reiner Lemoine Institut" @@ -51,7 +51,7 @@ $$ DECLARE vg250 record; BEGIN - FOR vg250 IN SELECT tablename FROM pg_tables WHERE schemaname = 'boundaries' AND tablename LIKE 'vg250_sta%' + FOR vg250 IN SELECT tablename FROM pg_tables WHERE schemaname = 'boundaries' AND tablename LIKE 'vg250_sta%%' LOOP EXECUTE E'INSERT INTO boundaries.bkg_vg250_1_sta SELECT to_date(SUBSTRING(' || quote_literal(vg250.tablename) || E', 11), \'YYYY-MM-DD\'), @@ -106,7 +106,7 @@ $$ DECLARE vg250 record; BEGIN - FOR vg250 IN SELECT tablename FROM pg_tables WHERE schemaname = 'boundaries' AND tablename LIKE 'vg250_lan%' + FOR vg250 IN SELECT tablename FROM pg_tables WHERE schemaname = 'boundaries' AND tablename LIKE 'vg250_lan%%' LOOP EXECUTE E'INSERT INTO boundaries.bkg_vg250_2_lan SELECT to_date(SUBSTRING(' || quote_literal(vg250.tablename) || E', 11), \'YYYY-MM-DD\'), @@ -161,7 +161,7 @@ $$ DECLARE vg250 record; BEGIN - FOR vg250 IN SELECT tablename FROM pg_tables WHERE schemaname = 'boundaries' AND tablename LIKE 'vg250_rbz%' + FOR vg250 IN SELECT tablename FROM pg_tables WHERE schemaname = 'boundaries' AND tablename LIKE 'vg250_rbz%%' LOOP EXECUTE E'INSERT INTO boundaries.bkg_vg250_3_rbz SELECT to_date(SUBSTRING(' || quote_literal(vg250.tablename) || E', 11), \'YYYY-MM-DD\'), @@ -216,7 +216,7 @@ $$ DECLARE vg250 record; BEGIN - FOR vg250 IN SELECT tablename FROM pg_tables WHERE schemaname = 'boundaries' AND tablename LIKE 'vg250_krs%' + FOR vg250 IN SELECT tablename FROM pg_tables WHERE schemaname = 'boundaries' AND tablename LIKE 'vg250_krs%%' LOOP EXECUTE E'INSERT INTO boundaries.bkg_vg250_4_krs SELECT to_date(SUBSTRING(' || quote_literal(vg250.tablename) || E', 11), \'YYYY-MM-DD\'), @@ -271,7 +271,7 @@ $$ DECLARE vg250 record; BEGIN - FOR vg250 IN SELECT tablename FROM pg_tables WHERE schemaname = 'boundaries' AND tablename LIKE 'vg250_vwg%' + FOR vg250 IN SELECT tablename FROM pg_tables WHERE schemaname = 'boundaries' AND tablename LIKE 'vg250_vwg%%' LOOP EXECUTE E'INSERT INTO boundaries.bkg_vg250_5_vwg SELECT to_date(SUBSTRING(' || quote_literal(vg250.tablename) || E', 11), \'YYYY-MM-DD\'), @@ -326,7 +326,7 @@ $$ DECLARE vg250 record; BEGIN - FOR vg250 IN SELECT tablename FROM pg_tables WHERE schemaname = 'boundaries' AND tablename LIKE 'vg250_gem%' + FOR vg250 IN SELECT tablename FROM pg_tables WHERE schemaname = 'boundaries' AND tablename LIKE 'vg250_gem%%' LOOP EXECUTE E'INSERT INTO boundaries.bkg_vg250_6_gem SELECT to_date(SUBSTRING(' || quote_literal(vg250.tablename) || E', 11), \'YYYY-MM-DD\'), @@ -340,12 +340,12 @@ $$; /* -- ego scenario log (version,io,schema_name,table_name,script_name,comment) -SELECT ego_scenario_log('v0.2.10','preprocessing','boundaries','bkg_vg250_1_sta','ego_pp_vg250_import.sql','setup vg250 tables'); -SELECT ego_scenario_log('v0.2.10','preprocessing','boundaries','bkg_vg250_2_lan','ego_pp_vg250_import.sql','setup vg250 tables'); -SELECT ego_scenario_log('v0.2.10','preprocessing','boundaries','bkg_vg250_3_rbz','ego_pp_vg250_import.sql','setup vg250 tables'); -SELECT ego_scenario_log('v0.2.10','preprocessing','boundaries','bkg_vg250_4_krs','ego_pp_vg250_import.sql','setup vg250 tables'); -SELECT ego_scenario_log('v0.2.10','preprocessing','boundaries','bkg_vg250_5_vwg','ego_pp_vg250_import.sql','setup vg250 tables'); -SELECT ego_scenario_log('v0.2.10','preprocessing','boundaries','bkg_vg250_6_gem','ego_pp_vg250_import.sql','setup vg250 tables'); +SELECT scenario_log('eGo_PP','PP1','preprocessing','boundaries','bkg_vg250_1_sta','ego_pp_vg250_import.sql','setup vg250 tables'); +SELECT scenario_log('eGo_PP','PP1','preprocessing','boundaries','bkg_vg250_2_lan','ego_pp_vg250_import.sql','setup vg250 tables'); +SELECT scenario_log('eGo_PP','PP1','preprocessing','boundaries','bkg_vg250_3_rbz','ego_pp_vg250_import.sql','setup vg250 tables'); +SELECT scenario_log('eGo_PP','PP1','preprocessing','boundaries','bkg_vg250_4_krs','ego_pp_vg250_import.sql','setup vg250 tables'); +SELECT scenario_log('eGo_PP','PP1','preprocessing','boundaries','bkg_vg250_5_vwg','ego_pp_vg250_import.sql','setup vg250 tables'); +SELECT scenario_log('eGo_PP','PP1','preprocessing','boundaries','bkg_vg250_6_gem','ego_pp_vg250_import.sql','setup vg250 tables'); */ -- drop tables @@ -354,7 +354,7 @@ $$ DECLARE vg250 record; BEGIN - FOR vg250 IN SELECT tablename FROM pg_tables WHERE schemaname = 'boundaries' AND tablename LIKE 'vg250_%' + FOR vg250 IN SELECT tablename FROM pg_tables WHERE schemaname = 'boundaries' AND tablename LIKE 'vg250_%%' LOOP EXECUTE E'DROP TABLE IF EXISTS boundaries.' || quote_ident(vg250.tablename) || ' ;'; END LOOP; diff --git a/preprocessing/sql_snippets/boundaries/ego_pp_vg250_metadata.sql b/preprocessing/vg250/ego_pp_vg250_metadata.sql similarity index 98% rename from preprocessing/sql_snippets/boundaries/ego_pp_vg250_metadata.sql rename to preprocessing/vg250/ego_pp_vg250_metadata.sql index 69c7286e..f6d77d50 100644 --- a/preprocessing/sql_snippets/boundaries/ego_pp_vg250_metadata.sql +++ b/preprocessing/vg250/ego_pp_vg250_metadata.sql @@ -455,9 +455,9 @@ COMMENT ON TABLE boundaries.bkg_vg250_6_gem IS '{ -- ego scenario log (version,io,schema_name,table_name,script_name,comment) -SELECT ego_scenario_log('v0.2.10','output','boundaries','bkg_vg250_1_sta','ego_pp_vg250_metadata.sql',''); -SELECT ego_scenario_log('v0.2.10','output','boundaries','bkg_vg250_2_lan','ego_pp_vg250_metadata.sql',''); -SELECT ego_scenario_log('v0.2.10','output','boundaries','bkg_vg250_3_rbz','ego_pp_vg250_metadata.sql',''); -SELECT ego_scenario_log('v0.2.10','output','boundaries','bkg_vg250_4_krs','ego_pp_vg250_metadata.sql',''); -SELECT ego_scenario_log('v0.2.10','output','boundaries','bkg_vg250_5_vwg','ego_pp_vg250_metadata.sql',''); -SELECT ego_scenario_log('v0.2.10','output','boundaries','bkg_vg250_6_gem','ego_pp_vg250_metadata.sql',''); +SELECT scenario_log('eGo_PP','PP1','output','boundaries','bkg_vg250_1_sta','ego_pp_vg250_metadata.sql',''); +SELECT scenario_log('eGo_PP','PP1','output','boundaries','bkg_vg250_2_lan','ego_pp_vg250_metadata.sql',''); +SELECT scenario_log('eGo_PP','PP1','output','boundaries','bkg_vg250_3_rbz','ego_pp_vg250_metadata.sql',''); +SELECT scenario_log('eGo_PP','PP1','output','boundaries','bkg_vg250_4_krs','ego_pp_vg250_metadata.sql',''); +SELECT scenario_log('eGo_PP','PP1','output','boundaries','bkg_vg250_5_vwg','ego_pp_vg250_metadata.sql',''); +SELECT scenario_log('eGo_PP','PP1','output','boundaries','bkg_vg250_6_gem','ego_pp_vg250_metadata.sql',''); diff --git a/preprocessing/sql_snippets/boundaries/ego_pp_vg250_mview.sql b/preprocessing/vg250/ego_pp_vg250_mview.sql similarity index 92% rename from preprocessing/sql_snippets/boundaries/ego_pp_vg250_mview.sql rename to preprocessing/vg250/ego_pp_vg250_mview.sql index 72017d1e..a5addf5f 100644 --- a/preprocessing/sql_snippets/boundaries/ego_pp_vg250_mview.sql +++ b/preprocessing/vg250/ego_pp_vg250_mview.sql @@ -1,4 +1,4 @@ --- Create MView with NUTS IDs +-- Create MView with NUTS IDs CREATE MATERIALIZED VIEW boundaries.bkg_vg250_2_lan_nuts_view AS SELECT lan.ags_0, diff --git a/preprocessing/vg250/import_vg250.py b/preprocessing/vg250/import_vg250.py new file mode 100644 index 00000000..18744ade --- /dev/null +++ b/preprocessing/vg250/import_vg250.py @@ -0,0 +1,27 @@ +import os +import zipfile +import subprocess + + +def certain_file_types(directory, extension): + return (f for f in os.listdir(directory) if f.endswith('.' + extension)) + + +def import_vg250(file, path, db_info, year=2019): + + zip = zipfile.ZipFile(os.path.join(path, file)) + zip.extractall(path=path) + file_stripped, _ = os.path.splitext(file) + file_splitted = file_stripped.split("_") + dir1 = file_splitted[0] + "_" + str(year) + "-" + "_".join(file_splitted[1:]) + vg250_shp_path = os.path.join(path, dir1, "vg250_ebenen") + table_date = "_" + "_".join(reversed(open(os.path.join(path, dir1, "aktualitaet.txt")).read().strip().split("."))) + + for f in certain_file_types(vg250_shp_path, "shp"): + table_base_name, ext = os.path.splitext(f) + tablename = table_base_name + table_date + ext + cmd_shp = 'shp2pgsql "' + os.path.join(vg250_shp_path, f) + '" boundaries.{}'.format(tablename) + cmd_pgql = ' | psql -h {host} -d {database} -U {user} -p {port}'.format(**db_info) + cmd = cmd_shp + cmd_pgql + + subprocess.run(cmd, shell=True, env={"PGPASSWORD": "egon"}) From d3e057eeeceb3b696afbcb27a85a4c5a6d83af2c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guido=20Ple=C3=9Fmann?= Date: Sun, 22 Mar 2020 20:19:14 +0100 Subject: [PATCH 30/84] Update list of data to process --- preprocessing/import.yml | 52 ++++++++++++++++++++-------------------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/preprocessing/import.yml b/preprocessing/import.yml index 7678ebc1..7d8e9eb4 100644 --- a/preprocessing/import.yml +++ b/preprocessing/import.yml @@ -1,24 +1,24 @@ -#scenario_log: -# scripts: -# - script: function_scenario_log.sql -# language: SQL -# - script: ego_pp_scenario.sql -# language: SQL -# - script: ego_pp_scenario_log.sql -# language: SQL -#openstreetmap: -# required_data: -# - url: https://download.geofabrik.de/europe/germany/bremen-latest.osm.pbf -# filename: bremen-latest.osm.pbf -# scripts: -# - script: osm2oedb.sh -# language: bash -# - script: ego_pp_osm_deu_import.sql -# language: SQL -# - script: ego_pp_osm_line_street_mview.sql -# language: SQL -# - script: ego_pp_osm_polygon_building_mview.sql -# language: SQL +scenario_log: + scripts: + - script: function_scenario_log.sql + language: SQL + - script: ego_pp_scenario.sql + language: SQL + - script: ego_pp_scenario_log.sql + language: SQL +openstreetmap: + required_data: + - url: https://download.geofabrik.de/europe/germany/bremen-latest.osm.pbf + filename: bremen-latest.osm.pbf + scripts: + - script: osm2oedb.sh + language: bash + - script: ego_pp_osm_deu_import.sql + language: SQL + - script: ego_pp_osm_line_street_mview.sql + language: SQL + - script: ego_pp_osm_polygon_building_mview.sql + language: SQL vg250: required_data: - url: https://daten.gdz.bkg.bund.de/produkte/vg/vg250_ebenen_0101/aktuell/vg250_01-01.gk3.shape.ebenen.zip @@ -26,11 +26,11 @@ vg250: scripts: - script: create_boundaries_schema.sql language: SQL -# - script: preprocessing.vg250.import_vg250::import_vg250 -# language: python -# filename: vg250_01-01.gk3.shape.ebenen.zip -# - script: ego_pp_vg250_import.sql -# language: SQL + - script: preprocessing.vg250.import_vg250::import_vg250 + language: python + filename: vg250_01-01.gk3.shape.ebenen.zip + - script: ego_pp_vg250_import.sql + language: SQL - script: ego_pp_vg250_metadata.sql language: SQL - script: ego_pp_vg250_mview.sql From 791a0384683a0505b90176db0f1a4e5b47b79327 Mon Sep 17 00:00:00 2001 From: IlkaCu Date: Mon, 23 Mar 2020 14:16:27 +0100 Subject: [PATCH 31/84] update powerflow section --- .../ego_dp_bpmn_section_powerflow.graphml | 42 +++++++++++++++---- 1 file changed, 33 insertions(+), 9 deletions(-) diff --git a/documentation/bpmn/ego_dp_bpmn_section_powerflow.graphml b/documentation/bpmn/ego_dp_bpmn_section_powerflow.graphml index 00cc2f62..6a84e0ec 100644 --- a/documentation/bpmn/ego_dp_bpmn_section_powerflow.graphml +++ b/documentation/bpmn/ego_dp_bpmn_section_powerflow.graphml @@ -1123,8 +1123,7 @@ model_draft.offshore_feedin_foreign - Preprocessing -ego_setup_renewable_feedin.sql + ego_setup_renewable_feedin.sql @@ -1336,10 +1335,10 @@ model_draft.ego_grid_mv_griddistrict model_draft.ego_demand_hvmv_demand + - @@ -1350,8 +1349,26 @@ model_draft.ego_grid_mv_griddistrict - model_draft.ego_demand_hv_largescaleconsumer -model_draft.ego_demand_loadarea + model_draft.ego_demand_hv_largescaleconsumer +model_draft.ego_demand_loadarea +BDEW SLP + + + + + + + + + + + + + + + + + BDEW SLP @@ -2249,7 +2266,6 @@ model_draft.ego_demand_loadarea - @@ -2260,7 +2276,6 @@ model_draft.ego_demand_loadarea - @@ -2271,7 +2286,6 @@ model_draft.ego_demand_loadarea - @@ -2282,7 +2296,6 @@ model_draft.ego_demand_loadarea - @@ -2292,6 +2305,17 @@ model_draft.ego_demand_loadarea + + + + + + + + + + From 26d76e6eae43f8c6d434d68d418b0bae9b3a17e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guido=20Ple=C3=9Fmann?= Date: Mon, 23 Mar 2020 19:49:01 +0100 Subject: [PATCH 32/84] Change box color back to green --- .../ego_dp-pre_bpmn_section_supply.graphml | 68 +++++++------------ 1 file changed, 23 insertions(+), 45 deletions(-) diff --git a/documentation/bpmn/ego_dp-pre_bpmn_section_supply.graphml b/documentation/bpmn/ego_dp-pre_bpmn_section_supply.graphml index e8c7a548..26c9bbd4 100644 --- a/documentation/bpmn/ego_dp-pre_bpmn_section_supply.graphml +++ b/documentation/bpmn/ego_dp-pre_bpmn_section_supply.graphml @@ -31,22 +31,22 @@ + + + - - - - - + + @@ -76,7 +76,7 @@ - + preprocessing/python_scripts/ renpass_gis/simple_feedin/simple_feedin.py @@ -216,10 +216,10 @@ renpass_gis/simple_feedin/simple_feedin.py model_draft.ego_dp_supply_res_powerplant + - @@ -250,10 +250,10 @@ renpass_gis/simple_feedin/simple_feedin.py model_draft.ego_supply_res_powerplant + - @@ -286,10 +286,10 @@ renpass_gis/simple_feedin/simple_feedin.py model_draft.ego_supply_conv_nep2035_temp + - @@ -320,10 +320,10 @@ model_draft.ego_supply_conv_nep2035_tempmodel_draft.ego_supply_conv_powerplant + - @@ -334,7 +334,7 @@ model_draft.ego_supply_conv_nep2035_temp - ego_dp_res_rea_by_scenario.sql + ego_dp_res_rea_by_scenario.sql @@ -356,10 +356,10 @@ model_draft.ego_supply_conv_nep2035_temp + - @@ -393,10 +393,10 @@ model_draft.ego_supply_scenario_capacities boundaries.bkg_vg250_6_gem_rs_mview + - @@ -407,7 +407,7 @@ boundaries.bkg_vg250_6_gem_rs_mview - ego_dp_preprocessing_res_powerplant_2035.sql + ego_dp_preprocessing_res_powerplant_2035.sql @@ -429,10 +429,10 @@ boundaries.bkg_vg250_6_gem_rs_mview + - @@ -464,10 +464,10 @@ boundaries.bkg_vg250_6_gem_rs_mview + - @@ -500,10 +500,10 @@ boundaries.bkg_vg250_6_gem_rs_mview + - @@ -514,7 +514,7 @@ boundaries.bkg_vg250_6_gem_rs_mview - Open Database License (ODbL) v1.0 + Open Database License (ODbL) v1.0 © OpenStreetMap contributors @@ -535,10 +535,10 @@ boundaries.bkg_vg250_6_gem_rs_mview + - @@ -571,10 +571,10 @@ boundaries.bkg_vg250_6_gem_rs_mview + - @@ -606,10 +606,10 @@ boundaries.bkg_vg250_6_gem_rs_mview + - @@ -643,10 +643,10 @@ model_draft.ego_dp_supply_res_powerplant + - @@ -680,10 +680,10 @@ coastdat.cosmoclmgrid + - @@ -727,7 +727,6 @@ coastdat.cosmoclmgrid - @@ -738,7 +737,6 @@ coastdat.cosmoclmgrid - @@ -749,7 +747,6 @@ coastdat.cosmoclmgrid - @@ -760,7 +757,6 @@ coastdat.cosmoclmgrid - @@ -771,7 +767,6 @@ coastdat.cosmoclmgrid - @@ -782,7 +777,6 @@ coastdat.cosmoclmgrid - @@ -793,7 +787,6 @@ coastdat.cosmoclmgrid - @@ -804,7 +797,6 @@ coastdat.cosmoclmgrid - @@ -815,7 +807,6 @@ coastdat.cosmoclmgrid - @@ -826,7 +817,6 @@ coastdat.cosmoclmgrid - @@ -837,7 +827,6 @@ coastdat.cosmoclmgrid - @@ -848,7 +837,6 @@ coastdat.cosmoclmgrid - @@ -859,7 +847,6 @@ coastdat.cosmoclmgrid - @@ -870,7 +857,6 @@ coastdat.cosmoclmgrid - @@ -881,7 +867,6 @@ coastdat.cosmoclmgrid - @@ -892,7 +877,6 @@ coastdat.cosmoclmgrid - @@ -903,7 +887,6 @@ coastdat.cosmoclmgrid - @@ -914,7 +897,6 @@ coastdat.cosmoclmgrid - @@ -925,7 +907,6 @@ coastdat.cosmoclmgrid - @@ -936,7 +917,6 @@ coastdat.cosmoclmgrid - @@ -947,7 +927,6 @@ coastdat.cosmoclmgrid - @@ -958,7 +937,6 @@ coastdat.cosmoclmgrid - From bda6542639575dbe08f29c1899dc969d88e63612 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guido=20Ple=C3=9Fmann?= Date: Fri, 27 Mar 2020 18:34:15 +0100 Subject: [PATCH 33/84] Pipe subprocess output to /dev/null --- preprocessing/vg250/import_vg250.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/preprocessing/vg250/import_vg250.py b/preprocessing/vg250/import_vg250.py index 18744ade..a379bd47 100644 --- a/preprocessing/vg250/import_vg250.py +++ b/preprocessing/vg250/import_vg250.py @@ -8,6 +8,7 @@ def certain_file_types(directory, extension): def import_vg250(file, path, db_info, year=2019): + FNULL = open(os.devnull, 'w') zip = zipfile.ZipFile(os.path.join(path, file)) zip.extractall(path=path) @@ -24,4 +25,4 @@ def import_vg250(file, path, db_info, year=2019): cmd_pgql = ' | psql -h {host} -d {database} -U {user} -p {port}'.format(**db_info) cmd = cmd_shp + cmd_pgql - subprocess.run(cmd, shell=True, env={"PGPASSWORD": "egon"}) + subprocess.run(cmd, shell=True, env={"PGPASSWORD": "egon"}, stdout=FNULL) From 834bbccf35e963f96e2a9106af95c0c9c1096161 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guido=20Ple=C3=9Fmann?= Date: Fri, 27 Mar 2020 18:35:12 +0100 Subject: [PATCH 34/84] Docker file changes * Increased shared memonry to 1GB * subdirectory for volume data --- docker-compose.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 2b856ed3..e4ae1841 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,4 +1,4 @@ -version: '3' +version: '3.5' services: data_processing_db: image: postgres:11-postgis @@ -7,6 +7,7 @@ services: build: context: . dockerfile: Dockerfile.postgis + shm_size: '1gb' ports: - "127.0.0.1:54321:5432" environment: @@ -14,5 +15,5 @@ services: POSTGRES_USER: oeuser POSTGRES_PASSWORD: egon volumes: - - $HOME/docker/volumes/postgres:/var/lib/postgresql/data + - $HOME/docker/volumes/postgres/data_processing:/var/lib/postgresql/data - ./utilities/entrypoints:/docker-entrypoint-initdb.d/ From 8fc539049249714d3e8b448847640bf6d33b3a0c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guido=20Ple=C3=9Fmann?= Date: Fri, 27 Mar 2020 18:36:34 +0100 Subject: [PATCH 35/84] Add zensus data import and zensus-post-vg250 data modifications --- preprocessing/eGo_import_and_preprocessing.py | 31 ++++- preprocessing/import.yml | 37 +++++- .../ego_dp_structure_boundaries_vg250.sql | 115 +++++++++++------- .../zensus/destatis_zensus_import_csv.py | 28 +++++ .../zensus/destatis_zensus_raw_import.sql | 57 +++++++++ .../ego_pp_destatis_zensus_import.sql | 9 +- .../ego_pp_destatis_zensus_insidevg250.sql | 3 +- .../ego_pp_destatis_zensus_metadata.sql | 2 +- 8 files changed, 226 insertions(+), 56 deletions(-) rename preprocessing/{sql_snippets => vg250}/ego_dp_structure_boundaries_vg250.sql (92%) create mode 100644 preprocessing/zensus/destatis_zensus_import_csv.py create mode 100644 preprocessing/zensus/destatis_zensus_raw_import.sql rename preprocessing/{sql_snippets/society => zensus}/ego_pp_destatis_zensus_import.sql (88%) rename preprocessing/{sql_snippets/society => zensus_vg250}/ego_pp_destatis_zensus_insidevg250.sql (98%) rename preprocessing/{sql_snippets/society => zensus_vg250}/ego_pp_destatis_zensus_metadata.sql (96%) diff --git a/preprocessing/eGo_import_and_preprocessing.py b/preprocessing/eGo_import_and_preprocessing.py index 5f475fdd..a792a6fc 100644 --- a/preprocessing/eGo_import_and_preprocessing.py +++ b/preprocessing/eGo_import_and_preprocessing.py @@ -40,6 +40,7 @@ SCENARIOLOG = True DOWNLOADDIR = os.path.join(os.path.expanduser("~"), ".egon-pre-processing-cached/") +FNULL = open(os.devnull, 'w') db_info = { "host": "localhost", @@ -98,7 +99,11 @@ def preprocessing(): # iterate over data sets for key, dataset in datasets.items(): for download in dataset.get("required_data", []): + logger.info("Downloading '{}' ...".format(script["required_data"])) + snippet_time = time.time() download_data(download["url"], download["filename"]) + logger.info('...successfully done in {:.2f} seconds.'.format( + time.time() - snippet_time)) for script in dataset.get("scripts", []): @@ -118,9 +123,19 @@ def preprocessing(): elif script["language"] == 'python': if len(script["script"].split("::")) > 1: module, func = script["script"].split("::") + + args = script.get("args", {}) + + if script["db"] == "info": + db_details = db_info + elif script["db"] == "conn": + db_details = engine_local + else: + "" + mod = importlib.import_module(module) func_to_call = getattr(mod, func) - func_to_call(script["filename"], DOWNLOADDIR, db_info=db_info) + func_to_call(script["filename"], DOWNLOADDIR, db_details, **args) else: filename = os.path.join(script_dir, key, script["script"]) script_str = open(filename, "rb").read() @@ -129,12 +144,18 @@ def preprocessing(): exec(compile(script_str, filename, 'exec')) elif script["language"] == 'bash': filename = os.path.join(script_dir, key, script["script"]) - script_str = open(filename, "rb").read() + script_str = script["script"] + + if "filename" in script: + script_str += script["filename"] # execute desired bash script - rc = subprocess.call(script_str, - cwd= os.path.join(script_dir, key), - shell=True) + # rc (script_str) + subprocess.run(script_str, + shell=True, + env={"PGPASSWORD": "egon"}, + stdout=FNULL, + cwd= os.path.join(script_dir, key)) else: raise NameError('{} is neither a python nor a sql script (at least it ' 'has not the right extension). Please add an extension ' diff --git a/preprocessing/import.yml b/preprocessing/import.yml index 7d8e9eb4..473c30df 100644 --- a/preprocessing/import.yml +++ b/preprocessing/import.yml @@ -8,17 +8,39 @@ scenario_log: language: SQL openstreetmap: required_data: - - url: https://download.geofabrik.de/europe/germany/bremen-latest.osm.pbf - filename: bremen-latest.osm.pbf +# - url: https://download.geofabrik.de/europe/germany/bremen-latest.osm.pbf + - url: https://download.geofabrik.de/europe/germany-latest.osm.pbf + filename: deu-latest.osm.pbf scripts: - - script: osm2oedb.sh +# - script: osm2oedb.sh +# language: bash +# - script: "export PGPASSWORD=egon" +# language: bash + - script: "export PGPASSWORD=egon\n osm2pgsql --create --slim --hstor6e-all --number-processes 4 --cache 4096 -H localhost -P 54321 -d dp -U oeuser -p osm_deu -S oedb.style /home/guido/.egon-pre-processing-cached/" language: bash + filename: deu-latest.osm.pbf - script: ego_pp_osm_deu_import.sql language: SQL - script: ego_pp_osm_line_street_mview.sql language: SQL - script: ego_pp_osm_polygon_building_mview.sql language: SQL +zensus: + required_data: + - url: https://next.rl-institut.de/s/Asoxps5oWGkGrZi/download + filename: Zensus_Bevoelkerung_100m-Gitter.zip + scripts: + - script: preprocessing.zensus.destatis_zensus_import_csv::import_zensus2011 + language: python + filename: Zensus_Bevoelkerung_100m-Gitter.zip + db: conn + args: + table: zensus_2011_pop_per_ha + schema: tmp_destatis + - script: destatis_zensus_raw_import.sql + language: SQL + - script: ego_pp_destatis_zensus_import.sql + language: SQL vg250: required_data: - url: https://daten.gdz.bkg.bund.de/produkte/vg/vg250_ebenen_0101/aktuell/vg250_01-01.gk3.shape.ebenen.zip @@ -29,9 +51,18 @@ vg250: - script: preprocessing.vg250.import_vg250::import_vg250 language: python filename: vg250_01-01.gk3.shape.ebenen.zip + db: info - script: ego_pp_vg250_import.sql language: SQL - script: ego_pp_vg250_metadata.sql language: SQL - script: ego_pp_vg250_mview.sql language: SQL + - script: ego_dp_structure_boundaries_vg250.sql + language: SQL +zensus_vg250: + scripts: + - script: ego_pp_destatis_zensus_insidevg250.sql + language: SQL + - script: ego_pp_destatis_zensus_metadata.sql + language: SQL diff --git a/preprocessing/sql_snippets/ego_dp_structure_boundaries_vg250.sql b/preprocessing/vg250/ego_dp_structure_boundaries_vg250.sql similarity index 92% rename from preprocessing/sql_snippets/ego_dp_structure_boundaries_vg250.sql rename to preprocessing/vg250/ego_dp_structure_boundaries_vg250.sql index a8329a9b..49016094 100644 --- a/preprocessing/sql_snippets/ego_dp_structure_boundaries_vg250.sql +++ b/preprocessing/vg250/ego_dp_structure_boundaries_vg250.sql @@ -14,24 +14,24 @@ __author__ = "Ludee" -- 1. Nationalstaat (sta) - country (cntry) -- ego scenario log (version,io,schema_name,table_name,script_name,comment) -SELECT ego_scenario_log('v0.3.0','input','boundaries','bkg_vg250_1_sta','ego_dp_structure_boundaries_vg250.sql',' '); +SELECT scenario_log('eGo_PP','PP1','input','boundaries','bkg_vg250_1_sta','ego_dp_structure_boundaries_vg250.sql',' '); -- 1. country - mview with tiny buffer because of intersection (in official data) DROP MATERIALIZED VIEW IF EXISTS boundaries.bkg_vg250_1_sta_mview CASCADE; CREATE MATERIALIZED VIEW boundaries.bkg_vg250_1_sta_mview AS SELECT vg.reference_date ::text, - vg.id ::integer, + vg.gid ::integer, vg.bez ::text, vg.gf ::double precision, ST_AREA(ST_TRANSFORM(vg.geom, 3035)) / 10000 ::double precision AS area_ha, ST_MULTI(ST_BUFFER(ST_TRANSFORM(vg.geom,3035),-0.001)) ::geometry(MultiPolygon,3035) AS geom FROM boundaries.bkg_vg250_1_sta AS vg WHERE vg.reference_date = '2016-01-01' - ORDER BY vg.id; + ORDER BY vg.gid; -- index (id) -CREATE UNIQUE INDEX bkg_vg250_1_sta_mview_id_idx - ON boundaries.bkg_vg250_1_sta_mview (id); +CREATE UNIQUE INDEX bkg_vg250_1_sta_mview_gid_idx + ON boundaries.bkg_vg250_1_sta_mview (gid); -- index GIST (geom) CREATE INDEX bkg_vg250_1_sta_mview_geom_idx @@ -112,18 +112,18 @@ COMMENT ON MATERIALIZED VIEW boundaries.bkg_vg250_1_sta_mview IS '{ SELECT obj_description('boundaries.bkg_vg250_1_sta_mview' ::regclass) ::json; -- ego scenario log (version,io,schema_name,table_name,script_name,comment) -SELECT ego_scenario_log('v0.3.0','output','boundaries','bkg_vg250_1_sta_mview','ego_dp_structure_boundaries_vg250.sql',' '); +SELECT scenario_log('eGo_PP','PP1','output','boundaries','bkg_vg250_1_sta_mview','ego_dp_structure_boundaries_vg250.sql',' '); -- 1. country - error geom DROP MATERIALIZED VIEW IF EXISTS boundaries.bkg_vg250_1_sta_error_geom_mview CASCADE; CREATE MATERIALIZED VIEW boundaries.bkg_vg250_1_sta_error_geom_mview AS - SELECT sub.id AS id, + SELECT sub.gid AS gid, sub.error AS error, sub.error_reason AS error_reason, ST_SETSRID(location(ST_IsValidDetail(sub.geom)),3035) ::geometry(Point,3035) AS geom FROM ( - SELECT source.id AS id, -- PK + SELECT source.gid AS gid, -- PK ST_IsValid(source.geom) AS error, reason(ST_IsValidDetail(source.geom)) AS error_reason, source.geom AS geom @@ -132,9 +132,9 @@ CREATE MATERIALIZED VIEW boundaries.bkg_vg250_1_sta_error_geom_mview AS ) AS sub WHERE sub.error = FALSE; --- index (id) -CREATE UNIQUE INDEX bkg_vg250_1_sta_error_geom_mview_id_idx - ON boundaries.bkg_vg250_1_sta_error_geom_mview (id); +-- index (gid) +CREATE UNIQUE INDEX bkg_vg250_1_sta_error_geom_mview_gid_idx + ON boundaries.bkg_vg250_1_sta_error_geom_mview (gid); -- index GIST (geom) CREATE INDEX bkg_vg250_1_sta_error_geom_mview_geom_idx @@ -157,7 +157,7 @@ COMMENT ON MATERIALIZED VIEW boundaries.bkg_vg250_1_sta_error_geom_mview IS '{ "Extend": "Germany; Nationalstaat (sta) - country (cntry)" }], "Description": ["Errors in country border"], "Column":[ - {"Name": "id", "Description": "Unique identifier", "Unit": " " }, + {"Name": "gid", "Description": "Unique identifier", "Unit": " " }, {"Name": "error", "Description": "Error", "Unit": " " }, {"Name": "error_reason", "Description": "Error reason", "Unit": " " }, {"Name": "geom", "Description": "Geometry", "Unit": " " } ], @@ -177,14 +177,14 @@ COMMENT ON MATERIALIZED VIEW boundaries.bkg_vg250_1_sta_error_geom_mview IS '{ SELECT obj_description('boundaries.bkg_vg250_1_sta_error_geom_mview' ::regclass) ::json; -- ego scenario log (version,io,schema_name,table_name,script_name,comment) -SELECT ego_scenario_log('v0.3.0','output','boundaries','bkg_vg250_1_sta_error_geom_mview','ego_dp_structure_boundaries_vg250.sql',' '); +SELECT scenario_log('eGo_PP','PP1','output','boundaries','bkg_vg250_1_sta_error_geom_mview','ego_dp_structure_boundaries_vg250.sql',' '); -- 1. country - union DROP MATERIALIZED VIEW IF EXISTS boundaries.bkg_vg250_1_sta_union_mview CASCADE; CREATE MATERIALIZED VIEW boundaries.bkg_vg250_1_sta_union_mview AS SELECT '2016-01-01' ::text AS reference_date, - '1' ::integer AS id, + '1' ::integer AS gid, 'Bundesrepublik' ::text AS bez, ST_AREA(un.geom) / 10000 ::double precision AS area_ha, un.geom ::geometry(MultiPolygon,3035) AS geom @@ -193,9 +193,9 @@ CREATE MATERIALIZED VIEW boundaries.bkg_vg250_1_sta_union_mview AS WHERE vg.bez = 'Bundesrepublik' AND reference_date = '2016-01-01' ) AS un; --- index (id) -CREATE UNIQUE INDEX bkg_vg250_1_sta_union_mview_id_idx - ON boundaries.bkg_vg250_1_sta_union_mview (id); +-- index (gid) +CREATE UNIQUE INDEX bkg_vg250_1_sta_union_mview_gid_idx + ON boundaries.bkg_vg250_1_sta_union_mview (gid); -- index GIST (geom) CREATE INDEX bkg_vg250_1_sta_union_mview_geom_idx @@ -219,7 +219,7 @@ COMMENT ON MATERIALIZED VIEW boundaries.bkg_vg250_1_sta_union_mview IS '{ "Description": ["Geometry union"], "Column":[ {"Name": "reference_date", "Description": "Reference Year", "Unit": " " }, - {"Name": "id", "Description": "Unique identifier", "Unit": " " }, + {"Name": "gid", "Description": "Unique identifier", "Unit": " " }, {"Name": "bez", "Description": "Bezeichnung der Verwaltungseinheit", "Unit": " " }, {"Name": "area_ha", "Description": "Area in ha", "Unit": "ha" }, {"Name": "geom", "Description": "Geometry", "Unit": " " } ], @@ -239,14 +239,14 @@ COMMENT ON MATERIALIZED VIEW boundaries.bkg_vg250_1_sta_union_mview IS '{ SELECT obj_description('boundaries.bkg_vg250_1_sta_union_mview' ::regclass) ::json; -- ego scenario log (version,io,schema_name,table_name,script_name,comment) -SELECT ego_scenario_log('v0.3.0','output','boundaries','bkg_vg250_1_sta_union_mview','ego_dp_structure_boundaries_vg250.sql',' '); +SELECT scenario_log('eGo_PP','PP1','output','boundaries','bkg_vg250_1_sta_union_mview','ego_dp_structure_boundaries_vg250.sql',' '); -- 1. state borders - bounding box DROP MATERIALIZED VIEW IF EXISTS boundaries.bkg_vg250_1_sta_bbox_mview CASCADE; CREATE MATERIALIZED VIEW boundaries.bkg_vg250_1_sta_bbox_mview AS SELECT '2016-01-01' ::text AS reference_date, - '1' ::integer AS id, + '1' ::integer AS gid, 'Bundesrepublik' ::text AS bez, ST_AREA(un.geom) / 10000 ::double precision AS area_ha, un.geom ::geometry(Polygon,3035) AS geom @@ -254,9 +254,9 @@ CREATE MATERIALIZED VIEW boundaries.bkg_vg250_1_sta_bbox_mview AS FROM boundaries.bkg_vg250_1_sta_union_mview AS vg ) AS un; --- index (id) -CREATE UNIQUE INDEX bkg_vg250_1_sta_bbox_mview_id_idx - ON boundaries.bkg_vg250_1_sta_bbox_mview (id); +-- index (gid) +CREATE UNIQUE INDEX bkg_vg250_1_sta_bbox_mview_gid_idx + ON boundaries.bkg_vg250_1_sta_bbox_mview (gid); -- index GIST (geom) CREATE INDEX bkg_vg250_1_sta_bbox_mview_geom_idx @@ -280,7 +280,7 @@ COMMENT ON MATERIALIZED VIEW boundaries.bkg_vg250_1_sta_bbox_mview IS '{ "Description": ["Geometry bounding box"], "Column":[ {"Name": "reference_date", "Description": "Reference Year", "Unit": " " }, - {"Name": "id", "Description": "Unique identifier", "Unit": " " }, + {"Name": "gid", "Description": "Unique identifier", "Unit": " " }, {"Name": "bez", "Description": "Bezeichnung der Verwaltungseinheit", "Unit": " " }, {"Name": "area_ha", "Description": "Area in ha", "Unit": "ha" }, {"Name": "geom", "Description": "Geometry", "Unit": " " } ], @@ -300,13 +300,13 @@ COMMENT ON MATERIALIZED VIEW boundaries.bkg_vg250_1_sta_bbox_mview IS '{ SELECT obj_description('boundaries.bkg_vg250_1_sta_bbox_mview' ::regclass) ::json; -- ego scenario log (version,io,schema_name,table_name,script_name,comment) -SELECT ego_scenario_log('v0.3.0','output','boundaries','bkg_vg250_1_sta_bbox_mview','ego_dp_structure_boundaries_vg250.sql',' '); +SELECT scenario_log('eGo_PP','PP1','output','boundaries','bkg_vg250_1_sta_bbox_mview','ego_dp_structure_boundaries_vg250.sql',' '); -- 2. Bundesland (lan) - federal state (fst) -- ego scenario log (version,io,schema_name,table_name,script_name,comment) -SELECT ego_scenario_log('v0.3.0','input','boundaries','bkg_vg250_2_lan','ego_dp_structure_boundaries_vg250.sql',' '); +SELECT scenario_log('eGo_PP','PP1','input','boundaries','bkg_vg250_2_lan','ego_dp_structure_boundaries_vg250.sql',' '); -- 2. federal state - mview with tiny buffer because of intersection (in official data) DROP MATERIALIZED VIEW IF EXISTS boundaries.bkg_vg250_2_lan_mview CASCADE; @@ -324,7 +324,7 @@ CREATE MATERIALIZED VIEW boundaries.bkg_vg250_2_lan_mview AS GROUP BY lan.ags_0,lan.gen ORDER BY lan.ags_0; --- index (id) +-- index (gid) CREATE UNIQUE INDEX bkg_vg250_2_lan_mview_ags_0_idx ON boundaries.bkg_vg250_2_lan_mview (ags_0); @@ -370,13 +370,13 @@ COMMENT ON MATERIALIZED VIEW boundaries.bkg_vg250_2_lan_mview IS '{ SELECT obj_description('boundaries.bkg_vg250_2_lan_mview' ::regclass) ::json; -- ego scenario log (version,io,schema_name,table_name,script_name,comment) -SELECT ego_scenario_log('v0.3.0','output','boundaries','bkg_vg250_2_lan_mview','ego_dp_structure_boundaries_vg250.sql',' '); +SELECT scenario_log('eGo_PP','PP1','output','boundaries','bkg_vg250_2_lan_mview','ego_dp_structure_boundaries_vg250.sql',' '); -- 4. Landkreis (krs) - district (dist) -- ego scenario log (version,io,schema_name,table_name,script_name,comment) -SELECT ego_scenario_log('v0.3.0','input','boundaries','bkg_vg250_4_krs','ego_dp_structure_boundaries_vg250.sql',' '); +SELECT scenario_log('eGo_PP','PP1','input','boundaries','bkg_vg250_4_krs','ego_dp_structure_boundaries_vg250.sql',' '); -- 4. district - mview DROP MATERIALIZED VIEW IF EXISTS boundaries.bkg_vg250_4_krs_mview CASCADE; @@ -394,7 +394,7 @@ CREATE MATERIALIZED VIEW boundaries.bkg_vg250_4_krs_mview AS WHERE vg.reference_date = '2016-01-01' ORDER BY vg.id; --- index (id) +-- index (gid) CREATE UNIQUE INDEX bkg_vg250_4_krs_mview_id_idx ON boundaries.bkg_vg250_4_krs_mview (id); @@ -444,12 +444,11 @@ COMMENT ON MATERIALIZED VIEW boundaries.bkg_vg250_4_krs_mview IS '{ SELECT obj_description('boundaries.bkg_vg250_4_krs_mview' ::regclass) ::json; -- ego scenario log (version,io,schema_name,table_name,script_name,comment) -SELECT ego_scenario_log('v0.3.0','output','boundaries','bkg_vg250_4_krs_mview','ego_dp_structure_boundaries_vg250.sql',' '); +SELECT scenario_log('eGo_PP','PP1','output','boundaries','bkg_vg250_4_krs_mview','ego_dp_structure_boundaries_vg250.sql',' '); -- 6. Gemeinde (gem) - municipality (mun) -/* -- census data per municipality DROP TABLE IF EXISTS society.destatis_zensus_population_per_bkg_vg250_6_gem; CREATE TABLE society.destatis_zensus_population_per_bkg_vg250_6_gem ( @@ -468,8 +467,40 @@ INSERT INTO society.destatis_zensus_population_per_bkg_vg250_6_gem (reference_da FROM boundaries.bkg_vg250_6_gem AS vg ORDER BY vg.id; --- ego scenario log (version,io,schema_name,table_name,script_name,comment) -SELECT ego_scenario_log('v0.3.0','input','society','destatis_zensus_population_per_ha_mview','ego_dp_structure_boundaries_vg250.sql',' '); + +-- zensus points with population (includes zensus points outside borders vg250) +DROP MATERIALIZED VIEW IF EXISTS society.destatis_zensus_population_per_ha_mview CASCADE; +CREATE MATERIALIZED VIEW society.destatis_zensus_population_per_ha_mview AS + SELECT a.gid, + a.population, + a.geom_point, + a.geom + FROM society.destatis_zensus_population_per_ha AS a + WHERE a.population >= 0 + ORDER BY a.gid; + +-- index +CREATE UNIQUE INDEX destatis_zensus_population_per_ha_mview_gid_idx + ON society.destatis_zensus_population_per_ha_mview (gid); + +CREATE INDEX destatis_zensus_population_per_ha_mview_geom_point_idx + ON society.destatis_zensus_population_per_ha_mview USING GIST (geom_point); + +CREATE INDEX destatis_zensus_population_per_ha_mview_geom_idx + ON society.destatis_zensus_population_per_ha_mview USING GIST (geom); + +-- access rights +ALTER TABLE society.destatis_zensus_population_per_ha_mview OWNER TO oeuser; + +-- metadata +COMMENT ON MATERIALIZED VIEW society.destatis_zensus_population_per_ha_mview IS '{ + "comment": "eGoPP - Temporary table", + "version": "PP1", + "published": "none" }'; + +-- scenario log (project,version,io,schema_name,table_name,script_name,comment) +SELECT scenario_log('eGo_PP','PP1','preprocessing','society','destatis_zensus_population_per_ha_mview','ego_pp_destatis_zensus_insidevg250.sql','none'); + -- zensus 2011 population UPDATE society.destatis_zensus_population_per_bkg_vg250_6_gem AS t1 @@ -501,10 +532,10 @@ UPDATE society.destatis_zensus_population_per_bkg_vg250_6_gem AS ce UPDATE society.destatis_zensus_population_per_bkg_vg250_6_gem AS ce SET census_density = '0' WHERE census_density IS NULL; - */ + -- ego scenario log (version,io,schema_name,table_name,script_name,comment) -SELECT ego_scenario_log('v0.3.0','input','boundaries','bkg_vg250_6_gem','ego_dp_structure_boundaries_vg250.sql',' '); +SELECT scenario_log('eGo_PP','PP1','input','boundaries','bkg_vg250_6_gem','ego_dp_structure_boundaries_vg250.sql',' '); -- 6. municipality - mview DROP MATERIALIZED VIEW IF EXISTS boundaries.bkg_vg250_6_gem_mview CASCADE; @@ -578,7 +609,7 @@ COMMENT ON MATERIALIZED VIEW boundaries.bkg_vg250_6_gem_mview IS '{ SELECT obj_description('boundaries.bkg_vg250_6_gem_mview' ::regclass) ::json; -- ego scenario log (version,io,schema_name,table_name,script_name,comment) -SELECT ego_scenario_log('v0.3.0','output','boundaries','bkg_vg250_6_gem_mview','ego_dp_structure_boundaries_vg250.sql',' '); +SELECT scenario_log('eGo_PP','PP1','output','boundaries','bkg_vg250_6_gem_mview','ego_dp_structure_boundaries_vg250.sql',' '); -- 6. municipality - error geom @@ -643,7 +674,7 @@ COMMENT ON MATERIALIZED VIEW boundaries.bkg_vg250_6_gem_error_geom_mview IS '{ SELECT obj_description('boundaries.bkg_vg250_6_gem_error_geom_mview' ::regclass) ::json; -- ego scenario log (version,io,schema_name,table_name,script_name,comment) -SELECT ego_scenario_log('v0.3.0','output','boundaries','bkg_vg250_6_gem_error_geom_mview','ego_dp_structure_boundaries_vg250.sql',' '); +SELECT scenario_log('eGo_PP','PP1','output','boundaries','bkg_vg250_6_gem_error_geom_mview','ego_dp_structure_boundaries_vg250.sql',' '); -- 6. municipality - dump @@ -723,7 +754,7 @@ COMMENT ON MATERIALIZED VIEW boundaries.bkg_vg250_6_gem_dump_mview IS '{ SELECT obj_description('boundaries.bkg_vg250_6_gem_dump_mview' ::regclass) ::json; -- ego scenario log (version,io,schema_name,table_name,script_name,comment) -SELECT ego_scenario_log('v0.3.0','output','boundaries','bkg_vg250_6_gem_dump_mview','ego_dp_structure_boundaries_vg250.sql',' '); +SELECT scenario_log('eGo_PP','PP1','output','boundaries','bkg_vg250_6_gem_dump_mview','ego_dp_structure_boundaries_vg250.sql',' '); -- ego @@ -840,7 +871,7 @@ COMMENT ON MATERIALIZED VIEW model_draft.ego_boundaries_bkg_vg250_6_gem_hole_mvi SELECT obj_description('model_draft.ego_boundaries_bkg_vg250_6_gem_hole_mview' ::regclass) ::json; -- ego scenario log (version,io,schema_name,table_name,script_name,comment) -SELECT ego_scenario_log('v0.3.0','output','model_draft','ego_boundaries_bkg_vg250_6_gem_hole_mview','ego_dp_structure_boundaries_vg250.sql',' '); +SELECT scenario_log('eGo_PP','PP1','output','model_draft','ego_boundaries_bkg_vg250_6_gem_hole_mview','ego_dp_structure_boundaries_vg250.sql',' '); -- update holes @@ -903,7 +934,7 @@ COMMENT ON TABLE model_draft.ego_boundaries_bkg_vg250_6_gem_clean IS '{ SELECT obj_description('model_draft.ego_boundaries_bkg_vg250_6_gem_clean' ::regclass) ::json; -- ego scenario log (version,io,schema_name,table_name,script_name,comment) -SELECT ego_scenario_log('v0.3.0','output','model_draft','ego_boundaries_bkg_vg250_6_gem_clean','ego_dp_structure_boundaries_vg250.sql',' '); +SELECT scenario_log('eGo_PP','PP1','output','model_draft','ego_boundaries_bkg_vg250_6_gem_clean','ego_dp_structure_boundaries_vg250.sql',' '); -- validation @@ -947,4 +978,4 @@ COMMENT ON VIEW boundaries.bkg_vg250_statistics_view IS '{ "version": "v0.3.0" }' ; -- ego scenario log (version,io,schema_name,table_name,script_name,comment) -SELECT ego_scenario_log('v0.3.0','output','boundaries','bkg_vg250_statistics_view','ego_dp_structure_boundaries_vg250.sql',' '); +SELECT scenario_log('eGo_PP','PP1','output','boundaries','bkg_vg250_statistics_view','ego_dp_structure_boundaries_vg250.sql',' '); diff --git a/preprocessing/zensus/destatis_zensus_import_csv.py b/preprocessing/zensus/destatis_zensus_import_csv.py new file mode 100644 index 00000000..72ad39e6 --- /dev/null +++ b/preprocessing/zensus/destatis_zensus_import_csv.py @@ -0,0 +1,28 @@ +import os +import zipfile +import pandas as pd +import sqlalchemy + + +def import_zensus2011(file, path, db, year=2019, **kwds): + + if not db.dialect.has_schema(db, kwds["schema"]): + db.execute(sqlalchemy.schema.CreateSchema(kwds["schema"])) + + csv_file = file.replace("zip", "csv") + + zip = zipfile.ZipFile(os.path.join(path, file)) + zip.extractall(path=path) + + # read with pandas + zensus_raw = pd.read_csv(os.path.join(path, file), delimiter=";") + zensus_raw.columns =["grid_id", "x_mp_100m", "y_mp_100m", "population"] + # zensus_raw = zensus_raw.iloc[:20000] + + # create table from geopandas dataframe + zensus_raw.to_sql(kwds["table"], + db, kwds["schema"], + index=False, + if_exists="replace", + chunksize=10000, + method="multi") diff --git a/preprocessing/zensus/destatis_zensus_raw_import.sql b/preprocessing/zensus/destatis_zensus_raw_import.sql new file mode 100644 index 00000000..2e667abc --- /dev/null +++ b/preprocessing/zensus/destatis_zensus_raw_import.sql @@ -0,0 +1,57 @@ +-- 01 +-- create table and insert (OK!) +DROP TABLE IF EXISTS tmp_destatis.zensus_population_per_ha; +CREATE TABLE tmp_destatis.zensus_population_per_ha ( + gid SERIAL NOT NULL, + grid_id character varying(254) NOT NULL, + x_mp numeric(10,0), + y_mp numeric(10,0), + population numeric(10,0), + geom geometry(Point,3035), + CONSTRAINT zensus_population_per_ha_pkey PRIMARY KEY (gid)); + +-- 02 +-- Insert (OK!) +INSERT INTO tmp_destatis.zensus_population_per_ha (grid_id,x_mp,y_mp,population,geom)( + SELECT + zensus.grid_id AS grid_id, + zensus.x_mp_100m AS x_mp, + zensus.y_mp_100m AS y_mp, + zensus.population AS population, + ST_SetSRID(ST_MakePoint(zensus.x_mp_100m, zensus.y_mp_100m),3035) AS geom + FROM tmp_destatis.zensus_2011_pop_per_ha AS zensus); + +-- 03 +-- Create index (OK!) +CREATE INDEX zensus_population_per_ha_geom_idx + ON tmp_destatis.zensus_population_per_ha + USING gist (geom); + +-- 04 +-- CREATE TABLE (OK!) +DROP TABLE IF EXISTS tmp_destatis.zensus_population_per_ha_grid; +CREATE TABLE tmp_destatis.zensus_population_per_ha_grid ( + gid INT, + grid_id character varying(254) NOT NULL, + x_mp numeric(10,0), + y_mp numeric(10,0), + population INT, + geom geometry(Polygon, 3035), + CONSTRAINT zensus_population_per_ha_grid_pkey PRIMARY KEY (gid)); + +-- 05 +-- INSERT GRID (OK!) +INSERT INTO tmp_destatis.zensus_population_per_ha_grid +SELECT pts.gid AS gid, + pts.grid_id AS grid_id, + pts.x_mp AS x_mp, + pts.y_mp AS y_mp, + pts.population AS population, + ST_SetSRID((ST_MakeEnvelope(pts.x_mp-50,pts.y_mp-50,pts.x_mp+50,pts.y_mp+50)),3035) AS geom +FROM tmp_destatis.zensus_population_per_ha AS pts; + +-- 06 +-- CREATE INDEX GIST (OK!) +CREATE INDEX zensus_population_per_ha_grid_geom_idx + ON tmp_destatis.zensus_population_per_ha_grid + USING GIST (geom); \ No newline at end of file diff --git a/preprocessing/sql_snippets/society/ego_pp_destatis_zensus_import.sql b/preprocessing/zensus/ego_pp_destatis_zensus_import.sql similarity index 88% rename from preprocessing/sql_snippets/society/ego_pp_destatis_zensus_import.sql rename to preprocessing/zensus/ego_pp_destatis_zensus_import.sql index 3de541c6..3ebd2654 100644 --- a/preprocessing/sql_snippets/society/ego_pp_destatis_zensus_import.sql +++ b/preprocessing/zensus/ego_pp_destatis_zensus_import.sql @@ -7,6 +7,7 @@ __url__ = "https://github.com/openego/data_processing/blob/master/LICENSE" __author__ = "Ludee" */ +CREATE SCHEMA IF NOT EXISTS society; -- zensus CREATE TABLE society.destatis_zensus_population_per_ha ( @@ -64,8 +65,10 @@ CREATE INDEX destatis_zensus_population_per_ha_geom_point_idx -- insert data INSERT INTO society.destatis_zensus_population_per_ha SELECT pnt.gid,pnt.grid_id,pnt.x_mp,pnt.y_mp,pnt.population,pnt.geom AS geom_point,poly.geom - FROM orig_destatis.zensus_population_per_ha AS pnt - JOIN orig_destatis.zensus_population_per_ha_grid AS poly ON (pnt.gid=poly.gid); + FROM tmp_destatis.zensus_population_per_ha AS pnt + JOIN tmp_destatis.zensus_population_per_ha_grid AS poly ON (pnt.gid=poly.gid); -- ego scenario log (version,io,schema_name,table_name,script_name,comment) -SELECT ego_scenario_log('v0.2.10','preprocessing','society','destatis_zensus_population_per_ha','ego_pp_destatis_zensus_import.sql','setup zensus table'); +SELECT scenario_log('eGo_PP','PP1','preprocessing','society','destatis_zensus_population_per_ha','ego_pp_destatis_zensus_import.sql','setup zensus table'); + +DROP SCHEMA tmp_destatis CASCADE; \ No newline at end of file diff --git a/preprocessing/sql_snippets/society/ego_pp_destatis_zensus_insidevg250.sql b/preprocessing/zensus_vg250/ego_pp_destatis_zensus_insidevg250.sql similarity index 98% rename from preprocessing/sql_snippets/society/ego_pp_destatis_zensus_insidevg250.sql rename to preprocessing/zensus_vg250/ego_pp_destatis_zensus_insidevg250.sql index c94fcb40..93c0a621 100644 --- a/preprocessing/sql_snippets/society/ego_pp_destatis_zensus_insidevg250.sql +++ b/preprocessing/zensus_vg250/ego_pp_destatis_zensus_insidevg250.sql @@ -47,8 +47,7 @@ COMMENT ON MATERIALIZED VIEW society.destatis_zensus_population_per_ha_mview IS "published": "none" }'; -- scenario log (project,version,io,schema_name,table_name,script_name,comment) -SELECT scenario_log('eGo_PP','PP1','output','society','destatis_zensus_population_per_ha_mview','ego_pp_destatis_zensus_insidevg250.sql','none'); - +SELECT scenario_log('eGo_PP','PP1','preprocessing','society','destatis_zensus_population_per_ha_mview','ego_pp_destatis_zensus_insidevg250.sql','none'); -- zensus points inside Germany (vg250) DROP TABLE IF EXISTS model_draft.destatis_zensus_population_per_ha_inside CASCADE; diff --git a/preprocessing/sql_snippets/society/ego_pp_destatis_zensus_metadata.sql b/preprocessing/zensus_vg250/ego_pp_destatis_zensus_metadata.sql similarity index 96% rename from preprocessing/sql_snippets/society/ego_pp_destatis_zensus_metadata.sql rename to preprocessing/zensus_vg250/ego_pp_destatis_zensus_metadata.sql index 1996b084..6cd9c03c 100644 --- a/preprocessing/sql_snippets/society/ego_pp_destatis_zensus_metadata.sql +++ b/preprocessing/zensus_vg250/ego_pp_destatis_zensus_metadata.sql @@ -58,4 +58,4 @@ COMMENT ON TABLE society.destatis_zensus_population_per_ha IS '{ "metadata_version": "1.3"}'; -- ego scenario log (version,io,schema_name,table_name,script_name,comment) -SELECT ego_scenario_log('v0.2.10','preprocessing','society','destatis_zensus_population_per_ha','ego_pp_destatis_zensus_import.sql','metadata'); +SELECT scenario_log('eGo_PP','PP1','preprocessing','society','destatis_zensus_population_per_ha','ego_pp_destatis_zensus_import.sql','metadata'); From 1f44cfdd682b7b5299df5d9686eaffd82475bf7f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guido=20Ple=C3=9Fmann?= Date: Fri, 27 Mar 2020 21:18:50 +0100 Subject: [PATCH 36/84] Do OSM DB insert by more flexible python script --- preprocessing/eGo_import_and_preprocessing.py | 2 +- preprocessing/import.yml | 9 +++------ preprocessing/openstreetmap/import_osm.py | 20 +++++++++++++++++++ preprocessing/openstreetmap/osm2oedb.sh | 2 -- 4 files changed, 24 insertions(+), 9 deletions(-) create mode 100644 preprocessing/openstreetmap/import_osm.py delete mode 100644 preprocessing/openstreetmap/osm2oedb.sh diff --git a/preprocessing/eGo_import_and_preprocessing.py b/preprocessing/eGo_import_and_preprocessing.py index a792a6fc..6485202b 100644 --- a/preprocessing/eGo_import_and_preprocessing.py +++ b/preprocessing/eGo_import_and_preprocessing.py @@ -99,7 +99,7 @@ def preprocessing(): # iterate over data sets for key, dataset in datasets.items(): for download in dataset.get("required_data", []): - logger.info("Downloading '{}' ...".format(script["required_data"])) + logger.info("Downloading '{}' ...".format(download["filename"])) snippet_time = time.time() download_data(download["url"], download["filename"]) logger.info('...successfully done in {:.2f} seconds.'.format( diff --git a/preprocessing/import.yml b/preprocessing/import.yml index 473c30df..e8da8651 100644 --- a/preprocessing/import.yml +++ b/preprocessing/import.yml @@ -12,13 +12,10 @@ openstreetmap: - url: https://download.geofabrik.de/europe/germany-latest.osm.pbf filename: deu-latest.osm.pbf scripts: -# - script: osm2oedb.sh -# language: bash -# - script: "export PGPASSWORD=egon" -# language: bash - - script: "export PGPASSWORD=egon\n osm2pgsql --create --slim --hstor6e-all --number-processes 4 --cache 4096 -H localhost -P 54321 -d dp -U oeuser -p osm_deu -S oedb.style /home/guido/.egon-pre-processing-cached/" - language: bash + - script: preprocessing.openstreetmap.import_osm::osm2postgres + language: python filename: deu-latest.osm.pbf + db: info - script: ego_pp_osm_deu_import.sql language: SQL - script: ego_pp_osm_line_street_mview.sql diff --git a/preprocessing/openstreetmap/import_osm.py b/preprocessing/openstreetmap/import_osm.py new file mode 100644 index 00000000..2a4195c6 --- /dev/null +++ b/preprocessing/openstreetmap/import_osm.py @@ -0,0 +1,20 @@ +import subprocess +import os + + +NUM_PROCESSES = 4 +CACHE_SIZE = 4096 + + +def osm2postgres(file, path, db_info): + + stylefile = os.path.join(os.path.dirname(os.path.abspath(__file__)), "oedb.style") + pbf_file = os.path.join(path, file) + + db_info.update({"stylefile": stylefile, "pbffile": pbf_file}) + + cmd = "osm2pgsql --create --slim --hstore-all --number-processes NUM_PROCESSES --cache CACHE_SIZE -H {host} -P {port} -d {database} -U {user} -p osm_deu -S {stylefile} {pbffile}".format(**db_info) + + subprocess.run(cmd, shell=True, env={"PGPASSWORD": "egon"}) + + diff --git a/preprocessing/openstreetmap/osm2oedb.sh b/preprocessing/openstreetmap/osm2oedb.sh deleted file mode 100644 index 02caf755..00000000 --- a/preprocessing/openstreetmap/osm2oedb.sh +++ /dev/null @@ -1,2 +0,0 @@ -export PGPASSWORD=egon -osm2pgsql --create --slim --hstore-all --number-processes 4 --cache 4096 -H localhost -P 54321 -d dp -U oeuser -p osm_deu -S oedb.style ~/.egon-pre-processing-cached/bremen-latest.osm.pbf \ No newline at end of file From d33f8444365078cf3ebc6cb3fc7a7f1792920003 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guido=20Ple=C3=9Fmann?= Date: Sat, 28 Mar 2020 18:56:58 +0100 Subject: [PATCH 37/84] Add download script for vernetzen wind ptential areas --- .../vernetzen_wind_potential_areas_to_shp.py | 64 +++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 preprocessing/utility/vernetzen_wind_potential_areas_to_shp.py diff --git a/preprocessing/utility/vernetzen_wind_potential_areas_to_shp.py b/preprocessing/utility/vernetzen_wind_potential_areas_to_shp.py new file mode 100644 index 00000000..9da70b49 --- /dev/null +++ b/preprocessing/utility/vernetzen_wind_potential_areas_to_shp.py @@ -0,0 +1,64 @@ +import pandas as pd +from egoio.tools import db +from egoio.db_tables.supply import VernetzenWindPotentialArea +from sqlalchemy.orm import sessionmaker +from sqlalchemy import func +import geopandas as gdp +from shapely import wkt +import os +import zipfile + + +DOWNLOADDIR = os.path.join(os.path.expanduser("~"), ".egon-pre-processing-cached/") +CRS = 4326 + + +def vernetzen_windpotential_areas2shp(): + + filename = "vernetzen_wind_potential_areas.shp" + oedbtable2shp(VernetzenWindPotentialArea, filename) + + + +def zipdir(path, ziph): + # ziph is zipfile handle + for root, dirs, files in os.walk(path): + for file in files: + ziph.write(os.path.join(root, file), arcname=file) + + +def oedbtable2shp(table_orm, filename): + + shp_dir = os.path.join(DOWNLOADDIR, filename.replace(".shp", "")) + os.makedirs(shp_dir, exist_ok=True) + shp_filename = os.path.join(shp_dir, filename) + zip_filename = os.path.join(DOWNLOADDIR, filename.replace(".shp", ".zip")) + + engine_oedb = db.connection(readonly=True) + session = sessionmaker(bind=engine_oedb)() + + table_query = session.query( + table_orm.region_key, + func.ST_AsText(func.ST_Transform(table_orm.geom, CRS)).label('geom')) + + table_df = pd.read_sql_query(table_query.statement, + session.bind, + index_col='region_key') + + table_df["geom"] = table_df["geom"].apply(wkt.loads) + table_gdf = gdp.GeoDataFrame(table_df, + geometry="geom", + crs={'init': 'epsg:{}'.format(CRS)}) + + table_gdf.to_file(shp_filename) + + zf = zipfile.ZipFile( + zip_filename, + mode='w', + compression=zipfile.ZIP_DEFLATED) + zipdir(shp_dir, zf) + zf.close() + + +if __name__== "__main__": + vernetzen_windpotential_areas2shp() \ No newline at end of file From c80c3e4f511b731a715d02091265ea3edfe3369b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guido=20Ple=C3=9Fmann?= Date: Sat, 28 Mar 2020 18:59:07 +0100 Subject: [PATCH 38/84] Add vernetzen wind areas import --- preprocessing/import.yml | 15 +++++ .../import_vernetzen_wind_potential_areas.py | 58 +++++++++++++++++++ .../vernetzen_wind_potential_area_v1.3.sql | 0 setup.py | 3 +- 4 files changed, 75 insertions(+), 1 deletion(-) create mode 100644 preprocessing/supply/import_vernetzen_wind_potential_areas.py rename preprocessing/{sql_snippets => supply}/vernetzen_wind_potential_area_v1.3.sql (100%) diff --git a/preprocessing/import.yml b/preprocessing/import.yml index e8da8651..a4ea34c1 100644 --- a/preprocessing/import.yml +++ b/preprocessing/import.yml @@ -63,3 +63,18 @@ zensus_vg250: language: SQL - script: ego_pp_destatis_zensus_metadata.sql language: SQL +supply: + required_data: + - url: https://next.rl-institut.de/s/TsjxR5P2Q6Z8Y7X/download + filename: vernetzen_wind_potential_areas.zip + scripts: + - script: preprocessing.supply.import_vernetzen_wind_potential_areas::import_vernetzen_wind_potential_areas + language: python + filename: vernetzen_wind_potential_areas.zip + db: conn + args: + schema: "supply" + table: "vernetzen_wind_potential_area" + - script: vernetzen_wind_potential_area_v1.3.sql + language: SQL + diff --git a/preprocessing/supply/import_vernetzen_wind_potential_areas.py b/preprocessing/supply/import_vernetzen_wind_potential_areas.py new file mode 100644 index 00000000..a4ff35e9 --- /dev/null +++ b/preprocessing/supply/import_vernetzen_wind_potential_areas.py @@ -0,0 +1,58 @@ +import os +import zipfile +import geopandas as gpd +import sqlalchemy +from geoalchemy2 import Geometry, WKTElement +from shapely.geometry import MultiPolygon + + +CRS = 4326 + + +# Function to generate WKB hex +def wkb_hexer(line): + return line.wkb_hex + + +def force_to_multipolygon(value): + + if isinstance(value, MultiPolygon): + return value + else: + return MultiPolygon([value]) + + +def import_vernetzen_wind_potential_areas(file, path, db, **kwds): + + if not db.dialect.has_schema(db, kwds["schema"]): + db.execute(sqlalchemy.schema.CreateSchema(kwds["schema"])) + + shp_file = file.replace(".zip", ".shp") + subdir = file.replace(".zip", "") + os.makedirs(os.path.join(path, subdir), exist_ok=True) + + + zf = zipfile.ZipFile(os.path.join(path, file)) + zf.extractall(path=os.path.join(path, subdir)) + + # read with pandas + wind_area = gpd.read_file(os.path.join(path, subdir, shp_file)) + + wind_area["geometry"] =wind_area["geometry"].apply(force_to_multipolygon) + wind_area['geom'] = wind_area['geometry'].apply(wkb_hexer) + wind_area.drop("geometry", axis=1, inplace=True) + + # create table from geopandas dataframe + wind_area.to_sql(kwds["table"], + db, kwds["schema"], + index=False, + if_exists="replace", + chunksize=10000, + method="multi", + dtype={'geom': Geometry('MULTIPOLYGON')} + ) + + create_pkey_constraint = "ALTER TABLE {schema}.{table} ADD CONSTRAINT " \ + "{table}_pkey PRIMARY KEY (region_key);".format(**kwds) + db.execution_options(autocommit=True).execute(create_pkey_constraint) + diff --git a/preprocessing/sql_snippets/vernetzen_wind_potential_area_v1.3.sql b/preprocessing/supply/vernetzen_wind_potential_area_v1.3.sql similarity index 100% rename from preprocessing/sql_snippets/vernetzen_wind_potential_area_v1.3.sql rename to preprocessing/supply/vernetzen_wind_potential_area_v1.3.sql diff --git a/setup.py b/setup.py index c2b3d559..08608391 100644 --- a/setup.py +++ b/setup.py @@ -20,7 +20,8 @@ 'demandlib', 'egoio >= 0.4.7', 'geoalchemy2', - "pyyaml" + "pyyaml", + "geopandas" ], extras_require={ 'docs': [ From 60fe3269662530c737314da2a849b480a7379826 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guido=20Ple=C3=9Fmann?= Date: Sun, 29 Mar 2020 15:09:59 +0200 Subject: [PATCH 39/84] Add SLP parameters creation script to pre-processing workflow --- preprocessing/import.yml | 4 ++++ .../{sql_snippets => slp}/ego_pre_slp_parameters.sql | 3 ++- 2 files changed, 6 insertions(+), 1 deletion(-) rename preprocessing/{sql_snippets => slp}/ego_pre_slp_parameters.sql (95%) diff --git a/preprocessing/import.yml b/preprocessing/import.yml index a4ea34c1..c70647d2 100644 --- a/preprocessing/import.yml +++ b/preprocessing/import.yml @@ -77,4 +77,8 @@ supply: table: "vernetzen_wind_potential_area" - script: vernetzen_wind_potential_area_v1.3.sql language: SQL +slp: + scripts: + - script: ego_pre_slp_parameters.sql + language: SQL diff --git a/preprocessing/sql_snippets/ego_pre_slp_parameters.sql b/preprocessing/slp/ego_pre_slp_parameters.sql similarity index 95% rename from preprocessing/sql_snippets/ego_pre_slp_parameters.sql rename to preprocessing/slp/ego_pre_slp_parameters.sql index 5dc6b424..b9cd6e68 100644 --- a/preprocessing/sql_snippets/ego_pre_slp_parameters.sql +++ b/preprocessing/slp/ego_pre_slp_parameters.sql @@ -10,6 +10,7 @@ __author__ = "gplssm" */ -- create empty parameters table +CREATE SCHEMA IF NOT EXISTS scenario; DROP TABLE IF EXISTS scenario.ego_slp_parameters CASCADE; CREATE TABLE scenario.ego_slp_parameters ( parameter text, @@ -72,4 +73,4 @@ SELECT obj_description('scenario.ego_slp_parameters' ::regclass) ::json; -- ego scenario log (version,io,schema_name,table_name,script_name,comment) -SELECT ego_scenario_log('v0.2.8','input','scenario','ego_slp_parameters','ego_slp_parameters.sql','Create SLP parameter table'); +SELECT scenario_log('eGo_PP','PP1','input','scenario','ego_slp_parameters','ego_slp_parameters.sql','Create SLP parameter table'); From 0ec21064d83dbaabc619f60521de0a3c8155a629 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guido=20Ple=C3=9Fmann?= Date: Sun, 29 Mar 2020 15:20:14 +0200 Subject: [PATCH 40/84] Add BPMN for SLP parameters creation script Fix #331 --- .../bpmn/ego_dp-pre_bpmn_section_slp.graphml | 244 ++++++++++++++++++ 1 file changed, 244 insertions(+) create mode 100644 documentation/bpmn/ego_dp-pre_bpmn_section_slp.graphml diff --git a/documentation/bpmn/ego_dp-pre_bpmn_section_slp.graphml b/documentation/bpmn/ego_dp-pre_bpmn_section_slp.graphml new file mode 100644 index 00000000..2504ad6a --- /dev/null +++ b/documentation/bpmn/ego_dp-pre_bpmn_section_slp.graphml @@ -0,0 +1,244 @@ + + + + + + + + + + + + + + + + + + + + + + + eGo Pre-Processing - slp + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ego_pre_slp_parameters.sql + + + + + + + + + + + + + + + + + + scenario.ego_slp_parameters + + + + + + + + + + + + + + + + + 0.01s + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Open Database License (ODbL) v1.0 +© OpenStreetMap contributors + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + From 28163b0797f2852ab1327192e099532734437679 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guido=20Ple=C3=9Fmann?= Date: Mon, 30 Mar 2020 09:57:18 +0200 Subject: [PATCH 41/84] Generalize SHP downloader --- preprocessing/__init__.py | 0 preprocessing/utility/__init__.py | 0 .../utility/download_oedb_data_to_shp.py | 51 ++++++++++++++++++ .../vernetzen_wind_potential_areas_to_shp.py | 52 +------------------ 4 files changed, 53 insertions(+), 50 deletions(-) create mode 100644 preprocessing/__init__.py create mode 100644 preprocessing/utility/__init__.py create mode 100644 preprocessing/utility/download_oedb_data_to_shp.py diff --git a/preprocessing/__init__.py b/preprocessing/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/preprocessing/utility/__init__.py b/preprocessing/utility/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/preprocessing/utility/download_oedb_data_to_shp.py b/preprocessing/utility/download_oedb_data_to_shp.py new file mode 100644 index 00000000..fbf928c7 --- /dev/null +++ b/preprocessing/utility/download_oedb_data_to_shp.py @@ -0,0 +1,51 @@ +import pandas as pd +from egoio.tools import db +from sqlalchemy.orm import sessionmaker +from sqlalchemy import func +import geopandas as gdp +from shapely import wkt +import zipfile +import os + + +DOWNLOADDIR = os.path.join(os.path.expanduser("~"), ".egon-pre-processing-cached/") + + +def zipdir(path, ziph): + # ziph is zipfile handle + for root, dirs, files in os.walk(path): + for file in files: + ziph.write(os.path.join(root, file), arcname=file) + + +def oedbtable2shp(table_orm, filename, crs, geometry_col, index_col): + + shp_dir = os.path.join(DOWNLOADDIR, filename.replace(".shp", "")) + os.makedirs(shp_dir, exist_ok=True) + shp_filename = os.path.join(shp_dir, filename) + zip_filename = os.path.join(DOWNLOADDIR, filename.replace(".shp", ".zip")) + + engine_oedb = db.connection(readonly=True) + session = sessionmaker(bind=engine_oedb)() + + table_query = session.query( + table_orm.region_key, + func.ST_AsText(func.ST_Transform(table_orm.geom, crs)).label(geometry_col)) + + table_df = pd.read_sql_query(table_query.statement, + session.bind, + index_col=index_col) + + table_df[geometry_col] = table_df[geometry_col].apply(wkt.loads) + table_gdf = gdp.GeoDataFrame(table_df, + geometry=geometry_col, + crs={'init': 'epsg:{}'.format(crs)}) + + table_gdf.to_file(shp_filename) + + zf = zipfile.ZipFile( + zip_filename, + mode='w', + compression=zipfile.ZIP_DEFLATED) + zipdir(shp_dir, zf) + zf.close() \ No newline at end of file diff --git a/preprocessing/utility/vernetzen_wind_potential_areas_to_shp.py b/preprocessing/utility/vernetzen_wind_potential_areas_to_shp.py index 9da70b49..6ebf01e7 100644 --- a/preprocessing/utility/vernetzen_wind_potential_areas_to_shp.py +++ b/preprocessing/utility/vernetzen_wind_potential_areas_to_shp.py @@ -1,63 +1,15 @@ -import pandas as pd -from egoio.tools import db from egoio.db_tables.supply import VernetzenWindPotentialArea -from sqlalchemy.orm import sessionmaker -from sqlalchemy import func -import geopandas as gdp -from shapely import wkt import os -import zipfile +from preprocessing.utility.download_oedb_data_to_shp import oedbtable2shp -DOWNLOADDIR = os.path.join(os.path.expanduser("~"), ".egon-pre-processing-cached/") CRS = 4326 def vernetzen_windpotential_areas2shp(): filename = "vernetzen_wind_potential_areas.shp" - oedbtable2shp(VernetzenWindPotentialArea, filename) - - - -def zipdir(path, ziph): - # ziph is zipfile handle - for root, dirs, files in os.walk(path): - for file in files: - ziph.write(os.path.join(root, file), arcname=file) - - -def oedbtable2shp(table_orm, filename): - - shp_dir = os.path.join(DOWNLOADDIR, filename.replace(".shp", "")) - os.makedirs(shp_dir, exist_ok=True) - shp_filename = os.path.join(shp_dir, filename) - zip_filename = os.path.join(DOWNLOADDIR, filename.replace(".shp", ".zip")) - - engine_oedb = db.connection(readonly=True) - session = sessionmaker(bind=engine_oedb)() - - table_query = session.query( - table_orm.region_key, - func.ST_AsText(func.ST_Transform(table_orm.geom, CRS)).label('geom')) - - table_df = pd.read_sql_query(table_query.statement, - session.bind, - index_col='region_key') - - table_df["geom"] = table_df["geom"].apply(wkt.loads) - table_gdf = gdp.GeoDataFrame(table_df, - geometry="geom", - crs={'init': 'epsg:{}'.format(CRS)}) - - table_gdf.to_file(shp_filename) - - zf = zipfile.ZipFile( - zip_filename, - mode='w', - compression=zipfile.ZIP_DEFLATED) - zipdir(shp_dir, zf) - zf.close() + oedbtable2shp(VernetzenWindPotentialArea, filename, CRS, "geom", "region_key") if __name__== "__main__": From be3c38100d8ac179af1026020b08718f2ce3390f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guido=20Ple=C3=9Fmann?= Date: Mon, 30 Mar 2020 10:56:13 +0200 Subject: [PATCH 42/84] Rework shp download scripts --- preprocessing/utility/cosmoclmgrid_to_shp.py | 26 +++++++++++++++++++ .../utility/download_oedb_data_to_shp.py | 9 ++----- .../vernetzen_wind_potential_areas_to_shp.py | 14 +++++++++- 3 files changed, 41 insertions(+), 8 deletions(-) create mode 100644 preprocessing/utility/cosmoclmgrid_to_shp.py diff --git a/preprocessing/utility/cosmoclmgrid_to_shp.py b/preprocessing/utility/cosmoclmgrid_to_shp.py new file mode 100644 index 00000000..9658a9ef --- /dev/null +++ b/preprocessing/utility/cosmoclmgrid_to_shp.py @@ -0,0 +1,26 @@ +from egoio.db_tables.climate import Cosmoclmgrid +from preprocessing.utility.download_oedb_data_to_shp import oedbtable2shp +from sqlalchemy import func +from sqlalchemy.orm import sessionmaker +from egoio.tools import db + + +CRS = 4326 + + +def cosmoclmgrid2shp(): + + filename = "cosmoclmgrid.shp" + + engine_oedb = db.connection(readonly=True) + session = sessionmaker(bind=engine_oedb)() + + query = session.query(Cosmoclmgrid.gid, + func.ST_AsText(func.ST_Transform(Cosmoclmgrid.geom, CRS)).label("geom")) + + oedbtable2shp(query, filename, CRS, "geom", "gid") + + + +if __name__== "__main__": + cosmoclmgrid2shp() \ No newline at end of file diff --git a/preprocessing/utility/download_oedb_data_to_shp.py b/preprocessing/utility/download_oedb_data_to_shp.py index fbf928c7..ba061a49 100644 --- a/preprocessing/utility/download_oedb_data_to_shp.py +++ b/preprocessing/utility/download_oedb_data_to_shp.py @@ -1,7 +1,6 @@ import pandas as pd from egoio.tools import db from sqlalchemy.orm import sessionmaker -from sqlalchemy import func import geopandas as gdp from shapely import wkt import zipfile @@ -18,7 +17,7 @@ def zipdir(path, ziph): ziph.write(os.path.join(root, file), arcname=file) -def oedbtable2shp(table_orm, filename, crs, geometry_col, index_col): +def oedbtable2shp(table_query, filename, crs, geometry_col, index_col): shp_dir = os.path.join(DOWNLOADDIR, filename.replace(".shp", "")) os.makedirs(shp_dir, exist_ok=True) @@ -28,10 +27,6 @@ def oedbtable2shp(table_orm, filename, crs, geometry_col, index_col): engine_oedb = db.connection(readonly=True) session = sessionmaker(bind=engine_oedb)() - table_query = session.query( - table_orm.region_key, - func.ST_AsText(func.ST_Transform(table_orm.geom, crs)).label(geometry_col)) - table_df = pd.read_sql_query(table_query.statement, session.bind, index_col=index_col) @@ -39,7 +34,7 @@ def oedbtable2shp(table_orm, filename, crs, geometry_col, index_col): table_df[geometry_col] = table_df[geometry_col].apply(wkt.loads) table_gdf = gdp.GeoDataFrame(table_df, geometry=geometry_col, - crs={'init': 'epsg:{}'.format(crs)}) + crs=crs) table_gdf.to_file(shp_filename) diff --git a/preprocessing/utility/vernetzen_wind_potential_areas_to_shp.py b/preprocessing/utility/vernetzen_wind_potential_areas_to_shp.py index 6ebf01e7..a53bfde8 100644 --- a/preprocessing/utility/vernetzen_wind_potential_areas_to_shp.py +++ b/preprocessing/utility/vernetzen_wind_potential_areas_to_shp.py @@ -1,6 +1,10 @@ from egoio.db_tables.supply import VernetzenWindPotentialArea import os from preprocessing.utility.download_oedb_data_to_shp import oedbtable2shp +from sqlalchemy import func +from sqlalchemy.orm import sessionmaker +from egoio.tools import db + CRS = 4326 @@ -9,7 +13,15 @@ def vernetzen_windpotential_areas2shp(): filename = "vernetzen_wind_potential_areas.shp" - oedbtable2shp(VernetzenWindPotentialArea, filename, CRS, "geom", "region_key") + + engine_oedb = db.connection(readonly=True) + session = sessionmaker(bind=engine_oedb)() + + table_query = session.query( + VernetzenWindPotentialArea.region_key, + func.ST_AsText(func.ST_Transform(VernetzenWindPotentialArea.geom, CRS)).label("geom")) + + oedbtable2shp(table_query, filename, CRS, "geom", "region_key") if __name__== "__main__": From b7baf90de2393d195e1ab652bf6f72c6b6bafa8a Mon Sep 17 00:00:00 2001 From: IlkaCu Date: Tue, 31 Mar 2020 12:16:24 +0200 Subject: [PATCH 43/84] include input data and mark missing information in red --- .../bpmn/ego_pp_bpmn_section_supply.graphml | 551 ++++++++++++++++++ 1 file changed, 551 insertions(+) create mode 100644 documentation/bpmn/ego_pp_bpmn_section_supply.graphml diff --git a/documentation/bpmn/ego_pp_bpmn_section_supply.graphml b/documentation/bpmn/ego_pp_bpmn_section_supply.graphml new file mode 100644 index 00000000..c1d0f127 --- /dev/null +++ b/documentation/bpmn/ego_pp_bpmn_section_supply.graphml @@ -0,0 +1,551 @@ + + + + + + + + + + + + + + + + + + + + + + + eGo Preprocessing - simple feedin + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + model_draft.ego_renewable_feedin +model_draft.ego_power_class + + + + + + + + + + + + + + + + + renpass_gis/simple_feedin.py + + + + + + + + + + + + + + + + + + model_draft.ego_supply_res_powerplant +model_draft.ego_supply_renewable_bnetza_full_attribute +model_draft.bnetza_eeganlagenstammdaten_wind_classification + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + OPSD beta Version of 2015/16 + + + + + + + + + + + + + + + + + ego_dp_preprocessing_conv_powerplant.sql + + + + + + + + + + + + + + + + + + model_draft.ego_dp_supply_conv_powerplant +model_draft.ego_supply_conv_nep2035_temp + + + + + + + + + + + + + + + + + model_draft.ego_dp_supply_conv_powerplant + + + + + + + + + + + + + + + + + opsd_powerdata.sql + + + + + + + + + + + + + + + + + + supply.ego_conventional_powerplant + + + + + + + + + + + + + + + + + supply_ego_renewable_power_plants_germany.sql + + + + + + + + + + + + + + + + + + supply.ego_renewable_powerplant + + + + + + + + + + + + + + + + + energymap +BNetzA + + + + + + + + + + + + + + + + + ego_dp_preprocessing_res_powerplant.sql + + + + + + + + + + + + + + + + + + model_draft.ego_dp_supply_res_powerplant + + + + + + + + + + + + + + + + + + model_draft.ego_supply_res_powerplant + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + From f371409dcbef77315f9a7ab99e91a8e816915ac1 Mon Sep 17 00:00:00 2001 From: IlkaCu Date: Tue, 31 Mar 2020 12:19:47 +0200 Subject: [PATCH 44/84] include additional input data and mark missing information in red --- .../ego_dp-pre_bpmn_section_supply.graphml | 629 ++++++++++-------- 1 file changed, 369 insertions(+), 260 deletions(-) diff --git a/documentation/bpmn/ego_dp-pre_bpmn_section_supply.graphml b/documentation/bpmn/ego_dp-pre_bpmn_section_supply.graphml index 26c9bbd4..372db54b 100644 --- a/documentation/bpmn/ego_dp-pre_bpmn_section_supply.graphml +++ b/documentation/bpmn/ego_dp-pre_bpmn_section_supply.graphml @@ -17,11 +17,11 @@ - + - eGo Pre-Processing - supply - + eGo Pre-Processing - supply + @@ -53,18 +53,18 @@ - + - + - + @@ -75,7 +75,7 @@ - + preprocessing/python_scripts/ @@ -94,10 +94,11 @@ renpass_gis/simple_feedin/simple_feedin.py - + - model_draft.ego_renewable_feedin + model_draft.ego_renewable_feedin +model_draft.ego_power_class @@ -109,25 +110,9 @@ renpass_gis/simple_feedin/simple_feedin.py - - - - - - ? - - - - - - - - - - - + @@ -148,10 +133,10 @@ renpass_gis/simple_feedin/simple_feedin.py - + - + @@ -172,54 +157,76 @@ renpass_gis/simple_feedin/simple_feedin.py + + + + + + + ego_dp_res_rea_by_scenario.sql + + + + + + + + + + + - - + + - Open Database License (ODbL) v1.0 -© OpenStreetMap contributors + model_draft.ego_dp_supply_res_powerplant + + - + - - - - - ego_dp_preprocessing_res_powerplant.sql + + + + + model_draft.ego_dp_supply_res_powerplant +boundaries.bkg_vg250_2_lan_nuts_view +model_draft.ego_supply_scenario_capacities +boundaries.bkg_vg250_6_gem_rs_mview - + - - + - - - - - model_draft.ego_dp_supply_res_powerplant + + + + + ego_dp_preprocessing_res_powerplant_2035.sql - + - + + @@ -227,16 +234,17 @@ renpass_gis/simple_feedin/simple_feedin.py - - + + - Open Database License (ODbL) v1.0 -© OpenStreetMap contributors + model_draft.ego_dp_supply_res_powerplant + + - + @@ -244,10 +252,11 @@ renpass_gis/simple_feedin/simple_feedin.py - + - model_draft.ego_supply_res_powerplant + model_draft.ego_dp_supply_res_powerplant + @@ -261,10 +270,10 @@ renpass_gis/simple_feedin/simple_feedin.py - + - ego_dp_preprocessing_conv_powerplant.sql + ego_dp_preprocessing_conv_powerplant_2035.sql @@ -279,11 +288,11 @@ renpass_gis/simple_feedin/simple_feedin.py - + - model_draft.ego_dp_supply_conv_powerplant -model_draft.ego_supply_conv_nep2035_temp + model_draft.ego_dp_supply_conv_powerplant_2035 + @@ -297,27 +306,11 @@ model_draft.ego_supply_conv_nep2035_temp - - - - Open Database License (ODbL) v1.0 -© OpenStreetMap contributors - - - - - - - - - - - - - + - model_draft.ego_supply_conv_powerplant + model_draft.ego_dp_supply_conv_powerplant + @@ -328,13 +321,13 @@ model_draft.ego_supply_conv_nep2035_temp - + - + - ego_dp_res_rea_by_scenario.sql + ego_dp_conv_by_scenario.sql @@ -346,13 +339,13 @@ model_draft.ego_supply_conv_nep2035_temp - + - + - model_draft.ego_dp_supply_res_powerplant + model_draft.ego_dp_supply_conv_powerplant @@ -364,33 +357,14 @@ model_draft.ego_supply_conv_nep2035_temp - - - - - - - Open Database License (ODbL) v1.0 -© OpenStreetMap contributors - - - - - - - - - - + - + - model_draft.ego_dp_supply_res_powerplant -boundaries.bkg_vg250_2_lan_nuts_view -model_draft.ego_supply_scenario_capacities -boundaries.bkg_vg250_6_gem_rs_mview + model_draft.ego_dp_supply_conv_powerplant + @@ -401,13 +375,13 @@ boundaries.bkg_vg250_6_gem_rs_mview - + - + - ego_dp_preprocessing_res_powerplant_2035.sql + ego_dp_supply_add_coastdat_gid.sql @@ -419,13 +393,14 @@ boundaries.bkg_vg250_6_gem_rs_mview - + - + - model_draft.ego_dp_supply_res_powerplant + model_draft.ego_dp_supply_conv_powerplant +model_draft.ego_dp_supply_res_powerplant @@ -437,90 +412,129 @@ boundaries.bkg_vg250_6_gem_rs_mview - + - - + + - Open Database License (ODbL) v1.0 -© OpenStreetMap contributors + model_draft.ego_dp_supply_conv_powerplant +model_draft.ego_dp_supply_res_powerplant +coastdat.cosmoclmgrid + + - + - + - + - model_draft.ego_dp_supply_res_powerplant - + model_draft.ego_supply_res_powerplant +model_draft.ego_supply_renewable_bnetza_full_attribute +model_draft.bnetza_eeganlagenstammdaten_wind_classification + + + + + + + + + + + + + + OPSD beta Version of 2015/16 + + + - + - + - ego_dp_preprocessing_conv_powerplant_2035.sql + ego_dp_preprocessing_conv_powerplant.sql - + - + - + - model_draft.ego_dp_supply_conv_powerplant_2035 - + model_draft.ego_dp_supply_conv_powerplant +model_draft.ego_supply_conv_nep2035_temp - + - + - - + + - Open Database License (ODbL) v1.0 -© OpenStreetMap contributors + model_draft.ego_dp_supply_conv_powerplant + + + + + + + + + + + + + + + + + opsd_powerdata.sql + - + + @@ -528,17 +542,16 @@ boundaries.bkg_vg250_6_gem_rs_mview - - + + - model_draft.ego_dp_supply_conv_powerplant - + supply.ego_conventional_powerplant - + @@ -546,17 +559,17 @@ boundaries.bkg_vg250_6_gem_rs_mview - + - ego_dp_conv_by_scenario.sql + supply_ego_renewable_power_plants_germany.sql - + @@ -564,17 +577,16 @@ boundaries.bkg_vg250_6_gem_rs_mview - + - model_draft.ego_dp_supply_conv_powerplant - + supply.ego_renewable_powerplant - + @@ -582,52 +594,53 @@ boundaries.bkg_vg250_6_gem_rs_mview - - + + - Open Database License (ODbL) v1.0 -© OpenStreetMap contributors + energymap +BNetzA - + + - - - - - model_draft.ego_dp_supply_conv_powerplant - + + + + + ego_dp_preprocessing_res_powerplant.sql - + - + + - - - - - ego_dp_supply_add_coastdat_gid.sql + + + + + model_draft.ego_dp_supply_res_powerplant + - - - + + @@ -635,74 +648,76 @@ boundaries.bkg_vg250_6_gem_rs_mview - + - model_draft.ego_dp_supply_conv_powerplant -model_draft.ego_dp_supply_res_powerplant - + model_draft.ego_supply_res_powerplant - + + + + + + + ? + + + + + + + + + + + + - - + + - Open Database License (ODbL) v1.0 -© OpenStreetMap contributors + model_draft.ego_supply_res_powerplant + - + + - + - - + + - model_draft.ego_dp_supply_conv_powerplant -model_draft.ego_dp_supply_res_powerplant -coastdat.cosmoclmgrid - + supply.ego_renewable_powerplant - + - - - - - - - - - - - - - + @@ -714,29 +729,27 @@ coastdat.cosmoclmgrid - + - + - - - - - - + + + + - + - - + + - + @@ -746,7 +759,7 @@ coastdat.cosmoclmgrid - + @@ -756,7 +769,7 @@ coastdat.cosmoclmgrid - + @@ -766,7 +779,7 @@ coastdat.cosmoclmgrid - + @@ -776,7 +789,7 @@ coastdat.cosmoclmgrid - + @@ -786,7 +799,7 @@ coastdat.cosmoclmgrid - + @@ -796,7 +809,7 @@ coastdat.cosmoclmgrid - + @@ -806,7 +819,7 @@ coastdat.cosmoclmgrid - + @@ -816,132 +829,228 @@ coastdat.cosmoclmgrid - + - - + + - + - - + + - + - - + + - + - - + + - + - - + + - + + - + - + + - - + + - + + - - + + - + + - - + + - + + - - + + - + + - - + + - + + - - + + - + + - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + From f2a812581df49273dee7d7872d5362b24d8dda9c Mon Sep 17 00:00:00 2001 From: IlkaCu Date: Wed, 1 Apr 2020 12:47:59 +0200 Subject: [PATCH 45/84] add additional scripts to supply section in preprocessing --- .../ego_dp-pre_bpmn_section_supply.graphml | 510 +++++++++++------- 1 file changed, 304 insertions(+), 206 deletions(-) diff --git a/documentation/bpmn/ego_dp-pre_bpmn_section_supply.graphml b/documentation/bpmn/ego_dp-pre_bpmn_section_supply.graphml index 372db54b..49696547 100644 --- a/documentation/bpmn/ego_dp-pre_bpmn_section_supply.graphml +++ b/documentation/bpmn/ego_dp-pre_bpmn_section_supply.graphml @@ -17,11 +17,11 @@ - + - eGo Pre-Processing - supply - + eGo Pre-Processing - supply + @@ -53,18 +53,18 @@ - + - + - + @@ -75,7 +75,7 @@ - + preprocessing/python_scripts/ @@ -94,7 +94,7 @@ renpass_gis/simple_feedin/simple_feedin.py - + model_draft.ego_renewable_feedin @@ -112,7 +112,7 @@ model_draft.ego_power_class - + @@ -136,7 +136,7 @@ model_draft.ego_power_class - + @@ -160,7 +160,7 @@ model_draft.ego_power_class - + ego_dp_res_rea_by_scenario.sql @@ -178,8 +178,8 @@ model_draft.ego_power_class - - + + model_draft.ego_dp_supply_res_powerplant @@ -196,12 +196,14 @@ model_draft.ego_power_class - + - model_draft.ego_dp_supply_res_powerplant -boundaries.bkg_vg250_2_lan_nuts_view + model_draft.ego_small_chp_plant_germany +model_draft.ego_dp_supply_res_powerplant +model_draft.ego_dp_supply_conv_powerplant model_draft.ego_supply_scenario_capacities +boundaries.bkg_vg250_2_lan_nuts_view boundaries.bkg_vg250_6_gem_rs_mview @@ -214,30 +216,12 @@ boundaries.bkg_vg250_6_gem_rs_mview - - - - - - ego_dp_preprocessing_res_powerplant_2035.sql - - - - - - - - - - - - - + - model_draft.ego_dp_supply_res_powerplant + model_draft.ego_supply_conv_powerplant_2035 @@ -249,13 +233,13 @@ boundaries.bkg_vg250_6_gem_rs_mview - + - + - model_draft.ego_dp_supply_res_powerplant + model_draft.ego_dp_supply_conv_powerplant @@ -267,13 +251,13 @@ boundaries.bkg_vg250_6_gem_rs_mview - + - + - ego_dp_preprocessing_conv_powerplant_2035.sql + ego_dp_conv_by_scenario.sql @@ -285,13 +269,13 @@ boundaries.bkg_vg250_6_gem_rs_mview - + - - + + - model_draft.ego_dp_supply_conv_powerplant_2035 + model_draft.ego_dp_supply_conv_powerplant @@ -303,14 +287,14 @@ boundaries.bkg_vg250_6_gem_rs_mview - + - + - model_draft.ego_dp_supply_conv_powerplant - + model_draft.ego_dp_supply_conv_powerplant +model_draft.ego_supply_conv_powerplant_2035 @@ -321,13 +305,13 @@ boundaries.bkg_vg250_6_gem_rs_mview - + - + - ego_dp_conv_by_scenario.sql + ego_dp_supply_add_coastdat_gid.sql @@ -339,13 +323,14 @@ boundaries.bkg_vg250_6_gem_rs_mview - + - - + + - model_draft.ego_dp_supply_conv_powerplant + model_draft.ego_dp_supply_conv_powerplant +model_draft.ego_dp_supply_res_powerplant @@ -357,13 +342,15 @@ boundaries.bkg_vg250_6_gem_rs_mview - + - + - model_draft.ego_dp_supply_conv_powerplant + model_draft.ego_dp_supply_conv_powerplant +model_draft.ego_dp_supply_res_powerplant +coastdat.cosmoclmgrid @@ -375,39 +362,56 @@ boundaries.bkg_vg250_6_gem_rs_mview - + - - - - - ego_dp_supply_add_coastdat_gid.sql + + + + + model_draft.ego_supply_res_powerplant +model_draft.ego_supply_renewable_bnetza_full_attribute +model_draft.bnetza_eeganlagenstammdaten_wind_classification - - - + + - + - - + + - model_draft.ego_dp_supply_conv_powerplant -model_draft.ego_dp_supply_res_powerplant - + OPSD beta Version of 2015/16 - + + + + + + + + + + + + ego_dp_preprocessing_conv_powerplant.sql + + + + + + + @@ -415,38 +419,35 @@ model_draft.ego_dp_supply_res_powerplant - - + + - model_draft.ego_dp_supply_conv_powerplant -model_draft.ego_dp_supply_res_powerplant -coastdat.cosmoclmgrid - + model_draft.ego_dp_supply_conv_powerplant +model_draft.ego_supply_conv_nep2035_temp - + - - - - - model_draft.ego_supply_res_powerplant -model_draft.ego_supply_renewable_bnetza_full_attribute -model_draft.bnetza_eeganlagenstammdaten_wind_classification + + + + + opsd_powerdata.sql + - - + + @@ -454,16 +455,16 @@ model_draft.bnetza_eeganlagenstammdaten_wind_classification - - + + - OPSD beta Version of 2015/16 + supply.ego_conventional_powerplant - + @@ -471,10 +472,10 @@ model_draft.bnetza_eeganlagenstammdaten_wind_classification - + - ego_dp_preprocessing_conv_powerplant.sql + supply_ego_renewable_power_plants_germany.sql @@ -489,11 +490,10 @@ model_draft.bnetza_eeganlagenstammdaten_wind_classification - + - model_draft.ego_dp_supply_conv_powerplant -model_draft.ego_supply_conv_nep2035_temp + supply.ego_renewable_powerplant @@ -507,16 +507,17 @@ model_draft.ego_supply_conv_nep2035_temp - - + + - model_draft.ego_dp_supply_conv_powerplant + energymap +BNetzA - + @@ -524,10 +525,10 @@ model_draft.ego_supply_conv_nep2035_temp - + - opsd_powerdata.sql + ego_dp_preprocessing_res_powerplant.sql @@ -542,10 +543,11 @@ model_draft.ego_supply_conv_nep2035_temp - + - supply.ego_conventional_powerplant + model_draft.ego_dp_supply_res_powerplant + @@ -559,10 +561,10 @@ model_draft.ego_supply_conv_nep2035_temp - + - supply_ego_renewable_power_plants_germany.sql + ? @@ -577,10 +579,11 @@ model_draft.ego_supply_conv_nep2035_temp - + - supply.ego_renewable_powerplant + model_draft.ego_supply_res_powerplant + @@ -594,11 +597,11 @@ model_draft.ego_supply_conv_nep2035_temp - - + + - energymap -BNetzA + model_draft.ego_dp_supply_res_powerplant + @@ -611,18 +614,17 @@ BNetzA - - - - - ego_dp_preprocessing_res_powerplant.sql + + + + + supply.ego_renewable_powerplant - - - + + @@ -630,17 +632,16 @@ BNetzA - - + + - model_draft.ego_dp_supply_res_powerplant - + model_draft.ego_dp_supply_conv_powerplant - + @@ -648,16 +649,16 @@ BNetzA - - + + - model_draft.ego_supply_res_powerplant + bkg_vg250 - + @@ -665,17 +666,17 @@ BNetzA - + - ? + ego_nep_2015_scenario_capacities.sql - + @@ -683,11 +684,10 @@ BNetzA - + - model_draft.ego_supply_res_powerplant - + model_draft.ego_supply_scenario_capacities @@ -701,16 +701,52 @@ BNetzA - - + + - supply.ego_renewable_powerplant + NEP 2015 +FlEnS - + + + + + + + + + + + + utility/ego_dp_preprocessing_conv_powerplant_2035.sql + + + + + + + + + + + + + + + + + + coastdat + + + + + + @@ -749,7 +785,7 @@ BNetzA - + @@ -759,7 +795,7 @@ BNetzA - + @@ -769,7 +805,7 @@ BNetzA - + @@ -779,7 +815,7 @@ BNetzA - + @@ -789,108 +825,115 @@ BNetzA - + - - + + - + - - + + - + + - + - + + - - + + - + + - - + + - + + - - + + - + + - - + + - + + - - + + - + + - - + + - + - + - + @@ -901,7 +944,7 @@ BNetzA - + @@ -912,29 +955,29 @@ BNetzA - + - - + + - + - - + + - + @@ -945,57 +988,57 @@ BNetzA - + - + - + - - + + - + - - + + - + - - + + - + - - + + @@ -1005,13 +1048,13 @@ BNetzA - - + + - + @@ -1022,7 +1065,7 @@ BNetzA - + @@ -1033,18 +1076,73 @@ BNetzA - + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + From 3fab1adc978b161ef48d98ee59c9729e453f4061 Mon Sep 17 00:00:00 2001 From: IlkaCu Date: Wed, 1 Apr 2020 12:53:27 +0200 Subject: [PATCH 46/84] move redundant script to utility folder --- .../{sql_snippets => utility}/ego_weather_points_per_scenario.sql | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename preprocessing/{sql_snippets => utility}/ego_weather_points_per_scenario.sql (100%) diff --git a/preprocessing/sql_snippets/ego_weather_points_per_scenario.sql b/preprocessing/utility/ego_weather_points_per_scenario.sql similarity index 100% rename from preprocessing/sql_snippets/ego_weather_points_per_scenario.sql rename to preprocessing/utility/ego_weather_points_per_scenario.sql From 28ffd76d30b8d38ec2d5b11ec9f91f738c509167 Mon Sep 17 00:00:00 2001 From: IlkaCu Date: Thu, 2 Apr 2020 17:27:57 +0200 Subject: [PATCH 47/84] mark problems in red --- .../ego_dp-pre_bpmn_section_supply.graphml | 490 ++++++++++++------ 1 file changed, 321 insertions(+), 169 deletions(-) diff --git a/documentation/bpmn/ego_dp-pre_bpmn_section_supply.graphml b/documentation/bpmn/ego_dp-pre_bpmn_section_supply.graphml index 49696547..ba8a7fd8 100644 --- a/documentation/bpmn/ego_dp-pre_bpmn_section_supply.graphml +++ b/documentation/bpmn/ego_dp-pre_bpmn_section_supply.graphml @@ -17,11 +17,11 @@ - + - eGo Pre-Processing - supply - + eGo Pre-Processing - supply + @@ -31,40 +31,40 @@ - - - - + + + + + - - + - + - + @@ -76,7 +76,7 @@ - + preprocessing/python_scripts/ renpass_gis/simple_feedin/simple_feedin.py @@ -112,7 +112,7 @@ model_draft.ego_power_class - + @@ -136,7 +136,7 @@ model_draft.ego_power_class - + @@ -199,7 +199,7 @@ model_draft.ego_power_class - model_draft.ego_small_chp_plant_germany + model_draft.ego_small_chp_plant_germany model_draft.ego_dp_supply_res_powerplant model_draft.ego_dp_supply_conv_powerplant model_draft.ego_supply_scenario_capacities @@ -218,7 +218,7 @@ boundaries.bkg_vg250_6_gem_rs_mview - + model_draft.ego_supply_conv_powerplant_2035 @@ -236,7 +236,7 @@ boundaries.bkg_vg250_6_gem_rs_mview - + model_draft.ego_dp_supply_conv_powerplant @@ -254,7 +254,7 @@ boundaries.bkg_vg250_6_gem_rs_mview - + ego_dp_conv_by_scenario.sql @@ -272,7 +272,7 @@ boundaries.bkg_vg250_6_gem_rs_mview - + model_draft.ego_dp_supply_conv_powerplant @@ -290,7 +290,7 @@ boundaries.bkg_vg250_6_gem_rs_mview - + model_draft.ego_dp_supply_conv_powerplant @@ -308,7 +308,7 @@ model_draft.ego_supply_conv_powerplant_2035 - + ego_dp_supply_add_coastdat_gid.sql @@ -326,7 +326,7 @@ model_draft.ego_supply_conv_powerplant_2035 - + model_draft.ego_dp_supply_conv_powerplant @@ -345,10 +345,10 @@ model_draft.ego_dp_supply_res_powerplant - + - model_draft.ego_dp_supply_conv_powerplant + model_draft.ego_dp_supply_conv_powerplant model_draft.ego_dp_supply_res_powerplant coastdat.cosmoclmgrid @@ -373,10 +373,10 @@ model_draft.ego_supply_renewable_bnetza_full_attribute model_draft.bnetza_eeganlagenstammdaten_wind_classification + - @@ -384,16 +384,16 @@ model_draft.bnetza_eeganlagenstammdaten_wind_classification - + OPSD beta Version of 2015/16 + - @@ -401,10 +401,11 @@ model_draft.bnetza_eeganlagenstammdaten_wind_classification - + - ego_dp_preprocessing_conv_powerplant.sql + ego_dp_preprocessing_conv_powerplant.sql +LOOP! @@ -419,17 +420,17 @@ model_draft.bnetza_eeganlagenstammdaten_wind_classification - + - model_draft.ego_dp_supply_conv_powerplant + model_draft.ego_dp_supply_conv_powerplant model_draft.ego_supply_conv_nep2035_temp + - @@ -437,10 +438,11 @@ model_draft.ego_supply_conv_nep2035_temp - - + + - opsd_powerdata.sql + supply_ego_renewable_power_plants_germany.sql +Nur CREATE keine Daten! @@ -455,16 +457,17 @@ model_draft.ego_supply_conv_nep2035_temp - - + + - supply.ego_conventional_powerplant + energymap +BNetzA + - @@ -472,10 +475,10 @@ model_draft.ego_supply_conv_nep2035_temp - + - supply_ego_renewable_power_plants_germany.sql + ego_dp_preprocessing_res_powerplant.sql @@ -490,16 +493,17 @@ model_draft.ego_supply_conv_nep2035_temp - + - supply.ego_renewable_powerplant + model_draft.ego_dp_supply_res_powerplant + + - @@ -507,35 +511,34 @@ model_draft.ego_supply_conv_nep2035_temp - - + + - energymap -BNetzA + model_draft.ego_supply_res_powerplant + + - - - - - - ego_dp_preprocessing_res_powerplant.sql + + + + + supply.ego_renewable_powerplant - + - - + @@ -543,53 +546,52 @@ BNetzA - - + + - model_draft.ego_dp_supply_res_powerplant - + supply.ego_conventional_powerplant +model_draft.ego_dp_supply_conv_powerplant + - - - - - - ? + + + + + bkg_vg250 - + - - + - - - - - model_draft.ego_supply_res_powerplant - + + + + + ego_nep_2015_scenario_capacities.sql + - - + + @@ -597,17 +599,16 @@ BNetzA - - + + - model_draft.ego_dp_supply_res_powerplant - + model_draft.ego_supply_scenario_capacities + - @@ -615,33 +616,35 @@ BNetzA - - + + - supply.ego_renewable_powerplant + NEP 2015 +FlEnS + - - - - - - model_draft.ego_dp_supply_conv_powerplant + + + + + utility/ego_dp_preprocessing_conv_powerplant_2035.sql + - - + + @@ -649,16 +652,16 @@ BNetzA - + - bkg_vg250 + coastdat + - @@ -666,10 +669,10 @@ BNetzA - + - ego_nep_2015_scenario_capacities.sql + ego_simple-feedin_per_scenario.sql @@ -684,16 +687,17 @@ BNetzA - + - model_draft.ego_supply_scenario_capacities + model_draft.ego_neighbours_offshore_point +model_draft.ego_weather_measurement_point + - @@ -701,17 +705,19 @@ BNetzA - - + + - NEP 2015 -FlEnS + climate.cosmoclmgrid +boundaries.bkg_vg250_1_sta +model_draft.renpass_gis_parameter_region +model_draft.ego_grid_hv_electrical_neighbours_bus + - @@ -719,28 +725,66 @@ FlEnS - + - utility/ego_dp_preprocessing_conv_powerplant_2035.sql + opsd_powerdata.sql +Nur CREATE keine Daten! - + + + + + + + ? Kein Skript vorhanden ? + + + + + + + + + + + + - - + + - coastdat + model_draft.ego_supply_res_powerplant +--> #120 redundant table + + + + + + + + + + + + + + + + + BNetzA +MA Marlon? @@ -846,7 +890,6 @@ FlEnS - @@ -857,7 +900,6 @@ FlEnS - @@ -867,8 +909,7 @@ FlEnS - - + @@ -878,8 +919,7 @@ FlEnS - - + @@ -889,8 +929,7 @@ FlEnS - - + @@ -900,52 +939,47 @@ FlEnS - - + - - + + - - + - - + + - - + - - + + - - + - - + + - - + @@ -955,8 +989,7 @@ FlEnS - - + @@ -966,8 +999,7 @@ FlEnS - - + @@ -977,8 +1009,7 @@ FlEnS - - + @@ -988,8 +1019,17 @@ FlEnS - - + + + + + + + + + + + @@ -999,19 +1039,17 @@ FlEnS - - + - + - - + @@ -1021,8 +1059,7 @@ FlEnS - - + @@ -1032,8 +1069,7 @@ FlEnS - - + @@ -1043,8 +1079,7 @@ FlEnS - - + @@ -1054,8 +1089,7 @@ FlEnS - - + @@ -1065,8 +1099,7 @@ FlEnS - - + @@ -1076,7 +1109,27 @@ FlEnS - + + + + + + + + + + + + + + + + + + + + + @@ -1087,7 +1140,7 @@ FlEnS - + @@ -1098,7 +1151,7 @@ FlEnS - + @@ -1109,7 +1162,7 @@ FlEnS - + @@ -1120,7 +1173,7 @@ FlEnS - + @@ -1131,7 +1184,18 @@ FlEnS - + + + + + + + + + + + + @@ -1142,7 +1206,95 @@ FlEnS - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + From 91d63dac68047fb0c42b7bd3e0439968370334f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guido=20Ple=C3=9Fmann?= Date: Thu, 9 Apr 2020 16:34:13 +0200 Subject: [PATCH 48/84] Information about dump and restore --- README.rst | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index e7fc61de..526cc5e6 100644 --- a/README.rst +++ b/README.rst @@ -54,7 +54,7 @@ Run Local database with docker ========================== -You need to have `docker-compose` installed. +You need to have `docker-compose` installed. And some more tools, see requirements_system.txt. ### Spin up database with docker @@ -77,4 +77,18 @@ Usage Pre-processing ************** -Execute `eGo_import_and_preprocessing.py` for starting pre-processing from scratch doing data import and a pre-processing of imported tables. \ No newline at end of file +Execute `eGo_import_and_preprocessing.py` for starting pre-processing from scratch doing data import and a pre-processing of imported tables. + +Once completed, export required schemas to SQL dump + +_You might have to install `postgresql-client-11` from additional source to have a compatible client with the v11 dockered PostgreSQL database._ + +``` + /usr/lib/postgresql/11/bin/pg_dump -d dp -U oeuser -p 54321 -h localhost -Fc -n model_draft -n openstreetmap -n boundaries -n society > `date -I`_eGo_data_pre-processing_bremen.backup +``` + +Import a dump by + +``` +/usr/lib/postgresql/11/bin/pg_restore -d dp -U oeuser -p 54321 -h localhost 2020-03-28_eGo_data_pre-processing_deu.backup +``` From 19b6554965243f3794f2343ac984b2a8490bb2aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guido=20Ple=C3=9Fmann?= Date: Thu, 9 Apr 2020 17:34:33 +0200 Subject: [PATCH 49/84] Finalize BPMN --- ..._dp-pre_bpmn_section_openstreetmap.graphml | 213 ++++++++++++++---- 1 file changed, 166 insertions(+), 47 deletions(-) diff --git a/documentation/bpmn/ego_dp-pre_bpmn_section_openstreetmap.graphml b/documentation/bpmn/ego_dp-pre_bpmn_section_openstreetmap.graphml index 4376dd84..d6821a1b 100644 --- a/documentation/bpmn/ego_dp-pre_bpmn_section_openstreetmap.graphml +++ b/documentation/bpmn/ego_dp-pre_bpmn_section_openstreetmap.graphml @@ -13,7 +13,7 @@ - + @@ -31,29 +31,29 @@ + + + - - - - - + + - + @@ -105,10 +105,10 @@ openstreetmap.osm_deu_roads openstreetmap.osm_deu_ways + - @@ -119,7 +119,7 @@ openstreetmap.osm_deu_ways< - ? + 1 h 12 min @@ -145,10 +145,10 @@ openstreetmap.osm_deu_ways< + - @@ -156,7 +156,7 @@ openstreetmap.osm_deu_ways< - + @@ -168,8 +168,8 @@ openstreetmap.osm_deu_ways< - + @@ -192,8 +192,8 @@ openstreetmap.osm_deu_ways< - + @@ -204,16 +204,16 @@ openstreetmap.osm_deu_ways< - + openstreetmap (osm2pgsql) + - @@ -238,7 +238,7 @@ openstreetmap.osm_deu_ways< - + Open Database License (ODbL) v1.0 @@ -279,10 +279,10 @@ openstreetmap.osm_deu_ways< openstreetmap.osm_deu_line_street_mview + - @@ -293,7 +293,7 @@ openstreetmap.osm_deu_ways< - ? + 3 min @@ -312,10 +312,10 @@ openstreetmap.osm_deu_ways< openstreetmap.osm_deu_line + - @@ -364,10 +364,10 @@ openstreetmap.osm_deu_ways< openstreetmap.osm_deu_polygon_building_mview + - @@ -378,7 +378,7 @@ openstreetmap.osm_deu_ways< - ? + 17 min @@ -397,10 +397,10 @@ openstreetmap.osm_deu_ways< openstreetmap.osm_deu_polygon + - @@ -422,6 +422,80 @@ openstreetmap.osm_deu_ways< + + + + + + + osm2postgres.py + + + + + + + + + + + + + + + + + + public.osm_deu_line +public.osm_deu_nodes +public.osm_deu_point +public.osm_deu_polygon +public.osm_deu_rels +public.osm_deu_roads +public.osm_deu_ways + + + + + + + + + + + + + + + + + Open Database License (ODbL) v1.0 +© OpenStreetMap contributors + + + + + + + + + + + + + + + + 4 h + + + + + + + + + @@ -460,10 +534,12 @@ openstreetmap.osm_deu_ways< - + - + + + @@ -472,26 +548,19 @@ openstreetmap.osm_deu_ways< - + - + - public.osm_deu_line -public.osm_deu_nodes -public.osm_deu_point -public.osm_deu_polygon -public.osm_deu_rels -public.osm_deu_roads -public.osm_deu_ways - + @@ -503,7 +572,7 @@ public.osm_deu_ways + @@ -515,7 +584,7 @@ public.osm_deu_ways + @@ -527,7 +596,7 @@ public.osm_deu_ways + @@ -539,7 +608,7 @@ public.osm_deu_ways + @@ -551,7 +620,7 @@ public.osm_deu_ways + @@ -563,8 +632,7 @@ public.osm_deu_ways - + @@ -574,8 +642,7 @@ public.osm_deu_ways - + @@ -585,8 +652,17 @@ public.osm_deu_ways - + + + + + + + + + + + @@ -596,7 +672,39 @@ public.osm_deu_ways + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -607,7 +715,7 @@ public.osm_deu_ways + @@ -618,6 +726,17 @@ public.osm_deu_ways + + + + + + + + + + From 86a9d2177e38373e377e03a4513b4eef094b00d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guido=20Ple=C3=9Fmann?= Date: Thu, 9 Apr 2020 19:00:59 +0200 Subject: [PATCH 50/84] Add BPMN for scenario_log part --- ...o_dp-pre_bpmn_section_scenario-log.graphml | 397 ++++++++++++++++++ 1 file changed, 397 insertions(+) create mode 100644 documentation/bpmn/ego_dp-pre_bpmn_section_scenario-log.graphml diff --git a/documentation/bpmn/ego_dp-pre_bpmn_section_scenario-log.graphml b/documentation/bpmn/ego_dp-pre_bpmn_section_scenario-log.graphml new file mode 100644 index 00000000..85db06bb --- /dev/null +++ b/documentation/bpmn/ego_dp-pre_bpmn_section_scenario-log.graphml @@ -0,0 +1,397 @@ + + + + + + + + + + + + + + + + + + + + + + + eGo Pre-Processing - scenario_log + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + function_scenario_log.sql + + + + + + + + + + + + + + + + + + 0.01 s + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ego_pp_scenario.sql + + + + + + + + + + + + + + + + + + model_draft.ego_scenario + + + + + + + + + + + + + + + + + 0.01 s + + + + + + + + + + + + + + + + Creative Commons Zero v1.0 Universal +Reiner Lemoine Institut; CC0-1.0 + + + + + + + + + + + + + + + + ego_pp_scenario.sql + + + + + + + + + + + + + + + + + + 0.01 s + + + + + + + + + + + + + + + + model_draft.ego_scenario + + + + + + + + + + + + + + + + + Creative Commons Zero v1.0 Universal +Reiner Lemoine Institut; CC0-1.0 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + From 50f3e98ccc84980373d3c3edfa04e7ce9875ca93 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guido=20Ple=C3=9Fmann?= Date: Thu, 9 Apr 2020 20:13:05 +0200 Subject: [PATCH 51/84] Updating BPMN according to PR review changes request --- .../ego_dp-pre_bpmn_section_zensus.graphml | 241 +++++++++++++----- 1 file changed, 171 insertions(+), 70 deletions(-) diff --git a/documentation/bpmn/ego_dp-pre_bpmn_section_zensus.graphml b/documentation/bpmn/ego_dp-pre_bpmn_section_zensus.graphml index 546f115c..3f242694 100644 --- a/documentation/bpmn/ego_dp-pre_bpmn_section_zensus.graphml +++ b/documentation/bpmn/ego_dp-pre_bpmn_section_zensus.graphml @@ -17,11 +17,11 @@ - + - eGo Pre-Processing - zensus - + eGo Pre-Processing - zensus + @@ -53,18 +53,18 @@ - + - + - + @@ -75,7 +75,7 @@ - + ego_pp_destatis_zensus_import.sql @@ -93,7 +93,7 @@ - + society.destatis_zensus_population_per_ha @@ -110,10 +110,10 @@ - + - ? + 5 min @@ -126,12 +126,12 @@ - + - orig_destatis.zensus_population_per_ha -orig_destatis.zensus_population_per_ha_grid + tmp_destatis.zensus_population_per_ha +tmp_destatis.zensus_population_per_ha_grid @@ -145,7 +145,7 @@ orig_destatis.zensus_population_per_ha_grid - + @@ -193,10 +193,10 @@ orig_destatis.zensus_population_per_ha_grid - + - ego_pp_destatis_zensus_insidevg250.sql + destatis_zensus_raw_import.sql @@ -211,10 +211,10 @@ orig_destatis.zensus_population_per_ha_grid - + - society.destatis_zensus_population_per_ha + tmp_destatis.zensus_2011_pop_per_ha @@ -244,30 +244,78 @@ orig_destatis.zensus_population_per_ha_grid - + + + + 5 min + + + + + + + + + + + + + + + + Datenlizenz Deutschland – Namensnennung – Version 2.0 +Statistisches Bundesamt, Wiesbaden, Genesis-Online, 2016; Datenlizenz by-2-0 + + + + + + + + + + + + + + + + destatis_zensus_import_csv.py + + + + + + + + + + + + + + + - society.destatis_zensus_population_per_ha_mview -model_draft.destatis_zensus_population_per_ha_inside -model_draft.destatis_zensus_population_per_ha_invg_mview -model_draft.destatis_zensus_population_per_ha_outvg_mview + tmp_destatis.zensus_2011_pop_per_ha - + - + - + - ? + 6 min @@ -277,13 +325,13 @@ model_draft.destatis_zensus_population_per_ha_outvg_mview - + - + - orig_destatis.zensus_population_per_ha + Zensus_Bevoelkerung_100m-Gitter.zip @@ -294,10 +342,10 @@ model_draft.destatis_zensus_population_per_ha_outvg_mview - + - + Datenlizenz Deutschland – Namensnennung – Version 2.0 @@ -311,10 +359,28 @@ Statistisches Bundesamt, Wiesbaden, Genesis-Online, 2016; Datenlizenz by-2-0 - + + + + + + + tmp_destatis.zensus_population_per_ha +tmp_destatis.zensus_population_per_ha_grid + + + + + + + + + + + - + Datenlizenz Deutschland – Namensnennung – Version 2.0 @@ -366,96 +432,131 @@ Statistisches Bundesamt, Wiesbaden, Genesis-Online, 2016; Datenlizenz by-2-0 - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - + + - + - + - - + + + + + + + + + + + - + + - - - - - + + + - + + - - + + - + + - - + + - + + - - + + - + + - - + + - + - + - - - - - - - - - From ea2df6eaa5ef73bfb4997bdec5f0868f4eba189f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guido=20Ple=C3=9Fmann?= Date: Thu, 9 Apr 2020 21:38:36 +0200 Subject: [PATCH 52/84] Update BPMN for vg250 according to #339 implementation --- .../ego_dp-pre_bpmn_section_bkg-vg250.graphml | 349 +++++++++++++++--- 1 file changed, 300 insertions(+), 49 deletions(-) diff --git a/documentation/bpmn/ego_dp-pre_bpmn_section_bkg-vg250.graphml b/documentation/bpmn/ego_dp-pre_bpmn_section_bkg-vg250.graphml index 7d47a0a5..8329d6cd 100644 --- a/documentation/bpmn/ego_dp-pre_bpmn_section_bkg-vg250.graphml +++ b/documentation/bpmn/ego_dp-pre_bpmn_section_bkg-vg250.graphml @@ -17,7 +17,7 @@ - + eGo Pre-Processing - bkg_vg250 @@ -53,13 +53,13 @@ - + - + @@ -75,7 +75,7 @@ - + ego_pp_vg250_import.sql @@ -93,7 +93,7 @@ - + boundaries.bkg_vg250_1_sta @@ -115,10 +115,10 @@ boundaries.bkg_vg250_6_gem< - + - ? + 1.6 s @@ -131,7 +131,7 @@ boundaries.bkg_vg250_6_gem< - + @@ -154,7 +154,7 @@ boundaries.bkg_vg250_gem_YYYY-MM-DD - + @@ -178,7 +178,7 @@ boundaries.bkg_vg250_gem_YYYY-MM-DD - + @@ -202,7 +202,7 @@ boundaries.bkg_vg250_gem_YYYY-MM-DD - + Geodatenzugangsgesetz (GeoZG) @@ -219,7 +219,7 @@ boundaries.bkg_vg250_gem_YYYY-MM-DD - + ego_dp_structure_boundaries_vg250.sql @@ -237,7 +237,7 @@ boundaries.bkg_vg250_gem_YYYY-MM-DD - + boundaries.bkg_vg250_1_sta @@ -275,7 +275,7 @@ society.destatis_zensus_population_per_ha_mview - + boundaries.bkg_vg250_1_sta_mview @@ -304,10 +304,10 @@ boundaries.bkg_vg250_statistics_view - + - ? + 3 min 42 s @@ -318,12 +318,85 @@ boundaries.bkg_vg250_statistics_view + + + + + + import_vg250.py + + + + + + + + + + + + + + + + + + boundaries.bkg_vg250_sta_YYYY-MM-DD +boundaries.bkg_vg250_lan_YYYY-MM-DD +boundaries.bkg_vg250_rbz_YYYY-MM-DD +boundaries.bkg_vg250_krs_YYYY-MM-DD +boundaries.bkg_vg250_vwg_YYYY-MM-DD +boundaries.bkg_vg250_gem_YYYY-MM-DD + + + + + + + + + + + - + + + + 15 s + + + + + + + + + + + + + + + + Geodatenzugangsgesetz (GeoZG) +© GeoBasis-DE / BKG 2016 (Daten verändert) + + + + + + + + + + + + + - boundaries.vg250_*_YYYY-MM-DD + vg250_01-01.gk3.shape.ebenen.zip @@ -334,6 +407,92 @@ boundaries.bkg_vg250_statistics_view + + + + + + + ego_pp_vg250_mview.sql + + + + + + + + + + + + + + + + + + boundaries.bkg_vg250_2_lan_nuts_view + + + + + + + + + + + + + + + + + 1.8 s + + + + + + + + + + + + + + + + + boundaries.bkg_vg250_2_lan + + + + + + + + + + + + + + + + + Geodatenzugangsgesetz (GeoZG) +© GeoBasis-DE / BKG 2016 (Daten verändert) + + + + + + + + + @@ -372,19 +531,7 @@ boundaries.bkg_vg250_statistics_view - - - - - - - - - - - - - + @@ -394,11 +541,11 @@ boundaries.bkg_vg250_statistics_view - + - + @@ -406,54 +553,67 @@ boundaries.bkg_vg250_statistics_view - + - + - + - - + + - + - - + + - + - + - + + + + + + + + + + + + + - - + + + + - - + - + @@ -466,6 +626,97 @@ boundaries.bkg_vg250_statistics_view + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + From 36d02565914848cdf850c9bf296e1351a61a0e72 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guido=20Ple=C3=9Fmann?= Date: Fri, 10 Apr 2020 10:03:07 +0200 Subject: [PATCH 53/84] Including script that merges vg250 and zensus 2011 --- .../ego_dp-pre_bpmn_section_bkg-vg250.graphml | 194 +++++++++++++++--- 1 file changed, 166 insertions(+), 28 deletions(-) diff --git a/documentation/bpmn/ego_dp-pre_bpmn_section_bkg-vg250.graphml b/documentation/bpmn/ego_dp-pre_bpmn_section_bkg-vg250.graphml index 8329d6cd..c35f9baf 100644 --- a/documentation/bpmn/ego_dp-pre_bpmn_section_bkg-vg250.graphml +++ b/documentation/bpmn/ego_dp-pre_bpmn_section_bkg-vg250.graphml @@ -17,7 +17,7 @@ - + eGo Pre-Processing - bkg_vg250 @@ -53,13 +53,13 @@ - + - + @@ -178,7 +178,7 @@ boundaries.bkg_vg250_gem_YYYY-MM-DD - + @@ -276,7 +276,7 @@ society.destatis_zensus_population_per_ha_mview - + boundaries.bkg_vg250_1_sta_mview boundaries.bkg_vg250_1_sta_error_geom_mview @@ -483,6 +483,96 @@ boundaries.bkg_vg250_gem_YYYY-MM-DD Geodatenzugangsgesetz (GeoZG) +© GeoBasis-DE / BKG 2016 (Daten verändert) + + + + + + + + + + + + + + + + ego_pp_destatis_zensus_insidevg250.sql + + + + + + + + + + + + + + + + + + society.destatis_zensus_population_per_ha_mview +model_draft.destatis_zensus_population_per_ha_inside +model_draft.destatis_zensus_population_per_ha_invg_mview +model_draft.destatis_zensus_population_per_ha_outvg_mview + + + + + + + + + + + + + + + + + 46 s + + + + + + + + + + + + + + + + + society.destatis_zensus_population_per_ha +boundaries.bkg_vg250_1_sta_union_mview + + + + + + + + + + + + + + + + + Geodatenzugangsgesetz (GeoZG) © GeoBasis-DE / BKG 2016 (Daten verändert) @@ -553,17 +643,7 @@ boundaries.bkg_vg250_gem_YYYY-MM-DD - - - - - - - - - - - + @@ -573,7 +653,7 @@ boundaries.bkg_vg250_gem_YYYY-MM-DD - + @@ -583,7 +663,7 @@ boundaries.bkg_vg250_gem_YYYY-MM-DD - + @@ -595,7 +675,7 @@ boundaries.bkg_vg250_gem_YYYY-MM-DD - + @@ -607,7 +687,7 @@ boundaries.bkg_vg250_gem_YYYY-MM-DD - + @@ -626,7 +706,7 @@ boundaries.bkg_vg250_gem_YYYY-MM-DD - + @@ -637,7 +717,7 @@ boundaries.bkg_vg250_gem_YYYY-MM-DD - + @@ -648,7 +728,7 @@ boundaries.bkg_vg250_gem_YYYY-MM-DD - + @@ -660,7 +740,7 @@ boundaries.bkg_vg250_gem_YYYY-MM-DD - + @@ -672,7 +752,7 @@ boundaries.bkg_vg250_gem_YYYY-MM-DD - + @@ -684,7 +764,7 @@ boundaries.bkg_vg250_gem_YYYY-MM-DD - + @@ -695,7 +775,7 @@ boundaries.bkg_vg250_gem_YYYY-MM-DD - + @@ -706,7 +786,7 @@ boundaries.bkg_vg250_gem_YYYY-MM-DD - + @@ -717,6 +797,64 @@ boundaries.bkg_vg250_gem_YYYY-MM-DD + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + From 818df1119551d0ae7eefa0deba392b2a67937484 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guido=20Ple=C3=9Fmann?= Date: Fri, 10 Apr 2020 11:36:22 +0200 Subject: [PATCH 54/84] Add BPMN for wind_potential_table data import --- ..._bpmn_section_wind_potential_areas.graphml | 378 ++++++++++++++++++ 1 file changed, 378 insertions(+) create mode 100644 documentation/bpmn/ego_dp-pre_bpmn_section_wind_potential_areas.graphml diff --git a/documentation/bpmn/ego_dp-pre_bpmn_section_wind_potential_areas.graphml b/documentation/bpmn/ego_dp-pre_bpmn_section_wind_potential_areas.graphml new file mode 100644 index 00000000..65920424 --- /dev/null +++ b/documentation/bpmn/ego_dp-pre_bpmn_section_wind_potential_areas.graphml @@ -0,0 +1,378 @@ + + + + + + + + + + + + + + + + + + + + + + + eGo Pre-Processing - wind potential areas + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + vernetzen_wind_potential_area_v1.3.sql + + + + + + + + + + + + + + + + + + ? + + + + + + + + + + + + + + + + import_vernetzen_wind_potential_areas.py + + + + + + + + + + + + + + + + + + supply.vernetzen_wind_potential_area + + + + + + + + + + + + + + + + + ? + + + + + + + + + + + + + + + + vernetzen_wind_potential_areas.zip + + + + + + + + + + + + + + + + + <no license> + + + + + + + + + + + + + + + + supply.vernetzen_wind_potential_area + + + + + + + + + + + + + + + + + Open Database License (ODbL) v1.0 +© OpenStreetMap contributors + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + From 5f45beeeb1568a93376fb6535267a47bbae90978 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guido=20Ple=C3=9Fmann?= Date: Fri, 10 Apr 2020 19:55:45 +0200 Subject: [PATCH 55/84] Import script for cosmoclmgrid SHP dump --- .../cosmoclmgrid/import_cosmoclmgrid.py | 48 +++++++++++++++++++ preprocessing/import.yml | 12 +++++ 2 files changed, 60 insertions(+) create mode 100644 preprocessing/cosmoclmgrid/import_cosmoclmgrid.py diff --git a/preprocessing/cosmoclmgrid/import_cosmoclmgrid.py b/preprocessing/cosmoclmgrid/import_cosmoclmgrid.py new file mode 100644 index 00000000..3aa9707d --- /dev/null +++ b/preprocessing/cosmoclmgrid/import_cosmoclmgrid.py @@ -0,0 +1,48 @@ +import os +import zipfile +import geopandas as gpd +import sqlalchemy +from geoalchemy2 import Geometry, WKTElement +from shapely.geometry import MultiPolygon + + +CRS = 4326 + + +# Function to generate WKB hex +def wkb_hexer(line): + return line.wkb_hex + + +def import_cosmoclmgrid(file, path, db, **kwds): + + if not db.dialect.has_schema(db, kwds["schema"]): + db.execute(sqlalchemy.schema.CreateSchema(kwds["schema"])) + + shp_file = file.replace(".zip", ".shp") + subdir = file.replace(".zip", "") + os.makedirs(os.path.join(path, subdir), exist_ok=True) + + zf = zipfile.ZipFile(os.path.join(path, file)) + zf.extractall(path=os.path.join(path, subdir)) + + # read with pandas + cosmoclmgrid = gpd.read_file(os.path.join(path, subdir, shp_file)) + + cosmoclmgrid['geom'] = cosmoclmgrid['geometry'].apply(wkb_hexer) + cosmoclmgrid.drop("geometry", axis=1, inplace=True) + + # create table from geopandas dataframe + cosmoclmgrid.to_sql(kwds["table"], + db, kwds["schema"], + index=False, + if_exists="replace", + chunksize=10000, + method="multi", + dtype={'geom': Geometry('POLYGON')} + ) + + create_pkey_constraint = "ALTER TABLE {schema}.{table} ADD CONSTRAINT " \ + "{table}_pkey PRIMARY KEY (gid);".format(**kwds) + db.execution_options(autocommit=True).execute(create_pkey_constraint) + diff --git a/preprocessing/import.yml b/preprocessing/import.yml index c70647d2..4f1494b1 100644 --- a/preprocessing/import.yml +++ b/preprocessing/import.yml @@ -63,6 +63,18 @@ zensus_vg250: language: SQL - script: ego_pp_destatis_zensus_metadata.sql language: SQL +cosmoclmgrid: + required_data: + - url: https://next.rl-institut.de/s/jX5ci9kimgjdXAM/download + filename: cosmoclmgrid.zip + scripts: + - script: preprocessing.cosmoclmgrid.import_cosmoclmgrid::import_cosmoclmgrid + language: python + filename: cosmoclmgrid.zip + db: conn + args: + schema: "cliamte" + table: "cosmoclmgrid" supply: required_data: - url: https://next.rl-institut.de/s/TsjxR5P2Q6Z8Y7X/download From 06ea7f5bb5a2b88fff3c3535033ff9993d7da92e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guido=20Ple=C3=9Fmann?= Date: Fri, 10 Apr 2020 20:01:28 +0200 Subject: [PATCH 56/84] Documenting COSMO CLM grid is imported from a .shp file --- ...o_dp-pre_bpmn_section_cosmoclmgrid.graphml | 25 +++++++++---------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/documentation/bpmn/ego_dp-pre_bpmn_section_cosmoclmgrid.graphml b/documentation/bpmn/ego_dp-pre_bpmn_section_cosmoclmgrid.graphml index e5a7a324..e8f6a654 100644 --- a/documentation/bpmn/ego_dp-pre_bpmn_section_cosmoclmgrid.graphml +++ b/documentation/bpmn/ego_dp-pre_bpmn_section_cosmoclmgrid.graphml @@ -31,29 +31,29 @@ + + + - - - - - + + - + @@ -78,7 +78,7 @@ - <manual import> + import_cosmoclmgrid.py @@ -110,10 +110,10 @@ - + - ? + 7 s @@ -126,10 +126,10 @@ - + - + cosmoclmgrid.zip @@ -194,7 +194,7 @@ - no license + <no metadata available> @@ -255,7 +255,6 @@ - From c7cba346cc6d4b87c71d6e88dfed69491456ff92 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guido=20Ple=C3=9Fmann?= Date: Tue, 21 Apr 2020 19:10:42 +0200 Subject: [PATCH 57/84] Include metadata for cosmoclmgrid --- .../cosmoclmgrid/import_cosmoclmgrid.py | 91 +++++++++++++++++-- preprocessing/import.yml | 2 +- 2 files changed, 83 insertions(+), 10 deletions(-) diff --git a/preprocessing/cosmoclmgrid/import_cosmoclmgrid.py b/preprocessing/cosmoclmgrid/import_cosmoclmgrid.py index 3aa9707d..844ea2df 100644 --- a/preprocessing/cosmoclmgrid/import_cosmoclmgrid.py +++ b/preprocessing/cosmoclmgrid/import_cosmoclmgrid.py @@ -2,8 +2,10 @@ import zipfile import geopandas as gpd import sqlalchemy -from geoalchemy2 import Geometry, WKTElement -from shapely.geometry import MultiPolygon +from geoalchemy2 import Geometry +import time +from dataprocessing.tools import metadata +import json CRS = 4326 @@ -34,15 +36,86 @@ def import_cosmoclmgrid(file, path, db, **kwds): # create table from geopandas dataframe cosmoclmgrid.to_sql(kwds["table"], - db, kwds["schema"], - index=False, - if_exists="replace", - chunksize=10000, - method="multi", - dtype={'geom': Geometry('POLYGON')} - ) + db, + kwds["schema"], + index=False, + if_exists="replace", + chunksize=10000, + method="multi", + dtype={'geom': Geometry('POLYGON')} + ) create_pkey_constraint = "ALTER TABLE {schema}.{table} ADD CONSTRAINT " \ "{table}_pkey PRIMARY KEY (gid);".format(**kwds) db.execution_options(autocommit=True).execute(create_pkey_constraint) + # create metadata json str + metadata_dict = {"name": "Spatial grid coastDat-2 re-analysis data set", + "title": "COSMO CLM grid", + "description": "This spatial grid provides reference polygons for coastDat-2 " + "re-analysis data.", + "language": ["EN"], + "spatial": { + "location": "0.22 ° x 0.22 °", + "extent": "Europe", + "resolution": "0.22 ° x 0.22 °" + }, + "temporal": { + "referenceDate": "2014", + "timeseries": { + "start": "", + "end": "", + "resolution": "", + "alignment": "", + "aggregationType": "" + } + }, + "sources": [ + { + "title": "coastDat-2 re-analysis data set", + "description": "coastDat-2 re-analysis data is a long-term backcasted " + "climate data set based on the regional weather forceasting " + "model COSMO CLM", + "path": "https://www.coastdat.de/about_us/index.php.en", + "licenses": [ + { + "name": "", + "title": "", + "path": "", + "instruction": "", + "attribution": "" + } + ] + } + ], + "licenses": [ + { + "name": "Open Data Commons Open Database License 1.0", + "title": "", + "path": "https://opendatacommons.org/licenses/odbl/1.0/", + "instruction": "You are free: To Share, To Create, To Adapt; As long as you: Attribute, Share-Alike, Keep open!", + "attribution": "© Reiner Lemoine Institut" + } + ], + "contributors": [ + { + "title": "Guido Pleßmann", + "email": "http://github.com/gplssm", + "date": time.strftime("%Y-%m-%d"), + "object": "", + "comment": "Only uploaded the data" + } + ], + "metaMetadata": { + "metadataVersion": "OEP-1.4.0", + "metadataLicense": { + "name": "CC0-1.0", + "title": "Creative Commons Zero v1.0 Universal", + "path": "https://creativecommons.org/publicdomain/zero/1.0/" + } + }, + } + + json_str = "'" + json.dumps(metadata_dict) + "'" + + metadata.submit_comment(db, json_str, kwds["schema"], kwds["table"]) diff --git a/preprocessing/import.yml b/preprocessing/import.yml index 4f1494b1..2064e104 100644 --- a/preprocessing/import.yml +++ b/preprocessing/import.yml @@ -73,7 +73,7 @@ cosmoclmgrid: filename: cosmoclmgrid.zip db: conn args: - schema: "cliamte" + schema: "climate" table: "cosmoclmgrid" supply: required_data: From 2794f327104ca3d6d35f51c738952302e4f25103 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guido=20Ple=C3=9Fmann?= Date: Tue, 28 Apr 2020 09:42:46 +0200 Subject: [PATCH 58/84] Fix copy&paste mistake --- .../bpmn/ego_dp-pre_bpmn_section_scenario-log.graphml | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/documentation/bpmn/ego_dp-pre_bpmn_section_scenario-log.graphml b/documentation/bpmn/ego_dp-pre_bpmn_section_scenario-log.graphml index 85db06bb..ea5d5619 100644 --- a/documentation/bpmn/ego_dp-pre_bpmn_section_scenario-log.graphml +++ b/documentation/bpmn/ego_dp-pre_bpmn_section_scenario-log.graphml @@ -228,7 +228,7 @@ Reiner Lemoine Institut; CC0-1.0 - ego_pp_scenario.sql + ego_pp_scenario_log.sql @@ -262,13 +262,13 @@ Reiner Lemoine Institut; CC0-1.0 - model_draft.ego_scenario + model_draft.scenario_log + - @@ -347,7 +347,6 @@ Reiner Lemoine Institut; CC0-1.0 - @@ -358,7 +357,6 @@ Reiner Lemoine Institut; CC0-1.0 - @@ -369,7 +367,6 @@ Reiner Lemoine Institut; CC0-1.0 - @@ -380,7 +377,6 @@ Reiner Lemoine Institut; CC0-1.0 - From 235b70e7934641e9d775d58e76913d4b95dff2c7 Mon Sep 17 00:00:00 2001 From: IlkaCu Date: Wed, 29 Apr 2020 12:51:28 +0200 Subject: [PATCH 59/84] create bpmn for section demand_federalstate --- ...e_bpmn_section_demand_federalstate.graphml | 317 ++++++++++++++++++ 1 file changed, 317 insertions(+) create mode 100644 documentation/bpmn/ego_dp-pre_bpmn_section_demand_federalstate.graphml diff --git a/documentation/bpmn/ego_dp-pre_bpmn_section_demand_federalstate.graphml b/documentation/bpmn/ego_dp-pre_bpmn_section_demand_federalstate.graphml new file mode 100644 index 00000000..b893816b --- /dev/null +++ b/documentation/bpmn/ego_dp-pre_bpmn_section_demand_federalstate.graphml @@ -0,0 +1,317 @@ + + + + + + + + + + + + + + + + + + + + + + + eGo Pre-Processing - demand federal states + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ego_dp_demand_setup.sql + + + + + + + + + + + + + + + + + + demand.ego_demand_federalstate + + + + + + + + + + + + + + + + + Open Data Commons Open Database License 1.0 + + + + + + + + + + + + + + + + regional statistical offices + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + VGR der Länder 2011 + + + + + + + + + + + + + + + + + demand.ego_demand_federalstate + + + + + + + + + + + + + + + + + "Open Data Commons Open Database License 1.0 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + From d351313644d2bf21dab1166d8a8aef743ac39d07 Mon Sep 17 00:00:00 2001 From: IlkaCu Date: Wed, 6 May 2020 13:41:31 +0200 Subject: [PATCH 60/84] include destatis gva per district in BPMN --- ...e_bpmn_section_demand_federalstate.graphml | 57 +++++++++++++++---- 1 file changed, 45 insertions(+), 12 deletions(-) diff --git a/documentation/bpmn/ego_dp-pre_bpmn_section_demand_federalstate.graphml b/documentation/bpmn/ego_dp-pre_bpmn_section_demand_federalstate.graphml index b893816b..3151c0e0 100644 --- a/documentation/bpmn/ego_dp-pre_bpmn_section_demand_federalstate.graphml +++ b/documentation/bpmn/ego_dp-pre_bpmn_section_demand_federalstate.graphml @@ -17,7 +17,7 @@ - + eGo Pre-Processing - demand federal states @@ -53,13 +53,13 @@ - + - + @@ -99,7 +99,7 @@ - + @@ -195,6 +195,7 @@ + destatis_gva_per_district.sql @@ -215,10 +216,10 @@ VGR der Länder 2011 + - @@ -229,13 +230,13 @@ - demand.ego_demand_federalstate + economy.destatis_gva_per_district + - @@ -262,7 +263,7 @@ - + @@ -274,8 +275,8 @@ - - + + @@ -300,12 +301,44 @@ + + + + + + + + + + - - + + + + + + + + + + + + + + + + + + + + + + + + From 8b09378eaabb040d2f6b315dd5cdc49b154ef71e Mon Sep 17 00:00:00 2001 From: IlkaCu Date: Wed, 6 May 2020 14:44:08 +0200 Subject: [PATCH 61/84] replace incorrect table .ego_supply_res_powerplant by supply.ego_renewable_powerplant in pp --- .../ego_dp_preprocessing_res_powerplant.sql | 21 ++++++++++--------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/preprocessing/sql_snippets/ego_dp_preprocessing_res_powerplant.sql b/preprocessing/sql_snippets/ego_dp_preprocessing_res_powerplant.sql index ed5db82b..a9be9cff 100644 --- a/preprocessing/sql_snippets/ego_dp_preprocessing_res_powerplant.sql +++ b/preprocessing/sql_snippets/ego_dp_preprocessing_res_powerplant.sql @@ -195,21 +195,22 @@ Insert into model_draft.ego_dp_supply_res_powerplant source || ' ego_dp' as source, comment || ' geom changes by rea' as comment , ST_Transform(geom,3035) as geom, - subst_id, - otg_id, - un_id, - voltage_level, - la_id, - mvlv_subst_id, - rea_sort, - rea_flag, + NULL as subst_id, + NULL as otg_id, + NULL as un_id, + NULL as voltage_level, + NULL as la_id, + NULL as mvlv_subst_id, + NULL as rea_sort, + NULL as rea_flag, NULL as rea_geom_line, Null as rea_geom_new, 'Status Quo'::text as scenario, 'constantly'::text as flag, - Null as nuts + NULL as nuts, + NULL as w-id FROM - model_draft.ego_supply_res_powerplant + supply.ego_renewable_powerplant WHERE geom is not NULL; From 215c80b0b5b2a6e43a2a1df065a366fc4a60d38e Mon Sep 17 00:00:00 2001 From: IlkaCu Date: Thu, 7 May 2020 14:50:41 +0200 Subject: [PATCH 62/84] Remove loop in conc powerplant script --- .../ego_dp_preprocessing_conv_powerplant.sql | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/preprocessing/sql_snippets/ego_dp_preprocessing_conv_powerplant.sql b/preprocessing/sql_snippets/ego_dp_preprocessing_conv_powerplant.sql index 03eab239..6e19474e 100644 --- a/preprocessing/sql_snippets/ego_dp_preprocessing_conv_powerplant.sql +++ b/preprocessing/sql_snippets/ego_dp_preprocessing_conv_powerplant.sql @@ -176,7 +176,7 @@ SELECT obj_description('model_draft.ego_dp_supply_conv_powerplant'::regclass)::j INSERT INTO model_draft.ego_dp_supply_conv_powerplant SELECT 'v0.3.0'::text as preversion, - id, + gid, bnetza_id, company, name, @@ -208,10 +208,10 @@ INSERT INTO model_draft.ego_dp_supply_conv_powerplant lon, comment, geom, - voltage_level, - subst_id, - otg_id, - un_id, + NULL as voltage_level, + NULL as subst_id, + NULL as otg_id, + NULL as un_id, NULL::int as la_id, 'Status Quo'::text as scenario, NULL::text as flag, From 9ee6811704ab3391c06ac5839d4221d86b65c7a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guido=20Ple=C3=9Fmann?= Date: Thu, 7 May 2020 17:03:56 +0200 Subject: [PATCH 63/84] Utility script to save osmTGmod data --- .../utility/download_oedb_data_to_shp.py | 18 +-- preprocessing/utility/osmTGmod_to_shp.py | 116 ++++++++++++++++++ 2 files changed, 127 insertions(+), 7 deletions(-) create mode 100644 preprocessing/utility/osmTGmod_to_shp.py diff --git a/preprocessing/utility/download_oedb_data_to_shp.py b/preprocessing/utility/download_oedb_data_to_shp.py index ba061a49..5ab27faa 100644 --- a/preprocessing/utility/download_oedb_data_to_shp.py +++ b/preprocessing/utility/download_oedb_data_to_shp.py @@ -19,10 +19,11 @@ def zipdir(path, ziph): def oedbtable2shp(table_query, filename, crs, geometry_col, index_col): - shp_dir = os.path.join(DOWNLOADDIR, filename.replace(".shp", "")) + file_basename = os.path.basename(os.path.splitext(filename)[0]) + shp_dir = os.path.join(DOWNLOADDIR, file_basename) os.makedirs(shp_dir, exist_ok=True) shp_filename = os.path.join(shp_dir, filename) - zip_filename = os.path.join(DOWNLOADDIR, filename.replace(".shp", ".zip")) + zip_filename = os.path.join(DOWNLOADDIR, file_basename + ".zip") engine_oedb = db.connection(readonly=True) session = sessionmaker(bind=engine_oedb)() @@ -31,12 +32,15 @@ def oedbtable2shp(table_query, filename, crs, geometry_col, index_col): session.bind, index_col=index_col) - table_df[geometry_col] = table_df[geometry_col].apply(wkt.loads) - table_gdf = gdp.GeoDataFrame(table_df, - geometry=geometry_col, - crs=crs) + if geometry_col: + table_df[geometry_col] = table_df[geometry_col].apply(wkt.loads) + table_gdf = gdp.GeoDataFrame(table_df, + geometry=geometry_col, + crs=crs) - table_gdf.to_file(shp_filename) + table_gdf.to_file(shp_filename) + else: + table_df.to_csv(shp_filename) zf = zipfile.ZipFile( zip_filename, diff --git a/preprocessing/utility/osmTGmod_to_shp.py b/preprocessing/utility/osmTGmod_to_shp.py new file mode 100644 index 00000000..3656849c --- /dev/null +++ b/preprocessing/utility/osmTGmod_to_shp.py @@ -0,0 +1,116 @@ +from egoio.db_tables.grid import OtgEhvhvBusDatum, OtgEhvhvBranchDatum, OtgEhvhvDclineDatum, OtgEhvhvResultsMetadatum +import os +from preprocessing.utility.download_oedb_data_to_shp import oedbtable2shp +from sqlalchemy import func +from sqlalchemy.orm import sessionmaker +from egoio.tools import db + + + +CRS = 4326 + + +def osmTGmod2shp(): + + filename_bus = "osmTGmod_bus.shp" + filename_branch = "osmTGmod_branch.shp" + filename_dcline = "osmTGmod_dcline.shp" + filename_results = "osmTGmod_results.csv" + + engine_oedb = db.connection(readonly=True) + session = sessionmaker(bind=engine_oedb)() + + bus_query = session.query( + OtgEhvhvBusDatum.result_id, + OtgEhvhvBusDatum.view_id, + OtgEhvhvBusDatum.bus_i, + OtgEhvhvBusDatum.bus_type, + OtgEhvhvBusDatum.pd, + OtgEhvhvBusDatum.qd, + OtgEhvhvBusDatum.gs, + OtgEhvhvBusDatum.bs, + OtgEhvhvBusDatum.bus_area, + OtgEhvhvBusDatum.vm, + OtgEhvhvBusDatum.va, + OtgEhvhvBusDatum.base_kv, + OtgEhvhvBusDatum.zone, + OtgEhvhvBusDatum.vmax, + OtgEhvhvBusDatum.vmin, + OtgEhvhvBusDatum.osm_substation_id, + OtgEhvhvBusDatum.cntr_id, + OtgEhvhvBusDatum.frequency, + OtgEhvhvBusDatum.osm_name, + OtgEhvhvBusDatum.result, + func.ST_AsText(func.ST_Transform(OtgEhvhvBusDatum.geom, CRS)).label("geom")) + + branch_query = session.query( + OtgEhvhvBranchDatum.result_id, + OtgEhvhvBranchDatum.view_id, + OtgEhvhvBranchDatum.branch_id, + OtgEhvhvBranchDatum.f_bus, + OtgEhvhvBranchDatum.t_bus, + OtgEhvhvBranchDatum.br_r, + OtgEhvhvBranchDatum.br_x, + OtgEhvhvBranchDatum.br_b, + OtgEhvhvBranchDatum.rate_a, + OtgEhvhvBranchDatum.rate_b, + OtgEhvhvBranchDatum.rate_c, + OtgEhvhvBranchDatum.tap, + OtgEhvhvBranchDatum.shift, + OtgEhvhvBranchDatum.br_status, + OtgEhvhvBranchDatum.link_type, + OtgEhvhvBranchDatum.branch_voltage, + OtgEhvhvBranchDatum.cables, + OtgEhvhvBranchDatum.frequency, + OtgEhvhvBranchDatum.result, + func.ST_AsText(func.ST_Transform(OtgEhvhvBranchDatum.geom, CRS)).label("geom"), + func.ST_AsText(func.ST_Transform(OtgEhvhvBranchDatum.topo, CRS)).label("topo") + ) + + dcline_query = session.query( + OtgEhvhvDclineDatum.result_id, + OtgEhvhvDclineDatum.view_id, + OtgEhvhvDclineDatum.dcline_id, + OtgEhvhvDclineDatum.f_bus, + OtgEhvhvDclineDatum.t_bus, + OtgEhvhvDclineDatum.br_status, + OtgEhvhvDclineDatum.pf, + OtgEhvhvDclineDatum.pt, + OtgEhvhvDclineDatum.qf, + OtgEhvhvDclineDatum.qt, + OtgEhvhvDclineDatum.vf, + OtgEhvhvDclineDatum.vt, + OtgEhvhvDclineDatum.pmin, + OtgEhvhvDclineDatum.pmax, + OtgEhvhvDclineDatum.qminf, + OtgEhvhvDclineDatum.qmaxf, + OtgEhvhvDclineDatum.qmint, + OtgEhvhvDclineDatum.qmaxt, + OtgEhvhvDclineDatum.loss0, + OtgEhvhvDclineDatum.loss1, + OtgEhvhvDclineDatum.link_type, + OtgEhvhvDclineDatum.branch_voltage, + OtgEhvhvDclineDatum.cables, + OtgEhvhvDclineDatum.frequency, + func.ST_AsText(func.ST_Transform(OtgEhvhvDclineDatum.geom, CRS)).label("geom"), + func.ST_AsText(func.ST_Transform(OtgEhvhvDclineDatum.topo, CRS)).label("topo"), + OtgEhvhvDclineDatum.result + ) + + results_query = session.query( + OtgEhvhvResultsMetadatum.id, + OtgEhvhvResultsMetadatum.osm_date, + OtgEhvhvResultsMetadatum.abstraction_date, + OtgEhvhvResultsMetadatum.applied_plans, + OtgEhvhvResultsMetadatum.applied_year, + OtgEhvhvResultsMetadatum.user_comment, + ) + + oedbtable2shp(bus_query, filename_bus, CRS, "geom", "view_id") + oedbtable2shp(branch_query, filename_branch, CRS, "geom", "view_id") + oedbtable2shp(dcline_query, filename_dcline, CRS, "geom", "view_id") + oedbtable2shp(results_query, filename_results, CRS, None, "id") + + +if __name__== "__main__": + osmTGmod2shp() \ No newline at end of file From 94ec37b0238838c6ca2da050913d27fceef89c29 Mon Sep 17 00:00:00 2001 From: eosram <> Date: Fri, 8 May 2020 15:05:36 +0200 Subject: [PATCH 64/84] Unfinished util to copy data from oedb to local db In order for it to work the variable `LOCAL` in `main` has to be set to the local database URL. `INPUT` specifies which tables should be copied from the oedb. --- preprocessing/utility/oedbtodb.py | 169 ++++++++++++++++++++++++++++++ 1 file changed, 169 insertions(+) create mode 100644 preprocessing/utility/oedbtodb.py diff --git a/preprocessing/utility/oedbtodb.py b/preprocessing/utility/oedbtodb.py new file mode 100644 index 00000000..ca650c2b --- /dev/null +++ b/preprocessing/utility/oedbtodb.py @@ -0,0 +1,169 @@ +# -*- coding: utf-8 -*- +""" + + +Sources +------- +https://github.com/OpenEnergyPlatform/oedialect/blob/master/doc/example/oedialec +https://stackoverflow.com/questions/21770829/sqlalchemy-copy-schema-and-data-of-subquery-to-another-database +https://geoalchemy-2.readthedocs.io/en/latest/elements.html + +""" + +import oedialect + +import sqlalchemy as sa + +from egoio.tools import Base +from importlib import import_module +from sqlalchemy import event +from sqlalchemy.orm import sessionmaker +from sqlalchemy.orm.session import make_transient +from geoalchemy2.elements import WKBElement + + +def meta_url(schema, table): + + base = "https://openenergy-platform.org/api/v0/" + return base + "schema/" + schema + "/tables/" + table + "/meta/" + + +def create_session(url): + """ Returns SQLAlchemy session object + + Parameters + ---------- + url : string + Database dialect and connection arguments + """ + engine = sa.create_engine(url) + return sessionmaker(bind=engine)() + + +def unique_schema(qualified): + """ Returns unique schema names + + Parameters + ---------- + qualified : list + List of qualified names in `schema.table` format + """ + + schemas = list(set([schema for schema, table in [i.split(".") for i in qualified]])) + + return schemas + + +def map_name_table(names, base): + """ Create map from `schema.table` to associated table object + + Parameters + ---------- + names : list + List of qualified names in `schema.table` format + base : `sqlalchemy.ext.declarative.api.Base` + + """ + mapper = {} + + for name, obj in base.metadata.tables.items(): + if name in names: + mapper[name] = obj + return mapper + + +def map_name_class(names, base): + """ Create map from `schema.table` to class + + Parameters + ---------- + names : list + List of qualified names in `schema.table` format + base : `sqlalchemy.ext.declarative.api.Base` + + Sources + ------- + https://stackoverflow.com/questions/11668355/sqlalchemy-get-model-from-table-name-this-may-imply-appending-some-function-to + + """ + mapper = {} + + for cls in Base._decl_class_registry.values(): + if hasattr(cls, "__table__"): + name = cls.__table__.schema + "." + cls.__tablename__ + if name in names: + mapper[name] = cls + return mapper + + +def main(): + """ """ + + LOCAL = "postgresql+psycopg2://user:password@localhost:5432/database" + REMOTE = "postgresql+oedialect://openenergy-platform.org" + + INPUT = { + "environment.dlm250_geb01_f": None, + "model_draft.wn_abw_ego_dp_hvmv_substation": {"version": "v0.3.0"}, + } + + source_session = create_session(REMOTE) + target_session = create_session(LOCAL) + + schemas = unique_schema(INPUT) + + # https://stackoverflow.com/questions/41678073/import-class-from-module-dynamically + + # by importing a submodule the class registry + # of `egoio.tools.Base` is extended with + # all classes of the imported submodule + for s in schemas: + packagename = "egoio.db_tables" + import_module(packagename + "." + s) + + class_map = map_name_class(INPUT.keys(), Base) + table_map = map_name_table(INPUT.keys(), Base) + + # create schema if not exists + # maybe as event hook? + # https://github.com/sqlalchemy/sqlalchemy/issues/3982 + for schema in schemas: + query = "CREATE SCHEMA IF NOT EXISTS {schema}".format(schema=schema) + target_session.execute(query) + target_session.commit() + + # create subset of database model in local database + metadata = Base.metadata + metadata.bind = target_session.connection().engine + + # https://stackoverflow.com/questions/19175311/how-to-create-only-one-table-with-sqlalchemy + metadata.create_all(tables=table_map.values(), checkfirst=True) + + # retrieve data from remote database + results = [] + + for name, v in INPUT.items(): + query = source_session.query(class_map[name]) + if v: + if "version" in v: + query = query.filter(class_map[name].version == v["version"]) + for ds in query: + make_transient(ds) + results.append(ds) + + # create WKBElement from WKBElement ;) + # otherwise database error about SRID being 0 + for r in results: + for k, v in vars(r).items(): + if isinstance(v, WKBElement): + wkb = WKBElement(v.data, srid=v.srid) + setattr(r, k, wkb) + target_session.add(r) + + target_session.commit() + source_session.close() + target_session.close() + + +if __name__ == "__main__": + main() From 8c2cccd336329f1dceeee27f8658e048cb8dcd70 Mon Sep 17 00:00:00 2001 From: eosram <> Date: Fri, 8 May 2020 15:51:38 +0200 Subject: [PATCH 65/84] Update requirements Atm oedialect requires `geoalchemy < 0.7.0`. --- setup.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 089d6c5d..6e674b77 100644 --- a/setup.py +++ b/setup.py @@ -18,8 +18,9 @@ 'workalendar', 'oemof.db', 'demandlib', - 'ego.io >=0.0.1rc4, <= 0.0.2', - 'geoalchemy2' + 'egoio @ git+https://github.com/openego/ego.io.git@features/use-one-Base-definition', + 'oedialect @ git+https://github.com/OpenEnergyPlatform/oedialect.git@4957e63ca46a976e35eecf84036466cb624e6bfe', + 'geoalchemy2 < 0.7.0', ], extras_require={ 'docs': [ From 9a60cb8e2d844e65fbaadadb9bb5494cfc7621f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guido=20Ple=C3=9Fmann?= Date: Wed, 13 May 2020 16:16:22 +0200 Subject: [PATCH 66/84] Add osmTGmod to DP-pre pipeline --- preprocessing/import.yml | 12 +++- preprocessing/osmtgmod/import_osmtgmod.py | 68 +++++++++++++++++++++++ preprocessing/utility/osmTGmod_to_shp.py | 34 ++++++++---- preprocessing/utility/utils.py | 15 +++++ 4 files changed, 118 insertions(+), 11 deletions(-) create mode 100644 preprocessing/osmtgmod/import_osmtgmod.py create mode 100644 preprocessing/utility/utils.py diff --git a/preprocessing/import.yml b/preprocessing/import.yml index 2064e104..ee8effbd 100644 --- a/preprocessing/import.yml +++ b/preprocessing/import.yml @@ -93,4 +93,14 @@ slp: scripts: - script: ego_pre_slp_parameters.sql language: SQL - +osmtgmod: + required_data: + - url: https://next.rl-institut.de/s/zb4yWoxHpKACw7Q/download + filename: otg_ehvhv_data.zip + scripts: + - script: preprocessing.osmtgmod.import_osmtgmod::import_osmtgmod + language: python + filename: otg_ehvhv_data.zip + db: conn + args: + schema: "grid" diff --git a/preprocessing/osmtgmod/import_osmtgmod.py b/preprocessing/osmtgmod/import_osmtgmod.py new file mode 100644 index 00000000..f7a6e594 --- /dev/null +++ b/preprocessing/osmtgmod/import_osmtgmod.py @@ -0,0 +1,68 @@ +import os +import zipfile +import sqlalchemy +import geopandas as gpd +import pandas as pd +from preprocessing.utility.utils import wkb_hexer +from geoalchemy2 import Geometry +from shapely.wkt import loads as wkt_loads + + +def certain_file_types(directory, extension): + return (f for f in os.listdir(directory) if f.endswith('.' + extension)) + + +def import_osmtgmod(file, path, db, **kwds): + + # Create schema if it does not exist + if not db.dialect.has_schema(db, kwds["schema"]): + db.execute(sqlalchemy.schema.CreateSchema(kwds["schema"])) + + # TODO: download and add metadata to osmtgmod zip + # TODO: write metadata to local table (would this be easier when ORMs are used locally?) + # TODO: Add primary keys to table + + # File names and paths + file_base, file_ext = os.path.splitext(os.path.join(path, file)) + extracted_path = os.path.join(path, file_base) + + # Extract data from zip archive + zip = zipfile.ZipFile(os.path.join(path, file)) + zip.extractall(path=extracted_path) + + # iterate through shapefiles and import to database + for shp_file in certain_file_types(extracted_path, "shp"): + df = gpd.read_file(os.path.join(extracted_path, shp_file)) + + # Store geometries in WKB hex format + df['geom'] = df['geometry'].apply(wkb_hexer) + df.drop("geometry", axis=1, inplace=True) + if "topo" in df.columns: + df['topo'] = df['topo'].apply(wkt_loads) + df['topo'] = df['topo'].apply(wkb_hexer) + + table = os.path.splitext(shp_file)[0] + + df.to_sql(table, + db, + kwds["schema"], + index=False, + if_exists="replace", + chunksize=10000, + method="multi", + dtype={'geom': Geometry(), + 'topo': Geometry()} + ) + + for csv_file in certain_file_types(extracted_path, "csv"): + df = pd.read_csv(os.path.join(extracted_path, csv_file), + parse_dates=["osm_date", "abstraction_date"]) + + table = os.path.splitext(csv_file)[0] + + df.to_sql(table, + db, + kwds["schema"], + index=False, + if_exists="replace" + ) diff --git a/preprocessing/utility/osmTGmod_to_shp.py b/preprocessing/utility/osmTGmod_to_shp.py index 3656849c..6cf529dc 100644 --- a/preprocessing/utility/osmTGmod_to_shp.py +++ b/preprocessing/utility/osmTGmod_to_shp.py @@ -1,10 +1,11 @@ from egoio.db_tables.grid import OtgEhvhvBusDatum, OtgEhvhvBranchDatum, OtgEhvhvDclineDatum, OtgEhvhvResultsMetadatum import os from preprocessing.utility.download_oedb_data_to_shp import oedbtable2shp +from preprocessing.utility.utils import DOWNLOADDIR, zipdir from sqlalchemy import func from sqlalchemy.orm import sessionmaker from egoio.tools import db - +import zipfile CRS = 4326 @@ -12,10 +13,11 @@ def osmTGmod2shp(): - filename_bus = "osmTGmod_bus.shp" - filename_branch = "osmTGmod_branch.shp" - filename_dcline = "osmTGmod_dcline.shp" - filename_results = "osmTGmod_results.csv" + filename_zip = "otg_ehvhv_data.zip" + filename_bus = "otg_ehvhv_bus_data.shp" + filename_branch = "otg_ehvhv_branch_data.shp" + filename_dcline = "otg_ehvhv_dcline_data.shp" + filename_results = "otg_ehvhv_results_metadata.csv" engine_oedb = db.connection(readonly=True) session = sessionmaker(bind=engine_oedb)() @@ -106,11 +108,23 @@ def osmTGmod2shp(): OtgEhvhvResultsMetadatum.user_comment, ) - oedbtable2shp(bus_query, filename_bus, CRS, "geom", "view_id") - oedbtable2shp(branch_query, filename_branch, CRS, "geom", "view_id") - oedbtable2shp(dcline_query, filename_dcline, CRS, "geom", "view_id") - oedbtable2shp(results_query, filename_results, CRS, None, "id") + oedbtable2shp(bus_query, filename_bus, CRS, "geom", "view_id", zip_it=False) + oedbtable2shp(branch_query, filename_branch, CRS, "geom", "view_id", zip_it=False) + oedbtable2shp(dcline_query, filename_dcline, CRS, "geom", "view_id", zip_it=False) + oedbtable2shp(results_query, filename_results, CRS, None, "id", zip_it=False) + + # zip all files into a single archive + directories = [os.path.join(DOWNLOADDIR, os.path.splitext(f)[0]) for f in + [filename_bus, filename_branch, filename_dcline, filename_results] + ] + with zipfile.ZipFile( + os.path.join(DOWNLOADDIR, filename_zip), + mode='w', + compression=zipfile.ZIP_DEFLATED) as zf: + for directory in directories: + zipdir(directory, zf) + zf.close() if __name__== "__main__": - osmTGmod2shp() \ No newline at end of file + osmTGmod2shp() diff --git a/preprocessing/utility/utils.py b/preprocessing/utility/utils.py new file mode 100644 index 00000000..b1fb33c8 --- /dev/null +++ b/preprocessing/utility/utils.py @@ -0,0 +1,15 @@ +import os + + +DOWNLOADDIR = os.path.join(os.path.expanduser("~"), ".egon-pre-processing-cached/") + + +def zipdir(path, ziph): + # ziph is zipfile handle + for root, dirs, files in os.walk(path): + for file in files: + ziph.write(os.path.join(root, file), arcname=file) + + +def wkb_hexer(line): + return line.wkb_hex \ No newline at end of file From 17ef1d59f63d2962d8253297c1c519837ee88940 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guido=20Ple=C3=9Fmann?= Date: Wed, 13 May 2020 16:17:10 +0200 Subject: [PATCH 67/84] Make zipping optional and use function from utils.py --- .../utility/download_oedb_data_to_shp.py | 26 +++++++------------ 1 file changed, 9 insertions(+), 17 deletions(-) diff --git a/preprocessing/utility/download_oedb_data_to_shp.py b/preprocessing/utility/download_oedb_data_to_shp.py index 5ab27faa..b6fcbda2 100644 --- a/preprocessing/utility/download_oedb_data_to_shp.py +++ b/preprocessing/utility/download_oedb_data_to_shp.py @@ -5,19 +5,10 @@ from shapely import wkt import zipfile import os +from preprocessing.utility.utils import DOWNLOADDIR, zipdir -DOWNLOADDIR = os.path.join(os.path.expanduser("~"), ".egon-pre-processing-cached/") - - -def zipdir(path, ziph): - # ziph is zipfile handle - for root, dirs, files in os.walk(path): - for file in files: - ziph.write(os.path.join(root, file), arcname=file) - - -def oedbtable2shp(table_query, filename, crs, geometry_col, index_col): +def oedbtable2shp(table_query, filename, crs, geometry_col, index_col, zip_it=True): file_basename = os.path.basename(os.path.splitext(filename)[0]) shp_dir = os.path.join(DOWNLOADDIR, file_basename) @@ -42,9 +33,10 @@ def oedbtable2shp(table_query, filename, crs, geometry_col, index_col): else: table_df.to_csv(shp_filename) - zf = zipfile.ZipFile( - zip_filename, - mode='w', - compression=zipfile.ZIP_DEFLATED) - zipdir(shp_dir, zf) - zf.close() \ No newline at end of file + if zip_it: + zf = zipfile.ZipFile( + zip_filename, + mode='w', + compression=zipfile.ZIP_DEFLATED) + zipdir(shp_dir, zf) + zf.close() \ No newline at end of file From c894c2ed604db23c943e3634b2bb11a307b1402b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guido=20Ple=C3=9Fmann?= Date: Mon, 25 May 2020 10:45:01 +0200 Subject: [PATCH 68/84] Use yaml.SafeLoader --- preprocessing/eGo_import_and_preprocessing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/preprocessing/eGo_import_and_preprocessing.py b/preprocessing/eGo_import_and_preprocessing.py index 6485202b..c45af395 100644 --- a/preprocessing/eGo_import_and_preprocessing.py +++ b/preprocessing/eGo_import_and_preprocessing.py @@ -88,7 +88,7 @@ def preprocessing(): 'ego_pre_voltage_level.sql', 'ego_pre_slp_parameters.sql' ] - datasets = yaml.load(open("import.yml"))#, Loader=Loader) + datasets = yaml.load(open("import.yml"), Loader=yaml.SafeLoader) # get database connection conn_oedb = db.connection(readonly=True).connect() From 155f9dcb6c1578c50087d6ecbd50a4e3526e78e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guido=20Ple=C3=9Fmann?= Date: Mon, 25 May 2020 12:12:24 +0200 Subject: [PATCH 69/84] Update DP-pre order BPMN --- .../bpmn/ego_dp-pre_bpmn_sections.graphml | 462 +++++++++++++++--- 1 file changed, 386 insertions(+), 76 deletions(-) diff --git a/documentation/bpmn/ego_dp-pre_bpmn_sections.graphml b/documentation/bpmn/ego_dp-pre_bpmn_sections.graphml index 32514ce6..a63c2d21 100644 --- a/documentation/bpmn/ego_dp-pre_bpmn_sections.graphml +++ b/documentation/bpmn/ego_dp-pre_bpmn_sections.graphml @@ -17,11 +17,11 @@ - + - eGo Data-PreProcessing - + eGo Data-PreProcessing + @@ -31,40 +31,40 @@ - - - - + + + + + - - + - + - + @@ -75,7 +75,7 @@ - + zensus @@ -93,10 +93,10 @@ - + - bkg_vg250 + zensus_vg250 @@ -109,30 +109,9 @@ - - - - - - boundaries.bkg_vg250_1_sta_union_mview -model_draft.ego_boundaries_bkg_vg250_6_gem_clean -political_boundary.bkg_vg250_4_krs_mview -boundaries.bkg_vg250_6_gem -political_boundary.bkg_vg250_1_sta_union_mview - - - - - - - - - - - - + @@ -153,10 +132,10 @@ political_boundary.bkg_vg250_1_sta_union_mview - + - + @@ -177,85 +156,416 @@ political_boundary.bkg_vg250_1_sta_union_mview + + + + + + + vg250 + + + + + + + + + + + - - - - + + + + + openstreetmap + + + + + + + + + + + + + + + + + + scenario_log + + + + + + + + + + + + + + + + + + SLP + + + + + + + + + + + + + + + + + + cosmoclmgrid + + + + + + + + + + + + + + + + + + wind potential areas - + - + + + + + + + + + + + + + renpassG!S + + + + + + + + + + + + + + + + + + supply_powerplant + + + + + + + + + + + + + + + + + + demand_federalstate + + + + + + + + + + + + + + + + + + osmTGmod + + + + + + + - + - - - - - + + - + - + + - + - - - - + + - + + - + - - - - + + - + + - + - - - - - - - - - + + + + - + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + From 11361dcbc2498e70687822332c27aa4db1fb5d2c Mon Sep 17 00:00:00 2001 From: IlkaCu Date: Wed, 3 Jun 2020 14:58:11 +0200 Subject: [PATCH 70/84] changes in supply docu --- .../ego_dp-pre_bpmn_section_supply.graphml | 560 ++++++++++++------ 1 file changed, 375 insertions(+), 185 deletions(-) diff --git a/documentation/bpmn/ego_dp-pre_bpmn_section_supply.graphml b/documentation/bpmn/ego_dp-pre_bpmn_section_supply.graphml index ba8a7fd8..0614221b 100644 --- a/documentation/bpmn/ego_dp-pre_bpmn_section_supply.graphml +++ b/documentation/bpmn/ego_dp-pre_bpmn_section_supply.graphml @@ -17,7 +17,7 @@ - + eGo Pre-Processing - supply @@ -31,35 +31,35 @@ + + + - - - - - + + - + - + @@ -112,7 +112,7 @@ model_draft.ego_power_class - + @@ -136,7 +136,7 @@ model_draft.ego_power_class - + @@ -160,7 +160,7 @@ model_draft.ego_power_class - + ego_dp_res_rea_by_scenario.sql @@ -178,7 +178,7 @@ model_draft.ego_power_class - + model_draft.ego_dp_supply_res_powerplant @@ -196,10 +196,10 @@ model_draft.ego_power_class - + - model_draft.ego_small_chp_plant_germany + model_draft.ego_small_chp_plant_germany model_draft.ego_dp_supply_res_powerplant model_draft.ego_dp_supply_conv_powerplant model_draft.ego_supply_scenario_capacities @@ -218,7 +218,7 @@ boundaries.bkg_vg250_6_gem_rs_mview - + model_draft.ego_supply_conv_powerplant_2035 @@ -236,7 +236,7 @@ boundaries.bkg_vg250_6_gem_rs_mview - + model_draft.ego_dp_supply_conv_powerplant @@ -254,7 +254,7 @@ boundaries.bkg_vg250_6_gem_rs_mview - + ego_dp_conv_by_scenario.sql @@ -272,7 +272,7 @@ boundaries.bkg_vg250_6_gem_rs_mview - + model_draft.ego_dp_supply_conv_powerplant @@ -290,7 +290,7 @@ boundaries.bkg_vg250_6_gem_rs_mview - + model_draft.ego_dp_supply_conv_powerplant @@ -308,7 +308,7 @@ model_draft.ego_supply_conv_powerplant_2035 - + ego_dp_supply_add_coastdat_gid.sql @@ -326,7 +326,7 @@ model_draft.ego_supply_conv_powerplant_2035 - + model_draft.ego_dp_supply_conv_powerplant @@ -345,7 +345,7 @@ model_draft.ego_dp_supply_res_powerplant - + model_draft.ego_dp_supply_conv_powerplant @@ -368,7 +368,7 @@ coastdat.cosmoclmgrid - model_draft.ego_supply_res_powerplant + model_draft.ego_dp_supply_res_powerplant model_draft.ego_supply_renewable_bnetza_full_attribute model_draft.bnetza_eeganlagenstammdaten_wind_classification @@ -382,30 +382,12 @@ model_draft.bnetza_eeganlagenstammdaten_wind_classification - - - - - - OPSD beta Version of 2015/16 - - - - - - - - - - - - + - ego_dp_preprocessing_conv_powerplant.sql -LOOP! + ego_dp_preprocessing_conv_powerplant.sql @@ -417,13 +399,13 @@ LOOP! - + - + - model_draft.ego_dp_supply_conv_powerplant + model_draft.ego_dp_supply_conv_powerplant model_draft.ego_supply_conv_nep2035_temp @@ -435,33 +417,13 @@ model_draft.ego_supply_conv_nep2035_temp - - - - - - - supply_ego_renewable_power_plants_germany.sql -Nur CREATE keine Daten! - - - - - - - - - - - - + - + - energymap -BNetzA + Dateiname @@ -472,10 +434,10 @@ BNetzA - + - + ego_dp_preprocessing_res_powerplant.sql @@ -490,10 +452,10 @@ BNetzA - + - + model_draft.ego_dp_supply_res_powerplant @@ -508,31 +470,13 @@ BNetzA - - - - - - - model_draft.ego_supply_res_powerplant - - - - - - - - - - - - + - + - supply.ego_renewable_powerplant + supply.ego_renewable_powerplant @@ -543,14 +487,13 @@ BNetzA - + - + - supply.ego_conventional_powerplant -model_draft.ego_dp_supply_conv_powerplant + supply.ego_conventional_powerplant @@ -561,10 +504,10 @@ model_draft.ego_dp_supply_conv_powerplant - + - + bkg_vg250 @@ -578,7 +521,7 @@ model_draft.ego_dp_supply_conv_powerplant - + @@ -596,7 +539,7 @@ model_draft.ego_dp_supply_conv_powerplant - + @@ -613,7 +556,7 @@ model_draft.ego_dp_supply_conv_powerplant - + @@ -631,10 +574,10 @@ FlEnS - + - + utility/ego_dp_preprocessing_conv_powerplant_2035.sql @@ -649,7 +592,7 @@ FlEnS - + @@ -666,11 +609,11 @@ FlEnS - + - + ego_simple-feedin_per_scenario.sql @@ -684,7 +627,7 @@ FlEnS - + @@ -702,7 +645,7 @@ model_draft.ego_weather_measurement_point - + @@ -722,14 +665,65 @@ model_draft.ego_grid_hv_electrical_neighbours_bus + + + + + + + IMPORT + + + + + + + + + + + + + + + + + + supply.ego_renewable_powerplant + + + + + + + + + + + + + + + + + Dateiname + + + + + + + + + + - - + + - opsd_powerdata.sql -Nur CREATE keine Daten! + IMPORT @@ -742,12 +736,46 @@ Nur CREATE keine Daten! + + + + + + supply.ego_conventional_powerplant + + + + + + + + + + + + + + + + + Dateiname + + + + + + + + + + + - + - ? Kein Skript vorhanden ? + IMPORT @@ -759,14 +787,13 @@ Nur CREATE keine Daten! - + - + - model_draft.ego_supply_res_powerplant ---> #120 redundant table + model_draft.ego_supply_renewable_bnetza_full_attribute @@ -777,14 +804,13 @@ Nur CREATE keine Daten! - + - - + + - BNetzA -MA Marlon? + Dateiname @@ -795,6 +821,93 @@ MA Marlon? < + + + + + + + IMPORT + + + + + + + + + + + + + + + + + + model_draft.bnetza_eeganlagenstammdaten_wind_classification + + + + + + + + + + + + + + + + + Dateiname + + + + + + + + + + + + + + + + + IMPORT + + + + + + + + + + + + + + + + + + model_draft.ego_small_chp_plant_germany + + + + + + + + + + @@ -899,7 +1012,7 @@ MA Marlon? < - + @@ -909,7 +1022,7 @@ MA Marlon? < - + @@ -919,17 +1032,17 @@ MA Marlon? < - + - - + + - + @@ -939,7 +1052,7 @@ MA Marlon? < - + @@ -949,7 +1062,7 @@ MA Marlon? < - + @@ -959,7 +1072,7 @@ MA Marlon? < - + @@ -969,7 +1082,7 @@ MA Marlon? < - + @@ -979,7 +1092,7 @@ MA Marlon? < - + @@ -989,7 +1102,7 @@ MA Marlon? < - + @@ -999,7 +1112,17 @@ MA Marlon? < - + + + + + + + + + + + @@ -1009,7 +1132,7 @@ MA Marlon? < - + @@ -1019,17 +1142,27 @@ MA Marlon? < - + - - + + + + + + + + + + + + - + @@ -1039,7 +1172,7 @@ MA Marlon? < - + @@ -1049,7 +1182,7 @@ MA Marlon? < - + @@ -1059,7 +1192,7 @@ MA Marlon? < - + @@ -1069,7 +1202,7 @@ MA Marlon? < - + @@ -1079,7 +1212,7 @@ MA Marlon? < - + @@ -1089,7 +1222,37 @@ MA Marlon? < - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -1099,7 +1262,8 @@ MA Marlon? < - + + @@ -1109,7 +1273,8 @@ MA Marlon? < - + + @@ -1119,7 +1284,8 @@ MA Marlon? < - + + @@ -1129,7 +1295,7 @@ MA Marlon? < - + @@ -1140,7 +1306,7 @@ MA Marlon? < - + @@ -1151,7 +1317,7 @@ MA Marlon? < - + @@ -1162,7 +1328,7 @@ MA Marlon? < - + @@ -1173,7 +1339,20 @@ MA Marlon? < - + + + + + + + + + + + + + + @@ -1184,18 +1363,18 @@ MA Marlon? < - + - + - + @@ -1206,95 +1385,106 @@ MA Marlon? < - + - - + + - + - - + + - + - - + + - + - - + + - + - - + + - + - - + + - + - - + + - + - - + + + + + + + + + + + + + - + From 1d8632a193aff9751bcf0db3a899719097e5cf3b Mon Sep 17 00:00:00 2001 From: IlkaCu Date: Thu, 4 Jun 2020 13:03:54 +0200 Subject: [PATCH 71/84] add BPMN for versioning section --- .../ego_dp_bpmn_section_versioning.graphml | 519 ++++++++++++++++++ 1 file changed, 519 insertions(+) create mode 100644 documentation/bpmn/ego_dp_bpmn_section_versioning.graphml diff --git a/documentation/bpmn/ego_dp_bpmn_section_versioning.graphml b/documentation/bpmn/ego_dp_bpmn_section_versioning.graphml new file mode 100644 index 00000000..e2d506b2 --- /dev/null +++ b/documentation/bpmn/ego_dp_bpmn_section_versioning.graphml @@ -0,0 +1,519 @@ + + + + + + + + + + + + + + + + + + + + + + + eGo DataProcessing - versioning + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ego_pp_nep2035_grid_variations.sql + + + + + + + + + + + + + + + + + + grid.ego_dp_ehv_substation +grid.ego_dp_hvmv_substation +grid.ego_dp_mvlv_substation +grid.ego_dp_ehv_griddistrict +grid.ego_dp_mv_griddistrict +grid.ego_dp_lv_griddistrict +demand.ego_dp_loadarea +supply.ego_dp_conv_powerplant +supply.ego_dp_res_powerplant +grid.ego_pf_hv_bus +grid.ego_pf_hv_generator +grid.ego_pf_hv_generator_pq_set +grid.ego_pf_hv_line +grid.ego_pf_hv_link +grid.ego_pf_hv_load +grid.ego_pf_hv_load_pq_set +grid.ego_pf_hv_source +grid.ego_pf_hv_storage +grid.ego_pf_hv_storage_pq_set +grid.ego_pf_hv_temp_resolution +grid.ego_pf_hv_transformer +grid.ego_pf_hv_extension_bus +grid.ego_pf_hv_extension_generator +grid.ego_pf_hv_extension_generator_pq_set +grid.ego_pf_hv_extension_line +grid.ego_pf_hv_extension_link +grid.ego_pf_hv_extension_load +grid.ego_pf_hv_extension_load_pq_set +grid.ego_pf_hv_extension_source +grid.ego_pf_hv_extension_storage +grid.ego_pf_hv_extension_storage_pq_set +grid.ego_pf_hv_extension_temp_resolution +grid.ego_pf_hv_extension_transformer +grid.ego_line_expansion_costs +supply.ego_renewable_feedin +supply.ego_power_class +supply.ego_aggr_weather +grid.ego_pf_hv_data_check + + + + + + + + + + + + + + + + + + model_draft.ego_grid_ehv_substation +model_draft.ego_grid_hvmv_substation +model_draft.ego_grid_mvlv_substation +model_draft.ego_grid_ehv_substation_voronoi +model_draft.ego_grid_mv_griddistrict +model_draft.ego_grid_lv_griddistrict +model_draft.ego_demand_loadarea +model_draft.ego_dp_supply_conv_powerplant +model_draft.ego_dp_supply_res_powerplant +model_draft.ego_grid_pf_hv_bus +model_draft.ego_grid_pf_hv_generator +model_draft.ego_grid_pf_hv_generator_pq_set +model_draft.ego_grid_pf_hv_line +model_draft.ego_grid_pf_hv_link +model_draft.ego_grid_pf_hv_load +model_draft.ego_grid_pf_hv_load_pq_set +model_draft.ego_grid_pf_hv_source +model_draft.ego_grid_pf_hv_storage +model_draft.ego_grid_pf_hv_storage_pq_set +model_draft.ego_grid_pf_hv_temp_resolution +model_draft.ego_grid_pf_hv_transformer +model_draft.ego_grid_pf_hv_extension_bus +model_draft.ego_grid_pf_hv_extension_generator +model_draft.ego_grid_pf_hv_extension_generator_pq_set +model_draft.ego_grid_pf_hv_extension_line +model_draft.ego_grid_pf_hv_extension_link +model_draft.ego_grid_pf_hv_extension_load +model_draft.ego_grid_pf_hv_extension_load_pq_set +model_draft.ego_grid_pf_hv_extension_source +model_draft.ego_grid_pf_hv_extension_storage +model_draft.ego_grid_pf_hv_extension_storage_pq_set +model_draft.ego_grid_pf_hv_extension_temp_resolution +model_draft.ego_grid_pf_hv_extension_transformer +model_draft.ego_grid_line_expansion_costs +model_draft.ego_renewable_feedin +model_draft.ego_power_class +model_draft.ego_supply_aggr_weather +model_draft.ego_grid_pf_hv_data_check + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ego_pp_nep2035_grid_variations.sql + + + + + + + + + + + + + + + + + + supply.ego_dp_conv_powerplant_sq_mview +supply.ego_dp_conv_powerplant_nep2035_mview +supply.ego_dp_conv_powerplant_ego100_mview +supply.ego_dp_res_powerplant_sq_mview +supply.ego_dp_res_powerplant_nep2035_mview +supply.ego_dp_res_powerplant_ego100_mview + + + + + + + + + + + + + + + + + + supply.ego_dp_conv_powerplant +supply.ego_dp_res_powerplant + + + + + + + + + + + + + + + + + DP griddistrict + + + + + + + + + + + + + + + + + DP powerflow + + + + + + + + + + + + + + + + + DP substation + + + + + + + + + + + + + + + + + DP loadarea + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + From 9d87d2e38eea00b381a28f64123d24cc832adafe Mon Sep 17 00:00:00 2001 From: IlkaCu Date: Thu, 30 Jul 2020 13:10:45 +0200 Subject: [PATCH 72/84] add BPMN for section Input data check --- ...o_dp_bpmn_section_input_data_check.graphml | 229 ++++++++++++++++++ 1 file changed, 229 insertions(+) create mode 100644 documentation/bpmn/ego_dp_bpmn_section_input_data_check.graphml diff --git a/documentation/bpmn/ego_dp_bpmn_section_input_data_check.graphml b/documentation/bpmn/ego_dp_bpmn_section_input_data_check.graphml new file mode 100644 index 00000000..341cbbca --- /dev/null +++ b/documentation/bpmn/ego_dp_bpmn_section_input_data_check.graphml @@ -0,0 +1,229 @@ + + + + + + + + + + + + + + + + + + + + + + + eGo DataProcessing - input data check + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +ego_dp_substation_hvmv.sql + + + + + + + + + + + + + + + + + + model_draft.ego_scenario_input + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + model_draft.ego_scenario_input + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + From 0e1f08fff6dd51c476bd261aca38855c263d8424 Mon Sep 17 00:00:00 2001 From: IlkaCu Date: Thu, 6 Aug 2020 14:04:32 +0200 Subject: [PATCH 73/84] complete BPMN documentation for supply section --- .../ego_dp-pre_bpmn_section_supply.graphml | 633 +++++------------- 1 file changed, 173 insertions(+), 460 deletions(-) diff --git a/documentation/bpmn/ego_dp-pre_bpmn_section_supply.graphml b/documentation/bpmn/ego_dp-pre_bpmn_section_supply.graphml index 0614221b..3af490e1 100644 --- a/documentation/bpmn/ego_dp-pre_bpmn_section_supply.graphml +++ b/documentation/bpmn/ego_dp-pre_bpmn_section_supply.graphml @@ -17,7 +17,7 @@ - + eGo Pre-Processing - supply @@ -53,13 +53,13 @@ - + - + @@ -75,7 +75,7 @@ - + preprocessing/python_scripts/ @@ -94,7 +94,7 @@ renpass_gis/simple_feedin/simple_feedin.py - + model_draft.ego_renewable_feedin @@ -112,7 +112,7 @@ model_draft.ego_power_class - + @@ -136,7 +136,7 @@ model_draft.ego_power_class - + @@ -160,10 +160,10 @@ model_draft.ego_power_class - + - ego_dp_res_rea_by_scenario.sql + ego_dp_res_rea_by_scenario.sql @@ -178,7 +178,7 @@ model_draft.ego_power_class - + model_draft.ego_dp_supply_res_powerplant @@ -196,7 +196,7 @@ model_draft.ego_power_class - + model_draft.ego_small_chp_plant_germany @@ -218,7 +218,7 @@ boundaries.bkg_vg250_6_gem_rs_mview - + model_draft.ego_supply_conv_powerplant_2035 @@ -236,7 +236,7 @@ boundaries.bkg_vg250_6_gem_rs_mview - + model_draft.ego_dp_supply_conv_powerplant @@ -254,7 +254,7 @@ boundaries.bkg_vg250_6_gem_rs_mview - + ego_dp_conv_by_scenario.sql @@ -272,7 +272,7 @@ boundaries.bkg_vg250_6_gem_rs_mview - + model_draft.ego_dp_supply_conv_powerplant @@ -290,7 +290,7 @@ boundaries.bkg_vg250_6_gem_rs_mview - + model_draft.ego_dp_supply_conv_powerplant @@ -308,7 +308,7 @@ model_draft.ego_supply_conv_powerplant_2035 - + ego_dp_supply_add_coastdat_gid.sql @@ -326,7 +326,7 @@ model_draft.ego_supply_conv_powerplant_2035 - + model_draft.ego_dp_supply_conv_powerplant @@ -345,10 +345,10 @@ model_draft.ego_dp_supply_res_powerplant - + - model_draft.ego_dp_supply_conv_powerplant + model_draft.ego_dp_supply_conv_powerplant model_draft.ego_dp_supply_res_powerplant coastdat.cosmoclmgrid @@ -365,12 +365,12 @@ coastdat.cosmoclmgrid - + model_draft.ego_dp_supply_res_powerplant -model_draft.ego_supply_renewable_bnetza_full_attribute -model_draft.bnetza_eeganlagenstammdaten_wind_classification +model_draft.bnetza_eeganlagenstammdaten_wind_classification +model_draft.ego_supply_renewable_bnetza_full_attribute @@ -384,7 +384,7 @@ model_draft.bnetza_eeganlagenstammdaten_wind_classification - + ego_dp_preprocessing_conv_powerplant.sql @@ -402,7 +402,7 @@ model_draft.bnetza_eeganlagenstammdaten_wind_classification - + model_draft.ego_dp_supply_conv_powerplant @@ -418,26 +418,9 @@ model_draft.ego_supply_conv_nep2035_temp - - - - - - Dateiname - - - - - - - - - - - - + ego_dp_preprocessing_res_powerplant.sql @@ -452,10 +435,10 @@ model_draft.ego_supply_conv_nep2035_temp - + - + model_draft.ego_dp_supply_res_powerplant @@ -470,11 +453,11 @@ model_draft.ego_supply_conv_nep2035_temp - + - - + + supply.ego_renewable_powerplant @@ -487,11 +470,11 @@ model_draft.ego_supply_conv_nep2035_temp - + - - + + supply.ego_conventional_powerplant @@ -504,10 +487,10 @@ model_draft.ego_supply_conv_nep2035_temp - + - + bkg_vg250 @@ -521,10 +504,10 @@ model_draft.ego_supply_conv_nep2035_temp - + - + ego_nep_2015_scenario_capacities.sql @@ -539,10 +522,10 @@ model_draft.ego_supply_conv_nep2035_temp - + - + model_draft.ego_supply_scenario_capacities @@ -556,10 +539,10 @@ model_draft.ego_supply_conv_nep2035_temp - + - + NEP 2015 @@ -574,10 +557,10 @@ FlEnS - + - + utility/ego_dp_preprocessing_conv_powerplant_2035.sql @@ -592,13 +575,13 @@ FlEnS - + - + - coastdat + coastdat @@ -609,28 +592,10 @@ FlEnS - - - - - - - ego_simple-feedin_per_scenario.sql - - - - - - - - - - - - + - + model_draft.ego_neighbours_offshore_point @@ -645,13 +610,13 @@ model_draft.ego_weather_measurement_point - + - + - climate.cosmoclmgrid + climate.cosmoclmgrid boundaries.bkg_vg250_1_sta model_draft.renpass_gis_parameter_region model_draft.ego_grid_hv_electrical_neighbours_bus @@ -665,152 +630,67 @@ model_draft.ego_grid_hv_electrical_neighbours_bus - - - - - - - IMPORT - - - - - - - - - - - - - - - - - - supply.ego_renewable_powerplant - - - - - - - - - - - + - + - Dateiname + model_draft.bnetza_eeganlagenstammdaten_wind_classification +model_draft.ego_supply_renewable_bnetza_full_attribute - - - - - - - - - - - - - - IMPORT - - - - - - - - - - - - - - - - - - supply.ego_conventional_powerplant - - - - + - + - Dateiname + model_draft.ego_small_chp_plant_germany + - - + - + - IMPORT + ego-simple-feedin_per_scenario.sql - + - - - - - - - model_draft.ego_supply_renewable_bnetza_full_attribute - - - - - - - - - - - + - + - Dateiname + model_draft.renpass_gis_parameter_region +model_draft.ego_grid_hv_electrical_neighbours_bus (DP) @@ -821,48 +701,13 @@ model_draft.ego_grid_hv_electrical_neighbours_bus - - - - - - - IMPORT - - - - - - - - - - - - - - - - - - model_draft.bnetza_eeganlagenstammdaten_wind_classification - - - - - - - - - - - + - - + + - Dateiname + supply.ego_renewable_powerplant @@ -873,37 +718,19 @@ model_draft.ego_grid_hv_electrical_neighbours_bus - - - - - - - IMPORT - - - - - - - - - - - - + - - + + - model_draft.ego_small_chp_plant_germany + supply.ego_conventional_powerplant - + @@ -914,7 +741,7 @@ model_draft.ego_grid_hv_electrical_neighbours_bus - + @@ -947,7 +774,7 @@ model_draft.ego_grid_hv_electrical_neighbours_bus - + @@ -957,7 +784,7 @@ model_draft.ego_grid_hv_electrical_neighbours_bus - + @@ -967,7 +794,7 @@ model_draft.ego_grid_hv_electrical_neighbours_bus - + @@ -977,7 +804,7 @@ model_draft.ego_grid_hv_electrical_neighbours_bus - + @@ -1006,7 +833,7 @@ model_draft.ego_grid_hv_electrical_neighbours_bus - + @@ -1022,7 +849,7 @@ model_draft.ego_grid_hv_electrical_neighbours_bus - + @@ -1032,32 +859,32 @@ model_draft.ego_grid_hv_electrical_neighbours_bus - + - - + + - + - - + + - + - - + + @@ -1066,23 +893,23 @@ model_draft.ego_grid_hv_electrical_neighbours_bus - - + + - + - - + + - + @@ -1092,27 +919,27 @@ model_draft.ego_grid_hv_electrical_neighbours_bus - + - - + + - + - - + + - + @@ -1122,22 +949,22 @@ model_draft.ego_grid_hv_electrical_neighbours_bus - + - - + + - + - - + + @@ -1146,8 +973,8 @@ model_draft.ego_grid_hv_electrical_neighbours_bus - - + + @@ -1156,18 +983,18 @@ model_draft.ego_grid_hv_electrical_neighbours_bus - - + + - + - - + + @@ -1176,53 +1003,53 @@ model_draft.ego_grid_hv_electrical_neighbours_bus - - + + - + - - - + + + - + - - - + + + - + - - + + - + - - + + - + @@ -1232,7 +1059,7 @@ model_draft.ego_grid_hv_electrical_neighbours_bus - + @@ -1242,10 +1069,10 @@ model_draft.ego_grid_hv_electrical_neighbours_bus - + - + @@ -1256,241 +1083,127 @@ model_draft.ego_grid_hv_electrical_neighbours_bus - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + - - + - - + + - - + - + - + - - + + - + - - + + - + - - + + - + - - + + - + - - + + - + - - + + - + - - + + - + - - + + - + - - + + From 8cf12305639e23d6220664de6c846a1e1641e239 Mon Sep 17 00:00:00 2001 From: IlkaCu Date: Thu, 6 Aug 2020 14:23:37 +0200 Subject: [PATCH 74/84] change color for external input data to darkred --- .../ego_dp-pre_bpmn_section_demand_federalstate.graphml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/documentation/bpmn/ego_dp-pre_bpmn_section_demand_federalstate.graphml b/documentation/bpmn/ego_dp-pre_bpmn_section_demand_federalstate.graphml index 3151c0e0..b48a5076 100644 --- a/documentation/bpmn/ego_dp-pre_bpmn_section_demand_federalstate.graphml +++ b/documentation/bpmn/ego_dp-pre_bpmn_section_demand_federalstate.graphml @@ -175,7 +175,7 @@ - + regional statistical offices @@ -211,7 +211,7 @@ - + VGR der Länder 2011 @@ -311,7 +311,6 @@ - @@ -322,7 +321,6 @@ - @@ -333,7 +331,6 @@ - From c3f181c5e7b0312a6485bde97c94515a6647a831 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guido=20Ple=C3=9Fmann?= Date: Fri, 21 Aug 2020 09:16:54 +0200 Subject: [PATCH 75/84] Add script that adds coast gid to power plant tables --- ...o_dp-pre_bpmn_section_cosmoclmgrid.graphml | 131 ++++++++++++++++-- 1 file changed, 116 insertions(+), 15 deletions(-) diff --git a/documentation/bpmn/ego_dp-pre_bpmn_section_cosmoclmgrid.graphml b/documentation/bpmn/ego_dp-pre_bpmn_section_cosmoclmgrid.graphml index e8f6a654..f7a293d1 100644 --- a/documentation/bpmn/ego_dp-pre_bpmn_section_cosmoclmgrid.graphml +++ b/documentation/bpmn/ego_dp-pre_bpmn_section_cosmoclmgrid.graphml @@ -17,11 +17,11 @@ - + - eGo Pre-Processing - cosmoclmgrid - + eGo Pre-Processing - cosmoclmgrid + @@ -53,7 +53,7 @@ - + @@ -64,7 +64,7 @@ - + @@ -75,7 +75,7 @@ - + import_cosmoclmgrid.py @@ -93,7 +93,7 @@ - + climate.cosmoclmgrid @@ -110,7 +110,7 @@ - + 7 s @@ -126,7 +126,7 @@ - + cosmoclmgrid.zip @@ -143,7 +143,7 @@ - + @@ -167,7 +167,7 @@ - + @@ -191,7 +191,7 @@ - + <no metadata available> @@ -204,6 +204,62 @@ + + + + + + + ego_dp_supply_add_coastdat_gid.sql + + + + + + + + + + + + + + + + + + model_draft.ego_dp_supply_conv_powerplant +model_draft.ego_dp_supply_res_powerplant +model_draft.ego_conv_powerplant_costdat_gid +model_draft.ego_res_powerplant_costdat_gid + + + + + + + + + + + + + + + + + model_draft.ego_dp_supply_conv_powerplant +model_draft.ego_dp_supply_res_powerplant + + + + + + + + + + @@ -254,12 +310,57 @@ - + + - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + From 33bbbe0b753801603b0842738e4f9b25c357c984 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guido=20Ple=C3=9Fmann?= Date: Fri, 21 Aug 2020 09:32:36 +0200 Subject: [PATCH 76/84] Add license of updated/created tables --- ...o_dp-pre_bpmn_section_cosmoclmgrid.graphml | 26 ++++++++++++++----- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/documentation/bpmn/ego_dp-pre_bpmn_section_cosmoclmgrid.graphml b/documentation/bpmn/ego_dp-pre_bpmn_section_cosmoclmgrid.graphml index f7a293d1..7f9ce1fc 100644 --- a/documentation/bpmn/ego_dp-pre_bpmn_section_cosmoclmgrid.graphml +++ b/documentation/bpmn/ego_dp-pre_bpmn_section_cosmoclmgrid.graphml @@ -234,10 +234,10 @@ model_draft.ego_conv_powerplant_costdat_gid model_draft.ego_res_powerplant_costdat_gid + - @@ -252,16 +252,33 @@ model_draft.ego_res_powerplant_costdat_gid + - + + + + + + + Open Database License (ODbL) v1.0 +© OpenStreetMap contributors + + + + + + + + + @@ -311,7 +328,6 @@ model_draft.ego_dp_supply_res_powerplant - @@ -322,7 +338,6 @@ model_draft.ego_dp_supply_res_powerplant - @@ -333,7 +348,6 @@ model_draft.ego_dp_supply_res_powerplant - @@ -344,7 +358,6 @@ model_draft.ego_dp_supply_res_powerplant - @@ -355,7 +368,6 @@ model_draft.ego_dp_supply_res_powerplant - From 6e38b3db434b3eb1b4a471e71a5940cc86939767 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guido=20Ple=C3=9Fmann?= Date: Fri, 21 Aug 2020 16:21:52 +0200 Subject: [PATCH 77/84] Add documentation for renpassG!S --- ...ego_dp-pre_bpmn_section_renpassGIS.graphml | 407 ++++++++++++++++++ 1 file changed, 407 insertions(+) create mode 100644 documentation/bpmn/ego_dp-pre_bpmn_section_renpassGIS.graphml diff --git a/documentation/bpmn/ego_dp-pre_bpmn_section_renpassGIS.graphml b/documentation/bpmn/ego_dp-pre_bpmn_section_renpassGIS.graphml new file mode 100644 index 00000000..2396d869 --- /dev/null +++ b/documentation/bpmn/ego_dp-pre_bpmn_section_renpassGIS.graphml @@ -0,0 +1,407 @@ + + + + + + + + + + + + + + + + + + + + + + + eGo Pre-Processing - renpassG!S + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + simple_feedin.py + + + + + + + + + + + + + + + + + + model_draft.ego_power_class + + + + + + + + + + + + + + + + + ? + + + + + + + + + + + + + + + + + model_draft.ego_supply_renewable_bnetza_full_attribute +model_draft.ego_dp_supply_res_powerplant + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + scenario_import.py + + + + + + + + + + + + + + + + + + calc_renpass_gis.renpass_gis_scenario +calc_renpass_gis.renpass_gis_linear_transformer +calc_renpass_gis.renpass_gis_source +calc_renpass_gis.renpass_gis_sink +calc_renpass_gis.renpass_gis_storage +calc_renpass_gis.renpass_gis_result + + + + + + + + + + + + + + + + + ? + + + + + + + + + + + + + + + + ? + + + + + + + + + + + + + + + + NODE_DATA.csv +SEQ_DATA.csv +RESULTS_DATA.csv + + + + + + + + + + + + + + + + + Open Database License (ODbL) v1.0 +© OpenStreetMap contributors + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + From 490680983a5aa56a15a115a9017bbb471fc8693b Mon Sep 17 00:00:00 2001 From: IlkaCu Date: Mon, 24 Aug 2020 11:15:56 +0200 Subject: [PATCH 78/84] add BPMN docu for osmthmod --- .../ego_dp-pre_bpmn_section_osmtgmod.graphml | 236 ++++++++++++++++++ 1 file changed, 236 insertions(+) create mode 100644 documentation/bpmn/ego_dp-pre_bpmn_section_osmtgmod.graphml diff --git a/documentation/bpmn/ego_dp-pre_bpmn_section_osmtgmod.graphml b/documentation/bpmn/ego_dp-pre_bpmn_section_osmtgmod.graphml new file mode 100644 index 00000000..c877738f --- /dev/null +++ b/documentation/bpmn/ego_dp-pre_bpmn_section_osmtgmod.graphml @@ -0,0 +1,236 @@ + + + + + + + + + + + + + + + + + + + + + + + eGo Pre-Processing - osmTGmod + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + osmTGmod - ego_otg.py + + + + + + + + + + + + + + + + + + grid.otg_ehvhv_branch_data +grid.otg_ehvhv_bus_data +grid.otg_ehvhv_dcline_data +grid-otg_ehvhv_results_metadata + + + + + + + + + + + + + + + + + OSM Import + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + From 68e5291d6218e59321cec5ac3547b12d7f7fd905 Mon Sep 17 00:00:00 2001 From: IlkaCu Date: Wed, 26 Aug 2020 11:43:30 +0200 Subject: [PATCH 79/84] replace dashed arrow by solid arrow --- .../bpmn/ego_dp-pre_bpmn_sections.graphml | 34 ++++--------------- 1 file changed, 7 insertions(+), 27 deletions(-) diff --git a/documentation/bpmn/ego_dp-pre_bpmn_sections.graphml b/documentation/bpmn/ego_dp-pre_bpmn_sections.graphml index a63c2d21..69f59134 100644 --- a/documentation/bpmn/ego_dp-pre_bpmn_sections.graphml +++ b/documentation/bpmn/ego_dp-pre_bpmn_sections.graphml @@ -31,22 +31,22 @@ + + + - - - - - + + @@ -138,7 +138,7 @@ - + @@ -351,7 +351,6 @@ - @@ -362,7 +361,6 @@ - @@ -373,7 +371,6 @@ - @@ -384,7 +381,6 @@ - @@ -395,7 +391,6 @@ - @@ -406,18 +401,16 @@ - - + - @@ -428,7 +421,6 @@ - @@ -439,7 +431,6 @@ - @@ -450,7 +441,6 @@ - @@ -461,7 +451,6 @@ - @@ -472,7 +461,6 @@ - @@ -483,7 +471,6 @@ - @@ -494,7 +481,6 @@ - @@ -505,7 +491,6 @@ - @@ -516,7 +501,6 @@ - @@ -527,7 +511,6 @@ - @@ -538,7 +521,6 @@ - @@ -549,7 +531,6 @@ - @@ -560,7 +541,6 @@ - From 1cab216a4a66c090a3fe63893e48424a438ade13 Mon Sep 17 00:00:00 2001 From: IlkaCu Date: Wed, 26 Aug 2020 13:05:40 +0200 Subject: [PATCH 80/84] Remove incorrect and redundant BPMN --- .../bpmn/ego_pp_bpmn_section_supply.graphml | 551 ------------------ 1 file changed, 551 deletions(-) delete mode 100644 documentation/bpmn/ego_pp_bpmn_section_supply.graphml diff --git a/documentation/bpmn/ego_pp_bpmn_section_supply.graphml b/documentation/bpmn/ego_pp_bpmn_section_supply.graphml deleted file mode 100644 index c1d0f127..00000000 --- a/documentation/bpmn/ego_pp_bpmn_section_supply.graphml +++ /dev/null @@ -1,551 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - eGo Preprocessing - simple feedin - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - model_draft.ego_renewable_feedin -model_draft.ego_power_class - - - - - - - - - - - - - - - - - renpass_gis/simple_feedin.py - - - - - - - - - - - - - - - - - - model_draft.ego_supply_res_powerplant -model_draft.ego_supply_renewable_bnetza_full_attribute -model_draft.bnetza_eeganlagenstammdaten_wind_classification - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - OPSD beta Version of 2015/16 - - - - - - - - - - - - - - - - - ego_dp_preprocessing_conv_powerplant.sql - - - - - - - - - - - - - - - - - - model_draft.ego_dp_supply_conv_powerplant -model_draft.ego_supply_conv_nep2035_temp - - - - - - - - - - - - - - - - - model_draft.ego_dp_supply_conv_powerplant - - - - - - - - - - - - - - - - - opsd_powerdata.sql - - - - - - - - - - - - - - - - - - supply.ego_conventional_powerplant - - - - - - - - - - - - - - - - - supply_ego_renewable_power_plants_germany.sql - - - - - - - - - - - - - - - - - - supply.ego_renewable_powerplant - - - - - - - - - - - - - - - - - energymap -BNetzA - - - - - - - - - - - - - - - - - ego_dp_preprocessing_res_powerplant.sql - - - - - - - - - - - - - - - - - - model_draft.ego_dp_supply_res_powerplant - - - - - - - - - - - - - - - - - - model_draft.ego_supply_res_powerplant - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - From f8ccb801a00f20d998f8c3f56aa1b478c6e214cc Mon Sep 17 00:00:00 2001 From: IlkaCu Date: Thu, 27 Aug 2020 09:26:43 +0200 Subject: [PATCH 81/84] remove ego_dp_supply_add_coastdat_gid.sql from BPMN --- ...o_dp-pre_bpmn_section_cosmoclmgrid.graphml | 122 +----------------- 1 file changed, 5 insertions(+), 117 deletions(-) diff --git a/documentation/bpmn/ego_dp-pre_bpmn_section_cosmoclmgrid.graphml b/documentation/bpmn/ego_dp-pre_bpmn_section_cosmoclmgrid.graphml index 7f9ce1fc..c086fb01 100644 --- a/documentation/bpmn/ego_dp-pre_bpmn_section_cosmoclmgrid.graphml +++ b/documentation/bpmn/ego_dp-pre_bpmn_section_cosmoclmgrid.graphml @@ -53,7 +53,7 @@ - + @@ -167,7 +167,7 @@ - + @@ -204,81 +204,8 @@ - - - - - - - ego_dp_supply_add_coastdat_gid.sql - - - - - - - - - - - - - - - - - - model_draft.ego_dp_supply_conv_powerplant -model_draft.ego_dp_supply_res_powerplant -model_draft.ego_conv_powerplant_costdat_gid -model_draft.ego_res_powerplant_costdat_gid - - - - - - - - - - - - - - - - - model_draft.ego_dp_supply_conv_powerplant -model_draft.ego_dp_supply_res_powerplant - - - - - - - - - - - - - - - - - Open Database License (ODbL) v1.0 -© OpenStreetMap contributors - - - - - - - - - @@ -327,52 +254,13 @@ model_draft.ego_dp_supply_res_powerplant - + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + From 8c5725f8b53ab4edd589d24be837a1a2e13d605c Mon Sep 17 00:00:00 2001 From: IlkaCu Date: Thu, 27 Aug 2020 09:38:25 +0200 Subject: [PATCH 82/84] move script ego_dp_supply_add_coastdat_gid.sql to utility folder --- .../{sql_snippets => utility}/ego_dp_supply_add_coastdat_gid.sql | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename preprocessing/{sql_snippets => utility}/ego_dp_supply_add_coastdat_gid.sql (100%) diff --git a/preprocessing/sql_snippets/ego_dp_supply_add_coastdat_gid.sql b/preprocessing/utility/ego_dp_supply_add_coastdat_gid.sql similarity index 100% rename from preprocessing/sql_snippets/ego_dp_supply_add_coastdat_gid.sql rename to preprocessing/utility/ego_dp_supply_add_coastdat_gid.sql From a1564648afca8b4f0e4729785e25f09ff5a380ce Mon Sep 17 00:00:00 2001 From: IlkaCu Date: Thu, 27 Aug 2020 10:55:23 +0200 Subject: [PATCH 83/84] add minor changes in bpmn for input data check --- ...o_dp_bpmn_section_input_data_check.graphml | 47 +++---------------- 1 file changed, 7 insertions(+), 40 deletions(-) diff --git a/documentation/bpmn/ego_dp_bpmn_section_input_data_check.graphml b/documentation/bpmn/ego_dp_bpmn_section_input_data_check.graphml index 341cbbca..42e7bf67 100644 --- a/documentation/bpmn/ego_dp_bpmn_section_input_data_check.graphml +++ b/documentation/bpmn/ego_dp_bpmn_section_input_data_check.graphml @@ -53,7 +53,7 @@ - + @@ -75,12 +75,10 @@ - + - - -ego_dp_substation_hvmv.sql + ego_dp_structure_input_verification.sql @@ -93,23 +91,6 @@ ego_dp_substation_hvmv.sql - - - - - - model_draft.ego_scenario_input - - - - - - - - - - - @@ -133,7 +114,7 @@ ego_dp_substation_hvmv.sql - + @@ -157,7 +138,7 @@ ego_dp_substation_hvmv.sql - + @@ -177,18 +158,6 @@ ego_dp_substation_hvmv.sql - - - - - - - - - - - - @@ -200,8 +169,7 @@ ego_dp_substation_hvmv.sql - - + @@ -211,8 +179,7 @@ ego_dp_substation_hvmv.sql - - + From 7e59b9467af2b9e2bcbc5c0d69d26588fec8f9a6 Mon Sep 17 00:00:00 2001 From: IlkaCu Date: Thu, 27 Aug 2020 10:58:42 +0200 Subject: [PATCH 84/84] add correct names for versioning scripts --- .../bpmn/ego_dp_bpmn_section_versioning.graphml | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/documentation/bpmn/ego_dp_bpmn_section_versioning.graphml b/documentation/bpmn/ego_dp_bpmn_section_versioning.graphml index e2d506b2..931fe27b 100644 --- a/documentation/bpmn/ego_dp_bpmn_section_versioning.graphml +++ b/documentation/bpmn/ego_dp_bpmn_section_versioning.graphml @@ -78,7 +78,7 @@ - ego_pp_nep2035_grid_variations.sql + ego_dp_versioning.sql @@ -254,7 +254,7 @@ model_draft.ego_grid_pf_hv_data_check - ego_pp_nep2035_grid_variations.sql + ego_dp_versioning_mviews.sql @@ -316,10 +316,10 @@ supply.ego_dp_res_powerplant DP griddistrict + - @@ -333,10 +333,10 @@ supply.ego_dp_res_powerplant DP powerflow + - @@ -350,10 +350,10 @@ supply.ego_dp_res_powerplant DP substation + - @@ -367,10 +367,10 @@ supply.ego_dp_res_powerplant DP loadarea + - @@ -458,7 +458,6 @@ supply.ego_dp_res_powerplant - @@ -469,7 +468,6 @@ supply.ego_dp_res_powerplant - @@ -480,7 +478,6 @@ supply.ego_dp_res_powerplant - @@ -491,7 +488,6 @@ supply.ego_dp_res_powerplant - @@ -502,7 +498,6 @@ supply.ego_dp_res_powerplant -