Skip to content

Commit 09d7b56

Browse files
LudeeLudee
committed
fix #151 input tables
1 parent d40948d commit 09d7b56

2 files changed

Lines changed: 123 additions & 52 deletions

File tree

‎dataprocessing/eGo_data_processing.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@ def data_processing():
4444
# 'ego_dp_vacuum_full.sql',
4545

4646
## INPUT DATA CHECK eGoPP
47-
# 'ego_dp_structure_input_verification.sql', # Check input tables and versions from eGoPP
47+
'ego_dp_structure_input_verification.sql', # Check input tables and versions from eGoPP
4848

4949
## SUBSTATION
5050
# 'ego_dp_substation_hvmv.sql', # Abstract HVMV Substations of the high voltage level from OSM

‎dataprocessing/sql_snippets/ego_dp_structure_input_verification.sql‎

Lines changed: 122 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -2,59 +2,130 @@
22
Check the necessary input tables.
33
Return version of input tables.
44
5-
__copyright__ = "Reiner Lemoine Institut"
6-
__license__ = "GNU Affero General Public License Version 3 (AGPL-3.0)"
7-
__url__ = "https://github.com/openego/data_processing/blob/master/LICENSE"
8-
__author__ = "Ludee"
5+
__copyright__ = "Reiner Lemoine Institut"
6+
__license__ = "GNU Affero General Public License Version 3 (AGPL-3.0)"
7+
__url__ = "https://github.com/openego/data_processing/blob/master/LICENSE"
8+
__author__ = "Ludee"
99
*/
1010

1111
-- ego scenario log (version,io,schema_name,table_name,script_name,comment)
12-
SELECT ego_scenario_log('v0.3.0','check','boundaries','bkg_vg250_1_sta','ego_dp_structure_versioning.sql','preprocessing verification');
13-
SELECT ego_scenario_log('v0.3.0','check','boundaries','bkg_vg250_2_lan','ego_dp_structure_versioning.sql','preprocessing verification');
14-
SELECT ego_scenario_log('v0.3.0','check','boundaries','bkg_vg250_4_krs','ego_dp_structure_versioning.sql','preprocessing verification');
15-
SELECT ego_scenario_log('v0.3.0','check','boundaries','bkg_vg250_6_gem','ego_dp_structure_versioning.sql','preprocessing verification');
16-
SELECT ego_scenario_log('v0.3.0','check','openstreetmap','osm_deu_polygon','ego_dp_structure_versioning.sql','preprocessing verification');
17-
SELECT ego_scenario_log('v0.3.0','check','openstreetmap','osm_deu_ways','ego_dp_structure_versioning.sql','preprocessing verification');
18-
SELECT ego_scenario_log('v0.3.0','check','openstreetmap','osm_deu_nodes','ego_dp_structure_versioning.sql','preprocessing verification');
19-
SELECT ego_scenario_log('v0.3.0','check','openstreetmap','osm_deu_line','ego_dp_structure_versioning.sql','preprocessing verification');
20-
SELECT ego_scenario_log('v0.3.0','check','society','destatis_zensus_population_per_ha','ego_dp_structure_versioning.sql','preprocessing verification');
21-
SELECT ego_scenario_log('v0.3.0','check','economy','destatis_gva_per_district','ego_dp_structure_versioning.sql','preprocessing verification');
22-
SELECT ego_scenario_log('v0.3.0','check','demand','ego_demand_federalstate','ego_dp_structure_versioning.sql','preprocessing verification');
23-
SELECT ego_scenario_log('v0.3.0','check','supply','vernetzen_wind_potential_area','ego_dp_structure_versioning.sql','preprocessing verification');
24-
SELECT ego_scenario_log('v0.3.0','check','model_draft','ego_supply_res_powerplant_sq_mview','ego_dp_structure_versioning.sql','preprocessing verification');
25-
SELECT ego_scenario_log('v0.3.0','check','model_draft','ego_supply_conv_powerplant_sq_mview','ego_dp_structure_versioning.sql','preprocessing verification');
12+
SELECT ego_scenario_log('v0.3.0','check','boundaries','bkg_vg250_1_sta','ego_dp_structure_input_verification.sql','preprocessing verification');
13+
SELECT ego_scenario_log('v0.3.0','check','boundaries','bkg_vg250_2_lan','ego_dp_structure_input_verification.sql','preprocessing verification');
14+
SELECT ego_scenario_log('v0.3.0','check','boundaries','bkg_vg250_4_krs','ego_dp_structure_input_verification.sql','preprocessing verification');
15+
SELECT ego_scenario_log('v0.3.0','check','boundaries','bkg_vg250_6_gem','ego_dp_structure_input_verification.sql','preprocessing verification');
16+
SELECT ego_scenario_log('v0.3.0','check','openstreetmap','osm_deu_polygon','ego_dp_structure_input_verification.sql','preprocessing verification');
17+
SELECT ego_scenario_log('v0.3.0','check','openstreetmap','osm_deu_ways','ego_dp_structure_input_verification.sql','preprocessing verification');
18+
SELECT ego_scenario_log('v0.3.0','check','openstreetmap','osm_deu_nodes','ego_dp_structure_input_verification.sql','preprocessing verification');
19+
SELECT ego_scenario_log('v0.3.0','check','openstreetmap','osm_deu_line','ego_dp_structure_input_verification.sql','preprocessing verification');
20+
SELECT ego_scenario_log('v0.3.0','check','society','destatis_zensus_population_per_ha','ego_dp_structure_input_verification.sql','preprocessing verification');
21+
SELECT ego_scenario_log('v0.3.0','check','economy','destatis_gva_per_district','ego_dp_structure_input_verification.sql','preprocessing verification');
22+
SELECT ego_scenario_log('v0.3.0','check','demand','ego_demand_federalstate','ego_dp_structure_input_verification.sql','preprocessing verification');
23+
SELECT ego_scenario_log('v0.3.0','check','supply','vernetzen_wind_potential_area','ego_dp_structure_input_verification.sql','preprocessing verification');
24+
SELECT ego_scenario_log('v0.3.0','check','model_draft','ego_supply_res_powerplant_sq_mview','ego_dp_structure_input_verification.sql','preprocessing verification');
25+
SELECT ego_scenario_log('v0.3.0','check','model_draft','ego_supply_conv_powerplant_sq_mview','ego_dp_structure_input_verification.sql','preprocessing verification');
2626

27+
/*
2728
-- input tables
28-
SELECT sub.oid,
29-
sub.database,
30-
sub.table_schema,
31-
sub.table_name,
32-
sub.path,
33-
sub.metadata::json ->>'title' AS metadata_title,
34-
sub.metadata::json #>>'{temporal,reference_date}' AS metadata_reference_date,
35-
sub.metadata
36-
FROM (SELECT st.relid AS oid,
37-
table_catalog AS database,
38-
i.table_schema AS table_schema,
39-
i.table_name AS table_name,
40-
i.table_schema ||'.'|| i.table_name AS path,
41-
pgd.description AS metadata
42-
FROM information_schema.tables AS i
43-
INNER JOIN pg_catalog.pg_statio_all_tables AS st ON (st.schemaname=i.table_schema and st.relname=i.table_name)
44-
INNER JOIN pg_catalog.pg_description AS pgd ON (pgd.objoid=st.relid)
45-
WHERE (table_schema='boundaries' AND table_name='bkg_vg250_1_sta') OR
46-
(table_schema='boundaries' AND table_name='bkg_vg250_2_lan') OR
47-
(table_schema='boundaries' AND table_name='bkg_vg250_4_krs') OR
48-
(table_schema='boundaries' AND table_name='bkg_vg250_6_gem') OR
49-
(table_schema='openstreetmap' AND table_name='osm_deu_polygon') OR
50-
(table_schema='openstreetmap' AND table_name='osm_deu_ways') OR
51-
(table_schema='openstreetmap' AND table_name='osm_deu_nodes') OR
52-
(table_schema='openstreetmap' AND table_name='osm_deu_line') OR
53-
(table_schema='society' AND table_name='destatis_zensus_population_per_ha') OR
54-
(table_schema='economy' AND table_name='destatis_gva_per_district') OR
55-
(table_schema='demand' AND table_name='ego_demand_federalstate') OR
56-
(table_schema='supply' AND table_name='vernetzen_wind_potential_area') OR
57-
(table_schema='model_draft' AND table_name='ego_supply_res_powerplant_sq_mview') OR
58-
(table_schema='model_draft' AND table_name='ego_supply_conv_powerplant_sq_mview')
59-
) AS sub
60-
ORDER BY table_schema, table_name;
29+
DROP TABLE IF EXISTS model_draft.ego_scenario_input CASCADE;
30+
CREATE TABLE model_draft.ego_scenario_input (
31+
id serial,
32+
version text,
33+
oid oid,
34+
database varchar,
35+
table_schema varchar,
36+
table_name varchar,
37+
path text,
38+
metadata_title text,
39+
metadata_reference_date text,
40+
metadata text,
41+
CONSTRAINT ego_scenario_input_pkey PRIMARY KEY (id));
42+
43+
-- grant (oeuser)
44+
ALTER TABLE model_draft.ego_scenario_input OWNER TO oeuser;
45+
46+
-- metadata template
47+
COMMENT ON TABLE model_draft.ego_scenario_input IS '{
48+
"title": "eGoDP input data verification",
49+
"description": "Checks input tables and gathers meta infos",
50+
"language": [ "eng" ],
51+
"spatial":
52+
{"location": "none",
53+
"extent": "none",
54+
"resolution": "none"},
55+
"temporal":
56+
{"reference_date": "none",
57+
"start": "none",
58+
"end": "none",
59+
"resolution": "none"},
60+
"sources": [
61+
{"name": "eGo dataprocessing", "description": "", "url": "https://github.com/openego/data_processing", "license": "GNU Affero General Public License Version 3 (AGPL-3.0)", "copyright": "© Reiner Lemoine Institut"},
62+
{"name": "", "description": "", "url": "", "license": "", "copyright": ""} ],
63+
"license":
64+
{"id": "ODbL-1.0",
65+
"name": "Open Data Commons Open Database License 1.0",
66+
"version": "1.0",
67+
"url": "https://opendatacommons.org/licenses/odbl/1.0/",
68+
"instruction": "You are free: To Share, To Create, To Adapt; As long as you: Attribute, Share-Alike, Keep open!",
69+
"copyright": "© Reiner Lemoine Institut"},
70+
"contributors": [
71+
{"name": "Ludee", "email": "", "date": "2017-10-26", "comment": "Create table and metadata" },
72+
{"name": "", "email": "", "date": "", "comment": ""} ],
73+
"resources": [
74+
{"name": "model_draft.ego_scenario_input",
75+
"format": "PostgreSQL",
76+
"fields": [
77+
{"name": "id", "description": "Unique identifier", "unit": "none"},
78+
{"name": "version", "description": "eGoDP version number", "unit": "none"},
79+
{"name": "oid", "description": "The OID of the object this description pertains to", "unit": "none"},
80+
{"name": "database", "description": "Database name", "unit": "none"},
81+
{"name": "table_schema", "description": "Schema name", "unit": "none"},
82+
{"name": "table_name", "description": "Table name", "unit": "none"},
83+
{"name": "path", "description": "Schema.Table", "unit": "none"},
84+
{"name": "metadata_title", "description": "Title from metadata", "unit": "none"},
85+
{"name": "metadata_reference_date", "description": "Reference date from metadata", "unit": "none"},
86+
{"name": "metadata", "description": "Full metadat", "unit": "none"} ] } ],
87+
"metadata_version": "1.3"}';
88+
89+
-- select description
90+
SELECT obj_description('model_draft.test_table' ::regclass) ::json;
91+
92+
-- ego scenario log (version,io,schema_name,table_name,script_name,comment)
93+
SELECT ego_scenario_log('v0.3.0','setup','model_draft','ego_scenario_input','ego_dp_structure_input_verification.sql','preprocessing verification');
94+
*/
95+
96+
-- insert version
97+
INSERT INTO model_draft.ego_scenario_input(version,oid,database,table_schema,table_name,path,metadata_title,metadata_reference_date,metadata)
98+
SELECT 'v0.3.0' AS version,
99+
sub.oid,
100+
sub.database,
101+
sub.table_schema,
102+
sub.table_name,
103+
sub.path,
104+
sub.metadata::json ->>'title' AS metadata_title,
105+
sub.metadata::json #>>'{temporal,reference_date}' AS metadata_reference_date,
106+
sub.metadata
107+
FROM (SELECT st.relid AS oid,
108+
table_catalog AS database,
109+
i.table_schema AS table_schema,
110+
i.table_name AS table_name,
111+
i.table_schema ||'.'|| i.table_name AS path,
112+
pgd.description AS metadata
113+
FROM information_schema.tables AS i
114+
INNER JOIN pg_catalog.pg_statio_all_tables AS st ON (st.schemaname=i.table_schema and st.relname=i.table_name)
115+
INNER JOIN pg_catalog.pg_description AS pgd ON (pgd.objoid=st.relid)
116+
WHERE (table_schema='boundaries' AND table_name='bkg_vg250_1_sta') OR
117+
(table_schema='boundaries' AND table_name='bkg_vg250_2_lan') OR
118+
(table_schema='boundaries' AND table_name='bkg_vg250_4_krs') OR
119+
(table_schema='boundaries' AND table_name='bkg_vg250_6_gem') OR
120+
(table_schema='openstreetmap' AND table_name='osm_deu_polygon') OR
121+
(table_schema='openstreetmap' AND table_name='osm_deu_ways') OR
122+
(table_schema='openstreetmap' AND table_name='osm_deu_nodes') OR
123+
(table_schema='openstreetmap' AND table_name='osm_deu_line') OR
124+
(table_schema='society' AND table_name='destatis_zensus_population_per_ha') OR
125+
(table_schema='economy' AND table_name='destatis_gva_per_district') OR
126+
(table_schema='demand' AND table_name='ego_demand_federalstate') OR
127+
(table_schema='supply' AND table_name='vernetzen_wind_potential_area') OR
128+
(table_schema='model_draft' AND table_name='ego_supply_res_powerplant_sq_mview') OR
129+
(table_schema='model_draft' AND table_name='ego_supply_conv_powerplant_sq_mview')
130+
) AS sub
131+
ORDER BY table_schema, table_name;

0 commit comments

Comments
 (0)