divide versioning skript

Author: IlkaCu

dataprocessing/eGo_data_processing.py

@@ -96,22 +96,23 @@ def data_processing():
 #	'ego_dp_powerflow_assignment_otgid.sql',			# assign otg_id to pp lists
 #	'ego_dp_powerflow_assignment_unid.sql',			# create a unified_id over all pp (res and conv) 
 #	'ego_dp_powerflow_create_pp_mview.sql',			# create mviews to display power plants per scenario
-	'ego_dp_powerflow_voronoi_weatherpoint.sql',		# Create voronoi cells based on weather points
-	'ego_dp_powerflow_hv_setup.sql',					# Set schema/tables for EHV/HV powerflow calculations up
-	'ego_dp_powerflow_osmtgmod_to_pypsa.sql',			# Include data from osmTGmod into EHV/HV powerflow schema
-	'ego_dp_powerflow_electrical_neighbour.sql',		# Create border crossing lines and buses in neighbouring countries
-	'ego_dp_powerflow_grid_future_scenarios.sql',		# Copy grid to future scenarios
-	'ego_dp_powerflow_assignment_generator.sql',  		# Assign generators to corresponding substation (SQ, NEP2035, eGo100)
-	'ego_dp_powerflow_assignment_load.sql',        		# Assign loads to their corresponding substation (SQ, NEP2035, eGo100)
-	'ego_dp_powerflow_assignment_storage.sql',		# Assign storages to their corresponding substation (SQ, NEP 2035, eGo 100)
-	'ego_dp_powerflow_timeseries_generator.sql',		# Transfer renpassG!S results into the corresponding powerflow table
-	'ego_dp_powerflow_griddistrict_demand.py',		# Demand per MV Griddistrict
-	'ego_dp_powerflow_timeseries_demand.sql',		# Insert demand series into corresponding powerflow table (SQ, NEP2035, eGo100)
-	'ego_dp_powerflow_lopf_data.sql',			# Set marginal costs for generators and storages
+#	'ego_dp_powerflow_voronoi_weatherpoint.sql',		# Create voronoi cells based on weather points
+#	'ego_dp_powerflow_hv_setup.sql',					# Set schema/tables for EHV/HV powerflow calculations up
+#	'ego_dp_powerflow_osmtgmod_to_pypsa.sql',			# Include data from osmTGmod into EHV/HV powerflow schema
+#	'ego_dp_powerflow_electrical_neighbour.sql',		# Create border crossing lines and buses in neighbouring countries
+#	'ego_dp_powerflow_grid_future_scenarios.sql',		# Copy grid to future scenarios
+#	'ego_dp_powerflow_assignment_generator.sql',  		# Assign generators to corresponding substation (SQ, NEP2035, eGo100)
+#	'ego_dp_powerflow_assignment_load.sql',        		# Assign loads to their corresponding substation (SQ, NEP2035, eGo100)
+#	'ego_dp_powerflow_assignment_storage.sql',		# Assign storages to their corresponding substation (SQ, NEP 2035, eGo 100)
+#	'ego_dp_powerflow_timeseries_generator.sql',		# Transfer renpassG!S results into the corresponding powerflow table
+#	'ego_dp_powerflow_griddistrict_demand.py',		# Demand per MV Griddistrict
+#	'ego_dp_powerflow_timeseries_demand.sql',		# Insert demand series into corresponding powerflow table (SQ, NEP2035, eGo100)
+#	'ego_dp_powerflow_lopf_data.sql',			# Set marginal costs for generators and storages
 
 
     ## VERSIONING
-#    	 'ego_dp_versioning.sql',						    # Versioning
+   	'ego_dp_versioning.sql',				# Versioning
+	'ego_dp_versioning_mviews.sql' ,			# Versioning of mviews
 
 	## VACUUM FULL
 #	 'ego_dp_vacuum_full.sql'

dataprocessing/sql_snippets/ego_dp_versioning.sql

@@ -524,240 +524,3 @@ UNION ALL
 	GROUP BY version
 	;
 
-
--- Create MView with full dataset per scenario and conventional power plants for schema supply 
-
--- MView for Status Quo
-DROP MATERIALIZED VIEW IF EXISTS  supply.ego_dp_conv_powerplant_sq_mview CASCADE;
-CREATE MATERIALIZED VIEW supply.ego_dp_conv_powerplant_sq_mview AS
-    SELECT *
-    FROM supply.ego_dp_conv_powerplant
-    WHERE scenario = 'Status Quo'
-    AND shutdown IS NULL or shutdown >= 2015
-    AND capacity > 0 
-    AND preversion = 'v0.3.0'
-    AND version = 'v0.3.0';
-
-ALTER MATERIALIZED VIEW supply.ego_dp_conv_powerplant_sq_mview
-    OWNER TO oeuser; 
-
--- MView for NEP 2035
-DROP MATERIALIZED VIEW IF EXISTS supply.ego_dp_conv_powerplant_nep2035_mview CASCADE;
-CREATE MATERIALIZED VIEW supply.ego_dp_conv_powerplant_nep2035_mview AS
-    SELECT *
-    FROM  supply.ego_dp_conv_powerplant
-    WHERE scenario = 'NEP 2035'
-    AND   capacity > 0 
-    AND   fuel not in ('hydro', 'run_of_river', 'reservoir')
-    AND   shutdown IS NULL or shutdown >= 2034
-    AND   preversion = 'v0.3.0'
-    AND version = 'v0.3.0';
-
-ALTER MATERIALIZED VIEW supply.ego_dp_conv_powerplant_nep2035_mview
-    OWNER TO oeuser;
-
--- MView for eGo 100
-DROP MATERIALIZED VIEW IF EXISTS  supply.ego_dp_conv_powerplant_ego100_mview CASCADE;
-CREATE MATERIALIZED VIEW supply.ego_dp_conv_powerplant_ego100_mview AS
-	SELECT 
-	  preversion,
-	  gid,
-	  bnetza_id,
-	  company,
-	  name,
-	  postcode,
-	  city,
-	  street,
-	  state,
-	  block,
-	  commissioned_original,
-	  commissioned,
-	  retrofit,
-	  shutdown,
-	  status,
-	  fuel,
-	  technology,
-	  type,
-	  eeg,
-	  chp,
-	  capacity,
-	  capacity_uba,
-	  chp_capacity_uba,
-	  efficiency_data,
-	  efficiency_estimate,
-	  network_node,
-	  voltage,
-	  network_operator,
-	  name_uba,
-	  lat,
-	  lon,
-	  'pumed storage for eGo 100'::text as comment,
-	  geom,
-	  voltage_level,
-	  subst_id,
-	  otg_id,
-	  un_id,
-	  la_id,
-	  'eGo 100'::text as scenario,
-	  'constantly'::text as flag,
-	  nuts
-	FROM supply.ego_dp_conv_powerplant
-	WHERE scenario in('NEP 2035')
-	AND fuel = 'pumped_storage'
-	AND capacity > 0
-	AND shutdown IS NULL or shutdown >= 2049
-	AND preversion = 'v0.3.0'
-	AND version = 'v0.3.0';
-
-ALTER MATERIALIZED VIEW supply.ego_dp_conv_powerplant_ego100_mview
-    OWNER TO oeuser;
-
--- ego scenario log (version,io,schema_name,table_name,script_name,comment)
-SELECT ego_scenario_log('v0.3.0','output','supply','ego_dp_conv_powerplant_sq_mview','ego_dp_powerflow_create_pp_mview.sql',' ');
-SELECT ego_scenario_log('v0.3.0','output','supply','ego_dp_conv_powerplant_nep2035_mview','ego_dp_powerflow_create_pp_mview.sql',' ');
-SELECT ego_scenario_log('v0.3.0','output','supply','ego_dp_conv_powerplant_ego100_mview','ego_dp_powerflow_create_pp_mview.sql',' ');
-
--- Create View with full dataset per scenario and renewables power plants
-
--- MView for Status Quo  
-DROP MATERIALIZED VIEW IF EXISTS supply.ego_dp_res_powerplant_sq_mview CASCADE;
-CREATE MATERIALIZED VIEW supply.ego_dp_res_powerplant_sq_mview AS
-    SELECT *
-    FROM supply.ego_dp_res_powerplant
-    WHERE scenario =  'Status Quo'
-    AND electrical_capacity > 0
-    AND preversion = 'v0.3.0'
-    AND version = 'v0.3.0';
-
-ALTER MATERIALIZED VIEW supply.ego_dp_res_powerplant_sq_mview
-    OWNER TO oeuser;
-
--- MView for NEP 2035
-DROP MATERIALIZED VIEW IF EXISTS supply.ego_dp_res_powerplant_nep2035_mview CASCADE;
-CREATE MATERIALIZED VIEW supply.ego_dp_res_powerplant_nep2035_mview AS
-	SELECT
-	sub.*
-	FROM  ( 
-		SELECT DISTINCT ON (id)
-		  *
-		FROM
-		  supply.ego_dp_res_powerplant
-			WHERE id not in (
-			SELECT id
-			FROM supply.ego_dp_res_powerplant
-			WHERE version = 'v0.3.0'
-			Group BY id
-			HAVING count(*) > 1
-			Order by id)
-		 AND scenario = 'Status Quo'
-		 And preversion = 'v0.3.0'
-		 AND electrical_capacity > 0
-		 ORDER BY id	
-		 ) as sub
-	UNION 
-	SELECT
-	sub2.*
-	FROM  ( 
-		SELECT  DISTINCT ON (id)
-		  *
-		FROM
-		  supply.ego_dp_res_powerplant
-		WHERE id in (
-			SELECT id
-			FROM supply.ego_dp_res_powerplant
-			WHERE scenario in ('NEP 2035')
-			AND version = 'v0.3.0'
-			Group BY id
-			Order by id)
-		 AND scenario in ('NEP 2035')
-		 And preversion = 'v0.3.0'
-		 AND version = 'v0.3.0'
-		 AND electrical_capacity > 0
-		 ORDER BY id	
-	) sub2
-	Order by id;
-
-ALTER MATERIALIZED VIEW supply.ego_dp_res_powerplant_nep2035_mview
-    OWNER TO oeuser;
-    
--- MView for eGo 100
-DROP MATERIALIZED VIEW IF EXISTS supply.ego_dp_res_powerplant_ego100_mview CASCADE;
-CREATE MATERIALIZED VIEW supply.ego_dp_res_powerplant_ego100_mview AS
-	SELECT DISTINCT ON (id)
-	sub.*
-	FROM  ( 
-		SELECT DISTINCT ON (id)
-		  *
-		FROM
-		  supply.ego_dp_res_powerplant
-			WHERE id not in (
-			SELECT id
-			FROM supply.ego_dp_res_powerplant
-			WHERE version = 'v0.3.0'
-			Group BY id
-			HAVING count(*) > 1
-			Order by id)
-		 AND scenario = 'Status Quo'
-		 AND preversion = 'v0.3.0'
-		 AND version = 'v0.3.0'
-		 AND electrical_capacity > 0
-		 AND generation_type in ('solar','wind')
-		 AND generation_subtype not in ('wind_offshore')
-		 ORDER BY id	
-		 ) as sub
-	UNION 
-	SELECT DISTINCT ON (id)
-	sub2.*
-	FROM  ( 
-		SELECT  DISTINCT ON (id)
-		  *
-		FROM
-		  supply.ego_dp_res_powerplant
-		WHERE id in (
-			SELECT id
-			FROM supply.ego_dp_res_powerplant
-			WHERE scenario in ('eGo 100')
-			AND generation_type not in ('gas')
-			AND version = 'v0.3.0'
-			AND flag in ('commissioning', 'repowering')
-			Group BY id
-			Order by id)
-		 AND scenario in ('eGo 100')
-		 And preversion = 'v0.3.0'
-		 AND version = 'v0.3.0'
-		 AND electrical_capacity > 0
-		 ORDER BY id	
-	) sub2
-        UNION 
-	SELECT DISTINCT ON (id)
-	sub3.*
-	FROM  ( 
-		SELECT  DISTINCT ON (id)
-		  *
-		FROM
-		  supply.ego_dp_res_powerplant
-		WHERE id in (
-			SELECT id
-			FROM supply.ego_dp_res_powerplant
-			WHERE scenario in ('NEP 2035')
-			AND version = 'v0.3.0'
-			AND generation_type not in ('biomass','gas','reservoir','run_of_river')
-			AND flag in ('commissioning', 'repowering')
-			AND generation_subtype not in ('wind_offshore')
-			Group BY id
-			Order by id)
-		 AND scenario in ('NEP 2035')
-		 And preversion = 'v0.3.0'
-		 AND version = 'v0.3.0'
-		 AND electrical_capacity > 0
-		 ORDER BY id	
-	) sub3
-	Order by id;
-
-ALTER MATERIALIZED VIEW supply.ego_dp_res_powerplant_ego100_mview
-    OWNER TO oeuser;
-
--- ego scenario log (version,io,schema_name,table_name,script_name,comment)
-SELECT ego_scenario_log('v0.3.0','output','supply','ego_dp_res_powerplant_sq_mview','ego_dp_powerflow_create_pp_mview.sql',' ');
-SELECT ego_scenario_log('v0.3.0','output','supply','ego_dp_res_powerplant_nep2035_mview','ego_dp_powerflow_create_pp_mview.sql',' ');
-SELECT ego_scenario_log('v0.3.0','output','supply','ego_dp_res_powerplant_ego100_mview','ego_dp_powerflow_create_pp_mview.sql',' ');