log splitting to avoid limit hit for GCP logging agent (#1913)

balit-raibot · web-flow · commit fb5aa3ec43b9 · 2026-03-12T02:07:33.000Z
diff --git a/statvar_imports/france_demographics/run.sh b/statvar_imports/france_demographics/run.sh
@@ -51,7 +51,7 @@ run_processor() {
         --pv_map="$pvmap" \
         --config_file="$CONFIG" \
         --existing_statvar_mcf="$MCF_PATH" \
-        --output_path="output/$out"
+        --output_path="output/$out" 2>&1 | split -b 200k --filter='cat; echo -e "\n--- LOG CHUNK BOUNDARY ---"'
 }
 
 datasets=(

Original file line number	Diff line number	Diff line change
`@@ -51,7 +51,7 @@ run_processor() {`
`51`	`51`	`--pv_map="$pvmap" \`
`52`	`52`	`--config_file="$CONFIG" \`
`53`	`53`	`--existing_statvar_mcf="$MCF_PATH" \`
`54`		`- --output_path="output/$out"`
	`54`	`+ --output_path="output/$out" 2>&1 \| split -b 200k --filter='cat; echo -e "\n--- LOG CHUNK BOUNDARY ---"'`
`55`	`55`	`}`
`56`	`56`
`57`	`57`	`datasets=(`