modified manifest and added run.sh for autorefresh (#1910)

* modified manifest and added run.sh for autorefresh

* Update statvar_imports/france_demographics/run.sh

Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>

* refactored for loop

---------

Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>

Author: balit-raibot

statvar_imports/france_demographics/README.md

@@ -8,22 +8,22 @@ This directory contains the scripts and configuration to import France demograph
 - **URL**: [INSEE Population Statistics](https://www.insee.fr/en/statistiques/8333211?sommaire=8333329)
 
 ## Directory Structure
-- `download_input_data.py`: Script to download source Excel files from INSEE.
+- `run.sh`: Primary entry point. Orchestrates downloading, renaming files, and processing.
 - `*_pvmap.csv`: Property-Value mapping files for the statvar processor.
 - `france_demographics_metadata.csv`: Metadata configuration for the import.
-- `manifest.json`: Import specification.
+- `manifest.json`: Import specification for the automation pipeline.
 
 ## Usage
 
-### 1. Download Data
-Run the download script to fetch the latest data files into `input_files/`.
+### 1. Automated Execution
+To ensure the pipeline works consistently across local environments and Google Cloud Batch, use the run.sh script. This script handles the downloading of INSEE files, renames them to match expected configuration, and runs the stat_var_processor.
 
-```bash
-python3 download_input_data.py
+```chmod +x run.sh
+./run.sh
 ```
 
-### 2. Process Data
-Run the `stat_var_processor.py` tool for each dataset to generate MCF and CSV files.
+### 2. Manual Processing
+If you already have the data files downloaded and renamed in input_files/, you can run the `stat_var_processor.py` tool for each dataset to generate MCF and CSV files.
 
 **Annual Population Components:**
 ```bash

statvar_imports/france_demographics/manifest.json

@@ -8,14 +8,7 @@
             "provenance_url": "https://www.insee.fr/en/statistiques/8333211?sommaire=8333329",
             "provenance_description": "Population data for France from INSEE",
             "scripts": [
-                "../../util/download_util_script.py --download_url=https://www.insee.fr/en/statistiques/fichier/8333211/Pop_annu_compo_evol_va.xlsx --output_folder=input_files/ && mv input_files/Pop_annu_compo_evol_va.xlsx input_files/annual_population_components.xlsx",
-                "../../util/download_util_script.py --download_url=https://www.insee.fr/en/statistiques/fichier/8333211/Pop1janv_age_va.xlsx --output_folder=input_files/ && mv input_files/Pop1janv_age_va.xlsx input_files/population_sex_detailed_age.xlsx",
-                "../../util/download_util_script.py --download_url=https://www.insee.fr/en/statistiques/fichier/8333211/Pop1janv_grages_va.xlsx --output_folder=input_files/ && mv input_files/Pop1janv_grages_va.xlsx input_files/population_sex_age_groups.xlsx",
-                "../../util/download_util_script.py --download_url=https://www.insee.fr/en/statistiques/fichier/8333211/Pop_age_moyen_median_va.xlsx --output_folder=input_files/ && mv input_files/Pop_age_moyen_median_va.xlsx input_files/average_median_age.xlsx",
-                "../../tools/statvar_importer/stat_var_processor.py --input_data=./input_files/annual_population_components.xlsx --pv_map=annual_population_components_pvmap.csv --config_file=france_demographics_metadata.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --output_path=output/annual_population_components_output",
-                "../../tools/statvar_importer/stat_var_processor.py --input_data=./input_files/population_sex_detailed_age.xlsx --pv_map=population_sex_detailed_age_pvmap.csv --config_file=france_demographics_metadata.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --output_path=output/population_sex_detailed_age_output",
-                "../../tools/statvar_importer/stat_var_processor.py --input_data=./input_files/population_sex_age_groups.xlsx --pv_map=population_sex_age_groups_pvmap.csv --config_file=france_demographics_metadata.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --output_path=output/population_sex_age_groups_output",
-                "../../tools/statvar_importer/stat_var_processor.py --input_data=./input_files/average_median_age.xlsx --pv_map=average_median_age_pvmap.csv --config_file=france_demographics_metadata.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --output_path=output/average_median_age_output"
+                "run.sh"
             ],
             "import_inputs": [
                 {

statvar_imports/france_demographics/run.sh

@@ -0,0 +1,68 @@
+#!/bin/bash
+
+# Exit immediately if a command exits with a non-zero status
+set -e
+
+echo "Starting France Demographics Import..."
+
+mkdir -p input_files
+mkdir -p output
+
+download_and_rename() {
+    local url=$1
+    local final_name=$2
+    local temp_name=$(basename "$url")
+    
+    echo "Downloading $temp_name..."
+    python3 ../../util/download_util_script.py --download_url="$url" --output_folder=input_files/
+    
+    echo "Renaming $temp_name to $final_name..."
+    mv "input_files/$temp_name" "input_files/$final_name"
+}
+
+BASE_URL="https://www.insee.fr/en/statistiques/fichier/8333211"
+declare -A files_to_download
+files_to_download["Pop_annu_compo_evol_va.xlsx"]="annual_population_components.xlsx"
+files_to_download["Pop1janv_age_va.xlsx"]="population_sex_detailed_age.xlsx"
+files_to_download["Pop1janv_grages_va.xlsx"]="population_sex_age_groups.xlsx"
+files_to_download["Pop_age_moyen_median_va.xlsx"]="average_median_age.xlsx"
+
+for source_file in "${!files_to_download[@]}"; do
+  url="${BASE_URL}/${source_file}"
+  dest_file="${files_to_download[$source_file]}"
+  download_and_rename "$url" "$dest_file"
+done
+
+echo "Verifying downloaded files:"
+ls -lh input_files/
+
+MCF_PATH="gs://unresolved_mcf/scripts/statvar/stat_vars.mcf"
+CONFIG="france_demographics_metadata.csv"
+STATVAR_PROCESSOR="../../tools/statvar_importer/stat_var_processor.py"
+
+run_processor() {
+    local input=$1
+    local pvmap=$2
+    local out=$3
+    
+    echo "Processing $input..."
+    python3 "$STATVAR_PROCESSOR" \
+        --input_data="input_files/$input" \
+        --pv_map="$pvmap" \
+        --config_file="$CONFIG" \
+        --existing_statvar_mcf="$MCF_PATH" \
+        --output_path="output/$out"
+}
+
+datasets=(
+    "annual_population_components"
+    "population_sex_detailed_age"
+    "population_sex_age_groups"
+    "average_median_age"
+)
+
+for dataset in "${datasets[@]}"; do
+    run_processor "${dataset}.xlsx" "${dataset}_pvmap.csv" "${dataset}_output"
+done
+
+echo "Import completed successfully."