Add new Data for WHO Bacteriologically Confirmed TB

Author: abhishekjaisw

scripts/us_census/acs5yr/subject_tables/S1251/S1251_spec.json

@@ -0,0 +1,48 @@
+# WHO Tuberculosis Dataset: Bacteriologically Confirmed Pulmonary TB
+
+## Overview
+This dataset provides the number of people diagnosed with a new episode of pulmonary TB whose disease was bacteriologically confirmed, sourced directly from the World Health Organization (WHO). 
+
+## Data Source
+
+**Source URL:**
+https://data.who.int/indicators/i/1891124/5D51DB1
+
+The data comes from the official WHO reporting database and includes comprehensive, country-level health metrics detailing annual Tuberculosis notifications and case classifications.
+
+## How To Download Input Data
+To download the data, you'll need to run the provided download script `download_tb_data.py`. This script automatically queries the WHO API for the indicator, merges it with the WHO geographical master list to append standard `iso3` country codes, and saves the cleaned `TB_Bacteriologically_Confirmed.csv` file inside an "source_files" folder.
+
+type of place: Country.
+
+statvars: Health / Tuberculosis.
+
+years: Historical to present.
+
+## Processing Instructions
+To process the WHO Tuberculosis data and generate statistical variables, use the following commands from your root `data` directory:
+
+**Download input file**
+```bash
+python3 statvar_imports/TB_Bacteriologically_Confirmed/download_tb_data.py
+```
+**For Test Data Run**
+```bash
+python3 tools/statvar_importer/stat_var_processor.py \
+  --input_data=statvar_imports/TB_Bacteriologically_Confirmed/test_data/TB_Bacteriologically_Confirmed_input.csv \
+  --pv_map=statvar_imports/TB_Bacteriologically_Confirmed/pulmonary_tb_bctpb_pvmap.csv \
+  --output_path=statvar_imports/TB_Bacteriologically_Confirmed/test_data/TB_Bacteriologically_Confirmed_output \
+  --config_file=statvar_imports/TB_Bacteriologically_Confirmed/pulmonary_tb_bctpb_metadata.csv \
+  --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf
+```
+
+**For Main data run**
+```bash
+python3 tools/statvar_importer/stat_var_processor.py \
+  --input_data=statvar_imports/TB_Bacteriologically_Confirmed/source_files/TB_Bacteriologically_Confirmed_input.csv \
+  --pv_map=statvar_imports/TB_Bacteriologically_Confirmed/pulmonary_tb_bctpb_pvmap.csv \
+  --output_path=statvar_imports/TB_Bacteriologically_Confirmed/TB_Bacteriologically_Confirmed_output \
+  --config_file=statvar_imports/TB_Bacteriologically_Confirmed/pulmonary_tb_bctpb_metadata.csv \
+  --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf
+```
+

scripts/us_census/acs5yr/subject_tables/S1251/test/S1251_cleaned.csv

@@ -0,0 +1,58 @@
+import os
+import requests
+import io
+import pandas as pd
+
+def download_who_tb_data_with_iso3():
+    # 1. Get the Clean Data from the API
+    api_url = "https://xmart-api-public.who.int/DATA_/RELAY_TB_DATA"
+    params = {
+        "$filter": "IND_ID eq '18911245D51DB1'",
+        "$select": "IND_ID,INDICATOR_NAME,YEAR,COUNTRY,VALUE",
+        "$format": "csv"
+    }
+    
+    print("1. Fetching clean indicator data from WHO API...")
+    api_response = requests.get(api_url, params=params)
+    
+    if api_response.status_code != 200:
+        print(f"Failed to fetch API data. HTTP {api_response.status_code}")
+        return
+        
+    # Load the clean API data into a pandas table
+    api_df = pd.read_csv(io.StringIO(api_response.text))
+    
+    # 2. Get ONLY the iso3 code from the master database
+    print("2. Fetching country iso3 codes from WHO master database...")
+    master_url = "https://extranet.who.int/tme/generateCSV.asp?ds=notifications"
+    
+    # We only pull the 'country' (for matching) and 'iso3' columns
+    geo_columns = ['country', 'iso3']
+    master_df = pd.read_csv(master_url, usecols=geo_columns).drop_duplicates()
+    
+    # 3. Merge the two datasets together based on the country name
+    print("3. Merging data and formatting...")
+    # The API uses uppercase 'COUNTRY', the master uses lowercase 'country'
+    merged_df = pd.merge(api_df, master_df, left_on='COUNTRY', right_on='country', how='left')
+    
+    # Drop the duplicate lowercase 'country' column used for joining
+    merged_df = merged_df.drop(columns=['country'])
+    
+    # Reorder columns so the iso3 code sits right next to the Country name
+    final_columns = [
+        'IND_ID', 'INDICATOR_NAME', 'YEAR', 'COUNTRY', 'iso3', 'VALUE'
+    ]
+    merged_df = merged_df[final_columns]
+    
+    # 4. Save to CSV
+    output_dir = "statvar_imports/TB_Bacteriologically_Confirmed/source_files"
+    filename = os.path.join(output_dir, "TB_Bacteriologically_Confirmed_input.csv")
+    
+    os.makedirs(output_dir, exist_ok=True)
+    
+    # Save without the pandas index column
+    merged_df.to_csv(filename, index=False)
+    print(f"Success! Data saved locally as '{filename}'")
+
+if __name__ == "__main__":
+    download_who_tb_data_with_iso3()

scripts/us_census/acs5yr/subject_tables/S1251/test/S1251_input.zip

@@ -0,0 +1,27 @@
+{
+    "import_specifications": [
+        {
+            "import_name": "TB_Bacteriologically_Confirmed",
+            "curator_emails": [
+                "support@datacommons.org"
+            ],
+            "provenance_url": "https://data.who.int/indicators/i/1891124/5D51DB1",
+            "provenance_description": "Number of people diagnosed with a new episode of pulmonary TB whose disease was bacteriologically confirmed",
+            "scripts": [
+                "download_tb_data.py",
+                "../../tools/statvar_importer/stat_var_processor.py --input_data=source_files/*.csv --pv_map=pulmonary_tb_bctpb_pvmap.csv --config_file=pulmonary_tb_bctpb_metadata.csv --output_path=TB_Bacteriologically_Confirmed_output"
+            ],
+            "source_files": [
+                "source_files/*.csv"
+            ],
+            "import_inputs": [
+                {
+                    "template_mcf": "TB_Bacteriologically_Confirmed_output.tmcf",
+                    "cleaned_csv": "TB_Bacteriologically_Confirmed_output.csv",
+                    "stat_var_mcf": "TB_Bacteriologically_Confirmed_output_stat_vars.mcf"
+                }
+            ],
+            "cron_schedule": "0 0 1 1,4,7,10 *"
+        }
+    ]
+}