addition of keys in pvmap and download code changes (#1887)

* Code changes to fetch 2025 montly file

* Code changes to fetch 2025 montly file

* fixed bug to consider monthly files until yearly archive

* dynamically fetch year

* modifying pvmap to add changed keys

---------

Co-authored-by: Tarun Bali <balit@google.com>

Author: niveditasing

statvar_imports/us_bls/cpi_category/cpi_category_download.py

@@ -404,7 +404,7 @@ def main():
     }
 
     logging.info("\n--- Starting download process for yearly ZIP archives ---")
-    years_to_download_zip = list(range(2010, 2024)) # Covers 2010 up to and including 2023
+    years_to_download_zip = list(range(2010, current_year)) # Covers 2010 up to and including previous year
 
     for year in years_to_download_zip:
         zip_url = generate_zip_url(year)
@@ -427,6 +427,22 @@ def main():
                 logging.fatal(f"CRITICAL ERROR: Unzipping or processing of {filename} failed. Cannot proceed.")
             else:
                 logging.info(f"Successfully processed ZIP for Year {year}.")
+        else:
+            logging.info(f"ZIP archive for {year} not found. Downloading all monthly files for cpi-u and cpi-w.")
+            for month in range(1, 13):
+                month_str = f"{month:02d}"
+                for category_prefix in ["cpi-u", "cpi-w"]:
+                    url = f"https://www.bls.gov/cpi/tables/supplemental-files/{category_prefix}-{year}{month_str}.xlsx"
+                    filename = url.split('/')[-1]
+                    if category_prefix == "cpi-u":
+                        file_save_path = os.path.join(cpi_u_folder, filename)
+                    else:
+                        file_save_path = os.path.join(cpi_w_folder, filename)
+                    
+                    try:
+                        download_file(url, file_save_path, timeout=args.timeout)
+                    except Exception as e:
+                        logging.warning(f"Failed to download monthly file {url} for year {year} (fallback): {e}")
 
     logging.info("\n--- Listing files in respective folders after all downloads/extractions ---")
     logging.info("Files in 'cpi-u' folder:")

statvar_imports/us_bls/cpi_category/cpi_u_pvmap.csv

@@ -219,6 +219,7 @@ Candy and chewing gum(5),consumerGoodsCategory,CandyAndChewingGum,unit,IndexPoin
 Candy and chewing gum(5)(6),consumerGoodsCategory,CandyAndChewingGum,unit,IndexPointBasePeriodDecember1997Equals100,,,,,,,,
 Canned fruits and vegetables(5),consumerGoodsCategory,CannedFruitsAndVegetables,unit,IndexPointBasePeriodDecember1997Equals100,,,,,,,,
 Canned fruits and vegetables(6),consumerGoodsCategory,CannedFruitsAndVegetables,unit,IndexPointBasePeriodDecember1997Equals100,,,,,,,,
+Canned fruits(4)(5)(6),consumerGoodsCategory,CannedFruits,unit,IndexPointBasePeriodDecember1997Equals100,,,,,,,,
 Canned fruits(5)(6),consumerGoodsCategory,CannedFruits,unit,IndexPointBasePeriodDecember1997Equals100,,,,,,,,
 Canned fruits(6)(7),consumerGoodsCategory,CannedFruits,unit,IndexPointBasePeriodDecember1997Equals100,,,,,,,,
 Canned vegetables(5)(6),consumerGoodsCategory,CannedVegetables,unit,IndexPointBasePeriodDecember1997Equals100,,,,,,,,
@@ -284,6 +285,7 @@ Delivery services(4)(5),consumerGoodsCategory,DeliveryServices,unit,IndexPointBa
 Delivery services(5),consumerGoodsCategory,DeliveryServices,unit,IndexPointBasePeriodDecember1997Equals100,,,,,,,,
 Delivery services(6),consumerGoodsCategory,DeliveryServices,unit,IndexPointBasePeriodDecember1997Equals100,,,,,,,,
 Dental services,consumerGoodsCategory,DentalServices,,,,,,,,,,
+Dental services(4),consumerGoodsCategory,DentalServices,,,,,,,,,,
 Dental services(11),consumerGoodsCategory,DentalServices,,,,,,,,,,
 Dishes and flatware(4)(5),consumerGoodsCategory,DishesAndFlatware,unit,IndexPointBasePeriodDecember1997Equals100,,,,,,,,
 Dishes and flatware(5)(6),consumerGoodsCategory,DishesAndFlatware,unit,IndexPointBasePeriodDecember1997Equals100,,,,,,,,
@@ -621,6 +623,7 @@ Nonprescription drugs(11),consumerGoodsCategory,NonprescriptionDrugs,unit,IndexP
 Nonprescription drugs(4)(11),consumerGoodsCategory,NonprescriptionDrugs,unit,IndexPointBasePeriodDecember2009Equals100,,,,,,,,
 Nonprescription drugs(4)(12),consumerGoodsCategory,NonprescriptionDrugs,unit,IndexPointBasePeriodDecember2009Equals100,,,,,,,,
 Nonprescription drugs(5)(12),consumerGoodsCategory,NonprescriptionDrugs,unit,IndexPointBasePeriodDecember2009Equals100,,,,,,,,
+Nursing homes and adult day services(4)(17),consumerGoodsCategory,NursingHomesAndAdultDayServices,unit,IndexPointBasePeriodDecember1996Equals100,,,,,,,,
 Nursing homes and adult day services(11)(17),consumerGoodsCategory,NursingHomesAndAdultDayServices,unit,IndexPointBasePeriodDecember1996Equals100,,,,,,,,
 Nursing homes and adult day services(16),consumerGoodsCategory,NursingHomesAndAdultDayServices,unit,IndexPointBasePeriodDecember1996Equals100,,,,,,,,
 Nursing homes and adult day services(17),consumerGoodsCategory,NursingHomesAndAdultDayServices,unit,IndexPointBasePeriodDecember1996Equals100,,,,,,,,
@@ -647,6 +650,7 @@ Other fats and oils including peanut butter(6),consumerGoodsCategory,OtherFatsAn
 Other food at home,consumerGoodsCategory,OtherFoodAtHome,,,,,,,,,,
 Other food away from home(4)(5),consumerGoodsCategory,OtherFoodAwayFromHome,unit,IndexPointBasePeriodDecember1997Equals100,,,,,,,,
 Other food away from home(5)(6),consumerGoodsCategory,OtherFoodAwayFromHome,unit,IndexPointBasePeriodDecember1997Equals100,,,,,,,,
+Other food away from home(5),consumerGoodsCategory,OtherFoodAwayFromHome,unit,IndexPointBasePeriodDecember1997Equals100,,,,,,,,
 Other foods,consumerGoodsCategory,OtherFoods,,,,,,,,,,
 Other fresh fruits(5),consumerGoodsCategory,OtherFreshFruits,unit,IndexPointBasePeriodDecember1997Equals100,,,,,,,,
 Other fresh fruits(6),consumerGoodsCategory,OtherFreshFruits,unit,IndexPointBasePeriodDecember1997Equals100,,,,,,,,
@@ -678,6 +682,7 @@ Other personal services(4)(12),consumerGoodsCategory,OtherPersonalServices,unit,
 Other personal services(5)(12),consumerGoodsCategory,OtherPersonalServices,unit,IndexPointBasePeriodDecember2009Equals100,,,,,,,,
 Other pork including roasts and picnics(5),consumerGoodsCategory,OtherPorkIncludingRoastAndPicnic,unit,IndexPointBasePeriodDecember1997Equals100,,,,,,,,
 "Other pork including roasts, steaks, and ribs(5)",consumerGoodsCategory,OtherPorkIncludingRoastsSteaksAndRibs,unit,IndexPointBasePeriodDecember1997Equals100,,,,,,,,
+"Other pork including roasts, steaks, and ribs(4)(5)",consumerGoodsCategory,OtherPorkIncludingRoastsSteaksAndRibs,unit,IndexPointBasePeriodDecember1997Equals100,,,,,,,,
 "Other pork including roasts, steaks, and ribs(6)",consumerGoodsCategory,OtherPorkIncludingRoastsSteaksAndRibs,unit,IndexPointBasePeriodDecember1997Equals100,,,,,,,,
 Other poultry including turkey(5),consumerGoodsCategory,OtherPoultryIncludingTurkey,unit,IndexPointBasePeriodDecember1997Equals100,,,,,,,,
 Other processed fruits and vegetables including dried(5),consumerGoodsCategory,OtherProcessedFruitsAndVegetablesIncludingDried,unit,IndexPointBasePeriodDecember1997Equals100,,,,,,,,
@@ -687,6 +692,7 @@ Other recreation services(6),consumerGoodsCategory,OtherRecreationServices,unit,
 Other recreational goods(5),consumerGoodsCategory,OtherRecreationalGoods,unit,IndexPointBasePeriodDecember1997Equals100,,,,,,,,
 Other recreational goods(6),consumerGoodsCategory,OtherRecreationalGoods,unit,IndexPointBasePeriodDecember1997Equals100,,,,,,,,
 Other services,consumerGoodsCategory,OtherServices,,,,,,,,,,
+Other sweets(4)(5),consumerGoodsCategory,OtherSweets,unit,IndexPointBasePeriodDecember1997Equals100,,,,,,,,
 Other sweets(5),consumerGoodsCategory,OtherSweets,unit,IndexPointBasePeriodDecember1997Equals100,,,,,,,,
 Other sweets(6),consumerGoodsCategory,OtherSweets,unit,IndexPointBasePeriodDecember1997Equals100,,,,,,,,
 Other uncooked poultry including turkey(5),consumerGoodsCategory,OtherUncookedPoultryIncludingTurkey,unit,IndexPointBasePeriodDecember1997Equals100,,,,,,,,
@@ -717,6 +723,7 @@ Parking fees and tolls(6)(7),consumerGoodsCategory,ParkingFeesAndTolls,unit,Inde
 Peanut butter(4)(5)(6),consumerGoodsCategory,PeanutButter,unit,IndexPointBasePeriodDecember1997Equals100,,,,,,,,
 Peanut butter(5)(6)(7),consumerGoodsCategory,PeanutButter,unit,IndexPointBasePeriodDecember1997Equals100,,,,,,,,
 Personal care,consumerGoodsCategory,PersonalCare,,,,,,,,,,
+Personal care products,consumerGoodsCategory,PersonalCareProducts,,,,,,,,,,
 Personal care products(4),consumerGoodsCategory,PersonalCareProducts,,,,,,,,,,
 Personal care products(5),consumerGoodsCategory,PersonalCareProducts,,,,,,,,,,
 Personal care services(4),consumerGoodsCategory,PersonalCareServices,,,,,,,,,,
@@ -745,6 +752,7 @@ Photographers and film processing(4)(5),consumerGoodsCategory,PhotographerAndFil
 Photographers and photo processing(4)(5),consumerGoodsCategory,PhotographersAndPhotoProcessing,unit,IndexPointBasePeriodDecember1997Equals100,,,,,,,,
 Photographers and photo processing(5)(6),consumerGoodsCategory,PhotographersAndPhotoProcessing,unit,IndexPointBasePeriodDecember1997Equals100,,,,,,,,
 Photographic equipment and supplies,consumerGoodsCategory,PhotographicEquipmentAndSupplies,,,,,,,,,,
+Photographic equipment and supplies(4),consumerGoodsCategory,PhotographicEquipmentAndSupplies,,,,,,,,,,
 Photographic equipment(4)(5)(6),consumerGoodsCategory,PhotographicEquipment,unit,IndexPointBasePeriodDecember1997Equals100,,,,,,,,
 Photographic equipment(5)(6),consumerGoodsCategory,PhotographicEquipment,unit,IndexPointBasePeriodDecember1997Equals100,,,,,,,,
 Photographic equipment(6)(7),consumerGoodsCategory,PhotographicEquipment,unit,IndexPointBasePeriodDecember1997Equals100,,,,,,,,
@@ -778,6 +786,7 @@ Processed fish and seafood(6),consumerGoodsCategory,ProcessedFishAndSeafood,unit
 Processed fruits and vegetables(5),consumerGoodsCategory,ProcessedFruitsAndVegetables,unit,IndexPointBasePeriodDecember1997Equals100,,,,,,,,
 Processed fruits and vegetables(6),consumerGoodsCategory,ProcessedFruitsAndVegetables,unit,IndexPointBasePeriodDecember1997Equals100,,,,,,,,
 Professional services,consumerGoodsCategory,ProfessionalServices,,,,,,,,,,
+Professional services(4),consumerGoodsCategory,ProfessionalServices,,,,,,,,,,
 "Propane, kerosene, and firewood(10)",consumerGoodsCategory,PropaneKeroseneAndFirewood,unit,IndexPointBasePeriodDecember1986Equals100,,,,,,,,
 "Propane, kerosene, and firewood(4)(9)",consumerGoodsCategory,PropaneKeroseneAndFirewood,unit,IndexPointBasePeriodDecember1986Equals100,,,,,,,,
 "Propane, kerosene, and firewood(9)",consumerGoodsCategory,PropaneKeroseneAndFirewood,unit,IndexPointBasePeriodDecember1986Equals100,,,,,,,,
@@ -908,6 +917,7 @@ Uncooked beef roasts(5)(6),consumerGoodsCategory,UncookedBeefRoasts,unit,IndexPo
 Uncooked beef steaks(4)(5),consumerGoodsCategory,UncookedBeefSteaks,unit,IndexPointBasePeriodDecember1997Equals100,,,,,,,,
 Uncooked beef steaks(5),consumerGoodsCategory,UncookedBeefSteaks,unit,IndexPointBasePeriodDecember1997Equals100,,,,,,,,
 Uncooked beef steaks(6),consumerGoodsCategory,UncookedBeefSteaks,unit,IndexPointBasePeriodDecember1997Equals100,,,,,,,,
+Uncooked ground beef,consumerGoodsCategory,UncookedGroundBeef,,,,,,,,,,
 Uncooked ground beef(4),consumerGoodsCategory,UncookedGroundBeef,,,,,,,,,,
 Uncooked ground beef(5),consumerGoodsCategory,UncookedGroundBeef,,,,,,,,,,
 Uncooked other beef and veal(4)(5),consumerGoodsCategory,UncookedOtherBeefAndVeal,unit,IndexPointBasePeriodDecember1997Equals100,,,,,,,,