Skip to content

Commit dc738c8

Browse files
addition of keys in pvmap and download code changes (#1887)
* Code changes to fetch 2025 montly file * Code changes to fetch 2025 montly file * fixed bug to consider monthly files until yearly archive * dynamically fetch year * modifying pvmap to add changed keys --------- Co-authored-by: Tarun Bali <balit@google.com>
1 parent 455f28d commit dc738c8

2 files changed

Lines changed: 27 additions & 1 deletion

File tree

statvar_imports/us_bls/cpi_category/cpi_category_download.py

Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -404,7 +404,7 @@ def main():
404404
}
405405

406406
logging.info("\n--- Starting download process for yearly ZIP archives ---")
407-
years_to_download_zip = list(range(2010, 2024)) # Covers 2010 up to and including 2023
407+
years_to_download_zip = list(range(2010, current_year)) # Covers 2010 up to and including previous year
408408

409409
for year in years_to_download_zip:
410410
zip_url = generate_zip_url(year)
@@ -427,6 +427,22 @@ def main():
427427
logging.fatal(f"CRITICAL ERROR: Unzipping or processing of {filename} failed. Cannot proceed.")
428428
else:
429429
logging.info(f"Successfully processed ZIP for Year {year}.")
430+
else:
431+
logging.info(f"ZIP archive for {year} not found. Downloading all monthly files for cpi-u and cpi-w.")
432+
for month in range(1, 13):
433+
month_str = f"{month:02d}"
434+
for category_prefix in ["cpi-u", "cpi-w"]:
435+
url = f"https://www.bls.gov/cpi/tables/supplemental-files/{category_prefix}-{year}{month_str}.xlsx"
436+
filename = url.split('/')[-1]
437+
if category_prefix == "cpi-u":
438+
file_save_path = os.path.join(cpi_u_folder, filename)
439+
else:
440+
file_save_path = os.path.join(cpi_w_folder, filename)
441+
442+
try:
443+
download_file(url, file_save_path, timeout=args.timeout)
444+
except Exception as e:
445+
logging.warning(f"Failed to download monthly file {url} for year {year} (fallback): {e}")
430446

431447
logging.info("\n--- Listing files in respective folders after all downloads/extractions ---")
432448
logging.info("Files in 'cpi-u' folder:")

statvar_imports/us_bls/cpi_category/cpi_u_pvmap.csv

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -219,6 +219,7 @@ Candy and chewing gum(5),consumerGoodsCategory,CandyAndChewingGum,unit,IndexPoin
219219
Candy and chewing gum(5)(6),consumerGoodsCategory,CandyAndChewingGum,unit,IndexPointBasePeriodDecember1997Equals100,,,,,,,,
220220
Canned fruits and vegetables(5),consumerGoodsCategory,CannedFruitsAndVegetables,unit,IndexPointBasePeriodDecember1997Equals100,,,,,,,,
221221
Canned fruits and vegetables(6),consumerGoodsCategory,CannedFruitsAndVegetables,unit,IndexPointBasePeriodDecember1997Equals100,,,,,,,,
222+
Canned fruits(4)(5)(6),consumerGoodsCategory,CannedFruits,unit,IndexPointBasePeriodDecember1997Equals100,,,,,,,,
222223
Canned fruits(5)(6),consumerGoodsCategory,CannedFruits,unit,IndexPointBasePeriodDecember1997Equals100,,,,,,,,
223224
Canned fruits(6)(7),consumerGoodsCategory,CannedFruits,unit,IndexPointBasePeriodDecember1997Equals100,,,,,,,,
224225
Canned vegetables(5)(6),consumerGoodsCategory,CannedVegetables,unit,IndexPointBasePeriodDecember1997Equals100,,,,,,,,
@@ -284,6 +285,7 @@ Delivery services(4)(5),consumerGoodsCategory,DeliveryServices,unit,IndexPointBa
284285
Delivery services(5),consumerGoodsCategory,DeliveryServices,unit,IndexPointBasePeriodDecember1997Equals100,,,,,,,,
285286
Delivery services(6),consumerGoodsCategory,DeliveryServices,unit,IndexPointBasePeriodDecember1997Equals100,,,,,,,,
286287
Dental services,consumerGoodsCategory,DentalServices,,,,,,,,,,
288+
Dental services(4),consumerGoodsCategory,DentalServices,,,,,,,,,,
287289
Dental services(11),consumerGoodsCategory,DentalServices,,,,,,,,,,
288290
Dishes and flatware(4)(5),consumerGoodsCategory,DishesAndFlatware,unit,IndexPointBasePeriodDecember1997Equals100,,,,,,,,
289291
Dishes and flatware(5)(6),consumerGoodsCategory,DishesAndFlatware,unit,IndexPointBasePeriodDecember1997Equals100,,,,,,,,
@@ -621,6 +623,7 @@ Nonprescription drugs(11),consumerGoodsCategory,NonprescriptionDrugs,unit,IndexP
621623
Nonprescription drugs(4)(11),consumerGoodsCategory,NonprescriptionDrugs,unit,IndexPointBasePeriodDecember2009Equals100,,,,,,,,
622624
Nonprescription drugs(4)(12),consumerGoodsCategory,NonprescriptionDrugs,unit,IndexPointBasePeriodDecember2009Equals100,,,,,,,,
623625
Nonprescription drugs(5)(12),consumerGoodsCategory,NonprescriptionDrugs,unit,IndexPointBasePeriodDecember2009Equals100,,,,,,,,
626+
Nursing homes and adult day services(4)(17),consumerGoodsCategory,NursingHomesAndAdultDayServices,unit,IndexPointBasePeriodDecember1996Equals100,,,,,,,,
624627
Nursing homes and adult day services(11)(17),consumerGoodsCategory,NursingHomesAndAdultDayServices,unit,IndexPointBasePeriodDecember1996Equals100,,,,,,,,
625628
Nursing homes and adult day services(16),consumerGoodsCategory,NursingHomesAndAdultDayServices,unit,IndexPointBasePeriodDecember1996Equals100,,,,,,,,
626629
Nursing homes and adult day services(17),consumerGoodsCategory,NursingHomesAndAdultDayServices,unit,IndexPointBasePeriodDecember1996Equals100,,,,,,,,
@@ -647,6 +650,7 @@ Other fats and oils including peanut butter(6),consumerGoodsCategory,OtherFatsAn
647650
Other food at home,consumerGoodsCategory,OtherFoodAtHome,,,,,,,,,,
648651
Other food away from home(4)(5),consumerGoodsCategory,OtherFoodAwayFromHome,unit,IndexPointBasePeriodDecember1997Equals100,,,,,,,,
649652
Other food away from home(5)(6),consumerGoodsCategory,OtherFoodAwayFromHome,unit,IndexPointBasePeriodDecember1997Equals100,,,,,,,,
653+
Other food away from home(5),consumerGoodsCategory,OtherFoodAwayFromHome,unit,IndexPointBasePeriodDecember1997Equals100,,,,,,,,
650654
Other foods,consumerGoodsCategory,OtherFoods,,,,,,,,,,
651655
Other fresh fruits(5),consumerGoodsCategory,OtherFreshFruits,unit,IndexPointBasePeriodDecember1997Equals100,,,,,,,,
652656
Other fresh fruits(6),consumerGoodsCategory,OtherFreshFruits,unit,IndexPointBasePeriodDecember1997Equals100,,,,,,,,
@@ -678,6 +682,7 @@ Other personal services(4)(12),consumerGoodsCategory,OtherPersonalServices,unit,
678682
Other personal services(5)(12),consumerGoodsCategory,OtherPersonalServices,unit,IndexPointBasePeriodDecember2009Equals100,,,,,,,,
679683
Other pork including roasts and picnics(5),consumerGoodsCategory,OtherPorkIncludingRoastAndPicnic,unit,IndexPointBasePeriodDecember1997Equals100,,,,,,,,
680684
"Other pork including roasts, steaks, and ribs(5)",consumerGoodsCategory,OtherPorkIncludingRoastsSteaksAndRibs,unit,IndexPointBasePeriodDecember1997Equals100,,,,,,,,
685+
"Other pork including roasts, steaks, and ribs(4)(5)",consumerGoodsCategory,OtherPorkIncludingRoastsSteaksAndRibs,unit,IndexPointBasePeriodDecember1997Equals100,,,,,,,,
681686
"Other pork including roasts, steaks, and ribs(6)",consumerGoodsCategory,OtherPorkIncludingRoastsSteaksAndRibs,unit,IndexPointBasePeriodDecember1997Equals100,,,,,,,,
682687
Other poultry including turkey(5),consumerGoodsCategory,OtherPoultryIncludingTurkey,unit,IndexPointBasePeriodDecember1997Equals100,,,,,,,,
683688
Other processed fruits and vegetables including dried(5),consumerGoodsCategory,OtherProcessedFruitsAndVegetablesIncludingDried,unit,IndexPointBasePeriodDecember1997Equals100,,,,,,,,
@@ -687,6 +692,7 @@ Other recreation services(6),consumerGoodsCategory,OtherRecreationServices,unit,
687692
Other recreational goods(5),consumerGoodsCategory,OtherRecreationalGoods,unit,IndexPointBasePeriodDecember1997Equals100,,,,,,,,
688693
Other recreational goods(6),consumerGoodsCategory,OtherRecreationalGoods,unit,IndexPointBasePeriodDecember1997Equals100,,,,,,,,
689694
Other services,consumerGoodsCategory,OtherServices,,,,,,,,,,
695+
Other sweets(4)(5),consumerGoodsCategory,OtherSweets,unit,IndexPointBasePeriodDecember1997Equals100,,,,,,,,
690696
Other sweets(5),consumerGoodsCategory,OtherSweets,unit,IndexPointBasePeriodDecember1997Equals100,,,,,,,,
691697
Other sweets(6),consumerGoodsCategory,OtherSweets,unit,IndexPointBasePeriodDecember1997Equals100,,,,,,,,
692698
Other uncooked poultry including turkey(5),consumerGoodsCategory,OtherUncookedPoultryIncludingTurkey,unit,IndexPointBasePeriodDecember1997Equals100,,,,,,,,
@@ -717,6 +723,7 @@ Parking fees and tolls(6)(7),consumerGoodsCategory,ParkingFeesAndTolls,unit,Inde
717723
Peanut butter(4)(5)(6),consumerGoodsCategory,PeanutButter,unit,IndexPointBasePeriodDecember1997Equals100,,,,,,,,
718724
Peanut butter(5)(6)(7),consumerGoodsCategory,PeanutButter,unit,IndexPointBasePeriodDecember1997Equals100,,,,,,,,
719725
Personal care,consumerGoodsCategory,PersonalCare,,,,,,,,,,
726+
Personal care products,consumerGoodsCategory,PersonalCareProducts,,,,,,,,,,
720727
Personal care products(4),consumerGoodsCategory,PersonalCareProducts,,,,,,,,,,
721728
Personal care products(5),consumerGoodsCategory,PersonalCareProducts,,,,,,,,,,
722729
Personal care services(4),consumerGoodsCategory,PersonalCareServices,,,,,,,,,,
@@ -745,6 +752,7 @@ Photographers and film processing(4)(5),consumerGoodsCategory,PhotographerAndFil
745752
Photographers and photo processing(4)(5),consumerGoodsCategory,PhotographersAndPhotoProcessing,unit,IndexPointBasePeriodDecember1997Equals100,,,,,,,,
746753
Photographers and photo processing(5)(6),consumerGoodsCategory,PhotographersAndPhotoProcessing,unit,IndexPointBasePeriodDecember1997Equals100,,,,,,,,
747754
Photographic equipment and supplies,consumerGoodsCategory,PhotographicEquipmentAndSupplies,,,,,,,,,,
755+
Photographic equipment and supplies(4),consumerGoodsCategory,PhotographicEquipmentAndSupplies,,,,,,,,,,
748756
Photographic equipment(4)(5)(6),consumerGoodsCategory,PhotographicEquipment,unit,IndexPointBasePeriodDecember1997Equals100,,,,,,,,
749757
Photographic equipment(5)(6),consumerGoodsCategory,PhotographicEquipment,unit,IndexPointBasePeriodDecember1997Equals100,,,,,,,,
750758
Photographic equipment(6)(7),consumerGoodsCategory,PhotographicEquipment,unit,IndexPointBasePeriodDecember1997Equals100,,,,,,,,
@@ -778,6 +786,7 @@ Processed fish and seafood(6),consumerGoodsCategory,ProcessedFishAndSeafood,unit
778786
Processed fruits and vegetables(5),consumerGoodsCategory,ProcessedFruitsAndVegetables,unit,IndexPointBasePeriodDecember1997Equals100,,,,,,,,
779787
Processed fruits and vegetables(6),consumerGoodsCategory,ProcessedFruitsAndVegetables,unit,IndexPointBasePeriodDecember1997Equals100,,,,,,,,
780788
Professional services,consumerGoodsCategory,ProfessionalServices,,,,,,,,,,
789+
Professional services(4),consumerGoodsCategory,ProfessionalServices,,,,,,,,,,
781790
"Propane, kerosene, and firewood(10)",consumerGoodsCategory,PropaneKeroseneAndFirewood,unit,IndexPointBasePeriodDecember1986Equals100,,,,,,,,
782791
"Propane, kerosene, and firewood(4)(9)",consumerGoodsCategory,PropaneKeroseneAndFirewood,unit,IndexPointBasePeriodDecember1986Equals100,,,,,,,,
783792
"Propane, kerosene, and firewood(9)",consumerGoodsCategory,PropaneKeroseneAndFirewood,unit,IndexPointBasePeriodDecember1986Equals100,,,,,,,,
@@ -908,6 +917,7 @@ Uncooked beef roasts(5)(6),consumerGoodsCategory,UncookedBeefRoasts,unit,IndexPo
908917
Uncooked beef steaks(4)(5),consumerGoodsCategory,UncookedBeefSteaks,unit,IndexPointBasePeriodDecember1997Equals100,,,,,,,,
909918
Uncooked beef steaks(5),consumerGoodsCategory,UncookedBeefSteaks,unit,IndexPointBasePeriodDecember1997Equals100,,,,,,,,
910919
Uncooked beef steaks(6),consumerGoodsCategory,UncookedBeefSteaks,unit,IndexPointBasePeriodDecember1997Equals100,,,,,,,,
920+
Uncooked ground beef,consumerGoodsCategory,UncookedGroundBeef,,,,,,,,,,
911921
Uncooked ground beef(4),consumerGoodsCategory,UncookedGroundBeef,,,,,,,,,,
912922
Uncooked ground beef(5),consumerGoodsCategory,UncookedGroundBeef,,,,,,,,,,
913923
Uncooked other beef and veal(4)(5),consumerGoodsCategory,UncookedOtherBeefAndVeal,unit,IndexPointBasePeriodDecember1997Equals100,,,,,,,,

0 commit comments

Comments
 (0)