fetch remote fees if fees dont exist

Author: jalbrekt85

fee_allocator/utils.py

@@ -6,6 +6,7 @@
 from web3 import Web3
 import os
 from dotenv import load_dotenv
+import json
 
 
 
@@ -96,3 +97,24 @@ def get_block_by_ts(timestamp, chain: "CorePoolChain", before=False):
         return int(data["result"])
     else:
         return chain.subgraph.get_first_block_after_utc_timestamp(timestamp)
+
+def fetch_collected_fees(start_date: str, end_date: str, fees_file_name: str = None) -> dict:
+    # If fees_file_name is provided, use that directly
+    if fees_file_name:
+        filename = fees_file_name
+    else:
+        filename = f"fees_{start_date}_{end_date}.json"
+    
+    local_path = f"fee_allocator/fees_collected/{filename}"
+    if os.path.exists(local_path):
+        with open(local_path) as f:
+            return json.load(f)
+            
+    v1_fees = f"https://raw.githubusercontent.com/BalancerMaxis/protocol_fee_allocator/main/fee_allocator/fees_collected/{filename}"
+    response = requests.get(v1_fees)
+    
+    if response.status_code == 200:
+        print(f"fetched collected fees from: {v1_fees}")
+        return response.json()
+        
+    raise FileNotFoundError(f"Could not find fees file {filename} locally or remotely")

main.py

@@ -1,12 +1,12 @@
 import argparse
-import json
 import os
 from datetime import datetime
+import pytz
 
 from dotenv import load_dotenv
 
 from fee_allocator.fee_allocator import FeeAllocator
-from fee_allocator.utils import get_last_thursday_odd_week
+from fee_allocator.utils import get_last_thursday_odd_week, fetch_collected_fees
 
 
 parser = argparse.ArgumentParser()
@@ -29,17 +29,17 @@
 
 def main() -> None:
     load_dotenv()
-    ts_now = parser.parse_args().ts_now or TS_NOW
-    ts_in_the_past = parser.parse_args().ts_in_the_past or TS_2_WEEKS_AGO
+    args = parser.parse_args()
+    ts_now = args.ts_now or TS_NOW
+    ts_in_the_past = args.ts_in_the_past or TS_2_WEEKS_AGO
     print(
         f"\n\n\n------\nRunning  from timestamps {ts_in_the_past} to {ts_now}\n------\n\n\n"
     )
-    fees_file_name = parser.parse_args().fees_file_name or "current_fees_collected.json"
-    input_fees_path = f"fee_allocator/fees_collected/{fees_file_name}"
+    
+    start_date = datetime.fromtimestamp(ts_in_the_past, tz=pytz.UTC).strftime("%Y-%m-%d")
+    end_date = datetime.fromtimestamp(ts_now, tz=pytz.UTC).strftime("%Y-%m-%d")
 
-    with open(input_fees_path) as f:
-        input_fees = json.load(f)
-        
+    input_fees = fetch_collected_fees(start_date, end_date, args.fees_file_name)
     date_range = (ts_in_the_past, ts_now)
 
     fee_allocator = FeeAllocator(input_fees, date_range)