added downloader of geomagnetic data

Author: wegar-2

README.md

@@ -17,6 +17,8 @@ List of currently available datasets:
 3. `pl_banking_stocks` - daily prices of stocks of select 
 Polish banks for period 2005-01-01 through 2024-12-31 
 (data source: [stooq.com](https://stooq.com/))
+4. `sunspots` - daily total sunspot number data as per [SILSO](https://www.sidc.be/SILSO/datafiles)
+5. `geomagnetic_activity` - data on geomagnetic activity as per [GFZ Centre for Geosciences](https://kp.gfz.de/en/data) 
 
 
 To install this package run:

moddata/_utils.py

@@ -15,7 +15,7 @@
     "btc",
     "pl_banking_stocks",
     "sunspots",
-    "geomagnetic_indexes"
+    "geomagnetic_activity"
 ]
 
 
@@ -73,6 +73,13 @@ def _load_sunspots() -> pd.DataFrame:
     ))
 
 
+def _load_geomagnetic_activity() -> pd.DataFrame:
+    return pd.read_parquet(str(
+        resources.files('moddata.data').joinpath(
+            'geomagnetic_activity.parquet')
+    ))
+
+
 def load_data(dataset: Dataset) -> pd.DataFrame | None:
     if dataset == "bankchurn":
         return _load_bankchurn()
@@ -82,6 +89,6 @@ def load_data(dataset: Dataset) -> pd.DataFrame | None:
         return _load_pl_banking_stocks()
     if dataset == "sunspots":
         raise _load_sunspots()
-    if dataset == "geomagnetic_indexes":
-        raise Exception()
+    if dataset == "geomagnetic_activity":
+        raise _load_geomagnetic_activity()
     raise ValueError(f"Encountered invalid dataset name: {dataset}")

moddata/data/geomagnetic_activity.parquet

@@ -1,4 +1,3 @@
-import pandas as pd
 
 
 class DownloadCboeDataExtractor:

moddata/extractor/download_cboe_data_extractor.py

@@ -1,15 +1,44 @@
+import logging
 from typing import Final
 
+import numpy as np
 import pandas as pd
+import requests
+
+logger = logging.getLogger(__name__)
 
 
 class DownloadGeomagneticIndexExtractor:
 
-    _DATA_URL: Final[str] = "https://kp.gfz.de/app/files/Kp_ap_Ap_SN_F107_since_1932.txt"
+    _DATA_URL: Final[str] = \
+        "https://kp.gfz.de/app/files/Kp_ap_Ap_SN_F107_since_1932.txt"
+    _FIRST_DATA_LINE: Final[int] = 41
+    _COLUMNS_TO_TYPES: Final[dict[str, type]] = {
+        "year": int, "month": int, "day": int, "days": int, "days_m": float,
+        "Bsr": int, "dB": int,
+        "Kp1": float, "Kp2": float, "Kp3": float, "Kp4": float,
+        "Kp5": float, "Kp6": float, "Kp7": float, "Kp8": float,
+        "ap1": int, "ap2": int, "ap3": int, "ap4": int, "ap5": int,
+        "ap6": int, "ap7": int,  "ap8": int,
+        "Ap": int, "SN": int,
+        "F10.7obs": float, "F10.7adj": float, "D": int
+    }
 
     def extract(self) -> pd.DataFrame:
-        pass
-
-
-if __name__ == '__main__':
-    DownloadGeomagneticIndexExtractor()
+        data = requests.get(self._DATA_URL)
+        lines: list[str] = str(data.content).split("\\n")
+        data_lines: list[pd.DataFrame] = []
+        for i, line in enumerate(lines[self._FIRST_DATA_LINE:], start=1):
+            if i % 250 == 0:
+                print(f"processing line {i}")
+            cols = [col for col in line.split(" ") if col != ""]
+            if len(cols) == 28:
+                row = pd.DataFrame(
+                    data=np.array(cols).reshape(-1, 28),
+                    columns=list(self._COLUMNS_TO_TYPES.keys())
+                )
+                row = row.astype(self._COLUMNS_TO_TYPES) # noqa
+                data_lines.append(
+                    row # noqa
+                )
+        return pd.concat(data_lines, axis=0).reset_index(drop=True)

moddata/extractor/download_geomagnetic_index_extractor.py

@@ -9,7 +9,8 @@
 
 class DownloadSunspotsExtractor:
 
-    _DAILY_TOTAL_SUNSPOT_NUMBER_URL: Final[str] = "https://www.sidc.be/SILSO/INFO/sndtotcsv.php"
+    _DAILY_TOTAL_SUNSPOT_NUMBER_URL: Final[str] = \
+        "https://www.sidc.be/SILSO/INFO/sndtotcsv.php"
 
     def extract(self) -> pd.DataFrame:
         data = pd.read_csv(
@@ -28,7 +29,7 @@ def extract(self) -> pd.DataFrame:
         data["day"] = (
                 data["year"].astype(str) + "-" +
                 data["month"].apply(lambda x: f"{x:02}") + "-" +
-                data["day"].apply( lambda x: f"{x:02}")
+                data["day"].apply(lambda x: f"{x:02}")
         )
         data = data[["day", "daily_sunspots_number"]]
         data["daily_sunspots_number"] = np.where(