Merge pull request #143 from oslokommune/validate-download-links

Validate S3 download URLs

Author: simenheg

CHANGELOG.md

@@ -1,6 +1,7 @@
 ## ?.?.? - Unreleased
 
 * Removed support for file system credentials caching.
+* S3 download URLs are now validated.
 
 ## 4.0.0 - 2026-01-22
 

okdata/sdk/config.py

@@ -18,6 +18,7 @@
     "keycloakServerUrl": "https://login-test.oslo.kommune.no/auth",
     "pipelineUrl": "https://api.data-dev.oslo.systems/pipeline",
     "s3BucketUrl": "https://s3.eu-west-1.amazonaws.com/ok-origo-dataplatform-dev",
+    "s3DownloadBaseUrl": "https://ok-origo-dataplatform-dev.s3.amazonaws.com",
     "tokenService": "https://api.data-dev.oslo.systems/token-service/token",
     "uploadUrl": "https://api.data-dev.oslo.systems/data-uploader",
     "statusApiUrl": "https://api.data-dev.oslo.systems/status-api/status",
@@ -33,6 +34,7 @@
     "keycloakServerUrl": "https://login.oslo.kommune.no/auth",
     "pipelineUrl": "https://api.data.oslo.systems/pipeline",
     "s3BucketUrl": "https://s3.eu-west-1.amazonaws.com/ok-origo-dataplatform-prod",
+    "s3DownloadBaseUrl": "https://ok-origo-dataplatform-prod.s3.amazonaws.com",
     "tokenService": "https://api.data.oslo.systems/token-service/token",
     "uploadUrl": "https://api.data.oslo.systems/data-uploader",
     "statusApiUrl": "https://api.data.oslo.systems/status-api/status",

okdata/sdk/data/download.py

@@ -7,6 +7,14 @@
 log = logging.getLogger()
 
 
+class DownloadURLAssertionError(Exception):
+    def __init__(self, url, expected_prefix):
+        super().__init__(
+            f"Aborting download because of an unexpected S3 download URL "
+            f"(should start with {expected_prefix}): {url}"
+        )
+
+
 class Download(SDK):
     def __init__(self, config=None, auth=None, env=None):
         self.__name__ = "download"
@@ -27,7 +35,13 @@ def download(self, dataset_id, version, edition, output_path, retries=0):
         downloaded_files = []
         for file in self.get_files(dataset_id, version, edition, retries=retries):
             file_name = file["key"].split("/")[-1]
-            file_content_response = requests.get(file["url"], stream=True)
+            file_url = file["url"]
+            base_url = self.config.get("s3DownloadBaseUrl")
+
+            if not file_url.startswith(base_url):
+                raise DownloadURLAssertionError(file_url, base_url)
+
+            file_content_response = requests.get(file_url, stream=True)
             file_content_response.raise_for_status()
 
             write_file_content(file_name, output_path, file_content_response.raw.read())

okdata/sdk/sdk.py

@@ -9,7 +9,7 @@
 log = logging.getLogger()
 
 
-class SDK(object):
+class SDK:
     def __init__(self, config=None, auth=None, env=None):
         self.config = config
         if self.config is None:

tests/data/download_test.py

@@ -1,6 +1,7 @@
+import json
 import os
+
 import pytest
-import json
 
 from okdata.sdk.data.download import Download
 
@@ -17,6 +18,7 @@
 
 def test_download(mock_home_dir, mock_http_calls):
     data_downloader = Download()
+    data_downloader.config.config["s3DownloadBaseUrl"] = download_url
     output_path = f"{os.environ['HOME']}/my/path"
     result = data_downloader.download(
         dataset_id, version, edition, output_path=output_path
@@ -29,6 +31,7 @@ def test_download(mock_home_dir, mock_http_calls):
 
 def test_download_public(mock_home_dir, mock_http_calls_public):
     data_downloader = Download()
+    data_downloader.config.config["s3DownloadBaseUrl"] = download_url
     data_downloader.auth.token_provider = None
     output_path = f"{os.environ['HOME']}/my/path"
     result = data_downloader.download(