diff --git a/CHANGELOG.md b/CHANGELOG.md index 3088011..7337ef9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,8 @@ and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ### Added * Support for downloading Sentinel-1 SLC granules from the Copernicus Data Space Ecosystem (CDSE) as an alternative to ASF. Metadata retrieval and bounding box checks continue to use ASF. Download from CDSE uses the compressed `.zip` archive directly, leveraging ISCE2's virtual file access to reduce download traffic. +* Sentinel-1D support: updated AUX_CAL downloads, DAAC ingest schema, and SLC localization to accept S1D data. AUX_CAL downloads for S1C/S1D gracefully skip with a warning if the files are not yet published. +* Added `S1D_MIN_DATE` (2026-04-17) placeholder to reject uncalibrated S1D acquisitions, mirroring the existing `S1C_MIN_DATE` check. Update once S1D calibration is officially confirmed. ## [1.0.2] diff --git a/Dockerfile b/Dockerfile index 5d293c1..f79a2e1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,7 +6,7 @@ ARG DEBIAN_FRONTEND=noninteractive ENV PYTHONDONTWRITEBYTECODE=true # Install libgl1-mesa-glx unzip vim -RUN apt-get update && apt-get install -y --no-install-recommends libgl1-mesa-glx unzip vim && \ +RUN apt-get update && apt-get install -y --no-install-recommends libgl1-mesa-glx unzip vim curl && \ apt-get clean && rm -rf /var/lib/apt/lists/* # run commands in a bash login shell @@ -33,6 +33,11 @@ COPY --chown=iscer:iscer . /home/ops/DockerizedTopsApp RUN mamba env create -f /home/ops/DockerizedTopsApp/environment.yml && \ conda clean -afy +# TODO: Remove this when ISCE2 releases S1D Fixes +RUN TARGET_DIR=$(conda run -n topsapp_env python -c "import isce; import os; print(os.path.dirname(isce.__file__))") && \ + curl -sSL -o "${TARGET_DIR}/components/isceobj/Sensor/TOPS/Sentinel1.py" \ + https://raw.githubusercontent.com/isce-framework/isce2/7bd57893ed9ae2d7ac0f7dd0e06625e337b0b993/components/isceobj/Sensor/TOPS/Sentinel1.py + # Ensure that environment is activated on startup RUN echo ". /opt/conda/etc/profile.d/conda.sh" >> ~/.profile && \ echo "conda activate topsapp_env" >> ~/.profile diff --git a/isce2_topsapp/iono_proc.py b/isce2_topsapp/iono_proc.py index 39c6723..24addf1 100644 --- a/isce2_topsapp/iono_proc.py +++ b/isce2_topsapp/iono_proc.py @@ -101,7 +101,7 @@ def iono_processing( ionosphere(topsapp, ionParam) else: # This mode is used for cross - # Sentinel-1A/B interferogram + # Sentinel-1A/B/C/D interferogram # runIon.ionSwathBySwath(topsapp, ionParam) ionSwathBySwath( topsapp, ionParam, use_bridging=True, conncomp_flag=conncomp_flag diff --git a/isce2_topsapp/localize_aux_cal.py b/isce2_topsapp/localize_aux_cal.py index 26a1d83..f50b551 100644 --- a/isce2_topsapp/localize_aux_cal.py +++ b/isce2_topsapp/localize_aux_cal.py @@ -8,6 +8,9 @@ S1A_AUX_URL = "https://d3g9emy65n853h.cloudfront.net/AUX_CAL/S1A_AUX_CAL_20241128.zip" S1B_AUX_URL = "https://d3g9emy65n853h.cloudfront.net/AUX_CAL/S1B_AUX_CAL_20241128.zip" +# SAR-MPC API for satellites not yet bundled on the ASF CloudFront CDN +SAR_MPC_API_URL = "https://sar-mpc.eu/api/v1/" + def _download_platform(url: str, aux_cal_dir: Path): """Download and remove nested structure of the aux cal files @@ -25,12 +28,80 @@ def _download_platform(url: str, aux_cal_dir: Path): zip_file.extract(zip_info, aux_cal_dir) +def _download_platform_from_sar_mpc(mission: str, aux_cal_dir: Path): + """Download all AUX_CAL files for a mission from the ESA SAR-MPC API. + + The ASF CloudFront bundles include all AUX_CAL versions (active + inactive) + because ISCE2 selects the correct calibration file based on acquisition date. + To be consistent, we download all entries from SAR-MPC, not just active ones. + + Each entry is a .SAFE.zip file. The zip contains files directly under + .SAFE/ (no additional nesting unlike the ASF bundles). + """ + results = [] + page_url = SAR_MPC_API_URL + params = { + "product_type": "AUX_CAL", + "sentinel1__mission": mission, + "mode": "extended", + "page_size": 100, + } + + # Paginate through all results + response = requests.get(page_url, params=params, timeout=120) + response.raise_for_status() + data = response.json() + results.extend(data.get("results", [])) + while data.get("next"): + response = requests.get(data["next"], timeout=120) + response.raise_for_status() + data = response.json() + results.extend(data.get("results", [])) + + if not results: + print(f"No AUX_CAL files found for {mission} on SAR-MPC - skipping") + return + + for entry in results: + download_url = entry["remote_url"] + product_name = entry["product_name"] + + # Skip if already extracted + safe_dir = aux_cal_dir / f"{product_name}.SAFE" + if safe_dir.exists(): + continue + + resp = requests.get(download_url, timeout=300) + resp.raise_for_status() + + content = BytesIO(resp.content) + with zipfile.ZipFile(content) as zip_file: + for zip_info in zip_file.infolist(): + if not zip_info.is_dir() and ".SAFE/" in zip_info.filename: + # ESA zips are flat: .SAFE/file - extract as-is + zip_file.extract(zip_info, aux_cal_dir) + + print(f"Downloaded {len(results)} AUX_CAL files for {mission} from SAR-MPC") + + def download_aux_cal(aux_cal_dir: Union[str, Path] = "aux_cal"): if not isinstance(aux_cal_dir, Path): aux_cal_dir = Path(aux_cal_dir) aux_cal_dir.mkdir(exist_ok=True, parents=True) + + # S1A and S1B: use ASF CloudFront bundles as baseline (fast, single download) for url in (S1A_AUX_URL, S1B_AUX_URL): _download_platform(url, aux_cal_dir) + # Supplement all missions from ESA SAR-MPC API. + # - S1A/S1B: picks up any entries newer than the ASF bundle date + # - S1C/S1D: primary source (no ASF bundle available yet) + # The skip-if-exists check avoids re-downloading what the ASF bundle provided. + for mission in ("S1A", "S1B", "S1C", "S1D"): + try: + _download_platform_from_sar_mpc(mission, aux_cal_dir) + except Exception as e: + print(f"Warning: could not download AUX_CAL for {mission}: {e}") + return {"aux_cal_dir": str(aux_cal_dir)} diff --git a/isce2_topsapp/localize_slc.py b/isce2_topsapp/localize_slc.py index 9621e8f..b40bb32 100644 --- a/isce2_topsapp/localize_slc.py +++ b/isce2_topsapp/localize_slc.py @@ -16,6 +16,9 @@ S1C_MIN_DATE = datetime.datetime( 2025, 5, 19, tzinfo=datetime.timezone.utc ) # https://sentinels.copernicus.eu/-/sentinel-1c-products-are-now-calibrated +S1D_MIN_DATE = datetime.datetime( + 2026, 4, 17, tzinfo=datetime.timezone.utc +) # placeholder - update when S1D calibration is confirmed def get_gunw_extent_from_frame_id(frame_id) -> Polygon: @@ -137,6 +140,29 @@ def check_if_s1c_has_valid_date(slc_ids: list, slc_properties: list) -> bool: return s1c_has_valid_date +def check_if_s1d_has_valid_date(slc_ids: list, slc_properties: list) -> bool: + assert len(slc_ids) == len(slc_properties) + s1d_filter_bool = [id.startswith("S1D") for id in slc_ids] + s1d_properties_filter = [ + prop for (k, prop) in enumerate(slc_properties) if s1d_filter_bool[k] + ] + # No s1d data + if not sum(s1d_filter_bool): + return True + s1d_ids = [id for (k, id) in enumerate(slc_ids) if s1d_filter_bool[k]] + s1d_dates = [parse(prop["startTime"]) for prop in s1d_properties_filter] + s1d_valid_data_filter = [date >= S1D_MIN_DATE for date in s1d_dates] + s1d_has_valid_date = all(s1d_valid_data_filter) + if not s1d_has_valid_date: + invalid_s1d_ids = [ + id for (k, id) in enumerate(s1d_ids) if not s1d_valid_data_filter[k] + ] + print( + f"The following S1D acquisitions were before {S1D_MIN_DATE}: {invalid_s1d_ids}" + ) + return s1d_has_valid_date + + def check_track_numbers(slc_properties: list): path_numbers = [prop["pathNumber"] for prop in slc_properties] path_numbers = sorted(list(set(path_numbers))) @@ -211,6 +237,13 @@ def download_slcs( f"The Sentinel-1C acquisitions provided were before {S1C_MIN_DATE}" ) + if not check_if_s1d_has_valid_date( + reference_ids + secondary_ids, reference_props + secondary_props + ): + raise ValueError( + f"The Sentinel-1D acquisitions provided were before {S1D_MIN_DATE}" + ) + # Check the number of objects is the same as inputs assert len(reference_obs) == len(reference_ids) assert len(secondary_obs) == len(secondary_ids) @@ -323,7 +356,7 @@ def get_slcs_for_date_and_frame(date: datetime.date, frame_id: int) -> list[str] for product_date in _get_dates(product) ): raise ValueError( - f"No Sentinel-1A/1B/1C SLCs found for date {date} and frame id {frame_id}." + f"No Sentinel-1A/1B/1C/1D SLCs found for date {date} and frame id {frame_id}." ) return [result.properties["sceneName"] for result in results] diff --git a/isce2_topsapp/templates/daac_ingest_schema.json b/isce2_topsapp/templates/daac_ingest_schema.json index ef6ff4e..c8c3ee4 100644 --- a/isce2_topsapp/templates/daac_ingest_schema.json +++ b/isce2_topsapp/templates/daac_ingest_schema.json @@ -91,7 +91,7 @@ "minItems": 1, "items": { "type": "string", - "enum": ["Sentinel-1A", "Sentinel-1B"] + "enum": ["Sentinel-1A", "Sentinel-1B", "Sentinel-1C", "Sentinel-1D"] } }, "beam_mode": { diff --git a/tests/test_localize_slc.py b/tests/test_localize_slc.py index 2c03bab..b43cbf7 100644 --- a/tests/test_localize_slc.py +++ b/tests/test_localize_slc.py @@ -7,6 +7,7 @@ from isce2_topsapp.localize_slc import ( check_date_order, check_flight_direction, + check_if_s1d_has_valid_date, check_track_numbers, download_slcs, get_asf_slc_objects, @@ -216,7 +217,9 @@ def test_localize_slc_with_valid_pairs(reference_ids, secondary_ids, frame_id): def test_get_slcs_by_date_and_frame(): - with pytest.raises(ValueError, match=r"^No Sentinel-1A/1B/1C SLCs found for date "): + with pytest.raises( + ValueError, match=r"^No Sentinel-1A/1B/1C/1D SLCs found for date " + ): get_slcs_for_date_and_frame(date(2018, 2, 17), 16584) assert get_slcs_for_date_and_frame(date(2018, 2, 18), 16584) == [ @@ -260,7 +263,9 @@ def test_get_slcs_by_date_and_frame(): ] # scenes close to midnight but not crossing - with pytest.raises(ValueError, match=r"^No Sentinel-1A/1B/1C SLCs found for date "): + with pytest.raises( + ValueError, match=r"^No Sentinel-1A/1B/1C/1D SLCs found for date " + ): get_slcs_for_date_and_frame(date(2025, 1, 4), 25671) assert get_slcs_for_date_and_frame(date(2025, 1, 3), 25671) == [ "S1A_IW_SLC__1SDV_20250103T235910_20250103T235937_057287_070C52_1291", @@ -296,3 +301,26 @@ def test_s1c_min_date(): "S1C_IW_SLC__1SDV_20250611T235952_20250612T000019_002742_005A5F_F563", ] download_slcs(slc_ids_ref, slc_ids_sec, 18830, dry_run=True) + + +def test_s1d_min_date(): + # Test check_if_s1d_has_valid_date directly since no real S1D SLCs + # exist in ASF yet (download_slcs would fail at ASF search) + slc_ids = [ + "S1D_IW_SLC__1SDV_20260401T120000_20260401T120027_000050_000050_BBBB", + ] + slc_props = [{"startTime": "2026-04-01T12:00:00.000000Z"}] + assert not check_if_s1d_has_valid_date(slc_ids, slc_props) + + slc_ids = [ + "S1D_IW_SLC__1SDV_20260417T120000_20260417T120027_000100_000100_AAAA", + ] + slc_props = [{"startTime": "2026-04-17T12:00:00.000000Z"}] + assert check_if_s1d_has_valid_date(slc_ids, slc_props) + + # Non-S1D data should always pass + slc_ids = [ + "S1A_IW_SLC__1SDV_20200101T120000_20200101T120027_000001_000001_AAAA", + ] + slc_props = [{"startTime": "2020-01-01T12:00:00.000000Z"}] + assert check_if_s1d_has_valid_date(slc_ids, slc_props)