SWE: l1a dependency updates to reduce reprocessing of science for HK bug (#1231)

Author: tech3371

sds_data_manager/lambda_code/SDSCode/pipeline_lambdas/dependency_config.csv

@@ -1130,9 +1130,9 @@ swapi, ancillary, energy-gf-sw-lut, swapi, l3b, combined, HARD, DOWNSTREAM
 
 # <---- SWE Dependencies ---->
 
-swe, l0, raw, swe, l1a, sci, HARD, DOWNSTREAM
-leapseconds, spice, historical, swe, l1a, sci, HARD_NO_TRIGGER, DOWNSTREAM
-spacecraft_clock, spice, historical, swe, l1a, sci, HARD_NO_TRIGGER, DOWNSTREAM
+swe, l0, raw, swe, l1a, all, HARD, DOWNSTREAM
+leapseconds, spice, historical, swe, l1a, all, HARD_NO_TRIGGER, DOWNSTREAM
+spacecraft_clock, spice, historical, swe, l1a, all, HARD_NO_TRIGGER, DOWNSTREAM
 
 swe, l1a, sci, swe, l1b, sci, HARD, DOWNSTREAM
 swe, ancillary, l1b-in-flight-cal, swe, l1b, sci, HARD, DOWNSTREAM

tests/lambda_endpoints/test_batch_starter.py

@@ -147,7 +147,7 @@ def test_lambda_handler(session, s3_client, mock_upload_request_success):
         lambda_handler(events, context)
         mock_batch_client.submit_job.assert_called_once()
         mock_batch_client.submit_job.assert_called_with(
-            jobName="swe-l1a-sci-job-1",
+            jobName="swe-l1a-all-job-1",
             jobQueue="ProcessingJobQueue",
             jobDefinition="ProcessingJob-swe",
             containerOverrides={
@@ -157,13 +157,13 @@ def test_lambda_handler(session, s3_client, mock_upload_request_success):
                     "--data-level",
                     "l1a",
                     "--descriptor",
-                    "sci",
+                    "all",
                     "--start-date",
                     "20240101",
                     "--version",
                     "v001",
                     "--dependency",
-                    "imap_swe_l1a_sci-c685cc19_20240101_v001.json",
+                    "imap_swe_l1a_all-c685cc19_20240101_v001.json",
                     "--upload-to-sdc",
                 ]
             },
@@ -184,7 +184,7 @@ def test_lambda_handler(session, s3_client, mock_upload_request_success):
         lambda_handler(events, context)
         mock_submit.assert_called_with(
             session,
-            {"data_source": "swe", "data_type": "l1a", "descriptor": "sci"},
+            {"data_source": "swe", "data_type": "l1a", "descriptor": "all"},
             "20240101",
             "v001",
             processing_input.serialize(),
@@ -807,7 +807,7 @@ def test_bulk_reprocessing_data_level(session, caplog, auth_event):
         "start_date": "20220101",
         "end_date": "20220301",
         "data_level": "l1a",
-        "descriptor": "sci",
+        "descriptor": "all",
     }
     # Create an authenticated event
     events = auth_event({"queryStringParameters": query_params})
@@ -1829,7 +1829,7 @@ def test_duplicate_job(session, first_status, second_status):
 def test_dependency_success():
     """Test the handler returns the expected dependency result."""
     dependencies = dependency.get_dependencies(
-        ("swe", "l1a", "sci"),
+        ("swe", "l1a", "all"),
         dependency_type="UPSTREAM",
         relationship="HARD",
     )
@@ -1887,7 +1887,7 @@ def test_dependency_success_empty(session):
     dependencies = dependency.get_jobs(
         data_source="swe",
         data_type="l1a",
-        descriptor="sci",
+        descriptor="all",
         dependency_type="UPSTREAM",
         relationship="HARD",
         start_date="20000101",

tests/lambda_endpoints/test_dependency_api.py

@@ -1648,14 +1648,13 @@ def test_calculate_crid(session):
     # The CRID associated with a file is made up of the filepath and the
     # Upstream file versions numbers packed into 2 bytes and sorted by the filename
     # imap_swe_l1b_sci_20240101_v001.cdf has a total of 4 upstream dependency files:
-    # - imap_swe_l1a_sci_20240101_v001.cdf.cdf
-    # - imap_swe_l0_raw_20240101_v001.pkts
-    # - imap_swe_l1a_sci_20240101_v010.cdf
-    # - imap_swe_esa-lut_20221231_v001.cdf
-    # - imap_swe_eu-conversion_20221231_v001.cdf
+    # - imap_swe_esa-lut_20221231_v001.cdf (v001)
+    # - imap_swe_eu-conversion_20221231_v001.cdf (v001)
+    # - imap_swe_l1a_sci_20240101_v010.cdf (v010)
+    # - imap_swe_l1b-in-flight-cal_20230102_v001.cdf (v001)
 
     # the upstream versions should be in order of the filenames alphabetically
-    upstream_versions = b"".join([v.to_bytes(2, "big") for v in [1, 1, 1, 10, 1]])
+    upstream_versions = b"".join([v.to_bytes(2, "big") for v in [1, 1, 10, 1]])
     expected_crid = base64.a85encode(record.file_path.encode() + upstream_versions)
     assert expected_crid.decode("ascii") == crid
 
@@ -1781,9 +1780,12 @@ def test_matching_crid(session):
     ]
     assert not matching_crids_exist(session, records)
     # Update the CRID of the science file to match the calculated CRID
-    records[
-        0
-    ].crid = "05t?ABJ4IG05593E*m[1ARB7.@UF1dBjWVL1,L[>0J[!Y0JG46@q90O!<<-#!<<H,!<"
+    # The CRID is calculated from the file path and upstream versions [1, 1, 10, 1]
+    upstream_versions = b"".join([v.to_bytes(2, "big") for v in [1, 1, 10, 1]])
+    expected_crid = base64.a85encode(
+        b"/path/to/imap_swe_l1b_sci_20240102_v001.cdf" + upstream_versions
+    ).decode("ascii")
+    records[0].crid = expected_crid
     assert matching_crids_exist(session, records)