Skip to content

Commit 4dc2f60

Browse files
lacoak21lacoak21
authored
IDEX l1b filename bug fix (#1190)
* skip daily file check for IDEX * remove todo * fix batch starter bug * pr comments
1 parent 2ff1f90 commit 4dc2f60

2 files changed

Lines changed: 154 additions & 42 deletions

File tree

‎sds_data_manager/lambda_code/SDSCode/pipeline_lambdas/batch_starter.py‎

Lines changed: 17 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -501,6 +501,21 @@ def submit_all_jobs(
501501
for filename in primary_science.filename_list:
502502
science_file = ScienceFilePath(filename)
503503
start_date, end_date = determine_date_range(session, science_file)
504+
if (
505+
job_node["data_source"] == "idex"
506+
and job_node["descriptor"] == "sci-1week"
507+
and job_node["data_type"] == "l1b"
508+
):
509+
# For idex l1b sci-1week jobs, we want to use a date range of 12 days ending
510+
# at the start date in the filename. Although the file is named as
511+
# 1week, it can actually contain over a week of data, so we want to
512+
# add a buffer to make sure we are getting all the spice coverage we need.
513+
query_start_date = (
514+
datetime.datetime.strptime(start_date, "%Y%m%d")
515+
- datetime.timedelta(days=12)
516+
).strftime("%Y%m%d")
517+
else:
518+
query_start_date = start_date
504519

505520
# Get the repointing number from the science file object
506521
job_repointing = science_file.repointing
@@ -516,7 +531,7 @@ def submit_all_jobs(
516531
descriptor=job_node["descriptor"],
517532
dependency_type="UPSTREAM",
518533
relationship="ALL",
519-
start_date=start_date,
534+
start_date=query_start_date,
520535
end_date=end_date,
521536
repoint=job_repointing,
522537
calculate_crids=False,
@@ -526,7 +541,7 @@ def submit_all_jobs(
526541
if not upstream_deps_for_job:
527542
logger.info(
528543
f"Skipping job submission for {job_node} with start_date: "
529-
f"{start_date} because of a missing upstream dependency."
544+
f"{query_start_date} because of a missing upstream dependency."
530545
)
531546
continue
532547
else:
@@ -698,20 +713,6 @@ def determine_date_range(session, file_obj):
698713
start_date, end_date = calculate_pointing_date_range(
699714
session, file_obj.repointing
700715
)
701-
elif (
702-
file_obj.instrument == "idex"
703-
and "sci-1week" in file_obj.descriptor
704-
and file_obj.data_level == "l1a"
705-
):
706-
# For idex l1b sci-1week jobs, we want to use a date range of 12 days ending
707-
# at the start date in the filename. Although the file is named as
708-
# 1week, it can actually contain over a week of data, so we want to
709-
# add a buffer to make sure we are getting all the spice coverage we need.
710-
end_date = file_obj.start_date
711-
start_date = (
712-
datetime.datetime.strptime(end_date, "%Y%m%d")
713-
- datetime.timedelta(days=12)
714-
).strftime("%Y%m%d")
715716
else:
716717
start_date = end_date = file_obj.start_date
717718
elif isinstance(file_obj, AncillaryFilePath):

‎tests/lambda_endpoints/test_batch_starter.py‎

Lines changed: 137 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@
3838
determine_date_range,
3939
determine_job_version,
4040
lambda_handler,
41+
submit_all_jobs,
4142
upload_dependency_file,
4243
)
4344

@@ -1312,6 +1313,142 @@ def test_def_cadence_map_event(
13121313
)
13131314

13141315

1316+
def test_idex_l1b(session, auth_event, mock_upload_request_success, caplog):
1317+
"""Tests ``submit_all_jobs`` for unique idex job with buffered query start date."""
1318+
_static_spice_files(session)
1319+
# Add idex l1a sci-1week file for 20251018 and spin file covering the date range.
1320+
session.add_all(
1321+
[
1322+
ScienceFiles(
1323+
file_path="/path/to/imap_idex_l1a_sci-1week_20251018_v001.cdf",
1324+
instrument="idex",
1325+
data_level="l1a",
1326+
descriptor="sci-1week",
1327+
start_date=datetime(2025, 10, 18),
1328+
version="v001",
1329+
extension="cdf",
1330+
ingestion_date=datetime.strptime(
1331+
"2024-01-25 23:35:26+00:00", "%Y-%m-%d %H:%M:%S%z"
1332+
),
1333+
),
1334+
# Spin file covering the buffered query window (12 days before 20251018)
1335+
SpinFiles(
1336+
file_path="imap_2025_291_2025_292_01.spin.csv",
1337+
start_date=datetime(2025, 10, 18),
1338+
end_date=datetime(2025, 10, 19),
1339+
version="01",
1340+
ingestion_date=datetime.strptime(
1341+
"2024-01-25 23:35:26+00:00", "%Y-%m-%d %H:%M:%S%z"
1342+
),
1343+
), # Spin file covering the rest of the window
1344+
SpinFiles(
1345+
file_path="imap_2025_285_2025_291_01.spin.csv",
1346+
start_date=datetime(2025, 10, 6),
1347+
end_date=datetime(2025, 10, 18),
1348+
version="01",
1349+
ingestion_date=datetime.strptime(
1350+
"2024-01-25 23:35:26+00:00", "%Y-%m-%d %H:%M:%S%z"
1351+
),
1352+
),
1353+
SPICEFiles(
1354+
file_path="path/to/imap_2000_055_2000_056_01.ah.bc",
1355+
file_name="imap_2000_055_2000_056_01.ah.bc",
1356+
ingestion_date=datetime.now(),
1357+
file_root="imap_2000_055_2000_056_.ah.bc",
1358+
kernel_type="attitude_history",
1359+
min_date_j2000=86400.1854936,
1360+
max_date_j2000=4575787269.1854936,
1361+
file_intervals_j2000=[[86400, 4575787269]],
1362+
min_date_datetime=datetime(2000, 1, 1),
1363+
max_date_datetime=datetime(2145, 1, 1),
1364+
file_intervals_datetime=[["0", "0"]],
1365+
min_date_sclk="",
1366+
max_date_sclk="",
1367+
file_intervals_sclk=[["0", "0"]],
1368+
sclk_kernel="imap_sclk_0001.tsc",
1369+
lsk_kernel="naif0012.tls",
1370+
version=1,
1371+
),
1372+
SPICEFiles(
1373+
file_path="path/to/imap_recon_20240101_20240101_v01.bsp",
1374+
file_name="imap_recon_20240101_20240101_v01.bsp",
1375+
ingestion_date=datetime.strptime(
1376+
"2024-01-25 23:35:26+00:00", "%Y-%m-%d %H:%M:%S%z"
1377+
),
1378+
file_root="imap_recon_20240101_20240101_.bsp",
1379+
kernel_type="ephemeris_reconstructed",
1380+
min_date_j2000=0,
1381+
max_date_j2000=4575787269.183866,
1382+
file_intervals_j2000=[[0, 4575787269.183866]],
1383+
min_date_datetime=datetime.strptime(
1384+
"2000-01-01 12:00:00+00:00", "%Y-%m-%d %H:%M:%S%z"
1385+
),
1386+
max_date_datetime=datetime.strptime(
1387+
"2145-01-01 00:00:00+00:00", "%Y-%m-%d %H:%M:%S%z"
1388+
),
1389+
file_intervals_datetime="[[2000-01-01T00:00:00, 2145-01-01T00:00:00]]",
1390+
min_date_sclk="1/0000000000:00000",
1391+
max_date_sclk="1/4285909749:39444",
1392+
file_intervals_sclk="[[1/00000000000:00000, 1/4285909749:39444]]",
1393+
sclk_kernel="/mnt/data/imap/spice/sclk/imap_sclk_0001.tsc",
1394+
lsk_kernel="/mnt/data/imap/spice/lsk/naif0012.tls",
1395+
version=1,
1396+
),
1397+
# Add ephemeris predicted file
1398+
SPICEFiles(
1399+
file_path="path/to/imap_pred_20240101_20240101_v01.bsp",
1400+
file_name="imap_pred_20240101_20240101_v01.bsp",
1401+
ingestion_date=datetime.strptime(
1402+
"2024-01-25 23:35:26+00:00", "%Y-%m-%d %H:%M:%S%z"
1403+
),
1404+
file_root="imap_pred_20240101_20240101_.bsp",
1405+
kernel_type="ephemeris_predicted",
1406+
min_date_j2000=0,
1407+
max_date_j2000=4575787269.183866,
1408+
file_intervals_j2000=[[0, 4575787269.183866]],
1409+
min_date_datetime=datetime.strptime(
1410+
"2000-01-01 12:00:00+00:00", "%Y-%m-%d %H:%M:%S%z"
1411+
),
1412+
max_date_datetime=datetime.strptime(
1413+
"2145-01-01 00:00:00+00:00", "%Y-%m-%d %H:%M:%S%z"
1414+
),
1415+
file_intervals_datetime="[[2000-01-01T00:00:00, 2145-01-01T00:00:00]]",
1416+
min_date_sclk="1/0000000000:00000",
1417+
max_date_sclk="1/4285909749:39444",
1418+
file_intervals_sclk="[[1/00000000000:00000, 1/4285909749:39444]]",
1419+
sclk_kernel="/mnt/data/imap/spice/sclk/imap_sclk_0001.tsc",
1420+
lsk_kernel="/mnt/data/imap/spice/lsk/naif0012.tls",
1421+
version=1,
1422+
),
1423+
]
1424+
)
1425+
1426+
session.commit()
1427+
job_node = {
1428+
"data_source": "idex",
1429+
"data_type": "l1b",
1430+
"descriptor": "sci-1week",
1431+
}
1432+
with (
1433+
patch.object(batch_starter, "try_to_submit_job") as mock_submit,
1434+
):
1435+
submit_all_jobs(
1436+
session,
1437+
job_node,
1438+
"20251018",
1439+
"20251018",
1440+
repoint=None,
1441+
calculate_crids=False,
1442+
)
1443+
1444+
# Verify that try_to_submit_job was called with the correct start date
1445+
assert mock_submit.call_count == 1
1446+
call_args = mock_submit.call_args
1447+
assert call_args[0][2] == "20251018", (
1448+
f"Expected start_date 20251018, got {call_args[0][2]}"
1449+
)
1450+
1451+
13151452
def test_idex_l2b(session, auth_event, mock_upload_request_success):
13161453
"""Tests ``lambda_handler` for unique idex l2b case."""
13171454
_static_spice_files(session)
@@ -2008,32 +2145,6 @@ def test_repoint_date_range(
20082145
)
20092146

20102147

2011-
def test_idex_l1a_date_range(session):
2012-
"""Test that the date range for IDEX l1a files is correct."""
2013-
_static_spice_files(session)
2014-
session.add(
2015-
ScienceFiles(
2016-
file_path="/path/to/imap_idex_l1a_sci-1week_20251020_v001.cdf",
2017-
instrument="idex",
2018-
data_level="l1a",
2019-
descriptor="sci-1week",
2020-
start_date=datetime(2025, 10, 20),
2021-
version="v001",
2022-
extension="cdf",
2023-
ingestion_date=datetime.strptime(
2024-
"2024-01-25 23:35:26+00:00", "%Y-%m-%d %H:%M:%S%z"
2025-
),
2026-
)
2027-
)
2028-
session.commit()
2029-
2030-
filename = "imap_idex_l1a_sci-1week_20251020_v001.cdf"
2031-
file_obj = imap_data_access.ScienceFilePath(filename)
2032-
date_range = determine_date_range(session, file_obj)
2033-
# Idex l1b jobs should have a date range of start date - 12 days - start date
2034-
assert date_range == ("20251008", "20251020")
2035-
2036-
20372148
def test_lambda_skip_processing_due_to_crid_check(session, caplog):
20382149
"""Test that processing stops when the calculated CRID is mismatched.
20392150

0 commit comments

Comments
 (0)