Skip to content

Commit 5321743

Browse files
laspsandovallaspsandoval
authored
I-ALiRT - replace fill values (IMAP-Science-Operations-Center#2660)
1 parent 130519c commit 5321743

4 files changed

Lines changed: 45 additions & 58 deletions

File tree

‎imap_processing/ialirt/l0/process_codice.py‎

Lines changed: 6 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -34,8 +34,6 @@
3434

3535
logger = logging.getLogger(__name__)
3636

37-
FILLVAL_UINT8 = 255
38-
FILLVAL_FLOAT32 = Decimal(str(-1.0e31))
3937
COD_LO_COUNTER = 232
4038
COD_HI_COUNTER = 197
4139
COD_LO_RANGE = range(0, 15)
@@ -357,9 +355,9 @@ def calculate_ratios(
357355
fe_over_o_abundance = Decimal(f"{float(fe_over_o_abundance):.3f}")
358356
else:
359357
c_over_o_abundance, mg_over_o_abundance, fe_over_o_abundance = (
360-
FILLVAL_FLOAT32,
361-
FILLVAL_FLOAT32,
362-
FILLVAL_FLOAT32,
358+
None,
359+
None,
360+
None,
363361
)
364362

365363
if float(pseudo_density_dict["cplus5"]) != 0:
@@ -369,23 +367,23 @@ def calculate_ratios(
369367

370368
c_plus_6_over_c_plus_5 = Decimal(f"{float(c_plus_6_over_c_plus_5):.3f}")
371369
else:
372-
c_plus_6_over_c_plus_5 = FILLVAL_FLOAT32
370+
c_plus_6_over_c_plus_5 = None
373371

374372
if float(pseudo_density_dict["oplus6"]) != 0:
375373
o_plus_7_over_o_plus_6 = (
376374
pseudo_density_dict["oplus7"] / pseudo_density_dict["oplus6"]
377375
)
378376
o_plus_7_over_o_plus_6 = Decimal(f"{float(o_plus_7_over_o_plus_6):.3f}")
379377
else:
380-
o_plus_7_over_o_plus_6 = FILLVAL_FLOAT32
378+
o_plus_7_over_o_plus_6 = None
381379

382380
if float(pseudo_density_dict["fe_hiq"]) != 0:
383381
fe_low_over_fe_high = (
384382
pseudo_density_dict["fe_loq"] / pseudo_density_dict["fe_hiq"]
385383
)
386384
fe_low_over_fe_high = Decimal(f"{float(fe_low_over_fe_high):.3f}")
387385
else:
388-
fe_low_over_fe_high = FILLVAL_FLOAT32
386+
fe_low_over_fe_high = None
389387

390388
return COD_LO_L2(
391389
c_over_o_abundance=c_over_o_abundance,

‎imap_processing/ialirt/l0/process_swapi.py‎

Lines changed: 27 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,6 @@
2222
logger = logging.getLogger(__name__)
2323

2424
NUM_IALIRT_ENERGY_STEPS = 63
25-
FILLVAL_FLOAT32 = -1.0e31
2625

2726

2827
def count_rate(
@@ -142,7 +141,7 @@ def optimize_pseudo_parameters(
142141
# report speed only if fit fails
143142
if sol is None:
144143
sol = initial_param_guess.copy()
145-
sol[1:] = FILLVAL_FLOAT32
144+
sol[1:] = np.nan
146145

147146
return sol
148147

@@ -187,6 +186,10 @@ def geometric_mean(
187186
& ~np.isnan(pseudo_proton_temperature_list)
188187
)
189188

189+
if not np.any(valid):
190+
avg_swapi_met = np.mean(met_arr)
191+
return avg_swapi_met, np.nan, np.nan, np.nan
192+
190193
pseudo_speed_arr = np.asarray(pseudo_speed_list)[valid]
191194
avg_pseudo_speed = np.exp(np.mean(np.log(pseudo_speed_arr)))
192195

@@ -324,35 +327,35 @@ def process_swapi_ialirt(
324327
pseudo_proton_temperature_list[-5:],
325328
)
326329

327-
# replace nans (resulting from geometric means that
328-
# include fill values) with fill values
329-
(
330-
avg_pseudo_proton_speed,
331-
avg_pseudo_proton_density,
332-
avg_pseudo_proton_temperature,
333-
) = np.nan_to_num(
334-
(
335-
avg_pseudo_proton_speed,
336-
avg_pseudo_proton_density,
337-
avg_pseudo_proton_temperature,
338-
),
339-
nan=FILLVAL_FLOAT32,
330+
avg_pseudo_proton_speed = (
331+
Decimal(f"{avg_pseudo_proton_speed:.3f}")
332+
if avg_pseudo_proton_speed is not None
333+
and np.isfinite(avg_pseudo_proton_speed)
334+
else None
335+
)
336+
337+
avg_pseudo_proton_density = (
338+
Decimal(f"{avg_pseudo_proton_density:.3f}")
339+
if avg_pseudo_proton_density is not None
340+
and np.isfinite(avg_pseudo_proton_density)
341+
else None
342+
)
343+
344+
avg_pseudo_proton_temperature = (
345+
Decimal(f"{avg_pseudo_proton_temperature:.3f}")
346+
if avg_pseudo_proton_temperature is not None
347+
and np.isfinite(avg_pseudo_proton_temperature)
348+
else None
340349
)
341350

342351
swapi_data.append(
343352
_populate_instrument_header_items(met)
344353
| {
345354
"instrument": "swapi",
346355
"swapi_epoch": int(met_to_ttj2000ns(avg_swapi_met)),
347-
"swapi_pseudo_proton_speed": Decimal(
348-
f"{avg_pseudo_proton_speed:.3f}"
349-
),
350-
"swapi_pseudo_proton_density": Decimal(
351-
f"{avg_pseudo_proton_density:.3f}"
352-
),
353-
"swapi_pseudo_proton_temperature": Decimal(
354-
f"{avg_pseudo_proton_temperature:.3f}"
355-
),
356+
"swapi_pseudo_proton_speed": avg_pseudo_proton_speed,
357+
"swapi_pseudo_proton_density": avg_pseudo_proton_density,
358+
"swapi_pseudo_proton_temperature": avg_pseudo_proton_temperature,
356359
}
357360
)
358361
if incomplete_groups:

‎imap_processing/tests/ialirt/unit/test_process_codice.py‎

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -29,8 +29,6 @@
2929
from imap_processing.ialirt.l0.process_codice import (
3030
COD_HI_COUNTER,
3131
COD_LO_COUNTER,
32-
FILLVAL_FLOAT32,
33-
FILLVAL_UINT8,
3432
concatenate_bytes,
3533
convert_to_intensities,
3634
create_xarray_dataset,
@@ -434,7 +432,7 @@ def test_group_and_decompress_ialirt_cod_lo(
434432

435433
# Verify that we grouped the values properly.
436434
counter_values = cod_lo_test_dataset["cod_lo_counter"].data
437-
valid_values = counter_values[counter_values != FILLVAL_UINT8]
435+
valid_values = counter_values[counter_values != 255]
438436
resets = np.where(valid_values == COD_LO_COUNTER)
439437

440438
count = increment = 0
@@ -520,7 +518,7 @@ def test_group_and_decompress_ialirt_cod_hi(
520518

521519
# Verify that we grouped the values properly.
522520
counter_values = cod_hi_test_dataset["cod_hi_counter"].data
523-
valid_values = counter_values[counter_values != FILLVAL_UINT8]
521+
valid_values = counter_values[counter_values != 255]
524522
resets = np.where(valid_values == COD_HI_COUNTER)
525523

526524
count = increment = 0
@@ -774,7 +772,7 @@ def test_process_codice_lo(
774772
assert len(cod_lo_data) == 9
775773

776774
for product in l2_products:
777-
assert cod_lo_data[0][product] == FILLVAL_FLOAT32
775+
assert cod_lo_data[0][product] is None
778776

779777

780778
@pytest.mark.external_test_data

‎imap_processing/tests/ialirt/unit/test_process_swapi.py‎

Lines changed: 9 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,6 @@
77

88
from imap_processing import imap_module_directory
99
from imap_processing.ialirt.l0.process_swapi import (
10-
FILLVAL_FLOAT32,
1110
Consts,
1211
count_rate,
1312
geometric_mean,
@@ -166,20 +165,9 @@ def test_process_swapi_ialirt(
166165

167166
swapi_result = process_swapi_ialirt(xarray_data, esa_unit_conversion_table)
168167

169-
key_names = [
170-
"apid",
171-
"met",
172-
"met_in_utc",
173-
"ttj2000ns",
174-
"swapi_pseudo_proton_density",
175-
"swapi_pseudo_proton_speed",
176-
"swapi_pseudo_proton_temperature",
177-
]
178-
179-
for key in key_names:
180-
assert swapi_result[0][key] is not None, (
181-
f"The expected attribute {key} was not filled in the result dict."
182-
)
168+
assert swapi_result[0]["swapi_pseudo_proton_speed"] is None
169+
assert swapi_result[0]["swapi_pseudo_proton_density"] is None
170+
assert swapi_result[0]["swapi_pseudo_proton_temperature"] is None
183171

184172

185173
def test_count_rate():
@@ -300,8 +288,8 @@ def test_optimize_parameters_exception_handling():
300288
)
301289

302290
np.testing.assert_allclose(speed, expected_speed, rtol=1e-6)
303-
np.testing.assert_allclose(density, FILLVAL_FLOAT32)
304-
np.testing.assert_allclose(temperature, FILLVAL_FLOAT32)
291+
np.testing.assert_allclose(density, np.nan)
292+
np.testing.assert_allclose(temperature, np.nan)
305293

306294

307295
def test_optimize_parameters_bad_fit_handling():
@@ -337,8 +325,8 @@ def test_optimize_parameters_bad_fit_handling():
337325
)
338326

339327
np.testing.assert_allclose(speed, expected_speed, rtol=1e-6)
340-
np.testing.assert_allclose(density, FILLVAL_FLOAT32)
341-
np.testing.assert_allclose(temperature, FILLVAL_FLOAT32)
328+
np.testing.assert_allclose(density, np.nan)
329+
np.testing.assert_allclose(temperature, np.nan)
342330

343331

344332
def test_optimize_parameters_bad_covariance_handling():
@@ -371,8 +359,8 @@ def test_optimize_parameters_bad_covariance_handling():
371359
)
372360

373361
np.testing.assert_allclose(speed, expected_speed, rtol=1e-6)
374-
np.testing.assert_allclose(density, FILLVAL_FLOAT32)
375-
np.testing.assert_allclose(temperature, FILLVAL_FLOAT32)
362+
np.testing.assert_allclose(density, np.nan)
363+
np.testing.assert_allclose(temperature, np.nan)
376364

377365

378366
def test_geometric_mean():

0 commit comments

Comments
 (0)