-
Notifications
You must be signed in to change notification settings - Fork 6
Expand file tree
/
Copy pathrun_local_with_upload_and_download.py
More file actions
262 lines (258 loc) · 20.9 KB
/
run_local_with_upload_and_download.py
File metadata and controls
262 lines (258 loc) · 20.9 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
import os
import shutil
import subprocess
import sys
from imap_l3_processing.glows.l3d.utils import PATH_TO_L3D_TOOLKIT
if __name__ == "__main__":
target = sys.argv[1]
subtarget = sys.argv[2] if len(sys.argv) > 2 else None
match target, subtarget:
case "hi", "combine-l2":
version = "v004"
subprocess.run([sys.executable,
'imap_l3_data_processor.py', '--instrument', 'hi', '--data-level', 'l3', '--start-date',
'20250415', '--version', version, '--descriptor', 'hic-ena-h-hf-nsp-full-hae-4deg-1yr',
'--dependency',
f"""[{{"type": "science", "files": ["imap_hi_l2_h45-ena-h-hf-nsp-anti-hae-4deg-1yr_20250415_{version}.cdf"]}},
{{"type": "science", "files": ["imap_hi_l2_h45-ena-h-hf-nsp-ram-hae-4deg-1yr_20250415_{version}.cdf"]}},
{{"type": "science", "files": ["imap_hi_l2_h90-ena-h-hf-nsp-anti-hae-4deg-1yr_20250415_{version}.cdf"]}},
{{"type": "science", "files": ["imap_hi_l2_h90-ena-h-hf-nsp-ram-hae-4deg-1yr_20250415_{version}.cdf"]}}]"""])
case "lo", "spx":
version = "v902"
subprocess.run([sys.executable,
'imap_l3_data_processor.py', '--instrument', 'lo', '--data-level', 'l3', '--start-date',
'20260101', '--version', version, '--descriptor', 'l090-spx-h-sf-sp-ram-hae-6deg-1yr',
'--dependency',
'[{"type": "science", "files": ["imap_lo_l3_l090-ena-h-sf-sp-ram-hae-6deg-1yr_20260101_v001.cdf"]}]'])
subprocess.run([sys.executable,
'imap_l3_data_processor.py', '--instrument', 'lo', '--data-level', 'l3', '--start-date',
'20260101', '--version', version, '--descriptor', 'l090-spx-h-sf-nsp-ram-hae-6deg-1yr',
'--dependency',
'[{"type": "science", "files": ["imap_lo_l2_l090-ena-h-sf-nsp-ram-hae-6deg-1yr_20260101_v902.cdf"]}]'])
subprocess.run([sys.executable,
'imap_l3_data_processor.py', '--instrument', 'lo', '--data-level', 'l3', '--start-date',
'20260101', '--version', version, '--descriptor', 'l090-spxnbs-h-sf-nsp-ram-hae-6deg-1yr',
'--dependency',
'[{"type": "science", "files": ["imap_lo_l2_l090-enanbs-h-sf-nsp-ram-hae-6deg-1yr_20260101_v900.cdf"]}]'])
case "hi", "spx":
subprocess.run([sys.executable, 'imap_l3_data_processor.py', '--instrument', 'hi', '--data-level', 'l3',
'--start-date', '20250415', '--version', 'v002', '--descriptor',
'h90-spx-h-hf-sp-full-hae-4deg-6mo', '--dependency',
'[{"type": "science", "files": ["imap_hi_l2_h90-ena-h-sf-nsp-ram-hae-4deg-6mo_20250415_v002.cdf"]}]'])
case "hi", "survival":
subprocess.run([sys.executable, 'imap_l3_data_processor.py', '--instrument', 'hi', '--data-level', 'l3',
'--start-date', '20250415', '--version', 'v002', '--descriptor',
'h90-ena-h-sf-sp-full-hae-4deg-6mo', '--dependency',
'[{"type": "science", "files": ["imap_hi_l2_h90-ena-h-sf-nsp-ram-hae-4deg-6mo_20250415_v001.cdf"]},\n{"type": "science", "files": ["imap_hi_l2_h90-ena-h-sf-nsp-anti-hae-4deg-6mo_20250415_v001.cdf"]}]'])
case "hit", "direct":
subprocess.run([sys.executable, 'imap_l3_data_processor.py', '--instrument', 'hit', '--data-level', 'l3',
'--start-date', '20100106', '--version', 'v001', '--descriptor',
'direct-events', '--dependency',
'[{"type": "science", "files": ["imap_hit_l1a_direct-events_20100106_v001.cdf"]},' +
'{"type": "ancillary", "files": ["imap_hit_range-2A-cosine-lookup_20250203_v000.csv"]},' +
'{"type": "ancillary", "files": ["imap_hit_range-2B-cosine-lookup_20250203_v000.csv"]},' +
'{"type": "ancillary", "files": ["imap_hit_range-3A-cosine-lookup_20250203_v000.csv"]},' +
'{"type": "ancillary", "files": ["imap_hit_range-3B-cosine-lookup_20250203_v000.csv"]},' +
'{"type": "ancillary", "files": ["imap_hit_range-4A-cosine-lookup_20250203_v000.csv"]},' +
'{"type": "ancillary", "files": ["imap_hit_range-4B-cosine-lookup_20250203_v000.csv"]},' +
'{"type": "ancillary", "files": ["imap_hit_hi-gain-lookup_20250203_v000.csv"]},' +
'{"type": "ancillary", "files": ["imap_hit_lo-gain-lookup_20250203_v000.csv"]},' +
'{"type": "ancillary", "files": ["imap_hit_hit-event-type-lookup_20250228_v000.csv"]},' +
'{"type": "ancillary", "files": ["imap_hit_range-2A-charge-fit-lookup_20250319_v000.csv"]},' +
'{"type": "ancillary", "files": ["imap_hit_range-3A-charge-fit-lookup_20250319_v000.csv"]},' +
'{"type": "ancillary", "files": ["imap_hit_range-4A-charge-fit-lookup_20250319_v000.csv"]},' +
'{"type": "ancillary", "files": ["imap_hit_range-2B-charge-fit-lookup_20250319_v000.csv"]},' +
'{"type": "ancillary", "files": ["imap_hit_range-3B-charge-fit-lookup_20250319_v000.csv"]},' +
'{"type": "ancillary", "files": ["imap_hit_range-4B-charge-fit-lookup_20250319_v000.csv"]}]'
])
case "hit", "macropixel":
subprocess.run([sys.executable, 'imap_l3_data_processor.py', '--instrument', 'hit', '--data-level', 'l3',
'--start-date', '20100106', '--version', 'v001', '--descriptor',
'macropixel', '--dependency',
'[{"type": "science", "files": ["imap_hit_l2_macropixel-intensity_20100106_v001.cdf"]},' +
'{"type": "science", "files": ["imap_mag_l1d_norm-dsrf_20100106_v001.cdf"]}]'
])
case "ultra", "survival":
subprocess.run([sys.executable, 'imap_l3_data_processor.py', '--instrument', 'ultra', '--data-level', 'l3',
'--start-date', '20250415', '--version', 'v010', '--descriptor',
'u90-ena-h-sf-sp-full-hae-4deg-3mo', '--dependency',
'[{"type": "science", "files": ["imap_ultra_l2_u90-ena-h-sf-nsp-full-hae-4deg-3mo_20250415_v010.cdf"]},' +
'{"type": "spice", "files": ["imap_sclk_0000.tsc"]},' +
'{"type": "spice", "files": ["naif0012.tls"]},' +
'{"type": "spice", "files": ["imap_science_0001.tf"]},' +
'{"type": "spice", "files": ["imap_dps_2025_105_2026_105_01.ah.bc"]}]'
])
case "ultra", "spx":
subprocess.run([sys.executable, 'imap_l3_data_processor.py', '--instrument', 'ultra', '--data-level', 'l3',
'--start-date', '20250415', '--version', 'v010', '--descriptor',
'u90-spx-h-sf-sp-full-hae-4deg-3mo', '--dependency',
'['
'{"type": "science", "files": ["imap_ultra_l3_u90-ena-h-sf-sp-full-hae-4deg-3mo_20250415_v010.cdf"]},'
'{"type": "ancillary", "files": ["imap_ultra_ulc-spx-energy-ranges_20250407_v000.dat"]}'
']'
])
case "glows", "l3b":
subprocess.run([sys.executable, 'imap_l3_data_processor.py', '--instrument', 'glows', '--data-level', 'l3b',
'--start-date', '20250415', '--version', 'v025', '--dependency',
'[{"type": "science", "files": ["imap_glows_l3a_hist_20250415-repoint01000_v026.cdf"]},'
'{"type": "ancillary", "files":["imap_glows_bad-days-list_20100101_v001.dat"]},'
'{"type": "ancillary", "files":["imap_glows_WawHelioIonMP_20100101_v002.json"]},'
'{"type": "ancillary", "files":["imap_glows_uv-anisotropy-1CR_20100101_v001.json"]},'
'{"type": "ancillary", "files":["imap_glows_pipeline-settings-l3bcde_19470303_v008.json"]},'
'{"type": "repoint", "files":["imap_2026_105_01.repoint.csv"]}]'])
case "glows", "l3d":
if os.path.exists(PATH_TO_L3D_TOOLKIT / 'data_l3b'): shutil.rmtree(PATH_TO_L3D_TOOLKIT / 'data_l3b')
if os.path.exists(PATH_TO_L3D_TOOLKIT / 'data_l3c'): shutil.rmtree(PATH_TO_L3D_TOOLKIT / 'data_l3c')
if os.path.exists(PATH_TO_L3D_TOOLKIT / 'data_l3d'): shutil.rmtree(PATH_TO_L3D_TOOLKIT / 'data_l3d')
if os.path.exists(PATH_TO_L3D_TOOLKIT / 'data_l3d_txt'): shutil.rmtree(PATH_TO_L3D_TOOLKIT / 'data_l3d_txt')
subprocess.run([sys.executable, 'imap_l3_data_processor.py', '--instrument', 'glows', '--data-level', 'l3d',
'--start-date', '20100101', '--version', 'v010', '--descriptor',
'solar-hist', '--dependency',
'[{"type": "science", "files": ["imap_glows_l3b_ion-rate-profile_20100422_v013.cdf"]},'
'{"type": "science", "files": ["imap_glows_l3c_sw-profile_20100422_v012.cdf"]},'
'{"type": "ancillary", "files": ["imap_glows_plasma-speed-2010a_20100101_v003.dat"]},'
'{"type": "ancillary", "files": ["imap_glows_proton-density-2010a_20100101_v003.dat"]},'
'{"type": "ancillary", "files": ["imap_glows_uv-anisotropy-2010a_20100101_v003.dat"]},'
'{"type": "ancillary", "files": ["imap_glows_photoion-2010a_20100101_v003.dat"]},'
'{"type": "ancillary", "files": ["imap_glows_electron-density-2010a_20100101_v003.dat"]},'
'{"type": "ancillary", "files": ["imap_glows_lya-2010a_20100101_v003.dat"]},'
'{"type": "ancillary", "files": ["imap_glows_l3b-archive_20100422_v011.zip"]},'
'{"type": "ancillary", "files": ["imap_glows_pipeline-settings-l3bcde_20100101_v006.json"]}]'
])
case "swapi", "proton":
subprocess.run([sys.executable, 'imap_l3_data_processor.py', '--instrument', 'swapi', '--data-level', 'l3a',
'--descriptor', 'proton-sw', '--start-date', '20250606', '--version', 'v000',
'--dependency',
'[{"type":"science","files":["imap_swapi_l2_sci_20251208_v006.cdf"]},'
'{"type":"ancillary","files":["imap_swapi_proton-density-temperature-lut_20240905_v000.dat"]},'
'{"type":"ancillary","files":["imap_swapi_alpha-density-temperature-lut_20240920_v000.dat"]},'
'{"type":"ancillary","files":["imap_swapi_clock-angle-and-flow-deflection-lut_20240918_v000.dat"]},'
'{"type":"ancillary","files":["imap_swapi_efficiency-lut_20241020_v000.dat"]},'
'{"type":"ancillary","files":["imap_swapi_energy-gf-pui-lut_20100101_v001.csv"]},'
'{"type":"ancillary","files":["imap_swapi_instrument-response-lut_20241023_v000.zip"]},'
'{"type":"ancillary","files":["imap_swapi_density-of-neutral-helium-lut_20241023_v000.dat"]},'
'{"type": "spice", "files": ["imap_sclk_0000.tsc"]},' +
'{"type": "spice", "files": ["naif0012.tls"]},' +
'{"type": "spice", "files": ["imap_science_108.tf"]},' +
'{"type": "spice", "files": ["imap_001.tf"]},' +
'{"type": "spice", "files": ["imap_2025_105_2026_105_01.ah.bc"]},' +
'{"type": "spice", "files": ["imap_dps_2025_105_2026_105_01.ah.bc"]}'
']'])
case "swapi", "alpha":
subprocess.run([sys.executable, 'imap_l3_data_processor.py', '--instrument', 'swapi', '--data-level', 'l3a',
'--descriptor', 'alpha-sw', '--start-date', '20250606', '--version', 'v000',
'--dependency',
'[{"type":"science","files":["imap_swapi_l2_sci_20250606_v001.cdf"]},'
'{"type":"ancillary","files":["imap_swapi_proton-density-temperature-lut_20240905_v000.dat"]},'
'{"type":"ancillary","files":["imap_swapi_alpha-density-temperature-lut_20240920_v000.dat"]},'
'{"type":"ancillary","files":["imap_swapi_clock-angle-and-flow-deflection-lut_20240918_v000.dat"]},'
'{"type":"ancillary","files":["imap_swapi_efficiency-lut_20241020_v000.dat"]},'
'{"type":"ancillary","files":["imap_swapi_energy-gf-pui-lut_20100101_v001.csv"]},'
'{"type":"ancillary","files":["imap_swapi_instrument-response-lut_20241023_v000.zip"]},'
'{"type":"ancillary","files":["imap_swapi_density-of-neutral-helium-lut_20241023_v000.dat"]}'
']'])
case "swapi", "pui":
subprocess.run([sys.executable, 'imap_l3_data_processor.py', '--instrument', 'swapi', '--data-level', 'l3a',
'--descriptor', 'pui-he', '--start-date', '20250606', '--version', 'v000',
'--dependency',
'[{"type":"science","files":["imap_swapi_l2_sci_20250606_v001.cdf"]},'
'{"type":"ancillary","files":["imap_swapi_proton-density-temperature-lut_20240905_v000.dat"]},'
'{"type":"ancillary","files":["imap_swapi_alpha-density-temperature-lut_20240920_v000.dat"]},'
'{"type":"ancillary","files":["imap_swapi_clock-angle-and-flow-deflection-lut_20240918_v000.dat"]},'
'{"type":"ancillary","files":["imap_swapi_efficiency-lut_20241020_v000.dat"]},'
'{"type":"ancillary","files":["imap_swapi_energy-gf-pui-lut_20100101_v001.csv"]},'
'{"type":"ancillary","files":["imap_swapi_instrument-response-lut_20241023_v000.zip"]},'
'{"type":"ancillary","files":["imap_swapi_density-of-neutral-helium-lut_20241023_v000.dat"]},'
'{"type": "spice", "files": ["imap_sclk_0000.tsc"]},' +
'{"type": "spice", "files": ["naif0012.tls"]},' +
'{"type": "spice", "files": ["imap_science_0001.tf"]},' +
'{"type": "spice", "files": ["imap_001.tf"]},' +
'{"type": "spice", "files": ["imap_2025_105_2026_105_01.ah.bc"]},' +
'{"type": "spice", "files": ["de440.bsp"]},' +
'{"type": "spice", "files": ["imap_recon_20250415_20260415_v01.bsp"]},' +
'{"type": "spice", "files": ["imap_dps_2025_105_2026_105_01.ah.bc"]}'
']'])
case "swapi", "l3b":
subprocess.run([sys.executable, 'imap_l3_data_processor.py', '--instrument', 'swapi', '--data-level', 'l3b',
'--start-date', '20250606', '--version', 'v003', '--dependency',
'[{"type": "science", "files": ["imap_swapi_l2_sci_20250606_v001.cdf"]},'
'{"type":"ancillary","files":["imap_swapi_energy-gf-sw-lut_20100101_v001.csv"]},'
'{"type":"ancillary","files":["imap_swapi_efficiency-lut_20241020_v000.dat"]}]'])
case "codice", "lo-direct-events":
subprocess.run(
[sys.executable, 'imap_l3_data_processor.py', '--instrument', 'codice', '--data-level', 'l3a',
'--descriptor', 'lo-direct-events', '--start-date', '20241110', '--version', 'v000',
'--dependency',
'['
'{"type": "science", "files": ["imap_codice_l1a_lo-sw-priority_20241110_v002.cdf"]},'
'{"type": "science", "files": ["imap_codice_l1a_lo-nsw-priority_20241110_v003.cdf"]},'
'{"type": "science", "files": ["imap_codice_l2_lo-direct-events_20241110_v006.cdf"]},'
'{"type": "ancillary", "files": ["imap_codice_mass-coefficient-lookup_20241110_v002.csv"]},'
'{"type": "ancillary", "files": ["imap_codice_lo-energy-per-charge_20241110_v001.csv"]}]'
])
case "codice", ("lo-partial-densities" | "partial-densities"):
subprocess.run(
[sys.executable, 'imap_l3_data_processor.py', '--instrument', 'codice', '--data-level', 'l3a',
'--descriptor', 'lo-partial-densities', '--start-date', '20241110', '--version', 'v000',
'--dependency',
'['
'{"type": "science", "files": ["imap_codice_l2_lo-sw-species_20241110_v002.cdf"]},'
'{"type": "ancillary", "files": ["imap_codice_mass-per-charge_20241110_v002.csv"]}]'
])
case "codice", ("lo-sw-ratios" | "ratios"):
subprocess.run(
[sys.executable, 'imap_l3_data_processor.py', '--instrument', 'codice', '--data-level', 'l3a',
'--descriptor', 'lo-sw-ratios', '--start-date', '20241110', '--version', 'v000',
'--dependency',
'['
'{"type": "science", "files": ["imap_codice_l3a_lo-partial-densities_20241110_v000.cdf"]}'
']'
])
case "codice", ("lo-sw-charge-state-distributions"):
subprocess.run(
[sys.executable, 'imap_l3_data_processor.py', '--instrument', 'codice', '--data-level', 'l3a',
'--descriptor', 'lo-sw-charge-state-distributions', '--start-date', '20241110', '--version', 'v000',
'--dependency',
'['
'{"type": "science", "files": ["imap_codice_l3a_lo-partial-densities_20241110_v000.cdf"]}'
']'
])
case "codice", "hi-direct-events":
subprocess.run(
[sys.executable, 'imap_l3_data_processor.py', '--instrument', 'codice', '--data-level', 'l3a',
'--descriptor', 'hi-direct-events', '--start-date', '20241110', '--version', 'v000',
'--dependency',
'['
'{"type": "science", "files": ["imap_codice_l2_hi-direct-events_20241110_v002.cdf"]},'
'{"type": "ancillary", "files": ["imap_codice_tof-lookup_20241110_v002.csv"]}]'
])
case "codice", "hi-pitch-angle":
subprocess.run(
[sys.executable, 'imap_l3_data_processor.py', '--instrument', 'codice', '--data-level', 'l3b',
'--descriptor', 'hi-pitch-angle', '--start-date', '20241110', '--version', 'v000',
'--dependency',
'['
'{"type": "science", "files": ["imap_codice_l2_hi-sectored_20241110_v002.cdf"]},'
'{"type": "science", "files": ["imap_mag_l1d_norm-dsrf_20250630_v001.cdf"]}'
']'
])
case "codice", descriptor if "3d-distribution" in descriptor:
species = descriptor.split("-")[1]
assert species in ["hplus", "heplus", "heplus2", "oplus6"], NotImplementedError(
"Target instrument or product not implemented")
subprocess.run(
[sys.executable, 'imap_l3_data_processor.py', '--instrument', 'codice', '--data-level', 'l3a',
'--descriptor', f'lo-{species}-3d-distribution', '--start-date', '20241110', '--version', 'v000',
'--dependency',
'['
'{"type": "science", "files": ["imap_codice_l1a_lo-sw-priority_20241110_v002.cdf"]},'
'{"type": "science", "files": ["imap_codice_l1a_lo-nsw-priority_20241110_v003.cdf"]},'
'{"type": "science", "files": ["imap_codice_l3a_lo-direct-events_20241110_v005.cdf"]},'
'{"type": "ancillary", "files": ["imap_codice_lo-mass-species-bin-lookup_20241110_v001.csv"]},'
'{"type": "ancillary", "files": ["imap_codice_lo-geometric-factors_20241110_v001.csv"]},'
'{"type": "ancillary", "files": ["imap_codice_lo-energy-per-charge_20241110_v001.csv"]},'
f'{{"type": "ancillary", "files": ["imap_codice_lo-{species}-efficiency_20241110_v001.csv"]}}'
']'
])
case _:
raise NotImplementedError("Target instrument or product not implemented")