Skip to content

Commit 78056df

Browse files
ulysses78ulysses78
authored
Merge pull request #318 from PollyNET/implement_martha
Implement martha
2 parents 8d7fdc2 + 36d5f51 commit 78056df

4 files changed

Lines changed: 161 additions & 12 deletions

File tree

‎lib/config/polly_global_config.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -267,6 +267,7 @@
267267
"yLim_depolConst_532": [0, 50],
268268
"yLim_depolConst_1064": [0, 10],
269269
"yLim_cloudinfo": [0, 2000],
270+
"yLim_overlap": [0, 3000],
270271

271272
"zLim_att_beta_355": [0, 15],
272273
"zLim_att_beta_532": [0, 5],

‎lib/interface/concat_pollyxt_lvl0.py‎

Lines changed: 133 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -78,7 +78,8 @@ def main():
7878
merging_flag = False
7979
merging_flag = concat_files()
8080

81-
get_pollyxt_logbook_files()
81+
if device != "martha":
82+
get_pollyxt_logbook_files()
8283

8384
if merging_flag == True:
8485
sys.exit(0) # Indicates success/True
@@ -94,7 +95,129 @@ def get_input_path(timestamp,device,raw_folder):
9495
print(input_path)
9596
return input_path
9697

98+
def get_input_path_martha(timestamp,device,raw_folder):
99+
#raw_folder="/data/level0/martha/MARTHA_DATA"
100+
YYYY=timestamp[0:4]
101+
MM=timestamp[4:6]
102+
#input_path = Path(raw_folder,device,"data_zip",f"{YYYY}{MM}")
103+
input_path = Path(raw_folder)
104+
print(input_path)
105+
return input_path
106+
97107
### start of function concat_pollyxt_files
108+
def get_martha_files():
109+
'''
110+
This function locates multiple pollyxt level0 nc-zip files from one day measurements,
111+
unzipps the files to output_path
112+
and returns a list of files to be merged
113+
and the title of the new merged nc-file
114+
'''
115+
input_path = get_input_path_martha(timestamp,device,raw_folder)
116+
path_exist = Path(input_path)
117+
118+
if path_exist.exists() == True:
119+
120+
## set the searchpattern for the zipped-nc-files:
121+
YYYY=timestamp[0:4]
122+
MM=timestamp[4:6]
123+
DD=timestamp[6:8]
124+
125+
zip_searchpattern = str(YYYY)+'_'+str(MM)+'_'+str(DD)+'*_*[0-9].nc'
126+
127+
polly_files = Path(r'{}'.format(input_path)).glob('{}'.format(zip_searchpattern))
128+
polly_zip_files_list0 = [x for x in polly_files if x.is_file()]
129+
130+
131+
132+
## convert type path to type string
133+
polly_zip_files_list = []
134+
for file in polly_zip_files_list0:
135+
polly_zip_files_list.append(str(file))
136+
137+
if len(polly_zip_files_list) < 1:
138+
print('no files found!')
139+
sys.exit()
140+
else:
141+
print(polly_zip_files_list)
142+
143+
# Ensure the destination directory exists
144+
Path(output_path).mkdir(parents=True, exist_ok=True)
145+
146+
polly_files_list = []
147+
to_unzip_list = []
148+
for zip_file in polly_zip_files_list:
149+
## check for size of zip-files to ensure to exclude bad measurement files with wrong timestamp e.g. 19700101
150+
f_size = os.path.getsize(zip_file)
151+
print(zip_file)
152+
if f_size > 150000:
153+
print(f_size)
154+
print("filesize passes")
155+
else:
156+
print(f_size)
157+
print("filesize too small, file will be skipped!")
158+
continue ## go to next file
159+
160+
# ## check if zipfile is a valid zip-file
161+
# if not is_zipfile(zip_file):
162+
# print(f"invalid zip-file: {zip_file}\nskipping file.")
163+
# #polly_zip_files_list.remove(zip_file)
164+
# continue
165+
# else:
166+
# pass
167+
168+
#unzipped_nc = Path(zip_file).name
169+
#unzipped_nc = Path(unzipped_nc)
170+
#unzipped_nc = Path(output_path,unzipped_nc)
171+
#print('here....')
172+
#polly_files_list.append(unzipped_nc)
173+
path = Path(zip_file)
174+
polly_files_list.append(path)
175+
176+
### check if unzipped files already exists in outputfolder
177+
#if path.is_file() == False:
178+
# to_unzip_list.append(zip_file)
179+
#if path.is_file() == True:
180+
# os.remove(unzipped_nc)
181+
# to_unzip_list.append(zip_file)
182+
183+
# ## unzipping
184+
# date_pattern = str(YYYY)+'_'+str(MM)+'_'+str(DD)
185+
# if len(to_unzip_list) > 0:
186+
# ## if working remotly on windows, copy zipped files first, than unzip
187+
# if os_name.lower() == 'windows':
188+
# print("\nCopy zipped files to local drive...")
189+
# for zip_file in to_unzip_list:
190+
# print(zip_file)
191+
# shutil.copy2(Path(zip_file), Path(output_path) / Path(zip_file).name)
192+
# print("\nUnzipping...")
193+
# for zip_file in Path(output_path).iterdir():
194+
# if zip_file.is_file() and date_pattern in zip_file.stem and zip_file.suffix == '.zip':
195+
# with ZipFile(zip_file, 'r') as zip_ref:
196+
# print("unzipping "+str(zip_file))
197+
# zip_ref.extractall(output_path)
198+
# print("Removing .zip file...")
199+
# os.remove(zip_file)
200+
#
201+
# else:
202+
# print("\nUnzipping...")
203+
# for zip_file in to_unzip_list:
204+
# with ZipFile(zip_file, 'r') as zip_ref:
205+
# print("unzipping "+zip_file)
206+
# zip_ref.extractall(output_path)
207+
208+
209+
## sort lists
210+
polly_files_list.sort()
211+
212+
print("\n"+str(len(polly_files_list))+" files found:\n")
213+
print(polly_files_list)
214+
print("\n")
215+
216+
else:
217+
print("\nNo data was found in {}. Correct path?\n".format(input_path))
218+
sys.exit()
219+
return polly_files_list
220+
98221
def get_pollyxt_files():
99222
'''
100223
This function locates multiple pollyxt level0 nc-zip files from one day measurements,
@@ -315,7 +438,10 @@ def checking_vars():
315438
'zenithangle'
316439
]
317440

318-
polly_files_list = get_pollyxt_files()
441+
if device == "martha":
442+
polly_files_list = get_martha_files()
443+
else:
444+
polly_files_list = get_pollyxt_files()
319445
if len(polly_files_list) == 1:
320446
return polly_files_list
321447

@@ -625,10 +751,11 @@ def write_netcdf(ds: xarray.Dataset, out_file: Path) -> None:
625751

626752
ds.close()
627753

628-
print("\ndeleting individual .nc files ...")
629-
for el in sel_polly_files_list:
630-
print(el)
631-
os.remove(el)
754+
if device != "martha":
755+
print("\ndeleting individual .nc files ...")
756+
for el in sel_polly_files_list:
757+
print(el)
758+
os.remove(el)
632759
destination_file = Path(output_path,filestring)
633760
if os.path.exists(destination_file):
634761
os.remove(destination_file) # Remove the existing destination file

‎lib/interface/picasso_go24.sh‎

Lines changed: 26 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ display_help() {
1212
echo " -d, --device specify device, e.g. pollyxt_lacros"
1313
echo " -c, --config_file specify Picasso configuration file, e.g.: ~/Pollynet_Processing_Chain/config/pollynet_processing_chain_config_rsd2_andi.json"
1414
# echo " -o, --output specify folder where to put merged nc-file, e.g.: ~/todo_filelist"
15+
echo " -b, --base_folder specify folder where level0 files are located; default is set to polly-folder, can be switched to e.g. martha-folder"
1516
echo " --force_merging specify whether files will be merged independently if attributes have changed or not; default: false"
1617
echo " --todolist write merged level0-file into todo-list (set to true or false); default: true"
1718
echo " --matlab specify location of matlab-executable; default is just set to: matlab"
@@ -27,6 +28,7 @@ display_help() {
2728
## initialize parameters
2829
FORCE_MERGING="false"
2930
MATLABEXEC="matlab"
31+
BASE_FOLDER="/data/level0/polly"
3032
PICASSO_CONFIG_FILE=""
3133
PICASSO_DIR_interface="$( cd "$(dirname "$0")" ; pwd -P )"
3234
PICASSO_DIR="$(dirname "$(dirname "$PICASSO_DIR_interface")")"
@@ -75,6 +77,13 @@ while :; do
7577
shift 2
7678
;;
7779

80+
-b | --base_folder)
81+
if [ $# -ne 0 ]; then
82+
BASE_FOLDER="$2"
83+
fi
84+
shift 2
85+
;;
86+
7887
--force_merging)
7988
if [ $# -ne 0 ]; then
8089
FORCE_MERGING="$2"
@@ -215,11 +224,16 @@ get_polly_filename() {
215224
local device=$1
216225
local date=$2
217226
# /data/level0/polly/pollyxt_lacros/data_zip/201907
218-
local YYYY=${date:0:4}
227+
local YYYY=${date:0:4}
219228
local MM=${date:4:2}
220229
local DD=${date:6:2}
221-
local input_path="/data/level0/polly/${device}/data_zip/${YYYY}${MM}"
222-
local searchpattern="${YYYY}_${MM}_${DD}*_*[0-9].nc.zip"
230+
if [[ "$DEVICE" == "martha" ]]; then
231+
local input_path="${BASE_FOLDER}"
232+
local searchpattern="${YYYY}_${MM}_${DD}*_*[0-9].nc"
233+
else
234+
local input_path="${BASE_FOLDER}/${device}/data_zip/${YYYY}${MM}"
235+
local searchpattern="${YYYY}_${MM}_${DD}*_*[0-9].nc.zip"
236+
fi
223237
local polly_files=`ls ${input_path}/${searchpattern}`
224238
# echo $polly_files
225239
# return ${polly_files}
@@ -238,7 +252,11 @@ get_polly_filename() {
238252
process_history() {
239253
DEVICE=$1
240254
DATE=$2
241-
local POLLY_FOLDER="/data/level0/polly/${DEVICE}"
255+
if [[ "$DEVICE" == "martha" ]]; then
256+
local POLLY_FOLDER="$BASE_FOLDER"
257+
else
258+
local POLLY_FOLDER="${BASE_FOLDER}/${DEVICE}"
259+
fi
242260
local POLLY_TYPE=$DEVICE
243261

244262
echo -e "\nSettings:\nPOLLY_FOLDER=$POLLY_FOLDER\nPOLLY_TYPE=$POLLY_TYPE\nPICASSO_CONFIG_FILE=$PICASSO_CONFIG_FILE\nSTART_DATE=$DATE\nEND_DATE=$DATE\n\n"
@@ -268,7 +286,7 @@ merging() {
268286
mkdir -p $OUTPUT_FOLDER ## create folder if not existing, else skip
269287
echo "start merging... "
270288

271-
"$PY_FOLDER"python "$PICASSO_DIR_interface"/concat_pollyxt_lvl0.py -t $DATE -d $DEVICE -o $OUTPUT_FOLDER -f ${FORCE_MERGING^}
289+
"$PY_FOLDER"python "$PICASSO_DIR_interface"/concat_pollyxt_lvl0.py -t $DATE -d $DEVICE -o $OUTPUT_FOLDER -r $BASE_FOLDER -f ${FORCE_MERGING^}
272290

273291
exit_status=$?
274292

@@ -287,7 +305,10 @@ write_job_into_todo_list() {
287305
DEVICE=$1
288306
DATE=$2
289307
local OUTPUT_FOLDER=$TODO_FOLDER/$DEVICE/data_zip/${DATE:0:6}
308+
#echo "$OUTPUT_FOLDER"
290309
local filename=$(get_polly_filename $DEVICE $DATE)
310+
311+
#echo "$filename"
291312
# echo $filename
292313
already_in_list=0
293314
if grep -q "$filename" $PICASSO_TODO_FILE

‎lib/visualization/pypolly_display_3d_plots.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2076,7 +2076,7 @@ def pollyDisplay_Overlap(nc_dict,config_dict,polly_conf_dict,outdir,donefilelist
20762076

20772077
## read from global config file
20782078
xLim = [-0.1, 1.1]
2079-
yLim = polly_conf_dict['yLim_att_beta_NR']
2079+
yLim = polly_conf_dict['yLim_overlap']
20802080
partnerLabel = polly_conf_dict['partnerLabel']
20812081
imgFormat = polly_conf_dict['imgFormat']
20822082

0 commit comments

Comments
 (0)