plot fixes

Author: pdevis

hypernets_processor/data_io/format/variables_SWIR.py

@@ -44,7 +44,7 @@ write_l2a: True
 
 plot_l0: False
 plot_l1a: True
-plot_l1a_diff: False
+plot_l1a_diff: True
 plot_l1b: True
 plot_l1c: True
 plot_l2a: True

hypernets_processor/etc/processor_2.config

@@ -1,17 +1,34 @@
 
 
 import scipy.interpolate
+import numpy as np
 
 class WaterNetworkInterpolationLinear:
     def function(self,output_time,times,variables):
         '''
         This function implements the measurement function.
         Each of the arguments can be either a scalar or a vector (1D-array).
         '''
-        print(times.shape,variables.shape)
-        irradiance_intfunc=scipy.interpolate.interp1d(times,variables, fill_value="extrapolate")
-
-        return irradiance_intfunc(output_time)
+        if hasattr(output_time,'__len__'):
+            out = np.empty((len(variables),len(output_time)))
+            for i in range(len(output_time)):
+                if output_time[i] > max(times):
+                    out[:,i] = variables[:,times == max(times)][:,0]
+                elif output_time[i] < min(times):
+                    out[:,i] = variables[:,times == min(times)][:,0]
+                else:
+                    irradiance_intfunc = scipy.interpolate.interp1d(times,
+                                                                    variables)
+                    out[:,i] = irradiance_intfunc(output_time[i])
+                return out
+        else:
+            if output_time > max(times):
+                return variables[:,times == max(times)][:,0]
+            elif output_time < min(times):
+                return variables[:,times == min(times)][:,0]
+            else:
+                irradiance_intfunc = scipy.interpolate.interp1d(times,variables)
+                return irradiance_intfunc(output_time)
 
     @staticmethod
     def get_name():

hypernets_processor/interpolation/measurement_functions/water_network_interpolation.py

@@ -202,6 +202,12 @@ def run(self):
             #     sequence_path)
             # self.context.logger.debug("Done")
 
+            l0_rad["digital_number"].values[:,0]=l0_rad["digital_number"].values[:,0]/1.5
+            l0_irr["digital_number"].values[:,0]=l0_irr["digital_number"].values[:,0]/1.5
+            l0_swir_rad["digital_number"].values[:,0]=l0_swir_rad["digital_number"].values[:,0]/1.5
+            l0_swir_irr["digital_number"].values[:,0]=l0_swir_irr["digital_number"].values[:,0]/1.5
+
+
             #Calibrate to L1a
             self.context.logger.debug("Processing to L1a...")
             L1a_rad = cal.calibrate_l1a("radiance",l0_rad,l0_bla)

hypernets_processor/main_processor_outdoortest.py

@@ -76,12 +76,14 @@ def plot_diff_scans(self,measurandstring,dataset,dataset_avg=None):
             mask=DatasetUtil.unpack_flags(dataset["quality_flag"])["outliers"]
             if dataset_avg is None:
                 ids_used = np.where((dataset['series_id'] == series_id[i]) & np.invert(
-                    DatasetUtil.unpack_flags(dataset["quality_flag"])["outliers"]))
+                    DatasetUtil.unpack_flags(dataset["quality_flag"])["outliers"]))[0]
                 ydata_subset_used = dataset[measurandstring].values[:,ids_used]
-                avgs=np.tile(np.mean(ydata_subset_used,axis=1)[...,None],len(ydata_subset[0]))
+                avgs=np.tile(np.mean(ydata_subset_used,axis=1)[...,None],len(ids_used))
+                mask=mask[ids_used]
             else:
                 avg_ids = np.where((dataset_avg['series_id'] == series_id[i]))[0]
-                avgs = np.tile(dataset_avg[measurandstring].values[:,avg_ids],len(ydata_subset[0]))
+                avgs = np.tile(dataset_avg[measurandstring].values[:,avg_ids],len(avg_ids))
+                mask=mask[avg_ids]
 
             self.plot_variable("relative difference",plotpath,dataset["wavelength"].values,
                                (ydata_subset-avgs)/avgs,ylim=[-0.2,0.2],mask=mask)
@@ -146,15 +148,16 @@ def plot_reflectance(self,plotpath,xdata,ydata,labels=None):
 
     def plot_other_var(self,measurandstring,plotpath,xdata,ydata,labels=None,ylim=None,mask=None):
         fig1,ax1 = plt.subplots(figsize=(10,5))
-        if labels is None and mask is None:
+        if labels is None and mask is None or len(np.where(mask))==0:
             ax1.plot(xdata,ydata,alpha=0.3)
         elif mask is None:
             for i in range(len(labels)):
                 ax1.plot(xdata,ydata[:,i],label=labels[i],alpha=0.3)
         else:
-            print(mask.shape,ydata[:,np.where(mask)].shape)
-            ax1.plot(xdata,ydata[:,np.where(mask)][:,0,0],label="masked",alpha=0.3)
-            ax1.plot(xdata,ydata[:,np.where(np.invert(mask))][:,0,0],label="used",alpha=0.3)
+            print(ydata.shape)
+            print(ydata[:,np.where(np.invert(mask))].shape,ydata[:,np.where(np.invert(mask))][:,0].shape)
+            ax1.plot(xdata,ydata[:,np.where(mask)].reshape((len(ydata),len(np.where(mask)[0]))),label="masked",alpha=0.3)
+            ax1.plot(xdata,ydata[:,np.where(np.invert(mask))].reshape((len(ydata),len(np.where(np.invert(mask))[0]))),label="used",alpha=0.3)
 
         if labels is not None or mask is not None:
             ax1.legend()

hypernets_processor/plotting/plotting.py

@@ -189,18 +189,18 @@ def cycleparse(self, rad, irr):
             if lu.scan[lu['quality_flag'] <= 0].count() < nbrlu:
 
                 for i in range(len(lu["scan"])):
-                    lu["quality_flag"][lu["scan"]==i] = du.set_flag(lu["quality_flag"][lu["scan"] == i], "min_nbrlu")
+                    lu["quality_flag"][lu["scan"]==i] = du.set_flag(lu["quality_flag"][lu["scan"] == i], "outliers")
                 self.context.logger.info(
                     "No enough upwelling radiance data for sequence {}".format(lu.attrs['sequence_id']))
             if lsky.scan[lsky['quality_flag'] <= 1].count() < nbrlsky:
                 for i in range(len(lsky["scan"])):
-                    lsky["quality_flag"][lsky["scan"] == i] = du.set_flag(lsky["quality_flag"][lsky["scan"] == i], "min_nbrlsky")
+                    lsky["quality_flag"][lsky["scan"] == i] = du.set_flag(lsky["quality_flag"][lsky["scan"] == i], "outliers")
                 self.context.logger.info(
                     "No enough downwelling radiance data for sequence {}".format(lsky.attrs['sequence_id']))
             if irr.scan[irr['quality_flag'] <= 1].count() < nbred:
                 for i in range(len(irr["scan"])):
                     irr["quality_flag"][irr["scan"] == i] = du.set_flag(irr["quality_flag"][irr["scan"] == i],
-                                                                          "min_nbred")
+                                                                          "outliers")
                 self.context.logger.info(
                     "No enough downwelling irradiance data for sequence {}".format(irr.attrs['sequence_id']))
 

hypernets_processor/rhymer/rhymer/hypstar/rhymer_hypstar.py

@@ -44,19 +44,21 @@ def process_l1d(self, dataset_l1c):
             dataset_l1d = self.templ.l1d_from_l1c_dataset(dataset_l1c)
 
             # add required correction factors here - but better to add them in the function factory???
-            epsilon, failSimil = self.rh.get_epsilon(dataset_l1d["reflectance_nosc"].values,
-                                                  dataset_l1d["wavelength"].values)
+            epsilon, failSimil = self.rh.get_epsilon(dataset_l1c["reflectance_nosc"].values,
+                                                  dataset_l1c["wavelength"].values)
             dataset_l1d["epsilon"].values = epsilon
 
             dataset_l1d["quality_flag"][dataset_l1d["scan"] == [i for i, x in enumerate(failSimil) if x]] = DatasetUtil.set_flag(
                 dataset_l1d["quality_flag"][dataset_l1d["scan"] == [i for i, x in enumerate(failSimil) if x]],
                 "simil_fail")
 
             input_vars = l1ctol1d_function.get_argument_names()
-            input_qty = self.find_input(input_vars, dataset_l1c)
-            u_random_input_qty = self.find_u_random_input(input_vars, dataset_l1c)
+            input_qty = self.find_input(input_vars, dataset_l1d)
+            u_random_input_qty = self.find_u_random_input(input_vars, dataset_l1d)
             u_systematic_input_qty,cov_systematic_input_qty = \
-                self.find_u_systematic_input(input_vars,dataset_l1c)
+                self.find_u_systematic_input(input_vars,dataset_l1d)
+
+            print("inp",input_qty[4],u_random_input_qty[4],u_systematic_input_qty[4],cov_systematic_input_qty[4])
 
             dataset_l1d = self.process_measurement_function(
                 ["water_leaving_radiance", "reflectance_nosc", "reflectance"],
@@ -191,8 +193,8 @@ def find_u_systematic_input(self, variables, dataset):
         covs_indep = []
         for var in variables:
             try:
-                inputs.append(dataset["u_systematic_indep_" + var].values)
                 covs_indep.append(punpy.convert_corr_to_cov(dataset["corr_systematic_indep_" + var].values,dataset["u_systematic_indep_" + var].values))
+                inputs.append(dataset["u_systematic_indep_" + var].values)
             except:
                 inputs.append(None)
                 covs_indep.append(None)
@@ -223,9 +225,6 @@ def process_measurement_function(self, measurandstrings, dataset, measurement_fu
                                                             output_vars=len(measurandstrings))
 
             if len(measurandstrings) > 1:
-                print(input_quantities[0].shape,u_systematic_input_quantities[0].shape)
-                print(input_quantities[1].shape)
-                print(u_systematic_input_quantities[1].shape)
                 u_systematic_measurand, corr_systematic_measurand, corr_between = \
                     self.prop.propagate_systematic(measurement_function,
                                                    input_quantities,