Merge branch 'master' into master

Author: westonslaughter

.RData

@@ -27,3 +27,4 @@ data/general/nhd_hr
 output/
 old_dataset_dirs.tar.gz
 old_logs
+vault/*

.gitignore

@@ -101,6 +101,16 @@ ms_init <- function(use_gpu = FALSE,
     successes <- 0
     which_machine <- 'unknown'
 
+    # example dev computer 'registration' code block
+    ## res <- try(setwd('~/your/file/path/to/macrosheds/data_processing'), silent=TRUE) # example
+    ## if(! 'try-error' %in% class(res)){
+    ##     successes <- successes + 1
+    ##     which_machine <- 'your_machine') # machine name is completely up to you, does not matter
+    ##     instance_type <- 'dev' # instance type is 'dev' for all personal computers
+    ##     machine_status <- 'n00b' # unless you have > 32GB of RAM and > 8 CPUS, your 'n00b'
+    ##     op_system <- 'mac' # whats your OS?
+    ## }
+
     res <- try(setwd('~/macrosheds_data_processing'), silent=TRUE) #DCC
     if(! 'try-error' %in% class(res)){
         successes <- successes + 1
@@ -127,6 +137,15 @@ ms_init <- function(use_gpu = FALSE,
         machine_status <- '1337'
         op_system <- 'windows'
     }
+    
+    res <- try(setwd('/Users/hectorontiveros/Applications/data_processing'), silent=FALSE) #Hector
+    if(! 'try-error' %in% class(res)){
+      successes <- successes + 1
+      which_machine <- 'hec'
+      instance_type <- 'dev'
+      machine_status <- 'n00b'
+      op_system <- 'macOS'
+    }
 
     res <- try(setwd('~/desktop/macrosheds/data_acquisition'), silent=TRUE) #spencer
     if(! 'try-error' %in% class(res)){
@@ -137,6 +156,15 @@ ms_init <- function(use_gpu = FALSE,
         op_system <- 'mac'
     }
 
+    res <- try(setwd('~/Desktop/MacroSheds/data_processing/src/data_acquisition'), silent=TRUE) #pranavi
+    if(! 'try-error' %in% class(res)){
+      successes <- successes + 1
+      which_machine <- 'Pranavi'
+      instance_type <- 'dev'
+      machine_status <- 'n00b'
+      op_system <- 'mac'
+    }
+    
     res <- try(setwd('C:/Users/gubbi/Documents/macrosheds/data_processing'), silent=TRUE) #Nick
     if(! 'try-error' %in% class(res)){
         successes <- successes + 1
@@ -172,6 +200,15 @@ ms_init <- function(use_gpu = FALSE,
     #     machine_status <- '1337'
     # }
 
+    res <- try(setwd('/home/weston/science/macrosheds/data_processing'), silent=TRUE) # wes
+    if(! 'try-error' %in% class(res)){
+        successes <- successes + 1
+        which_machine <- 'wes'
+        instance_type <- 'dev'
+        machine_status <- '1337'
+        op_system <- 'linux'
+    }
+
     res <- try(setwd('/home/macrosheds/data_acquisition'), silent=TRUE) #server
     if(! 'try-error' %in% class(res)){
         successes <- successes + 1
@@ -218,24 +255,29 @@ ms_instance <- ms_init(use_ms_error_handling = FALSE,
                        config_storage_location = 'remote')
 
 #load authorization file for macrosheds google sheets
-googlesheets4::gs4_auth(path = 'googlesheet_service_accnt.json')
+## googlesheets4::gs4_auth(path = 'googlesheet_service_accnt.json')
 
 #read in secrets
 conf <- jsonlite::fromJSON('config.json',
                            simplifyDataFrame = FALSE)
 
+
 #connect rgee to earth engine and python
 gee_login <- case_when(
     ms_instance$which_machine %in% c('Mike', 'BM1') ~ conf$gee_login_mike,
-    ms_instance$which_machine %in% c('Spencer', 'BM0', 'BM2') ~ conf$gee_login_spencer,
-    ms_instance$which_machine %in% c('Nick') ~ conf$gee_login_spencer,
+    ms_instance$which_machine %in% c('Spencer', 'BM0', 'BM2', 'Nick') ~ conf$gee_login_spencer,
+    ms_instance$which_machine %in% c('Hector','Biniam','Pranavi', 'Wes') ~conf$gee_login_ms,
     TRUE ~ 'UNKNOWN')
 
-# try(rgee::ee_Initialize(user = gee_login,
-#                         drive = TRUE))
-
+#load authorization file for macrosheds google sheets and drive
+#same account must have GEE and GDrive access
+googlesheets4::gs4_auth(email = gee_login)
 googledrive::drive_auth(email = gee_login)
 
+#initialize and authorize GEE account
+try(rgee::ee_Initialize(user = gee_login,
+                        drive = TRUE))
+                        
 #set up global logger. network-domain loggers are set up later
 logging::basicConfig()
 logging::addHandler(logging::writeToFile,
@@ -249,7 +291,7 @@ if(ms_instance$use_ms_error_handling){
     source_decoratees('src/global/global_helpers.R') #parse decorators
 }
 
-#puts ms_vars, site_data, ws_delin_specs, univ_products into the global environment
+#puts (google sheets) ms_vars, site_data, ws_delin_specs, univ_products into the global environment
 load_config_datasets(from_where = ms_instance$config_data_storage)
 
 
@@ -274,8 +316,6 @@ ms_globals <- c(ls(all.names = TRUE), 'ms_globals')
 
 dir.create('logs', showWarnings = FALSE)
 
-dmnrow = 29
-# print(network_domain, n=50)
 for(dmnrow in 1:nrow(network_domain)){
 
     # drop_automated_entries('.') #use with caution!
@@ -284,17 +324,19 @@ for(dmnrow in 1:nrow(network_domain)){
     network <- network_domain$network[dmnrow]
     domain <- network_domain$domain[dmnrow]
 
-    # held_data = get_data_tracker(network, domain)
+    held_data = get_data_tracker(network, domain)
 
-    # held_data = invalidate_tracked_data(network, domain, 'munge')
-    # owrite_tracker(network, domain)
-    # held_data = invalidate_tracked_data(network, domain, 'derive')
-    # owrite_tracker(network, domain)
+    ## dangerous lines - use at your own risk!    :0
+    ## held_data = invalidate_tracked_data(network, domain, 'munge')
+    ## owrite_tracker(network, domain)
+    ## held_data = invalidate_tracked_data(network, domain, 'derive')
+    ## owrite_tracker(network, domain)
 
-    # held_data = invalidate_tracked_data(network, domain, 'munge', 'precipitation')
-    # owrite_tracker(network, domain)
-    # held_data = invalidate_tracked_data(network, domain, 'derive', 'stream_flux_inst')
-    # owrite_tracker(network, domain)
+    ## less dangerous version below, clears tracker for just a specified product
+    ## held_data = invalidate_tracked_data(network, domain, 'munge', 'stream_chemistry')
+    ## owrite_tracker(network, domain)
+    ## held_data = invalidate_tracked_data(network, domain, 'derive', 'stream_flux_inst')
+    ## owrite_tracker(network, domain)
 
     logger_module <- set_up_logger(network = network,
                                    domain = domain)
@@ -306,23 +348,27 @@ for(dmnrow in 1:nrow(network_domain)){
 
     update_product_statuses(network = network,
                            domain = domain)
+
     get_all_local_helpers(network = network,
-                         domain = domain)
+                          domain = domain)
 
     ms_retrieve(network = network,
                 # prodname_filter = c('stream_chemistry'),
                 domain = domain)
+
     ms_munge(network = network,
              prodname_filter = c('stream_chemistry'),
              domain = domain)
+
     if(domain != 'mcmurdo'){
         sw(ms_delineate(network = network,
                         domain = domain,
                         dev_machine_status = ms_instance$machine_status,
                         verbose = TRUE))
     }
+
     ms_derive(network = network,
-              prodname_filter = c('discharge'),
+              prodname_filter = c('precip_pchem_pflux'),
               domain = domain)
 
     if(domain != 'mcmurdo'){
@@ -356,3 +402,4 @@ if(length(email_err_msgs)){
 
 loginfo(msg = 'Run complete',
         logger = logger_module)
+

src/.RData

@@ -1423,3 +1423,17 @@ insert_retrieval_datetimes <- function(){
         write_lines(rt, f)
     }
 }
+
+get_nonnumerics <- function(d){
+
+    #gets unique nonnumeric values by row. useful for identifying quality codes
+    #within data columns
+
+    nonnumerics = apply(d, 2, function(x){
+        xx = as.numeric(x)
+        nonnumerics = is.na(xx)
+        out = unique(x[nonnumerics])
+    })
+
+    return(nonnumerics)
+}

src/acquisition_master.R

@@ -50,5 +50,9 @@ map = purrr::map
 map2 = purrr::map
 st_read = sf::st_read
 errors = errors::errors
+drop_errors = errors::drop_errors
+set_errors = errors::set_errors
 pivot_wider = tidyr::pivot_wider
 pivot_longer = tidyr::pivot_longer
+rename = dplyr::rename
+where = tidyselect:::where

src/dev/dev_helpers.R

@@ -1822,36 +1822,24 @@ process_3_ms824 <- function(network, domain, prodname_ms, site_code,
 
   googledrive::drive_rm('GEE/rgee.csv', verbose = FALSE)
 
-
-  final <- fin_table %>%
+  fin_table <- fin_table %>%
     select(date, site_code, dayl, prcp, srad, swe, tmax, tmin, vp)
 
-  if(nrow(final) == 0){
+  if(nrow(fin_table) == 0){
     return(generate_ms_exception(glue('No data was retrived for {s}',
                                       s = site_code)))
   }
 
   dir.create(glue('data/{n}/{d}/ws_traits/daymet/',
                   n = network,
-                  d = domain))
+                  d = domain),
+             showWarnings = FALSE)
 
   file_path <- glue('data/{n}/{d}/ws_traits/daymet/domain_climate.feather',
                     n = network,
                     d = domain)
 
-  write_feather(final, file_path)
-
-  # type <- str_split_fixed(prodname_ms, '__', n = Inf)[,1]
-  #
-  # dir <- glue('data/{n}/{d}/ws_traits/{v}/',
-  #             n = network, d = domain, v = type)
-  #
-  # final <- append_unprod_prefix(final, prodname_ms)
-  # final_sum <- append_unprod_prefix(final_sum, prodname_ms)
-  #
-  # save_general_files(final_file = final_sum,
-  #                    raw_file = final,
-  #                    domain_dir = dir)
+  write_feather(fin_table, file_path)
 
   return()
 }