merged webb updates, gitignored .RData

Author: vlahm

.RData

@@ -29,3 +29,5 @@ old_dataset_dirs.tar.gz
 old_logs
 eml/data_links
 eml/eml_out
+vault/*
+.Rdata

.gitignore

@@ -103,6 +103,16 @@ ms_init <- function(use_gpu = FALSE,
     successes <- 0
     which_machine <- 'unknown'
 
+    # example dev computer 'registration' code block
+    ## res <- try(setwd('~/your/file/path/to/macrosheds/data_processing'), silent=TRUE) # example
+    ## if(! 'try-error' %in% class(res)){
+    ##     successes <- successes + 1
+    ##     which_machine <- 'your_machine') # machine name is completely up to you, does not matter
+    ##     instance_type <- 'dev' # instance type is 'dev' for all personal computers
+    ##     machine_status <- 'n00b' # unless you have > 32GB of RAM and > 8 CPUS, your 'n00b'
+    ##     op_system <- 'mac' # whats your OS?
+    ## }
+
     res <- try(setwd('~/macrosheds_data_processing'), silent=TRUE) #DCC
     if(! 'try-error' %in% class(res)){
         successes <- successes + 1
@@ -129,6 +139,15 @@ ms_init <- function(use_gpu = FALSE,
         machine_status <- '1337'
         op_system <- 'windows'
     }
+    
+    res <- try(setwd('/Users/hectorontiveros/Applications/data_processing'), silent=FALSE) #Hector
+    if(! 'try-error' %in% class(res)){
+      successes <- successes + 1
+      which_machine <- 'hec'
+      instance_type <- 'dev'
+      machine_status <- 'n00b'
+      op_system <- 'macOS'
+    }
 
     res <- try(setwd('~/desktop/macrosheds/data_acquisition'), silent=TRUE) #spencer
     if(! 'try-error' %in% class(res)){
@@ -139,6 +158,15 @@ ms_init <- function(use_gpu = FALSE,
         op_system <- 'mac'
     }
 
+    res <- try(setwd('~/Desktop/MacroSheds/data_processing/src/data_acquisition'), silent=TRUE) #pranavi
+    if(! 'try-error' %in% class(res)){
+      successes <- successes + 1
+      which_machine <- 'Pranavi'
+      instance_type <- 'dev'
+      machine_status <- 'n00b'
+      op_system <- 'mac'
+    }
+    
     res <- try(setwd('C:/Users/gubbi/Documents/macrosheds/data_processing'), silent=TRUE) #Nick
     if(! 'try-error' %in% class(res)){
         successes <- successes + 1
@@ -174,6 +202,15 @@ ms_init <- function(use_gpu = FALSE,
     #     machine_status <- '1337'
     # }
 
+    res <- try(setwd('/home/weston/science/macrosheds/data_processing'), silent=TRUE) # wes
+    if(! 'try-error' %in% class(res)){
+        successes <- successes + 1
+        which_machine <- 'wes'
+        instance_type <- 'dev'
+        machine_status <- '1337'
+        op_system <- 'linux'
+    }
+
     res <- try(setwd('/home/macrosheds/data_acquisition'), silent=TRUE) #server
     if(! 'try-error' %in% class(res)){
         successes <- successes + 1
@@ -183,6 +220,16 @@ ms_init <- function(use_gpu = FALSE,
         op_system <- NA
     }
 
+  
+    res <- try(setwd('C:/Users/Dell/Documents/Projects/data_processing'), silent=TRUE) #server
+    if(! 'try-error' %in% class(res)){
+      successes <- successes + 1
+      which_machine <- 'bini'
+      instance_type <- 'dev'
+      machine_status <- 'noob'
+      op_system <- 'windows'
+    }
+    
     if(successes > 1){
         stop(glue('more than one working directory was available. must set the ',
                   'correct one manually'))
@@ -210,24 +257,29 @@ ms_instance <- ms_init(use_ms_error_handling = FALSE,
                        config_storage_location = 'remote')
 
 #load authorization file for macrosheds google sheets
-googlesheets4::gs4_auth(path = 'googlesheet_service_accnt.json')
+## googlesheets4::gs4_auth(path = 'googlesheet_service_accnt.json')
 
 #read in secrets
 conf <- jsonlite::fromJSON('config.json',
                            simplifyDataFrame = FALSE)
 
+
 #connect rgee to earth engine and python
 gee_login <- case_when(
     ms_instance$which_machine %in% c('Mike', 'BM1') ~ conf$gee_login_mike,
-    ms_instance$which_machine %in% c('Spencer', 'BM0', 'BM2') ~ conf$gee_login_spencer,
-    ms_instance$which_machine %in% c('Nick') ~ conf$gee_login_spencer,
+    ms_instance$which_machine %in% c('Spencer', 'BM0', 'BM2', 'Nick') ~ conf$gee_login_spencer,
+    ms_instance$which_machine %in% c('Hector','Biniam','Pranavi', 'Wes') ~conf$gee_login_ms,
     TRUE ~ 'UNKNOWN')
 
-try(rgee::ee_Initialize(user = gee_login,
-                        drive = TRUE))
-
+#load authorization file for macrosheds google sheets and drive
+#same account must have GEE and GDrive access
+googlesheets4::gs4_auth(email = gee_login)
 googledrive::drive_auth(email = gee_login)
 
+#initialize and authorize GEE account
+try(rgee::ee_Initialize(user = gee_login,
+                        drive = TRUE))
+                        
 #set up global logger. network-domain loggers are set up later
 logging::basicConfig()
 logging::addHandler(logging::writeToFile,
@@ -241,7 +293,7 @@ if(ms_instance$use_ms_error_handling){
     source_decoratees('src/global/global_helpers.R') #parse decorators
 }
 
-#puts ms_vars, site_data, ws_delin_specs, univ_products into the global environment
+#puts (google sheets) ms_vars, site_data, ws_delin_specs, univ_products into the global environment
 load_config_datasets(from_where = ms_instance$config_data_storage)
 
 
@@ -266,8 +318,13 @@ ms_globals <- c(ls(all.names = TRUE), 'ms_globals')
 
 dir.create('logs', showWarnings = FALSE)
 
-# dmnrow = 8
-# print(network_domain, n=50)
+# NOTE: this should be moved I believe, and made to work with the raw data
+# dcumentation of the latest iteration...
+# this function will update the citation sheet with the data and url of raw data download
+scrape_data_download_urls()
+
+## change string in line below to find row index of your desired domain
+## dmnrow <- which(network_domain$domain == 'loch_vale')
 for(dmnrow in 1:nrow(network_domain)){
 
     # drop_automated_entries('.') #use with caution!
@@ -276,17 +333,19 @@ for(dmnrow in 1:nrow(network_domain)){
     network <- network_domain$network[dmnrow]
     domain <- network_domain$domain[dmnrow]
 
-    # held_data = get_data_tracker(network, domain)
+    held_data = get_data_tracker(network, domain)
 
-    # held_data = invalidate_tracked_data(network, domain, 'munge')
-    # owrite_tracker(network, domain)
-    # held_data = invalidate_tracked_data(network, domain, 'derive')
-    # owrite_tracker(network, domain)
+    ## dangerous lines - use at your own risk!    :0
+    ## held_data = invalidate_tracked_data(network, domain, 'munge')
+    ## owrite_tracker(network, domain)
+    ## held_data = invalidate_tracked_data(network, domain, 'derive')
+    ## owrite_tracker(network, domain)
 
-    # held_data = invalidate_tracked_data(network, domain, 'munge', 'stream_chemistry')
-    # owrite_tracker(network, domain)
-    # held_data = invalidate_tracked_data(network, domain, 'derive', 'stream_flux_inst')
-    # owrite_tracker(network, domain)
+    ## less dangerous version below, clears tracker for just a specified product
+    ## held_data = invalidate_tracked_data(network, domain, 'munge', 'stream_chemistry')
+    ## owrite_tracker(network, domain)
+    ## held_data = invalidate_tracked_data(network, domain, 'derive', 'stream_flux_inst')
+    ## owrite_tracker(network, domain)
 
     logger_module <- set_up_logger(network = network,
                                    domain = domain)
@@ -296,23 +355,30 @@ for(dmnrow in 1:nrow(network_domain)){
                        n = network,
                        d = domain))
 
+    # this should only run when you have your producs.csv
+    # and processing kernels prod information matching
     update_product_statuses(network = network,
                            domain = domain)
+
     get_all_local_helpers(network = network,
                           domain = domain)
 
+    # stop here and go to processing_kernels.R to continue
     ms_retrieve(network = network,
                 # prodname_filter = c('stream_chemistry'),
                 domain = domain)
+
     ms_munge(network = network,
              # prodname_filter = c('stream_chemistry'),
              domain = domain)
+
     if(domain != 'mcmurdo'){
         sw(ms_delineate(network = network,
                         domain = domain,
                         dev_machine_status = ms_instance$machine_status,
                         verbose = TRUE))
     }
+
     ms_derive(network = network,
               prodname_filter = c('stream_chemistry'),
               domain = domain)
@@ -348,3 +414,4 @@ if(length(email_err_msgs)){
 
 loginfo(msg = 'Run complete',
         logger = logger_module)
+

src/acquisition_master.R

@@ -9372,7 +9372,7 @@ ms_write_confdata <- function(x,
 
     type_string <- case_when(
         which_dataset == 'ms_vars' ~ 'cccccccnnccnn',
-        which_dataset == 'site_data' ~ 'ccccccccnnnnncccc',
+        which_dataset == 'site_data' ~ 'ccccccccnnnnnccccc',
         which_dataset == 'ws_delin_specs' ~ 'cccncnnccl',
         TRUE ~ 'placeholder')
 
@@ -12130,6 +12130,7 @@ retrieve_versionless_product <- function(network,
 
     rt <- tracker[[prodname_ms]][[site_code]]$retrieve
 
+    ## i = 1
     for(i in 1:nrow(rt)){
 
         held_dt <- as.POSIXct(rt$held_version[i],
@@ -15177,3 +15178,79 @@ reformat_camels_for_ms <- function(){
         }
     }
 }
+
+scrape_data_download_urls <- function() {
+  # connecting to gsheet
+  citation_gsheet <- googlesheets4::read_sheet(
+            conf$citation_gsheet,
+            na = c('', 'NA'),
+            col_types = citation_cols <- paste0(rep('c', 24), collapse=''))
+      colnames(citation_gsheet) <- citation_gsheet[6,]
+
+  raw_fp <- "./vault/raw_documentation_files/"
+
+  networks <- list.files(raw_fp)
+
+  for(network in networks) {
+    ## network <- networks[5]
+    domains <- list.files(file.path(raw_fp, network))
+
+    for(domain in domains) {
+      ## domain <- domains[4]
+      writeLines(paste('reading documentation for data source:', domain))
+
+      # list all files in this domain of this network
+      product_names <- list.files(file.path(raw_fp, network, domain, "raw", "documentation"))
+      product_docs <- list.files(file.path(raw_fp, network, domain, "raw", "documentation"), full.names = TRUE)
+
+
+      # filter gsheet to domain, get all prodcodes
+      dmn_citation_gsheet <- citation_gsheet %>%
+        filter(network == !!network,
+               domain == !!domain)
+
+      dmn_prodcodes <- dmn_citation_gsheet$macrosheds_prodcode
+      dmn_prodcodes_grep <- paste0(dmn_prodcodes, collapse="|")
+
+      # filter product docs to only those with prodcode text matching gsheet
+      cited_products <- product_docs[ifelse(grepl(dmn_prodcodes_grep, product_docs), TRUE, FALSE)]
+
+      for(file in cited_products) {
+        ## file <- cited_products[1]
+
+        cited_filename <- stringr::str_split(file, "__", simplify =TRUE)[2]
+        cited_prodcode <- stringr::str_split(cited_filename, ".txt", simplify =TRUE)[1]
+
+        data_source_doc <- readLines(file)
+        data_source_link <- trimws(stringr::str_split(data_source_doc, "[^)][0-9]{4}\\-", simplify =TRUE)[1])
+        data_source_dt <- trimws(stringr::str_extract_all(data_source_doc, "[0-9]{4}\\-.*[^)]", simplify =TRUE))[1]
+
+        print(paste(network, domain, data_source_dt))
+
+        if(grepl("https://", data_source_link)) {
+          data_source_link <- stringr::str_split(data_source_link, " ", simplify = TRUE)[1]
+        }
+
+        # NOTE: documentation files are not mecha-standardized, this function scrapes the best standard
+        # this *should* capture all prodcodes in citation gsheet, giving NA for versionlesss
+
+        # now, for this prodcode in the citation_gsheet df, we put "docs" in the "link" column
+        citation_gsheet <- citation_gsheet %>%
+          mutate(
+            link = case_when(
+              ifelse(
+                grepl(
+                  cited_prodcode, macrosheds_prodcode), TRUE, FALSE) & domain == !!domain ~ data_source_link, TRUE ~ link),
+            link_download_datetime = case_when(
+              ifelse(
+                grepl(
+                  cited_prodcode, macrosheds_prodcode), TRUE, FALSE) & domain == !!domain ~ data_source_dt, TRUE ~ link_download_datetime)
+            ## link = case_when(link == "NA" ~ NA, TRUE ~ link)
+          )
+      }
+    }
+  }
+  # then, we write the edited df to the actual google sheet
+  googlesheets4::sheet_write(citation_gsheet, ss = conf$citation_gsheet, sheet = "timeseries")
+  ## return(citation_gsheet)
+}