add webb network helper, load data for pkernels

Author: westonslaughter

src/acquisition_master.R

@@ -292,6 +292,13 @@ ms_globals <- c(ls(all.names = TRUE), 'ms_globals')
 
 dir.create('logs', showWarnings = FALSE)
 
+# NOTE: this should be moved I believe, and made to work with the raw data
+# dcumentation of the latest iteration...
+# this function will update the citation sheet with the data and url of raw data download
+scrape_data_download_urls()
+
+## change string in line below to find row index of your desired domain
+## dmnrow <- which(network_domain$domain == 'loch_vale')
 for(dmnrow in 1:nrow(network_domain)){
 
     # drop_automated_entries('.') #use with caution!
@@ -322,12 +329,15 @@ for(dmnrow in 1:nrow(network_domain)){
                        n = network,
                        d = domain))
 
+    # this should only run when you have your producs.csv
+    # and processing kernels prod information matching
     update_product_statuses(network = network,
                            domain = domain)
 
     get_all_local_helpers(network = network,
                           domain = domain)
 
+    # stop here and go to processing_kernels.R to continue
     ms_retrieve(network = network,
                 # prodname_filter = c('stream_chemistry'),
                 domain = domain)

src/webb/loch_vale/domain_helpers.R

@@ -0,0 +1,73 @@
+retrieve_sleepers_product <- function(network,
+                                 domain,
+                                 prodname_ms,
+                                 site_code,
+                                 tracker,
+                                 url){
+    # creating a string which matches the names of processing kernels
+    processing_func <- get(paste0('process_0_',
+                                  # these names or based off of prod names in products.csv
+                                  prodcode_from_prodname_ms(prodname_ms)))
+
+    # tracking the "version" of the product
+    rt <- tracker[[prodname_ms]][[site_code]]$retrieve
+
+    held_dt <- as.POSIXct(rt$held_version,
+                          tz = 'UTC')
+
+    # "deets" is a list of all the information originally from a row in products.csv
+    deets <- list(prodname_ms = prodname_ms,
+                  site_code = site_code,
+                  component = rt$component,
+                  last_mod_dt = held_dt,
+                  url = url)
+
+    # these "deets" are fed as arguments to wwhatever processing kernel is currently being called
+    # remember, this "retrieve_product" function is being called, in the retrieve.R script,
+    # in a loop over the product names from products.csv -- this is why the products.csv prod names
+    # must match the end of the procesing kernels which are written to retrieve that product
+
+    # if you're working on pkernels and not actually running this func, uncomment and run these lines:
+    ## set_details = deets
+    ## network = network
+    ## domain = domain
+
+    result <- do.call(processing_func,
+                      args = list(set_details = deets,
+                                  network = network,
+                                  domain = domain))
+
+
+    new_status <- evaluate_result_status(result)
+
+    if('access_time' %in% names(result) && any(! is.na(result$access_time))){
+        deets$last_mod_dt <- result$access_time[! is.na(result$access_time)][1]
+    }
+
+    update_data_tracker_r(network = network,
+                          domain = domain,
+                          tracker_name = 'held_data',
+                          set_details = deets,
+                          new_status = new_status)
+
+    source_urls <- get_source_urls(result_obj = result,
+                                   processing_func = processing_func)
+
+    write_metadata_r(murl = source_urls,
+                     network = network,
+                     domain = domain,
+                     prodname_ms = prodname_ms)
+
+}
+
+retrieve_usgs_sleeper_daily_q <- function(set_details) {
+  if(grepl("w5", set_details$component) == TRUE) {
+    q <- dataRetrieval::readNWISdv(siteNumbers = "01135300",
+                                   parameterCd = "00060")
+  } else if(grepl("w3", set_details$component) == TRUE) {
+    q <- dataRetrieval::readNWISdv(siteNumbers = "01135150",
+                                   parameterCd = "00060")
+  }
+
+  return(q)
+}