small fixes

vlahm · vlahm · commit 55f98d756219 · 2023-01-17T19:53:58.000-07:00
diff --git a/src/acquisition_master.R b/src/acquisition_master.R
@@ -320,7 +320,7 @@ ms_globals <- c(ls(all.names = TRUE), 'ms_globals')
 dir.create('logs', showWarnings = FALSE)
 
 ## change string in line below to find row index of your desired domain
-dmnrow <- which(network_domain$domain == 'calhoun')
+# dmnrow <- which(network_domain$domain == 'suef')
 for(dmnrow in 1:nrow(network_domain)){
 
     # drop_automated_entries('.') #use with caution!
@@ -359,7 +359,7 @@ for(dmnrow in 1:nrow(network_domain)){
     get_all_local_helpers(network = network,
                           domain = domain)
 
-    # stop here and go to processing_kernels.R to continue
+    stop here and go to processing_kernels.R to continue
     ms_retrieve(network = network,
                 # prodname_filter = c('stream_chemistry'),
                 domain = domain)
@@ -383,11 +383,8 @@ for(dmnrow in 1:nrow(network_domain)){
 
         ms_general(network = network,
                    domain = domain,
-                   get_missing_only = F,
-                   # general_prod_filter = NULL)
-                   general_prod_filter = c('prism_precip', 'prism_temp_mean', 'et_ref'))
-                   # general_prod_filter = c('prism_temp_mean', 'et_ref'))
-                   general_prod_filter = c('et_ref'))
+                   get_missing_only = FALSE,
+                   general_prod_filter = NULL)
     }
 
     retain_ms_globals(ms_globals)
diff --git a/src/global/global_helpers.R b/src/global/global_helpers.R
@@ -10014,12 +10014,26 @@ postprocess_entire_dataset <- function(site_data,
         # legal_details_scrape(dataset_version = dataset_version)
 
         warning('TEMPORARY: removing all remaining NEON data, flux data, and in-progress domains')
-        remove_flux_neon_etc(where)
+        remove_flux_neon_etc(where = fs_dir)
 
         log_with_indent(glue('Uploading dataset v{vv} to Figshare',
                              vv = dataset_version),
                         logger = logger_module)
         # upload_dataset_to_figshare(dataset_version = dataset_version)
+
+        warning('IMPROVE THE FOLLOWING system calls (fix these issues upstream)')
+        system(glue("find {fs_dir} -name '*.csv' | xargs sed -e 's/cloased_shrub/closed_shrub/g' -i"))
+        system(glue("find {fs_dir} -name '*.csv' | xargs sed -e 's/lg_lncd/lg_nlcd/g' -i"))
+        system(glue("find {fs_dir} -name '*.csv' | xargs sed -e 's/ci_mean_annual_et/ck_mean_annual_et/g' -i"))
+        system("find ../portal -name '*.csv' | xargs sed -e 's/cloased_shrub/closed_shrub/g' -i")
+        system("find ../portal -name '*.csv' | xargs sed -e 's/lg_lncd/lg_nlcd/g' -i")
+        system("find ../portal -name '*.csv' | xargs sed -e 's/ci_mean_annual_et/ck_mean_annual_et/g' -i")
+        system("find ../portal -name '*.csv' | xargs sed -e 's/idbp/igbp/g' -i")
+        read_feather('../portal/data/general/biplot/year.feather') %>%
+            mutate(var = ifelse(var == 'lb_igbp_cloased_shrub', 'lb_igbp_closed_shrub', var),
+                   var = ifelse(var == 'lg_lncd_lichens', 'lg_nlcd_lichens', var)) %>%
+            write_feather('../portal/data/general/biplot/year.feather')
+
         upload_dataset_to_figshare_packageversion(dataset_version = dataset_version)
     } else {
         log_with_indent('NOT pushing data to Figshare.',
@@ -10894,7 +10908,8 @@ figshare_create_article <- function(title,
                                     keywords,
                                     category_ids,
                                     authors,
-                                    token){
+                                    token,
+                                    verbose = FALSE){
 
     if(is.character(keywords)) keywords <- as.list(keywords)
     if(is.numeric(category_ids)) category_ids <- as.list(category_ids)
@@ -10913,6 +10928,12 @@ figshare_create_article <- function(title,
                                   authors = authors),
                              auto_unbox = TRUE)
 
+    if(verbose){
+        print(request)
+        print(header)
+        print(meta)
+    }
+
     post <- expo_backoff(
         expr = {
             httr::POST(request,
@@ -11445,7 +11466,15 @@ upload_dataset_to_figshare_packageversion <- function(dataset_version){
     ### CREATE, UPLOAD, PUBLISH SITES, VARS, LEGAL STUFF, SPATIAL DATA, AND DOCUMENTATION
     other_uploadsA <- list.files('../portal/data/general/spatial_downloadables',
                                  full.names = TRUE)
-    other_uploadsA = grep('spatial_timeseries', other_uploadsA, invert = TRUE, value = TRUE) #patch. see upload_dataset_to_figshare()
+    # other_uploadsA = grep('spatial_timeseries', other_uploadsA, invert = TRUE, value = TRUE) #patch. see upload_dataset_to_figshare()
+    # rmneon = grep('spatial_timeseries', other_uploadsA, value = T)
+    # for(dd in rmneon){
+    #     read_csv(dd) %>% filter(domain != 'neon') %>% write_csv(dd)
+    # }
+    # fff <- list.files('macrosheds_figshare_v1/1_watershed_attribute_data/ws_attr_timeseries', full.names = TRUE)
+    # for(dd in fff){
+    #     read_csv(dd) %>% filter(domain != 'neon') %>% write_csv(dd)
+    # }
     names(other_uploadsA) <- rep('watershed_attributes', length(other_uploadsA))
     titlesA <- str_match(other_uploadsA, '/([^/]+)\\.csv(?:\\.zip)?$')[, 2]
 
@@ -11479,6 +11508,16 @@ upload_dataset_to_figshare_packageversion <- function(dataset_version){
 
     print(paste('uploading extras'))
 
+    #removing items that should now be accessed via EDI portal
+    rms <- (names(other_uploads) == 'metadata' | grepl('(?:columns|codes)\\.(?:txt|csv)$', other_uploads))
+    other_uploads <- other_uploads[! rms]
+    titles <- titles[! rms]
+
+    #variable catalog can be included with package data
+    ms_var_catalog <- paste0('macrosheds_figshare_v', dataset_version, '/macrosheds_documentation_packageformat/variable_catalog.csv')
+    save(ms_var_catalog, file = '../r_package/data/ms_var_catalog.RData')
+
+    file_ids_for_r_package2 <- tibble()
     for(i in seq_along(other_uploads)){
 
         uf <- other_uploads[i]
@@ -11500,19 +11539,23 @@ upload_dataset_to_figshare_packageversion <- function(dataset_version){
         #if existing article, delete old version
         if(ut %in% existing_extras_deets$title){
 
-            fls <- figshare_list_article_files(fs_id,
-                                               token = token)
+            for(fsid_ in fs_id){
 
+                fls <- figshare_list_article_files(fsid_,
+                                                   token = token)
 
-            if(length(fls) >= 1){
-                for(j in seq_along(fls)){
-                    figshare_delete_article_file(fs_id,
-                                                 file_id = fls[[j]]$id,
-                                                 token = token)
+                if(length(fls) >= 1){
+                    for(j in seq_along(fls)){
+                        figshare_delete_article_file(fsid_,
+                                                     file_id = fls[[j]]$id,
+                                                     token = token)
+                    }
                 }
             }
         }
 
+        fs_id <- fs_id[1]
+
         figshare_upload_article(fs_id,
                                 file = unname(uf),
                                 token = token)
@@ -11524,42 +11567,48 @@ upload_dataset_to_figshare_packageversion <- function(dataset_version){
         fls <- figshare_list_article_files(fs_id,
                                            token = token)
 
-        if(ut == 'site_metadata'){
-            sysout <- system(paste0("sed -r 's/files\\/[0-9]+/files\\/",
-                                    fls[[1]]$id,
-                                    "/g' ../r_package/R/ms_download_site_data.R -i"),
-                             intern = TRUE,
-                             ignore.stdout = FALSE,
-                             ignore.stderr = FALSE)
-            if(length(sysout)) stop('cannot update file ID in r_package/R/ms_download_site_data.R. maybe your path is different?')
-        }
-
-        if(ut == 'variable_metadata'){
-            sysout <- system(paste0("sed -r 's/files\\/[0-9]+/files\\/",
-                                    fls[[1]]$id,
-                                    "/g' ../r_package/R/ms_download_variables.R -i"),
-                             intern = TRUE,
-                             ignore.stdout = FALSE,
-                             ignore.stderr = FALSE)
-            if(length(sysout)) stop('cannot update file ID in r_package/R/ms_download_variables.R or ms_conversions.R. maybe your path is different?')
-            sysout <- system(paste0("sed -r 's/files\\/[0-9]+/files\\/",
-                                    fls[[1]]$id,
-                                    "/g' ../r_package/R/ms_conversions.R -i"),
-                             intern = TRUE,
-                             ignore.stdout = FALSE,
-                             ignore.stderr = FALSE)
-        }
-
-        if(ut == 'variable_catalog'){
-            sysout <- system(paste0("sed -r 's/files\\/[0-9]+/files\\/",
-                                    fls[[1]]$id,
-                                    "/g' ../r_package/R/ms_catalog.R -i"),
-                             intern = TRUE,
-                             ignore.stdout = FALSE,
-                             ignore.stderr = FALSE)
-            if(length(sysout)) stop('cannot update file ID in r_package/R/ms_catalog.R maybe your path is different?')
-        }
-    }
+        file_ids_for_r_package2 <- bind_rows(
+            file_ids_for_r_package2,
+            tibble(ut, fig_code = fls[[1]]$id))
+        # if(ut == 'site_metadata'){
+        #     sysout <- system(paste0("sed -r 's/files\\/[0-9]+/files\\/",
+        #                             fls[[1]]$id,
+        #                             "/g' ../r_package/R/ms_download_site_data.R -i"),
+        #                      intern = TRUE,
+        #                      ignore.stdout = FALSE,
+        #                      ignore.stderr = FALSE)
+        #     if(length(sysout)) stop('cannot update file ID in r_package/R/ms_download_site_data.R. maybe your path is different?')
+        # }
+
+        # if(ut == 'variable_metadata'){
+        #     sysout <- system(paste0("sed -r 's/files\\/[0-9]+/files\\/",
+        #                             fls[[1]]$id,
+        #                             "/g' ../r_package/R/ms_download_variables.R -i"),
+        #                      intern = TRUE,
+        #                      ignore.stdout = FALSE,
+        #                      ignore.stderr = FALSE)
+        #     if(length(sysout)) stop('cannot update file ID in r_package/R/ms_download_variables.R or ms_conversions.R. maybe your path is different?')
+        #     sysout <- system(paste0("sed -r 's/files\\/[0-9]+/files\\/",
+        #                             fls[[1]]$id,
+        #                             "/g' ../r_package/R/ms_conversions.R -i"),
+        #                      intern = TRUE,
+        #                      ignore.stdout = FALSE,
+        #                      ignore.stderr = FALSE)
+        # }
+
+        # if(ut == 'variable_catalog'){
+        #     sysout <- system(paste0("sed -r 's/files\\/[0-9]+/files\\/",
+        #                             fls[[1]]$id,
+        #                             "/g' ../r_package/R/ms_catalog.R -i"),
+        #                      intern = TRUE,
+        #                      ignore.stdout = FALSE,
+        #                      ignore.stderr = FALSE)
+        #     if(length(sysout)) stop('cannot update file ID in r_package/R/ms_catalog.R maybe your path is different?')
+        # }
+    }
+
+    save(file_ids_for_r_package2,
+         file = '../r_package/data/sysdata2.RData')
 }
 
 detrmin_mean_record_length <- function(df){
@@ -14049,9 +14098,10 @@ generate_watershed_summaries <- function(){
 
     precip <- map_dfr(precip_files, read_feather) %>%
         filter(year != substr(Sys.Date(), 0, 4),
-               var == 'cc_cumulative_precip') %>%
+               var == 'cc_cumulative_precip',
+               val < 30000) %>%
         group_by(site_code) %>%
-        summarize(cc_mean_annual_precip = mean(val, na.arm = TRUE)) %>%
+        summarize(cc_mean_annual_precip = mean(val, na.rm = TRUE)) %>%
         filter(!is.na(cc_mean_annual_precip))
 
     # Prism temp
@@ -14062,7 +14112,7 @@ generate_watershed_summaries <- function(){
         filter(year != substr(Sys.Date(), 0, 4),
                var == 'cc_temp_mean') %>%
         group_by(site_code) %>%
-        summarize(cc_mean_annual_temp = mean(val, na.arm = TRUE)) %>%
+        summarize(cc_mean_annual_temp = mean(val, na.rm = TRUE)) %>%
         filter(!is.na(cc_mean_annual_temp))
 
     # start of season
@@ -14072,7 +14122,7 @@ generate_watershed_summaries <- function(){
         filter(year != substr(Sys.Date(), 0, 4),
                var == 'vd_sos_mean') %>%
         group_by(site_code) %>%
-        summarize(vd_mean_sos = mean(val, na.arm = TRUE)) %>%
+        summarize(vd_mean_sos = mean(val, na.rm = TRUE)) %>%
         filter(!is.na(vd_mean_sos))
 
     # end of season
@@ -14082,7 +14132,7 @@ generate_watershed_summaries <- function(){
         filter(year != substr(Sys.Date(), 0, 4),
                var == 'vd_eos_mean') %>%
         group_by(site_code) %>%
-        summarize(vd_mean_eos = mean(val, na.arm = TRUE)) %>%
+        summarize(vd_mean_eos = mean(val, na.rm = TRUE)) %>%
         filter(!is.na(vd_mean_eos))
 
     # length of season
@@ -14092,7 +14142,7 @@ generate_watershed_summaries <- function(){
         filter(year != substr(Sys.Date(), 0, 4),
                var == 'vd_los_mean') %>%
         group_by(site_code) %>%
-        summarize(vd_mean_los = mean(val, na.arm = TRUE)) %>%
+        summarize(vd_mean_los = mean(val, na.rm = TRUE)) %>%
         filter(!is.na(vd_mean_los))
 
     # maximum day of photosynthesis
@@ -14102,7 +14152,7 @@ generate_watershed_summaries <- function(){
         filter(year != substr(Sys.Date(), 0, 4),
                var == 'vd_mos_mean') %>%
         group_by(site_code) %>%
-        summarize(vd_mean_mos = mean(val, na.arm = TRUE)) %>%
+        summarize(vd_mean_mos = mean(val, na.rm = TRUE)) %>%
         filter(!is.na(vd_mean_mos))
 
     # gpp
@@ -14113,7 +14163,7 @@ generate_watershed_summaries <- function(){
         filter(year != substr(Sys.Date(), 0, 4),
                var == 'va_gpp_sum') %>%
         group_by(site_code) %>%
-        summarize(va_mean_annual_gpp = mean(val, na.arm = TRUE)) %>%
+        summarize(va_mean_annual_gpp = mean(val, na.rm = TRUE)) %>%
         filter(!is.na(va_mean_annual_gpp))
 
     # npp
@@ -14123,7 +14173,7 @@ generate_watershed_summaries <- function(){
         filter(year != substr(Sys.Date(), 0, 4),
                var == 'va_npp_median') %>%
         group_by(site_code) %>%
-        summarize(va_mean_annual_npp = mean(val, na.arm = TRUE)) %>%
+        summarize(va_mean_annual_npp = mean(val, na.rm = TRUE)) %>%
         filter(! is.na(va_mean_annual_npp))
 
     # terrain
@@ -14524,6 +14574,8 @@ compute_yearly_summary <- function(filter_ms_interp = FALSE,
     # this and compute_yearly_summary_ws should probably be combined at some point, but for now,
     # compute_yearly_summary_ws() appends compute_yearly_summary with ws_traits
 
+    #does not affect published dataset, only portal data, so no worries about filter settings.
+
     #df = default sites for each domain
     df <- site_data %>%
         filter(site_type != 'rain_gauge') %>%
diff --git a/src/templates/figshare_docfiles/packageformat_readme.txt b/src/templates/figshare_docfiles/packageformat_readme.txt
@@ -1,7 +1,7 @@
 This dataset is intended to be used through the "macrosheds" package for R
 (https://github.com/MacroSHEDS/macrosheds). It is not documented or arranged to be
-accessed directly by end-users, but you're welcome to use it to download data one
-MacroSheds domain at a time. The official MacroSheds dataset is available at https://doi.org/10.6084/m9.figshare.c.5621740.
+accessed directly by end-users. The official MacroSheds dataset is available on the data portal of
+the Environmental Data Initiative (EDI). Publication is currently pending.
 
 PLEASE carefully consider the data policy. Most of the data included here are (re)distributed
 under CC BY 4.0, so you're free to share and adapt with attribution, but a few of our
diff --git a/src/templates/macrosheds_citation.bib b/src/templates/macrosheds_citation.bib
@@ -0,0 +1,12 @@
+@article{vlah_etal_macrosheds_2023,
+    title = {MacroSheds: a synthesis of long-term biogeochemical, hydroclimatic, and geospatial data from small watershed ecosystem studies},
+    author = {Vlah, M.J. and Rhea, S. and Bernhardt, E.S. and Slaughter, W. and Gubbins, N. and DelVecchia, A.G. and Thellman, A. and Ross, M.R.V.},
+    year = {2023},
+    journal = {Limnology and Oceanography Letters},
+%    volume = {},
+%    number = {},
+%    pages = {5--18},
+%    publisher = {Elsevier},
+}
+
+
