data fully restructured. just making metadata now

Author: vlahm

eml/eml_templates/attributes_CAMELS_compliant_ws_attr.txt

@@ -0,0 +1,34 @@
+attributeName	attributeDefinition	class	unit	dateTimeFormatString	missingValueCode	missingValueCodeExplanation
+p_mean		numeric	!Add units here!			
+pet_mean		numeric	!Add units here!			
+aridity		numeric	!Add units here!			
+p_seasonality		numeric	!Add units here!			
+frac_snow		numeric	!Add units here!			
+high_prec_freq		numeric	!Add units here!			
+high_prec_dur		numeric	!Add units here!			
+high_prec_timing		categorical				
+low_prec_freq		numeric	!Add units here!			
+low_prec_dur		numeric	!Add units here!			
+low_prec_timing		categorical				
+site_code		character				
+geol_1st_class		categorical				
+glim_1st_class_frac		numeric	!Add units here!			
+geol_2nd_class		categorical				
+glim_2nd_class_frac		numeric	!Add units here!			
+carbonate_rocks_frac		numeric	!Add units here!			
+geol_porosity		numeric	!Add units here!			
+geol_permeability		numeric	!Add units here!			
+sand_frac		numeric	!Add units here!			
+silt_frac		numeric	!Add units here!			
+clay_frac		numeric	!Add units here!			
+organic_frac		numeric	!Add units here!			
+gauge_lat		numeric	!Add units here!			
+gauge_lon		numeric	!Add units here!			
+area		numeric	!Add units here!			
+elev_mean		numeric	!Add units here!			
+slope_mean		numeric	!Add units here!			
+frac_forest		numeric	!Add units here!			
+dom_land_cover_frac		numeric	!Add units here!			
+dom_land_cover		categorical				
+root_depth_50		numeric	!Add units here!			
+root_depth_99		numeric	!Add units here!			

eml/eml_templates/attributes_ws_attr_timeseries.txt

@@ -0,0 +1,8 @@
+attributeName	attributeDefinition	class	unit	dateTimeFormatString	missingValueCode	missingValueCodeExplanation
+network		categorical				
+domain		categorical				
+site_code		categorical				
+var		categorical				
+date		Date		!Add datetime specifier here!		
+val		numeric	!Add units here!			
+pctCellErr		numeric	!Add units here!			

src/global/global_helpers.R

@@ -9963,7 +9963,6 @@ postprocess_entire_dataset <- function(site_data,
         prepare_for_edi(where = edi_dir,
                         dataset_version = dataset_version)
 
-        remove_more_neon_stuff_temporarily()
         manually_edit_eml()
 
         log_with_indent(glue('Uploading dataset v{vv} to EDI',
@@ -9984,27 +9983,46 @@ remove_more_neon_stuff_temporarily <- function(){
 
 manually_edit_eml <- function(){
 
-    if(.Platform$OS.type == 'windows') stop('this will not work on windows')
+    if(.Platform$OS.type == 'windows') stop('this will not work on windows (but can be adapted quickly)')
 
     att <- read_tsv('eml/eml_templates/attributes_ws_attr_summaries.txt')
 
     most_recent_eml <- system('ls -t eml/eml_out | head -n 1', intern = TRUE)
     eml <- read_lines(file.path('eml/eml_out', most_recent_eml))
 
+    new_eml_chunk <- c('\t<methods>', '\t\t<methodStep>', '\t\t\t<description>',
+                       NA, '\t\t\t</description>', '\t\t</methodStep>', '\t</methods>')
+
     mvclines <- grep('</missingValueCode>', eml)
     attlines <- grep('</attribute>', eml)
+    atnlines <- grep('<attributeName>', eml)
 
     for(i in rev(seq_along(mvclines))){
 
         mvcl <- mvclines[i]
 
         if((mvcl + 1) %in% attlines){
-            #HERE: figure out which variable this is, look up its details in att.
-            #if no details, next. otherwise, insert a methods-methodStep-description block
 
             attl <- mvcl + 1
-            eml <- c(eml[1:mvcl], neweml, eml[attl:length(eml)])
+            attdif <- atnlines - attl
+            attdif <- attdif[attdif < 0]
+            atnl <- atnlines[which.max(attdif)]
+            varn <- str_match(eml[atnl], '\\<attributeName\\>([^\\<]+)\\<\\/attributeName\\>$')[, 2]
+
+            if(length(varn) != 1 || is.na(varn)) stop('problem with varn')
+
+            if(! varn %in% att$attributeName) next
+
+            attdeets <- pull(att[att$attributeName == varn, 'details'])
+
+            if(is.na(attdeets)) next
 
+            new_eml_chunk[4] <- attdeets
+            eml <- c(eml[1:mvcl], new_eml_chunk, eml[attl:length(eml)])
+        }
+    }
+
+    write_lines(eml, file.path('eml/eml_out', most_recent_eml))
 }
 
 make_figshare_docs_skeleton <- function(where){
@@ -10340,7 +10358,7 @@ prepare_for_figshare <- function(where, dataset_version){
     #prepare documentation and metadata
     make_figshare_docs_skeleton(where = where)
     prepare_site_metadata_for_figshare(outfile = file.path(where, 'macrosheds_documentation/04_site_documentation/04a_site_metadata.csv'))
-    prepare_variable_metadata_for_figshare(outfile = file.path(where, '/macrosheds_documentation/variable_metadata.csv'),
+    prepare_variable_meta`data_for_figshare(outfile = file.path(where, '/macrosheds_documentation/variable_metadata.csv'),
                                            fs_format = 'new')
     assemble_misc_docs_figshare(where = where)
 
@@ -10415,7 +10433,7 @@ combine_ts_csvs <- function(where){
 
         domain_combined %>%
             arrange(site_code, var_category, var) %>%
-            write_csv(file.path(network_dir, paste0(d, '.csv')))
+            write_csv(file.path(network_dir, paste0('timeseries_', d, '.csv')))
     }
 }
 
@@ -10502,11 +10520,75 @@ prepare_for_edi <- function(where, dataset_version){
                     logger = logger_module)
     combine_daymet_csvs(file.path(where, '4_CAMELS-compliant_Daymet_forcings'))
 
+    log_with_indent('Combining ws attrs (separately for ms and camels-compliant)',
+                    indent = 2,
+                    logger = logger_module)
+    combine_ws_attrs(where)
+
     log_with_indent('Combining spatial objects by domain',
                     indent = 2,
                     logger = logger_module)
     combine_and_move_spatial_objects(from = file.path(where, '2_timeseries_data'),
                                      to = file.path(where, '5_shapefiles'))
+
+    #TEMPORARY
+    remove_more_neon_stuff_temporarily()
+
+    eml_misc(where)
+}
+
+combine_ws_attrs <- function(){
+
+    #ms-standard watershed attributes
+    ws_attrs <- list.files(glue('{where}/1_watershed_attribute_data/ws_attr_timeseries'),
+                           full.names = TRUE)
+
+    map_dfr(ws_attrs, read_csv) %>%
+        write_csv(glue('{where}/1_watershed_attribute_data/ws_attr_timeseries.csv'))
+
+    file.remove(ws_attrs)
+    file.remove(glue('{where}/1_watershed_attribute_data/ws_attr_timeseries'))
+
+    #camels-compliant watershed attributes
+    ws_attrs <- list.files(glue('{where}/3_CAMELS-compliant_watershed_attributes'),
+                           full.names = TRUE)
+
+    d <- read_csv(ws_attrs[1])
+    for(i in 2:length(ws_attrs)){
+        d <- full_join(d, read_csv(ws_attrs[i]), by = 'site_code')
+    }
+
+    write_csv(d, glue('{where}/3_CAMELS-compliant_watershed_attributes/CAMELS_compliant_ws_attr.csv'))
+
+    file.remove(ws_attrs)
+}
+
+eml_misc <- function(where){
+
+    ## rename some files to clarify what they are in the absence of dir structure
+
+    # fs <- list.files(glue('{where}/1_watershed_attribute_data/ws_attr_timeseries'),
+    #                  full.names = TRUE)
+    # file.rename(fs, sub('ws_attr_timeseries/', 'ws_attr_timeseries/ws_attr_ts_', fs))
+    #
+    # fs <- list.files(glue('{where}/3_CAMELS-compliant_watershed_attributes'),
+    #                  full.names = TRUE)
+    # file.rename(fs, sub('watershed_attributes/', 'watershed_attributes/CAMELS-compliant_ws_attr_ts_', fs))
+
+    file.rename(glue('{where}/4_CAMELS-compliant_Daymet_forcings/CAMELS-compliant_Daymet_forcings.csv'),
+                glue('{where}/4_CAMELS-compliant_Daymet_forcings/CAMELS_compliant_Daymet_forcings.csv'))
+
+    ## link shapefiles to eml loading dock and zip them together
+
+    dir.create('eml/data_links/shapefiles', showWarnings = FALSE)
+
+    sfs <- list.files(glue('{where}/5_shapefiles'), full.names = TRUE)
+    sfs_basenames <- basename(sfs)
+    file.link(sfs, file.path('eml/data_links/shapefiles', sfs_basenames))
+
+    zip(zipfile = 'eml/data_links/shapefiles.zip',
+        files = 'eml/data_links/shapefiles',
+        flags = '-r9Xq')
 }
 
 prepare_for_figshare_packageformat <- function(where, dataset_version){

src/global/one-off/build_eml_templates.R

@@ -1,14 +1,21 @@
 #don't source this file. it was used in the creation of macrosheds/data_processing/eml,
 #but everything in there should now be edited manually or piecemeal
 
+stop('if this is for macrosheds > v1, generalize paths')
+
 library(EMLassemblyline)
+library(tidyverse)
 
 setwd('~/git/macrosheds/data_acquisition')
 
+conf <- jsonlite::fromJSON('config.json',
+                           simplifyDataFrame = FALSE)
+
 wd <- file.path('eml', 'eml_templates')
 ed <- file.path('eml', 'eml_out')
 dd <- file.path('eml', 'data_links')
 
+unlink(dd, recursive = TRUE)
 dir.create(wd, recursive = TRUE)
 dir.create(ed, recursive = TRUE)
 dir.create(dd, recursive = TRUE)
@@ -42,31 +49,58 @@ template_annotations()
 
 # template_arguments()
 
-files_to_link <- c('macrosheds_figshare_v1/1_watershed_attribute_data/ws_attr_summaries.csv',
-                   'macrosheds_figshare_v1/2_timeseries_data/bear/bear/discharge/EB.csv')
+ts_tables <- list.files('macrosheds_figshare_v1/2_timeseries_data', pattern = '\\.csv$',
+                        recursive = TRUE, full.names = TRUE)
+
+files_to_link <- c(ts_tables,
+                   list.files('macrosheds_figshare_v1/1_watershed_attribute_data',
+                              full.names = TRUE, recursive = TRUE),
+                   'macrosheds_figshare_v1/3_CAMELS-compliant_watershed_attributes/CAMELS_compliant_ws_attr.csv',
+                   'macrosheds_figshare_v1/4_CAMELS-compliant_Daymet_forcings/CAMELS_compliant_Daymet_forcings.csv')
 basenames <- basename(files_to_link)
 link_locs <- file.path(dd, basenames)
 
+descriptions <- basenames
+descriptions <- str_replace(descriptions,
+                            '^timeseries_([a-z_]+)\\.csv$',
+                            'Time-series (streamflow, precip if available, chemistry) for domain: \\1')
+descriptions <- str_replace(descriptions,
+                            '^ws_attr_summaries\\.csv$',
+                            'Watershed attribute data, summarized across time, for all domains')
+descriptions <- str_replace(descriptions,
+                            '^ws_attr_timeseries\\.csv$',
+                            'Watershed attribute data, temporally explicit, for all domains')
+descriptions <- str_replace(descriptions,
+                            '^CAMELS_compliant_ws_attr\\.csv$',
+                            'Watershed attribute data, temporally explicit, for all domains, and interoperable with the CAMELS dataset (https://ral.ucar.edu/solutions/products/camels)')
+descriptions <- str_replace(descriptions,
+                            '^CAMELS_compliant_Daymet_forcings\\.csv$',
+                            'Daymet climate forcings for all domains; interoperable with the CAMELS dataset (https://ral.ucar.edu/solutions/products/camels)')
+
 for(i in seq_along(files_to_link)){
     suppressWarnings(file.link(files_to_link[i], link_locs[i]))
 }
 
+
+# temporal_coverage <- map(ts_tables, ~range(read_csv(.)$datetime)) %>%
+#     reduce(~c(min(c(.x[1], .y[1])), max(c(.x[2], .y[2]))))
+temporal_coverage <- c("1945-07-01", "2022-04-16")
+
 make_eml(wd, dd, ed,
          dataset.title = 'MacroSheds',
-         temporal.coverage = c('2012-05-01', '2014-11-30'),
+         temporal.coverage = as.Date(temporal_coverage),
          geographic.description = NULL,#not needed if geographic_coverage.txt exists,
          geographic.coordinates = NULL,#same,
          maintenance.description = 'ongoing',
          data.table = basenames,
-         # data.table.name = data.table, #takes care of itself
-         data.table.description = rep('placeholder', length(files_to_link)),
+         # data.table.name = data.table,
+         data.table.description = descriptions,
          data.table.quote.character = rep('"', length(files_to_link)),
          data.table.url = NULL,
-         other.entity = NULL, #list any non-table (zip, shp, R, etc) file here
+         other.entity = 'eml/data_links/shapefiles.zip',
          # other.entity.name = other.entity,
-         other.entity.description = NULL,
+         other.entity.description = 'Watershed boundaries, stream gauge locations, and precip gauge locations, for all domains',
          other.entity.url = NULL,
-         # provenance = c('knb-lter-cap.46.3'), #deprecated. use template_provenance()
-         user.id = NULL,
-         user.domain = NULL, #pretty sure this doesn't apply to me
+         user.id = conf$edi_user_id,
+         user.domain = NULL, #pretty sure this doesn't apply to us
          package.id = NULL)