finished building metadata

Author: vlahm

src/dev/dev_helpers.R

@@ -1291,6 +1291,11 @@ correct_all_geometries <- function(path, dir_pattern = 'ws_boundary'){
     }
 }
 
+# correct_all_geometries(path = '~/git/macrosheds/data_acquisition/data')
+# correct_all_geometries(path = '~/git/macrosheds/portal/data')
+# correct_all_geometries(path = '~/git/macrosheds/data_acquisition/macrosheds_dataset_v1')
+# correct_all_geometries(path = '~/git/macrosheds/data_acquisition/macrosheds_figshare_v1')
+
 rebuild_portal_data_before_postprocessing <- function(network_domain, backup = TRUE){
 
     dir_wrapper <- function(path, keyword){
@@ -1393,7 +1398,28 @@ rebuild_portal_data_before_postprocessing <- function(network_domain, backup = T
                   'sure it looks good before and after postprocessing. especially portal/data/general'))
 }
 
-# correct_all_geometries(path = '~/git/macrosheds/data_acquisition/data')
-# correct_all_geometries(path = '~/git/macrosheds/portal/data')
-# correct_all_geometries(path = '~/git/macrosheds/data_acquisition/macrosheds_dataset_v1')
-# correct_all_geometries(path = '~/git/macrosheds/data_acquisition/macrosheds_figshare_v1')
+insert_retrieval_datetimes <- function(){
+
+    #this inserts the last modification datetime of each raw documentation file as
+    #the presumed retrieval datetime for the corresponding raw data product.
+
+    #used in a pinch. hopefully a real recording method has been implemented by now.
+
+    fs <- list.files('data',
+                     recursive = TRUE, full.names = TRUE)
+    fs <- fs[grepl('/raw/', fs)]
+    fs <- fs[grepl('/documentation/', fs)]
+
+    for(f in fs){
+
+        rt <- read_lines(f)
+        if(length(rt) != 1) stop('sup with this')
+        if(grepl('UTC\\)$', rt)) next
+        rt <- glue(
+            rt, ' (',
+            as.character(lubridate::with_tz(file.info(f)$mtime, 'UTC')),
+            ' UTC)')
+
+        write_lines(rt, f)
+    }
+}

src/global/global_helpers.R

@@ -9568,10 +9568,6 @@ postprocess_entire_dataset <- function(site_data,
                         logger = logger_module)
     }
 
-    log_with_indent('adding legal metadata to each domain directory',
-                    logger = logger_module)
-    # legal_details_scrape(dataset_version = dataset_version)
-
     log_with_indent(glue('Generating output dataset v',
                          dataset_version),
                     logger = logger_module)
@@ -9594,13 +9590,6 @@ postprocess_entire_dataset <- function(site_data,
     # log_with_indent(glue('Removing unneeded files from portal dataset.',
     #                 logger = logger_module)
     # clean_portal_dataset()
-    
-    if(reformat_camels) {
-        log_with_indent('Reformatting CAMELS attributes to MacroSheds format', logger = logger_module)
-        reformat_camels_for_ms()
-    } else{
-        log_with_indent('NOT reformatting CAMELS attributes to MacroSheds format', logger = logger_module)
-    }
 
     log_with_indent('Generating spatial summary data',
                     logger = logger_module)
@@ -9634,6 +9623,11 @@ postprocess_entire_dataset <- function(site_data,
                              dataset_version = dataset_version)
         prepare_for_figshare_packageformat(where = fs_dir,
                                            dataset_version = dataset_version)
+        reformat_camels_for_ms()
+
+        # log_with_indent('adding legal metadata to each domain directory',
+        #                 logger = logger_module)
+        # legal_details_scrape(dataset_version = dataset_version)
 
         log_with_indent(glue('Uploading dataset v{vv} to Figshare',
                              vv = dataset_version),
@@ -9654,6 +9648,7 @@ make_figshare_docs_skeleton <- function(where){
     dir.create(file.path(where, 'macrosheds_documentation', '04_site_documentation'), showWarnings = FALSE)
     dir.create(file.path(where, 'macrosheds_documentation', '05_timeseries_documentation'), showWarnings = FALSE)
     dir.create(file.path(where, 'macrosheds_documentation', '06_ws_attr_documentation'), showWarnings = FALSE)
+    dir.create(file.path(where, 'macrosheds_documentation', '07_CAMELS-compliant_datasets_documentation'), showWarnings = FALSE)
     dir.create(file.path(where, 'macrosheds_documentation_packageformat'), showWarnings = FALSE)
 }
 
@@ -9825,11 +9820,18 @@ assemble_misc_docs_figshare <- function(where){
                                 overwrite = TRUE)
     select(domain_detection_limits, -precision, -sigfigs, -added_programmatically) %>%
         write_csv(file.path(docs_dir, '05_timeseries_documentation', '05f_detection_limits_and_precision.csv'))
-    file.copy('src/templates/figshare_docfiles/05g_detection_limits_and_precision_column_descriptions.txt', docs_dir)
+    file.copy('src/templates/figshare_docfiles/05g_detection_limits_and_precision_column_descriptions.txt',
+              file.path(docs_dir, '05_timeseries_documentation'))
     file.copy('/home/mike/git/macrosheds/papers/release_paper/tables/timeseries_refs.bib',
               file.path(docs_dir, '05_timeseries_documentation', '05h_timeseries_refs.bib'))
     file.copy('/home/mike/git/macrosheds/papers/release_paper/tables/ws_attr_refs.bib',
               file.path(docs_dir, '06_ws_attr_documentation', '06h_ws_attr_refs.bib'))
+    file.copy('src/templates/figshare_docfiles/07a_CAMELS-compliant_datasets_metadata.txt',
+              file.path(docs_dir, '07_CAMELS-compliant_datasets_documentation'))
+    file.copy('src/templates/figshare_docfiles/07b_CAMELS-compliant_ws_attributes_column_descriptions.txt',
+              file.path(docs_dir, '07_CAMELS-compliant_datasets_documentation'))
+    file.copy('src/templates/figshare_docfiles/07c_CAMELS-compliant_Daymet_forcings_column_descriptions.txt',
+              file.path(docs_dir, '07_CAMELS-compliant_datasets_documentation'))
     file.copy('src/templates/figshare_docfiles/02_glossary.txt', docs_dir)
     file.copy('src/templates/figshare_docfiles/03_changelog.txt', docs_dir)
     file.copy('/home/mike/git/macrosheds/data_acquisition/src/templates/figshare_docfiles/04b_site_metadata_column_descriptions.txt',
@@ -9981,6 +9983,14 @@ prepare_for_figshare <- function(where, dataset_version){
     prepare_ts_data_for_figshare(where = where,
                                  dataset_version = dataset_version)
     prepare_ws_attr_data_for_figshare(where = where)
+
+    #decided to change some dirnames. easiest to just do that as a patch here
+    file.rename(file.path(where, 'macrosheds_documentation'),
+                file.path(where, '0_documentation_and_metadata'))
+    file.rename(file.path(where, 'macrosheds_watershed_attribute_data'),
+                file.path(where, '1_watershed_attribute_data'))
+    file.rename(file.path(where, 'macrosheds_timeseries_data'),
+                file.path(where, '2_timeseries_data'))
 }
 
 prepare_for_figshare_packageformat <- function(where, dataset_version){
@@ -11570,7 +11580,7 @@ get_source_urls <- function(result_obj, processing_func){
 
     if(uses_gdrive_func){
 
-        source_urls <- 'MacroSheds drive; not yet public'
+        source_urls <- 'MacroSheds drive (contact us for original source): https://drive.google.com/drive/folders/1gugTmDybtMTbmKRq2WQvw2K1WkJjcmJr?usp=sharing'
 
     } else if('url' %in% names(result_obj)){
 
@@ -14489,41 +14499,42 @@ legal_details_scrape <- function(dataset_version){
 }
 
 reformat_camels_for_ms <- function(){
-    
-    # This function will reformat the camels metrics computed for MacroSheds 
-    # watersheds to the MacroSheds format.
-    
-    ms_attributes_dir <- '../timeseries_experimentation/neon_camels_attr/data/ms_attributes'
-    
-    all_files <- list.files(ms_attributes_dir, recursive = T, full.names = TRUE)
-    
-    
-    
+
+    # ms_attributes_dir <- '../timeseries_experimentation/neon_camels_attr/data/ms_attributes'
+    ms_attributes_dir <- '../qa_experimentation/data/ms_in_camels_format'
+
+    all_files <- list.files(ms_attributes_dir, recursive = TRUE, full.names = TRUE)
+
     soil_files <- all_files[grep('soil.feather', all_files)]
     clim_files <- all_files[grep('clim.feather', all_files)]
     topo_files <- all_files[grep('topo.feather', all_files)]
+    vege_files <- all_files[grep('vege.feather', all_files)]
     geol_files <- all_files[grep('geol.feather', all_files)]
     daymet_files <- all_files[grep('daymet_full_climate.feather', all_files)]
-    
+    daymet_files <- daymet_files[! grepl('/NC/', daymet_files)]
+    warning('removing NC daymet data from camels set')
+
     soil <- map_dfr(soil_files, read_feather)
     clim <- map_dfr(clim_files, read_feather)
     topo <- map_dfr(topo_files, read_feather)
     geol <- map_dfr(geol_files, read_feather)
-    
-    dir.create('data/camels_compliant')
-    dir.create('data/camels_compliant/daymet')
-    
-    
-    write_csv(soil, 'data/camels_compliant/soil.csv')
-    write_csv(clim, 'data/camels_compliant/clim.csv')
-    write_csv(topo, 'data/camels_compliant/topo.csv')
-    write_csv(geol, 'data/camels_compliant/geol.csv')
-    
+    vege <- map_dfr(vege_files, read_feather)
+
+    dir.create('macrosheds_figshare_v1/3_CAMELS-compliant_watershed_attributes')
+    dir.create('macrosheds_figshare_v1/4_CAMELS-compliant_Daymet_forcings')
+
+    write_csv(soil, 'macrosheds_figshare_v1/3_CAMELS-compliant_watershed_attributes/soil.csv')
+    write_csv(clim, 'macrosheds_figshare_v1/3_CAMELS-compliant_watershed_attributes/clim.csv')
+    write_csv(topo, 'macrosheds_figshare_v1/3_CAMELS-compliant_watershed_attributes/topo.csv')
+    write_csv(geol, 'macrosheds_figshare_v1/3_CAMELS-compliant_watershed_attributes/geol.csv')
+    write_csv(vege, 'macrosheds_figshare_v1/3_CAMELS-compliant_watershed_attributes/vege.csv')
+
     for(i in 1:length(daymet_files)){
+
         this_daymet <- read_feather(daymet_files[i])
-        
+
         sites <- unique(this_daymet$site_code)
-        
+
         for(s in 1:length(sites)){
             this_site <- this_daymet %>%
                 filter(site_code == !!sites[s]) %>%
@@ -14534,13 +14545,10 @@ reformat_camels_for_ms <- function(){
                        `tmax(C)` = tmax,
                        `tmin(C)` = tmin,
                        `vp(Pa)` = vp,
-                       `pet(mm)` = pet) 
-            
-            write_csv(this_site, glue('data/camels_compliant/daymet/{s}.csv',
+                       `pet(mm)` = pet)
+
+            write_csv(this_site, glue('macrosheds_figshare_v1/4_CAMELS-compliant_Daymet_forcings/{s}.csv',
                                       s = sites[s]))
         }
     }
 }
-
-
-

src/templates/figshare_docfiles/00_MAIN_README.txt

@@ -8,16 +8,30 @@ Details are in 01a_data_use_agreements.docx, alongside this readme. For citation
 by domain and product, see 01b_attribution_and_intellectual_rights_complete.xlsx.
 And please contact us (mail@macrosheds.org) if you feel any MacroSheds content should be amended.
 
-Otherwise, just refer to this file, and the many others it describes, as needed. And
-don't forget about macrosheds.org for visualizing the MacroSheds dataset, and the "macrosheds"
-R package for working with it (https://github.com/MacroSHEDS/macrosheds).
+For a thorough description of the methods involved in building this dataset, see the
+corresponding publication (pending). Also note that the easiest way to explore
+our data catalog is use the interactive catalogs on macrosheds.org (Data tab).
+You can also visualize the dataset there.
+
+Otherwise, just refer to this file, and the many others it names, as needed. And don't forget
+about the "macrosheds" R package, which helps with retrieving and working with MacroSheds data
+(https://github.com/MacroSHEDS/macrosheds). All project code is available
+on GitHub: https://github.com/MacroSHEDS
 
 ---
 
 MacroSheds dataset contents:
 
-DOCUMENTATION (this directory)
+NOTE: in addition to the files in 0_documentation_and_metadata, there are
+individual "documentation" directories in 2_timeseries_data/<n>/<d>,
+where <n> is any MacroSheds network and <d> is any MacroSheds domain within
+<n>. These additional documentation directories contain information on dataset
+retrieval locations, and code used to retrieve, munge, and derive each
+timeseries data product.
+
+0_documentation_and_metadata (this directory)
 ├── 01a_data_use_agreements.docx
+├── 01b_attribution_and_intellectual_rights_complete.xlsx
 ├── 02_glossary.txt
 ├── 03_changelog.txt
 ├── 04_site_documentation
@@ -27,27 +41,53 @@ DOCUMENTATION (this directory)
 │   ├── 05b_timeseries_variable_metadata.csv
 │   ├── 05c_timeseries_variable_metadata_column_descriptions.txt
 │   ├── 05d_timeseries_column_descriptions.txt
-│   └── 05e_range_check_limits.csv
+│   ├── 05e_range_check_limits.csv
+│   ├── 05f_detection_limits_and_precision.csv
+│   ├── 05g_detection_limits_and_precision_column_descriptions.txt
+│   └── 05h_timeseries_refs.bib
 ├── 06_ws_attr_documentation
 │   ├── 06b_ws_attr_variable_metadata.csv
 │   ├── 06c_ws_attr_variable_metadata_column_descriptions.txt
 │   ├── 06d_ws_attr_variable_category_codes.csv
 │   ├── 06e_ws_attr_data_source_codes.csv
 │   ├── 06f_ws_attr_summary_column_descriptions.csv
-│   └── 06g_ws_attr_timeseries_column_descriptions.txt
+│   ├── 06g_ws_attr_timeseries_column_descriptions.txt
+│   └── 06h_ws_attr_refs.bib
+├── 07_CAMELS-compliant_datasets_documentation
+│   ├── 07a_CAMELS-compliant_datasets_metadata.txt
+│   ├── 07b_CAMELS-compliant_ws_attributes_column_descriptions.txt
+│   └── 07c_CAMELS-compliant_Daymet_forcings_column_descriptions.txt
 └── 08_data_irregularities.csv
 
-WATERSHED ATTRIBUTES
-summaries
-timeseries by category, 1-6
-maybe add some readmes that refer back to the master readmes
+1_watershed_attribute_data
+├── ws_attr_summaries.csv
+└── ws_attr_timeseries
+    ├── climate.csv
+    ├── hydrology.csv
+    ├── landcover.csv
+    ├── parentmaterial.csv
+    ├── terrain.csv
+    └── vegetation.csv
+
+2_timeseries_data
+└── separate directories for each MacroSheds network
+    └── separate directories for each MacroSheds domain within that network
+        ├── documentation (of retrieval locations and code used to retrieve/munge/derive each product)
+        ├── discharge (if available) 
+        ├── precipitation (if available)
+        ├── stream_chemistry (if available)
+        ├── precip chemistry (if available)
+        ├── stream_gauge_locations
+        ├── precip_gauge_locations (if available)
+        └── ws_boundary (except McMurdo)
 
-TIME SERIES
-network-domain
-   should this documentation be moved to the huge docs section?
-these readmes should refer back to the master readmes
+3_CAMELS-compliant_watershed_attributes
+├── clim.csv
+├── geol.csv
+├── soil.csv
+├── topo.csv
+└── vege.csv
 
-CAMELS-STYLE TIME SERIES AND WATERSHED ATTRIBUTES
-camels-style attributes
-camels-style timeseries data
+4_CAMELS-compliant_Daymet_forcings
+└── forcings for each site in MacroSheds
 

src/templates/figshare_docfiles/07a_CAMELS-compliant_datasets_metadata.txt

@@ -0,0 +1,4 @@
+The subset of MacroSheds that relates to streamflow and climate forcings makes it a valuable supplement to existing datasets like CAMELS (671 sites) and GAGES-II  (9067 sites). Using CAMELS methods, we have compiled watershed attributes and Daymet forcings, for each MacroSheds site, that are immediately commensurable with the published CAMELS dataset, enhancing the predictive power of the combined set, especially for small watersheds. Of the 178 sites with discharge data that MacroSheds adds to this corpus (as of version 1), 122 have watershed areas of 10 km2 or less, and 68 have areas of 1 km2 or less. For CAMELS, these numbers are 8 and 0, respectively. For GAGES-II, they are 207 and 2 (see Figure 2 in the MacroSheds data paper, in review at the time of this writing).
+
+Please note that we used gSSURGO (Soil Survey Staff 2022) instead of the superseded STATSGO dataset for soil characteristics. Two other CAMELS watershed attributes, pet_mean and aridity, were also computed differently for MacroSheds watersheds. For these, we solved the Priestly-Taylor formulation by using a gridded 𝛼 product (Aschonitis et al. 2017), rather than calibrating 𝛼 ourselves.
+

src/templates/figshare_docfiles/07b_CAMELS-compliant_ws_attributes_column_descriptions.txt

@@ -0,0 +1,17 @@
+MacroSheds CAMELS-compliant watershed attribute data
+Column descriptions
+
+Except for the deviations below, we direct the user of these data to the variable descriptions in the original CAMELS dataset
+webpage: https://ral.ucar.edu/solutions/products/camels
+CAMELS attributes download URL: https://ral.ucar.edu/sites/default/files/public/product-tool/camels-catchment-attributes-and-meteorology-for-large-sample-studies-dataset-downloads/camels_attributes_v2.0.zip
+
+See the file called camels_attributes_v2.0.xlsx
+
+Deviations from CAMELS:
+We used gSSURGO (Soil Survey Staff 2022) instead of the superseded STATSGO dataset for soil characteristics, namely sand_frac, clay_frac, silt_frac, and organic_frac. We have omitted the following variables that are included with the original CAMELS dataset: soil_depth_pelletier, soil_depth_statsgo, soil_porosity, soil_conductivity, max_water_content, water_frac, other_frac. However, note that equivalents of these variables are included with the core MacroSheds watershed attribute dataset.
+
+Two other CAMELS watershed attributes, pet_mean and aridity, were also computed differently for MacroSheds watersheds. For these, we solved the Priestly-Taylor formulation by using a gridded 𝛼 product (Aschonitis et al. 2017), rather than calibrating 𝛼 ourselves.
+
+References:
+Aschonitis, V. G., Papamichail, D., Demertzi, K., Colombani, N., Mastrocicco, M., Ghirardini, A., Castaldelli, G., & Fano, E.-A. (2017). High resolution global grids of revised Priestley-Taylor and Hargreaves-Samani coefficients for assessing ASCE-standardized reference crop evapotranspiration and solar radiation, links to ESRI-grid files [Data set].
+Soil Survey Staff. (2022). National Value Added Look Up (valu) Table Database for the Gridded Soil Survey Geographic (gSSURGO) Database for the United States of America and the Territories, Commonwealths, and Island Nations served by the USDA-NRCS. United States Department of Agriculture, Natural Resources Conservation Service. https://gdg.sc.egov.usda.gov/

src/templates/figshare_docfiles/07c_CAMELS-compliant_Daymet_forcings_column_descriptions.txt

@@ -0,0 +1,10 @@
+MacroSheds CAMELS-compliant Daymet climate forcings
+
+The Daymet forcings accompanying the core MacroSheds dataset represent watershed averages of published, gridded Daymet products (Thornton et al. 2020). Note that our format is a little different from what CAMELS provides. Our Daymet files are CSVs with a single datetime column and only one header. For site locations and elevations (and lots of other site information) see 04_site_documentation.
+
+We've included timeseries of potential evapotranspiration (pet) in this dataset, though pet is not a Daymet variable per se. The pet product was also computed differently for MacroSheds watersheds than for CAMELS in that we solved the Priestly-Taylor formulation by using a gridded 𝛼 product (Aschonitis et al. 2017), rather than calibrating 𝛼 ourselves.
+
+References:
+Aschonitis, V. G., Papamichail, D., Demertzi, K., Colombani, N., Mastrocicco, M., Ghirardini, A., Castaldelli, G., & Fano, E.-A. (2017). High resolution global grids of revised Priestley-Taylor and Hargreaves-Samani coefficients for assessing ASCE-standardized reference crop evapotranspiration and solar radiation, links to ESRI-grid files [Data set].
+Thornton, M. M., Shrestha, R., Wei, Y., Thornton, P. E., Kao, S., & Wilson, B. E. (2020). DaymetDaymet: Daily Surface Weather Data on a 1-km Grid for North America, Version 4 [NetCDF]. 0 MB. https://doi.org/10.3334/ORNLDAAC/1840
+

src/templates/write_metadata_d_boilerplate.txt

@@ -3,7 +3,8 @@ This MacroSheds data product ({p}) was generated from the following munged Macro
 
 {mp}
 
-Source data were retrieved from the following web page(s), static file(s), or web-API endpoint(s):
+Source data were retrieved from the following web page(s), static file(s), or web-API endpoint(s)
+on the datetime in parentheses:
 
 {ru}