Skip to content

Commit daaa4d4

Browse files
vlahmvlahm
committed
precaution to prevent data loss in unforeseen scenarios (ws_delin)
1 parent e200083 commit daaa4d4

5 files changed

Lines changed: 113 additions & 102 deletions

File tree

‎src/acquisition_master.R‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -264,7 +264,7 @@ ms_globals <- c(ls(all.names = TRUE), 'ms_globals')
264264

265265
dir.create('logs', showWarnings = FALSE)
266266

267-
# dmnrow = 1
267+
# dmnrow = 20
268268
# print(network_domain, n=50)
269269
for(dmnrow in 1:nrow(network_domain)){
270270

@@ -303,7 +303,7 @@ for(dmnrow in 1:nrow(network_domain)){
303303
# prodname_filter = c('stream_chemistry'),
304304
domain = domain)
305305
ms_munge(network = network,
306-
# prodname_filter = c('stream_chemistry'),
306+
# prodname_filter = c('discharge'),
307307
domain = domain)
308308
if(domain != 'mcmurdo'){
309309
sw(ms_delineate(network = network,
@@ -312,9 +312,9 @@ for(dmnrow in 1:nrow(network_domain)){
312312
verbose = TRUE))
313313
}
314314
ms_derive(network = network,
315-
# prodname_filter = c('precip_pchem_pflux'),
315+
prodname_filter = c('discharge'),
316316
domain = domain)
317-
317+
318318
if(domain != 'mcmurdo'){
319319
ms_general(network = network,
320320
domain = domain,

‎src/dev/compare_points_to_nhd.R‎

Lines changed: 60 additions & 69 deletions
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,7 @@ buf <- function(site, buf_dist){
4646
site_csv <- suppressMessages(googlesheets4::read_sheet(
4747
'https://docs.google.com/spreadsheets/d/1Xd38tvB0upHjDRDa5qalGN2Ors6HHWZKpT4Bw0dDqU4/edit?usp=drive_web&ouid=111793152718629438266',
4848
na = c('', 'NA'),
49-
col_types = 'ccccccccnnnnnccc'
49+
col_types = 'ccccccccnnnnncccc'
5050
))
5151

5252
sites <- filter(site_csv,
@@ -56,66 +56,65 @@ sites <- filter(site_csv,
5656
sites$NHD_COMID <- '?'
5757
manual_input <- 1:233
5858
total_len <- nrow(sites)
59-
sites$NHD_COMID[manual_input] <- c('HR only', 'too small', 'HR only', '6729679',
60-
'HR only', 'HR only', '6729787', 'HR only',
61-
'too small', '23773411', 'HR only', 'HR only',
62-
'HR only', 'HR only', 'HR only', 'HR only',
63-
'HR only', 'HR only', '23774053', '3424530',
64-
'18548462', '18211220', '1239639', '3649284',
65-
'8444872', '698676', '22050327', '7690025',#28
66-
'24505800', '22048111', 'HR only', '20440650',
67-
'2889452', '2964310', '1306285', '23773423',#COMO, BLDE, PRIN, MCRA,
68-
'nonCONUS', 'nonCONUS', '11722717', 'nonCONUS',#first two AK, last PR: '800035089'
69-
'nonCONUS', 'HR only', '18841314', '18208464', #first PR 800026322
70-
'HR only', '22144520', '18841358', 'HR only',#48
71-
'18841356', 'HR only', '11689212', '11688596',#... GFCP, GFGB
72-
'11688596', '11689106', 'too small', '11689186',#GFVN on same reach as GFGB
73-
'11688418', 'too small', '11689206', '2889280',
74-
'2889770', '2889186', 'too small', 'HR only',#GREEN4...
75-
'too small', '17595459', '17595361', '17595359',#68
76-
'17595433', '17594763', '17594741', '17595453', #same as next
77-
'17595453', '17594769', '17594785', '17595305',
78-
'17596097', '17595477', '17595369', '17595473',
79-
'17596159', '17595473', '17596161', 'nonCONUS', #bonanza, then mcmurdo
80-
'nonCONUS', 'nonCONUS', 'nonCONUS', 'nonCONUS',
81-
'nonCONUS', 'nonCONUS', 'nonCONUS', 'nonCONUS',
82-
'nonCONUS', 'nonCONUS', 'nonCONUS', 'nonCONUS',
83-
'nonCONUS', 'nonCONUS', 'nonCONUS', 'nonCONUS',
84-
'nonCONUS', 'nonCONUS', 'nonCONUS', 'nonCONUS',
85-
'nonCONUS', 'nonCONUS', '5860599', '5862611', #plum
86-
'5862611', '5862581', 'nonCONUS', 'nonCONUS',#112
87-
'nonCONUS', 'nonCONUS', 'nonCONUS', 'nonCONUS', #arctic
88-
'nonCONUS', 'nonCONUS', 'nonCONUS', 'nonCONUS',
89-
'nonCONUS', 'nonCONUS', 'nonCONUS', 'nonCONUS',
90-
'nonCONUS', 'nonCONUS', 'nonCONUS', 'nonCONUS',
91-
'nonCONUS', 'nonCONUS', 'nonCONUS', 'nonCONUS',
92-
'nonCONUS', 'nonCONUS', 'nonCONUS', 'nonCONUS',
93-
'nonCONUS', 'nonCONUS', 'nonCONUS', 'nonCONUS',#140
94-
'nonCONUS', 'nonCONUS', '2889384', '2889360', #boulder
95-
'2889410', '2891254', 'too small', '17827556',
96-
'17827556', '17826162', '17826162', '17827558',
97-
'17826228', 'HR only', 'HR only', 'too small',
98-
'HR only', '17826228', 'HR only', 'HR only',#160
99-
'HR only', 'HR only', 'nonCONUS', 'nonCONUS', #luquillo
100-
'nonCONUS', 'nonCONUS', 'nonCONUS', 'nonCONUS',
101-
'nonCONUS', 'HR only', 'too small', 'too small',
102-
'3775221', 'too small', 'too small', 'too small',
103-
'HR only', 'too small', 'HR only', '1332754',#180
104-
'1332754', '1332672', '1332674', '1332674',
105-
'HR only', 'HR only', '1332198', '1332190',
106-
'1332186', '1332186', 'HR only', '1332204',
107-
'1332224', '13633173', 'HR only', 'HR only',
108-
'HR only', 'HR only', 'HR only', 'HR only',#200
109-
'HR only', '22050299', '22050299', '22050323', #krew. some questionable point-segment associations in here
110-
'23903201', 'HR only', 'HR only', 'HR only',
111-
'9643235', '9643251', '9643235', 'HR only',
112-
'nonCONUS', 'nonCONUS', 'nonCONUS', 'nonCONUS', #krycklan
113-
'nonCONUS', 'nonCONUS', 'nonCONUS', 'nonCONUS',
114-
'nonCONUS', 'nonCONUS', 'nonCONUS', 'nonCONUS',#224
115-
'nonCONUS', '22125024', 'HR only', '4681928',
116-
'4682266', '4682628', 'too small', '2679458',
117-
'2679458')
118-
# for(i in seq_len(nrow(sites))){
59+
# sites$NHD_COMID[manual_input] <- c('HR only', 'too small', 'HR only', '6729679',
60+
# 'HR only', 'HR only', '6729787', 'HR only',
61+
# 'too small', '23773411', 'HR only', 'HR only',
62+
# 'HR only', 'HR only', 'HR only', 'HR only',
63+
# 'HR only', 'HR only', '23774053', '3424530',
64+
# '18548462', '18211220', '1239639', '3649284',
65+
# '8444872', '698676', '22050327', '7690025',#28
66+
# '24505800', '22048111', 'HR only', '20440650',
67+
# '2889452', '2964310', '1306285', '23773423',#COMO, BLDE, PRIN, MCRA,
68+
# 'nonCONUS', 'nonCONUS', '11722717', 'nonCONUS',#first two AK, last PR: '800035089'
69+
# 'nonCONUS', 'HR only', '18841314', '18208464', #first PR 800026322
70+
# 'HR only', '22144520', '18841358', 'HR only',#48
71+
# '18841356', 'HR only', '11689212', '11688596',#... GFCP, GFGB
72+
# '11688596', '11689106', 'too small', '11689186',#GFVN on same reach as GFGB
73+
# '11688418', 'too small', '11689206', '2889280',
74+
# '2889770', '2889186', 'too small', 'HR only',#GREEN4...
75+
# 'too small', '17595459', '17595361', '17595359',#68
76+
# '17595433', '17594763', '17594741', '17595453', #same as next
77+
# '17595453', '17594769', '17594785', '17595305',
78+
# '17596097', '17595477', '17595369', '17595473',
79+
# '17596159', '17595473', '17596161', 'nonCONUS', #bonanza, then mcmurdo
80+
# 'nonCONUS', 'nonCONUS', 'nonCONUS', 'nonCONUS',
81+
# 'nonCONUS', 'nonCONUS', 'nonCONUS', 'nonCONUS',
82+
# 'nonCONUS', 'nonCONUS', 'nonCONUS', 'nonCONUS',
83+
# 'nonCONUS', 'nonCONUS', 'nonCONUS', 'nonCONUS',
84+
# 'nonCONUS', 'nonCONUS', 'nonCONUS', 'nonCONUS',
85+
# 'nonCONUS', 'nonCONUS', '5860599', '5862611', #plum
86+
# '5862611', '5862581', 'nonCONUS', 'nonCONUS',#112
87+
# 'nonCONUS', 'nonCONUS', 'nonCONUS', 'nonCONUS', #arctic
88+
# 'nonCONUS', 'nonCONUS', 'nonCONUS', 'nonCONUS',
89+
# 'nonCONUS', 'nonCONUS', 'nonCONUS', 'nonCONUS',
90+
# 'nonCONUS', 'nonCONUS', 'nonCONUS', 'nonCONUS',
91+
# 'nonCONUS', 'nonCONUS', 'nonCONUS', 'nonCONUS',
92+
# 'nonCONUS', 'nonCONUS', 'nonCONUS', 'nonCONUS',
93+
# 'nonCONUS', 'nonCONUS', 'nonCONUS', 'nonCONUS',#140
94+
# 'nonCONUS', 'nonCONUS', '2889384', '2889360', #boulder
95+
# '2889410', '2891254', 'too small', '17827556',
96+
# '17827556', '17826162', '17826162', '17827558',
97+
# '17826228', 'HR only', 'HR only', 'too small',
98+
# 'HR only', '17826228', 'HR only', 'HR only',#160
99+
# 'HR only', 'HR only', 'nonCONUS', 'nonCONUS', #luquillo
100+
# 'nonCONUS', 'nonCONUS', 'nonCONUS', 'nonCONUS',
101+
# 'nonCONUS', 'HR only', 'too small', 'too small',
102+
# '3775221', 'too small', 'too small', 'too small',
103+
# 'HR only', 'too small', 'HR only', '1332754',#180
104+
# '1332754', '1332672', '1332674', '1332674',
105+
# 'HR only', 'HR only', '1332198', '1332190',
106+
# '1332186', '1332186', 'HR only', '1332204',
107+
# '1332224', '13633173', 'HR only', 'HR only',
108+
# 'HR only', 'HR only', 'HR only', 'HR only',#200
109+
# 'HR only', '22050299', '22050299', '22050323', #krew. some questionable point-segment associations in here
110+
# '23903201', 'HR only', 'HR only', 'HR only',
111+
# '9643235', '9643251', '9643235', 'HR only',
112+
# 'nonCONUS', 'nonCONUS', 'nonCONUS', 'nonCONUS', #krycklan
113+
# 'nonCONUS', 'nonCONUS', 'nonCONUS', 'nonCONUS',
114+
# 'nonCONUS', 'nonCONUS', 'nonCONUS', 'nonCONUS',#224
115+
# 'nonCONUS', '22125024', 'HR only', '4681928',
116+
# '4682266', '4682628', 'too small', '2679458',
117+
# '2679458')
119118

120119
# loop 1: NHDPlusV2 or NHD-HR (kinda obsolete) ####
121120

@@ -202,14 +201,6 @@ for(i in 1:total_len){
202201
print(paste('map saved to', mapview_save_path))
203202
print(xx)
204203

205-
# gg <- ggplot() +
206-
# geom_sf(data = NHD_HR, color = 'darkslategray3') +
207-
# geom_sf(data = NHDPlus, color = 'deepskyblue4') +
208-
# geom_sf(data = site, color = 'red') +
209-
# coord_sf()
210-
#
211-
# print(gg)
212-
213204
system('spd-say "chili chili chili"')
214205
x <- readline(cat('This point is on: [A] an NHDPlus flowline, [B] an NHD_HR flowline, or [C] neither >\n'))
215206

‎src/dev/prepare_ws_attrs_for_domain.R‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
library(tidyverse)
2+
library(feather)
3+
4+
setwd('/home/mike/git/macrosheds/data_acquisition/macrosheds_figshare_v1/1_watershed_attribute_data')
5+
6+
d = map_dfr(list.files('ws_attr_timeseries', full.names = TRUE), read_csv)
7+
filter(d, domain == 'hbef') %>%
8+
write_feather('~/Desktop/hbef_ws_attr_timeseries.feather')
9+
10+
d = map_dfr(list.files('../4_CAMELS-compliant_Daymet_forcings/', full.names = TRUE), read_csv)
11+
filter(d, site_code %in% paste0('w', 1:9)) %>%
12+
write_feather('~/Desktop/hbef_ws_attr_timeseries_daymet.feather')
13+
14+
d = map_dfr(list.files('../3_CAMELS-compliant_watershed_attributes/', full.names = TRUE), read_csv)
15+
filter(d, site_code %in% paste0('w', 1:9)) %>%
16+
select(site_code, everything()) %>%
17+
write_feather('~/Desktop/hbef_ws_attr_camels.csv')
18+
19+
d = read_csv('../1_watershed_attribute_data/ws_attr_summaries.csv')
20+
filter(d, domain == 'hbef') %>%
21+
write_feather('~/Desktop/hbef_ws_attr.csv')
22+

‎src/global/global_helpers.R‎

Lines changed: 5 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -3252,16 +3252,14 @@ delineate_watershed_apriori_recurse <- function(lat,
32523252
allow_alphanumeric_response = FALSE)
32533253

32543254
if('n' %in% resp){
3255-
unlink(write_dir,
3256-
recursive = TRUE)
3255+
unlink(write_dir)
32573256
print(glue('Moving on. You haven\'t seen the last of {s}!',
32583257
s = site_code))
32593258
return(1)
32603259
}
32613260

32623261
if('a' %in% resp){
3263-
unlink(write_dir,
3264-
recursive = TRUE)
3262+
unlink(write_dir)
32653263
print(glue('Aborted. Any completed delineations have been saved.'))
32663264
return(2)
32673265
}
@@ -7611,24 +7609,10 @@ write_metadata_r <- function(murl, network, domain, prodname_ms){
76117609
rd = raw_dir,
76127610
p = prodname_ms)
76137611

7614-
readr::write_file(murl,
7612+
readr::write_file(paste0(murl, ' (retrieved ',
7613+
lubridate::with_tz(Sys.time(), 'UTC'),
7614+
')'),
76157615
file = data_acq_file)
7616-
7617-
# #create portal directory if necessary
7618-
# portal_dir <- glue('../portal/data/{d}/{p}', #portal ignores network
7619-
# d = domain,
7620-
# p = strsplit(prodname_ms, '__')[[1]][1])
7621-
# dir.create(portal_dir,
7622-
# showWarnings = FALSE,
7623-
# recursive = TRUE)
7624-
#
7625-
# #hardlink file
7626-
# portal_file <- glue(portal_dir, '/raw_data_documentation_url.txt')
7627-
# unlink(portal_file)
7628-
# invisible(sw(file.link(to = portal_file,
7629-
# from = data_acq_file)))
7630-
7631-
#return()
76327616
}
76337617

76347618
read_metadata_r <- function(network, domain, prodname_ms){

‎src/neon/neon/processing_kernels.R‎

Lines changed: 22 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -848,19 +848,33 @@ process_1_DP4.00130 <-function(network, domain, prodname_ms, site_code,
848848

849849
rawfiles <- list.files(rawdir)
850850

851-
relevant_file <- 'csd_continuousDischarge.feather'
851+
if(site_code == 'TOMB'){
852+
relevant_file <- 'csd_continuousDischargeUSGS.feather'
853+
} else {
854+
relevant_file <- 'csd_continuousDischarge.feather'
855+
}
852856

853857
if(relevant_file %in% rawfiles) {
854858

855859
rawd <- read_feather(glue(rawdir, '/', relevant_file))
856860

857-
out_sub <- rawd %>%
858-
mutate(dischargeFinalQFSciRvw = ifelse(is.na(dischargeFinalQFSciRvw), 0, dischargeFinalQFSciRvw),
859-
dischargeFinalQF = ifelse(is.na(dischargeFinalQF), 0, dischargeFinalQF)) %>%
860-
mutate(ms_status = ifelse(dischargeFinalQF == 1 | dischargeFinalQFSciRvw == 1,
861-
1, 0)) %>%
862-
mutate(var = 'discharge') %>%
863-
select(site_code = siteID, datetime = endDate, var, val = maxpostDischarge, ms_status)
861+
if(site_code == 'TOMB'){
862+
863+
out_sub <- rawd %>%
864+
mutate(ms_status = ifelse(is.na(dischargeFinalQFSciRvw), 0, dischargeFinalQFSciRvw)) %>%
865+
mutate(var = 'discharge') %>%
866+
select(site_code = siteID, datetime = endDate, var, val = usgsDischarge, ms_status)
867+
868+
} else {
869+
870+
out_sub <- rawd %>%
871+
mutate(dischargeFinalQFSciRvw = ifelse(is.na(dischargeFinalQFSciRvw), 0, dischargeFinalQFSciRvw),
872+
dischargeFinalQF = ifelse(is.na(dischargeFinalQF), 0, dischargeFinalQF)) %>%
873+
mutate(ms_status = ifelse(dischargeFinalQF == 1 | dischargeFinalQFSciRvw == 1,
874+
1, 0)) %>%
875+
mutate(var = 'discharge') %>%
876+
select(site_code = siteID, datetime = endDate, var, val = maxpostDischarge, ms_status)
877+
}
864878

865879
} else {
866880
return(generate_ms_exception('Missing discharge files'))

0 commit comments

Comments
 (0)