Skip to content

Commit 48af61f

Browse files
westonslaughterwestonslaughter
committed
2 parents e1f0176 + e2604ab commit 48af61f

27 files changed

Lines changed: 1135 additions & 491 deletions

‎src/acquisition_master.R‎

Lines changed: 11 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@ suppressPackageStartupMessages({
3939
library(doParallel) #replaced by doFuture, but still needed on BM1
4040
library(doFuture)
4141
library(googlesheets4)
42+
library(googledrive)
4243
library(rgee) #requires geojsonio package
4344
library(osmdata)
4445

@@ -291,8 +292,13 @@ ms_globals <- c(ls(all.names = TRUE), 'ms_globals')
291292

292293
dir.create('logs', showWarnings = FALSE)
293294

295+
<<<<<<< HEAD
294296
dmnrow = 20
295297
## print(network_domain, n=50)
298+
=======
299+
# dmnrow = 12
300+
# print(network_domain, n=50)
301+
>>>>>>> e2604ab9dbc8d0a005d9012d15cfa35daa6e4c0e
296302
for(dmnrow in 1:nrow(network_domain)){
297303

298304
# drop_automated_entries('.') #use with caution!
@@ -327,14 +333,14 @@ for(dmnrow in 1:nrow(network_domain)){
327333
domain = domain)
328334

329335
get_all_local_helpers(network = network,
330-
domain = domain)
336+
domain = domain)
331337

332338
ms_retrieve(network = network,
333339
# prodname_filter = c('stream_chemistry'),
334340
domain = domain)
335341

336342
ms_munge(network = network,
337-
# prodname_filter = c('stream_chemistry'),
343+
prodname_filter = c('stream_chemistry'),
338344
domain = domain)
339345

340346
if(domain != 'mcmurdo'){
@@ -345,13 +351,13 @@ for(dmnrow in 1:nrow(network_domain)){
345351
}
346352

347353
ms_derive(network = network,
348-
# prodname_filter = c('precip_pchem_pflux'),
354+
prodname_filter = c('precip_pchem_pflux'),
349355
domain = domain)
350-
356+
351357
if(domain != 'mcmurdo'){
352358
ms_general(network = network,
353359
domain = domain,
354-
get_missing_only = F)
360+
get_missing_only = TRUE)
355361
}
356362

357363
retain_ms_globals(ms_globals)

‎src/czo/network_helpers.R‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -115,9 +115,9 @@ populate_set_details <- function(tracker, prodname_ms, site_code, avail){
115115
s = sites,
116116
c = components)
117117

118-
email_err(msgs = update_msg,
119-
addrs = conf$report_emails,
120-
pw = conf$gmail_pw)
118+
#email_err(msgs = update_msg,
119+
# addrs = conf$report_emails,
120+
# pw = conf$gmail_pw)
121121
}
122122
}
123123

‎src/dev/compare_points_to_nhd.R‎

Lines changed: 60 additions & 69 deletions
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,7 @@ buf <- function(site, buf_dist){
4646
site_csv <- suppressMessages(googlesheets4::read_sheet(
4747
'https://docs.google.com/spreadsheets/d/1Xd38tvB0upHjDRDa5qalGN2Ors6HHWZKpT4Bw0dDqU4/edit?usp=drive_web&ouid=111793152718629438266',
4848
na = c('', 'NA'),
49-
col_types = 'ccccccccnnnnnccc'
49+
col_types = 'ccccccccnnnnncccc'
5050
))
5151

5252
sites <- filter(site_csv,
@@ -56,66 +56,65 @@ sites <- filter(site_csv,
5656
sites$NHD_COMID <- '?'
5757
manual_input <- 1:233
5858
total_len <- nrow(sites)
59-
sites$NHD_COMID[manual_input] <- c('HR only', 'too small', 'HR only', '6729679',
60-
'HR only', 'HR only', '6729787', 'HR only',
61-
'too small', '23773411', 'HR only', 'HR only',
62-
'HR only', 'HR only', 'HR only', 'HR only',
63-
'HR only', 'HR only', '23774053', '3424530',
64-
'18548462', '18211220', '1239639', '3649284',
65-
'8444872', '698676', '22050327', '7690025',#28
66-
'24505800', '22048111', 'HR only', '20440650',
67-
'2889452', '2964310', '1306285', '23773423',#COMO, BLDE, PRIN, MCRA,
68-
'nonCONUS', 'nonCONUS', '11722717', 'nonCONUS',#first two AK, last PR: '800035089'
69-
'nonCONUS', 'HR only', '18841314', '18208464', #first PR 800026322
70-
'HR only', '22144520', '18841358', 'HR only',#48
71-
'18841356', 'HR only', '11689212', '11688596',#... GFCP, GFGB
72-
'11688596', '11689106', 'too small', '11689186',#GFVN on same reach as GFGB
73-
'11688418', 'too small', '11689206', '2889280',
74-
'2889770', '2889186', 'too small', 'HR only',#GREEN4...
75-
'too small', '17595459', '17595361', '17595359',#68
76-
'17595433', '17594763', '17594741', '17595453', #same as next
77-
'17595453', '17594769', '17594785', '17595305',
78-
'17596097', '17595477', '17595369', '17595473',
79-
'17596159', '17595473', '17596161', 'nonCONUS', #bonanza, then mcmurdo
80-
'nonCONUS', 'nonCONUS', 'nonCONUS', 'nonCONUS',
81-
'nonCONUS', 'nonCONUS', 'nonCONUS', 'nonCONUS',
82-
'nonCONUS', 'nonCONUS', 'nonCONUS', 'nonCONUS',
83-
'nonCONUS', 'nonCONUS', 'nonCONUS', 'nonCONUS',
84-
'nonCONUS', 'nonCONUS', 'nonCONUS', 'nonCONUS',
85-
'nonCONUS', 'nonCONUS', '5860599', '5862611', #plum
86-
'5862611', '5862581', 'nonCONUS', 'nonCONUS',#112
87-
'nonCONUS', 'nonCONUS', 'nonCONUS', 'nonCONUS', #arctic
88-
'nonCONUS', 'nonCONUS', 'nonCONUS', 'nonCONUS',
89-
'nonCONUS', 'nonCONUS', 'nonCONUS', 'nonCONUS',
90-
'nonCONUS', 'nonCONUS', 'nonCONUS', 'nonCONUS',
91-
'nonCONUS', 'nonCONUS', 'nonCONUS', 'nonCONUS',
92-
'nonCONUS', 'nonCONUS', 'nonCONUS', 'nonCONUS',
93-
'nonCONUS', 'nonCONUS', 'nonCONUS', 'nonCONUS',#140
94-
'nonCONUS', 'nonCONUS', '2889384', '2889360', #boulder
95-
'2889410', '2891254', 'too small', '17827556',
96-
'17827556', '17826162', '17826162', '17827558',
97-
'17826228', 'HR only', 'HR only', 'too small',
98-
'HR only', '17826228', 'HR only', 'HR only',#160
99-
'HR only', 'HR only', 'nonCONUS', 'nonCONUS', #luquillo
100-
'nonCONUS', 'nonCONUS', 'nonCONUS', 'nonCONUS',
101-
'nonCONUS', 'HR only', 'too small', 'too small',
102-
'3775221', 'too small', 'too small', 'too small',
103-
'HR only', 'too small', 'HR only', '1332754',#180
104-
'1332754', '1332672', '1332674', '1332674',
105-
'HR only', 'HR only', '1332198', '1332190',
106-
'1332186', '1332186', 'HR only', '1332204',
107-
'1332224', '13633173', 'HR only', 'HR only',
108-
'HR only', 'HR only', 'HR only', 'HR only',#200
109-
'HR only', '22050299', '22050299', '22050323', #krew. some questionable point-segment associations in here
110-
'23903201', 'HR only', 'HR only', 'HR only',
111-
'9643235', '9643251', '9643235', 'HR only',
112-
'nonCONUS', 'nonCONUS', 'nonCONUS', 'nonCONUS', #krycklan
113-
'nonCONUS', 'nonCONUS', 'nonCONUS', 'nonCONUS',
114-
'nonCONUS', 'nonCONUS', 'nonCONUS', 'nonCONUS',#224
115-
'nonCONUS', '22125024', 'HR only', '4681928',
116-
'4682266', '4682628', 'too small', '2679458',
117-
'2679458')
118-
# for(i in seq_len(nrow(sites))){
59+
# sites$NHD_COMID[manual_input] <- c('HR only', 'too small', 'HR only', '6729679',
60+
# 'HR only', 'HR only', '6729787', 'HR only',
61+
# 'too small', '23773411', 'HR only', 'HR only',
62+
# 'HR only', 'HR only', 'HR only', 'HR only',
63+
# 'HR only', 'HR only', '23774053', '3424530',
64+
# '18548462', '18211220', '1239639', '3649284',
65+
# '8444872', '698676', '22050327', '7690025',#28
66+
# '24505800', '22048111', 'HR only', '20440650',
67+
# '2889452', '2964310', '1306285', '23773423',#COMO, BLDE, PRIN, MCRA,
68+
# 'nonCONUS', 'nonCONUS', '11722717', 'nonCONUS',#first two AK, last PR: '800035089'
69+
# 'nonCONUS', 'HR only', '18841314', '18208464', #first PR 800026322
70+
# 'HR only', '22144520', '18841358', 'HR only',#48
71+
# '18841356', 'HR only', '11689212', '11688596',#... GFCP, GFGB
72+
# '11688596', '11689106', 'too small', '11689186',#GFVN on same reach as GFGB
73+
# '11688418', 'too small', '11689206', '2889280',
74+
# '2889770', '2889186', 'too small', 'HR only',#GREEN4...
75+
# 'too small', '17595459', '17595361', '17595359',#68
76+
# '17595433', '17594763', '17594741', '17595453', #same as next
77+
# '17595453', '17594769', '17594785', '17595305',
78+
# '17596097', '17595477', '17595369', '17595473',
79+
# '17596159', '17595473', '17596161', 'nonCONUS', #bonanza, then mcmurdo
80+
# 'nonCONUS', 'nonCONUS', 'nonCONUS', 'nonCONUS',
81+
# 'nonCONUS', 'nonCONUS', 'nonCONUS', 'nonCONUS',
82+
# 'nonCONUS', 'nonCONUS', 'nonCONUS', 'nonCONUS',
83+
# 'nonCONUS', 'nonCONUS', 'nonCONUS', 'nonCONUS',
84+
# 'nonCONUS', 'nonCONUS', 'nonCONUS', 'nonCONUS',
85+
# 'nonCONUS', 'nonCONUS', '5860599', '5862611', #plum
86+
# '5862611', '5862581', 'nonCONUS', 'nonCONUS',#112
87+
# 'nonCONUS', 'nonCONUS', 'nonCONUS', 'nonCONUS', #arctic
88+
# 'nonCONUS', 'nonCONUS', 'nonCONUS', 'nonCONUS',
89+
# 'nonCONUS', 'nonCONUS', 'nonCONUS', 'nonCONUS',
90+
# 'nonCONUS', 'nonCONUS', 'nonCONUS', 'nonCONUS',
91+
# 'nonCONUS', 'nonCONUS', 'nonCONUS', 'nonCONUS',
92+
# 'nonCONUS', 'nonCONUS', 'nonCONUS', 'nonCONUS',
93+
# 'nonCONUS', 'nonCONUS', 'nonCONUS', 'nonCONUS',#140
94+
# 'nonCONUS', 'nonCONUS', '2889384', '2889360', #boulder
95+
# '2889410', '2891254', 'too small', '17827556',
96+
# '17827556', '17826162', '17826162', '17827558',
97+
# '17826228', 'HR only', 'HR only', 'too small',
98+
# 'HR only', '17826228', 'HR only', 'HR only',#160
99+
# 'HR only', 'HR only', 'nonCONUS', 'nonCONUS', #luquillo
100+
# 'nonCONUS', 'nonCONUS', 'nonCONUS', 'nonCONUS',
101+
# 'nonCONUS', 'HR only', 'too small', 'too small',
102+
# '3775221', 'too small', 'too small', 'too small',
103+
# 'HR only', 'too small', 'HR only', '1332754',#180
104+
# '1332754', '1332672', '1332674', '1332674',
105+
# 'HR only', 'HR only', '1332198', '1332190',
106+
# '1332186', '1332186', 'HR only', '1332204',
107+
# '1332224', '13633173', 'HR only', 'HR only',
108+
# 'HR only', 'HR only', 'HR only', 'HR only',#200
109+
# 'HR only', '22050299', '22050299', '22050323', #krew. some questionable point-segment associations in here
110+
# '23903201', 'HR only', 'HR only', 'HR only',
111+
# '9643235', '9643251', '9643235', 'HR only',
112+
# 'nonCONUS', 'nonCONUS', 'nonCONUS', 'nonCONUS', #krycklan
113+
# 'nonCONUS', 'nonCONUS', 'nonCONUS', 'nonCONUS',
114+
# 'nonCONUS', 'nonCONUS', 'nonCONUS', 'nonCONUS',#224
115+
# 'nonCONUS', '22125024', 'HR only', '4681928',
116+
# '4682266', '4682628', 'too small', '2679458',
117+
# '2679458')
119118

120119
# loop 1: NHDPlusV2 or NHD-HR (kinda obsolete) ####
121120

@@ -202,14 +201,6 @@ for(i in 1:total_len){
202201
print(paste('map saved to', mapview_save_path))
203202
print(xx)
204203

205-
# gg <- ggplot() +
206-
# geom_sf(data = NHD_HR, color = 'darkslategray3') +
207-
# geom_sf(data = NHDPlus, color = 'deepskyblue4') +
208-
# geom_sf(data = site, color = 'red') +
209-
# coord_sf()
210-
#
211-
# print(gg)
212-
213204
system('spd-say "chili chili chili"')
214205
x <- readline(cat('This point is on: [A] an NHDPlus flowline, [B] an NHD_HR flowline, or [C] neither >\n'))
215206

‎src/dev/dev_helpers.R‎

Lines changed: 44 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1291,6 +1291,11 @@ correct_all_geometries <- function(path, dir_pattern = 'ws_boundary'){
12911291
}
12921292
}
12931293

1294+
# correct_all_geometries(path = '~/git/macrosheds/data_acquisition/data')
1295+
# correct_all_geometries(path = '~/git/macrosheds/portal/data')
1296+
# correct_all_geometries(path = '~/git/macrosheds/data_acquisition/macrosheds_dataset_v1')
1297+
# correct_all_geometries(path = '~/git/macrosheds/data_acquisition/macrosheds_figshare_v1')
1298+
12941299
rebuild_portal_data_before_postprocessing <- function(network_domain, backup = TRUE){
12951300

12961301
dir_wrapper <- function(path, keyword){
@@ -1393,7 +1398,42 @@ rebuild_portal_data_before_postprocessing <- function(network_domain, backup = T
13931398
'sure it looks good before and after postprocessing. especially portal/data/general'))
13941399
}
13951400

1396-
# correct_all_geometries(path = '~/git/macrosheds/data_acquisition/data')
1397-
# correct_all_geometries(path = '~/git/macrosheds/portal/data')
1398-
# correct_all_geometries(path = '~/git/macrosheds/data_acquisition/macrosheds_dataset_v1')
1399-
# correct_all_geometries(path = '~/git/macrosheds/data_acquisition/macrosheds_figshare_v1')
1401+
insert_retrieval_datetimes <- function(){
1402+
1403+
#this inserts the last modification datetime of each raw documentation file as
1404+
#the presumed retrieval datetime for the corresponding raw data product.
1405+
1406+
#used in a pinch. hopefully a real recording method has been implemented by now.
1407+
1408+
fs <- list.files('data',
1409+
recursive = TRUE, full.names = TRUE)
1410+
fs <- fs[grepl('/raw/', fs)]
1411+
fs <- fs[grepl('/documentation/', fs)]
1412+
1413+
for(f in fs){
1414+
1415+
rt <- read_lines(f)
1416+
if(length(rt) != 1) stop('sup with this')
1417+
if(grepl('UTC\\)$', rt)) next
1418+
rt <- glue(
1419+
rt, ' (',
1420+
as.character(lubridate::with_tz(file.info(f)$mtime, 'UTC')),
1421+
' UTC)')
1422+
1423+
write_lines(rt, f)
1424+
}
1425+
}
1426+
1427+
get_nonnumerics <- function(d){
1428+
1429+
#gets unique nonnumeric values by row. useful for identifying quality codes
1430+
#within data columns
1431+
1432+
nonnumerics = apply(d, 2, function(x){
1433+
xx = as.numeric(x)
1434+
nonnumerics = is.na(xx)
1435+
out = unique(x[nonnumerics])
1436+
})
1437+
1438+
return(nonnumerics)
1439+
}

0 commit comments

Comments
 (0)