Confluence integration

BUSBOI/.ipynb_checkpoints/config-checkpoint.R

@@ -0,0 +1,160 @@
+#' BUSBOI Configuration File
+#' 
+#' This file contains all configurable parameters for BUSBOI runs.
+#' Modify these values to change BUSBOI behavior without editing source code.
+
+# ==============================================================================
+# BUSBOI ALGORITHM PARAMETERS
+# ==============================================================================
+# Check for required packages
+required_packages <- c("dplyr", "tidyr", "RNetCDF", "optimx", "ggplot2", "deSolve", "hydroGOF", "jsonlite", "optparse")
+
+for(pkg in required_packages) {
+    if(!require(pkg, character.only = TRUE, quietly = TRUE)) {
+        stop(paste0("Required package '", pkg, "' is not installed. Please install it with: install.packages('", pkg, "')"))
+    }
+}
+# Q Prior Type
+# Options: 'daily' or 'monthly'
+# 'daily' - Use daily machine learning discharge priors from LSTM ensemble
+# 'monthly' - Use monthly static discharge priors from SOS
+Q_PRIOR <- 'monthly'
+
+# TULIP (Two-stage Uncertainty-aware Low-flow Inference Process)
+# Options: 'ON' or 'OFF'
+# 'ON' - Use TULIP for improved low-flow estimation
+# 'OFF' - Standard BUSBOI processing
+TULIP <- 'OFF'
+
+# Gradually Varied Flow (GVF) correction
+# Options: 0 or 1
+# 0 - GVF correction disabled
+# 1 - GVF correction enabled (improves hydraulic geometry)
+GVF_ON <- 0
+
+# Bed elevation estimation mode
+# Options: 0, 1, or 2
+# 0 - 5-point bed estimation (full flexibility, one bed per node)
+# 1 - 1-point bed estimation (single bed value for entire reach)
+# 2 - Fixed bed (use prior bed values, no optimization)
+FIX_BED <- 0
+
+# ==============================================================================
+# INPUT/OUTPUT DIRECTORIES
+# ==============================================================================
+
+# Input directory (Confluence standard)
+IN_DIR <- "/mnt/data/input"
+
+# Output directory (Confluence standard)
+OUT_DIR <- "/mnt/data/output"
+
+# ==============================================================================
+# DATA FILE PATTERNS
+# ==============================================================================
+
+# SWOT data file pattern
+SWOT_FILE_PATTERN <- "_SWOT.nc"
+
+# SOS prior file patterns by continent
+SOS_FILES <- c(
+  'eu' = 'eu_sword_v17b_SOS_priors.nc',
+  'na' = 'na_sword_v17b_SOS_priors.nc',
+  'sa' = 'sa_sword_v17b_SOS_priors.nc',
+  'oc' = 'oc_sword_v17b_SOS_priors.nc',
+  'af' = 'af_sword_v17b_SOS_priors.nc',
+  'as' = 'as_sword_v17b_SOS_priors.nc'
+)
+
+# SWORD file patterns by continent
+SWORD_FILES <- c(
+  'eu' = 'eu_sword_v17b.nc',
+  'na' = 'na_sword_v17b.nc',
+  'sa' = 'sa_sword_v17b.nc',
+  'oc' = 'oc_sword_v17b.nc',
+  'af' = 'af_sword_v17b.nc',
+  'as' = 'as_sword_v17b.nc'
+)
+
+# Continent code to continent name mapping
+CONTINENT_MAP <- list(
+  '1' = 'af',  # Africa
+  '2' = 'eu',  # Europe
+  '3' = 'as',  # Asia (Siberia)
+  '4' = 'as',  # Asia
+  '5' = 'oc',  # Oceania
+  '6' = 'sa',  # South America
+  '7' = 'na',  # North America (North)
+  '8' = 'na',  # North America (South)
+  '9' = 'na'   # North America (Arctic)
+)
+
+# ==============================================================================
+# NETCDF OUTPUT SETTINGS
+# ==============================================================================
+
+# Fill value for invalid/missing data in NetCDF files
+NC_FILL_VALUE <- -999999999999
+
+# Output file suffix
+OUTPUT_FILE_SUFFIX <- "_busboi.nc"
+
+# ==============================================================================
+# ALGORITHM-SPECIFIC SETTINGS
+# ==============================================================================
+
+# Q prior adjustment factors (for daily prior mode)
+Q_PRIOR_MIN_FACTOR <- 0.6  # Minimum Q = min(ML_Q) * this factor
+Q_PRIOR_MAX_FACTOR <- 1.4  # Maximum Q = max(ML_Q) * this factor
+
+# ==============================================================================
+# LOGGING AND DIAGNOSTICS
+# ==============================================================================
+
+# Verbose output
+VERBOSE <- TRUE
+
+# Save intermediate results (RDS files for debugging)
+SAVE_RDS <- TRUE
+
+# ==============================================================================
+# HELPER FUNCTIONS
+# ==============================================================================
+
+#' Get configuration as a named list
+#' @return list of all configuration parameters
+get_config <- function() {
+  return(list(
+    Q_PRIOR = Q_PRIOR,
+    TULIP = TULIP,
+    GVF_ON = GVF_ON,
+    FIX_BED = FIX_BED,
+    IN_DIR = IN_DIR,
+    OUT_DIR = OUT_DIR,
+    SWOT_FILE_PATTERN = SWOT_FILE_PATTERN,
+    SOS_FILES = SOS_FILES,
+    SWORD_FILES = SWORD_FILES,
+    CONTINENT_MAP = CONTINENT_MAP,
+    NC_FILL_VALUE = NC_FILL_VALUE,
+    OUTPUT_FILE_SUFFIX = OUTPUT_FILE_SUFFIX,
+    Q_PRIOR_MIN_FACTOR = Q_PRIOR_MIN_FACTOR,
+    Q_PRIOR_MAX_FACTOR = Q_PRIOR_MAX_FACTOR,
+    VERBOSE = VERBOSE,
+    SAVE_RDS = SAVE_RDS
+  ))
+}
+
+#' Print current configuration
+print_config <- function() {
+  cat("========================================\n")
+  cat("BUSBOI Configuration\n")
+  cat("========================================\n")
+  cat(sprintf("Q Prior Type:      %s\n", Q_PRIOR))
+  cat(sprintf("TULIP:             %s\n", TULIP))
+  cat(sprintf("GVF Correction:    %s\n", ifelse(GVF_ON == 1, "ON", "OFF")))
+  cat(sprintf("Bed Mode:          %s\n", 
+      c("5-point", "1-point", "Fixed")[FIX_BED + 1]))
+  cat(sprintf("Input Directory:   %s\n", IN_DIR))
+  cat(sprintf("Output Directory:  %s\n", OUT_DIR))
+  cat("========================================\n")
+}

BUSBOI/.ipynb_checkpoints/debug_BUSBOI-checkpoint.ipynb

@@ -16,13 +16,15 @@
     "#version D\n",
     "\n",
     "#######\n",
-    "single spline\n",
+    "# single spline\n",
+    "#monotonics from 0.85 to 0.5\n",
+    "#rmese from 1 to 2\n",
     "\n"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 15,
+   "execution_count": 2,
    "id": "aecda730-57e7-40d8-a247-89464c099d77",
    "metadata": {},
    "outputs": [],
@@ -38,7 +40,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 16,
+   "execution_count": 8,
    "id": "abaf26a7-5be6-4626-8864-86f1f14d910c",
    "metadata": {},
    "outputs": [],
@@ -55,7 +57,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 44,
+   "execution_count": 13,
    "id": "cf510f41-cda1-4203-afda-0fa0a2819d54",
    "metadata": {},
    "outputs": [
@@ -79,7 +81,7 @@
     }
    ],
    "source": [
-    "output_path='/nas/cee-water/cjgleason/colin/BUSBOI/debug tests/daily_single_splineD/'\n",
+    "output_path='/nas/cee-water/cjgleason/colin/BUSBOI/debug tests/monthly_single_splineD/'\n",
     "run_ids=substr(list.files(output_path),1,11)\n",
     "unrun= reach_ids[!reach_ids %in% run_ids]\n",
     "length(unrun)"
@@ -140,12 +142,20 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 10,
    "id": "c37b79aa-89fe-4c2c-92fc-8f311cc2df09",
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Time difference of 55.27635 secs\n"
+     ]
+    }
+   ],
    "source": [
-    "source('/nas/cee-water/cjgleason/colin/BUSBOI/BUSBOI/main_function.R')\n",
+    " source('/nas/cee-water/cjgleason/colin/BUSBOI/BUSBOI/main_function.R')\n",
     "# suppressWarnings({\n",
     "# for (i in 19:26){\n",
     "\n",
@@ -174,7 +184,7 @@
     "                   output_path=output_path,\n",
     "                   swot_base=swot_base,\n",
     "                   sos_base=sos_base,\n",
-    "                   Q_prior='daily', #or 'monthly'\n",
+    "                   Q_prior='monthly', #or 'monthly'\n",
     "                   tulip='OFF', #or 'OFF'\n",
     "                   GVF_on=0,\n",
     "                   fix_bed=0) # 0 = 5 pt, 1 = 1pt, 2 = fixed\n",
@@ -184,25 +194,15 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 10,
-   "id": "4cdef4c5-2a3b-4fa7-8be9-d678d18aa646",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "stopCluster(clust)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 14,
+   "execution_count": 12,
    "id": "a17e5fb4-78b4-4e13-aacb-f67b297fc468",
    "metadata": {},
    "outputs": [
     {
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "Submitted batch job 53564801\n",
+      "Submitted batch job 53876514\n",
       "\n"
      ]
     }

BUSBOI/.ipynb_checkpoints/drive_BUSBOI-checkpoint.R

@@ -0,0 +1,96 @@
+#!/usr/bin/env Rscript
+
+# BUSBOI Confluence Driver
+# Reads from /mnt/data/input and writes to /mnt/data/output
+
+suppressMessages({
+    library(dplyr)
+    library(tidyr)
+    library(RNetCDF)
+    library(optimx)
+    library(ggplot2)
+    library(deSolve)
+    library(hydroGOF)
+    library(jsonlite)
+    library(optparse)
+})
+
+# Parse command line arguments
+option_list <- list(
+    make_option(c("-r", "--reachfile"), type="character", default="reaches.json",
+                help="Path to reaches JSON file [default %default]", metavar="character"),
+    make_option(c("-i", "--index"), type="integer", default=0,
+                help="Array index for reach to process [default %default]", metavar="integer")
+)
+
+opt_parser <- OptionParser(option_list=option_list)
+opt <- parse_args(opt_parser)
+
+# Get BUSBOI directory
+args <- commandArgs(trailingOnly = FALSE)
+script_path <- sub("--file=", "", args[grep("--file=", args)])
+BUSBOI_DIR <- dirname(script_path)
+if(length(BUSBOI_DIR) == 0 || BUSBOI_DIR == "") {
+    BUSBOI_DIR <- getwd()
+}
+
+# Load configuration first
+source(file.path(BUSBOI_DIR, 'config.R'))
+
+# Print configuration if verbose
+if(VERBOSE) {
+    print_config()
+}
+
+# Source all BUSBOI functions
+source(file.path(BUSBOI_DIR, 'input.R'))
+source(file.path(BUSBOI_DIR, 'jeff_tulip.R'))
+source(file.path(BUSBOI_DIR, 'calcHgivenparams.R'))
+source(file.path(BUSBOI_DIR, 'calcHgivenparams_fixedbed.R'))
+source(file.path(BUSBOI_DIR, 'calcHgivenparams_bedonly.R'))
+source(file.path(BUSBOI_DIR, 'GVF.R'))
+source(file.path(BUSBOI_DIR, 'calculate_cum_dist.R'))
+source(file.path(BUSBOI_DIR, 'calc_Sf_newH.R'))
+source(file.path(BUSBOI_DIR, 'calculate_spline.R'))
+source(file.path(BUSBOI_DIR, 'fit_hydraulics.R'))
+source(file.path(BUSBOI_DIR, 'get_Q_prior.R'))
+source(file.path(BUSBOI_DIR, 'rejection_sample.R'))
+source(file.path(BUSBOI_DIR, 'Jeff_solver.R'))
+source(file.path(BUSBOI_DIR, 'Jeff_solver_bedthenQ.R'))
+source(file.path(BUSBOI_DIR, 'read_LSTM_ensemble.R'))
+source(file.path(BUSBOI_DIR, 'run_BUSBOI.R'))
+source(file.path(BUSBOI_DIR, 'Slope_empirical.R'))
+source(file.path(BUSBOI_DIR, 'main_function.R'))
+source(file.path(BUSBOI_DIR, 'write_output.R'))
+
+# Read reaches from JSON file
+reach_json_path <- file.path(IN_DIR, opt$reachfile)
+if(!file.exists(reach_json_path)) {
+    stop(paste0("Reaches file not found: ", reach_json_path))
+}
+
+reaches <- fromJSON(reach_json_path)
+
+# Get reach_id from index
+if(opt$index < 0 || opt$index >= length(reaches)) {
+    stop(paste0("Index ", opt$index, " out of bounds. Valid range: 0-", length(reaches)-1))
+}
+
+reach_id <- reaches[[opt$index + 1]]  # R is 1-indexed
+
+cat(paste0("Processing reach [", opt$index, "]: ", reach_id, "\n"))
+
+# Run BUSBOI with config parameters
+result <- main_function(
+    this_reach_id = reach_id,
+    swot_base = paste0(IN_DIR, "/"),
+    sos_base = paste0(IN_DIR, "/"),
+    sword_base = paste0(IN_DIR, "/"),
+    output_path = paste0(OUT_DIR, "/"),
+    fix_bed = FIX_BED,
+    GVF_on = GVF_ON,
+    Q_prior = Q_PRIOR,
+    tulip = TULIP
+)
+
+cat("BUSBOI processing complete\n")