run_daily.R

# ============================================================================
# run_daily.R — Daily Deception+ Analysis Using Per-Pitcher Models
# ----------------------------------------------------------------------------
# Runs daily Deception+ analysis for the previous day's MLB games using
# per-pitcher models (each pitcher evaluated against their own patterns).
#
# Approach:
#   1. Load historical data to get each pitcher's last N pitches
#   2. Load today's test data
#   3. For each pitcher: train model on their history, evaluate today's pitches
#   4. Standardize using fixed baseline (from compute_baseline.R)
#   5. Generate social media graphics
#
# Output:
#   - CSV: output/{season}/{month}/{day}.csv
#   - Social media graphics: output/{season}/{month}/visualizations/
# ============================================================================

source("pitch_ppi.R")

# ============================================================================
# CONFIGURATION
# ============================================================================

# Get the date to analyze (default: yesterday)
args <- commandArgs(trailingOnly = TRUE)
if (length(args) > 0 && !grepl("^--", args[1])) {
  TARGET_DATE <- as.Date(args[1])
} else {
  TARGET_DATE <- Sys.Date() - 1
}

# Parse optional arguments
get_arg <- function(flag, default = NULL) {
  hit <- which(args == flag)
  if (length(hit) == 1 && hit < length(args)) args[hit + 1] else default
}

# Level selection (MLB or AAA)
LEVEL <- get_arg("--level", "MLB")

# Number of historical pitches per pitcher for training
N_HISTORY_PITCHES <- as.integer(get_arg("--n_history", "500"))

# Minimum historical pitches required to evaluate a pitcher
MIN_HISTORY_PITCHES <- as.integer(get_arg("--min_history", "100"))

# Minimum test pitches for social media graphics (role-specific thresholds)
MIN_PITCHES_STARTER_SOCIAL  <- 50
MIN_PITCHES_RELIEVER_SOCIAL <- 10

# Baseline parameters file (generated by compute_baseline.R)
BASELINE_FILE <- get_arg("--baseline", "baseline_params.rds")

# Features for per-pitcher models (simpler set for individual models)
FEATURE_NAMES <- c(
  "balls", "strikes", "two_strikes", "ahead_in_count",
  "outs", "is_risp", "stand", "last_pitch_type"
)

# ============================================================================
# DATE CALCULATIONS
# ============================================================================

target_year <- as.integer(format(TARGET_DATE, "%Y"))
target_month <- as.integer(format(TARGET_DATE, "%m"))
target_day <- format(TARGET_DATE, "%d")
target_month_name <- tolower(format(TARGET_DATE, "%B"))

# ============================================================================
# OUTPUT PATHS
# ============================================================================

output_base <- file.path("output", target_year, target_month_name)
viz_output <- file.path(output_base, "visualizations")

if (!dir.exists(output_base)) {
  dir.create(output_base, recursive = TRUE)
  message("Created output directory: ", output_base)
}

if (!dir.exists(viz_output)) {
  dir.create(viz_output, recursive = TRUE)
}

OUT_CSV <- file.path(output_base, paste0(target_day, ".csv"))
OUT_MODEL <- file.path("models", sprintf("daily_%s_%s.rds", LEVEL, TARGET_DATE))

# ============================================================================
# EXECUTION
# ============================================================================

cat("\n")
cat("============================================================\n")
cat("  Deception+ Daily Analysis (Per-Pitcher Models)\n")
cat("============================================================\n")
cat("Target Date:       ", as.character(TARGET_DATE), "\n")
cat("Level:             ", LEVEL, "\n")
cat("History Pitches:   ", N_HISTORY_PITCHES, " per pitcher\n")
cat("Min History:       ", MIN_HISTORY_PITCHES, "\n")
cat("Baseline File:     ", BASELINE_FILE, "\n")
cat("Output CSV:        ", OUT_CSV, "\n")
cat("============================================================\n\n")

ensure_directories()

# ============================================================================
# LOAD BASELINE PARAMETERS
# ============================================================================

if (!file.exists(BASELINE_FILE)) {
  warning("Baseline file not found: ", BASELINE_FILE)
  warning("Using fallback values. Run compute_baseline.R for stable baseline.")
  baseline_params <- list(mu = 1.0, sd = 0.1)  # Fallback
} else {
  message("Loading baseline parameters from: ", BASELINE_FILE)
  baseline_params <- load_baseline_params(BASELINE_FILE)
  message("  Baseline μ = ", round(baseline_params$mu, 4),
          ", σ = ", round(baseline_params$sd, 4))
  message("  Computed from: ", baseline_params$data_period)
}

# ============================================================================
# LOAD TEST DATA (Today's Games)
# ============================================================================

message("\nLoading test data for ", TARGET_DATE, "...")

TRAIN_LEVEL <- LEVEL
TEST_LEVEL <- LEVEL

test_cachefile <- sprintf("cache/savant_raw_%s_%s_R_%s.Rds",
                          TARGET_DATE, TARGET_DATE, LEVEL)

if (file.exists(test_cachefile)) {
  message("Using cached test data: ", test_cachefile)
  raw_test <- readRDS(test_cachefile)
} else {
  message("Downloading test data...")
  raw_test <- load_statcast_range(as.character(TARGET_DATE),
                                   as.character(TARGET_DATE),
                                   game_type = "R", level = LEVEL, verbose = TRUE)
  if (!is.null(raw_test) && nrow(raw_test) > 0) {
    saveRDS(raw_test, test_cachefile)
    message("Cached to: ", test_cachefile)
  }
}

if (is.null(raw_test) || nrow(raw_test) == 0) {
  message("No games found for ", TARGET_DATE, ". This may be an off-day.")
  quit(save = "no", status = 0)
}

message("Test data: ", nrow(raw_test), " pitches")

df_test <- engineer_features(raw_test, include_batter_metrics = FALSE)
df_test <- df_test %>% dplyr::filter(!is.na(pitcher_id))

message("Test features: ", nrow(df_test), " pitches from ",
        length(unique(df_test$pitcher_id)), " pitchers")

# Get unique pitchers from today's games
today_pitcher_ids <- unique(df_test$pitcher_id)
message("Pitchers to evaluate: ", length(today_pitcher_ids))

# ============================================================================
# LOAD HISTORICAL DATA
# ============================================================================

message("\nLoading historical data for pitcher histories...")

# We need to load enough historical data to get last N_HISTORY_PITCHES per pitcher
# Load data from the past ~2 years to ensure coverage
history_start <- as.Date(TARGET_DATE) - 730  # ~2 years back
history_end <- as.Date(TARGET_DATE) - 1      # Through yesterday

# Try to load cached historical data, or download
# For efficiency, we cache by year
historical_data <- NULL
current_year <- lubridate::year(history_end)

for (year in seq(lubridate::year(history_start), current_year)) {
  year_start <- max(as.Date(sprintf("%d-03-01", year)), history_start)
  year_end <- min(as.Date(sprintf("%d-11-30", year)), history_end)

  if (year_end < year_start) next

  if (year == current_year) {
    # Current (incomplete) season: use a stable filename and update incrementally.
    # The date-range filename changes every day, so instead we cache by year and
    # download only the delta since the last cached date.
    year_cachefile <- sprintf("cache/savant_partial_%d_R_%s.Rds", year, LEVEL)

    if (file.exists(year_cachefile)) {
      message("  Loading cached partial ", year, " data...")
      year_data <- readRDS(year_cachefile)

      # Determine how stale the cache is and fetch only new pitches
      cached_max_date <- tryCatch(
        as.Date(max(as.character(year_data$game_date), na.rm = TRUE)),
        error = function(e) as.Date(NA)
      )

      if (!is.na(cached_max_date) && cached_max_date < year_end) {
        delta_start <- cached_max_date + 1
        message("  Fetching new data: ", delta_start, " to ", year_end, "...")
        new_data <- load_statcast_range(
          as.character(delta_start), as.character(year_end),
          game_type = "R", level = LEVEL, verbose = FALSE
        )
        if (!is.null(new_data) && nrow(new_data) > 0) {
          year_data <- dplyr::bind_rows(year_data, new_data)
          saveRDS(year_data, year_cachefile)
          message("  Cache updated: added ", nrow(new_data), " pitches through ", year_end)
        } else {
          message("  No new pitches found since ", cached_max_date)
        }
      } else {
        message("  Cache is current through ", year_end)
      }
    } else {
      message("  Downloading partial ", year, " data (", year_start, " to ", year_end, ")...")
      year_data <- load_statcast_range(
        as.character(year_start), as.character(year_end),
        game_type = "R", level = LEVEL, verbose = FALSE
      )
      if (!is.null(year_data) && nrow(year_data) > 0) {
        saveRDS(year_data, year_cachefile)
        message("  Cached to: ", year_cachefile)
      }
    }
  } else {
    # Complete past season: filename is stable (end date never changes once season ends)
    year_cachefile <- sprintf("cache/savant_raw_%s_%s_R_%s.Rds",
                              year_start, year_end, LEVEL)

    if (file.exists(year_cachefile)) {
      message("  Loading cached ", year, " data...")
      year_data <- readRDS(year_cachefile)
    } else {
      message("  Downloading ", year, " data...")
      year_data <- load_statcast_range(as.character(year_start),
                                        as.character(year_end),
                                        game_type = "R", level = LEVEL, verbose = FALSE)
      if (!is.null(year_data) && nrow(year_data) > 0) {
        saveRDS(year_data, year_cachefile)
      }
    }
  }

  if (!is.null(year_data) && nrow(year_data) > 0) {
    if (is.null(historical_data)) {
      historical_data <- year_data
    } else {
      historical_data <- dplyr::bind_rows(historical_data, year_data)
    }
  }
}

if (is.null(historical_data) || nrow(historical_data) == 0) {
  stop("No historical data available")
}

message("Total historical data: ", nrow(historical_data), " pitches")

# Pre-filter historical data to only pitchers who appeared today before
# running expensive feature engineering on the full dataset
message("\nPre-filtering historical data to today's pitchers...")
pitcher_id_col <- coalesce_pitcher_id(historical_data)
historical_data <- historical_data[!is.na(pitcher_id_col) & pitcher_id_col %in% today_pitcher_ids, ]
message("  Filtered to ", nrow(historical_data), " pitches for today's pitchers")

# Engineer features on filtered historical data
df_history_all <- engineer_features(historical_data, include_batter_metrics = FALSE)
df_history_all <- df_history_all %>% dplyr::filter(!is.na(pitcher_id))

# Get last N_HISTORY_PITCHES for each pitcher who appeared today
message("\nExtracting last ", N_HISTORY_PITCHES, " pitches per pitcher...")

df_history <- get_pitcher_history(df_history_all, today_pitcher_ids, N_HISTORY_PITCHES)

message("  History data: ", nrow(df_history), " pitches from ",
        length(unique(df_history$pitcher_id)), " pitchers")

# ============================================================================
# EVALUATE PER-PITCHER UNPREDICTABILITY
# ============================================================================

message("\nEvaluating per-pitcher unpredictability...")

# Use the same baseline keys that were used to compute baseline_params (mu/sigma)
BASELINE_KEYS <- baseline_params$baseline_keys %||% c("balls", "strikes", "stand", "two_strikes")

eval_result <- evaluate_per_pitcher(
  df_history = df_history,
  df_test = df_test,
  min_train_pitches = MIN_HISTORY_PITCHES,
  min_test_pitches = 1,  # Include all pitchers with any test pitches
  feature_names = FEATURE_NAMES,
  baseline_keys = BASELINE_KEYS,
  verbose = TRUE
)

# Extract results and excluded pitchers
per_pitcher_results <- eval_result$results
excluded_pitchers <- eval_result$excluded

if (nrow(per_pitcher_results) == 0) {
  message("No pitchers with sufficient data to evaluate.")
  quit(save = "no", status = 0)
}

# ============================================================================
# STANDARDIZE TO PREDICT+
# ============================================================================

message("\nStandardizing to Deception+ scale...")

# Calculate Deception+ using fixed baseline parameters
per_pitcher_results <- per_pitcher_results %>%
  dplyr::mutate(
    ppi = 1 - (mean_surp_model / pmax(mean_surp_base, 1e-9)),
    ppi = pmin(pmax(ppi, -1), 1),
    deception_plus = 100 + 10 * ((unpredictability_ratio - baseline_params$mu) /
                                pmax(baseline_params$sd, 1e-9))
  )

# Identify starter/reliever roles from test data
message("Identifying starter/reliever roles...")
pitcher_roles <- get_daily_pitcher_roles(df_test)
n_starters <- sum(pitcher_roles$role == "starter")
n_relievers <- sum(pitcher_roles$role == "reliever")
message("  ", n_starters, " starters, ", n_relievers, " relievers")

# Resolve pitcher names (for both evaluated and excluded pitchers)
message("Resolving pitcher names...")
all_pitcher_ids <- dplyr::bind_rows(
  per_pitcher_results %>% dplyr::select(pitcher_id),
  excluded_pitchers %>% dplyr::select(pitcher_id)
) %>% dplyr::distinct()

name_map <- resolve_pitcher_names_with_fallback(all_pitcher_ids,
                                                 cache_file = "cache/mlbam_name_cache.csv",
                                                 verbose = TRUE)

# Create evaluated pitchers output
evaluated_ppi <- per_pitcher_results %>%
  dplyr::left_join(name_map, by = "pitcher_id") %>%
  dplyr::left_join(pitcher_roles, by = "pitcher_id") %>%
  dplyr::mutate(
    pitcher_name = dplyr::if_else(is.na(pitcher_name),
                                   paste0("Pitcher_", pitcher_id),
                                   pitcher_name),
    total_pitches = n_history,
    n_pitches_test = n_test,
    status = "evaluated",
    role = dplyr::coalesce(role, "unknown")
  ) %>%
  dplyr::select(
    pitcher_id, pitcher_name, role, total_pitches, n_pitches_test,
    mean_surp_model, mean_surp_base, ppi,
    unpredictability_ratio, deception_plus, status
  ) %>%
  dplyr::arrange(dplyr::desc(deception_plus))

# Create excluded pitchers output (debuts, insufficient history)
excluded_ppi <- excluded_pitchers %>%
  dplyr::left_join(name_map, by = "pitcher_id") %>%
  dplyr::left_join(pitcher_roles, by = "pitcher_id") %>%
  dplyr::mutate(
    pitcher_name = dplyr::if_else(is.na(pitcher_name),
                                   paste0("Pitcher_", pitcher_id),
                                   pitcher_name),
    total_pitches = n_history,
    n_pitches_test = n_test,
    mean_surp_model = NA_real_,
    mean_surp_base = NA_real_,
    ppi = NA_real_,
    unpredictability_ratio = NA_real_,
    deception_plus = NA_real_,
    role = dplyr::coalesce(role, "unknown")
  ) %>%
  dplyr::select(
    pitcher_id, pitcher_name, role, total_pitches, n_pitches_test,
    mean_surp_model, mean_surp_base, ppi,
    unpredictability_ratio, deception_plus, status
  )

# Combine evaluated and excluded pitchers
pitcher_ppi <- dplyr::bind_rows(evaluated_ppi, excluded_ppi) %>%
  dplyr::arrange(dplyr::desc(deception_plus), status)

# ============================================================================
# SAVE RESULTS
# ============================================================================

message("\nSaving results...")
readr::write_csv(pitcher_ppi, OUT_CSV)
message("CSV saved: ", OUT_CSV)

# Create result object for visualization functions
res <- list(
  pitcher_ppi = pitcher_ppi,
  baseline_params = baseline_params,
  split_method = "per_pitcher",
  test_period = as.character(TARGET_DATE),
  n_history_pitches = N_HISTORY_PITCHES
)

# Count evaluated vs excluded
n_evaluated <- sum(pitcher_ppi$status == "evaluated")
n_debuts <- sum(pitcher_ppi$status == "debut_no_history")
n_insufficient <- sum(pitcher_ppi$status == "insufficient_history")

# Save model metadata
saveRDS(list(
  baseline_params = baseline_params,
  split_method = "per_pitcher",
  test_date = as.character(TARGET_DATE),
  level = LEVEL,
  n_history_pitches = N_HISTORY_PITCHES,
  min_history_pitches = MIN_HISTORY_PITCHES,
  feature_names = FEATURE_NAMES,
  n_pitchers_evaluated = n_evaluated,
  n_pitchers_debut = n_debuts,
  n_pitchers_insufficient = n_insufficient
), OUT_MODEL)
message("Model metadata saved: ", OUT_MODEL)

# ============================================================================
# SOCIAL MEDIA GRAPHICS
# ============================================================================

message("\nGenerating social media graphics...")
create_social_media_graphics(
  res = res,
  game_date = as.character(TARGET_DATE),
  min_pitches_starter  = MIN_PITCHES_STARTER_SOCIAL,
  min_pitches_reliever = MIN_PITCHES_RELIEVER_SOCIAL,
  output_dir = viz_output,
  top_n = 5
)

# ============================================================================
# SUMMARY
# ============================================================================

cat("\n")
cat("============================================================\n")
cat("  Daily Analysis Complete!\n")
cat("============================================================\n")
cat("Date:              ", as.character(TARGET_DATE), "\n")
cat("Level:             ", LEVEL, "\n")
cat("Pitchers Evaluated:", n_evaluated, "\n")
if (n_debuts > 0) {
  cat("Debuts (no data):  ", n_debuts, "\n")
}
if (n_insufficient > 0) {
  cat("Insufficient Hist: ", n_insufficient, " (<", MIN_HISTORY_PITCHES, " pitches)\n", sep = "")
}
cat("Total Test Pitches:", sum(pitcher_ppi$n_pitches_test, na.rm = TRUE), "\n")
cat("Baseline:          μ=", round(baseline_params$mu, 4),
    " σ=", round(baseline_params$sd, 4), "\n")
cat("\n")

# Show top 5 and bottom 5 using role-specific thresholds (only evaluated pitchers)
# Starters: >= MIN_PITCHES_STARTER_SOCIAL, Relievers: >= MIN_PITCHES_RELIEVER_SOCIAL
qualified_starters <- pitcher_ppi %>%
  dplyr::filter(status == "evaluated", role == "starter",
                n_pitches_test >= MIN_PITCHES_STARTER_SOCIAL)

qualified_relievers <- pitcher_ppi %>%
  dplyr::filter(status == "evaluated", role == "reliever",
                n_pitches_test >= MIN_PITCHES_RELIEVER_SOCIAL)

# Helper function to print rankings
print_rankings <- function(data, title_top, title_bottom, n = 5) {
  if (nrow(data) == 0) {
    cat("No pitchers qualified.\n")
    return()
  }

  cat(title_top, ":\n", sep = "")
  top_n <- data %>%
    dplyr::arrange(dplyr::desc(deception_plus)) %>%
    head(n) %>%
    dplyr::mutate(rank = dplyr::row_number()) %>%
    dplyr::select(rank, pitcher_name, n_pitches_test, deception_plus)
  print(as.data.frame(top_n), row.names = FALSE)

  cat("\n", title_bottom, ":\n", sep = "")
  bottom_n <- data %>%
    dplyr::arrange(deception_plus) %>%
    head(n) %>%
    dplyr::mutate(rank = dplyr::row_number()) %>%
    dplyr::select(rank, pitcher_name, n_pitches_test, deception_plus)
  print(as.data.frame(bottom_n), row.names = FALSE)
}

# Starters
cat("--- STARTERS (>= ", MIN_PITCHES_STARTER_SOCIAL, " pitches) ---\n", sep = "")
if (nrow(qualified_starters) > 0) {
  print_rankings(
    qualified_starters,
    paste0("Top 5 Least Predictable Starters"),
    paste0("Top 5 Most Predictable Starters")
  )
} else {
  cat("No starters with >= ", MIN_PITCHES_STARTER_SOCIAL, " pitches today.\n", sep = "")
}

cat("\n")

# Relievers
cat("--- RELIEVERS (>= ", MIN_PITCHES_RELIEVER_SOCIAL, " pitches) ---\n", sep = "")
if (nrow(qualified_relievers) > 0) {
  print_rankings(
    qualified_relievers,
    paste0("Top 5 Least Predictable Relievers"),
    paste0("Top 5 Most Predictable Relievers")
  )
} else {
  cat("No relievers with >= ", MIN_PITCHES_RELIEVER_SOCIAL, " pitches today.\n", sep = "")
}

cat("\n")
cat("============================================================\n")
cat("Output files:\n")
cat("  CSV:     ", OUT_CSV, "\n")
cat("  Model:   ", OUT_MODEL, "\n")
cat("  Visuals: ", viz_output, "/\n", sep = "")
cat("============================================================\n\n")

# ============================================================================
# BALTIMORE ORIOLES PITCHER SUMMARY
# ============================================================================

if (LEVEL == "MLB") {
  orioles_ppi <- NULL

  if (all(c("home_team", "away_team", "inning_topbot") %in% names(df_test))) {
    # Determine each pitcher's team from test data:
    # TOP of inning → home team is pitching; BOT of inning → away team is pitching
    pitcher_team_map <- df_test %>%
      dplyr::mutate(
        pitcher_team = dplyr::if_else(
          stringr::str_to_upper(.data$inning_topbot) == "TOP",
          as.character(.data$home_team),
          as.character(.data$away_team)
        )
      ) %>%
      dplyr::filter(!is.na(.data$pitcher_id), !is.na(.data$pitcher_team)) %>%
      dplyr::distinct(.data$pitcher_id, .data$pitcher_team) %>%
      dplyr::group_by(.data$pitcher_id) %>%
      dplyr::slice(1) %>%
      dplyr::ungroup()

    orioles_pitcher_ids <- pitcher_team_map %>%
      dplyr::filter(.data$pitcher_team == "BAL") %>%
      dplyr::pull(.data$pitcher_id)

    if (length(orioles_pitcher_ids) > 0) {
      # Include ALL Orioles pitchers regardless of pitch count
      orioles_ppi <- pitcher_ppi %>%
        dplyr::filter(.data$pitcher_id %in% orioles_pitcher_ids) %>%
        dplyr::arrange(dplyr::desc(.data$deception_plus))
    }
  }

  cat("============================================================\n")
  cat("  Baltimore Orioles Pitchers (all pitchers)\n")
  cat("============================================================\n")

  if (!is.null(orioles_ppi) && nrow(orioles_ppi) > 0) {
    # Print markdown-style table to console
    orioles_display <- orioles_ppi %>%
      dplyr::select(
        .data$pitcher_name, .data$role, .data$n_pitches_test,
        .data$deception_plus, .data$status
      )
    print(as.data.frame(orioles_display), row.names = FALSE)

    # Generate PNG graphic with orange/black gradient
    create_orioles_graphic(
      orioles_data = orioles_ppi,
      game_date    = as.character(TARGET_DATE),
      output_dir   = viz_output
    )
  } else if (!all(c("home_team", "away_team") %in% names(df_test))) {
    cat("Team data not available in Statcast download.\n")
  } else {
    cat("No Orioles pitchers found in today's games.\n")
  }

  cat("============================================================\n\n")
}