diff --git a/bin/local-dev/README.md b/bin/local-dev/README.md index a560f9098b8..e1a1f175fcb 100644 --- a/bin/local-dev/README.md +++ b/bin/local-dev/README.md @@ -35,6 +35,40 @@ bin/local-dev.sh --help # full reference Everything in this folder is wired up by the wrapper at `bin/local-dev.sh`; you should not need to invoke any file inside `bin/local-dev/` directly. +## Supported platforms + +macOS and Linux. Where the two differ, `main.sh` picks the right probe by +platform — there is nothing to configure: + +| Concern | macOS | Linux | +| --- | --- | --- | +| Host LAN IP (the MinIO endpoint) | `route get default`, `ipconfig getifaddr` | `ip route show default`, `ip -4 addr show scope global` | +| Artifact mtime (`watch`'s ARTIFACT MTIME column) | BSD `stat -f` | GNU `stat -c` | +| Port → PID | `lsof` | `lsof`, falling back to `ss` | + +On top of the toolchain in [AGENTS.md](../../AGENTS.md) — JDK 17, Scala 2.13, +Python 3.12, Node 24 — plus sbt and docker, Linux needs: + +* **iproute2** (`ip`, `ss`). Essential on every mainstream distro, so normally + already installed. `ip` is required: without it the host LAN IP cannot be + detected and `up` refuses to start rather than publish an endpoint that only + works from one side. +* **docker with the compose v2 plugin.** On stock Ubuntu that is + `apt install docker.io docker-compose-v2`; `docker-compose-plugin` only + exists in Docker's own apt repository. +* **your user in the `docker` group** — `sudo usermod -aG docker "$USER"`, + then log in again. Group membership does not apply to shells that are + already running, so a `docker compose` permission error right after that + command is expected. + +`HOST_LAN_IP` is detected for you; export it only to override the choice, e.g. +to pin one interface on a multi-homed host. The address has to be reachable +from **both** the host JVMs and the containers, which is why the scan skips +container bridges (`docker0`, `br-*`, `veth*`) and overlay/VPN interfaces +(tailscale, zerotier, wireguard): `172.17.0.1` looks like a perfectly good +local IPv4 but is not reachable from inside another container's network +namespace. + ## Layout ``` diff --git a/bin/local-dev/main.sh b/bin/local-dev/main.sh index 8ee11a256bd..402d69a520b 100755 --- a/bin/local-dev/main.sh +++ b/bin/local-dev/main.sh @@ -40,6 +40,7 @@ # bin/local-dev.sh up [service] # [--fresh|--build|--skip-build] [--skip=svc1,svc2] [--json] # [--worktree=PATH | --branch=NAME] +# [--install-missing | --no-install] # Default: skip build if no source/lock # changes since last build. --build forces # incremental sbt dist + yarn/bun install. @@ -72,11 +73,22 @@ # branch and run its own local-dev.sh # instead (a warning is printed when such # drift is detected). +# TOOLCHAIN: a missing JDK 17 / Node / +# Python 3.12 is offered for install +# (distro package manager for the JDK, +# nvm for Node, pyenv for Python) after +# asking. --install-missing answers yes +# without asking, --no-install only ever +# prints the hint. Without a TTY the +# prompt is skipped entirely, so scripted +# and CI runs behave as before. Same flags +# on `auto`. # bin/local-dev.sh down [service] [--skip=svc1,svc2] [--json] # stop every non-skipped service, or just # (--json: summary JSON on # stdout). # bin/local-dev.sh auto [--skip=svc1,svc2] [--json] +# [--install-missing | --no-install] # rebuild + bounce only the services whose # source changed since the last build. # bin/local-dev.sh logs tail this service's log. @@ -288,6 +300,115 @@ amap_append() { eval "$_var=\"\${$_var:-}\$_suffix\"" } +# --------- toolchain versions + consented install --------- +# The versions this repo needs (AGENTS.md's table, and the CI matrices). +# Overridable so a contributor can try a newer runtime without editing this. +TEXERA_PYTHON_VERSION="${TEXERA_PYTHON_VERSION:-3.12}" +TEXERA_NODE_VERSION="${TEXERA_NODE_VERSION:-24}" + +# This block sits above the JDK probe on purpose: the probe is the first thing +# that can fail on a fresh machine, and it should be able to offer a fix. That +# also means no colour variables here — those are defined further down, with +# the rest of the TUI helpers. + +# The package manager we'd install with, or non-zero if we don't recognise one. +_pkg_manager() { + case "$(uname -s 2>/dev/null)" in + Darwin) + command -v brew >/dev/null 2>&1 && { printf 'brew\n'; return 0; } + ;; + Linux) + local m="" + for m in apt-get dnf yum pacman zypper; do + command -v "$m" >/dev/null 2>&1 && { printf '%s\n' "$m"; return 0; } + done + ;; + esac + return 1 +} + +# What we would run to install $1. Prints the command and never executes it — +# that split is what lets us show the user the exact command before asking, and +# lets CI assert on the decision without installing anything. +# +# Java prefers the distro package manager: it is system-wide and the distro +# patches it. SDKMAN is the fallback when there's no package manager (or no +# root). Node goes through nvm and Python through pyenv, because the versions +# this repo needs are newer than most distros ship — and both bootstrap +# themselves first if absent. +# +# docker and sbt are deliberately absent. Docker needs a daemon, group +# membership and a re-login; that is not something to do behind a y/n prompt. +_install_cmd_for() { + local tool="${1:-}" pm="" + pm=$(_pkg_manager) || pm="" + case "$tool" in + java) + case "$pm" in + apt-get) printf 'sudo apt-get install -y openjdk-17-jdk\n' ;; + dnf) printf 'sudo dnf install -y java-17-openjdk-devel\n' ;; + yum) printf 'sudo yum install -y java-17-openjdk-devel\n' ;; + pacman) printf 'sudo pacman -S --noconfirm jdk17-openjdk\n' ;; + zypper) printf 'sudo zypper install -y java-17-openjdk-devel\n' ;; + brew) printf 'brew install openjdk@17\n' ;; + *) printf 'curl -s https://get.sdkman.io | bash && sdk install java 17.0.13-tem\n' ;; + esac + ;; + node) + if command -v nvm >/dev/null 2>&1 || [[ -s "${NVM_DIR:-$HOME/.nvm}/nvm.sh" ]]; then + printf 'nvm install %s && nvm alias default %s\n' \ + "$TEXERA_NODE_VERSION" "$TEXERA_NODE_VERSION" + else + printf 'curl -fsSL https://raw.githubusercontent.com/nvm-sh/nvm/v0.40.3/install.sh | bash && . "$HOME/.nvm/nvm.sh" && nvm install %s && nvm alias default %s\n' \ + "$TEXERA_NODE_VERSION" "$TEXERA_NODE_VERSION" + fi + ;; + python) + if command -v pyenv >/dev/null 2>&1; then + printf 'pyenv install -s %s\n' "$TEXERA_PYTHON_VERSION" + else + printf 'curl -fsSL https://pyenv.run | bash && export PYENV_ROOT="$HOME/.pyenv" && export PATH="$PYENV_ROOT/bin:$PATH" && pyenv install -s %s\n' \ + "$TEXERA_PYTHON_VERSION" + fi + ;; + *) return 1 ;; + esac +} + +# Ask before installing anything. Never prompts without a TTY: `up` has to stay +# scriptable, and AGENTS.md points agents at the non-interactive subcommands, so +# a non-interactive run keeps the old behaviour of printing a hint and failing. +# --install-missing / TEXERA_INSTALL_MISSING=1 assume yes +# --no-install / TEXERA_INSTALL_MISSING=0 never install +_consent_to_install() { + local tool="${1:-}" cmd="${2:-}" reply="" + case "${TEXERA_INSTALL_MISSING:-ask}" in + 1|y|yes|true) return 0 ;; + 0|n|no|false) return 1 ;; + esac + [[ -t 0 && -t 1 ]] || return 1 + printf "\n %s is missing. I can run:\n\n" "$tool" >&2 + printf " %s\n\n" "$cmd" >&2 + printf " Run it now? [y/N] " >&2 + read -r reply || return 1 + [[ "$reply" == [Yy]* ]] +} + +# Offer to install $1, then report whether it's actually there afterwards. +# Runs through `bash -lc` so shell functions like nvm / pyenv / sdk resolve. +_offer_install() { + local tool="${1:-}" cmd="" + cmd=$(_install_cmd_for "$tool") || return 1 + _consent_to_install "$tool" "$cmd" || return 1 + printf " installing %s ...\n" "$tool" >&2 + if ! bash -lc "$cmd" >&2; then + printf " %s: install command failed\n" "$tool" >&2 + return 1 + fi + printf " %s: installed\n" "$tool" >&2 + return 0 +} + # --------- toolchain (JDK 17 + node) --------- # Detect a JDK 17 installation rather than pinning one path. We try, in # order: (1) caller-set $JAVA_HOME if it really is 17, (2) macOS's official @@ -423,7 +544,13 @@ _diagnose_jdk17() { echo "" >&2 } -JAVA_HOME_DETECTED="$(_find_jdk17)" || { +JAVA_HOME_DETECTED="$(_find_jdk17)" || JAVA_HOME_DETECTED="" +# Offer to install it rather than only describing how. Declines, and every +# non-interactive run, fall through to the hint-and-exit below unchanged. +if [[ -z "$JAVA_HOME_DETECTED" ]] && _offer_install java; then + JAVA_HOME_DETECTED="$(_find_jdk17)" || JAVA_HOME_DETECTED="" +fi +if [[ -z "$JAVA_HOME_DETECTED" ]]; then echo "FATAL: could not find a JDK 17 install." >&2 _diagnose_jdk17 echo " fix:" >&2 @@ -433,7 +560,7 @@ JAVA_HOME_DETECTED="$(_find_jdk17)" || { echo " or set JAVA_HOME=/path/to/jdk-17 explicitly" >&2 echo "" >&2 exit 1 -} +fi export JAVA_HOME="$JAVA_HOME_DETECTED" export PATH="$JAVA_HOME/bin:$PATH" @@ -462,22 +589,61 @@ fi # `localhost` only works for the host. `texera-minio` only works inside the # docker network. The host's LAN IP works from BOTH (host loopback for the # host, docker NAT'd out-and-back for the container). -_detect_host_lan_ip() { +# +# Both platform probes follow the same two steps: the interface backing the +# default route first (most reliable on a laptop that may have wifi + +# thunderbolt + tailscale all active), then a scan as a fallback. +_detect_host_lan_ip_darwin() { local iface="" ip="" - # 1. The interface backing the default route — most reliable on a - # laptop that may have wifi + thunderbolt + tailscale all active. iface=$(route get default 2>/dev/null | awk '/interface:/{print $2; exit}') if [[ -n "$iface" ]]; then ip=$(ipconfig getifaddr "$iface" 2>/dev/null) [[ -n "$ip" && "$ip" != 127.* ]] && { printf '%s\n' "$ip"; return 0; } fi - # 2. Fallback: linux `hostname -I`-equivalent walk over en*. for iface in en0 en1 en2 en3 en4 en5 en6 en7 en8 en9 en10; do ip=$(ipconfig getifaddr "$iface" 2>/dev/null) [[ -n "$ip" && "$ip" != 127.* ]] && { printf '%s\n' "$ip"; return 0; } done return 1 } + +_detect_host_lan_ip_linux() { + command -v ip >/dev/null 2>&1 || return 1 + local iface="" addr="" idx="" dev="" fam="" cidr="" + # 1. Interface backing the default route — take its first global IPv4, + # stripping the /prefix that `ip -o addr` appends. + iface=$(ip -4 route show default 2>/dev/null \ + | awk '{ for (i = 1; i < NF; i++) if ($i == "dev") { print $(i+1); exit } }') + if [[ -n "$iface" ]]; then + addr=$(ip -4 -o addr show dev "$iface" scope global 2>/dev/null \ + | awk '{ split($4, a, "/"); print a[1]; exit }') + [[ -n "$addr" && "$addr" != 127.* ]] && { printf '%s\n' "$addr"; return 0; } + fi + # 2. Scan every global IPv4, skipping the interfaces that would defeat the + # purpose of this address. A container bridge (docker0, br-*, veth*) is + # reachable from the host but not from inside another container's + # network namespace the way MinIO needs; an overlay/VPN address + # (tailscale, zerotier) is not reachable from the docker bridge at all. + while read -r idx dev fam cidr _rest; do + [[ "$fam" == "inet" ]] || continue + case "$dev" in + lo|docker*|br-*|bridge*|virbr*|veth*|tun*|tap*|cni*|flannel*|cali*|kube*|tailscale*|zt*|wg*) + continue ;; + esac + addr="${cidr%%/*}" + [[ -n "$addr" && "$addr" != 127.* ]] && { printf '%s\n' "$addr"; return 0; } + done < <(ip -4 -o addr show scope global 2>/dev/null) + return 1 +} + +_detect_host_lan_ip() { + case "$(uname -s 2>/dev/null)" in + Darwin) _detect_host_lan_ip_darwin ;; + Linux) _detect_host_lan_ip_linux ;; + # Anything else (BSD, WSL oddities): try both rather than give up. + *) _detect_host_lan_ip_darwin || _detect_host_lan_ip_linux ;; + esac +} # Lazy resolver — called from subcommands that actually need to publish a # host-reachable S3 endpoint (cmd_up, cmd_auto). Subcommands like # `version`, `status`, `--help`, or `-i` don't talk to MinIO and shouldn't @@ -486,10 +652,17 @@ _require_host_lan_ip() { [[ -n "${HOST_LAN_IP:-}" ]] && return 0 HOST_LAN_IP="$(_detect_host_lan_ip)" || HOST_LAN_IP="" if [[ -z "$HOST_LAN_IP" ]]; then + local probes="" + case "$(uname -s 2>/dev/null)" in + Darwin) probes="\`route get default\` / en0-en10" ;; + Linux) probes="\`ip route show default\` / \`ip -4 addr show scope global\`" ;; + *) probes="the macOS and Linux probes" ;; + esac echo "FATAL: could not detect a host LAN IP." >&2 echo " MinIO needs an address reachable from both docker (lakekeeper" >&2 echo " does S3 ops) and the host (JVMs read signed URLs back); none" >&2 - echo " of \`route get default\` / en0-en10 had a non-loopback IPv4." >&2 + echo " of $probes offered a non-loopback IPv4" >&2 + echo " outside the container bridges." >&2 echo " Connect to a network or export HOST_LAN_IP= explicitly." >&2 exit 1 fi @@ -519,7 +692,10 @@ export STORAGE_LAKEFS_ENDPOINT="${STORAGE_LAKEFS_ENDPOINT:-http://localhost:8000 export STORAGE_LAKEFS_AUTH_USERNAME="${STORAGE_LAKEFS_AUTH_USERNAME:-AKIAIOSFOLKFSSAMPLES}" export STORAGE_LAKEFS_AUTH_PASSWORD="${STORAGE_LAKEFS_AUTH_PASSWORD:-wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY}" export STORAGE_LAKEFS_AUTH_API_SECRET="${STORAGE_LAKEFS_AUTH_API_SECRET:-random_string_for_lakefs}" -export UDF_PYTHON_PATH="${UDF_PYTHON_PATH:-$(command -v python3 2>/dev/null || command -v python 2>/dev/null)}" +# UDF_PYTHON_PATH is resolved lazily by _require_udf_python (see below) rather +# than defaulted to `command -v python3` here: the system interpreter almost +# never has amber/requirements.txt installed, and picking it silently turned a +# toolchain problem into import errors inside a Python worker. export TEXERA_DASHBOARD_SERVICE_ENDPOINT="${TEXERA_DASHBOARD_SERVICE_ENDPOINT:-http://localhost:8080}" export WORKFLOW_COMPILING_SERVICE_ENDPOINT="${WORKFLOW_COMPILING_SERVICE_ENDPOINT:-http://localhost:9090}" export WORKFLOW_EXECUTION_SERVICE_ENDPOINT="${WORKFLOW_EXECUTION_SERVICE_ENDPOINT:-http://localhost:8085}" @@ -1133,20 +1309,23 @@ _install_hint() { node) printf " ${BOLD}install Node 20+ (needed for frontend & agent-service):${RESET}\n" printf " macOS: brew install node\n" + printf " Linux: use a version manager below — distro packages\n" + printf " (apt/dnf nodejs) are older than the frontend needs\n" printf " nvm: nvm install --lts && nvm use --lts\n" printf " fnm: fnm install --lts\n" printf " volta: volta install node\n" ;; yarn) printf " ${BOLD}install yarn (needed for the frontend):${RESET}\n" + printf " any OS: corepack enable ${DIM}# ships with Node; frontend pins yarn@4${RESET}\n" printf " macOS: brew install yarn\n" - printf " npm: npm install -g yarn\n" - printf " corepack: corepack enable\n" + printf " Linux: npm install -g yarn\n" ;; bun) printf " ${BOLD}install bun (needed for agent-service):${RESET}\n" printf " macOS: brew install oven-sh/bun/bun\n" - printf " curl: curl -fsSL https://bun.sh/install | bash\n" + printf " Linux: curl -fsSL https://bun.sh/install | bash\n" + printf " npm: npm install -g bun\n" ;; sbt) printf " ${BOLD}install sbt (needed to build the JVM services):${RESET}\n" @@ -1156,7 +1335,8 @@ _install_hint() { docker) printf " ${BOLD}install Docker (needed for postgres/minio/lakefs/lakekeeper/litellm):${RESET}\n" printf " macOS: download Docker Desktop from https://docker.com/products/docker-desktop\n" - printf " Linux: apt install docker.io docker-compose-plugin\n" + printf " Linux: apt install docker.io docker-compose-v2 ${DIM}# dnf: moby-engine docker-compose${RESET}\n" + printf " then sudo usermod -aG docker \"\$USER\" and log back in\n" ;; *) printf " ${DIM}no install hint for: %s${RESET}\n" "$tool" @@ -1193,10 +1373,188 @@ _diagnose_node() { printf "\n" } +# --------- python for UDF workers --------- +# "3.12" for an interpreter, or non-zero if it can't be run at all. +_python_version_of() { + local py="${1:-}" + [[ -n "$py" && -x "$py" ]] || return 1 + "$py" -c 'import sys; print("%d.%d" % sys.version_info[:2])' 2>/dev/null +} + +_python_version_ok() { + [[ "$(_python_version_of "${1:-}" 2>/dev/null)" == "$TEXERA_PYTHON_VERSION" ]] +} + +# Can this interpreter actually run a Python UDF worker? pyarrow, pandas and +# loguru are the heavy three from amber/requirements.txt — if they import, the +# environment was populated. +_python_deps_ok() { + local py="${1:-}" + [[ -n "$py" && -x "$py" ]] || return 1 + "$py" -c 'import pyarrow, pandas, loguru' >/dev/null 2>&1 +} + +# Interpreters that might run UDF workers, best first. Prints paths only; +# suitability is the caller's business. Deduplicated, because the sibling-venv +# and pyenv guesses overlap on some layouts. +_udf_python_candidates() { + local seen="" base="" root="" d="" + # Collapse `..` segments so the path we print and export is the one a + # contributor would recognise. `realpath` isn't portable to a stock macOS, + # and only the directory is resolved — the final component stays a symlink, + # which is what we want for a venv's bin/python. + _abspath() { + local dir="" leaf="" + dir=$(dirname "$1"); leaf=$(basename "$1") + [[ -d "$dir" ]] || { printf '%s\n' "$1"; return 0; } + printf '%s/%s\n' "$(cd "$dir" && pwd -P)" "$leaf" + } + _emit() { [[ -n "${1:-}" ]] || return 0; local p=""; p=$(_abspath "$1"); case "$seen" in *"|$p|"*) return 0 ;; esac; seen="$seen|$p|"; printf '%s\n' "$p"; } + # An activated venv is the most explicit statement of intent available. + [[ -n "${VIRTUAL_ENV:-}" ]] && _emit "$VIRTUAL_ENV/bin/python" + # AGENTS.md's layout: /{texera, texera-worktrees/, venv312} + for base in "$REPO_ROOT/.." "$REPO_ROOT/../.." "$SELF_ROOT/.." "$SELF_ROOT/../.."; do + _emit "$base/venv312/bin/python" + done + if command -v pyenv >/dev/null 2>&1; then + root=$(pyenv root 2>/dev/null) || root="$HOME/.pyenv" + while IFS= read -r d; do + [[ -n "$d" ]] && _emit "$d/bin/python" + done < <(shopt -s nullglob; printf '%s\n' "$root/versions/$TEXERA_PYTHON_VERSION"*) + fi + _emit "$(command -v "python$TEXERA_PYTHON_VERSION" 2>/dev/null || true)" + _emit "$(command -v python3 2>/dev/null || true)" +} + +# Lazy resolver, mirroring _require_host_lan_ip: only the subcommands that +# actually launch services pay for it, so `status` / `logs` / `--help` don't +# spawn an interpreter per candidate. +# +# Never fatal. The JVM services run fine without a Python toolchain; only +# Python UDFs don't. So this warns, offers a fix, and carries on. +_require_udf_python() { + [[ -n "${_UDF_PYTHON_RESOLVED:-}" ]] && return 0 + _UDF_PYTHON_RESOLVED=1 + # An explicit UDF_PYTHON_PATH always wins — it's the documented override — + # but say so if it can't import what a worker needs. + if [[ -n "${UDF_PYTHON_PATH:-}" ]]; then + if ! _python_deps_ok "$UDF_PYTHON_PATH"; then + tui_warn "python: UDF_PYTHON_PATH=$UDF_PYTHON_PATH can't import amber's deps" + tui_warn "python: Python UDFs will fail at worker launch" + fi + export UDF_PYTHON_PATH + return 0 + fi + local c="" no_deps="" + while IFS= read -r c; do + [[ -n "$c" ]] || continue + _python_version_ok "$c" || continue + if _python_deps_ok "$c"; then + export UDF_PYTHON_PATH="$c" + tui_ok "python: $c ${DIM}(runs Python UDFs)${RESET}" + return 0 + fi + [[ -z "$no_deps" ]] && no_deps="$c" + done < <(_udf_python_candidates) + # Distinguish "no 3.12 anywhere" from "3.12 without amber's deps": they need + # different fixes, and the old default silently picked the second one. + if [[ -n "$no_deps" ]]; then + tui_warn "python: $no_deps is $TEXERA_PYTHON_VERSION but can't import amber's deps" + printf " ${BOLD}populate it:${RESET}\n" + printf " %s -m pip install -r %s/amber/requirements.txt -r %s/amber/operator-requirements.txt\n" \ + "$no_deps" "$REPO_ROOT" "$REPO_ROOT" + export UDF_PYTHON_PATH="$no_deps" + return 0 + fi + tui_warn "python: no Python $TEXERA_PYTHON_VERSION found — Python UDFs will not run" + # Deliberately not `_install_hint python`: that hint is about the `-i` + # dashboard's textual dependency, which is a different interpreter and a + # different requirements file. + printf " ${BOLD}looked in:${RESET} \$VIRTUAL_ENV, /venv312, pyenv, PATH\n" + printf " ${BOLD}or point at one yourself:${RESET} export UDF_PYTHON_PATH=/path/to/python%s\n" \ + "$TEXERA_PYTHON_VERSION" + if _offer_install python; then + while IFS= read -r c; do + [[ -n "$c" ]] || continue + if _python_version_ok "$c"; then + export UDF_PYTHON_PATH="$c" + tui_ok "python: $c" + tui_warn "python: now install amber's deps into it:" + printf " %s -m pip install -r %s/amber/requirements.txt -r %s/amber/operator-requirements.txt\n" \ + "$c" "$REPO_ROOT" "$REPO_ROOT" + return 0 + fi + done < <(_udf_python_candidates) + fi + return 0 +} + +# Translate --install-missing / --no-install into TEXERA_INSTALL_MISSING. The +# flags beat the environment variable; the contradiction is refused rather than +# silently resolved one way. +_set_install_flag() { + local want="" + case "${1:-}" in + --install-missing) want=1 ;; + --no-install) want=0 ;; + *) return 0 ;; + esac + if [[ -n "${_INSTALL_FLAG_SEEN:-}" && "${_INSTALL_FLAG_SEEN}" != "$want" ]]; then + tui_err "--install-missing and --no-install are mutually exclusive" >&2 + exit 2 + fi + _INSTALL_FLAG_SEEN="$want" + export TEXERA_INSTALL_MISSING="$want" +} + # --------- helpers --------- +# PID of whatever is listening on a TCP port, or nothing. Always exits 0 — +# "empty output" is the contract for "nothing is listening", and callers +# (svc_running_pid, wait_for_port, the status table) rely on that. +# +# lsof first: it ships with macOS and is what this has always used. But it is +# not a default package on Debian/Ubuntu/Fedora, and when it is missing every +# native service reads as stopped — `up` then relaunches services that are +# already running and `down` silently no-ops. So fall back to `ss` from +# iproute2, which is essential on any modern Linux. listen_pid_for_port() { - # || true so pipefail doesn't kill us when nothing is listening - lsof -nP -iTCP:"$1" -sTCP:LISTEN -t 2>/dev/null | head -1 || true + local port="$1" pid="" + if command -v lsof >/dev/null 2>&1; then + # || true so pipefail doesn't kill us when nothing is listening + pid=$(lsof -nP -iTCP:"$port" -sTCP:LISTEN -t 2>/dev/null | head -1 || true) + [[ -n "$pid" ]] && { printf '%s\n' "$pid"; return 0; } + fi + if command -v ss >/dev/null 2>&1; then + # `ss -lntp` prints e.g. + # LISTEN 0 511 127.0.0.1:8080 0.0.0.0:* users:(("java",pid=4827,fd=9)) + # Match on the local-address column ending in : so :18080 doesn't + # answer for :8080. No -H: it postdates the iproute2 in older distros. + pid=$(ss -lntp 2>/dev/null \ + | awk -v suffix=":$port" '$4 ~ suffix"$" { print; exit }' \ + | grep -o 'pid=[0-9]*' | head -1 | cut -d= -f2 || true) + [[ -n "$pid" ]] && { printf '%s\n' "$pid"; return 0; } + fi + return 0 +} + +# Modification time of a file as "YYYY-MM-DD HH:MM", in whichever stat dialect +# is present. Non-zero and silent if the file is missing or unreadable, so +# callers can substitute their own placeholder. +# +# GNU coreutils and BSD disagree completely here: BSD's `stat -f FMT -t TIMEFMT` +# reads `-f` as "file system info" under GNU, which then treats the format +# strings as filenames — it writes a diagnostic to stderr, prints filesystem +# fields on stdout and exits 1. Probe rather than branch on `uname` so a +# GNU-coreutils macOS also works. +_file_mtime_str() { + local f="${1:-}" + [[ -n "$f" && -e "$f" ]] || return 1 + if stat -c '%y' "$f" >/dev/null 2>&1; then + # GNU: "2026-07-29 20:06:12.123456789 +0000" → keep date + HH:MM + stat -c '%y' "$f" 2>/dev/null | cut -c1-16 + else + stat -f "%Sm" -t "%Y-%m-%d %H:%M" "$f" 2>/dev/null + fi } # Branch + short-sha of the deploy source ($REPO_ROOT), tab-separated, each @@ -1671,14 +2029,14 @@ svc_artifact_mtime() { done <<< "$globbed" fi if [[ ${#main_jars[@]} -gt 0 ]]; then - stat -f "%Sm" -t "%Y-%m-%d %H:%M" "${main_jars[0]}" + _file_mtime_str "${main_jars[0]}" || echo "—" return fi fi echo "—" ;; - bun) stat -f "%Sm" -t "%Y-%m-%d %H:%M" "$(amap_get SVC_CWD "$svc")/bun.lock" 2>/dev/null || echo "—" ;; - yarn) stat -f "%Sm" -t "%Y-%m-%d %H:%M" "$(amap_get SVC_CWD "$svc")/yarn.lock" 2>/dev/null || echo "—" ;; + bun) _file_mtime_str "$(amap_get SVC_CWD "$svc")/bun.lock" || echo "—" ;; + yarn) _file_mtime_str "$(amap_get SVC_CWD "$svc")/yarn.lock" || echo "—" ;; docker) echo "—" ;; esac } @@ -2333,6 +2691,7 @@ cmd_up() { --skip-build) BUILD=no ;; --no-build) tui_err "--no-build was renamed to --skip-build" >&2; exit 2 ;; --json) JSON_OUT=true ;; + --install-missing|--no-install) _set_install_flag "$1" ;; # Deploy-target selectors are resolved at startup (they must precede # the build.sbt parse); accept and ignore them here. --worktree=*|--branch=*) selector_seen=true ;; @@ -2365,6 +2724,10 @@ cmd_up() { return $ec fi + # Resolve the interpreter the JVM services hand to Python UDF workers + # before anything is launched with that environment. + _require_udf_python + local n_skip=0 [[ -n "$SKIP_LIST" ]] && n_skip=$(echo "$SKIP_LIST" | tr ',' '\n' | wc -l | tr -d ' ') local skip_label="none" @@ -2525,6 +2888,7 @@ cmd_auto() { case "$1" in --skip=*) SKIP_LIST="${1#--skip=}" ;; --json) JSON_OUT=true ;; + --install-missing|--no-install) _set_install_flag "$1" ;; # Deploy-target selectors are resolved at startup; accept here. --worktree=*|--branch=*) ;; *) tui_err "unknown flag: $1" >&2; exit 2 ;; @@ -2534,6 +2898,8 @@ cmd_auto() { # See cmd_up: human progress to stderr, JSON summary on real stdout (fd 3). if $JSON_OUT; then exec 3>&1 1>&2; fi + _require_udf_python + tui_banner "Texera Local Dev — auto bounce" \ "rebuild + bounce only what changed since last build" if [[ "$REPO_ROOT" != "$SELF_ROOT" ]]; then @@ -2839,6 +3205,8 @@ cmd_up_one() { fi local type="" type=$(amap_get SVC_TYPE "$svc") + # Docker services never launch Python workers; don't pay for the probe. + [[ "$type" == "docker" ]] || _require_udf_python case "$type" in docker) # No source to build — --build degrades to a restart, everything diff --git a/bin/local-dev/tests/test_local_dev_sh.sh b/bin/local-dev/tests/test_local_dev_sh.sh index c8d5e1d4179..b5b330cbc68 100755 --- a/bin/local-dev/tests/test_local_dev_sh.sh +++ b/bin/local-dev/tests/test_local_dev_sh.sh @@ -20,10 +20,14 @@ # bash bin/local-dev/tests/test_local_dev_sh.sh # Exits 0 if every check passes, 1 otherwise. # -# Kept deliberately small: bringing up the actual stack needs Docker / -# sbt / a Mac and is out of scope for CI here. We cover the things that -# regress quietly — script syntax, version-detection, the subcommand -# dispatch, and graceful failure on garbage input. +# Kept deliberately small: bringing up the actual stack needs Docker, sbt and +# the rest of the toolchain, which is out of scope for CI here. We cover the +# things that regress quietly — script syntax, version-detection, the +# subcommand dispatch, and graceful failure on garbage input. +# +# Runs on macOS and Linux alike (CI's `infra` job covers both). Anything that +# can only hold on one of them is guarded by a `uname` check or by probing for +# the tool involved, and reports a skip rather than a pass. set -u @@ -593,5 +597,539 @@ else _fail "changelog references missing files:$missing_files" fi +# 29) Host LAN IP detection on Linux. The macOS probes (`route get default`, +# `ipconfig getifaddr en0..en10`) do not exist there, so `up`/`auto` used to +# abort with "could not detect a host LAN IP" on every Linux box (#7065). +# Driven against a fake `ip` so the assertions don't depend on the test +# machine's interfaces. The docker-bridge case is the one that matters most: +# the whole point of HOST_LAN_IP is an address reachable from both the host +# and the lakekeeper container, and 172.17.0.1 is reachable from neither +# side the way we need. +lanip_fn=$(awk '/^_detect_host_lan_ip_linux\(\)/{f=1} f{print} f&&/^}/{exit}' "$MAIN_SH") +if [[ -z "$lanip_fn" ]]; then + _fail "_detect_host_lan_ip_linux helper missing" +else + _lan_dir=$(mktemp -d 2>/dev/null || mktemp -d -t ldlan) + # Fake `ip`: answers `route show default` from ip.route and + # `-o addr show [dev X] scope global` from ip.addr. + cat > "$_lan_dir/ip" <<'FAKE_IP' +#!/usr/bin/env bash +mode=""; want_dev="" +while [[ $# -gt 0 ]]; do + case "$1" in + route) mode=route ;; + addr) mode=addr ;; + dev) shift; want_dev="${1:-}" ;; + esac + shift +done +case "$mode" in + route) cat "$0.route" 2>/dev/null ;; + addr) + if [[ -n "$want_dev" ]]; then + awk -v d="$want_dev" '$2 == d' "$0.addr" 2>/dev/null + else + cat "$0.addr" 2>/dev/null + fi + ;; +esac +FAKE_IP + chmod +x "$_lan_dir/ip" + _lan_check() { # $1=label $2=expected stdout ("" = must fail) $3=route $4=addr + printf '%s\n' "$3" > "$_lan_dir/ip.route" + printf '%s\n' "$4" > "$_lan_dir/ip.addr" + local got="" rc=0 + got=$(PATH="$_lan_dir:$PATH"; eval "$lanip_fn"; _detect_host_lan_ip_linux) || rc=$? + if [[ -z "$2" ]]; then + if (( rc != 0 )) && [[ -z "$got" ]]; then + _pass "linux LAN IP: $1 (refuses)" + else + _fail "linux LAN IP: $1 should refuse" "rc=$rc got='$got'" + fi + elif [[ "$got" == "$2" ]]; then + _pass "linux LAN IP: $1 → $got" + else + _fail "linux LAN IP: $1" "expected '$2' got '$got' (rc=$rc)" + fi + } + _lan_check "default route interface wins" "10.10.10.30" \ + 'default via 10.10.10.1 dev eth0 proto static metric 100' \ + '2: eth0 inet 10.10.10.30/24 brd 10.10.10.255 scope global eth0 +3: docker0 inet 172.17.0.1/16 brd 172.17.255.255 scope global docker0' + _lan_check "no default route: skips the docker bridge" "10.10.10.30" \ + '' \ + '3: docker0 inet 172.17.0.1/16 brd 172.17.255.255 scope global docker0 +2: eth0 inet 10.10.10.30/24 brd 10.10.10.255 scope global eth0' + _lan_check "no default route: skips bridges, veth and tailscale" "192.168.1.42" \ + '' \ + '4: br-abc123 inet 172.18.0.1/16 scope global br-abc123 +5: veth9f2 inet 172.19.0.1/16 scope global veth9f2 +6: tailscale0 inet 100.101.102.103/32 scope global tailscale0 +2: enp5s0 inet 192.168.1.42/24 scope global enp5s0' + _lan_check "default route interface has no global v4: falls back to scan" "192.168.1.42" \ + 'default via 10.0.0.1 dev ppp0 proto static' \ + '2: enp5s0 inet 192.168.1.42/24 scope global enp5s0' + _lan_check "nothing but loopback" "" '' '' + _lan_check "only excluded interfaces" "" \ + '' \ + '3: docker0 inet 172.17.0.1/16 scope global docker0' + # Negative: no iproute2 at all must fail cleanly, not emit a bogus address. + _lan_no_ip=$(mktemp -d 2>/dev/null || mktemp -d -t ldlan2) + got=""; rc=0 + got=$(PATH="$_lan_no_ip"; eval "$lanip_fn"; _detect_host_lan_ip_linux) || rc=$? + if (( rc != 0 )) && [[ -z "$got" ]]; then + _pass "linux LAN IP: no \`ip\` on PATH (refuses)" + else + _fail "linux LAN IP: no \`ip\` should refuse" "rc=$rc got='$got'" + fi + rm -rf "$_lan_dir" "$_lan_no_ip" +fi + +# 30) The dispatcher must keep the macOS probes for Darwin and only use the +# Linux ones on Linux — the fix must not regress the platform that worked. +dispatch_fn=$(awk '/^_detect_host_lan_ip\(\)/{f=1} f{print} f&&/^}/{exit}' "$MAIN_SH") +if [[ "$dispatch_fn" == *"Darwin"* ]] \ + && [[ "$dispatch_fn" == *"_detect_host_lan_ip_darwin"* ]] \ + && [[ "$dispatch_fn" == *"_detect_host_lan_ip_linux"* ]]; then + _pass "_detect_host_lan_ip dispatches per platform" +else + _fail "_detect_host_lan_ip doesn't dispatch to both platform probes" +fi +# And the FATAL must stop naming only the macOS probes, since on Linux none of +# them ever ran. +if ! grep -q 'of \\`route get default\\` / en0-en10 had a non-loopback IPv4' "$MAIN_SH"; then + _pass "host-LAN-IP failure message is not macOS-only" +else + _fail "host-LAN-IP failure message still blames route get/en0-en10 on every platform" +fi + +# 31) Artifact mtime must be readable with either stat dialect. GNU coreutils +# rejects BSD's `stat -f "%Sm" -t FMT` outright (it reads -f as "file system +# info" and the format strings as filenames), so on Linux the ARTIFACT MTIME +# column rendered stat's error text instead of a timestamp. +mtime_fn=$(awk '/^_file_mtime_str\(\)/{f=1} f{print} f&&/^}/{exit}' "$MAIN_SH") +if [[ -z "$mtime_fn" ]]; then + _fail "_file_mtime_str helper missing" +else + _mt_dir=$(mktemp -d 2>/dev/null || mktemp -d -t ldmt) + : > "$_mt_dir/artifact.jar" + got=$(eval "$mtime_fn"; _file_mtime_str "$_mt_dir/artifact.jar") + if [[ "$got" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}\ [0-9]{2}:[0-9]{2}$ ]]; then + _pass "_file_mtime_str formats YYYY-MM-DD HH:MM ($got)" + else + _fail "_file_mtime_str wrong format" "got '$got'" + fi + # Negative: a missing file must fail rather than print stat's diagnostics. + got=""; rc=0 + got=$(eval "$mtime_fn"; _file_mtime_str "$_mt_dir/nope.jar" 2>&1) || rc=$? + if (( rc != 0 )) && [[ -z "$got" ]]; then + _pass "_file_mtime_str refuses a missing file quietly" + else + _fail "_file_mtime_str should fail quietly on a missing file" "rc=$rc got='$got'" + fi + got=""; rc=0 + got=$(eval "$mtime_fn"; _file_mtime_str 2>&1) || rc=$? + if (( rc != 0 )) && [[ -z "$got" ]]; then + _pass "_file_mtime_str refuses a missing argument quietly" + else + _fail "_file_mtime_str should fail quietly with no argument" "rc=$rc got='$got'" + fi + rm -rf "$_mt_dir" +fi +# The BSD spelling is allowed to survive inside _file_mtime_str — that's the +# whole point of the helper — but nowhere else, or the next call site silently +# breaks on Linux again. +main_outside_helper=$(awk ' + /^_file_mtime_str\(\)/ { skip = 1 } + skip { if ($0 == "}") skip = 0; next } + { print } +' "$MAIN_SH") +if ! printf '%s\n' "$main_outside_helper" | grep -qE 'stat -f "%Sm" -t'; then + _pass "BSD stat is confined to _file_mtime_str" +else + _fail "main.sh still calls BSD-only \`stat -f \"%Sm\" -t\` outside _file_mtime_str" \ + "$(printf '%s\n' "$main_outside_helper" | grep -nE 'stat -f "%Sm" -t' | head -3 | tr '\n' '|')" +fi + +# 32) Port→PID lookup must not depend on lsof. It is standard on macOS but not +# installed by default on Debian/Ubuntu/Fedora, and without it every native +# service read as "stopped" — so `up` relaunched live services and `down` +# silently no-opped. Tested against a real listener, with lsof shimmed out. +listen_fn=$(awk '/^listen_pid_for_port\(\)/{f=1} f{print} f&&/^}/{exit}' "$MAIN_SH") +if [[ -z "$listen_fn" ]]; then + _fail "listen_pid_for_port helper missing" +elif ! command -v python3 >/dev/null 2>&1; then + _pass "skip: python3 not on PATH (port listener check)" +else + _lp_dir=$(mktemp -d 2>/dev/null || mktemp -d -t ldlp) + # Bind an ephemeral port, print it, then hold it open until killed. + python3 -c ' +import socket, sys, time +s = socket.socket(); s.bind(("127.0.0.1", 0)); s.listen(1) +print(s.getsockname()[1], flush=True) +time.sleep(120) +' > "$_lp_dir/port" 2>/dev/null & + _lp_pid=$! + _lp_port="" + for _ in 1 2 3 4 5 6 7 8 9 10; do + _lp_port=$(head -1 "$_lp_dir/port" 2>/dev/null) + [[ -n "$_lp_port" ]] && break + sleep 0.3 + done + if [[ -z "$_lp_port" ]]; then + _fail "port listener check: could not start a listener" + else + got=$(eval "$listen_fn"; listen_pid_for_port "$_lp_port") + if [[ "$got" == "$_lp_pid" ]]; then + _pass "listen_pid_for_port finds the listener ($_lp_port → $got)" + else + _fail "listen_pid_for_port didn't find the listener" \ + "port=$_lp_port expected=$_lp_pid got='$got'" + fi + # Same answer with lsof unavailable — the Linux-without-lsof box. The + # fallback is `ss` from iproute2, which only exists on Linux; macOS has + # no equivalent and ships lsof in the base system, so there is nothing + # to fall back to (or to test) there. + if command -v ss >/dev/null 2>&1; then + printf '#!/bin/sh\nexit 127\n' > "$_lp_dir/lsof"; chmod +x "$_lp_dir/lsof" + got=$(PATH="$_lp_dir:$PATH"; eval "$listen_fn"; listen_pid_for_port "$_lp_port") + if [[ "$got" == "$_lp_pid" ]]; then + _pass "listen_pid_for_port works without lsof ($got)" + else + _fail "listen_pid_for_port needs lsof" \ + "port=$_lp_port expected=$_lp_pid got='$got'" + fi + else + _pass "skip: no \`ss\` on this platform (lsof fallback is Linux-only)" + fi + # Negative: a port nobody listens on yields empty output, not noise. + got=$(eval "$listen_fn"; listen_pid_for_port 1 2>&1) + if [[ -z "$got" ]]; then + _pass "listen_pid_for_port: unused port yields nothing" + else + _fail "listen_pid_for_port: unused port produced output" "got='$got'" + fi + fi + kill "$_lp_pid" 2>/dev/null + wait "$_lp_pid" 2>/dev/null + rm -rf "$_lp_dir" +fi + +# 33) Install hints have to be actionable on Linux: `docker-compose-plugin` +# only exists in Docker's own apt repo (stock Ubuntu ships +# `docker-compose-v2`), and node/yarn/bun had no Linux line at all. +hint_body=$(awk '/^_install_hint\(\)/{f=1} f{print} f&&/^}/{exit}' "$MAIN_SH") +if [[ "$hint_body" != *"docker-compose-plugin"* ]] \ + && [[ "$hint_body" == *"docker-compose-v2"* ]]; then + _pass "docker hint names the distro compose package" +else + _fail "docker hint still points at docker-compose-plugin" +fi +hint_missing="" +for tool in java python node yarn bun sbt docker; do + case_body=$(printf '%s\n' "$hint_body" | awk -v t="$tool" ' + $0 ~ "^[[:space:]]*"t"\\)" {f=1} + f {print} + f && /;;/ {exit}') + [[ "$case_body" == *"Linux"* ]] || hint_missing+=" $tool" +done +if [[ -z "$hint_missing" ]]; then + _pass "every install hint has a Linux line" +else + _fail "install hints with no Linux line:$hint_missing" +fi + +# -------------------------------------------------------------------------- +# Toolchain detection + consented install (#7066). +# Everything below goes through the *decision* helpers, which are pure: they +# print what would be done and never install anything. That separation is the +# reason this is testable in CI at all. +# -------------------------------------------------------------------------- +_extract_fn() { # $1=function name → its definition, or empty + awk -v fn="$1" 'index($0, fn "()") == 1 {f=1} f{print} f && /^}/{exit}' "$MAIN_SH" +} + +# 34) Which package manager we'd use. On Linux the first of apt-get/dnf/yum/ +# pacman/zypper on PATH wins; with none present it must refuse rather than +# emit a bogus name that a caller would then try to run. +pm_fn=$(_extract_fn _pkg_manager) +if [[ -z "$pm_fn" ]]; then + _fail "_pkg_manager helper missing" +else + _pm_dir=$(mktemp -d 2>/dev/null || mktemp -d -t ldpm) + _pm_check() { # $1=label $2=expected ("" = must refuse) $3..=fake tools + local label="$1" expect="$2"; shift 2 + rm -f "$_pm_dir"/* + local t="" + for t in "$@"; do printf '#!/bin/sh\nexit 0\n' > "$_pm_dir/$t"; chmod +x "$_pm_dir/$t"; done + # PATH is stripped to the fakes so the host's real apt/dnf can't answer, + # but _pkg_manager still needs `uname` to know the platform. + ln -sf "$(command -v uname)" "$_pm_dir/uname" + local got="" rc=0 + got=$(PATH="$_pm_dir"; eval "$pm_fn"; _pkg_manager) || rc=$? + if [[ -z "$expect" ]]; then + if (( rc != 0 )) && [[ -z "$got" ]]; then + _pass "pkg manager: $label (refuses)" + else + _fail "pkg manager: $label should refuse" "rc=$rc got='$got'" + fi + elif [[ "$got" == "$expect" ]]; then + _pass "pkg manager: $label → $got" + else + _fail "pkg manager: $label" "expected '$expect' got '$got'" + fi + } + if [[ "$(uname -s)" == "Linux" ]]; then + _pm_check "apt-get" "apt-get" apt-get + _pm_check "dnf only" "dnf" dnf + _pm_check "pacman only" "pacman" pacman + _pm_check "apt-get preferred over dnf" "apt-get" apt-get dnf + _pm_check "nothing installed" "" + else + _pass "skip: package-manager PATH probing is Linux-only on this host" + fi + rm -rf "$_pm_dir" + if [[ "$pm_fn" == *brew* && "$pm_fn" == *Darwin* ]]; then + _pass "_pkg_manager keeps the Darwin/brew branch" + else + _fail "_pkg_manager lost the Darwin/brew branch" + fi +fi + +# 35) The command we'd run per tool. Java goes through the distro package +# manager first with SDKMAN as the no-sudo fallback; node uses nvm; python +# uses pyenv and must pin 3.12, the version AGENTS.md and the CI matrix +# both specify. Unknown or deliberately-unsupported tools must refuse +# instead of printing something a caller would run. +cmd_fn=$(_extract_fn _install_cmd_for) +if [[ -z "$cmd_fn" || -z "$pm_fn" ]]; then + _fail "_install_cmd_for helper missing" +else + _ic_dir=$(mktemp -d 2>/dev/null || mktemp -d -t ldic) + _ic() { # $1=tool $2..=fake pkg managers on PATH → prints the command + local tool="$1"; shift + rm -f "$_ic_dir"/* + local t="" + for t in "$@"; do printf '#!/bin/sh\nexit 0\n' > "$_ic_dir/$t"; chmod +x "$_ic_dir/$t"; done + ln -sf "$(command -v uname)" "$_ic_dir/uname" + ( PATH="$_ic_dir" + TEXERA_NODE_VERSION=24 TEXERA_PYTHON_VERSION=3.12 + eval "$pm_fn"; eval "$cmd_fn"; _install_cmd_for "$tool" ) 2>/dev/null + } + _ic_expect() { # $1=label $2=needle $3=tool $4..=fake managers + local label="$1" needle="$2" tool="$3"; shift 3 + local got="" + got=$(_ic "$tool" "$@") + if [[ "$got" == *"$needle"* ]]; then + _pass "install cmd: $label" + else + _fail "install cmd: $label" "expected to contain '$needle', got '$got'" + fi + } + # Which manager answers is platform-dependent: _pkg_manager's Darwin branch + # only ever looks for brew, so a faked apt-get on PATH is correctly ignored + # there. Assert each platform's own branch. + if [[ "$(uname -s)" == "Linux" ]]; then + _ic_expect "java via apt names openjdk-17-jdk" "openjdk-17-jdk" java apt-get + _ic_expect "java via apt asks for sudo explicitly" "sudo" java apt-get + _ic_expect "java via dnf names java-17-openjdk-devel" "java-17-openjdk-devel" java dnf + else + _ic_expect "java via brew" "brew install openjdk@17" java brew + fi + # Platform-independent: with no manager at all we fall back to SDKMAN. + _ic_expect "java with no package manager falls back to SDKMAN" "sdk" java + _ic_expect "node uses nvm" "nvm install" node apt-get + _ic_expect "python uses pyenv" "pyenv install" python apt-get + _ic_expect "python pins 3.12" "3.12" python apt-get + for tool in docker sbt definitely-not-a-tool; do + got=""; rc=0 + got=$( PATH="$_ic_dir" + TEXERA_NODE_VERSION=24 TEXERA_PYTHON_VERSION=3.12 + eval "$pm_fn"; eval "$cmd_fn"; _install_cmd_for "$tool" 2>/dev/null ) || rc=$? + if (( rc != 0 )) && [[ -z "$got" ]]; then + _pass "install cmd: refuses '$tool'" + else + _fail "install cmd: should refuse '$tool'" "rc=$rc got='$got'" + fi + done + rm -rf "$_ic_dir" +fi + +# 36) Consent. The prompt is for humans; a non-interactive run (CI, cron, an +# agent following AGENTS.md's non-interactive subcommands) must keep the old +# behaviour and never block on a read. This is the most important test in +# this PR: a regression here hangs every automated `up`. +consent_fn=$(_extract_fn _consent_to_install) +if [[ -z "$consent_fn" ]]; then + _fail "_consent_to_install helper missing" +else + rc=0 + ( eval "$consent_fn"; _consent_to_install java "sudo apt-get install -y openjdk-17-jdk" ) \ + /dev/null 2>&1 || rc=$? + if (( rc != 0 )); then + _pass "consent: refuses without a TTY (never prompts)" + else + _fail "consent: granted install without a TTY" + fi + rc=0 + ( TEXERA_INSTALL_MISSING=1; eval "$consent_fn"; _consent_to_install java "cmd" ) \ + /dev/null 2>&1 || rc=$? + if (( rc == 0 )); then + _pass "consent: TEXERA_INSTALL_MISSING=1 assumes yes" + else + _fail "consent: TEXERA_INSTALL_MISSING=1 didn't assume yes" "rc=$rc" + fi + rc=0 + ( TEXERA_INSTALL_MISSING=0; eval "$consent_fn"; _consent_to_install java "cmd" ) \ + /dev/null 2>&1 || rc=$? + if (( rc != 0 )); then + _pass "consent: TEXERA_INSTALL_MISSING=0 refuses" + else + _fail "consent: TEXERA_INSTALL_MISSING=0 didn't refuse" + fi +fi + +# 37) Picking the interpreter that runs Python UDFs. The old default was +# `command -v python3` — the system interpreter, which has none of +# amber/requirements.txt — so UDFs died at worker launch on import errors +# that pointed nowhere near the interpreter choice. +ver_fn=$(_extract_fn _python_version_of) +vok_fn=$(_extract_fn _python_version_ok) +dok_fn=$(_extract_fn _python_deps_ok) +if [[ -z "$ver_fn" || -z "$vok_fn" || -z "$dok_fn" ]]; then + _fail "python probe helpers missing (_python_version_of/_python_version_ok/_python_deps_ok)" +else + _py_dir=$(mktemp -d 2>/dev/null || mktemp -d -t ldpy) + # `python -c ...` is only ever asked for the version string or an import + # check, so scripts that echo a version / set an exit code drive both probes. + printf '#!/bin/sh\necho 3.12\n' > "$_py_dir/py312" + printf '#!/bin/sh\necho 3.11\n' > "$_py_dir/py311" + printf '#!/bin/sh\nexit 0\n' > "$_py_dir/deps-ok" + printf '#!/bin/sh\nexit 1\n' > "$_py_dir/deps-missing" + chmod +x "$_py_dir"/* + _py_probe() { # $1=fn $2=path → rc + local rc=0 + ( TEXERA_PYTHON_VERSION=3.12 + eval "$ver_fn"; eval "$vok_fn"; eval "$dok_fn" + "$1" "$2" ) >/dev/null 2>&1 || rc=$? + return $rc + } + if _py_probe _python_version_ok "$_py_dir/py312"; then + _pass "python probe: 3.12 accepted" + else + _fail "python probe: 3.12 rejected" + fi + if ! _py_probe _python_version_ok "$_py_dir/py311"; then + _pass "python probe: 3.11 rejected" + else + _fail "python probe: 3.11 accepted (must pin 3.12)" + fi + if ! _py_probe _python_version_ok "$_py_dir/nonexistent"; then + _pass "python probe: missing interpreter rejected" + else + _fail "python probe: missing interpreter accepted" + fi + if _py_probe _python_deps_ok "$_py_dir/deps-ok"; then + _pass "python probe: importable deps accepted" + else + _fail "python probe: importable deps rejected" + fi + if ! _py_probe _python_deps_ok "$_py_dir/deps-missing"; then + _pass "python probe: missing deps rejected" + else + _fail "python probe: missing deps accepted" + fi + if ! _py_probe _python_deps_ok ""; then + _pass "python probe: empty path rejected" + else + _fail "python probe: empty path accepted" + fi + rm -rf "$_py_dir" +fi + +# 38) Candidate order. An activated venv must beat the sibling venv312 from +# AGENTS.md's layout, and both must beat whatever `python3` happens to be on +# PATH — that last one being the old, broken default. +cand_fn=$(_extract_fn _udf_python_candidates) +if [[ -z "$cand_fn" ]]; then + _fail "_udf_python_candidates helper missing" +else + _cd_dir=$(mktemp -d 2>/dev/null || mktemp -d -t ldcd) + mkdir -p "$_cd_dir/active/bin" "$_cd_dir/ws/venv312/bin" "$_cd_dir/ws/texera" + : > "$_cd_dir/active/bin/python"; chmod +x "$_cd_dir/active/bin/python" + : > "$_cd_dir/ws/venv312/bin/python"; chmod +x "$_cd_dir/ws/venv312/bin/python" + order=$( VIRTUAL_ENV="$_cd_dir/active" REPO_ROOT="$_cd_dir/ws/texera" \ + SELF_ROOT="$_cd_dir/ws/texera" TEXERA_PYTHON_VERSION=3.12 + eval "$cand_fn"; _udf_python_candidates 2>/dev/null ) + pos_active=$(printf '%s\n' "$order" | grep -n "active/bin/python" | head -1 | cut -d: -f1) + pos_venv=$(printf '%s\n' "$order" | grep -n "venv312/bin/python" | head -1 | cut -d: -f1) + pos_sys=$(printf '%s\n' "$order" | grep -nE "/python3$" | tail -1 | cut -d: -f1) + if [[ -n "$pos_active" && -n "$pos_venv" ]] && (( pos_active < pos_venv )); then + _pass "python candidates: \$VIRTUAL_ENV before sibling venv312" + else + _fail "python candidates: wrong order" \ + "active=$pos_active venv312=$pos_venv order=$(printf '%s' "$order" | tr '\n' '|')" + fi + if [[ -z "$pos_sys" ]] || { [[ -n "$pos_venv" ]] && (( pos_venv < pos_sys )); }; then + _pass "python candidates: venv312 before bare python3 on PATH" + else + _fail "python candidates: bare python3 outranks venv312" \ + "venv312=$pos_venv sys=$pos_sys order=$(printf '%s' "$order" | tr '\n' '|')" + fi + rm -rf "$_cd_dir" +fi + +# 39) The resolver must be wired into the paths that launch services, and must +# NOT run at source time — `status` / `--help` shouldn't pay for spawning +# interpreters. +if [[ -n "$(_extract_fn _require_udf_python)" ]]; then + _pass "_require_udf_python helper is defined" +else + _fail "_require_udf_python helper missing" +fi +for fn in cmd_up cmd_auto cmd_up_one; do + body=$(_extract_fn "$fn") + if [[ "$body" == *"_require_udf_python"* ]]; then + _pass "$fn resolves the UDF interpreter before launching" + else + _fail "$fn doesn't call _require_udf_python" + fi +done +# The UDF interpreter must not be advertised with the `-i` dashboard's hint: +# that one is about textual and amber/dev-requirements.txt, a different +# interpreter and a different requirements file. +# Comments are stripped: the code explains *why* it doesn't use that hint, and +# the explanation naturally names it. +udf_body=$(_extract_fn _require_udf_python | sed 's/^[[:space:]]*#.*$//') +if [[ "$udf_body" != *"_install_hint python"* ]] \ + && [[ "$udf_body" == *"UDF_PYTHON_PATH"* ]] \ + && [[ "$udf_body" == *"amber/requirements.txt"* ]]; then + _pass "_require_udf_python points at amber's requirements, not the TUI hint" +else + _fail "_require_udf_python reuses the TUI python hint or omits amber's requirements" +fi +if ! grep -qE '^export UDF_PYTHON_PATH="\$\{UDF_PYTHON_PATH:-\$\(command -v python3' "$MAIN_SH"; then + _pass "UDF_PYTHON_PATH no longer defaults to bare python3 at source time" +else + _fail "UDF_PYTHON_PATH still defaults to \`command -v python3\` at source time" +fi + +# 40) The new knobs are discoverable, and contradictory ones are refused rather +# than silently resolved one way. +help_out=$("$SCRIPT" --help 2>&1) +if [[ "$help_out" == *"--install-missing"* && "$help_out" == *"--no-install"* ]]; then + _pass "--help documents --install-missing / --no-install" +else + _fail "--help doesn't document the install flags" +fi +_im_state=$(mktemp -d 2>/dev/null || mktemp -d -t ldim) +out=$(TEXERA_LOCAL_DEV_DIR="$_im_state" "$SCRIPT" up --install-missing --no-install 2>&1); rc=$? +if (( rc == 2 )); then + _pass "up rejects --install-missing together with --no-install (rc=2)" +else + _fail "up accepted contradictory install flags" "rc=$rc out=$(echo "$out" | head -1)" +fi +rm -rf "$_im_state" + printf "\n%d passed, %d failed\n" "$PASS" "$FAIL" (( FAIL == 0 ))