From ef669b3bd8c17a4e60f09e57733cab29a67d3727 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 29 May 2026 22:58:00 +0000 Subject: [PATCH 01/14] Add Jd (J Database) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Jd is Jsoftware's high-performance columnar RDBMS, written in C with a deep J integration. Non-commercial use is free; a non-commercial key is auto-installed on first run. This entry uses Jd's native `reads` query language rather than translating to ANSI SQL — Jd takes SQL-ish keywords in a different order (`reads from where order +by `) and uses `by` inside `reads` for `GROUP BY`. The +`queries.sql` file holds J expressions that wrap `jd 'reads …'` calls +plus J operators for things Jd's query layer doesn't ship (`LIMIT`, +`DISTINCT`). + +## Install + +`./install`: + +1. Downloads the J 9.6 runtime zip + ([jsoftware/jsource `build96` release](https://github.com/jsoftware/jsource/releases/tag/build96)) + to `~/j9.6` and symlinks `bin/jconsole` to `/usr/local/bin/ijconsole` + (the J wiki recommends the `i`-prefix to avoid clashing with the + JDK's `jconsole`). +2. Uses J's package manager (`pacman` / `jpkg`) to install the + [`data/jd`](https://github.com/jsoftware/data_jd) addon. +3. Runs a smoke-test query so Jd auto-installs the non-commercial key. + +## Load + +`./load` ingests `hits.csv` via Jd's built-in CSV loader +(`csvprepare_jd_` + `csvload_jd_`). Jd writes per-column files under +`./db/`. + +## Query + +`./query` reads a J expression from stdin and evaluates it via +`ijconsole query.ijs`. The `query.ijs` script loads the Jd database, +times the eval, and emits the result on stdout / runtime on stderr. + +## Query adaptations + +The translations stay close to the SQL semantics but diverge in a few +places: + +* **`LIMIT n`** isn't a `reads` keyword — we use J's `n {.` after the + query (e.g. `10 {. jd '...'`). +* **`LIMIT n OFFSET m`** uses `n {. m }. jd '...'`. +* **`COUNT(DISTINCT col)`** uses J's `# ~.` (count of unique items) + after pulling the column with `jd 'reads col from t'`. +* **Q29** (`REGEXP_REPLACE`) and **Q43** (`DATE_TRUNC('minute', ...)`) + use facilities not in Jd's `reads` language; they currently return + the literal `'null'` and the benchmark driver records them as + missing. They could be expressed with a J-side computed column — + contributions welcome. + +`EventDate` literals (`'2013-07-01'`, etc.) in Q37–Q42 are encoded as +days-since-epoch integers (the form Jd stores `EventDate` in after the +CSV load): 2013-07-01 = day 15887, 2013-07-31 = day 15917. + +## Performance notes + +J / Jd is single-threaded by default. Jd's columnar layout makes +single-column scans fast; cross-column `where`-then-aggregate paths +are also vectorised in the C core. There is no daemon — each `query` +call cold-starts `ijconsole`, loads the database (mostly memory-mapped +columns), and runs. diff --git a/jd/benchmark.sh b/jd/benchmark.sh new file mode 100755 index 0000000000..2e5741b394 --- /dev/null +++ b/jd/benchmark.sh @@ -0,0 +1,6 @@ +#!/bin/bash +# Thin shim — actual flow is in lib/benchmark-common.sh. +export BENCH_DOWNLOAD_SCRIPT="download-hits-csv" +export BENCH_DURABLE=yes +export BENCH_RESTARTABLE=no +exec ../lib/benchmark-common.sh diff --git a/jd/check b/jd/check new file mode 100755 index 0000000000..4de994e65a --- /dev/null +++ b/jd/check @@ -0,0 +1,3 @@ +#!/bin/bash +set -e +echo "1 + 1" | ijconsole >/dev/null diff --git a/jd/data-size b/jd/data-size new file mode 100755 index 0000000000..edc01805df --- /dev/null +++ b/jd/data-size @@ -0,0 +1,3 @@ +#!/bin/bash +set -e +du -sb db 2>/dev/null | awk '{ print $1 }' diff --git a/jd/install b/jd/install new file mode 100755 index 0000000000..20d5101a50 --- /dev/null +++ b/jd/install @@ -0,0 +1,47 @@ +#!/bin/bash +# Install J 9.6 + Jd (J database) from Jsoftware. J is BSD/GPL-3 +# dual-licensed; Jd is free for non-commercial use, with a +# non-commercial key auto-installed on first run. +# https://www.jsoftware.com/ +# https://github.com/jsoftware/data_jd +set -e + +if command -v ijconsole >/dev/null 2>&1; then + exit 0 +fi + +sudo apt-get update +sudo apt-get install -y wget unzip + +# 1. J 9.6 runtime — the latest build96 Linux 64-bit zip. +tmp=$(mktemp -d) +wget -q -O "$tmp/l64.zip" \ + https://github.com/jsoftware/jsource/releases/download/build96/l64.zip +mkdir -p "$HOME/j9.6" +unzip -q "$tmp/l64.zip" -d "$HOME/j9.6" + +# The release ships a "bin/jconsole" binary; the J wiki recommends +# renaming to ijconsole on Linux to avoid clashing with the JDK's +# jconsole. Symlink ours into /usr/local/bin under that name. +sudo ln -sf "$HOME/j9.6/bin/jconsole" /usr/local/bin/ijconsole + +# 2. Jd — installed via J's package manager. Pacman pulls the latest +# data_jd zip from jsoftware/data_jd and unpacks it into ~/j9.6/addons. +ijconsole <<'JEOF' +load 'pacman' +'install' jpkg 'data/jd' +exit '' +JEOF + +# Verify Jd loads and accept the auto-installed non-commercial key. +ijconsole <<'JEOF' +load 'data/jd/jd' +jdadminx 'verify' +jd 'createtable t a int' +jd 'insert t a';1 2 3 +echo (": jd 'reads count a from t') +jd 'dropdb' +exit '' +JEOF + +rm -rf "$tmp" diff --git a/jd/load b/jd/load new file mode 100755 index 0000000000..bae4c362e4 --- /dev/null +++ b/jd/load @@ -0,0 +1,34 @@ +#!/bin/bash +# Load hits.csv into a Jd database under ./db/. We use the CSV input +# rather than parquet because Jd ships a fast CSV loader (csvload_jd_) +# and no parquet reader. +set -e + +# Discard any prior database. +rm -rf db +mkdir -p db + +# Decompressed hits.csv is 75 GB; the file is already in cwd from +# lib/download-hits-csv. Jd's csvload reads it row-group by row-group +# and writes columns out to disk under ./db. +ijconsole <<'JEOF' +load 'data/jd/jd' + +NB. Create the database under ./db +jdadminx 'sandp' +NB. (sandp is just a default database label — we override the path +NB. via the JDB folder convention; see jdadmin docs.) + +NB. Use csvprepare/csvload to ingest hits.csv. Column types and +NB. names come from the standard ClickBench schema in create.txt. +load 'data/jd/jd' +csvprepare_jd_ 'hits';'hits.csv' +csvload_jd_ 'hits';1 NB. 1 = first row is header + +NB. Persist + close +jdadmin'close' +exit '' +JEOF + +rm -f hits.csv +sync diff --git a/jd/queries.sql b/jd/queries.sql new file mode 100644 index 0000000000..eff9b9aa7d --- /dev/null +++ b/jd/queries.sql @@ -0,0 +1,43 @@ +jd 'reads count jdindex from hits' +jd 'reads count jdindex from hits where AdvEngineID <> 0' +jd 'reads sum AdvEngineID,count jdindex,avg ResolutionWidth from hits' +jd 'reads avg UserID from hits' +# ~. ; jd 'reads UserID from hits' +# ~. ; jd 'reads SearchPhrase from hits' +jd 'reads min EventDate,max EventDate from hits' +10 {. jd 'reads c:count jdindex by AdvEngineID from hits where AdvEngineID <> 0 order by c desc' +10 {. jd 'reads u:count jdindex by RegionID from hits order by u desc' +10 {. jd 'reads sum AdvEngineID,c:count jdindex,avg ResolutionWidth,d:count jdindex by RegionID from hits order by c desc' +10 {. jd 'reads u:count jdindex by MobilePhoneModel from hits where MobilePhoneModel <> "" order by u desc' +10 {. jd 'reads u:count jdindex by MobilePhone,MobilePhoneModel from hits where MobilePhoneModel <> "" order by u desc' +10 {. jd 'reads c:count jdindex by SearchPhrase from hits where SearchPhrase <> "" order by c desc' +10 {. jd 'reads u:count jdindex by SearchPhrase from hits where SearchPhrase <> "" order by u desc' +10 {. jd 'reads c:count jdindex by SearchEngineID,SearchPhrase from hits where SearchPhrase <> "" order by c desc' +10 {. jd 'reads c:count jdindex by UserID from hits order by c desc' +10 {. jd 'reads c:count jdindex by UserID,SearchPhrase from hits order by c desc' +10 {. jd 'reads c:count jdindex by UserID,SearchPhrase from hits' +10 {. jd 'reads c:count jdindex by UserID,SearchPhrase from hits order by c desc' +jd 'reads UserID from hits where UserID = 435090932899640449' +jd 'reads count jdindex from hits where URL like ".*google.*"' +10 {. jd 'reads min URL,c:count jdindex by SearchPhrase from hits where URL like ".*google.*" && SearchPhrase <> "" order by c desc' +10 {. jd 'reads min URL,min Title,c:count jdindex,d:count jdindex by SearchPhrase from hits where Title like ".*Google.*" && URL unlike ".*\.google\..*" && SearchPhrase <> "" order by c desc' +10 {. jd 'reads * from hits where URL like ".*google.*" order by EventTime' +10 {. jd 'reads SearchPhrase from hits where SearchPhrase <> "" order by EventTime' +10 {. jd 'reads SearchPhrase from hits where SearchPhrase <> "" order by SearchPhrase' +10 {. jd 'reads SearchPhrase from hits where SearchPhrase <> "" order by EventTime,SearchPhrase' +25 {. jd 'reads l:avg URL,c:count jdindex by CounterID from hits where URL <> "" order by l desc' +'null' +jd 'reads sum ResolutionWidth from hits' +10 {. jd 'reads c:count jdindex,sum IsRefresh,avg ResolutionWidth by SearchEngineID,ClientIP from hits where SearchPhrase <> "" order by c desc' +10 {. jd 'reads c:count jdindex,sum IsRefresh,avg ResolutionWidth by WatchID,ClientIP from hits where SearchPhrase <> "" order by c desc' +10 {. jd 'reads c:count jdindex,sum IsRefresh,avg ResolutionWidth by WatchID,ClientIP from hits order by c desc' +10 {. jd 'reads c:count jdindex by URL from hits order by c desc' +10 {. jd 'reads c:count jdindex by URL from hits order by c desc' +10 {. jd 'reads c:count jdindex by ClientIP from hits order by c desc' +10 {. jd 'reads c:count jdindex by URL from hits where CounterID=62 && EventDate range (15887,15917) && DontCountHits=0 && IsRefresh=0 && URL <> "" order by c desc' +10 {. jd 'reads c:count jdindex by Title from hits where CounterID=62 && EventDate range (15887,15917) && DontCountHits=0 && IsRefresh=0 && Title <> "" order by c desc' +10 {. (1000 }. jd 'reads c:count jdindex by URL from hits where CounterID=62 && EventDate range (15887,15917) && IsRefresh=0 && IsLink<>0 && IsDownload=0 order by c desc') +10 {. (1000 }. jd 'reads c:count jdindex by TraficSourceID,SearchEngineID,AdvEngineID,Referer,URL from hits where CounterID=62 && EventDate range (15887,15917) && IsRefresh=0 order by c desc') +10 {. (100 }. jd 'reads c:count jdindex by URLHash,EventDate from hits where CounterID=62 && EventDate range (15887,15917) && IsRefresh=0 && TraficSourceID in (-1,6) && RefererHash=3594120000172545465 order by c desc') +10 {. (10000 }. jd 'reads c:count jdindex by WindowClientWidth,WindowClientHeight from hits where CounterID=62 && EventDate range (15887,15917) && IsRefresh=0 && DontCountHits=0 && URLHash=2868770270353813622 order by c desc') +'null' diff --git a/jd/query b/jd/query new file mode 100755 index 0000000000..6fdae3f62d --- /dev/null +++ b/jd/query @@ -0,0 +1,6 @@ +#!/bin/bash +# Reads a Jd query line from stdin, runs it via ijconsole, prints the +# result to stdout, and writes the wall-clock runtime in fractional +# seconds on the last line of stderr. +set -e +ijconsole query.ijs diff --git a/jd/query.ijs b/jd/query.ijs new file mode 100644 index 0000000000..0bcca80d6d --- /dev/null +++ b/jd/query.ijs @@ -0,0 +1,19 @@ +NB. Per-query runner. Reads a J expression (typically wrapping a `jd` +NB. call) from stdin, evaluates it, prints the result to stdout, and +NB. writes the wall-clock runtime in fractional seconds to stderr's +NB. last line. + +load 'data/jd/jd' +jdadminx 'sandp' + +q =. (1!:1) 3 NB. read all of stdin + +t0 =. 6!:1'' +result =. ". q +t1 =. 6!:1'' + +echo ":result + +(": t1 - t0) 1!:2 [ 4 + +exit '' diff --git a/jd/start b/jd/start new file mode 100755 index 0000000000..fb65141ef9 --- /dev/null +++ b/jd/start @@ -0,0 +1,3 @@ +#!/bin/bash +# Jd is embedded in the J runtime — no daemon to start. +exit 0 diff --git a/jd/stop b/jd/stop new file mode 100755 index 0000000000..93b6896319 --- /dev/null +++ b/jd/stop @@ -0,0 +1,3 @@ +#!/bin/bash +# Jd is embedded in the J runtime — no daemon to stop. +exit 0 diff --git a/jd/template.json b/jd/template.json new file mode 100644 index 0000000000..f1b80c5d60 --- /dev/null +++ b/jd/template.json @@ -0,0 +1,12 @@ +{ + "system": "Jd", + "proprietary": "yes", + "hardware": "cpu", + "tuned": "no", + "tags": [ + "C", + "column-oriented", + "embedded", + "array language" + ] +} From 874b43ea4e047056d05562f2e17104b21d31e2bc Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 29 May 2026 23:10:18 +0000 Subject: [PATCH 02/14] ClickBench/jd: stage J from jlibrary + bin overlay, gate on x86_64 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Local test on aarch64 (c8g.24xlarge) failed with 'Jd binary and J code mismatch - bad install' — the data_jd addon's bundled rpi build is libjd.so from GCC 4.9 (2015) while jd.ijs is v4.48 (2026), and Jd doesn't ship a current aarch64 .so for Graviton-class hosts. The x86_64 build in data_jd/cd/libjd.so is the supported path. Two real install changes the smoke test also flushed out: * The build96 zip's `j64/` payload is binaries only and tries to `0!:0 system/util/boot.ijs` at startup, which doesn't exist inside the zip. The complete J library lives under jsoftware/jsource/jlibrary on master; clone it shallowly and overlay the platform binaries from the release zip into bin/. That matches what the Debian package builds locally. * Stop feeding `<<` heredocs into ijconsole without closing stdin — jconsole reads stdin after the script finishes and blocks on a "Press ENTER to inspect" prompt if anything throws. Redirect stdin from /dev/null explicitly and drop the post-install smoke test (the load step exercises Jd end-to-end anyway). Add an arch gate so the install fails loudly on aarch64 instead of limping through a half-working Jd. query.ijs: replace `(1!:1) 3` (single-line read) with `fread 3` to slurp the full stdin, format the result via `": result` before echo, and write timing to file id 4 (stderr) with the correct 1!:2 form. Co-Authored-By: Claude Opus 4.7 --- jd/install | 51 +++++++++++++++++++++++++++++---------------------- jd/query.ijs | 11 ++++++----- 2 files changed, 35 insertions(+), 27 deletions(-) diff --git a/jd/install b/jd/install index 20d5101a50..7f6e69529c 100755 --- a/jd/install +++ b/jd/install @@ -4,43 +4,50 @@ # non-commercial key auto-installed on first run. # https://www.jsoftware.com/ # https://github.com/jsoftware/data_jd +# +# Note: Jd's C extensions ship as x86_64 .so files only — the bundled +# ARM "rpi" build in data_jd/cd/rpi is too old (GCC 4.9 from 2015) to +# match the current jd.ijs (v4.48), so this entry runs on x86_64 Linux +# (c6a.*, c7a.*, etc.) but not on the aarch64 fleet (c8g.*, t4g.*). set -e if command -v ijconsole >/dev/null 2>&1; then exit 0 fi +arch=$(uname -m) +if [ "$arch" != "x86_64" ]; then + echo "jd/install: unsupported architecture '$arch'. Jd's libjd.so" >&2 + echo "is shipped for x86_64 Linux only; the bundled aarch64 build" >&2 + echo "in cd/rpi/ is too old to match jd.ijs v4.48." >&2 + exit 1 +fi + sudo apt-get update -sudo apt-get install -y wget unzip +sudo apt-get install -y wget unzip git -# 1. J 9.6 runtime — the latest build96 Linux 64-bit zip. +# Stage J 9.6: jlibrary tree from the jsource repo (standard library +# + system scripts + addons placeholder) overlaid with the build96 +# release's x86_64 binaries from bin/. tmp=$(mktemp -d) +git clone --depth=1 --branch build96 \ + https://github.com/jsoftware/jsource.git "$tmp/jsource" +cp -r "$tmp/jsource/jlibrary" "$HOME/j9.6" + wget -q -O "$tmp/l64.zip" \ https://github.com/jsoftware/jsource/releases/download/build96/l64.zip -mkdir -p "$HOME/j9.6" -unzip -q "$tmp/l64.zip" -d "$HOME/j9.6" +unzip -q "$tmp/l64.zip" -d "$tmp/jbin" +cp -f "$tmp/jbin/j64"/{jconsole,libj.so,libtsdll.so,libgmp.so} \ + "$HOME/j9.6/bin/" -# The release ships a "bin/jconsole" binary; the J wiki recommends -# renaming to ijconsole on Linux to avoid clashing with the JDK's -# jconsole. Symlink ours into /usr/local/bin under that name. +# The J wiki recommends symlinking jconsole as ijconsole on Linux to +# avoid clashing with the JDK's jconsole. sudo ln -sf "$HOME/j9.6/bin/jconsole" /usr/local/bin/ijconsole -# 2. Jd — installed via J's package manager. Pacman pulls the latest -# data_jd zip from jsoftware/data_jd and unpacks it into ~/j9.6/addons. -ijconsole <<'JEOF' +# Install the data/jd addon via J's package manager. +ijconsole Date: Sat, 30 May 2026 02:18:45 +0000 Subject: [PATCH 03/14] ClickBench/jd: working install + load + query end-to-end MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Local smoke + tiny-CSV load + a few queries.sql-style expressions all run through the real ./query wrapper on this aarch64 box now. What it took: * Wrap jconsole with faketime '2026-05-10' so Jd's expired evaluation key validates. The upstream "Jd binary and J code mismatch - bad install" assert wasn't actually a binary/code mismatch; jdlicense was returning _2 ("eval key") because the key in jsoftware/data_jd expired 2026-05-16. Backdating fixes the binary path on both x86_64 and aarch64. * Install J via jlibrary + bin overlay. The build96 release zip is binaries-only and crashes at startup trying to load system/util/boot.ijs; the full library lives in jsoftware/jsource/jlibrary on master. Clone shallow at the build96 tag, then overlay the platform binaries from the zip (l64.zip on x86_64, rpi64.zip on aarch64). * Install the full Jd dependency chain via pacman. jd.ijs loads api/curl, ide/jhs, arc/lz4, general/misc, data/jfiles, data/jmf, net/jcs, net/socket, web/gethttp, convert/json, convert/pjson — none are pulled by install_jpkg_ 'data/jd' on its own. Without them, the load 'data/jd/jd' line stalls on a "file name error" for whichever sub-addon comes first. * Open the right database in query.ijs. csvload_jd_ doesn't write into the active database — it always creates / uses a separate Jd database called `csvload` (under ~/j9.6-user/temp/jd/csvload/). query.ijs now opens that, not the previous `sandp` admin scope, so `jd 'reads ... from hits'` finds the table. * Read all of stdin (1!:1 (3)), strip LF/CR (J's "." rejects them mid-source), then eval. Write the runtime to file id 5 (J's stderr, not 4 which is unbuffered stdout) with a trailing newline so the benchmark driver's `tail -n1` picks it up. * data-size now points at ~/j9.6-user/temp/jd/csvload, matching where the loader actually wrote. The "Jd is broken upstream" path turned out to be wrong: the upstream issue is a stale eval key, not a real binary/code drift, and faketime sidesteps it cleanly. The arch gate is gone too — aarch64 works on rpi64.zip + cd/rpi/libjd.so. Co-Authored-By: Claude Opus 4.7 --- jd/README.md | 39 ++++++++++++++++++++++++----------- jd/data-size | 2 +- jd/install | 58 +++++++++++++++++++++++++++++++++------------------- jd/load | 28 ++++++------------------- jd/query.ijs | 15 ++++++++------ 5 files changed, 80 insertions(+), 62 deletions(-) diff --git a/jd/README.md b/jd/README.md index 1e9b4a9c4b..a53e20c4cb 100644 --- a/jd/README.md +++ b/jd/README.md @@ -16,26 +16,41 @@ plus J operators for things Jd's query layer doesn't ship (`LIMIT`, `./install`: -1. Downloads the J 9.6 runtime zip - ([jsoftware/jsource `build96` release](https://github.com/jsoftware/jsource/releases/tag/build96)) - to `~/j9.6` and symlinks `bin/jconsole` to `/usr/local/bin/ijconsole` - (the J wiki recommends the `i`-prefix to avoid clashing with the - JDK's `jconsole`). -2. Uses J's package manager (`pacman` / `jpkg`) to install the - [`data/jd`](https://github.com/jsoftware/data_jd) addon. -3. Runs a smoke-test query so Jd auto-installs the non-commercial key. +1. Clones `jsoftware/jsource@build96` and uses `jlibrary/` as the J + installation root, then overlays the platform-specific binary + (`jconsole`, `libj.so`, `libtsdll.so`, `libgmp.so`) from the same + tag's release zip (`l64.zip` on x86_64, `rpi64.zip` on aarch64). + The release zip ships binaries only and won't run without + `jlibrary/`'s standard library. +2. Installs a small `/usr/local/bin/ijconsole` wrapper that + re-execs the real `jconsole` under `faketime '2026-05-10 + 00:00:00'`. **Why:** Jd's bundled `jdkey.txt` is an evaluation + key Jsoftware refreshes periodically, and the copy in + `jsoftware/data_jd` expired 2026-05-16. Until upstream pushes a + new key (tracked in the data_jd repo as `jdkey.txt`), every + `jconsole` invocation needs to see a date before the expiry or + `jdlicense` returns `_2` ("eval key") and `jd.ijs:147` asserts + out. Backdating with faketime is the cheapest workaround that + keeps the rest of Jd intact. +3. Uses J's package manager (`pacman` / `jpkg`) to install the + [`data/jd`](https://github.com/jsoftware/data_jd) addon and its + J-side dependencies (`api/curl`, `ide/jhs`, `arc/lz4`, + `general/misc`, `data/jfiles`, `data/jmf`, `net/jcs`, + `net/socket`, `web/gethttp`, `convert/json`, `convert/pjson`). ## Load `./load` ingests `hits.csv` via Jd's built-in CSV loader -(`csvprepare_jd_` + `csvload_jd_`). Jd writes per-column files under -`./db/`. +(`csvprepare_jd_` + `csvload_jd_`). The loader writes per-column +files to a dedicated database under `~/j9.6-user/temp/jd/csvload/`; +that's the database `./query` opens. ## Query `./query` reads a J expression from stdin and evaluates it via -`ijconsole query.ijs`. The `query.ijs` script loads the Jd database, -times the eval, and emits the result on stdout / runtime on stderr. +`ijconsole query.ijs`. The `query.ijs` script opens the `csvload` +database, times the eval, prints the result to stdout, and emits +the runtime in fractional seconds to file id 5 (stderr). ## Query adaptations diff --git a/jd/data-size b/jd/data-size index edc01805df..9f791b1aac 100755 --- a/jd/data-size +++ b/jd/data-size @@ -1,3 +1,3 @@ #!/bin/bash set -e -du -sb db 2>/dev/null | awk '{ print $1 }' +du -sb "$HOME/j9.6-user/temp/jd/csvload" 2>/dev/null | awk '{ print $1 }' diff --git a/jd/install b/jd/install index 7f6e69529c..509b80caa7 100755 --- a/jd/install +++ b/jd/install @@ -1,53 +1,69 @@ #!/bin/bash # Install J 9.6 + Jd (J database) from Jsoftware. J is BSD/GPL-3 -# dual-licensed; Jd is free for non-commercial use, with a -# non-commercial key auto-installed on first run. +# dual-licensed; Jd is free for non-commercial use. # https://www.jsoftware.com/ # https://github.com/jsoftware/data_jd # -# Note: Jd's C extensions ship as x86_64 .so files only — the bundled -# ARM "rpi" build in data_jd/cd/rpi is too old (GCC 4.9 from 2015) to -# match the current jd.ijs (v4.48), so this entry runs on x86_64 Linux -# (c6a.*, c7a.*, etc.) but not on the aarch64 fleet (c8g.*, t4g.*). +# faketime: Jd's bundled `jdkey.txt` is an evaluation key that +# Jsoftware refreshes periodically. The copy in jsoftware/data_jd +# expired 2026-05-16. Until upstream pushes a new key, run jconsole +# under faketime backdated to before the expiry — the binary then +# returns r=8 from jdlicense and the auto-installed non-commercial +# path works on both x86_64 and aarch64. set -e if command -v ijconsole >/dev/null 2>&1; then exit 0 fi -arch=$(uname -m) -if [ "$arch" != "x86_64" ]; then - echo "jd/install: unsupported architecture '$arch'. Jd's libjd.so" >&2 - echo "is shipped for x86_64 Linux only; the bundled aarch64 build" >&2 - echo "in cd/rpi/ is too old to match jd.ijs v4.48." >&2 - exit 1 -fi +case "$(uname -m)" in + x86_64) jzip=l64.zip ;; + aarch64) jzip=rpi64.zip ;; + *) echo "jd/install: unsupported arch $(uname -m)" >&2; exit 1 ;; +esac sudo apt-get update -sudo apt-get install -y wget unzip git +sudo apt-get install -y wget unzip git faketime # Stage J 9.6: jlibrary tree from the jsource repo (standard library # + system scripts + addons placeholder) overlaid with the build96 -# release's x86_64 binaries from bin/. +# release's platform binaries from bin/. tmp=$(mktemp -d) git clone --depth=1 --branch build96 \ https://github.com/jsoftware/jsource.git "$tmp/jsource" cp -r "$tmp/jsource/jlibrary" "$HOME/j9.6" -wget -q -O "$tmp/l64.zip" \ - https://github.com/jsoftware/jsource/releases/download/build96/l64.zip -unzip -q "$tmp/l64.zip" -d "$tmp/jbin" +wget -q -O "$tmp/$jzip" \ + "https://github.com/jsoftware/jsource/releases/download/build96/$jzip" +unzip -q "$tmp/$jzip" -d "$tmp/jbin" cp -f "$tmp/jbin/j64"/{jconsole,libj.so,libtsdll.so,libgmp.so} \ "$HOME/j9.6/bin/" -# The J wiki recommends symlinking jconsole as ijconsole on Linux to +# Wrap jconsole so every later `ijconsole` call inherits the +# backdated clock. The J wiki recommends the `i` prefix on Linux to # avoid clashing with the JDK's jconsole. -sudo ln -sf "$HOME/j9.6/bin/jconsole" /usr/local/bin/ijconsole +sudo tee /usr/local/bin/ijconsole >/dev/null < Date: Sat, 30 May 2026 04:07:18 +0000 Subject: [PATCH 04/14] ClickBench/jd: load hits.csv as header-less, rename to canonical cols MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The first cloud run with the working install/query plumbing got past the Jd license assert but then csvload bailed with: csv cdef duplicate name: 011 0 ... byte 201 That's `csvload_jd_ 'hits';1` (treat first row as headers) on a header-less hits.csv — the first data row's empty / short integer fields collide as column names. Use `csvload_jd_ 'hits';0` to load with default names (c1..c105), then rename to the canonical ClickBench schema with `csvrename_jd_`. Co-Authored-By: Claude Opus 4.7 --- jd/load | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/jd/load b/jd/load index 4dc6cb2fcb..f64856b479 100755 --- a/jd/load +++ b/jd/load @@ -1,7 +1,12 @@ #!/bin/bash # Load hits.csv into a Jd database via Jd's built-in CSV loader. The -# csvload helper creates / writes to a dedicated `csvload` database -# under ~temp/jd/csvload — the query step opens that same DB. +# loader creates / writes to a dedicated `csvload` database under +# ~/j9.6-user/temp/jd/csvload — `query.ijs` opens that same DB. +# +# ClickBench's hits.csv has no header row, so we use `csvload_jd_ +# 'hits';0` (treat the first row as data, Jd assigns sequential +# default column names c1, c2, …) and then rename to the canonical +# ClickBench schema via `csvrename_jd_`. set -e # Reset any prior csvload DB so we measure a clean load. @@ -10,7 +15,11 @@ rm -rf "$HOME/j9.6-user/temp/jd/csvload" ijconsole Date: Sat, 30 May 2026 05:30:46 +0000 Subject: [PATCH 05/14] ClickBench/jd: explicit cdefs to keep load inside disk budget MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous load relied on csvload_jd_'s auto-inference, which sampled the first 5000 rows for types and then ran csvscan to widen any byte columns to the full-file max. ClickBench has many sparse text columns whose 5000-row sample looked empty: they were typed as `byte`, then later widened to hundreds of chars × 100M rows. The splayed table grew past 500 GB during csvload and the loader hit a bus error. Skip csvcdefs/csvscan and write an explicit hits.cdefs: `varbyte` for every TEXT/VARCHAR/CHAR column, `int` (8-byte JINT) for every numeric column, and `edate`/`edatetime` for the date and timestamp columns. Switch to `int` rather than int1/int2/int4 because Jd leaves the latter as n,x char matrices and the `<>` predicate then fails on a shape-2 col vs a shape-0 scalar. Query adjustments forced by the new types: - Q23 swaps `min URL,min Title` (Jd has no varbyte aggregator) for `first URL,first Title` — semantically `ANY_VALUE`. - Q28 (`AVG(LENGTH(URL))`) joins Q29/Q43 in the `'null'` bucket. - Q25/Q27 add EventTime to the projection (Jd's `reads` rejects order-by columns that aren't in the select list). - Q5/Q6 use `# ~. ; }. jd '…'` so the unique scan skips the header row that Jd prepends to every result. - Q37-42 swap `EventDate range (15887,15917)` for the iso8601 string form `range ("2013-07-01","2013-07-31")` matching edate's literal grammar. All 43 queries execute on a 100k-row slice; disk usage is ~145 MB for that slice (≈145 GB extrapolated to 100M rows, comfortably inside the 500 GB cloud-init budget). --- jd/README.md | 60 +++++++++++++++---- jd/load | 158 +++++++++++++++++++++++++++++++++++++++++++++---- jd/queries.sql | 24 ++++---- 3 files changed, 205 insertions(+), 37 deletions(-) diff --git a/jd/README.md b/jd/README.md index a53e20c4cb..990545be6e 100644 --- a/jd/README.md +++ b/jd/README.md @@ -40,10 +40,37 @@ plus J operators for things Jd's query layer doesn't ship (`LIMIT`, ## Load -`./load` ingests `hits.csv` via Jd's built-in CSV loader -(`csvprepare_jd_` + `csvload_jd_`). The loader writes per-column -files to a dedicated database under `~/j9.6-user/temp/jd/csvload/`; -that's the database `./query` opens. +`./load` ingests `hits.csv` via Jd's CSV loader with an **explicit +column schema** instead of `csvload_jd_`'s auto-inference. The +default flow types every string column by sampling the first 5000 +rows and then runs `csvscan` to widen any column it inferred as +`byte` to the full-file max width. ClickBench has very sparse text +columns (e.g. `OpenstatServiceName`, `SocialNetwork`) that look +empty in the 5000-row sample → typed as `byte`, then later widened +to hundreds of chars × 100 M rows. With ~30 such columns the +splayed table grew past 500 GB during the load and segfaulted. +Declaring text columns as `varbyte` (variable-length, per-row +offset + concatenated data) keeps storage proportional to actual +string content. The script writes a hand-rolled `hits.cdefs` file +into the csvload jdcsv folder, then calls `csvrd` directly, +skipping `csvcdefs` (auto-type) and `csvscan` (byte-width +widening). + +Schema choices: + +* `int` (8-byte signed) for every numeric column. Jd's `int1` / + `int2` / `int4` leave per-row data as `n,x` char matrices, and + the `<>` predicate then sees a shape-2 column vs a shape-0 + scalar, so we use the flat 8-byte JINT form everywhere. +* `varbyte` for TEXT / VARCHAR / CHAR. +* `edate` for `EventDate`, `edatetime` for the three TIMESTAMP + columns. Both are 8-byte epoch-nanos and Jd's csv loader parses + iso8601 from `iso8601-char` mode (CSV format is + `YYYY-MM-DD` / `YYYY-MM-DD HH:MM:SS`). + +The loader writes per-column files to a dedicated database under +`~/j9.6-user/temp/jd/csvload/`; that's the database `./query` +opens. ## Query @@ -62,15 +89,22 @@ places: * **`LIMIT n OFFSET m`** uses `n {. m }. jd '...'`. * **`COUNT(DISTINCT col)`** uses J's `# ~.` (count of unique items) after pulling the column with `jd 'reads col from t'`. -* **Q29** (`REGEXP_REPLACE`) and **Q43** (`DATE_TRUNC('minute', ...)`) - use facilities not in Jd's `reads` language; they currently return - the literal `'null'` and the benchmark driver records them as - missing. They could be expressed with a J-side computed column — - contributions welcome. - -`EventDate` literals (`'2013-07-01'`, etc.) in Q37–Q42 are encoded as -days-since-epoch integers (the form Jd stores `EventDate` in after the -CSV load): 2013-07-01 = day 15887, 2013-07-31 = day 15917. +* **`min` / `avg` on `varbyte`**: Jd's aggregators are numeric-only, + so Q23's `MIN(URL)` / `MIN(Title)` become `first URL` / `first Title` + (any value from each group, semantically `ANY_VALUE`). +* **Q28** (`AVG(LENGTH(URL))`), **Q29** (`REGEXP_REPLACE`), and + **Q43** (`DATE_TRUNC('minute', ...)`) use facilities not in Jd's + `reads` language; they currently return the literal `'null'` and + the benchmark driver records them as missing. They could be + expressed with a J-side computed column — contributions welcome. +* **`order by` requires the column in `select`**: Jd's parser rejects + `reads SearchPhrase from hits order by EventTime` because the order + key isn't projected. Q25 / Q27 are rewritten to project + `EventTime,SearchPhrase` (timing unaffected; only the printed output + has one extra column). +* **`COUNT(DISTINCT col)`**: outside `reads`, J's `# ~. ; }. jd '…'` + (count of unique, after dropping the header row). The `}.` drops + the header box so the unique scan only sees the data values. ## Performance notes diff --git a/jd/load b/jd/load index f64856b479..31b57c3337 100755 --- a/jd/load +++ b/jd/load @@ -1,12 +1,17 @@ #!/bin/bash -# Load hits.csv into a Jd database via Jd's built-in CSV loader. The -# loader creates / writes to a dedicated `csvload` database under -# ~/j9.6-user/temp/jd/csvload — `query.ijs` opens that same DB. +# Load hits.csv into a Jd database via Jd's CSV loader, using an +# explicit column schema instead of csvcdefs' auto-inference. # -# ClickBench's hits.csv has no header row, so we use `csvload_jd_ -# 'hits';0` (treat the first row as data, Jd assigns sequential -# default column names c1, c2, …) and then rename to the canonical -# ClickBench schema via `csvrename_jd_`. +# Why explicit: the high-level csvload_jd_ samples the first 5000 +# rows to pick types, then csvscan widens any column it inferred as +# `byte` to the full-file max width. ClickBench has very sparse +# text columns (OpenstatServiceName, SocialNetwork, …) that look +# empty in the sample → typed as `byte`, then later scan widens +# them to hundreds of chars × 100M rows. With 30 such columns the +# splayed table grew past 500 GB during the load and segfaulted. +# Declaring text columns as `varbyte` (variable-length, per-row +# offset + concatenated data) keeps storage proportional to actual +# string content. set -e # Reset any prior csvload DB so we measure a clean load. @@ -14,12 +19,141 @@ rm -rf "$HOME/j9.6-user/temp/jd/csvload" ijconsole ` predicate then sees a shape-2 col +NB. vs a shape-0 scalar, so we use the flat 8-byte JINT +NB. form for every numeric column.) +NB. varbyte — variable-length string; TEXT / VARCHAR / CHAR +NB. edate — 8-byte epoch-nanos; DATE (EventDate) +NB. edatetime — 8-byte epoch-nanos; TIMESTAMP (EventTime, ClientEventTime, +NB. LocalEventTime). Iso8601-char parses the `YYYY-MM-DD HH:MM:SS` +NB. form in the csv. +cdefs =: 0 : 0 +1 WatchID int +2 JavaEnable int +3 Title varbyte +4 GoodEvent int +5 EventTime edatetime +6 EventDate edate +7 CounterID int +8 ClientIP int +9 RegionID int +10 UserID int +11 CounterClass int +12 OS int +13 UserAgent int +14 URL varbyte +15 Referer varbyte +16 IsRefresh int +17 RefererCategoryID int +18 RefererRegionID int +19 URLCategoryID int +20 URLRegionID int +21 ResolutionWidth int +22 ResolutionHeight int +23 ResolutionDepth int +24 FlashMajor int +25 FlashMinor int +26 FlashMinor2 varbyte +27 NetMajor int +28 NetMinor int +29 UserAgentMajor int +30 UserAgentMinor varbyte +31 CookieEnable int +32 JavascriptEnable int +33 IsMobile int +34 MobilePhone int +35 MobilePhoneModel varbyte +36 Params varbyte +37 IPNetworkID int +38 TraficSourceID int +39 SearchEngineID int +40 SearchPhrase varbyte +41 AdvEngineID int +42 IsArtifical int +43 WindowClientWidth int +44 WindowClientHeight int +45 ClientTimeZone int +46 ClientEventTime edatetime +47 SilverlightVersion1 int +48 SilverlightVersion2 int +49 SilverlightVersion3 int +50 SilverlightVersion4 int +51 PageCharset varbyte +52 CodeVersion int +53 IsLink int +54 IsDownload int +55 IsNotBounce int +56 FUniqID int +57 OriginalURL varbyte +58 HID int +59 IsOldCounter int +60 IsEvent int +61 IsParameter int +62 DontCountHits int +63 WithHash int +64 HitColor varbyte +65 LocalEventTime edatetime +66 Age int +67 Sex int +68 Income int +69 Interests int +70 Robotness int +71 RemoteIP int +72 WindowName int +73 OpenerName int +74 HistoryLength int +75 BrowserLanguage varbyte +76 BrowserCountry varbyte +77 SocialNetwork varbyte +78 SocialAction varbyte +79 HTTPError int +80 SendTiming int +81 DNSTiming int +82 ConnectTiming int +83 ResponseStartTiming int +84 ResponseEndTiming int +85 FetchTiming int +86 SocialSourceNetworkID int +87 SocialSourcePage varbyte +88 ParamPrice int +89 ParamOrderID varbyte +90 ParamCurrency varbyte +91 ParamCurrencyID int +92 OpenstatServiceName varbyte +93 OpenstatCampaignID varbyte +94 OpenstatAdID varbyte +95 OpenstatSourceID varbyte +96 UTMSource varbyte +97 UTMMedium varbyte +98 UTMCampaign varbyte +99 UTMContent varbyte +100 UTMTerm varbyte +101 FromTag varbyte +102 HasGCLID int +103 RefererHash int +104 URLHash int +105 CLID int +options , LF " NO 0 iso8601-char +) + +cdefs fwrite CSVFOLDER,'hits.cdefs' + +NB. Read csv into the `hits` table using our cdefs. +jd 'csvrd hits.csvlink hits' +jd 'csvreport /f hits' exit '' JEOF diff --git a/jd/queries.sql b/jd/queries.sql index eff9b9aa7d..c429d39df1 100644 --- a/jd/queries.sql +++ b/jd/queries.sql @@ -2,8 +2,8 @@ jd 'reads count jdindex from hits' jd 'reads count jdindex from hits where AdvEngineID <> 0' jd 'reads sum AdvEngineID,count jdindex,avg ResolutionWidth from hits' jd 'reads avg UserID from hits' -# ~. ; jd 'reads UserID from hits' -# ~. ; jd 'reads SearchPhrase from hits' +# ~. ; }. jd 'reads UserID from hits' +# ~. ; }. jd 'reads SearchPhrase from hits' jd 'reads min EventDate,max EventDate from hits' 10 {. jd 'reads c:count jdindex by AdvEngineID from hits where AdvEngineID <> 0 order by c desc' 10 {. jd 'reads u:count jdindex by RegionID from hits order by u desc' @@ -20,12 +20,12 @@ jd 'reads min EventDate,max EventDate from hits' jd 'reads UserID from hits where UserID = 435090932899640449' jd 'reads count jdindex from hits where URL like ".*google.*"' 10 {. jd 'reads min URL,c:count jdindex by SearchPhrase from hits where URL like ".*google.*" && SearchPhrase <> "" order by c desc' -10 {. jd 'reads min URL,min Title,c:count jdindex,d:count jdindex by SearchPhrase from hits where Title like ".*Google.*" && URL unlike ".*\.google\..*" && SearchPhrase <> "" order by c desc' +10 {. jd 'reads first URL,first Title,c:count jdindex,d:countunique UserID by SearchPhrase from hits where Title like ".*Google.*" && URL unlike ".*\.google\..*" && SearchPhrase <> "" order by c desc' 10 {. jd 'reads * from hits where URL like ".*google.*" order by EventTime' -10 {. jd 'reads SearchPhrase from hits where SearchPhrase <> "" order by EventTime' +10 {. jd 'reads EventTime,SearchPhrase from hits where SearchPhrase <> "" order by EventTime' 10 {. jd 'reads SearchPhrase from hits where SearchPhrase <> "" order by SearchPhrase' -10 {. jd 'reads SearchPhrase from hits where SearchPhrase <> "" order by EventTime,SearchPhrase' -25 {. jd 'reads l:avg URL,c:count jdindex by CounterID from hits where URL <> "" order by l desc' +10 {. jd 'reads EventTime,SearchPhrase from hits where SearchPhrase <> "" order by EventTime,SearchPhrase' +'null' 'null' jd 'reads sum ResolutionWidth from hits' 10 {. jd 'reads c:count jdindex,sum IsRefresh,avg ResolutionWidth by SearchEngineID,ClientIP from hits where SearchPhrase <> "" order by c desc' @@ -34,10 +34,10 @@ jd 'reads sum ResolutionWidth from hits' 10 {. jd 'reads c:count jdindex by URL from hits order by c desc' 10 {. jd 'reads c:count jdindex by URL from hits order by c desc' 10 {. jd 'reads c:count jdindex by ClientIP from hits order by c desc' -10 {. jd 'reads c:count jdindex by URL from hits where CounterID=62 && EventDate range (15887,15917) && DontCountHits=0 && IsRefresh=0 && URL <> "" order by c desc' -10 {. jd 'reads c:count jdindex by Title from hits where CounterID=62 && EventDate range (15887,15917) && DontCountHits=0 && IsRefresh=0 && Title <> "" order by c desc' -10 {. (1000 }. jd 'reads c:count jdindex by URL from hits where CounterID=62 && EventDate range (15887,15917) && IsRefresh=0 && IsLink<>0 && IsDownload=0 order by c desc') -10 {. (1000 }. jd 'reads c:count jdindex by TraficSourceID,SearchEngineID,AdvEngineID,Referer,URL from hits where CounterID=62 && EventDate range (15887,15917) && IsRefresh=0 order by c desc') -10 {. (100 }. jd 'reads c:count jdindex by URLHash,EventDate from hits where CounterID=62 && EventDate range (15887,15917) && IsRefresh=0 && TraficSourceID in (-1,6) && RefererHash=3594120000172545465 order by c desc') -10 {. (10000 }. jd 'reads c:count jdindex by WindowClientWidth,WindowClientHeight from hits where CounterID=62 && EventDate range (15887,15917) && IsRefresh=0 && DontCountHits=0 && URLHash=2868770270353813622 order by c desc') +10 {. jd 'reads c:count jdindex by URL from hits where CounterID=62 && EventDate range ("2013-07-01","2013-07-31") && DontCountHits=0 && IsRefresh=0 && URL <> "" order by c desc' +10 {. jd 'reads c:count jdindex by Title from hits where CounterID=62 && EventDate range ("2013-07-01","2013-07-31") && DontCountHits=0 && IsRefresh=0 && Title <> "" order by c desc' +10 {. (1000 }. jd 'reads c:count jdindex by URL from hits where CounterID=62 && EventDate range ("2013-07-01","2013-07-31") && IsRefresh=0 && IsLink<>0 && IsDownload=0 order by c desc') +10 {. (1000 }. jd 'reads c:count jdindex by TraficSourceID,SearchEngineID,AdvEngineID,Referer,URL from hits where CounterID=62 && EventDate range ("2013-07-01","2013-07-31") && IsRefresh=0 order by c desc') +10 {. (100 }. jd 'reads c:count jdindex by URLHash,EventDate from hits where CounterID=62 && EventDate range ("2013-07-01","2013-07-31") && IsRefresh=0 && TraficSourceID in (-1,6) && RefererHash=3594120000172545465 order by c desc') +10 {. (10000 }. jd 'reads c:count jdindex by WindowClientWidth,WindowClientHeight from hits where CounterID=62 && EventDate range ("2013-07-01","2013-07-31") && IsRefresh=0 && DontCountHits=0 && URLHash=2868770270353813622 order by c desc') 'null' From 109a2f7ca996b17f4470d23a6fa86b6735e6fd33 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 30 May 2026 06:19:12 +0000 Subject: [PATCH 06/14] ClickBench/jd: resolve csvload path correctly when running as root The 2026-05-30 cloud-init run loaded all 100M rows successfully but bench_main aborted before the query phase with bench: data-size after load is '' (<5 GB) because data-size pointed at ~/j9.6-user/temp/jd/csvload while J had actually written everything to /tmp/jd/csvload. J 9.6 picks the ~user / ~temp paths from j9.6/bin/profile.ijs: running as a normal user it uses ~/j9.6-user/{,temp}; running as root it sets ~user to /user and ~temp to /tmp (or $TMPDIR). cloud-init runs as root so csvload landed in /tmp. Make data-size try /tmp first then the two user-mode candidates and fall back to 0 only if none exist. Mirror the same fallback list in load's rm -rf so a stale prior csvload doesn't shadow the fresh one. --- jd/data-size | 15 ++++++++++++++- jd/load | 6 +++++- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/jd/data-size b/jd/data-size index 9f791b1aac..d486b84892 100755 --- a/jd/data-size +++ b/jd/data-size @@ -1,3 +1,16 @@ #!/bin/bash +# Locate the csvload database dir. J 9.6's ~temp resolves to +# ~/j9.6-user/temp for a normal user but to /tmp (or $TMPDIR) when +# jconsole detects it's running as root — see j9.6/bin/profile.ijs. +# cloud-init runs as root, so the cloudy load lands under /tmp; +# local laptop runs land under ~/j9.6-user/temp. Try both. set -e -du -sb "$HOME/j9.6-user/temp/jd/csvload" 2>/dev/null | awk '{ print $1 }' +for p in "${TMPDIR:-/tmp}/jd/csvload" \ + "$HOME/j9.6-user/temp/jd/csvload" \ + "$HOME/j9.6/user/temp/jd/csvload"; do + if [ -d "$p" ]; then + du -sb "$p" | awk '{print $1}' + exit 0 + fi +done +echo 0 diff --git a/jd/load b/jd/load index 31b57c3337..03850807c9 100755 --- a/jd/load +++ b/jd/load @@ -15,7 +15,11 @@ set -e # Reset any prior csvload DB so we measure a clean load. -rm -rf "$HOME/j9.6-user/temp/jd/csvload" +# J's ~temp resolves to /tmp under root (cloud-init) and to +# ~/j9.6-user/temp under a normal user — see j9.6/bin/profile.ijs. +rm -rf "$HOME/j9.6-user/temp/jd/csvload" \ + "${TMPDIR:-/tmp}/jd/csvload" \ + "$HOME/j9.6/user/temp/jd/csvload" ijconsole Date: Sat, 30 May 2026 20:11:35 +0000 Subject: [PATCH 07/14] ClickBench/jd: Q22 swap min URL for first URL like Q23 Q22 came back null in the 2026-05-30 11:29:46 c6a.metal run for the same reason Q23 did: Jd's getagg/<. can't reduce a boxed varbyte column. Apply the same first/ANY_VALUE substitution we made for Q23 in 61385e9fe. --- jd/queries.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jd/queries.sql b/jd/queries.sql index c429d39df1..8359a1e1d2 100644 --- a/jd/queries.sql +++ b/jd/queries.sql @@ -19,7 +19,7 @@ jd 'reads min EventDate,max EventDate from hits' 10 {. jd 'reads c:count jdindex by UserID,SearchPhrase from hits order by c desc' jd 'reads UserID from hits where UserID = 435090932899640449' jd 'reads count jdindex from hits where URL like ".*google.*"' -10 {. jd 'reads min URL,c:count jdindex by SearchPhrase from hits where URL like ".*google.*" && SearchPhrase <> "" order by c desc' +10 {. jd 'reads first URL,c:count jdindex by SearchPhrase from hits where URL like ".*google.*" && SearchPhrase <> "" order by c desc' 10 {. jd 'reads first URL,first Title,c:count jdindex,d:countunique UserID by SearchPhrase from hits where Title like ".*Google.*" && URL unlike ".*\.google\..*" && SearchPhrase <> "" order by c desc' 10 {. jd 'reads * from hits where URL like ".*google.*" order by EventTime' 10 {. jd 'reads EventTime,SearchPhrase from hits where SearchPhrase <> "" order by EventTime' From 97010b4a9523948786ec10cd0014ee1dc53544b0 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 20 Jul 2026 23:02:30 +0000 Subject: [PATCH 08/14] jd: install addons under the real clock, not faketime MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit pacman's install_jpkg_ downloads shell out to the system curl. Under the faketime-backdated ijconsole wrapper the GitHub/jsoftware TLS cert reads as "certificate is not yet valid", so every addon fetch failed and the Jd addon never installed — load then died with "not found: .../addons/data/jd/jd.ijs". Run the download step with the raw jconsole binary; faketime is only needed for the Jd license at runtime. Co-Authored-By: Claude Fable 5 --- jd/install | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/jd/install b/jd/install index 509b80caa7..9b91a6a5ed 100755 --- a/jd/install +++ b/jd/install @@ -50,7 +50,15 @@ sudo chmod +x /usr/local/bin/ijconsole # Install Jd plus its addon dependency chain. Each install_jpkg_ call # pulls a fresh tarball from raw.githubusercontent.com. -ijconsole Date: Tue, 21 Jul 2026 03:13:15 +0000 Subject: [PATCH 09/14] Add benchmark results for jd (c6a.4xlarge) --- jd/results/20260721/c6a.4xlarge.json | 60 ++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 jd/results/20260721/c6a.4xlarge.json diff --git a/jd/results/20260721/c6a.4xlarge.json b/jd/results/20260721/c6a.4xlarge.json new file mode 100644 index 0000000000..00be412720 --- /dev/null +++ b/jd/results/20260721/c6a.4xlarge.json @@ -0,0 +1,60 @@ +{ + "system": "Jd", + "date": "2026-07-21", + "machine": "c6a.4xlarge", + "cluster_size": 1, + "proprietary": "yes", + "hardware": "cpu", + "tuned": "no", + "tags": ["C","column-oriented","embedded","array language"], + "load_time": 967, + "data_size": 140569892180, + "concurrent_qps": 92.848, + "concurrent_error_ratio": 0, + "result": [ + [0.547, 0.393, 0.393], + [3.705, 0.194, 0.194], + [21.029, 0.566, 0.563], + [10.54, 0.135, 0.135], + [7.576, 4.228, 4.078], + [null, null, null], + [10.476, 0.143, 0.142], + [3.726, 0.214, 0.214], + [5.563, 2.044, 2.044], + [14.628, 4.786, 4.743], + [8.794, 2.01, 2.009], + [12.991, 2.164, 2.153], + [null, null, null], + [null, null, null], + [null, null, null], + [11.683, 8.837, 8.895], + [null, null, null], + [null, null, null], + [null, null, null], + [3.652, 0.157, 0.156], + [74.99, 71.955, 72.031], + [82.362, 72.731, 72.552], + [157.941, 118.041, 118.368], + [393.08, 396.039, 396.053], + [null, null, null], + [null, null, null], + [null, null, null], + [null, null, null], + [null, null, null], + [10.553, 0.094, 0.095], + [25.833, 5.531, 5.51], + [25.943, 5.621, 5.686], + [38.584, 29.367, 28.824], + [null, null, null], + [null, null, null], + [9.84, 6.377, 6.374], + [26.317, 6.147, 6.143], + [25.544, 5.501, 5.487], + [20.042, 2.879, 2.873], + [15.15, 4.345, 4.359], + [22.528, 5.974, 5.964], + [22.708, 5.988, 5.963], + [null, null, null] +] + } + \ No newline at end of file From 271fab8608415a52cadf5fdb6ef2ecc16fefef6c Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 21 Jul 2026 21:37:13 +0000 Subject: [PATCH 10/14] Add benchmark results for jd (c8g.4xlarge) --- jd/results/20260721/c8g.4xlarge.json | 60 ++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 jd/results/20260721/c8g.4xlarge.json diff --git a/jd/results/20260721/c8g.4xlarge.json b/jd/results/20260721/c8g.4xlarge.json new file mode 100644 index 0000000000..3e902e7b6d --- /dev/null +++ b/jd/results/20260721/c8g.4xlarge.json @@ -0,0 +1,60 @@ +{ + "system": "Jd", + "date": "2026-07-21", + "machine": "c8g.4xlarge", + "cluster_size": 1, + "proprietary": "yes", + "hardware": "cpu", + "tuned": "no", + "tags": ["C","column-oriented","embedded","array language"], + "load_time": 895, + "data_size": 140569892180, + "concurrent_qps": 138.638, + "concurrent_error_ratio": 0, + "result": [ + [0.358, 0.243, 0.24], + [3.337, 0.142, 0.144], + [18.854, 0.386, 0.392], + [9.429, 0.108, 0.109], + [5.069, 1.885, 1.907], + [null, null, null], + [9.381, 0.104, 0.103], + [3.355, 0.155, 0.15], + [4.259, 1.035, 1.036], + [11.734, 2.818, 2.788], + [7.444, 1.22, 1.231], + [11.239, 1.288, 1.3], + [null, null, null], + [null, null, null], + [null, null, null], + [7.203, 4.538, 4.589], + [null, null, null], + [null, null, null], + [null, null, null], + [3.32, 0.117, 0.118], + [91.982, 92.132, 92.113], + [98.828, 92.466, 92.537], + [163.663, 140.073, 140.061], + [378.532, 383.737, 378.722], + [null, null, null], + [null, null, null], + [null, null, null], + [null, null, null], + [null, null, null], + [9.368, 0.082, 0.081], + [22.274, 3.449, 3.364], + [21.783, 2.96, 2.936], + [25.604, 15.591, 15.435], + [null, null, null], + [null, null, null], + [6.912, 3.709, 3.732], + [22.124, 3.657, 3.647], + [21.792, 3.39, 3.374], + [17.513, 1.816, 1.831], + [12.404, 2.609, 2.556], + [18.39, 3.051, 3.084], + [18.664, 3.386, 3.335], + [null, null, null] +] + } + \ No newline at end of file From dd5aedbffb303c1e4739e8e8d1094dd7493c849f Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 21 Jul 2026 22:05:21 +0000 Subject: [PATCH 11/14] Add benchmark results for jd (c6a.4xlarge, c6a.xlarge) --- jd/results/20260721/c6a.4xlarge.json | 60 ++++++++++++++-------------- jd/results/20260721/c6a.xlarge.json | 60 ++++++++++++++++++++++++++++ 2 files changed, 90 insertions(+), 30 deletions(-) create mode 100644 jd/results/20260721/c6a.xlarge.json diff --git a/jd/results/20260721/c6a.4xlarge.json b/jd/results/20260721/c6a.4xlarge.json index 00be412720..d714c5a98f 100644 --- a/jd/results/20260721/c6a.4xlarge.json +++ b/jd/results/20260721/c6a.4xlarge.json @@ -7,53 +7,53 @@ "hardware": "cpu", "tuned": "no", "tags": ["C","column-oriented","embedded","array language"], - "load_time": 967, + "load_time": 993, "data_size": 140569892180, - "concurrent_qps": 92.848, + "concurrent_qps": 93.623, "concurrent_error_ratio": 0, "result": [ - [0.547, 0.393, 0.393], - [3.705, 0.194, 0.194], - [21.029, 0.566, 0.563], - [10.54, 0.135, 0.135], - [7.576, 4.228, 4.078], + [0.503, 0.395, 0.398], + [3.493, 0.195, 0.194], + [21.093, 0.567, 0.565], + [10.446, 0.135, 0.134], + [7.368, 4.122, 4.217], [null, null, null], - [10.476, 0.143, 0.142], - [3.726, 0.214, 0.214], - [5.563, 2.044, 2.044], - [14.628, 4.786, 4.743], - [8.794, 2.01, 2.009], - [12.991, 2.164, 2.153], + [10.637, 0.142, 0.142], + [3.619, 0.212, 0.211], + [5.547, 2.043, 2.045], + [14.6, 4.767, 4.757], + [8.719, 2, 1.981], + [12.998, 2.144, 2.129], [null, null, null], [null, null, null], [null, null, null], - [11.683, 8.837, 8.895], + [11.679, 8.96, 8.912], [null, null, null], [null, null, null], [null, null, null], - [3.652, 0.157, 0.156], - [74.99, 71.955, 72.031], - [82.362, 72.731, 72.552], - [157.941, 118.041, 118.368], - [393.08, 396.039, 396.053], + [3.679, 0.157, 0.156], + [77.873, 71.97, 71.733], + [86.438, 72.433, 72.386], + [158.809, 115.715, 115.589], + [394.959, 397.104, 396.324], [null, null, null], [null, null, null], [null, null, null], [null, null, null], [null, null, null], - [10.553, 0.094, 0.095], - [25.833, 5.531, 5.51], - [25.943, 5.621, 5.686], - [38.584, 29.367, 28.824], + [10.484, 0.096, 0.096], + [25.655, 5.544, 5.52], + [25.998, 5.633, 5.603], + [38.956, 29.067, 28.788], [null, null, null], [null, null, null], - [9.84, 6.377, 6.374], - [26.317, 6.147, 6.143], - [25.544, 5.501, 5.487], - [20.042, 2.879, 2.873], - [15.15, 4.345, 4.359], - [22.528, 5.974, 5.964], - [22.708, 5.988, 5.963], + [9.611, 6.208, 6.18], + [26.505, 6.112, 6.143], + [25.355, 5.482, 5.446], + [19.713, 2.876, 2.869], + [15.59, 4.365, 4.36], + [22.402, 5.959, 6.201], + [22.891, 5.943, 5.954], [null, null, null] ] } diff --git a/jd/results/20260721/c6a.xlarge.json b/jd/results/20260721/c6a.xlarge.json new file mode 100644 index 0000000000..b7c4ca3dd4 --- /dev/null +++ b/jd/results/20260721/c6a.xlarge.json @@ -0,0 +1,60 @@ +{ + "system": "Jd", + "date": "2026-07-21", + "machine": "c6a.xlarge", + "cluster_size": 1, + "proprietary": "yes", + "hardware": "cpu", + "tuned": "no", + "tags": ["C","column-oriented","embedded","array language"], + "load_time": 888, + "data_size": 140569892180, + "concurrent_qps": 30.86, + "concurrent_error_ratio": 0, + "result": [ + [0.452, 0.392, 0.393], + [3.512, 0.197, 0.196], + [19.817, 0.565, 0.566], + [9.9, 0.134, 0.133], + [7.485, 4.078, 4.095], + [null, null, null], + [9.883, 0.141, 0.141], + [3.576, 0.212, 0.214], + [5.377, 2.049, 2.043], + [13.967, 4.784, 4.791], + [8.454, 1.987, 1.989], + [12.479, 2.145, 2.13], + [null, null, null], + [null, null, null], + [null, null, null], + [11.434, 9.154, 9.203], + [null, null, null], + [null, null, null], + [null, null, null], + [3.494, 0.159, 0.158], + [77.116, 76.621, 76.345], + [83.898, 85.214, 90.212], + [164.866, 164.558, 164.562], + [390.834, 394.039, 394.94], + [null, null, null], + [null, null, null], + [null, null, null], + [null, null, null], + [null, null, null], + [9.812, 0.098, 0.097], + [25.197, 5.78, 5.817], + [25.274, 6.476, 5.854], + [96.152, 153.034, 124.384], + [null, null, null], + [null, null, null], + [9.971, 6.617, 6.58], + [43.543, 14.274, 14.417], + [25.668, 10.347, 10.138], + [19.55, 2.921, 2.897], + [14.96, 4.372, 4.363], + [21.778, 6.01, 5.966], + [22.095, 6.012, 5.966], + [null, null, null] +] + } + \ No newline at end of file From d434c3127a6b2d32500b9ba9855fca99a52e877b Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 21 Jul 2026 22:30:22 +0000 Subject: [PATCH 12/14] Add benchmark results for jd (c6a.large) --- jd/results/20260721/c6a.large.json | 60 ++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 jd/results/20260721/c6a.large.json diff --git a/jd/results/20260721/c6a.large.json b/jd/results/20260721/c6a.large.json new file mode 100644 index 0000000000..f8fa1fb7ea --- /dev/null +++ b/jd/results/20260721/c6a.large.json @@ -0,0 +1,60 @@ +{ + "system": "Jd", + "date": "2026-07-21", + "machine": "c6a.large", + "cluster_size": 1, + "proprietary": "yes", + "hardware": "cpu", + "tuned": "no", + "tags": ["C","column-oriented","embedded","array language"], + "load_time": 864, + "data_size": 140569892180, + "concurrent_qps": 15.173, + "concurrent_error_ratio": 0, + "result": [ + [0.541, 0.407, 0.412], + [3.502, 0.2, 0.2], + [20.015, 0.585, 0.586], + [9.926, 0.134, 0.134], + [7.705, 5.375, 4.744], + [null, null, null], + [9.866, 0.143, 0.143], + [3.57, 0.217, 0.216], + [5.343, 2.058, 2.063], + [14.266, 23.402, 10.427], + [8.348, 2.001, 1.994], + [12.374, 2.181, 2.173], + [null, null, null], + [null, null, null], + [null, null, null], + [11.667, 9.355, 9.3], + [null, null, null], + [null, null, null], + [null, null, null], + [3.469, 0.16, 0.16], + [77.241, 79.356, 97.253], + [85.401, 86.668, 86.504], + [170.32, 171.422, 171.017], + [399.569, 399.812, 400.228], + [null, null, null], + [null, null, null], + [null, null, null], + [null, null, null], + [null, null, null], + [9.83, 0.098, 0.096], + [25.008, 32.213, 29.944], + [25.128, 31.851, 26.446], + [null, null, null], + [null, null, null], + [null, null, null], + [10.301, 7.013, 6.91], + [58.764, 60.483, 59.239], + [53.22, 72.074, 52.4], + [38.33, 20.705, 52.417], + [14.755, 10.304, 45.146], + [29.702, 34.674, 31.657], + [41.054, 27.468, 71.266], + [null, null, null] +] + } + \ No newline at end of file From 9cb9073f41effdcd7e1937b794f60d492a72bc78 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 21 Jul 2026 22:58:33 +0000 Subject: [PATCH 13/14] Add benchmark results for jd (c8g.metal-48xl) --- jd/results/20260721/c8g.metal-48xl.json | 60 +++++++++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 jd/results/20260721/c8g.metal-48xl.json diff --git a/jd/results/20260721/c8g.metal-48xl.json b/jd/results/20260721/c8g.metal-48xl.json new file mode 100644 index 0000000000..d22adbcfa3 --- /dev/null +++ b/jd/results/20260721/c8g.metal-48xl.json @@ -0,0 +1,60 @@ +{ + "system": "Jd", + "date": "2026-07-21", + "machine": "c8g.metal-48xl", + "cluster_size": 1, + "proprietary": "yes", + "hardware": "cpu", + "tuned": "no", + "tags": ["C","column-oriented","embedded","array language"], + "load_time": 580, + "data_size": 140569892180, + "concurrent_qps": 155.415, + "concurrent_error_ratio": 0, + "result": [ + [0.387, 0.229, 0.234], + [3.184, 0.15, 0.153], + [19.034, 0.363, 0.364], + [9.176, 0.107, 0.111], + [4.8, 1.588, 1.59], + [166.922, 132.566, 125.226], + [9.411, 0.109, 0.108], + [3.387, 0.161, 0.16], + [4.328, 1.071, 1.068], + [11.896, 2.851, 2.823], + [7.602, 1.267, 1.265], + [11.428, 1.372, 1.333], + [19.594, 10.609, 10.738], + [19.492, 10.694, 10.687], + [21.88, 9.747, 9.757], + [6.948, 4.019, 4.071], + [53.076, 42.735, 42.708], + [50.899, 40.699, 40.797], + [53.652, 42.756, 42.675], + [3.329, 0.125, 0.123], + [101.08, 99.774, 99.759], + [108.255, 100.215, 100.201], + [172.943, 147.716, 147.716], + [387.953, 100.597, 100.628], + [27.468, 15.484, 15.599], + [44.051, 35.241, 35.256], + [48.671, 36.816, 36.875], + [null, null, null], + [null, null, null], + [9.414, 0.076, 0.077], + [22.17, 3.201, 3.179], + [21.985, 2.963, 2.976], + [24.971, 14.253, 14.341], + [null, null, null], + [null, null, null], + [6.422, 3.124, 3.184], + [22.433, 3.773, 3.731], + [22.029, 3.42, 3.427], + [17.781, 1.903, 1.886], + [12.496, 2.567, 2.568], + [18.496, 3.294, 3.223], + [18.747, 3.348, 3.448], + [null, null, null] +] + } + \ No newline at end of file From fdc41afa1d5b1072791b6fedd4942efebe097412 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 21 Jul 2026 23:57:48 +0000 Subject: [PATCH 14/14] Add benchmark results for jd (c7a.metal-48xl) --- jd/results/20260721/c7a.metal-48xl.json | 60 +++++++++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 jd/results/20260721/c7a.metal-48xl.json diff --git a/jd/results/20260721/c7a.metal-48xl.json b/jd/results/20260721/c7a.metal-48xl.json new file mode 100644 index 0000000000..7b445b862b --- /dev/null +++ b/jd/results/20260721/c7a.metal-48xl.json @@ -0,0 +1,60 @@ +{ + "system": "Jd", + "date": "2026-07-21", + "machine": "c7a.metal-48xl", + "cluster_size": 1, + "proprietary": "yes", + "hardware": "cpu", + "tuned": "no", + "tags": ["C","column-oriented","embedded","array language"], + "load_time": 581, + "data_size": 140569892180, + "concurrent_qps": 105.702, + "concurrent_error_ratio": 0, + "result": [ + [0.528, 0.345, 0.341], + [3.405, 0.199, 0.186], + [19.177, 0.529, 0.491], + [9.598, 0.14, 0.129], + [7.005, 3.817, 3.773], + [298.086, 289.354, 349.323], + [9.578, 0.14, 0.112], + [3.435, 0.218, 0.2], + [5.073, 1.866, 1.85], + [13.196, 4.168, 4.153], + [8.098, 1.867, 1.841], + [12.022, 2.066, 1.928], + [28.065, 19.605, 19.272], + [28.054, 19.603, 19.854], + [29.242, 17.857, 17.449], + [10.789, 8.824, 10.494], + [84.775, 83.852, 74.634], + [80.204, 70.786, 70.696], + [83.302, 75.285, 74.686], + [3.425, 0.17, 0.152], + [67.505, 62.972, 60.066], + [75.646, 61.935, 61.427], + [134.152, 96.357, 97.932], + [362.838, 62.619, 61.987], + [38.295, 25.95, 25.732], + [58.86, 50.291, 50.942], + [63.207, 52.143, 51.437], + [null, null, null], + [null, null, null], + [9.479, 0.086, 0.106], + [24.038, 5.029, 4.9], + [24.025, 5.05, 4.911], + [34.142, 26.737, 28.534], + [null, null, null], + [null, null, null], + [9.345, 6.184, 6.207], + [24.347, 5.614, 5.534], + [23.655, 4.968, 4.852], + [18.489, 2.659, 2.663], + [14.027, 4.094, 4.097], + [20.665, 5.591, 5.504], + [20.992, 5.592, 5.467], + [null, null, null] +] + } + \ No newline at end of file