From ed40926492e17a6d5fc3b82d55d58aa9f08bf56a Mon Sep 17 00:00:00 2001 From: Thomas Honeyman Date: Mon, 6 Jul 2026 09:47:45 -0400 Subject: [PATCH 1/5] Give large package decodes an aggregate memory budget v0.9.11 serialised decodes of package docs JSON files of 5MB or more, but concurrent decodes of files just below that cutoff can still exhaust the heap: scrapers crawling the ~1,900 module pages of next-purs-rsc (a 4.4MB file) repeatedly took the server down by stacking four or five decodes of it at once (RTS heap-overflow aborts, systemd status=251). Decodes of files of 1MB or more now share a 16MB in-flight byte budget instead of a single >=5MB lock: a decode is admitted once the total size of large files currently being decoded fits within the budget, or immediately when it is the only large decode running, so files bigger than the budget itself are still served. The bounded queue and its 503 fast-fail behaviour are unchanged, as is the search index regeneration's exemption from the bound. --- CHANGELOG.md | 13 +++++++++ pursuit.cabal | 1 + src/Application.hs | 2 +- src/Foundation.hs | 10 +++---- src/Handler/Database.hs | 65 ++++++++++++++++++++++++++++++----------- 5 files changed, 68 insertions(+), 23 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3570528..3e7b0c5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,19 @@ the most up-to-date version of this file. ## Unreleased +- Give large package decodes an aggregate memory budget (@thomashoneyman) + + v0.9.11 serialised decodes of package docs JSON files of 5MB or more, but + concurrent decodes of files just below that cutoff could still exhaust the + heap: scrapers crawling the ~1,900 module pages of next-purs-rsc (a 4.4MB + file) repeatedly took the server down by stacking four or five decodes of + it at once. Decodes of files of 1MB or more now share a 16MB in-flight + budget instead: a decode starts once the total size of large files being + decoded fits within the budget (or immediately if it is the only large + decode running, so files bigger than the budget itself are still served). + The queue remains bounded, failing fast with a 503 when full, and the + search index regeneration remains exempt from the bound. + ## v0.9.11 - Serialise decoding of large package files (@thomashoneyman) diff --git a/pursuit.cabal b/pursuit.cabal index 4a67c9f..2cbad13 100644 --- a/pursuit.cabal +++ b/pursuit.cabal @@ -88,6 +88,7 @@ library , http-conduit , deepseq , directory + , stm , warp , data-default , aeson diff --git a/src/Application.hs b/src/Application.hs index 02ce7fc..5359bc2 100644 --- a/src/Application.hs +++ b/src/Application.hs @@ -59,7 +59,7 @@ makeFoundation appSettings = do appHttpManager <- newManager appLogger <- newStdoutLoggerSet defaultBufSize >>= makeYesodLogger appSearchIndex <- newTVarIO emptySearchIndex - appLargeDecodeLock <- newMVar () + appDecodeBytesInFlight <- newTVarIO 0 appLargeDecodeWaiters <- newTVarIO 0 let foundation = App{..} void (startRegenThread foundation) diff --git a/src/Foundation.hs b/src/Foundation.hs index 25f6d9d..9c7f802 100644 --- a/src/Foundation.hs +++ b/src/Foundation.hs @@ -68,12 +68,12 @@ data App = App , appHttpManager :: Manager , appLogger :: Logger , appSearchIndex :: TVar SearchIndex - , appLargeDecodeLock :: MVar () - -- ^ Held while decoding a large package's JSON; see - -- 'Handler.Database.lookupPackage'. + , appDecodeBytesInFlight :: TVar Integer + -- ^ The total file size of the large package JSONs currently being + -- decoded; see 'Handler.Database.lookupPackage'. , appLargeDecodeWaiters :: TVar Int - -- ^ The number of threads holding or waiting for 'appLargeDecodeLock', - -- used to bound the queue. + -- ^ The number of threads holding or waiting for a share of the decode + -- budget, used to bound the queue. } instance HasHttpManager App where diff --git a/src/Handler/Database.hs b/src/Handler/Database.hs index c468871..4a53271 100644 --- a/src/Handler/Database.hs +++ b/src/Handler/Database.hs @@ -14,6 +14,7 @@ module Handler.Database import Import import Language.PureScript.CoreFn.FromJSON (parseVersion') +import qualified Control.Concurrent.STM as STM import qualified Data.Aeson as A import qualified Data.NonNull as NN import qualified Data.Text as T @@ -101,7 +102,7 @@ lookupPackageWithPolicy policy pkgName version = do Nothing -> return Nothing Just size | size >= largeDecodeThreshold -> - withLargeDecodeLock policy (decodeFrom file) + withLargeDecodeBudget policy size (decodeFrom file) | otherwise -> decodeFrom file case mpkg of @@ -112,11 +113,11 @@ lookupPackageWithPolicy policy pkgName version = do dirExists <- liftIO $ doesDirectoryExist dir return $ Left $ if dirExists then NoSuchPackageVersion else NoSuchPackage where - -- The file is read inside the lock (rather than before queueing for it) so - -- that threads waiting their turn do not each pin a copy of a large file's - -- bytes. The decoded package is forced before the lock is released; the - -- decode is eager in practice, but we make certain the allocation spike - -- stays inside the lock. + -- The file is read inside the budget (rather than before queueing for it) + -- so that threads waiting their turn do not each pin a copy of a large + -- file's bytes. The decoded package is forced before the budget share is + -- released; the decode is eager in practice, but we make certain the + -- allocation spike stays inside the budget. decodeFrom :: String -> Handler (Maybe D.VerifiedPackage) decodeFrom file = do mcontents <- liftIO (readFileMay file) @@ -127,22 +128,37 @@ lookupPackageWithPolicy policy pkgName version = do pkg `deepseq` return (Just pkg) -- | Decoding a package's docs JSON transiently needs tens of times the file's --- size in memory, and a few generated packages (react-icons, elmish-html, --- deku, ...) have files of 10MB or more, so a handful of concurrent requests --- for their (rarely cached, because there are so many of them) documentation --- pages can exhaust the heap. Decodes of large files therefore run one at a --- time; packages below the threshold - nearly all of them - are unaffected. +-- size in memory, so concurrent requests for (rarely cached, because there +-- are so many of them) documentation pages of packages with large files can +-- exhaust the heap. Serialising only the decodes of files of 5MB or more +-- turned out not to be enough: a crawler fetching many pages of a package +-- whose file sits just below whatever cutoff is chosen (next-purs-rsc, at +-- 4.4MB, has ~1,900 module pages) stacks enough concurrent decodes to +-- exhaust the heap anyway. Decodes of large files therefore share an +-- aggregate budget: a decode is admitted once the total size of large files +-- being decoded fits within 'largeDecodeBudget', or when no other large +-- decode is running (so a single file bigger than the whole budget is still +-- served, by itself). Packages below the threshold - nearly all of them - +-- are unaffected. -- --- The queue for the lock is bounded: once 'maxLargeDecodeWaiters' threads --- hold or await the lock, further requests fail fast with a 503 rather than +-- The budget is measured in file bytes as a proxy for heap use, and it +-- bounds the decode spike, not the decoded package, which lives on a little +-- longer while the page renders. Admission is not first-come-first-served: a +-- large file's decode can be overtaken by smaller ones that fit the +-- remaining budget, which is acceptable because waiters are bounded and +-- contention is rare. +-- +-- The queue is bounded: once 'maxLargeDecodeWaiters' threads hold or await a +-- share of the budget, further requests fail fast with a 503 rather than -- accumulating without limit while clients time out. The hourly search index -- regeneration ('WaitWhenBusy') is exempt from the bound - it must not -- silently omit a package from the index just because the server is busy, and -- as a single sequential thread it adds at most one waiter. -withLargeDecodeLock :: LargeDecodePolicy -> Handler a -> Handler a -withLargeDecodeLock policy action = do +withLargeDecodeBudget :: LargeDecodePolicy -> Integer -> Handler a -> Handler a +withLargeDecodeBudget policy size action = do app <- getYesod let counter = appLargeDecodeWaiters app + let inFlight = appDecodeBytesInFlight app admitted <- case policy of WaitWhenBusy -> do atomically (modifyTVar' counter (+ 1)) @@ -156,7 +172,7 @@ withLargeDecodeLock policy action = do return True if admitted then - withMVar (appLargeDecodeLock app) (const action) + bracket_ (acquireBudget inFlight) (releaseBudget inFlight) action `finally` atomically (modifyTVar' counter (subtract 1)) else sendResponseStatus serviceUnavailable503 @@ -164,8 +180,23 @@ withLargeDecodeLock policy action = do where maxLargeDecodeWaiters = 16 :: Int + acquireBudget inFlight = atomically $ do + inUse <- readTVar inFlight + STM.check (inUse == 0 || inUse + size <= largeDecodeBudget) + writeTVar inFlight (inUse + size) + + releaseBudget inFlight = + atomically (modifyTVar' inFlight (subtract size)) + +-- | Files at least this large count against 'largeDecodeBudget' while they +-- are being decoded. largeDecodeThreshold :: Integer -largeDecodeThreshold = 5 * 1024 * 1024 +largeDecodeThreshold = 1024 * 1024 + +-- | The maximum total size of large package files being decoded at any one +-- time. +largeDecodeBudget :: Integer +largeDecodeBudget = 16 * 1024 * 1024 availableVersionsFor :: PackageName -> Handler [Version] availableVersionsFor pkgName = do From fcd398e18d4c79f93a91a21134578d2e4fe8774a Mon Sep 17 00:00:00 2001 From: Thomas Honeyman Date: Mon, 6 Jul 2026 09:50:40 -0400 Subject: [PATCH 2/5] Challenge scraper traffic to package pages with Anubis The OOM incidents of early July 2026 were driven by distributed scraper fleets - on 2026-07-06, ~4,800 distinct IPs presenting spoofed Chrome/Edge user agents crawled the ~1,900 module pages of next-purs-rsc - which robots.txt cannot address. Deploys now install Anubis (https://github.com/TecharoHQ/anubis), a challenge proxy, pinned by version and checksum in remote.sh with its config generated on first deploy. nginx routes cache misses under /packages/ (the decode-triggering routes) through Anubis on the way to the backend. Clients with browser-like user agents must solve a JavaScript proof-of-work challenge once and then carry a cookie; non-browser clients (curl, package uploads, editor tooling, badge fetchers) and known-good search crawlers pass through untouched. Cache hits, /search (including the DigitalOcean uptime check, which must observe the real backend), POST /packages uploads, and every other route bypass Anubis entirely. The bot policy is a verbatim copy of the upstream v1.25.0 default, kept in the repo so deploys are reproducible and changes reviewable. --- CHANGELOG.md | 14 ++ deploy/SERVER.md | 45 +++++- deploy/anubis.botPolicies.yaml | 247 +++++++++++++++++++++++++++++++++ deploy/nginx.conf | 39 ++++++ deploy/remote.sh | 35 +++++ 5 files changed, 374 insertions(+), 6 deletions(-) create mode 100644 deploy/anubis.botPolicies.yaml diff --git a/CHANGELOG.md b/CHANGELOG.md index 3e7b0c5..f895da0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,20 @@ the most up-to-date version of this file. ## Unreleased +- Challenge browser-like scraper traffic to package pages with Anubis + (@thomashoneyman) + + Distributed scraper fleets (thousands of IPs presenting spoofed browser + user agents, so robots.txt does not apply) crawl the module documentation + pages of large generated packages hard enough to take the server down. + Deploys now install [Anubis](https://github.com/TecharoHQ/anubis), a + challenge proxy, and nginx routes cache misses under `/packages/` through + it: clients with browser-like user agents must pass a JavaScript + proof-of-work challenge once, while non-browser clients (curl, package + uploads, editor tooling, badge fetchers) and known-good search engine + crawlers pass through untouched. Cached package pages, `/search`, and all + other routes bypass Anubis entirely. + - Give large package decodes an aggregate memory budget (@thomashoneyman) v0.9.11 serialised decodes of package docs JSON files of 5MB or more, but diff --git a/deploy/SERVER.md b/deploy/SERVER.md index 9818465..3f6ee85 100644 --- a/deploy/SERVER.md +++ b/deploy/SERVER.md @@ -2,7 +2,7 @@ This file documents the state that lives only on the server or in the DigitalOcean account, which no deploy touches and which would otherwise be -tribal knowledge. Last verified: 2026-07-05. +tribal knowledge. Last verified: 2026-07-06. ## The droplet @@ -30,6 +30,37 @@ tribal knowledge. Last verified: 2026-07-05. - **Cache eviction**: `/etc/cron.weekly/pursuit-cache-eviction` (installed by `remote.sh` from `deploy/cache-eviction.sh`) deletes cache files not accessed in 90 days so the cache cannot fill the disk. +- **Anubis signing key**: `/etc/anubis/pursuit.env` is generated by + `remote.sh` on first deploy and then left alone; it contains the ed25519 + key that signs challenge-pass cookies. Deleting or regenerating it just + makes every browser re-solve a challenge once. + +## Anubis (bot challenge proxy) + +Scraper fleets - thousands of IPs, spoofed browser user agents, immune to +robots.txt - crawl the module docs pages of large generated packages hard +enough to exhaust the backend's heap (2026-07-06: ~4,800 IPs crawling +next-purs-rsc's ~1,900 module pages caused repeated heap-overflow restarts). +[Anubis](https://github.com/TecharoHQ/anubis) counters exactly this: requests +with browser-like user agents must pass a JavaScript proof-of-work challenge +once (then carry a cookie for a week); non-browser clients (curl, `pursuit` +uploads, editor tooling, GitHub's badge fetcher) and known-good search +crawlers pass through untouched. + +- Deployed by `remote.sh`: pinned `.deb` from upstream releases, config in + `/etc/anubis/pursuit.env`, policy installed on every deploy from + `deploy/anubis.botPolicies.yaml` (a verbatim copy of the upstream default). +- Runs as `anubis@pursuit.service` on `127.0.0.1:8923` (metrics on + `127.0.0.1:9823`), forwarding admitted traffic to the backend on `:3000`. +- nginx routes only *cache misses under `/packages/`* (plus Anubis's own + `/.within.website/` endpoints) through it - the expensive decode-triggering + routes. Cache hits, `/search` (including the uptime check, which must see + the real backend), `POST /packages` uploads, and everything else go + straight to the backend. +- **Emergency bypass**: if Anubis itself misbehaves, edit + `/etc/nginx/sites-enabled/pursuit.conf` to point the `@anubis` location's + `proxy_pass` at `http://127.0.0.1:3000`, then `systemctl reload nginx`. + (The next deploy restores the routing.) ## DigitalOcean account state @@ -49,9 +80,11 @@ tribal knowledge. Last verified: 2026-07-05. The whole-package docs JSON is the unit of storage; rendering any single documentation page decodes the entire file for that package version, which transiently needs tens of times the file size in heap. A few generated -packages (react-icons, elmish-html, deku, google-apps, ...) have 10-25MB -files, so concurrent uncached requests for their pages can exhaust the heap. -`Handler.Database.lookupPackage` therefore serialises decodes of files over -5MB, and the search index is built one package at a time +packages (react-icons, elmish-html, deku, next-purs-rsc, google-apps, ...) +have 4-25MB files, so concurrent uncached requests for their pages can +exhaust the heap. `Handler.Database.lookupPackage` therefore admits decodes +of files of 1MB or more against a shared 16MB in-flight budget, and the +search index is built one package at a time (`createSearchIndexFromDatabase`). If heap-exhaustion restarts reappear in -the journal (`journalctl -u pursuit | grep "Heap exhausted"`), start there. +the journal (`journalctl -u pursuit | grep -E "Heap exhausted|status=251"`), +start there. diff --git a/deploy/anubis.botPolicies.yaml b/deploy/anubis.botPolicies.yaml new file mode 100644 index 0000000..e84bf55 --- /dev/null +++ b/deploy/anubis.botPolicies.yaml @@ -0,0 +1,247 @@ +## Bot policy for the Anubis instance protecting pursuit.purescript.org. +## +## This is a verbatim copy of the default policy shipped with Anubis v1.25.0 +## (https://github.com/TecharoHQ/anubis/blob/v1.25.0/data/botPolicies.yaml), +## kept in the repo so deploys are reproducible and changes are reviewable. +## When upgrading the pinned Anubis version in remote.sh, diff this file +## against the new release's default and fold in upstream changes. +## +## remote.sh installs this to /etc/anubis/pursuit.botPolicies.yaml on every +## deploy; local edits on the server will be overwritten. + +## Anubis has the ability to let you import snippets of configuration into the main +## configuration file. This allows you to break up your config into smaller parts +## that get logically assembled into one big file. +## +## Of note, a bot rule can either have inline bot configuration or import a +## bot config snippet. You cannot do both in a single bot rule. +## +## Import paths can either be prefixed with (data) to import from the common/shared +## rules in the data folder in the Anubis source tree or will point to absolute/relative +## paths in your filesystem. If you don't have access to the Anubis source tree, check +## /usr/share/docs/anubis/data or in the tarball you extracted Anubis from. + +bots: + # You can import the entire default config with this macro: + # - import: (data)/meta/default-config.yaml + + # Pathological bots to deny + - # This correlates to data/bots/_deny-pathological.yaml in the source tree + # https://github.com/TecharoHQ/anubis/blob/main/data/bots/_deny-pathological.yaml + import: (data)/bots/_deny-pathological.yaml + - import: (data)/bots/aggressive-brazilian-scrapers.yaml + + # Aggressively block AI/LLM related bots/agents by default + - import: (data)/meta/ai-block-aggressive.yaml + + # Consider replacing the aggressive AI policy with more selective policies: + # - import: (data)/meta/ai-block-moderate.yaml + # - import: (data)/meta/ai-block-permissive.yaml + + # Search engine crawlers to allow, defaults to: + # - Google (so they don't try to bypass Anubis) + # - Apple + # - Bing + # - DuckDuckGo + # - Qwant + # - The Internet Archive + # - Kagi + # - Marginalia + # - Mojeek + - import: (data)/crawlers/_allow-good.yaml + # Challenge Firefox AI previews + - import: (data)/clients/x-firefox-ai.yaml + + # Allow common "keeping the internet working" routes (well-known, favicon, robots.txt) + - import: (data)/common/keep-internet-working.yaml + + # # Punish any bot with "bot" in the user-agent string + # # This is known to have a high false-positive rate, use at your own risk + # - name: generic-bot-catchall + # user_agent_regex: (?i:bot|crawler) + # action: CHALLENGE + # challenge: + # difficulty: 16 # impossible + # algorithm: slow # intentionally waste CPU cycles and time + + # Requires a subscription to Thoth to use, see + # https://anubis.techaro.lol/docs/admin/thoth#geoip-based-filtering + - name: countries-with-aggressive-scrapers + action: WEIGH + geoip: + countries: + - BR + - CN + weight: + adjust: 10 + + # Requires a subscription to Thoth to use, see + # https://anubis.techaro.lol/docs/admin/thoth#asn-based-filtering + - name: aggressive-asns-without-functional-abuse-contact + action: WEIGH + asns: + match: + - 13335 # Cloudflare + - 136907 # Huawei Cloud + - 45102 # Alibaba Cloud + weight: + adjust: 10 + + # ## System load based checks. + # # If the system is under high load, add weight. + # - name: high-load-average + # action: WEIGH + # expression: load_1m >= 10.0 # make sure to end the load comparison in a .0 + # weight: + # adjust: 20 + + ## If your backend service is running on the same operating system as Anubis, + ## you can uncomment this rule to make the challenge easier when the system is + ## under low load. + ## + ## If it is not, remove weight. + # - name: low-load-average + # action: WEIGH + # expression: load_15m <= 4.0 # make sure to end the load comparison in a .0 + # weight: + # adjust: -10 + + # Generic catchall rule + - name: generic-browser + user_agent_regex: >- + Mozilla|Opera + action: WEIGH + weight: + adjust: 10 + +dnsbl: false + +# # +# impressum: +# # Displayed at the bottom of every page rendered by Anubis. +# footer: >- +# This website is hosted by Zombocom. If you have any complaints or notes +# about the service, please contact +# contact@domainhere.example +# and we will assist you as soon as possible. + +# # The imprint page that will be linked to at the footer of every Anubis page. +# page: +# # The HTML of the page +# title: Imprint and Privacy Policy +# # The HTML contents of the page. The exact contents of this page can +# # and will vary by locale. Please consult with a lawyer if you are not +# # sure what to put here +# body: >- +# <p>Last updated: June 2025</p> + +# <h2>Information that is gathered from visitors</h2> + +# <p>In common with other websites, log files are stored on the web server saving details such as the visitor's IP address, browser type, referring page and time of visit.</p> + +# <p>Cookies may be used to remember visitor preferences when interacting with the website.</p> + +# <p>Where registration is required, the visitor's email and a username will be stored on the server.</p> + +# <!-- ... --> + +# Open Graph passthrough configuration, see here for more information: +# https://anubis.techaro.lol/docs/admin/configuration/open-graph/ +openGraph: + # Enables Open Graph passthrough + enabled: false + # Enables the use of the HTTP host in the cache key, this enables + # caching metadata for multiple http hosts at once. + considerHost: false + # How long cached OpenGraph metadata should last in memory + ttl: 24h + # # If set, return these opengraph values instead of looking them up with + # # the target service. + # # + # # Correlates to properties in https://ogp.me/ + # override: + # # og:title is required, it is the title of the website + # "og:title": "Techaro Anubis" + # "og:description": >- + # Anubis is a Web AI Firewall Utility that helps you fight the bots + # away so that you can maintain uptime at work! + # "description": >- + # Anubis is a Web AI Firewall Utility that helps you fight the bots + # away so that you can maintain uptime at work! + +# By default, send HTTP 200 back to clients that either get issued a challenge +# or a denial. This seems weird, but this is load-bearing due to the fact that +# the most aggressive scraper bots seem to really, really, want an HTTP 200 and +# will stop sending requests once they get it. +status_codes: + CHALLENGE: 200 + DENY: 200 + +# Anubis can store temporary data in one of a few backends. See the storage +# backends section of the docs for more information: +# +# https://anubis.techaro.lol/docs/admin/policies#storage-backends +store: + backend: memory + parameters: {} + +# The weight thresholds for when to trigger individual challenges. Any +# CHALLENGE will take precedence over this. +# +# A threshold has four configuration options: +# +# - name: the name that is reported down the stack and used for metrics +# - expression: A CEL expression with the request weight in the variable +# weight +# - action: the Anubis action to apply, similar to in a bot policy +# - challenge: which challenge to send to the user, similar to in a bot policy +# +# See https://anubis.techaro.lol/docs/admin/configuration/thresholds for more +# information. +thresholds: + # By default Anubis ships with the following thresholds: + - name: minimal-suspicion # This client is likely fine, its soul is lighter than a feather + expression: weight <= 0 # a feather weighs zero units + action: ALLOW # Allow the traffic through + # For clients that had some weight reduced through custom rules, give them a + # lightweight challenge. + - name: mild-suspicion + expression: + all: + - weight > 0 + - weight < 10 + action: CHALLENGE + challenge: + # https://anubis.techaro.lol/docs/admin/configuration/challenges/metarefresh + algorithm: metarefresh + difficulty: 1 + # For clients that are browser-like but have either gained points from custom rules or + # report as a standard browser. + - name: moderate-suspicion + expression: + all: + - weight >= 10 + - weight < 20 + action: CHALLENGE + challenge: + # https://anubis.techaro.lol/docs/admin/configuration/challenges/proof-of-work + algorithm: fast + difficulty: 2 # two leading zeros, very fast for most clients + - name: mild-proof-of-work + expression: + all: + - weight >= 20 + - weight < 30 + action: CHALLENGE + challenge: + # https://anubis.techaro.lol/docs/admin/configuration/challenges/proof-of-work + algorithm: fast + difficulty: 4 + # For clients that are browser like and have gained many points from custom rules + - name: extreme-suspicion + expression: weight >= 30 + action: CHALLENGE + challenge: + # https://anubis.techaro.lol/docs/admin/configuration/challenges/proof-of-work + algorithm: fast + difficulty: 6 diff --git a/deploy/nginx.conf b/deploy/nginx.conf index ce932db..69f8f5d 100644 --- a/deploy/nginx.conf +++ b/deploy/nginx.conf @@ -72,6 +72,45 @@ server { try_files $uri/index.$file_suffix @backend; } + # Uncached package pages are the expensive routes - rendering one decodes + # the package's whole docs JSON - and scraper fleets with spoofed browser + # user agents crawl them hard enough to take the backend down, so their + # cache misses detour through Anubis (a challenge proxy; see SERVER.md) + # on the way to the backend. Cache hits are still served directly, and + # every other route - including `POST /packages` uploads (which match + # `location /`, not this prefix), /search, and the /search uptime check - + # bypasses Anubis entirely, so tooling and monitoring see the backend + # itself. All routes under /packages/ are GET-only, but non-GET requests + # keep the same escape hatch as `location /` just in case. + location /packages/ { + error_page 418 = @backend; + recursive_error_pages on; + + if ($request_method != GET) { + return 418; + } + + root /var/www/pursuit/data/cache; + + try_files $uri/index.$file_suffix @anubis; + } + + # Anubis serves its challenge pages' scripts and check-in endpoints under + # this prefix. + location /.within.website/ { + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_pass http://127.0.0.1:8923; + } + + location @anubis { + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_pass http://127.0.0.1:8923; + } + location @backend { proxy_pass http://127.0.0.1:3000; } diff --git a/deploy/remote.sh b/deploy/remote.sh index 93c0550..fcabbb2 100755 --- a/deploy/remote.sh +++ b/deploy/remote.sh @@ -52,6 +52,41 @@ install /var/www/pursuit/pursuit /usr/local/bin/pursuit popd rm -r "$tmpdir" +# install anubis, the challenge proxy that nginx routes uncached package page +# requests through (see SERVER.md). Installed before the nginx config so the +# config never points at a proxy that isn't there yet. +anubis_version="1.25.0" +anubis_sha256="93e083461f43c8fd92b95a9d6b2a88a80131ecfbef15e894a3fd576cdc5749f3" +if [ "$(dpkg-query --showformat='${Version}' --show anubis 2>/dev/null)" != "$anubis_version" ] +then + anubis_tmpdir="$(mktemp -d)" + wget -O "$anubis_tmpdir/anubis.deb" "https://github.com/TecharoHQ/anubis/releases/download/v${anubis_version}/anubis_${anubis_version}_amd64.deb" + echo "$anubis_sha256 $anubis_tmpdir/anubis.deb" | sha256sum --check + apt-get install --yes "$anubis_tmpdir/anubis.deb" + rm -r "$anubis_tmpdir" +fi + +# The env file holds the instance's signing key (challenge-pass cookies are +# invalidated whenever it changes), so it is generated once and kept. +if [ ! -f /etc/anubis/pursuit.env ] +then + touch /etc/anubis/pursuit.env + chmod 600 /etc/anubis/pursuit.env + cat > /etc/anubis/pursuit.env <<EOF +BIND=127.0.0.1:8923 +BIND_NETWORK=tcp +METRICS_BIND=127.0.0.1:9823 +METRICS_BIND_NETWORK=tcp +TARGET=http://127.0.0.1:3000 +POLICY_FNAME=/etc/anubis/pursuit.botPolicies.yaml +ED25519_PRIVATE_KEY_HEX=$(openssl rand -hex 32) +EOF +fi + +cp /var/www/pursuit/deploy/anubis.botPolicies.yaml /etc/anubis/pursuit.botPolicies.yaml +systemctl enable anubis@pursuit +systemctl restart anubis@pursuit + # install nginx config cp /var/www/pursuit/deploy/nginx.conf /etc/nginx/sites-enabled/pursuit.conf systemctl reload nginx From 94ee164fb8a69caf4883c97f6eab8e040817198e Mon Sep 17 00:00:00 2001 From: Thomas Honeyman <admin@thomashoneyman.com> Date: Mon, 6 Jul 2026 10:07:52 -0400 Subject: [PATCH 3/5] Document the measured basis for the 16MB decode budget Measured with the exact decode path (strict read, aeson decode, deepseq) against real package files: decoding transiently holds ~11x the file size in live heap and ~24x in heap blocks, consistently across a 1.4MB (halogen 7.0.0), 4.4MB (next-purs-rsc 0.1.0) and 25MB (react-icons 1.0.2) file. A full 16MB budget therefore costs ~400MB of heap and the worst single admission (react-icons, alone) ~660MB, both well within the ~1.4GB of headroom under the 3.4GB RTS cap. --- src/Handler/Database.hs | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/Handler/Database.hs b/src/Handler/Database.hs index 4a53271..6d0caa3 100644 --- a/src/Handler/Database.hs +++ b/src/Handler/Database.hs @@ -195,6 +195,18 @@ largeDecodeThreshold = 1024 * 1024 -- | The maximum total size of large package files being decoded at any one -- time. +-- +-- Measured with this module's exact decode path (strict read, aeson decode, +-- deepseq; GHC 9.2.5, 64-bit, -A64m): the decode transiently holds ~11x the +-- file size in live heap while the parse tree is alive, which is ~24x the +-- file size in heap blocks under the copying collector - consistently across +-- a 1.4MB (halogen), 4.4MB (next-purs-rsc) and 25MB (react-icons) file. A +-- full budget therefore costs roughly 16MB x 24 ~= 400MB of heap, and the +-- worst single admission - react-icons at 25MB, admitted alone - costs +-- ~660MB, both well within the ~1.4GB the process has to spare under its +-- 3.4GB RTS cap. The budget's job is to keep mid-size files from stacking +-- past that known-survivable single-giant level, while still letting a few +-- of them decode concurrently. largeDecodeBudget :: Integer largeDecodeBudget = 16 * 1024 * 1024 From 4f0c39e93ddb0e3e99009ef7dfc10cfa57e48f75 Mon Sep 17 00:00:00 2001 From: Thomas Honeyman <admin@thomashoneyman.com> Date: Mon, 6 Jul 2026 16:38:06 -0400 Subject: [PATCH 4/5] Fix cache misses for browser Accept headers; route HEAD like GET Adversarial review findings against the Anubis routing: The Accept-header map had no default, so the composite Accept strings real browsers send never matched an entry and try_files probed for a file literally named "index." - meaning browsers never hit the nginx page cache and every browser page view was re-rendered by the backend (verified against the live site: an exact "Accept: text/html" returns the cached file with an etag, a real Chrome Accept string returns a fresh Yesod render with a session cookie). That also made the /packages/ Anubis detour apply to all browser traffic rather than genuine cache misses, and made an Anubis outage a 502 for every browser package view. Cache lookups now default to html. HEAD requests took the non-GET escape hatch straight to the backend, where Yesod runs the full GET handler for them - a full-cost decode that skipped the challenge entirely, handing scrapers a free bypass. HEAD now takes the same cache/challenge path as GET in both locations. Also fail the deploy loudly if anubis is not running shortly after restart (a broken policy file makes it exit after startup while systemctl restart reports success), rather than leaving a redeploy serving 502s. --- CHANGELOG.md | 11 +++++++++++ deploy/SERVER.md | 3 ++- deploy/nginx.conf | 32 +++++++++++++++++++++++--------- deploy/remote.sh | 7 +++++++ 4 files changed, 43 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4dc11b8..0e33366 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,17 @@ the most up-to-date version of this file. crawlers pass through untouched. Cached package pages, `/search`, and all other routes bypass Anubis entirely. +- Serve cached pages to browsers, and treat HEAD like GET (@thomashoneyman) + + The nginx Accept-header map had no default, so the composite Accept + strings real browsers send ("text/html,application/xhtml+xml,...") never + matched the page cache, and every browser page view was re-rendered by the + backend. Cache lookups now default to the HTML representation, making + cache hits real for browsers - and confining the Anubis detour above to + genuine cache misses. HEAD requests now take the same cache/challenge path + as GET instead of going straight to the backend, where Yesod runs the full + GET handler for them - previously a full-cost decode that looked free. + - Give large package decodes an aggregate memory budget (@thomashoneyman) v0.9.11 serialised decodes of package docs JSON files of 5MB or more, but diff --git a/deploy/SERVER.md b/deploy/SERVER.md index 3f6ee85..ce68d6a 100644 --- a/deploy/SERVER.md +++ b/deploy/SERVER.md @@ -56,7 +56,8 @@ crawlers pass through untouched. `/.within.website/` endpoints) through it - the expensive decode-triggering routes. Cache hits, `/search` (including the uptime check, which must see the real backend), `POST /packages` uploads, and everything else go - straight to the backend. + straight to the backend. If Anubis is down, those cache misses return 502 + (cached pages keep serving) - see the emergency bypass below. - **Emergency bypass**: if Anubis itself misbehaves, edit `/etc/nginx/sites-enabled/pursuit.conf` to point the `@anubis` location's `proxy_pass` at `http://127.0.0.1:3000`, then `systemctl reload nginx`. diff --git a/deploy/nginx.conf b/deploy/nginx.conf index 69f8f5d..5f70ba4 100644 --- a/deploy/nginx.conf +++ b/deploy/nginx.conf @@ -1,10 +1,18 @@ -# For caching, see below +# For caching, see below. A request is served from the page cache when a file +# named for the Accept header's mapped suffix exists. Browsers send composite +# Accept strings ("text/html,application/xhtml+xml,...;q=0.9") that match +# none of the exact entries, so without a default they would miss the cache +# and have the backend re-render every page view; default to html so that +# ordinary page views are cache hits. Clients wanting a non-HTML +# representation of a cached page must send that exact Accept value, as +# before. map $http_accept $file_suffix { - text/html html; + default html; + text/html html; application/json json; - text/svg svg; - text/plain txt; + text/svg svg; + text/plain txt; } server { @@ -59,11 +67,14 @@ server { # this is really gross. sorry # it's here because nginx will return 405 Not Allowed by default # if you try to access a static file using the POST method, and - # we need to be able to do `POST /packages`. + # we need to be able to do `POST /packages`. HEAD is served like GET + # (from the cache when possible): Yesod runs the full GET handler for + # HEAD requests, so sending them straight to the backend would make + # them as expensive as a render while looking free. error_page 418 = @backend; recursive_error_pages on; - if ($request_method != GET) { + if ($request_method !~ ^(GET|HEAD)$) { return 418; } @@ -80,13 +91,16 @@ server { # every other route - including `POST /packages` uploads (which match # `location /`, not this prefix), /search, and the /search uptime check - # bypasses Anubis entirely, so tooling and monitoring see the backend - # itself. All routes under /packages/ are GET-only, but non-GET requests - # keep the same escape hatch as `location /` just in case. + # itself. HEAD must take the same path as GET: Yesod runs the full GET + # handler for HEAD requests, so letting HEAD skip Anubis would give + # scrapers a full-cost decode for free. All routes under /packages/ are + # GET-only, but other methods keep the same escape hatch as `location /` + # just in case. location /packages/ { error_page 418 = @backend; recursive_error_pages on; - if ($request_method != GET) { + if ($request_method !~ ^(GET|HEAD)$) { return 418; } diff --git a/deploy/remote.sh b/deploy/remote.sh index fcabbb2..ca2dc29 100755 --- a/deploy/remote.sh +++ b/deploy/remote.sh @@ -87,6 +87,13 @@ cp /var/www/pursuit/deploy/anubis.botPolicies.yaml /etc/anubis/pursuit.botPolici systemctl enable anubis@pursuit systemctl restart anubis@pursuit +# A broken policy file makes anubis exit shortly after starting (restart +# reports success regardless), and on a redeploy nginx is already routing +# package pages to it. Fail the deploy loudly here rather than leaving that +# to be discovered as 502s. +sleep 2 +systemctl is-active --quiet anubis@pursuit + # install nginx config cp /var/www/pursuit/deploy/nginx.conf /etc/nginx/sites-enabled/pursuit.conf systemctl reload nginx From c2ac0325956244d7c489540015801efb7e7a9c7d Mon Sep 17 00:00:00 2001 From: Thomas Honeyman <admin@thomashoneyman.com> Date: Mon, 6 Jul 2026 12:43:15 -0400 Subject: [PATCH 5/5] Queue large decode admissions first-come-first-served The decode budget admits a file bigger than the whole budget only once no other large decode is in flight, and STM retries impose no arrival order, so under a steady stream of smaller decodes such a file - and the search index regeneration queued behind it - can be overtaken indefinitely. Load-testing a crawler burst against the startup regeneration starved the index build for 71 minutes; a single react-icons page request against sustained sub-budget traffic reliably never completes. Admission now happens while holding an MVar, whose wakeup order is first-come-first-served: later arrivals queue behind a waiting oversize decode, the in-flight total drains within seconds, and the same probe is served in about two seconds. --- CHANGELOG.md | 6 ++++++ src/Application.hs | 1 + src/Foundation.hs | 4 ++++ src/Handler/Database.hs | 24 +++++++++++++++--------- 4 files changed, 26 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0e33366..fd64f18 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -42,6 +42,12 @@ the most up-to-date version of this file. decode running, so files bigger than the budget itself are still served). The queue remains bounded, failing fast with a 503 when full, and the search index regeneration remains exempt from the bound. + + Budget admission is first-come-first-served: without an ordering, a file + too large to share the budget (which must wait until nothing else is in + flight) can be overtaken indefinitely by a steady stream of smaller + decodes - load testing showed a crawler burst starving the search index + build behind a react-icons decode for over an hour. - Type search queries containing type variables now also match more concrete types: `a -> HTMLElement` finds `HTMLAnchorElement -> HTMLElement` the same diff --git a/src/Application.hs b/src/Application.hs index 5359bc2..96722b7 100644 --- a/src/Application.hs +++ b/src/Application.hs @@ -60,6 +60,7 @@ makeFoundation appSettings = do appLogger <- newStdoutLoggerSet defaultBufSize >>= makeYesodLogger appSearchIndex <- newTVarIO emptySearchIndex appDecodeBytesInFlight <- newTVarIO 0 + appLargeDecodeAdmission <- newMVar () appLargeDecodeWaiters <- newTVarIO 0 let foundation = App{..} void (startRegenThread foundation) diff --git a/src/Foundation.hs b/src/Foundation.hs index 9c7f802..b290ea8 100644 --- a/src/Foundation.hs +++ b/src/Foundation.hs @@ -71,6 +71,10 @@ data App = App , appDecodeBytesInFlight :: TVar Integer -- ^ The total file size of the large package JSONs currently being -- decoded; see 'Handler.Database.lookupPackage'. + , appLargeDecodeAdmission :: MVar () + -- ^ Held while acquiring a share of the decode budget, so that admission + -- is first-come-first-served and a file too large to share the budget + -- cannot be overtaken indefinitely by smaller decodes. , appLargeDecodeWaiters :: TVar Int -- ^ The number of threads holding or waiting for a share of the decode -- budget, used to bound the queue. diff --git a/src/Handler/Database.hs b/src/Handler/Database.hs index 6d0caa3..f271909 100644 --- a/src/Handler/Database.hs +++ b/src/Handler/Database.hs @@ -143,10 +143,14 @@ lookupPackageWithPolicy policy pkgName version = do -- -- The budget is measured in file bytes as a proxy for heap use, and it -- bounds the decode spike, not the decoded package, which lives on a little --- longer while the page renders. Admission is not first-come-first-served: a --- large file's decode can be overtaken by smaller ones that fit the --- remaining budget, which is acceptable because waiters are bounded and --- contention is rare. +-- longer while the page renders. Admission is first-come-first-served, which +-- matters for files too large to share the budget: they are only admitted +-- once nothing else is in flight, and under a steady stream of smaller +-- decodes the in-flight total never reaches zero, so without the admission +-- queue such a file's decode (and the search index regeneration behind it) +-- is overtaken indefinitely - observed as an hour-long stall. Holding +-- 'appLargeDecodeAdmission' while waiting makes later arrivals queue behind +-- the waiting decode instead, and the budget drains within seconds. -- -- The queue is bounded: once 'maxLargeDecodeWaiters' threads hold or await a -- share of the budget, further requests fail fast with a 503 rather than @@ -159,6 +163,7 @@ withLargeDecodeBudget policy size action = do app <- getYesod let counter = appLargeDecodeWaiters app let inFlight = appDecodeBytesInFlight app + let admission = appLargeDecodeAdmission app admitted <- case policy of WaitWhenBusy -> do atomically (modifyTVar' counter (+ 1)) @@ -172,7 +177,7 @@ withLargeDecodeBudget policy size action = do return True if admitted then - bracket_ (acquireBudget inFlight) (releaseBudget inFlight) action + bracket_ (acquireBudget admission inFlight) (releaseBudget inFlight) action `finally` atomically (modifyTVar' counter (subtract 1)) else sendResponseStatus serviceUnavailable503 @@ -180,10 +185,11 @@ withLargeDecodeBudget policy size action = do where maxLargeDecodeWaiters = 16 :: Int - acquireBudget inFlight = atomically $ do - inUse <- readTVar inFlight - STM.check (inUse == 0 || inUse + size <= largeDecodeBudget) - writeTVar inFlight (inUse + size) + acquireBudget admission inFlight = withMVar admission $ \_ -> + atomically $ do + inUse <- readTVar inFlight + STM.check (inUse == 0 || inUse + size <= largeDecodeBudget) + writeTVar inFlight (inUse + size) releaseBudget inFlight = atomically (modifyTVar' inFlight (subtract size))