feat(datasource/graphene): supervoxel splitting

.github/workflows/deploy_gae.yml

@@ -0,0 +1,79 @@
+name: Build
+
+on: [push, pull_request]
+
+jobs:
+  build-and-deploy:
+    permissions:
+      contents: "read"
+      id-token: "write"
+      deployments: "write"
+    strategy:
+      matrix:
+        os:
+          - "ubuntu-latest"
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          # Need full history to determine version number.
+          fetch-depth: 0
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 22.x
+          cache: "npm"
+          cache-dependency-path: |
+            package-lock.json
+            examples/**/package-lock.json
+      - run: npm install
+      - name: Typecheck with TypeScript
+        run: npm run typecheck
+      - name: Get branch name (merge)
+        if: github.event_name != 'pull_request'
+        shell: bash
+        run: echo "BRANCH_NAME=$(echo ${GITHUB_REF#refs/heads/} | tr / -)" >> $GITHUB_ENV
+      - name: Get branch name (pull request)
+        if: github.event_name == 'pull_request'
+        shell: bash
+        run: echo "BRANCH_NAME=$(echo ${GITHUB_HEAD_REF} | tr / -)" >> $GITHUB_ENV
+      - run: echo "BRANCH_NAME_URL=$(echo ${{ env.BRANCH_NAME }} | tr / - | tr _ -)" >> $GITHUB_ENV
+      - name: Get build info
+        run: echo "BUILD_INFO={\"tag\":\"$(git describe --always --tags)\", \"url\":\"https://github.com/${{github.repository}}/commit/$(git rev-parse HEAD)\", \"timestamp\":\"$(date)\", \"branch\":\"${{github.repository}}/${{env.BRANCH_NAME}}\"}" >> $GITHUB_ENV
+        shell: bash
+      # - name: Check for dirty working directory
+      #   run: git diff --exit-code
+      - name: Build client bundles
+        run: npm run build -- --no-typecheck --no-lint --define STATE_SERVERS=$(cat config/state_servers.json | tr -d " \t\n\r") --define NEUROGLANCER_BUILD_INFO='${{ env.BUILD_INFO }}' --define NEUROGLANCER_CUSTOM_INPUT_BINDINGS=$(cat config/custom-keybinds.json | tr -d " \t\n\r") --define NEUROGLANCER_SEGMENT_LIST_COLOR_WIDGET=true
+      - name: Write build info
+        run: echo $BUILD_INFO > ./dist/client/version.json
+        shell: bash
+      - run: cp -r ./dist/client appengine/frontend/static/
+      - name: start deployment
+        uses: bobheadxi/deployments@v1
+        id: deployment
+        with:
+          step: start
+          token: ${{ secrets.GITHUB_TOKEN }}
+          env: ${{ env.BRANCH_NAME }}
+          desc: Setting up staging deployment for ${{ env.BRANCH_NAME }}
+      - id: "auth"
+        uses: "google-github-actions/auth@v1"
+        with:
+          workload_identity_provider: "projects/483670036293/locations/global/workloadIdentityPools/neuroglancer-github/providers/github"
+          service_account: "chris-apps-deploy@seung-lab.iam.gserviceaccount.com"
+      - id: deploy
+        uses: google-github-actions/deploy-appengine@main
+        with:
+          version: ${{ env.BRANCH_NAME_URL }}
+          deliverables: appengine/frontend/app.yaml
+          promote: false
+      - name: update deployment status
+        uses: bobheadxi/deployments@v1
+        if: always()
+        with:
+          step: finish
+          token: ${{ secrets.GITHUB_TOKEN }}
+          env: ${{ steps.deployment.outputs.env }}
+          env_url: ${{ steps.deploy.outputs.url }}
+          status: ${{ job.status }}
+          deployment_id: ${{ steps.deployment.outputs.deployment_id }}

appengine/frontend/app.yaml

@@ -0,0 +1,19 @@
+runtime: python312
+
+service: neuroglancer
+
+handlers:
+  # Handle the main page by serving the index page.
+  # Note the $ to specify the end of the path, since app.yaml does prefix matching.
+  - url: /$
+    static_files: static/index.html
+    upload: static/index.html
+    login: optional
+    secure: always
+    redirect_http_response_code: 301
+
+  - url: /
+    static_dir: static
+    login: optional
+    secure: always
+    redirect_http_response_code: 301

config/custom-keybinds.json

@@ -0,0 +1,39 @@
+{
+  "keym": {
+    "action": {
+      "layerType": "segmentation",
+      "tool": "grapheneMergeSegments",
+      "provider": "graphene"
+    }
+  },
+  "keyc": {
+    "action": {
+      "layerType": "segmentation",
+      "tool": "grapheneMulticutSegments",
+      "provider": "graphene"
+    }
+  },
+  "keyf": {
+    "action": {
+      "layerType": "segmentation",
+      "tool": "grapheneFindPath",
+      "provider": "graphene"
+    }
+  },
+  "keyx": {
+    "action": false
+  },
+  "control+shift+keyx": {
+    "action": "clear-segments"
+  },
+  "bracketleft": [
+    { "action": false, "context": "sliceView" },
+    { "action": false, "context": "perspectiveView" },
+    { "action": "select-previous" }
+  ],
+  "bracketright": [
+    { "action": false, "context": "sliceView" },
+    { "action": false, "context": "perspectiveView" },
+    { "action": "select-next" }
+  ]
+}

config/state_servers.json

@@ -0,0 +1,6 @@
+{
+  "cave": {
+    "url": "middleauth+https://global.daf-apis.com/nglstate/api/v1/post",
+    "default": true
+  }
+}

package.json

@@ -428,6 +428,12 @@
       "neuroglancer/kvstore/icechunk:disabled": "./src/util/false.ts",
       "default": "./src/kvstore/icechunk/register_backend.ts"
     },
+    "#kvstore/kvstack/register": {
+      "neuroglancer/kvstore/kvstack:enabled": "./src/kvstore/kvstack/register.ts",
+      "neuroglancer/kvstore:none_by_default": "./src/util/false.ts",
+      "neuroglancer/kvstore/kvstack:disabled": "./src/util/false.ts",
+      "default": "./src/kvstore/kvstack/register.ts"
+    },
     "#kvstore/middleauth/register_frontend": {
       "neuroglancer/kvstore/middleauth:enabled": "./src/kvstore/middleauth/register_frontend.ts",
       "neuroglancer/kvstore:none_by_default": "./src/util/false.ts",

src/chunk_manager/generic_file_source.ts

@@ -69,6 +69,22 @@ export class SimpleAsyncCache<Key, Value> extends ChunkSourceBase {
     progressOptions: ProgressOptions,
   ) => Promise<{ size: number; data: Value }>;
 
+  invalidate(key: Key) {
+    const encodedKey = this.encodeKeyFunction(key);
+    const chunk = this.chunks.get(encodedKey);
+    if (chunk !== undefined) {
+      chunk.freeSystemMemory();
+      this.chunkManager.queueManager.updateChunkState(chunk, ChunkState.QUEUED);
+    }
+  }
+
+  invalidateAll() {
+    for (const chunk of this.chunks.values()) {
+      chunk.freeSystemMemory();
+      this.chunkManager.queueManager.updateChunkState(chunk, ChunkState.QUEUED);
+    }
+  }
+
   get(key: Key, options: Partial<ProgressOptions>): Promise<Value> {
     const encodedKey = this.encodeKeyFunction(key);
     let chunk = this.chunks.get(encodedKey);

src/datasource/graphene/backend.ts

@@ -28,6 +28,7 @@ import type {
 } from "#src/datasource/graphene/base.js";
 import {
   getGrapheneFragmentKey,
+  GRAPHENE_INVALIDATE_OCDBT_RPC_ID,
   GRAPHENE_MESH_NEW_SEGMENT_RPC_ID,
   ChunkedGraphSourceParameters,
   MeshSourceParameters,
@@ -42,6 +43,7 @@ import { decodeManifestChunk } from "#src/datasource/precomputed/backend.js";
 import { WithSharedKvStoreContextCounterpart } from "#src/kvstore/backend.js";
 import type { KvStoreWithPath, ReadResponse } from "#src/kvstore/index.js";
 import { readKvStore } from "#src/kvstore/index.js";
+import { invalidateOcdbtCaches } from "#src/kvstore/ocdbt/metadata_cache.js";
 import type { FragmentChunk, ManifestChunk } from "#src/mesh/backend.js";
 import { assignMeshFragmentData, MeshSource } from "#src/mesh/backend.js";
 import { decodeDraco } from "#src/mesh/draco/index.js";
@@ -642,3 +644,8 @@ registerRPC(GRAPHENE_MESH_NEW_SEGMENT_RPC_ID, function (x) {
   const obj = <GrapheneMeshSource>this.get(x.rpcId);
   obj.addNewSegment(x.segment);
 });
+
+registerRPC(GRAPHENE_INVALIDATE_OCDBT_RPC_ID, function (x) {
+  const source = this.get(x.layerId) as GrapheneChunkedGraphChunkSource;
+  invalidateOcdbtCaches(source.sharedKvStoreContext);
+});

src/datasource/graphene/base.ts

@@ -32,6 +32,7 @@ import type { FetchOk, HttpError } from "#src/util/http_request.js";
 
 export const PYCG_APP_VERSION = 1;
 export const GRAPHENE_MESH_NEW_SEGMENT_RPC_ID = "GrapheneMeshSource:NewSegment";
+export const GRAPHENE_INVALIDATE_OCDBT_RPC_ID = "Graphene:InvalidateOcdbt";
 
 export enum VolumeChunkEncoding {
   RAW = 0,