Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 26 additions & 8 deletions .env.example
Original file line number Diff line number Diff line change
Expand Up @@ -31,20 +31,38 @@ VAST_API_KEY=your-vast-api-key-here
# Vast.ai template hash for creating Contemplant instances.
# This should be a template containing the Contemplant image.
# The default is the maintained Contemplant template.
VAST_TEMPLATE_HASH=819fdf2e42fc8ceb32f295465b5bb21e
VAST_TEMPLATE_HASH=ca595bee8049f9316883ce8dcfdcf245

# Number of Contemplant instances to maintain.
# Magister will continuously ensure this many instances are running on Vast.ai.
NUMBER_INSTANCES=1

# Prover type: "cpu" or "cuda" (default: "cpu").
# CPU proving is slower but doesn't require GPU-specific setup.
CONTEMPLANT_PROVER_TYPE=cuda
# Which ZK VM(s) each spawned Contemplant should serve. Comma-separated list;
# valid values are "sp1" and "risc0". The default is both so that Scriptory's
# fibonacci-sp1 and fibonacci-risc0 services each have a worker to assign to.
# Magister reads this only when no [[contemplant.provers]] entries were
# declared in magister.toml; the TOML wins when both are supplied.
CONTEMPLANT_VMS=sp1,risc0

# Moongate CUDA prover endpoint (default: none).
# Only used when prover_type is "cuda". If not set, Contemplant will spin up
# a moongate-server Docker container automatically.
CONTEMPLANT_MOONGATE_ENDPOINT=http://localhost:3000/twirp/
# Per-VM backend. Defaults to "cpu" when unset. Use "cuda" to have the spawned
# Contemplant run with GPU acceleration. The Vast.ai template referenced by
# VAST_TEMPLATE_HASH must satisfy SP1's sm_89+ requirement when SP1 CUDA is
# in play.
CONTEMPLANT_SP1_BACKEND=cuda
CONTEMPLANT_RISC0_BACKEND=cuda

# Opts the spawned RISC Zero worker into producing onchain Groth16 wrapped
# proofs (default: false). Required for the RISC Zero fibonacci test's
# groth16 wrap path. The released Contemplant image already ships the
# vendored Groth16 prover assets.
CONTEMPLANT_RISC0_GROTH16=true

# Moongate CUDA prover endpoint for SP1 (default: none).
# Only meaningful when CONTEMPLANT_SP1_BACKEND=cuda. If unset, sp1-sdk spins
# up a dockerized moongate-server inside the Vast.ai instance. The URL must
# terminate in `/twirp/`; the Contemplant appends it automatically when
# missing, so either form is accepted.
MOONGATE_ENDPOINT=http://localhost:3000/twirp/

# SSH public keys for debugging access (default: none).
# Allows SSH access to Contemplant instances on port 2222 for debugging.
Expand Down
64 changes: 43 additions & 21 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,11 @@ HIEROPHANT_IMAGE ?= unattended/hierophant:latest
MAGISTER_IMAGE ?= unattended/magister:latest
COMPOSE_FILE ?= docker-compose.yml

# Export variables for docker-compose to use.
# Export for docker-compose's variable substitution. HIEROPHANT_IMAGE
# reaches the SP1 fibonacci Dockerfile's `COPY --from=${HIEROPHANT_IMAGE}`
# step that pulls SP1 circuit artifacts out of the Hierophant image rather
# than re-vendoring them; the same value also picks the Hierophant
# container the docker-compose stack runs.
export BUILD_IMAGE
export HIEROPHANT_IMAGE
export MAGISTER_IMAGE
Expand Down Expand Up @@ -58,15 +62,18 @@ clean:

.PHONY: build
build:
@echo "Building fibonacci Docker image ..."
docker compose build fibonacci
@echo "Building both fibonacci Docker images ..."
docker compose --profile all build
@echo "Build complete."

# `make test` runs the full end-to-end fibonacci suite (both SP1 and RISC
# Zero) through docker-compose. There are no host-side unit tests to run;
# the host's rustc + SP1 / RISC Zero toolchains generally do not match the
# pinned versions petros ships, so a host `cargo test` would just trip
# rustc-version mismatches against fresh transitive deps. Run inside
# petros (via the docker-compose flow) where the toolchains are pinned.
.PHONY: test
test:
@echo "Running fibonacci tests ..."
cd src/fibonacci && cargo test --release
@echo "Tests completed."
test: scriptory

.PHONY: docker
docker: build
Expand All @@ -84,23 +91,37 @@ run-m:
@echo "Starting Magister (and Hierophant if needed) ..."
docker compose up magister

.PHONY: run-f
run-f:
@echo "Starting fibonacci test (and dependencies if needed) ..."
docker compose up fibonacci
# `test-sp1` and `test-risc0` activate the matching docker-compose profile
# so only the selected fibonacci service runs alongside Hierophant and
# Magister. Both targets force `--build` so a stale fibonacci image doesn't
# silently mask a recent edit. Use the env-level overrides documented in
# `.env.example` (SP1_PROOF_SYSTEM, RISC0_PROOF_MODE, RISC0_WRAP_SNARK) to
# pick which proving mode each test exercises.
.PHONY: test-sp1
test-sp1:
@echo "Starting SP1 fibonacci test ..."
docker compose --profile sp1 up --build

.PHONY: test-risc0
test-risc0:
@echo "Starting RISC Zero fibonacci test ..."
docker compose --profile risc0 up --build

.PHONY: run
run: scriptory

# `make scriptory` runs both fibonacci tests in the same compose stack so a
# user with a dual-VM Contemplant pool sees both proof flows light up at
# once. Use `make test-sp1` or `make test-risc0` to drive only one VM.
.PHONY: scriptory
scriptory:
@echo "Starting scriptory services ..."
docker compose up --build
@echo "Starting scriptory services (SP1 + RISC Zero fibonacci) ..."
docker compose --profile all up --build

.PHONY: scriptory-d
scriptory-d:
@echo "Starting scriptory services in detached mode ..."
docker compose up --build -d
docker compose --profile all up --build -d
@echo "Services started. Use 'make logs' to view output."

.PHONY: stop
Expand Down Expand Up @@ -128,15 +149,16 @@ help:
@echo "Targets:"
@echo " init Initialize config from examples."
@echo " clean Clean volumes."
@echo " build Build the fibonacci Docker image."
@echo " test Run fibonacci tests."
@echo " docker Build the fibonacci Docker image."
@echo " ci Build the fibonacci Docker image."
@echo " run Run all Scriptory services."
@echo " build Build both fibonacci Docker images (sp1 and risc0)."
@echo " test Alias for scriptory. Runs both end-to-end tests."
@echo " docker Build both fibonacci Docker images."
@echo " ci Build both fibonacci Docker images."
@echo " test-sp1 Run the SP1 fibonacci end-to-end test."
@echo " test-risc0 Run the RISC Zero fibonacci end-to-end test."
@echo " run Alias for scriptory."
@echo " run-h Run just Hierophant."
@echo " run-m Run Magister (starts Hierophant if needed)."
@echo " run-f Run fibonacci test (starts all dependencies)."
@echo " scriptory Start all services in foreground."
@echo " scriptory Start all services (both VM tests) in foreground."
@echo " scriptory-d Start all services in background."
@echo " stop Stop all services."
@echo " restart Restart all services."
Expand Down
36 changes: 21 additions & 15 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

> Write the vision; make it plain on tablets, so he may run who reads it.

A simple solution for hosted proof production using [Hierophant](https://github.com/unattended-backpack/hierophant), [Magister](https://github.com/unattended-backpack/magister), and Contemplant. This Docker Compose setup runs a complete zkVM prover network with GPU-accelerated proof generation on Vast, demonstrated with a fibonacci example program.
A simple solution for hosted proof production using [Hierophant](https://github.com/unattended-backpack/hierophant), [Magister](https://github.com/unattended-backpack/magister), and Contemplant. This Docker Compose setup runs a complete zkVM prover network with GPU-accelerated proof generation on Vast, demonstrated with a fibonacci example program for each of the two supported zkVMs: [SP1](https://github.com/succinctlabs/sp1) (over Hierophant's SP1 prover network gRPC) and [RISC Zero](https://risczero.com/) (over Hierophant's [Bonsai](https://dev.bonsai.xyz/)-compatible REST surface).

## Running

Expand All @@ -21,8 +21,11 @@ The `.env` file contains the minimal configuration required to run Scriptory. Yo
The remaining values in `.env` have sensible defaults:
- `VAST_TEMPLATE_HASH`: Template for creating Contemplant instances (default provided).
- `NUMBER_INSTANCES`: Number of Contemplant instances to maintain (default: 1).
- `CONTEMPLANT_PROVER_TYPE`: Whether the Contemplant instances should use `cuda` proving or `cpu` proving (default: `cuda`).
- `CONTEMPLANT_MOONGATE_ENDPOINT`: The endpoint a Contemplant instance should use for a Moongate prover (default provided).
- `CONTEMPLANT_VMS`: Comma-separated list of zkVMs each spawned Contemplant should serve (default: `sp1,risc0`). Declaring both produces a Contemplant that advertises both VMs to Hierophant and serves either kind of proof as it becomes idle.
- `CONTEMPLANT_SP1_BACKEND`: SP1 backend per Contemplant (`cpu` or `cuda`, default `cuda`).
- `CONTEMPLANT_RISC0_BACKEND`: RISC Zero backend per Contemplant (`cpu` or `cuda`, default `cuda`).
- `CONTEMPLANT_RISC0_GROTH16`: Whether the RISC Zero worker accepts onchain Groth16 wrap requests (`true` or `false`, default `true`). Required for the RISC Zero fibonacci test's groth16 wrap path.
- `MOONGATE_ENDPOINT`: External moongate URL for SP1 CUDA (optional; if unset, the Contemplant spins one up locally inside the Vast.ai instance).
- `CONTEMPLANT_SSH_AUTHORIZED_KEYS`: Any SSH public keys for gaining debug access to Contemplant instances.

The TOML files (`hierophant.toml` and `magister.toml`) contain detailed service configuration and work out of the box for Docker Compose deployments. Values in `.env` will override TOML settings where applicable. Review the TOML files if you need to customize advanced settings such as worker timeouts, Vast.ai query parameters, or artifact storage limits.
Expand All @@ -31,19 +34,22 @@ The TOML files (`hierophant.toml` and `magister.toml`) contain detailed service

Once configuration is complete, start the services with `make scriptory` (foreground) or `make scriptory-d` (detached). This command will:

1. Build the fibonacci example Docker image.
2. Start Hierophant on ports 9000 (gRPC) and 9010 (HTTP/WebSocket).
1. Build both fibonacci example Docker images (SP1 and RISC Zero variants).
2. Start Hierophant on ports 9000 (SP1 gRPC) and 9010 (HTTP/WebSocket plus the Bonsai REST surface used by the RISC Zero fibonacci test).
3. Start Magister on port 8555, which will:
- Connect to Hierophant.
- Create and maintain the configured number of Contemplant instances on Vast.
- Create and maintain the configured number of Contemplant instances on Vast, each declaring both SP1 and RISC Zero capability per the `[[contemplant.provers]]` array in `magister.toml`.
- Monitor instances and replace any that fail.
4. Run the fibonacci example program, which will:
- Generate a fibonacci computation proof request.
- Submit the request to Hierophant.
- Wait for a Contemplant to pick up the work and generate a proof.
- Retrieve and verify the completed proof.
4. Run the SP1 fibonacci example program against Hierophant's SP1 gRPC.
5. Run the RISC Zero fibonacci example program against Hierophant's Bonsai REST surface.

The fibonacci example demonstrates the complete proof generation pipeline. You can watch the logs to see the proof request, witness generation, proof generation on GPU, and final verification.
Each fibonacci example submits a proof request, waits for a Contemplant to pick up the work, retrieves the completed proof, and verifies it. Logs from all services interleave on the foreground command so you can watch both proof flows progress simultaneously.

To exercise only one zkVM at a time, use `make test-sp1` or `make test-risc0` instead. Both forms accept the following overrides via `.env`:

- `SP1_PROOF_SYSTEM` selects which SP1 proving mode the test requests: `core`, `compressed`, `plonk` (default), or `groth16`.
- `RISC0_PROOF_MODE` selects the RISC Zero session mode: `composite` (default), `succinct`, or `groth16`.
- `RISC0_WRAP_SNARK=true` flips on the canonical Bonsai composite-then-Groth16-wrap flow. Requires Contemplants spawned with `CONTEMPLANT_RISC0_GROTH16=true` (the default).

### Managing Services

Expand All @@ -55,9 +61,9 @@ Additional targets are available for building images, running tests, and cleanin

Scriptory orchestrates three core components:

- **Hierophant**: The prover network coordinator that manages proof requests, worker registration, and artifact storage. Clients submit proof requests to Hierophant and retrieve completed proofs.
- **Magister**: The Vast instance manager that automatically creates, monitors, and maintains Contemplant workers on GPU instances. Magister ensures the configured number of instances are always available.
- **Contemplant**: GPU-accelerated proof generation workers that connect to Hierophant via WebSocket, receive proof tasks, and generate zkVM proofs using CUDA acceleration.
- **Hierophant**: The prover network coordinator that manages proof requests, worker registration, and artifact storage. SP1 clients submit proof requests via the gRPC surface on port 9000 (`sp1-sdk`-compatible); RISC Zero clients submit via the Bonsai-compatible REST surface at `/bonsai/` on port 9010 (`bonsai-sdk`-compatible).
- **Magister**: The Vast instance manager that automatically creates, monitors, and maintains Contemplant workers on GPU instances. Magister ensures the configured number of instances are always available, and tells each spawned Contemplant which zkVM(s) to serve via the `[[contemplant.provers]]` array in `magister.toml`.
- **Contemplant**: GPU-accelerated proof generation workers that connect to Hierophant via WebSocket, receive proof tasks, and generate zkVM proofs using CUDA acceleration. A single Contemplant can serve both SP1 and RISC Zero proof requests; it processes one proof at a time regardless of how many VMs it advertises.

## Requirements

Expand Down
97 changes: 75 additions & 22 deletions docker-compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,12 @@ services:
hierophant:
image: ${HIEROPHANT_IMAGE:-unattended/hierophant:latest}
container_name: hierophant
networks:
- network
ports:
- "9000:9000" # gRPC
- "9010:9010" # HTTP + WebSocket

# Host networking: hierophant binds 9000 + 9010 directly on the host so
# Vast.ai contemplants reach them via the router-forwarded WAN IP and
# local fibonacci clients reach them via localhost. Eliminates Docker
# bridge + published-port indirection that complicates NAT hairpin.
network_mode: host
volumes:
- ./hierophant.toml:/home/hierophant/hierophant.toml:ro
env_file:
Expand All @@ -23,10 +24,7 @@ services:
magister:
image: ${MAGISTER_IMAGE:-unattended/magister:latest}
container_name: magister
networks:
- network
ports:
- "8555:8555"
network_mode: host
volumes:
- ./magister.toml:/home/magister/magister.toml:ro
env_file:
Expand All @@ -43,32 +41,87 @@ services:
retries: 10
start_period: 10s

fibonacci:
# SP1 fibonacci proof. Talks to Hierophant's SP1 prover network gRPC on
# :9000 via sp1-sdk's network prover. SP1_PROOF_SYSTEM picks the SP1
# proving mode and defaults to plonk; override via .env to exercise core,
# compressed, plonk, or groth16. Selected via the `sp1` compose profile;
# `make test-sp1` activates it, `make scriptory` activates both this and
# fibonacci-risc0 via the `all` profile alias.
# The SP1 fibonacci demo client. Generates a proof and emits a Solidity-
# verifier-compatible JSON fixture; verification of the proof itself
# happens server-side at Hierophant before the proof is returned, so this
# client doesn't carry the SP1 circuit artifacts that a client.verify
# path would need. Build context is the repo root so the Dockerfile's
# `COPY src/sp1-fibonacci/...` and `COPY src/fibonacci/` paths resolve.
fibonacci-sp1:
profiles: ["sp1", "all"]
build:
context: ./src/fibonacci
dockerfile: Dockerfile
container_name: fibonacci
networks:
- network
context: .
dockerfile: src/sp1-fibonacci/Dockerfile
args:
BUILD_IMAGE: ${BUILD_IMAGE:-unattended/petros:latest}
container_name: fibonacci-sp1
network_mode: host
depends_on:
hierophant:
condition: service_healthy
magister:
condition: service_healthy
environment:
- RUST_LOG=info
- SP1_PROOF_SYSTEM=${SP1_PROOF_SYSTEM:-plonk}

# Override the Dockerfile-baked NETWORK_RPC_URL=http://hierophant:9000;
# with host networking the Docker DNS name no longer resolves.
- NETWORK_RPC_URL=http://localhost:9000
command:
- bash
- -c
- |
echo "Waiting for at least one idle Contemplant..."
while ! (exec 3<>/dev/tcp/hierophant/9010 && echo -e 'GET /contemplants HTTP/1.1\r\nHost: hierophant\r\nConnection: close\r\n\r\n' >&3 && cat <&3 | grep -q "Idle"); do
echo "Waiting for an SP1-capable idle Contemplant..."
while ! (exec 3<>/dev/tcp/localhost/9010 && echo -e 'GET /contemplants HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n' >&3 && cat <&3 | grep -q "Idle"); do
echo "No idle Contemplants available yet, waiting..."
sleep 5
done
echo "Idle Contemplant found, starting fibonacci proof..."
cargo run --release --bin evm -- --n 10 --system plonk
echo "Idle Contemplant found, starting SP1 fibonacci proof (system=$$SP1_PROOF_SYSTEM)..."
cargo run --release --bin evm -- --n 10 --system "$$SP1_PROOF_SYSTEM"

networks:
network:
driver: bridge
# RISC Zero fibonacci proof. Talks to Hierophant's Bonsai-shaped REST
# surface at :9010/bonsai/ via bonsai-sdk. PROOF_MODE picks the STARK
# variant (composite / succinct / groth16) and WRAP_SNARK flips on the
# canonical Bonsai composite -> Groth16 wrap flow. WRAP_SNARK=true
# requires the spawned Contemplant to advertise groth16_enabled. The
# compose profile `risc0` opts this service in.
# The RISC Zero fibonacci Dockerfile is taken verbatim from hierophant's
# integration test. It uses the shared `src/fibonacci/` no_std crate
# (path-dep'd from both guest and host), so the build context must be
# the repo root rather than `src/risc0-fibonacci/`.
fibonacci-risc0:
profiles: ["risc0", "all"]
build:
context: .
dockerfile: src/risc0-fibonacci/Dockerfile
args:
BUILD_IMAGE: ${BUILD_IMAGE:-unattended/petros:latest}
container_name: fibonacci-risc0
network_mode: host
depends_on:
hierophant:
condition: service_healthy
magister:
condition: service_healthy
environment:
- RUST_LOG=info
- PROOF_MODE=${RISC0_PROOF_MODE:-composite}
- WRAP_SNARK=${RISC0_WRAP_SNARK:-false}
command:
- bash
- -c
- |
echo "Waiting for a RISC Zero-capable idle Contemplant..."
while ! (exec 3<>/dev/tcp/localhost/9010 && echo -e 'GET /contemplants HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n' >&3 && cat <&3 | grep -q "Idle"); do
echo "No idle Contemplants available yet, waiting..."
sleep 5
done
echo "Idle Contemplant found, starting RISC Zero fibonacci proof (mode=$$PROOF_MODE, wrap=$$WRAP_SNARK)..."
cargo run --release --bin evm -- --n 10 --bonsai-url http://localhost:9010/bonsai
6 changes: 6 additions & 0 deletions hierophant.example.toml
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,15 @@
this_hierophant_ip = "hierophant"

# OPTIONAL: Port for gRPC service (default: 9000).
# Used by SP1 clients via sp1-sdk's network prover.
# grpc_port = 9000

# OPTIONAL: Port for HTTP and WebSocket service (default: 9010).
# Carries three things on a single port:
# 1. WebSocket connections from Contemplants registering with this Hierophant.
# 2. HTTP endpoints for client artifact upload/download and operator status.
# 3. The Bonsai-compatible REST surface at /bonsai/ used by RISC Zero
# clients via bonsai-sdk.
# http_port = 9010

# OPTIONAL: Timeout in seconds for worker responses (default: 30).
Expand Down
Loading