From 700c6a6f853d74c1d0bcde65f07b67a7b884b4e5 Mon Sep 17 00:00:00 2001 From: Nathan Flurry Date: Wed, 8 Jul 2026 00:22:55 -0700 Subject: [PATCH] ci: speed up publish workflow --- .github/workflows/ci.yml | 3 +- .github/workflows/publish.yaml | 220 ++++++++---------- .gitignore | 3 +- docker/build/darwin.Dockerfile | 34 ++- docker/build/linux-gnu.Dockerfile | 106 +++++++++ .../build/linux-gnu.Dockerfile.dockerignore | 8 + packages/agentos-plugin/scripts/build.mjs | 7 + .../src/generated/actor-actions.generated.ts | 164 +++++++++++++ packages/sidecar-binary/scripts/build.mjs | 7 + turbo.json | 15 ++ 10 files changed, 436 insertions(+), 131 deletions(-) create mode 100644 docker/build/linux-gnu.Dockerfile create mode 100644 docker/build/linux-gnu.Dockerfile.dockerignore create mode 100644 packages/agentos/src/generated/actor-actions.generated.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7d927bacf2..a731c55d3a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -63,7 +63,8 @@ jobs: targets: wasm32-wasip1 - uses: Swatinem/rust-cache@v2 with: - workspaces: registry/native -> registry/native/target + workspaces: registry/native -> target + cache-workspace-crates: true key: wasm-commands-${{ hashFiles('registry/native/Cargo.lock') }} - run: pnpm install --frozen-lockfile --filter '@rivet-dev/agentos-runtime-core...' - run: make -C registry/native commands diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml index 1c77cd71dc..dd7cb0a5f7 100644 --- a/.github/workflows/publish.yaml +++ b/.github/workflows/publish.yaml @@ -21,7 +21,7 @@ env: R2_BUCKET: rivet-releases R2_ENDPOINT: https://2a94c6a0ced8d35ea63cddc86c2681e7.r2.cloudflarestorage.com SIDECAR_PLATFORMS: "linux-x64-gnu linux-arm64-gnu darwin-x64 darwin-arm64" - PUBLISH_INCLUDE_REGISTRY_PACKAGES: "1" + RUST_TOOLCHAIN: "1.91.1" LINUX_GNU_LLVM_VERSION: "22" LINUX_GNU_SYSROOT_TAG: "sysroot-20250207" @@ -66,7 +66,8 @@ jobs: targets: wasm32-wasip1 - uses: Swatinem/rust-cache@v2 with: - workspaces: registry/native -> registry/native/target + workspaces: registry/native -> target + cache-workspace-crates: true key: wasm-commands-${{ hashFiles('registry/native/Cargo.lock') }} - run: pnpm install --frozen-lockfile --filter '@rivet-dev/agentos-runtime-core...' - run: make -C registry/native commands @@ -79,7 +80,7 @@ jobs: build-sidecar: needs: [context] - name: "Build sidecars (${{ matrix.platform }})" + name: "Build linux native artifacts (${{ matrix.platform }})" strategy: fail-fast: false matrix: @@ -88,132 +89,79 @@ jobs: runner: [self-hosted, agentos-builder] target: x86_64-unknown-linux-gnu - platform: linux-arm64-gnu - runner: ubuntu-22.04-arm + runner: [self-hosted, agentos-builder-arm64] target: aarch64-unknown-linux-gnu runs-on: ${{ matrix.runner }} steps: - uses: actions/checkout@v4 - - uses: pnpm/action-setup@v4 - - uses: actions/setup-node@v4 - with: - node-version: 22 - cache: pnpm - cache-dependency-path: pnpm-lock.yaml - - uses: dtolnay/rust-toolchain@stable - with: - targets: ${{ matrix.target }} - - uses: Swatinem/rust-cache@v2 - with: - workspaces: . -> target - key: ${{ matrix.target }}-${{ env.LINUX_GNU_SYSROOT_TAG }}-llvm-${{ env.LINUX_GNU_LLVM_VERSION }}-${{ needs.context.outputs.trigger }} - - uses: actions/cache@v4 + - uses: docker/setup-buildx-action@v3 with: - path: ~/.cargo/.rusty_v8 - key: ${{ runner.os }}-${{ matrix.target }}-rusty-v8-${{ hashFiles('Cargo.lock') }} - restore-keys: | - ${{ runner.os }}-${{ matrix.target }}-rusty-v8- - - run: pnpm install --frozen-lockfile --filter='!@agentos/website' - - name: Set up Linux GNU sysroot - run: scripts/ci/setup-linux-gnu-sysroot.sh - - name: Build sidecar binaries - id: build + name: agentos-${{ matrix.platform }} + keep-state: ${{ contains(matrix.runner, 'self-hosted') }} + - name: Resolve build profile + id: mode run: | set -euo pipefail - out="target/sidecar-artifacts/${{ matrix.platform }}" - mkdir -p "$out" if [ "${{ needs.context.outputs.trigger }}" = "release" ]; then - flag="--release" - profile="release" + echo "profile=release" >> "$GITHUB_OUTPUT" else - flag="" - profile="debug" + echo "profile=debug" >> "$GITHUB_OUTPUT" fi - cargo build $flag -p agentos-sidecar -p agentos-native-sidecar --target ${{ matrix.target }} - cp "target/${{ matrix.target }}/${profile}/agentos-sidecar" "$out/agentos-sidecar" - cp "target/${{ matrix.target }}/${profile}/agentos-native-sidecar" "$out/agentos-native-sidecar" - echo "dir=$out" >> "$GITHUB_OUTPUT" + - name: Build linux artifacts + uses: docker/build-push-action@v6 + with: + context: . + file: docker/build/linux-gnu.Dockerfile + build-args: | + TARGET=${{ matrix.target }} + BUILD_PROFILE=${{ steps.mode.outputs.profile }} + CACHE_PLATFORM=${{ matrix.platform }} + RUST_TOOLCHAIN=${{ env.RUST_TOOLCHAIN }} + LINUX_GNU_LLVM_VERSION=${{ env.LINUX_GNU_LLVM_VERSION }} + LINUX_GNU_SYSROOT_TAG=${{ env.LINUX_GNU_SYSROOT_TAG }} + tags: agentos-linux-${{ matrix.platform }} + load: true + - name: Extract linux artifacts + id: extract + run: | + set -euo pipefail + sidecar_out="target/sidecar-artifacts/${{ matrix.platform }}" + plugin_out="target/plugin-artifacts/${{ matrix.platform }}" + mkdir -p "$sidecar_out" "$plugin_out" + cid=$(docker create agentos-linux-${{ matrix.platform }}) + docker cp "$cid:/artifacts/agentos-sidecar" "$sidecar_out/agentos-sidecar" + docker cp "$cid:/artifacts/agentos-native-sidecar" "$sidecar_out/agentos-native-sidecar" + docker cp "$cid:/artifacts/libagentos_actor_plugin.so" "$plugin_out/libagentos_actor_plugin.so" + docker rm "$cid" + test -f "$sidecar_out/agentos-sidecar" + test -f "$sidecar_out/agentos-native-sidecar" + test -f "$plugin_out/libagentos_actor_plugin.so" + echo "sidecar_dir=$sidecar_out" >> "$GITHUB_OUTPUT" + echo "plugin_dir=$plugin_out" >> "$GITHUB_OUTPUT" - name: Check glibc floor run: | scripts/ci/check-linux-glibc-floor.sh \ - "${{ steps.build.outputs.dir }}/agentos-sidecar" \ - "${{ steps.build.outputs.dir }}/agentos-native-sidecar" - - name: Smoke-run in old Linux containers + "${{ steps.extract.outputs.sidecar_dir }}/agentos-sidecar" \ + "${{ steps.extract.outputs.sidecar_dir }}/agentos-native-sidecar" \ + "${{ steps.extract.outputs.plugin_dir }}/libagentos_actor_plugin.so" + - name: Smoke-run binaries in old Linux containers run: | scripts/ci/smoke-linux-artifacts.sh binary \ - "${{ steps.build.outputs.dir }}/agentos-sidecar" \ - "${{ steps.build.outputs.dir }}/agentos-native-sidecar" + "${{ steps.extract.outputs.sidecar_dir }}/agentos-sidecar" \ + "${{ steps.extract.outputs.sidecar_dir }}/agentos-native-sidecar" + - name: Smoke-run plugin in old Linux containers + run: | + scripts/ci/smoke-linux-artifacts.sh shared-library \ + "${{ steps.extract.outputs.plugin_dir }}/libagentos_actor_plugin.so" - uses: actions/upload-artifact@v4 with: name: sidecar-${{ matrix.platform }} - path: ${{ steps.build.outputs.dir }} + path: ${{ steps.extract.outputs.sidecar_dir }} if-no-files-found: error - - build-plugin: - needs: [context] - name: "Build plugin (${{ matrix.platform }})" - strategy: - fail-fast: false - matrix: - include: - - platform: linux-x64-gnu - runner: ubuntu-22.04 - target: x86_64-unknown-linux-gnu - - platform: linux-arm64-gnu - runner: ubuntu-22.04-arm - target: aarch64-unknown-linux-gnu - runs-on: ${{ matrix.runner }} - steps: - - uses: actions/checkout@v4 - - uses: pnpm/action-setup@v4 - - uses: actions/setup-node@v4 - with: - node-version: 22 - cache: pnpm - cache-dependency-path: pnpm-lock.yaml - - uses: dtolnay/rust-toolchain@stable - with: - targets: ${{ matrix.target }} - - uses: Swatinem/rust-cache@v2 - with: - workspaces: . -> target - key: plugin-${{ matrix.target }}-${{ env.LINUX_GNU_SYSROOT_TAG }}-llvm-${{ env.LINUX_GNU_LLVM_VERSION }}-${{ needs.context.outputs.trigger }} - - uses: actions/cache@v4 - with: - path: ~/.cargo/.rusty_v8 - key: ${{ runner.os }}-${{ matrix.target }}-rusty-v8-${{ hashFiles('Cargo.lock') }} - restore-keys: | - ${{ runner.os }}-${{ matrix.target }}-rusty-v8- - - run: pnpm install --frozen-lockfile --filter='!@agentos/website' - - name: Set up Linux GNU sysroot - run: scripts/ci/setup-linux-gnu-sysroot.sh - - name: Build plugin cdylib - id: build - run: | - set -euo pipefail - out="target/plugin-artifacts/${{ matrix.platform }}" - mkdir -p "$out" - if [ "${{ needs.context.outputs.trigger }}" = "release" ]; then - cargo build --release -p agentos-actor-plugin --target ${{ matrix.target }} - profile="release" - else - cargo build -p agentos-actor-plugin --target ${{ matrix.target }} - profile="debug" - fi - cp "target/${{ matrix.target }}/${profile}/libagentos_actor_plugin.so" \ - "$out/libagentos_actor_plugin.so" - echo "dir=$out" >> "$GITHUB_OUTPUT" - - name: Check glibc floor - run: | - scripts/ci/check-linux-glibc-floor.sh \ - "${{ steps.build.outputs.dir }}/libagentos_actor_plugin.so" - - name: Smoke-run in old Linux containers - run: | - scripts/ci/smoke-linux-artifacts.sh shared-library \ - "${{ steps.build.outputs.dir }}/libagentos_actor_plugin.so" - uses: actions/upload-artifact@v4 with: name: plugin-${{ matrix.platform }} - path: ${{ steps.build.outputs.dir }} + path: ${{ steps.extract.outputs.plugin_dir }} if-no-files-found: error build-sidecar-darwin: @@ -224,27 +172,47 @@ jobs: matrix: include: - platform: darwin-x64 + runner: [self-hosted, agentos-builder] target: x86_64-apple-darwin clang: x86_64-apple-darwin20.4 - platform: darwin-arm64 + runner: [self-hosted, agentos-builder] target: aarch64-apple-darwin clang: aarch64-apple-darwin20.4 - runs-on: ubuntu-22.04 + runs-on: ${{ matrix.runner }} steps: - uses: actions/checkout@v4 + - uses: docker/setup-buildx-action@v3 + with: + name: agentos-${{ matrix.platform }} + keep-state: true - name: Log in to ghcr.io run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin + - name: Compute build profile + id: mode + run: | + set -euo pipefail + if [ "${{ needs.context.outputs.trigger }}" = "release" ]; then + echo "profile=release" >> "$GITHUB_OUTPUT" + else + echo "profile=debug" >> "$GITHUB_OUTPUT" + fi - name: Cross-compile via osxcross + uses: docker/build-push-action@v6 + with: + context: . + file: docker/build/darwin.Dockerfile + build-args: | + TARGET=${{ matrix.target }} + CLANG=${{ matrix.clang }} + BUILD_PROFILE=${{ steps.mode.outputs.profile }} + CACHE_PLATFORM=${{ matrix.platform }} + RUST_TOOLCHAIN=${{ env.RUST_TOOLCHAIN }} + tags: agentos-darwin-${{ matrix.platform }} + load: true + - name: Extract darwin artifacts run: | set -euo pipefail - if [ "${{ needs.context.outputs.trigger }}" = "release" ]; then profile=release; else profile=debug; fi - DOCKER_BUILDKIT=1 docker build \ - -f docker/build/darwin.Dockerfile \ - --build-arg TARGET=${{ matrix.target }} \ - --build-arg CLANG=${{ matrix.clang }} \ - --build-arg BUILD_PROFILE="${profile}" \ - -t agentos-darwin-${{ matrix.platform }} . - sidecar_out="target/sidecar-artifacts/${{ matrix.platform }}" plugin_out="target/plugin-artifacts/${{ matrix.platform }}" mkdir -p "$sidecar_out" "$plugin_out" @@ -256,6 +224,14 @@ jobs: test -f "$sidecar_out/agentos-sidecar" test -f "$sidecar_out/agentos-native-sidecar" test -f "$plugin_out/libagentos_actor_plugin.dylib" + case "${{ matrix.platform }}" in + darwin-x64) expected_arch="x86_64" ;; + darwin-arm64) expected_arch="arm64" ;; + *) echo "unknown darwin platform: ${{ matrix.platform }}" >&2; exit 2 ;; + esac + file "$sidecar_out/agentos-sidecar" | grep -F "$expected_arch" + file "$sidecar_out/agentos-native-sidecar" | grep -F "$expected_arch" + file "$plugin_out/libagentos_actor_plugin.dylib" | grep -F "$expected_arch" - uses: actions/upload-artifact@v4 with: name: sidecar-${{ matrix.platform }} @@ -268,9 +244,9 @@ jobs: if-no-files-found: error publish-npm: - needs: [context, wasm-commands, build-sidecar, build-plugin, build-sidecar-darwin] + needs: [context, wasm-commands, build-sidecar, build-sidecar-darwin] name: Publish npm - if: ${{ !cancelled() && needs.wasm-commands.result == 'success' && needs.build-sidecar.result == 'success' && needs.build-plugin.result == 'success' && needs.build-sidecar-darwin.result == 'success' }} + if: ${{ !cancelled() && needs.wasm-commands.result == 'success' && needs.build-sidecar.result == 'success' && needs.build-sidecar-darwin.result == 'success' }} runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -330,14 +306,13 @@ jobs: pnpm --filter=publish exec tsx src/ci/bin.ts bump-versions \ --version ${{ needs.context.outputs.version }} \ --version-only - - name: Install wasm-pack - run: | - rustup target add wasm32-unknown-unknown - curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh - run: node packages/runtime-core/scripts/copy-wasm-commands.mjs --require - name: Build TypeScript packages + env: + AGENTOS_SKIP_NATIVE_META_BUILD: "1" run: | npx turbo build \ + --filter='!@rivet-dev/agentos-plugin' \ --filter='!@rivet-dev/agentos-playground' \ --filter='!./examples/*' \ --filter='!./examples/quickstart/*' @@ -394,6 +369,7 @@ jobs: cp "artifacts/sidecar-${p}/agentos-sidecar" "release-assets/agentos-sidecar-${target}" cp "artifacts/sidecar-${p}/agentos-native-sidecar" "release-assets/agentos-native-sidecar-${target}" done + chmod +x release-assets/agentos-sidecar-* release-assets/agentos-native-sidecar-* if [ -d crates/execution/assets/pyodide ]; then mkdir -p release-assets/pyodide cp -R crates/execution/assets/pyodide/. release-assets/pyodide/ diff --git a/.gitignore b/.gitignore index 5da228eb1d..985e5c514c 100644 --- a/.gitignore +++ b/.gitignore @@ -15,7 +15,8 @@ target/ # TypeScript *.tsbuildinfo dist/ -packages/agentos/src/generated/ +packages/agentos/src/generated/* +!packages/agentos/src/generated/actor-actions.generated.ts # IDE .vscode/ diff --git a/docker/build/darwin.Dockerfile b/docker/build/darwin.Dockerfile index e2187c7010..7e89f5a052 100644 --- a/docker/build/darwin.Dockerfile +++ b/docker/build/darwin.Dockerfile @@ -10,21 +10,40 @@ FROM ghcr.io/rivet-dev/rivet/builder-base-osxcross:0e33ceb98 ARG TARGET=aarch64-apple-darwin ARG CLANG=aarch64-apple-darwin20.4 ARG BUILD_PROFILE=debug +ARG CACHE_PLATFORM=darwin-arm64 +ARG RUST_TOOLCHAIN=1.91.1 ENV SDK=/root/osxcross/target/SDK/MacOSX11.3.sdk \ - RUSTC_WRAPPER= + RUSTC_WRAPPER=sccache \ + SCCACHE_DIR=/root/.cache/sccache \ + SCCACHE_IDLE_TIMEOUT=0 WORKDIR /build COPY . . -RUN rustup toolchain install stable --profile minimal && \ - rustup default stable && \ +RUN rustup toolchain install "$RUST_TOOLCHAIN" --profile minimal && \ + rustup default "$RUST_TOOLCHAIN" && \ rustup target add "$TARGET" -RUN corepack enable && \ - pnpm install --no-frozen-lockfile --filter='!@agentos/website' +RUN --mount=type=cache,id=pnpm-store-agentos-darwin,target=/root/.local/share/pnpm/store,sharing=locked \ + corepack enable && \ + pnpm config set store-dir /root/.local/share/pnpm/store && \ + pnpm install --no-frozen-lockfile --filter='!@agentos/website' -RUN tu=$(echo "$TARGET" | tr 'a-z-' 'A-Z_') && \ +RUN --mount=type=cache,id=cargo-registry-agentos-darwin,target=/usr/local/cargo/registry,sharing=locked \ + --mount=type=cache,id=cargo-git-agentos-darwin,target=/usr/local/cargo/git,sharing=locked \ + --mount=type=cache,id=cargo-target-agentos-${CACHE_PLATFORM},target=/build/target,sharing=locked \ + --mount=type=cache,id=sccache-agentos-${CACHE_PLATFORM},target=/root/.cache/sccache,sharing=locked \ + if ! command -v sccache >/dev/null 2>&1; then \ + unset RUSTC_WRAPPER; \ + elif ! (sccache --start-server 2>/tmp/sccache-start.err && sccache --show-stats >/dev/null 2>&1); then \ + echo "[sccache] unavailable, disabling:"; cat /tmp/sccache-start.err 2>/dev/null || true; \ + sccache --stop-server >/dev/null 2>&1 || true; \ + unset RUSTC_WRAPPER SCCACHE_DIR; \ + else \ + echo "[sccache] enabled via local BuildKit cache"; \ + fi && \ + tu=$(echo "$TARGET" | tr 'a-z-' 'A-Z_') && \ tl=$(echo "$TARGET" | tr - _) && \ export BINDGEN_EXTRA_CLANG_ARGS_${tl}="--sysroot=$SDK -isystem $SDK/usr/include" && \ export CFLAGS_${tl}="-B/root/osxcross/target/bin" && \ @@ -39,6 +58,7 @@ RUN tu=$(echo "$TARGET" | tr 'a-z-' 'A-Z_') && \ mkdir -p /artifacts && \ cp "target/$TARGET/$PROF/agentos-sidecar" /artifacts/agentos-sidecar && \ cp "target/$TARGET/$PROF/agentos-native-sidecar" /artifacts/agentos-native-sidecar && \ - cp "target/$TARGET/$PROF/libagentos_actor_plugin.dylib" /artifacts/libagentos_actor_plugin.dylib + cp "target/$TARGET/$PROF/libagentos_actor_plugin.dylib" /artifacts/libagentos_actor_plugin.dylib && \ + (sccache --show-stats 2>/dev/null || true) CMD ["ls", "-la", "/artifacts"] diff --git a/docker/build/linux-gnu.Dockerfile b/docker/build/linux-gnu.Dockerfile new file mode 100644 index 0000000000..08de122893 --- /dev/null +++ b/docker/build/linux-gnu.Dockerfile @@ -0,0 +1,106 @@ +# syntax=docker/dockerfile:1.10.0 +# +# Build AgentOS Linux GNU native artifacts in one cached container. The sysroot +# setup mirrors scripts/ci/setup-linux-gnu-sysroot.sh, but runs inside Docker so +# Cargo caches can be persisted with BuildKit/GHA like the Darwin build. +FROM ubuntu:24.04 + +ARG RUST_TOOLCHAIN=1.91.1 +ARG TARGET=x86_64-unknown-linux-gnu +ARG BUILD_PROFILE=debug +ARG CACHE_PLATFORM=linux-x64-gnu +ARG NODE_MAJOR=22 +ARG LINUX_GNU_LLVM_VERSION=22 +ARG LINUX_GNU_SYSROOT_TAG=sysroot-20250207 + +ENV DEBIAN_FRONTEND=noninteractive \ + CARGO_HOME=/usr/local/cargo \ + COREPACK_ENABLE_DOWNLOAD_PROMPT=0 \ + PNPM_HOME=/usr/local/pnpm \ + RUSTUP_HOME=/usr/local/rustup \ + PATH=/usr/local/cargo/bin:/usr/local/pnpm:/usr/local/rustup/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \ + RUSTC_WRAPPER=sccache \ + SCCACHE_DIR=/root/.cache/sccache \ + SCCACHE_IDLE_TIMEOUT=0 + +WORKDIR /build + +COPY scripts/ci/deno-memfd-create-shim.c scripts/ci/agentos-gettid-shim.c /tmp/agentos-ci/ + +RUN set -eux; \ + . /etc/os-release; \ + codename="${VERSION_CODENAME:?missing VERSION_CODENAME}"; \ + apt-get update; \ + apt-get install -y --no-install-recommends ca-certificates curl gnupg xz-utils binutils sccache build-essential pkg-config libssl-dev; \ + curl -fsSL "https://deb.nodesource.com/setup_${NODE_MAJOR}.x" | bash -; \ + apt-get install -y --no-install-recommends nodejs; \ + corepack enable; \ + corepack prepare pnpm@10.13.1 --activate; \ + echo "deb http://apt.llvm.org/${codename}/ llvm-toolchain-${codename}-${LINUX_GNU_LLVM_VERSION} main" \ + > "/etc/apt/sources.list.d/llvm-toolchain-${codename}-${LINUX_GNU_LLVM_VERSION}.list"; \ + curl -fsSL https://apt.llvm.org/llvm-snapshot.gpg.key \ + | gpg --dearmor \ + > /etc/apt/trusted.gpg.d/llvm-snapshot.gpg; \ + apt-get update; \ + apt-get install -y --no-install-recommends \ + "clang-${LINUX_GNU_LLVM_VERSION}" \ + "lld-${LINUX_GNU_LLVM_VERSION}"; \ + "clang-${LINUX_GNU_LLVM_VERSION}" -c -o /tmp/agentos_memfd_create_shim.o \ + /tmp/agentos-ci/deno-memfd-create-shim.c -fPIC; \ + "clang-${LINUX_GNU_LLVM_VERSION}" -c -o /tmp/agentos_gettid_shim.o \ + /tmp/agentos-ci/agentos-gettid-shim.c -fPIC; \ + if [ ! -x /usr/local/cargo/bin/rustup ]; then \ + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs \ + | sh -s -- -y --default-toolchain "${RUST_TOOLCHAIN}" --profile minimal --no-modify-path; \ + fi; \ + rustup default "${RUST_TOOLCHAIN}"; \ + rustup target add "${TARGET}"; \ + sysroot_arch="$(uname -m)"; \ + sysroot_url="https://github.com/denoland/deno_sysroot_build/releases/download/${LINUX_GNU_SYSROOT_TAG}/sysroot-${sysroot_arch}.tar.xz"; \ + curl -fsSL "${sysroot_url}" -o /tmp/agentos-sysroot.tar.xz; \ + rm -rf /sysroot; \ + cd /; \ + xzcat /tmp/agentos-sysroot.tar.xz | tar -x; \ + rm -rf /var/lib/apt/lists/* /tmp/agentos-sysroot.tar.xz + +COPY . . + +RUN --mount=type=cache,id=cargo-registry-agentos-${CACHE_PLATFORM},target=/usr/local/cargo/registry,sharing=locked \ + --mount=type=cache,id=cargo-git-agentos-${CACHE_PLATFORM},target=/usr/local/cargo/git,sharing=locked \ + --mount=type=cache,id=cargo-target-agentos-${CACHE_PLATFORM},target=/build/target,sharing=locked \ + --mount=type=cache,id=pnpm-store-agentos-${CACHE_PLATFORM},target=/root/.local/share/pnpm/store,sharing=locked \ + --mount=type=cache,id=rusty-v8-agentos-${CACHE_PLATFORM},target=/root/.cargo/.rusty_v8,sharing=locked \ + --mount=type=cache,id=sccache-agentos-${CACHE_PLATFORM},target=/root/.cache/sccache,sharing=locked \ + set -eux; \ + pnpm install --frozen-lockfile --filter '@rivet-dev/agentos-build-tools'; \ + . /sysroot/.env; \ + if ! command -v sccache >/dev/null 2>&1; then \ + unset RUSTC_WRAPPER; \ + elif ! (sccache --start-server 2>/tmp/sccache-start.err && sccache --show-stats >/dev/null 2>&1); then \ + echo "[sccache] unavailable, disabling:"; cat /tmp/sccache-start.err 2>/dev/null || true; \ + sccache --stop-server >/dev/null 2>&1 || true; \ + unset RUSTC_WRAPPER SCCACHE_DIR; \ + else \ + echo "[sccache] enabled via local BuildKit cache"; \ + fi; \ + export CC="/usr/bin/clang-${LINUX_GNU_LLVM_VERSION}"; \ + export CFLAGS="${CFLAGS:-}"; \ + export RUSTFLAGS="-C linker-plugin-lto=true \ + -C linker=clang-${LINUX_GNU_LLVM_VERSION} \ + -C link-arg=-fuse-ld=lld-${LINUX_GNU_LLVM_VERSION} \ + -C link-arg=-ldl \ + -C link-arg=-Wl,--allow-shlib-undefined \ + -C link-arg=-Wl,--thinlto-cache-dir=/build/target/release/lto-cache \ + -C link-arg=-Wl,--thinlto-cache-policy,cache_size_bytes=700m \ + -C link-arg=/tmp/agentos_memfd_create_shim.o \ + -C link-arg=/tmp/agentos_gettid_shim.o \ + ${RUSTFLAGS:-}"; \ + if [ "$BUILD_PROFILE" = "release" ]; then FLAG="--release"; PROF=release; else FLAG=""; PROF=debug; fi; \ + cargo build $FLAG -p agentos-sidecar -p agentos-native-sidecar -p agentos-actor-plugin --target "$TARGET"; \ + mkdir -p /artifacts; \ + cp "target/$TARGET/$PROF/agentos-sidecar" /artifacts/agentos-sidecar; \ + cp "target/$TARGET/$PROF/agentos-native-sidecar" /artifacts/agentos-native-sidecar; \ + cp "target/$TARGET/$PROF/libagentos_actor_plugin.so" /artifacts/libagentos_actor_plugin.so; \ + (sccache --show-stats 2>/dev/null || true) + +CMD ["ls", "-la", "/artifacts"] diff --git a/docker/build/linux-gnu.Dockerfile.dockerignore b/docker/build/linux-gnu.Dockerfile.dockerignore new file mode 100644 index 0000000000..4b06a65cd2 --- /dev/null +++ b/docker/build/linux-gnu.Dockerfile.dockerignore @@ -0,0 +1,8 @@ +# Permissive context for the Linux native build. Send source, drop generated +# artifacts that the in-container build regenerates. +.git +**/node_modules +**/target +**/dist +**/.astro +**/.turbo diff --git a/packages/agentos-plugin/scripts/build.mjs b/packages/agentos-plugin/scripts/build.mjs index 271b0ffbc4..5460529bb2 100644 --- a/packages/agentos-plugin/scripts/build.mjs +++ b/packages/agentos-plugin/scripts/build.mjs @@ -5,6 +5,13 @@ import { execFileSync } from "node:child_process"; const release = process.argv.includes("--release"); +if (process.env.AGENTOS_SKIP_NATIVE_META_BUILD === "1") { + console.log( + "Skipping agentos-actor-plugin cargo build; publish CI already staged platform plugin artifacts.", + ); + process.exit(0); +} + execFileSync( "cargo", ["build", "-p", "agentos-actor-plugin", ...(release ? ["--release"] : [])], diff --git a/packages/agentos/src/generated/actor-actions.generated.ts b/packages/agentos/src/generated/actor-actions.generated.ts new file mode 100644 index 0000000000..9671fd7470 --- /dev/null +++ b/packages/agentos/src/generated/actor-actions.generated.ts @@ -0,0 +1,164 @@ +// @generated by agentos-actor-plugin. Do not edit. +// This file is committed so package builds do not need to compile the native +// plugin just to regenerate TypeScript action types. +import type { + ExecResult, + PermissionReply, + ProcessInfo, + ProcessTreeNode, + SpawnedProcessInfo, + VirtualStat, +} from "@rivet-dev/agentos-core"; +import type { + PersistedSessionEvent, + PersistedSessionRecord, + PromptResult, + SerializableCronJobInfo, + SerializableCronJobOptions, +} from "../types.js"; + +// The leading actor context arg; stripped from the client-facing method. +// biome-ignore lint/suspicious/noExplicitAny: ctx is server-side only and never reaches the typed client surface. +type Ctx = any; + +export interface DirEntry { + path: string; + type: "file" | "directory" | "symlink"; + size: number; +} + +export interface ReaddirEntry { + name: string; + isDirectory: boolean; + isSymbolicLink: boolean; +} + +export interface SpawnedProcess { + pid: number; +} + +export interface ExecActionOptions { + env?: Record; + cwd?: string; +} + +export interface SpawnActionOptions { + env?: Record; + cwd?: string; + streamStdin?: boolean; +} + +export interface ScheduledCronJob { + id: string; +} + +export interface VmFetchOptions { + method?: string; + headers?: Record; + body?: string | Uint8Array; +} + +export interface VmFetchResponse { + status: number; + statusText: string; + headers: Record; + body: Uint8Array; +} + +export interface CreateSessionOptions { + cwd?: string; + env?: Record; + skipOsInstructions?: boolean; + additionalInstructions?: string; +} + +export interface SignedPreviewUrl { + path: string; + token: string; + port: number; + expiresAt: number; +} + +export interface OpenShellActionOptions { + command?: string; + args?: string[]; + env?: Record; + cwd?: string; + cols?: number; + rows?: number; +} + +export interface OpenShellResult { + shellId: string; +} + +export interface WriteFileResult { + path: string; + success: boolean; + error?: string; +} + +export interface ReadFileResult { + path: string; + content?: Uint8Array; + error?: string; +} + +export interface MountInfo { + path: string; + kind: "host_dir" | "s3" | "google_drive" | "sandbox_agent"; + config: unknown; + readOnly: boolean; +} + +export interface SoftwareInfo { + package: string; + kind: "wasm-commands" | "agent" | "tool"; + version: string | null; + commands: string[]; +} + +export type AgentOsActions = { + readFile: (c: Ctx, path: string) => Promise; + writeFile: (c: Ctx, path: string, content: string | Uint8Array) => Promise; + stat: (c: Ctx, path: string) => Promise; + mkdir: (c: Ctx, path: string) => Promise; + readdir: (c: Ctx, path: string) => Promise; + readdirEntries: (c: Ctx, path: string) => Promise; + exists: (c: Ctx, path: string) => Promise; + move: (c: Ctx, from: string, to: string) => Promise; + deleteFile: (c: Ctx, path: string, options?: { recursive?: boolean }) => Promise; + writeFiles: ( c: Ctx, entries: { path: string; content: string | Uint8Array }[], ) => Promise; + readFiles: (c: Ctx, paths: string[]) => Promise; + readdirRecursive: (c: Ctx, path: string) => Promise; + exec: ( c: Ctx, command: string, options?: ExecActionOptions, ) => Promise; + spawn: ( c: Ctx, command: string, args: string[], options?: SpawnActionOptions, ) => Promise; + waitProcess: (c: Ctx, pid: number) => Promise; + killProcess: (c: Ctx, pid: number) => Promise; + stopProcess: (c: Ctx, pid: number) => Promise; + listProcesses: (c: Ctx) => Promise; + allProcesses: (c: Ctx) => Promise; + processTree: (c: Ctx) => Promise; + getProcess: (c: Ctx, pid: number) => Promise; + writeProcessStdin: (c: Ctx, pid: number, data: string | Uint8Array) => Promise; + closeProcessStdin: (c: Ctx, pid: number) => Promise; + openShell: (c: Ctx, options?: OpenShellActionOptions) => Promise; + writeShell: (c: Ctx, shellId: string, data: string | Uint8Array) => Promise; + resizeShell: (c: Ctx, shellId: string, cols: number, rows: number) => Promise; + closeShell: (c: Ctx, shellId: string) => Promise; + waitShell: (c: Ctx, shellId: string) => Promise; + vmFetch: ( c: Ctx, port: number, url: string, options?: VmFetchOptions, ) => Promise; + scheduleCron: (c: Ctx, options: SerializableCronJobOptions) => Promise; + listCronJobs: (c: Ctx) => Promise; + cancelCronJob: (c: Ctx, id: string) => Promise; + createSession: (c: Ctx, agentType: string, options?: CreateSessionOptions) => Promise; + sendPrompt: (c: Ctx, sessionId: string, text: string) => Promise; + closeSession: (c: Ctx, sessionId: string) => Promise; + listPersistedSessions: (c: Ctx) => Promise; + getSessionEvents: (c: Ctx, sessionId: string) => Promise; + respondPermission: ( c: Ctx, sessionId: string, permissionId: string, reply: PermissionReply, ) => Promise; + createSignedPreviewUrl: (c: Ctx, port: number, ttlSeconds: number) => Promise; + expireSignedPreviewUrl: (c: Ctx, token: string) => Promise; + listMounts: (c: Ctx) => Promise; + listSoftware: (c: Ctx) => Promise; +}; diff --git a/packages/sidecar-binary/scripts/build.mjs b/packages/sidecar-binary/scripts/build.mjs index 876f84c2b1..4bd56280fa 100644 --- a/packages/sidecar-binary/scripts/build.mjs +++ b/packages/sidecar-binary/scripts/build.mjs @@ -5,6 +5,13 @@ import { execFileSync } from "node:child_process"; const release = process.argv.includes("--release"); +if (process.env.AGENTOS_SKIP_NATIVE_META_BUILD === "1") { + console.log( + "Skipping agentos-sidecar cargo build; publish CI already staged platform sidecar artifacts.", + ); + process.exit(0); +} + execFileSync( "cargo", ["build", "-p", "agentos-sidecar", ...(release ? ["--release"] : [])], diff --git a/turbo.json b/turbo.json index 754ab26ac4..fac7639fd5 100644 --- a/turbo.json +++ b/turbo.json @@ -1,5 +1,10 @@ { "$schema": "https://turbo.build/schema.json", + "globalEnv": ["AGENTOS_SKIP_NATIVE_META_BUILD"], + "globalDependencies": [ + "crates/sidecar-protocol/protocol/**", + "packages/build-tools/scripts/compile-sidecar-protocol.mjs" + ], "tasks": { "//#fmt": { "cache": false @@ -29,6 +34,16 @@ "@rivet-dev/agentos-sidecar#build": { "cache": false }, + "@rivet-dev/agentos-runtime-core#build": { + "dependsOn": ["^build"], + "inputs": [ + "src/**", + "scripts/**", + "tsconfig.json", + "package.json" + ], + "outputs": ["dist/**", "commands/**"] + }, "@rivet-dev/agentos-plugin#build": { "cache": false }