diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 789c0c0..75fc6e3 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -7,6 +7,23 @@ on: branches: [main] release: types: [published] + workflow_dispatch: + inputs: + include_duckdb: + description: 'Build images with DuckDB support enabled' + required: false + type: boolean + default: false + deploy_phala: + description: 'Deploy the dstack image to Phala after build' + required: false + type: boolean + default: false + image_version: + description: 'Manual image version to tag/deploy, without leading v. Defaults to the selected ref name.' + required: false + type: string + default: '' env: REGISTRY: ghcr.io @@ -25,38 +42,53 @@ jobs: uses: docker/setup-buildx-action@v3 - name: Log in to GHCR - if: github.event_name == 'release' || (github.event_name == 'push' && github.ref == 'refs/heads/main') + if: github.event_name == 'release' || github.event_name == 'workflow_dispatch' || (github.event_name == 'push' && github.ref == 'refs/heads/main') uses: docker/login-action@v3 with: registry: ${{ env.REGISTRY }} username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} + - name: Resolve optional build features + id: build_features + run: | + FEATURES="" + IMAGE_SUFFIX="" + if [ "${{ github.event_name }}" = "workflow_dispatch" ] && [ "${{ inputs.include_duckdb }}" = "true" ]; then + FEATURES="duckdb" + IMAGE_SUFFIX="-duckdb" + fi + echo "cargo_features=${FEATURES}" >> "$GITHUB_OUTPUT" + echo "image_suffix=${IMAGE_SUFFIX}" >> "$GITHUB_OUTPUT" + echo "manual_version=${{ inputs.image_version }}" >> "$GITHUB_OUTPUT" + - name: Extract metadata (tags, labels) id: meta uses: docker/metadata-action@v5 with: images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} tags: | - type=semver,pattern={{version}} - type=semver,pattern={{major}}.{{minor}} - type=semver,pattern={{major}} + type=semver,pattern={{version}},suffix=${{ steps.build_features.outputs.image_suffix }} + type=semver,pattern={{major}}.{{minor}},suffix=${{ steps.build_features.outputs.image_suffix }} + type=semver,pattern={{major}},suffix=${{ steps.build_features.outputs.image_suffix }} + type=raw,value=${{ steps.build_features.outputs.manual_version }}${{ steps.build_features.outputs.image_suffix }},enable=${{ github.event_name == 'workflow_dispatch' && steps.build_features.outputs.manual_version != '' }} type=sha,prefix= - type=raw,value=latest,enable={{is_default_branch}} + type=raw,value=latest${{ steps.build_features.outputs.image_suffix }},enable={{is_default_branch}} - name: Build and push Docker image uses: docker/build-push-action@v5 with: context: . - push: ${{ github.event_name == 'release' || (github.event_name == 'push' && github.ref == 'refs/heads/main') }} + push: ${{ github.event_name == 'release' || github.event_name == 'workflow_dispatch' || (github.event_name == 'push' && github.ref == 'refs/heads/main') }} tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} + build-args: CARGO_FEATURES=${{ steps.build_features.outputs.cargo_features }} cache-from: type=gha cache-to: type=gha,mode=max build-dstack: runs-on: ubuntu-latest - if: github.event_name == 'release' || (github.event_name == 'push' && github.ref == 'refs/heads/main') + if: github.event_name == 'release' || github.event_name == 'workflow_dispatch' || (github.event_name == 'push' && github.ref == 'refs/heads/main') permissions: contents: read packages: write @@ -73,17 +105,34 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} + - name: Resolve optional build features + id: build_features + run: | + FEATURES="dstack" + IMAGE_SUFFIX="" + SHA_PREFIX="dstack-" + if [ "${{ github.event_name }}" = "workflow_dispatch" ] && [ "${{ inputs.include_duckdb }}" = "true" ]; then + FEATURES="dstack duckdb" + IMAGE_SUFFIX="-duckdb" + SHA_PREFIX="dstack-duckdb-" + fi + echo "cargo_features=${FEATURES}" >> "$GITHUB_OUTPUT" + echo "image_suffix=${IMAGE_SUFFIX}" >> "$GITHUB_OUTPUT" + echo "sha_prefix=${SHA_PREFIX}" >> "$GITHUB_OUTPUT" + echo "manual_version=${{ inputs.image_version }}" >> "$GITHUB_OUTPUT" + - name: Extract metadata (tags, labels) id: meta uses: docker/metadata-action@v5 with: images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} tags: | - type=semver,pattern={{version}},suffix=-dstack - type=semver,pattern={{major}}.{{minor}},suffix=-dstack - type=semver,pattern={{major}},suffix=-dstack - type=sha,prefix=dstack- - type=raw,value=dstack,enable={{is_default_branch}} + type=semver,pattern={{version}},suffix=-dstack${{ steps.build_features.outputs.image_suffix }} + type=semver,pattern={{major}}.{{minor}},suffix=-dstack${{ steps.build_features.outputs.image_suffix }} + type=semver,pattern={{major}},suffix=-dstack${{ steps.build_features.outputs.image_suffix }} + type=raw,value=${{ steps.build_features.outputs.manual_version }}-dstack${{ steps.build_features.outputs.image_suffix }},enable=${{ github.event_name == 'workflow_dispatch' && steps.build_features.outputs.manual_version != '' }} + type=sha,prefix=${{ steps.build_features.outputs.sha_prefix }} + type=raw,value=dstack${{ steps.build_features.outputs.image_suffix }},enable={{is_default_branch}} - name: Build and push dstack Docker image uses: docker/build-push-action@v5 @@ -92,14 +141,14 @@ jobs: push: true tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} - build-args: CARGO_FEATURES=dstack + build-args: CARGO_FEATURES=${{ steps.build_features.outputs.cargo_features }} cache-from: type=gha,scope=dstack cache-to: type=gha,mode=max,scope=dstack deploy-phala: runs-on: ubuntu-latest needs: [build, build-dstack] - if: github.event_name == 'release' + if: github.event_name == 'release' || (github.event_name == 'workflow_dispatch' && inputs.deploy_phala) steps: - uses: actions/checkout@v4 @@ -113,15 +162,31 @@ jobs: - name: Update compose with release tag run: | - TAG="${{ github.event.release.tag_name }}" + if [ "${{ github.event_name }}" = "release" ]; then + TAG="${{ github.event.release.tag_name }}" + else + TAG="${{ inputs.image_version }}" + if [ -z "${TAG}" ]; then + if [ "${GITHUB_REF_TYPE}" != "tag" ]; then + echo "::error::workflow_dispatch deploy_phala requires image_version unless the workflow is run from a tag" + exit 1 + fi + TAG="${GITHUB_REF_NAME}" + fi + fi # metadata-action emits {{version}} without the leading v (e.g. 1.3.0). # Strip a leading v from the tag to match the pushed image tag. VERSION="${TAG#v}" + DUCKDB_SUFFIX="" + if [ "${{ github.event_name }}" = "workflow_dispatch" ] && [ "${{ inputs.include_duckdb }}" = "true" ]; then + DUCKDB_SUFFIX="-duckdb" + fi # The prod CVM uses the dstack-suffixed image. Replace the floating # ":dstack" tag in the checked-in compose with the versioned tag - # built by the build-dstack job (e.g. ":1.3.0-dstack"). - sed -i "s|ghcr.io/tinycloudlabs/tinycloud-node:dstack|ghcr.io/tinycloudlabs/tinycloud-node:${VERSION}-dstack|g" docker-compose.dstack-postgres.yaml - echo "Resolved image tag: ghcr.io/tinycloudlabs/tinycloud-node:${VERSION}-dstack" + # built by the build-dstack job (e.g. ":1.3.0-dstack" or + # ":1.3.0-dstack-duckdb"). + sed -i "s|ghcr.io/tinycloudlabs/tinycloud-node:dstack|ghcr.io/tinycloudlabs/tinycloud-node:${VERSION}-dstack${DUCKDB_SUFFIX}|g" docker-compose.dstack-postgres.yaml + echo "Resolved image tag: ghcr.io/tinycloudlabs/tinycloud-node:${VERSION}-dstack${DUCKDB_SUFFIX}" cat docker-compose.dstack-postgres.yaml - name: Deploy to Phala Cloud diff --git a/Dockerfile b/Dockerfile index ac8f5fc..222daee 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ # Build argument to select runtime base (default scratch) ARG RUNTIME_BASE=scratch -# Optional: pass "dstack" to enable TEE support +# Optional: pass "dstack", "duckdb", or "dstack duckdb" to enable build features. ARG CARGO_FEATURES="" FROM rust:alpine AS chef @@ -27,12 +27,20 @@ ARG CARGO_FEATURES="" COPY --from=planner /app/recipe.json recipe.json RUN --mount=type=cache,target=/usr/local/cargo/registry \ --mount=type=cache,target=/app/target \ - cargo chef cook --release --recipe-path recipe.json ${CARGO_FEATURES:+--features $CARGO_FEATURES} + if [ -n "$CARGO_FEATURES" ]; then \ + cargo chef cook --release --recipe-path recipe.json --features "$CARGO_FEATURES"; \ + else \ + cargo chef cook --release --recipe-path recipe.json; \ + fi COPY --from=planner /app/ ./ RUN chmod +x ./scripts/init-tinycloud-data.sh && ./scripts/init-tinycloud-data.sh RUN --mount=type=cache,target=/usr/local/cargo/registry \ --mount=type=cache,target=/app/target \ - cargo build --release -p tinycloud-node ${CARGO_FEATURES:+--features $CARGO_FEATURES} && \ + if [ -n "$CARGO_FEATURES" ]; then \ + cargo build --release -p tinycloud-node --features "$CARGO_FEATURES"; \ + else \ + cargo build --release -p tinycloud-node; \ + fi && \ cp /app/target/release/tinycloud /app/tinycloud RUN addgroup -g 1000 tinycloud && adduser -u 1000 -G tinycloud -s /bin/sh -D tinycloud RUN mkdir -p /scratch-tmp && chmod 1777 /scratch-tmp diff --git a/docker-compose.dstack-postgres.yaml b/docker-compose.dstack-postgres.yaml index f9293c3..0bce0db 100644 --- a/docker-compose.dstack-postgres.yaml +++ b/docker-compose.dstack-postgres.yaml @@ -8,6 +8,8 @@ services: tinycloud: + # DuckDB-enabled images are opt-in and use the -duckdb suffix, for example: + # ghcr.io/tinycloudlabs/tinycloud-node:1.4.0-dstack-duckdb image: ghcr.io/tinycloudlabs/tinycloud-node:dstack restart: unless-stopped ports: diff --git a/docker-compose.dstack.yaml b/docker-compose.dstack.yaml index 06cccf0..394497f 100644 --- a/docker-compose.dstack.yaml +++ b/docker-compose.dstack.yaml @@ -3,6 +3,8 @@ # # Deploy on dstack: # docker build --build-arg CARGO_FEATURES=dstack -t tinycloud-dstack . +# # DuckDB is opt-in: +# docker build --build-arg "CARGO_FEATURES=dstack duckdb" -t tinycloud-dstack-duckdb . # # Or use the pre-built image from ghcr.io # # The same image works on dstack (TEE) and classic (non-TEE) deployments. diff --git a/tinycloud-core/Cargo.toml b/tinycloud-core/Cargo.toml index 99fc16c..95ac8d9 100644 --- a/tinycloud-core/Cargo.toml +++ b/tinycloud-core/Cargo.toml @@ -10,6 +10,7 @@ default = ["sqlite", "postgres", "mysql", "tokio"] sqlite = ["sea-orm/sqlx-sqlite"] postgres = ["sea-orm/sqlx-postgres"] mysql = ["sea-orm/sqlx-mysql"] +duckdb = ["dep:arrow", "dep:duckdb"] tokio = ["sea-orm/runtime-tokio-rustls"] async-std = ["sea-orm/runtime-async-std-rustls"] @@ -34,8 +35,8 @@ rusqlite.workspace = true sqlparser.workspace = true tracing.workspace = true tokio.workspace = true -duckdb = { version = "1.1", features = ["bundled", "appender-arrow"] } -arrow = { version = "56", features = ["ipc"] } +duckdb = { version = "1.1", features = ["bundled", "appender-arrow"], optional = true } +arrow = { version = "56", features = ["ipc"], optional = true } tempfile = "3" aes-gcm = "0.10" rand = "0.8" diff --git a/tinycloud-core/src/lib.rs b/tinycloud-core/src/lib.rs index 2bd9e8c..8b6b7d9 100644 --- a/tinycloud-core/src/lib.rs +++ b/tinycloud-core/src/lib.rs @@ -1,5 +1,6 @@ pub mod database_artifacts; pub mod db; +#[cfg(feature = "duckdb")] pub mod duckdb; pub mod encryption; pub mod encryption_network; diff --git a/tinycloud-node-server/Cargo.toml b/tinycloud-node-server/Cargo.toml index d15792f..fdd7f06 100644 --- a/tinycloud-node-server/Cargo.toml +++ b/tinycloud-node-server/Cargo.toml @@ -54,6 +54,7 @@ tracing-subscriber = { version = "0.3.20", features = ["env-filter", "json"] } [dependencies.tinycloud-core] path = "../tinycloud-core/" +default-features = false features = ["sqlite", "postgres", "mysql", "tokio"] [dependencies.tinycloud-auth] @@ -62,6 +63,7 @@ path = "../tinycloud-auth/" [features] default = [] dstack = [] +duckdb = ["tinycloud-core/duckdb"] # cargo-binstall support: allows `cargo binstall tinycloud-node` [package.metadata.binstall] diff --git a/tinycloud-node-server/src/lib.rs b/tinycloud-node-server/src/lib.rs index 5ce0bfc..2d72850 100644 --- a/tinycloud-node-server/src/lib.rs +++ b/tinycloud-node-server/src/lib.rs @@ -49,9 +49,10 @@ use storage::{ s3::{S3BlockConfig, S3BlockStore}, }; use tee::TeeContext; +#[cfg(feature = "duckdb")] +use tinycloud_core::duckdb::DuckDbService; use tinycloud_core::{ database_artifacts::SeaOrmDatabaseArtifactRepository, - duckdb::DuckDbService, encryption_network::{EncryptionService, LocalOneOfOneBackend}, keys::{SecretsSetup, StaticSecret}, sea_orm::{ConnectOptions, Database, DatabaseConnection}, @@ -239,6 +240,7 @@ pub async fn app(config: &Figment) -> Result> { database_artifact_repository.clone(), ); + #[cfg(feature = "duckdb")] let duckdb_service = DuckDbService::new( tinycloud_config .storage @@ -276,8 +278,10 @@ pub async fn app(config: &Figment) -> Result> { header_name: tinycloud_config.log.tracing.traceheader, }) .manage(tinycloud) - .manage(sql_service) - .manage(duckdb_service) + .manage(sql_service); + #[cfg(feature = "duckdb")] + let rocket = rocket.manage(duckdb_service); + let rocket = rocket .manage(quota_cache) .manage(hook_runtime) .manage(signed_url_runtime) @@ -384,12 +388,13 @@ async fn ensure_local_dirs(storage: &config::Storage) -> Result<()> { } } - // SQL and DuckDB storage paths are always local filesystem + // SQL storage paths are always local filesystem. if let Some(ref sql_path) = storage.sql.path { tokio::fs::create_dir_all(sql_path) .await .with_context(|| format!("creating SQL storage directory: {}", sql_path))?; } + #[cfg(feature = "duckdb")] if let Some(ref duckdb_path) = storage.duckdb.path { tokio::fs::create_dir_all(duckdb_path) .await diff --git a/tinycloud-node-server/src/routes/hooks.rs b/tinycloud-node-server/src/routes/hooks.rs index 2a6c14a..8e1a266 100644 --- a/tinycloud-node-server/src/routes/hooks.rs +++ b/tinycloud-node-server/src/routes/hooks.rs @@ -349,14 +349,15 @@ pub async fn delete_webhook( } fn validate_subscription(subscription: &HookSubscription) -> Result<(), (Status, String)> { - if !matches!(subscription.service.as_str(), "kv" | "sql" | "duckdb") { + let service = subscription.service.as_str(); + if !(matches!(service, "kv" | "sql") || cfg!(feature = "duckdb") && service == "duckdb") { return Err((Status::BadRequest, "Unsupported hook service".to_string())); } - let allowed_abilities: &[&str] = match subscription.service.as_str() { + let allowed_abilities: &[&str] = match service { "kv" => &["tinycloud.kv/put", "tinycloud.kv/del"], "sql" => &["tinycloud.sql/write"], - "duckdb" => &["tinycloud.duckdb/write"], + "duckdb" if cfg!(feature = "duckdb") => &["tinycloud.duckdb/write"], _ => unreachable!(), }; @@ -833,7 +834,7 @@ mod tests { } #[tokio::test] - async fn accepts_sql_and_duckdb_subscription_filters() { + async fn accepts_sql_subscription_filters() { validate_subscription(&HookSubscription { space: "tinycloud:space".to_string(), service: "sql".to_string(), @@ -841,7 +842,11 @@ mod tests { abilities: vec!["tinycloud.sql/write".to_string()], }) .expect("sql subscription should be allowed"); + } + #[cfg(feature = "duckdb")] + #[tokio::test] + async fn accepts_duckdb_subscription_filters() { validate_subscription(&HookSubscription { space: "tinycloud:space".to_string(), service: "duckdb".to_string(), diff --git a/tinycloud-node-server/src/routes/mod.rs b/tinycloud-node-server/src/routes/mod.rs index c937c09..ecefef2 100644 --- a/tinycloud-node-server/src/routes/mod.rs +++ b/tinycloud-node-server/src/routes/mod.rs @@ -21,8 +21,11 @@ use crate::{ tracing::TracingSpan, BlockStage, BlockStores, TinyCloud, }; +#[cfg(feature = "duckdb")] +use tinycloud_core::duckdb::{ + DuckDbCaveats, DuckDbError, DuckDbRequest, DuckDbResponse, DuckDbService, +}; use tinycloud_core::{ - duckdb::{DuckDbCaveats, DuckDbError, DuckDbRequest, DuckDbResponse, DuckDbService}, encryption_network::EncryptionService, events::Invocation, models::{hook_delivery, hook_subscription, kv_delete, kv_write}, @@ -63,16 +66,10 @@ fn build_info( encryption: &State, ) -> NodeInfo { #[allow(unused_mut)] - let mut features = vec![ - "kv", - "delegation", - "sharing", - "sql", - "duckdb", - "hooks", - "signed-urls", - "encryption", - ]; + let mut features = vec!["kv", "delegation", "sharing", "sql"]; + #[cfg(feature = "duckdb")] + features.push("duckdb"); + features.extend(["hooks", "signed-urls", "encryption"]); #[cfg(feature = "dstack")] features.push("tee"); NodeInfo { @@ -250,6 +247,7 @@ pub async fn delegate( } #[post("/invoke", data = "")] +#[cfg(feature = "duckdb")] #[allow(clippy::too_many_arguments)] pub async fn invoke( i: AuthHeaderGetter, @@ -263,6 +261,74 @@ pub async fn invoke( sql_service: &State, duckdb_service: &State, hook_runtime: &State, +) -> Result::Readable>, (Status, String)> { + invoke_impl( + i, + req_span, + headers, + data, + staging, + tinycloud, + config, + quota_cache, + sql_service, + duckdb_service, + hook_runtime, + ) + .await +} + +#[post("/invoke", data = "")] +#[cfg(not(feature = "duckdb"))] +#[allow(clippy::too_many_arguments)] +pub async fn invoke( + i: AuthHeaderGetter, + req_span: TracingSpan, + headers: ObjectHeaders, + data: DataIn<'_>, + staging: &State, + tinycloud: &State, + config: &State, + quota_cache: &State, + sql_service: &State, + hook_runtime: &State, +) -> Result::Readable>, (Status, String)> { + invoke_impl( + i, + req_span, + headers, + data, + staging, + tinycloud, + config, + quota_cache, + sql_service, + (), + hook_runtime, + ) + .await +} + +#[cfg(feature = "duckdb")] +type DuckDbInvokeState<'a> = &'a State; +#[cfg(not(feature = "duckdb"))] +type DuckDbInvokeState<'a> = (); + +#[allow(clippy::too_many_arguments)] +async fn invoke_impl( + i: AuthHeaderGetter, + req_span: TracingSpan, + headers: ObjectHeaders, + data: DataIn<'_>, + staging: &State, + tinycloud: &State, + config: &State, + quota_cache: &State, + sql_service: &State, + #[cfg_attr(not(feature = "duckdb"), allow(unused_variables))] duckdb_service: DuckDbInvokeState< + '_, + >, + hook_runtime: &State, ) -> Result::Readable>, (Status, String)> { let action_label = "invocation"; let span = info_span!(parent: &req_span.0, "invoke", action = %action_label); @@ -306,43 +372,62 @@ pub async fn invoke( return result; } - // Check for DuckDB capabilities - let duckdb_caps: Vec<_> = - i.0 .0 - .capabilities - .iter() - .filter_map(|c| match (&c.resource, c.ability.as_ref().as_ref()) { - (Resource::TinyCloud(r), ability) - if r.service().as_str() == "duckdb" - && ability.starts_with("tinycloud.duckdb/") => - { - Some(( - r.space().clone(), - r.path().map(|p| p.to_string()), - ability.to_string(), - )) - } - _ => None, - }) - .collect(); + #[cfg(feature = "duckdb")] + { + // Check for DuckDB capabilities + let duckdb_caps: Vec<_> = + i.0 .0 + .capabilities + .iter() + .filter_map(|c| match (&c.resource, c.ability.as_ref().as_ref()) { + (Resource::TinyCloud(r), ability) + if r.service().as_str() == "duckdb" + && ability.starts_with("tinycloud.duckdb/") => + { + Some(( + r.space().clone(), + r.path().map(|p| p.to_string()), + ability.to_string(), + )) + } + _ => None, + }) + .collect(); - if !duckdb_caps.is_empty() { - let arrow_format = headers.0 .0.iter().any(|(k, v)| { - k.eq_ignore_ascii_case("accept") - && v.contains("application/vnd.apache.arrow.stream") - }); - let result = handle_duckdb_invoke( - i, - data, - tinycloud, - duckdb_service, - hook_runtime, - &duckdb_caps, - arrow_format, + if !duckdb_caps.is_empty() { + let arrow_format = headers.0 .0.iter().any(|(k, v)| { + k.eq_ignore_ascii_case("accept") + && v.contains("application/vnd.apache.arrow.stream") + }); + let result = handle_duckdb_invoke( + i, + data, + tinycloud, + duckdb_service, + hook_runtime, + &duckdb_caps, + arrow_format, + ) + .await; + timer.observe_duration(); + return result; + } + } + + #[cfg(not(feature = "duckdb"))] + if i.0 .0.capabilities.iter().any(|c| { + matches!( + (&c.resource, c.ability.as_ref().as_ref()), + (Resource::TinyCloud(r), ability) + if r.service().as_str() == "duckdb" + && ability.starts_with("tinycloud.duckdb/") ) - .await; + }) { timer.observe_duration(); - return result; + return Err(( + Status::NotImplemented, + "DuckDB support is not enabled on this node".to_string(), + )); } let mut put_iter = i.0 .0.capabilities.iter().filter_map(|c| { @@ -721,6 +806,7 @@ fn sql_error_to_status(err: &SqlError) -> Status { } } +#[cfg(feature = "duckdb")] async fn handle_duckdb_invoke( i: AuthHeaderGetter, data: DataIn<'_>, @@ -851,6 +937,7 @@ async fn handle_duckdb_invoke( } } +#[cfg(feature = "duckdb")] fn duckdb_error_to_status(err: &DuckDbError) -> Status { match err { DuckDbError::DuckDb(_) => Status::BadRequest,