refactor(object-store): share one URL registry across Python, Java and DuckDB

.github/workflows/ci.yml

@@ -246,7 +246,7 @@ jobs:
             target: wasm32-unknown-unknown
             env:
               rustflags: "RUSTFLAGS='-A warnings --cfg getrandom_backend=\"unsupported\"'"
-            args: "--target wasm32-unknown-unknown --exclude vortex --exclude vortex-cuda --exclude vortex-cub --exclude vortex-nvcomp --exclude vortex-datafusion --exclude vortex-duckdb --exclude vortex-tui --exclude vortex-zstd --exclude vortex-test-e2e-cuda --exclude vortex-python-cuda --exclude vortex-sqllogictest --exclude vortex-parquet-variant"
+            args: "--target wasm32-unknown-unknown --exclude vortex --exclude vortex-cuda --exclude vortex-cub --exclude vortex-nvcomp --exclude vortex-datafusion --exclude vortex-duckdb --exclude vortex-tui --exclude vortex-zstd --exclude vortex-test-e2e-cuda --exclude vortex-python-cuda --exclude vortex-sqllogictest --exclude vortex-parquet-variant --exclude vortex-object-store-opendal"
     steps:
       - uses: runs-on/action@v2
         if: github.repository == 'vortex-data/vortex'

Cargo.toml

@@ -95,7 +95,7 @@ rust-version = "1.91.0"
 version = "0.1.0"
 
 [workspace.dependencies]
-anyhow = "1.0.97"
+anyhow = "1.0.100"
 arbitrary = "1.3.2"
 arc-swap = "1.9"
 arcref = "0.2.0"
@@ -173,7 +173,7 @@ indicatif = "0.18.0"
 insta = "1.43"
 inventory = "0.3.20"
 itertools = "0.14.0"
-jiff = "0.2.0"
+jiff = "0.2.17"
 jni = { version = "0.22.0" }
 kanal = "0.1.1"
 lasso = { version = "0.7", features = ["multi-threaded", "inline-more"] }
@@ -191,9 +191,11 @@ noodles-vcf = { version = "0.88.0", features = ["async"] }
 num-traits = "0.2.19"
 num_enum = { version = "0.7.3", default-features = false }
 object_store = { version = "0.13.2", default-features = false }
+object_store_opendal = "0.57.0"
 once_cell = "1.21"
 oneshot = { version = "0.2.0", features = ["async"] }
 onpair = "0.1.1"
+opendal = { version = "0.57.0", default-features = false }
 opentelemetry = "0.32.0"
 opentelemetry-otlp = "0.32.0"
 opentelemetry_sdk = "0.32.0"
@@ -232,7 +234,7 @@ rstest = "0.26.1"
 rstest_reuse = "0.7.0"
 rustc-hash = "2.1.1"
 rustix = { version = "1.1", features = ["fs"] }
-serde = "1.0.220"
+serde = "1.0.221"
 serde_json = "1.0.138"
 serde_test = "1.0.176"
 sha2 = "0.11.0"
@@ -275,7 +277,7 @@ tracing-perfetto = "0.1.5"
 tracing-subscriber = "0.3"
 url = "2.5.7"
 uuid = { version = "1.23", features = ["js"] }
-wasm-bindgen-futures = "0.4.54"
+wasm-bindgen-futures = "0.4.58"
 wkb = "0.9.2"
 xshell = "0.2.6"
 zigzag = "0.1.0"
@@ -310,6 +312,7 @@ vortex-json = { version = "0.1.0", path = "./vortex-json", default-features = fa
 vortex-layout = { version = "0.1.0", path = "./vortex-layout", default-features = false }
 vortex-mask = { version = "0.1.0", path = "./vortex-mask", default-features = false }
 vortex-metrics = { version = "0.1.0", path = "./vortex-metrics", default-features = false }
+vortex-object-store-opendal = { version = "0.1.0", path = "./vortex-object-store-opendal", default-features = false }
 vortex-onpair = { version = "0.1.0", path = "./encodings/onpair", default-features = false }
 vortex-parquet-variant = { version = "0.1.0", path = "./encodings/parquet-variant" }
 vortex-pco = { version = "0.1.0", path = "./encodings/pco", default-features = false }

docs/api/python/store/opendal.rst

@@ -1,13 +1,29 @@
-=============
-OpenDAL (COS)
-=============
+==================
+OpenDAL (COS, OSS)
+==================
 
-Vortex can read from and write to Tencent Cloud COS through
+Vortex can read from and write to Tencent Cloud COS and Alibaba Cloud OSS through
 `OpenDAL <https://opendal.apache.org/>`_, which provides native service support.
 
-This store is available only when Vortex is built with the ``opendal`` feature
+These stores are available only when Vortex is built with the ``opendal`` feature
 (e.g. ``maturin develop --features opendal`` or ``cargo build -p vortex-jni --features opendal``).
 
+.. list-table::
+   :header-rows: 1
+
+   * - Scheme
+     - Service
+     - Endpoint variable
+     - Credential variables
+   * - ``cos://``
+     - Tencent Cloud COS
+     - ``COS_ENDPOINT``
+     - ``TENCENTCLOUD_SECRET_ID``, ``TENCENTCLOUD_SECRET_KEY``
+   * - ``oss://``
+     - Alibaba Cloud OSS
+     - ``OSS_ENDPOINT``
+     - ``ALIBABA_CLOUD_ACCESS_KEY_ID``, ``ALIBABA_CLOUD_ACCESS_KEY_SECRET``
+
 :class:`vortex.store.CosStore`
 ==============================
 
@@ -85,3 +101,16 @@ exactly like the built-in S3/Azure/GCS stores:
    # When `store=` is supplied, the path is resolved as a key within the store, so the scheme
    # and bucket are not part of the path passed to read_url.
    a = read_url("path/to/dataset.vortex", store=store)
+
+Reading from OSS
+================
+
+Alibaba Cloud OSS is reachable by URL. There is no standalone ``OssStore`` class yet, so
+configuration comes from the environment variables OpenDAL's OSS builder reads
+(``ALIBABA_CLOUD_ACCESS_KEY_ID``, ``ALIBABA_CLOUD_ACCESS_KEY_SECRET`` and ``OSS_ENDPOINT``):
+
+.. code-block:: python
+
+   import vortex as vx
+
+   a = vx.io.read_url("oss://my-bucket/path/to/dataset.vortex")

vortex-duckdb/Cargo.toml

@@ -38,10 +38,20 @@ static_assertions = { workspace = true }
 tracing = { workspace = true }
 tracing-subscriber = { workspace = true }
 url = { workspace = true }
-vortex = { workspace = true, features = ["files", "tokio", "object_store"] }
+vortex = { workspace = true, features = [
+    "files",
+    "tokio",
+    "object_store",
+    "object_store_registry",
+] }
 vortex-geo = { workspace = true }
 vortex-utils = { workspace = true, features = ["dashmap"] }
 
+[features]
+# OpenDAL-backed schemes (`cos://`, `oss://`) in the object store registry. Pulls in `opendal`, so
+# it is opt-in.
+opendal = ["vortex/opendal"]
+
 [dev-dependencies]
 anyhow = { workspace = true }
 divan = { workspace = true }

vortex-duckdb/src/multi_file.rs

@@ -2,11 +2,10 @@
 // SPDX-FileCopyrightText: Copyright the Vortex contributors
 
 use std::sync::Arc;
+use std::sync::LazyLock;
 
 use itertools::Itertools;
-use object_store::ObjectStore;
-use object_store::aws::AmazonS3Builder;
-use object_store::local::LocalFileSystem;
+use object_store::registry::ObjectStoreRegistry;
 use url::Url;
 use vortex::error::VortexResult;
 use vortex::error::vortex_bail;
@@ -16,6 +15,7 @@ use vortex::file::multi::parse_uri_or_path;
 use vortex::io::compat::Compat;
 use vortex::io::filesystem::FileSystemRef;
 use vortex::io::object_store::ObjectStoreFileSystem;
+use vortex::io::object_store::Registry;
 use vortex::io::runtime::BlockingRuntime;
 use vortex::layout::scan::multi::MultiLayoutDataSource;
 use vortex_utils::aliases::hash_map::HashMap;
@@ -25,18 +25,15 @@ use crate::SESSION;
 use crate::duckdb::BindInputRef;
 use crate::duckdb::ExtractedValue;
 
+/// Process-wide registry, so repeated scans against the same bucket share one client.
+static REGISTRY: LazyLock<Registry> = LazyLock::new(Registry::new);
+
 fn resolve_filesystem(base_url: &Url) -> VortexResult<FileSystemRef> {
-    let object_store: Arc<dyn ObjectStore> = match base_url.scheme() {
-        "file" => Arc::new(LocalFileSystem::new()),
-        "s3" => Arc::new(
-            AmazonS3Builder::from_env()
-                .with_bucket_name(base_url.host_str().ok_or_else(|| {
-                    vortex_err!("Failed to extract bucket name from URL: {base_url}")
-                })?)
-                .build()?,
-        ),
-        other => vortex_bail!("Unsupported URL scheme '{other}'"),
-    };
+    // `base_url` has its path cleared by the caller, so the resolved path is empty and only the
+    // store matters here. Going through the shared registry means DuckDB resolves the same set of
+    // schemes as the Python and Java bindings, including the OpenDAL-backed ones when the
+    // `opendal` feature is on.
+    let (object_store, _) = REGISTRY.resolve(base_url)?;
 
     Ok(Arc::new(ObjectStoreFileSystem::new(
         Arc::new(Compat::new(object_store)),

vortex-io/Cargo.toml

@@ -34,10 +34,12 @@ tokio = { workspace = true, features = [
     "sync",
 ] } # this is the maximum subset of fetaures that is safe for wasm32 targets
 tracing = { workspace = true }
+url = { workspace = true, optional = true }
 vortex-array = { workspace = true }
 vortex-buffer = { workspace = true }
 vortex-error = { workspace = true }
 vortex-metrics = { workspace = true }
+vortex-object-store-opendal = { workspace = true, optional = true }
 vortex-session = { workspace = true }
 vortex-utils = { workspace = true }
 
@@ -62,6 +64,23 @@ tokio = { workspace = true, features = ["full"] }
 
 [features]
 object_store = ["dep:object_store", "vortex-error/object_store"]
+# The URL -> ObjectStore registry, plus the cloud backends it resolves URLs to. Kept separate from
+# `object_store` so that plain file IO does not pull in the cloud HTTP stack.
+object_store_registry = [
+    "object_store",
+    "dep:url",
+    "object_store/aws",
+    "object_store/azure",
+    "object_store/gcp",
+    "object_store/http",
+]
+# OpenDAL-backed schemes (`cos://`, `oss://`) in the object store registry.
+opendal = [
+    "object_store_registry",
+    "dep:vortex-object-store-opendal",
+    "vortex-object-store-opendal/cos",
+    "vortex-object-store-opendal/oss",
+]
 tokio = ["tokio/fs", "tokio/rt-multi-thread"]
 
 [lints]

vortex-io/src/object_store/mod.rs

@@ -3,8 +3,12 @@
 
 mod filesystem;
 mod read_at;
+#[cfg(feature = "object_store_registry")]
+mod registry;
 mod write;
 
 pub use filesystem::*;
 pub use read_at::*;
+#[cfg(feature = "object_store_registry")]
+pub use registry::*;
 pub use write::*;