Replace the rust usage with a somewhat hacky call to uvx

.github/workflows/ci.yml

@@ -407,6 +407,8 @@ jobs:
       - uses: ./.github/actions/setup-prebuild
         with:
           enable-sccache: "true"
+      - name: Install uv
+        uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b  # v8.1.0
       - name: Rust Tests (Windows)
         run: |
           cargo nextest run --cargo-profile ci --locked --workspace --all-features --no-fail-fast `

.github/workflows/musl.yml

@@ -54,6 +54,9 @@ jobs:
 
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7
 
+      - name: Install uv
+        uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b  # v8.1.0
+
       - name: Install nextest
         shell: bash
         # Prebuilt static musl nextest binary; building it from source would

Cargo.toml

@@ -266,15 +266,12 @@ thiserror = "2.0.3"
 tokio = { version = "1.52" }
 tokio-stream = "0.1.17"
 tokio-util = "0.7.17"
-vortex-array-macros = { version = "0.1.0", path = "./vortex-array-macros" }
-# Pull these into non-public crates to support DF 58
-tpchgen = { version = "2.0.2", git = "https://github.com/clflushopt/tpchgen-rs.git", rev = "438e9c2dbc25b2fff82c0efc08b3f13b5707874f" }
-tpchgen-arrow = { version = "2.0.2", git = "https://github.com/clflushopt/tpchgen-rs.git", rev = "438e9c2dbc25b2fff82c0efc08b3f13b5707874f" }
 tracing = { version = "0.1.41", default-features = false }
 tracing-perfetto = "0.1.5"
 tracing-subscriber = "0.3"
 url = "2.5.7"
 uuid = { version = "1.23", features = ["js"] }
+vortex-array-macros = { version = "0.1.0", path = "./vortex-array-macros" }
 wasm-bindgen-futures = "0.4.54"
 wkb = "0.9.2"
 xshell = "0.2.6"

vortex-bench/Cargo.toml

@@ -68,8 +68,6 @@ target-lexicon = { workspace = true }
 tokio = { workspace = true, features = ["full"] }
 tokio-stream = { workspace = true }
 tokio-util = { workspace = true }
-tpchgen = { workspace = true }
-tpchgen-arrow = { workspace = true }
 tracing = { workspace = true }
 tracing-perfetto = { workspace = true }
 tracing-subscriber = { workspace = true, features = [

vortex-bench/sql/tpch/README.md

@@ -6,8 +6,8 @@ It stresses scan throughput, filter pushdown, joins, and aggregations over mostl
 and date columns.
 
 The queries in this directory (`q1.sql` ... `q22.sql`) are the standard TPC-H queries.
-Data is generated deterministically by [`tpchgen`](../../src/tpch/tpchgen.rs); the benchmark
-harness lives in [`src/tpch`](../../src/tpch).
+Data is generated deterministically as Parquet via [`tpchgen-cli`](../../src/tpch/tpchgen.rs);
+the benchmark harness lives in [`src/tpch`](../../src/tpch).
 
 ## CI variants
 

vortex-bench/src/conversions.rs

@@ -235,26 +235,28 @@ pub async fn convert_parquet_directory_to_vortex(
     let parquet_path = input_path.join(Format::Parquet.name());
     create_dir_all(&vortex_dir).await?;
 
-    let parquet_inputs = fs::read_dir(&parquet_path)?.collect::<std::io::Result<Vec<_>>>()?;
+    let parquet_inputs = fs::read_dir(&parquet_path)?
+        .map(|entry| entry.map(|entry| entry.path()))
+        .filter(|entry| {
+            entry
+                .as_ref()
+                .is_ok_and(|path| path.extension().is_some_and(|e| e == "parquet"))
+        })
+        .collect::<std::io::Result<Vec<_>>>()?;
     trace!(
         "Found {} parquet files in {}",
         parquet_inputs.len(),
         parquet_path.to_str().unwrap()
     );
 
-    let iter = parquet_inputs
-        .iter()
-        .filter(|entry| entry.path().extension().is_some_and(|e| e == "parquet"));
-
     let concurrency = calculate_concurrency();
-    futures::stream::iter(iter)
-        .map(|dir_entry| {
+    futures::stream::iter(parquet_inputs)
+        .map(|parquet_file_path| {
             let filename = {
-                let mut temp = dir_entry.path();
+                let mut temp = parquet_file_path.clone();
                 temp.set_extension("");
                 temp.file_name().unwrap().to_str().unwrap().to_string()
             };
-            let parquet_file_path = parquet_path.join(format!("{filename}.parquet"));
             let output_path = vortex_dir.join(format!("{filename}.{}", format.ext()));
 
             tokio::spawn(

vortex-bench/src/datasets/tpch_l_comment.rs

@@ -18,9 +18,11 @@ use vortex::expr::col;
 use vortex::expr::pack;
 use vortex::file::OpenOptionsSessionExt;
 
+use crate::CompactionStrategy;
 use crate::Format;
 use crate::IdempotentPath;
 use crate::SESSION;
+use crate::conversions::convert_parquet_directory_to_vortex;
 use crate::datasets::Dataset;
 use crate::tpch::tpchgen::TpchGenOptions;
 use crate::tpch::tpchgen::generate_tpch_tables;
@@ -58,13 +60,12 @@ impl Dataset for TPCHLCommentChunked {
         let scale_factor_dir = base_path.join("1.0");
         let data_dir = scale_factor_dir.join(Format::OnDiskVortex.name());
 
-        // Generate TPC-H CSV data if it doesn't exist
         if !data_dir.exists() {
-            // Use blocking call like TPC-H benchmark does
-            let options = TpchGenOptions::new("1.0".to_string(), scale_factor_dir)
-                .with_format(Format::OnDiskVortex);
-
-            futures::executor::block_on(generate_tpch_tables(options))?;
+            let options = TpchGenOptions::new("1.0".to_string(), scale_factor_dir.clone())
+                .with_format(Format::Parquet);
+            generate_tpch_tables(options).await?;
+            convert_parquet_directory_to_vortex(&scale_factor_dir, CompactionStrategy::Default)
+                .await?;
         }
 
         let mut chunks: Vec<ArrayRef> = vec![];

vortex-bench/src/tpch/benchmark.rs

@@ -93,8 +93,7 @@ impl Benchmark for TpcHBenchmark {
             .to_file_path()
             .map_err(|_| anyhow::anyhow!("Invalid file URL: {}", self.data_url.as_str()))?;
 
-        let options = TpchGenOptions::new(self.scale_factor.clone(), base_data_dir)
-            .with_max_file_size_mb(Some(600));
+        let options = TpchGenOptions::new(self.scale_factor.clone(), base_data_dir);
 
         tpchgen::generate_tpch_tables(options).await?;