feat: db:transform command

Author: joshcanhelp

docs/devlog/2025-05-14.md

@@ -20,4 +20,35 @@ export default {
 	  Transformation2() 
 	]
 }
-```
+```
+
+- [Vite `Plugin` interface](https://github.com/vitejs/vite/blob/c7c17434968848f1471179c10a5fc9d2804add8b/packages/vite/src/node/plugin.ts#L82)
+- [Sample plugin](https://github.com/mammadataei/vite-plugin-graphiql/blob/792548d23402cfa30e2bc758b35c62a0f3bfdffa/src/node/index.ts#L14)
+
+Alright, this shape feels good to me so far and allows for options and customization in the config to be passed in. Good starting point 👍
+
+Thinking through the process naively:
+
+1. start with an existing table with a shape like `api` + `endpoint` or a straight table name so we're able to transform and link from transformed tables
+2. indicate the columns with the data we want to use
+3. indicate how the data needs to change; this will happen in-flight so DuckDB can infer the type when creating a new table
+4. create a new table with a specific name and insert
+
+I have to keep reminding myself that the config file is 100% created and maintained by data owners but the transformations could be provided by contributors and used by data owners. I keep caught up in the web of different ways this can be configured (or could in the future). Just focus on the happy path use case!
+
+This is working well so far and I'm seeing places where I can write an interface for plugins instead of allowing the full DB. Running into this problem:
+
+https://github.com/duckdb/duckdb/discussions/9558
+
+I can pull results out as JSON but can't seem to create a table from those results.
+
+I refactored everything to just use interfaces and it's working really well, IMHO. I need to build a few more of these to make sure it all makes sense but Im happy with where this landed and am looking forward to working on the output based on this. 
+
+```ts
+export interface DbTransformation {
+  getSourceTable: () => string;
+  getSourceColumns: () => string[];
+  transform: (data: object) => object;
+  getDestinationTable: () => string;
+}
+```

src/commands/db/transform.ts

@@ -1,20 +1,60 @@
+import { Connection } from "duckdb-async";
+
 import { DbBaseCommand } from "./_base.js";
 import getConfig from "../../utils/config.js";
+import path from "path";
+import { removeFile, writeFile } from "../../utils/fs.js";
+
+export interface DbTransformation {
+  getSourceTable: () => string;
+  getSourceColumns: () => string[];
+  transform: (data: object) => object;
+  getDestinationTable: () => string;
+}
+
+export interface DuckDBConnection extends Connection {}
 
 export default class TransformDb extends DbBaseCommand<typeof DbBaseCommand> {
   static override summary = "Build new tables from existing or transformed data";
 
   static override examples = ["<%= config.bin %> <%= command.id %>"];
 
   public async run(): Promise<void> {
-    const { transformations } = getConfig();
+    const { transformations, dbOutputDir } = getConfig();
 
     if (!transformations.length) {
-      throw new Error("No trasformations to run.");
+      throw new Error("No transformations to run.");
     }
 
     for (const transformation of transformations) {
-      console.log(transformation);
+      const sourceTable = transformation.getSourceTable();
+      const sourceCols = transformation.getSourceColumns();
+      const results = await this.dbConn.all(`
+        SELECT ${sourceCols.join(", ")}
+        FROM '${sourceTable}'
+        `);
+      const transformed = transformation.transform(results);
+      const destinationTable = transformation.getDestinationTable();
+
+      // DuckDB does not support reading from a stringified JSON object
+      // https://github.com/duckdb/duckdb/discussions/9558
+      const jsonTmpFile = path.join(dbOutputDir, `${destinationTable}.json`);
+      writeFile(jsonTmpFile, JSON.stringify(transformed));
+
+      await this.dbConn.all(`
+        DROP TABLE IF EXISTS "${destinationTable}"
+      `);
+
+      await this.dbConn.all(`
+        CREATE TABLE "${destinationTable}" AS
+          SELECT *
+          FROM read_json('${jsonTmpFile}')
+      `);
+
+      removeFile(jsonTmpFile);
+      console.log(
+        `Created table ${destinationTable} from ${sourceTable} with ${results.length} rows`
+      );
     }
   }
 }