Implement backend API routes start-export-to-seafowl and await-export-to-seafowl-task

The `start-export-to-seafowl` route takes a list of source
tables from Splitgraph (list of `{namespace,repository,table}`),
and starts a task to export them to Seafowl. It returns a list
of objects `{taskId: string; tableName: string;}`, where each
item represents the currently exporting table (and `tableName` is
the source table name).

The `await-export-to-seafowl-task` route takes a single `taskId`
parameter and returns its status, i.e. `{completed: boolean; ...otherInfo}`

Author: milesrichardson

examples/nextjs-import-airbyte-github-export-seafowl/pages/api/await-export-to-seafowl-task.ts

@@ -0,0 +1,76 @@
+import type { NextApiRequest, NextApiResponse } from "next";
+import { makeAuthenticatedSplitgraphDb } from "../../lib-backend/splitgraph-db";
+import type { DeferredSplitgraphExportTask } from "@madatdata/db-splitgraph/plugins/exporters/splitgraph-base-export-plugin";
+
+type ResponseData =
+  | {
+      completed: boolean;
+      jobStatus: DeferredSplitgraphExportTask["response"];
+    }
+  | { error: string; completed: false };
+
+/**
+ * To manually send a request, example:
+
+```bash
+curl -i \
+  -H "Content-Type: application/json" http://localhost:3000/api/await-export-to-seafowl-task \
+  -d '{ "taskId": "2923fd6f-2197-495a-9df1-2428a9ca8dee" }'
+```
+ */
+export default async function handler(
+  req: NextApiRequest,
+  res: NextApiResponse<ResponseData>
+) {
+  if (!req.body["taskId"]) {
+    res.status(400).json({
+      error: "Missing required key: taskId",
+      completed: false,
+    });
+    return;
+  }
+
+  const { taskId } = req.body;
+
+  try {
+    const maybeCompletedTask = await pollImport({
+      splitgraphTaskId: taskId,
+    });
+
+    if (maybeCompletedTask.error) {
+      throw new Error(JSON.stringify(maybeCompletedTask.error));
+    }
+
+    res.status(200).json(maybeCompletedTask);
+    return;
+  } catch (err) {
+    res.status(400).json({
+      error: err.message,
+      completed: false,
+    });
+    return;
+  }
+}
+
+const pollImport = async ({
+  splitgraphTaskId,
+}: {
+  splitgraphTaskId: string;
+}) => {
+  const db = makeAuthenticatedSplitgraphDb();
+
+  // NOTE: We must call this, or else requests will fail silently
+  await db.fetchAccessToken();
+
+  const maybeCompletedTask = (await db.pollDeferredTask("export-to-seafowl", {
+    taskId: splitgraphTaskId,
+  })) as DeferredSplitgraphExportTask;
+
+  // NOTE: We do not include the jobLog, in case it could leak the GitHub PAT
+  // (remember we're using our PAT on behalf of the users of this app)
+  return {
+    completed: maybeCompletedTask?.completed ?? false,
+    jobStatus: maybeCompletedTask?.response,
+    error: maybeCompletedTask?.error ?? undefined,
+  };
+};

examples/nextjs-import-airbyte-github-export-seafowl/pages/api/start-export-to-seafowl.ts

@@ -0,0 +1,103 @@
+import type { NextApiRequest, NextApiResponse } from "next";
+import { makeAuthenticatedSplitgraphDb } from "../../lib-backend/splitgraph-db";
+
+type ResponseData =
+  | {
+      tables: {
+        tableName: string;
+        taskId: string;
+      }[];
+    }
+  | { error: string };
+
+type TableInput = { namespace: string; repository: string; table: string };
+
+/**
+ * To manually send a request, example:
+
+```bash
+curl -i \
+  -H "Content-Type: application/json" http://localhost:3000/api/start-export-to-seafowl \
+  -d '{ "tables": [{"namespace": "miles", "repository": "import-via-nextjs", "table": "stargazers"}] }'
+```
+ */
+export default async function handler(
+  req: NextApiRequest,
+  res: NextApiResponse<ResponseData>
+) {
+  const db = makeAuthenticatedSplitgraphDb();
+  const { tables } = req.body;
+
+  if (
+    !tables ||
+    !tables.length ||
+    !tables.every(
+      (t: TableInput) =>
+        t.namespace &&
+        t.repository &&
+        t.table &&
+        typeof t.namespace === "string" &&
+        typeof t.repository === "string" &&
+        typeof t.table === "string"
+    )
+  ) {
+    res.status(400).json({ error: "invalid tables input in request body" });
+    return;
+  }
+
+  try {
+    const exportingTables = await startExport({
+      db,
+      tables,
+    });
+    res.status(200).json({
+      tables: exportingTables,
+    });
+  } catch (err) {
+    res.status(400).json({
+      error: err.message,
+    });
+  }
+}
+
+const startExport = async ({
+  db,
+  tables,
+}: {
+  db: ReturnType<typeof makeAuthenticatedSplitgraphDb>;
+  tables: TableInput[];
+}) => {
+  await db.fetchAccessToken();
+
+  const response = await db.exportData(
+    "export-to-seafowl",
+    {
+      tables: tables.map((splitgraphSource) => ({
+        source: {
+          repository: splitgraphSource.repository,
+          namespace: splitgraphSource.namespace,
+          table: splitgraphSource.table,
+        },
+      })),
+    },
+    {
+      // Empty instance will trigger Splitgraph to export to demo.seafowl.cloud
+      seafowlInstance: {},
+    },
+    { defer: true }
+  );
+
+  if (response.error) {
+    throw new Error(JSON.stringify(response.error));
+  }
+
+  const loadingTables: { taskId: string; tableName: string }[] =
+    response.taskIds.tables.map(
+      (t: { jobId: string; sourceTable: string }) => ({
+        taskId: t.jobId,
+        tableName: t.sourceTable,
+      })
+    );
+
+  return loadingTables;
+};