Add backend config and API routes for starting, awaiting import task

Author: milesrichardson

examples/nextjs-import-airbyte-github-export-seafowl/.env.test.local

@@ -0,0 +1,18 @@
+# IMPORTANT: Put your own values in `.env.local` (a git-ignored file) when running this locally
+# Configure them in Vercel settings when running in production
+# This file is mostly to show which variables exist, since it's the only one checked into the repo.
+# SEE: https://nextjs.org/docs/app/building-your-application/configuring/environment-variables
+
+# Create your own API key and secret: https://www.splitgraph.com/connect
+SPLITGRAPH_API_KEY="********************************"
+SPLITGRAPH_API_SECRET="********************************"
+
+# Create a GitHub token that can query the repositories you want to connect
+# For example, a token with read-only access to public repos is sufficient
+# CREATE ONE HERE: https://github.com/settings/personal-access-tokens/new
+GITHUB_PAT_SECRET="github_pat_**********************_***********************************************************"
+
+# OPTIONAL: Set this environment variable to a proxy address to capture requests from API routes
+# e.g. To intercept requests to Splitgraph API sent from madatdata libraries in API routes
+# You can also set this by running: yarn dev-mitm (see package.json)
+# MITMPROXY_ADDRESS="http://localhost:7979"

examples/nextjs-import-airbyte-github-export-seafowl/.gitignore

@@ -1 +1,4 @@
 .next
+.env.local
+.env.*.local
+!.env.test.local

examples/nextjs-import-airbyte-github-export-seafowl/env-vars.d.ts

@@ -0,0 +1,41 @@
+namespace NodeJS {
+  interface ProcessEnv {
+    /**
+     * The API key of an existing Splitgraph account.
+     *
+     * This should be defined in `.env.local` (a git-ignored file) or in Vercel settings.
+     *
+     * Get credentials: https://www.splitgraph.com/connect
+     */
+    SPLITGRAPH_API_KEY: string;
+
+    /**
+     * The API secret of an existing Splitgraph account.
+     *
+     * This should be defined in `.env.local` (a git-ignored file) or in Vercel settings.
+     *
+     * Get credentials: https://www.splitgraph.com/connect
+     */
+    SPLITGRAPH_API_SECRET: string;
+
+    /**
+     * A GitHub personal access token that can be used for importing repositories.
+     * It will be passed to the Airbyte connector that runs on Splitgraph servers
+     * and ingests data from GitHub into Splitgraph.
+     *
+     * This should be defined in `.env.local` (a git-ignored file) or in Vercel settings.
+     *
+     * Create one here: https://github.com/settings/personal-access-tokens/new
+     */
+    GITHUB_PAT_SECRET: string;
+
+    /**
+     * Optional environment variable containing the address of a proxy instance
+     * through which to forward requests from API routes. See next.config.js
+     * for where it's setup.
+     *
+     * This is useful for debugging and development.
+     */
+    MITMPROXY_ADDRESS?: string;
+  }
+}

examples/nextjs-import-airbyte-github-export-seafowl/lib-backend/splitgraph-db.ts

@@ -0,0 +1,52 @@
+import { makeSplitgraphDb } from "@madatdata/core";
+
+// TODO: fix plugin exports
+import { makeDefaultPluginList } from "@madatdata/db-splitgraph";
+import { defaultSplitgraphHost } from "@madatdata/core";
+
+const SPLITGRAPH_API_KEY = process.env.SPLITGRAPH_API_KEY;
+const SPLITGRAPH_API_SECRET = process.env.SPLITGRAPH_API_SECRET;
+
+if (!SPLITGRAPH_API_KEY || !SPLITGRAPH_API_SECRET) {
+  throw new Error(
+    "Environment variable SPLITGRAPH_API_KEY or SPLITGRAPH_API_SECRET is not set." +
+      " See env-vars.d.ts for instructions."
+  );
+}
+
+const authenticatedCredential: Parameters<
+  typeof makeSplitgraphDb
+>[0]["authenticatedCredential"] = {
+  apiKey: SPLITGRAPH_API_KEY,
+  apiSecret: SPLITGRAPH_API_SECRET,
+  anonymous: false,
+};
+
+// TODO: The access token can expire and silently fail?
+
+export const makeAuthenticatedSplitgraphDb = () =>
+  makeSplitgraphDb({
+    authenticatedCredential,
+    plugins: makeDefaultPluginList({
+      graphqlEndpoint: defaultSplitgraphHost.baseUrls.gql,
+      authenticatedCredential,
+    }),
+  });
+
+// TODO: export this utility function from the library
+export const claimsFromJWT = (jwt?: string) => {
+  if (!jwt) {
+    return {};
+  }
+
+  const [_header, claims, _signature] = jwt
+    .split(".")
+    .map(fromBase64)
+    .slice(0, -1) // Signature is not parseable JSON
+    .map((o) => JSON.parse(o));
+
+  return claims;
+};
+
+const fromBase64 = (input: string) =>
+  !!globalThis.Buffer ? Buffer.from(input, "base64").toString() : atob(input);

examples/nextjs-import-airbyte-github-export-seafowl/next.config.js

@@ -0,0 +1,33 @@
+const { ProxyAgent, setGlobalDispatcher } = require("undici");
+
+// If running `yarn dev-mitm`, then setup the proxy with MITMPROXY_ADDRESS
+// NOTE(FIXME): not all madatdata requests get sent through here for some reason
+const setupProxy = () => {
+  if (!process.env.MITMPROXY_ADDRESS) {
+    return;
+  }
+
+  const MITM = process.env.MITMPROXY_ADDRESS;
+
+  console.log("MITM SETUP:", MITM);
+
+  if (!process.env.GLOBAL_AGENT_HTTP_PROXY) {
+    process.env["GLOBAL_AGENT_HTTP_PROXY"] = MITM;
+  }
+
+  process.env["NODE_TLS_REJECT_UNAUTHORIZED"] = "0";
+
+  const mitmProxyOpts = {
+    uri: MITM,
+    connect: {
+      rejectUnauthorized: false,
+      requestCert: false,
+    },
+  };
+
+  setGlobalDispatcher(new ProxyAgent(mitmProxyOpts));
+};
+
+setupProxy();
+
+module.exports = {};

examples/nextjs-import-airbyte-github-export-seafowl/package.json

@@ -2,6 +2,7 @@
   "private": true,
   "scripts": {
     "dev": "yarn next",
+    "dev-mitm": "MITMPROXY_ADDRESS=http://localhost:7979 yarn next",
     "build": "yarn next build",
     "start": "yarn next start"
   },

examples/nextjs-import-airbyte-github-export-seafowl/pages/api/await-import-from-github.ts

@@ -0,0 +1,89 @@
+import type { NextApiRequest, NextApiResponse } from "next";
+import { makeAuthenticatedSplitgraphDb } from "../../lib-backend/splitgraph-db";
+import type { DeferredSplitgraphImportTask } from "@madatdata/db-splitgraph/plugins/importers/splitgraph-base-import-plugin";
+
+type ResponseData =
+  | {
+      completed: boolean;
+      jobStatus: DeferredSplitgraphImportTask["response"]["jobStatus"];
+    }
+  | { error: string; completed: false };
+
+/**
+ * To manually send a request, example:
+
+```bash
+curl -i \
+  -H "Content-Type: application/json" http://localhost:3000/api/await-import-from-github \
+  -d '{ "taskId": "xxxx", "splitgraphNamespace": "xxx", "splitgraphRepo": "yyy" }'
+```
+ */
+export default async function handler(
+  req: NextApiRequest,
+  res: NextApiResponse<ResponseData>
+) {
+  const missing = [
+    "taskId",
+    "splitgraphNamespace",
+    "splitgraphRepository",
+  ].filter((expKey) => !req.body[expKey]);
+  if (missing.length > 0) {
+    res.status(400).json({
+      error: `Missing required keys: ${missing.join(", ")}`,
+      completed: false,
+    });
+    return;
+  }
+
+  const { taskId, splitgraphNamespace, splitgraphRepository } = req.body;
+
+  try {
+    const maybeCompletedTask = await pollImport({
+      splitgraphTaskId: taskId,
+      splitgraphDestinationNamespace: splitgraphNamespace,
+      splitgraphDestinationRepository: splitgraphRepository,
+    });
+
+    if (maybeCompletedTask.error) {
+      throw new Error(JSON.stringify(maybeCompletedTask.error));
+    }
+
+    res.status(200).json(maybeCompletedTask);
+    return;
+  } catch (err) {
+    res.status(400).json({
+      error: err.message,
+      completed: false,
+    });
+    return;
+  }
+}
+
+const pollImport = async ({
+  splitgraphTaskId,
+  splitgraphDestinationNamespace,
+  splitgraphDestinationRepository,
+}: {
+  splitgraphDestinationNamespace: string;
+  splitgraphDestinationRepository: string;
+  splitgraphTaskId: string;
+}) => {
+  const db = makeAuthenticatedSplitgraphDb();
+
+  // NOTE: We must call this, or else requests will fail silently
+  await db.fetchAccessToken();
+
+  const maybeCompletedTask = (await db.pollDeferredTask("csv", {
+    taskId: splitgraphTaskId,
+    namespace: splitgraphDestinationNamespace,
+    repository: splitgraphDestinationRepository,
+  })) as DeferredSplitgraphImportTask;
+
+  // NOTE: We do not include the jobLog, in case it could leak the GitHub PAT
+  // (remember we're using our PAT on behalf of the users of this app)
+  return {
+    completed: maybeCompletedTask?.completed ?? false,
+    jobStatus: maybeCompletedTask?.response.jobStatus,
+    error: maybeCompletedTask?.error ?? undefined,
+  };
+};

examples/nextjs-import-airbyte-github-export-seafowl/pages/api/start-import-from-github.ts

@@ -0,0 +1,117 @@
+import type { NextApiRequest, NextApiResponse } from "next";
+import {
+  makeAuthenticatedSplitgraphDb,
+  claimsFromJWT,
+} from "../../lib-backend/splitgraph-db";
+
+const GITHUB_PAT_SECRET = process.env.GITHUB_PAT_SECRET;
+
+type ResponseData =
+  | {
+      destination: {
+        splitgraphNamespace: string;
+        splitgraphRepository: string;
+      };
+      taskId: string;
+    }
+  | { error: string };
+
+/**
+ * To manually send a request, example:
+
+```bash
+curl -i \
+  -H "Content-Type: application/json" http://localhost:3000/api/start-import-from-github \
+  -d '{ "githubSourceRepository": "splitgraph/seafowl", "splitgraphDestinationRepository": "import-via-nextjs" }'
+```
+ */
+export default async function handler(
+  req: NextApiRequest,
+  res: NextApiResponse<ResponseData>
+) {
+  const db = makeAuthenticatedSplitgraphDb();
+  const { username } = claimsFromJWT((await db.fetchAccessToken()).token);
+
+  const { githubSourceRepository } = req.body;
+
+  if (!githubSourceRepository) {
+    res.status(400).json({ error: "githubSourceRepository is required" });
+    return;
+  }
+
+  const splitgraphDestinationRepository =
+    req.body.splitgraphDestinationRepository ??
+    `github-import-${githubSourceRepository.replaceAll("/", "-")}`;
+
+  try {
+    const taskId = await startImport({
+      db,
+      githubSourceRepository,
+      splitgraphDestinationRepository,
+      githubStartDate: req.body.githubStartDate,
+    });
+    res.status(200).json({
+      destination: {
+        splitgraphNamespace: username,
+        splitgraphRepository: splitgraphDestinationRepository,
+      },
+      taskId,
+    });
+  } catch (err) {
+    res.status(400).json({
+      error: err.message,
+    });
+  }
+}
+
+const startImport = async ({
+  db,
+  githubSourceRepository,
+  splitgraphDestinationRepository,
+  githubStartDate,
+}: {
+  db: ReturnType<typeof makeAuthenticatedSplitgraphDb>;
+  githubSourceRepository: string;
+  splitgraphDestinationRepository: string;
+  /**
+   * Optional start date for ingestion, must be in format like: 2021-06-01T00:00:00Z
+   * Defaults to 2020-01-01T00:00:00Z
+   * */
+  githubStartDate?: string;
+}) => {
+  const { username: splitgraphNamespace } = claimsFromJWT(
+    (await db.fetchAccessToken()).token
+  );
+
+  const { taskId } = await db.importData(
+    "airbyte-github",
+    {
+      credentials: {
+        credentials: {
+          personal_access_token: GITHUB_PAT_SECRET,
+        },
+      },
+      params: {
+        repository: githubSourceRepository,
+        start_date: githubStartDate ?? "2020-01-01T00:00:00Z",
+      },
+    },
+    {
+      namespace: splitgraphNamespace,
+      repository: splitgraphDestinationRepository,
+      tables: [
+        {
+          name: "stargazers",
+          options: {
+            airbyte_cursor_field: ["starred_at"],
+            airbyte_primary_key_field: [],
+          },
+          schema: [],
+        },
+      ],
+    },
+    { defer: true }
+  );
+
+  return taskId;
+};

examples/nextjs-import-airbyte-github-export-seafowl/tsconfig.json

@@ -15,6 +15,6 @@
     "isolatedModules": true,
     "jsx": "preserve"
   },
-  "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx"],
+  "include": ["next-env.d.ts", "env-vars.d.ts", "**/*.ts", "**/*.tsx"],
   "exclude": ["node_modules"]
 }