Track completed import/exports in a meta repository on Splitgraph

After an import/export has completed, insert a row into the meta table,
which we will also use to fetch the previously imported repositories
from the client side when rendering the sidebar. We don't have transactional
guarantees on the DDN, so we can't do `INSERT ON CONFLICT`, so instead we
avoid duplicate rows by first selecting the existing row, and returning `204`
if it's already been inserted into the `completed_repositories` table.

However, I did notice that when I inserted the same row twice, it only
showed up once when I made a selection in the Console. I don't know if
this was due to a race condition, a bug, or because it's using the entire
row as a compound primary key and for some reason requiring that it be unique.

Author: milesrichardson

examples/nextjs-import-airbyte-github-export-seafowl/.env.test.local

@@ -7,6 +7,9 @@
 SPLITGRAPH_API_KEY="********************************"
 SPLITGRAPH_API_SECRET="********************************"
 
+# This should match the username associated with the API key
+SPLITGRAPH_GITHUB_ANALYTICS_META_NAMESPACE="*****"
+
 # Create a GitHub token that can query the repositories you want to connect
 # For example, a token with read-only access to public repos is sufficient
 # CREATE ONE HERE: https://github.com/settings/personal-access-tokens/new

examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportPanel.tsx

@@ -45,6 +45,8 @@ export const ExportPanel = () => {
   );
 
   const handleStartExport = useCallback(async () => {
+    const abortController = new AbortController();
+
     try {
       const response = await fetch("/api/start-export-to-seafowl", {
         method: "POST",
@@ -55,6 +57,7 @@ export const ExportPanel = () => {
         headers: {
           "Content-Type": "application/json",
         },
+        signal: abortController.signal,
       });
       const data = (await response.json()) as StartExportToSeafowlResponseData;
 
@@ -87,8 +90,14 @@ export const ExportPanel = () => {
         ],
       });
     } catch (error) {
+      if (error.name === "AbortError") {
+        return;
+      }
+
       dispatch({ type: "export_error", error: error.message });
     }
+
+    return () => abortController.abort();
   }, [queriesToExport, tablesToExport, dispatch]);
 
   return (

examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/StepperContext.tsx

@@ -1,5 +1,5 @@
 // StepperContext.tsx
-import React, { useReducer, useContext, ReactNode } from "react";
+import React, { useContext, ReactNode } from "react";
 import {
   StepperState,
   StepperAction,

examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/stepper-states.ts

@@ -45,11 +45,6 @@ export type StepperAction =
   | { type: "reset" }
   | { type: "initialize_from_url"; parsedFromUrl: StepperState };
 
-type ExtractStepperAction<T extends StepperAction["type"]> = Extract<
-  StepperAction,
-  { type: T }
->;
-
 const initialState: StepperState = {
   stepperState: "unstarted",
   repository: null,
@@ -62,19 +57,6 @@ const initialState: StepperState = {
   exportError: null,
 };
 
-// FOR DEBUGGING: uncomment for hardcoded state initialization
-// export const initialState: StepperState = {
-//   ...normalInitialState,
-//   stepperState: "import_complete",
-//   splitgraphNamespace: "miles",
-//   splitgraphRepository: "import-via-nextjs",
-// };
-
-type ActionParams<T extends StepperAction["type"]> = Omit<
-  ExtractStepperAction<T>,
-  "type"
->;
-
 const getQueryParamAsString = <T extends string = string>(
   query: ParsedUrlQuery,
   key: string
@@ -310,13 +292,95 @@ const urlNeedsChange = (state: StepperState, router: NextRouter) => {
   );
 };
 
+/**
+ * When the export has completed, send a request to /api/mark-import-export-complete
+ * which will insert the repository into the metadata table, which we query to
+ * render the sidebar
+ */
+const useMarkAsComplete = (
+  state: StepperState,
+  dispatch: React.Dispatch<StepperAction>
+) => {
+  useEffect(() => {
+    if (state.stepperState !== "export_complete") {
+      return;
+    }
+
+    const {
+      repository: {
+        namespace: githubSourceNamespace,
+        repository: githubSourceRepository,
+      },
+      splitgraphRepository: splitgraphDestinationRepository,
+    } = state;
+
+    // NOTE: Make sure to abort request so that in React 18 development mode,
+    // when effect runs twice, the second request is aborted and we don't have
+    // a race condition with two requests inserting into the table (where we have no transactional
+    // integrity and manually do a SELECT before the INSERT to check if the row already exists)
+    const abortController = new AbortController();
+
+    const markImportExportComplete = async () => {
+      try {
+        const response = await fetch("/api/mark-import-export-complete", {
+          method: "POST",
+          headers: {
+            "Content-Type": "application/json",
+          },
+          body: JSON.stringify({
+            githubSourceNamespace,
+            githubSourceRepository,
+            splitgraphDestinationRepository,
+          }),
+          signal: abortController.signal,
+        });
+
+        if (!response.ok) {
+          throw new Error("Failed to mark import/export as complete");
+        }
+
+        const data = await response.json();
+
+        if (!data.status) {
+          throw new Error(
+            "Got unexpected resposne shape when marking import/export complete"
+          );
+        }
+
+        if (data.error) {
+          throw new Error(
+            `Failed to mark import/export complete: ${data.error}`
+          );
+        }
+
+        console.log("Marked import/export as complete");
+      } catch (error) {
+        if (error.name === "AbortError") {
+          return;
+        }
+
+        dispatch({
+          type: "export_error",
+          error: error.message ?? error.toString(),
+        });
+      }
+    };
+
+    markImportExportComplete();
+
+    return () => abortController.abort();
+  }, [state, dispatch]);
+};
+
 export const useStepperReducer = () => {
   const router = useRouter();
   const [state, dispatch] = useReducer(stepperReducer, {
     ...initialState,
     stepperState: "uninitialized",
   });
 
+  useMarkAsComplete(state, dispatch);
+
   useEffect(() => {
     dispatch({
       type: "initialize_from_url",

examples/nextjs-import-airbyte-github-export-seafowl/env-vars.d.ts

@@ -37,5 +37,58 @@ namespace NodeJS {
      * This is useful for debugging and development.
      */
     MITMPROXY_ADDRESS?: string;
+
+    /**
+     * The namespace of the repository in Splitgraph where metadata is stored
+     * containing the state of imported GitHub repositories, which should contain
+     * the repository `SPLITGRAPH_GITHUB_ANALYTICS_META_REPOSITORY`.
+     *
+     * This should be defined in `.env.local`, since it's not checked into Git
+     * and can vary between users. It should match the username associated with
+     * the `SPLITGRAPH_API_KEY`
+     *
+     * Example:
+     *
+     * ```
+     * miles/splitgraph-github-analytics.completed_repositories
+     * ^^^^^
+     * SPLITGRAPH_GITHUB_ANALYTICS_META_NAMESPACE=miles
+     * ```
+     */
+    SPLITGRAPH_GITHUB_ANALYTICS_META_NAMESPACE: string;
+
+    /**
+     * The repository (no namespace) in Splitgraph where metadata is stored
+     * containing the state of imported GitHub repositories, which should be a
+     * repository contained inside `SPLITGRAPH_GITHUB_ANALYTICS_META_NAMESPACE`.
+     *
+     * This is defined by default in `.env` which is checked into Git.
+     *
+     * * Example:
+     *
+     * ```
+     * miles/splitgraph-github-analytics.completed_repositories
+     *       ^^^^^^^^^^^^^^^^^^^^^^^^^^^
+     *       SPLITGRAPH_GITHUB_ANALYTICS_META_REPOSITORY=splitgraph-github-analytics
+     * ```
+     */
+    SPLITGRAPH_GITHUB_ANALYTICS_META_REPOSITORY: string;
+
+    /**
+     * The name of the table containing completed repositories, which are inserted
+     * when the import/export is complete, and which can be queried to render the
+     * sidebar containing previously imported github repositories.
+     *
+     * This is defined by default in `.env` which is checked into Git.
+     *
+     * Example:
+     *
+     * ```
+     * miles/splitgraph-github-analytics.completed_repositories
+     *                                   ^^^^^^^^^^^^^^^^^^^^^^
+     *       SPLITGRAPH_GITHUB_ANALYTICS_META_COMPLETED_TABLE=completed_repositories
+     * ```
+     */
+    SPLITGRAPH_GITHUB_ANALYTICS_META_COMPLETED_TABLE: string;
   }
 }

examples/nextjs-import-airbyte-github-export-seafowl/lib/backend/splitgraph-db.ts

@@ -1,4 +1,4 @@
-import { makeSplitgraphDb } from "@madatdata/core";
+import { makeSplitgraphDb, makeSplitgraphHTTPContext } from "@madatdata/core";
 
 // TODO: fix plugin exports
 import { makeDefaultPluginList } from "@madatdata/db-splitgraph";
@@ -33,6 +33,15 @@ export const makeAuthenticatedSplitgraphDb = () =>
     }),
   });
 
+export const makeAuthenticatedSplitgraphHTTPContext = () =>
+  makeSplitgraphHTTPContext({
+    authenticatedCredential,
+    plugins: makeDefaultPluginList({
+      graphqlEndpoint: defaultSplitgraphHost.baseUrls.gql,
+      authenticatedCredential,
+    }),
+  });
+
 // TODO: export this utility function from the library
 export const claimsFromJWT = (jwt?: string) => {
   if (!jwt) {