Add integration test that ingests Seafowl GitHub repo with airbyte-github

If environment variable `VITE_TEST_GITHUB_PAT_SECRET` is defined, then
run an integration test which ingests data from the Seafowl GitHub
repository into a Splitgraph repo `madatdata-test-github-ingestion`
under the namespace of the username associated with the integration
test `API_KEY` and `API_SECRET`.

Technically, it's hardcoded to be disabled right now so hot reload
testing doesn't spam ingestion, but it works, e.g. see this repository
which was ingested with the new code in `db-splitgraph.test.ts`:

https://www.splitgraph.com/miles/madatdata-test-github-ingestion/latest/-/tables

Author: milesrichardson

CONTRIBUTING.md

@@ -42,6 +42,7 @@ VITE_TEST_INTEGRATION=1
 VITE_TEST_DDN_API_KEY=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
 VITE_TEST_DDN_API_SECRET=yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy
 VITE_TEST_SEAFOWL_SECRET=zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz
+VITE_TEST_GITHUB_PAT=uuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuu
 ```
 
 Then simply append `--mode integration` flag to any variant of `yarn test` that

packages/db-splitgraph/db-splitgraph.test.ts

@@ -5,7 +5,10 @@ import { makeDb } from "./db-splitgraph";
 import { SplitgraphImportCSVPlugin } from "./plugins/importers/splitgraph-import-csv-plugin";
 import { ExportQueryPlugin } from "./plugins/exporters/export-query-plugin";
 
-import { shouldSkipIntegrationTests } from "@madatdata/test-helpers/env-config";
+import {
+  shouldSkipIntegrationTests,
+  shouldSkipIntegrationTestsForGitHubExternalDataSource,
+} from "@madatdata/test-helpers/env-config";
 import { setupMswServerTestHooks } from "@madatdata/test-helpers/msw-server-hooks";
 import { setupMemo } from "@madatdata/test-helpers/setup-memo";
 import { compose, graphql, rest, type DefaultBodyType } from "msw";
@@ -113,26 +116,69 @@ const createRealDb = () => {
         graphqlEndpoint: defaultHost.baseUrls.gql,
       }),
 
+      new AirbyteGithubImportPlugin({
+        graphqlEndpoint: defaultHost.baseUrls.gql,
+      }),
+
       new ExportQueryPlugin({
         graphqlEndpoint: defaultHost.baseUrls.gql,
       }),
     ],
   });
 };
 
-// describe("importData for AirbyeGitHubImportPlugin", () => {
-//   it("can use the plugin", async () => {
-//     const db = createDb();
+// @ts-expect-error https://stackoverflow.com/a/70711231
+const GITHUB_PAT_SECRET = import.meta.env.VITE_TEST_GITHUB_PAT_SECRET;
 
-//     // db.importData("airbyte-github", {params: {repository: "madatdata/test-repo"}})
+describe.skipIf(shouldSkipIntegrationTestsForGitHubExternalDataSource())(
+  "importData for AirbyeGitHubImportPlugin",
+  () => {
+    it("can use the plugin", async () => {
+      const db = createRealDb();
 
-//     // await db.importData("airbyte-github", { params: }
+      const { username: namespace } = await fetchToken(db);
 
-//     // db.importData("csv", {data})
+      const res = await db.importData(
+        "airbyte-github",
+        {
+          credentials: {
+            credentials: {
+              personal_access_token: GITHUB_PAT_SECRET,
+            },
+          },
+          params: {
+            repository: "splitgraph/seafowl",
+            start_date: "2021-06-01T00:00:00Z",
+          },
+        },
+        {
+          namespace: namespace,
+          repository: "madatdata-test-github-ingestion",
+          tables: [
+            {
+              name: "stargazers",
+              options: {
+                airbyte_cursor_field: ["starred_at"],
+                airbyte_primary_key_field: [],
+              },
+              schema: [],
+            },
+          ],
+        }
+      );
 
-//     // db.importData("airbyte-github"
-//   });
-// });
+      expect(res.response.status).toEqual(200);
+
+      // db.importData("airbyte-github", {}, })
+
+      // await db.importData("airbyte-github", { params: }
+
+      // db.importData("csv", {data})
+
+      // db.importData("airbyte-github"
+    }, 60_000);
+  }
+);
 
 // Useful when writing initial tests against real server (where anon is allowed)
 // const _makeAnonymousDb = () => {

packages/db-splitgraph/plugins/importers/airbyte-github-plugin.ts

@@ -4,19 +4,14 @@ import type { AirbyteGithubCredentialsSchema } from "./generated/airbyte-github/
 
 import { makeGeneratedImportPlugin } from "./splitgraph-generated-import-plugin";
 
-import type { ImportDestOptions } from "./base-import-plugin";
+// import type { ImportDestOptions } from "./base-import-plugin";
 // import { SplitgraphImportCSVPlugin } from "./splitgraph-import-csv-plugin";
 
 export const AirbyteGithubImportPlugin = makeGeneratedImportPlugin<
   "airbyte-github",
   AirbyteGithubParamsSchema,
   AirbyteGithubTableParamsSchema,
-  AirbyteGithubCredentialsSchema,
-  ImportDestOptions<
-    AirbyteGithubTableParamsSchema,
-    AirbyteGithubCredentialsSchema
-  >,
-  { params: AirbyteGithubParamsSchema }
+  AirbyteGithubCredentialsSchema
 >("airbyte-github");
 
 // const plugin = new AirbyteGithubImportPlugin({

packages/db-splitgraph/plugins/importers/base-import-plugin.ts

@@ -16,23 +16,9 @@ import type {
 export type SplitgraphDestOptions = {
   namespace: string;
   repository: string;
-  tableName: string;
-};
-
-export interface ImportDestOptions<
-  TableParamsSchema extends object,
-  CredentialsSchema extends object
-> extends SplitgraphDestOptions {
-  // TODO: note this is duplicated (idk why lol... think it's as a hack while supporting only 1 table)
-  params?: TableParamsSchema;
-  tableName: SplitgraphDestOptions["tableName"];
-
-  // TODO: support > 1 table
-  tableParams?: TableParamsSchema;
-  credentials?: CredentialsSchema;
-  /* default private */
+  /* default is public */
   initialPermissions?: StartExternalRepositoryLoadMutationVariables["initialPermissions"];
-}
+};
 
 export interface SplitgraphImportPluginOptions {
   graphqlEndpoint: string;
@@ -55,6 +41,14 @@ const retryOptions = {
   exponentialOption: { maxInterval: MAX_BACKOFF_INTERVAL, multiplier: 2 },
 };
 
+type ProvidedExternalLoadMutationVariables = Pick<
+  StartExternalRepositoryLoadMutationVariables,
+  "tables"
+> &
+  Partial<
+    Omit<StartExternalRepositoryLoadMutationVariables, "tables" | "pluginName">
+  >;
+
 export abstract class SplitgraphImportPlugin<
   PluginName extends string,
   /** The "params" schema for the plugin, i.e. provided by auto-generated type */
@@ -73,10 +67,7 @@ export abstract class SplitgraphImportPlugin<
     ConcreteImportDestOptions,
     ConcreteImportSourceOptions
   >,
-  ConcreteImportDestOptions extends ImportDestOptions<
-    PluginTableParamsSchema,
-    PluginCredentialsSchema
-  > = ImportDestOptions<PluginTableParamsSchema, PluginCredentialsSchema>,
+  ConcreteImportDestOptions extends SplitgraphDestOptions = SplitgraphDestOptions,
   ConcreteImportSourceOptions extends object = Record<string, never>
 > implements
     ImportPlugin<PluginName>,
@@ -143,16 +134,15 @@ export abstract class SplitgraphImportPlugin<
   protected abstract makeLoadMutationVariables(
     sourceOptions: ConcreteImportSourceOptions,
     destOptions: ConcreteImportDestOptions
-  ): Pick<
-    StartExternalRepositoryLoadMutationVariables,
-    "params" | "tables" | "pluginName"
-  >;
+  ): ProvidedExternalLoadMutationVariables;
+
+  // TODO: preview step should return available table names
 
   private async startLoad(
     sourceOptions: ConcreteImportSourceOptions,
     destOptions: ConcreteImportDestOptions
   ) {
-    const { params, tables, pluginName } = this.makeLoadMutationVariables(
+    const { tables, ...optionalVariables } = this.makeLoadMutationVariables(
       sourceOptions,
       destOptions
     );
@@ -170,6 +160,7 @@ export abstract class SplitgraphImportPlugin<
           $pluginName: String
           $params: JSON
           $credentialId: String
+          $credentialData: JSON
           $sync: Boolean
         ) {
           startExternalRepositoryLoad(
@@ -180,22 +171,27 @@ export abstract class SplitgraphImportPlugin<
             initialPermissions: $initialPermissions
             tables: $tables
             credentialId: $credentialId
+            credentialData: $credentialData
             sync: $sync
           ) {
             taskId
           }
         }
       `,
       {
-        initialPermissions: destOptions.initialPermissions,
-        // NOTE: Optional params are required for typescript, ignored when sent
-        credentialId: undefined,
-        sync: undefined,
-        namespace: destOptions.namespace,
-        repository: destOptions.repository,
-        params,
+        // Only required variable, but can be empty list to indicate loading all tables
         tables,
-        pluginName,
+        // Not required or passable, because it must be same as this.__name
+        pluginName: this.__name,
+        // These are required, but we default to destOptions.{namespace,repository}
+        namespace: optionalVariables.namespace ?? destOptions.namespace,
+        repository: optionalVariables.repository ?? destOptions.repository,
+        // Truly optional variables
+        params: optionalVariables.params ?? {},
+        credentialData: optionalVariables.credentialData ?? undefined,
+        initialPermissions: destOptions.initialPermissions ?? undefined,
+        credentialId: optionalVariables.credentialId ?? undefined,
+        sync: optionalVariables.sync ?? undefined,
       }
     );
   }
@@ -374,6 +370,9 @@ export abstract class SplitgraphImportPlugin<
     };
   }
 
+  // TODO: Clean this up to use an intermediate private member inside the derived class
+  // instead of passing some magical object through some magical pipeline
+
   /**
    * Derived classes should implement this method to perform any pre-import steps,
    * such as uploading a CSV file to object storage. It should return sourceOptions

packages/db-splitgraph/plugins/importers/splitgraph-generated-import-plugin.ts

@@ -1,23 +1,64 @@
 import type { ImportPlugin, WithOptionsInterface } from "@madatdata/base-db";
-import type { ImportDestOptions } from "./base-import-plugin";
 
 import { SplitgraphImportPlugin } from "./base-import-plugin";
+import type { SplitgraphDestOptions } from "./base-import-plugin";
 import type { SplitgraphImportPluginOptions } from "./base-import-plugin";
 
-interface GeneratedImportSourceOptions<PluginParamsSchema extends object> {
+import type { ExternalTableColumnInput } from "../../gql-client/unified-types";
+
+export interface BaseGeneratedImportSourceOptions<
+  PluginParamsSchema extends object
+> {
   params: PluginParamsSchema;
+  sync?: boolean;
+}
+
+export interface GeneratedImportSourceOptionsWithInlineCredentialData<
+  PluginParamsSchema extends object,
+  PluginCredentialsSchema extends object
+> extends BaseGeneratedImportSourceOptions<PluginParamsSchema> {
+  credentials: PluginCredentialsSchema;
+}
+
+export interface GeneratedImportSourceOptionsWithSavedCredentialId<
+  PluginParamsSchema extends object
+> extends BaseGeneratedImportSourceOptions<PluginParamsSchema> {
+  credentialId: string;
+}
+
+export type GeneratedImportSourceOptions<
+  PluginParamsSchema extends object,
+  PluginCredentialsSchema extends object
+> =
+  | GeneratedImportSourceOptionsWithInlineCredentialData<
+      PluginParamsSchema,
+      PluginCredentialsSchema
+    >
+  | GeneratedImportSourceOptionsWithSavedCredentialId<PluginParamsSchema>;
+
+export interface GeneratedImportDestOptions<TableParamsSchema extends object>
+  extends SplitgraphDestOptions {
+  tables?: {
+    name: string;
+    options: TableParamsSchema;
+    /**
+     * Array containing which columns to include in the ingestion. Set to an
+     * empty array (`[]`) to default to including all columns.
+     * */
+    schema: ExternalTableColumnInput[];
+  }[];
 }
 
 export function makeGeneratedImportPlugin<
   PluginName extends string,
   ParamsSchema extends object,
   TableParamsSchema extends object,
   CredentialsSchema extends object,
-  ConcreteImportDestOptions extends ImportDestOptions<
-    TableParamsSchema,
+  ConcreteImportDestOptions extends GeneratedImportDestOptions<TableParamsSchema> = GeneratedImportDestOptions<TableParamsSchema>,
+  ConcreteImportSourceOptions extends GeneratedImportSourceOptions<
+    ParamsSchema,
     CredentialsSchema
-  > = ImportDestOptions<TableParamsSchema, CredentialsSchema>,
-  ConcreteImportSourceOptions extends GeneratedImportSourceOptions<ParamsSchema> = GeneratedImportSourceOptions<ParamsSchema>
+  > = GeneratedImportSourceOptions<ParamsSchema, CredentialsSchema>
 >(
   pluginName: PluginName
 ): new (opts: SplitgraphImportPluginOptions) => ImportPlugin<
@@ -51,21 +92,22 @@ export function makeGeneratedImportPlugin<
       sourceOptions: ConcreteImportSourceOptions,
       destOptions: ConcreteImportDestOptions
     ) {
+      // NOTE: only need to return variables that aren't already defaulted e.g. from destOptions
+      //       which is why we can skip namespace, repository, and pluginName
       return {
         params: JSON.stringify({
           ...sourceOptions.params,
         }),
-        tables: [
-          {
-            name: destOptions.tableName,
-            options: JSON.stringify({
-              ...destOptions.tableParams,
-            }),
-            // TODO: allow user to specify schema in destOptions
-            schema: [],
-          },
-        ],
-        pluginName: "csv",
+        tables: destOptions.tables ?? [],
+        credentialData:
+          "credentials" in sourceOptions
+            ? JSON.stringify(sourceOptions.credentials)
+            : undefined,
+        credentialId:
+          "credentialId" in sourceOptions
+            ? sourceOptions.credentialId
+            : undefined,
+        sync: "sync" in sourceOptions ? sourceOptions.sync : undefined,
       };
     }
   }