Auto-generate a PluginClass for every plugin in a plugin.ts file for its plugin directory

Actually auto-generate the class for the plugin (as a script that
calls a factory function to create the class) in the `plugin.ts`
file within each plugin directory.

Author: milesrichardson

packages/db-splitgraph/db-splitgraph.test.ts

@@ -16,7 +16,7 @@ import { compose, graphql, rest, type DefaultBodyType } from "msw";
 import { defaultHost } from "@madatdata/base-client";
 
 import { faker } from "@faker-js/faker";
-import { AirbyteGithubImportPlugin } from "./plugins/importers/airbyte-github-plugin";
+import { SplitgraphAirbyteGithubImportPlugin } from "./plugins/importers/generated/airbyte-github/plugin";
 
 describe("importData", () => {
   it("returns false for unknown plugin", async () => {
@@ -80,7 +80,7 @@ const createDb = () => {
         graphqlEndpoint: defaultHost.baseUrls.gql,
         transformRequestHeaders,
       }),
-      new AirbyteGithubImportPlugin({
+      new SplitgraphAirbyteGithubImportPlugin({
         graphqlEndpoint: defaultHost.baseUrls.gql,
         transformRequestHeaders,
       }),
@@ -116,7 +116,7 @@ const createRealDb = () => {
         graphqlEndpoint: defaultHost.baseUrls.gql,
       }),
 
-      new AirbyteGithubImportPlugin({
+      new SplitgraphAirbyteGithubImportPlugin({
         graphqlEndpoint: defaultHost.baseUrls.gql,
       }),
 
@@ -138,7 +138,14 @@ describe.skipIf(shouldSkipIntegrationTestsForGitHubExternalDataSource())(
 
       const { username: namespace } = await fetchToken(db);
 
-      const res = await db.importData(
+      // NOTE: not actually asserting anything here atm, and these tests
+      // should usually be skipped until we have a better way of integration
+      // testing that doesn't require spam-ingesting GitHub repos into Splitgraph,
+      // or at least has the capability to delete them afterward
+      // SEE: packages/test-helpers/env-config.ts for hardcoded skip logic
+
+      // For now this is just a way to manually check everything is working
+      await db.importData(
         "airbyte-github",
         {
           credentials: {
@@ -166,16 +173,6 @@ describe.skipIf(shouldSkipIntegrationTestsForGitHubExternalDataSource())(
           ],
         }
       );
-
-      expect(res.response.status).toEqual(200);
-
-      // db.importData("airbyte-github", {}, })
-
-      // await db.importData("airbyte-github", { params: }
-
-      // db.importData("csv", {data})
-
-      // db.importData("airbyte-github"
     }, 60_000);
   }
 );

packages/db-splitgraph/plugins/generate-plugins.script.ts

@@ -14,35 +14,43 @@ const targetDir = path.join(thisSourceFileDir, "importers", "generated");
 const generateTypes = async () => {
   const allPlugins = await fetchSchemas();
 
-  // Store list of used interface names to ensure global uniqueness so that we
+  // Store set of used interface names to ensure global uniqueness so that we
   // can export * from generated files. However, note this may not work because
   // the intermediate types are also exported, and those will have collisions
   const usedInterfaceNames = new Set<string>();
 
+  // Set of plugin class names (same comment as above applies, i.e. uniqueness is expected anyway)
+  const usedPluginClassNames = new Set<string>();
+
   for (let plugin of allPlugins.externalPlugins) {
     log("generateTypes:", plugin.pluginName);
     let pluginTargetDir = path.join(targetDir, plugin.pluginName);
 
     log("mkdir:", fdir(pluginTargetDir));
     await mkdir(pluginTargetDir, { recursive: true });
 
-    let schemas = [
+    let schemas: [schema: any, schemaName: string, interfaceName: string][] = [
       [plugin.credentialsSchema, "CredentialsSchema"],
       [plugin.paramsSchema, "ParamsSchema"],
       [plugin.tableParamsSchema, "TableParamsSchema"],
-    ];
-
-    for (let [schema, schemaName] of schemas) {
+    ].map(([schema, schemaName]) => [
+      schema,
+      schemaName,
+      (() => {
+        // Generate a name that's safe for interface type name and hasn't been used yet
+        // We don't expect collisions, since pluginName is unique, but just in case
+        const interfaceName = generateName(
+          plugin.pluginName + schemaName,
+          usedInterfaceNames
+        );
+        usedInterfaceNames.add(interfaceName);
+
+        return interfaceName;
+      })(),
+    ]);
+
+    for (let [schema, schemaName, interfaceName] of schemas) {
       let schemaOutFile = path.join(pluginTargetDir, `${schemaName}.ts`);
-
-      // Generate a name that's safe for interface type name and hasn't been used yet
-      // We don't expect collisions, since pluginName is unique, but just in case
-      let interfaceName = generateName(
-        plugin.pluginName + schemaName,
-        usedInterfaceNames
-      );
-      usedInterfaceNames.add(interfaceName);
-
       log("interfaceName:", interfaceName);
 
       let generatedTypescript = await compile(schema, interfaceName, {
@@ -52,6 +60,40 @@ const generateTypes = async () => {
       log("write schema:", fdir(schemaOutFile));
       await writeFile(schemaOutFile, generatedTypescript);
     }
+
+    const [
+      [_cs, _csn, credentialsSchemaInterfaceName],
+      [_ps, _psn, paramsSchemaInterfaceName],
+      [_tps, _tpsn, tableParamsSchemaInterfaceName],
+    ] = schemas;
+
+    // NOTE: Concatenate "Splitgraph" outside of generated name to ensure PascalCasing
+    // (otherwise we get "SplitgraphairbyteGitHubImportPlugin" instead of "SplitgraphAirbyteGitHubImportPlugin")
+    let pluginClassName =
+      "Splitgraph" +
+      generateName(plugin.pluginName + "ImportPlugin", usedPluginClassNames);
+    let pluginOutFile = path.join(pluginTargetDir, "plugin.ts");
+
+    log("create plugin: ", pluginClassName, "in", pluginOutFile);
+
+    await writeFile(
+      pluginOutFile,
+      `/** Auto-generated plugin **/
+
+import type { ${paramsSchemaInterfaceName} } from "./ParamsSchema";
+import type { ${tableParamsSchemaInterfaceName} } from "./TableParamsSchema";
+import type { ${credentialsSchemaInterfaceName} } from "./CredentialsSchema";
+import { makeGeneratedImportPlugin } from "../../splitgraph-generated-import-plugin";
+
+export const ${pluginClassName} = makeGeneratedImportPlugin<
+  "${plugin.pluginName}",
+  ${paramsSchemaInterfaceName},
+  ${tableParamsSchemaInterfaceName},
+  ${credentialsSchemaInterfaceName}
+>("${plugin.pluginName}");
+
+`
+    );
   }
 };
 

packages/db-splitgraph/plugins/importers/airbyte-github-plugin.ts

@@ -81,16 +81,16 @@ export const shouldSkipIntegrationTests = () => {
 export const shouldSkipIntegrationTestsForGitHubExternalDataSource = () => {
   // TODO: Temporarily hardcoded to avoid ingesting a bunch of data without deleting it
   return true;
+
+  // const environmentHasGitHubPATSecret = () => {
+  //   return (
+  //     // @ts-expect-error https://stackoverflow.com/a/70711231
+  //     !!import.meta.env.VITE_TEST_GITHUB_PAT_SECRET
+  //   );
+  // };
   // return shouldSkipIntegrationTests() || !environmentHasGitHubPATSecret();
 };
 
 export const shouldSkipSeafowlTests = () => {
   return !environmentHasSeafowlCredential();
 };
-
-// const environmentHasGitHubPATSecret = () => {
-//   return (
-//     // @ts-expect-error https://stackoverflow.com/a/70711231
-//     !!import.meta.env.VITE_TEST_GITHUB_PAT_SECRET
-//   );
-// };