Squash of eng-294-create-supabase-insertupdate-route (#176)

* Squash of eng-294-create-supabase-insertupdate-route
Work by Sid: Hyde utility, embedding route, supabase insert routes
Marc-Antoine: introduce generated types, further work on insert routes

* Adopt most coderabbit suggestions.

* 'revert' most changes outside of  apps/website/app/

* minor eslint changes

* normalize imports

* will never create an abstract agent, use concrete paths

* add NODE_ENV

* Remove most comments, arrow-ify a few functions, isolate more ItemValidators

* some more comment removal, some more typing exposed a bug

* generic arrow functions

* adjustments to model for uniqueness

* Use upsert and specific uniqueOn in getOrCreateEntity. Use named parameters.

* improvements to error handling

* Use upsert in batches. Generalize error handling.

* remove insert from route

* Getters and deleters

* repair vercel

* comments and warnings

* coderabbit comments

* replace GetOrCreateEntityResult, BatchProcessResult by PostgrestResponse

* Add table parameter to default handlers

* More renames, fix dangling agent

* more corrections

* do not copy supabase file

* correction to path; adding ref

* only run supabase locally

* last review comments

Author: maparent

apps/website/app/api/embeddings/openai/small/route.ts

@@ -0,0 +1,98 @@
+import { NextRequest, NextResponse } from "next/server";
+import OpenAI from "openai";
+import cors from "~/utils/llm/cors";
+
+const apiKey = process.env.OPENAI_API_KEY;
+
+if (!apiKey) {
+  console.error(
+    "Missing OPENAI_API_KEY environment variable. The embeddings API will not function.",
+  );
+}
+
+const openai = apiKey ? new OpenAI({ apiKey }) : null;
+
+type RequestBody = {
+  input: string | string[];
+  model?: string;
+  dimensions?: number;
+  encoding_format?: "float" | "base64";
+};
+
+const OPENAI_REQUEST_TIMEOUT_MS = 30000;
+
+export const POST = async (req: NextRequest): Promise<NextResponse> => {
+  let response: NextResponse;
+
+  if (!apiKey) {
+    response = NextResponse.json(
+      {
+        error: "Server configuration error.",
+        details: "Embeddings service is not configured.",
+      },
+      { status: 500 },
+    );
+    return cors(req, response) as NextResponse;
+  }
+
+  try {
+    const body: RequestBody = await req.json();
+    const {
+      input,
+      model = "text-embedding-3-small",
+      dimensions,
+      encoding_format = "float",
+    } = body;
+
+    if (!input || (Array.isArray(input) && input.length === 0)) {
+      response = NextResponse.json(
+        { error: "Input text cannot be empty." },
+        { status: 400 },
+      );
+      return cors(req, response) as NextResponse;
+    }
+
+    const options: OpenAI.EmbeddingCreateParams = {
+      model,
+      input,
+      dimensions,
+      encoding_format,
+    };
+
+    const embeddingsPromise = openai!.embeddings.create(options);
+    const timeoutPromise = new Promise<never>((_, reject) =>
+      setTimeout(
+        () => reject(new Error("OpenAI API request timeout")),
+        OPENAI_REQUEST_TIMEOUT_MS,
+      ),
+    );
+
+    const openAIResponse = (await Promise.race([
+      embeddingsPromise,
+      timeoutPromise,
+    ])) as OpenAI.CreateEmbeddingResponse;
+
+    response = NextResponse.json(openAIResponse, { status: 200 });
+  } catch (error: unknown) {
+    console.error("Error calling OpenAI Embeddings API:", error);
+    const errorMessage =
+      process.env.NODE_ENV === "development"
+        ? error instanceof Error
+          ? error.message
+          : "Unknown error"
+        : "Internal server error";
+    response = NextResponse.json(
+      {
+        error: "Failed to generate embeddings.",
+        details: errorMessage,
+      },
+      { status: 500 },
+    );
+  }
+
+  return cors(req, response) as NextResponse;
+};
+
+export const OPTIONS = async (req: NextRequest): Promise<NextResponse> => {
+  return cors(req, new NextResponse(null, { status: 204 })) as NextResponse;
+};

apps/website/app/api/supabase/account/[id].ts

@@ -0,0 +1,11 @@
+import {
+  defaultOptionsHandler,
+  makeDefaultGetHandler,
+  makeDefaultDeleteHandler,
+} from "~/utils/supabase/apiUtils";
+
+export const GET = makeDefaultGetHandler("Account");
+
+export const OPTIONS = defaultOptionsHandler;
+
+export const DELETE = makeDefaultDeleteHandler("Account");

apps/website/app/api/supabase/account/route.ts

@@ -0,0 +1,70 @@
+import { NextResponse, NextRequest } from "next/server";
+import type { PostgrestSingleResponse } from "@supabase/supabase-js";
+
+import { createClient } from "~/utils/supabase/server";
+import { getOrCreateEntity, ItemValidator } from "~/utils/supabase/dbUtils";
+import {
+  createApiResponse,
+  handleRouteError,
+  defaultOptionsHandler,
+  asPostgrestFailure,
+} from "~/utils/supabase/apiUtils";
+import { Tables, TablesInsert } from "@repo/database/types.gen.ts";
+
+type AccountDataInput = TablesInsert<"Account">;
+type AccountRecord = Tables<"Account">;
+
+const validateAccount: ItemValidator<AccountDataInput> = (account) => {
+  if (!account || typeof account !== "object")
+    return "Invalid request body: expected a JSON object.";
+  if (!account.agent_id) return "Missing required agent_id";
+  if (!account.platform_id) return "Missing required platform_id";
+  return null;
+};
+
+const getOrCreateAccount = async (
+  supabasePromise: ReturnType<typeof createClient>,
+  accountData: AccountDataInput,
+): Promise<PostgrestSingleResponse<AccountRecord>> => {
+  const {
+    agent_id,
+    platform_id,
+    active = true,
+    write_permission = true,
+    account_local_id,
+  } = accountData;
+
+  const error = validateAccount(accountData);
+  if (error !== null) return asPostgrestFailure(error, "invalid");
+
+  const supabase = await supabasePromise;
+
+  const result = await getOrCreateEntity<"Account">({
+    supabase,
+    tableName: "Account",
+    insertData: {
+      agent_id,
+      platform_id,
+      active,
+      write_permission,
+      account_local_id,
+    },
+    uniqueOn: ["agent_id", "platform_id"],
+  });
+  return result;
+};
+
+export const POST = async (request: NextRequest): Promise<NextResponse> => {
+  const supabasePromise = createClient();
+
+  try {
+    const body: AccountDataInput = await request.json();
+    const result = await getOrCreateAccount(supabasePromise, body);
+
+    return createApiResponse(request, result);
+  } catch (e: unknown) {
+    return handleRouteError(request, e, "/api/supabase/account");
+  }
+};
+
+export const OPTIONS = defaultOptionsHandler;

apps/website/app/api/supabase/content-embedding/[id].ts

@@ -0,0 +1,19 @@
+import {
+  defaultOptionsHandler,
+  makeDefaultGetHandler,
+  makeDefaultDeleteHandler,
+} from "~/utils/supabase/apiUtils";
+
+// TODO: Make model agnostic
+
+export const GET = makeDefaultGetHandler(
+  "ContentEmbedding_openai_text_embedding_3_small_1536",
+  "targetId",
+);
+
+export const DELETE = makeDefaultDeleteHandler(
+  "ContentEmbedding_openai_text_embedding_3_small_1536",
+  "targetId",
+);
+
+export const OPTIONS = defaultOptionsHandler;

apps/website/app/api/supabase/content-embedding/batch/route.ts

@@ -0,0 +1,132 @@
+import { NextResponse, NextRequest } from "next/server";
+import type { PostgrestResponse } from "@supabase/supabase-js";
+
+import { createClient } from "~/utils/supabase/server";
+import {
+  createApiResponse,
+  handleRouteError,
+  defaultOptionsHandler,
+  asPostgrestFailure,
+} from "~/utils/supabase/apiUtils";
+import {
+  processAndInsertBatch,
+  KNOWN_EMBEDDING_TABLES,
+} from "~/utils/supabase/dbUtils";
+import {
+  ApiInputEmbeddingItem,
+  ApiOutputEmbeddingRecord,
+  embeddingInputProcessing,
+  embeddingOutputProcessing,
+} from "~/utils/supabase/validators";
+
+const DEFAULT_MODEL = "openai_text_embedding_3_small_1536";
+
+const batchInsertEmbeddingsProcess = async (
+  supabase: Awaited<ReturnType<typeof createClient>>,
+  embeddingItems: ApiInputEmbeddingItem[],
+): Promise<PostgrestResponse<ApiOutputEmbeddingRecord>> => {
+  // groupBy is node21 only, we are using 20. Group by model, by hand.
+  // Note: This means that later index values may be totally wrong.
+  // Note2: The key is a ModelName, but I cannot use an enum as a key.
+  const byModel: { [key: string]: ApiInputEmbeddingItem[] } = {};
+  try {
+    embeddingItems.reduce((acc, item) => {
+      const model = item?.model || DEFAULT_MODEL;
+      if (acc[model] === undefined) {
+        acc[model] = [];
+      }
+      acc[model]!.push(item);
+      return acc;
+    }, byModel);
+  } catch (error) {
+    if (error instanceof Error) {
+      return asPostgrestFailure(error.message, "exception");
+    }
+    throw error;
+  }
+
+  const globalResults: ApiOutputEmbeddingRecord[] = [];
+  const partialErrors: string[] = [];
+  let created = false,
+    count = 0,
+    has_400 = false;
+  for (const modelName of Object.keys(byModel)) {
+    const embeddingItemsSet = byModel[modelName];
+    if (embeddingItemsSet === undefined) continue;
+    const tableData = KNOWN_EMBEDDING_TABLES[modelName];
+    if (tableData === undefined) continue;
+    const results = await processAndInsertBatch<
+      // any ContentEmbedding table for type checking purposes only
+      "ContentEmbedding_openai_text_embedding_3_small_1536",
+      ApiInputEmbeddingItem,
+      ApiOutputEmbeddingRecord
+    >({
+      supabase,
+      items: embeddingItemsSet,
+      tableName: tableData.tableName,
+      inputProcessor: embeddingInputProcessing,
+      outputProcessor: embeddingOutputProcessing,
+    });
+    if (results.data) {
+      count += results.data.length;
+      globalResults.push(...results.data);
+      created = created || results.status === 201;
+    } else {
+      partialErrors.push(results.error.message);
+      if (results.status === 400) has_400 = true;
+    }
+  }
+  if (count > 0) {
+    if (partialErrors.length > 0) {
+      return {
+        data: globalResults,
+        error: null,
+        status: has_400 ? 400 : 500,
+        count,
+        statusText: partialErrors.join("; "),
+      };
+    } else
+      return {
+        data: globalResults,
+        error: null,
+        status: created ? 201 : 200,
+        count,
+        statusText: created ? "created" : "success",
+      };
+  } else {
+    return asPostgrestFailure(
+      partialErrors.join("; "),
+      "multiple",
+      has_400 ? 400 : 500,
+    );
+  }
+};
+
+export const POST = async (request: NextRequest): Promise<NextResponse> => {
+  const supabase = await createClient();
+
+  try {
+    const body: ApiInputEmbeddingItem[] = await request.json();
+    if (!Array.isArray(body)) {
+      return createApiResponse(
+        request,
+        asPostgrestFailure(
+          "Request body must be an array of embedding items.",
+          "empty",
+        ),
+      );
+    }
+
+    const result = await batchInsertEmbeddingsProcess(supabase, body);
+
+    return createApiResponse(request, result);
+  } catch (e: unknown) {
+    return handleRouteError(
+      request,
+      e,
+      `/api/supabase/content-embedding/batch`,
+    );
+  }
+};
+
+export const OPTIONS = defaultOptionsHandler;