More migration stuff

api/db/migration-state-queries.ts

@@ -45,7 +45,7 @@ export async function backfillMigrationState(
     name: 'backfill_migration_state',
     text: `insert into migration_state (platform_membership_id, membership_id, state) VALUES ($1, $2, $3)
 on conflict (platform_membership_id) do update set state = migration_state.state returning state`,
-    values: [platformMembershipId, bungieMembershipId, MigrationState.Stately],
+    values: [platformMembershipId, bungieMembershipId, MigrationState.Postgres],
   });
 
   return result.rows[0].state;

api/routes/update.ts

@@ -121,7 +121,7 @@ export const updateHandler = asyncHandler(async (req, res) => {
   // that we've seen this user and they are in the Stately migration state. This
   // makes sure new users get put into the migration table while we're
   // backfilling.
-  let migrationState: MigrationState = MigrationState.Stately;
+  let migrationState: MigrationState = MigrationState.Postgres;
   if (platformMembershipId) {
     migrationState = await transaction(async (client) =>
       backfillMigrationState(client, platformMembershipId ?? profileIds[0], bungieMembershipId),

api/stately/init/migrate-stately-to-postgres.ts

@@ -210,6 +210,47 @@ function toErrorMessage(error: unknown): string {
   return String(error).slice(0, 500);
 }
 
+function hasUnpairedSurrogates(value: string): boolean {
+  for (let i = 0; i < value.length; i++) {
+    const code = value.charCodeAt(i);
+    if (code >= 0xd800 && code <= 0xdbff) {
+      const next = value.charCodeAt(i + 1);
+      if (!(next >= 0xdc00 && next <= 0xdfff)) {
+        return true;
+      }
+      i += 1;
+      continue;
+    }
+
+    if (code >= 0xdc00 && code <= 0xdfff) {
+      return true;
+    }
+  }
+
+  return false;
+}
+
+function isWellFormedUnicode(value: string): boolean {
+  const candidate = value as string & { isWellFormed?: () => boolean };
+  if (typeof candidate.isWellFormed === 'function') {
+    return candidate.isWellFormed();
+  }
+
+  return !hasUnpairedSurrogates(value);
+}
+
+function getUnsafePostgresTextReason(value: string): string | undefined {
+  if (!isWellFormedUnicode(value)) {
+    return 'string contains unpaired UTF-16 surrogate code units';
+  }
+
+  if (value.includes('\u0000')) {
+    return 'string contains NUL (\\u0000), which Postgres text cannot store';
+  }
+
+  return undefined;
+}
+
 async function migrateOneClaimedUser(
   pgClient: ClientBase,
   bungieMembershipId: number | undefined,
@@ -252,17 +293,32 @@ async function migrateOneClaimedUser(
   }
 
   for (const searchData of searches) {
-    await importSearch(
-      pgClient,
-      bungieMembershipId,
-      platformMembershipId,
-      searchData.destinyVersion,
-      searchData.search.query,
-      searchData.search.saved,
-      searchData.search.lastUsage,
-      searchData.search.usageCount,
-      searchData.search.type,
-    );
+    try {
+      // if the query isn't valid UTF-8, the importSearch function will throw. In that case we want to skip it and continue with the rest of the migration instead of failing the whole migration.
+      const invalidReason = getUnsafePostgresTextReason(searchData.search.query);
+      if (invalidReason) {
+        console.warn(
+          `Skipping search with invalid query for ${platformMembershipId} (${invalidReason}):`,
+          searchData.search.query,
+        );
+        continue;
+      }
+
+      await importSearch(
+        pgClient,
+        bungieMembershipId,
+        platformMembershipId,
+        searchData.destinyVersion,
+        searchData.search.query,
+        searchData.search.saved,
+        searchData.search.lastUsage,
+        searchData.search.usageCount,
+        searchData.search.type,
+      );
+    } catch (error) {
+      console.error(`Failed to import search for ${platformMembershipId}`, searchData, error);
+      throw error;
+    }
   }
 }
 
@@ -311,7 +367,7 @@ try {
         console.log(`Migration finished for ${platformMembershipId}`);
       } catch (error) {
         const errorMessage = toErrorMessage(error);
-        console.error(`Migration failed for ${platformMembershipId}:`, errorMessage);
+        console.error(`Migration failed for ${platformMembershipId}:`, error);
         await withRetry(`abortMigration:${platformMembershipId}`, () =>
           transaction(async (pgClient) => {
             await abortMigrationToPostgres(

kubernetes/dim-api-migration-worker-deployment.yaml

@@ -5,7 +5,7 @@ metadata:
   labels:
     app: dim-api-migration-worker
 spec:
-  replicas: 2
+  replicas: 1
   selector:
     matchLabels:
       app: dim-api-migration-worker
@@ -20,7 +20,7 @@ spec:
       containers:
         - name: dim-api-migration-worker
           image: destinyitemmanager/dim-api:$COMMITHASH
-          imagePullPolicy: IfNotPresent
+          imagePullPolicy: Always
           command: ['node']
           args:
             - --enable-source-maps