From 6b15f5937a0ce43ba9a087fa9c63c82cdb38ad8a Mon Sep 17 00:00:00 2001 From: Ben Hollis Date: Tue, 28 Apr 2026 21:03:50 -0700 Subject: [PATCH 1/6] Handle stately shenanigans --- api/db/migration-state-queries.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/api/db/migration-state-queries.ts b/api/db/migration-state-queries.ts index 191e82e..0419ac9 100644 --- a/api/db/migration-state-queries.ts +++ b/api/db/migration-state-queries.ts @@ -73,6 +73,7 @@ export async function claimMigrationWork( select platform_membership_id from migration_state where state = $1 + and platform_membership_id = '4611686018433092312' and attempt_count < $2 order by last_state_change_at asc limit $3 From 315b28a04fb33bfeb862595e381065b26c95b3ad Mon Sep 17 00:00:00 2001 From: Ben Hollis Date: Tue, 28 Apr 2026 21:25:57 -0700 Subject: [PATCH 2/6] Remove the restriction on platform membership id --- api/db/migration-state-queries.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/api/db/migration-state-queries.ts b/api/db/migration-state-queries.ts index 0419ac9..191e82e 100644 --- a/api/db/migration-state-queries.ts +++ b/api/db/migration-state-queries.ts @@ -73,7 +73,6 @@ export async function claimMigrationWork( select platform_membership_id from migration_state where state = $1 - and platform_membership_id = '4611686018433092312' and attempt_count < $2 order by last_state_change_at asc limit $3 From 40a1117a34ac0b376e12391da7055115df708db1 Mon Sep 17 00:00:00 2001 From: Ben Hollis Date: Mon, 4 May 2026 19:46:52 -0700 Subject: [PATCH 3/6] Skip bad search --- .../init/migrate-stately-to-postgres.ts | 32 ++++++++++++------- .../dim-api-migration-worker-deployment.yaml | 4 +-- 2 files changed, 22 insertions(+), 14 deletions(-) diff --git a/api/stately/init/migrate-stately-to-postgres.ts b/api/stately/init/migrate-stately-to-postgres.ts index b9e5b3c..6141884 100644 --- a/api/stately/init/migrate-stately-to-postgres.ts +++ b/api/stately/init/migrate-stately-to-postgres.ts @@ -252,17 +252,25 @@ async function migrateOneClaimedUser( } for (const searchData of searches) { - await importSearch( - pgClient, - bungieMembershipId, - platformMembershipId, - searchData.destinyVersion, - searchData.search.query, - searchData.search.saved, - searchData.search.lastUsage, - searchData.search.usageCount, - searchData.search.type, - ); + try { + await importSearch( + pgClient, + bungieMembershipId, + platformMembershipId, + searchData.destinyVersion, + searchData.search.query, + searchData.search.saved, + searchData.search.lastUsage, + searchData.search.usageCount, + searchData.search.type, + ); + } catch (error) { + if (error instanceof Error && error.message.includes('invalid byte sequence')) { + } else { + console.error(`Failed to import search for ${platformMembershipId}`, searchData, error); + throw error; + } + } } } @@ -311,7 +319,7 @@ try { console.log(`Migration finished for ${platformMembershipId}`); } catch (error) { const errorMessage = toErrorMessage(error); - console.error(`Migration failed for ${platformMembershipId}:`, errorMessage); + console.error(`Migration failed for ${platformMembershipId}:`, error); await withRetry(`abortMigration:${platformMembershipId}`, () => transaction(async (pgClient) => { await abortMigrationToPostgres( diff --git a/kubernetes/dim-api-migration-worker-deployment.yaml b/kubernetes/dim-api-migration-worker-deployment.yaml index d0f8fb6..8c900aa 100644 --- a/kubernetes/dim-api-migration-worker-deployment.yaml +++ b/kubernetes/dim-api-migration-worker-deployment.yaml @@ -5,7 +5,7 @@ metadata: labels: app: dim-api-migration-worker spec: - replicas: 2 + replicas: 1 selector: matchLabels: app: dim-api-migration-worker @@ -20,7 +20,7 @@ spec: containers: - name: dim-api-migration-worker image: destinyitemmanager/dim-api:$COMMITHASH - imagePullPolicy: IfNotPresent + imagePullPolicy: Always command: ['node'] args: - --enable-source-maps From 6db87e0ed391cf37d6066a226b99ec648aee06f5 Mon Sep 17 00:00:00 2001 From: Ben Hollis Date: Mon, 4 May 2026 19:55:12 -0700 Subject: [PATCH 4/6] Skip invalid --- api/stately/init/migrate-stately-to-postgres.ts | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/api/stately/init/migrate-stately-to-postgres.ts b/api/stately/init/migrate-stately-to-postgres.ts index 6141884..1147260 100644 --- a/api/stately/init/migrate-stately-to-postgres.ts +++ b/api/stately/init/migrate-stately-to-postgres.ts @@ -253,6 +253,15 @@ async function migrateOneClaimedUser( for (const searchData of searches) { try { + // if the query isn't valid UTF-8, the importSearch function will throw. In that case we want to skip it and continue with the rest of the migration instead of failing the whole migration. + if (!searchData.search.query.isWellFormed()) { + console.warn( + `Skipping search with invalid UTF-8 query for ${platformMembershipId}:`, + searchData.search.query, + ); + continue; + } + await importSearch( pgClient, bungieMembershipId, @@ -265,11 +274,8 @@ async function migrateOneClaimedUser( searchData.search.type, ); } catch (error) { - if (error instanceof Error && error.message.includes('invalid byte sequence')) { - } else { - console.error(`Failed to import search for ${platformMembershipId}`, searchData, error); - throw error; - } + console.error(`Failed to import search for ${platformMembershipId}`, searchData, error); + throw error; } } } From 3ca72e5ddf4498b8444ce3f2be9125a72066bca1 Mon Sep 17 00:00:00 2001 From: Ben Hollis Date: Mon, 4 May 2026 19:58:58 -0700 Subject: [PATCH 5/6] Better check --- .../init/migrate-stately-to-postgres.ts | 46 ++++++++++++++++++- 1 file changed, 44 insertions(+), 2 deletions(-) diff --git a/api/stately/init/migrate-stately-to-postgres.ts b/api/stately/init/migrate-stately-to-postgres.ts index 1147260..ec3834c 100644 --- a/api/stately/init/migrate-stately-to-postgres.ts +++ b/api/stately/init/migrate-stately-to-postgres.ts @@ -210,6 +210,47 @@ function toErrorMessage(error: unknown): string { return String(error).slice(0, 500); } +function hasUnpairedSurrogates(value: string): boolean { + for (let i = 0; i < value.length; i++) { + const code = value.charCodeAt(i); + if (code >= 0xd800 && code <= 0xdbff) { + const next = value.charCodeAt(i + 1); + if (!(next >= 0xdc00 && next <= 0xdfff)) { + return true; + } + i += 1; + continue; + } + + if (code >= 0xdc00 && code <= 0xdfff) { + return true; + } + } + + return false; +} + +function isWellFormedUnicode(value: string): boolean { + const candidate = value as string & { isWellFormed?: () => boolean }; + if (typeof candidate.isWellFormed === 'function') { + return candidate.isWellFormed(); + } + + return !hasUnpairedSurrogates(value); +} + +function getUnsafePostgresTextReason(value: string): string | undefined { + if (!isWellFormedUnicode(value)) { + return 'string contains unpaired UTF-16 surrogate code units'; + } + + if (value.includes('\u0000')) { + return 'string contains NUL (\\u0000), which Postgres text cannot store'; + } + + return undefined; +} + async function migrateOneClaimedUser( pgClient: ClientBase, bungieMembershipId: number | undefined, @@ -254,9 +295,10 @@ async function migrateOneClaimedUser( for (const searchData of searches) { try { // if the query isn't valid UTF-8, the importSearch function will throw. In that case we want to skip it and continue with the rest of the migration instead of failing the whole migration. - if (!searchData.search.query.isWellFormed()) { + const invalidReason = getUnsafePostgresTextReason(searchData.search.query); + if (invalidReason) { console.warn( - `Skipping search with invalid UTF-8 query for ${platformMembershipId}:`, + `Skipping search with invalid query for ${platformMembershipId} (${invalidReason}):`, searchData.search.query, ); continue; From 8ee9c34f8f4dcb1ff505f6df232674f6edd5a8f2 Mon Sep 17 00:00:00 2001 From: Ben Hollis Date: Mon, 4 May 2026 20:01:29 -0700 Subject: [PATCH 6/6] Change migration state default --- api/db/migration-state-queries.ts | 2 +- api/routes/update.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/api/db/migration-state-queries.ts b/api/db/migration-state-queries.ts index 191e82e..16bee48 100644 --- a/api/db/migration-state-queries.ts +++ b/api/db/migration-state-queries.ts @@ -45,7 +45,7 @@ export async function backfillMigrationState( name: 'backfill_migration_state', text: `insert into migration_state (platform_membership_id, membership_id, state) VALUES ($1, $2, $3) on conflict (platform_membership_id) do update set state = migration_state.state returning state`, - values: [platformMembershipId, bungieMembershipId, MigrationState.Stately], + values: [platformMembershipId, bungieMembershipId, MigrationState.Postgres], }); return result.rows[0].state; diff --git a/api/routes/update.ts b/api/routes/update.ts index d319d38..547c4fb 100644 --- a/api/routes/update.ts +++ b/api/routes/update.ts @@ -121,7 +121,7 @@ export const updateHandler = asyncHandler(async (req, res) => { // that we've seen this user and they are in the Stately migration state. This // makes sure new users get put into the migration table while we're // backfilling. - let migrationState: MigrationState = MigrationState.Stately; + let migrationState: MigrationState = MigrationState.Postgres; if (platformMembershipId) { migrationState = await transaction(async (client) => backfillMigrationState(client, platformMembershipId ?? profileIds[0], bungieMembershipId),