diff --git a/api/db/migration-state-queries.ts b/api/db/migration-state-queries.ts index 191e82e..16bee48 100644 --- a/api/db/migration-state-queries.ts +++ b/api/db/migration-state-queries.ts @@ -45,7 +45,7 @@ export async function backfillMigrationState( name: 'backfill_migration_state', text: `insert into migration_state (platform_membership_id, membership_id, state) VALUES ($1, $2, $3) on conflict (platform_membership_id) do update set state = migration_state.state returning state`, - values: [platformMembershipId, bungieMembershipId, MigrationState.Stately], + values: [platformMembershipId, bungieMembershipId, MigrationState.Postgres], }); return result.rows[0].state; diff --git a/api/routes/update.ts b/api/routes/update.ts index d319d38..547c4fb 100644 --- a/api/routes/update.ts +++ b/api/routes/update.ts @@ -121,7 +121,7 @@ export const updateHandler = asyncHandler(async (req, res) => { // that we've seen this user and they are in the Stately migration state. This // makes sure new users get put into the migration table while we're // backfilling. - let migrationState: MigrationState = MigrationState.Stately; + let migrationState: MigrationState = MigrationState.Postgres; if (platformMembershipId) { migrationState = await transaction(async (client) => backfillMigrationState(client, platformMembershipId ?? profileIds[0], bungieMembershipId), diff --git a/api/stately/init/migrate-stately-to-postgres.ts b/api/stately/init/migrate-stately-to-postgres.ts index b9e5b3c..ec3834c 100644 --- a/api/stately/init/migrate-stately-to-postgres.ts +++ b/api/stately/init/migrate-stately-to-postgres.ts @@ -210,6 +210,47 @@ function toErrorMessage(error: unknown): string { return String(error).slice(0, 500); } +function hasUnpairedSurrogates(value: string): boolean { + for (let i = 0; i < value.length; i++) { + const code = value.charCodeAt(i); + if (code >= 0xd800 && code <= 0xdbff) { + const next = value.charCodeAt(i + 1); + if (!(next >= 0xdc00 && next <= 0xdfff)) { + return true; + } + i += 1; + continue; + } + + if (code >= 0xdc00 && code <= 0xdfff) { + return true; + } + } + + return false; +} + +function isWellFormedUnicode(value: string): boolean { + const candidate = value as string & { isWellFormed?: () => boolean }; + if (typeof candidate.isWellFormed === 'function') { + return candidate.isWellFormed(); + } + + return !hasUnpairedSurrogates(value); +} + +function getUnsafePostgresTextReason(value: string): string | undefined { + if (!isWellFormedUnicode(value)) { + return 'string contains unpaired UTF-16 surrogate code units'; + } + + if (value.includes('\u0000')) { + return 'string contains NUL (\\u0000), which Postgres text cannot store'; + } + + return undefined; +} + async function migrateOneClaimedUser( pgClient: ClientBase, bungieMembershipId: number | undefined, @@ -252,17 +293,32 @@ async function migrateOneClaimedUser( } for (const searchData of searches) { - await importSearch( - pgClient, - bungieMembershipId, - platformMembershipId, - searchData.destinyVersion, - searchData.search.query, - searchData.search.saved, - searchData.search.lastUsage, - searchData.search.usageCount, - searchData.search.type, - ); + try { + // if the query isn't valid UTF-8, the importSearch function will throw. In that case we want to skip it and continue with the rest of the migration instead of failing the whole migration. + const invalidReason = getUnsafePostgresTextReason(searchData.search.query); + if (invalidReason) { + console.warn( + `Skipping search with invalid query for ${platformMembershipId} (${invalidReason}):`, + searchData.search.query, + ); + continue; + } + + await importSearch( + pgClient, + bungieMembershipId, + platformMembershipId, + searchData.destinyVersion, + searchData.search.query, + searchData.search.saved, + searchData.search.lastUsage, + searchData.search.usageCount, + searchData.search.type, + ); + } catch (error) { + console.error(`Failed to import search for ${platformMembershipId}`, searchData, error); + throw error; + } } } @@ -311,7 +367,7 @@ try { console.log(`Migration finished for ${platformMembershipId}`); } catch (error) { const errorMessage = toErrorMessage(error); - console.error(`Migration failed for ${platformMembershipId}:`, errorMessage); + console.error(`Migration failed for ${platformMembershipId}:`, error); await withRetry(`abortMigration:${platformMembershipId}`, () => transaction(async (pgClient) => { await abortMigrationToPostgres( diff --git a/kubernetes/dim-api-migration-worker-deployment.yaml b/kubernetes/dim-api-migration-worker-deployment.yaml index d0f8fb6..8c900aa 100644 --- a/kubernetes/dim-api-migration-worker-deployment.yaml +++ b/kubernetes/dim-api-migration-worker-deployment.yaml @@ -5,7 +5,7 @@ metadata: labels: app: dim-api-migration-worker spec: - replicas: 2 + replicas: 1 selector: matchLabels: app: dim-api-migration-worker @@ -20,7 +20,7 @@ spec: containers: - name: dim-api-migration-worker image: destinyitemmanager/dim-api:$COMMITHASH - imagePullPolicy: IfNotPresent + imagePullPolicy: Always command: ['node'] args: - --enable-source-maps