From e2a73a475618660ca6dbd07dd662dbbd99ac2e0d Mon Sep 17 00:00:00 2001 From: Ben Hollis Date: Thu, 7 May 2026 22:12:51 -0700 Subject: [PATCH 1/6] Postgres only settings, sync --- api/db/settings-queries.ts | 26 ++++++++++++++++++++++++++ api/routes/profile.ts | 29 +++++++++++------------------ 2 files changed, 37 insertions(+), 18 deletions(-) diff --git a/api/db/settings-queries.ts b/api/db/settings-queries.ts index c66a823..2f4405f 100644 --- a/api/db/settings-queries.ts +++ b/api/db/settings-queries.ts @@ -26,6 +26,32 @@ export async function getSettings( : undefined; } +/** + * Get settings for a particular account. + */ +export async function syncSettings( + client: ClientBase, + bungieMembershipId: number, + syncTimestamp: number, +): Promise<{ settings: Partial; deleted: boolean; lastModifiedAt: number } | undefined> { + const results = await client.query<{ + settings: Partial; + deleted_at: Date | null; + last_updated_at: Date; + }>({ + name: 'get_settings', + text: 'SELECT settings, deleted_at, last_updated_at FROM settings WHERE membership_id = $1 and last_updated_at > $2', + values: [bungieMembershipId, new Date(syncTimestamp)], + }); + return results.rows.length > 0 + ? { + settings: results.rows[0].settings, + deleted: Boolean(results.rows[0].deleted_at), + lastModifiedAt: results.rows[0].last_updated_at.getTime(), + } + : undefined; +} + /** * Insert or update (upsert) an entire settings tree, totally replacing whatever's there. */ diff --git a/api/routes/profile.ts b/api/routes/profile.ts index 0ba9ae6..7b5c1b2 100644 --- a/api/routes/profile.ts +++ b/api/routes/profile.ts @@ -14,7 +14,7 @@ import { import { getLoadoutsForProfile, syncLoadoutsForProfile } from '../db/loadouts-queries.js'; import { getMigrationState, MigrationState } from '../db/migration-state-queries.js'; import { getSearchesForProfile, syncSearchesForProfile } from '../db/searches-queries.js'; -import { getSettings } from '../db/settings-queries.js'; +import { getSettings, syncSettings } from '../db/settings-queries.js'; import { getTrackedTriumphsForProfile, syncTrackedTriumphsForProfile, @@ -27,7 +27,6 @@ import { Search, SearchType } from '../shapes/search.js'; import { defaultSettings } from '../shapes/settings.js'; import { UserInfo } from '../shapes/user.js'; import { getProfile, syncProfile } from '../stately/bulk-queries.js'; -import { querySettings, syncSettings } from '../stately/settings-queries.js'; import { badRequest, checkPlatformMembershipId, isValidPlatformMembershipId } from '../utils.js'; type ProfileComponent = 'settings' | 'loadouts' | 'tags' | 'hashtags' | 'triumphs' | 'searches'; @@ -256,28 +255,22 @@ async function loadProfile( // TODO: should settings be stored under profile too?? maybe primary profile ID? promises.push( (async () => { - // Load settings from Postgres. If they're there, you're done. Otherwise load from Stately. const start = new Date(); - const now = Date.now(); + const tokenData = getSyncToken('s'); // TODO: Should add the token to the query to avoid fetching if unchanged const pgSettings = await readTransaction(async (pgClient) => - getSettings(pgClient, bungieMembershipId), + tokenData + ? syncSettings(pgClient, bungieMembershipId, tokenData) + : getSettings(pgClient, bungieMembershipId), ); - if (pgSettings) { - const tokenData = getSyncToken('s'); - if (tokenData === undefined || pgSettings.lastModifiedAt > tokenData) { - response.settings = { ...defaultSettings, ...pgSettings.settings }; - } - addSyncToken('s', { canSync: true, tokenData: now }); - } else { - const tokenData = getSyncToken('settings'); - const { settings: storedSettings, token: settingsToken } = tokenData - ? await syncSettings(tokenData) - : await querySettings(bungieMembershipId); - response.settings = storedSettings; - addSyncToken('settings', settingsToken); + if ( + tokenData === undefined || + (pgSettings !== undefined && pgSettings.lastModifiedAt > tokenData) + ) { + response.settings = { ...defaultSettings, ...pgSettings?.settings }; } + addSyncToken('s', { canSync: true, tokenData: now }); metrics.timing(`${timerPrefix}.settings`, start); })(), From 1561022b9ef424106349aade3df58412b1482f90 Mon Sep 17 00:00:00 2001 From: Ben Hollis Date: Thu, 7 May 2026 22:16:15 -0700 Subject: [PATCH 2/6] Remove stately from profile --- api/routes/profile.ts | 356 ++++++++++++++++-------------------------- 1 file changed, 132 insertions(+), 224 deletions(-) diff --git a/api/routes/profile.ts b/api/routes/profile.ts index 7b5c1b2..d905b63 100644 --- a/api/routes/profile.ts +++ b/api/routes/profile.ts @@ -1,5 +1,4 @@ import * as Sentry from '@sentry/node'; -import { ListToken } from '@stately-cloud/client'; import express from 'express'; import asyncHandler from 'express-async-handler'; import { readTransaction } from '../db/index.js'; @@ -12,7 +11,6 @@ import { syncItemHashTagsForProfile, } from '../db/item-hash-tags-queries.js'; import { getLoadoutsForProfile, syncLoadoutsForProfile } from '../db/loadouts-queries.js'; -import { getMigrationState, MigrationState } from '../db/migration-state-queries.js'; import { getSearchesForProfile, syncSearchesForProfile } from '../db/searches-queries.js'; import { getSettings, syncSettings } from '../db/settings-queries.js'; import { @@ -26,7 +24,6 @@ import { ProfileResponse } from '../shapes/profile.js'; import { Search, SearchType } from '../shapes/search.js'; import { defaultSettings } from '../shapes/settings.js'; import { UserInfo } from '../shapes/user.js'; -import { getProfile, syncProfile } from '../stately/bulk-queries.js'; import { badRequest, checkPlatformMembershipId, isValidPlatformMembershipId } from '../utils.js'; type ProfileComponent = 'settings' | 'loadouts' | 'tags' | 'hashtags' | 'triumphs' | 'searches'; @@ -223,21 +220,15 @@ async function loadProfile( destinyVersion: DestinyVersion, incomingSyncTokens?: { [component: string]: Buffer | number }, ) { - let response: ProfileResponse = { + const response: ProfileResponse = { sync: Boolean(incomingSyncTokens), }; const timerPrefix = response.sync ? 'profileSync' : 'profileStately'; const counterPrefix = response.sync ? 'sync' : 'stately'; - const syncTokens: { [component: string]: string | number } = {}; - const addSyncToken = ( - name: string, - token: ListToken | { canSync: boolean; tokenData: number }, - ) => { + const syncTokens: { [component: string]: number } = {}; + const addSyncToken = (name: string, token: { canSync: boolean; tokenData: number }) => { if (token.canSync) { - syncTokens[name] = - token.tokenData instanceof Uint8Array - ? Buffer.from(token.tokenData).toString('base64') - : token.tokenData; + syncTokens[name] = token.tokenData; } }; const getSyncToken = (name: string) => { @@ -277,234 +268,151 @@ async function loadProfile( ); } - let loadFromPostgres = false; - if ( - platformMembershipId && - (['loadouts', 'tags', 'hashtags', 'triumphs', 'searches'] as const).some((c) => - components.includes(c), - ) - ) { - const { state: migrationState } = await readTransaction(async (client) => - getMigrationState(client, platformMembershipId), - ); - - if (migrationState === MigrationState.Postgres) { - loadFromPostgres = true; - } + if (!platformMembershipId) { + badRequest(res, `Need a platformMembershipId to return ${components.join(', ')}`); + return; } - - if (loadFromPostgres) { - if (!platformMembershipId) { - badRequest(res, `Need a platformMembershipId to return ${components.join(', ')}`); - return; - } - promises.push( - (async () => { - const now = Date.now(); - await readTransaction(async (client) => { - // TODO: Special case: DIM wants everything, so we can get it in a single query - - if (components.includes('loadouts')) { - const start = new Date(); - const tokenData = getSyncToken('loadouts'); - if (tokenData) { - const { updated, deletedLoadoutIds } = await syncLoadoutsForProfile( - client, - platformMembershipId, - destinyVersion, - tokenData, - ); - if (updated.length) { - response.loadouts = updated; - } - if (deletedLoadoutIds.length) { - response.deletedLoadoutIds = deletedLoadoutIds; - } - } else { - const loadouts = await getLoadoutsForProfile( - client, - platformMembershipId, - destinyVersion, - ); - response.loadouts = loadouts; + promises.push( + (async () => { + const now = Date.now(); + await readTransaction(async (client) => { + // TODO: Special case: DIM wants everything, so we can get it in a single query + + if (components.includes('loadouts')) { + const start = new Date(); + const tokenData = getSyncToken('loadouts'); + if (tokenData) { + const { updated, deletedLoadoutIds } = await syncLoadoutsForProfile( + client, + platformMembershipId, + destinyVersion, + tokenData, + ); + if (updated.length) { + response.loadouts = updated; + } + if (deletedLoadoutIds.length) { + response.deletedLoadoutIds = deletedLoadoutIds; } - addSyncToken('loadouts', { - canSync: true, - tokenData: now, - }); - metrics.timing(`${timerPrefix}.loadouts`, start); + } else { + const loadouts = await getLoadoutsForProfile( + client, + platformMembershipId, + destinyVersion, + ); + response.loadouts = loadouts; } + addSyncToken('loadouts', { + canSync: true, + tokenData: now, + }); + metrics.timing(`${timerPrefix}.loadouts`, start); + } - if (components.includes('tags')) { - const start = new Date(); - const tokenData = getSyncToken('tags'); - if (tokenData) { - const { updated, deletedItemIds } = await syncItemAnnotationsForProfile( - client, - platformMembershipId, - destinyVersion, - tokenData, - ); - if (updated.length) { - response.tags = updated; - } - if (deletedItemIds.length) { - response.deletedTagsIds = deletedItemIds; - } - } else { - const tags = await getItemAnnotationsForProfile( - client, - platformMembershipId, - destinyVersion, - ); - response.tags = tags; + if (components.includes('tags')) { + const start = new Date(); + const tokenData = getSyncToken('tags'); + if (tokenData) { + const { updated, deletedItemIds } = await syncItemAnnotationsForProfile( + client, + platformMembershipId, + destinyVersion, + tokenData, + ); + if (updated.length) { + response.tags = updated; } - addSyncToken('tags', { canSync: true, tokenData: now }); - metrics.timing(`${timerPrefix}.tags`, start); + if (deletedItemIds.length) { + response.deletedTagsIds = deletedItemIds; + } + } else { + const tags = await getItemAnnotationsForProfile( + client, + platformMembershipId, + destinyVersion, + ); + response.tags = tags; } + addSyncToken('tags', { canSync: true, tokenData: now }); + metrics.timing(`${timerPrefix}.tags`, start); + } - if (components.includes('hashtags')) { - const start = new Date(); - const tokenData = getSyncToken('hashtags'); - if (tokenData) { - const { updated, deletedItemHashes } = await syncItemHashTagsForProfile( - client, - platformMembershipId, - tokenData, - ); - if (updated.length) { - response.itemHashTags = updated; - } - if (deletedItemHashes.length) { - response.deletedItemHashTagHashes = deletedItemHashes; - } - } else { - const tags = await getItemHashTagsForProfile(client, platformMembershipId); - response.itemHashTags = tags; + if (components.includes('hashtags')) { + const start = new Date(); + const tokenData = getSyncToken('hashtags'); + if (tokenData) { + const { updated, deletedItemHashes } = await syncItemHashTagsForProfile( + client, + platformMembershipId, + tokenData, + ); + if (updated.length) { + response.itemHashTags = updated; } - addSyncToken('hashtags', { canSync: true, tokenData: now }); - metrics.timing(`${timerPrefix}.hashtags`, start); + if (deletedItemHashes.length) { + response.deletedItemHashTagHashes = deletedItemHashes; + } + } else { + const tags = await getItemHashTagsForProfile(client, platformMembershipId); + response.itemHashTags = tags; } + addSyncToken('hashtags', { canSync: true, tokenData: now }); + metrics.timing(`${timerPrefix}.hashtags`, start); + } - if (components.includes('triumphs') && destinyVersion === 2) { - const start = new Date(); - const tokenData = getSyncToken('triumphs'); - if (tokenData) { - const { updated, deleted } = await syncTrackedTriumphsForProfile( - client, - platformMembershipId, - tokenData, - ); - if (updated.length) { - response.triumphs = updated; - } - if (deleted.length) { - response.deletedTriumphs = deleted; - } - } else { - const triumphs = await getTrackedTriumphsForProfile(client, platformMembershipId); - response.triumphs = triumphs; + if (components.includes('triumphs') && destinyVersion === 2) { + const start = new Date(); + const tokenData = getSyncToken('triumphs'); + if (tokenData) { + const { updated, deleted } = await syncTrackedTriumphsForProfile( + client, + platformMembershipId, + tokenData, + ); + if (updated.length) { + response.triumphs = updated; } - addSyncToken('triumphs', { canSync: true, tokenData: now }); - metrics.timing(`${timerPrefix}.triumphs`, start); + if (deleted.length) { + response.deletedTriumphs = deleted; + } + } else { + const triumphs = await getTrackedTriumphsForProfile(client, platformMembershipId); + response.triumphs = triumphs; } + addSyncToken('triumphs', { canSync: true, tokenData: now }); + metrics.timing(`${timerPrefix}.triumphs`, start); + } - if (components.includes('searches')) { - const start = new Date(); - const tokenData = getSyncToken('searches'); - if (tokenData) { - const { updated, deletedSearchHashes } = await syncSearchesForProfile( - client, - platformMembershipId, - destinyVersion, - tokenData, - ); - if (updated.length) { - response.searches = updated; - } - if (deletedSearchHashes.length) { - response.deletedSearchHashes = deletedSearchHashes; - } - } else { - const searches = await getSearchesForProfile( - client, - platformMembershipId, - destinyVersion, - ); - response.searches = searches; + if (components.includes('searches')) { + const start = new Date(); + const tokenData = getSyncToken('searches'); + if (tokenData) { + const { updated, deletedSearchHashes } = await syncSearchesForProfile( + client, + platformMembershipId, + destinyVersion, + tokenData, + ); + if (updated.length) { + response.searches = updated; } - addSyncToken('searches', { canSync: true, tokenData: now }); - metrics.timing(`${timerPrefix}.searches`, start); + if (deletedSearchHashes.length) { + response.deletedSearchHashes = deletedSearchHashes; + } + } else { + const searches = await getSearchesForProfile( + client, + platformMembershipId, + destinyVersion, + ); + response.searches = searches; } - }); - })(), - ); - } else { - // Special case: DIM wants everything, so we can get it in a single query - if ( - platformMembershipId && - (['loadouts', 'tags', 'hashtags', 'triumphs', 'searches'] as const).every((c) => - components.includes(c), - ) - ) { - // Replace the individual components with a bulk fetch - components = components.includes('settings') ? ['settings', 'p'] : ['p']; - } - - const loadComponent = ( - name: Exclude | 'p', - suffix: string, - handleEmpty: () => void, - ) => { - if (components.includes(name)) { - if (!platformMembershipId) { - badRequest(res, `Need a platformMembershipId to return ${name}`); - return; + addSyncToken('searches', { canSync: true, tokenData: now }); + metrics.timing(`${timerPrefix}.searches`, start); } - promises.push( - (async () => { - const start = new Date(); - const tokenData = getSyncToken(name); - const { profile, token } = tokenData - ? await syncProfile(tokenData) - : await getProfile(platformMembershipId, destinyVersion, suffix); - response = { ...response, ...profile }; - if (!tokenData) { - handleEmpty(); - } - addSyncToken(name, token); - metrics.timing(`${timerPrefix}.${name}`, start); - })(), - ); - } - }; - - loadComponent('p', '', () => { - response.loadouts ??= []; - response.searches ??= []; - response.tags ??= []; - response.itemHashTags ??= []; - response.triumphs ??= []; - response.searches ??= []; - }); - loadComponent('loadouts', '/loadout', () => { - response.loadouts ??= []; - }); - loadComponent('tags', '/ia', () => { - response.tags ??= []; - }); - if (destinyVersion === 2) { - loadComponent('hashtags', '/iht', () => { - response.itemHashTags ??= []; }); - } - loadComponent('triumphs', '/triumph', () => { - response.triumphs ??= []; - }); - loadComponent('searches', '/search', () => { - response.searches ??= []; - }); - } + })(), + ); await Promise.all(promises); From 11b6d04f070a758467b42ef863bc3ea2021362f9 Mon Sep 17 00:00:00 2001 From: Ben Hollis Date: Thu, 7 May 2026 22:18:30 -0700 Subject: [PATCH 3/6] Remove Stately from update --- api/routes/update.ts | 268 ++----------------------------------------- 1 file changed, 8 insertions(+), 260 deletions(-) diff --git a/api/routes/update.ts b/api/routes/update.ts index 547c4fb..b806051 100644 --- a/api/routes/update.ts +++ b/api/routes/update.ts @@ -1,9 +1,9 @@ import { captureMessage } from '@sentry/node'; -import { chunk, groupBy, partition, sortBy } from 'es-toolkit'; +import { sortBy } from 'es-toolkit'; import express from 'express'; import asyncHandler from 'express-async-handler'; import { ClientBase } from 'pg'; -import { readTransaction, transaction } from '../db/index.js'; +import { transaction } from '../db/index.js'; import { deleteItemAnnotationList, updateItemAnnotation as updateItemAnnotationInDb, @@ -13,17 +13,12 @@ import { deleteLoadout as deleteLoadoutInDb, updateLoadout as updateLoadoutInDb, } from '../db/loadouts-queries.js'; -import { backfillMigrationState, MigrationState } from '../db/migration-state-queries.js'; import { deleteSearch as deleteSearchInDb, saveSearch as saveSearchInDb, updateUsedSearch, } from '../db/searches-queries.js'; -import { - getSettings, - replaceSettings, - setSetting as setSettingInPostgres, -} from '../db/settings-queries.js'; +import { setSetting as setSettingInPostgres } from '../db/settings-queries.js'; import { trackTriumph as trackTriumphInDb, unTrackTriumph } from '../db/triumphs-queries.js'; import { metrics } from '../metrics/index.js'; import { ApiApp } from '../shapes/app.js'; @@ -31,52 +26,24 @@ import { DestinyVersion } from '../shapes/general.js'; import { ItemAnnotation, ItemHashTag } from '../shapes/item-annotations.js'; import { Loadout } from '../shapes/loadouts.js'; import { - DeleteLoadoutUpdate, DeleteSearchUpdate, ItemHashTagUpdate, - LoadoutUpdate, ProfileUpdate, ProfileUpdateRequest, ProfileUpdateResult, SavedSearchUpdate, SettingUpdate, - TagCleanupUpdate, - TagUpdate, TrackTriumphUpdate, UsedSearchUpdate, } from '../shapes/profile.js'; import { SearchType } from '../shapes/search.js'; -import { defaultSettings, Settings } from '../shapes/settings.js'; +import { Settings } from '../shapes/settings.js'; import { UserInfo } from '../shapes/user.js'; -import { client as statelyClient } from '../stately/client.js'; -import { - deleteItemAnnotation as deleteItemAnnotationListStately, - updateItemAnnotation as updateItemAnnotationInStately, -} from '../stately/item-annotations-queries.js'; -import { updateItemHashTag as updateItemHashTagInStately } from '../stately/item-hash-tags-queries.js'; -import { - deleteLoadout as deleteLoadoutInStately, - updateLoadout as updateLoadoutInStately, -} from '../stately/loadouts-queries.js'; -import { - deleteSearch as deleteSearchInStately, - UpdateSearch, - updateSearches, -} from '../stately/searches-queries.js'; -import { - getSettingsForUpdate, - getSettings as getSettingsStately, - keyFor, -} from '../stately/settings-queries.js'; -import { Transaction } from '../stately/stately-utils.js'; -import { trackUntrackTriumphs } from '../stately/triumphs-queries.js'; import { badRequest, checkPlatformMembershipId, - delay, isValidItemId, isValidPlatformMembershipId, - subtractObject, } from '../utils.js'; /** @@ -117,42 +84,11 @@ export const updateHandler = asyncHandler(async (req, res) => { return; } - // Do a conditional update of the migration state table in Postgres to mark - // that we've seen this user and they are in the Stately migration state. This - // makes sure new users get put into the migration table while we're - // backfilling. - let migrationState: MigrationState = MigrationState.Postgres; - if (platformMembershipId) { - migrationState = await transaction(async (client) => - backfillMigrationState(client, platformMembershipId ?? profileIds[0], bungieMembershipId), - ); - } - - if ( - migrationState === MigrationState.MigratingToPostgres && - updates.some((u) => u.action !== 'setting') - ) { - res.status(503).header('Retry-After', '60').send({ - error: 'MigrationInProgress', - message: `This account is being migrated. Try again in a little bit.`, - }); - return; - } - const results: ProfileUpdateResult[] = validateUpdates(req, updates, platformMembershipId, appId); // Only attempt updates that pass validation const updatesToApply = updates.filter((_, index) => results[index].status === 'Success'); - if (migrationState === MigrationState.Postgres) { - await pgUpdate(updatesToApply, bungieMembershipId, platformMembershipId, destinyVersion); - } else { - await statelyUpdate( - updatesToApply, - bungieMembershipId, - platformMembershipId ?? profileIds[0], - destinyVersion, - ); - } + await pgUpdate(updatesToApply, bungieMembershipId, platformMembershipId, destinyVersion); res.send({ results, @@ -243,127 +179,6 @@ function validateUpdates( return results; } -async function statelyUpdate( - updates: ProfileUpdate[], - bungieMembershipId: number, - platformMembershipId: string | undefined, - destinyVersion: DestinyVersion, -) { - // We want to group save/delete search and search updates together - const actionKey = (u: ProfileUpdate) => - u.action === 'save_search' || u.action === 'delete_search' ? 'search' : u.action; - - const sortedUpdates = sortBy(updates, [actionKey]).flatMap((u): ProfileUpdate[] => { - // Separate out tag_cleanup updates into individual updates - if (u.action === 'tag_cleanup') { - return u.payload.map((p) => ({ action: 'tag_cleanup', payload: [p] })); - } - return [u]; - }); - - const tagIds = new Set(); - for (const update of sortedUpdates) { - if (update.action === 'tag') { - tagIds.add(update.payload.id); - } - } - - for (const updateChunk of chunk(sortedUpdates, 25)) { - await statelyClient.transaction(async (txn) => { - for (const [action, group] of Object.entries(groupBy(updateChunk, actionKey))) { - switch (action) { - case 'setting': { - await settingsUpdates(group as SettingUpdate[], bungieMembershipId, txn); - break; - } - - case 'loadout': - await updateLoadoutInStately( - txn, - platformMembershipId!, - destinyVersion, - (group as LoadoutUpdate[]).map((u) => u.payload), - ); - break; - - case 'delete_loadout': - await deleteLoadoutInStately( - txn, - platformMembershipId!, - destinyVersion, - (group as DeleteLoadoutUpdate[]).map((u) => u.payload), - ); - break; - - case 'tag': - await updateItemAnnotationInStately( - txn, - platformMembershipId!, - destinyVersion, - (group as TagUpdate[]).map((u) => u.payload), - ); - break; - - case 'tag_cleanup': { - const instanceIds = (group as TagCleanupUpdate[]) - .flatMap((u) => u.payload) - .filter( - (id) => - // We've seen a problem where DIM sends a tag_cleanup and a tag for the same item in the same update - !tagIds.has(id) && isValidItemId(id), - ); - if (instanceIds.length) { - await deleteItemAnnotationListStately( - txn, - platformMembershipId!, - destinyVersion, - instanceIds, - ); - } - break; - } - - case 'item_hash_tag': - for (const update of group as ItemHashTagUpdate[]) { - // TODO: Batch this one too - await updateItemHashTagInStately(txn, platformMembershipId!, update.payload); - } - break; - - case 'track_triumph': - await trackUntrackTriumphs( - txn, - platformMembershipId!, - (group as TrackTriumphUpdate[]).map((u) => u.payload), - ); - break; - - // saved searches and used searches are collectively "searches" - case 'search': { - const searchUpdates = consolidateSearchUpdates( - group as (UsedSearchUpdate | SavedSearchUpdate | DeleteSearchUpdate)[], - ); - const [deletes, updates] = partition(searchUpdates, (u) => u.deleted); - if (deletes.length) { - await deleteSearchInStately( - txn, - platformMembershipId!, - destinyVersion, - deletes.map((u) => u.query), - ); - } - if (updates.length) { - await updateSearches(txn, platformMembershipId!, destinyVersion, updates); - } - break; - } - } - } - }); - await delay(100); // sleep to let transaction flush - } -} - async function pgUpdate( updates: ProfileUpdate[], bungieMembershipId: number, @@ -393,7 +208,7 @@ async function pgUpdate( for (const update of updates) { switch (update.action) { case 'setting': - await settingsUpdates([update], bungieMembershipId, undefined, client); + await settingsUpdates([update], bungieMembershipId, client); break; case 'loadout': @@ -815,38 +630,10 @@ async function updateItemHashTag( metrics.timing('update.updateItemHashTag', start); } -function consolidateSearchUpdates( - updates: (UsedSearchUpdate | SavedSearchUpdate | DeleteSearchUpdate)[], -) { - const updatesByQuery = groupBy(updates, (u) => u.payload.query); - return Object.values(updatesByQuery).map((group) => { - const u: UpdateSearch = { - query: group[0].payload.query, - type: group[0].payload.type ?? SearchType.Item, - incrementUsed: 0, - deleted: false, - }; - for (const update of group) { - if (update.action === 'save_search') { - u.deleted = false; - u.saved = update.payload.saved; - } else if (update.action === 'delete_search') { - u.deleted = true; - u.incrementUsed = 0; - } else { - u.deleted = false; - u.incrementUsed++; - } - } - return u; - }); -} - async function settingsUpdates( group: SettingUpdate[], bungieMembershipId: number, - txn?: Transaction, - client?: ClientBase, + client: ClientBase, ) { // The DIM reducer already combines settings updates, but just in case... let mergedSettings: Partial = group.shift()!.payload; @@ -854,44 +641,5 @@ async function settingsUpdates( mergedSettings = { ...mergedSettings, ...update.payload }; } - // TODO: Remove the check for settings in Postgres once we're fully migrated off Stately - const pgSettings = await (client - ? getSettings(client, bungieMembershipId) - : readTransaction((client) => getSettings(client, bungieMembershipId))); - if (pgSettings) { - await (client - ? setSettingInPostgres(client, bungieMembershipId, mergedSettings) - : transaction(async (client) => { - await setSettingInPostgres(client, bungieMembershipId, mergedSettings); - })); - } else { - const statelySettings = await (txn - ? getSettingsForUpdate(txn, bungieMembershipId) - : getSettingsStately(bungieMembershipId)); - if (statelySettings) { - mergedSettings = { ...statelySettings, ...mergedSettings }; - await (client - ? replaceSettings( - client, - bungieMembershipId, - subtractObject(mergedSettings, defaultSettings), - ) - : transaction(async (client) => { - replaceSettings( - client, - bungieMembershipId, - subtractObject(mergedSettings, defaultSettings), - ); - })); - await (txn - ? txn.del(keyFor(bungieMembershipId)) - : statelyClient.del(keyFor(bungieMembershipId))); - } else { - await (client - ? setSettingInPostgres(client, bungieMembershipId, mergedSettings) - : transaction(async (client) => { - await setSettingInPostgres(client, bungieMembershipId, mergedSettings); - })); - } - } + await setSettingInPostgres(client, bungieMembershipId, mergedSettings); } From 2c7be7ff6d4b705550ffe42fef99ae332d721338 Mon Sep 17 00:00:00 2001 From: Ben Hollis Date: Thu, 7 May 2026 22:20:50 -0700 Subject: [PATCH 4/6] Remove Stately from import/export --- api/routes/export.ts | 32 ++++++-------------------------- api/routes/import.ts | 22 +--------------------- 2 files changed, 7 insertions(+), 47 deletions(-) diff --git a/api/routes/export.ts b/api/routes/export.ts index 0a83a97..f154e74 100644 --- a/api/routes/export.ts +++ b/api/routes/export.ts @@ -4,17 +4,12 @@ import { readTransaction } from '../db/index.js'; import { getItemAnnotationsForProfile } from '../db/item-annotations-queries.js'; import { getItemHashTagsForProfile } from '../db/item-hash-tags-queries.js'; import { getLoadoutsForProfile } from '../db/loadouts-queries.js'; -import { getMigrationState, MigrationState } from '../db/migration-state-queries.js'; import { getSearchesForProfile } from '../db/searches-queries.js'; import { getSettings as getSettingsFromPostgres } from '../db/settings-queries.js'; import { getTrackedTriumphsForProfile } from '../db/triumphs-queries.js'; import { ExportResponse } from '../shapes/export.js'; import { DestinyVersion } from '../shapes/general.js'; -import { defaultSettings, Settings } from '../shapes/settings.js'; import { UserInfo } from '../shapes/user.js'; -import { exportDataForProfile } from '../stately/bulk-queries.js'; -import { getSettings } from '../stately/settings-queries.js'; -import { subtractObject } from '../utils.js'; export const exportHandler = asyncHandler(async (req, res) => { const { bungieMembershipId, profileIds } = req.user as UserInfo; @@ -31,20 +26,11 @@ export const exportHandler = asyncHandler(async (req, res) => { }; for (const profileId of profileIds) { - const migrationState = await readTransaction(async (client) => - getMigrationState(client, profileId), - ); - - let partialResponse: ExportResponse; - if (migrationState.state === MigrationState.Postgres) { - partialResponse = await readTransaction(async (client) => { - const d1Response = await pgExport(client, profileId, 1); - const d2Response = await pgExport(client, profileId, 2); - return mergeResponses(d1Response, d2Response); - }); - } else { - partialResponse = await exportDataForProfile(profileId); - } + const partialResponse = await readTransaction(async (client) => { + const d1Response = await pgExport(client, profileId, 1); + const d2Response = await pgExport(client, profileId, 2); + return mergeResponses(d1Response, d2Response); + }); response = mergeResponses(response, partialResponse); } @@ -71,16 +57,10 @@ function mergeResponses(base: ExportResponse, addition: ExportResponse): ExportR export async function exportSettings( bungieMembershipId: number, ): Promise { - let settings: Partial; const pgSettings = await readTransaction((client) => getSettingsFromPostgres(client, bungieMembershipId), ); - if (pgSettings) { - settings = pgSettings.settings; - } else { - settings = subtractObject((await getSettings(bungieMembershipId)) ?? {}, defaultSettings); - } - return settings; + return pgSettings?.settings ?? {}; } export async function pgExport( diff --git a/api/routes/import.ts b/api/routes/import.ts index 9bae79f..b12b032 100644 --- a/api/routes/import.ts +++ b/api/routes/import.ts @@ -7,7 +7,6 @@ import { } from '../db/item-annotations-queries.js'; import { softDeleteAllItemHashTags, updateItemHashTag } from '../db/item-hash-tags-queries.js'; import { softDeleteAllLoadouts, updateLoadout } from '../db/loadouts-queries.js'; -import { doMigration, getMigrationState, MigrationState } from '../db/migration-state-queries.js'; import { importSearch, softDeleteAllSearches } from '../db/searches-queries.js'; import { replaceSettings } from '../db/settings-queries.js'; import { softDeleteAllTrackedTriumphs, trackTriumph } from '../db/triumphs-queries.js'; @@ -18,7 +17,6 @@ import { ItemAnnotation, ItemHashTag } from '../shapes/item-annotations.js'; import { Loadout } from '../shapes/loadouts.js'; import { defaultSettings, Settings } from '../shapes/settings.js'; import { UserInfo } from '../shapes/user.js'; -import { deleteAllDataForUser } from '../stately/bulk-queries.js'; import { badRequest, subtractObject } from '../utils.js'; export const importHandler = asyncHandler(async (req, res) => { @@ -120,25 +118,7 @@ export const importHandler = asyncHandler(async (req, res) => { response.itemHashTags += importResp.itemHashTags; }; - const migrationState = await transaction(async (client) => - getMigrationState(client, profileId), - ); - - if (migrationState.state === MigrationState.MigratingToPostgres) { - badRequest( - res, - `Unable to import data for profile ${profileId} - migration in progress. Please wait a bit and try again.`, - ); - return; - } - - if (migrationState.state === MigrationState.Stately) { - await doMigration(bungieMembershipId, profileId, doImport, async () => - deleteAllDataForUser(bungieMembershipId, [profileId]), - ); - } else { - await doImport(); - } + await doImport(); } // default 200 OK From 0b15092602421080f11bc4064dbd9ec14fe5877c Mon Sep 17 00:00:00 2001 From: Ben Hollis Date: Thu, 7 May 2026 22:22:05 -0700 Subject: [PATCH 5/6] Remove Stately from delete-all-data --- api/routes/delete-all-data.ts | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/api/routes/delete-all-data.ts b/api/routes/delete-all-data.ts index a1bba4f..7eff93f 100644 --- a/api/routes/delete-all-data.ts +++ b/api/routes/delete-all-data.ts @@ -11,7 +11,6 @@ import { softDeleteAllTrackedTriumphs } from '../db/triumphs-queries.js'; import { DeleteAllResponse } from '../shapes/delete-all.js'; import { DestinyVersion } from '../shapes/general.js'; import { UserInfo } from '../shapes/user.js'; -import { deleteAllDataForUser } from '../stately/bulk-queries.js'; /** * Delete My Data - this allows a user to wipe all their data from DIM storage. @@ -19,7 +18,14 @@ import { deleteAllDataForUser } from '../stately/bulk-queries.js'; export const deleteAllDataHandler = asyncHandler(async (req, res) => { const { bungieMembershipId, profileIds } = req.user as UserInfo; - let result = await deleteAllDataForUser(bungieMembershipId, profileIds); + let result = { + settings: 1, + loadouts: 0, + tags: 0, + itemHashTags: 0, + triumphs: 0, + searches: 0, + }; await transaction(async (client) => { await deleteSettings(client, bungieMembershipId); From 09ca10bdece609207460e24fcec9a3619177a0ec Mon Sep 17 00:00:00 2001 From: Ben Hollis Date: Thu, 7 May 2026 22:32:21 -0700 Subject: [PATCH 6/6] Update tests --- api/db/settings-queries.ts | 2 +- api/routes/profile.ts | 269 +++++++++++++++++++------------------ api/server.test.ts | 54 +------- 3 files changed, 140 insertions(+), 185 deletions(-) diff --git a/api/db/settings-queries.ts b/api/db/settings-queries.ts index 2f4405f..0f9badf 100644 --- a/api/db/settings-queries.ts +++ b/api/db/settings-queries.ts @@ -39,7 +39,7 @@ export async function syncSettings( deleted_at: Date | null; last_updated_at: Date; }>({ - name: 'get_settings', + name: 'sync_settings', text: 'SELECT settings, deleted_at, last_updated_at FROM settings WHERE membership_id = $1 and last_updated_at > $2', values: [bungieMembershipId, new Date(syncTimestamp)], }); diff --git a/api/routes/profile.ts b/api/routes/profile.ts index d905b63..29aa523 100644 --- a/api/routes/profile.ts +++ b/api/routes/profile.ts @@ -268,151 +268,158 @@ async function loadProfile( ); } - if (!platformMembershipId) { - badRequest(res, `Need a platformMembershipId to return ${components.join(', ')}`); - return; - } - promises.push( - (async () => { - const now = Date.now(); - await readTransaction(async (client) => { - // TODO: Special case: DIM wants everything, so we can get it in a single query - - if (components.includes('loadouts')) { - const start = new Date(); - const tokenData = getSyncToken('loadouts'); - if (tokenData) { - const { updated, deletedLoadoutIds } = await syncLoadoutsForProfile( - client, - platformMembershipId, - destinyVersion, - tokenData, - ); - if (updated.length) { - response.loadouts = updated; - } - if (deletedLoadoutIds.length) { - response.deletedLoadoutIds = deletedLoadoutIds; + if ( + (['loadouts', 'tags', 'hashtags', 'triumphs', 'searches'] as const).some((c) => + components.includes(c), + ) + ) { + if (!platformMembershipId) { + badRequest(res, `Need a platformMembershipId to return ${components.join(', ')}`); + return; + } + + promises.push( + (async () => { + const now = Date.now(); + await readTransaction(async (client) => { + // TODO: Special case: DIM wants everything, so we can get it in a single query + + if (components.includes('loadouts')) { + const start = new Date(); + const tokenData = getSyncToken('loadouts'); + if (tokenData) { + const { updated, deletedLoadoutIds } = await syncLoadoutsForProfile( + client, + platformMembershipId, + destinyVersion, + tokenData, + ); + if (updated.length) { + response.loadouts = updated; + } + if (deletedLoadoutIds.length) { + response.deletedLoadoutIds = deletedLoadoutIds; + } + } else { + const loadouts = await getLoadoutsForProfile( + client, + platformMembershipId, + destinyVersion, + ); + response.loadouts = loadouts; } - } else { - const loadouts = await getLoadoutsForProfile( - client, - platformMembershipId, - destinyVersion, - ); - response.loadouts = loadouts; + addSyncToken('loadouts', { + canSync: true, + tokenData: now, + }); + metrics.timing(`${timerPrefix}.loadouts`, start); } - addSyncToken('loadouts', { - canSync: true, - tokenData: now, - }); - metrics.timing(`${timerPrefix}.loadouts`, start); - } - if (components.includes('tags')) { - const start = new Date(); - const tokenData = getSyncToken('tags'); - if (tokenData) { - const { updated, deletedItemIds } = await syncItemAnnotationsForProfile( - client, - platformMembershipId, - destinyVersion, - tokenData, - ); - if (updated.length) { - response.tags = updated; - } - if (deletedItemIds.length) { - response.deletedTagsIds = deletedItemIds; + if (components.includes('tags')) { + const start = new Date(); + const tokenData = getSyncToken('tags'); + if (tokenData) { + const { updated, deletedItemIds } = await syncItemAnnotationsForProfile( + client, + platformMembershipId, + destinyVersion, + tokenData, + ); + if (updated.length) { + response.tags = updated; + } + if (deletedItemIds.length) { + response.deletedTagsIds = deletedItemIds; + } + } else { + const tags = await getItemAnnotationsForProfile( + client, + platformMembershipId, + destinyVersion, + ); + response.tags = tags; } - } else { - const tags = await getItemAnnotationsForProfile( - client, - platformMembershipId, - destinyVersion, - ); - response.tags = tags; + addSyncToken('tags', { canSync: true, tokenData: now }); + metrics.timing(`${timerPrefix}.tags`, start); } - addSyncToken('tags', { canSync: true, tokenData: now }); - metrics.timing(`${timerPrefix}.tags`, start); - } - if (components.includes('hashtags')) { - const start = new Date(); - const tokenData = getSyncToken('hashtags'); - if (tokenData) { - const { updated, deletedItemHashes } = await syncItemHashTagsForProfile( - client, - platformMembershipId, - tokenData, - ); - if (updated.length) { - response.itemHashTags = updated; - } - if (deletedItemHashes.length) { - response.deletedItemHashTagHashes = deletedItemHashes; + if (components.includes('hashtags')) { + const start = new Date(); + const tokenData = getSyncToken('hashtags'); + if (tokenData) { + const { updated, deletedItemHashes } = await syncItemHashTagsForProfile( + client, + platformMembershipId, + tokenData, + ); + if (updated.length) { + response.itemHashTags = updated; + } + if (deletedItemHashes.length) { + response.deletedItemHashTagHashes = deletedItemHashes; + } + } else { + const tags = await getItemHashTagsForProfile(client, platformMembershipId); + response.itemHashTags = tags; } - } else { - const tags = await getItemHashTagsForProfile(client, platformMembershipId); - response.itemHashTags = tags; + addSyncToken('hashtags', { canSync: true, tokenData: now }); + metrics.timing(`${timerPrefix}.hashtags`, start); } - addSyncToken('hashtags', { canSync: true, tokenData: now }); - metrics.timing(`${timerPrefix}.hashtags`, start); - } - if (components.includes('triumphs') && destinyVersion === 2) { - const start = new Date(); - const tokenData = getSyncToken('triumphs'); - if (tokenData) { - const { updated, deleted } = await syncTrackedTriumphsForProfile( - client, - platformMembershipId, - tokenData, - ); - if (updated.length) { - response.triumphs = updated; + if (components.includes('triumphs') && destinyVersion === 2) { + const start = new Date(); + const tokenData = getSyncToken('triumphs'); + if (tokenData) { + const { updated, deleted } = await syncTrackedTriumphsForProfile( + client, + platformMembershipId, + tokenData, + ); + if (updated.length) { + response.triumphs = updated; + } + if (deleted.length) { + response.deletedTriumphs = deleted; + } + } else { + const triumphs = await getTrackedTriumphsForProfile(client, platformMembershipId); + response.triumphs = triumphs; } - if (deleted.length) { - response.deletedTriumphs = deleted; - } - } else { - const triumphs = await getTrackedTriumphsForProfile(client, platformMembershipId); - response.triumphs = triumphs; + addSyncToken('triumphs', { canSync: true, tokenData: now }); + metrics.timing(`${timerPrefix}.triumphs`, start); } - addSyncToken('triumphs', { canSync: true, tokenData: now }); - metrics.timing(`${timerPrefix}.triumphs`, start); - } - if (components.includes('searches')) { - const start = new Date(); - const tokenData = getSyncToken('searches'); - if (tokenData) { - const { updated, deletedSearchHashes } = await syncSearchesForProfile( - client, - platformMembershipId, - destinyVersion, - tokenData, - ); - if (updated.length) { - response.searches = updated; - } - if (deletedSearchHashes.length) { - response.deletedSearchHashes = deletedSearchHashes; + if (components.includes('searches')) { + const start = new Date(); + const tokenData = getSyncToken('searches'); + if (tokenData) { + const { updated, deletedSearchHashes } = await syncSearchesForProfile( + client, + platformMembershipId, + destinyVersion, + tokenData, + ); + if (updated.length) { + response.searches = updated; + } + if (deletedSearchHashes.length) { + response.deletedSearchHashes = deletedSearchHashes; + } + } else { + const searches = await getSearchesForProfile( + client, + platformMembershipId, + destinyVersion, + ); + response.searches = searches; } - } else { - const searches = await getSearchesForProfile( - client, - platformMembershipId, - destinyVersion, - ); - response.searches = searches; + addSyncToken('searches', { canSync: true, tokenData: now }); + metrics.timing(`${timerPrefix}.searches`, start); } - addSyncToken('searches', { canSync: true, tokenData: now }); - metrics.timing(`${timerPrefix}.searches`, start); - } - }); - })(), - ); + }); + })(), + ); + } await Promise.all(promises); diff --git a/api/server.test.ts b/api/server.test.ts index c97e353..cc67390 100644 --- a/api/server.test.ts +++ b/api/server.test.ts @@ -5,11 +5,8 @@ import { makeFetch } from 'supertest-fetch'; import { promisify } from 'util'; import { v4 as uuid } from 'uuid'; import { refreshApps, stopAppsRefresh } from './apps/index.js'; -import { setGlobalSettings } from './db/global-settings-queries.js'; import { closeDbPool, transaction } from './db/index.js'; import { MigrationState, setMigrationStateForTest } from './db/migration-state-queries.js'; -import { replaceSettings as replaceSettingsPostgres } from './db/settings-queries.js'; -import { extractImportData } from './routes/import.js'; import { app } from './server.js'; import { ApiApp } from './shapes/app.js'; import { DeleteAllResponse } from './shapes/delete-all.js'; @@ -21,47 +18,11 @@ import { Loadout, LoadoutItem } from './shapes/loadouts.js'; import { ProfileResponse, ProfileUpdateRequest, ProfileUpdateResponse } from './shapes/profile.js'; import { SearchType } from './shapes/search.js'; import { defaultSettings } from './shapes/settings.js'; -import { statelyImport } from './stately/bulk-queries.js'; const fetch = makeFetch(app); // Test backend configurations const backendConfigs = [ - { - backend: 'Stately', - state: MigrationState.Stately, - bungieMembershipId: 1234, - platformMembershipId: '4611686018433092312', - async setup() { - await transaction(async (client) => { - await setMigrationStateForTest( - client, - this.platformMembershipId, - this.bungieMembershipId, - this.state, - ); - await replaceSettingsPostgres(client, this.bungieMembershipId, defaultSettings); - }); - }, - importer: async () => { - const file = JSON.parse( - (await promisify(readFile)('./dim-data.json')).toString(), - ) as ExportResponse; - - const data = extractImportData(file); - - await statelyImport( - 1234, - ['4611686018433092312'], - data.settings, - data.loadouts, - data.itemAnnotations, - data.triumphs, - data.searches, - data.itemHashTags, - ); - }, - }, { backend: 'Postgres', state: MigrationState.Postgres, @@ -88,19 +49,6 @@ beforeAll(async () => { testApiKey = appResponse.dimApiKey; expect(testApiKey).toBeDefined(); await refreshApps(); - - // Make sure we have global settings in Stately - for (const stage of ['dev', 'beta', 'app']) { - await setGlobalSettings(stage, { - dimApiEnabled: true, - destinyProfileMinimumRefreshInterval: 15, - destinyProfileRefreshInterval: 120, - autoRefresh: true, - refreshProfileOnVisible: true, - dimProfileMinimumRefreshInterval: 600, - showIssueBanner: false, - }); - } }); afterAll(async () => { @@ -199,7 +147,7 @@ describe.each(backendConfigs)('$backend backend', (backend) => { describe('profile', () => { // Applies only to tests in this describe block - beforeEach(backend.importer ?? importData); + beforeEach(importData); it('can retrieve all profile data', async () => { const profileResponse = (await getRequestAuthed(