diff --git a/database_admin/config.go b/database_admin/config.go index edda9832c..1195d0fb0 100644 --- a/database_admin/config.go +++ b/database_admin/config.go @@ -19,4 +19,6 @@ var ( updateDBConfig = utils.PodConfig.GetBool("update_db_config", false) // Terminate lockUsers sessions after NOLOGIN (for major DDL migrations) terminateDBSessions = utils.PodConfig.GetBool("terminate_db_sessions", false) + // One-off: replace corrupt system_advisories_0 and clear bucket-0 advisory caches + repairSystemAdvisories0 = utils.PodConfig.GetBool("repair_system_advisories_0", false) ) diff --git a/database_admin/schema/repair_system_advisories_0.sql b/database_admin/schema/repair_system_advisories_0.sql new file mode 100644 index 000000000..a1673b607 --- /dev/null +++ b/database_admin/schema/repair_system_advisories_0.sql @@ -0,0 +1,44 @@ +-- One-off repair for corrupt system_advisories_0 (pg_xact / unreadable heap). +-- Gated by database_admin flag repair_system_advisories_0=true (default false). +-- Irreversible: drops partition data for hash remainder 0 and clears related caches. +-- Safe to re-run only when intentionally wiping bucket 0 again (e.g. failed cutover). + +BEGIN; + +-- Detach if still a partition of system_advisories (no-op when already detached/missing). +DO $$ +BEGIN + IF EXISTS ( + SELECT 1 + FROM pg_inherits i + JOIN pg_class child ON child.oid = i.inhrelid + JOIN pg_class parent ON parent.oid = i.inhparent + WHERE parent.relname = 'system_advisories' + AND child.relname = 'system_advisories_0' + ) THEN + EXECUTE 'ALTER TABLE system_advisories DETACH PARTITION system_advisories_0'; + END IF; +END $$; + +DROP TABLE IF EXISTS system_advisories_0; + +CREATE TABLE system_advisories_0 + PARTITION OF system_advisories + FOR VALUES WITH (MODULUS 32, REMAINDER 0) + WITH (fillfactor = '70', autovacuum_vacuum_scale_factor = '0.05'); + +GRANT SELECT, INSERT, UPDATE, DELETE ON system_advisories_0 TO evaluator; +GRANT UPDATE, DELETE ON system_advisories_0 TO manager; +GRANT SELECT, INSERT, UPDATE, DELETE ON system_advisories_0 TO listener; +GRANT SELECT, DELETE ON system_advisories_0 TO vmaas_sync; + +-- Clear denormalized counts for accounts that hash into remainder 0 (do not read old _0). +DELETE FROM advisory_account_data aad + WHERE satisfies_hash_partition( + 'system_advisories'::regclass, 32, 0, aad.rh_account_id); + +DELETE FROM account_advisory aa + WHERE satisfies_hash_partition( + 'system_advisories'::regclass, 32, 0, aa.rh_account_id); + +COMMIT; diff --git a/database_admin/update.go b/database_admin/update.go index 0b67e230b..d8bffefab 100644 --- a/database_admin/update.go +++ b/database_admin/update.go @@ -184,6 +184,15 @@ func startMigration(conn database.Driver, db *sql.DB, migrationFilesURL string) unblockUsers(db) } +func repairSystemAdvisories0Partition(db *sql.DB) { + log.Info("Repairing system_advisories_0 (detach/drop/create + clear bucket-0 caches)") + prepareForMigration(db) + execFromFile(db, "./database_admin/schema/repair_system_advisories_0.sql") + log.Info("system_advisories_0 repair finished") + log.Info("Reverting components privileges after system_advisories_0 repair") + unblockUsers(db) +} + func dbConn() (database.Driver, *sql.DB) { sslModeCert := utils.CoreCfg.DBSslMode if utils.CoreCfg.DBSslRootCert != "" { @@ -236,6 +245,10 @@ func UpdateDB(migrationFilesURL string) { startMigration(conn, db, migrationFilesURL) } + if repairSystemAdvisories0 { + repairSystemAdvisories0Partition(db) + } + if updateUsers { log.Info("Setting user passwords") // Set specific password for each user. If the users are already created, change the password. diff --git a/deploy/clowdapp.yaml b/deploy/clowdapp.yaml index 2834554b7..f12214436 100644 --- a/deploy/clowdapp.yaml +++ b/deploy/clowdapp.yaml @@ -896,6 +896,7 @@ parameters: - {name: DATABASE_ADMIN_CONFIG, value: ''} # Set 'schema_version=XXX' if need specific database schema # 'terminate_db_sessions=true' to kill app DB sessions before DDL # 'force_schema_version=XXX' to reset the dirty flag to false and force the specific version, it will follow up with the schema upgrade defined by schema_version + # 'repair_system_advisories_0=true' one-off: replace corrupt system_advisories_0 and clear bucket-0 AAD/AA (default false; remove after cutover) # Common parameters - {name: IMAGE, value: quay.io/redhat-services-prod/insights-management-tenant/insights-patch/patchman-engine} diff --git a/docs/md/major-migration-runbook.md b/docs/md/major-migration-runbook.md index 20744581c..d5a5d3344 100644 --- a/docs/md/major-migration-runbook.md +++ b/docs/md/major-migration-runbook.md @@ -147,6 +147,18 @@ Config keys are defined in `database_admin/config.go`. ClowdApp comments in `dep `NOLOGIN` alone does not close existing connections — that is why this flag exists. +### `repair_system_advisories_0` + +| | | +|---|---| +| **Config key** | `repair_system_advisories_0` (boolean, default `false`) | +| **Where** | `DATABASE_ADMIN_CONFIG` on the **db-migration Job** only | +| **Effect** | After migrate CONTINUE/MIGRATE, runs `prepareForMigration`, then `database_admin/schema/repair_system_advisories_0.sql`: detach/drop/recreate empty `system_advisories_0`, clear bucket-0 `advisory_account_data` and `account_advisory`. **Destructive** for hash remainder 0. | + +**Enable when:** one-off recovery from corrupt/unreadable `system_advisories_0`. Combine with `terminate_db_sessions=true` if detach/drop is blocked by app sessions. + +**Leave off for all normal deploys.** Remove after the cutover succeeds. Do not enable on manager/listener/evaluator pods. + --- ## During deploy