RHINENG-26348: add ability to repair system_advisories_0 partition

database_admin/config.go

@@ -19,4 +19,6 @@ var (
 	updateDBConfig = utils.PodConfig.GetBool("update_db_config", false)
 	// Terminate lockUsers sessions after NOLOGIN (for major DDL migrations)
 	terminateDBSessions = utils.PodConfig.GetBool("terminate_db_sessions", false)
+	// One-off: replace corrupt system_advisories_0 and clear bucket-0 advisory caches
+	repairSystemAdvisories0 = utils.PodConfig.GetBool("repair_system_advisories_0", false)
 )

database_admin/schema/repair_system_advisories_0.sql

@@ -0,0 +1,44 @@
+-- One-off repair for corrupt system_advisories_0 (pg_xact / unreadable heap).
+-- Gated by database_admin flag repair_system_advisories_0=true (default false).
+-- Irreversible: drops partition data for hash remainder 0 and clears related caches.
+-- Safe to re-run only when intentionally wiping bucket 0 again (e.g. failed cutover).
+
+BEGIN;
+
+-- Detach if still a partition of system_advisories (no-op when already detached/missing).
+DO $$
+BEGIN
+    IF EXISTS (
+        SELECT 1
+          FROM pg_inherits i
+          JOIN pg_class child ON child.oid = i.inhrelid
+          JOIN pg_class parent ON parent.oid = i.inhparent
+         WHERE parent.relname = 'system_advisories'
+           AND child.relname = 'system_advisories_0'
+    ) THEN
+        EXECUTE 'ALTER TABLE system_advisories DETACH PARTITION system_advisories_0';
+    END IF;
+END $$;
+
+DROP TABLE IF EXISTS system_advisories_0;
+
+CREATE TABLE system_advisories_0
+    PARTITION OF system_advisories
+    FOR VALUES WITH (MODULUS 32, REMAINDER 0)
+    WITH (fillfactor = '70', autovacuum_vacuum_scale_factor = '0.05');
+
+GRANT SELECT, INSERT, UPDATE, DELETE ON system_advisories_0 TO evaluator;
+GRANT UPDATE, DELETE ON system_advisories_0 TO manager;
+GRANT SELECT, INSERT, UPDATE, DELETE ON system_advisories_0 TO listener;
+GRANT SELECT, DELETE ON system_advisories_0 TO vmaas_sync;
+
+-- Clear denormalized counts for accounts that hash into remainder 0 (do not read old _0).
+DELETE FROM advisory_account_data aad
+ WHERE satisfies_hash_partition(
+         'system_advisories'::regclass, 32, 0, aad.rh_account_id);
+
+DELETE FROM account_advisory aa
+ WHERE satisfies_hash_partition(
+         'system_advisories'::regclass, 32, 0, aa.rh_account_id);
+
+COMMIT;

database_admin/update.go

@@ -184,6 +184,15 @@ func startMigration(conn database.Driver, db *sql.DB, migrationFilesURL string)
 	unblockUsers(db)
 }
 
+func repairSystemAdvisories0Partition(db *sql.DB) {
+	log.Info("Repairing system_advisories_0 (detach/drop/create + clear bucket-0 caches)")
+	prepareForMigration(db)
+	execFromFile(db, "./database_admin/schema/repair_system_advisories_0.sql")
+	log.Info("system_advisories_0 repair finished")
+	log.Info("Reverting components privileges after system_advisories_0 repair")
+	unblockUsers(db)
+}
+
 func dbConn() (database.Driver, *sql.DB) {
 	sslModeCert := utils.CoreCfg.DBSslMode
 	if utils.CoreCfg.DBSslRootCert != "" {
@@ -236,6 +245,10 @@ func UpdateDB(migrationFilesURL string) {
 		startMigration(conn, db, migrationFilesURL)
 	}
 
+	if repairSystemAdvisories0 {
+		repairSystemAdvisories0Partition(db)
+	}
+
 	if updateUsers {
 		log.Info("Setting user passwords")
 		// Set specific password for each user. If the users are already created, change the password.

deploy/clowdapp.yaml

@@ -896,6 +896,7 @@ parameters:
 - {name: DATABASE_ADMIN_CONFIG, value: ''} # Set 'schema_version=XXX' if need specific database schema
                                            # 'terminate_db_sessions=true' to kill app DB sessions before DDL
                                            # 'force_schema_version=XXX' to reset the dirty flag to false and force the specific version, it will follow up with the schema upgrade defined by schema_version
+                                           # 'repair_system_advisories_0=true' one-off: replace corrupt system_advisories_0 and clear bucket-0 AAD/AA (default false; remove after cutover)
 
 # Common parameters
 - {name: IMAGE, value: quay.io/redhat-services-prod/insights-management-tenant/insights-patch/patchman-engine}

docs/md/major-migration-runbook.md

@@ -147,6 +147,18 @@ Config keys are defined in `database_admin/config.go`. ClowdApp comments in `dep
 
 `NOLOGIN` alone does not close existing connections — that is why this flag exists.
 
+### `repair_system_advisories_0`
+
+| | |
+|---|---|
+| **Config key** | `repair_system_advisories_0` (boolean, default `false`) |
+| **Where** | `DATABASE_ADMIN_CONFIG` on the **db-migration Job** only |
+| **Effect** | After migrate CONTINUE/MIGRATE, runs `prepareForMigration`, then `database_admin/schema/repair_system_advisories_0.sql`: detach/drop/recreate empty `system_advisories_0`, clear bucket-0 `advisory_account_data` and `account_advisory`. **Destructive** for hash remainder 0. |
+
+**Enable when:** one-off recovery from corrupt/unreadable `system_advisories_0`. Combine with `terminate_db_sessions=true` if detach/drop is blocked by app sessions.
+
+**Leave off for all normal deploys.** Remove after the cutover succeeds. Do not enable on manager/listener/evaluator pods.
+
 ---
 
 ## During deploy