Space leak report improvements

- Improves concurrency handling.
- Fixes cases where the report might fail due to locked tables. The main scenario is a race with table deletion.

[Substrate:756e67c89c7312d1302bad38360ba6cb5e127418]

Author: Alexandre Costa

dev/ese/src/_res/jetmsg.mc

@@ -2620,6 +2620,8 @@ Space used by root space tree split buffers: %34 page(s) (%35 bytes, %36%%).%n
 Number of cached primary objects: %37%n
 Number of uncached primary objects: %38%n
 Correction applied to space owned by primary objects: %46 page(s) (%47 bytes, %48%%).%n
+Enumeration conflicts resolved successfully: %49%n
+Enumeration conflicts not resolved successfully: %50%n
 Performance: %39 page(s) read, %40 page(s) preread, %41 page(s) referenced, %42 page(s) dirtied, %43 page(s) re-dirtied.%n
 Duration: %44 minute(s) and %45 second(s).%n
 .

dev/ese/src/ese/_log/logredo.cxx

@@ -980,12 +980,12 @@ LOCAL ERR ErrLGRICreateFucb(
     const BOOL  fSpace,
     FUCB        **ppfucb )
 {
-    ERR         err = JET_errSuccess;
-    FUCB        *pfucb  = pfucbNil;
-    FCB         *pfcb = pfcbNil;
-    BOOL        fState;
-    ULONG       cRetries = 0;
-    BOOL        fCreatedNewFCB = fFalse;
+    ERR             err = JET_errSuccess;
+    FUCB            *pfucb = pfucbNil;
+    FCB             *pfcb = pfcbNil;
+    FCBStateFlags   fcbsf;
+    ULONG           cRetries = 0;
+    BOOL            fCreatedNewFCB = fFalse;
 
     //  create fucb
     //
@@ -1005,8 +1005,8 @@ LOCAL ERR ErrLGRICreateFucb(
 
     //  get fcb for table, if one exists
     //
-    pfcb = FCB::PfcbFCBGet( ifmp, pgnoFDP, &fState, fTrue /* FIncrementRefCount */, fTrue /* fInitForRecovery */);
-    Assert( pfcbNil == pfcb || fFCBStateInitialized == fState );
+    pfcb = FCB::PfcbFCBGet( ifmp, pgnoFDP, &fcbsf, fTrue /* FIncrementRefCount */, fTrue /* fInitForRecovery */);
+    Assert( pfcbNil == pfcb || ( fcbsf & fcbsfInitialized ) );
     if ( pfcbNil == pfcb )
     {
         //  there exists no fcb for FDP
@@ -1173,8 +1173,7 @@ LOCAL ERR ErrLGRIPurgeFcbs( const IFMP ifmp, const PGNO pgnoFDP, FDPTYPE fFDPTyp
 
     Assert( NULL != pctablehash );
 
-    BOOL    fState;
-    FCB *   pfcb    = FCB::PfcbFCBGet( ifmp, pgnoFDP, &fState );
+    FCB *   pfcb    = FCB::PfcbFCBGet( ifmp, pgnoFDP );
 
     if ( pfcbNil == pfcb )
     {
@@ -1385,7 +1384,7 @@ LOCAL ERR ErrLGRIPurgeFcbs( const IFMP ifmp, const PGNO pgnoFDP, FDPTYPE fFDPTyp
         pfcb->Purge();
     }
 
-    Assert( FCB::PfcbFCBGet( ifmp, pgnoFDP, &fState ) == pfcbNil );
+    Assert( FCB::PfcbFCBGet( ifmp, pgnoFDP ) == pfcbNil );
 
     return JET_errSuccess;
 }
@@ -10957,10 +10956,9 @@ ERR LOG::ErrLGRIRedoRootPageMove( PIB* const ppib, const DBTIME dbtime )
     OnDebug( rm.AssertValid( fFalse /* fBeforeMove */, fTrue /* fRedo */ ) );
 
     // We must not have reloaded these.
-    BOOL fUnused = fFalse;
-    Assert( FCB::PfcbFCBGet( ifmp, rm.pgnoFDP, &fUnused ) == pfcbNil );
-    Assert( FCB::PfcbFCBGet( ifmp, pgnoFDPMSO, &fUnused ) == pfcbNil );
-    Assert( FCB::PfcbFCBGet( ifmp, pgnoFDPMSOShadow, &fUnused ) == pfcbNil );
+    Assert( FCB::PfcbFCBGet( ifmp, rm.pgnoFDP ) == pfcbNil );
+    Assert( FCB::PfcbFCBGet( ifmp, pgnoFDPMSO ) == pfcbNil );
+    Assert( FCB::PfcbFCBGet( ifmp, pgnoFDPMSOShadow ) == pfcbNil );
 
     // We will log the root page move to the RBS even if PerformRootMove is skipped for the page as it is possible page was patched and it's was skipped.
     // Note: It is important that we capture this into the snapshot after we capture the preimage of the pages being moved.

dev/ese/src/ese/bt.cxx

@@ -7778,7 +7778,7 @@ ERR ErrBTIOpen(
 {
     ERR             err;
     FCB             *pfcb;
-    INT             fState;
+    FCBStateFlags   fcbsf;
     ULONG           cRetries = 0;
     PIBTraceContextScope tcScope = ppib->InitTraceContextScope( );
     tcScope->iorReason.SetIors( iorsBTOpen );
@@ -7788,13 +7788,13 @@ ERR ErrBTIOpen(
 
     //  get the FCB for the given ifmp/pgnoFDP
 
-    pfcb = FCB::PfcbFCBGet( ifmp, pgnoFDP, &fState, fTrue, !fWillInitFCB );
+    pfcb = FCB::PfcbFCBGet( ifmp, pgnoFDP, &fcbsf, fTrue, !fWillInitFCB );
     if ( pfcb == pfcbNil )
     {
 
         //  the FCB does not exist
 
-        Assert( fFCBStateNull == fState );
+        Assert( fcbsfNone == fcbsf );
 
         //  try to create a new B-tree which will cause the creation of the new FCB
 
@@ -7821,7 +7821,7 @@ ERR ErrBTIOpen(
     {
         tcScope->nParentObjectClass = pfcb->TCE();
 
-        if ( fFCBStateInitialized == fState )
+        if ( fcbsf & fcbsfInitialized )
         {
             Assert( pfcb->WRefCount() >= 1);
             err = ErrBTOpen( ppib, pfcb, ppfucb );

dev/ese/src/ese/cat.cxx

@@ -10179,7 +10179,7 @@ ERR ErrCATRenameTable(
 //  ================================================================
 {
     ERR             err;
-    INT             fState              = fFCBStateNull;
+    FCBStateFlags   fcbsf               = fcbsfNone;
     FCB             * pfcbTable         = pfcbNil;
     OBJID           objidTable;
     PGNO            pgnoFDPTable;
@@ -10192,10 +10192,10 @@ ERR ErrCATRenameTable(
     //  check to see if the FCB is present and initialized
     //  if its not present we can just update the catalog
 
-    pfcbTable = FCB::PfcbFCBGet( ifmp, pgnoFDPTable, &fState );
+    pfcbTable = FCB::PfcbFCBGet( ifmp, pgnoFDPTable, &fcbsf );
     if( pfcbNil != pfcbTable )
     {
-        if( fFCBStateInitialized != fState )
+        if( !( fcbsf & fcbsfInitialized ) )
         {
 
             //  this should only happen if this is called in a multi-threaded scenario
@@ -15241,6 +15241,7 @@ ERR ErrCATGetNextRootObject(
     _In_ const BOOL         fSortedByObjId,
     _Inout_ FUCB** const    ppfucbCatalog,
     _Out_ OBJID* const      pobjid,
+    _Out_ PGNO* const       ppgnoFDP,
     _Out_writes_opt_z_( JET_cbNameMost + 1 ) CHAR* const szObjectName )
 //  ================================================================
 {
@@ -15251,6 +15252,7 @@ ERR ErrCATGetNextRootObject(
 
     ERR err = JET_errSuccess;
     OBJID objid = objidNil;
+    PGNO pgnoFDP = pgnoNull;
     FUCB* pfucbCatalog = *ppfucbCatalog;
     BOOL fInitilializedFucb = fFalse;
     BOOL fCatalogLatched = fFalse;
@@ -15387,6 +15389,15 @@ ERR ErrCATGetNextRootObject(
     objid = *( (UnalignedLittleEndian<OBJID>*)dataField.Pv() );
     Assert( objid != objidNil );
 
+    if ( ppgnoFDP != NULL )
+    {
+        // Retrieve the pgnoFDP.
+        Call( ErrRECIRetrieveFixedColumn( pfcbNil, pfucbCatalog->u.pfcb->Ptdb(), fidMSO_PgnoFDP, pfucbCatalog->kdfCurr.data, &dataField ) );
+        Assert( dataField.Cb() == sizeof( pgnoFDP ) );
+        pgnoFDP = *( ( UnalignedLittleEndian<PGNO>* )dataField.Pv() );
+        Assert( pgnoFDP != pgnoNull );
+    }
+
     if ( szObjectName != NULL )
     {
         // Retrieve the object name.
@@ -15406,6 +15417,10 @@ ERR ErrCATGetNextRootObject(
         {
             OSStrCbCopyA( szObjectName, sizeof( szObjectNameT ), szObjectNameT );
         }
+        if ( ppgnoFDP != NULL )
+        {
+            *ppgnoFDP = pgnoFDP;
+        }
     }
     else
     {
@@ -15414,6 +15429,10 @@ ERR ErrCATGetNextRootObject(
         {
             *szObjectName = '\0';
         }
+        if ( ppgnoFDP != NULL )
+        {
+            *ppgnoFDP = pgnoNull;
+        }
     }
 
     if ( fCatalogLatched )

dev/ese/src/ese/dbutil.cxx

@@ -5153,6 +5153,8 @@ LOCAL VOID DBUTLIReportSpaceLeakEstimationSucceeded(
     const CPG cpgSplitBuffers,
     const ULONG cCachedPrimary,
     const ULONG cUncachedPrimary,
+    const ULONG cEnumerationConflictsSucceeded,
+    const ULONG cEnumerationConflictsFailed,
     const JET_THREADSTATS& jts,
     const ULONG ulMinElapsed,
     const double dblSecElapsed )
@@ -5189,7 +5191,9 @@ LOCAL VOID DBUTLIReportSpaceLeakEstimationSucceeded(
         OSFormatW( L"%u", cUncachedPrimary ),
         OSFormatW( L"%u", jts.cPageRead ), OSFormatW( L"%u", jts.cPagePreread ), OSFormatW( L"%u", jts.cPageReferenced ), OSFormatW( L"%u", jts.cPageDirtied ), OSFormatW( L"%u", jts.cPageRedirtied ),
         OSFormatW( L"%u", ulMinElapsed ), OSFormatW( L"%.3f", dblSecElapsed ),
-        OSFormatW( L"%d", cpgOwnedPrimaryCorrection ), OSFormatW( L"%I64d", pfmp->CbOfCpgSigned( cpgOwnedPrimaryCorrection ) ), ( ( cpgOwnedPrimaryOriginal != 0 ) ? OSFormatW( L"%.3f", ( 100.0 * (double)cpgOwnedPrimaryCorrection ) / (double)cpgOwnedPrimaryOriginal ) : L"-" )
+        OSFormatW( L"%d", cpgOwnedPrimaryCorrection ), OSFormatW( L"%I64d", pfmp->CbOfCpgSigned( cpgOwnedPrimaryCorrection ) ), ( ( cpgOwnedPrimaryOriginal != 0 ) ? OSFormatW( L"%.3f", ( 100.0 * (double)cpgOwnedPrimaryCorrection ) / (double)cpgOwnedPrimaryOriginal ) : L"-" ),
+        OSFormatW( L"%u", cEnumerationConflictsSucceeded ),
+        OSFormatW( L"%u", cEnumerationConflictsFailed )
     };
     UtilReportEvent(
         eventInformation,
@@ -5245,6 +5249,8 @@ LOCAL ERR ErrDBUTLIEstimateRootSpaceLeak( PIB* const ppib, const IFMP ifmp )
     BOOL fRunning = fFalse;
     JET_THREADSTATS jtsStart = { 0 }, jtsEnd = { 0 };
     OBJID objidLast = objidNil;
+    PGNO pgnoFDPLast = pgnoNull;
+    ULONG cEnumerationConflictsFailed = 0, cEnumerationConflictsSucceeded = 0;
     CPG cpgOwnedPrimary = 0, cpgOwnedPrimaryCorrection = 0;
     ULONG cCachedPrimary = 0, cUncachedPrimary = 0;
     CPG cpgUsedRoot = 0, cpgUsedOe = 0, cpgUsedAe = 0;
@@ -5277,9 +5283,9 @@ LOCAL ERR ErrDBUTLIEstimateRootSpaceLeak( PIB* const ppib, const IFMP ifmp )
     ppib->SetFSessionLeakReport();
 
     CHAR szObjectName[ JET_cbNameMost + 1 ];
-    for ( err = ErrCATGetNextRootObject( ppib, ifmp, fTrue, &pfucbCatalog, &objidLast, szObjectName );
+    for ( err = ErrCATGetNextRootObject( ppib, ifmp, fTrue, &pfucbCatalog, &objidLast, &pgnoFDPLast, szObjectName );
         ( err >= JET_errSuccess ) && ( objidLast != objidNil );
-        err = ErrCATGetNextRootObject( ppib, ifmp, fTrue, &pfucbCatalog, &objidLast, szObjectName ) )
+        err = ErrCATGetNextRootObject( ppib, ifmp, fTrue, &pfucbCatalog, &objidLast, &pgnoFDPLast, szObjectName ) )
     {
 #ifdef DEBUG
         Assert( objidLast != objidSystemRoot );  // Root object is not supposed to be returned here.
@@ -5307,37 +5313,111 @@ LOCAL ERR ErrDBUTLIEstimateRootSpaceLeak( PIB* const ppib, const IFMP ifmp )
             // Test injection.
             OnDebug( while ( objidLast >= (OBJID)UlConfigOverrideInjection( 48550, objidFDPOverMax ) ) );
 
-            err = ErrFILEOpenTable( ppib, ifmp, &pfucbTable, szObjectName, JET_bitTableReadOnly | JET_bitTableTryPurgeOnClose );
-            if ( err == JET_errObjectNotFound )
+            BOOL fRetried = fFalse, fRetry = fFalse;
+            const BOOL fInfiniteRetries = OnDebugOrRetail( fTrue, fFalse );
+            ERR errRetry = JET_errSuccess;
+            const CHAR* wszRetryReason = "";
+            do
             {
-                err = JET_errSuccess;
-            }
-            else
-            {
-                Call( err );
+                fRetried = fRetry;
+                if ( fRetry )
+                {
+                    UtilSleep( 10 );
+                    fRetry = fFalse;
+                }
 
-                cUncachedPrimary++;
-                
-                // Test injection.
-                OnDebug( while ( objidLast >= (OBJID)UlConfigOverrideInjection( 57894, objidFDPOverMax ) ) );
+                err = ErrFILEOpenTable( ppib, ifmp, &pfucbTable, szObjectName, JET_bitTableReadOnly | JET_bitTableTryPurgeOnClose );
+                if ( ( err == JET_errObjectNotFound ) || ( err == JET_errTableLocked ) )
+                {
+                    // We are probably racing with table deletion.
+                    FCBStateFlags fcbsf = fcbsfNone;
+                    const BOOL fFoundFcb = ( FCB::PfcbFCBGet( ifmp, pgnoFDPLast, &fcbsf, fFalse /* fIncrementRefCount */, fTrue /* fInitForRecovery */ ) != pfcbNil );
+                    const BOOL fDeletePending = fFoundFcb && ( fcbsf & fcbsfDeletePending );
 
-                Call( ErrSPGetInfo(
-                    ppib,
-                    ifmp,
-                    pfucbTable,
-                    (BYTE*)&cpgPrimaryObject,
-                    sizeof( cpgPrimaryObject ),
-                    fSPOwnedExtent,
-                    gci::Allow ) );
+                    if ( fFoundFcb && !fDeletePending )
+                    {
+                        // This is unexpected if we know the table is actually getting deleted.
+                        Assert( err != JET_errObjectNotFound );
+                        fRetry = fTrue;
+                        wszRetryReason = "DelNotPending";
+                    }
+                    else if ( fFoundFcb && fDeletePending )
+                    {
+                        // Table deletion is still pending.
+                        fRetry = fTrue;
+                        wszRetryReason = "DelPending";
 
-                cpgOwnedPrimary += cpgPrimaryObject;
+                        // Perform cleanup.
+                        (void)PverFromPpib( ppib )->ErrVERRCEClean( ifmp );
+                    }
+                    else
+                    {
+                        Assert( !fFoundFcb );
+                        if ( err == JET_errTableLocked )
+                        {
+                            // Either the version store entry for the table deletion has cleared,
+                            // or an exclusive user released the table and the FCB got purged.
+                            fRetry = fTrue;
+                            wszRetryReason = "FcbNotFound";
+                        }
+                        else
+                        {
+                            // Version store entry for the table deletion has cleared.
+                            // No need to retry.
+                            Assert( err == JET_errObjectNotFound );
+                        }
+                    }
 
-                Call( ErrFILECloseTable( ppib, pfucbTable ) );
-                pfucbTable = pfucbNil;
-            }
+                    errRetry = err;
+                    err = JET_errSuccess;
+                }
+                else
+                {
+                    Assert( !fRetry );
+                    Call( err );
+
+                    cUncachedPrimary++;
 
-            Assert( pfucbTable == pfucbNil );
+                    // Test injection.
+                    OnDebug( while ( objidLast >= (OBJID)UlConfigOverrideInjection( 57894, objidFDPOverMax ) ) );
+
+                    Call( ErrSPGetInfo(
+                        ppib,
+                        ifmp,
+                        pfucbTable,
+                        (BYTE*)&cpgPrimaryObject,
+                        sizeof( cpgPrimaryObject ),
+                        fSPOwnedExtent,
+                        gci::Allow ) );
+
+                    cpgOwnedPrimary += cpgPrimaryObject;
+
+                    Call( ErrFILECloseTable( ppib, pfucbTable ) );
+                    pfucbTable = pfucbNil;
+                }
+
+                if ( fRetried )
+                {
+                    if ( fRetry )
+                    {
+                        cEnumerationConflictsFailed++;
+                        if ( !fInfiniteRetries )
+                        {
+                            FireWall( OSFormat( "LeakReportConflict:%s:%d", wszRetryReason, errRetry ) );
+                        }
+                    }
+                    else
+                    {
+                        cEnumerationConflictsSucceeded++;
+                    }
+                }
+
+                Assert( pfucbTable == pfucbNil );
+                Assert( err >= JET_errSuccess );
+            }
+            while ( fRetry && ( !fRetried || fInfiniteRetries ) );
         }
+
         pfmp->SetOjidLeakEstimation( objidLast );
 
 #ifdef DEBUG
@@ -5506,6 +5586,8 @@ LOCAL ERR ErrDBUTLIEstimateRootSpaceLeak( PIB* const ppib, const IFMP ifmp )
             rgcpgRootSpaceInfo[ 2 ],    // cpgSplitBuffers
             cCachedPrimary,
             cUncachedPrimary,
+            cEnumerationConflictsSucceeded,
+            cEnumerationConflictsFailed,
             jts,
             ulMinElapsed,
             dblSecElapsed );