Optionally Unique Indices and JetGetLock( Key )

Support 2 knew functions:
1) JetGetLock can take a lock on the currently made key in an index rather than just the key of the current row.
2) An optionally unique index, which, if no other actions are taken, acts like a non-unique index.  If a grbit is supplied during JetUpdate(), optionally unique indices are enforced as unique indices.
3) Includes an EFV because of the persisted bit indicating new index type.

[Substrate:ab66b31e27e10fa2f58508106cf3f4ee54c6046c]

Author: TAW

dev/ese/published/inc/jethdr.w

@@ -678,6 +678,7 @@ typedef void (JET_API *JET_SPCATCALLBACK)( _In_ const unsigned long pgno, _In_ c
 #define JET_efvIndexDeferredPopulate                        9540    //  Adds support for deferred population of indices.
 #define JET_efvReservedTags                                 9560    //  Allows adding additional reserved tags to cpage.
 #define JET_efvRBSTooSoonDeletes                            9580    //  Allows to decide if we can now perform non-revertable delete even if root page of table was moved recently by shrink or created recently.
+#define JET_efvOptionallyUniqueIndices                      9600    //  Allows creation of optionally unique indices.
 
 // Special format specifiers here
 #define JET_efvUseEngineDefault             (0x40000001)    //  Instructs the engine to use the maximal default supported Engine Format Version. (default)
@@ -4629,6 +4630,11 @@ typedef struct
 
 #define JET_bitReadLock                 0x00000001
 #define JET_bitWriteLock                0x00000002
+// end_PubEsent
+#if ( JET_VERSION >= 0x0A01 )
+#define JET_bitKeyLock                  0x00000004
+#endif // JET_VERSION >= 0x0A01
+// begin_PubEsent
 
     /* Constants for JetMove */
 
@@ -4705,6 +4711,9 @@ typedef struct
 #define JET_bitIndexDeferredPopulateCreate  0x00100000  // Only create the index, don't actually populate it.
 #define JET_bitIndexDeferredPopulateProcess 0x00200000  // Populate an index that was previously created with JET_bitIndexDeferredPopulateCreate
 #endif // JET_VERSION >= 0x0A00
+#if ( JET_VERSION >= 0x0A01 )
+#define JET_bitIndexOptionallyUnique    0x00400000  // Index uniqueness is only enforced on updates using JET_bitUpdateEnforceOptionallyUniqueIndices
+#endif // JET_VERSION >= 0x0A01
 
 // These are not persisted anywhere. These are bits used by the 'Isam layer', a simpler C#-based
 // interface to access ESE databases.
@@ -4958,6 +4967,9 @@ typedef struct
 #endif // JET_VERSION >= 0x0502
 // end_PubEsent
 #define JET_bitUpdateNoVersion                  0x00000002  //  do not create rollback or versioning information for update
+#if ( JET_VERSION >= 0x0A01 )
+#define JET_bitUpdateEnforceOptionallyUniqueIndices 0x00000004 // Enforce optionally unique indices.
+#endif // JET_VERSION >= 0x0A01
 // begin_PubEsent
 
     /* Flags for JetEscrowUpdate */

dev/ese/src/ese/bt.cxx

@@ -5161,7 +5161,7 @@ ERR ErrBTIGetReachablePageCount( FUCB* const pfucb, CPG* const pcpg )
 //  UNDONE: we don't need to latch the page at all. just create
 //          the version using the bookmark in the FUCB
 //
-ERR ErrBTLock( FUCB *pfucb, DIRLOCK dirlock )
+ERR ErrBTLock( FUCB *pfucb, DIRLOCK dirlock, BOOKMARK &bm )
 {
     Assert( dirlock == writeLock
             || dirlock == readLock );
@@ -5194,7 +5194,7 @@ ERR ErrBTLock( FUCB *pfucb, DIRLOCK dirlock )
         Call( pver->ErrVERCheckTransactionSize( pfucb->ppib ) );
         Call( pver->ErrVERModify(
                 pfucb,
-                pfucb->bmCurr,
+                bm,
                 oper,
                 &prce,
                 NULL ) );

dev/ese/src/ese/dir.cxx

@@ -1490,6 +1490,20 @@ ERR ErrDIRReplace( FUCB *pfucb, const DATA& data, DIRFLAG dirflag )
 
 //  lock record by calling BTLockRecord
 //
+ERR ErrDIRGetLock( FUCB *pfucb, DIRLOCK dirlock, BOOKMARK &bm )
+{
+    ERR     err = JET_errSuccess;
+
+    Call( ErrBTLock( pfucb, dirlock, bm ) );
+    CallS( err );
+
+    return err;
+
+HandleError:
+    Assert( !Pcsr( pfucb )->FLatched() );
+    return err;
+}
+
 ERR ErrDIRGetLock( FUCB *pfucb, DIRLOCK dirlock )
 {
     ERR     err = JET_errSuccess;
@@ -1519,10 +1533,10 @@ ERR ErrDIRGetLock( FUCB *pfucb, DIRLOCK dirlock )
                     pfucb->locLogical == locAfterLast ||
                     pfucb->locLogical == locBeforeFirst ||
                     pfucb->locLogical == locOnSeekBM );
-            return( ErrERRCheck( JET_errNoCurrentRecord ) );
+            Error( ErrERRCheck( JET_errNoCurrentRecord ) );
     }
 
-    Call( ErrBTLock( pfucb, dirlock ) );
+    Call( ErrBTLock( pfucb, dirlock, pfucb->bmCurr ) );
     CallS( err );
 
     return err;

dev/ese/src/ese/fcreate.cxx

@@ -1440,6 +1440,7 @@ LOCAL ERR ErrFILEIValidateCreateIndex(
     const BOOL                      fConditional        = ( pidxcreate->cConditionalColumn > 0 );
     const BOOL                      fPrimary            = ( grbit & JET_bitIndexPrimary );
     const BOOL                      fDeferredPopulate   = ( grbit & JET_bitIndexDeferredPopulateCreate );
+    const BOOL                      fOptionallyUnique   = ( grbit & JET_bitIndexOptionallyUnique );
     const BOOL                      fUnique             = ( grbit & ( JET_bitIndexUnique|JET_bitIndexPrimary ) );
     const BOOL                      fDisallowNull       = ( grbit & JET_bitIndexDisallowNull );
     const BOOL                      fIgnoreNull         = ( grbit & JET_bitIndexIgnoreNull );
@@ -1451,9 +1452,9 @@ LOCAL ERR ErrFILEIValidateCreateIndex(
     const BOOL                      fCustomTupleLimits  = ( grbit & JET_bitIndexTupleLimits );
     const BOOL                      fTuples             = ( ( grbit & JET_bitIndexTuples )
                                                             || fCustomTupleLimits );
-    const BOOL                      fCrossProduct   = ( grbit & JET_bitIndexCrossProduct );
-    const BOOL                      fNestedTable    = ( grbit & JET_bitIndexNestedTable );
-    const BOOL                      fDotNetGuid     = ( grbit & JET_bitIndexDotNetGuid );
+    const BOOL                      fCrossProduct       = ( grbit & JET_bitIndexCrossProduct );
+    const BOOL                      fNestedTable        = ( grbit & JET_bitIndexNestedTable );
+    const BOOL                      fDotNetGuid         = ( grbit & JET_bitIndexDotNetGuid );
     const BOOL                      fKeyMost            = ( grbit & JET_bitIndexKeyMost );
     const BOOL                      fDisallowTruncation = ( grbit & JET_bitIndexDisallowTruncation );
 
@@ -1462,6 +1463,12 @@ LOCAL ERR ErrFILEIValidateCreateIndex(
     USHORT                          cbKeyMost           = 0;
     USHORT                          cbVarSegMac         = 0;
 
+    // grbits that rely on an EFV
+    if ( fOptionallyUnique )
+    {
+        CallR( g_rgfmp[ ifmp ].ErrDBFormatFeatureEnabled( JET_efvOptionallyUniqueIndices ) );
+    }
+
     //  if index has custom key size set then set in IDB
     //  otherwise default to historic maximum key size of 255 bytes
     //
@@ -1499,6 +1506,13 @@ LOCAL ERR ErrFILEIValidateCreateIndex(
         return err;
     }
 
+    // Optionally Unique is only allowed on non-unique secondary indices.
+    if ( fOptionallyUnique && ( fPrimary || fUnique ) )
+    {
+        err = ErrERRCheck( JET_errInvalidGrbit );
+        return err;
+    }
+
     //  do not allow both linear and cross product
     //
     if ( fCrossProduct && fNestedTable )
@@ -1777,6 +1791,11 @@ LOCAL ERR ErrFILEIValidateCreateIndex(
         pidb->SetFDeferredPopulate();
     }
 
+    if ( fOptionallyUnique )
+    {
+        pidb->SetFOptionallyUnique();
+    }
+
     //  not both linear and cross product
     //
     Assert( !( fCrossProduct && fNestedTable ) );
@@ -9554,6 +9573,7 @@ VOID IDB::SetFlagsFromGrbit( const JET_GRBIT grbit )
     const BOOL  fDotNetGuid         = ( grbit & JET_bitIndexDotNetGuid );
     const BOOL  fDisallowTruncation = ( grbit & JET_bitIndexDisallowTruncation );
     const BOOL  fDeferredPopulate   = ( grbit & JET_bitIndexDeferredPopulateCreate );
+    const BOOL  fOptionallyUnique   = ( grbit & JET_bitIndexOptionallyUnique );
 
     ResetFlags();
 
@@ -9597,6 +9617,11 @@ VOID IDB::SetFlagsFromGrbit( const JET_GRBIT grbit )
         SetFDeferredPopulate();
     }
 
+    if ( fOptionallyUnique )
+    {
+        SetFOptionallyUnique();
+    }
+
     //  not both linear and cross product
     //
     Assert( !( fCrossProduct && fNestedTable ) );

dev/ese/src/ese/rec.cxx

@@ -38,6 +38,7 @@ ERR VTAPI ErrIsamGetLock( JET_SESID sesid, JET_VTID vtid, JET_GRBIT grbit )
 {
     PIB     *ppib = reinterpret_cast<PIB *>( sesid );
     FUCB    *pfucb = reinterpret_cast<FUCB *>( vtid );
+    DIRLOCK dirlock;
     ERR     err;
 
     CallR( ErrPIBCheck( ppib ) );
@@ -47,25 +48,71 @@ ERR VTAPI ErrIsamGetLock( JET_SESID sesid, JET_VTID vtid, JET_GRBIT grbit )
     if ( ppib->Level() <= 0 )
         return ErrERRCheck( JET_errNotInTransaction );
 
-    if ( JET_bitReadLock == grbit )
+    switch ( grbit & ( JET_bitWriteLock | JET_bitReadLock ) )
     {
-        Call( ErrDIRGetLock( pfucb, readLock ) );
+        case JET_bitReadLock:
+            dirlock = readLock;
+            break;
+
+        case JET_bitWriteLock:
+            //  ensure that table is updatable
+            //
+            CallR( ErrFUCBCheckUpdatable( pfucb )  );
+            if ( !FFMPIsTempDB( pfucb->ifmp ) )
+            {
+                CallR( ErrPIBCheckUpdatable( ppib ) );
+            }
+            dirlock = writeLock;
+            break;
+
+        default:
+            Error( ErrERRCheck( JET_errInvalidGrbit ));
+            break;
     }
-    else if ( JET_bitWriteLock == grbit )
+
+    if ( grbit & JET_bitKeyLock )
     {
-        //  ensure that table is updatable
-        //
-        CallR( ErrFUCBCheckUpdatable( pfucb )  );
-        if ( !FFMPIsTempDB( pfucb->ifmp ) )
+        FUCB     *pfucbUsed;
+        BOOKMARK bm;
+
+        if ( pfucbNil == pfucb->pfucbCurIndex )
+        {
+            // We're using the primary index.
+            pfucbUsed = pfucb;
+            Assert( pfucbUsed->u.pfcb->FPrimaryIndex() );
+            Assert( pfucbUsed->u.pfcb->Pidb() == pidbNil
+                    || pfucbUsed->u.pfcb->Pidb()->FPrimary() );
+        }
+        else
         {
-            CallR( ErrPIBCheckUpdatable( ppib ) );
+            // We're using a 2ndary index.
+            pfucbUsed = pfucb->pfucbCurIndex;
+            Assert( pfucbUsed->u.pfcb->FTypeSecondaryIndex() );
+            Assert( pfucbUsed->u.pfcb->Pidb() != pidbNil );
+            Assert( !pfucbUsed->u.pfcb->Pidb()->FPrimary() );
         }
 
-        Call( ErrDIRGetLock( pfucb, writeLock ) );
+        // Caller wants us to lock based on the search key.
+        if ( !FKSPrepared( pfucbUsed ) )
+        {
+            Error( ErrERRCheck( JET_errKeyNotMade ) );
+        }
+
+        if ( pfucbUsed->dataSearchKey.FNull() ||
+             NULL == pfucbUsed->dataSearchKey.Pv() )
+        {
+            Expected( fFalse );
+            Error( ErrERRCheck( JET_errKeyNotMade ) );
+        }
+
+        bm.Nullify();
+        bm.key.suffix = pfucbUsed->dataSearchKey;
+
+        Call( ErrDIRGetLock( pfucbUsed, dirlock, bm ) );
     }
     else
     {
-        err = ErrERRCheck( JET_errInvalidGrbit );
+        Call( ErrDIRGetLock( pfucb, dirlock ) );
     }
 
 HandleError:

dev/ese/src/ese/recupd.cxx

@@ -956,6 +956,11 @@ ERR VTAPI ErrIsamUpdate(
     CheckTable( ppib, pfucb );
     CheckSecondary( pfucb );
 
+    if ( grbit & JET_bitUpdateEnforceOptionallyUniqueIndices )
+    {
+        ppib->SetFEnforceOptionallyUniqueIndices();
+    }
+
     if ( FFUCBReplacePrepared( pfucb ) )
     {
         BOOKMARK *pbm;
@@ -997,6 +1002,8 @@ ERR VTAPI ErrIsamUpdate(
         RECIFreeCopyBuffer( pfucb );
     }
 
+    ppib->ResetFEnforceOptionallyUniqueIndices();
+
     AssertDIRMaybeNoLatch( ppib, pfucb );
     Assert( err != JET_errNoCurrentRecord );
     return err;
@@ -2055,6 +2062,63 @@ ERR ErrRECIInsertIndexEntry(
 
     PERFOpt( PERFIncCounterTable( cRECIndexInserts, PinstFromPfucb( pfucbIdx ), TceFromFUCB( pfucbIdx ) ) );
 
+    if ( pidb->FOptionallyUnique() && pfucbIdx->ppib->FEnforceOptionallyUniqueIndices() )
+    {
+        DIB      dib;
+        BOOKMARK bm;
+
+        bm.Nullify();
+        bm.key = keyToInsert;
+        dib.pos = posDown;
+        dib.pbm = &bm;
+        dib.dirflag = fDIRExact;
+
+        // First, globally lock the key so no one else can insert it.
+        // Note: this lock is automatically released if the JetUpdate this call is part of
+        // fails.
+        Call( ErrDIRGetLock( pfucbIdx, writeLock, bm ) );
+
+        // Now, make sure the key is not already in use somewhere.
+        err = ErrDIRDown( pfucbIdx, &dib );
+
+        // Massage some internal "warnings".  fDIRExact does not always result in the error code
+        // you'd expect.
+        switch ( err )
+        {
+            case wrnNDFoundLess:
+            case wrnNDFoundGreater:
+                // Although we landed on something, it wasn't what we wanted.
+                DIRUp( pfucbIdx );
+                err = ErrERRCheck( JET_errRecordNotFound );
+                break;
+
+            default:
+                break;
+        }
+
+        // Ignore warnings.
+        switch ( min( err, JET_errSuccess ) )
+        {
+            case JET_errSuccess:
+                // But, report any warnings.
+                CallS( err );
+
+                // There's already something with this key.
+                Error( ErrERRCheck( JET_errKeyDuplicate ) );
+
+            case JET_errNoCurrentRecord:
+            case JET_errRecordNotFound:
+                // Nope, not already in use.
+                err = JET_errSuccess;
+                break;
+
+            default:
+                // We don't really expect any other errors.
+                Assert( fFalse );
+                Error( err );
+        }
+    }
+
     err = ErrDIRInsert(
         pfucbIdx,
         keyToInsert,

dev/ese/src/ese/sysver.cxx

@@ -96,6 +96,7 @@ constexpr FormatVersions g_rgfmtversEngine[] = {
     { JET_efvIndexDeferredPopulate, /* 9540 */          { 1568,260,540 }, { 8,120,260 }, { 3,0,0 } }, // [2022/03/02]  
     { JET_efvReservedTags, /* 9560 */                   { 1568,270,560 }, { 8,120,260 }, { 3,0,0 } }, // [2022/06/17]
     { JET_efvRBSTooSoonDeletes, /* 9580 */              { 1568,280,580 }, { 8,130,280 }, { 3,0,0 } }, // [2022/07/14]
+    { JET_efvOptionallyUniqueIndices, /* 9600 */        { 1568,290,600 }, { 8,130,280 }, { 3,0,0 } }, // [2022/07/14]
 };
 
 constexpr INT g_cfmtversEngine = std::size( g_rgfmtversEngine );

dev/ese/src/inc/bt.hxx

@@ -159,7 +159,7 @@ ERR ErrBTContainsPage( FUCB* const pfucb, const BOOKMARK& bm, const PGNO pgno, c
 //  **************************************
 //  update operations
 //
-ERR ErrBTLock( FUCB *pfucb, DIRLOCK dirlock );
+ERR ErrBTLock( FUCB *pfucb, DIRLOCK dirlock, BOOKMARK& bookmark );
 ERR ErrBTReplace( FUCB * const pfucb, const DATA& data, const DIRFLAG dirflags );
 
 template< typename TDelta >

dev/ese/src/inc/dir.hxx

@@ -189,7 +189,8 @@ ERR ErrDIRTermAppend( FUCB *pfucb );
 ERR ErrDIRDelete( FUCB *pfucb, DIRFLAG dirflag, RCE *prcePrimary = prceNil );
 
 ERR ErrDIRReplace( FUCB *pfucb, const DATA& data, DIRFLAG dirflag );
-ERR ErrDIRGetLock( FUCB *pfucb, DIRLOCK dirlock );
+ERR ErrDIRGetLock( FUCB *pfucb, DIRLOCK dirlock );                      // Pre-existing default is to use bmCurr.  Use an overload
+ERR ErrDIRGetLock( FUCB *pfucb, DIRLOCK dirlock, BOOKMARK& bmToLock );  //         to allow supplying bookmark other than that.
 
 template< typename TDelta >
 ERR ErrDIRDelta(

dev/ese/src/inc/idb.hxx

@@ -71,6 +71,7 @@ typedef USHORT  IDXFLAG;
 const IDXFLAG   fIDXExtendedColumns  = 0x0001;   // IDXSEGs are comprised of JET_COLUMNIDs, not FIDs
 const IDXFLAG   fIDXDotNetGuid       = 0x0002;   // GUIDs sort according to .Net rules
 const IDXFLAG   fIDXDeferredPopulate = 0x0004;   // Index is not completely populated and so is not normally usable.
+const IDXFLAG   fIDXOptionallyUnique = 0x0008;   // Index can be used to enforce uniqueness, but doesn't require it.
 
 INLINE BOOL FIDXExtendedColumns( const IDXFLAG idxflag )        { return ( idxflag & fIDXExtendedColumns ); }
 INLINE BOOL FIDXDotNetGUID( const IDXFLAG idxflag )     { return ( idxflag & fIDXDotNetGuid ); }
@@ -552,6 +553,10 @@ class IDB
         INLINE VOID SetFDeferredPopulate()              { m_fidxPersisted |= fIDXDeferredPopulate; }
         INLINE VOID ResetFDeferredPopulate()            { m_fidxPersisted &= ~fIDXDeferredPopulate; }
 
+        INLINE BOOL FOptionallyUnique() const           { return ( m_fidxPersisted & fIDXOptionallyUnique ); }
+        INLINE VOID SetFOptionallyUnique()              { m_fidxPersisted |= fIDXOptionallyUnique; }
+        INLINE VOID ResetFOptionallyUnique()            { m_fidxPersisted &= ~fIDXOptionallyUnique; }
+
         // Whether the user specified a locale when creating the index. The default locale may
         // still be used and persisted, but FLocaleSet() will be false in that case.
         // Also see FLocalizedText().