fpta: update libmdbx to 0.11.2.22 (MDBX_SHRINK_ALLOWED / MDBX_ACCEDE collision).

Author: erthink

externals/libmdbx/CMakeLists.txt

@@ -434,7 +434,7 @@ endif()
 #           ####   #          #       #     ####   #    #   ####
 #
 
-set(MDBX_BUILD_OPTIONS ENABLE_ASAN MDBX_USE_VALGRIND ENABLE_GPROF ENABLE_GCOV)
+set(MDBX_BUILD_OPTIONS ENABLE_UBSAN ENABLE_ASAN MDBX_USE_VALGRIND ENABLE_GPROF ENABLE_GCOV)
 macro(add_mdbx_option NAME DESCRIPTION DEFAULT)
   list(APPEND MDBX_BUILD_OPTIONS ${NAME})
   if(NOT ${DEFAULT} STREQUAL "AUTO")
@@ -863,7 +863,7 @@ else()
 endif()
 
 # options
-set(options VERSION C_COMPILER CXX_COMPILER MDBX_BUILD_TARGET MDBX_BUILD_TYPE)
+set(options VERSION C_COMPILER CXX_COMPILER MDBX_BUILD_TARGET MDBX_BUILD_TYPE ${MDBX_BUILD_OPTIONS})
 foreach(item IN LISTS options)
   if(DEFINED ${item})
     set(value "${${item}}")
@@ -874,7 +874,7 @@ foreach(item IN LISTS options)
     set(item CMAKE_${item})
     set(value "${${item}}")
   else()
-    set(value "undefined")
+    set(value "AUTO (not pre-defined explicitly)")
   endif()
   message(STATUS "${item}: ${value}")
 endforeach(item)

externals/libmdbx/ChangeLog.md

@@ -23,19 +23,28 @@ ChangeLog
 Acknowledgements:
 
  - [gcxfd <i@rmw.link>](https://github.com/gcxfd) for reporting, contributing and testing.
+ - [장세연 (Чан Се Ен)](https://github.com/sasgas) for reporting and testing.
+
+New features, extensions and improvements:
+
+  - [Added](https://github.com/erthink/libmdbx/issues/236) `mdbx_cursor_get_batch()`.
+  - [Added](https://github.com/erthink/libmdbx/issues/250) `MDBX_SET_UPPERBOUND`.
 
 Fixes:
 
   - Disabled C++20 concepts for stupid AppleClang 13.x
+  - Fixed internal collision of `MDBX_SHRINK_ALLOWED` with `MDBX_ACCEDE`.
 
 Minors:
 
- - Fixed returning `MDBX_RESULT_TRUE` (-1) from `mdbx_env_set_option()`.
+ - Fixed returning `MDBX_RESULT_TRUE` (unexpected -1) from `mdbx_env_set_option()`.
  - Added `mdbx_env_get_syncbytes()` and `mdbx_env_get_syncperiod()`.
  - [Clarified](https://github.com/erthink/libmdbx/pull/249) description of `MDBX_INTEGERKEY`.
  - Reworked/simplified `mdbx_env_sync_internal()`.
  - [Fixed](https://github.com/erthink/libmdbx/issues/248) extra assertion inside `mdbx_cursor_put()` for `MDBX_DUPFIXED` cases.
  - Avoiding extra looping inside `mdbx_env_info_ex()`.
+ - Explicitly enabled core dumps from stochastic tests scripts on Linux.
+ - [Fixed](https://github.com/erthink/libmdbx/issues/253) `mdbx_override_meta()` to avoid false-positive assertions.
 
 
 ## v0.11.2 at 2021-12-02
@@ -106,7 +115,7 @@ Fixes:
 Minors:
 
  - Refined handling of weak or invalid meta-pages while a DB opening.
- - Refined providing information for the @MAIN and @GC sub-databases of a last committed modification transaction's ID.
+ - Refined providing information for the `@MAIN` and `@GC` sub-databases of a last committed modification transaction's ID.
 
 
 ## v0.10.4 at 2021-10-10

externals/libmdbx/VERSION.txt

@@ -1 +1 @@
-0.11.2.15
+0.11.2.22

externals/libmdbx/mdbx.c

@@ -12,7 +12,7 @@
  * <http://www.OpenLDAP.org/license.html>. */
 
 #define xMDBX_ALLOY 1
-#define MDBX_BUILD_SOURCERY c5b9a3e9ba7c98987e7554f0c84c1e12e9f82d0d470ad881d54923ecf3599c17_v0_11_2_15_gca197965
+#define MDBX_BUILD_SOURCERY eceee647cf2c0dcedb1c09cb1f7c82a14d9efc74380e3954b5a3f158d5b8c29b_v0_11_2_22_gc70d2d62
 #ifdef MDBX_CONFIG_H
 #include MDBX_CONFIG_H
 #endif
@@ -7091,7 +7091,8 @@ static void mdbx_node_del(MDBX_cursor *mc, size_t ksize);
 static void mdbx_node_shrink(MDBX_page *mp, unsigned indx);
 static int __must_check_result mdbx_node_move(MDBX_cursor *csrc,
                                               MDBX_cursor *cdst, bool fromleft);
-static int __must_check_result mdbx_node_read(MDBX_cursor *mc, MDBX_node *leaf,
+static int __must_check_result mdbx_node_read(MDBX_cursor *mc,
+                                              const MDBX_node *leaf,
                                               MDBX_val *data,
                                               const txnid_t front);
 static int __must_check_result mdbx_rebalance(MDBX_cursor *mc);
@@ -10350,7 +10351,7 @@ __cold static int mdbx_env_sync_internal(MDBX_env *env, bool force,
   int rc = MDBX_RESULT_TRUE /* means "nothing to sync" */;
 
 retry:;
-  unsigned flags = env->me_flags & ~MDBX_NOMETASYNC;
+  unsigned flags = env->me_flags & ~(MDBX_NOMETASYNC | MDBX_SHRINK_ALLOWED);
   if (unlikely((flags & (MDBX_RDONLY | MDBX_FATAL_ERROR | MDBX_ENV_ACTIVE)) !=
                MDBX_ENV_ACTIVE)) {
     rc = MDBX_EACCESS;
@@ -10434,17 +10435,19 @@ retry:;
     mdbx_assert(env, head_txnid == meta_txnid(env, head, false));
     mdbx_assert(env, head_txnid == mdbx_recent_committed_txnid(env));
     mdbx_find_oldest(env->me_txn0);
+    flags |= MDBX_SHRINK_ALLOWED;
   }
 
   mdbx_assert(env, inside_txn || locked);
+  mdbx_assert(env, !inside_txn || (flags & MDBX_SHRINK_ALLOWED) == 0);
 
   if (!META_IS_STEADY(head) ||
       ((flags & MDBX_SAFE_NOSYNC) == 0 && unsynced_pages)) {
     mdbx_debug("meta-head %" PRIaPGNO ", %s, sync_pending %" PRIaPGNO,
                data_page(head)->mp_pgno, mdbx_durable_str(head),
                unsynced_pages);
     MDBX_meta meta = *head;
-    rc = mdbx_sync_locked(env, flags | MDBX_SHRINK_ALLOWED, &meta);
+    rc = mdbx_sync_locked(env, flags, &meta);
     if (unlikely(rc != MDBX_SUCCESS))
       goto bailout;
   }
@@ -14092,6 +14095,10 @@ static int mdbx_sync_locked(MDBX_env *env, unsigned flags,
                        unaligned_peek_u64(4, pending->mm_txnid_a))) {
             const txnid_t txnid =
                 safe64_txnid_next(unaligned_peek_u64(4, pending->mm_txnid_a));
+            mdbx_notice("force-forward pending-txn %" PRIaTXN " -> %" PRIaTXN,
+                        unaligned_peek_u64(4, pending->mm_txnid_a), txnid);
+            mdbx_ensure(env, env->me_txn0->mt_owner != mdbx_thread_self() &&
+                                 !env->me_txn);
             if (unlikely(txnid > MAX_TXNID)) {
               rc = MDBX_TXN_FULL;
               mdbx_error("txnid overflow, raise %d", rc);
@@ -15681,16 +15688,12 @@ __cold static int __must_check_result mdbx_override_meta(
                     MDBX_SYNC_DATA | MDBX_SYNC_IODQ);
     if (unlikely(rc != MDBX_SUCCESS))
       return rc;
-    MDBX_meta *live = METAPAGE(env, target);
-    mdbx_meta_update_begin(env, live, unaligned_peek_u64(4, model->mm_txnid_a));
+    /* mdbx_override_meta() called only while current process have exclusive
+     * lock of a DB file. So meta-page could be updated directly without
+     * clearing consistency flag by mdbx_meta_update_begin() */
+    memcpy(pgno2page(env, target), page, env->me_psize);
     mdbx_flush_incoherent_cpu_writeback();
-    mdbx_meta_update_begin(env, model,
-                           unaligned_peek_u64(4, model->mm_txnid_a));
-    unaligned_poke_u64(4, model->mm_datasync_sign, MDBX_DATASIGN_WEAK);
-    memcpy((void *)data_page(live), page, env->me_psize);
-    mdbx_meta_update_end(env, live, unaligned_peek_u64(4, model->mm_txnid_b));
-    mdbx_flush_incoherent_cpu_writeback();
-    rc = mdbx_msync(&env->me_dxb_mmap, 0, pgno_align2os_bytes(env, target),
+    rc = mdbx_msync(&env->me_dxb_mmap, 0, pgno_align2os_bytes(env, target + 1),
                     MDBX_SYNC_DATA | MDBX_SYNC_IODQ);
   } else {
     const mdbx_filehandle_t fd = (env->me_dsync_fd != INVALID_HANDLE_VALUE)
@@ -16941,8 +16944,9 @@ __hot static int mdbx_page_search(MDBX_cursor *mc, const MDBX_val *key,
  * [out] data   Updated to point to the node's data.
  *
  * Returns 0 on success, non-zero on failure. */
-static __always_inline int mdbx_node_read(MDBX_cursor *mc, MDBX_node *node,
-                                          MDBX_val *data, const txnid_t front) {
+static __always_inline int mdbx_node_read(MDBX_cursor *mc,
+                                          const MDBX_node *node, MDBX_val *data,
+                                          const txnid_t front) {
   data->iov_len = node_ds(node);
   data->iov_base = node_data(node);
   if (unlikely(F_ISSET(node_flags(node), F_BIGDATA))) {
@@ -17848,7 +17852,7 @@ int mdbx_cursor_get(MDBX_cursor *mc, MDBX_val *key, MDBX_val *data,
     break;
   case MDBX_FIRST_DUP:
     mfunc = mdbx_cursor_first;
-  mmove:
+  move:
     if (unlikely(data == NULL || !(mc->mc_flags & C_INITIALIZED)))
       return MDBX_EINVAL;
     if (unlikely(mc->mc_xcursor == NULL))
@@ -17876,7 +17880,8 @@ int mdbx_cursor_get(MDBX_cursor *mc, MDBX_val *key, MDBX_val *data,
     break;
   case MDBX_LAST_DUP:
     mfunc = mdbx_cursor_last;
-    goto mmove;
+    goto move;
+  case MDBX_SET_UPPERBOUND: /* mostly same as MDBX_SET_LOWERBOUND */
   case MDBX_SET_LOWERBOUND: {
     if (unlikely(key == NULL || data == NULL))
       return MDBX_EINVAL;
@@ -17909,6 +17914,15 @@ int mdbx_cursor_get(MDBX_cursor *mc, MDBX_val *key, MDBX_val *data,
     }
     if (rc == MDBX_SUCCESS && !csr.exact)
       rc = MDBX_RESULT_TRUE;
+    if (unlikely(op == MDBX_SET_UPPERBOUND)) {
+      /* minor fixups for MDBX_SET_UPPERBOUND */
+      if (rc == MDBX_RESULT_TRUE)
+        /* already at great-than by MDBX_SET_LOWERBOUND */
+        rc = MDBX_SUCCESS;
+      else if (rc == MDBX_SUCCESS)
+        /* exactly match, going next */
+        rc = mdbx_cursor_next(mc, key, data, MDBX_NEXT);
+    }
     break;
   }
   default:
@@ -17920,6 +17934,122 @@ int mdbx_cursor_get(MDBX_cursor *mc, MDBX_val *key, MDBX_val *data,
   return rc;
 }
 
+static int cursor_first_batch(MDBX_cursor *mc) {
+  if (!(mc->mc_flags & C_INITIALIZED) || mc->mc_top) {
+    int err = mdbx_page_search(mc, NULL, MDBX_PS_FIRST);
+    if (unlikely(err != MDBX_SUCCESS))
+      return err;
+  }
+  mdbx_cassert(mc, IS_LEAF(mc->mc_pg[mc->mc_top]));
+
+  mc->mc_flags |= C_INITIALIZED;
+  mc->mc_flags &= ~C_EOF;
+  mc->mc_ki[mc->mc_top] = 0;
+  return MDBX_SUCCESS;
+}
+
+static int cursor_next_batch(MDBX_cursor *mc) {
+  if (unlikely(!(mc->mc_flags & C_INITIALIZED)))
+    return cursor_first_batch(mc);
+
+  MDBX_page *mp = mc->mc_pg[mc->mc_top];
+  if (unlikely(mc->mc_flags & C_EOF)) {
+    if ((unsigned)mc->mc_ki[mc->mc_top] + 1 >= page_numkeys(mp))
+      return MDBX_NOTFOUND;
+    mc->mc_flags ^= C_EOF;
+  }
+
+  int ki = mc->mc_ki[mc->mc_top];
+  mc->mc_ki[mc->mc_top] = (indx_t)++ki;
+  const int numkeys = page_numkeys(mp);
+  if (likely(ki >= numkeys)) {
+    mdbx_debug("%s", "=====> move to next sibling page");
+    mc->mc_ki[mc->mc_top] = (indx_t)(numkeys - 1);
+    int err = mdbx_cursor_sibling(mc, SIBLING_RIGHT);
+    if (unlikely(err != MDBX_SUCCESS)) {
+      mc->mc_flags |= C_EOF;
+      return err;
+    }
+    mp = mc->mc_pg[mc->mc_top];
+    mdbx_debug("next page is %" PRIaPGNO ", key index %u", mp->mp_pgno,
+               mc->mc_ki[mc->mc_top]);
+  }
+  return MDBX_SUCCESS;
+}
+
+int mdbx_cursor_get_batch(MDBX_cursor *mc, size_t *count, MDBX_val *pairs,
+                          size_t limit, MDBX_cursor_op op) {
+  if (unlikely(mc == NULL || count == NULL || limit < 4))
+    return MDBX_EINVAL;
+
+  if (unlikely(mc->mc_signature != MDBX_MC_LIVE))
+    return (mc->mc_signature == MDBX_MC_READY4CLOSE) ? MDBX_EINVAL
+                                                     : MDBX_EBADSIGN;
+
+  int rc = check_txn(mc->mc_txn, MDBX_TXN_BLOCKED);
+  if (unlikely(rc != MDBX_SUCCESS))
+    return rc;
+
+  if (unlikely(mc->mc_db->md_flags & MDBX_DUPSORT))
+    return MDBX_INCOMPATIBLE /* must be a non-dupsort subDB */;
+
+  switch (op) {
+  case MDBX_FIRST:
+    rc = cursor_first_batch(mc);
+    break;
+  case MDBX_NEXT:
+    rc = cursor_next_batch(mc);
+    break;
+  case MDBX_GET_CURRENT:
+    rc = likely(mc->mc_flags & C_INITIALIZED) ? MDBX_SUCCESS : MDBX_ENODATA;
+    break;
+  default:
+    mdbx_debug("unhandled/unimplemented cursor operation %u", op);
+    rc = EINVAL;
+    break;
+  }
+
+  if (unlikely(rc != MDBX_SUCCESS)) {
+    *count = 0;
+    return rc;
+  }
+
+  const MDBX_page *const page = mc->mc_pg[mc->mc_top];
+  const unsigned nkeys = page_numkeys(page);
+  unsigned i = mc->mc_ki[mc->mc_top], n = 0;
+  if (unlikely(i >= nkeys)) {
+    mdbx_cassert(mc, op == MDBX_GET_CURRENT);
+    mdbx_cassert(mc, mdbx_cursor_on_last(mc) == MDBX_RESULT_TRUE);
+    *count = 0;
+    if (mc->mc_flags & C_EOF) {
+      mdbx_cassert(mc, mdbx_cursor_on_last(mc) == MDBX_RESULT_TRUE);
+      return MDBX_ENODATA;
+    }
+    if (mdbx_cursor_on_last(mc) != MDBX_RESULT_TRUE)
+      return MDBX_EINVAL /* again MDBX_GET_CURRENT after MDBX_GET_CURRENT */;
+    mc->mc_flags |= C_EOF;
+    return MDBX_NOTFOUND;
+  }
+
+  const txnid_t pp_txnid = pp_txnid4chk(page, mc->mc_txn);
+  do {
+    if (unlikely(n + 2 > limit)) {
+      rc = MDBX_RESULT_TRUE;
+      break;
+    }
+    const MDBX_node *leaf = page_node(page, i);
+    get_key(leaf, &pairs[n]);
+    rc = mdbx_node_read(mc, leaf, &pairs[n + 1], pp_txnid);
+    if (unlikely(rc != MDBX_SUCCESS))
+      break;
+    n += 2;
+  } while (++i < nkeys);
+
+  mc->mc_ki[mc->mc_top] = (indx_t)i;
+  *count = n;
+  return rc;
+}
+
 static int mdbx_touch_dbi(MDBX_cursor *mc) {
   mdbx_cassert(mc, (*mc->mc_dbistate & DBI_DIRTY) == 0);
   *mc->mc_dbistate |= DBI_DIRTY;
@@ -28535,9 +28665,9 @@ __dll_export
         0,
         11,
         2,
-        15,
-        {"2021-12-09T15:54:16+03:00", "b9a59a0f8051a9596747d2cdc7b4dd9f07ffea68", "ca19796514122e7980cb42c9cf9aeccb0e544705",
-         "v0.11.2-15-gca197965"},
+        22,
+        {"2021-12-13T13:03:49+03:00", "85a340e46868508e5b9d6b93c83fec220f8a7930", "c70d2d62d1d5fe5659790703390d3fd2d4c5ff60",
+         "v0.11.2-22-gc70d2d62"},
         sourcery};
 
 __dll_export

externals/libmdbx/mdbx.c++

@@ -12,7 +12,7 @@
  * <http://www.OpenLDAP.org/license.html>. */
 
 #define xMDBX_ALLOY 1
-#define MDBX_BUILD_SOURCERY c5b9a3e9ba7c98987e7554f0c84c1e12e9f82d0d470ad881d54923ecf3599c17_v0_11_2_15_gca197965
+#define MDBX_BUILD_SOURCERY eceee647cf2c0dcedb1c09cb1f7c82a14d9efc74380e3954b5a3f158d5b8c29b_v0_11_2_22_gc70d2d62
 #ifdef MDBX_CONFIG_H
 #include MDBX_CONFIG_H
 #endif