Merge remote-tracking branch 'origin/mdb.RE/0.9' into OPENLDAP_REL_ENG_2_4

libraries/liblmdb/CHANGES

 LMDB 0.9 Change Log
 
+LMDB 0.9.29 Release (2021/03/16)
+	ITS#9461 refix ITS#9376
+	ITS#9500 fix regression from ITS#8662
+
+LMDB 0.9.28 Release (2021/02/04)
+	ITS#8662 add -a append option to mdb_load
+
 LMDB 0.9.27 Release (2020/10/26)
 	ITS#9376 fix repeated DUPSORT cursor deletes
 

libraries/liblmdb/lmdb.h

@@ -200,7 +200,7 @@ typedef int mdb_filehandle_t;
 /** Library minor version */
 #define MDB_VERSION_MINOR	9
 /** Library patch version */
-#define MDB_VERSION_PATCH	27
+#define MDB_VERSION_PATCH	29
 
 /** Combine args a,b,c into a single integer for easy version comparisons */
 #define MDB_VERINT(a,b,c)	(((a) << 24) | ((b) << 16) | (c))
@@ -210,7 +210,7 @@ typedef int mdb_filehandle_t;
 	MDB_VERINT(MDB_VERSION_MAJOR,MDB_VERSION_MINOR,MDB_VERSION_PATCH)
 
 /** The release date of this library version */
-#define MDB_VERSION_DATE	"October 26, 2020"
+#define MDB_VERSION_DATE	"March 16, 2021"
 
 /** A stringifier for the version info */
 #define MDB_VERSTR(a,b,c,d)	"LMDB " #a "." #b "." #c ": (" d ")"
@@ -576,7 +576,7 @@ int  mdb_env_create(MDB_env **env);
 	 *	<li>#MDB_NOTLS
 	 *		Don't use Thread-Local Storage. Tie reader locktable slots to
 	 *		#MDB_txn objects instead of to threads. I.e. #mdb_txn_reset() keeps
-	 *		the slot reseved for the #MDB_txn object. A thread may use parallel
+	 *		the slot reserved for the #MDB_txn object. A thread may use parallel
 	 *		read-only transactions. A read-only transaction may span threads if
 	 *		the user synchronizes its use. Applications that multiplex many
 	 *		user threads over individual OS threads need this option. Such an
@@ -928,7 +928,7 @@ void *mdb_env_get_userctx(MDB_env *env);
 typedef void MDB_assert_func(MDB_env *env, const char *msg);
 
 	/** Set or reset the assert() callback of the environment.
-	 * Disabled if liblmdb is buillt with NDEBUG.
+	 * Disabled if liblmdb is built with NDEBUG.
 	 * @note This hack should become obsolete as lmdb's error handling matures.
 	 * @param[in] env An environment handle returned by #mdb_env_create().
 	 * @param[in] func An #MDB_assert_func function, or 0.

libraries/liblmdb/mdb.c

@@ -493,7 +493,7 @@ static txnid_t mdb_debug_start;
 	 *	The string is printed literally, with no format processing.
 	 */
 #define DPUTS(arg)	DPRINTF(("%s", arg))
-	/** Debuging output value of a cursor DBI: Negative in a sub-cursor. */
+	/** Debugging output value of a cursor DBI: Negative in a sub-cursor. */
 #define DDBI(mc) \
 	(((mc)->mc_flags & C_SUB) ? -(int)(mc)->mc_dbi : (int)(mc)->mc_dbi)
 /** @} */
@@ -6597,7 +6597,7 @@ mdb_cursor_put(MDB_cursor *mc, MDB_val *key, MDB_val *data,
 
 	dkey.mv_size = 0;
 
-	if (flags == MDB_CURRENT) {
+	if (flags & MDB_CURRENT) {
 		if (!(mc->mc_flags & C_INITIALIZED))
 			return EINVAL;
 		rc = MDB_SUCCESS;
@@ -6992,7 +6992,7 @@ put_sub:
 			xdata.mv_size = 0;
 			xdata.mv_data = "";
 			leaf = NODEPTR(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top]);
-			if (flags & MDB_CURRENT) {
+			if ((flags & (MDB_CURRENT|MDB_APPENDDUP)) == MDB_CURRENT) {
 				xflags = MDB_CURRENT|MDB_NOSPILL;
 			} else {
 				mdb_xcursor_init1(mc, leaf);
@@ -8481,7 +8481,6 @@ mdb_cursor_del0(MDB_cursor *mc)
 		return rc;
 	}
 
-	ki = mc->mc_ki[mc->mc_top];
 	mp = mc->mc_pg[mc->mc_top];
 	nkeys = NUMKEYS(mp);
 
@@ -8493,19 +8492,18 @@ mdb_cursor_del0(MDB_cursor *mc)
 		if (m3->mc_snum < mc->mc_snum)
 			continue;
 		if (m3->mc_pg[mc->mc_top] == mp) {
+			if (m3->mc_ki[mc->mc_top] >= mc->mc_ki[mc->mc_top]) {
 			/* if m3 points past last node in page, find next sibling */
-			if (m3->mc_ki[mc->mc_top] >= nkeys) {
-				rc = mdb_cursor_sibling(m3, 1);
-				if (rc == MDB_NOTFOUND) {
-					m3->mc_flags |= C_EOF;
-					rc = MDB_SUCCESS;
-					continue;
+				if (m3->mc_ki[mc->mc_top] >= nkeys) {
+					rc = mdb_cursor_sibling(m3, 1);
+					if (rc == MDB_NOTFOUND) {
+						m3->mc_flags |= C_EOF;
+						rc = MDB_SUCCESS;
+						continue;
+					}
+					if (rc)
+						goto fail;
 				}
-				if (rc)
-					goto fail;
-			}
-			if (m3->mc_ki[mc->mc_top] >= ki ||
-				/* moved to right sibling */ m3->mc_pg[mc->mc_top] != mp) {
 				if (m3->mc_xcursor && !(m3->mc_flags & C_EOF)) {
 					MDB_node *node = NODEPTR(m3->mc_pg[m3->mc_top], m3->mc_ki[m3->mc_top]);
 					/* If this node has dupdata, it may need to be reinited
@@ -8527,10 +8525,10 @@ mdb_cursor_del0(MDB_cursor *mc)
 					}
 					m3->mc_xcursor->mx_cursor.mc_flags |= C_DEL;
 				}
-				m3->mc_flags |= C_DEL;
 			}
 		}
 	}
+	mc->mc_flags |= C_DEL;
 
 fail:
 	if (rc)

libraries/liblmdb/mdb_load.1

@@ -37,6 +37,13 @@ option below.
 .BR \-V
 Write the library version number to the standard output, and exit.
 .TP
+.BR \-a
+Append all records in the order they appear in the input. The input is assumed to already be
+in correctly sorted order and no sorting or checking for redundant values will be performed.
+This option must be used to reload data that was produced by running
+.B mdb_dump
+on a database that uses custom compare functions.
+.TP
 .BR \-f \ file
 Read from the specified file instead of from the standard input.
 .TP

libraries/liblmdb/mdb_load.c

@@ -37,6 +37,7 @@ static int Eof;
 static MDB_envinfo info;
 
 static MDB_val kbuf, dbuf;
+static MDB_val k0buf;
 
 #ifdef _WIN32
 #define Z	"I"
@@ -278,10 +279,15 @@ badend:
 
 static void usage(void)
 {
-	fprintf(stderr, "usage: %s [-V] [-f input] [-n] [-s name] [-N] [-T] dbpath\n", prog);
+	fprintf(stderr, "usage: %s [-V] [-a] [-f input] [-n] [-s name] [-N] [-T] dbpath\n", prog);
 	exit(EXIT_FAILURE);
 }
 
+static int greater(const MDB_val *a, const MDB_val *b)
+{
+	return 1;
+}
+
 int main(int argc, char *argv[])
 {
 	int i, rc;
@@ -290,8 +296,9 @@ int main(int argc, char *argv[])
 	MDB_cursor *mc;
 	MDB_dbi dbi;
 	char *envname;
-	int envflags = 0, putflags = 0;
-	int dohdr = 0;
+	int envflags = MDB_NOSYNC, putflags = 0;
+	int dohdr = 0, append = 0;
+	MDB_val prevk;
 
 	prog = argv[0];
 
@@ -299,19 +306,23 @@ int main(int argc, char *argv[])
 		usage();
 	}
 
-	/* -f: load file instead of stdin
+	/* -a: append records in input order
+	 * -f: load file instead of stdin
 	 * -n: use NOSUBDIR flag on env_open
 	 * -s: load into named subDB
 	 * -N: use NOOVERWRITE on puts
 	 * -T: read plaintext
 	 * -V: print version and exit
 	 */
-	while ((i = getopt(argc, argv, "f:ns:NTV")) != EOF) {
+	while ((i = getopt(argc, argv, "af:ns:NTV")) != EOF) {
 		switch(i) {
 		case 'V':
 			printf("%s\n", MDB_VERSION_STRING);
 			exit(0);
 			break;
+		case 'a':
+			append = 1;
+			break;
 		case 'f':
 			if (freopen(optarg, "r", stdin) == NULL) {
 				fprintf(stderr, "%s: %s: reopen: %s\n",
@@ -370,11 +381,16 @@ int main(int argc, char *argv[])
 	}
 
 	kbuf.mv_size = mdb_env_get_maxkeysize(env) * 2 + 2;
-	kbuf.mv_data = malloc(kbuf.mv_size);
+	kbuf.mv_data = malloc(kbuf.mv_size * 2);
+	k0buf.mv_size = kbuf.mv_size;
+	k0buf.mv_data = (char *)kbuf.mv_data + kbuf.mv_size;
+	prevk.mv_data = k0buf.mv_data;
 
 	while(!Eof) {
 		MDB_val key, data;
 		int batch = 0;
+		flags = 0;
+		int appflag;
 
 		if (!dohdr) {
 			dohdr = 1;
@@ -392,6 +408,12 @@ int main(int argc, char *argv[])
 			fprintf(stderr, "mdb_open failed, error %d %s\n", rc, mdb_strerror(rc));
 			goto txn_abort;
 		}
+		prevk.mv_size = 0;
+		if (append) {
+			mdb_set_compare(txn, dbi, greater);
+			if (flags & MDB_DUPSORT)
+				mdb_set_dupsort(txn, dbi, greater);
+		}
 
 		rc = mdb_cursor_open(txn, dbi, &mc);
 		if (rc) {
@@ -410,7 +432,20 @@ int main(int argc, char *argv[])
 				goto txn_abort;
 			}
 
-			rc = mdb_cursor_put(mc, &key, &data, putflags);
+			if (append) {
+				appflag = MDB_APPEND;
+				if (flags & MDB_DUPSORT) {
+					if (prevk.mv_size == key.mv_size && !memcmp(prevk.mv_data, key.mv_data, key.mv_size))
+						appflag = MDB_CURRENT|MDB_APPENDDUP;
+					else {
+						memcpy(prevk.mv_data, key.mv_data, key.mv_size);
+						prevk.mv_size = key.mv_size;
+					}
+				}
+			} else {
+				appflag = 0;
+			}
+			rc = mdb_cursor_put(mc, &key, &data, putflags|appflag);
 			if (rc == MDB_KEYEXIST && putflags)
 				continue;
 			if (rc) {
@@ -435,6 +470,10 @@ int main(int argc, char *argv[])
 					fprintf(stderr, "mdb_cursor_open failed, error %d %s\n", rc, mdb_strerror(rc));
 					goto txn_abort;
 				}
+				if (appflag & MDB_APPENDDUP) {
+					MDB_val k, d;
+					mdb_cursor_get(mc, &k, &d, MDB_LAST);
+				}
 				batch = 0;
 			}
 		}