Merge remote branch 'origin/mdb.master' into OPENLDAP_REL_ENG_2_4

libraries/libmdb/.gitignore

 mtest
 mtest[23456]
 testdb
+mdb_copy
 mdb_stat
 *.[ao]
 *.so

libraries/libmdb/Makefile

@@ -5,7 +5,7 @@ CFLAGS	= -pthread $(OPT) $(W) $(XCFLAGS)
 LDLIBS	=
 SOLIBS	=
 
-PROGS	= mdb_stat mtest mtest2 mtest3 mtest4 mtest5
+PROGS	= mdb_stat mdb_copy mtest mtest2 mtest3 mtest4 mtest5
 all:	libmdb.a libmdb.so $(PROGS)
 
 clean:
@@ -22,6 +22,7 @@ libmdb.so:	mdb.o midl.o
 	gcc -pthread -shared -o $@ mdb.o midl.o $(SOLIBS)
 
 mdb_stat: mdb_stat.o libmdb.a
+mdb_copy: mdb_copy.o libmdb.a
 mtest:    mtest.o    libmdb.a
 mtest2:	mtest2.o libmdb.a
 mtest3:	mtest3.o libmdb.a

libraries/libmdb/mdb.h

@@ -38,6 +38,63 @@
  *	corrupt the database. Of course if your application code is known to
  *	be bug-free (...) then this is not an issue.
  *
+ *	Troubleshooting the lock file, plus semaphores on BSD systems:
+ *
+ *	- A broken lockfile can cause sync issues.
+ *	  Stale reader transactions left behind by an aborted program
+ *	  cause further writes to grow the database quickly, and
+ *	  stale locks can block further operation.
+ *
+ *	  Fix: Terminate all programs using the database, or make
+ *	  them close it.  Next database user will reset the lockfile.
+ *
+ *	- On BSD systems or others configured with MDB_USE_POSIX_SEM,
+ *	  startup can fail due to semaphores owned by another userid.
+ *
+ *	  Fix: Open and close the database as the user which owns the
+ *	  semaphores (likely last user) or as root, while no other
+ *	  process is using the database.
+ *
+ *	Restrictions/caveats (in addition to those listed for some functions):
+ *
+ *	- Only the database owner should normally use the database on
+ *	  BSD systems or when otherwise configured with MDB_USE_POSIX_SEM.
+ *	  Multiple users can cause startup to fail later, as noted above.
+ *
+ *	- A thread can only use one transaction at a time, plus any child
+ *	  transactions.  Each transaction belongs to one thread.  See below.
+ *
+ *	- Use an MDB_env* in the process which opened it, without fork()ing.
+ *
+ *	- Do not have open an MDB database twice in the same process at
+ *	  the same time.  Not even from a plain open() call - close()ing it
+ *	  breaks flock() advisory locking.
+ *
+ *	- Avoid long-lived transactions.  Read transactions prevent
+ *	  reuse of pages freed by newer write transactions, thus the
+ *	  database can grow quickly.  Write transactions prevent
+ *	  other write transactions, since writes are serialized.
+ *
+ *	...when several processes can use a database concurrently:
+ *
+ *	- Avoid suspending a process with active transactions.  These
+ *	  would then be "long-lived" as above.
+ *
+ *	- Avoid aborting a process with an active transaction.
+ *	  The transaction becomes "long-lived" as above until the lockfile
+ *	  is reset, since the process may not remove it from the lockfile.
+ *
+ *	- If you do that anyway, close the environment once in a while,
+ *	  so the lockfile can get reset.
+ *
+ *	- Do not use MDB databases on remote filesystems, even between
+ *	  processes on the same host.  This breaks flock() on some OSes,
+ *	  possibly memory map sync, and certainly sync between programs
+ *	  on different hosts.
+ *
+ *	- Opening a database can fail if another process is opening or
+ *	  closing it at exactly the same time.
+ *
  *	@author	Howard Chu, Symas Corporation.
  *
  *	@copyright Copyright 2011-2012 Howard Chu, Symas Corp. All rights reserved.
@@ -301,6 +358,16 @@ typedef struct MDB_stat {
 	size_t		ms_entries;			/**< Number of data items */
 } MDB_stat;
 
+/** @brief Information about the environment */
+typedef struct MDB_envinfo {
+	void	*me_mapaddr;			/**< Address of map, if fixed */
+	size_t	me_mapsize;				/**< Size of the data memory map */
+	size_t	me_last_pgno;			/**< ID of the last used page */
+	size_t	me_last_txnid;			/**< ID of the last committed transaction */
+	unsigned int me_maxreaders;		/**< maximum number of threads for the environment */
+	unsigned int me_numreaders;		/**< maximum number of threads used in the environment */
+} MDB_envinfo;
+
 	/** @brief Return the mdb library version information.
 	 *
 	 * @param[out] major if non-NULL, the library major version number is copied here
@@ -344,6 +411,7 @@ int  mdb_env_create(MDB_env **env);
 	 * @param[in] flags Special options for this environment. This parameter
 	 * must be set to 0 or by bitwise OR'ing together one or more of the
 	 * values described here.
+	 * Flags set by mdb_env_set_flags() are also used.
 	 * <ul>
 	 *	<li>#MDB_FIXEDMAP
 	 *      use a fixed address for the mmap region. This flag must be specified
@@ -393,6 +461,18 @@ int  mdb_env_create(MDB_env **env);
 	 */
 int  mdb_env_open(MDB_env *env, const char *path, unsigned int flags, mode_t mode);
 
+	/** @brief Copy an MDB environment to the specified path.
+	 *
+	 * This function may be used to make a backup of an existing environment.
+	 * @param[in] env An environment handle returned by #mdb_env_create(). It
+	 * must have already been opened successfully.
+	 * @param[in] path The directory in which the copy will reside. This
+	 * directory must already exist and be writable but must otherwise be
+	 * empty.
+	 * @return A non-zero error value on failure and 0 on success.
+	 */
+int  mdb_env_copy(MDB_env *env, const char *path);
+
 	/** @brief Return statistics about the MDB environment.
 	 *
 	 * @param[in] env An environment handle returned by #mdb_env_create()
@@ -401,6 +481,14 @@ int  mdb_env_open(MDB_env *env, const char *path, unsigned int flags, mode_t mod
 	 */
 int  mdb_env_stat(MDB_env *env, MDB_stat *stat);
 
+	/** @brief Return information about the MDB environment.
+	 *
+	 * @param[in] env An environment handle returned by #mdb_env_create()
+	 * @param[out] stat The address of an #MDB_envinfo structure
+	 * 	where the information will be copied
+	 */
+int  mdb_env_info(MDB_env *env, MDB_envinfo *stat);
+
 	/** @brief Flush the data buffers to disk.
 	 *
 	 * Data is always written to disk when #mdb_txn_commit() is called,
@@ -432,7 +520,7 @@ void mdb_env_close(MDB_env *env);
 
 	/** @brief Set environment flags.
 	 *
-	 * This may be used to set some flags that weren't already set during
+	 * This may be used to set some flags in addition to those from
 	 * #mdb_env_open(), or to unset these flags.
 	 * @param[in] env An environment handle returned by #mdb_env_create()
 	 * @param[in] flags The flags to change, bitwise OR'ed together

libraries/libmdb/mdb_copy.c

+/* mdb_copy.c - memory-mapped database backup tool */
+/*
+ * Copyright 2012 Howard Chu, Symas Corp.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted only as authorized by the OpenLDAP
+ * Public License.
+ *
+ * A copy of this license is available in the file LICENSE in the
+ * top-level directory of the distribution or, alternatively, at
+ * <http://www.OpenLDAP.org/license.html>.
+ */
+#include <stdio.h>
+#include <stdlib.h>
+#include <time.h>
+#include "mdb.h"
+
+int main(int argc,char * argv[])
+{
+	int rc;
+	MDB_env *env;
+	char *envname = argv[1];
+
+	if (argc != 3) {
+		fprintf(stderr, "usage: %s srcpath dstpath\n", argv[0]);
+		exit(EXIT_FAILURE);
+	}
+
+	rc = mdb_env_create(&env);
+
+	rc = mdb_env_open(env, envname, MDB_RDONLY, 0);
+	if (rc) {
+		printf("mdb_env_open failed, error %d %s\n", rc, mdb_strerror(rc));
+	} else {
+		rc = mdb_env_copy(env, argv[2]);
+		if (rc)
+			printf("mdb_env_copy failed, error %d %s\n", rc, mdb_strerror(rc));
+	}
+	mdb_env_close(env);
+
+	return rc ? EXIT_FAILURE : EXIT_SUCCESS;
+}

libraries/libmdb/mdb_stat.c

@@ -13,49 +13,179 @@
  */
 #include <stdio.h>
 #include <stdlib.h>
-#include <time.h>
+#include <string.h>
+#include <unistd.h>
 #include "mdb.h"
 
-int main(int argc,char * argv[])
+static void prstat(MDB_stat *ms)
 {
-	int rc;
+#if 0
+	printf("  Page size: %u\n", ms->ms_psize);
+#endif
+	printf("  Tree depth: %u\n", ms->ms_depth);
+	printf("  Branch pages: %zu\n", ms->ms_branch_pages);
+	printf("  Leaf pages: %zu\n", ms->ms_leaf_pages);
+	printf("  Overflow pages: %zu\n", ms->ms_overflow_pages);
+	printf("  Entries: %zu\n", ms->ms_entries);
+}
+
+static void usage(char *prog)
+{
+	fprintf(stderr, "usage: %s dbpath [-e] [-f] [-n] [-a|-s subdb]\n", prog);
+	exit(EXIT_FAILURE);
+}
+
+int main(int argc, char *argv[])
+{
+	int i, rc;
 	MDB_env *env;
 	MDB_txn *txn;
 	MDB_dbi dbi;
 	MDB_stat mst;
-	char *envname = argv[1];
+	MDB_envinfo mei;
+	char *prog = argv[0];
+	char *envname;
 	char *subname = NULL;
+	int alldbs = 0, envinfo = 0, envflags = 0, freinfo = 0;
 
+	if (argc < 2) {
+		usage(prog);
+	}
+
+	/* -a: print stat of main DB and all subDBs
+	 * -s: print stat of only the named subDB
+	 * -e: print env info
+	 * -f: print freelist info
+	 * -n: use NOSUBDIR flag on env_open
+	 * (default) print stat of only the main DB
+	 */
+	while ((i = getopt(argc, argv, "aefns:")) != EOF) {
+		switch(i) {
+		case 'a':
+			alldbs++;
+			break;
+		case 'e':
+			envinfo++;
+			break;
+		case 'f':
+			freinfo++;
+			break;
+		case 'n':
+			envflags |= MDB_NOSUBDIR;
+			break;
+		case 's':
+			subname = optarg;
+			break;
+		default:
+			usage(prog);
+		}
+	}
+
+	if (optind != argc - 1)
+		usage(prog);
+
+	envname = argv[optind];
 	rc = mdb_env_create(&env);
 
-	if (argc > 2) {
+	if (alldbs || subname) {
 		mdb_env_set_maxdbs(env, 4);
-		subname = argv[2];
 	}
 
-	rc = mdb_env_open(env, envname, MDB_RDONLY, 0);
+	rc = mdb_env_open(env, envname, envflags | MDB_RDONLY, 0664);
 	if (rc) {
-		printf("mdb_env_open failed, error %d\n", rc);
+		printf("mdb_env_open failed, error %d %s\n", rc, mdb_strerror(rc));
 		goto env_close;
 	}
 	rc = mdb_txn_begin(env, NULL, MDB_RDONLY, &txn);
 	if (rc) {
-		printf("mdb_txn_begin failed, error %d\n", rc);
+		printf("mdb_txn_begin failed, error %d %s\n", rc, mdb_strerror(rc));
 		goto env_close;
 	}
+
+	if (envinfo) {
+		rc = mdb_env_stat(env, &mst);
+		rc = mdb_env_info(env, &mei);
+		printf("Environment Info\n");
+		printf("  Map address: %p\n", mei.me_mapaddr);
+		printf("  Map size: %zu\n", mei.me_mapsize);
+		printf("  Page size: %u\n", mst.ms_psize);
+		printf("  Max pages: %zu\n", mei.me_mapsize / mst.ms_psize);
+		printf("  Number of pages used: %zu\n", mei.me_last_pgno+1);
+		printf("  Last transaction ID: %zu\n", mei.me_last_txnid);
+		printf("  Max readers: %u\n", mei.me_maxreaders);
+		printf("  Number of readers used: %u\n", mei.me_numreaders);
+	}
+
+	if (freinfo) {
+		MDB_cursor *cursor;
+		MDB_val data;
+		size_t pages = 0, *iptr;
+
+		printf("Freelist Status\n");
+		dbi = 0;
+		rc = mdb_cursor_open(txn, dbi, &cursor);
+		if (rc) {
+			printf("mdb_cursor_open failed, error %d %s\n", rc, mdb_strerror(rc));
+			goto txn_abort;
+		}
+		rc = mdb_stat(txn, dbi, &mst);
+		if (rc) {
+			printf("mdb_stat failed, error %d %s\n", rc, mdb_strerror(rc));
+			goto txn_abort;
+		}
+		while ((rc = mdb_cursor_get(cursor, NULL, &data, MDB_NEXT)) == 0) {
+			iptr = data.mv_data;
+			pages += *iptr;
+		}
+		mdb_cursor_close(cursor);
+		prstat(&mst);
+		printf("  Free pages: %zu\n", pages);
+	}
+
 	rc = mdb_open(txn, subname, 0, &dbi);
 	if (rc) {
-		printf("mdb_open failed, error %d\n", rc);
+		printf("mdb_open failed, error %d %s\n", rc, mdb_strerror(rc));
 		goto txn_abort;
 	}
    
 	rc = mdb_stat(txn, dbi, &mst);
-	printf("Page size: %u\n", mst.ms_psize);
-	printf("Tree depth: %u\n", mst.ms_depth);
-	printf("Branch pages: %zu\n", mst.ms_branch_pages);
-	printf("Leaf pages: %zu\n", mst.ms_leaf_pages);
-	printf("Overflow pages: %zu\n", mst.ms_overflow_pages);
-	printf("Entries: %zu\n", mst.ms_entries);
+	if (rc) {
+		printf("mdb_stat failed, error %d %s\n", rc, mdb_strerror(rc));
+		goto txn_abort;
+	}
+	printf("Status of %s\n", subname ? subname : "Main DB");
+	prstat(&mst);
+
+	if (alldbs) {
+		MDB_cursor *cursor;
+		MDB_val key;
+
+		rc = mdb_cursor_open(txn, dbi, &cursor);
+		if (rc) {
+			printf("mdb_cursor_open failed, error %d %s\n", rc, mdb_strerror(rc));
+			goto txn_abort;
+		}
+		while ((rc = mdb_cursor_get(cursor, &key, NULL, MDB_NEXT)) == 0) {
+			char *str = malloc(key.mv_size+1);
+			MDB_dbi db2;
+			memcpy(str, key.mv_data, key.mv_size);
+			str[key.mv_size] = '\0';
+			rc = mdb_open(txn, str, 0, &db2);
+			if (rc == MDB_SUCCESS)
+				printf("Status of %s\n", str);
+			free(str);
+			if (rc) continue;
+			rc = mdb_stat(txn, db2, &mst);
+			if (rc) {
+				printf("mdb_stat failed, error %d %s\n", rc, mdb_strerror(rc));
+				goto txn_abort;
+			}
+			prstat(&mst);
+			mdb_close(env, db2);
+		}
+		mdb_cursor_close(cursor);
+	}
+
 	mdb_close(env, dbi);
 txn_abort:
 	mdb_txn_abort(txn);

libraries/libmdb/mdb_stata.c

-/* mdb_stat.c - memory-mapped database status tool */
-/*
- * Copyright 2011 Howard Chu, Symas Corp.
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted only as authorized by the OpenLDAP
- * Public License.
- *
- * A copy of this license is available in the file LICENSE in the
- * top-level directory of the distribution or, alternatively, at
- * <http://www.OpenLDAP.org/license.html>.
- */
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include "mdb.h"
-
-int main(int argc,char * argv[])
-{
-	int rc;
-	MDB_env *env;
-	MDB_txn *txn;
-	MDB_dbi dbi;
-	MDB_stat mst;
-	MDB_cursor *cursor;
-	MDB_val key;
-	char *envname = argv[1];
-
-	rc = mdb_env_create(&env);
-
-	mdb_env_set_maxdbs(env, 4);
-
-	rc = mdb_env_open(env, envname, MDB_RDONLY, 0);
-	if (rc) {
-		printf("mdb_env_open failed, error %d\n", rc);
-		goto env_close;
-	}
-	rc = mdb_txn_begin(env, NULL, MDB_RDONLY, &txn);
-	if (rc) {
-		printf("mdb_txn_begin failed, error %d\n", rc);
-		goto env_close;
-	}
-	rc = mdb_open(txn, NULL, 0, &dbi);
-	if (rc) {
-		printf("mdb_open failed, error %d\n", rc);
-		goto txn_abort;
-	}
-   
-	rc = mdb_stat(txn, dbi, &mst);
-	printf("Page size: %u\n", mst.ms_psize);
-	printf("Tree depth: %u\n", mst.ms_depth);
-	printf("Branch pages: %zu\n", mst.ms_branch_pages);
-	printf("Leaf pages: %zu\n", mst.ms_leaf_pages);
-	printf("Overflow pages: %zu\n", mst.ms_overflow_pages);
-	printf("Entries: %zu\n", mst.ms_entries);
-
-	rc = mdb_cursor_open(txn, dbi, &cursor);
-	while ((rc = mdb_cursor_get(cursor, &key, NULL, MDB_NEXT)) == 0) {
-		char *str = malloc(key.mv_size+1);
-		MDB_dbi db2;
-		memcpy(str, key.mv_data, key.mv_size);
-		str[key.mv_size] = '\0';
-		printf("\n%s\n", str);
-		rc = mdb_open(txn, str, 0, &db2);
-		if (rc) break;
-		free(str);
-		rc = mdb_stat(txn, db2, &mst);
-		printf("Tree depth: %u\n", mst.ms_depth);
-		printf("Branch pages: %zu\n", mst.ms_branch_pages);
-		printf("Leaf pages: %zu\n", mst.ms_leaf_pages);
-		printf("Overflow pages: %zu\n", mst.ms_overflow_pages);
-		printf("Entries: %zu\n", mst.ms_entries);
-		mdb_close(env, db2);
-	}
-	mdb_cursor_close(cursor);
-	mdb_close(env, dbi);
-txn_abort:
-	mdb_txn_abort(txn);
-env_close:
-	mdb_env_close(env);
-
-	return rc ? EXIT_FAILURE : EXIT_SUCCESS;
-}