dbpagesize, checksum keywords

CHANGES

@@ -7,6 +7,8 @@ OpenLDAP 2.4.13 Engineering
 	Added slapd private databases in global overlays (ITS#5735,ITS#5736)
 	Added slapd support for certificateListExactMatch (ITS#5700)
 	Fixed slapd-bdb/hdb invalid db crash (ITS#5698)
+	Added slapd-bdb/hdb dbpagesize keyword
+	Added slapd-bdb/hdb checksum keyword
 	Fixed slapo-chain/translucent back-config support (ITS#5736)
 	Added slapo-constraint support for LDAP URI constraints (ITS#5704)
 	Added slapo-constraint support for constraining rename (ITS#5703)

doc/man/man5/slapd-bdb.5

@@ -60,6 +60,10 @@ the \fI<min>\fP argument is non-zero, an internal task will run every
 \fI<min>\fP minutes to perform the checkpoint.
 See the Berkeley DB reference guide for more details.
 .TP
+.B checksum
+Enable checksum validation of DB pages whenever they are read from disk.
+This setting can only be configured before any database files are created.
+.TP
 .BI cryptfile \ <file>
 Specify the pathname of a file containing an encryption key to use for
 encrypting the database. Encryption is performed using Berkeley DB's
@@ -111,6 +115,19 @@ Enabling this option may improve performance at the expense of data
 security.
 See the Berkeley DB reference guide for more details.
 .TP
+\fBdbpagesize \fR \fI<dbfile> <size>\fR
+Specify the page size to use for a particular database file, in units
+of 1024 bytes. The default for the
+.B id2entry
+file is 16, the default for all other files depends on the size of the
+underlying filesystem's block size (typically 4 or 8).
+The maximum that BerkeleyDB supports is 64. This
+setting usually should not need to be changed, but if BerkeleyDB's
+"db_stat -d" shows a large amount of overflow pages in use in a file,
+setting a larger size may increase performance at the expense of
+data integrity. This setting only takes effect when a database is
+being newly created. See the Berkeley DB reference guide for more details.
+.TP
 .BI directory \ <directory>
 Specify the directory where the BDB files containing this database and
 associated indexes live.

servers/slapd/back-bdb/back-bdb.h

@@ -54,10 +54,6 @@ LDAP_BEGIN_DECL
 #define	BDB_ID2ENTRY_PAGESIZE	16384
 #endif
 
-#ifndef BDB_PAGESIZE
-#define	BDB_PAGESIZE	4096	/* BDB's original default */
-#endif
-
 #define DEFAULT_CACHE_SIZE     1000
 
 /* The default search IDL stack cache depth */
@@ -156,6 +152,12 @@ struct bdb_db_info {
 	DB			*bdi_db;
 };
 
+struct bdb_db_pgsize {
+	struct bdb_db_pgsize *bdp_next;
+	struct berval	bdp_name;
+	int	bdp_size;
+};
+
 #ifdef LDAP_DEVEL
 #define BDB_MONITOR_IDX
 #endif /* LDAP_DEVEL */
@@ -178,9 +180,10 @@ struct bdb_info {
 	int			bi_dbenv_mode;
 
 	int			bi_ndatabases;
+	int		bi_db_opflags;	/* db-specific flags */
 	struct bdb_db_info **bi_databases;
 	ldap_pvt_thread_mutex_t	bi_database_mutex;
-	int		bi_db_opflags;	/* db-specific flags */
+	struct bdb_db_pgsize *bi_pagesizes;
 
 	slap_mask_t	bi_defaultmask;
 	Cache		bi_cache;
@@ -226,6 +229,7 @@ struct bdb_info {
 #define	BDB_UPD_CONFIG	0x04
 #define	BDB_DEL_INDEX	0x08
 #define	BDB_RE_OPEN		0x10
+#define BDB_CHKSUM		0x20
 #ifdef BDB_HIER
 	int		bi_modrdns;		/* number of modrdns completed */
 	ldap_pvt_thread_mutex_t	bi_modrdns_mutex;

servers/slapd/back-bdb/config.c

@@ -50,7 +50,9 @@ enum {
 	BDB_INDEX,
 	BDB_LOCKD,
 	BDB_SSTACK,
-	BDB_MODE
+	BDB_MODE,
+	BDB_PGSIZE,
+	BDB_CHECKSUM
 };
 
 static ConfigTable bdbcfg[] = {
@@ -73,6 +75,10 @@ static ConfigTable bdbcfg[] = {
 		bdb_cf_gen, "( OLcfgDbAt:1.2 NAME 'olcDbCheckpoint' "
 			"DESC 'Database checkpoint interval in kbytes and minutes' "
 			"SYNTAX OMsDirectoryString SINGLE-VALUE )",NULL, NULL },
+	{ "checksum", NULL, 1, 2, 0, ARG_ON_OFF|ARG_MAGIC|BDB_CHECKSUM,
+		bdb_cf_gen, "( OLcfgDbAt:1.16 NAME 'olcDbChecksum' "
+			"DESC 'Enable database checksum validation' "
+			"SYNTAX OMsBoolean SINGLE-VALUE )", NULL, NULL },
 	{ "cryptfile", "file", 2, 2, 0, ARG_STRING|ARG_MAGIC|BDB_CRYPTFILE,
 		bdb_cf_gen, "( OLcfgDbAt:1.13 NAME 'olcDbCryptFile' "
 			"DESC 'Pathname of file containing the DB encryption key' "
@@ -89,6 +95,11 @@ static ConfigTable bdbcfg[] = {
 		bdb_cf_gen, "( OLcfgDbAt:1.4 NAME 'olcDbNoSync' "
 			"DESC 'Disable synchronous database writes' "
 			"SYNTAX OMsBoolean SINGLE-VALUE )", NULL, NULL },
+	{ "dbpagesize", "db> <size", 3, 3, 0, ARG_MAGIC|BDB_PGSIZE,
+		bdb_cf_gen, "( OLcfgDbAt:1.15 NAME 'olcDbPageSize' "
+			"DESC 'Page size of specified DB, in Kbytes' "
+			"EQUALITY caseExactMatch "
+			"SYNTAX OMsDirectoryString )", NULL, NULL },
 	{ "dirtyread", NULL, 1, 2, 0,
 #ifdef SLAP_BDB_ALLOW_DIRTY_READ
 		ARG_ON_OFF|ARG_MAGIC|BDB_DIRTYR, bdb_cf_gen,
@@ -157,7 +168,7 @@ static ConfigOCs bdbocs[] = {
 		"olcDbNoSync $ olcDbDirtyRead $ olcDbIDLcacheSize $ "
 		"olcDbIndex $ olcDbLinearIndex $ olcDbLockDetect $ "
 		"olcDbMode $ olcDbSearchStack $ olcDbShmKey $ "
-		"olcDbCacheFree $ olcDbDNcacheSize ) )",
+		"olcDbCacheFree $ olcDbDNcacheSize $ olcDbPageSize ) )",
 		 	Cft_Database, bdbcfg },
 	{ NULL, 0, NULL }
 };
@@ -461,6 +472,11 @@ bdb_cf_gen( ConfigArgs *c )
 				c->value_int = 1;
 			break;
 			
+		case BDB_CHECKSUM:
+			if ( bdb->bi_flags & BDB_CHKSUM )
+				c->value_int = 1;
+			break;
+
 		case BDB_INDEX:
 			bdb_attr_index_unparse( bdb, &c->rvalue_vals );
 			if ( !c->rvalue_vals ) rc = 1;
@@ -483,6 +499,23 @@ bdb_cf_gen( ConfigArgs *c )
 		case BDB_SSTACK:
 			c->value_int = bdb->bi_search_stack_depth;
 			break;
+
+		case BDB_PGSIZE: {
+				struct bdb_db_pgsize *ps;
+				char buf[SLAP_TEXT_BUFLEN];
+				struct berval bv;
+				int rc = 1;
+
+				bv.bv_val = buf;
+				for ( ps = bdb->bi_pagesizes; ps; ps = ps->bdp_next ) {
+					bv.bv_len = sprintf( buf, "%s %d", ps->bdp_name.bv_val,
+						ps->bdp_size / 1024 );
+					value_add_one( &c->rvalue_vals, &bv );
+					rc = 0;
+
+				}
+				break;
+			}
 		}
 		return rc;
 	} else if ( c->op == LDAP_MOD_DELETE ) {
@@ -554,6 +587,9 @@ bdb_cf_gen( ConfigArgs *c )
 		case BDB_NOSYNC:
 			bdb->bi_dbenv->set_flags( bdb->bi_dbenv, DB_TXN_NOSYNC, 0 );
 			break;
+		case BDB_CHECKSUM:
+			bdb->bi_flags &= ~BDB_CHKSUM;
+			break;
 		case BDB_INDEX:
 			if ( c->valx == -1 ) {
 				int i;
@@ -608,6 +644,24 @@ bdb_cf_gen( ConfigArgs *c )
 				}
 			}
 			break;
+		/* doesn't make sense on the fly; the DB file must be
+		 * recreated
+		 */
+		case BDB_PGSIZE: {
+				struct bdb_db_pgsize *ps, **prev;
+				int i;
+
+				for ( i = 0, prev = &bdb->bi_pagesizes, ps = *prev; ps;
+					prev = &ps->bdp_next, ps = ps->bdp_next, i++ ) {
+					if ( c->valx == -1 || i == c->valx ) {
+						*prev = ps->bdp_next;
+						ch_free( ps );
+						ps = *prev;
+						if ( i == c->valx ) break;
+					}
+				}
+			}
+			break;
 		}
 		return rc;
 	}
@@ -798,6 +852,13 @@ bdb_cf_gen( ConfigArgs *c )
 		}
 		break;
 
+	case BDB_CHECKSUM:
+		if ( c->value_int )
+			bdb->bi_flags |= BDB_CHKSUM;
+		else
+			bdb->bi_flags &= ~BDB_CHKSUM;
+		break;
+
 	case BDB_INDEX:
 		rc = bdb_attr_index_config( bdb, c->fname, c->lineno,
 			c->argc - 1, &c->argv[1], &c->reply);
@@ -841,6 +902,31 @@ bdb_cf_gen( ConfigArgs *c )
 		}
 		bdb->bi_search_stack_depth = c->value_int;
 		break;
+
+	case BDB_PGSIZE: {
+		struct bdb_db_pgsize *ps, **prev;
+		int i, s;
+		
+		s = atoi(c->argv[2]);
+		if ( s < 1 || s > 64 ) {
+			snprintf( c->cr_msg, sizeof( c->cr_msg ),
+				"%s: size must be > 0 and <= 64: %d",
+				c->log, s );
+			Debug( LDAP_DEBUG_ANY, "%s\n", c->cr_msg, 0, 0 );
+			return -1;
+		}
+		i = strlen(c->argv[1]);
+		ps = ch_malloc( sizeof(struct bdb_db_pgsize) + i + 1 );
+		ps->bdp_next = NULL;
+		ps->bdp_name.bv_len = i;
+		ps->bdp_name.bv_val = (char *)(ps+1);
+		strcpy( ps->bdp_name.bv_val, c->argv[1] );
+		ps->bdp_size = s * 1024;
+		for ( prev = &bdb->bi_pagesizes; *prev; prev = &(*prev)->bdp_next )
+			;
+		*prev = ps;
+		}
+		break;
 	}
 	return 0;
 }

servers/slapd/back-bdb/dbcache.c

@@ -57,6 +57,29 @@ bdb_db_hash(
 #define	BDB_INDEXTYPE	DB_BTREE
 #endif
 
+/* If a configured size is found, return it, otherwise return 0 */
+int
+bdb_db_findsize(
+	struct bdb_info *bdb,
+	struct berval *name
+)
+{
+	struct bdb_db_pgsize *bp;
+	int rc;
+
+	for ( bp = bdb->bi_pagesizes; bp; bp=bp->bdp_next ) {
+		rc = strncmp( name->bv_val, bp->bdp_name.bv_val, name->bv_len );
+		if ( !rc ) {
+			if ( name->bv_len == bp->bdp_name.bv_len )
+				return bp->bdp_size;
+			if ( name->bv_len < bp->bdp_name.bv_len &&
+				bp->bdp_name.bv_val[name->bv_len] == '.' )
+				return bp->bdp_size;
+		}
+	}
+	return 0;
+}
+
 int
 bdb_db_cache(
 	Backend	*be,
@@ -121,7 +144,24 @@ bdb_db_cache(
 		}
 	}
 
-	rc = db->bdi_db->set_pagesize( db->bdi_db, BDB_PAGESIZE );
+	if( bdb->bi_flags & BDB_CHKSUM ) {
+		rc = db->bdi_db->set_flags( db->bdi_db, DB_CHKSUM );
+		if ( rc ) {
+			Debug( LDAP_DEBUG_ANY,
+				"bdb_db_cache: db set_flags(DB_CHKSUM)(%s) failed: %s (%d)\n",
+				bdb->bi_dbenv_home, db_strerror(rc), rc );
+			ldap_pvt_thread_mutex_unlock( &bdb->bi_database_mutex );
+			db->bdi_db->close( db->bdi_db, 0 );
+			ch_free( db );
+			return rc;
+		}
+	}
+
+	/* If no explicit size set, use the FS default */
+	flags = bdb_db_findsize( bdb, name );
+	if ( flags )
+		rc = db->bdi_db->set_pagesize( db->bdi_db, flags );
+
 #ifdef BDB_INDEX_USE_HASH
 	rc = db->bdi_db->set_h_hash( db->bdi_db, bdb_db_hash );
 #endif

servers/slapd/back-bdb/init.c

@@ -416,19 +416,40 @@ shm_retry:
 			}
 		}
 
+		if( bdb->bi_flags & BDB_CHKSUM ) {
+			rc = db->bdi_db->set_flags( db->bdi_db, DB_CHKSUM );
+			if ( rc ) {
+				snprintf(cr->msg, sizeof(cr->msg),
+					"database \"%s\": db set_flags(DB_CHKSUM)(%s) failed: %s (%d).",
+					be->be_suffix[0].bv_val, 
+					bdb->bi_dbenv_home, db_strerror(rc), rc );
+				Debug( LDAP_DEBUG_ANY,
+					LDAP_XSTRING(bdb_db_open) ": %s\n",
+					cr->msg, 0, 0 );
+				goto fail;
+			}
+		}
+
+		rc = bdb_db_findsize( bdb, (struct berval *)&bdbi_databases[i].name );
+
 		if( i == BDB_ID2ENTRY ) {
+			if ( !rc ) rc = BDB_ID2ENTRY_PAGESIZE;
+			rc = db->bdi_db->set_pagesize( db->bdi_db, rc );
+
 			if ( slapMode & SLAP_TOOL_MODE )
 				db->bdi_db->mpf->set_priority( db->bdi_db->mpf,
 					DB_PRIORITY_VERY_LOW );
 
-			rc = db->bdi_db->set_pagesize( db->bdi_db,
-				BDB_ID2ENTRY_PAGESIZE );
 			if ( slapMode & SLAP_TOOL_READMAIN ) {
 				flags |= DB_RDONLY;
 			} else {
 				flags |= DB_CREATE;
 			}
 		} else {
+			/* Use FS default size if not configured */
+			if ( rc )
+				rc = db->bdi_db->set_pagesize( db->bdi_db, rc );
+
 			rc = db->bdi_db->set_flags( db->bdi_db, 
 				DB_DUP | DB_DUPSORT );
 #ifndef BDB_HIER
@@ -446,8 +467,6 @@ shm_retry:
 				flags |= DB_CREATE;
 			}
 #endif
-			rc = db->bdi_db->set_pagesize( db->bdi_db,
-				BDB_PAGESIZE );
 		}
 
 #ifdef HAVE_EBCDIC

servers/slapd/back-bdb/proto-bdb.h

@@ -70,6 +70,7 @@ int bdb_back_init_cf( BackendInfo *bi );
  * dbcache.c
  */
 #define bdb_db_cache				BDB_SYMBOL(db_cache)
+#define bdb_db_findsize				BDB_SYMBOL(db_findsize)
 
 int
 bdb_db_cache(
@@ -77,6 +78,11 @@ bdb_db_cache(
     struct berval *name,
 	DB **db );
 
+int
+bdb_db_findsize(
+	struct bdb_info *bdb,
+	struct berval *name );
+
 /*
  * dn2entry.c
  */