ITS#9270 Additional information on indexing

doc/guide/admin/tuning.sdf

@@ -64,9 +64,20 @@ If the filter term has not been indexed, then the search must read every single
  entry in the target scope and test to see if each entry matches the filter. 
 Obviously indexing can save a lot of work when it's used correctly.
 
+In back-mdb, indexes can only track a certain number of entries per key (by
+default that number is 2^16 = 65536). If more entries' values hash to this
+key, some/all of them will have to be represented by a range of candidates,
+making the index less useful over time as deletions cannot usually be tracked
+accurately.
+
 H3: What to index
 
-You should create indices to match the actual filter terms used in
+As a general rule, to make any use of indexes, you must set up an equality
+index on objectClass:
+
+>        index objectClass eq
+
+Then you should create indices to match the actual filter terms used in
 search queries. 
 
 >        index cn,sn,givenname,mail eq
@@ -86,7 +97,8 @@ all of those entries are going to be read anyway, because they are valid
 members of the result set. In a subtree where 100% of the
 entries are going to contain the same attributes, the presence index does
 absolutely NOTHING to benefit the search, because 100% of the entries match
-that presence filter. 
+that presence filter. As an example, setting a presence index on objectClass
+provides no benefit since it is present on every entry.
 
 So the resource cost of generating the index is a
 complete waste of CPU time, disk, and memory. Don't do it unless you know
@@ -101,6 +113,31 @@ not be done, it's just wasted overhead.
 See the {{Logging}} section below on what to watch out for if you have a frequently searched
 for attribute that is unindexed.
 
+H3: Equality indexing
+
+Similarly to presence indexes, equality indexes are most useful if the
+values searched for are uncommon. Most OpenLDAP indexes work by hashing
+the normalised value and using the hash as the key. Hashing behaviour
+depends on the matching rule syntax, some matching rules also implement
+indexers that help speed up inequality (lower than, ...) queries.
+
+Check the documentation and other parts of this guide if some indexes are
+mandatory - e.g. to enable replication, it is expected you index certain
+operational attributes, similar if you rely of filters in ACL processing.
+
+Approximate indexes are usually identical to equality indexes unless
+a matching rule explicitly implements it. As of OpenLDAP 2.5, only
+directoryStringApproxMatch and IA5StringApproxMatch (soundex) matchers
+and indexers are implemented.
+
+H3: Substring indexing
+
+Substring indexes work on spliting the value into short chunks and then
+indexing those in a similar way to how equality index does. The storage
+space needed to store all of this data is analogous to the amount of data
+being indexed, which makes the indexes extremely heavy-handed in most
+scenarios.
+
 
 H2: Logging
 

include/ldif.h

@@ -33,7 +33,7 @@ LDAP_BEGIN_DECL
 /* This is NOT a bogus extern declaration (unlike ldap_debug) */
 LDAP_LDIF_V (int) ldif_debug;
 
-#define LDIF_LINE_WIDTH      76      /* default maximum length of LDIF lines */
+#define LDIF_LINE_WIDTH      78      /* default maximum length of LDIF lines */
 #define LDIF_LINE_WIDTH_MAX  ((ber_len_t)-1) /* maximum length of LDIF lines */
 #define LDIF_LINE_WIDTH_WRAP(wrap) ((wrap) == 0 ? LDIF_LINE_WIDTH : (wrap))
 
@@ -50,9 +50,7 @@ LDAP_LDIF_V (int) ldif_debug;
  * first newline + base64 value + continued lines.  Each continued line
  * needs room for a newline and a leading space character.
  */
-#define LDIF_SIZE_NEEDED(nlen,vlen) \
-    ((nlen) + 4 + LDIF_BASE64_LEN(vlen) \
-    + ((LDIF_BASE64_LEN(vlen) + (nlen) + 3) / (LDIF_LINE_WIDTH-1) * 2 ))
+#define LDIF_SIZE_NEEDED(nlen,vlen) LDIF_SIZE_NEEDED_WRAP(nlen, vlen, 0)
 
 #define LDIF_SIZE_NEEDED_WRAP(nlen,vlen,wrap) \
     ((nlen) + 4 + LDIF_BASE64_LEN(vlen) \

libraries/libldap/ldif.c

@@ -432,9 +432,6 @@ ldif_must_b64_encode( LDAP_CONST char *s )
 	return 0;
 }
 
-/* compatibility with U-Mich off by two bug */
-#define LDIF_KLUDGE 2
-
 /* NOTE: only preserved for binary compatibility */
 void
 ldif_sput(
@@ -444,7 +441,7 @@ ldif_sput(
 	LDAP_CONST char *val,
 	ber_len_t vlen )
 {
-	ldif_sput_wrap( out, type, name, val, vlen, LDIF_LINE_WIDTH+LDIF_KLUDGE );
+	ldif_sput_wrap( out, type, name, val, vlen, 0 );
 }
 
 void
@@ -468,7 +465,7 @@ ldif_sput_wrap(
 	ber_len_t i;
 
 	if ( !wrap )
-		wrap = LDIF_LINE_WIDTH+LDIF_KLUDGE;
+		wrap = LDIF_LINE_WIDTH;
 
 	/* prefix */
 	switch( type ) {
@@ -664,7 +661,7 @@ ldif_put(
 	LDAP_CONST char *val,
 	ber_len_t vlen )
 {
-	return ldif_put_wrap( type, name, val, vlen, LDIF_LINE_WIDTH );
+	return ldif_put_wrap( type, name, val, vlen, 0 );
 }
 
 char *