First cut of LDBM substrings indexing. Needs testing.

servers/slapd/back-ldbm/attr.c

@@ -23,7 +23,7 @@ typedef struct ldbm_attrinfo {
 #else
 	char *ai_desc;
 #endif
-	slap_index ai_indexmask;	/* how the attr is indexed	*/
+	slap_mask_t ai_indexmask;	/* how the attr is indexed	*/
 } AttrInfo;
 
 static int
@@ -64,7 +64,7 @@ attr_mask(
 #else
     const char *desc,
 #endif
-    slap_index *indexmask )
+    slap_mask_t *indexmask )
 {
 	AttrInfo	*a;
 
@@ -84,7 +84,7 @@ attr_index_config(
 {
 	int rc;
 	int	i;
-	slap_index mask;
+	slap_mask_t mask;
 	char **attrs;
 	char **indexes = NULL;
 
@@ -115,7 +115,7 @@ attr_index_config(
 		mask = 0;
 
 		for ( i = 0; indexes[i] != NULL; i++ ) {
-			slap_index index;
+			slap_mask_t index;
 			rc = slap_str2index( indexes[i], &index );
 
 			if( rc != LDAP_SUCCESS ) {

servers/slapd/back-ldbm/back-ldbm.h

@@ -107,7 +107,7 @@ struct ldbminfo {
 	ldap_pvt_thread_mutex_t		li_root_mutex;
 	ldap_pvt_thread_mutex_t		li_add_mutex;
 	int			li_mode;
-	slap_index	li_defaultmask;
+	slap_mask_t	li_defaultmask;
 	char			*li_directory;
 	Cache		li_cache;
 	Avlnode			*li_attrs;

servers/slapd/back-ldbm/filterindex.c

@@ -24,7 +24,7 @@ static ID_BLOCK	*approx_candidates(
 	Backend *be, AttributeAssertion *ava );
 static ID_BLOCK	*substring_candidates(
 	Backend *be,
-	Filter *f );
+	SubstringsAssertion *sub );
 static ID_BLOCK	*list_candidates(
 	Backend *be,
 	Filter *flist,
@@ -69,7 +69,7 @@ filter_candidates(
 
 	case LDAP_FILTER_SUBSTRINGS:
 		Debug( LDAP_DEBUG_FILTER, "\tSUBSTRINGS\n", 0, 0, 0 );
-		result = substring_candidates( be, f );
+		result = substring_candidates( be, f->f_sub );
 		break;
 
 	case LDAP_FILTER_GE:
@@ -117,7 +117,7 @@ presence_candidates(
 	DBCache	*db;
 	int rc;
 	char *dbname;
-	slap_index mask;
+	slap_mask_t mask;
 	struct berval *prefix;
 
 	Debug( LDAP_DEBUG_TRACE, "=> presence_candidates\n", 0, 0, 0 );
@@ -128,11 +128,17 @@ presence_candidates(
 		&dbname, &mask, &prefix );
 
 	if( rc != LDAP_SUCCESS ) {
+		Debug( LDAP_DEBUG_ANY,
+		    "<= presence_candidates: index_param failed (%d)\n",
+			rc, 0, 0 );
 		return idl;
 	}
 
 	if( dbname == NULL ) {
 		/* not indexed */
+		Debug( LDAP_DEBUG_TRACE,
+		    "<= presense_candidates: not indexed\n",
+			0, 0, 0 );
 		ber_bvfree( prefix );
 		return idl;
 	}
@@ -184,7 +190,7 @@ equality_candidates(
 	int i;
 	int rc;
 	char *dbname;
-	slap_index mask;
+	slap_mask_t mask;
 	struct berval *prefix;
 	struct berval **keys = NULL;
 	MatchingRule *mr;
@@ -197,11 +203,17 @@ equality_candidates(
 		&dbname, &mask, &prefix );
 
 	if( rc != LDAP_SUCCESS ) {
+		Debug( LDAP_DEBUG_ANY,
+		    "<= equality_candidates: index_param failed (%d)\n",
+			rc, 0, 0 );
 		return idl;
 	}
 
 	if( dbname == NULL ) {
 		/* not indexed */
+		Debug( LDAP_DEBUG_TRACE,
+		    "<= equality_candidates: not indexed\n",
+			0, 0, 0 );
 		ber_bvfree( prefix );
 		return idl;
 	}
@@ -209,7 +221,6 @@ equality_candidates(
 	mr = ava->aa_desc->ad_type->sat_equality;
 	if( !mr ) {
 		ber_bvfree( prefix );
-		/* return LDAP_INAPPROPRIATE_MATCHING; */
 		return idl;
 	}
 
@@ -220,6 +231,7 @@ equality_candidates(
 
 	rc = (mr->smr_filter)(
 		LDAP_FILTER_EQUALITY,
+		mask,
 		ava->aa_desc->ad_type->sat_syntax,
 		mr,
 		prefix,
@@ -229,6 +241,16 @@ equality_candidates(
 	ber_bvfree( prefix );
 
 	if( rc != LDAP_SUCCESS ) {
+		Debug( LDAP_DEBUG_TRACE,
+		    "<= equality_candidates: (%s%s) MR filter failed (%d)\n",
+			dbname, LDBM_SUFFIX, rc );
+		return idl;
+	}
+
+	if( keys == NULL ) {
+		Debug( LDAP_DEBUG_TRACE,
+		    "<= equality_candidates: no keys (%s%s)\n",
+			dbname, LDBM_SUFFIX, 0 );
 		return idl;
 	}
 
@@ -294,7 +316,7 @@ approx_candidates(
 	int i;
 	int rc;
 	char *dbname;
-	slap_index mask;
+	slap_mask_t mask;
 	struct berval *prefix;
 	struct berval **keys = NULL;
 	MatchingRule *mr;
@@ -303,28 +325,33 @@ approx_candidates(
 
 	idl = idl_allids( be );
 
-	rc = index_param( be, ava->aa_desc, LDAP_FILTER_EQUALITY,
+	rc = index_param( be, ava->aa_desc, LDAP_FILTER_APPROX,
 		&dbname, &mask, &prefix );
 
 	if( rc != LDAP_SUCCESS ) {
+		Debug( LDAP_DEBUG_ANY,
+		    "<= approx_candidates: index_param failed (%d)\n",
+			rc, 0, 0 );
 		return idl;
 	}
 
 	if( dbname == NULL ) {
 		/* not indexed */
+		Debug( LDAP_DEBUG_ANY,
+		    "<= approx_candidates: not indexed\n",
+			0, 0, 0 );
 		ber_bvfree( prefix );
 		return idl;
 	}
 
 	mr = ava->aa_desc->ad_type->sat_approx;
-	if( mr == NULL ) {
+	if( !mr ) {
 		/* no approx matching rule, try equality matching rule */
 		mr = ava->aa_desc->ad_type->sat_equality;
 	}
 
 	if( !mr ) {
 		ber_bvfree( prefix );
-		/* return LDAP_INAPPROPRIATE_MATCHING; */
 		return idl;
 	}
 
@@ -334,7 +361,8 @@ approx_candidates(
 	}
 
 	rc = (mr->smr_filter)(
-		LDAP_FILTER_EQUALITY,
+		LDAP_FILTER_APPROX,
+		mask,
 		ava->aa_desc->ad_type->sat_syntax,
 		mr,
 		prefix,
@@ -344,6 +372,16 @@ approx_candidates(
 	ber_bvfree( prefix );
 
 	if( rc != LDAP_SUCCESS ) {
+		Debug( LDAP_DEBUG_TRACE,
+		    "<= approx_candidates: (%s%s) MR filter failed (%d)\n",
+			dbname, LDBM_SUFFIX, rc );
+		return idl;
+	}
+
+	if( keys == NULL ) {
+		Debug( LDAP_DEBUG_TRACE,
+		    "<= approx_candidates: no keys (%s%s)\n",
+			dbname, LDBM_SUFFIX, 0 );
 		return idl;
 	}
 
@@ -438,16 +476,122 @@ list_candidates(
 static ID_BLOCK *
 substring_candidates(
     Backend	*be,
-    Filter	*f
+    SubstringsAssertion	*sub
 )
 {
 	ID_BLOCK *idl;
+	DBCache	*db;
+	int i;
+	int rc;
+	char *dbname;
+	slap_mask_t mask;
+	struct berval *prefix;
+	struct berval **keys = NULL;
+	MatchingRule *mr;
 
-	Debug( LDAP_DEBUG_TRACE, "=> substring_candidates\n", 0, 0, 0 );
+	Debug( LDAP_DEBUG_TRACE, "=> substrings_candidates\n", 0, 0, 0 );
 
 	idl = idl_allids( be );
-	Debug( LDAP_DEBUG_TRACE, "<= substring_candidates %ld\n",
+
+	rc = index_param( be, sub->sa_desc, LDAP_FILTER_SUBSTRINGS,
+		&dbname, &mask, &prefix );
+
+	if( rc != LDAP_SUCCESS ) {
+		Debug( LDAP_DEBUG_ANY,
+		    "<= substrings_candidates: index_param failed (%d)\n",
+			rc, 0, 0 );
+		return idl;
+	}
+
+	if( dbname == NULL ) {
+		/* not indexed */
+		Debug( LDAP_DEBUG_ANY,
+		    "<= substrings_candidates: not indexed\n",
+			0, 0, 0 );
+		ber_bvfree( prefix );
+		return idl;
+	}
+
+	mr = sub->sa_desc->ad_type->sat_substr;
+
+	if( !mr ) {
+		ber_bvfree( prefix );
+		return idl;
+	}
+
+	if( !mr->smr_filter ) {
+		ber_bvfree( prefix );
+		return idl;
+	}
+
+	rc = (mr->smr_filter)(
+		LDAP_FILTER_SUBSTRINGS,
+		mask,
+		sub->sa_desc->ad_type->sat_syntax,
+		mr,
+		prefix,
+		sub,
+		&keys );
+
+	ber_bvfree( prefix );
+
+	if( rc != LDAP_SUCCESS ) {
+		Debug( LDAP_DEBUG_TRACE,
+		    "<= substrings_candidates: (%s%s) MR filter failed (%d)\n",
+			dbname, LDBM_SUFFIX, rc );
+		return idl;
+	}
+
+	if( keys == NULL ) {
+		Debug( LDAP_DEBUG_TRACE,
+		    "<= substrings_candidates: (0x%04lx) no keys (%s%s)\n",
+			mask, dbname, LDBM_SUFFIX );
+		return idl;
+	}
+
+	db = ldbm_cache_open( be, dbname, LDBM_SUFFIX, LDBM_READER );
+	
+	if ( db == NULL ) {
+		Debug( LDAP_DEBUG_ANY,
+		    "<= substrings_candidates db open failed (%s%s)\n",
+			dbname, LDBM_SUFFIX, 0 );
+		return idl;
+	}
+
+	for ( i= 0; keys[i] != NULL; i++ ) {
+		ID_BLOCK *save;
+		ID_BLOCK *tmp;
+
+		rc = key_read( be, db, keys[i], &tmp );
+
+		if( rc != LDAP_SUCCESS ) {
+			idl_free( idl );
+			idl = NULL;
+			Debug( LDAP_DEBUG_TRACE, "<= substrings_candidates key read failed (%d)\n",
+			    rc, 0, 0 );
+			break;
+		}
+
+		if( tmp == NULL ) {
+			idl_free( idl );
+			idl = NULL;
+			Debug( LDAP_DEBUG_TRACE, "<= substrings_candidates NULL\n",
+			    0, 0, 0 );
+			break;
+		}
+
+		save = idl;
+		idl = idl_intersection( be, idl, tmp );
+		idl_free( save );
+
+		if( idl == NULL ) break;
+	}
+
+	ber_bvecfree( keys );
+
+	ldbm_cache_close( be, db );
+
+	Debug( LDAP_DEBUG_TRACE, "<= substrings_candidates %ld\n",
 	    idl ? ID_BLOCK_NIDS(idl) : 0, 0, 0 );
 	return( idl );
 }
-

servers/slapd/back-ldbm/index.c

@@ -22,7 +22,7 @@ static index_mask(
 	char **atname )
 {
 	AttributeType *at;
-	slap_index mask = 0;
+	slap_mask_t mask = 0;
 
 	/* we do support indexing of binary attributes */
 	if( slap_ad_is_binary( desc ) ) return 0;
@@ -77,10 +77,10 @@ int index_param(
 	AttributeDescription *desc,
 	int ftype,
 	char **dbnamep,
-	slap_index *maskp,
+	slap_mask_t *maskp,
 	struct berval **prefixp )
 {
-	slap_index mask;
+	slap_mask_t mask;
 	char *dbname;
 	char *atname;
 
@@ -135,7 +135,7 @@ static int indexer(
 	struct berval **vals,
 	ID id,
 	int op,
-	slap_index mask )
+	slap_mask_t mask )
 {
 	int rc, i;
 	const char *text;
@@ -169,6 +169,7 @@ static int indexer(
 
 	if( IS_SLAP_INDEX( mask, SLAP_INDEX_EQUALITY ) ) {
 		rc = ad->ad_type->sat_equality->smr_indexer(
+			LDAP_FILTER_EQUALITY,
 			mask,
 			ad->ad_type->sat_syntax,
 			ad->ad_type->sat_equality,
@@ -184,6 +185,7 @@ static int indexer(
 
 	if( IS_SLAP_INDEX( mask, SLAP_INDEX_APPROX ) ) {
 		rc = ad->ad_type->sat_approx->smr_indexer(
+			LDAP_FILTER_APPROX,
 			mask,
 			ad->ad_type->sat_syntax,
 			ad->ad_type->sat_approx,
@@ -199,6 +201,7 @@ static int indexer(
 
 	if( IS_SLAP_INDEX( mask, SLAP_INDEX_SUBSTR ) ) {
 		rc = ad->ad_type->sat_substr->smr_indexer(
+			LDAP_FILTER_SUBSTRINGS,
 			mask,
 			ad->ad_type->sat_syntax,
 			ad->ad_type->sat_substr,
@@ -225,10 +228,10 @@ static int index_at_values(
 	ID id,
 	int op,
 	char ** dbnamep,
-	slap_index *maskp )
+	slap_mask_t *maskp )
 {
-	slap_index mask;
-	slap_index tmpmask = 0;
+	slap_mask_t mask;
+	slap_mask_t tmpmask = 0;
 	int lindex = 0;
 
 	if( type->sat_sup ) {
@@ -291,7 +294,7 @@ int index_values(
 	int op )
 {
 	char *dbname = NULL;
-	slap_index mask;
+	slap_mask_t mask;
 
 	if( slap_ad_is_binary( desc ) ) {
 		/* binary attributes have no index capabilities */

servers/slapd/back-ldbm/proto-back-ldbm.h

@@ -35,7 +35,7 @@ Entry *deref_internal_r LDAP_P((
 
 void attr_mask LDAP_P(( struct ldbminfo *li,
 	const char *desc,
-	slap_index *indexmask ));
+	slap_mask_t *indexmask ));
 
 int attr_index_config LDAP_P(( struct ldbminfo *li,
 	const char *fname, int lineno,
@@ -139,7 +139,7 @@ index_param LDAP_P((
 	AttributeDescription *desc,
 	int ftype,
 	char **dbname,
-	slap_index *mask,
+	slap_mask_t *mask,
 	struct berval **prefix ));
 
 extern int

servers/slapd/index.c

@@ -34,7 +34,7 @@ slap_index2prefix( int indextype )
 	return( prefix );
 }
 
-int slap_str2index( const char *str, slap_index *idx )
+int slap_str2index( const char *str, slap_mask_t *idx )
 {
 	if ( strcasecmp( str, "pres" ) == 0 ) {
 		*idx = SLAP_INDEX_PRESENT;

servers/slapd/proto-slap.h

@@ -252,7 +252,7 @@ LDAP_SLAPD_F (int) read_config LDAP_P(( const char *fname ));
  * index.c
  */
 LDAP_SLAPD_F (int) slap_index2prefix LDAP_P(( int indextype ));
-LDAP_SLAPD_F (int) slap_str2index LDAP_P(( const char *str, slap_index *idx ));
+LDAP_SLAPD_F (int) slap_str2index LDAP_P(( const char *str, slap_mask_t *idx ));
 
 /*
  * connection.c

servers/slapd/schema_init.c

@@ -88,7 +88,7 @@
 static int
 octetStringMatch(
 	int *matchp,
-	unsigned flags,
+	slap_mask_t flags,
 	Syntax *syntax,
 	MatchingRule *mr,
 	struct berval *value,
@@ -108,7 +108,8 @@ octetStringMatch(
 
 /* Index generation function */
 int octetStringIndexer(
-	unsigned flags,
+	slap_mask_t use,
+	slap_mask_t flags,
 	Syntax *syntax,
 	MatchingRule *mr,
 	struct berval *prefix,
@@ -161,7 +162,8 @@ int octetStringIndexer(
 
 /* Index generation function */
 int octetStringFilter(
-	unsigned flags,
+	slap_mask_t use,
+	slap_mask_t flags,
 	Syntax *syntax,
 	MatchingRule *mr,
 	struct berval *prefix,
@@ -254,7 +256,7 @@ dnNormalize(
 static int
 dnMatch(
 	int *matchp,
-	unsigned flags,
+	slap_mask_t flags,
 	Syntax *syntax,
 	MatchingRule *mr,
 	struct berval *value,
@@ -329,7 +331,7 @@ booleanValidate(
 static int
 booleanMatch(
 	int *matchp,
-	unsigned flags,
+	slap_mask_t flags,
 	Syntax *syntax,
 	MatchingRule *mr,
 	struct berval *value,
@@ -628,7 +630,7 @@ IA5StringNormalize(
 static int
 caseExactIA5Match(
 	int *matchp,
-	unsigned flags,
+	slap_mask_t flags,
 	Syntax *syntax,
 	MatchingRule *mr,
 	struct berval *value,
@@ -649,7 +651,7 @@ caseExactIA5Match(
 static int
 caseExactIA5SubstringsMatch(
 	int *matchp,
-	unsigned flags,
+	slap_mask_t flags,
 	Syntax *syntax,
 	MatchingRule *mr,
 	struct berval *value,
@@ -773,7 +775,8 @@ done:
 
 /* Index generation function */
 int caseExactIA5Indexer(
-	unsigned flags,
+	slap_mask_t use,
+	slap_mask_t flags,
 	Syntax *syntax,
 	MatchingRule *mr,
 	struct berval *prefix,
@@ -826,7 +829,8 @@ int caseExactIA5Indexer(
 
 /* Index generation function */
 int caseExactIA5Filter(
-	unsigned flags,
+	slap_mask_t use,
+	slap_mask_t flags,
 	Syntax *syntax,
 	MatchingRule *mr,
 	struct berval *prefix,
@@ -871,7 +875,8 @@ int caseExactIA5Filter(
 
 /* Substrings Index generation function */
 int caseExactIA5SubstringsIndexer(
-	unsigned flags,
+	slap_mask_t use,
+	slap_mask_t flags,
 	Syntax *syntax,
 	MatchingRule *mr,
 	struct berval *prefix,
@@ -894,7 +899,7 @@ int caseExactIA5SubstringsIndexer(
 			continue;
 		}
 
-		if( flags & SLAP_MR_SUBSTR_INITIAL ) {
+		if( flags & SLAP_INDEX_SUBSTR_INITIAL ) {
 			if( values[i]->bv_len >= SLAP_INDEX_SUBSTR_MAXLEN ) {
 				nkeys += SLAP_INDEX_SUBSTR_MAXLEN -
 					( SLAP_INDEX_SUBSTR_MINLEN - 1);
@@ -903,13 +908,13 @@ int caseExactIA5SubstringsIndexer(
 			}
 		}
 
-		if( flags & SLAP_MR_SUBSTR_ANY ) {
+		if( flags & SLAP_INDEX_SUBSTR_ANY ) {
 			if( values[i]->bv_len >= SLAP_INDEX_SUBSTR_MAXLEN ) {
 				nkeys += values[i]->bv_len - ( SLAP_INDEX_SUBSTR_MAXLEN - 1 );
 			}
 		}
 
-		if( flags & SLAP_MR_SUBSTR_FINAL ) {
+		if( flags & SLAP_INDEX_SUBSTR_FINAL ) {
 			if( values[i]->bv_len >= SLAP_INDEX_SUBSTR_MAXLEN ) {
 				nkeys += SLAP_INDEX_SUBSTR_MAXLEN -
 					( SLAP_INDEX_SUBSTR_MINLEN - 1);
@@ -939,7 +944,7 @@ int caseExactIA5SubstringsIndexer(
 		value = values[i];
 		if( value->bv_len < SLAP_INDEX_SUBSTR_MINLEN ) continue;
 
-		if( ( flags & SLAP_MR_SUBSTR_ANY ) &&
+		if( ( flags & SLAP_INDEX_SUBSTR_ANY ) &&
 			( value->bv_len >= SLAP_INDEX_SUBSTR_MAXLEN ) )
 		{
 			char pre = SLAP_INDEX_SUBSTR_PREFIX;
@@ -973,7 +978,7 @@ int caseExactIA5SubstringsIndexer(
 		for( j=SLAP_INDEX_SUBSTR_MINLEN; j<=max; j++ ) {
 			char pre;
 
-			if( flags & SLAP_MR_SUBSTR_INITIAL ) {
+			if( flags & SLAP_INDEX_SUBSTR_INITIAL ) {
 				pre = SLAP_INDEX_SUBSTR_INITIAL_PREFIX;
 				lutil_MD5Init( &MD5context );
 				if( prefix != NULL && prefix->bv_len > 0 ) {
@@ -993,7 +998,7 @@ int caseExactIA5SubstringsIndexer(
 				keys[nkeys++] = ber_bvdup( &digest );
 			}
 
-			if( flags & SLAP_MR_SUBSTR_FINAL ) {
+			if( flags & SLAP_INDEX_SUBSTR_FINAL ) {
 				pre = SLAP_INDEX_SUBSTR_FINAL_PREFIX;
 				lutil_MD5Init( &MD5context );
 				if( prefix != NULL && prefix->bv_len > 0 ) {
@@ -1016,13 +1021,20 @@ int caseExactIA5SubstringsIndexer(
 		}
 	}
 
-	keys[nkeys] = NULL;
-	*keysp = keys;
+	if( nkeys > 0 ) {
+		keys[nkeys] = NULL;
+		*keysp = keys;
+	} else {
+		ch_free( keys );
+		*keysp = NULL;
+	}
+
 	return LDAP_SUCCESS;
 }
 
 int caseExactIA5SubstringsFilter(
-	unsigned flags,
+	slap_mask_t use,
+	slap_mask_t flags,
 	Syntax *syntax,
 	MatchingRule *mr,
 	struct berval *prefix,
@@ -1039,13 +1051,13 @@ int caseExactIA5SubstringsFilter(
 	struct berval *value;
 	struct berval digest;
 
-	if( sa->sa_initial != NULL &&
+	if( flags & SLAP_INDEX_SUBSTR_INITIAL && sa->sa_initial != NULL &&
 		sa->sa_initial->bv_len >= SLAP_INDEX_SUBSTR_MINLEN )
 	{