schema_init.c

/* schema_init.c - init builtin schema */
/* $OpenLDAP$ */
/*
 * Copyright 1998-2000 The OpenLDAP Foundation, All Rights Reserved.
 * COPYING RESTRICTIONS APPLY, see COPYRIGHT file
 */

#include "portable.h"

#include <stdio.h>
#include <limits.h>

#include <ac/ctype.h>
#include <ac/errno.h>
#include <ac/string.h>
#include <ac/socket.h>

#include "slap.h"
#include "ldap_pvt.h"

#include "ldap_utf8.h"

#include "lutil_hash.h"
/* We should replace MD5 with a faster hash */
#define HASH_BYTES				LUTIL_HASH_BYTES
#define HASH_CONTEXT			lutil_HASH_CTX
#define HASH_Init(c)			lutil_HASHInit(c)
#define HASH_Update(c,buf,len)	lutil_HASHUpdate(c,buf,len)
#define HASH_Final(d,c)			lutil_HASHFinal(d,c)

/* recycled validatation routines */
#define berValidate						blobValidate

/* unimplemented pretters */
#ifndef USE_LDAP_DN_PARSING
#define dnPretty						NULL
#endif /* !USE_LDAP_DN_PARSING */
#define integerPretty					NULL

/* recycled matching routines */
#define bitStringMatch					octetStringMatch
#define numericStringMatch				caseIgnoreIA5Match
#define objectIdentifierMatch			caseIgnoreIA5Match
#define telephoneNumberMatch			caseIgnoreIA5Match
#define telephoneNumberSubstringsMatch	caseIgnoreIA5SubstringsMatch
#define generalizedTimeMatch			caseIgnoreIA5Match
#define generalizedTimeOrderingMatch	caseIgnoreIA5Match
#define uniqueMemberMatch				dnMatch

/* approx matching rules */
#define directoryStringApproxMatchOID	"1.3.6.1.4.1.4203.666.4.4"
#define directoryStringApproxMatch	approxMatch
#define directoryStringApproxIndexer	approxIndexer
#define directoryStringApproxFilter	approxFilter
#define IA5StringApproxMatchOID			"1.3.6.1.4.1.4203.666.4.5"
#define IA5StringApproxMatch			approxMatch
#define IA5StringApproxIndexer			approxIndexer
#define IA5StringApproxFilter			approxFilter

/* orderring matching rules */
#define caseIgnoreOrderingMatch			caseIgnoreMatch
#define caseExactOrderingMatch			caseExactMatch

/* unimplemented matching routines */
#define caseIgnoreListMatch				NULL
#define caseIgnoreListSubstringsMatch	NULL
#define protocolInformationMatch		NULL
#define integerFirstComponentMatch		NULL

#define OpenLDAPaciMatch				NULL
#define authPasswordMatch				NULL

/* recycled indexing/filtering routines */
#define dnIndexer				caseExactIgnoreIndexer
#define dnFilter				caseExactIgnoreFilter
#define bitStringFilter			octetStringFilter
#define bitStringIndexer		octetStringIndexer

#define telephoneNumberIndexer			caseIgnoreIA5Indexer
#define telephoneNumberFilter			caseIgnoreIA5Filter
#define telephoneNumberSubstringsIndexer	caseIgnoreIA5SubstringsIndexer
#define telephoneNumberSubstringsFilter		caseIgnoreIA5SubstringsFilter

/* must match OIDs below */
#define caseExactMatchOID			"2.5.13.5"
#define caseExactSubstringsMatchOID		"2.5.13.7"

static char *strcasechr( const char *str, int c )
{
	char *lower = strchr( str, TOLOWER(c) );
	char *upper = strchr( str, TOUPPER(c) );

	if( lower && upper ) {
		return lower < upper ? lower : upper;
	} else if ( lower ) {
		return lower;
	} else {
		return upper;
	}
}

static int
octetStringMatch(
	int *matchp,
	slap_mask_t flags,
	Syntax *syntax,
	MatchingRule *mr,
	struct berval *value,
	void *assertedValue )
{
	int match = value->bv_len - ((struct berval *) assertedValue)->bv_len;

	if( match == 0 ) {
		match = memcmp( value->bv_val,
			((struct berval *) assertedValue)->bv_val,
			value->bv_len );
	}

	*matchp = match;
	return LDAP_SUCCESS;
}

/* Index generation function */
int octetStringIndexer(
	slap_mask_t use,
	slap_mask_t flags,
	Syntax *syntax,
	MatchingRule *mr,
	struct berval *prefix,
	struct berval **values,
	struct berval ***keysp )
{
	int i;
	size_t slen, mlen;
	struct berval **keys;
	HASH_CONTEXT   HASHcontext;
	unsigned char	HASHdigest[HASH_BYTES];
	struct berval digest;
	digest.bv_val = HASHdigest;
	digest.bv_len = sizeof(HASHdigest);

	for( i=0; values[i] != NULL; i++ ) {
		/* just count them */
	}

	/* we should have at least one value at this point */
	assert( i > 0 );

	keys = ch_malloc( sizeof( struct berval * ) * (i+1) );

	slen = strlen( syntax->ssyn_oid );
	mlen = strlen( mr->smr_oid );

	for( i=0; values[i] != NULL; i++ ) {
		HASH_Init( &HASHcontext );
		if( prefix != NULL && prefix->bv_len > 0 ) {
			HASH_Update( &HASHcontext,
				prefix->bv_val, prefix->bv_len );
		}
		HASH_Update( &HASHcontext,
			syntax->ssyn_oid, slen );
		HASH_Update( &HASHcontext,
			mr->smr_oid, mlen );
		HASH_Update( &HASHcontext,
			values[i]->bv_val, values[i]->bv_len );
		HASH_Final( HASHdigest, &HASHcontext );

		keys[i] = ber_bvdup( &digest );
	}

	keys[i] = NULL;

	*keysp = keys;

	return LDAP_SUCCESS;
}

/* Index generation function */
int octetStringFilter(
	slap_mask_t use,
	slap_mask_t flags,
	Syntax *syntax,
	MatchingRule *mr,
	struct berval *prefix,
	void * assertValue,
	struct berval ***keysp )
{
	size_t slen, mlen;
	struct berval **keys;
	HASH_CONTEXT   HASHcontext;
	unsigned char	HASHdigest[HASH_BYTES];
	struct berval *value = (struct berval *) assertValue;
	struct berval digest;
	digest.bv_val = HASHdigest;
	digest.bv_len = sizeof(HASHdigest);

	slen = strlen( syntax->ssyn_oid );
	mlen = strlen( mr->smr_oid );

	keys = ch_malloc( sizeof( struct berval * ) * 2 );

	HASH_Init( &HASHcontext );
	if( prefix != NULL && prefix->bv_len > 0 ) {
		HASH_Update( &HASHcontext,
			prefix->bv_val, prefix->bv_len );
	}
	HASH_Update( &HASHcontext,
		syntax->ssyn_oid, slen );
	HASH_Update( &HASHcontext,
		mr->smr_oid, mlen );
	HASH_Update( &HASHcontext,
		value->bv_val, value->bv_len );
	HASH_Final( HASHdigest, &HASHcontext );

	keys[0] = ber_bvdup( &digest );
	keys[1] = NULL;

	*keysp = keys;

	return LDAP_SUCCESS;
}

#ifdef USE_LDAP_DN_PARSING
static int
dnValidate(
	Syntax *syntax,
	struct berval *in )
{
	int		rc;
	LDAPDN		*dn = NULL;

	if ( in->bv_len == 0 ) {
		return( LDAP_SUCCESS );
	}

	rc = ldap_str2dn( in->bv_val, &dn, LDAP_DN_FORMAT_LDAPV3 );
	ldapava_free_dn( dn );
	
	if ( rc != LDAP_SUCCESS ) {
		return( LDAP_INVALID_SYNTAX );
	}

	return( LDAP_SUCCESS );
}

static void
AVA_Sort( LDAPRDN *rdn, int iAVA )
{
	int		i;
	LDAPAVA		*ava_in = rdn[ iAVA ][ 0 ];
	
	for ( i = 0; i < iAVA; i++ ) {
		LDAPAVA		*ava = rdn[ i ][ 0 ];
		int		a, j;

		a = strcmp( ava_in->la_attr->bv_val, ava->la_attr->bv_val );

		if ( a > 0 ) {
			break;
		}

		while ( a == 0 ) {
			int		v, d;

			d = ava_in->la_value->bv_len - ava->la_value->bv_len;

			v = memcmp( ava_in->la_value->bv_val, 
					ava->la_value->bv_val,
					d <= 0 ? ava_in->la_value->bv_len 
						: ava->la_value->bv_len );

			if ( v == 0 && d != 0 ) {
				v = d;
			}

			if ( v <= 0 ) {
				/* 
				 * got it!
				 */
				break;
			}

			if ( ++i == iAVA ) {
				/*
				 * already sorted
				 */
				return;
			}

			ava = rdn[ i ][ 0 ];
			a = strcmp( ava_in->la_value->bv_val, 
					ava->la_value->bv_val );
		}

		/*
		 * move ahead
		 */
		for ( j = iAVA; j > i; j-- ) {
			rdn[ j ][ 0 ] = rdn[ j - 1 ][ 0 ];
		}
		rdn[ i ][ 0 ] = ava_in;

		return;
	}
}

/*
 * In-place normalization of the structural representation 
 * of a distinguished name
 */
static int
DN_Normalize( LDAPDN *dn )
{
	int 		iRDN;
	int 		rc;

	assert( dn );

	for ( iRDN = 0; dn[ iRDN ]; iRDN++ ) {
		LDAPRDN		*rdn = dn[ iRDN ][ 0 ];
		int		iAVA;

		for ( iAVA = 0; rdn[ iAVA ]; iAVA++ ) {
			LDAPAVA			*ava = rdn[ iAVA ][ 0 ];
			AttributeDescription	*ad = NULL;
			const char		*text = NULL;
			slap_syntax_transform_func *nf = NULL;
			struct berval		*bv = NULL;

			rc = slap_bv2ad( ava->la_attr, &ad, &text );
			if ( rc != LDAP_SUCCESS ) {
				return( LDAP_INVALID_SYNTAX );
			}

			/* 
			 * FIXME: is this required? 
			 */
			ber_bvfree( ava->la_attr );
			ava->la_attr = ber_bvdup( &ad->ad_cname );
				
			/*
			 * FIXME: What is this intended for?
			 */
			nf = ad->ad_type->sat_syntax->ssyn_normalize;
			if ( !nf ) {
				break;
			}
				
			rc = ( *nf )( ad->ad_type->sat_syntax,
				 ava->la_value, &bv );
			
			if ( rc != LDAP_SUCCESS ) {
				return( LDAP_INVALID_SYNTAX );
			}

			/*
			 * FIXME: shouldn't this happen inside 
			 * ssyn_normalize if the syntax is case 
			 * insensitive?
			 */
			if ( !( ava->la_flags & LDAP_AVA_BINARY ) ) {
				struct berval *s = bv;
				
				bv = ber_bvstr( UTF8normalize( bv, 
						UTF8_CASEFOLD ) );
				ber_bvfree( s );
			}

			ber_bvfree( ava->la_value );
			ava->la_value = bv;

			AVA_Sort( rdn, iAVA );
		}
	}

	return( LDAP_SUCCESS );
}

int
dnNormalize(
	Syntax *syntax,
	struct berval *val,
	struct berval **normalized )
{
	struct berval *out = NULL;

	Debug( LDAP_DEBUG_TRACE, ">>> dnNormalize: %s\n", val->bv_val, 0, 0 );

	if ( val->bv_len != 0 ) {
		LDAPDN		*dn = NULL;
		char		*dn_out = NULL;
		int		rc;

		/*
		 * Go to structural representation
		 */
		rc = ldap_str2dn( val->bv_val, &dn, LDAP_DN_FORMAT_LDAPV3 );
		if ( rc != LDAP_SUCCESS ) {
			return( LDAP_INVALID_SYNTAX );
		}

		/*
		 * Add schema-aware normalization stuff
		 */
		if ( DN_Normalize( dn ) != LDAP_SUCCESS ) {
			goto error_return;
		}

		/*
		 * Back to string representation
		 */
		rc = ldap_dn2str( dn, &dn_out, LDAP_DN_FORMAT_LDAPV3 );

		if ( rc != LDAP_SUCCESS ) {
error_return:;
			ldapava_free_dn( dn );
			return( LDAP_INVALID_SYNTAX );
		}

		ldapava_free_dn( dn );

		out = ber_bvstr( dn_out );

	} else {
		out = ber_bvdup( val );
	}

	Debug( LDAP_DEBUG_TRACE, "<<< dnNormalize: %s\n", out->bv_val, 0, 0 );

	*normalized = out;

	return( LDAP_SUCCESS );
}

int
dnPretty(
	Syntax *syntax,
	struct berval *val,
	struct berval **pretty)
{
	struct berval *out = NULL;

	if ( val->bv_len != 0 ) {
		LDAPDN		*dn = NULL;
		char		*dn_out = NULL;
		int		rc;

		/* FIXME: should be liberal in what we accept */
		rc = ldap_str2dn( val->bv_val, &dn, LDAP_DN_FORMAT_LDAPV3 );
		if ( rc != LDAP_SUCCESS ) {
			return( LDAP_INVALID_SYNTAX );
		}

		/* Add LDAPDN pretty code here
		 *   pretty'ing MUST preserve values
		 *   but MAY use alternative representation of value.
		 *
		 * That is, in addition to changes made by the
		 * str->LDAPDN->str processing, this code can
		 *	 convert values from BER to LDAP string representation
		 *	 	(or other direction if appropriate)
		 *	 "pretty" values
		 *	 normalize attribute descriptions
		 *	 reorder AVAs in RDNs
		 */

		/* FIXME: not sure why the default isn't pretty */
		rc = ldap_dn2str( dn, &dn_out,
			LDAP_DN_FORMAT_LDAPV3 | LDAP_DN_PRETTY );
		ldapava_free_dn( dn );

		if ( rc != LDAP_SUCCESS ) {
			return( LDAP_INVALID_SYNTAX );
		}

		out = ber_bvstr( dn_out );

	} else {
		out = ber_bvdup( val );
	}

	*pretty = out;

	return( LDAP_SUCCESS );
}

int
dnMatch(
	int *matchp,
	slap_mask_t flags,
	Syntax *syntax,
	MatchingRule *mr,
	struct berval *value,
	void *assertedValue )
{
	int match;
	struct berval *asserted = (struct berval *) assertedValue;
	
	match = value->bv_len - asserted->bv_len;

	if ( match == 0 ) {
#ifdef USE_DN_NORMALIZE
		match = strcmp( value->bv_val, asserted->bv_val );
		fprintf(stderr, "USE_DN_NORMALIZE :(\n");
#else
		match = strcasecmp( value->bv_val, asserted->bv_val );
		fprintf(stderr, "!USE_DN_NORMALIZE :)\n");
#endif
	}

#ifdef NEW_LOGGING
	LDAP_LOG(( "schema", LDAP_LEVEL_ENTRY,
		"dnMatch: %d\n    %s\n    %s\n", match,
		value->bv_val, asserted->bv_val ));
#else
	Debug( LDAP_DEBUG_ARGS, "dnMatch %d\n\t\"%s\"\n\t\"%s\"\n",
		match, value->bv_val, asserted->bv_val );
#endif

	*matchp = match;
	return( LDAP_SUCCESS );
}

#else /* !USE_LDAP_DN_PARSING */

static int
dnValidate(
	Syntax *syntax,
	struct berval *in )
{
	int rc;
	char *dn;

	if( in->bv_len == 0 ) return LDAP_SUCCESS;

	dn = ch_strdup( in->bv_val );

	if( dn == NULL ) {
		return LDAP_INVALID_SYNTAX;

	} else if ( strlen( in->bv_val ) != in->bv_len ) {
		rc = LDAP_INVALID_SYNTAX;

	} else if ( dn_validate( dn ) == NULL ) {
		rc = LDAP_INVALID_SYNTAX;

	} else {
		rc = LDAP_SUCCESS;
	}

	ch_free( dn );
	return rc;
}

int
dnNormalize(
	Syntax *syntax,
	struct berval *val,
	struct berval **normalized )
{
	struct berval *out;

	if ( val->bv_len != 0 ) {
		char *dn;
		out = ber_bvstr( UTF8normalize( val, UTF8_CASEFOLD ) );

		dn = dn_validate( out->bv_val );

		if( dn == NULL ) {
			ber_bvfree( out );
			return LDAP_INVALID_SYNTAX;
		}

		out->bv_val = dn;
		out->bv_len = strlen( dn );
	} else {
		out = ber_bvdup( val );
	}

	*normalized = out;
	return LDAP_SUCCESS;
}

int
dnMatch(
	int *matchp,
	slap_mask_t flags,
	Syntax *syntax,
	MatchingRule *mr,
	struct berval *value,
	void *assertedValue )
{
	int match;
	struct berval *asserted = (struct berval *) assertedValue;
	
	match = value->bv_len - asserted->bv_len;

	if( match == 0 ) {
#ifdef USE_DN_NORMALIZE
		match = strcmp( value->bv_val, asserted->bv_val );
#else
		match = strcasecmp( value->bv_val, asserted->bv_val );
#endif
	}

#ifdef NEW_LOGGING
	LDAP_LOG(( "schema", LDAP_LEVEL_ENTRY,
		"dnMatch: %d\n    %s\n    %s\n", match,
		value->bv_val, asserted->bv_val ));
#else
	Debug( LDAP_DEBUG_ARGS, "dnMatch %d\n\t\"%s\"\n\t\"%s\"\n",
		match, value->bv_val, asserted->bv_val );
#endif


	*matchp = match;
	return LDAP_SUCCESS;
}

#endif /* !USE_LDAP_DN_PARSING */

static int
nameUIDValidate(
	Syntax *syntax,
	struct berval *in )
{
	int rc;
	struct berval *dn;

	if( in->bv_len == 0 ) return LDAP_SUCCESS;

	dn = ber_bvdup( in );

	if( dn->bv_val[dn->bv_len-1] == '\'' ) {
		/* assume presence of optional UID */
		ber_len_t i;

		for(i=dn->bv_len-2; i>2; i--) {
			if( dn->bv_val[i] != '0' &&	dn->bv_val[i] != '1' ) {
				break;
			}
		}
		if( dn->bv_val[i] != '\'' ||
		    dn->bv_val[i-1] != 'B' ||
		    dn->bv_val[i-2] != '#' ) {
			ber_bvfree( dn );
			return LDAP_INVALID_SYNTAX;
		}

		/* trim the UID to allow use of dn_validate */
		dn->bv_val[i-2] = '\0';
	}

	rc = dn_validate( dn->bv_val ) == NULL
		? LDAP_INVALID_SYNTAX : LDAP_SUCCESS;

	ber_bvfree( dn );
	return rc;
}

static int
nameUIDNormalize(
	Syntax *syntax,
	struct berval *val,
	struct berval **normalized )
{
	struct berval *out = ber_bvdup( val );

	if( out->bv_len != 0 ) {
		char *dn;
		ber_len_t dnlen;
		char *uid = NULL;
		ber_len_t uidlen = 0;

		if( out->bv_val[out->bv_len-1] == '\'' ) {
			/* assume presence of optional UID */
			uid = strrchr( out->bv_val, '#' );

			if( uid == NULL ) {
				ber_bvfree( out );
				return LDAP_INVALID_SYNTAX;
			}

			uidlen = out->bv_len - (out->bv_val - uid);
			/* temporarily trim the UID */
			*uid = '\0';
		}

#ifdef USE_DN_NORMALIZE
		dn = dn_normalize( out->bv_val );
#else
		dn = dn_validate( out->bv_val );
#endif

		if( dn == NULL ) {
			ber_bvfree( out );
			return LDAP_INVALID_SYNTAX;
		}

		dnlen = strlen(dn);

		if( uidlen ) {
			/* restore the separator */
			*uid = '#';
			/* shift the UID */
			SAFEMEMCPY( &dn[dnlen], uid, uidlen );
		}

		out->bv_val = dn;
		out->bv_len = dnlen + uidlen;
	}

	*normalized = out;
	return LDAP_SUCCESS;
}

static int
inValidate(
	Syntax *syntax,
	struct berval *in )
{
	/* any value allowed */
	return LDAP_OTHER;
}

static int
blobValidate(
	Syntax *syntax,
	struct berval *in )
{
	/* any value allowed */
	return LDAP_SUCCESS;
}

static int
bitStringValidate(
	Syntax *syntax,
	struct berval *in )
{
	ber_len_t i;

	/* very unforgiving validation, requires no normalization
	 * before simplistic matching
	 */
	if( in->bv_len < 3 ) {
		return LDAP_INVALID_SYNTAX;
	}

	/*
	 * rfc 2252 section 6.3 Bit String
	 * bitstring = "'" *binary-digit "'"
	 * binary-digit = "0" / "1"
	 * example: '0101111101'B
	 */
	
	if( in->bv_val[0] != '\'' ||
		in->bv_val[in->bv_len-2] != '\'' ||
		in->bv_val[in->bv_len-1] != 'B' )
	{
		return LDAP_INVALID_SYNTAX;
	}

	for( i=in->bv_len-3; i>0; i-- ) {
		if( in->bv_val[i] != '0' && in->bv_val[i] != '1' ) {
			return LDAP_INVALID_SYNTAX;
		}
	}

	return LDAP_SUCCESS;
}

static int
bitStringNormalize(
	Syntax *syntax,
	struct berval *val,
	struct berval **normalized )
{
	/*
     * A normalized bitString is has no extaneous (leading) zero bits.
	 * That is, '00010'B is normalized to '10'B
	 * However, as a special case, '0'B requires no normalization.
     */
	struct berval *newval;
	char *p;

	/* start at the first bit */
	p = &val->bv_val[1];

	/* Find the first non-zero bit */
	while ( *p == '0' ) p++;

	newval = (struct berval *) ch_malloc( sizeof(struct berval) );

	if( *p == '\'' ) {
		/* no non-zero bits */
		newval->bv_val = ch_strdup("\'0\'B");
		newval->bv_len = sizeof("\'0\'B") - 1;
		goto done;
	}

	newval->bv_val = ch_malloc( val->bv_len + 1 );

	newval->bv_val[0] = '\'';
	newval->bv_len = 1;

	for( ; *p != '\0'; p++ ) {
		newval->bv_val[newval->bv_len++] = *p;
	}

	newval->bv_val[newval->bv_len] = '\0';

done:
	*normalized = newval;
	return LDAP_SUCCESS;
}

/*
 * Handling boolean syntax and matching is quite rigid.
 * A more flexible approach would be to allow a variety
 * of strings to be normalized and prettied into TRUE
 * and FALSE.
 */
static int
booleanValidate(
	Syntax *syntax,
	struct berval *in )
{
	/* very unforgiving validation, requires no normalization
	 * before simplistic matching
	 */

	if( in->bv_len == 4 ) {
		if( !memcmp( in->bv_val, "TRUE", 4 ) ) {
			return LDAP_SUCCESS;
		}
	} else if( in->bv_len == 5 ) {
		if( !memcmp( in->bv_val, "FALSE", 5 ) ) {
			return LDAP_SUCCESS;
		}
	}

	return LDAP_INVALID_SYNTAX;
}

static int
booleanMatch(
	int *matchp,
	slap_mask_t flags,
	Syntax *syntax,
	MatchingRule *mr,
	struct berval *value,
	void *assertedValue )
{
	/* simplistic matching allowed by rigid validation */
	struct berval *asserted = (struct berval *) assertedValue;
	*matchp = value->bv_len != asserted->bv_len;
	return LDAP_SUCCESS;
}

static int
UTF8StringValidate(
	Syntax *syntax,
	struct berval *in )
{
	ber_len_t count;
	int len;
	unsigned char *u = in->bv_val;

	if( !in->bv_len ) return LDAP_INVALID_SYNTAX;

	for( count = in->bv_len; count > 0; count-=len, u+=len ) {
		/* get the length indicated by the first byte */
		len = LDAP_UTF8_CHARLEN( u );

		/* should not be zero */
		if( len == 0 ) return LDAP_INVALID_SYNTAX;

		/* make sure len corresponds with the offset
			to the next character */
		if( LDAP_UTF8_OFFSET( u ) != len ) return LDAP_INVALID_SYNTAX;
	}

	if( count != 0 ) return LDAP_INVALID_SYNTAX;

	return LDAP_SUCCESS;
}

static int
UTF8StringNormalize(
	Syntax *syntax,
	struct berval *val,
	struct berval **normalized )
{
	struct berval *newval;
	char *p, *q, *s;

	newval = ch_malloc( sizeof( struct berval ) );

	p = val->bv_val;

	/* Ignore initial whitespace */
	while ( ldap_utf8_isspace( p ) ) {
		LDAP_UTF8_INCR( p );
	}

	if( *p == '\0' ) {
		ch_free( newval );
		return LDAP_INVALID_SYNTAX;
	}

	newval->bv_val = ch_strdup( p );
	p = q = newval->bv_val;
	s = NULL;

	while ( *p ) {
		int len;

		if ( ldap_utf8_isspace( p ) ) {
			len = LDAP_UTF8_COPY(q,p);
			s=q;
			p+=len;
			q+=len;

			/* Ignore the extra whitespace */
			while ( ldap_utf8_isspace( p ) ) {
				LDAP_UTF8_INCR( p );
			}
		} else {
			len = LDAP_UTF8_COPY(q,p);
			s=NULL;
			p+=len;
			q+=len;
		}
	}

	assert( *newval->bv_val );
	assert( newval->bv_val < p );
	assert( q <= p );

	/* cannot start with a space */
	assert( !ldap_utf8_isspace(newval->bv_val) );

	/*
	 * If the string ended in space, backup the pointer one
	 * position.  One is enough because the above loop collapsed
	 * all whitespace to a single space.
	 */

	if ( s != NULL ) {
		q = s;
	}

	/* cannot end with a space */
	assert( !ldap_utf8_isspace( LDAP_UTF8_PREV(q) ) );

	/* null terminate */
	*q = '\0';

	newval->bv_len = q - newval->bv_val;
	*normalized = newval;

	return LDAP_SUCCESS;
}

/* Returns Unicode cannonically normalized copy of a substring assertion
 * Skipping attribute description */
SubstringsAssertion *
UTF8SubstringsassertionNormalize(
	SubstringsAssertion *sa,
	char casefold )
{
	SubstringsAssertion *nsa;
	int i;

	nsa = (SubstringsAssertion *)ch_calloc( 1, sizeof(SubstringsAssertion) );
	if( nsa == NULL ) {
		return NULL;
	}

	if( sa->sa_initial != NULL ) {
		nsa->sa_initial = ber_bvstr( UTF8normalize( sa->sa_initial, casefold ) );
		if( nsa->sa_initial == NULL ) {
			goto err;
		}
	}

	if( sa->sa_any != NULL ) {
		for( i=0; sa->sa_any[i] != NULL; i++ ) {
			/* empty */
		}
		nsa->sa_any = (struct berval **)ch_malloc( (i + 1) * sizeof(struct berval *) );
		for( i=0; sa->sa_any[i] != NULL; i++ ) {
			nsa->sa_any[i] = ber_bvstr( UTF8normalize( sa->sa_any[i], casefold ) );
			if( nsa->sa_any[i] == NULL ) {
				goto err;
			}
		}
		nsa->sa_any[i] = NULL;
	}

	if( sa->sa_final != NULL ) {
		nsa->sa_final = ber_bvstr( UTF8normalize( sa->sa_final, casefold ) );
		if( nsa->sa_final == NULL ) {
			goto err;
		}
	}

	return nsa;

err:
	ber_bvfree( nsa->sa_final );
	ber_bvecfree( nsa->sa_any );
	ber_bvfree( nsa->sa_initial );
	ch_free( nsa );
	return NULL;
}

/* Strip characters with the 8th bit set */
char *
strip8bitChars(
	char *in )      
{
	char *p = in, *q;
  
	if( in == NULL ) {
		return NULL;
	}
	while( *p ) {
		if( *p & 0x80 ) {
			q = p;
			while( *++q & 0x80 ) {
				/* empty */
			}
			p = memmove(p, q, strlen(q) + 1);
		} else {
			p++;
		}
	}
	return in;
}

#ifndef SLAPD_APPROX_OLDSINGLESTRING

#if defined(SLAPD_APPROX_INITIALS)
#define SLAPD_APPROX_DELIMITER "._ "
#define SLAPD_APPROX_WORDLEN 2
#else
#define SLAPD_APPROX_DELIMITER " "
#define SLAPD_APPROX_WORDLEN 1
#endif

static int
approxMatch(
	int *matchp,
	slap_mask_t flags,
	Syntax *syntax,
	MatchingRule *mr,
	struct berval *value,
	void *assertedValue )
{
	char *val, *nval, *assertv, **values, **words, *c;
	int i, count, len, nextchunk=0, nextavail=0;
	size_t avlen;

	/* Yes, this is necessary */
	nval = UTF8normalize( value, UTF8_NOCASEFOLD );
	if( nval == NULL ) {
		*matchp = 1;
		return LDAP_SUCCESS;
	}
	strip8bitChars( nval );

	/* Yes, this is necessary */
	assertv = UTF8normalize( ((struct berval *)assertedValue),
				 UTF8_NOCASEFOLD );
	if( assertv == NULL ) {
		ch_free( nval );
		*matchp = 1;
		return LDAP_SUCCESS;
	}
	strip8bitChars( assertv );
	avlen = strlen( assertv );

	/* Isolate how many words there are */
	for( c=nval,count=1; *c; c++ ) {
		c = strpbrk( c, SLAPD_APPROX_DELIMITER );
		if ( c == NULL ) break;
		*c = '\0';
		count++;
	}

	/* Get a phonetic copy of each word */
	words = (char **)ch_malloc( count * sizeof(char *) );
	values = (char **)ch_malloc( count * sizeof(char *) );
	for( c=nval,i=0;  i<count;  i++,c+=strlen(c)+1 ) {
		words[i] = c;
		values[i] = phonetic(c);
	}

	/* Work through the asserted value's words, to see if at least some
	   of the words are there, in the same order. */
	len = 0;
	while ( nextchunk < avlen ) {
		len = strcspn( assertv + nextchunk, SLAPD_APPROX_DELIMITER);
		if( len == 0 ) {
			nextchunk++;
			continue;
		}
#if defined(SLAPD_APPROX_INITIALS)
		else if( len == 1 ) {
			/* Single letter words need to at least match one word's initial */
			for( i=nextavail; i<count; i++ )
				if( !strncasecmp( assertv+nextchunk, words[i], 1 )) {
					nextavail=i+1;
					break;
				}
		}
#endif
		else {
			/* Isolate the next word in the asserted value and phonetic it */
			assertv[nextchunk+len] = '\0';
			val = phonetic( assertv + nextchunk );

			/* See if this phonetic chunk is in the remaining words of *value */
			for( i=nextavail; i<count; i++ ){
				if( !strcmp( val, values[i] ) ){
					nextavail = i+1;
					break;
				}
			}
			ch_free( val );
		}

		/* This chunk in the asserted value was NOT within the *value. */
		if( i >= count ) {
			nextavail=-1;
			break;
		}

		/* Go on to the next word in the asserted value */
		nextchunk += len+1;
	}

	/* If some of the words were seen, call it a match */
	if( nextavail > 0 ) {
		*matchp = 0;
	}
	else {
		*matchp = 1;
	}

	/* Cleanup allocs */
	free( assertv );
	for( i=0; i<count; i++ ) {
		ch_free( values[i] );
	}
	ch_free( values );
	ch_free( words );
	ch_free( nval );

	return LDAP_SUCCESS;
}

int 
approxIndexer(
	slap_mask_t use,
	slap_mask_t flags,
	Syntax *syntax,
	MatchingRule *mr,
	struct berval *prefix,
	struct berval **values,
	struct berval ***keysp )
{
	char *val, *c;
	int i,j, len, wordcount, keycount=0;
	struct berval **newkeys, **keys=NULL;

	for( j=0; values[j] != NULL; j++ ) {
		/* Yes, this is necessary */
		val = UTF8normalize( values[j], UTF8_NOCASEFOLD );
		strip8bitChars( val );

		/* Isolate how many words there are. There will be a key for each */
		for( wordcount=0,c=val;	 *c;  c++) {
			len = strcspn(c, SLAPD_APPROX_DELIMITER);
			if( len >= SLAPD_APPROX_WORDLEN ) wordcount++;
			c+= len;
			if (*c == '\0') break;
			*c = '\0';
		}

		/* Allocate/increase storage to account for new keys */
		newkeys = (struct berval **)ch_malloc( (keycount + wordcount + 1) 
			* sizeof(struct berval *) );
		memcpy( newkeys, keys, keycount * sizeof(struct berval *) );
		if( keys ) ch_free( keys );
		keys = newkeys;

		/* Get a phonetic copy of each word */
		for( c=val,i=0;	 i<wordcount;  c+=len+1	 ) {
			len = strlen( c );
			if( len < SLAPD_APPROX_WORDLEN ) continue;
			keys[keycount] = (struct berval *)ch_malloc( sizeof(struct berval) );
			keys[keycount]->bv_val = phonetic( c );
			keys[keycount]->bv_len = strlen( keys[keycount]->bv_val );
			keycount++;
			i++;
		}

		free( val );
	}
	keys[keycount] = NULL;
	*keysp = keys;

	return LDAP_SUCCESS;
}

int 
approxFilter(
	slap_mask_t use,
	slap_mask_t flags,
	Syntax *syntax,
	MatchingRule *mr,
	struct berval *prefix,
	void * assertValue,
	struct berval ***keysp )
{
	char *val, *c;
	int i, count, len;
	struct berval **keys;

	/* Yes, this is necessary */
	val = UTF8normalize( ((struct berval *)assertValue),
			     UTF8_NOCASEFOLD );
	if( val == NULL ) {
		keys = (struct berval **)ch_malloc( sizeof(struct berval *) );
		keys[0] = NULL;
		*keysp = keys;
		return LDAP_SUCCESS;
	}
	strip8bitChars( val );

	/* Isolate how many words there are. There will be a key for each */
	for( count=0,c=val;  *c;  c++) {
		len = strcspn(c, SLAPD_APPROX_DELIMITER);
		if( len >= SLAPD_APPROX_WORDLEN ) count++;
		c+= len;
		if (*c == '\0') break;
		*c = '\0';
	}

	/* Allocate storage for new keys */
	keys = (struct berval **)ch_malloc( (count + 1) * sizeof(struct berval *) );

	/* Get a phonetic copy of each word */
	for( c=val,i=0;	 i<count; c+=len+1 ) {
		len = strlen(c);
		if( len < SLAPD_APPROX_WORDLEN ) continue;
		keys[i] = ber_bvstr( phonetic( c ) );
		i++;
	}

	free( val );

	keys[count] = NULL;
	*keysp = keys;

	return LDAP_SUCCESS;
}


#else
/* No other form of Approximate Matching is defined */

static int
approxMatch(
	int *matchp,
	slap_mask_t flags,
	Syntax *syntax,
	MatchingRule *mr,
	struct berval *value,
	void *assertedValue )
{
	char *vapprox, *avapprox;
	char *s, *t;

	/* Yes, this is necessary */
	s = UTF8normalize( value, UTF8_NOCASEFOLD );
	if( s == NULL ) {
		*matchp = 1;
		return LDAP_SUCCESS;
	}

	/* Yes, this is necessary */
	t = UTF8normalize( ((struct berval *)assertedValue),
			   UTF8_NOCASEFOLD );
	if( t == NULL ) {
		free( s );
		*matchp = -1;
		return LDAP_SUCCESS;
	}

	vapprox = phonetic( strip8bitChars( s ) );
	avapprox = phonetic( strip8bitChars( t ) );

	free( s );
	free( t );

	*matchp = strcmp( vapprox, avapprox );

	ch_free( vapprox );
	ch_free( avapprox );

	return LDAP_SUCCESS;
}

int 
approxIndexer(
	slap_mask_t use,
	slap_mask_t flags,
	Syntax *syntax,
	MatchingRule *mr,
	struct berval *prefix,
	struct berval **values,
	struct berval ***keysp )
{
	int i;
	struct berval **keys;
	char *s;

	for( i=0; values[i] != NULL; i++ ) {
		/* empty - just count them */
	}

	/* we should have at least one value at this point */
	assert( i > 0 );

	keys = (struct berval **)ch_malloc( sizeof( struct berval * ) * (i+1) );

	/* Copy each value and run it through phonetic() */
	for( i=0; values[i] != NULL; i++ ) {
		/* Yes, this is necessary */
		s = UTF8normalize( values[i], UTF8_NOCASEFOLD );

		/* strip 8-bit chars and run through phonetic() */
		keys[i] = ber_bvstr( phonetic( strip8bitChars( s ) ) );
		free( s );
	}
	keys[i] = NULL;

	*keysp = keys;
	return LDAP_SUCCESS;
}


int 
approxFilter(
	slap_mask_t use,
	slap_mask_t flags,
	Syntax *syntax,
	MatchingRule *mr,
	struct berval *prefix,
	void * assertValue,
	struct berval ***keysp )
{
	struct berval **keys;
	char *s;

	keys = (struct berval **)ch_malloc( sizeof( struct berval * ) * 2 );

	/* Yes, this is necessary */
	s = UTF8normalize( ((struct berval *)assertValue),
			     UTF8_NOCASEFOLD );
	if( s == NULL ) {
		keys[0] = NULL;
	} else {
		/* strip 8-bit chars and run through phonetic() */
		keys[0] = ber_bvstr( phonetic( strip8bitChars( s ) ) );
		free( s );
		keys[1] = NULL;
	}

	*keysp = keys;
	return LDAP_SUCCESS;
}
#endif


static int
caseExactMatch(
	int *matchp,
	slap_mask_t flags,
	Syntax *syntax,
	MatchingRule *mr,
	struct berval *value,
	void *assertedValue )
{
	*matchp = UTF8normcmp( value->bv_val,
		((struct berval *) assertedValue)->bv_val,
		UTF8_NOCASEFOLD );
	return LDAP_SUCCESS;
}

static int
caseExactIgnoreSubstringsMatch(
	int *matchp,
	slap_mask_t flags,
	Syntax *syntax,
	MatchingRule *mr,
	struct berval *value,
	void *assertedValue )
{
	int match = 0;
	SubstringsAssertion *sub = NULL;
	struct berval left;
	int i;
	ber_len_t inlen=0;
	char *nav, casefold;

	casefold = strcmp( mr->smr_oid, caseExactSubstringsMatchOID )
		? UTF8_CASEFOLD : UTF8_NOCASEFOLD;

	nav = UTF8normalize( value, casefold );