Integer indexing, sync with HEAD

include/lutil.h

@@ -304,9 +304,8 @@ lutil_atoulx( unsigned long *v, const char *s, int x );
 #define lutil_atol(v, s)	lutil_atolx((v), (s), 10)
 #define lutil_atoul(v, s)	lutil_atoulx((v), (s), 10)
 
-/* Parse and unparse time intervals */
 LDAP_LUTIL_F (int)
-lutil_str2bin( struct berval *in, struct berval *out );
+lutil_str2bin( struct berval *in, struct berval *out, void *ctx );
 
 /* Parse and unparse time intervals */
 LDAP_LUTIL_F (int)

libraries/liblutil/utils.c

@@ -35,6 +35,7 @@
 #include "lutil.h"
 #include "ldap_defaults.h"
 #include "ldap_pvt.h"
+#include "lber_pvt.h"
 
 #ifdef HAVE_EBCDIC
 int _trans_argv = 1;
@@ -671,7 +672,7 @@ scale( int new, lutil_int_decnum *prev, unsigned char *tmp )
  * any hex input.
  */
 int
-lutil_str2bin( struct berval *in, struct berval *out )
+lutil_str2bin( struct berval *in, struct berval *out, void *ctx )
 {
 	char *pin, *pout, ctmp;
 	char *end;
@@ -747,7 +748,7 @@ lutil_str2bin( struct berval *in, struct berval *out )
 
 		/* tmp must be at least as large as outbuf */
 		if ( out->bv_len > sizeof(tmpbuf)) {
-			tmp = ber_memalloc( out->bv_len );
+			tmp = ber_memalloc_x( out->bv_len, ctx );
 		} else {
 			tmp = tmpbuf;
 		}
@@ -795,7 +796,7 @@ lutil_str2bin( struct berval *in, struct berval *out )
 		out->bv_len = num.len;
 decfail:
 		if ( tmp != tmpbuf ) {
-			ber_memfree( tmp );
+			ber_memfree_x( tmp, ctx );
 		}
 	}
 	return rc;

servers/slapd/bconfig.c

@@ -1160,6 +1160,8 @@ config_generic(ConfigArgs *c) {
 
 		case CFG_IX_INTLEN:
 			index_intlen = SLAP_INDEX_INTLEN_DEFAULT;
+			index_intlen_strlen = SLAP_INDEX_INTLEN_STRLEN(
+				SLAP_INDEX_INTLEN_DEFAULT );
 			break;
 
 		case CFG_ACL:
@@ -1511,6 +1513,8 @@ config_generic(ConfigArgs *c) {
 			else if ( c->value_int > 255 )
 				c->value_int = 255;
 			index_intlen = c->value_int;
+			index_intlen_strlen = SLAP_INDEX_INTLEN_STRLEN(
+				index_intlen );
 			break;
 			
 		case CFG_SORTVALS: {

servers/slapd/proto-slap.h

@@ -1877,6 +1877,10 @@ LDAP_SLAPD_V (unsigned int) index_substr_if_maxlen;
 LDAP_SLAPD_V (unsigned int) index_substr_any_len;
 LDAP_SLAPD_V (unsigned int) index_substr_any_step;
 LDAP_SLAPD_V (unsigned int) index_intlen;
+/* all signed integers from strings of this size need more than intlen bytes */
+/* i.e. log(10)*(index_intlen_strlen-2) > log(2)*(8*(index_intlen)-1) */
+LDAP_SLAPD_V (unsigned int) index_intlen_strlen;
+#define SLAP_INDEX_INTLEN_STRLEN(intlen) ((8*(intlen)-1) * 146/485 + 3)
 
 LDAP_SLAPD_V (ber_len_t) sockbuf_max_incoming;
 LDAP_SLAPD_V (ber_len_t) sockbuf_max_incoming_auth;

servers/slapd/schema_init.c

@@ -63,6 +63,8 @@ unsigned int index_substr_any_len = SLAP_INDEX_SUBSTR_ANY_LEN_DEFAULT;
 unsigned int index_substr_any_step = SLAP_INDEX_SUBSTR_ANY_STEP_DEFAULT;
 
 unsigned int index_intlen = SLAP_INDEX_INTLEN_DEFAULT;
+unsigned int index_intlen_strlen = SLAP_INDEX_INTLEN_STRLEN(
+	SLAP_INDEX_INTLEN_DEFAULT );
 
 ldap_pvt_thread_mutex_t	ad_undef_mutex;
 ldap_pvt_thread_mutex_t	oc_undef_mutex;
@@ -2112,51 +2114,72 @@ integerMatch(
 	return LDAP_SUCCESS;
 }
 
+/* 10**Chop < 256**Chopbytes and Chop > Chopbytes<<1 (for sign bit and itmp) */
+#define INDEX_INTLEN_CHOP 7
+#define INDEX_INTLEN_CHOPBYTES 3
+
 static int
 integerVal2Key(
-	struct berval *val,
+	struct berval *in,
 	struct berval *key,
-	struct berval *tmp
-)
+	struct berval *tmp,
+	void *ctx )
 {
-	struct berval iv;
-	int neg;
+	/* index format:
+	 * only if too large: one's complement <sign*exponent (chopped bytes)>,
+	 * two's complement value (sign-extended or chopped as needed),
+	 * however the top <number of exponent-bytes + 1> bits of first byte
+	 * above is the inverse sign.   The next bit is the sign as delimiter.
+	 */
+	ber_slen_t k = index_intlen_strlen;
+	ber_len_t chop = 0;
+	unsigned signmask = ~0x7fU;
+	unsigned char lenbuf[sizeof(k) + 2], *lenp, neg = 0xff;
+	struct berval val = *in, itmp = *tmp;
 
-	iv = *tmp;
-	if ( lutil_str2bin( val, &iv )) {
-		return LDAP_INVALID_SYNTAX;
+	if ( val.bv_val[0] != '-' ) {
+		neg = 0;
+		--k;
 	}
 
-	neg = iv.bv_val[0] & 0x80;
-
-	/* Omit leading 0 pad byte */
-	if ( !iv.bv_val[0] ) {
-		iv.bv_val++;
-		iv.bv_len--;
+	/* Chop least significant digits, increase length instead */
+	if ( val.bv_len > (ber_len_t) k ) {
+		chop = (val.bv_len-k+2)/INDEX_INTLEN_CHOP; /* 2 fewer digits */
+		val.bv_len -= chop * INDEX_INTLEN_CHOP;	/* #digits chopped */
+		chop *= INDEX_INTLEN_CHOPBYTES;		/* #bytes added */
 	}
 
-	/* If too small, sign-extend */
-	if ( iv.bv_len < index_intlen ) {
-		int j, k, pad;
-		key->bv_val[0] = index_intlen;
-		k = index_intlen - iv.bv_len + 1;
-		if ( neg )
-			pad = 0xff;
-		else
-			pad = 0;
-		for ( j=1; j<k; j++)
-			key->bv_val[j] = pad;
-		for ( j = 0; j<iv.bv_len; j++ )
-			key->bv_val[j+k] = iv.bv_val[j];
-	} else {
-		key->bv_val[0] = iv.bv_len;
-		memcpy( key->bv_val+1, iv.bv_val, index_intlen );
-	}
-	if ( neg ) {
-		key->bv_val[0] = -key->bv_val[0];
+	if ( lutil_str2bin( &val, &itmp, ctx )) {
+		return LDAP_INVALID_SYNTAX;
 	}
-	/* convert signed to unsigned */
-	key->bv_val[0] ^= 0x80;
+
+	/* Omit leading sign byte */
+	if ( itmp.bv_val[0] == neg ) {
+		itmp.bv_val++;
+		itmp.bv_len--;
+	}
+
+	k = (ber_slen_t) index_intlen - (ber_slen_t) (itmp.bv_len + chop);
+	if ( k > 0 ) {
+		assert( chop == 0 );
+		memset( key->bv_val, neg, k );	/* sign-extend */
+	} else if ( k != 0 || ((itmp.bv_val[0] ^ neg) & 0xc0) ) {
+		lenp = lenbuf + sizeof(lenbuf);
+		chop = - (ber_len_t) k;
+		do {
+			*--lenp = ((unsigned char) chop & 0xff) ^ neg;
+			signmask >>= 1;
+		} while ( (chop >>= 8) != 0 || (signmask >> 1) & (*lenp ^ neg) );
+		/* With n bytes in lenbuf, the top n+1 bits of (signmask&0xff)
+		 * are 1, and the top n+2 bits of lenp[] are the sign bit. */
+		k = (lenbuf + sizeof(lenbuf)) - lenp;
+		if ( k > (ber_slen_t) index_intlen )
+			k = index_intlen;
+		memcpy( key->bv_val, lenp, k );
+		itmp.bv_len = index_intlen - k;
+	}
+	memcpy( key->bv_val + k, itmp.bv_val, itmp.bv_len );
+	key->bv_val[0] ^= (unsigned char) signmask & 0xff; /* invert sign */
 	return 0;
 }
 
@@ -2175,36 +2198,46 @@ integerIndexer(
 	char ibuf[64];
 	struct berval itmp;
 	BerVarray keys;
+	ber_len_t vlen;
 	int i, rc;
+	unsigned maxstrlen = index_intlen_strlen + INDEX_INTLEN_CHOP-1;
 
-	for( i=0; !BER_BVISNULL( &values[i] ); i++ ) {
-		/* just count them */
+	/* count the values and find max needed length */
+	vlen = 0;
+	for( i = 0; !BER_BVISNULL( &values[i] ); i++ ) {
+		if ( vlen < values[i].bv_len )
+			vlen = values[i].bv_len;
 	}
+	if ( vlen > maxstrlen )
+		vlen = maxstrlen;
 
 	/* we should have at least one value at this point */
 	assert( i > 0 );
 
 	keys = slap_sl_malloc( sizeof( struct berval ) * (i+1), ctx );
 	for ( i = 0; !BER_BVISNULL( &values[i] ); i++ ) {
-		keys[i].bv_len = index_intlen+1;
-		keys[i].bv_val = slap_sl_malloc( index_intlen+1, ctx );
+		keys[i].bv_len = index_intlen;
+		keys[i].bv_val = slap_sl_malloc( index_intlen, ctx );
 	}
 	keys[i].bv_len = 0;
 	keys[i].bv_val = NULL;
 
-	itmp.bv_val = ibuf;
+	if ( vlen > sizeof(ibuf) ) {
+		itmp.bv_val = slap_sl_malloc( vlen, ctx );
+	} else {
+		itmp.bv_val = ibuf;
+	}
 	itmp.bv_len = sizeof(ibuf);
 
 	for ( i=0; !BER_BVISNULL( &values[i] ); i++ ) {
-		if ( values[i].bv_len > itmp.bv_len ) {
+		if ( itmp.bv_val != ibuf ) {
 			itmp.bv_len = values[i].bv_len;
-			if ( itmp.bv_val == ibuf ) {
-				itmp.bv_val = slap_sl_malloc( itmp.bv_len, ctx );
-			} else {
-				itmp.bv_val = slap_sl_realloc( itmp.bv_val, itmp.bv_len, ctx );
-			}
+			if ( itmp.bv_len <= sizeof(ibuf) )
+				itmp.bv_len = sizeof(ibuf);
+			else if ( itmp.bv_len > maxstrlen )
+				itmp.bv_len = maxstrlen;
 		}
-		rc = integerVal2Key( &values[i], &keys[i], &itmp );
+		rc = integerVal2Key( &values[i], &keys[i], &itmp, ctx );
 		if ( rc )
 			goto leave;
 	}
@@ -2238,18 +2271,19 @@ integerFilter(
 
 	keys = slap_sl_malloc( sizeof( struct berval ) * 2, ctx );
 
-	keys[0].bv_len = index_intlen + 1;
-	keys[0].bv_val = slap_sl_malloc( index_intlen+1, ctx );
+	keys[0].bv_len = index_intlen;
+	keys[0].bv_val = slap_sl_malloc( index_intlen, ctx );
 
-	if ( value->bv_len > sizeof( ibuf )) {
-		iv.bv_val = slap_sl_malloc( value->bv_len, ctx );
-		iv.bv_len = value->bv_len;
+	iv.bv_len = value->bv_len < index_intlen_strlen + INDEX_INTLEN_CHOP-1
+		? value->bv_len : index_intlen_strlen + INDEX_INTLEN_CHOP-1;
+	if ( iv.bv_len > (int) sizeof(ibuf) ) {
+		iv.bv_val = slap_sl_malloc( iv.bv_len, ctx );
 	} else {
 		iv.bv_val = ibuf;
 		iv.bv_len = sizeof(ibuf);
 	}
 
-	rc = integerVal2Key( value, keys, &iv );
+	rc = integerVal2Key( value, keys, &iv, ctx );
 	if ( rc == 0 )
 		*keysp = keys;
 
@@ -3106,7 +3140,7 @@ serialNumberAndIssuerNormalize(
 	}
 	sn2.bv_val = stmp;
 	sn2.bv_len = sn.bv_len;
-	if ( lutil_str2bin( &sn, &sn2 )) {
+	if ( lutil_str2bin( &sn, &sn2, ctx )) {
 		rc = LDAP_INVALID_SYNTAX;
 		goto leave;
 	}