ITS#5239 -- Ordered indexing for integer attributes

CHANGES

 OpenLDAP 2.4 Change Log
 
 OpenLDAP 2.4.7 Engineering
+	Added slapd ordered indexing of integer attributes (ITS#5239)
 	Fixed slapd paged results control handling (ITS#5191)
 	Fixed slapd sasl-host parsing (ITS#5209)
 	Fixed slapd filter normalization (ITS#5212)

doc/man/man5/slapd-config.5

@@ -445,7 +445,7 @@ server
 .B with another database,
 without disrupting the currently active clients.
 The default is FALSE.  You may wish to use
-.B olcIdletTmeout
+.B olcIdleTimeout
 along with this option.
 .TP
 .B olcIdleTimeout: <integer>
@@ -453,6 +453,11 @@ Specify the number of seconds to wait before forcibly closing
 an idle client connection.  A setting of 0 disables this
 feature.  The default is 0.
 .TP
+.B olcIndexIntLen: <integer>
+Specify the key length for ordered integer indices. The most significant
+bytes of the binary integer will be used for index keys. The default
+value is 4, which provides exact indexing for 32 bit values.
+.TP
 .B olcIndexSubstrIfMaxlen: <integer>
 Specify the maximum length for subinitial and subfinal indices. Only
 this many characters of an attribute value will be processed by the
@@ -479,7 +484,11 @@ lookup. The default is 2. For example, with the default values, a search
 using this filter "cn=*abcdefgh*" would generate index lookups for
 "abcd", "cdef", and "efgh".
 
-Note: Indexing support depends on the particular backend in use.
+.LP
+Note: Indexing support depends on the particular backend in use. Also,
+changing these settings will generally require deleting any indices that
+depend on these parameters and recreating them with
+.BR slapindex (8).
 
 .TP
 .B olcLocalSSF: <SSF>

doc/man/man5/slapd.conf.5

@@ -470,6 +470,11 @@ feature.  The default is 0.
 Read additional configuration information from the given file before
 continuing with the next line of the current file.
 .TP
+.B index_intlen <integer>
+Specify the key length for ordered integer indices. The most significant
+bytes of the binary integer will be used for index keys. The default
+value is 4, which provides exact indexing for 32 bit values.
+.TP
 .B index_substr_if_minlen <integer>
 Specify the minimum length for subinitial and subfinal indices. An
 attribute value must have at least this many characters in order to be
@@ -496,7 +501,11 @@ lookup. The default is 2. For example, with the default values, a search
 using this filter "cn=*abcdefgh*" would generate index lookups for
 "abcd", "cdef", and "efgh".
 
-Note: Indexing support depends on the particular backend in use.
+.LP
+Note: Indexing support depends on the particular backend in use. Also,
+changing these settings will generally require deleting any indices that
+depend on these parameters and recreating them with
+.BR slapindex (8).
 
 .TP
 .B localSSF <SSF>

servers/slapd/bconfig.c

@@ -179,6 +179,7 @@ enum {
 	CFG_MONITORING,
 	CFG_SERVERID,
 	CFG_SORTVALS,
+	CFG_IX_INTLEN,
 
 	CFG_LAST
 };
@@ -372,6 +373,9 @@ static ConfigTable config_back_cf_table[] = {
 	{ "index_substr_any_step", "step", 2, 2, 0, ARG_INT|ARG_NONZERO,
 		&index_substr_any_step, "( OLcfgGlAt:23 NAME 'olcIndexSubstrAnyStep' "
 			"SYNTAX OMsInteger SINGLE-VALUE )", NULL, NULL },
+	{ "index_intlen", "len", 2, 2, 0, ARG_INT|ARG_MAGIC|CFG_IX_INTLEN,
+		&config_generic, "( OLcfgGlAt:84 NAME 'olcIndexIntLen' "
+			"SYNTAX OMsInteger SINGLE-VALUE )", NULL, NULL },
 	{ "lastmod", "on|off", 2, 2, 0, ARG_DB|ARG_ON_OFF|ARG_MAGIC|CFG_LASTMOD,
 		&config_generic, "( OLcfgDbAt:0.4 NAME 'olcLastMod' "
 			"SYNTAX OMsBoolean SINGLE-VALUE )", NULL, NULL },
@@ -709,8 +713,8 @@ static ConfigOCs cf_ocs[] = {
 		 "olcConnMaxPending $ olcConnMaxPendingAuth $ "
 		 "olcDisallows $ olcGentleHUP $ olcIdleTimeout $ "
 		 "olcIndexSubstrIfMaxLen $ olcIndexSubstrIfMinLen $ "
-		 "olcIndexSubstrAnyLen $ olcIndexSubstrAnyStep $ olcLocalSSF $ "
-		 "olcLogLevel $ "
+		 "olcIndexSubstrAnyLen $ olcIndexSubstrAnyStep $ olcIndexIntLen $ "
+		 "olcLocalSSF $ olcLogLevel $ "
 		 "olcPasswordCryptSaltFormat $ olcPasswordHash $ olcPidFile $ "
 		 "olcPluginLogFile $ olcReadOnly $ olcReferral $ "
 		 "olcReplogFile $ olcRequires $ olcRestrict $ olcReverseLookup $ "
@@ -1014,6 +1018,9 @@ config_generic(ConfigArgs *c) {
 		case CFG_SSTR_IF_MIN:
 			c->value_int = index_substr_if_minlen;
 			break;
+		case CFG_IX_INTLEN:
+			c->value_int = index_intlen;
+			break;
 		case CFG_SORTVALS: {
 			ADlist *sv;
 			rc = 1;
@@ -1151,6 +1158,10 @@ config_generic(ConfigArgs *c) {
 			c->be->be_flags &= ~SLAP_DBFLAG_HIDDEN;
 			break;
 
+		case CFG_IX_INTLEN:
+			index_intlen = SLAP_INDEX_INTLEN_DEFAULT;
+			break;
+
 		case CFG_ACL:
 			if ( c->valx < 0 ) {
 				AccessControl *end;
@@ -1494,6 +1505,14 @@ config_generic(ConfigArgs *c) {
 					return(1);
 			break;
 
+		case CFG_IX_INTLEN:
+			if ( c->value_int < SLAP_INDEX_INTLEN_DEFAULT )
+				c->value_int = SLAP_INDEX_INTLEN_DEFAULT;
+			else if ( c->value_int > 255 )
+				c->value_int = 255;
+			index_intlen = c->value_int;
+			break;
+			
 		case CFG_SORTVALS: {
 			ADlist *svnew = NULL, *svtail, *sv;
 

servers/slapd/proto-slap.h

@@ -707,12 +707,12 @@ LDAP_SLAPD_F (int) connections_shutdown LDAP_P((void));
 LDAP_SLAPD_F (int) connections_destroy LDAP_P((void));
 LDAP_SLAPD_F (int) connections_timeout_idle LDAP_P((time_t));
 
-LDAP_SLAPD_F (int) connection_client_setup LDAP_P((
+LDAP_SLAPD_F (Connection *) connection_client_setup LDAP_P((
 	ber_socket_t s,
 	ldap_pvt_thread_start_t *func,
 	void *arg ));
-LDAP_SLAPD_F (void) connection_client_enable LDAP_P(( ber_socket_t s ));
-LDAP_SLAPD_F (void) connection_client_stop LDAP_P(( ber_socket_t s ));
+LDAP_SLAPD_F (void) connection_client_enable LDAP_P(( Connection *c ));
+LDAP_SLAPD_F (void) connection_client_stop LDAP_P(( Connection *c ));
 
 #ifdef LDAP_PF_LOCAL_SENDMSG
 #define LDAP_PF_LOCAL_SENDMSG_ARG(arg)	, arg
@@ -822,6 +822,19 @@ LDAP_SLAPD_V (const char *) slapd_slp_attrs;
 LDAP_SLAPD_V (slap_ssf_t) local_ssf;
 LDAP_SLAPD_V (struct runqueue_s) slapd_rq;
 
+#ifdef HAVE_WINSOCK
+LDAP_SLAPD_F (ber_socket_t) slapd_socknew(ber_socket_t s);
+LDAP_SLAPD_F (ber_socket_t) slapd_sock2fd(ber_socket_t s);
+LDAP_SLAPD_V (SOCKET *) slapd_ws_sockets;
+#define	SLAP_FD2SOCK(s)	slapd_ws_sockets[s]
+#define	SLAP_SOCK2FD(s)	slapd_sock2fd(s)
+#define	SLAP_SOCKNEW(s)	slapd_socknew(s)
+#else
+#define	SLAP_FD2SOCK(s)	s
+#define	SLAP_SOCK2FD(s)	s
+#define	SLAP_SOCKNEW(s)	s
+#endif
+
 /*
  * dn.c
  */
@@ -1863,6 +1876,7 @@ LDAP_SLAPD_V (unsigned int) index_substr_if_minlen;
 LDAP_SLAPD_V (unsigned int) index_substr_if_maxlen;
 LDAP_SLAPD_V (unsigned int) index_substr_any_len;
 LDAP_SLAPD_V (unsigned int) index_substr_any_step;
+LDAP_SLAPD_V (unsigned int) index_intlen;
 
 LDAP_SLAPD_V (ber_len_t) sockbuf_max_incoming;
 LDAP_SLAPD_V (ber_len_t) sockbuf_max_incoming_auth;

servers/slapd/schema_init.c

@@ -62,6 +62,8 @@ unsigned int index_substr_if_maxlen = SLAP_INDEX_SUBSTR_IF_MAXLEN_DEFAULT;
 unsigned int index_substr_any_len = SLAP_INDEX_SUBSTR_ANY_LEN_DEFAULT;
 unsigned int index_substr_any_step = SLAP_INDEX_SUBSTR_ANY_STEP_DEFAULT;
 
+unsigned int index_intlen = SLAP_INDEX_INTLEN_DEFAULT;
+
 ldap_pvt_thread_mutex_t	ad_undef_mutex;
 ldap_pvt_thread_mutex_t	oc_undef_mutex;
 
@@ -2109,7 +2111,143 @@ integerMatch(
 	*matchp = match;
 	return LDAP_SUCCESS;
 }
-	
+
+/* Index generation function */
+static int
+integerIndexer(
+	slap_mask_t use,
+	slap_mask_t flags,
+	Syntax *syntax,
+	MatchingRule *mr,
+	struct berval *prefix,
+	BerVarray values,
+	BerVarray *keysp,
+	void *ctx )
+{
+	char ibuf[64];
+	struct berval iv, itmp;
+	BerVarray keys;
+	int i, rc;
+
+	for( i=0; !BER_BVISNULL( &values[i] ); i++ ) {
+		/* just count them */
+	}
+
+	/* we should have at least one value at this point */
+	assert( i > 0 );
+
+	keys = slap_sl_malloc( sizeof( struct berval ) * (i+1), ctx );
+	for ( i = 0; !BER_BVISNULL( &values[i] ); i++ ) {
+		keys[i].bv_len = index_intlen+1;
+		keys[i].bv_val = slap_sl_malloc( index_intlen+1, ctx );
+	}
+	keys[i].bv_len = 0;
+	keys[i].bv_val = NULL;
+
+	itmp.bv_val = ibuf;
+	itmp.bv_len = sizeof(ibuf);
+
+	for ( i=0; !BER_BVISNULL( &values[i] ); i++ ) {
+		if ( values[i].bv_len > itmp.bv_len ) {
+			itmp.bv_len = values[i].bv_len;
+			if ( itmp.bv_val == ibuf ) {
+				itmp.bv_val = slap_sl_malloc( itmp.bv_len, ctx );
+			} else {
+				itmp.bv_val = slap_sl_realloc( itmp.bv_val, itmp.bv_len, ctx );
+			}
+		}
+		iv = itmp;
+		if ( lutil_str2bin( &values[i], &iv )) {
+			rc = LDAP_INVALID_SYNTAX;
+			goto leave;
+		}
+		/* If too small, pad with zeros */
+		if ( iv.bv_len < index_intlen ) {
+			int j, k;
+			keys[i].bv_val[0] = index_intlen;
+			k = index_intlen - iv.bv_len + 1;
+			for ( j=1; j<k; j++)
+				keys[i].bv_val[j] = 0;
+			for ( j = 0; j<iv.bv_len; j++ )
+				keys[i].bv_val[j+k] = iv.bv_val[j];
+		} else {
+			keys[i].bv_val[0] = iv.bv_len;
+			memcpy( keys[i].bv_val+1, iv.bv_val, index_intlen );
+		}
+		/* convert signed to unsigned */
+		keys[i].bv_val[1] ^= 0x80;
+	}
+	*keysp = keys;
+	rc = 0;
+leave:
+	if ( itmp.bv_val != ibuf ) {
+		slap_sl_free( itmp.bv_val, ctx );
+	}
+	return rc;
+}
+
+/* Index generation function */
+static int
+integerFilter(
+	slap_mask_t use,
+	slap_mask_t flags,
+	Syntax *syntax,
+	MatchingRule *mr,
+	struct berval *prefix,
+	void * assertedValue,
+	BerVarray *keysp,
+	void *ctx )
+{
+	char ibuf[64];
+	struct berval iv;
+	BerVarray keys;
+	struct berval *value;
+	int rc;
+
+	value = (struct berval *) assertedValue;
+
+	keys = slap_sl_malloc( sizeof( struct berval ) * 2, ctx );
+
+	keys[0].bv_len = index_intlen + 1;
+	keys[0].bv_val = slap_sl_malloc( index_intlen+1, ctx );
+
+	if ( value->bv_len > sizeof( ibuf )) {
+		iv.bv_val = slap_sl_malloc( value->bv_len, ctx );
+		iv.bv_len = value->bv_len;
+	} else {
+		iv.bv_val = ibuf;
+		iv.bv_len = sizeof(ibuf);
+	}
+
+	if ( lutil_str2bin( value, &iv )) {
+		rc = LDAP_INVALID_SYNTAX;
+		goto leave;
+	}
+	/* If too small, pad with zeros */
+	if ( iv.bv_len < index_intlen ) {
+		int j, k;
+		keys[0].bv_val[0] = index_intlen;
+		k = index_intlen - iv.bv_len + 1;
+		for ( j=1; j<k; j++)
+			keys[0].bv_val[j] = 0;
+		for ( j = 0; j<iv.bv_len; j++ )
+			keys[0].bv_val[j+k] = iv.bv_val[j];
+	} else {
+		keys[0].bv_val[0] = iv.bv_len;
+		memcpy( keys[0].bv_val+1, iv.bv_val, index_intlen );
+	}
+	/* convert signed to unsigned */
+	keys[0].bv_val[1] ^= 0x80;
+
+	rc = 0;
+	*keysp = keys;
+leave:
+	if ( iv.bv_val != ibuf ) {
+		slap_sl_free( iv.bv_val, ctx );
+	}
+	return rc;
+}
+
 static int
 countryStringValidate(
 	Syntax *syntax,
@@ -2932,7 +3070,6 @@ serialNumberAndIssuerNormalize(
 	char sbuf[64], *stmp = sbuf;
 	int rc;
 	ber_len_t n;
-	int is_hex = 0;
 
 	assert( in != NULL );
 	assert( out != NULL );
@@ -2983,10 +3120,11 @@ serialNumberAndIssuerNormalize(
 	AC_MEMCPY( &out->bv_val[n], sn.bv_val, sn.bv_len );
 	{
 		int j;
-		unsigned char *v = sn2.bv_val;
+		unsigned char *v = (unsigned char *)sn2.bv_val;
 		out->bv_val[n++] = '\'';
 		for ( j = 0; j < sn2.bv_len; j++ ) {
-			sprintf( &out->bv_val[n], "%02X", v[j] );
+			snprintf( &out->bv_val[n], out->bv_len - n + 1,
+				"%02X", v[j] );
 			n += 2;
 		}
 		out->bv_val[n++] = '\'';
@@ -4650,14 +4788,14 @@ static slap_mrule_defs_rec mrule_defs[] = {
 
 	{"( 2.5.13.14 NAME 'integerMatch' "
 		"SYNTAX 1.3.6.1.4.1.1466.115.121.1.27 )",
-		SLAP_MR_EQUALITY | SLAP_MR_EXT, NULL,
+		SLAP_MR_EQUALITY | SLAP_MR_EXT | SLAP_MR_ORDERED_INDEX, NULL,
 		NULL, NULL, integerMatch,
-		octetStringIndexer, octetStringFilter,
+		integerIndexer, integerFilter,
 		NULL },
 
 	{"( 2.5.13.15 NAME 'integerOrderingMatch' "
 		"SYNTAX 1.3.6.1.4.1.1466.115.121.1.27 )",
-		SLAP_MR_ORDERING, NULL,
+		SLAP_MR_ORDERING | SLAP_MR_ORDERED_INDEX, NULL,
 		NULL, NULL, integerMatch,
 		NULL, NULL,
 		"integerMatch" },