Tests for monitoring support

tests/data/lloadd-empty.conf

+# Load balancer config -- for testing
+# $OpenLDAP$
+## This work is part of OpenLDAP Software <http://www.openldap.org/>.
+##
+## Copyright 1998-2020 The OpenLDAP Foundation.
+## All rights reserved.
+##
+## Redistribution and use in source and binary forms, with or without
+## modification, are permitted only as authorized by the OpenLDAP
+## Public License.
+##
+## A copy of this license is available in the file LICENSE in the
+## top-level directory of the distribution or, alternatively, at
+## <http://www.OpenLDAP.org/license.html>.
+
+# allow big PDUs from anonymous (for testing purposes)
+sockbuf_max_incoming_client 4194303
+sockbuf_max_incoming_upstream 4194303
+
+feature proxyauthz
+
+bindconf
+    bindmethod=simple
+    binddn="cn=Manager,dc=example,dc=com"
+    credentials=secret

tests/data/lloadd/monitor.ldif

+# empty lloadd
+dn: cn=Load Balancer,cn=Backends,cn=Monitor
+objectClass: olmBalancer
+olmIncomingConnections: 0
+olmOutgoingConnections: 0
+
+dn: cn=Incoming Connections,cn=Load Balancer,cn=Backends,cn=Monitor
+objectClass: monitorContainer
+
+dn: cn=Operations,cn=Load Balancer,cn=Backends,cn=Monitor
+objectClass: monitorContainer
+
+dn: cn=Bind,cn=Operations,cn=Load Balancer,cn=Backends,cn=Monitor
+objectClass: olmBalancerOperation
+olmReceivedOps: 1
+olmForwardedOps: 0
+olmRejectedOps: 1
+olmCompletedOps: 0
+olmFailedOps: 0
+
+dn: cn=Other,cn=Operations,cn=Load Balancer,cn=Backends,cn=Monitor
+objectClass: olmBalancerOperation
+olmReceivedOps: 1
+olmForwardedOps: 0
+olmRejectedOps: 0
+olmCompletedOps: 0
+olmFailedOps: 0
+
+dn: cn=Backend Servers,cn=Load Balancer,cn=Backends,cn=Monitor
+objectClass: monitorContainer
+
+
+# with first backend
+dn: cn=Load Balancer,cn=Backends,cn=Monitor
+objectClass: olmBalancer
+olmIncomingConnections: 0
+olmOutgoingConnections: 4
+
+dn: cn=Incoming Connections,cn=Load Balancer,cn=Backends,cn=Monitor
+objectClass: monitorContainer
+
+dn: cn=Operations,cn=Load Balancer,cn=Backends,cn=Monitor
+objectClass: monitorContainer
+
+dn: cn=Bind,cn=Operations,cn=Load Balancer,cn=Backends,cn=Monitor
+objectClass: olmBalancerOperation
+olmReceivedOps: 1
+olmForwardedOps: 0
+olmRejectedOps: 1
+olmCompletedOps: 0
+olmFailedOps: 0
+
+dn: cn=Other,cn=Operations,cn=Load Balancer,cn=Backends,cn=Monitor
+objectClass: olmBalancerOperation
+olmReceivedOps: 1
+olmForwardedOps: 0
+olmRejectedOps: 0
+olmCompletedOps: 0
+olmFailedOps: 0
+
+dn: cn=Backend Servers,cn=Load Balancer,cn=Backends,cn=Monitor
+objectClass: monitorContainer
+
+dn: cn=first,cn=Backend Servers,cn=Load Balancer,cn=Backends,cn=Monitor
+objectClass: olmBalancerServer
+olmServerURI: ldap://localhost:9012/
+olmActiveConnections: 4
+olmPendingConnections: 0
+olmPendingOps: 0
+olmReceivedOps: 0
+olmCompletedOps: 0
+olmFailedOps: 0
+
+dn: cn=Connection 1,cn=first,cn=Backend Servers,cn=Load Balancer,cn=Backends,c
+ n=Monitor
+objectClass: olmBalancerConnection
+olmConnectionType: regular
+olmPendingOps: 0
+olmReceivedOps: 0
+olmCompletedOps: 0
+olmFailedOps: 0
+
+dn: cn=Connection 3,cn=first,cn=Backend Servers,cn=Load Balancer,cn=Backends,c
+ n=Monitor
+objectClass: olmBalancerConnection
+olmConnectionType: regular
+olmPendingOps: 0
+olmReceivedOps: 0
+olmCompletedOps: 0
+olmFailedOps: 0
+
+dn: cn=Connection 2,cn=first,cn=Backend Servers,cn=Load Balancer,cn=Backends,c
+ n=Monitor
+objectClass: olmBalancerConnection
+olmConnectionType: bind
+olmPendingOps: 0
+olmReceivedOps: 0
+olmCompletedOps: 0
+olmFailedOps: 0
+
+dn: cn=Connection 4,cn=first,cn=Backend Servers,cn=Load Balancer,cn=Backends,c
+ n=Monitor
+objectClass: olmBalancerConnection
+olmConnectionType: bind
+olmPendingOps: 0
+olmReceivedOps: 0
+olmCompletedOps: 0
+olmFailedOps: 0
+
+
+# second backend and a search+WhoAmI?
+dn: cn=Load Balancer,cn=Backends,cn=Monitor
+objectClass: olmBalancer
+olmIncomingConnections: 0
+olmOutgoingConnections: 13
+
+dn: cn=Incoming Connections,cn=Load Balancer,cn=Backends,cn=Monitor
+objectClass: monitorContainer
+
+dn: cn=Operations,cn=Load Balancer,cn=Backends,cn=Monitor
+objectClass: monitorContainer
+
+dn: cn=Bind,cn=Operations,cn=Load Balancer,cn=Backends,cn=Monitor
+objectClass: olmBalancerOperation
+olmReceivedOps: 3
+olmForwardedOps: 2
+olmRejectedOps: 1
+olmCompletedOps: 2
+olmFailedOps: 0
+
+dn: cn=Other,cn=Operations,cn=Load Balancer,cn=Backends,cn=Monitor
+objectClass: olmBalancerOperation
+olmReceivedOps: 5
+olmForwardedOps: 2
+olmRejectedOps: 0
+olmCompletedOps: 2
+olmFailedOps: 0
+
+dn: cn=Backend Servers,cn=Load Balancer,cn=Backends,cn=Monitor
+objectClass: monitorContainer
+
+dn: cn=first,cn=Backend Servers,cn=Load Balancer,cn=Backends,cn=Monitor
+objectClass: olmBalancerServer
+olmServerURI: ldap://localhost:9012/
+olmActiveConnections: 4
+olmPendingConnections: 0
+olmPendingOps: 0
+olmReceivedOps: 2
+olmCompletedOps: 2
+olmFailedOps: 0
+
+dn: cn=Connection 1,cn=first,cn=Backend Servers,cn=Load Balancer,cn=Backends,c
+ n=Monitor
+objectClass: olmBalancerConnection
+olmConnectionType: regular
+olmPendingOps: 0
+olmReceivedOps: 0
+olmCompletedOps: 0
+olmFailedOps: 0
+
+dn: cn=Connection 3,cn=first,cn=Backend Servers,cn=Load Balancer,cn=Backends,c
+ n=Monitor
+objectClass: olmBalancerConnection
+olmConnectionType: regular
+olmPendingOps: 0
+olmReceivedOps: 0
+olmCompletedOps: 0
+olmFailedOps: 0
+
+dn: cn=Connection 2,cn=first,cn=Backend Servers,cn=Load Balancer,cn=Backends,c
+ n=Monitor
+objectClass: olmBalancerConnection
+olmConnectionType: bind
+olmPendingOps: 0
+olmReceivedOps: 1
+olmCompletedOps: 1
+olmFailedOps: 0
+
+dn: cn=Connection 4,cn=first,cn=Backend Servers,cn=Load Balancer,cn=Backends,c
+ n=Monitor
+objectClass: olmBalancerConnection
+olmConnectionType: bind
+olmPendingOps: 0
+olmReceivedOps: 1
+olmCompletedOps: 1
+olmFailedOps: 0
+
+dn: cn=server 2,cn=Backend Servers,cn=Load Balancer,cn=Backends,cn=Monitor
+objectClass: olmBalancerServer
+olmServerURI: ldap://localhost:9013/
+olmActiveConnections: 9
+olmPendingConnections: 0
+olmPendingOps: 0
+olmReceivedOps: 2
+olmCompletedOps: 2
+olmFailedOps: 0
+
+dn: cn=Connection 5,cn=server 2,cn=Backend Servers,cn=Load Balancer,cn=Backend
+ s,cn=Monitor
+objectClass: olmBalancerConnection
+olmConnectionType: regular
+olmPendingOps: 0
+olmReceivedOps: 1
+olmCompletedOps: 1
+olmFailedOps: 0
+
+dn: cn=Connection 7,cn=server 2,cn=Backend Servers,cn=Load Balancer,cn=Backend
+ s,cn=Monitor
+objectClass: olmBalancerConnection
+olmConnectionType: regular
+olmPendingOps: 0
+olmReceivedOps: 1
+olmCompletedOps: 1
+olmFailedOps: 0
+
+dn: cn=Connection 8,cn=server 2,cn=Backend Servers,cn=Load Balancer,cn=Backend
+ s,cn=Monitor
+objectClass: olmBalancerConnection
+olmConnectionType: regular
+olmPendingOps: 0
+olmReceivedOps: 0
+olmCompletedOps: 0
+olmFailedOps: 0
+
+dn: cn=Connection 9,cn=server 2,cn=Backend Servers,cn=Load Balancer,cn=Backend
+ s,cn=Monitor
+objectClass: olmBalancerConnection
+olmConnectionType: regular
+olmPendingOps: 0
+olmReceivedOps: 0
+olmCompletedOps: 0
+olmFailedOps: 0
+
+dn: cn=Connection 6,cn=server 2,cn=Backend Servers,cn=Load Balancer,cn=Backend
+ s,cn=Monitor
+objectClass: olmBalancerConnection
+olmConnectionType: bind
+olmPendingOps: 0
+olmReceivedOps: 0
+olmCompletedOps: 0
+olmFailedOps: 0
+
+dn: cn=Connection 10,cn=server 2,cn=Backend Servers,cn=Load Balancer,cn=Backen
+ ds,cn=Monitor
+objectClass: olmBalancerConnection
+olmConnectionType: bind
+olmPendingOps: 0
+olmReceivedOps: 0
+olmCompletedOps: 0
+olmFailedOps: 0
+
+dn: cn=Connection 11,cn=server 2,cn=Backend Servers,cn=Load Balancer,cn=Backen
+ ds,cn=Monitor
+objectClass: olmBalancerConnection
+olmConnectionType: bind
+olmPendingOps: 0
+olmReceivedOps: 0
+olmCompletedOps: 0
+olmFailedOps: 0
+
+dn: cn=Connection 12,cn=server 2,cn=Backend Servers,cn=Load Balancer,cn=Backen
+ ds,cn=Monitor
+objectClass: olmBalancerConnection
+olmConnectionType: bind
+olmPendingOps: 0
+olmReceivedOps: 0
+olmCompletedOps: 0
+olmFailedOps: 0
+
+dn: cn=Connection 13,cn=server 2,cn=Backend Servers,cn=Load Balancer,cn=Backen
+ ds,cn=Monitor
+objectClass: olmBalancerConnection
+olmConnectionType: bind
+olmPendingOps: 0
+olmReceivedOps: 0
+olmCompletedOps: 0
+olmFailedOps: 0
+

tests/data/slapd-lload.conf

@@ -38,3 +38,4 @@ database config
 include		@TESTDIR@/configpw.conf
 
 database	monitor
+access to * by * read

tests/scripts/defines.sh

@@ -153,6 +153,7 @@ DYNAMICCONF=$DATADIR/slapd-dynamic.ldif
 
 SLAPDLLOADCONF=$DATADIR/slapd-lload.conf
 LLOADDCONF=$DATADIR/lloadd.conf
+LLOADDEMPTYCONF=$DATADIR/lloadd-empty.conf
 LLOADDANONCONF=$DATADIR/lloadd-anon.conf
 LLOADDUNREACHABLECONF=$DATADIR/lloadd-backend-issues.conf
 

tests/scripts/lloadd/test004-monitor

+#! /bin/sh
+# $OpenLDAP$
+## This work is part of OpenLDAP Software <http://www.openldap.org/>.
+##
+## Copyright 1998-2020 The OpenLDAP Foundation.
+## All rights reserved.
+##
+## Redistribution and use in source and binary forms, with or without
+## modification, are permitted only as authorized by the OpenLDAP
+## Public License.
+##
+## A copy of this license is available in the file LICENSE in the
+## top-level directory of the distribution or, alternatively, at
+## <http://www.OpenLDAP.org/license.html>.
+
+echo "running defines.sh"
+. $SRCDIR/scripts/defines.sh
+
+mkdir -p $TESTDIR $DBDIR1 $DBDIR2
+
+$SLAPPASSWD -g -n >$CONFIGPWF
+echo "rootpw `$SLAPPASSWD -T $CONFIGPWF`" >$TESTDIR/configpw.conf
+
+if test $AC_lloadd = lloaddyes ; then
+    echo "Load balancer module not available, skipping..."
+    exit 0
+fi
+
+# Monitor counts are unstable in the face of concurrency, since different
+# clients may get different upstreams assigned for their operations. This might
+# also change later when tiered load balancing is available.
+# Another constraint is that some global counts are updated by the statistics
+# collection task scheduled to run every second.
+#
+# This test assumes current round-robin policy:
+# - default backend is rotated every time we successfully pick an upstream
+# - upstream connections within the same backend are rotated in the same way
+# - the monitor entry order for upstream connections reflects the connection
+#   order within its CIRCLEQ_
+
+echo "Starting the first slapd on TCP/IP port $PORT2..."
+. $CONFFILTER $BACKEND < $CONF > $CONF2
+$SLAPADD -f $CONF2 -l $LDIFORDERED
+RC=$?
+if test $RC != 0 ; then
+    echo "slapadd failed ($RC)!"
+    exit $RC
+fi
+
+echo "Running slapindex to index slapd database..."
+$SLAPINDEX -f $CONF2
+RC=$?
+if test $RC != 0 ; then
+    echo "warning: slapindex failed ($RC)"
+    echo "  assuming no indexing support"
+fi
+
+$SLAPD -f $CONF2 -h $URI2 -d $LVL > $LOG2 2>&1 &
+PID=$!
+if test $WAIT != 0 ; then
+    echo PID $PID
+    read foo
+fi
+PID2="$PID"
+KILLPIDS="$PID"
+
+echo "Testing slapd searching..."
+for i in 0 1 2 3 4 5; do
+    $LDAPSEARCH -s base -b "$MONITOR" -H $URI2 \
+        '(objectclass=*)' > /dev/null 2>&1
+    RC=$?
+    if test $RC = 0 ; then
+        break
+    fi
+    echo "Waiting $SLEEP1 seconds for slapd to start..."
+    sleep $SLEEP1
+done
+if test $RC != 0 ; then
+    echo "ldapsearch failed ($RC)!"
+    test $KILLSERVERS != no && kill -HUP $KILLPIDS
+    exit $RC
+fi
+
+echo "Running slapadd to build slapd database..."
+. $CONFFILTER $BACKEND < $CONFTWO > $CONF3
+$SLAPADD -f $CONF3 -l $LDIFORDERED
+RC=$?
+if test $RC != 0 ; then
+    echo "slapadd failed ($RC)!"
+    exit $RC
+fi
+
+echo "Running slapindex to index slapd database..."
+$SLAPINDEX -f $CONF3
+RC=$?
+if test $RC != 0 ; then
+    echo "warning: slapindex failed ($RC)"
+    echo "  assuming no indexing support"
+fi
+
+echo "Starting second slapd on TCP/IP port $PORT3..."
+$SLAPD -f $CONF3 -h $URI3 -d $LVL > $LOG3 2>&1 &
+PID=$!
+if test $WAIT != 0 ; then
+    echo PID $PID
+    read foo
+fi
+PID3="$PID"
+KILLPIDS="$KILLPIDS $PID"
+
+sleep $SLEEP0
+
+echo "Testing slapd searching..."
+for i in 0 1 2 3 4 5; do
+    $LDAPSEARCH -s base -b "$MONITOR" -H $URI3 \
+        '(objectclass=*)' > /dev/null 2>&1
+    RC=$?
+    if test $RC = 0 ; then
+        break
+    fi
+    echo "Waiting $SLEEP1 seconds for slapd to start..."
+    sleep $SLEEP1
+done
+if test $RC != 0 ; then
+    echo "ldapsearch failed ($RC)!"
+    test $KILLSERVERS != no && kill -HUP $KILLPIDS
+    exit $RC
+fi
+
+echo "Starting lloadd on TCP/IP port $PORT1..."
+. $CONFFILTER $BACKEND < $LLOADDEMPTYCONF > $CONF1.lloadd
+. $CONFFILTER $BACKEND < $SLAPDLLOADCONF > $CONF1.slapd
+$SLAPD -f $CONF1.slapd -h $URI6 -d $LVL > $LOG1 2>&1 &
+PID=$!
+if test $WAIT != 0 ; then
+    echo PID $PID
+    read foo
+fi
+KILLPIDS="$KILLPIDS $PID"
+
+echo "Testing slapd searching..."
+for i in 0 1 2 3 4 5; do
+    $LDAPSEARCH -s base -b "$MONITOR" -H $URI6 \
+        '(objectclass=*)' > /dev/null 2>&1
+    RC=$?
+    if test $RC = 0 ; then
+        break
+    fi
+    echo "Waiting $SLEEP1 seconds for lloadd to start..."
+    sleep $SLEEP1
+done
+
+if test $RC != 0 ; then
+    echo "ldapsearch failed ($RC)!"
+    test $KILLSERVERS != no && kill -HUP $KILLPIDS
+    exit $RC
+fi
+
+echo "Sending a search request to prime the counters..."
+$LDAPSEARCH -b "$BASEDN" -s base -H $URI1 >> $TESTOUT 2>&1
+RC=$?
+if test $RC != 52 ; then
+    echo "ldapsearch should have failed ($RC != 52)!"
+    test $KILLSERVERS != no && kill -HUP $KILLPIDS
+    exit $RC
+fi
+
+echo "Retrieving data from cn=monitor..."
+echo "# Retrieving data from an empty lload's cn=monitor..." >>$SEARCHOUT
+echo "# Operations received:" >>$SEARCHOUT
+echo "#     Bind: 1 (0 forwarded)" >>$SEARCHOUT
+echo "#     Search: 0" >>$SEARCHOUT
+echo "#     Unbind: 1" >>$SEARCHOUT
+$LDAPSEARCH -b "cn=Load Balancer,cn=Backends,cn=monitor" -H $URI6 \
+    olmBalancer olmBalancerServer olmBalancerOperation olmBalancerConnection >> $SEARCHOUT 2>&1
+RC=$?
+if test $RC != 0 ; then
+    echo "ldapsearch failed ($RC)!"
+    test $KILLSERVERS != no && kill -HUP $KILLPIDS
+    exit $RC
+fi
+
+echo "Adding first backend server..."
+$LDAPMODIFY -D cn=config -H $URI6 -y $CONFIGPWF <<EOF >> $TESTOUT 2>&1
+dn: cn=first,olcBackend={0}lload,cn=config
+changetype: add
+objectClass: olcBkLloadBackendConfig
+olcBkLloadBackendUri: $URI2
+olcBkLloadMaxPendingConns: 3
+olcBkLloadMaxPendingOps: 5
+olcBkLloadRetry: 1000
+olcBkLloadNumconns: 2
+olcBkLloadBindconns: 2
+EOF
+RC=$?
+if test $RC != 0 ; then
+    echo "ldapadd failed for backend ($RC)!"
+    test $KILLSERVERS != no && kill -HUP $KILLPIDS
+    exit $RC
+fi
+
+# At the moment, the global counters are updated by a recurring job,
+# wait for it to settle
+echo "Waiting until connections are established..."
+for i in 0 1 2 3 4 5; do
+    $LDAPCOMPARE "cn=Load Balancer,cn=Backends,cn=monitor" -H $URI6 \
+        'olmOutgoingConnections:4' > /dev/null 2>&1
+    RC=$?
+    if test $RC = 6 ; then
+        break
+    fi
+    echo "Waiting $SLEEP1 seconds until connections are established..."
+    sleep $SLEEP1
+done
+if test $RC != 6 ; then
+    echo "ldapcompare failed ($RC)!"
+    test $KILLSERVERS != no && kill -HUP $KILLPIDS
+    exit $RC
+fi
+
+echo "Retrieving data from cn=monitor again..."
+echo >>$SEARCHOUT
+echo "# Retrieving data from lload's cn=monitor..." >>$SEARCHOUT
+$LDAPSEARCH -b "cn=Load Balancer,cn=Backends,cn=monitor" -H $URI6 \
+    olmBalancer olmBalancerServer olmBalancerOperation olmBalancerConnection >> $SEARCHOUT 2>&1
+RC=$?
+if test $RC != 0 ; then
+    echo "ldapsearch failed ($RC)!"
+    test $KILLSERVERS != no && kill -HUP $KILLPIDS
+    exit $RC
+fi
+
+echo "Adding another backend server..."
+$LDAPMODIFY -D cn=config -H $URI6 -y $CONFIGPWF <<EOF >> $TESTOUT 2>&1
+dn: cn=server 2,olcBackend={0}lload,cn=config
+changetype: add
+objectClass: olcBkLloadBackendConfig
+olcBkLloadBackendUri: $URI3
+olcBkLloadMaxPendingConns: 3
+olcBkLloadMaxPendingOps: 5
+olcBkLloadRetry: 1000
+olcBkLloadNumconns: 4
+olcBkLloadBindconns: 5
+EOF
+RC=$?
+if test $RC != 0 ; then
+    echo "ldapadd failed for backend ($RC)!"
+    test $KILLSERVERS != no && kill -HUP $KILLPIDS
+    exit $RC
+fi
+
+# At the moment, the global counters are updated by a recurring job,
+# wait for it to settle
+echo "Waiting until connections are established..."
+for i in 0 1 2 3 4 5; do
+    $LDAPCOMPARE "cn=Load Balancer,cn=Backends,cn=monitor" -H $URI6 \
+        'olmOutgoingConnections:13' > /dev/null 2>&1
+    RC=$?
+    if test $RC = 6 ; then
+        break
+    fi
+    echo "Waiting $SLEEP1 seconds until connections are established..."
+    sleep $SLEEP1
+done
+if test $RC != 6 ; then
+    echo "ldapcompare failed ($RC)!"
+    test $KILLSERVERS != no && kill -HUP $KILLPIDS
+    exit $RC
+fi
+
+echo "Sending another search request..."
+$LDAPSEARCH -b "$BASEDN" -s base -H $URI1 >> $TESTOUT 2>&1
+RC=$?
+if test $RC != 0 ; then
+    echo "ldapsearch failed ($RC)!"
+    test $KILLSERVERS != no && kill -HUP $KILLPIDS
+    exit $RC
+fi
+
+echo "Sending a WhoAmI? request..."
+$LDAPWHOAMI -D "$BABSDN" -w bjensen -H $URI1 >> $TESTOUT 2>&1
+RC=$?
+if test $RC != 0 ; then
+    echo "ldapwhoami failed ($RC)!"
+    test $KILLSERVERS != no && kill -HUP $KILLPIDS
+    exit $RC
+fi
+
+# At the moment, the global counters are updated by a recurring job,
+# wait for it to settle
+echo "Waiting until global counters are updated..."
+for i in 0 1 2 3 4 5; do
+    $LDAPCOMPARE "cn=Other,cn=Operations,cn=Load Balancer,cn=Backends,cn=monitor" -H $URI6 \
+        'olmCompletedOps:2' > /dev/null 2>&1
+    RC=$?
+    if test $RC = 6 ; then
+        break
+    fi
+    echo "Waiting $SLEEP1 seconds until connections are established..."
+    sleep $SLEEP1
+done
+if test $RC != 6 ; then
+    echo "ldapcompare failed ($RC)!"
+    test $KILLSERVERS != no && kill -HUP $KILLPIDS
+    exit $RC
+fi
+
+echo "Retrieving data from cn=monitor again..."
+echo >>$SEARCHOUT
+echo "# Retrieving data after a search+WhoAmI?..." >>$SEARCHOUT
+echo "# Operations received:" >>$SEARCHOUT
+echo "#     Bind: 3 (2 forwarded)" >>$SEARCHOUT