test050-syncrepl-multimaster

#! /bin/sh
# $OpenLDAP$
## This work is part of OpenLDAP Software <http://www.openldap.org/>.
##
## Copyright 1998-2009 The OpenLDAP Foundation.
## All rights reserved.
##
## Redistribution and use in source and binary forms, with or without
## modification, are permitted only as authorized by the OpenLDAP
## Public License.
##
## A copy of this license is available in the file LICENSE in the
## top-level directory of the distribution or, alternatively, at
## <http://www.OpenLDAP.org/license.html>.

echo "running defines.sh"
. $SRCDIR/scripts/defines.sh

if test $SYNCPROV = syncprovno; then 
	echo "Syncrepl provider overlay not available, test skipped"
	exit 0
fi 

MMR=${MMR-4}

if [ $MMR -gt 9 ]; then
MMR=9
fi

XDIR=$TESTDIR/srv
TMP=$TESTDIR/tmp

mkdir -p $TESTDIR

$SLAPPASSWD -g -n >$CONFIGPWF

if test x"$SYNCMODE" = x ; then
	SYNCMODE=rp
fi
case "$SYNCMODE" in
	ro)
		SYNCTYPE="type=refreshOnly interval=00:00:00:03"
		;;
	rp)
		SYNCTYPE="type=refreshAndPersist"
		;;
	*)
		echo "unknown sync mode $SYNCMODE"
		exit 1;
		;;
esac

#
# Test replication of dynamic config:
# - start servers
# - configure over ldap
# - populate over ldap
# - configure syncrepl over ldap
# - retrieve database over ldap and compare against expected results
#

echo "Initializing server configurations..."
n=1
while [ $n -le $MMR ]; do

DBDIR=${XDIR}$n/db
CFDIR=${XDIR}$n/slapd.d

mkdir -p ${XDIR}$n $DBDIR $CFDIR

$SLAPADD -F $CFDIR -n 0 <<EOF
dn: cn=config
objectClass: olcGlobal
cn: config
olcServerID: $n

dn: olcDatabase={0}config,cn=config
objectClass: olcDatabaseConfig
olcDatabase: {0}config
olcRootPW:< file://$CONFIGPWF
EOF

n=`expr $n + 1`
done

echo "Starting server 1 on TCP/IP port $PORT1..."
cd ${XDIR}1
$SLAPD -F slapd.d -h $URI1 -d $LVL $TIMING > $LOG1 2>&1 &
PID=$!
if test $WAIT != 0 ; then
    echo PID $PID
    read foo
fi
KILLPIDS="$PID"
cd $TESTWD

sleep 1

echo "Using ldapsearch to check that server 1 is running..."
for i in 0 1 2 3 4 5; do
	$LDAPSEARCH -s base -b "" -H $URI1 \
		'objectclass=*' > /dev/null 2>&1
	RC=$?
	if test $RC = 0 ; then
		break
	fi
	echo "Waiting 5 seconds for slapd to start..."
	sleep 5
done

if test $RC != 0 ; then
	echo "ldapsearch failed ($RC)!"
	test $KILLSERVERS != no && kill -HUP $KILLPIDS
	exit $RC
fi

echo "Inserting syncprov overlay on server 1..."
echo "" > $TMP
if [ "$SYNCPROV" = syncprovmod ]; then
cat <<EOF >> $TMP
dn: cn=module,cn=config
changetype: add
objectClass: olcModuleList
cn: module
olcModulePath: ../../../servers/slapd/overlays
olcModuleLoad: syncprov.la

EOF
fi
#
# Note that we configure a timeout here; it's possible for both
# servers to attempt to bind to each other while a modify to
# cn=config is in progress. When the modify pauses the thread pool
# neither server will progress. The timeout will drop the syncrepl
# attempt and allow the modifies to complete.
#
read CONFIGPW < $CONFIGPWF
echo "dn: cn=config" >> $TMP
echo "changetype: modify" >> $TMP
echo "replace: olcServerID" >> $TMP
n=1
while [ $n -le $MMR ]; do
PORT=`expr $BASEPORT + $n`
URI="ldap://${LOCALHOST}:$PORT/"
echo "olcServerID: $n $URI" >> $TMP
n=`expr $n + 1`
done

cat <<EOF >> $TMP

dn: olcOverlay=syncprov,olcDatabase={0}config,cn=config
changetype: add
objectClass: olcOverlayConfig
objectClass: olcSyncProvConfig
olcOverlay: syncprov

dn: olcDatabase={0}config,cn=config
changetype: modify
add: olcSyncRepl
EOF

n=1
while [ $n -le $MMR ]; do
PORT=`expr $BASEPORT + $n`
URI="ldap://${LOCALHOST}:$PORT/"
echo "olcSyncRepl: rid=00$n provider=$URI binddn=\"cn=config\" bindmethod=simple" >> $TMP
echo "  credentials=$CONFIGPW searchbase=\"cn=config\" type=refreshAndPersist" >> $TMP
echo "  retry=\"3 10 300 5\" timeout=3" >> $TMP
n=`expr $n + 1`
done
echo "-" >> $TMP
echo "add: olcMirrorMode" >> $TMP
echo "olcMirrorMode: TRUE" >> $TMP
$LDAPMODIFY -D cn=config -H $URI1 -y $CONFIGPWF < $TMP >> $TESTOUT 2>&1
RC=$?
if test $RC != 0 ; then
	echo "ldapmodify failed for syncrepl config ($RC)!"
	test $KILLSERVERS != no && kill -HUP $KILLPIDS
	exit $RC
fi

n=2
while [ $n -le $MMR ]; do
PORT=`expr $BASEPORT + $n`
URI="ldap://${LOCALHOST}:$PORT/"
LOG=$TESTDIR/slapd.$n.log
echo "Starting server $n on TCP/IP port $PORT..."
cd ${XDIR}$n
$SLAPD -F ./slapd.d -h $URI -d $LVL $TIMING > $LOG 2>&1 &
SLAVEPID=$!
if test $WAIT != 0 ; then
    echo SLAVEPID $SLAVEPID
    read foo
fi
KILLPIDS="$KILLPIDS $SLAVEPID"
cd $TESTWD

sleep 1

echo "Using ldapsearch to check that server $n is running..."
for i in 0 1 2 3 4 5; do
	$LDAPSEARCH -s base -b "" -H $URI \
		'objectclass=*' > /dev/null 2>&1
	RC=$?
	if test $RC = 0 ; then
		break
	fi
	echo "Waiting 5 seconds for slapd to start..."
	sleep 5
done

if test $RC != 0 ; then
	echo "ldapsearch failed ($RC)!"
	test $KILLSERVERS != no && kill -HUP $KILLPIDS
	exit $RC
fi

echo "Configuring syncrepl on server $n..."
cat <<EOF > $TMP
dn: olcDatabase={0}config,cn=config
changetype: modify
add: olcSyncRepl
EOF
j=1
while [ $j -le $MMR ]; do
P2=`expr $BASEPORT + $j`
U2="ldap://${LOCALHOST}:$P2/"
echo "olcSyncRepl: rid=00$j provider=$U2 binddn=\"cn=config\" bindmethod=simple" >> $TMP
echo "  credentials=$CONFIGPW searchbase=\"cn=config\" type=refreshAndPersist" >> $TMP
echo "  retry=\"3 10 300 5\" timeout=3" >> $TMP
j=`expr $j + 1`
done
cat <<EOF >> $TMP
-
add: olcMirrorMode
olcMirrorMode: TRUE
EOF
$LDAPMODIFY -D cn=config -H $URI -y $CONFIGPWF < $TMP >>$TESTOUT 2>&1
n=`expr $n + 1`
done

echo "Adding schema and databases on server 1..."
$LDAPADD -D cn=config -H $URI1 -y $CONFIGPWF <<EOF >>$TESTOUT 2>&1
include: file://$ABS_SCHEMADIR/core.ldif

include: file://$ABS_SCHEMADIR/cosine.ldif

include: file://$ABS_SCHEMADIR/inetorgperson.ldif

include: file://$ABS_SCHEMADIR/openldap.ldif

include: file://$ABS_SCHEMADIR/nis.ldif
EOF
RC=$?
if test $RC != 0 ; then
	echo "ldapadd failed for schema config ($RC)!"
	test $KILLSERVERS != no && kill -HUP $KILLPIDS
	exit $RC
fi

echo "" > $TMP
if [ "$BACKENDTYPE" = mod ]; then
cat <<EOF >> $TMP
dn: cn=module,cn=config
objectClass: olcModuleList
cn: module
olcModulePath: ../../../servers/slapd/back-$BACKEND
olcModuleLoad: back_$BACKEND.la

EOF
fi

cat <<EOF >> $TMP
dn: olcDatabase={1}$BACKEND,cn=config
objectClass: olcDatabaseConfig
objectClass: olc${BACKEND}Config
olcDatabase: {1}$BACKEND
olcSuffix: $BASEDN
olcDbDirectory: ./db
olcRootDN: $MANAGERDN
olcRootPW: $PASSWD
EOF

n=1
while [ $n -le $MMR ]; do
PORT=`expr $BASEPORT + $n`
URI="ldap://${LOCALHOST}:$PORT/"

echo "olcSyncRepl: rid=01$n provider=$URI binddn=\"$MANAGERDN\" bindmethod=simple" >> $TMP
echo "  credentials=$PASSWD searchbase=\"$BASEDN\" $SYNCTYPE" >> $TMP
echo "  retry=\"3 10 300 5\" timeout=3" >> $TMP
n=`expr $n + 1`
done

cat <<EOF >> $TMP
olcMirrorMode: TRUE

dn: olcOverlay=syncprov,olcDatabase={1}${BACKEND},cn=config
objectClass: olcOverlayConfig
objectClass: olcSyncProvConfig
olcOverlay: syncprov
EOF
$LDAPADD -D cn=config -H $URI1 -y $CONFIGPWF < $TMP >>$TESTOUT 2>&1
RC=$?
if test $RC != 0 ; then
	echo "ldapadd failed for database config ($RC)!"
	test $KILLSERVERS != no && kill -HUP $KILLPIDS
	exit $RC
fi

case $BACKEND in
bdb | hdb)
	$LDAPMODIFY -D cn=config -H $URI1 -y $CONFIGPWF <<EOF >>$TESTOUT 2>&1
dn: olcDatabase={1}$BACKEND,cn=config
changetype: modify
add: olcDbIndex
olcDbIndex: objectClass,entryUUID,entryCSN eq
olcDbIndex: cn,uid pres,eq,sub
EOF
	RC=$?
	if test $RC != 0 ; then
		echo "ldapadd modify for database config ($RC)!"
		test $KILLSERVERS != no && kill -HUP $KILLPIDS
		exit $RC
	fi
	;;
esac

echo "Using ldapadd to populate server 1..."
$LDAPADD -D "$MANAGERDN" -H $URI1 -w $PASSWD -f $LDIFORDERED \
	>> $TESTOUT 2>&1
RC=$?
if test $RC != 0 ; then
	echo "ldapadd failed for server 1 database ($RC)!"
	test $KILLSERVERS != no && kill -HUP $KILLPIDS
	exit $RC
fi

echo "Waiting $SLEEP2 seconds for syncrepl to receive changes..."
sleep $SLEEP2

n=1
while [ $n -le $MMR ]; do
PORT=`expr $BASEPORT + $n`
URI="ldap://${LOCALHOST}:$PORT/"

echo "Using ldapsearch to read config from server $n..."
$LDAPSEARCH -b cn=config -D cn=config -H $URI -y $CONFIGPWF  \
	'objectclass=*' > $TESTDIR/server$n.out 2>&1
RC=$?

if test $RC != 0 ; then
	echo "ldapsearch failed at server $n ($RC)!"
	test $KILLSERVERS != no && kill -HUP $KILLPIDS
	exit $RC
fi

. $LDIFFILTER < $TESTDIR/server$n.out > $TESTDIR/server$n.flt

n=`expr $n + 1`
done

n=2
while [ $n -le $MMR ]; do
echo "Comparing retrieved configs from server 1 and server $n..."
$CMP $MASTERFLT $TESTDIR/server$n.flt > $CMPOUT

if test $? != 0 ; then
	echo "test failed - server 1 and server $n configs differ"
	test $KILLSERVERS != no && kill -HUP $KILLPIDS
	exit 1
fi
n=`expr $n + 1`
done

n=1
while [ $n -le $MMR ]; do
PORT=`expr $BASEPORT + $n`
URI="ldap://${LOCALHOST}:$PORT/"

echo "Using ldapsearch to read all the entries from server $n..."
$LDAPSEARCH -S "" -b "$BASEDN" -D "$MANAGERDN" -H $URI -w $PASSWD  \
	'objectclass=*' > $TESTDIR/server$n.out 2>&1
RC=$?

if test $RC != 0 ; then
	echo "ldapsearch failed at server $n ($RC)!"
	test $KILLSERVERS != no && kill -HUP $KILLPIDS
	exit $RC
fi
. $LDIFFILTER < $TESTDIR/server$n.out > $TESTDIR/server$n.flt
n=`expr $n + 1`
done

n=2
while [ $n -le $MMR ]; do
echo "Comparing retrieved entries from server 1 and server $n..."
$CMP $MASTERFLT $TESTDIR/server$n.flt > $CMPOUT

if test $? != 0 ; then
	echo "test failed - server 1 and server $n databases differ"
	test $KILLSERVERS != no && kill -HUP $KILLPIDS
	exit 1
fi
n=`expr $n + 1`
done

echo "Using ldapadd to populate server 2..."
$LDAPADD -D "$MANAGERDN" -H $URI2 -w $PASSWD -f $LDIFADD1 \
	>> $TESTOUT 2>&1
RC=$?
if test $RC != 0 ; then
	echo "ldapadd failed for server 2 database ($RC)!"
	test $KILLSERVERS != no && kill -HUP $KILLPIDS
	exit $RC
fi

echo "Waiting $SLEEP1 seconds for syncrepl to receive changes..."
sleep $SLEEP1

n=1
while [ $n -le $MMR ]; do
PORT=`expr $BASEPORT + $n`
URI="ldap://${LOCALHOST}:$PORT/"

echo "Using ldapsearch to read all the entries from server $n..."
$LDAPSEARCH -S "" -b "$BASEDN" -D "$MANAGERDN" -H $URI -w $PASSWD  \
	'objectclass=*' > $TESTDIR/server$n.out 2>&1
RC=$?

if test $RC != 0 ; then
	echo "ldapsearch failed at server $n ($RC)!"
	test $KILLSERVERS != no && kill -HUP $KILLPIDS
	exit $RC
fi
. $LDIFFILTER < $TESTDIR/server$n.out > $TESTDIR/server$n.flt
n=`expr $n + 1`
done

n=2
while [ $n -le $MMR ]; do
echo "Comparing retrieved entries from server 1 and server $n..."
$CMP $MASTERFLT $TESTDIR/server$n.flt > $CMPOUT

if test $? != 0 ; then
	echo "test failed - server 1 and server $n databases differ"
	test $KILLSERVERS != no && kill -HUP $KILLPIDS
	exit 1
fi
n=`expr $n + 1`
done

echo "Using ldapadd to populate server 3..."
$LDAPADD -D "$MANAGERDN" -H $URI3 -w $PASSWD \
	<< EOMODS >> $TESTOUT 2>&1
dn: cn=Server 3 Test,dc=example,dc=com
changetype: add
objectClass: device
cn: Server 3 Test
EOMODS
RC=$?
if test $RC != 0 ; then
	echo "ldapadd failed for server 3 database ($RC)!"
	test $KILLSERVERS != no && kill -HUP $KILLPIDS
	exit $RC
fi

echo "Waiting $SLEEP1 seconds for syncrepl to receive changes..."
sleep $SLEEP1

n=1
while [ $n -le $MMR ]; do
PORT=`expr $BASEPORT + $n`
URI="ldap://${LOCALHOST}:$PORT/"

echo "Using ldapsearch to read all the entries from server $n..."
$LDAPSEARCH -S "" -b "$BASEDN" -D "$MANAGERDN" -H $URI -w $PASSWD  \
	'objectclass=*' > $TESTDIR/server$n.out 2>&1
RC=$?

if test $RC != 0 ; then
	echo "ldapsearch failed at server $n ($RC)!"
	test $KILLSERVERS != no && kill -HUP $KILLPIDS
	exit $RC
fi
. $LDIFFILTER < $TESTDIR/server$n.out > $TESTDIR/server$n.flt
n=`expr $n + 1`
done

n=2
while [ $n -le $MMR ]; do
echo "Comparing retrieved entries from server 1 and server $n..."
$CMP $MASTERFLT $TESTDIR/server$n.flt > $CMPOUT

if test $? != 0 ; then
	echo "test failed - server 1 and server $n databases differ"
	test $KILLSERVERS != no && kill -HUP $KILLPIDS
	exit 1
fi
n=`expr $n + 1`
done

echo "Using ldapmodify to add to server 1 entries that will be deleted..."
$LDAPMODIFY -D "$MANAGERDN" -H $URI1 -w $PASSWD \
	>> $TESTOUT 2>&1 << EOADDS
dn: cn=To be deleted by server 1,dc=example,dc=com
changetype: add
objectClass: device
# no distinguished values, will be added by DSA

dn: cn=To be deleted by server 2,dc=example,dc=com
changetype: add
objectClass: device
# no distinguished values, will be added by DSA

dn: cn=To be deleted by server 3,dc=example,dc=com
changetype: add
objectClass: device
# no distinguished values, will be added by DSA

dn: cn=To be deleted by server 1,dc=example,dc=com
changetype: delete
EOADDS
RC=$?
if test $RC != 0 ; then
	echo "ldapmodify failed for server 1 database ($RC)!"
	test $KILLSERVERS != no && kill -HUP $KILLPIDS
	exit $RC
fi

echo "Waiting $SLEEP1 seconds for syncrepl to receive changes..."
sleep $SLEEP1

n=1
while [ $n -le $MMR ]; do
PORT=`expr $BASEPORT + $n`
URI="ldap://${LOCALHOST}:$PORT/"

echo "Using ldapsearch to read all the entries from server $n..."
$LDAPSEARCH -S "" -b "$BASEDN" -D "$MANAGERDN" -H $URI -w $PASSWD  \
	'objectclass=*' > $TESTDIR/server$n.out 2>&1
RC=$?

if test $RC != 0 ; then
	echo "ldapsearch failed at server $n ($RC)!"
	test $KILLSERVERS != no && kill -HUP $KILLPIDS
	exit $RC
fi
. $LDIFFILTER < $TESTDIR/server$n.out > $TESTDIR/server$n.flt
n=`expr $n + 1`
done

n=2
while [ $n -le $MMR ]; do
echo "Comparing retrieved entries from server 1 and server $n..."
$CMP $MASTERFLT $TESTDIR/server$n.flt > $CMPOUT

if test $? != 0 ; then
	echo "test failed - server 1 and server $n databases differ"
	test $KILLSERVERS != no && kill -HUP $KILLPIDS
	exit 1
fi
n=`expr $n + 1`
done

echo "Using ldapmodify to delete entries from server 2..."
$LDAPMODIFY -D "$MANAGERDN" -H $URI2 -w $PASSWD \
	>> $TESTOUT 2>&1 << EOADDS
dn: cn=To be deleted by server 2,dc=example,dc=com
changetype: delete
EOADDS
RC=$?
if test $RC != 0 ; then
	echo "ldapmodify failed for server 2 database ($RC)!"
	test $KILLSERVERS != no && kill -HUP $KILLPIDS
	exit $RC
fi

echo "Waiting $SLEEP1 seconds for syncrepl to receive changes..."
sleep $SLEEP1

echo "Using ldapmodify to delete entries from server 3..."
$LDAPMODIFY -D "$MANAGERDN" -H $URI3 -w $PASSWD \
	>> $TESTOUT 2>&1 << EOADDS
dn: cn=To be deleted by server 3,dc=example,dc=com
changetype: delete
EOADDS
RC=$?
if test $RC != 0 ; then
	echo "ldapmodify failed for server 3 database ($RC)!"
	test $KILLSERVERS != no && kill -HUP $KILLPIDS
	exit $RC
fi

echo "Waiting $SLEEP1 seconds for syncrepl to receive changes..."
sleep $SLEEP1

n=1
while [ $n -le $MMR ]; do
PORT=`expr $BASEPORT + $n`
URI="ldap://${LOCALHOST}:$PORT/"

echo "Using ldapsearch to read all the entries from server $n..."
$LDAPSEARCH -S "" -b "$BASEDN" -D "$MANAGERDN" -H $URI -w $PASSWD  \
	'objectclass=*' > $TESTDIR/server$n.out 2>&1
RC=$?

if test $RC != 0 ; then
	echo "ldapsearch failed at server $n ($RC)!"
	test $KILLSERVERS != no && kill -HUP $KILLPIDS
	exit $RC
fi
. $LDIFFILTER < $TESTDIR/server$n.out > $TESTDIR/server$n.flt
n=`expr $n + 1`
done

n=2
while [ $n -le $MMR ]; do
echo "Comparing retrieved entries from server 1 and server $n..."
$CMP $MASTERFLT $TESTDIR/server$n.flt > $CMPOUT

if test $? != 0 ; then
	echo "test failed - server 1 and server $n databases differ"
	test $KILLSERVERS != no && kill -HUP $KILLPIDS
	exit 1
fi
n=`expr $n + 1`
done

# kill!
# test $KILLSERVERS != no && kill -HUP $KILLPIDS
kill -HUP $KILLPIDS

# kill!
# test $KILLSERVERS != no && wait
wait

echo "Restarting servers..."
KILLPIDS=""
n=1
while [ $n -le $MMR ]; do
PORT=`expr $BASEPORT + $n`
URI="ldap://${LOCALHOST}:$PORT/"
LOG=$TESTDIR/slapd.$n.log
echo "Starting server $n on TCP/IP port $PORT..."
cd ${XDIR}$n
echo "======================= RESTART =======================" >> $LOG
$SLAPD -F ./slapd.d -h $URI -d $LVL $TIMING >> $LOG 2>&1 &
PID=$!
if test $WAIT != 0 ; then
    echo PID $PID
    read foo
fi
KILLPIDS="$KILLPIDS $PID"
cd $TESTWD
echo "Using ldapsearch to check that server $n is running..."
for i in 0 1 2 3 4 5; do
	$LDAPSEARCH -s base -b "" -H $URI \
		'objectclass=*' > /dev/null 2>&1
	RC=$?
	if test $RC = 0 ; then
		break
	fi
	echo "Waiting 5 seconds for slapd to start..."
	sleep 5
done

if test $RC != 0 ; then
	echo "ldapsearch failed ($RC)!"
	test $KILLSERVERS != no && kill -HUP $KILLPIDS
	exit $RC
fi
n=`expr $n + 1`
done

# Insert modifications and more tests here.
echo "Waiting $SLEEP2 seconds for servers to resync..."
sleep $SLEEP2

echo "Using ldapmodify to add/modify/delete entries from server 1..."
for i in 1 2 3 4 5 6 7 8 9 10; do
echo "  iteration $i"
$LDAPMODIFY -D "$MANAGERDN" -H $URI1 -w $PASSWD \
	>> $TESTOUT 2>&1 << EOMODS
dn: cn=Add-Mod-Del,dc=example,dc=com
changetype: add
cn: Add/Mod/Del
objectclass: organizationalRole

dn: cn=Add-Mod-Del,dc=example,dc=com
changetype: modify
replace: description
description: guinea pig
-

dn: cn=Add-Mod-Del,dc=example,dc=com
changetype: delete
EOMODS
RC=$?
if test $RC != 0 ; then
	echo "ldapmodify failed for server 1 database ($RC)!"
	test $KILLSERVERS != no && kill -HUP $KILLPIDS
	exit $RC
fi
done

echo "Waiting $SLEEP1 seconds for servers to resync..."
sleep $SLEEP1

n=1
while [ $n -le $MMR ]; do
PORT=`expr $BASEPORT + $n`
URI="ldap://${LOCALHOST}:$PORT/"

echo "Using ldapsearch to read all the entries from server $n..."
$LDAPSEARCH -S "" -b "$BASEDN" -D "$MANAGERDN" -H $URI -w $PASSWD  \
	'objectclass=*' > $TESTDIR/server$n.out 2>&1
RC=$?

if test $RC != 0 ; then
	echo "ldapsearch failed at server $n ($RC)!"
	test $KILLSERVERS != no && kill -HUP $KILLPIDS
	exit $RC
fi
. $LDIFFILTER < $TESTDIR/server$n.out > $TESTDIR/server$n.flt
n=`expr $n + 1`
done

n=2
while [ $n -le $MMR ]; do
echo "Comparing retrieved entries from server 1 and server $n..."
$CMP $MASTERFLT $TESTDIR/server$n.flt > $CMPOUT

if test $? != 0 ; then
	echo "test failed - server 1 and server $n databases differ"
	test $KILLSERVERS != no && kill -HUP $KILLPIDS
	exit 1
fi
n=`expr $n + 1`
done

test $KILLSERVERS != no && kill -HUP $KILLPIDS

echo ">>>>> Test succeeded"

test $KILLSERVERS != no && wait

exit 0