From 69de6c94cd549770b8579d85501c5e41b4cf1c6b Mon Sep 17 00:00:00 2001 From: Dimitar Stoychev Date: Thu, 19 May 2022 06:45:50 -0700 Subject: [PATCH 1/2] ITS#9823 Update test043 to check deltasync recovery after accesslog has been purged --- tests/data/slapd-deltasync-provider.conf | 4 + tests/scripts/test043-delta-syncrepl | 131 ++++++++++++++++++++++- 2 files changed, 134 insertions(+), 1 deletion(-) diff --git a/tests/data/slapd-deltasync-provider.conf b/tests/data/slapd-deltasync-provider.conf index c38b9ef3d0..844eed5d6e 100644 --- a/tests/data/slapd-deltasync-provider.conf +++ b/tests/data/slapd-deltasync-provider.conf @@ -29,6 +29,10 @@ argsfile @TESTDIR@/slapd.1.args #accesslogmod#modulepath ../servers/slapd/overlays/ #accesslogmod#moduleload accesslog.la +database config +include @TESTDIR@/configpw.conf + + ####################################################################### # provider database definitions ####################################################################### diff --git a/tests/scripts/test043-delta-syncrepl b/tests/scripts/test043-delta-syncrepl index c919478459..c5c5a413b2 100755 --- a/tests/scripts/test043-delta-syncrepl +++ b/tests/scripts/test043-delta-syncrepl @@ -34,6 +34,8 @@ mkdir -p $TESTDIR $DBDIR1A $DBDIR1B $DBDIR2 SPEC="mdb=a" +$SLAPPASSWD -g -n >$CONFIGPWF +echo "rootpw `$SLAPPASSWD -T $CONFIGPWF`" >$TESTDIR/configpw.conf # # Test replication: # - start provider @@ -129,6 +131,7 @@ sleep $SLEEP1 echo "Stopping the provider, sleeping 10 seconds and restarting it..." kill -HUP "$PID" +wait $PID sleep 10 echo "RESTART" >> $LOG1 $SLAPD -f $CONF1 -h $URI1 -d $LVL >> $LOG1 2>&1 & @@ -296,7 +299,8 @@ fi echo "Stopping consumer to test recovery..." kill -HUP $CONSUMERPID -sleep 10 +wait $CONSUMERPID +KILLPIDS="$PID" echo "Modifying more entries on the provider..." $LDAPMODIFY -v -D "$BJORNSDN" -H $URI1 -w bjorn >> \ @@ -388,6 +392,131 @@ if test $RC != 0 ; then exit $RC fi +echo "Filtering provider results..." +$LDIFFILTER -b $BACKEND -s $SPEC < $PROVIDEROUT | grep -iv "^auditcontext:" > $PROVIDERFLT +echo "Filtering consumer results..." +$LDIFFILTER -b $BACKEND -s $SPEC < $CONSUMEROUT | grep -iv "^auditcontext:" > $CONSUMERFLT + +echo "Comparing retrieved entries from provider and consumer..." +$CMP $PROVIDERFLT $CONSUMERFLT > $CMPOUT + +if test $? != 0 ; then + echo "test failed - provider and consumer databases differ" + test $KILLSERVERS != no && kill -HUP $KILLPIDS + exit 1 +fi + +echo "Stopping consumer to test recovery after logpurge expired..." +kill -HUP $CONSUMERPID +wait $CONSUMERPID +KILLPIDS="$PID" + +echo "Modifying even more entries on the provider..." +$LDAPMODIFY -v -D "$BJORNSDN" -H $URI1 -w bjorn >> \ + $TESTOUT 2>&1 << EOMODS +dn: cn=Dorothy Stevens,ou=Alumni Association,ou=People,dc=example,dc=com +changetype: delete + +dn: cn=Bjorn Jensen, ou=Information Technology Division, ou=People, dc=example,dc=com +changetype: modify +add: drink +drink: Sangria + +dn: cn=George D. Stevens, ou=Retired, ou=People, dc=example,dc=com +changetype: add +objectclass: OpenLDAPperson +sn: Stevens +uid: gstevens +cn: George D. Stevens + +dn: cn=Barbara Jensen,ou=Information Technology Division,ou=People,dc=example, + dc=com +changetype: modify +replace: drink +drink: cold water + +dn: cn=Some Staff,ou=Groups,dc=example,dc=com +changetype: modrdn +newrdn: cn=More Staff +deleteoldrdn: 1 + +EOMODS + +echo "Configuring logpurge of 1 second..." +$LDAPMODIFY -v -D cn=config -H $URI1 -y $CONFIGPWF >> \ + $TESTOUT 2>&1 << EOMODS + +dn: olcOverlay={1}accesslog,olcDatabase={2}$BACKEND,cn=config +changetype: modify +replace: olcAccessLogPurge +olcAccessLogPurge: 0+00:00:02 0+00:00:01 +- + +EOMODS +RC=$? +if test $RC != 0 ; then + echo "ldapmodify failed ($RC)!" + test $KILLSERVERS != no && kill -HUP $KILLPIDS + exit $RC +fi + +echo "Waiting 4 seconds for accesslog to be purged..." +sleep 4 + +echo "Using ldapsearch to check if accesslog is empty..." +for i in 0 1 2 3 4 5; do + $LDAPSEARCH -b "cn=log" -H $URI1 -z 1 \ + > $SEARCHOUT 2>&1 + RC=$? + if test $RC = 0 ; then + break + fi + echo "Waiting 3 seconds for accesslog to be purged..." + sleep 3 +done + +if test $RC != 0; then + echo "Accesslog did not purge in time" + test $KILLSERVERS != no && kill -HUP $KILLPIDS + exit 1 +fi + + +echo "Restarting consumer..." +echo "RESTART" >> $LOG2 +$SLAPD -f $CONF2 -h $URI2 -d $LVL >> $LOG2 2>&1 & +CONSUMERPID=$! +if test $WAIT != 0 ; then + echo CONSUMERPID $CONSUMERPID + read foo +fi +KILLPIDS="$PID $CONSUMERPID" + +echo "Waiting $SLEEP1 seconds for syncrepl to receive changes..." +sleep $SLEEP1 + +echo "Using ldapsearch to read all the entries from the provider..." +$LDAPSEARCH -S "" -b "$BASEDN" -H $URI1 \ + 'objectclass=*' \* + > $PROVIDEROUT 2>&1 +RC=$? + +if test $RC != 0 ; then + echo "ldapsearch failed at provider ($RC)!" + test $KILLSERVERS != no && kill -HUP $KILLPIDS + exit $RC +fi + +echo "Using ldapsearch to read all the entries from the consumer..." +$LDAPSEARCH -S "" -b "$BASEDN" -H $URI2 \ + 'objectclass=*' \* + > $CONSUMEROUT 2>&1 +RC=$? + +if test $RC != 0 ; then + echo "ldapsearch failed at consumer ($RC)!" + test $KILLSERVERS != no && kill -HUP $KILLPIDS + exit $RC +fi + test $KILLSERVERS != no && kill -HUP $KILLPIDS echo "Filtering provider results..." -- GitLab From c64e663518988afbe5c5414ebb1a06a1864cf414 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Kuzn=C3=ADk?= Date: Fri, 22 Apr 2022 14:14:16 +0100 Subject: [PATCH 2/2] ITS#9823 Check minCSN when setting up delta-log replay --- servers/slapd/overlays/syncprov.c | 59 ++++++++++++++++++++++++++----- 1 file changed, 51 insertions(+), 8 deletions(-) diff --git a/servers/slapd/overlays/syncprov.c b/servers/slapd/overlays/syncprov.c index 4b6358fdd1..f6ab09949d 100644 --- a/servers/slapd/overlays/syncprov.c +++ b/servers/slapd/overlays/syncprov.c @@ -3235,14 +3235,8 @@ aband: if (srs->sr_state.numcsns != numcsns) { /* consumer doesn't have the right number of CSNs */ Debug( LDAP_DEBUG_SYNC, "%s syncprov_op_search: " - "consumer cookie is missing a csn we track%s\n", - op->o_log_prefix, si->si_nopres ? ", rejecting" : "" ); - - if ( si->si_nopres ) { - rs->sr_err = LDAP_SYNC_REFRESH_REQUIRED; - rs->sr_text = "not enough information to resync, please use other means"; - goto bailout; - } + "consumer cookie is missing a csn we track\n", + op->o_log_prefix ); changed = SS_CHANGED; if ( srs->sr_state.ctxcsn ) { @@ -3342,7 +3336,55 @@ no_change: if ( !(op->o_sync_mode & SLAP_SYNC_PERSIST) ) { numcsns, sids, &mincsn, minsid ) ) { do_present = SS_PRESENT; } + } else if ( si->si_nopres && si->si_usehint ) { + /* We are instructed to trust minCSN if it exists. */ + Entry *e; + Attribute *a = NULL; + int rc; + + /* + * ITS#9580 FIXME: when we've figured out and split the + * sessionlog/deltalog tracking, use the appropriate attribute + */ + rc = overlay_entry_get_ov( op, &op->o_bd->be_nsuffix[0], NULL, + ad_minCSN, 0, &e, on ); + if ( rc == LDAP_SUCCESS && e != NULL ) { + a = attr_find( e->e_attrs, ad_minCSN ); + } + + if ( a != NULL ) { + int *minsids; + + minsids = slap_parse_csn_sids( a->a_vals, a->a_numvals, op->o_tmpmemctx ); + slap_sort_csn_sids( a->a_vals, minsids, a->a_numvals, op->o_tmpmemctx ); + + for ( i=0, j=0; i < a->a_numvals; i++ ) { + while ( j < numcsns && minsids[i] > sids[j] ) j++; + if ( j < numcsns && minsids[i] == sids[j] && + ber_bvcmp( &a->a_vals[i], &srs->sr_state.ctxcsn[j] ) <= 0 ) { + /* minCSN for this serverID is contained, keep going */ + continue; + } + /* + * Log DB's minCSN claims we can only replay from a certain + * CSN for this serverID, but consumer's cookie hasn't met that + * threshold: they need to refresh + */ + Debug( LDAP_DEBUG_SYNC, "%s syncprov_op_search: " + "consumer not within recorded mincsn for DB's mincsn=%s\n", + op->o_log_prefix, a->a_vals[i].bv_val ); + rs->sr_err = LDAP_SYNC_REFRESH_REQUIRED; + rs->sr_text = "sync cookie is stale"; + slap_sl_free( minsids, op->o_tmpmemctx ); + overlay_entry_release_ov( op, e, 0, on ); + goto bailout; + } + slap_sl_free( minsids, op->o_tmpmemctx ); + } + if ( e != NULL ) + overlay_entry_release_ov( op, e, 0, on ); } + /* * If sessionlog wasn't useful, see if we can find at least one entry * that hasn't changed based on the cookie. @@ -3787,6 +3829,7 @@ sp_cf_gen(ConfigArgs *c) break; case SP_USEHINT: si->si_usehint = c->value_int; + rc = syncprov_setup_accesslog(); break; case SP_LOGDB: if ( si->si_logs ) { -- GitLab