Fixed slapo-syncprov incomplete sync on restart issues (ITS#4622)

(synced syncprov.c with HEAD)

CHANGES

@@ -3,10 +3,11 @@ OpenLDAP 2.3 Change Log
 OpenLDAP 2.3.26 Engineering
 	Fixed libldap dnssrv bug with "not present" positive statement (ITS#4610)
 	Fixed slapd incorrect rebuilding of replica URI (ITS#4633)
+	Fixed slapd-monitor operations order via callbacks (ITS#4631)
 	Fixed slapo-accesslog purge task during shutdown
 	Fixed slapo-ppolicy handling of default policy (ITS#4634)
 	Fixed slapo-ppolicy logging verbosity when using default policy
-	Fixed back-monitor operations order via callbacks (ITS#4631)
+	Fixed slapo-syncprov incomplete sync on restart issues (ITS#4622)
 
 OpenLDAP 2.3.25 Release
 	Fixed liblber ber_bvreplace_x argument checks

include/ldap.h

@@ -948,9 +948,9 @@ ldap_abandon_ext LDAP_P((
 	LDAPControl		**serverctrls,
 	LDAPControl		**clientctrls ));
 
-#if LDAP_DEPRECATED
+#if LDAP_DEPRECATED	
 LDAP_F( int )
-ldap_abandon LDAP_P((	/* deprecated */
+ldap_abandon LDAP_P((	/* deprecated, use ldap_abandon_ext */
 	LDAP *ld,
 	int msgid ));
 #endif
@@ -978,13 +978,13 @@ ldap_add_ext_s LDAP_P((
 
 #if LDAP_DEPRECATED
 LDAP_F( int )
-ldap_add LDAP_P((	/* deprecated */
+ldap_add LDAP_P((	/* deprecated, use ldap_add_ext */
 	LDAP *ld,
 	LDAP_CONST char *dn,
 	LDAPMod **attrs ));
 
 LDAP_F( int )
-ldap_add_s LDAP_P((	/* deprecated */
+ldap_add_s LDAP_P((	/* deprecated, use ldap_add_ext_s */
 	LDAP *ld,
 	LDAP_CONST char *dn,
 	LDAPMod **attrs ));
@@ -1057,14 +1057,14 @@ ldap_parse_sasl_bind_result LDAP_P((
  *	(deprecated)
  */
 LDAP_F( int )
-ldap_bind LDAP_P((	/* deprecated */
+ldap_bind LDAP_P((	/* deprecated, use ldap_sasl_bind */
 	LDAP *ld,
 	LDAP_CONST char *who,
 	LDAP_CONST char *passwd,
 	int authmethod ));
 
 LDAP_F( int )
-ldap_bind_s LDAP_P((	/* deprecated */
+ldap_bind_s LDAP_P((	/* deprecated, use ldap_sasl_bind_s */
 	LDAP *ld,
 	LDAP_CONST char *who,
 	LDAP_CONST char *cred,
@@ -1074,13 +1074,13 @@ ldap_bind_s LDAP_P((	/* deprecated */
  * in sbind.c:
  */
 LDAP_F( int )
-ldap_simple_bind LDAP_P(( /* deprecated */
+ldap_simple_bind LDAP_P(( /* deprecated, use ldap_sasl_bind */
 	LDAP *ld,
 	LDAP_CONST char *who,
 	LDAP_CONST char *passwd ));
 
 LDAP_F( int )
-ldap_simple_bind_s LDAP_P(( /* deprecated */
+ldap_simple_bind_s LDAP_P(( /* deprecated, use ldap_sasl_bind_s */
 	LDAP *ld,
 	LDAP_CONST char *who,
 	LDAP_CONST char *passwd ));
@@ -1088,7 +1088,7 @@ ldap_simple_bind_s LDAP_P(( /* deprecated */
 
 /*
  * in kbind.c:
- *	(deprecated)
+ *	(deprecated - use SASL instead)
  */
 LDAP_F( int )
 ldap_kerberos_bind_s LDAP_P((	/* deprecated */
@@ -1141,14 +1141,14 @@ ldap_compare_ext_s LDAP_P((
 
 #if LDAP_DEPRECATED
 LDAP_F( int )
-ldap_compare LDAP_P((	/* deprecated */
+ldap_compare LDAP_P((	/* deprecated, use ldap_compare_ext */
 	LDAP *ld,
 	LDAP_CONST char *dn,
 	LDAP_CONST char *attr,
 	LDAP_CONST char *value ));
 
 LDAP_F( int )
-ldap_compare_s LDAP_P((	/* deprecated */
+ldap_compare_s LDAP_P((	/* deprecated, use ldap_compare_ext_s */
 	LDAP *ld,
 	LDAP_CONST char *dn,
 	LDAP_CONST char *attr,
@@ -1176,12 +1176,12 @@ ldap_delete_ext_s LDAP_P((
 
 #if LDAP_DEPRECATED
 LDAP_F( int )
-ldap_delete LDAP_P((	/* deprecated */
+ldap_delete LDAP_P((	/* deprecated, use ldap_delete_ext */
 	LDAP *ld,
 	LDAP_CONST char *dn ));
 
 LDAP_F( int )
-ldap_delete_s LDAP_P((	/* deprecated */
+ldap_delete_s LDAP_P((	/* deprecated, use ldap_delete_ext_s */
 	LDAP *ld,
 	LDAP_CONST char *dn ));
 #endif
@@ -1207,13 +1207,13 @@ ldap_err2string LDAP_P((
 
 #if LDAP_DEPRECATED
 LDAP_F( int )
-ldap_result2error LDAP_P((	/* deprecated */
+ldap_result2error LDAP_P((	/* deprecated, use ldap_parse_result */
 	LDAP *ld,
 	LDAPMessage *r,
 	int freeit ));
 
 LDAP_F( void )
-ldap_perror LDAP_P((	/* deprecated */
+ldap_perror LDAP_P((	/* deprecated, use ldap_err2string */
 	LDAP *ld,
 	LDAP_CONST char *s ));
 #endif
@@ -1241,13 +1241,13 @@ ldap_modify_ext_s LDAP_P((
 
 #if LDAP_DEPRECATED
 LDAP_F( int )
-ldap_modify LDAP_P((	/* deprecated */
+ldap_modify LDAP_P((	/* deprecated, use ldap_modify_ext */
 	LDAP *ld,
 	LDAP_CONST char *dn,
 	LDAPMod **mods ));
 
 LDAP_F( int )
-ldap_modify_s LDAP_P((	/* deprecated */
+ldap_modify_s LDAP_P((	/* deprecated, use ldap_modify_ext_s */
 	LDAP *ld,
 	LDAP_CONST char *dn,
 	LDAPMod **mods ));
@@ -1280,7 +1280,7 @@ ldap_rename_s LDAP_P((
 
 #if LDAP_DEPRECATED
 LDAP_F( int )
-ldap_rename2 LDAP_P((	/* deprecated */
+ldap_rename2 LDAP_P((	/* deprecated, use ldap_rename */
 	LDAP *ld,
 	LDAP_CONST char *dn,
 	LDAP_CONST char *newrdn,
@@ -1288,7 +1288,7 @@ ldap_rename2 LDAP_P((	/* deprecated */
 	int deleteoldrdn ));
 
 LDAP_F( int )
-ldap_rename2_s LDAP_P((	/* deprecated */
+ldap_rename2_s LDAP_P((	/* deprecated, use ldap_rename_s */
 	LDAP *ld,
 	LDAP_CONST char *dn,
 	LDAP_CONST char *newrdn,
@@ -1296,26 +1296,26 @@ ldap_rename2_s LDAP_P((	/* deprecated */
 	int deleteoldrdn ));
 
 LDAP_F( int )
-ldap_modrdn LDAP_P((	/* deprecated */
+ldap_modrdn LDAP_P((	/* deprecated, use ldap_rename */
 	LDAP *ld,
 	LDAP_CONST char *dn,
 	LDAP_CONST char *newrdn ));
 
 LDAP_F( int )
-ldap_modrdn_s LDAP_P((	/* deprecated */
+ldap_modrdn_s LDAP_P((	/* deprecated, use ldap_rename_s */
 	LDAP *ld,
 	LDAP_CONST char *dn,
 	LDAP_CONST char *newrdn ));
 
 LDAP_F( int )
-ldap_modrdn2 LDAP_P((	/* deprecated */
+ldap_modrdn2 LDAP_P((	/* deprecated, use ldap_rename */
 	LDAP *ld,
 	LDAP_CONST char *dn,
 	LDAP_CONST char *newrdn,
 	int deleteoldrdn ));
 
 LDAP_F( int )
-ldap_modrdn2_s LDAP_P((	/* deprecated */
+ldap_modrdn2_s LDAP_P((	/* deprecated, use ldap_rename_s */
 	LDAP *ld,
 	LDAP_CONST char *dn,
 	LDAP_CONST char *newrdn,
@@ -1328,12 +1328,12 @@ ldap_modrdn2_s LDAP_P((	/* deprecated */
  */
 #if LDAP_DEPRECATED
 LDAP_F( LDAP * )
-ldap_init LDAP_P(( /* deprecated */
+ldap_init LDAP_P(( /* deprecated, use ldap_create or ldap_initialize */
 	LDAP_CONST char *host,
 	int port ));
 
 LDAP_F( LDAP * )
-ldap_open LDAP_P((	/* deprecated */
+ldap_open LDAP_P((	/* deprecated, use ldap_create or ldap_initialize */
 	LDAP_CONST char *host,
 	int port ));
 #endif
@@ -1557,16 +1557,16 @@ ldap_dn_normalize LDAP_P((
 	char **out, unsigned oflags ));
 
 LDAP_F( char * )
-ldap_dn2ufn LDAP_P(( /* deprecated */
+ldap_dn2ufn LDAP_P(( /* deprecated, use ldap_str2dn/dn2str */
 	LDAP_CONST char *dn ));
 
 LDAP_F( char ** )
-ldap_explode_dn LDAP_P(( /* deprecated */
+ldap_explode_dn LDAP_P(( /* deprecated, ldap_str2dn */
 	LDAP_CONST char *dn,
 	int notypes ));
 
 LDAP_F( char ** )
-ldap_explode_rdn LDAP_P(( /* deprecated */
+ldap_explode_rdn LDAP_P(( /* deprecated, ldap_str2rdn */
 	LDAP_CONST char *rdn,
 	int notypes ));
 
@@ -1578,13 +1578,16 @@ ldap_X509dn2bv LDAP_P(( void *x509_name, struct berval *dn,
 	LDAPDN_rewrite_func *func, unsigned flags ));
 
 LDAP_F( char * )
-ldap_dn2dcedn LDAP_P(( LDAP_CONST char *dn ));	/* deprecated */
+ldap_dn2dcedn LDAP_P(( /* deprecated, ldap_str2dn/dn2str */
+	LDAP_CONST char *dn ));
 
 LDAP_F( char * )
-ldap_dcedn2dn LDAP_P(( LDAP_CONST char *dce ));	/* deprecated */
+ldap_dcedn2dn LDAP_P(( /* deprecated, ldap_str2dn/dn2str */
+	LDAP_CONST char *dce ));
 
 LDAP_F( char * )
-ldap_dn2ad_canonical LDAP_P(( LDAP_CONST char *dn ));	/* deprecated */
+ldap_dn2ad_canonical LDAP_P(( /* deprecated, ldap_str2dn/dn2str */
+	LDAP_CONST char *dn ));
 
 LDAP_F( int )
 ldap_get_dn_ber LDAP_P((
@@ -1630,17 +1633,17 @@ ldap_value_free_len LDAP_P((
 
 #if LDAP_DEPRECATED
 LDAP_F( char ** )
-ldap_get_values LDAP_P((	/* deprecated */
+ldap_get_values LDAP_P((	/* deprecated, use ldap_get_values_len */
 	LDAP *ld,
 	LDAPMessage *entry,
 	LDAP_CONST char *target ));
 
 LDAP_F( int )
-ldap_count_values LDAP_P((	/* deprecated */
+ldap_count_values LDAP_P((	/* deprecated, use ldap_count_values_len */
 	char **vals ));
 
 LDAP_F( void )
-ldap_value_free LDAP_P((	/* deprecated */
+ldap_value_free LDAP_P((	/* deprecated, use ldap_values_free_len */
 	char **vals ));
 #endif
 
@@ -1711,7 +1714,7 @@ ldap_search_ext_s LDAP_P((
 
 #if LDAP_DEPRECATED
 LDAP_F( int )
-ldap_search LDAP_P((	/* deprecated */
+ldap_search LDAP_P((	/* deprecated, use ldap_search_ext */
 	LDAP *ld,
 	LDAP_CONST char *base,
 	int scope,
@@ -1720,7 +1723,7 @@ ldap_search LDAP_P((	/* deprecated */
 	int attrsonly ));
 
 LDAP_F( int )
-ldap_search_s LDAP_P((	/* deprecated */
+ldap_search_s LDAP_P((	/* deprecated, use ldap_search_ext_s */
 	LDAP *ld,
 	LDAP_CONST char *base,
 	int scope,
@@ -1730,7 +1733,7 @@ ldap_search_s LDAP_P((	/* deprecated */
 	LDAPMessage **res ));
 
 LDAP_F( int )
-ldap_search_st LDAP_P((	/* deprecated */
+ldap_search_st LDAP_P((	/* deprecated, use ldap_search_ext_s */
 	LDAP *ld,
 	LDAP_CONST char *base,
 	int scope,
@@ -1758,11 +1761,11 @@ ldap_unbind_ext_s LDAP_P((
 
 #if LDAP_DEPRECATED
 LDAP_F( int )
-ldap_unbind LDAP_P(( /* deprecated */
+ldap_unbind LDAP_P(( /* deprecated, use ldap_unbind_ext */
 	LDAP *ld ));
 
 LDAP_F( int )
-ldap_unbind_s LDAP_P(( /* deprecated */
+ldap_unbind_s LDAP_P(( /* deprecated, use ldap_unbind_ext_s */
 	LDAP *ld ));
 #endif
 
@@ -1812,7 +1815,7 @@ ldap_mods_free LDAP_P((
 
 #if LDAP_DEPRECATED
 /*
- * in sort.c (deprecated)
+ * in sort.c (deprecated, use custom code instead)
  */
 typedef int (LDAP_SORT_AD_CMP_PROC) LDAP_P(( /* deprecated */
 	LDAP_CONST char *left,

servers/slapd/ctxcsn.c

@@ -32,8 +32,7 @@ const struct berval slap_ldapsync_cn_bv = BER_BVC("cn=ldapsync");
 void
 slap_get_commit_csn(
 	Operation *op,
-	struct berval *maxcsn,
-	struct berval *curcsn
+	struct berval *maxcsn
 )
 {
 	struct slap_csn_entry *csne, *committed_csne = NULL;
@@ -46,7 +45,6 @@ slap_get_commit_csn(
 
 	LDAP_TAILQ_FOREACH( csne, op->o_bd->be_pending_csn_list, ce_csn_link ) {
 		if ( csne->ce_opid == op->o_opid && csne->ce_connid == op->o_connid ) {
-			if ( curcsn ) *curcsn = csne->ce_csn;
 			csne->ce_state = SLAP_CSN_COMMIT;
 			break;
 		}

servers/slapd/overlays/syncprov.c

@@ -911,6 +911,33 @@ syncprov_qtask( void *ctx, void *arg )
 	return NULL;
 }
 
+/* Start the task to play back queued psearch responses */
+static void
+syncprov_qstart( syncops *so )
+{
+	int wake=0;
+	ldap_pvt_thread_mutex_lock( &slapd_rq.rq_mutex );
+	if ( !so->s_qtask ) {
+		so->s_qtask = ldap_pvt_runqueue_insert( &slapd_rq, RUNQ_INTERVAL,
+			syncprov_qtask, so, "syncprov_qtask",
+			so->s_op->o_conn->c_peer_name.bv_val );
+		++so->s_inuse;
+		wake = 1;
+	} else {
+		if (!ldap_pvt_runqueue_isrunning( &slapd_rq, so->s_qtask ) &&
+			!so->s_qtask->next_sched.tv_sec ) {
+			so->s_qtask->interval.tv_sec = 0;
+			ldap_pvt_runqueue_resched( &slapd_rq, so->s_qtask, 0 );
+			so->s_qtask->interval.tv_sec = RUNQ_INTERVAL;
+			++so->s_inuse;
+			wake = 1;
+		}
+	}
+	ldap_pvt_thread_mutex_unlock( &slapd_rq.rq_mutex );
+	if ( wake )
+		slap_wake_listener();
+}
+
 /* Queue a persistent search response */
 static int
 syncprov_qresp( opcookie *opc, syncops *so, int mode )
@@ -949,27 +976,7 @@ syncprov_qresp( opcookie *opc, syncops *so, int mode )
 		so->s_flags |= PS_FIND_BASE;
 	}
 	if ( so->s_flags & PS_IS_DETACHED ) {
-		int wake=0;
-		ldap_pvt_thread_mutex_lock( &slapd_rq.rq_mutex );
-		if ( !so->s_qtask ) {
-			so->s_qtask = ldap_pvt_runqueue_insert( &slapd_rq, RUNQ_INTERVAL,
-				syncprov_qtask, so, "syncprov_qtask",
-				so->s_op->o_conn->c_peer_name.bv_val );
-			++so->s_inuse;
-			wake = 1;
-		} else {
-			if (!ldap_pvt_runqueue_isrunning( &slapd_rq, so->s_qtask ) &&
-				!so->s_qtask->next_sched.tv_sec ) {
-				so->s_qtask->interval.tv_sec = 0;
-				ldap_pvt_runqueue_resched( &slapd_rq, so->s_qtask, 0 );
-				so->s_qtask->interval.tv_sec = RUNQ_INTERVAL;
-				++so->s_inuse;
-				wake = 1;
-			}
-		}
-		ldap_pvt_thread_mutex_unlock( &slapd_rq.rq_mutex );
-		if ( wake )
-			slap_wake_listener();
+		syncprov_qstart( so );
 	}
 	ldap_pvt_thread_mutex_unlock( &so->s_mutex );
 	return LDAP_SUCCESS;
@@ -1489,7 +1496,7 @@ syncprov_op_response( Operation *op, SlapReply *rs )
 		/* Update our context CSN */
 		cbuf[0] = '\0';
 		ldap_pvt_thread_mutex_lock( &si->si_csn_mutex );
-		slap_get_commit_csn( op, &maxcsn, NULL );
+		slap_get_commit_csn( op, &maxcsn );
 		if ( !BER_BVISNULL( &maxcsn ) ) {
 			strcpy( cbuf, maxcsn.bv_val );
 			if ( ber_bvcmp( &maxcsn, &si->si_ctxcsn ) > 0 ) {
@@ -1877,7 +1884,7 @@ syncprov_search_response( Operation *op, SlapReply *rs )
 			op->o_tmpfree( cookie.bv_val, op->o_tmpmemctx );
 		} else {
 		/* It's RefreshAndPersist, transition to Persist phase */
-			syncprov_sendinfo( op, rs, ( ss->ss_present && rs->sr_nentries ) ?
+			syncprov_sendinfo( op, rs, ss->ss_present ?
 	 			LDAP_TAG_SYNC_REFRESH_PRESENT : LDAP_TAG_SYNC_REFRESH_DELETE,
 				&cookie, 1, NULL, 0 );
 			op->o_tmpfree( cookie.bv_val, op->o_tmpmemctx );
@@ -1889,6 +1896,10 @@ syncprov_search_response( Operation *op, SlapReply *rs )
 			ss->ss_so->s_flags ^= PS_IS_REFRESHING;
 
 			syncprov_detach_op( op, ss->ss_so, on );
+
+			/* If there are queued responses, fire them off */
+			if ( ss->ss_so->s_res )
+				syncprov_qstart( ss->ss_so );
 			ldap_pvt_thread_mutex_unlock( &ss->ss_so->s_mutex );
 
 			return LDAP_SUCCESS;
@@ -1996,7 +2007,10 @@ syncprov_op_search( Operation *op, SlapReply *rs )
 		sl=si->si_logs;
 		if ( sl ) {
 			ldap_pvt_thread_mutex_lock( &sl->sl_mutex );
-			if ( ber_bvcmp( &srs->sr_state.ctxcsn, &sl->sl_mincsn ) >= 0 ) {
+			/* Are there any log entries, and is the consumer state
+			 * present in the session log?
+			 */
+			if ( sl->sl_num > 0 && ber_bvcmp( &srs->sr_state.ctxcsn, &sl->sl_mincsn ) >= 0 ) {
 				do_present = 0;
 				/* mutex is unlocked in playlog */
 				syncprov_playlog( op, rs, sl, srs, &ctxcsn );

servers/slapd/proto-slap.h

@@ -705,7 +705,7 @@ LDAP_SLAPD_F (ContentRule *) cr_bvfind LDAP_P((
 LDAP_SLAPD_V( const struct berval ) slap_ldapsync_bv;
 LDAP_SLAPD_V( const struct berval ) slap_ldapsync_cn_bv;
 LDAP_SLAPD_F (void) slap_get_commit_csn LDAP_P((
-	Operation *, struct berval *maxcsn, struct berval *curcsn ));
+	Operation *, struct berval *maxcsn ));
 LDAP_SLAPD_F (void) slap_rewind_commit_csn LDAP_P(( Operation * ));
 LDAP_SLAPD_F (void) slap_graduate_commit_csn LDAP_P(( Operation * ));
 LDAP_SLAPD_F (Entry *) slap_create_context_csn_entry LDAP_P(( Backend *, struct berval *));