syncrepl.c 111 KB
Newer Older
Kurt Zeilenga's avatar
Kurt Zeilenga committed
1
/* syncrepl.c -- Replication Engine which uses the LDAP Sync protocol */
2
/* $OpenLDAP$ */
Kurt Zeilenga's avatar
Kurt Zeilenga committed
3
/* This work is part of OpenLDAP Software <http://www.openldap.org/>.
4
 *
Kurt Zeilenga's avatar
Kurt Zeilenga committed
5
 * Copyright 2003-2008 The OpenLDAP Foundation.
Kurt Zeilenga's avatar
Kurt Zeilenga committed
6
7
8
 * Portions Copyright 2003 by IBM Corporation.
 * Portions Copyright 2003 by Howard Chu, Symas Corporation.
 * All rights reserved.
9
 *
Kurt Zeilenga's avatar
Kurt Zeilenga committed
10
11
12
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted only as authorized by the OpenLDAP
 * Public License.
13
 *
Kurt Zeilenga's avatar
Kurt Zeilenga committed
14
15
16
 * A copy of this license is available in the file LICENSE in the
 * top-level directory of the distribution or, alternatively, at
 * <http://www.OpenLDAP.org/license.html>.
17
 */
18
19
20
21
22
23
24
25
26
27
28
29

#include "portable.h"

#include <stdio.h>

#include <ac/string.h>
#include <ac/socket.h>

#include "lutil.h"
#include "slap.h"
#include "lutil_ldap.h"

30
31
#include "config.h"

32
33
#include "ldap_rq.h"

34
35
36
37
38
39
struct nonpresent_entry {
	struct berval *npe_name;
	struct berval *npe_nname;
	LDAP_LIST_ENTRY(nonpresent_entry) npe_link;
};

40
41
42
43
44
45
46
47
typedef struct cookie_state {
	ldap_pvt_thread_mutex_t	cs_mutex;
	int	cs_num;
	int cs_age;
	struct berval *cs_vals;
	int *cs_sids;
} cookie_state;
	
48
49
50
51
#define	SYNCDATA_DEFAULT	0	/* entries are plain LDAP entries */
#define	SYNCDATA_ACCESSLOG	1	/* entries are accesslog format */
#define	SYNCDATA_CHANGELOG	2	/* entries are changelog format */

52
53
54
#define	SYNCLOG_LOGGING		0	/* doing a log-based update */
#define	SYNCLOG_FALLBACK	1	/* doing a full refresh */

55
56
57
58
59
#define RETRYNUM_FOREVER	(-1)	/* retry forever */
#define RETRYNUM_TAIL		(-2)	/* end of retrynum array */
#define RETRYNUM_VALID(n)	((n) >= RETRYNUM_FOREVER)	/* valid retrynum */
#define RETRYNUM_FINITE(n)	((n) > RETRYNUM_FOREVER)	/* not forever */

60
typedef struct syncinfo_s {
61
	struct syncinfo_s	*si_next;
62
63
64
65
	BackendDB		*si_be;
	BackendDB		*si_wbe;
	struct re_s		*si_re;
	int			si_rid;
Quanah Gibson-Mount's avatar
Quanah Gibson-Mount committed
66
	char			si_ridtxt[ STRLENOF("rid=999") + 1 ];
Howard Chu's avatar
Howard Chu committed
67
68
	slap_bindconf		si_bindconf;
	struct berval		si_base;
69
70
71
	struct berval		si_logbase;
	struct berval		si_filterstr;
	struct berval		si_logfilterstr;
72
73
74
	int			si_scope;
	int			si_attrsonly;
	char			*si_anfile;
Howard Chu's avatar
Howard Chu committed
75
76
	AttributeName		*si_anlist;
	AttributeName		*si_exanlist;
77
78
79
80
81
82
83
84
85
86
87
	char 			**si_attrs;
	char			**si_exattrs;
	int			si_allattrs;
	int			si_allopattrs;
	int			si_schemachecking;
	int			si_type;	/* the active type */
	int			si_ctype;	/* the configured type */
	time_t			si_interval;
	time_t			*si_retryinterval;
	int			*si_retrynum_init;
	int			*si_retrynum;
Howard Chu's avatar
Howard Chu committed
88
	struct sync_cookie	si_syncCookie;
89
	cookie_state		*si_cookieState;
90
91
92
93
94
95
	int			si_cookieAge;
	int			si_manageDSAit;
	int			si_slimit;
	int			si_tlimit;
	int			si_refreshDelete;
	int			si_refreshPresent;
Quanah Gibson-Mount's avatar
Quanah Gibson-Mount committed
96
	int			si_refreshDone;
97
98
	int			si_syncdata;
	int			si_logstate;
Quanah Gibson-Mount's avatar
Quanah Gibson-Mount committed
99
	ber_int_t	si_msgid;
100
101
	Avlnode			*si_presentlist;
	LDAP			*si_ld;
102
	Connection		*si_conn;
103
	LDAP_LIST_HEAD(np, nonpresent_entry)	si_nonpresentlist;
Howard Chu's avatar
Howard Chu committed
104
	ldap_pvt_thread_mutex_t	si_mutex;
105
106
} syncinfo_t;

107
static int syncuuid_cmp( const void *, const void * );
Quanah Gibson-Mount's avatar
Quanah Gibson-Mount committed
108
static int avl_presentlist_insert( syncinfo_t* si, struct berval *syncUUID );
109
static void syncrepl_del_nonpresent( Operation *, syncinfo_t *, BerVarray, struct berval * );
110
111
static int syncrepl_message_to_op(
					syncinfo_t *, Operation *, LDAPMessage * );
112
113
114
115
116
static int syncrepl_message_to_entry(
					syncinfo_t *, Operation *, LDAPMessage *,
					Modifications **, Entry **, int );
static int syncrepl_entry(
					syncinfo_t *, Operation*, Entry*,
Pierangelo Masarati's avatar
Pierangelo Masarati committed
117
118
					Modifications**,int, struct berval*,
					struct berval *cookieCSN );
119
static int syncrepl_updateCookie(
120
121
122
123
					syncinfo_t *, Operation *, struct berval *,
					struct sync_cookie * );
static struct berval * slap_uuidstr_from_normalized(
					struct berval *, struct berval *, void * );
124

125
/* callback functions */
126
127
128
static int dn_callback( Operation *, SlapReply * );
static int nonpresent_callback( Operation *, SlapReply * );
static int null_callback( Operation *, SlapReply * );
129

130
static AttributeDescription *sync_descs[4];
131

132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
static const char *
syncrepl_state2str( int state )
{
	switch ( state ) {
	case LDAP_SYNC_PRESENT:
		return "PRESENT";

	case LDAP_SYNC_ADD:
		return "ADD";

	case LDAP_SYNC_MODIFY:
		return "MODIFY";

	case LDAP_SYNC_DELETE:
		return "DELETE";
	}

	return "UNKNOWN";
}

152
static void
153
init_syncrepl(syncinfo_t *si)
154
{
Jong Hyuk Choi's avatar
Jong Hyuk Choi committed
155
	int i, j, k, l, n;
156
	char **attrs, **exattrs;
157
158
159
160
161
162
163
164

	if ( !sync_descs[0] ) {
		sync_descs[0] = slap_schema.si_ad_objectClass;
		sync_descs[1] = slap_schema.si_ad_structuralObjectClass;
		sync_descs[2] = slap_schema.si_ad_entryCSN;
		sync_descs[3] = NULL;
	}

165
166
167
168
	if ( si->si_allattrs && si->si_allopattrs )
		attrs = NULL;
	else
		attrs = anlist2attrs( si->si_anlist );
169

170
171
172
173
	if ( attrs ) {
		if ( si->si_allattrs ) {
			i = 0;
			while ( attrs[i] ) {
174
				if ( !is_at_operational( at_find( attrs[i] ) ) ) {
175
176
177
178
					for ( j = i; attrs[j] != NULL; j++ ) {
						if ( j == i )
							ch_free( attrs[i] );
						attrs[j] = attrs[j+1];
179
					}
180
181
				} else {
					i++;
182
183
				}
			}
184
			attrs = ( char ** ) ch_realloc( attrs, (i + 2)*sizeof( char * ) );
185
186
			attrs[i] = ch_strdup("*");
			attrs[i + 1] = NULL;
187

188
189
190
		} else if ( si->si_allopattrs ) {
			i = 0;
			while ( attrs[i] ) {
191
				if ( is_at_operational( at_find( attrs[i] ) ) ) {
192
193
194
195
196
197
198
199
200
					for ( j = i; attrs[j] != NULL; j++ ) {
						if ( j == i )
							ch_free( attrs[i] );
						attrs[j] = attrs[j+1];
					}
				} else {
					i++;
				}
			}
201
			attrs = ( char ** ) ch_realloc( attrs, (i + 2)*sizeof( char * ) );
202
203
			attrs[i] = ch_strdup("+");
			attrs[i + 1] = NULL;
204
		}
Kurt Zeilenga's avatar
Kurt Zeilenga committed
205

206
207
208
		for ( i = 0; sync_descs[i] != NULL; i++ ) {
			j = 0;
			while ( attrs[j] ) {
209
				if ( !strcmp( attrs[j], sync_descs[i]->ad_cname.bv_val ) ) {
210
211
212
213
					for ( k = j; attrs[k] != NULL; k++ ) {
						if ( k == j )
							ch_free( attrs[k] );
						attrs[k] = attrs[k+1];
214
					}
215
216
				} else {
					j++;
217
218
				}
			}
219
220
221
222
223
		}

		for ( n = 0; attrs[ n ] != NULL; n++ ) /* empty */;

		if ( si->si_allopattrs ) {
224
			attrs = ( char ** ) ch_realloc( attrs, (n + 2)*sizeof( char * ) );
225
		} else {
226
			attrs = ( char ** ) ch_realloc( attrs, (n + 4)*sizeof( char * ) );
227
228
229
230
231
232
		}

		/* Add Attributes */
		if ( si->si_allopattrs ) {
			attrs[n++] = ch_strdup( sync_descs[0]->ad_cname.bv_val );
		} else {
233
234
235
			for ( i = 0; sync_descs[ i ] != NULL; i++ ) {
				attrs[ n++ ] = ch_strdup ( sync_descs[i]->ad_cname.bv_val );
			}
236
		}
237
		attrs[ n ] = NULL;
238

239
	} else {
Jong Hyuk Choi's avatar
Jong Hyuk Choi committed
240

241
		i = 0;
242
243
		if ( si->si_allattrs == si->si_allopattrs ) {
			attrs = (char**) ch_malloc( 3 * sizeof(char*) );
244
245
			attrs[i++] = ch_strdup( "*" );
			attrs[i++] = ch_strdup( "+" );
246
247
248
249
250
251
		} else if ( si->si_allattrs && !si->si_allopattrs ) {
			for ( n = 0; sync_descs[ n ] != NULL; n++ ) ;
			attrs = (char**) ch_malloc( (n+1)* sizeof(char*) );
			attrs[i++] = ch_strdup( "*" );
			for ( j = 1; sync_descs[ j ] != NULL; j++ ) {
				attrs[i++] = ch_strdup ( sync_descs[j]->ad_cname.bv_val );
Jong Hyuk Choi's avatar
Jong Hyuk Choi committed
252
			}
253
254
255
256
		} else if ( !si->si_allattrs && si->si_allopattrs ) {
			attrs = (char**) ch_malloc( 3 * sizeof(char*) );
			attrs[i++] = ch_strdup( "+" );
			attrs[i++] = ch_strdup( sync_descs[0]->ad_cname.bv_val );
257
258
		}
		attrs[i] = NULL;
259
260
	}
	
261
262
263
264
265
	si->si_attrs = attrs;

	exattrs = anlist2attrs( si->si_exanlist );

	if ( exattrs ) {
266
267
		for ( n = 0; exattrs[n] != NULL; n++ ) ;

Jong Hyuk Choi's avatar
Jong Hyuk Choi committed
268
		for ( i = 0; sync_descs[i] != NULL; i++ ) {
269
270
			j = 0;
			while ( exattrs[j] != NULL ) {
271
				if ( !strcmp( exattrs[j], sync_descs[i]->ad_cname.bv_val ) ) {
Howard Chu's avatar
Cleanup    
Howard Chu committed
272
					ch_free( exattrs[j] );
273
274
					for ( k = j; exattrs[k] != NULL; k++ ) {
						exattrs[k] = exattrs[k+1];
Jong Hyuk Choi's avatar
Jong Hyuk Choi committed
275
					}
276
277
				} else {
					j++;
Jong Hyuk Choi's avatar
Jong Hyuk Choi committed
278
279
280
				}
			}
		}
281
282
283

		for ( i = 0; exattrs[i] != NULL; i++ ) {
			for ( j = 0; si->si_anlist[j].an_name.bv_val; j++ ) {
284
285
				ObjectClass	*oc;
				if ( ( oc = si->si_anlist[j].an_oc ) ) {
286
287
288
					k = 0;
					while ( oc->soc_required[k] ) {
						if ( !strcmp( exattrs[i],
289
							 oc->soc_required[k]->sat_cname.bv_val ) ) {
Howard Chu's avatar
Cleanup    
Howard Chu committed
290
							ch_free( exattrs[i] );
291
292
293
294
295
							for ( l = i; exattrs[l]; l++ ) {
								exattrs[l] = exattrs[l+1];
							}
						} else {
							k++;
Jong Hyuk Choi's avatar
Jong Hyuk Choi committed
296
297
298
299
300
						}
					}
				}
			}
		}
301
302

		for ( i = 0; exattrs[i] != NULL; i++ ) ;
303
304

		if ( i != n )
305
			exattrs = (char **) ch_realloc( exattrs, (i + 1)*sizeof(char *) );
Jong Hyuk Choi's avatar
Jong Hyuk Choi committed
306
	}
307
308

	si->si_exattrs = exattrs;	
309
310
}

Howard Chu's avatar
Howard Chu committed
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
typedef struct logschema {
	struct berval ls_dn;
	struct berval ls_req;
	struct berval ls_mod;
	struct berval ls_newRdn;
	struct berval ls_delRdn;
	struct berval ls_newSup;
} logschema;

static logschema changelog_sc = {
	BER_BVC("targetDN"),
	BER_BVC("changeType"),
	BER_BVC("changes"),
	BER_BVC("newRDN"),
	BER_BVC("deleteOldRDN"),
	BER_BVC("newSuperior")
};

static logschema accesslog_sc = {
	BER_BVC("reqDN"),
	BER_BVC("reqType"),
	BER_BVC("reqMod"),
	BER_BVC("reqNewRDN"),
	BER_BVC("reqDeleteOldRDN"),
	BER_BVC("reqNewSuperior")
};
337

338
static int
Jong Hyuk Choi's avatar
Jong Hyuk Choi committed
339
340
ldap_sync_search(
	syncinfo_t *si,
Kurt Zeilenga's avatar
Kurt Zeilenga committed
341
	void *ctx )
Jong Hyuk Choi's avatar
Jong Hyuk Choi committed
342
{
343
344
345
346
	BerElementBuffer berbuf;
	BerElement *ber = (BerElement *)&berbuf;
	LDAPControl c[2], *ctrls[3];
	int rc;
347
348
	int rhint;
	char *base;
Howard Chu's avatar
Howard Chu committed
349
	char **attrs, *lattrs[8];
350
351
352
	char *filter;
	int attrsonly;
	int scope;
Jong Hyuk Choi's avatar
Jong Hyuk Choi committed
353

354
355
356
	/* setup LDAP SYNC control */
	ber_init2( ber, NULL, LBER_USE_DER );
	ber_set_option( ber, LBER_OPT_BER_MEMCTX, &ctx );
Jong Hyuk Choi's avatar
Jong Hyuk Choi committed
357

358
359
360
	/* If we're using a log but we have no state, then fallback to
	 * normal mode for a full refresh.
	 */
361
	if ( si->si_syncdata && !si->si_syncCookie.numcsns ) {
362
		si->si_logstate = SYNCLOG_FALLBACK;
363
	}
364
365
366

	/* Use the log parameters if we're in log mode */
	if ( si->si_syncdata && si->si_logstate == SYNCLOG_LOGGING ) {
Howard Chu's avatar
Howard Chu committed
367
368
369
370
371
372
373
374
375
376
377
378
379
380
		logschema *ls;
		if ( si->si_syncdata == SYNCDATA_ACCESSLOG )
			ls = &accesslog_sc;
		else
			ls = &changelog_sc;
		lattrs[0] = ls->ls_dn.bv_val;
		lattrs[1] = ls->ls_req.bv_val;
		lattrs[2] = ls->ls_mod.bv_val;
		lattrs[3] = ls->ls_newRdn.bv_val;
		lattrs[4] = ls->ls_delRdn.bv_val;
		lattrs[5] = ls->ls_newSup.bv_val;
		lattrs[6] = slap_schema.si_ad_entryCSN->ad_cname.bv_val;
		lattrs[7] = NULL;

381
382
383
		rhint = 0;
		base = si->si_logbase.bv_val;
		filter = si->si_logfilterstr.bv_val;
Howard Chu's avatar
Howard Chu committed
384
		attrs = lattrs;
385
		attrsonly = 0;
386
		scope = LDAP_SCOPE_SUBTREE;
387
388
389
390
391
392
393
394
395
	} else {
		rhint = 1;
		base = si->si_base.bv_val;
		filter = si->si_filterstr.bv_val;
		attrs = si->si_attrs;
		attrsonly = si->si_attrsonly;
		scope = si->si_scope;
	}
	if ( si->si_syncdata && si->si_logstate == SYNCLOG_FALLBACK ) {
Howard Chu's avatar
Howard Chu committed
396
		si->si_type = LDAP_SYNC_REFRESH_ONLY;
397
	} else {
Howard Chu's avatar
Howard Chu committed
398
		si->si_type = si->si_ctype;
399
400
	}

401
	if ( !BER_BVISNULL( &si->si_syncCookie.octet_str ) )
Kurt Zeilenga's avatar
Kurt Zeilenga committed
402
	{
Howard Chu's avatar
Howard Chu committed
403
		ber_printf( ber, "{eOb}",
Howard Chu's avatar
Howard Chu committed
404
			abs(si->si_type), &si->si_syncCookie.octet_str, rhint );
405
	} else {
Howard Chu's avatar
Howard Chu committed
406
		ber_printf( ber, "{eb}",
Howard Chu's avatar
Howard Chu committed
407
			abs(si->si_type), rhint );
408
	}
Jong Hyuk Choi's avatar
Jong Hyuk Choi committed
409

410
	if ( (rc = ber_flatten2( ber, &c[0].ldctl_value, 0 ) ) == LBER_ERROR ) {
411
412
413
		ber_free_buf( ber );
		return rc;
	}
Jong Hyuk Choi's avatar
Jong Hyuk Choi committed
414

415
	c[0].ldctl_oid = LDAP_CONTROL_SYNC;
416
	c[0].ldctl_iscritical = si->si_type < 0;
417
	ctrls[0] = &c[0];
Jong Hyuk Choi's avatar
Jong Hyuk Choi committed
418

419
	if ( !BER_BVISNULL( &si->si_bindconf.sb_authzId ) ) {
420
		c[1].ldctl_oid = LDAP_CONTROL_PROXY_AUTHZ;
421
		c[1].ldctl_value = si->si_bindconf.sb_authzId;
422
423
424
425
426
427
		c[1].ldctl_iscritical = 1;
		ctrls[1] = &c[1];
		ctrls[2] = NULL;
	} else {
		ctrls[1] = NULL;
	}
Jong Hyuk Choi's avatar
Jong Hyuk Choi committed
428

429
	rc = ldap_search_ext( si->si_ld, base, scope, filter, attrs, attrsonly,
Quanah Gibson-Mount's avatar
Quanah Gibson-Mount committed
430
		ctrls, NULL, NULL, si->si_slimit, &si->si_msgid );
431
	ber_free_buf( ber );
Jong Hyuk Choi's avatar
Jong Hyuk Choi committed
432
433
434
	return rc;
}

435
436
437
438
static int
do_syncrep1(
	Operation *op,
	syncinfo_t *si )
439
440
{
	int	rc;
441
	int cmdline_cookie_found = 0;
442

443
	struct sync_cookie	*sc = NULL;
444
445
446
#ifdef HAVE_TLS
	void	*ssl;
#endif
447

448
	rc = slap_client_connect( &si->si_ld, &si->si_bindconf );
Jong Hyuk Choi's avatar
Jong Hyuk Choi committed
449
	if ( rc != LDAP_SUCCESS ) {
450
		goto done;
451
	}
452
	op->o_protocol = LDAP_VERSION3;
453

454
455
456
	/* Set SSF to strongest of TLS, SASL SSFs */
	op->o_sasl_ssf = 0;
	op->o_tls_ssf = 0;
457
	op->o_transport_ssf = 0;
458
#ifdef HAVE_TLS
459
460
461
	if ( ldap_get_option( si->si_ld, LDAP_OPT_X_TLS_SSL_CTX, &ssl )
		== LDAP_SUCCESS && ssl != NULL )
	{
462
463
464
465
		op->o_tls_ssf = ldap_pvt_tls_get_strength( ssl );
	}
#endif /* HAVE_TLS */
	ldap_get_option( si->si_ld, LDAP_OPT_X_SASL_SSF, &op->o_sasl_ssf );
466
467
	op->o_ssf = ( op->o_sasl_ssf > op->o_tls_ssf )
		?  op->o_sasl_ssf : op->o_tls_ssf;
468

469
470
	ldap_set_option( si->si_ld, LDAP_OPT_TIMELIMIT, &si->si_tlimit );

471
472
473
	/* We've just started up, or the remote server hasn't sent us
	 * any meaningful state.
	 */
474
	if ( BER_BVISNULL( &si->si_syncCookie.octet_str ) ) {
475
		int i;
476

477
		si->si_syncCookie.rid = si->si_rid;
478
479
		si->si_syncCookie.sid = SLAP_SINGLE_SHADOW( si->si_be ) ? -1 :
			slap_serverID;
480
481
482
483
484
485
486
487
488
489

		LDAP_STAILQ_FOREACH( sc, &slap_sync_cookie, sc_next ) {
			if ( si->si_rid == sc->rid ) {
				cmdline_cookie_found = 1;
				break;
			}
		}

		if ( cmdline_cookie_found ) {
			/* cookie is supplied in the command line */
490

491
492
			LDAP_STAILQ_REMOVE( &slap_sync_cookie, sc, sync_cookie, sc_next );

493
494
			/* ctxcsn wasn't parsed yet, do it now */
			slap_parse_sync_cookie( sc, op->o_tmpmemctx );
495
			slap_sync_cookie_free( &si->si_syncCookie, 0 );
496
497
			slap_dup_sync_cookie( &si->si_syncCookie, sc );
			slap_sync_cookie_free( sc, 1 );
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
		} else {
			ldap_pvt_thread_mutex_lock( &si->si_cookieState->cs_mutex );
			if ( !si->si_cookieState->cs_num ) {
				/* get contextCSN shadow replica from database */
				BerVarray csn = NULL;
				void *ctx = op->o_tmpmemctx;

				op->o_req_ndn = op->o_bd->be_nsuffix[0];
				op->o_req_dn = op->o_req_ndn;

				/* try to read stored contextCSN */
				op->o_tmpmemctx = NULL;
				backend_attribute( op, NULL, &op->o_req_ndn,
					slap_schema.si_ad_contextCSN, &csn, ACL_READ );
				op->o_tmpmemctx = ctx;
				if ( csn ) {
					si->si_cookieState->cs_vals = csn;
					for (i=0; !BER_BVISNULL( &csn[i] ); i++);
					si->si_cookieState->cs_num = i;
Howard Chu's avatar
Howard Chu committed
517
					si->si_cookieState->cs_sids = slap_parse_csn_sids( csn, i, NULL );
518
519
520
521
522
523
524
				}
			}
			if ( si->si_cookieState->cs_num ) {
				ber_bvarray_free( si->si_syncCookie.ctxcsn );
				if ( ber_bvarray_dup_x( &si->si_syncCookie.ctxcsn,
					si->si_cookieState->cs_vals, NULL )) {
					rc = LDAP_NO_MEMORY;
Quanah Gibson-Mount's avatar
Quanah Gibson-Mount committed
525
					ldap_pvt_thread_mutex_unlock( &si->si_cookieState->cs_mutex );
526
527
528
529
530
531
532
533
534
					goto done;
				}
				si->si_syncCookie.numcsns = si->si_cookieState->cs_num;
				si->si_syncCookie.sids = ch_malloc( si->si_cookieState->cs_num *
					sizeof(int) );
				for ( i=0; i<si->si_syncCookie.numcsns; i++ )
					si->si_syncCookie.sids[i] = si->si_cookieState->cs_sids[i];
			}
			ldap_pvt_thread_mutex_unlock( &si->si_cookieState->cs_mutex );
535
		}
Kurt Zeilenga's avatar
Kurt Zeilenga committed
536

Howard Chu's avatar
Howard Chu committed
537
		slap_compose_sync_cookie( NULL, &si->si_syncCookie.octet_str,
538
			si->si_syncCookie.ctxcsn, si->si_syncCookie.rid,
539
			si->si_syncCookie.sid );
540
	} else {
541
		AttributeName at[2];
542
		Attribute a = {0};
543
544
545
546
547
548
549
550
551
552
		Entry e = {0};
		SlapReply rs = {0};
		int i, j, changed = 0;

		/* Look for contextCSN from syncprov overlay. If
		 * there's no overlay, this will be a no-op. That means
		 * this is a pure consumer, so local changes will not be
		 * allowed, and all changes will already be reflected in
		 * the cookieState.
		 */
553
		a.a_desc = slap_schema.si_ad_contextCSN;
554
555
556
557
558
559
560
561
		e.e_attrs = &a;
		e.e_name = si->si_wbe->be_suffix[0];
		e.e_nname = si->si_wbe->be_nsuffix[0];
		rs.sr_entry = &e;
		rs.sr_flags = REP_ENTRY_MODIFIABLE;
		at[0].an_name = a.a_desc->ad_cname;
		at[0].an_desc = a.a_desc;
		BER_BVZERO( &at[1].an_name );
562
563
		op->o_req_dn = e.e_name;
		op->o_req_ndn = e.e_nname;
564

565
		ldap_pvt_thread_mutex_lock( &si->si_cookieState->cs_mutex );
566
567
		rc = backend_operational( op, &rs );
		if ( rc == LDAP_SUCCESS && a.a_vals ) {
568
			int num = a.a_numvals;
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
			/* check for differences */
			if ( num != si->si_cookieState->cs_num ) {
				changed = 1;
			} else {
				for ( i=0; i<num; i++ ) {
					if ( ber_bvcmp( &a.a_vals[i],
						&si->si_cookieState->cs_vals[i] )) {
						changed =1;
						break;
					}
				}
			}
			if ( changed ) {
				ber_bvarray_free( si->si_cookieState->cs_vals );
				ch_free( si->si_cookieState->cs_sids );
				si->si_cookieState->cs_num = num;
				si->si_cookieState->cs_vals = a.a_vals;
				si->si_cookieState->cs_sids = slap_parse_csn_sids( a.a_vals,
Howard Chu's avatar
Howard Chu committed
587
					num, NULL );
588
589
590
591
592
593
594
595
596
597
				si->si_cookieState->cs_age++;
			} else {
				ber_bvarray_free( a.a_vals );
			}
			changed = 0;
		}
		/* See if the cookieState has changed due to anything outside
		 * this particular consumer. That includes other consumers in
		 * the same context, or local changes detected above.
		 */
598
599
600
601
602
603
604
605
606
		if ( si->si_cookieState->cs_num > 1 && si->si_cookieAge !=
			si->si_cookieState->cs_age ) {

			for (i=0; !BER_BVISNULL( &si->si_syncCookie.ctxcsn[i] ); i++) {
				/* bogus, just dup everything */
				if ( si->si_syncCookie.sids[i] == -1 ) {
					ber_bvarray_free( si->si_syncCookie.ctxcsn );
					ber_bvarray_dup_x( &si->si_syncCookie.ctxcsn,
						si->si_cookieState->cs_vals, NULL );
607
					changed = 1;
608
609
610
611
612
613
					break;
				}
				for (j=0; j<si->si_cookieState->cs_num; j++) {
					if ( si->si_syncCookie.sids[i] !=
						si->si_cookieState->cs_sids[j] )
						continue;
614
615
616
					if ( bvmatch( &si->si_syncCookie.ctxcsn[i],
						&si->si_cookieState->cs_vals[j] ))
						break;
617
618
					ber_bvreplace( &si->si_syncCookie.ctxcsn[i],
						&si->si_cookieState->cs_vals[j] );
619
					changed = 1;
620
621
622
					break;
				}
			}
623
624
625
626
627
628
			if ( changed ) {
				ch_free( si->si_syncCookie.octet_str.bv_val );
				slap_compose_sync_cookie( NULL, &si->si_syncCookie.octet_str,
					si->si_syncCookie.ctxcsn, si->si_syncCookie.rid,
					SLAP_SINGLE_SHADOW( si->si_be ) ? -1 : slap_serverID );
			}
629
630
		}
		ldap_pvt_thread_mutex_unlock( &si->si_cookieState->cs_mutex );
Jong Hyuk Choi's avatar
Jong Hyuk Choi committed
631
632
	}

Quanah Gibson-Mount's avatar
Quanah Gibson-Mount committed
633
634
	si->si_refreshDone = 0;

Jong Hyuk Choi's avatar
Jong Hyuk Choi committed
635
	rc = ldap_sync_search( si, op->o_tmpmemctx );
636

637
	if( rc != LDAP_SUCCESS ) {
638
		Debug( LDAP_DEBUG_ANY, "do_syncrep1: %s "
639
			"ldap_search_ext: %s (%d)\n",
640
			si->si_ridtxt, ldap_err2string( rc ), rc );
641
642
643
644
645
	}

done:
	if ( rc ) {
		if ( si->si_ld ) {
646
			ldap_unbind_ext( si->si_ld, NULL, NULL );
647
648
649
650
651
652
653
			si->si_ld = NULL;
		}
	}

	return rc;
}

654
655
656
657
658
659
660
661
static int
compare_csns( struct sync_cookie *sc1, struct sync_cookie *sc2, int *which )
{
	int i, j, match = 0;
	const char *text;

	*which = 0;

662
663
664
665
666
	if ( sc1->numcsns < sc2->numcsns ) {
		*which = sc1->numcsns;
		return -1;
	}

667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
	for (i=0; !BER_BVISNULL( &sc1->ctxcsn[i] ); i++) {
		for (j=0; !BER_BVISNULL( &sc2->ctxcsn[j] ); j++) {
			if ( sc1->sids[i] != sc2->sids[j] )
				continue;
			value_match( &match, slap_schema.si_ad_entryCSN,
				slap_schema.si_ad_entryCSN->ad_type->sat_ordering,
				SLAP_MR_VALUE_OF_ATTRIBUTE_SYNTAX,
				&sc1->ctxcsn[i], &sc2->ctxcsn[i], &text );
			if ( match < 0 ) {
				*which = j;
				return match;
			}
			break;
		}
	}
	return match;
}

685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
static int
do_syncrep2(
	Operation *op,
	syncinfo_t *si )
{
	LDAPControl	**rctrls = NULL;
	LDAPControl	*rctrlp;

	BerElementBuffer berbuf;
	BerElement	*ber = (BerElement *)&berbuf;

	LDAPMessage	*res = NULL;
	LDAPMessage	*msg = NULL;

	char		*retoid = NULL;
	struct berval	*retdata = NULL;

	Entry		*entry = NULL;

	int		syncstate;
Kurt Zeilenga's avatar
Kurt Zeilenga committed
705
	struct berval	syncUUID = BER_BVNULL;
706
707
	struct sync_cookie	syncCookie = { NULL };
	struct sync_cookie	syncCookie_req = { NULL };
Kurt Zeilenga's avatar
Kurt Zeilenga committed
708
	struct berval		cookie = BER_BVNULL;
709

710
711
	int		rc,
			err = LDAP_SUCCESS;
712
713
714
715
716
	ber_len_t	len;

	struct berval	*psub;
	Modifications	*modlist = NULL;

717
	int				match, m;
718
719
720
721

	struct timeval *tout_p = NULL;
	struct timeval tout = { 0, 0 };

722
	int		refreshDeletes = 0;
723
	BerVarray syncUUIDs = NULL;
724
725
	ber_tag_t si_tag;

726
	if ( slapd_shutdown ) {
727
		rc = -2;
728
		goto done;
729
730
	}

731
732
733
	ber_init2( ber, NULL, LBER_USE_DER );
	ber_set_option( ber, LBER_OPT_BER_MEMCTX, &op->o_tmpmemctx );

734
	Debug( LDAP_DEBUG_TRACE, "=>do_syncrep2 %s\n", si->si_ridtxt, 0, 0 );
735
736
737

	psub = &si->si_be->be_nsuffix[0];

Jong Hyuk Choi's avatar
Jong Hyuk Choi committed
738
	slap_dup_sync_cookie( &syncCookie_req, &si->si_syncCookie );
739

Kurt Zeilenga's avatar
Kurt Zeilenga committed
740
	if ( abs(si->si_type) == LDAP_SYNC_REFRESH_AND_PERSIST ) {
741
742
743
744
745
		tout_p = &tout;
	} else {
		tout_p = NULL;
	}

Quanah Gibson-Mount's avatar
Quanah Gibson-Mount committed
746
	while ( ( rc = ldap_result( si->si_ld, si->si_msgid, LDAP_MSG_ONE,
747
		tout_p, &res ) ) > 0 )
748
	{
749
		if ( slapd_shutdown ) {
750
751
			rc = -2;
			goto done;
752
		}
753
		for( msg = ldap_first_message( si->si_ld, res );
Kurt Zeilenga's avatar
Kurt Zeilenga committed
754
755
			msg != NULL;
			msg = ldap_next_message( si->si_ld, msg ) )
756
		{
757
758
759
760
			if ( slapd_shutdown ) {
				rc = -2;
				goto done;
			}
761
762
			switch( ldap_msgtype( msg ) ) {
			case LDAP_RES_SEARCH_ENTRY:
763
				ldap_get_entry_controls( si->si_ld, msg, &rctrls );
Howard Chu's avatar
Howard Chu committed
764
				/* we can't work without the control */
765
766
				rctrlp = NULL;
				if ( rctrls ) {
767
					LDAPControl **next;
768
769
770
					/* NOTE: make sure we use the right one;
					 * a better approach would be to run thru
					 * the whole list and take care of all */
771
772
773
774
775
776
777
778
779
					rctrlp = ldap_control_find( LDAP_CONTROL_SYNC_STATE, rctrls, &next );
					if ( next && ldap_control_find( LDAP_CONTROL_SYNC_STATE, next, NULL ) )
					{
						Debug( LDAP_DEBUG_ANY, "do_syncrep2: %s "
							"got search entry with multiple "
							"Sync State control\n", si->si_ridtxt, 0, 0 );
						rc = -1;
						goto done;
					}
780
781
				}
				if ( rctrlp == NULL ) {
782
					Debug( LDAP_DEBUG_ANY, "do_syncrep2: %s "
783
						"got search entry without "
784
						"Sync State control\n", si->si_ridtxt, 0, 0 );
Howard Chu's avatar
Howard Chu committed
785
786
787
788
					rc = -1;
					goto done;
				}
				ber_init2( ber, &rctrlp->ldctl_value, LBER_USE_DER );
Kurt Zeilenga's avatar
Kurt Zeilenga committed
789
				ber_scanf( ber, "{em" /*"}"*/, &syncstate, &syncUUID );
790
791
792
				/* FIXME: what if syncUUID is NULL or empty?
				 * (happens with back-sql...) */
				if ( BER_BVISEMPTY( &syncUUID ) ) {
793
					Debug( LDAP_DEBUG_ANY, "do_syncrep2: %s "
794
795
796
						"got empty syncUUID with LDAP_SYNC_%s\n",
						si->si_ridtxt,
						syncrepl_state2str( syncstate ), 0 );
Pierangelo Masarati's avatar
Pierangelo Masarati committed
797
					ldap_controls_free( rctrls );
798
799
800
					rc = -1;
					goto done;
				}
801
				if ( ber_peek_tag( ber, &len ) == LDAP_TAG_SYNC_COOKIE ) {
Kurt Zeilenga's avatar
Kurt Zeilenga committed
802
					ber_scanf( ber, /*"{"*/ "m}", &cookie );
Quanah Gibson-Mount's avatar
Quanah Gibson-Mount committed
803
804
805
806

					Debug( LDAP_DEBUG_SYNC, "do_syncrep2: cookie=%s\n",
						BER_BVISNULL( &cookie ) ? "" : cookie.bv_val, 0, 0 );

807
					if ( !BER_BVISNULL( &cookie ) ) {
808
809
						ch_free( syncCookie.octet_str.bv_val );
						ber_dupbv( &syncCookie.octet_str, &cookie );
Jong Hyuk Choi's avatar
Jong Hyuk Choi committed
810
					}
811
					if ( !BER_BVISNULL( &syncCookie.octet_str ) )
812
					{
Howard Chu's avatar
Howard Chu committed
813
						slap_parse_sync_cookie( &syncCookie, NULL );
Quanah Gibson-Mount's avatar
Quanah Gibson-Mount committed
814
815
816
817
818
819
820
821
822
823
824
825
826
						if ( syncCookie.ctxcsn ) {
							int i, sid = slap_parse_csn_sid( syncCookie.ctxcsn );
							for ( i =0; i<si->si_cookieState->cs_num; i++ ) {
								if ( si->si_cookieState->cs_sids[i] == sid && 
									ber_bvcmp( syncCookie.ctxcsn, &si->si_cookieState->cs_vals[i] ) <= 0 ) {
									Debug( LDAP_DEBUG_SYNC, "do_syncrep2: %s CSN too old, ignoring %s\n",
										si->si_ridtxt, syncCookie.ctxcsn->bv_val, 0 );
									ldap_controls_free( rctrls );
									rc = 0;
									goto done;
								}
							}
						}
827
					}
828
				}
829
				rc = 0;
830
				if ( si->si_syncdata && si->si_logstate == SYNCLOG_LOGGING ) {
831
					modlist = NULL;
832
					if ( ( rc = syncrepl_message_to_op( si, op, msg ) ) == LDAP_SUCCESS &&
833
						syncCookie.ctxcsn )
834
					{
835
						rc = syncrepl_updateCookie( si, op, psub, &syncCookie );
Quanah Gibson-Mount's avatar
Quanah Gibson-Mount committed
836
837
838
839
					} else if ( rc == LDAP_NO_SUCH_OBJECT ) {
						rc = LDAP_SYNC_REFRESH_REQUIRED;
						si->si_logstate = SYNCLOG_FALLBACK;
						ldap_abandon_ext( si->si_ld, si->si_msgid, NULL, NULL );
840
					}
841
842
843
844
				} else if ( ( rc = syncrepl_message_to_entry( si, op, msg,
					&modlist, &entry, syncstate ) ) == LDAP_SUCCESS )
				{
					if ( ( rc = syncrepl_entry( si, op, entry, &modlist,
Pierangelo Masarati's avatar
Pierangelo Masarati committed
845
						syncstate, &syncUUID, syncCookie.ctxcsn ) ) == LDAP_SUCCESS &&
846
						syncCookie.ctxcsn )
847
					{
848
						rc = syncrepl_updateCookie( si, op, psub, &syncCookie );
849
					}
850
				}
Howard Chu's avatar
Howard Chu committed
851
				ldap_controls_free( rctrls );
Jong Hyuk Choi's avatar
Jong Hyuk Choi committed
852
				if ( modlist ) {
853
					slap_mods_free( modlist, 1 );
Jong Hyuk Choi's avatar
Jong Hyuk Choi committed
854
				}
855
856
				if ( rc )
					goto done;
857
858
859
860
				break;

			case LDAP_RES_SEARCH_REFERENCE:
				Debug( LDAP_DEBUG_ANY,
861
862
					"do_syncrep2: %s reference received error\n",
					si->si_ridtxt, 0, 0 );
863
864
865
				break;

			case LDAP_RES_SEARCH_RESULT:
Jong Hyuk Choi's avatar
Jong Hyuk Choi committed
866
				Debug( LDAP_DEBUG_SYNC,
867
868
					"do_syncrep2: %s LDAP_RES_SEARCH_RESULT\n",
					si->si_ridtxt, 0, 0 );
869
				ldap_parse_result( si->si_ld, msg, &err, NULL, NULL, NULL,
870
					&rctrls, 0 );
871
872
873
874
875
876
#ifdef LDAP_X_SYNC_REFRESH_REQUIRED
				if ( err == LDAP_X_SYNC_REFRESH_REQUIRED ) {
					/* map old result code to registered code */
					err = LDAP_SYNC_REFRESH_REQUIRED;
				}
#endif
877
878
879
880
881
				if ( err == LDAP_SYNC_REFRESH_REQUIRED ) {
					if ( si->si_logstate == SYNCLOG_LOGGING ) {
						si->si_logstate = SYNCLOG_FALLBACK;
					}
					rc = err;
Howard Chu's avatar
Howard Chu committed
882
					goto done;
883
				}
884
885
				if ( rctrls ) {
					rctrlp = *rctrls;
886
					ber_init2( ber, &rctrlp->ldctl_value, LBER_USE_DER );
Jong Hyuk Choi's avatar
Jong Hyuk Choi committed
887

888
					ber_scanf( ber, "{" /*"}"*/);
Kurt Zeilenga's avatar
Kurt Zeilenga committed
889
					if ( ber_peek_tag( ber, &len ) == LDAP_TAG_SYNC_COOKIE ) {
Jong Hyuk Choi's avatar
Jong Hyuk Choi committed
890
						ber_scanf( ber, "m", &cookie );
Quanah Gibson-Mount's avatar
Quanah Gibson-Mount committed
891
892
893
894

						Debug( LDAP_DEBUG_SYNC, "do_syncrep2: cookie=%s\n",
							BER_BVISNULL( &cookie ) ? "" : cookie.bv_val, 0, 0 );

895
						if ( !BER_BVISNULL( &cookie ) ) {
896
897
							ch_free( syncCookie.octet_str.bv_val );
							ber_dupbv( &syncCookie.octet_str, &cookie);
Jong Hyuk Choi's avatar
Jong Hyuk Choi committed
898
						}
899
						if ( !BER_BVISNULL( &syncCookie.octet_str ) )
Kurt Zeilenga's avatar
Kurt Zeilenga committed
900
						{
Howard Chu's avatar
Howard Chu committed
901
							slap_parse_sync_cookie( &syncCookie, NULL );
Kurt Zeilenga's avatar
Kurt Zeilenga committed
902
						}
903
					}
904
905
906
907
					if ( ber_peek_tag( ber, &len ) == LDAP_TAG_REFRESHDELETES )
					{
						ber_scanf( ber, "b", &refreshDeletes );
					}
Kurt Zeilenga's avatar
Kurt Zeilenga committed
908
					ber_scanf( ber, /*"{"*/ "}" );
909
				}
Howard Chu's avatar
Howard Chu committed
910
911
912
				if ( !syncCookie.ctxcsn ) {
					match = 1;
				} else if ( !syncCookie_req.ctxcsn ) {
Jong Hyuk Choi's avatar
Jong Hyuk Choi committed
913
					match = -1;
Howard Chu's avatar
Howard Chu committed
914
					m = 0;
Jong Hyuk Choi's avatar
Jong Hyuk Choi committed
915
				} else {
916
					match = compare_csns( &syncCookie_req, &syncCookie, &m );
Jong Hyuk Choi's avatar
Jong Hyuk Choi committed
917
				}
918
919
920
921
				if ( rctrls ) {
					ldap_controls_free( rctrls );
				}
				if (si->si_type != LDAP_SYNC_REFRESH_AND_PERSIST) {
Jong Hyuk Choi's avatar
Jong Hyuk Choi committed
922
					/* FIXME : different error behaviors according to
923
924
925
					 *	1) err code : LDAP_BUSY ...
					 *	2) on err policy : stop service, stop sync, retry
					 */
926
					if ( refreshDeletes == 0 && match < 0 &&
Kurt Zeilenga's avatar
Kurt Zeilenga committed
927
928
						err == LDAP_SUCCESS )
					{
929
930
						syncrepl_del_nonpresent( op, si, NULL,
							&syncCookie.ctxcsn[m] );
931
					} else {
Quanah Gibson-Mount's avatar
Quanah Gibson-Mount committed
932
						avl_free( si->si_presentlist, ch_free );
933
						si->si_presentlist = NULL;
Jong Hyuk Choi's avatar
Jong Hyuk Choi committed
934
					}
935
				}
936
				if ( syncCookie.ctxcsn && match < 0 && err == LDAP_SUCCESS )
937
938
939
				{
					rc = syncrepl_updateCookie( si, op, psub, &syncCookie );
				}
940
941
942
943
944
945
946
				if ( err == LDAP_SUCCESS
					&& si->si_logstate == SYNCLOG_FALLBACK ) {
					si->si_logstate = SYNCLOG_LOGGING;
					rc = LDAP_SYNC_REFRESH_REQUIRED;
				} else {
					rc = -2;
				}
947
				goto done;
948
949
				break;

950
			case LDAP_RES_INTERMEDIATE:
951
				rc = ldap_parse_intermediate( si->si_ld, msg,
Kurt Zeilenga's avatar
Kurt Zeilenga committed
952
953
					&retoid, &retdata, NULL, 0 );
				if ( !rc && !strcmp( retoid, LDAP_SYNC_INFO ) ) {
954
					ber_init2( ber, retdata, LBER_USE_DER );
955

956
					switch ( si_tag = ber_peek_tag( ber, &len ) ) {
957
958
					ber_tag_t tag;
					case LDAP_TAG_SYNC_NEW_COOKIE:
Jong Hyuk Choi's avatar
Jong Hyuk Choi committed
959
						Debug( LDAP_DEBUG_SYNC,
960
961
							"do_syncrep2: %s %s - %s\n", 
							si->si_ridtxt,
Jong Hyuk Choi's avatar
Jong Hyuk Choi committed
962
							"LDAP_RES_INTERMEDIATE", 
963
							"NEW_COOKIE" );
964
965
966
967
						ber_scanf( ber, "tm", &tag, &cookie );
						break;
					case LDAP_TAG_SYNC_REFRESH_DELETE:
					case LDAP_TAG_SYNC_REFRESH_PRESENT:
Jong Hyuk Choi's avatar
Jong Hyuk Choi committed
968
						Debug( LDAP_DEBUG_SYNC,
969
970
							"do_syncrep2: %s %s - %s\n", 
							si->si_ridtxt,
Jong Hyuk Choi's avatar
Jong Hyuk Choi committed
971
972
							"LDAP_RES_INTERMEDIATE", 
							si_tag == LDAP_TAG_SYNC_REFRESH_PRESENT ?
973
							"REFRESH_PRESENT" : "REFRESH_DELETE" );
Howard Chu's avatar
Howard Chu committed
974
						if ( si_tag == LDAP_TAG_SYNC_REFRESH_DELETE ) {
Howard Chu's avatar
Howard Chu committed
975
976
977
978
							si->si_refreshDelete = 1;
						} else {
							si->si_refreshPresent = 1;
						}
Kurt Zeilenga's avatar
Kurt Zeilenga committed
979
						ber_scanf( ber, "t{" /*"}"*/, &tag );
980
981
						if ( ber_peek_tag( ber, &len ) == LDAP_TAG_SYNC_COOKIE )
						{
982
							ber_scanf( ber, "m", &cookie );
Quanah Gibson-Mount's avatar
Quanah Gibson-Mount committed
983
984
985
986

							Debug( LDAP_DEBUG_SYNC, "do_syncrep2: cookie=%s\n",
								BER_BVISNULL( &cookie ) ? "" : cookie.bv_val, 0, 0 );

987
							if ( !BER_BVISNULL( &cookie ) ) {
988
989
								ch_free( syncCookie.octet_str.bv_val );
								ber_dupbv( &syncCookie.octet_str, &cookie );