syncrepl.c 126 KB
Newer Older
Kurt Zeilenga's avatar
Kurt Zeilenga committed
1
/* syncrepl.c -- Replication Engine which uses the LDAP Sync protocol */
2
/* $OpenLDAP$ */
Kurt Zeilenga's avatar
Kurt Zeilenga committed
3
/* This work is part of OpenLDAP Software <http://www.openldap.org/>.
4
 *
5
 * Copyright 2003-2009 The OpenLDAP Foundation.
Kurt Zeilenga's avatar
Kurt Zeilenga committed
6
 * Portions Copyright 2003 by IBM Corporation.
Quanah Gibson-Mount's avatar
Quanah Gibson-Mount committed
7
 * Portions Copyright 2003-2008 by Howard Chu, Symas Corporation.
Kurt Zeilenga's avatar
Kurt Zeilenga committed
8
 * All rights reserved.
9
 *
Kurt Zeilenga's avatar
Kurt Zeilenga committed
10
11
12
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted only as authorized by the OpenLDAP
 * Public License.
13
 *
Kurt Zeilenga's avatar
Kurt Zeilenga committed
14
15
16
 * A copy of this license is available in the file LICENSE in the
 * top-level directory of the distribution or, alternatively, at
 * <http://www.OpenLDAP.org/license.html>.
17
 */
18
19
20
21
22
23
24
25
26
27
28
29

#include "portable.h"

#include <stdio.h>

#include <ac/string.h>
#include <ac/socket.h>

#include "lutil.h"
#include "slap.h"
#include "lutil_ldap.h"

30
31
#include "config.h"

32
33
#include "ldap_rq.h"

34
35
36
37
38
39
struct nonpresent_entry {
	struct berval *npe_name;
	struct berval *npe_nname;
	LDAP_LIST_ENTRY(nonpresent_entry) npe_link;
};

40
41
42
43
typedef struct cookie_state {
	ldap_pvt_thread_mutex_t	cs_mutex;
	int	cs_num;
	int cs_age;
Quanah Gibson-Mount's avatar
Quanah Gibson-Mount committed
44
	int cs_ref;
45
46
47
48
	struct berval *cs_vals;
	int *cs_sids;
} cookie_state;
	
49
50
51
52
#define	SYNCDATA_DEFAULT	0	/* entries are plain LDAP entries */
#define	SYNCDATA_ACCESSLOG	1	/* entries are accesslog format */
#define	SYNCDATA_CHANGELOG	2	/* entries are changelog format */

53
54
55
#define	SYNCLOG_LOGGING		0	/* doing a log-based update */
#define	SYNCLOG_FALLBACK	1	/* doing a full refresh */

56
57
58
59
60
#define RETRYNUM_FOREVER	(-1)	/* retry forever */
#define RETRYNUM_TAIL		(-2)	/* end of retrynum array */
#define RETRYNUM_VALID(n)	((n) >= RETRYNUM_FOREVER)	/* valid retrynum */
#define RETRYNUM_FINITE(n)	((n) > RETRYNUM_FOREVER)	/* not forever */

61
typedef struct syncinfo_s {
62
	struct syncinfo_s	*si_next;
63
64
65
66
	BackendDB		*si_be;
	BackendDB		*si_wbe;
	struct re_s		*si_re;
	int			si_rid;
Quanah Gibson-Mount's avatar
Quanah Gibson-Mount committed
67
	char			si_ridtxt[ STRLENOF("rid=999") + 1 ];
Howard Chu's avatar
Howard Chu committed
68
69
	slap_bindconf		si_bindconf;
	struct berval		si_base;
70
71
72
	struct berval		si_logbase;
	struct berval		si_filterstr;
	struct berval		si_logfilterstr;
73
74
75
	int			si_scope;
	int			si_attrsonly;
	char			*si_anfile;
Howard Chu's avatar
Howard Chu committed
76
77
	AttributeName		*si_anlist;
	AttributeName		*si_exanlist;
78
79
80
81
82
83
84
85
86
87
88
	char 			**si_attrs;
	char			**si_exattrs;
	int			si_allattrs;
	int			si_allopattrs;
	int			si_schemachecking;
	int			si_type;	/* the active type */
	int			si_ctype;	/* the configured type */
	time_t			si_interval;
	time_t			*si_retryinterval;
	int			*si_retrynum_init;
	int			*si_retrynum;
Howard Chu's avatar
Howard Chu committed
89
	struct sync_cookie	si_syncCookie;
90
	cookie_state		*si_cookieState;
91
92
93
94
95
96
	int			si_cookieAge;
	int			si_manageDSAit;
	int			si_slimit;
	int			si_tlimit;
	int			si_refreshDelete;
	int			si_refreshPresent;
Quanah Gibson-Mount's avatar
Quanah Gibson-Mount committed
97
	int			si_refreshDone;
98
99
	int			si_syncdata;
	int			si_logstate;
Quanah Gibson-Mount's avatar
Quanah Gibson-Mount committed
100
	int			si_got;
Quanah Gibson-Mount's avatar
Quanah Gibson-Mount committed
101
	ber_int_t	si_msgid;
102
103
	Avlnode			*si_presentlist;
	LDAP			*si_ld;
104
	Connection		*si_conn;
105
	LDAP_LIST_HEAD(np, nonpresent_entry)	si_nonpresentlist;
Howard Chu's avatar
Howard Chu committed
106
	ldap_pvt_thread_mutex_t	si_mutex;
107
108
} syncinfo_t;

109
static int syncuuid_cmp( const void *, const void * );
Quanah Gibson-Mount's avatar
Quanah Gibson-Mount committed
110
static int avl_presentlist_insert( syncinfo_t* si, struct berval *syncUUID );
Quanah Gibson-Mount's avatar
Quanah Gibson-Mount committed
111
static void syncrepl_del_nonpresent( Operation *, syncinfo_t *, BerVarray, struct sync_cookie *, int );
112
113
static int syncrepl_message_to_op(
					syncinfo_t *, Operation *, LDAPMessage * );
114
115
116
117
118
static int syncrepl_message_to_entry(
					syncinfo_t *, Operation *, LDAPMessage *,
					Modifications **, Entry **, int );
static int syncrepl_entry(
					syncinfo_t *, Operation*, Entry*,
Pierangelo Masarati's avatar
Pierangelo Masarati committed
119
120
					Modifications**,int, struct berval*,
					struct berval *cookieCSN );
121
static int syncrepl_updateCookie(
122
123
124
125
					syncinfo_t *, Operation *, struct berval *,
					struct sync_cookie * );
static struct berval * slap_uuidstr_from_normalized(
					struct berval *, struct berval *, void * );
126

127
/* callback functions */
128
129
130
static int dn_callback( Operation *, SlapReply * );
static int nonpresent_callback( Operation *, SlapReply * );
static int null_callback( Operation *, SlapReply * );
131

132
static AttributeDescription *sync_descs[4];
133

134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
static const char *
syncrepl_state2str( int state )
{
	switch ( state ) {
	case LDAP_SYNC_PRESENT:
		return "PRESENT";

	case LDAP_SYNC_ADD:
		return "ADD";

	case LDAP_SYNC_MODIFY:
		return "MODIFY";

	case LDAP_SYNC_DELETE:
		return "DELETE";
	}

	return "UNKNOWN";
}

154
static void
155
init_syncrepl(syncinfo_t *si)
156
{
Jong Hyuk Choi's avatar
Jong Hyuk Choi committed
157
	int i, j, k, l, n;
158
	char **attrs, **exattrs;
159
160
161
162
163
164
165
166

	if ( !sync_descs[0] ) {
		sync_descs[0] = slap_schema.si_ad_objectClass;
		sync_descs[1] = slap_schema.si_ad_structuralObjectClass;
		sync_descs[2] = slap_schema.si_ad_entryCSN;
		sync_descs[3] = NULL;
	}

167
168
169
170
	if ( si->si_allattrs && si->si_allopattrs )
		attrs = NULL;
	else
		attrs = anlist2attrs( si->si_anlist );
171

172
173
174
175
	if ( attrs ) {
		if ( si->si_allattrs ) {
			i = 0;
			while ( attrs[i] ) {
176
				if ( !is_at_operational( at_find( attrs[i] ) ) ) {
177
178
179
180
					for ( j = i; attrs[j] != NULL; j++ ) {
						if ( j == i )
							ch_free( attrs[i] );
						attrs[j] = attrs[j+1];
181
					}
182
183
				} else {
					i++;
184
185
				}
			}
186
			attrs = ( char ** ) ch_realloc( attrs, (i + 2)*sizeof( char * ) );
187
188
			attrs[i] = ch_strdup("*");
			attrs[i + 1] = NULL;
189

190
191
192
		} else if ( si->si_allopattrs ) {
			i = 0;
			while ( attrs[i] ) {
193
				if ( is_at_operational( at_find( attrs[i] ) ) ) {
194
195
196
197
198
199
200
201
202
					for ( j = i; attrs[j] != NULL; j++ ) {
						if ( j == i )
							ch_free( attrs[i] );
						attrs[j] = attrs[j+1];
					}
				} else {
					i++;
				}
			}
203
			attrs = ( char ** ) ch_realloc( attrs, (i + 2)*sizeof( char * ) );
204
205
			attrs[i] = ch_strdup("+");
			attrs[i + 1] = NULL;
206
		}
Kurt Zeilenga's avatar
Kurt Zeilenga committed
207

208
209
210
		for ( i = 0; sync_descs[i] != NULL; i++ ) {
			j = 0;
			while ( attrs[j] ) {
211
				if ( !strcmp( attrs[j], sync_descs[i]->ad_cname.bv_val ) ) {
212
213
214
215
					for ( k = j; attrs[k] != NULL; k++ ) {
						if ( k == j )
							ch_free( attrs[k] );
						attrs[k] = attrs[k+1];
216
					}
217
218
				} else {
					j++;
219
220
				}
			}
221
222
223
224
225
		}

		for ( n = 0; attrs[ n ] != NULL; n++ ) /* empty */;

		if ( si->si_allopattrs ) {
226
			attrs = ( char ** ) ch_realloc( attrs, (n + 2)*sizeof( char * ) );
227
		} else {
228
			attrs = ( char ** ) ch_realloc( attrs, (n + 4)*sizeof( char * ) );
229
230
231
232
233
234
		}

		/* Add Attributes */
		if ( si->si_allopattrs ) {
			attrs[n++] = ch_strdup( sync_descs[0]->ad_cname.bv_val );
		} else {
235
236
237
			for ( i = 0; sync_descs[ i ] != NULL; i++ ) {
				attrs[ n++ ] = ch_strdup ( sync_descs[i]->ad_cname.bv_val );
			}
238
		}
239
		attrs[ n ] = NULL;
240

241
	} else {
Jong Hyuk Choi's avatar
Jong Hyuk Choi committed
242

243
		i = 0;
244
245
		if ( si->si_allattrs == si->si_allopattrs ) {
			attrs = (char**) ch_malloc( 3 * sizeof(char*) );
246
247
			attrs[i++] = ch_strdup( "*" );
			attrs[i++] = ch_strdup( "+" );
248
249
250
251
252
253
		} else if ( si->si_allattrs && !si->si_allopattrs ) {
			for ( n = 0; sync_descs[ n ] != NULL; n++ ) ;
			attrs = (char**) ch_malloc( (n+1)* sizeof(char*) );
			attrs[i++] = ch_strdup( "*" );
			for ( j = 1; sync_descs[ j ] != NULL; j++ ) {
				attrs[i++] = ch_strdup ( sync_descs[j]->ad_cname.bv_val );
Jong Hyuk Choi's avatar
Jong Hyuk Choi committed
254
			}
255
256
257
258
		} else if ( !si->si_allattrs && si->si_allopattrs ) {
			attrs = (char**) ch_malloc( 3 * sizeof(char*) );
			attrs[i++] = ch_strdup( "+" );
			attrs[i++] = ch_strdup( sync_descs[0]->ad_cname.bv_val );
259
260
		}
		attrs[i] = NULL;
261
262
	}
	
263
264
265
266
267
	si->si_attrs = attrs;

	exattrs = anlist2attrs( si->si_exanlist );

	if ( exattrs ) {
268
269
		for ( n = 0; exattrs[n] != NULL; n++ ) ;

Jong Hyuk Choi's avatar
Jong Hyuk Choi committed
270
		for ( i = 0; sync_descs[i] != NULL; i++ ) {
271
272
			j = 0;
			while ( exattrs[j] != NULL ) {
273
				if ( !strcmp( exattrs[j], sync_descs[i]->ad_cname.bv_val ) ) {
Howard Chu's avatar
Cleanup    
Howard Chu committed
274
					ch_free( exattrs[j] );
275
276
					for ( k = j; exattrs[k] != NULL; k++ ) {
						exattrs[k] = exattrs[k+1];
Jong Hyuk Choi's avatar
Jong Hyuk Choi committed
277
					}
278
279
				} else {
					j++;
Jong Hyuk Choi's avatar
Jong Hyuk Choi committed
280
281
282
				}
			}
		}
283
284
285

		for ( i = 0; exattrs[i] != NULL; i++ ) {
			for ( j = 0; si->si_anlist[j].an_name.bv_val; j++ ) {
286
287
				ObjectClass	*oc;
				if ( ( oc = si->si_anlist[j].an_oc ) ) {
288
289
290
					k = 0;
					while ( oc->soc_required[k] ) {
						if ( !strcmp( exattrs[i],
291
							 oc->soc_required[k]->sat_cname.bv_val ) ) {
Howard Chu's avatar
Cleanup    
Howard Chu committed
292
							ch_free( exattrs[i] );
293
294
295
296
297
							for ( l = i; exattrs[l]; l++ ) {
								exattrs[l] = exattrs[l+1];
							}
						} else {
							k++;
Jong Hyuk Choi's avatar
Jong Hyuk Choi committed
298
299
300
301
302
						}
					}
				}
			}
		}
303
304

		for ( i = 0; exattrs[i] != NULL; i++ ) ;
305
306

		if ( i != n )
307
			exattrs = (char **) ch_realloc( exattrs, (i + 1)*sizeof(char *) );
Jong Hyuk Choi's avatar
Jong Hyuk Choi committed
308
	}
309
310

	si->si_exattrs = exattrs;	
311
312
}

Howard Chu's avatar
Howard Chu committed
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
typedef struct logschema {
	struct berval ls_dn;
	struct berval ls_req;
	struct berval ls_mod;
	struct berval ls_newRdn;
	struct berval ls_delRdn;
	struct berval ls_newSup;
} logschema;

static logschema changelog_sc = {
	BER_BVC("targetDN"),
	BER_BVC("changeType"),
	BER_BVC("changes"),
	BER_BVC("newRDN"),
	BER_BVC("deleteOldRDN"),
	BER_BVC("newSuperior")
};

static logschema accesslog_sc = {
	BER_BVC("reqDN"),
	BER_BVC("reqType"),
	BER_BVC("reqMod"),
	BER_BVC("reqNewRDN"),
	BER_BVC("reqDeleteOldRDN"),
	BER_BVC("reqNewSuperior")
};
339

340
static int
Jong Hyuk Choi's avatar
Jong Hyuk Choi committed
341
342
ldap_sync_search(
	syncinfo_t *si,
Kurt Zeilenga's avatar
Kurt Zeilenga committed
343
	void *ctx )
Jong Hyuk Choi's avatar
Jong Hyuk Choi committed
344
{
345
346
	BerElementBuffer berbuf;
	BerElement *ber = (BerElement *)&berbuf;
Quanah Gibson-Mount's avatar
Quanah Gibson-Mount committed
347
	LDAPControl c[3], *ctrls[4];
348
	int rc;
349
350
	int rhint;
	char *base;
Howard Chu's avatar
Howard Chu committed
351
	char **attrs, *lattrs[8];
352
353
354
	char *filter;
	int attrsonly;
	int scope;
Jong Hyuk Choi's avatar
Jong Hyuk Choi committed
355

356
357
358
	/* setup LDAP SYNC control */
	ber_init2( ber, NULL, LBER_USE_DER );
	ber_set_option( ber, LBER_OPT_BER_MEMCTX, &ctx );
Jong Hyuk Choi's avatar
Jong Hyuk Choi committed
359

360
361
362
	/* If we're using a log but we have no state, then fallback to
	 * normal mode for a full refresh.
	 */
363
	if ( si->si_syncdata && !si->si_syncCookie.numcsns ) {
364
		si->si_logstate = SYNCLOG_FALLBACK;
365
	}
366
367
368

	/* Use the log parameters if we're in log mode */
	if ( si->si_syncdata && si->si_logstate == SYNCLOG_LOGGING ) {
Howard Chu's avatar
Howard Chu committed
369
370
371
372
373
374
375
376
377
378
379
380
381
382
		logschema *ls;
		if ( si->si_syncdata == SYNCDATA_ACCESSLOG )
			ls = &accesslog_sc;
		else
			ls = &changelog_sc;
		lattrs[0] = ls->ls_dn.bv_val;
		lattrs[1] = ls->ls_req.bv_val;
		lattrs[2] = ls->ls_mod.bv_val;
		lattrs[3] = ls->ls_newRdn.bv_val;
		lattrs[4] = ls->ls_delRdn.bv_val;
		lattrs[5] = ls->ls_newSup.bv_val;
		lattrs[6] = slap_schema.si_ad_entryCSN->ad_cname.bv_val;
		lattrs[7] = NULL;

383
384
385
		rhint = 0;
		base = si->si_logbase.bv_val;
		filter = si->si_logfilterstr.bv_val;
Howard Chu's avatar
Howard Chu committed
386
		attrs = lattrs;
387
		attrsonly = 0;
388
		scope = LDAP_SCOPE_SUBTREE;
389
390
391
392
393
394
395
396
397
	} else {
		rhint = 1;
		base = si->si_base.bv_val;
		filter = si->si_filterstr.bv_val;
		attrs = si->si_attrs;
		attrsonly = si->si_attrsonly;
		scope = si->si_scope;
	}
	if ( si->si_syncdata && si->si_logstate == SYNCLOG_FALLBACK ) {
Howard Chu's avatar
Howard Chu committed
398
		si->si_type = LDAP_SYNC_REFRESH_ONLY;
399
	} else {
Howard Chu's avatar
Howard Chu committed
400
		si->si_type = si->si_ctype;
401
402
	}

403
	if ( !BER_BVISNULL( &si->si_syncCookie.octet_str ) )
Kurt Zeilenga's avatar
Kurt Zeilenga committed
404
	{
Howard Chu's avatar
Howard Chu committed
405
		ber_printf( ber, "{eOb}",
Howard Chu's avatar
Howard Chu committed
406
			abs(si->si_type), &si->si_syncCookie.octet_str, rhint );
407
	} else {
Howard Chu's avatar
Howard Chu committed
408
		ber_printf( ber, "{eb}",
Howard Chu's avatar
Howard Chu committed
409
			abs(si->si_type), rhint );
410
	}
Jong Hyuk Choi's avatar
Jong Hyuk Choi committed
411

Quanah Gibson-Mount's avatar
Quanah Gibson-Mount committed
412
	if ( (rc = ber_flatten2( ber, &c[0].ldctl_value, 0 ) ) == -1 ) {
413
414
415
		ber_free_buf( ber );
		return rc;
	}
Jong Hyuk Choi's avatar
Jong Hyuk Choi committed
416

417
	c[0].ldctl_oid = LDAP_CONTROL_SYNC;
418
	c[0].ldctl_iscritical = si->si_type < 0;
419
	ctrls[0] = &c[0];
Jong Hyuk Choi's avatar
Jong Hyuk Choi committed
420

Quanah Gibson-Mount's avatar
Quanah Gibson-Mount committed
421
422
423
424
425
	c[1].ldctl_oid = LDAP_CONTROL_MANAGEDSAIT;
	BER_BVZERO( &c[1].ldctl_value );
	c[1].ldctl_iscritical = 1;
	ctrls[1] = &c[1];

426
	if ( !BER_BVISNULL( &si->si_bindconf.sb_authzId ) ) {
Quanah Gibson-Mount's avatar
Quanah Gibson-Mount committed
427
428
429
430
431
		c[2].ldctl_oid = LDAP_CONTROL_PROXY_AUTHZ;
		c[2].ldctl_value = si->si_bindconf.sb_authzId;
		c[2].ldctl_iscritical = 1;
		ctrls[2] = &c[2];
		ctrls[3] = NULL;
432
	} else {
Quanah Gibson-Mount's avatar
Quanah Gibson-Mount committed
433
		ctrls[2] = NULL;
434
	}
Jong Hyuk Choi's avatar
Jong Hyuk Choi committed
435

436
	rc = ldap_search_ext( si->si_ld, base, scope, filter, attrs, attrsonly,
Quanah Gibson-Mount's avatar
Quanah Gibson-Mount committed
437
		ctrls, NULL, NULL, si->si_slimit, &si->si_msgid );
438
	ber_free_buf( ber );
Jong Hyuk Choi's avatar
Jong Hyuk Choi committed
439
440
441
	return rc;
}

Quanah Gibson-Mount's avatar
Quanah Gibson-Mount committed
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
static int
check_syncprov(
	Operation *op,
	syncinfo_t *si )
{
	AttributeName at[2];
	Attribute a = {0};
	Entry e = {0};
	SlapReply rs = {0};
	int i, j, changed = 0;

	/* Look for contextCSN from syncprov overlay. If
	 * there's no overlay, this will be a no-op. That means
	 * this is a pure consumer, so local changes will not be
	 * allowed, and all changes will already be reflected in
	 * the cookieState.
	 */
	a.a_desc = slap_schema.si_ad_contextCSN;
	e.e_attrs = &a;
	e.e_name = op->o_bd->be_suffix[0];
	e.e_nname = op->o_bd->be_nsuffix[0];
	at[0].an_name = a.a_desc->ad_cname;
	at[0].an_desc = a.a_desc;
	BER_BVZERO( &at[1].an_name );
	rs.sr_entry = &e;
	rs.sr_flags = REP_ENTRY_MODIFIABLE;
	rs.sr_attrs = at;
	op->o_req_dn = e.e_name;
	op->o_req_ndn = e.e_nname;

	ldap_pvt_thread_mutex_lock( &si->si_cookieState->cs_mutex );
	i = backend_operational( op, &rs );
	if ( i == LDAP_SUCCESS && a.a_nvals ) {
		int num = a.a_numvals;
		/* check for differences */
		if ( num != si->si_cookieState->cs_num ) {
			changed = 1;
		} else {
			for ( i=0; i<num; i++ ) {
				if ( ber_bvcmp( &a.a_nvals[i],
					&si->si_cookieState->cs_vals[i] )) {
Quanah Gibson-Mount's avatar
Quanah Gibson-Mount committed
483
					changed = 1;
Quanah Gibson-Mount's avatar
Quanah Gibson-Mount committed
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
					break;
				}
			}
		}
		if ( changed ) {
			ber_bvarray_free( si->si_cookieState->cs_vals );
			ch_free( si->si_cookieState->cs_sids );
			si->si_cookieState->cs_num = num;
			si->si_cookieState->cs_vals = a.a_nvals;
			si->si_cookieState->cs_sids = slap_parse_csn_sids( a.a_nvals,
				num, NULL );
			si->si_cookieState->cs_age++;
		} else {
			ber_bvarray_free( a.a_nvals );
		}
		ber_bvarray_free( a.a_vals );
	}
	/* See if the cookieState has changed due to anything outside
	 * this particular consumer. That includes other consumers in
	 * the same context, or local changes detected above.
	 */
	if ( si->si_cookieState->cs_num > 0 && si->si_cookieAge !=
		si->si_cookieState->cs_age ) {
		if ( !si->si_syncCookie.numcsns ) {
			ber_bvarray_free( si->si_syncCookie.ctxcsn );
			ber_bvarray_dup_x( &si->si_syncCookie.ctxcsn,
				si->si_cookieState->cs_vals, NULL );
			changed = 1;
		} else {
			for (i=0; !BER_BVISNULL( &si->si_syncCookie.ctxcsn[i] ); i++) {
				/* bogus, just dup everything */
				if ( si->si_syncCookie.sids[i] == -1 ) {
					ber_bvarray_free( si->si_syncCookie.ctxcsn );
					ber_bvarray_dup_x( &si->si_syncCookie.ctxcsn,
						si->si_cookieState->cs_vals, NULL );
					changed = 1;
					break;
				}
				for (j=0; j<si->si_cookieState->cs_num; j++) {
					if ( si->si_syncCookie.sids[i] !=
						si->si_cookieState->cs_sids[j] )
						continue;
					if ( bvmatch( &si->si_syncCookie.ctxcsn[i],
						&si->si_cookieState->cs_vals[j] ))
						break;
					ber_bvreplace( &si->si_syncCookie.ctxcsn[i],
						&si->si_cookieState->cs_vals[j] );
					changed = 1;
					break;
				}
			}
		}
	}
	if ( changed ) {
		si->si_cookieAge = si->si_cookieState->cs_age;
		ch_free( si->si_syncCookie.octet_str.bv_val );
		slap_compose_sync_cookie( NULL, &si->si_syncCookie.octet_str,
			si->si_syncCookie.ctxcsn, si->si_syncCookie.rid,
			si->si_syncCookie.sid );
	}
	ldap_pvt_thread_mutex_unlock( &si->si_cookieState->cs_mutex );
	return changed;
}

548
549
550
551
static int
do_syncrep1(
	Operation *op,
	syncinfo_t *si )
552
553
{
	int	rc;
554
	int cmdline_cookie_found = 0;
555

556
	struct sync_cookie	*sc = NULL;
557
558
559
#ifdef HAVE_TLS
	void	*ssl;
#endif
560

561
	rc = slap_client_connect( &si->si_ld, &si->si_bindconf );
Jong Hyuk Choi's avatar
Jong Hyuk Choi committed
562
	if ( rc != LDAP_SUCCESS ) {
563
		goto done;
564
	}
565
	op->o_protocol = LDAP_VERSION3;
566

567
568
569
	/* Set SSF to strongest of TLS, SASL SSFs */
	op->o_sasl_ssf = 0;
	op->o_tls_ssf = 0;
570
	op->o_transport_ssf = 0;
571
#ifdef HAVE_TLS
572
573
574
	if ( ldap_get_option( si->si_ld, LDAP_OPT_X_TLS_SSL_CTX, &ssl )
		== LDAP_SUCCESS && ssl != NULL )
	{
575
576
577
		op->o_tls_ssf = ldap_pvt_tls_get_strength( ssl );
	}
#endif /* HAVE_TLS */
Quanah Gibson-Mount's avatar
Quanah Gibson-Mount committed
578
579
580
581
582
583
	{
		ber_len_t ssf; /* ITS#5403, 3864 LDAP_OPT_X_SASL_SSF probably ought
						  to use sasl_ssf_t but currently uses ber_len_t */
		ldap_get_option( si->si_ld, LDAP_OPT_X_SASL_SSF, &ssf );
		op->o_sasl_ssf = ssf;
	}
584
585
	op->o_ssf = ( op->o_sasl_ssf > op->o_tls_ssf )
		?  op->o_sasl_ssf : op->o_tls_ssf;
586

587
588
	ldap_set_option( si->si_ld, LDAP_OPT_TIMELIMIT, &si->si_tlimit );

Quanah Gibson-Mount's avatar
Quanah Gibson-Mount committed
589
590
591
	rc = LDAP_DEREF_NEVER;	/* actually could allow DEREF_FINDING */
	ldap_set_option( si->si_ld, LDAP_OPT_DEREF, &rc );

Quanah Gibson-Mount's avatar
Quanah Gibson-Mount committed
592
593
	ldap_set_option( si->si_ld, LDAP_OPT_REFERRALS, LDAP_OPT_OFF );

Howard Chu's avatar
Howard Chu committed
594
	si->si_syncCookie.rid = si->si_rid;
Quanah Gibson-Mount's avatar
Quanah Gibson-Mount committed
595
596
597
598

	/* whenever there are multiple data sources possible, advertise sid */
	si->si_syncCookie.sid = ( SLAP_MULTIMASTER( si->si_be ) || si->si_be != si->si_wbe ) ?
		slap_serverID : -1;
Howard Chu's avatar
Howard Chu committed
599

600
601
602
	/* We've just started up, or the remote server hasn't sent us
	 * any meaningful state.
	 */
603
	if ( BER_BVISNULL( &si->si_syncCookie.octet_str ) ) {
604
		int i;
605

606
607
608
609
610
611
612
613
614
		LDAP_STAILQ_FOREACH( sc, &slap_sync_cookie, sc_next ) {
			if ( si->si_rid == sc->rid ) {
				cmdline_cookie_found = 1;
				break;
			}
		}

		if ( cmdline_cookie_found ) {
			/* cookie is supplied in the command line */
615

616
617
			LDAP_STAILQ_REMOVE( &slap_sync_cookie, sc, sync_cookie, sc_next );

618
619
			/* ctxcsn wasn't parsed yet, do it now */
			slap_parse_sync_cookie( sc, op->o_tmpmemctx );
620
			slap_sync_cookie_free( &si->si_syncCookie, 0 );
621
622
			slap_dup_sync_cookie( &si->si_syncCookie, sc );
			slap_sync_cookie_free( sc, 1 );
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
		} else {
			ldap_pvt_thread_mutex_lock( &si->si_cookieState->cs_mutex );
			if ( !si->si_cookieState->cs_num ) {
				/* get contextCSN shadow replica from database */
				BerVarray csn = NULL;
				void *ctx = op->o_tmpmemctx;

				op->o_req_ndn = op->o_bd->be_nsuffix[0];
				op->o_req_dn = op->o_req_ndn;

				/* try to read stored contextCSN */
				op->o_tmpmemctx = NULL;
				backend_attribute( op, NULL, &op->o_req_ndn,
					slap_schema.si_ad_contextCSN, &csn, ACL_READ );
				op->o_tmpmemctx = ctx;
				if ( csn ) {
					si->si_cookieState->cs_vals = csn;
					for (i=0; !BER_BVISNULL( &csn[i] ); i++);
					si->si_cookieState->cs_num = i;
Howard Chu's avatar
Howard Chu committed
642
					si->si_cookieState->cs_sids = slap_parse_csn_sids( csn, i, NULL );
643
644
645
646
647
648
649
				}
			}
			if ( si->si_cookieState->cs_num ) {
				ber_bvarray_free( si->si_syncCookie.ctxcsn );
				if ( ber_bvarray_dup_x( &si->si_syncCookie.ctxcsn,
					si->si_cookieState->cs_vals, NULL )) {
					rc = LDAP_NO_MEMORY;
Quanah Gibson-Mount's avatar
Quanah Gibson-Mount committed
650
					ldap_pvt_thread_mutex_unlock( &si->si_cookieState->cs_mutex );
651
652
653
654
655
656
657
658
659
					goto done;
				}
				si->si_syncCookie.numcsns = si->si_cookieState->cs_num;
				si->si_syncCookie.sids = ch_malloc( si->si_cookieState->cs_num *
					sizeof(int) );
				for ( i=0; i<si->si_syncCookie.numcsns; i++ )
					si->si_syncCookie.sids[i] = si->si_cookieState->cs_sids[i];
			}
			ldap_pvt_thread_mutex_unlock( &si->si_cookieState->cs_mutex );
660
		}
Kurt Zeilenga's avatar
Kurt Zeilenga committed
661

Howard Chu's avatar
Howard Chu committed
662
		slap_compose_sync_cookie( NULL, &si->si_syncCookie.octet_str,
663
			si->si_syncCookie.ctxcsn, si->si_syncCookie.rid,
664
			si->si_syncCookie.sid );
665
	} else {
Quanah Gibson-Mount's avatar
Quanah Gibson-Mount committed
666
667
		/* Look for contextCSN from syncprov overlay. */
		check_syncprov( op, si );
Jong Hyuk Choi's avatar
Jong Hyuk Choi committed
668
669
	}

Quanah Gibson-Mount's avatar
Quanah Gibson-Mount committed
670
671
	si->si_refreshDone = 0;

Jong Hyuk Choi's avatar
Jong Hyuk Choi committed
672
	rc = ldap_sync_search( si, op->o_tmpmemctx );
673

674
	if( rc != LDAP_SUCCESS ) {
675
		Debug( LDAP_DEBUG_ANY, "do_syncrep1: %s "
676
			"ldap_search_ext: %s (%d)\n",
677
			si->si_ridtxt, ldap_err2string( rc ), rc );
678
679
680
681
682
	}

done:
	if ( rc ) {
		if ( si->si_ld ) {
683
			ldap_unbind_ext( si->si_ld, NULL, NULL );
684
685
686
687
688
689
690
			si->si_ld = NULL;
		}
	}

	return rc;
}

691
692
693
694
695
696
697
698
static int
compare_csns( struct sync_cookie *sc1, struct sync_cookie *sc2, int *which )
{
	int i, j, match = 0;
	const char *text;

	*which = 0;

699
700
701
702
703
	if ( sc1->numcsns < sc2->numcsns ) {
		*which = sc1->numcsns;
		return -1;
	}

Quanah Gibson-Mount's avatar
Quanah Gibson-Mount committed
704
705
	for (j=0; j<sc2->numcsns; j++) {
		for (i=0; i<sc1->numcsns; i++) {
706
707
708
709
710
			if ( sc1->sids[i] != sc2->sids[j] )
				continue;
			value_match( &match, slap_schema.si_ad_entryCSN,
				slap_schema.si_ad_entryCSN->ad_type->sat_ordering,
				SLAP_MR_VALUE_OF_ATTRIBUTE_SYNTAX,
Howard Chu's avatar
Howard Chu committed
711
				&sc1->ctxcsn[i], &sc2->ctxcsn[j], &text );
712
713
714
715
716
717
			if ( match < 0 ) {
				*which = j;
				return match;
			}
			break;
		}
Quanah Gibson-Mount's avatar
Quanah Gibson-Mount committed
718
719
720
721
722
		if ( i == sc1->numcsns ) {
			/* sc2 has a sid sc1 lacks */
			*which = j;
			return -1;
		}
723
724
725
726
	}
	return match;
}

Howard Chu's avatar
Howard Chu committed
727
728
#define	SYNC_PAUSED	-3

729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
static int
do_syncrep2(
	Operation *op,
	syncinfo_t *si )
{
	LDAPControl	**rctrls = NULL;

	BerElementBuffer berbuf;
	BerElement	*ber = (BerElement *)&berbuf;

	LDAPMessage	*msg = NULL;

	char		*retoid = NULL;
	struct berval	*retdata = NULL;

	Entry		*entry = NULL;

	int		syncstate;
Kurt Zeilenga's avatar
Kurt Zeilenga committed
747
	struct berval	syncUUID = BER_BVNULL;
748
749
	struct sync_cookie	syncCookie = { NULL };
	struct sync_cookie	syncCookie_req = { NULL };
Kurt Zeilenga's avatar
Kurt Zeilenga committed
750
	struct berval		cookie = BER_BVNULL;
751

752
753
	int		rc,
			err = LDAP_SUCCESS;
754
755
756
757
758
	ber_len_t	len;

	struct berval	*psub;
	Modifications	*modlist = NULL;

759
	int				match, m;
760
761
762
763

	struct timeval *tout_p = NULL;
	struct timeval tout = { 0, 0 };

764
	int		refreshDeletes = 0;
765
	BerVarray syncUUIDs = NULL;
766
767
	ber_tag_t si_tag;

768
	if ( slapd_shutdown ) {
769
		rc = -2;
770
		goto done;
771
772
	}

773
774
775
	ber_init2( ber, NULL, LBER_USE_DER );
	ber_set_option( ber, LBER_OPT_BER_MEMCTX, &op->o_tmpmemctx );

776
	Debug( LDAP_DEBUG_TRACE, "=>do_syncrep2 %s\n", si->si_ridtxt, 0, 0 );
777
778
779

	psub = &si->si_be->be_nsuffix[0];

Jong Hyuk Choi's avatar
Jong Hyuk Choi committed
780
	slap_dup_sync_cookie( &syncCookie_req, &si->si_syncCookie );
781

Kurt Zeilenga's avatar
Kurt Zeilenga committed
782
	if ( abs(si->si_type) == LDAP_SYNC_REFRESH_AND_PERSIST ) {
783
784
785
786
787
		tout_p = &tout;
	} else {
		tout_p = NULL;
	}

Quanah Gibson-Mount's avatar
Quanah Gibson-Mount committed
788
	while ( ( rc = ldap_result( si->si_ld, si->si_msgid, LDAP_MSG_ONE,
Howard Chu's avatar
Howard Chu committed
789
		tout_p, &msg ) ) > 0 )
790
	{
Quanah Gibson-Mount's avatar
Quanah Gibson-Mount committed
791
792
		LDAPControl	*rctrlp = NULL;

793
		if ( slapd_shutdown ) {
794
795
			rc = -2;
			goto done;
796
		}
Howard Chu's avatar
Howard Chu committed
797
798
799
800
801
802
803
804
805
		switch( ldap_msgtype( msg ) ) {
		case LDAP_RES_SEARCH_ENTRY:
			ldap_get_entry_controls( si->si_ld, msg, &rctrls );
			/* we can't work without the control */
			if ( rctrls ) {
				LDAPControl **next;
				/* NOTE: make sure we use the right one;
				 * a better approach would be to run thru
				 * the whole list and take care of all */
Quanah Gibson-Mount's avatar
Quanah Gibson-Mount committed
806
807
808
809
				/* NOTE: since we issue the search request,
				 * we should know what controls to expect,
				 * and there should be none apart from the
				 * sync-related control */
Howard Chu's avatar
Howard Chu committed
810
811
812
				rctrlp = ldap_control_find( LDAP_CONTROL_SYNC_STATE, rctrls, &next );
				if ( next && ldap_control_find( LDAP_CONTROL_SYNC_STATE, next, NULL ) )
				{
813
					Debug( LDAP_DEBUG_ANY, "do_syncrep2: %s "
Howard Chu's avatar
Howard Chu committed
814
						"got search entry with multiple "
815
						"Sync State control\n", si->si_ridtxt, 0, 0 );
Quanah Gibson-Mount's avatar
Quanah Gibson-Mount committed
816
					ldap_controls_free( rctrls );
Howard Chu's avatar
Howard Chu committed
817
818
819
					rc = -1;
					goto done;
				}
Howard Chu's avatar
Howard Chu committed
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
			}
			if ( rctrlp == NULL ) {
				Debug( LDAP_DEBUG_ANY, "do_syncrep2: %s "
					"got search entry without "
					"Sync State control\n", si->si_ridtxt, 0, 0 );
				rc = -1;
				goto done;
			}
			ber_init2( ber, &rctrlp->ldctl_value, LBER_USE_DER );
			ber_scanf( ber, "{em" /*"}"*/, &syncstate, &syncUUID );
			/* FIXME: what if syncUUID is NULL or empty?
			 * (happens with back-sql...) */
			if ( BER_BVISEMPTY( &syncUUID ) ) {
				Debug( LDAP_DEBUG_ANY, "do_syncrep2: %s "
					"got empty syncUUID with LDAP_SYNC_%s\n",
					si->si_ridtxt,
					syncrepl_state2str( syncstate ), 0 );
				ldap_controls_free( rctrls );
				rc = -1;
				goto done;
			}
			if ( ber_peek_tag( ber, &len ) == LDAP_TAG_SYNC_COOKIE ) {
				ber_scanf( ber, /*"{"*/ "m}", &cookie );

				Debug( LDAP_DEBUG_SYNC, "do_syncrep2: cookie=%s\n",
					BER_BVISNULL( &cookie ) ? "" : cookie.bv_val, 0, 0 );

				if ( !BER_BVISNULL( &cookie ) ) {
					ch_free( syncCookie.octet_str.bv_val );
					ber_dupbv( &syncCookie.octet_str, &cookie );
				}
				if ( !BER_BVISNULL( &syncCookie.octet_str ) )
				{
					slap_parse_sync_cookie( &syncCookie, NULL );
					if ( syncCookie.ctxcsn ) {
						int i, sid = slap_parse_csn_sid( syncCookie.ctxcsn );
						for ( i =0; i<si->si_cookieState->cs_num; i++ ) {
							if ( si->si_cookieState->cs_sids[i] == sid && 
								ber_bvcmp( syncCookie.ctxcsn, &si->si_cookieState->cs_vals[i] ) <= 0 ) {
								Debug( LDAP_DEBUG_SYNC, "do_syncrep2: %s CSN too old, ignoring %s\n",
									si->si_ridtxt, syncCookie.ctxcsn->bv_val, 0 );
								ldap_controls_free( rctrls );
								rc = 0;
								goto done;
							}
						}
					}
Quanah Gibson-Mount's avatar
Quanah Gibson-Mount committed
867
					op->o_controls[slap_cids.sc_LDAPsync] = &syncCookie;
Howard Chu's avatar
Howard Chu committed
868
869
870
871
872
873
874
875
876
				}
			}
			rc = 0;
			if ( si->si_syncdata && si->si_logstate == SYNCLOG_LOGGING ) {
				modlist = NULL;
				if ( ( rc = syncrepl_message_to_op( si, op, msg ) ) == LDAP_SUCCESS &&
					syncCookie.ctxcsn )
				{
					rc = syncrepl_updateCookie( si, op, psub, &syncCookie );
877
878
879
880
881
882
883
884
885
886
887
				} else switch ( rc ) {
					case LDAP_ALREADY_EXISTS:
					case LDAP_NO_SUCH_OBJECT:
					case LDAP_NO_SUCH_ATTRIBUTE:
					case LDAP_TYPE_OR_VALUE_EXISTS:
						rc = LDAP_SYNC_REFRESH_REQUIRED;
						si->si_logstate = SYNCLOG_FALLBACK;
						ldap_abandon_ext( si->si_ld, si->si_msgid, NULL, NULL );
						break;
					default:
						break;
Howard Chu's avatar
Howard Chu committed
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
				}
			} else if ( ( rc = syncrepl_message_to_entry( si, op, msg,
				&modlist, &entry, syncstate ) ) == LDAP_SUCCESS )
			{
				if ( ( rc = syncrepl_entry( si, op, entry, &modlist,
					syncstate, &syncUUID, syncCookie.ctxcsn ) ) == LDAP_SUCCESS &&
					syncCookie.ctxcsn )
				{
					rc = syncrepl_updateCookie( si, op, psub, &syncCookie );
				}
			}
			ldap_controls_free( rctrls );
			if ( modlist ) {
				slap_mods_free( modlist, 1 );
			}
			if ( rc )
				goto done;
			break;

		case LDAP_RES_SEARCH_REFERENCE:
			Debug( LDAP_DEBUG_ANY,
				"do_syncrep2: %s reference received error\n",
				si->si_ridtxt, 0, 0 );
			break;

		case LDAP_RES_SEARCH_RESULT:
			Debug( LDAP_DEBUG_SYNC,
				"do_syncrep2: %s LDAP_RES_SEARCH_RESULT\n",
				si->si_ridtxt, 0, 0 );
			ldap_parse_result( si->si_ld, msg, &err, NULL, NULL, NULL,
				&rctrls, 0 );
#ifdef LDAP_X_SYNC_REFRESH_REQUIRED
			if ( err == LDAP_X_SYNC_REFRESH_REQUIRED ) {
				/* map old result code to registered code */
				err = LDAP_SYNC_REFRESH_REQUIRED;
			}
#endif
			if ( err == LDAP_SYNC_REFRESH_REQUIRED ) {
				if ( si->si_logstate == SYNCLOG_LOGGING ) {
					si->si_logstate = SYNCLOG_FALLBACK;
928
				}
Howard Chu's avatar
Howard Chu committed
929
930
931
				rc = err;
				goto done;
			}
Quanah Gibson-Mount's avatar
Quanah Gibson-Mount committed
932
933
934
935
936
			if ( err ) {
				Debug( LDAP_DEBUG_ANY,
					"do_syncrep2: %s LDAP_RES_SEARCH_RESULT (%d) %s\n",
					si->si_ridtxt, err, ldap_err2string( err ) );
			}
Howard Chu's avatar
Howard Chu committed
937
			if ( rctrls ) {
Quanah Gibson-Mount's avatar
Quanah Gibson-Mount committed
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
				LDAPControl **next;
				/* NOTE: make sure we use the right one;
				 * a better approach would be to run thru
				 * the whole list and take care of all */
				/* NOTE: since we issue the search request,
				 * we should know what controls to expect,
				 * and there should be none apart from the
				 * sync-related control */
				rctrlp = ldap_control_find( LDAP_CONTROL_SYNC_DONE, rctrls, &next );
				if ( next && ldap_control_find( LDAP_CONTROL_SYNC_DONE, next, NULL ) )
				{
					Debug( LDAP_DEBUG_ANY, "do_syncrep2: %s "
						"got search result with multiple "
						"Sync State control\n", si->si_ridtxt, 0, 0 );
					ldap_controls_free( rctrls );
					rc = -1;
					goto done;
				}
			}
			if ( rctrlp ) {
Howard Chu's avatar
Howard Chu committed
958
959
960
				ber_init2( ber, &rctrlp->ldctl_value, LBER_USE_DER );

				ber_scanf( ber, "{" /*"}"*/);
961
				if ( ber_peek_tag( ber, &len ) == LDAP_TAG_SYNC_COOKIE ) {
Howard Chu's avatar
Howard Chu committed
962
					ber_scanf( ber, "m", &cookie );
Quanah Gibson-Mount's avatar
Quanah Gibson-Mount committed
963
964
965
966

					Debug( LDAP_DEBUG_SYNC, "do_syncrep2: cookie=%s\n",
						BER_BVISNULL( &cookie ) ? "" : cookie.bv_val, 0, 0 );

967
					if ( !BER_BVISNULL( &cookie ) ) {
968
						ch_free( syncCookie.octet_str.bv_val );
Howard Chu's avatar
Howard Chu committed
969
						ber_dupbv( &syncCookie.octet_str, &cookie);
Jong Hyuk Choi's avatar
Jong Hyuk Choi committed
970
					}
971
					if ( !BER_BVISNULL( &syncCookie.octet_str ) )
972
					{
Howard Chu's avatar
Howard Chu committed
973
						slap_parse_sync_cookie( &syncCookie, NULL );
Quanah Gibson-Mount's avatar
Quanah Gibson-Mount committed
974
						op->o_controls[slap_cids.sc_LDAPsync] = &syncCookie;
975
					}
976
				}
Howard Chu's avatar
Howard Chu committed
977
				if ( ber_peek_tag( ber, &len ) == LDAP_TAG_REFRESHDELETES )
978
				{
Howard Chu's avatar
Howard Chu committed
979
					ber_scanf( ber, "b", &refreshDeletes );
980
				}
Howard Chu's avatar
Howard Chu committed
981
982
				ber_scanf( ber, /*"{"*/ "}" );
			}
Quanah Gibson-Mount's avatar
Quanah Gibson-Mount committed
983
984
985
986
			if ( SLAP_MULTIMASTER( op->o_bd ) && check_syncprov( op, si )) {
				slap_sync_cookie_free( &syncCookie_req, 0 );
				slap_dup_sync_cookie( &syncCookie_req, &si->si_syncCookie );
			}
Howard Chu's avatar
Howard Chu committed
987
988
989
990
991
992
993
994
995
			if ( !syncCookie.ctxcsn ) {
				match = 1;
			} else if ( !syncCookie_req.ctxcsn ) {
				match = -1;
				m = 0;
			} else {
				match = compare_csns( &syncCookie_req, &syncCookie, &m );
			}
			if ( rctrls ) {
Howard Chu's avatar
Howard Chu committed
996
				ldap_controls_free( rctrls );
Howard Chu's avatar
Howard Chu committed
997
998
999
1000
			}
			if (si->si_type != LDAP_SYNC_REFRESH_AND_PERSIST) {
				/* FIXME : different error behaviors according to
				 *	1) err code : LDAP_BUSY ...