syncrepl.c 114 KB
Newer Older
Kurt Zeilenga's avatar
Kurt Zeilenga committed
1
/* syncrepl.c -- Replication Engine which uses the LDAP Sync protocol */
2
/* $OpenLDAP$ */
Kurt Zeilenga's avatar
Kurt Zeilenga committed
3
/* This work is part of OpenLDAP Software <http://www.openldap.org/>.
4
 *
Kurt Zeilenga's avatar
Kurt Zeilenga committed
5
 * Copyright 2003-2008 The OpenLDAP Foundation.
Kurt Zeilenga's avatar
Kurt Zeilenga committed
6
7
8
 * Portions Copyright 2003 by IBM Corporation.
 * Portions Copyright 2003 by Howard Chu, Symas Corporation.
 * All rights reserved.
9
 *
Kurt Zeilenga's avatar
Kurt Zeilenga committed
10
11
12
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted only as authorized by the OpenLDAP
 * Public License.
13
 *
Kurt Zeilenga's avatar
Kurt Zeilenga committed
14
15
16
 * A copy of this license is available in the file LICENSE in the
 * top-level directory of the distribution or, alternatively, at
 * <http://www.OpenLDAP.org/license.html>.
17
 */
18
19
20
21
22
23
24
25
26
27
28
29

#include "portable.h"

#include <stdio.h>

#include <ac/string.h>
#include <ac/socket.h>

#include "lutil.h"
#include "slap.h"
#include "lutil_ldap.h"

30
31
#include "config.h"

32
33
#include "ldap_rq.h"

34
35
36
37
38
39
struct nonpresent_entry {
	struct berval *npe_name;
	struct berval *npe_nname;
	LDAP_LIST_ENTRY(nonpresent_entry) npe_link;
};

40
41
42
43
44
45
46
47
typedef struct cookie_state {
	ldap_pvt_thread_mutex_t	cs_mutex;
	int	cs_num;
	int cs_age;
	struct berval *cs_vals;
	int *cs_sids;
} cookie_state;
	
48
49
50
51
#define	SYNCDATA_DEFAULT	0	/* entries are plain LDAP entries */
#define	SYNCDATA_ACCESSLOG	1	/* entries are accesslog format */
#define	SYNCDATA_CHANGELOG	2	/* entries are changelog format */

52
53
54
#define	SYNCLOG_LOGGING		0	/* doing a log-based update */
#define	SYNCLOG_FALLBACK	1	/* doing a full refresh */

55
56
57
58
59
#define RETRYNUM_FOREVER	(-1)	/* retry forever */
#define RETRYNUM_TAIL		(-2)	/* end of retrynum array */
#define RETRYNUM_VALID(n)	((n) >= RETRYNUM_FOREVER)	/* valid retrynum */
#define RETRYNUM_FINITE(n)	((n) > RETRYNUM_FOREVER)	/* not forever */

60
typedef struct syncinfo_s {
61
	struct syncinfo_s	*si_next;
62
63
64
65
	BackendDB		*si_be;
	BackendDB		*si_wbe;
	struct re_s		*si_re;
	int			si_rid;
Quanah Gibson-Mount's avatar
Quanah Gibson-Mount committed
66
	char			si_ridtxt[ STRLENOF("rid=999") + 1 ];
Howard Chu's avatar
Howard Chu committed
67
68
	slap_bindconf		si_bindconf;
	struct berval		si_base;
69
70
71
	struct berval		si_logbase;
	struct berval		si_filterstr;
	struct berval		si_logfilterstr;
72
73
74
	int			si_scope;
	int			si_attrsonly;
	char			*si_anfile;
Howard Chu's avatar
Howard Chu committed
75
76
	AttributeName		*si_anlist;
	AttributeName		*si_exanlist;
77
78
79
80
81
82
83
84
85
86
87
	char 			**si_attrs;
	char			**si_exattrs;
	int			si_allattrs;
	int			si_allopattrs;
	int			si_schemachecking;
	int			si_type;	/* the active type */
	int			si_ctype;	/* the configured type */
	time_t			si_interval;
	time_t			*si_retryinterval;
	int			*si_retrynum_init;
	int			*si_retrynum;
Howard Chu's avatar
Howard Chu committed
88
	struct sync_cookie	si_syncCookie;
89
	cookie_state		*si_cookieState;
90
91
92
93
94
95
	int			si_cookieAge;
	int			si_manageDSAit;
	int			si_slimit;
	int			si_tlimit;
	int			si_refreshDelete;
	int			si_refreshPresent;
Quanah Gibson-Mount's avatar
Quanah Gibson-Mount committed
96
	int			si_refreshDone;
97
98
	int			si_syncdata;
	int			si_logstate;
Quanah Gibson-Mount's avatar
Quanah Gibson-Mount committed
99
	ber_int_t	si_msgid;
100
101
	Avlnode			*si_presentlist;
	LDAP			*si_ld;
102
	Connection		*si_conn;
103
	LDAP_LIST_HEAD(np, nonpresent_entry)	si_nonpresentlist;
Howard Chu's avatar
Howard Chu committed
104
	ldap_pvt_thread_mutex_t	si_mutex;
105
106
} syncinfo_t;

107
static int syncuuid_cmp( const void *, const void * );
Quanah Gibson-Mount's avatar
Quanah Gibson-Mount committed
108
static int avl_presentlist_insert( syncinfo_t* si, struct berval *syncUUID );
Quanah Gibson-Mount's avatar
Quanah Gibson-Mount committed
109
static void syncrepl_del_nonpresent( Operation *, syncinfo_t *, BerVarray, struct sync_cookie *, int );
110
111
static int syncrepl_message_to_op(
					syncinfo_t *, Operation *, LDAPMessage * );
112
113
114
115
116
static int syncrepl_message_to_entry(
					syncinfo_t *, Operation *, LDAPMessage *,
					Modifications **, Entry **, int );
static int syncrepl_entry(
					syncinfo_t *, Operation*, Entry*,
Pierangelo Masarati's avatar
Pierangelo Masarati committed
117
118
					Modifications**,int, struct berval*,
					struct berval *cookieCSN );
119
static int syncrepl_updateCookie(
120
121
122
123
					syncinfo_t *, Operation *, struct berval *,
					struct sync_cookie * );
static struct berval * slap_uuidstr_from_normalized(
					struct berval *, struct berval *, void * );
124

125
/* callback functions */
126
127
128
static int dn_callback( Operation *, SlapReply * );
static int nonpresent_callback( Operation *, SlapReply * );
static int null_callback( Operation *, SlapReply * );
129

130
static AttributeDescription *sync_descs[4];
131

132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
static const char *
syncrepl_state2str( int state )
{
	switch ( state ) {
	case LDAP_SYNC_PRESENT:
		return "PRESENT";

	case LDAP_SYNC_ADD:
		return "ADD";

	case LDAP_SYNC_MODIFY:
		return "MODIFY";

	case LDAP_SYNC_DELETE:
		return "DELETE";
	}

	return "UNKNOWN";
}

152
static void
153
init_syncrepl(syncinfo_t *si)
154
{
Jong Hyuk Choi's avatar
Jong Hyuk Choi committed
155
	int i, j, k, l, n;
156
	char **attrs, **exattrs;
157
158
159
160
161
162
163
164

	if ( !sync_descs[0] ) {
		sync_descs[0] = slap_schema.si_ad_objectClass;
		sync_descs[1] = slap_schema.si_ad_structuralObjectClass;
		sync_descs[2] = slap_schema.si_ad_entryCSN;
		sync_descs[3] = NULL;
	}

165
166
167
168
	if ( si->si_allattrs && si->si_allopattrs )
		attrs = NULL;
	else
		attrs = anlist2attrs( si->si_anlist );
169

170
171
172
173
	if ( attrs ) {
		if ( si->si_allattrs ) {
			i = 0;
			while ( attrs[i] ) {
174
				if ( !is_at_operational( at_find( attrs[i] ) ) ) {
175
176
177
178
					for ( j = i; attrs[j] != NULL; j++ ) {
						if ( j == i )
							ch_free( attrs[i] );
						attrs[j] = attrs[j+1];
179
					}
180
181
				} else {
					i++;
182
183
				}
			}
184
			attrs = ( char ** ) ch_realloc( attrs, (i + 2)*sizeof( char * ) );
185
186
			attrs[i] = ch_strdup("*");
			attrs[i + 1] = NULL;
187

188
189
190
		} else if ( si->si_allopattrs ) {
			i = 0;
			while ( attrs[i] ) {
191
				if ( is_at_operational( at_find( attrs[i] ) ) ) {
192
193
194
195
196
197
198
199
200
					for ( j = i; attrs[j] != NULL; j++ ) {
						if ( j == i )
							ch_free( attrs[i] );
						attrs[j] = attrs[j+1];
					}
				} else {
					i++;
				}
			}
201
			attrs = ( char ** ) ch_realloc( attrs, (i + 2)*sizeof( char * ) );
202
203
			attrs[i] = ch_strdup("+");
			attrs[i + 1] = NULL;
204
		}
Kurt Zeilenga's avatar
Kurt Zeilenga committed
205

206
207
208
		for ( i = 0; sync_descs[i] != NULL; i++ ) {
			j = 0;
			while ( attrs[j] ) {
209
				if ( !strcmp( attrs[j], sync_descs[i]->ad_cname.bv_val ) ) {
210
211
212
213
					for ( k = j; attrs[k] != NULL; k++ ) {
						if ( k == j )
							ch_free( attrs[k] );
						attrs[k] = attrs[k+1];
214
					}
215
216
				} else {
					j++;
217
218
				}
			}
219
220
221
222
223
		}

		for ( n = 0; attrs[ n ] != NULL; n++ ) /* empty */;

		if ( si->si_allopattrs ) {
224
			attrs = ( char ** ) ch_realloc( attrs, (n + 2)*sizeof( char * ) );
225
		} else {
226
			attrs = ( char ** ) ch_realloc( attrs, (n + 4)*sizeof( char * ) );
227
228
229
230
231
232
		}

		/* Add Attributes */
		if ( si->si_allopattrs ) {
			attrs[n++] = ch_strdup( sync_descs[0]->ad_cname.bv_val );
		} else {
233
234
235
			for ( i = 0; sync_descs[ i ] != NULL; i++ ) {
				attrs[ n++ ] = ch_strdup ( sync_descs[i]->ad_cname.bv_val );
			}
236
		}
237
		attrs[ n ] = NULL;
238

239
	} else {
Jong Hyuk Choi's avatar
Jong Hyuk Choi committed
240

241
		i = 0;
242
243
		if ( si->si_allattrs == si->si_allopattrs ) {
			attrs = (char**) ch_malloc( 3 * sizeof(char*) );
244
245
			attrs[i++] = ch_strdup( "*" );
			attrs[i++] = ch_strdup( "+" );
246
247
248
249
250
251
		} else if ( si->si_allattrs && !si->si_allopattrs ) {
			for ( n = 0; sync_descs[ n ] != NULL; n++ ) ;
			attrs = (char**) ch_malloc( (n+1)* sizeof(char*) );
			attrs[i++] = ch_strdup( "*" );
			for ( j = 1; sync_descs[ j ] != NULL; j++ ) {
				attrs[i++] = ch_strdup ( sync_descs[j]->ad_cname.bv_val );
Jong Hyuk Choi's avatar
Jong Hyuk Choi committed
252
			}
253
254
255
256
		} else if ( !si->si_allattrs && si->si_allopattrs ) {
			attrs = (char**) ch_malloc( 3 * sizeof(char*) );
			attrs[i++] = ch_strdup( "+" );
			attrs[i++] = ch_strdup( sync_descs[0]->ad_cname.bv_val );
257
258
		}
		attrs[i] = NULL;
259
260
	}
	
261
262
263
264
265
	si->si_attrs = attrs;

	exattrs = anlist2attrs( si->si_exanlist );

	if ( exattrs ) {
266
267
		for ( n = 0; exattrs[n] != NULL; n++ ) ;

Jong Hyuk Choi's avatar
Jong Hyuk Choi committed
268
		for ( i = 0; sync_descs[i] != NULL; i++ ) {
269
270
			j = 0;
			while ( exattrs[j] != NULL ) {
271
				if ( !strcmp( exattrs[j], sync_descs[i]->ad_cname.bv_val ) ) {
Howard Chu's avatar
Cleanup    
Howard Chu committed
272
					ch_free( exattrs[j] );
273
274
					for ( k = j; exattrs[k] != NULL; k++ ) {
						exattrs[k] = exattrs[k+1];
Jong Hyuk Choi's avatar
Jong Hyuk Choi committed
275
					}
276
277
				} else {
					j++;
Jong Hyuk Choi's avatar
Jong Hyuk Choi committed
278
279
280
				}
			}
		}
281
282
283

		for ( i = 0; exattrs[i] != NULL; i++ ) {
			for ( j = 0; si->si_anlist[j].an_name.bv_val; j++ ) {
284
285
				ObjectClass	*oc;
				if ( ( oc = si->si_anlist[j].an_oc ) ) {
286
287
288
					k = 0;
					while ( oc->soc_required[k] ) {
						if ( !strcmp( exattrs[i],
289
							 oc->soc_required[k]->sat_cname.bv_val ) ) {
Howard Chu's avatar
Cleanup    
Howard Chu committed
290
							ch_free( exattrs[i] );
291
292
293
294
295
							for ( l = i; exattrs[l]; l++ ) {
								exattrs[l] = exattrs[l+1];
							}
						} else {
							k++;
Jong Hyuk Choi's avatar
Jong Hyuk Choi committed
296
297
298
299
300
						}
					}
				}
			}
		}
301
302

		for ( i = 0; exattrs[i] != NULL; i++ ) ;
303
304

		if ( i != n )
305
			exattrs = (char **) ch_realloc( exattrs, (i + 1)*sizeof(char *) );
Jong Hyuk Choi's avatar
Jong Hyuk Choi committed
306
	}
307
308

	si->si_exattrs = exattrs;	
309
310
}

Howard Chu's avatar
Howard Chu committed
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
typedef struct logschema {
	struct berval ls_dn;
	struct berval ls_req;
	struct berval ls_mod;
	struct berval ls_newRdn;
	struct berval ls_delRdn;
	struct berval ls_newSup;
} logschema;

static logschema changelog_sc = {
	BER_BVC("targetDN"),
	BER_BVC("changeType"),
	BER_BVC("changes"),
	BER_BVC("newRDN"),
	BER_BVC("deleteOldRDN"),
	BER_BVC("newSuperior")
};

static logschema accesslog_sc = {
	BER_BVC("reqDN"),
	BER_BVC("reqType"),
	BER_BVC("reqMod"),
	BER_BVC("reqNewRDN"),
	BER_BVC("reqDeleteOldRDN"),
	BER_BVC("reqNewSuperior")
};
337

338
static int
Jong Hyuk Choi's avatar
Jong Hyuk Choi committed
339
340
ldap_sync_search(
	syncinfo_t *si,
Kurt Zeilenga's avatar
Kurt Zeilenga committed
341
	void *ctx )
Jong Hyuk Choi's avatar
Jong Hyuk Choi committed
342
{
343
344
345
346
	BerElementBuffer berbuf;
	BerElement *ber = (BerElement *)&berbuf;
	LDAPControl c[2], *ctrls[3];
	int rc;
347
348
	int rhint;
	char *base;
Howard Chu's avatar
Howard Chu committed
349
	char **attrs, *lattrs[8];
350
351
352
	char *filter;
	int attrsonly;
	int scope;
Jong Hyuk Choi's avatar
Jong Hyuk Choi committed
353

354
355
356
	/* setup LDAP SYNC control */
	ber_init2( ber, NULL, LBER_USE_DER );
	ber_set_option( ber, LBER_OPT_BER_MEMCTX, &ctx );
Jong Hyuk Choi's avatar
Jong Hyuk Choi committed
357

358
359
360
	/* If we're using a log but we have no state, then fallback to
	 * normal mode for a full refresh.
	 */
361
	if ( si->si_syncdata && !si->si_syncCookie.numcsns ) {
362
		si->si_logstate = SYNCLOG_FALLBACK;
363
	}
364
365
366

	/* Use the log parameters if we're in log mode */
	if ( si->si_syncdata && si->si_logstate == SYNCLOG_LOGGING ) {
Howard Chu's avatar
Howard Chu committed
367
368
369
370
371
372
373
374
375
376
377
378
379
380
		logschema *ls;
		if ( si->si_syncdata == SYNCDATA_ACCESSLOG )
			ls = &accesslog_sc;
		else
			ls = &changelog_sc;
		lattrs[0] = ls->ls_dn.bv_val;
		lattrs[1] = ls->ls_req.bv_val;
		lattrs[2] = ls->ls_mod.bv_val;
		lattrs[3] = ls->ls_newRdn.bv_val;
		lattrs[4] = ls->ls_delRdn.bv_val;
		lattrs[5] = ls->ls_newSup.bv_val;
		lattrs[6] = slap_schema.si_ad_entryCSN->ad_cname.bv_val;
		lattrs[7] = NULL;

381
382
383
		rhint = 0;
		base = si->si_logbase.bv_val;
		filter = si->si_logfilterstr.bv_val;
Howard Chu's avatar
Howard Chu committed
384
		attrs = lattrs;
385
		attrsonly = 0;
386
		scope = LDAP_SCOPE_SUBTREE;
387
388
389
390
391
392
393
394
395
	} else {
		rhint = 1;
		base = si->si_base.bv_val;
		filter = si->si_filterstr.bv_val;
		attrs = si->si_attrs;
		attrsonly = si->si_attrsonly;
		scope = si->si_scope;
	}
	if ( si->si_syncdata && si->si_logstate == SYNCLOG_FALLBACK ) {
Howard Chu's avatar
Howard Chu committed
396
		si->si_type = LDAP_SYNC_REFRESH_ONLY;
397
	} else {
Howard Chu's avatar
Howard Chu committed
398
		si->si_type = si->si_ctype;
399
400
	}

401
	if ( !BER_BVISNULL( &si->si_syncCookie.octet_str ) )
Kurt Zeilenga's avatar
Kurt Zeilenga committed
402
	{
Howard Chu's avatar
Howard Chu committed
403
		ber_printf( ber, "{eOb}",
Howard Chu's avatar
Howard Chu committed
404
			abs(si->si_type), &si->si_syncCookie.octet_str, rhint );
405
	} else {
Howard Chu's avatar
Howard Chu committed
406
		ber_printf( ber, "{eb}",
Howard Chu's avatar
Howard Chu committed
407
			abs(si->si_type), rhint );
408
	}
Jong Hyuk Choi's avatar
Jong Hyuk Choi committed
409

410
	if ( (rc = ber_flatten2( ber, &c[0].ldctl_value, 0 ) ) == LBER_ERROR ) {
411
412
413
		ber_free_buf( ber );
		return rc;
	}
Jong Hyuk Choi's avatar
Jong Hyuk Choi committed
414

415
	c[0].ldctl_oid = LDAP_CONTROL_SYNC;
416
	c[0].ldctl_iscritical = si->si_type < 0;
417
	ctrls[0] = &c[0];
Jong Hyuk Choi's avatar
Jong Hyuk Choi committed
418

419
	if ( !BER_BVISNULL( &si->si_bindconf.sb_authzId ) ) {
420
		c[1].ldctl_oid = LDAP_CONTROL_PROXY_AUTHZ;
421
		c[1].ldctl_value = si->si_bindconf.sb_authzId;
422
423
424
425
426
427
		c[1].ldctl_iscritical = 1;
		ctrls[1] = &c[1];
		ctrls[2] = NULL;
	} else {
		ctrls[1] = NULL;
	}
Jong Hyuk Choi's avatar
Jong Hyuk Choi committed
428

429
	rc = ldap_search_ext( si->si_ld, base, scope, filter, attrs, attrsonly,
Quanah Gibson-Mount's avatar
Quanah Gibson-Mount committed
430
		ctrls, NULL, NULL, si->si_slimit, &si->si_msgid );
431
	ber_free_buf( ber );
Jong Hyuk Choi's avatar
Jong Hyuk Choi committed
432
433
434
	return rc;
}

Quanah Gibson-Mount's avatar
Quanah Gibson-Mount committed
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
static int
check_syncprov(
	Operation *op,
	syncinfo_t *si )
{
	AttributeName at[2];
	Attribute a = {0};
	Entry e = {0};
	SlapReply rs = {0};
	int i, j, changed = 0;

	/* Look for contextCSN from syncprov overlay. If
	 * there's no overlay, this will be a no-op. That means
	 * this is a pure consumer, so local changes will not be
	 * allowed, and all changes will already be reflected in
	 * the cookieState.
	 */
	a.a_desc = slap_schema.si_ad_contextCSN;
	e.e_attrs = &a;
	e.e_name = op->o_bd->be_suffix[0];
	e.e_nname = op->o_bd->be_nsuffix[0];
	at[0].an_name = a.a_desc->ad_cname;
	at[0].an_desc = a.a_desc;
	BER_BVZERO( &at[1].an_name );
	rs.sr_entry = &e;
	rs.sr_flags = REP_ENTRY_MODIFIABLE;
	rs.sr_attrs = at;
	op->o_req_dn = e.e_name;
	op->o_req_ndn = e.e_nname;

	ldap_pvt_thread_mutex_lock( &si->si_cookieState->cs_mutex );
	i = backend_operational( op, &rs );
	if ( i == LDAP_SUCCESS && a.a_nvals ) {
		int num = a.a_numvals;
		/* check for differences */
		if ( num != si->si_cookieState->cs_num ) {
			changed = 1;
		} else {
			for ( i=0; i<num; i++ ) {
				if ( ber_bvcmp( &a.a_nvals[i],
					&si->si_cookieState->cs_vals[i] )) {
					changed =1;
					break;
				}
			}
		}
		if ( changed ) {
			ber_bvarray_free( si->si_cookieState->cs_vals );
			ch_free( si->si_cookieState->cs_sids );
			si->si_cookieState->cs_num = num;
			si->si_cookieState->cs_vals = a.a_nvals;
			si->si_cookieState->cs_sids = slap_parse_csn_sids( a.a_nvals,
				num, NULL );
			si->si_cookieState->cs_age++;
		} else {
			ber_bvarray_free( a.a_nvals );
		}
		ber_bvarray_free( a.a_vals );
	}
	/* See if the cookieState has changed due to anything outside
	 * this particular consumer. That includes other consumers in
	 * the same context, or local changes detected above.
	 */
	if ( si->si_cookieState->cs_num > 0 && si->si_cookieAge !=
		si->si_cookieState->cs_age ) {
		if ( !si->si_syncCookie.numcsns ) {
			ber_bvarray_free( si->si_syncCookie.ctxcsn );
			ber_bvarray_dup_x( &si->si_syncCookie.ctxcsn,
				si->si_cookieState->cs_vals, NULL );
			changed = 1;
		} else {
			for (i=0; !BER_BVISNULL( &si->si_syncCookie.ctxcsn[i] ); i++) {
				/* bogus, just dup everything */
				if ( si->si_syncCookie.sids[i] == -1 ) {
					ber_bvarray_free( si->si_syncCookie.ctxcsn );
					ber_bvarray_dup_x( &si->si_syncCookie.ctxcsn,
						si->si_cookieState->cs_vals, NULL );
					changed = 1;
					break;
				}
				for (j=0; j<si->si_cookieState->cs_num; j++) {
					if ( si->si_syncCookie.sids[i] !=
						si->si_cookieState->cs_sids[j] )
						continue;
					if ( bvmatch( &si->si_syncCookie.ctxcsn[i],
						&si->si_cookieState->cs_vals[j] ))
						break;
					ber_bvreplace( &si->si_syncCookie.ctxcsn[i],
						&si->si_cookieState->cs_vals[j] );
					changed = 1;
					break;
				}
			}
		}
	}
	if ( changed ) {
		si->si_cookieAge = si->si_cookieState->cs_age;
		ch_free( si->si_syncCookie.octet_str.bv_val );
		slap_compose_sync_cookie( NULL, &si->si_syncCookie.octet_str,
			si->si_syncCookie.ctxcsn, si->si_syncCookie.rid,
			si->si_syncCookie.sid );
	}
	ldap_pvt_thread_mutex_unlock( &si->si_cookieState->cs_mutex );
	return changed;
}

541
542
543
544
static int
do_syncrep1(
	Operation *op,
	syncinfo_t *si )
545
546
{
	int	rc;
547
	int cmdline_cookie_found = 0;
548

549
	struct sync_cookie	*sc = NULL;
550
551
552
#ifdef HAVE_TLS
	void	*ssl;
#endif
553

554
	rc = slap_client_connect( &si->si_ld, &si->si_bindconf );
Jong Hyuk Choi's avatar
Jong Hyuk Choi committed
555
	if ( rc != LDAP_SUCCESS ) {
556
		goto done;
557
	}
558
	op->o_protocol = LDAP_VERSION3;
559

560
561
562
	/* Set SSF to strongest of TLS, SASL SSFs */
	op->o_sasl_ssf = 0;
	op->o_tls_ssf = 0;
563
	op->o_transport_ssf = 0;
564
#ifdef HAVE_TLS
565
566
567
	if ( ldap_get_option( si->si_ld, LDAP_OPT_X_TLS_SSL_CTX, &ssl )
		== LDAP_SUCCESS && ssl != NULL )
	{
568
569
570
		op->o_tls_ssf = ldap_pvt_tls_get_strength( ssl );
	}
#endif /* HAVE_TLS */
Quanah Gibson-Mount's avatar
Quanah Gibson-Mount committed
571
572
573
574
575
576
	{
		ber_len_t ssf; /* ITS#5403, 3864 LDAP_OPT_X_SASL_SSF probably ought
						  to use sasl_ssf_t but currently uses ber_len_t */
		ldap_get_option( si->si_ld, LDAP_OPT_X_SASL_SSF, &ssf );
		op->o_sasl_ssf = ssf;
	}
577
578
	op->o_ssf = ( op->o_sasl_ssf > op->o_tls_ssf )
		?  op->o_sasl_ssf : op->o_tls_ssf;
579

580
581
	ldap_set_option( si->si_ld, LDAP_OPT_TIMELIMIT, &si->si_tlimit );

Howard Chu's avatar
Howard Chu committed
582
583
584
585
	si->si_syncCookie.rid = si->si_rid;
	si->si_syncCookie.sid = SLAP_SINGLE_SHADOW( si->si_be ) ? -1 :
		slap_serverID;

586
587
588
	/* We've just started up, or the remote server hasn't sent us
	 * any meaningful state.
	 */
589
	if ( BER_BVISNULL( &si->si_syncCookie.octet_str ) ) {
590
		int i;
591

592
593
594
595
596
597
598
599
600
		LDAP_STAILQ_FOREACH( sc, &slap_sync_cookie, sc_next ) {
			if ( si->si_rid == sc->rid ) {
				cmdline_cookie_found = 1;
				break;
			}
		}

		if ( cmdline_cookie_found ) {
			/* cookie is supplied in the command line */
601

602
603
			LDAP_STAILQ_REMOVE( &slap_sync_cookie, sc, sync_cookie, sc_next );

604
605
			/* ctxcsn wasn't parsed yet, do it now */
			slap_parse_sync_cookie( sc, op->o_tmpmemctx );
606
			slap_sync_cookie_free( &si->si_syncCookie, 0 );
607
608
			slap_dup_sync_cookie( &si->si_syncCookie, sc );
			slap_sync_cookie_free( sc, 1 );
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
		} else {
			ldap_pvt_thread_mutex_lock( &si->si_cookieState->cs_mutex );
			if ( !si->si_cookieState->cs_num ) {
				/* get contextCSN shadow replica from database */
				BerVarray csn = NULL;
				void *ctx = op->o_tmpmemctx;

				op->o_req_ndn = op->o_bd->be_nsuffix[0];
				op->o_req_dn = op->o_req_ndn;

				/* try to read stored contextCSN */
				op->o_tmpmemctx = NULL;
				backend_attribute( op, NULL, &op->o_req_ndn,
					slap_schema.si_ad_contextCSN, &csn, ACL_READ );
				op->o_tmpmemctx = ctx;
				if ( csn ) {
					si->si_cookieState->cs_vals = csn;
					for (i=0; !BER_BVISNULL( &csn[i] ); i++);
					si->si_cookieState->cs_num = i;
Howard Chu's avatar
Howard Chu committed
628
					si->si_cookieState->cs_sids = slap_parse_csn_sids( csn, i, NULL );
629
630
631
632
633
634
635
				}
			}
			if ( si->si_cookieState->cs_num ) {
				ber_bvarray_free( si->si_syncCookie.ctxcsn );
				if ( ber_bvarray_dup_x( &si->si_syncCookie.ctxcsn,
					si->si_cookieState->cs_vals, NULL )) {
					rc = LDAP_NO_MEMORY;
Quanah Gibson-Mount's avatar
Quanah Gibson-Mount committed
636
					ldap_pvt_thread_mutex_unlock( &si->si_cookieState->cs_mutex );
637
638
639
640
641
642
643
644
645
					goto done;
				}
				si->si_syncCookie.numcsns = si->si_cookieState->cs_num;
				si->si_syncCookie.sids = ch_malloc( si->si_cookieState->cs_num *
					sizeof(int) );
				for ( i=0; i<si->si_syncCookie.numcsns; i++ )
					si->si_syncCookie.sids[i] = si->si_cookieState->cs_sids[i];
			}
			ldap_pvt_thread_mutex_unlock( &si->si_cookieState->cs_mutex );
646
		}
Kurt Zeilenga's avatar
Kurt Zeilenga committed
647

Howard Chu's avatar
Howard Chu committed
648
		slap_compose_sync_cookie( NULL, &si->si_syncCookie.octet_str,
649
			si->si_syncCookie.ctxcsn, si->si_syncCookie.rid,
650
			si->si_syncCookie.sid );
651
	} else {
Quanah Gibson-Mount's avatar
Quanah Gibson-Mount committed
652
653
		/* Look for contextCSN from syncprov overlay. */
		check_syncprov( op, si );
Jong Hyuk Choi's avatar
Jong Hyuk Choi committed
654
655
	}

Quanah Gibson-Mount's avatar
Quanah Gibson-Mount committed
656
657
	si->si_refreshDone = 0;

Jong Hyuk Choi's avatar
Jong Hyuk Choi committed
658
	rc = ldap_sync_search( si, op->o_tmpmemctx );
659

660
	if( rc != LDAP_SUCCESS ) {
661
		Debug( LDAP_DEBUG_ANY, "do_syncrep1: %s "
662
			"ldap_search_ext: %s (%d)\n",
663
			si->si_ridtxt, ldap_err2string( rc ), rc );
664
665
666
667
668
	}

done:
	if ( rc ) {
		if ( si->si_ld ) {
669
			ldap_unbind_ext( si->si_ld, NULL, NULL );
670
671
672
673
674
675
676
			si->si_ld = NULL;
		}
	}

	return rc;
}

677
678
679
680
681
682
683
684
static int
compare_csns( struct sync_cookie *sc1, struct sync_cookie *sc2, int *which )
{
	int i, j, match = 0;
	const char *text;

	*which = 0;

685
686
687
688
689
	if ( sc1->numcsns < sc2->numcsns ) {
		*which = sc1->numcsns;
		return -1;
	}

Howard Chu's avatar
Howard Chu committed
690
	for (i=0; i<sc1->numcsns; i++) {
Howard Chu's avatar
Howard Chu committed
691
		for (j=0; j<sc2->numcsns; j++) {
692
693
694
695
696
			if ( sc1->sids[i] != sc2->sids[j] )
				continue;
			value_match( &match, slap_schema.si_ad_entryCSN,
				slap_schema.si_ad_entryCSN->ad_type->sat_ordering,
				SLAP_MR_VALUE_OF_ATTRIBUTE_SYNTAX,
Howard Chu's avatar
Howard Chu committed
697
				&sc1->ctxcsn[i], &sc2->ctxcsn[j], &text );
698
699
700
701
702
703
704
705
706
707
			if ( match < 0 ) {
				*which = j;
				return match;
			}
			break;
		}
	}
	return match;
}

Howard Chu's avatar
Howard Chu committed
708
709
#define	SYNC_PAUSED	-3

710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
static int
do_syncrep2(
	Operation *op,
	syncinfo_t *si )
{
	LDAPControl	**rctrls = NULL;
	LDAPControl	*rctrlp;

	BerElementBuffer berbuf;
	BerElement	*ber = (BerElement *)&berbuf;

	LDAPMessage	*msg = NULL;

	char		*retoid = NULL;
	struct berval	*retdata = NULL;

	Entry		*entry = NULL;

	int		syncstate;
Kurt Zeilenga's avatar
Kurt Zeilenga committed
729
	struct berval	syncUUID = BER_BVNULL;
730
731
	struct sync_cookie	syncCookie = { NULL };
	struct sync_cookie	syncCookie_req = { NULL };
Kurt Zeilenga's avatar
Kurt Zeilenga committed
732
	struct berval		cookie = BER_BVNULL;
733

734
735
	int		rc,
			err = LDAP_SUCCESS;
736
737
738
739
740
	ber_len_t	len;

	struct berval	*psub;
	Modifications	*modlist = NULL;

741
	int				match, m;
742
743
744
745

	struct timeval *tout_p = NULL;
	struct timeval tout = { 0, 0 };

746
	int		refreshDeletes = 0;
747
	BerVarray syncUUIDs = NULL;
748
749
	ber_tag_t si_tag;

750
	if ( slapd_shutdown ) {
751
		rc = -2;
752
		goto done;
753
754
	}

755
756
757
	ber_init2( ber, NULL, LBER_USE_DER );
	ber_set_option( ber, LBER_OPT_BER_MEMCTX, &op->o_tmpmemctx );

758
	Debug( LDAP_DEBUG_TRACE, "=>do_syncrep2 %s\n", si->si_ridtxt, 0, 0 );
759
760
761

	psub = &si->si_be->be_nsuffix[0];

Jong Hyuk Choi's avatar
Jong Hyuk Choi committed
762
	slap_dup_sync_cookie( &syncCookie_req, &si->si_syncCookie );
763

Kurt Zeilenga's avatar
Kurt Zeilenga committed
764
	if ( abs(si->si_type) == LDAP_SYNC_REFRESH_AND_PERSIST ) {
765
766
767
768
769
		tout_p = &tout;
	} else {
		tout_p = NULL;
	}

Quanah Gibson-Mount's avatar
Quanah Gibson-Mount committed
770
	while ( ( rc = ldap_result( si->si_ld, si->si_msgid, LDAP_MSG_ONE,
Howard Chu's avatar
Howard Chu committed
771
		tout_p, &msg ) ) > 0 )
772
	{
773
		if ( slapd_shutdown ) {
774
775
			rc = -2;
			goto done;
776
		}
Howard Chu's avatar
Howard Chu committed
777
778
779
780
781
782
783
784
785
786
787
788
789
		switch( ldap_msgtype( msg ) ) {
		case LDAP_RES_SEARCH_ENTRY:
			ldap_get_entry_controls( si->si_ld, msg, &rctrls );
			/* we can't work without the control */
			rctrlp = NULL;
			if ( rctrls ) {
				LDAPControl **next;
				/* NOTE: make sure we use the right one;
				 * a better approach would be to run thru
				 * the whole list and take care of all */
				rctrlp = ldap_control_find( LDAP_CONTROL_SYNC_STATE, rctrls, &next );
				if ( next && ldap_control_find( LDAP_CONTROL_SYNC_STATE, next, NULL ) )
				{
790
					Debug( LDAP_DEBUG_ANY, "do_syncrep2: %s "
Howard Chu's avatar
Howard Chu committed
791
						"got search entry with multiple "
792
						"Sync State control\n", si->si_ridtxt, 0, 0 );
Howard Chu's avatar
Howard Chu committed
793
794
795
					rc = -1;
					goto done;
				}
Howard Chu's avatar
Howard Chu committed
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
			}
			if ( rctrlp == NULL ) {
				Debug( LDAP_DEBUG_ANY, "do_syncrep2: %s "
					"got search entry without "
					"Sync State control\n", si->si_ridtxt, 0, 0 );
				rc = -1;
				goto done;
			}
			ber_init2( ber, &rctrlp->ldctl_value, LBER_USE_DER );
			ber_scanf( ber, "{em" /*"}"*/, &syncstate, &syncUUID );
			/* FIXME: what if syncUUID is NULL or empty?
			 * (happens with back-sql...) */
			if ( BER_BVISEMPTY( &syncUUID ) ) {
				Debug( LDAP_DEBUG_ANY, "do_syncrep2: %s "
					"got empty syncUUID with LDAP_SYNC_%s\n",
					si->si_ridtxt,
					syncrepl_state2str( syncstate ), 0 );
				ldap_controls_free( rctrls );
				rc = -1;
				goto done;
			}
			if ( ber_peek_tag( ber, &len ) == LDAP_TAG_SYNC_COOKIE ) {
				ber_scanf( ber, /*"{"*/ "m}", &cookie );

				Debug( LDAP_DEBUG_SYNC, "do_syncrep2: cookie=%s\n",
					BER_BVISNULL( &cookie ) ? "" : cookie.bv_val, 0, 0 );

				if ( !BER_BVISNULL( &cookie ) ) {
					ch_free( syncCookie.octet_str.bv_val );
					ber_dupbv( &syncCookie.octet_str, &cookie );
				}
				if ( !BER_BVISNULL( &syncCookie.octet_str ) )
				{
					slap_parse_sync_cookie( &syncCookie, NULL );
					if ( syncCookie.ctxcsn ) {
						int i, sid = slap_parse_csn_sid( syncCookie.ctxcsn );
						for ( i =0; i<si->si_cookieState->cs_num; i++ ) {
							if ( si->si_cookieState->cs_sids[i] == sid && 
								ber_bvcmp( syncCookie.ctxcsn, &si->si_cookieState->cs_vals[i] ) <= 0 ) {
								Debug( LDAP_DEBUG_SYNC, "do_syncrep2: %s CSN too old, ignoring %s\n",
									si->si_ridtxt, syncCookie.ctxcsn->bv_val, 0 );
								ldap_controls_free( rctrls );
								rc = 0;
								goto done;
							}
						}
					}
				}
			}
			rc = 0;
			if ( si->si_syncdata && si->si_logstate == SYNCLOG_LOGGING ) {
				modlist = NULL;
				if ( ( rc = syncrepl_message_to_op( si, op, msg ) ) == LDAP_SUCCESS &&
					syncCookie.ctxcsn )
				{
					rc = syncrepl_updateCookie( si, op, psub, &syncCookie );
852
853
854
855
856
857
858
859
860
861
862
				} else switch ( rc ) {
					case LDAP_ALREADY_EXISTS:
					case LDAP_NO_SUCH_OBJECT:
					case LDAP_NO_SUCH_ATTRIBUTE:
					case LDAP_TYPE_OR_VALUE_EXISTS:
						rc = LDAP_SYNC_REFRESH_REQUIRED;
						si->si_logstate = SYNCLOG_FALLBACK;
						ldap_abandon_ext( si->si_ld, si->si_msgid, NULL, NULL );
						break;
					default:
						break;
Howard Chu's avatar
Howard Chu committed
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
				}
			} else if ( ( rc = syncrepl_message_to_entry( si, op, msg,
				&modlist, &entry, syncstate ) ) == LDAP_SUCCESS )
			{
				if ( ( rc = syncrepl_entry( si, op, entry, &modlist,
					syncstate, &syncUUID, syncCookie.ctxcsn ) ) == LDAP_SUCCESS &&
					syncCookie.ctxcsn )
				{
					rc = syncrepl_updateCookie( si, op, psub, &syncCookie );
				}
			}
			ldap_controls_free( rctrls );
			if ( modlist ) {
				slap_mods_free( modlist, 1 );
			}
			if ( rc )
				goto done;
			break;

		case LDAP_RES_SEARCH_REFERENCE:
			Debug( LDAP_DEBUG_ANY,
				"do_syncrep2: %s reference received error\n",
				si->si_ridtxt, 0, 0 );
			break;

		case LDAP_RES_SEARCH_RESULT:
			Debug( LDAP_DEBUG_SYNC,
				"do_syncrep2: %s LDAP_RES_SEARCH_RESULT\n",
				si->si_ridtxt, 0, 0 );
			ldap_parse_result( si->si_ld, msg, &err, NULL, NULL, NULL,
				&rctrls, 0 );
#ifdef LDAP_X_SYNC_REFRESH_REQUIRED
			if ( err == LDAP_X_SYNC_REFRESH_REQUIRED ) {
				/* map old result code to registered code */
				err = LDAP_SYNC_REFRESH_REQUIRED;
			}
#endif
			if ( err == LDAP_SYNC_REFRESH_REQUIRED ) {
				if ( si->si_logstate == SYNCLOG_LOGGING ) {
					si->si_logstate = SYNCLOG_FALLBACK;
903
				}
Howard Chu's avatar
Howard Chu committed
904
905
906
907
908
909
910
911
				rc = err;
				goto done;
			}
			if ( rctrls ) {
				rctrlp = *rctrls;
				ber_init2( ber, &rctrlp->ldctl_value, LBER_USE_DER );

				ber_scanf( ber, "{" /*"}"*/);
912
				if ( ber_peek_tag( ber, &len ) == LDAP_TAG_SYNC_COOKIE ) {
Howard Chu's avatar
Howard Chu committed
913
					ber_scanf( ber, "m", &cookie );
Quanah Gibson-Mount's avatar
Quanah Gibson-Mount committed
914
915
916
917

					Debug( LDAP_DEBUG_SYNC, "do_syncrep2: cookie=%s\n",
						BER_BVISNULL( &cookie ) ? "" : cookie.bv_val, 0, 0 );

918
					if ( !BER_BVISNULL( &cookie ) ) {
919
						ch_free( syncCookie.octet_str.bv_val );
Howard Chu's avatar
Howard Chu committed
920
						ber_dupbv( &syncCookie.octet_str, &cookie);
Jong Hyuk Choi's avatar
Jong Hyuk Choi committed
921
					}
922
					if ( !BER_BVISNULL( &syncCookie.octet_str ) )
923
					{
Howard Chu's avatar
Howard Chu committed
924
						slap_parse_sync_cookie( &syncCookie, NULL );
925
					}
926
				}
Howard Chu's avatar
Howard Chu committed
927
				if ( ber_peek_tag( ber, &len ) == LDAP_TAG_REFRESHDELETES )
928
				{
Howard Chu's avatar
Howard Chu committed
929
					ber_scanf( ber, "b", &refreshDeletes );
930
				}
Howard Chu's avatar
Howard Chu committed
931
932
				ber_scanf( ber, /*"{"*/ "}" );
			}
Quanah Gibson-Mount's avatar
Quanah Gibson-Mount committed
933
934
935
936
			if ( SLAP_MULTIMASTER( op->o_bd ) && check_syncprov( op, si )) {
				slap_sync_cookie_free( &syncCookie_req, 0 );
				slap_dup_sync_cookie( &syncCookie_req, &si->si_syncCookie );
			}
Howard Chu's avatar
Howard Chu committed
937
938
939
940
941
942
943
944
945
			if ( !syncCookie.ctxcsn ) {
				match = 1;
			} else if ( !syncCookie_req.ctxcsn ) {
				match = -1;
				m = 0;
			} else {
				match = compare_csns( &syncCookie_req, &syncCookie, &m );
			}
			if ( rctrls ) {
Howard Chu's avatar
Howard Chu committed
946
				ldap_controls_free( rctrls );
Howard Chu's avatar
Howard Chu committed
947
948
949
950
951
952
953
			}
			if (si->si_type != LDAP_SYNC_REFRESH_AND_PERSIST) {
				/* FIXME : different error behaviors according to
				 *	1) err code : LDAP_BUSY ...
				 *	2) on err policy : stop service, stop sync, retry
				 */
				if ( refreshDeletes == 0 && match < 0 &&
Quanah Gibson-Mount's avatar
Quanah Gibson-Mount committed
954
955
					err == LDAP_SUCCESS &&
					syncCookie_req.numcsns == syncCookie.numcsns )
Howard Chu's avatar
Howard Chu committed
956
957
				{
					syncrepl_del_nonpresent( op, si, NULL,
Quanah Gibson-Mount's avatar
Quanah Gibson-Mount committed
958
						&syncCookie, m );
Howard Chu's avatar
Howard Chu committed
959
960
961
				} else {
					avl_free( si->si_presentlist, ch_free );
					si->si_presentlist = NULL;
Jong Hyuk Choi's avatar
Jong Hyuk Choi committed
962
				}
Howard Chu's avatar
Howard Chu committed
963
964
965
966
967
968
969
970
971
972
973
974
975
976
			}
			if ( syncCookie.ctxcsn && match < 0 && err == LDAP_SUCCESS )
			{
				rc = syncrepl_updateCookie( si, op, psub, &syncCookie );
			}
			if ( err == LDAP_SUCCESS
				&& si->si_logstate == SYNCLOG_FALLBACK ) {
				si->si_logstate = SYNCLOG_LOGGING;
				rc = LDAP_SYNC_REFRESH_REQUIRED;
			} else {
				rc = -2;
			}
			goto done;
			break;
977

Howard Chu's avatar
Howard Chu committed
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
		case LDAP_RES_INTERMEDIATE:
			rc = ldap_parse_intermediate( si->si_ld, msg,
				&retoid, &retdata, NULL, 0 );
			if ( !rc && !strcmp( retoid, LDAP_SYNC_INFO ) ) {
				ber_init2( ber, retdata, LBER_USE_DER );

				switch ( si_tag = ber_peek_tag( ber, &len ) ) {
				ber_tag_t tag;
				case LDAP_TAG_SYNC_NEW_COOKIE:
					Debug( LDAP_DEBUG_SYNC,
						"do_syncrep2: %s %s - %s\n", 
						si->si_ridtxt,
						"LDAP_RES_INTERMEDIATE", 
						"NEW_COOKIE" );
					ber_scanf( ber, "tm", &tag, &cookie );
					break;
				case LDAP_TAG_SYNC_REFRESH_DELETE:
				case LDAP_TAG_SYNC_REFRESH_PRESENT:
					Debug( LDAP_DEBUG_SYNC,
						"do_syncrep2: %s %s - %s\n", 
						si->si_ridtxt,
						"LDAP_RES_INTERMEDIATE", 
						si_tag == LDAP_TAG_SYNC_REFRESH_PRESENT ?
For faster browsing, not all history is shown. View entire blame