syncrepl.c 107 KB
Newer Older
Kurt Zeilenga's avatar
Kurt Zeilenga committed
1
/* syncrepl.c -- Replication Engine which uses the LDAP Sync protocol */
2
/* $OpenLDAP$ */
Kurt Zeilenga's avatar
Kurt Zeilenga committed
3
/* This work is part of OpenLDAP Software <http://www.openldap.org/>.
4
 *
Kurt Zeilenga's avatar
Kurt Zeilenga committed
5
 * Copyright 2003-2007 The OpenLDAP Foundation.
Kurt Zeilenga's avatar
Kurt Zeilenga committed
6
7
8
 * Portions Copyright 2003 by IBM Corporation.
 * Portions Copyright 2003 by Howard Chu, Symas Corporation.
 * All rights reserved.
9
 *
Kurt Zeilenga's avatar
Kurt Zeilenga committed
10
11
12
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted only as authorized by the OpenLDAP
 * Public License.
13
 *
Kurt Zeilenga's avatar
Kurt Zeilenga committed
14
15
16
 * A copy of this license is available in the file LICENSE in the
 * top-level directory of the distribution or, alternatively, at
 * <http://www.OpenLDAP.org/license.html>.
17
 */
18
19
20
21
22
23
24
25
26
27
28
29

#include "portable.h"

#include <stdio.h>

#include <ac/string.h>
#include <ac/socket.h>

#include "lutil.h"
#include "slap.h"
#include "lutil_ldap.h"

30
31
#include "config.h"

32
33
#include "ldap_rq.h"

34
35
36
37
38
39
struct nonpresent_entry {
	struct berval *npe_name;
	struct berval *npe_nname;
	LDAP_LIST_ENTRY(nonpresent_entry) npe_link;
};

40
41
42
43
44
45
46
47
typedef struct cookie_state {
	ldap_pvt_thread_mutex_t	cs_mutex;
	int	cs_num;
	int cs_age;
	struct berval *cs_vals;
	int *cs_sids;
} cookie_state;
	
48
49
50
51
#define	SYNCDATA_DEFAULT	0	/* entries are plain LDAP entries */
#define	SYNCDATA_ACCESSLOG	1	/* entries are accesslog format */
#define	SYNCDATA_CHANGELOG	2	/* entries are changelog format */

52
53
54
#define	SYNCLOG_LOGGING		0	/* doing a log-based update */
#define	SYNCLOG_FALLBACK	1	/* doing a full refresh */

55
56
57
58
59
#define RETRYNUM_FOREVER	(-1)	/* retry forever */
#define RETRYNUM_TAIL		(-2)	/* end of retrynum array */
#define RETRYNUM_VALID(n)	((n) >= RETRYNUM_FOREVER)	/* valid retrynum */
#define RETRYNUM_FINITE(n)	((n) > RETRYNUM_FOREVER)	/* not forever */

60
typedef struct syncinfo_s {
61
	struct syncinfo_s	*si_next;
Howard Chu's avatar
Howard Chu committed
62
	struct slap_backend_db *si_be;
63
	struct slap_backend_db *si_wbe;
Howard Chu's avatar
Howard Chu committed
64
	struct re_s			*si_re;
65
66
	int					si_rid;
	char				si_ridtxt[8];
Howard Chu's avatar
Howard Chu committed
67
68
	slap_bindconf		si_bindconf;
	struct berval		si_base;
69
70
71
	struct berval		si_logbase;
	struct berval		si_filterstr;
	struct berval		si_logfilterstr;
Howard Chu's avatar
Howard Chu committed
72
73
74
75
76
77
78
79
80
81
	int					si_scope;
	int					si_attrsonly;
	char				*si_anfile;
	AttributeName		*si_anlist;
	AttributeName		*si_exanlist;
	char 				**si_attrs;
	char				**si_exattrs;
	int					si_allattrs;
	int					si_allopattrs;
	int					si_schemachecking;
Howard Chu's avatar
Howard Chu committed
82
83
	int					si_type;	/* the active type */
	int					si_ctype;	/* the configured type */
Howard Chu's avatar
Howard Chu committed
84
85
86
87
88
	time_t				si_interval;
	time_t				*si_retryinterval;
	int					*si_retrynum_init;
	int					*si_retrynum;
	struct sync_cookie	si_syncCookie;
89
90
	cookie_state		*si_cookieState;
	int					si_cookieAge;
Howard Chu's avatar
Howard Chu committed
91
92
93
94
95
	int					si_manageDSAit;
	int					si_slimit;
	int					si_tlimit;
	int					si_refreshDelete;
	int					si_refreshPresent;
96
	int					si_syncdata;
97
	int					si_logstate;
Howard Chu's avatar
Howard Chu committed
98
	int					si_conn_setup;
Howard Chu's avatar
Howard Chu committed
99
100
101
102
	Avlnode				*si_presentlist;
	LDAP				*si_ld;
	LDAP_LIST_HEAD(np, nonpresent_entry) si_nonpresentlist;
	ldap_pvt_thread_mutex_t	si_mutex;
103
104
} syncinfo_t;

105
static int syncuuid_cmp( const void *, const void * );
106
static void avl_ber_bvfree( void * );
107
static void syncrepl_del_nonpresent( Operation *, syncinfo_t *, BerVarray, struct berval * );
108
109
static int syncrepl_message_to_op(
					syncinfo_t *, Operation *, LDAPMessage * );
110
111
112
113
114
static int syncrepl_message_to_entry(
					syncinfo_t *, Operation *, LDAPMessage *,
					Modifications **, Entry **, int );
static int syncrepl_entry(
					syncinfo_t *, Operation*, Entry*,
115
					Modifications**,int, struct berval* );
116
static int syncrepl_updateCookie(
117
118
119
120
					syncinfo_t *, Operation *, struct berval *,
					struct sync_cookie * );
static struct berval * slap_uuidstr_from_normalized(
					struct berval *, struct berval *, void * );
121

122
/* callback functions */
Jong Hyuk Choi's avatar
Jong Hyuk Choi committed
123
static int dn_callback( struct slap_op *, struct slap_rep * );
124
125
static int nonpresent_callback( struct slap_op *, struct slap_rep * );
static int null_callback( struct slap_op *, struct slap_rep * );
126

127
static AttributeDescription *sync_descs[4];
128

129
static void
130
init_syncrepl(syncinfo_t *si)
131
{
Jong Hyuk Choi's avatar
Jong Hyuk Choi committed
132
	int i, j, k, l, n;
133
	char **attrs, **exattrs;
134
135
136
137
138
139
140
141

	if ( !sync_descs[0] ) {
		sync_descs[0] = slap_schema.si_ad_objectClass;
		sync_descs[1] = slap_schema.si_ad_structuralObjectClass;
		sync_descs[2] = slap_schema.si_ad_entryCSN;
		sync_descs[3] = NULL;
	}

142
143
144
145
	if ( si->si_allattrs && si->si_allopattrs )
		attrs = NULL;
	else
		attrs = anlist2attrs( si->si_anlist );
146

147
148
149
150
	if ( attrs ) {
		if ( si->si_allattrs ) {
			i = 0;
			while ( attrs[i] ) {
151
				if ( !is_at_operational( at_find( attrs[i] ) ) ) {
152
153
154
155
					for ( j = i; attrs[j] != NULL; j++ ) {
						if ( j == i )
							ch_free( attrs[i] );
						attrs[j] = attrs[j+1];
156
					}
157
158
				} else {
					i++;
159
160
				}
			}
161
			attrs = ( char ** ) ch_realloc( attrs, (i + 2)*sizeof( char * ) );
162
163
			attrs[i] = ch_strdup("*");
			attrs[i + 1] = NULL;
164

165
166
167
		} else if ( si->si_allopattrs ) {
			i = 0;
			while ( attrs[i] ) {
168
				if ( is_at_operational( at_find( attrs[i] ) ) ) {
169
170
171
172
173
174
175
176
177
					for ( j = i; attrs[j] != NULL; j++ ) {
						if ( j == i )
							ch_free( attrs[i] );
						attrs[j] = attrs[j+1];
					}
				} else {
					i++;
				}
			}
178
			attrs = ( char ** ) ch_realloc( attrs, (i + 2)*sizeof( char * ) );
179
180
			attrs[i] = ch_strdup("+");
			attrs[i + 1] = NULL;
181
		}
Kurt Zeilenga's avatar
Kurt Zeilenga committed
182

183
184
185
		for ( i = 0; sync_descs[i] != NULL; i++ ) {
			j = 0;
			while ( attrs[j] ) {
186
				if ( !strcmp( attrs[j], sync_descs[i]->ad_cname.bv_val ) ) {
187
188
189
190
					for ( k = j; attrs[k] != NULL; k++ ) {
						if ( k == j )
							ch_free( attrs[k] );
						attrs[k] = attrs[k+1];
191
					}
192
193
				} else {
					j++;
194
195
				}
			}
196
197
198
199
200
		}

		for ( n = 0; attrs[ n ] != NULL; n++ ) /* empty */;

		if ( si->si_allopattrs ) {
201
			attrs = ( char ** ) ch_realloc( attrs, (n + 2)*sizeof( char * ) );
202
		} else {
203
			attrs = ( char ** ) ch_realloc( attrs, (n + 4)*sizeof( char * ) );
204
205
206
		}

		if ( attrs == NULL ) {
207
			Debug( LDAP_DEBUG_ANY, "out of memory\n", 0, 0, 0 );
208
209
210
211
212
213
		}

		/* Add Attributes */
		if ( si->si_allopattrs ) {
			attrs[n++] = ch_strdup( sync_descs[0]->ad_cname.bv_val );
		} else {
214
215
216
			for ( i = 0; sync_descs[ i ] != NULL; i++ ) {
				attrs[ n++ ] = ch_strdup ( sync_descs[i]->ad_cname.bv_val );
			}
217
		}
218
		attrs[ n ] = NULL;
219

220
	} else {
Jong Hyuk Choi's avatar
Jong Hyuk Choi committed
221

222
		i = 0;
223
224
		if ( si->si_allattrs == si->si_allopattrs ) {
			attrs = (char**) ch_malloc( 3 * sizeof(char*) );
225
226
			attrs[i++] = ch_strdup( "*" );
			attrs[i++] = ch_strdup( "+" );
227
228
229
230
231
232
		} else if ( si->si_allattrs && !si->si_allopattrs ) {
			for ( n = 0; sync_descs[ n ] != NULL; n++ ) ;
			attrs = (char**) ch_malloc( (n+1)* sizeof(char*) );
			attrs[i++] = ch_strdup( "*" );
			for ( j = 1; sync_descs[ j ] != NULL; j++ ) {
				attrs[i++] = ch_strdup ( sync_descs[j]->ad_cname.bv_val );
Jong Hyuk Choi's avatar
Jong Hyuk Choi committed
233
			}
234
235
236
237
		} else if ( !si->si_allattrs && si->si_allopattrs ) {
			attrs = (char**) ch_malloc( 3 * sizeof(char*) );
			attrs[i++] = ch_strdup( "+" );
			attrs[i++] = ch_strdup( sync_descs[0]->ad_cname.bv_val );
238
239
		}
		attrs[i] = NULL;
240
241
	}
	
242
243
244
245
246
	si->si_attrs = attrs;

	exattrs = anlist2attrs( si->si_exanlist );

	if ( exattrs ) {
247
248
		for ( n = 0; exattrs[n] != NULL; n++ ) ;

Jong Hyuk Choi's avatar
Jong Hyuk Choi committed
249
		for ( i = 0; sync_descs[i] != NULL; i++ ) {
250
251
			j = 0;
			while ( exattrs[j] != NULL ) {
252
				if ( !strcmp( exattrs[j], sync_descs[i]->ad_cname.bv_val ) ) {
Howard Chu's avatar
Cleanup    
Howard Chu committed
253
					ch_free( exattrs[j] );
254
255
					for ( k = j; exattrs[k] != NULL; k++ ) {
						exattrs[k] = exattrs[k+1];
Jong Hyuk Choi's avatar
Jong Hyuk Choi committed
256
					}
257
258
				} else {
					j++;
Jong Hyuk Choi's avatar
Jong Hyuk Choi committed
259
260
261
				}
			}
		}
262
263
264

		for ( i = 0; exattrs[i] != NULL; i++ ) {
			for ( j = 0; si->si_anlist[j].an_name.bv_val; j++ ) {
265
266
				ObjectClass	*oc;
				if ( ( oc = si->si_anlist[j].an_oc ) ) {
267
268
269
					k = 0;
					while ( oc->soc_required[k] ) {
						if ( !strcmp( exattrs[i],
270
							 oc->soc_required[k]->sat_cname.bv_val ) ) {
Howard Chu's avatar
Cleanup    
Howard Chu committed
271
							ch_free( exattrs[i] );
272
273
274
275
276
							for ( l = i; exattrs[l]; l++ ) {
								exattrs[l] = exattrs[l+1];
							}
						} else {
							k++;
Jong Hyuk Choi's avatar
Jong Hyuk Choi committed
277
278
279
280
281
						}
					}
				}
			}
		}
282
283

		for ( i = 0; exattrs[i] != NULL; i++ ) ;
284
285

		if ( i != n )
286
			exattrs = (char **) ch_realloc( exattrs, (i + 1)*sizeof(char *) );
Jong Hyuk Choi's avatar
Jong Hyuk Choi committed
287
	}
288
289

	si->si_exattrs = exattrs;	
290
291
}

Howard Chu's avatar
Howard Chu committed
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
typedef struct logschema {
	struct berval ls_dn;
	struct berval ls_req;
	struct berval ls_mod;
	struct berval ls_newRdn;
	struct berval ls_delRdn;
	struct berval ls_newSup;
} logschema;

static logschema changelog_sc = {
	BER_BVC("targetDN"),
	BER_BVC("changeType"),
	BER_BVC("changes"),
	BER_BVC("newRDN"),
	BER_BVC("deleteOldRDN"),
	BER_BVC("newSuperior")
};

static logschema accesslog_sc = {
	BER_BVC("reqDN"),
	BER_BVC("reqType"),
	BER_BVC("reqMod"),
	BER_BVC("reqNewRDN"),
	BER_BVC("reqDeleteOldRDN"),
	BER_BVC("reqNewSuperior")
};
318

319
static int
Jong Hyuk Choi's avatar
Jong Hyuk Choi committed
320
321
ldap_sync_search(
	syncinfo_t *si,
Kurt Zeilenga's avatar
Kurt Zeilenga committed
322
	void *ctx )
Jong Hyuk Choi's avatar
Jong Hyuk Choi committed
323
{
324
325
326
	BerElementBuffer berbuf;
	BerElement *ber = (BerElement *)&berbuf;
	LDAPControl c[2], *ctrls[3];
327
	struct timeval timeout;
328
	ber_int_t	msgid;
329
	int rc;
330
331
	int rhint;
	char *base;
Howard Chu's avatar
Howard Chu committed
332
	char **attrs, *lattrs[8];
333
334
335
	char *filter;
	int attrsonly;
	int scope;
Jong Hyuk Choi's avatar
Jong Hyuk Choi committed
336

337
338
339
	/* setup LDAP SYNC control */
	ber_init2( ber, NULL, LBER_USE_DER );
	ber_set_option( ber, LBER_OPT_BER_MEMCTX, &ctx );
Jong Hyuk Choi's avatar
Jong Hyuk Choi committed
340

341
342
343
	/* If we're using a log but we have no state, then fallback to
	 * normal mode for a full refresh.
	 */
344
	if ( si->si_syncdata && !si->si_syncCookie.numcsns ) {
345
		si->si_logstate = SYNCLOG_FALLBACK;
346
	}
347
348
349

	/* Use the log parameters if we're in log mode */
	if ( si->si_syncdata && si->si_logstate == SYNCLOG_LOGGING ) {
Howard Chu's avatar
Howard Chu committed
350
351
352
353
354
355
356
357
358
359
360
361
362
363
		logschema *ls;
		if ( si->si_syncdata == SYNCDATA_ACCESSLOG )
			ls = &accesslog_sc;
		else
			ls = &changelog_sc;
		lattrs[0] = ls->ls_dn.bv_val;
		lattrs[1] = ls->ls_req.bv_val;
		lattrs[2] = ls->ls_mod.bv_val;
		lattrs[3] = ls->ls_newRdn.bv_val;
		lattrs[4] = ls->ls_delRdn.bv_val;
		lattrs[5] = ls->ls_newSup.bv_val;
		lattrs[6] = slap_schema.si_ad_entryCSN->ad_cname.bv_val;
		lattrs[7] = NULL;

364
365
366
		rhint = 0;
		base = si->si_logbase.bv_val;
		filter = si->si_logfilterstr.bv_val;
Howard Chu's avatar
Howard Chu committed
367
		attrs = lattrs;
368
		attrsonly = 0;
369
		scope = LDAP_SCOPE_SUBTREE;
370
371
372
373
374
375
376
377
378
	} else {
		rhint = 1;
		base = si->si_base.bv_val;
		filter = si->si_filterstr.bv_val;
		attrs = si->si_attrs;
		attrsonly = si->si_attrsonly;
		scope = si->si_scope;
	}
	if ( si->si_syncdata && si->si_logstate == SYNCLOG_FALLBACK ) {
Howard Chu's avatar
Howard Chu committed
379
		si->si_type = LDAP_SYNC_REFRESH_ONLY;
380
	} else {
Howard Chu's avatar
Howard Chu committed
381
		si->si_type = si->si_ctype;
382
383
	}

384
	if ( !BER_BVISNULL( &si->si_syncCookie.octet_str ) )
Kurt Zeilenga's avatar
Kurt Zeilenga committed
385
	{
Howard Chu's avatar
Howard Chu committed
386
		ber_printf( ber, "{eOb}",
Howard Chu's avatar
Howard Chu committed
387
			abs(si->si_type), &si->si_syncCookie.octet_str, rhint );
388
	} else {
Howard Chu's avatar
Howard Chu committed
389
		ber_printf( ber, "{eb}",
Howard Chu's avatar
Howard Chu committed
390
			abs(si->si_type), rhint );
391
	}
Jong Hyuk Choi's avatar
Jong Hyuk Choi committed
392

393
	if ( (rc = ber_flatten2( ber, &c[0].ldctl_value, 0 ) ) == LBER_ERROR ) {
394
395
396
		ber_free_buf( ber );
		return rc;
	}
Jong Hyuk Choi's avatar
Jong Hyuk Choi committed
397

398
	c[0].ldctl_oid = LDAP_CONTROL_SYNC;
399
	c[0].ldctl_iscritical = si->si_type < 0;
400
	ctrls[0] = &c[0];
Jong Hyuk Choi's avatar
Jong Hyuk Choi committed
401

402
	if ( !BER_BVISNULL( &si->si_bindconf.sb_authzId ) ) {
403
		c[1].ldctl_oid = LDAP_CONTROL_PROXY_AUTHZ;
404
		c[1].ldctl_value = si->si_bindconf.sb_authzId;
405
406
407
408
409
410
		c[1].ldctl_iscritical = 1;
		ctrls[1] = &c[1];
		ctrls[2] = NULL;
	} else {
		ctrls[1] = NULL;
	}
Jong Hyuk Choi's avatar
Jong Hyuk Choi committed
411

Jong Hyuk Choi's avatar
Jong Hyuk Choi committed
412
	timeout.tv_sec = si->si_tlimit;
413
	timeout.tv_usec = 0;
414

415
	rc = ldap_search_ext( si->si_ld, base, scope, filter, attrs, attrsonly,
Jong Hyuk Choi's avatar
Jong Hyuk Choi committed
416
		ctrls, NULL, si->si_tlimit > 0 ? &timeout : NULL,
417
		si->si_slimit, &msgid );
418
	ber_free_buf( ber );
Jong Hyuk Choi's avatar
Jong Hyuk Choi committed
419
420
421
	return rc;
}

422
423
424
425
static int
do_syncrep1(
	Operation *op,
	syncinfo_t *si )
426
427
{
	int	rc;
428
	int cmdline_cookie_found = 0;
429

430
	struct sync_cookie	*sc = NULL;
431
432
433
#ifdef HAVE_TLS
	void	*ssl;
#endif
434

435
	rc = slap_client_connect( &si->si_ld, &si->si_bindconf );
Jong Hyuk Choi's avatar
Jong Hyuk Choi committed
436
	if ( rc != LDAP_SUCCESS ) {
437
		goto done;
438
	}
439
	op->o_protocol = LDAP_VERSION3;
440

441
442
443
	/* Set SSF to strongest of TLS, SASL SSFs */
	op->o_sasl_ssf = 0;
	op->o_tls_ssf = 0;
444
	op->o_transport_ssf = 0;
445
#ifdef HAVE_TLS
446
447
448
	if ( ldap_get_option( si->si_ld, LDAP_OPT_X_TLS_SSL_CTX, &ssl )
		== LDAP_SUCCESS && ssl != NULL )
	{
449
450
451
452
		op->o_tls_ssf = ldap_pvt_tls_get_strength( ssl );
	}
#endif /* HAVE_TLS */
	ldap_get_option( si->si_ld, LDAP_OPT_X_SASL_SSF, &op->o_sasl_ssf );
453
454
	op->o_ssf = ( op->o_sasl_ssf > op->o_tls_ssf )
		?  op->o_sasl_ssf : op->o_tls_ssf;
455

456
457
458
	/* We've just started up, or the remote server hasn't sent us
	 * any meaningful state.
	 */
459
	if ( BER_BVISNULL( &si->si_syncCookie.octet_str ) ) {
460
		int i;
461

462
		si->si_syncCookie.rid = si->si_rid;
463
464
		si->si_syncCookie.sid = SLAP_SINGLE_SHADOW( si->si_be ) ? -1 :
			slap_serverID;
465
466
467
468
469
470
471
472
473
474

		LDAP_STAILQ_FOREACH( sc, &slap_sync_cookie, sc_next ) {
			if ( si->si_rid == sc->rid ) {
				cmdline_cookie_found = 1;
				break;
			}
		}

		if ( cmdline_cookie_found ) {
			/* cookie is supplied in the command line */
475

476
477
			LDAP_STAILQ_REMOVE( &slap_sync_cookie, sc, sync_cookie, sc_next );

478
479
			/* ctxcsn wasn't parsed yet, do it now */
			slap_parse_sync_cookie( sc, op->o_tmpmemctx );
480
			slap_sync_cookie_free( &si->si_syncCookie, 0 );
481
482
			slap_dup_sync_cookie( &si->si_syncCookie, sc );
			slap_sync_cookie_free( sc, 1 );
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
		} else {
			ldap_pvt_thread_mutex_lock( &si->si_cookieState->cs_mutex );
			if ( !si->si_cookieState->cs_num ) {
				/* get contextCSN shadow replica from database */
				BerVarray csn = NULL;
				void *ctx = op->o_tmpmemctx;

				op->o_req_ndn = op->o_bd->be_nsuffix[0];
				op->o_req_dn = op->o_req_ndn;

				/* try to read stored contextCSN */
				op->o_tmpmemctx = NULL;
				backend_attribute( op, NULL, &op->o_req_ndn,
					slap_schema.si_ad_contextCSN, &csn, ACL_READ );
				op->o_tmpmemctx = ctx;
				if ( csn ) {
					si->si_cookieState->cs_vals = csn;
					for (i=0; !BER_BVISNULL( &csn[i] ); i++);
					si->si_cookieState->cs_num = i;
Howard Chu's avatar
Howard Chu committed
502
					si->si_cookieState->cs_sids = slap_parse_csn_sids( csn, i, NULL );
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
				}
			}
			if ( si->si_cookieState->cs_num ) {
				ber_bvarray_free( si->si_syncCookie.ctxcsn );
				if ( ber_bvarray_dup_x( &si->si_syncCookie.ctxcsn,
					si->si_cookieState->cs_vals, NULL )) {
					rc = LDAP_NO_MEMORY;
					goto done;
				}
				si->si_syncCookie.numcsns = si->si_cookieState->cs_num;
				si->si_syncCookie.sids = ch_malloc( si->si_cookieState->cs_num *
					sizeof(int) );
				for ( i=0; i<si->si_syncCookie.numcsns; i++ )
					si->si_syncCookie.sids[i] = si->si_cookieState->cs_sids[i];
			}
			ldap_pvt_thread_mutex_unlock( &si->si_cookieState->cs_mutex );
519
		}
Kurt Zeilenga's avatar
Kurt Zeilenga committed
520

Howard Chu's avatar
Howard Chu committed
521
		slap_compose_sync_cookie( NULL, &si->si_syncCookie.octet_str,
522
			si->si_syncCookie.ctxcsn, si->si_syncCookie.rid,
523
			si->si_syncCookie.sid );
524
	} else {
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
		AttributeName at[2];
		Attribute a = { slap_schema.si_ad_contextCSN };
		Entry e = {0};
		SlapReply rs = {0};
		int i, j, changed = 0;

		/* Look for contextCSN from syncprov overlay. If
		 * there's no overlay, this will be a no-op. That means
		 * this is a pure consumer, so local changes will not be
		 * allowed, and all changes will already be reflected in
		 * the cookieState.
		 */
		e.e_attrs = &a;
		e.e_name = si->si_wbe->be_suffix[0];
		e.e_nname = si->si_wbe->be_nsuffix[0];
		rs.sr_entry = &e;
		rs.sr_flags = REP_ENTRY_MODIFIABLE;
		at[0].an_name = a.a_desc->ad_cname;
		at[0].an_desc = a.a_desc;
		BER_BVZERO( &at[1].an_name );

546
		ldap_pvt_thread_mutex_lock( &si->si_cookieState->cs_mutex );
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
		rc = backend_operational( op, &rs );
		if ( rc == LDAP_SUCCESS && a.a_vals ) {
			int num;
			for (i=0; !BER_BVISNULL( &a.a_vals[i] ); i++) ;
			num = i;
			/* check for differences */
			if ( num != si->si_cookieState->cs_num ) {
				changed = 1;
			} else {
				for ( i=0; i<num; i++ ) {
					if ( ber_bvcmp( &a.a_vals[i],
						&si->si_cookieState->cs_vals[i] )) {
						changed =1;
						break;
					}
				}
			}
			if ( changed ) {
				ber_bvarray_free( si->si_cookieState->cs_vals );
				ch_free( si->si_cookieState->cs_sids );
				si->si_cookieState->cs_num = num;
				si->si_cookieState->cs_vals = a.a_vals;
				si->si_cookieState->cs_sids = slap_parse_csn_sids( a.a_vals,
Howard Chu's avatar
Howard Chu committed
570
					num, NULL );
571
572
573
574
575
576
577
578
579
580
				si->si_cookieState->cs_age++;
			} else {
				ber_bvarray_free( a.a_vals );
			}
			changed = 0;
		}
		/* See if the cookieState has changed due to anything outside
		 * this particular consumer. That includes other consumers in
		 * the same context, or local changes detected above.
		 */
581
582
583
584
585
586
587
588
589
		if ( si->si_cookieState->cs_num > 1 && si->si_cookieAge !=
			si->si_cookieState->cs_age ) {

			for (i=0; !BER_BVISNULL( &si->si_syncCookie.ctxcsn[i] ); i++) {
				/* bogus, just dup everything */
				if ( si->si_syncCookie.sids[i] == -1 ) {
					ber_bvarray_free( si->si_syncCookie.ctxcsn );
					ber_bvarray_dup_x( &si->si_syncCookie.ctxcsn,
						si->si_cookieState->cs_vals, NULL );
590
					changed = 1;
591
592
593
594
595
596
					break;
				}
				for (j=0; j<si->si_cookieState->cs_num; j++) {
					if ( si->si_syncCookie.sids[i] !=
						si->si_cookieState->cs_sids[j] )
						continue;
597
598
599
					if ( bvmatch( &si->si_syncCookie.ctxcsn[i],
						&si->si_cookieState->cs_vals[j] ))
						break;
600
601
					ber_bvreplace( &si->si_syncCookie.ctxcsn[i],
						&si->si_cookieState->cs_vals[j] );
602
					changed = 1;
603
604
605
					break;
				}
			}
606
607
608
609
610
611
			if ( changed ) {
				ch_free( si->si_syncCookie.octet_str.bv_val );
				slap_compose_sync_cookie( NULL, &si->si_syncCookie.octet_str,
					si->si_syncCookie.ctxcsn, si->si_syncCookie.rid,
					SLAP_SINGLE_SHADOW( si->si_be ) ? -1 : slap_serverID );
			}
612
613
		}
		ldap_pvt_thread_mutex_unlock( &si->si_cookieState->cs_mutex );
Jong Hyuk Choi's avatar
Jong Hyuk Choi committed
614
615
616
	}

	rc = ldap_sync_search( si, op->o_tmpmemctx );
617

618
	if( rc != LDAP_SUCCESS ) {
619
		Debug( LDAP_DEBUG_ANY, "do_syncrep1: %s "
620
			"ldap_search_ext: %s (%d)\n",
621
			si->si_ridtxt, ldap_err2string( rc ), rc );
622
623
624
625
626
	}

done:
	if ( rc ) {
		if ( si->si_ld ) {
627
			ldap_unbind_ext( si->si_ld, NULL, NULL );
628
629
630
631
632
633
634
			si->si_ld = NULL;
		}
	}

	return rc;
}

635
636
637
638
639
640
641
642
static int
compare_csns( struct sync_cookie *sc1, struct sync_cookie *sc2, int *which )
{
	int i, j, match = 0;
	const char *text;

	*which = 0;

643
644
645
646
647
	if ( sc1->numcsns < sc2->numcsns ) {
		*which = sc1->numcsns;
		return -1;
	}

648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
	for (i=0; !BER_BVISNULL( &sc1->ctxcsn[i] ); i++) {
		for (j=0; !BER_BVISNULL( &sc2->ctxcsn[j] ); j++) {
			if ( sc1->sids[i] != sc2->sids[j] )
				continue;
			value_match( &match, slap_schema.si_ad_entryCSN,
				slap_schema.si_ad_entryCSN->ad_type->sat_ordering,
				SLAP_MR_VALUE_OF_ATTRIBUTE_SYNTAX,
				&sc1->ctxcsn[i], &sc2->ctxcsn[i], &text );
			if ( match < 0 ) {
				*which = j;
				return match;
			}
			break;
		}
	}
	return match;
}

666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
static int
do_syncrep2(
	Operation *op,
	syncinfo_t *si )
{
	LDAPControl	**rctrls = NULL;
	LDAPControl	*rctrlp;

	BerElementBuffer berbuf;
	BerElement	*ber = (BerElement *)&berbuf;

	LDAPMessage	*res = NULL;
	LDAPMessage	*msg = NULL;

	char		*retoid = NULL;
	struct berval	*retdata = NULL;

	Entry		*entry = NULL;

	int		syncstate;
Kurt Zeilenga's avatar
Kurt Zeilenga committed
686
	struct berval	syncUUID = BER_BVNULL;
687
688
	struct sync_cookie	syncCookie = { NULL };
	struct sync_cookie	syncCookie_req = { NULL };
Kurt Zeilenga's avatar
Kurt Zeilenga committed
689
	struct berval		cookie = BER_BVNULL;
690

691
	int	rc, err;
692
693
694
695
696
	ber_len_t	len;

	struct berval	*psub;
	Modifications	*modlist = NULL;

697
	int				match, m;
698
699
700
701

	struct timeval *tout_p = NULL;
	struct timeval tout = { 0, 0 };

702
703
	int		refreshDeletes = 0;
	int		refreshDone = 1;
704
	BerVarray syncUUIDs = NULL;
705
706
	ber_tag_t si_tag;

707
	if ( slapd_shutdown ) {
708
		rc = -2;
709
		goto done;
710
711
	}

712
713
714
	ber_init2( ber, NULL, LBER_USE_DER );
	ber_set_option( ber, LBER_OPT_BER_MEMCTX, &op->o_tmpmemctx );

715
	Debug( LDAP_DEBUG_TRACE, "=>do_syncrep2 %s\n", si->si_ridtxt, 0, 0 );
716
717
718

	psub = &si->si_be->be_nsuffix[0];

Jong Hyuk Choi's avatar
Jong Hyuk Choi committed
719
	slap_dup_sync_cookie( &syncCookie_req, &si->si_syncCookie );
720

Kurt Zeilenga's avatar
Kurt Zeilenga committed
721
	if ( abs(si->si_type) == LDAP_SYNC_REFRESH_AND_PERSIST ) {
722
723
724
725
726
		tout_p = &tout;
	} else {
		tout_p = NULL;
	}

727
728
	while ( ( rc = ldap_result( si->si_ld, LDAP_RES_ANY, LDAP_MSG_ONE,
		tout_p, &res ) ) > 0 )
729
	{
730
		if ( slapd_shutdown ) {
731
732
			rc = -2;
			goto done;
733
		}
734
		for( msg = ldap_first_message( si->si_ld, res );
Kurt Zeilenga's avatar
Kurt Zeilenga committed
735
736
			msg != NULL;
			msg = ldap_next_message( si->si_ld, msg ) )
737
		{
738
739
740
741
			if ( slapd_shutdown ) {
				rc = -2;
				goto done;
			}
742
743
			switch( ldap_msgtype( msg ) ) {
			case LDAP_RES_SEARCH_ENTRY:
744
				ldap_get_entry_controls( si->si_ld, msg, &rctrls );
Howard Chu's avatar
Howard Chu committed
745
				/* we can't work without the control */
746
747
748
749
750
751
752
753
				rctrlp = NULL;
				if ( rctrls ) {
					/* NOTE: make sure we use the right one;
					 * a better approach would be to run thru
					 * the whole list and take care of all */
					rctrlp = ldap_find_control( LDAP_CONTROL_SYNC_STATE, rctrls );
				}
				if ( rctrlp == NULL ) {
754
					Debug( LDAP_DEBUG_ANY, "do_syncrep2: %s "
755
						"got search entry without "
756
						"Sync State control\n", si->si_ridtxt, 0, 0 );
Howard Chu's avatar
Howard Chu committed
757
758
759
760
					rc = -1;
					goto done;
				}
				ber_init2( ber, &rctrlp->ldctl_value, LBER_USE_DER );
Kurt Zeilenga's avatar
Kurt Zeilenga committed
761
				ber_scanf( ber, "{em" /*"}"*/, &syncstate, &syncUUID );
762
763
764
				/* FIXME: what if syncUUID is NULL or empty?
				 * (happens with back-sql...) */
				if ( BER_BVISEMPTY( &syncUUID ) ) {
765
766
					Debug( LDAP_DEBUG_ANY, "do_syncrep2: %s "
						"got empty syncUUID\n", si->si_ridtxt, 0, 0 );
Pierangelo Masarati's avatar
Pierangelo Masarati committed
767
					ldap_controls_free( rctrls );
768
769
770
					rc = -1;
					goto done;
				}
771
				if ( ber_peek_tag( ber, &len ) == LDAP_TAG_SYNC_COOKIE ) {
Kurt Zeilenga's avatar
Kurt Zeilenga committed
772
					ber_scanf( ber, /*"{"*/ "m}", &cookie );
773
					if ( !BER_BVISNULL( &cookie ) ) {
774
775
						ch_free( syncCookie.octet_str.bv_val );
						ber_dupbv( &syncCookie.octet_str, &cookie );
Jong Hyuk Choi's avatar
Jong Hyuk Choi committed
776
					}
777
					if ( !BER_BVISNULL( &syncCookie.octet_str ) )
778
					{
Howard Chu's avatar
Howard Chu committed
779
						slap_parse_sync_cookie( &syncCookie, NULL );
780
					}
781
				}
782
				rc = 0;
783
				if ( si->si_syncdata && si->si_logstate == SYNCLOG_LOGGING ) {
784
					modlist = NULL;
785
					if ( ( rc = syncrepl_message_to_op( si, op, msg ) ) == LDAP_SUCCESS &&
786
						syncCookie.ctxcsn )
787
					{
788
						rc = syncrepl_updateCookie( si, op, psub, &syncCookie );
789
					}
790
791
792
793
				} else if ( ( rc = syncrepl_message_to_entry( si, op, msg,
					&modlist, &entry, syncstate ) ) == LDAP_SUCCESS )
				{
					if ( ( rc = syncrepl_entry( si, op, entry, &modlist,
794
						syncstate, &syncUUID ) ) == LDAP_SUCCESS &&
795
						syncCookie.ctxcsn )
796
					{
797
						rc = syncrepl_updateCookie( si, op, psub, &syncCookie );
798
					}
799
				}
Howard Chu's avatar
Howard Chu committed
800
				ldap_controls_free( rctrls );
Jong Hyuk Choi's avatar
Jong Hyuk Choi committed
801
				if ( modlist ) {
802
					slap_mods_free( modlist, 1 );
Jong Hyuk Choi's avatar
Jong Hyuk Choi committed
803
				}
804
805
				if ( rc )
					goto done;
806
807
808
809
				break;

			case LDAP_RES_SEARCH_REFERENCE:
				Debug( LDAP_DEBUG_ANY,
810
811
					"do_syncrep2: %s reference received error\n",
					si->si_ridtxt, 0, 0 );
812
813
814
				break;

			case LDAP_RES_SEARCH_RESULT:
Jong Hyuk Choi's avatar
Jong Hyuk Choi committed
815
				Debug( LDAP_DEBUG_SYNC,
816
817
					"do_syncrep2: %s LDAP_RES_SEARCH_RESULT\n",
					si->si_ridtxt, 0, 0 );
818
				ldap_parse_result( si->si_ld, msg, &err, NULL, NULL, NULL,
819
					&rctrls, 0 );
820
821
822
823
824
825
#ifdef LDAP_X_SYNC_REFRESH_REQUIRED
				if ( err == LDAP_X_SYNC_REFRESH_REQUIRED ) {
					/* map old result code to registered code */
					err = LDAP_SYNC_REFRESH_REQUIRED;
				}
#endif
826
827
828
829
830
				if ( err == LDAP_SYNC_REFRESH_REQUIRED ) {
					if ( si->si_logstate == SYNCLOG_LOGGING ) {
						si->si_logstate = SYNCLOG_FALLBACK;
					}
					rc = err;
Howard Chu's avatar
Howard Chu committed
831
					goto done;
832
				}
833
834
				if ( rctrls ) {
					rctrlp = *rctrls;
835
					ber_init2( ber, &rctrlp->ldctl_value, LBER_USE_DER );
Jong Hyuk Choi's avatar
Jong Hyuk Choi committed
836

837
					ber_scanf( ber, "{" /*"}"*/);
Kurt Zeilenga's avatar
Kurt Zeilenga committed
838
					if ( ber_peek_tag( ber, &len ) == LDAP_TAG_SYNC_COOKIE ) {
Jong Hyuk Choi's avatar
Jong Hyuk Choi committed
839
						ber_scanf( ber, "m", &cookie );
840
						if ( !BER_BVISNULL( &cookie ) ) {
841
842
							ch_free( syncCookie.octet_str.bv_val );
							ber_dupbv( &syncCookie.octet_str, &cookie);
Jong Hyuk Choi's avatar
Jong Hyuk Choi committed
843
						}
844
						if ( !BER_BVISNULL( &syncCookie.octet_str ) )
Kurt Zeilenga's avatar
Kurt Zeilenga committed
845
						{
Howard Chu's avatar
Howard Chu committed
846
							slap_parse_sync_cookie( &syncCookie, NULL );
Kurt Zeilenga's avatar
Kurt Zeilenga committed
847
						}
848
					}
849
850
851
852
					if ( ber_peek_tag( ber, &len ) == LDAP_TAG_REFRESHDELETES )
					{
						ber_scanf( ber, "b", &refreshDeletes );
					}
Kurt Zeilenga's avatar
Kurt Zeilenga committed
853
					ber_scanf( ber, /*"{"*/ "}" );
854
				}
Howard Chu's avatar
Howard Chu committed
855
856
857
				if ( !syncCookie.ctxcsn ) {
					match = 1;
				} else if ( !syncCookie_req.ctxcsn ) {
Jong Hyuk Choi's avatar
Jong Hyuk Choi committed
858
					match = -1;
Howard Chu's avatar
Howard Chu committed
859
					m = 0;
Jong Hyuk Choi's avatar
Jong Hyuk Choi committed
860
				} else {
861
					match = compare_csns( &syncCookie_req, &syncCookie, &m );
Jong Hyuk Choi's avatar
Jong Hyuk Choi committed
862
				}
863
864
865
866
				if ( rctrls ) {
					ldap_controls_free( rctrls );
				}
				if (si->si_type != LDAP_SYNC_REFRESH_AND_PERSIST) {
Jong Hyuk Choi's avatar
Jong Hyuk Choi committed
867
					/* FIXME : different error behaviors according to
868
869
870
					 *	1) err code : LDAP_BUSY ...
					 *	2) on err policy : stop service, stop sync, retry
					 */
871
					if ( refreshDeletes == 0 && match < 0 &&
Kurt Zeilenga's avatar
Kurt Zeilenga committed
872
873
						err == LDAP_SUCCESS )
					{
874
875
						syncrepl_del_nonpresent( op, si, NULL,
							&syncCookie.ctxcsn[m] );
876
877
878
					} else {
						avl_free( si->si_presentlist, avl_ber_bvfree );
						si->si_presentlist = NULL;
Jong Hyuk Choi's avatar
Jong Hyuk Choi committed
879
					}
880
				}
881
				if ( syncCookie.ctxcsn && match < 0 && err == LDAP_SUCCESS )
882
883
884
				{
					rc = syncrepl_updateCookie( si, op, psub, &syncCookie );
				}
885
886
887
888
889
890
891
				if ( err == LDAP_SUCCESS
					&& si->si_logstate == SYNCLOG_FALLBACK ) {
					si->si_logstate = SYNCLOG_LOGGING;
					rc = LDAP_SYNC_REFRESH_REQUIRED;
				} else {
					rc = -2;
				}
892
				goto done;
893
894
				break;

895
			case LDAP_RES_INTERMEDIATE:
896
				rc = ldap_parse_intermediate( si->si_ld, msg,
Kurt Zeilenga's avatar
Kurt Zeilenga committed
897
898
					&retoid, &retdata, NULL, 0 );
				if ( !rc && !strcmp( retoid, LDAP_SYNC_INFO ) ) {
899
					ber_init2( ber, retdata, LBER_USE_DER );
900

901
					switch ( si_tag = ber_peek_tag( ber, &len ) ) {
902
903
					ber_tag_t tag;
					case LDAP_TAG_SYNC_NEW_COOKIE:
Jong Hyuk Choi's avatar
Jong Hyuk Choi committed
904
						Debug( LDAP_DEBUG_SYNC,
905
906
							"do_syncrep2: %s %s - %s\n", 
							si->si_ridtxt,
Jong Hyuk Choi's avatar
Jong Hyuk Choi committed
907
							"LDAP_RES_INTERMEDIATE", 
908
							"NEW_COOKIE" );
909
910
911
912
						ber_scanf( ber, "tm", &tag, &cookie );
						break;
					case LDAP_TAG_SYNC_REFRESH_DELETE:
					case LDAP_TAG_SYNC_REFRESH_PRESENT:
Jong Hyuk Choi's avatar
Jong Hyuk Choi committed
913
						Debug( LDAP_DEBUG_SYNC,
914
915
							"do_syncrep2: %s %s - %s\n", 
							si->si_ridtxt,
Jong Hyuk Choi's avatar
Jong Hyuk Choi committed
916
917
							"LDAP_RES_INTERMEDIATE", 
							si_tag == LDAP_TAG_SYNC_REFRESH_PRESENT ?
918
							"REFRESH_PRESENT" : "REFRESH_DELETE" );
Howard Chu's avatar
Howard Chu committed
919
						if ( si_tag == LDAP_TAG_SYNC_REFRESH_DELETE ) {
Howard Chu's avatar
Howard Chu committed
920
921
922
923
							si->si_refreshDelete = 1;
						} else {
							si->si_refreshPresent = 1;
						}
Kurt Zeilenga's avatar
Kurt Zeilenga committed
924
						ber_scanf( ber, "t{" /*"}"*/, &tag );
925
926
						if ( ber_peek_tag( ber, &len ) == LDAP_TAG_SYNC_COOKIE )
						{
927
							ber_scanf( ber, "m", &cookie );
928
							if ( !BER_BVISNULL( &cookie ) ) {
929
930
								ch_free( syncCookie.octet_str.bv_val );
								ber_dupbv( &syncCookie.octet_str, &cookie );
931
							}
932
							if ( !BER_BVISNULL( &syncCookie.octet_str ) )
Kurt Zeilenga's avatar
Kurt Zeilenga committed
933
							{
Howard Chu's avatar
Howard Chu committed
934
								slap_parse_sync_cookie( &syncCookie, NULL );
Kurt Zeilenga's avatar
Kurt Zeilenga committed
935
							}
Jong Hyuk Choi's avatar
Jong Hyuk Choi committed
936
						}
937
						if ( ber_peek_tag( ber, &len ) ==
Kurt Zeilenga's avatar
Kurt Zeilenga committed
938
							LDAP_TAG_REFRESHDONE )
939
940
941
						{
							ber_scanf( ber, "b", &refreshDone );
						}
Kurt Zeilenga's avatar
Kurt Zeilenga committed
942
						ber_scanf( ber, /*"{"*/ "}" );
943
944
						break;
					case LDAP_TAG_SYNC_ID_SET:
Jong Hyuk Choi's avatar
Jong Hyuk Choi committed
945
						Debug( LDAP_DEBUG_SYNC,
946
947
							"do_syncrep2: %s %s - %s\n", 
							si->si_ridtxt,
Jong Hyuk Choi's avatar
Jong Hyuk Choi committed
948
							"LDAP_RES_INTERMEDIATE", 
949
							"SYNC_ID_SET" );
Kurt Zeilenga's avatar
Kurt Zeilenga committed
950
						ber_scanf( ber, "t{" /*"}"*/, &tag );
951
						if ( ber_peek_tag( ber, &len ) ==
Kurt Zeilenga's avatar
Kurt Zeilenga committed
952
953
							LDAP_TAG_SYNC_COOKIE )
						{
954
							ber_scanf( ber, "m", &cookie );
955
							if ( !BER_BVISNULL( &cookie ) ) {
956
957
								ch_free( syncCookie.octet_str.bv_val );
								ber_dupbv( &syncCookie.octet_str, &cookie );
958
							}
959
							if ( !BER_BVISNULL( &syncCookie.octet_str ) )
Kurt Zeilenga's avatar
Kurt Zeilenga committed
960
							{
Howard Chu's avatar
Howard Chu committed
961
								slap_parse_sync_cookie( &syncCookie, NULL );
962
								compare_csns( &syncCookie_req, &syncCookie, &m );
Kurt Zeilenga's avatar
Kurt Zeilenga committed
963