syncprov.c 85 KB
Newer Older
Howard Chu's avatar
Howard Chu committed
1
/* $OpenLDAP$ */
Howard Chu's avatar
Howard Chu committed
2
3
4
/* syncprov.c - syncrepl provider */
/* This work is part of OpenLDAP Software <http://www.openldap.org/>.
 *
5
 * Copyright 2004-2009 The OpenLDAP Foundation.
Howard Chu's avatar
Howard Chu committed
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted only as authorized by the OpenLDAP
 * Public License.
 *
 * A copy of this license is available in the file LICENSE in the
 * top-level directory of the distribution or, alternatively, at
 * <http://www.OpenLDAP.org/license.html>.
 */
/* ACKNOWLEDGEMENTS:
 * This work was initially developed by Howard Chu for inclusion in
 * OpenLDAP Software.
 */

#include "portable.h"

#ifdef SLAPD_OVER_SYNCPROV

Howard Chu's avatar
Howard Chu committed
25
26
#include <ac/string.h>
#include "lutil.h"
Howard Chu's avatar
Howard Chu committed
27
#include "slap.h"
Howard Chu's avatar
Howard Chu committed
28
#include "config.h"
29
#include "ldap_rq.h"
Howard Chu's avatar
Howard Chu committed
30

Quanah Gibson-Mount's avatar
Quanah Gibson-Mount committed
31
32
33
34
#ifdef LDAP_DEVEL
#define	CHECK_CSN	1
#endif

35
36
37
38
39
40
41
42
43
44
45
46
47
/* A modify request on a particular entry */
typedef struct modinst {
	struct modinst *mi_next;
	Operation *mi_op;
} modinst;

typedef struct modtarget {
	struct modinst *mt_mods;
	struct modinst *mt_tail;
	Operation *mt_op;
	ldap_pvt_thread_mutex_t mt_mutex;
} modtarget;

Howard Chu's avatar
Howard Chu committed
48
49
50
/* A queued result of a persistent search */
typedef struct syncres {
	struct syncres *s_next;
51
	struct berval s_dn;
Howard Chu's avatar
Howard Chu committed
52
53
	struct berval s_ndn;
	struct berval s_uuid;
54
55
56
	struct berval s_csn;
	char s_mode;
	char s_isreference;
Howard Chu's avatar
Howard Chu committed
57
58
} syncres;

Howard Chu's avatar
Howard Chu committed
59
60
61
62
63
64
/* Record of a persistent search */
typedef struct syncops {
	struct syncops *s_next;
	struct berval	s_base;		/* ndn of search base */
	ID		s_eid;		/* entryID of search base */
	Operation	*s_op;		/* search op */
65
	int		s_rid;
66
	int		s_sid;
Howard Chu's avatar
Howard Chu committed
67
	struct berval s_filterstr;
Howard Chu's avatar
Howard Chu committed
68
	int		s_flags;	/* search status */
Howard Chu's avatar
Howard Chu committed
69
70
71
72
#define	PS_IS_REFRESHING	0x01
#define	PS_IS_DETACHED		0x02
#define	PS_WROTE_BASE		0x04
#define	PS_FIND_BASE		0x08
73
#define	PS_FIX_FILTER		0x10
Quanah Gibson-Mount's avatar
Quanah Gibson-Mount committed
74
#define	PS_TASK_QUEUED		0x20
Howard Chu's avatar
Howard Chu committed
75

Howard Chu's avatar
Howard Chu committed
76
	int		s_inuse;	/* reference count */
Howard Chu's avatar
Howard Chu committed
77
78
79
	struct syncres *s_res;
	struct syncres *s_restail;
	ldap_pvt_thread_mutex_t	s_mutex;
Howard Chu's avatar
Howard Chu committed
80
81
} syncops;

82
83
84
85
86
87
/* A received sync control */
typedef struct sync_control {
	struct sync_cookie sr_state;
	int sr_rhint;
} sync_control;

88
89
90
#if 0 /* moved back to slap.h */
#define	o_sync	o_ctrlflag[slap_cids.sc_LDAPsync]
#endif
91
/* o_sync_mode uses data bits of o_sync */
92
#define	o_sync_mode	o_ctrlflag[slap_cids.sc_LDAPsync]
93
94
95
96
97
98

#define SLAP_SYNC_NONE					(LDAP_SYNC_NONE<<SLAP_CONTROL_SHIFT)
#define SLAP_SYNC_REFRESH				(LDAP_SYNC_REFRESH_ONLY<<SLAP_CONTROL_SHIFT)
#define SLAP_SYNC_PERSIST				(LDAP_SYNC_RESERVED<<SLAP_CONTROL_SHIFT)
#define SLAP_SYNC_REFRESH_AND_PERSIST	(LDAP_SYNC_REFRESH_AND_PERSIST<<SLAP_CONTROL_SHIFT)

Howard Chu's avatar
Howard Chu committed
99
100
101
102
103
104
/* Record of which searches matched at premodify step */
typedef struct syncmatches {
	struct syncmatches *sm_next;
	syncops *sm_op;
} syncmatches;

105
106
107
108
109
/* Session log data */
typedef struct slog_entry {
	struct slog_entry *se_next;
	struct berval se_uuid;
	struct berval se_csn;
110
	int	se_sid;
111
112
113
114
115
116
117
118
119
120
121
122
	ber_tag_t	se_tag;
} slog_entry;

typedef struct sessionlog {
	struct berval	sl_mincsn;
	int		sl_num;
	int		sl_size;
	slog_entry *sl_head;
	slog_entry *sl_tail;
	ldap_pvt_thread_mutex_t sl_mutex;
} sessionlog;

Howard Chu's avatar
Cleanup    
Howard Chu committed
123
/* The main state for this overlay */
Howard Chu's avatar
Howard Chu committed
124
125
typedef struct syncprov_info_t {
	syncops		*si_ops;
126
127
128
	BerVarray	si_ctxcsn;	/* ldapsync context */
	int		*si_sids;
	int		si_numcsns;
129
130
131
	int		si_chkops;	/* checkpointing info */
	int		si_chktime;
	int		si_numops;	/* number of ops since last checkpoint */
132
	int		si_nopres;	/* Skip present phase */
133
	int		si_usehint;	/* use reload hint */
134
	time_t	si_chklast;	/* time of last checkpoint */
135
	Avlnode	*si_mods;	/* entries being modified */
136
	sessionlog	*si_logs;
Howard Chu's avatar
Howard Chu committed
137
	ldap_pvt_thread_rdwr_t	si_csn_rwlock;
Howard Chu's avatar
Howard Chu committed
138
	ldap_pvt_thread_mutex_t	si_ops_mutex;
Howard Chu's avatar
Howard Chu committed
139
	ldap_pvt_thread_mutex_t	si_mods_mutex;
Quanah Gibson-Mount's avatar
Quanah Gibson-Mount committed
140
	ldap_pvt_thread_mutex_t	si_resp_mutex;
Howard Chu's avatar
Howard Chu committed
141
142
143
144
145
} syncprov_info_t;

typedef struct opcookie {
	slap_overinst *son;
	syncmatches *smatches;
Quanah Gibson-Mount's avatar
Quanah Gibson-Mount committed
146
	modtarget *smt;
147
148
149
150
	struct berval sdn;	/* DN of entry, for deletes */
	struct berval sndn;
	struct berval suuid;	/* UUID of entry */
	struct berval sctxcsn;
Quanah Gibson-Mount's avatar
Quanah Gibson-Mount committed
151
152
153
	short osid;	/* sid of op csn */
	short rsid;	/* sid of relay */
	short sreference;	/* Is the entry a reference? */
Howard Chu's avatar
Howard Chu committed
154
155
} opcookie;

156
typedef struct fbase_cookie {
157
158
159
160
	struct berval *fdn;	/* DN of a modified entry, for scope testing */
	syncops *fss;	/* persistent search we're testing against */
	int fbase;	/* if TRUE we found the search base and it's still valid */
	int fscope;	/* if TRUE then fdn is within the psearch scope */
161
} fbase_cookie;
Howard Chu's avatar
Howard Chu committed
162

163
static AttributeName csn_anlist[3];
Howard Chu's avatar
Howard Chu committed
164
165
static AttributeName uuid_anlist[2];

Howard Chu's avatar
Cleanup    
Howard Chu committed
166
/* Build a LDAPsync intermediate state control */
167
168
169
170
171
static int
syncprov_state_ctrl(
	Operation	*op,
	SlapReply	*rs,
	Entry		*e,
172
	int		entry_sync_state,
173
	LDAPControl	**ctrls,
174
175
176
	int		num_ctrls,
	int		send_cookie,
	struct berval	*cookie )
177
178
179
180
181
182
183
{
	Attribute* a;
	int ret;

	BerElementBuffer berbuf;
	BerElement *ber = (BerElement *)&berbuf;

184
	struct berval	entryuuid_bv = BER_BVNULL;
185
186
187
188

	ber_init2( ber, 0, LBER_USE_DER );
	ber_set_option( ber, LBER_OPT_BER_MEMCTX, &op->o_tmpmemctx );

Howard Chu's avatar
Howard Chu committed
189
	ctrls[num_ctrls] = op->o_tmpalloc( sizeof ( LDAPControl ), op->o_tmpmemctx );
190

191
192
193
194
195
	for ( a = e->e_attrs; a != NULL; a = a->a_next ) {
		AttributeDescription *desc = a->a_desc;
		if ( desc == slap_schema.si_ad_entryUUID ) {
			entryuuid_bv = a->a_nvals[0];
			break;
196
		}
197
	}
198

199
200
	/* FIXME: what if entryuuid is NULL or empty ? */

201
202
203
204
205
206
	if ( send_cookie && cookie ) {
		ber_printf( ber, "{eOON}",
			entry_sync_state, &entryuuid_bv, cookie );
	} else {
		ber_printf( ber, "{eON}",
			entry_sync_state, &entryuuid_bv );
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
	}

	ctrls[num_ctrls]->ldctl_oid = LDAP_CONTROL_SYNC_STATE;
	ctrls[num_ctrls]->ldctl_iscritical = (op->o_sync == SLAP_CONTROL_CRITICAL);
	ret = ber_flatten2( ber, &ctrls[num_ctrls]->ldctl_value, 1 );

	ber_free_buf( ber );

	if ( ret < 0 ) {
		Debug( LDAP_DEBUG_TRACE,
			"slap_build_sync_ctrl: ber_flatten2 failed\n",
			0, 0, 0 );
		send_ldap_error( op, rs, LDAP_OTHER, "internal error" );
		return ret;
	}

	return LDAP_SUCCESS;
}

Howard Chu's avatar
Cleanup    
Howard Chu committed
226
/* Build a LDAPsync final state control */
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
static int
syncprov_done_ctrl(
	Operation	*op,
	SlapReply	*rs,
	LDAPControl	**ctrls,
	int			num_ctrls,
	int			send_cookie,
	struct berval *cookie,
	int			refreshDeletes )
{
	int ret;
	BerElementBuffer berbuf;
	BerElement *ber = (BerElement *)&berbuf;

	ber_init2( ber, NULL, LBER_USE_DER );
	ber_set_option( ber, LBER_OPT_BER_MEMCTX, &op->o_tmpmemctx );

Howard Chu's avatar
Howard Chu committed
244
	ctrls[num_ctrls] = op->o_tmpalloc( sizeof ( LDAPControl ), op->o_tmpmemctx );
245
246
247
248
249
250
251
252

	ber_printf( ber, "{" );
	if ( send_cookie && cookie ) {
		ber_printf( ber, "O", cookie );
	}
	if ( refreshDeletes == LDAP_SYNC_REFRESH_DELETES ) {
		ber_printf( ber, "b", refreshDeletes );
	}
253
	ber_printf( ber, "N}" );
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271

	ctrls[num_ctrls]->ldctl_oid = LDAP_CONTROL_SYNC_DONE;
	ctrls[num_ctrls]->ldctl_iscritical = (op->o_sync == SLAP_CONTROL_CRITICAL);
	ret = ber_flatten2( ber, &ctrls[num_ctrls]->ldctl_value, 1 );

	ber_free_buf( ber );

	if ( ret < 0 ) {
		Debug( LDAP_DEBUG_TRACE,
			"syncprov_done_ctrl: ber_flatten2 failed\n",
			0, 0, 0 );
		send_ldap_error( op, rs, LDAP_OTHER, "internal error" );
		return ret;
	}

	return LDAP_SUCCESS;
}

Howard Chu's avatar
Howard Chu committed
272
static int
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
syncprov_sendinfo(
	Operation	*op,
	SlapReply	*rs,
	int			type,
	struct berval *cookie,
	int			refreshDone,
	BerVarray	syncUUIDs,
	int			refreshDeletes )
{
	BerElementBuffer berbuf;
	BerElement *ber = (BerElement *)&berbuf;
	struct berval rspdata;

	int ret;

	ber_init2( ber, NULL, LBER_USE_DER );
	ber_set_option( ber, LBER_OPT_BER_MEMCTX, &op->o_tmpmemctx );

	if ( type ) {
		switch ( type ) {
		case LDAP_TAG_SYNC_NEW_COOKIE:
			ber_printf( ber, "tO", type, cookie );
			break;
		case LDAP_TAG_SYNC_REFRESH_DELETE:
		case LDAP_TAG_SYNC_REFRESH_PRESENT:
			ber_printf( ber, "t{", type );
			if ( cookie ) {
				ber_printf( ber, "O", cookie );
			}
			if ( refreshDone == 0 ) {
				ber_printf( ber, "b", refreshDone );
			}
			ber_printf( ber, "N}" );
			break;
		case LDAP_TAG_SYNC_ID_SET:
			ber_printf( ber, "t{", type );
			if ( cookie ) {
				ber_printf( ber, "O", cookie );
			}
			if ( refreshDeletes == 1 ) {
				ber_printf( ber, "b", refreshDeletes );
			}
			ber_printf( ber, "[W]", syncUUIDs );
			ber_printf( ber, "N}" );
			break;
		default:
			Debug( LDAP_DEBUG_TRACE,
				"syncprov_sendinfo: invalid syncinfo type (%d)\n",
				type, 0, 0 );
			return LDAP_OTHER;
		}
	}

	ret = ber_flatten2( ber, &rspdata, 0 );

	if ( ret < 0 ) {
		Debug( LDAP_DEBUG_TRACE,
			"syncprov_sendinfo: ber_flatten2 failed\n",
			0, 0, 0 );
		send_ldap_error( op, rs, LDAP_OTHER, "internal error" );
		return ret;
	}

Howard Chu's avatar
Cleanup    
Howard Chu committed
336
	rs->sr_rspoid = LDAP_SYNC_INFO;
337
338
339
340
341
342
343
	rs->sr_rspdata = &rspdata;
	send_ldap_intermediate( op, rs );
	rs->sr_rspdata = NULL;
	ber_free_buf( ber );

	return LDAP_SUCCESS;
}
344

Howard Chu's avatar
Cleanup    
Howard Chu committed
345
/* Find a modtarget in an AVL tree */
346
347
348
349
350
351
352
353
354
355
356
357
358
static int
sp_avl_cmp( const void *c1, const void *c2 )
{
	const modtarget *m1, *m2;
	int rc;

	m1 = c1; m2 = c2;
	rc = m1->mt_op->o_req_ndn.bv_len - m2->mt_op->o_req_ndn.bv_len;

	if ( rc ) return rc;
	return ber_bvcmp( &m1->mt_op->o_req_ndn, &m2->mt_op->o_req_ndn );
}

359
360
361
362
363
364
365
366
367
/* syncprov_findbase:
 *   finds the true DN of the base of a search (with alias dereferencing) and
 * checks to make sure the base entry doesn't get replaced with a different
 * entry (e.g., swapping trees via ModDN, or retargeting an alias). If a
 * change is detected, any persistent search on this base must be terminated /
 * reloaded.
 *   On the first call, we just save the DN and entryID. On subsequent calls
 * we compare the DN and entryID with the saved values.
 */
Howard Chu's avatar
Howard Chu committed
368
369
370
371
372
373
static int
findbase_cb( Operation *op, SlapReply *rs )
{
	slap_callback *sc = op->o_callback;

	if ( rs->sr_type == REP_SEARCH && rs->sr_err == LDAP_SUCCESS ) {
374
375
376
377
378
379
		fbase_cookie *fc = sc->sc_private;

		/* If no entryID, we're looking for the first time.
		 * Just store whatever we got.
		 */
		if ( fc->fss->s_eid == NOID ) {
Howard Chu's avatar
Howard Chu committed
380
			fc->fbase = 2;
381
382
			fc->fss->s_eid = rs->sr_entry->e_id;
			ber_dupbv( &fc->fss->s_base, &rs->sr_entry->e_nname );
383

384
		} else if ( rs->sr_entry->e_id == fc->fss->s_eid &&
Howard Chu's avatar
Howard Chu committed
385
			dn_match( &rs->sr_entry->e_nname, &fc->fss->s_base )) {
386

Howard Chu's avatar
Howard Chu committed
387
		/* OK, the DN is the same and the entryID is the same. */
Howard Chu's avatar
Howard Chu committed
388
389
			fc->fbase = 1;
		}
Howard Chu's avatar
Howard Chu committed
390
	}
391
392
393
	if ( rs->sr_err != LDAP_SUCCESS ) {
		Debug( LDAP_DEBUG_ANY, "findbase failed! %d\n", rs->sr_err,0,0 );
	}
Howard Chu's avatar
Howard Chu committed
394
395
396
	return LDAP_SUCCESS;
}

Howard Chu's avatar
Howard Chu committed
397
398
399
static Filter generic_filter = { LDAP_FILTER_PRESENT, { 0 }, NULL };
static struct berval generic_filterstr = BER_BVC("(objectclass=*)");

Howard Chu's avatar
Howard Chu committed
400
static int
401
syncprov_findbase( Operation *op, fbase_cookie *fc )
Howard Chu's avatar
Howard Chu committed
402
{
Howard Chu's avatar
Howard Chu committed
403
404
405
	/* Use basic parameters from syncrepl search, but use
	 * current op's threadctx / tmpmemctx
	 */
Howard Chu's avatar
Howard Chu committed
406
407
408
409
410
411
	ldap_pvt_thread_mutex_lock( &fc->fss->s_mutex );
	if ( fc->fss->s_flags & PS_FIND_BASE ) {
		slap_callback cb = {0};
		Operation fop;
		SlapReply frs = { REP_RESULT };
		int rc;
Howard Chu's avatar
Howard Chu committed
412

Howard Chu's avatar
Howard Chu committed
413
414
		fc->fss->s_flags ^= PS_FIND_BASE;
		ldap_pvt_thread_mutex_unlock( &fc->fss->s_mutex );
Howard Chu's avatar
Howard Chu committed
415

Howard Chu's avatar
Howard Chu committed
416
		fop = *fc->fss->s_op;
Howard Chu's avatar
Howard Chu committed
417

Quanah Gibson-Mount's avatar
Quanah Gibson-Mount committed
418
		fop.o_bd = fop.o_bd->bd_self;
Howard Chu's avatar
Howard Chu committed
419
420
421
		fop.o_hdr = op->o_hdr;
		fop.o_time = op->o_time;
		fop.o_tincr = op->o_tincr;
Howard Chu's avatar
Howard Chu committed
422

Howard Chu's avatar
Howard Chu committed
423
424
		cb.sc_response = findbase_cb;
		cb.sc_private = fc;
Howard Chu's avatar
Howard Chu committed
425

Howard Chu's avatar
Howard Chu committed
426
427
428
429
430
431
432
433
434
435
		fop.o_sync_mode = 0;	/* turn off sync mode */
		fop.o_managedsait = SLAP_CONTROL_CRITICAL;
		fop.o_callback = &cb;
		fop.o_tag = LDAP_REQ_SEARCH;
		fop.ors_scope = LDAP_SCOPE_BASE;
		fop.ors_limit = NULL;
		fop.ors_slimit = 1;
		fop.ors_tlimit = SLAP_NO_LIMIT;
		fop.ors_attrs = slap_anlist_no_attrs;
		fop.ors_attrsonly = 1;
Howard Chu's avatar
Howard Chu committed
436
437
		fop.ors_filter = &generic_filter;
		fop.ors_filterstr = generic_filterstr;
Howard Chu's avatar
Howard Chu committed
438

Quanah Gibson-Mount's avatar
Quanah Gibson-Mount committed
439
		rc = fop.o_bd->be_search( &fop, &frs );
Howard Chu's avatar
Howard Chu committed
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
	} else {
		ldap_pvt_thread_mutex_unlock( &fc->fss->s_mutex );
		fc->fbase = 1;
	}

	/* After the first call, see if the fdn resides in the scope */
	if ( fc->fbase == 1 ) {
		switch ( fc->fss->s_op->ors_scope ) {
		case LDAP_SCOPE_BASE:
			fc->fscope = dn_match( fc->fdn, &fc->fss->s_base );
			break;
		case LDAP_SCOPE_ONELEVEL: {
			struct berval pdn;
			dnParent( fc->fdn, &pdn );
			fc->fscope = dn_match( &pdn, &fc->fss->s_base );
			break; }
		case LDAP_SCOPE_SUBTREE:
			fc->fscope = dnIsSuffix( fc->fdn, &fc->fss->s_base );
			break;
		case LDAP_SCOPE_SUBORDINATE:
			fc->fscope = dnIsSuffix( fc->fdn, &fc->fss->s_base ) &&
				!dn_match( fc->fdn, &fc->fss->s_base );
			break;
		}
	}

	if ( fc->fbase )
		return LDAP_SUCCESS;
Howard Chu's avatar
Howard Chu committed
468
469
470
471
472
473

	/* If entryID has changed, then the base of this search has
	 * changed. Invalidate the psearch.
	 */
	return LDAP_NO_SUCH_OBJECT;
}
Howard Chu's avatar
Howard Chu committed
474

475
/* syncprov_findcsn:
476
 *   This function has three different purposes, but they all use a search
477
 * that filters on entryCSN so they're combined here.
478
479
480
481
482
 * 1: at startup time, after a contextCSN has been read from the database,
 * we search for all entries with CSN >= contextCSN in case the contextCSN
 * was not checkpointed at the previous shutdown.
 *
 * 2: when the current contextCSN is known and we have a sync cookie, we search
Howard Chu's avatar
Howard Chu committed
483
484
 * for one entry with CSN = the cookie CSN. If not found, try <= cookie CSN.
 * If an entry is found, the cookie CSN is valid, otherwise it is stale.
485
 *
486
 * 3: during a refresh phase, we search for all entries with CSN <= the cookie
487
488
489
 * CSN, and generate Present records for them. We always collect this result
 * in SyncID sets, even if there's only one match.
 */
490
491
492
493
494
typedef enum find_csn_t {
	FIND_MAXCSN	= 1,
	FIND_CSN	= 2,
	FIND_PRESENT	= 3
} find_csn_t;
495
496
497
498
499
500
501
502
503

static int
findmax_cb( Operation *op, SlapReply *rs )
{
	if ( rs->sr_type == REP_SEARCH && rs->sr_err == LDAP_SUCCESS ) {
		struct berval *maxcsn = op->o_callback->sc_private;
		Attribute *a = attr_find( rs->sr_entry->e_attrs,
			slap_schema.si_ad_entryCSN );

Quanah Gibson-Mount's avatar
Quanah Gibson-Mount committed
504
505
		if ( a && ber_bvcmp( &a->a_vals[0], maxcsn ) > 0 &&
			slap_parse_csn_sid( &a->a_vals[0] ) == slap_serverID ) {
506
507
508
509
510
511
			maxcsn->bv_len = a->a_vals[0].bv_len;
			strcpy( maxcsn->bv_val, a->a_vals[0].bv_val );
		}
	}
	return LDAP_SUCCESS;
}
Howard Chu's avatar
Howard Chu committed
512
513
514
515
516
517

static int
findcsn_cb( Operation *op, SlapReply *rs )
{
	slap_callback *sc = op->o_callback;

Howard Chu's avatar
Howard Chu committed
518
519
520
521
522
	/* We just want to know that at least one exists, so it's OK if
	 * we exceed the unchecked limit.
	 */
	if ( rs->sr_err == LDAP_ADMINLIMIT_EXCEEDED ||
		(rs->sr_type == REP_SEARCH && rs->sr_err == LDAP_SUCCESS )) {
523
		sc->sc_private = (void *)1;
Howard Chu's avatar
Howard Chu committed
524
525
526
527
	}
	return LDAP_SUCCESS;
}

528
529
/* Build a list of entryUUIDs for sending in a SyncID set */

530
531
#define UUID_LEN	16

Howard Chu's avatar
Howard Chu committed
532
533
534
typedef struct fpres_cookie {
	int num;
	BerVarray uuids;
535
	char *last;
Howard Chu's avatar
Howard Chu committed
536
537
538
539
540
541
542
} fpres_cookie;

static int
findpres_cb( Operation *op, SlapReply *rs )
{
	slap_callback *sc = op->o_callback;
	fpres_cookie *pc = sc->sc_private;
543
	Attribute *a;
Howard Chu's avatar
Howard Chu committed
544
	int ret = SLAP_CB_CONTINUE;
Howard Chu's avatar
Howard Chu committed
545

546
547
548
	switch ( rs->sr_type ) {
	case REP_SEARCH:
		a = attr_find( rs->sr_entry->e_attrs, slap_schema.si_ad_entryUUID );
549
550
551
552
		if ( a ) {
			pc->uuids[pc->num].bv_val = pc->last;
			AC_MEMCPY( pc->uuids[pc->num].bv_val, a->a_nvals[0].bv_val,
				pc->uuids[pc->num].bv_len );
Howard Chu's avatar
Howard Chu committed
553
			pc->num++;
554
555
			pc->last = pc->uuids[pc->num].bv_val;
			pc->uuids[pc->num].bv_val = NULL;
Howard Chu's avatar
Howard Chu committed
556
		}
557
		ret = LDAP_SUCCESS;
558
559
560
561
		if ( pc->num != SLAP_SYNCUUID_SET_SIZE )
			break;
		/* FALLTHRU */
	case REP_RESULT:
Howard Chu's avatar
Howard Chu committed
562
563
		ret = rs->sr_err;
		if ( pc->num ) {
564
			ret = syncprov_sendinfo( op, rs, LDAP_TAG_SYNC_ID_SET, NULL,
Howard Chu's avatar
Howard Chu committed
565
				0, pc->uuids, 0 );
566
			pc->uuids[pc->num].bv_val = pc->last;
Howard Chu's avatar
Howard Chu committed
567
			pc->num = 0;
568
			pc->last = pc->uuids[0].bv_val;
Howard Chu's avatar
Howard Chu committed
569
		}
570
571
572
		break;
	default:
		break;
Howard Chu's avatar
Howard Chu committed
573
574
575
576
577
	}
	return ret;
}

static int
578
syncprov_findcsn( Operation *op, find_csn_t mode )
Howard Chu's avatar
Howard Chu committed
579
580
581
582
583
584
585
586
587
{
	slap_overinst		*on = (slap_overinst *)op->o_bd->bd_info;
	syncprov_info_t		*si = on->on_bi.bi_private;

	slap_callback cb = {0};
	Operation fop;
	SlapReply frs = { REP_RESULT };
	char buf[LDAP_LUTIL_CSNSTR_BUFSIZE + STRLENOF("(entryCSN<=)")];
	char cbuf[LDAP_LUTIL_CSNSTR_BUFSIZE];
588
	struct berval maxcsn;
589
	Filter cf;
Ralf Haferkamp's avatar
Ralf Haferkamp committed
590
	AttributeAssertion eq = ATTRIBUTEASSERTION_INIT;
Howard Chu's avatar
Howard Chu committed
591
	fpres_cookie pcookie;
Pierangelo Masarati's avatar
Pierangelo Masarati committed
592
	sync_control *srs = NULL;
593
594
	struct slap_limits_set fc_limits;
	int i, rc = LDAP_SUCCESS, findcsn_retry = 1;
Quanah Gibson-Mount's avatar
Quanah Gibson-Mount committed
595
	int maxid;
596
597
598

	if ( mode != FIND_MAXCSN ) {
		srs = op->o_controls[slap_cids.sc_LDAPsync];
Howard Chu's avatar
Howard Chu committed
599
600
601
	}

	fop = *op;
602
	fop.o_sync_mode &= SLAP_CONTROL_MASK;	/* turn off sync_mode */
603
604
	/* We want pure entries, not referrals */
	fop.o_managedsait = SLAP_CONTROL_CRITICAL;
Howard Chu's avatar
Howard Chu committed
605

606
607
	cf.f_ava = &eq;
	cf.f_av_desc = slap_schema.si_ad_entryCSN;
608
	BER_BVZERO( &cf.f_av_value );
609
610
	cf.f_next = NULL;

611
612
613
614
	fop.o_callback = &cb;
	fop.ors_limit = NULL;
	fop.ors_tlimit = SLAP_NO_LIMIT;
	fop.ors_filter = &cf;
615
	fop.ors_filterstr.bv_val = buf;
616

Howard Chu's avatar
Howard Chu committed
617
again:
618
619
620
	switch( mode ) {
	case FIND_MAXCSN:
		cf.f_choice = LDAP_FILTER_GE;
Quanah Gibson-Mount's avatar
Quanah Gibson-Mount committed
621
622
623
		/* If there are multiple CSNs, use the one with our serverID */
		for ( i=0; i<si->si_numcsns; i++) {
			if ( slap_serverID == si->si_sids[i] ) {
624
				maxid = i;
Quanah Gibson-Mount's avatar
Quanah Gibson-Mount committed
625
				break;
626
627
			}
		}
Quanah Gibson-Mount's avatar
Quanah Gibson-Mount committed
628
629
630
631
632
633
634
		if ( i == si->si_numcsns ) {
			/* No match: this is multimaster, and none of the content in the DB
			 * originated locally. Treat like no CSN.
			 */
			return LDAP_NO_SUCH_OBJECT;
		}
		cf.f_av_value = si->si_ctxcsn[maxid];
635
636
		fop.ors_filterstr.bv_len = snprintf( buf, sizeof( buf ),
			"(entryCSN>=%s)", cf.f_av_value.bv_val );
637
		if ( fop.ors_filterstr.bv_len >= sizeof( buf ) ) {
638
639
			return LDAP_OTHER;
		}
640
641
642
643
644
		fop.ors_attrsonly = 0;
		fop.ors_attrs = csn_anlist;
		fop.ors_slimit = SLAP_NO_LIMIT;
		cb.sc_private = &maxcsn;
		cb.sc_response = findmax_cb;
645
		strcpy( cbuf, cf.f_av_value.bv_val );
646
		maxcsn.bv_val = cbuf;
647
		maxcsn.bv_len = cf.f_av_value.bv_len;
648
649
		break;
	case FIND_CSN:
650
651
652
653
654
655
656
657
658
659
		if ( BER_BVISEMPTY( &cf.f_av_value )) {
			cf.f_av_value = srs->sr_state.ctxcsn[0];
			/* If there are multiple CSNs, use the smallest */
			for ( i=1; i<srs->sr_state.numcsns; i++ ) {
				if ( ber_bvcmp( &cf.f_av_value, &srs->sr_state.ctxcsn[i] )
					> 0 ) {
					cf.f_av_value = srs->sr_state.ctxcsn[i];
				}
			}
		}
Howard Chu's avatar
Howard Chu committed
660
661
662
		/* Look for exact match the first time */
		if ( findcsn_retry ) {
			cf.f_choice = LDAP_FILTER_EQUALITY;
663
664
			fop.ors_filterstr.bv_len = snprintf( buf, sizeof( buf ),
				"(entryCSN=%s)", cf.f_av_value.bv_val );
Howard Chu's avatar
Howard Chu committed
665
666
667
		/* On retry, look for <= */
		} else {
			cf.f_choice = LDAP_FILTER_LE;
668
			fop.ors_limit = &fc_limits;
669
			memset( &fc_limits, 0, sizeof( fc_limits ));
670
			fc_limits.lms_s_unchecked = 1;
671
672
673
			fop.ors_filterstr.bv_len = snprintf( buf, sizeof( buf ),
				"(entryCSN<=%s)", cf.f_av_value.bv_val );
		}
674
		if ( fop.ors_filterstr.bv_len >= sizeof( buf ) ) {
675
			return LDAP_OTHER;
Howard Chu's avatar
Howard Chu committed
676
		}
677
678
679
680
		fop.ors_attrsonly = 1;
		fop.ors_attrs = slap_anlist_no_attrs;
		fop.ors_slimit = 1;
		cb.sc_private = NULL;
Howard Chu's avatar
Howard Chu committed
681
		cb.sc_response = findcsn_cb;
682
683
		break;
	case FIND_PRESENT:
684
685
		fop.ors_filter = op->ors_filter;
		fop.ors_filterstr = op->ors_filterstr;
Howard Chu's avatar
Howard Chu committed
686
687
688
		fop.ors_attrsonly = 0;
		fop.ors_attrs = uuid_anlist;
		fop.ors_slimit = SLAP_NO_LIMIT;
Howard Chu's avatar
Howard Chu committed
689
		cb.sc_private = &pcookie;
Howard Chu's avatar
Howard Chu committed
690
691
		cb.sc_response = findpres_cb;
		pcookie.num = 0;
692
693
694
695
696
697
698
699
700
701
702
703

		/* preallocate storage for a full set */
		pcookie.uuids = op->o_tmpalloc( (SLAP_SYNCUUID_SET_SIZE+1) *
			sizeof(struct berval) + SLAP_SYNCUUID_SET_SIZE * UUID_LEN,
			op->o_tmpmemctx );
		pcookie.last = (char *)(pcookie.uuids + SLAP_SYNCUUID_SET_SIZE+1);
		pcookie.uuids[0].bv_val = pcookie.last;
		pcookie.uuids[0].bv_len = UUID_LEN;
		for (i=1; i<SLAP_SYNCUUID_SET_SIZE; i++) {
			pcookie.uuids[i].bv_val = pcookie.uuids[i-1].bv_val + UUID_LEN;
			pcookie.uuids[i].bv_len = UUID_LEN;
		}
704
		break;
Howard Chu's avatar
Howard Chu committed
705
706
	}

707
	fop.o_bd->bd_info = (BackendInfo *)on->on_info;
708
	fop.o_bd->be_search( &fop, &frs );
709
	fop.o_bd->bd_info = (BackendInfo *)on;
Howard Chu's avatar
Howard Chu committed
710

711
712
	switch( mode ) {
	case FIND_MAXCSN:
713
		if ( ber_bvcmp( &si->si_ctxcsn[maxid], &maxcsn )) {
Quanah Gibson-Mount's avatar
Quanah Gibson-Mount committed
714
715
716
717
#ifdef CHECK_CSN
			Syntax *syn = slap_schema.si_ad_contextCSN->ad_type->sat_syntax;
			assert( !syn->ssyn_validate( syn, &maxcsn ));
#endif
718
719
720
			ber_bvreplace( &si->si_ctxcsn[maxid], &maxcsn );
			si->si_numops++;	/* ensure a checkpoint */
		}
721
722
723
		break;
	case FIND_CSN:
		/* If matching CSN was not found, invalidate the context. */
Howard Chu's avatar
Howard Chu committed
724
725
726
727
728
729
730
731
		if ( !cb.sc_private ) {
			/* If we didn't find an exact match, then try for <= */
			if ( findcsn_retry ) {
				findcsn_retry = 0;
				goto again;
			}
			rc = LDAP_NO_SUCH_OBJECT;
		}
732
733
		break;
	case FIND_PRESENT:
734
		op->o_tmpfree( pcookie.uuids, op->o_tmpmemctx );
735
		break;
Howard Chu's avatar
Howard Chu committed
736
737
	}

738
	return rc;
Howard Chu's avatar
Howard Chu committed
739
740
}

741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
static void
syncprov_free_syncop( syncops *so )
{
	syncres *sr, *srnext;
	GroupAssertion *ga, *gnext;

	ldap_pvt_thread_mutex_lock( &so->s_mutex );
	if ( --so->s_inuse > 0 ) {
		ldap_pvt_thread_mutex_unlock( &so->s_mutex );
		return;
	}
	ldap_pvt_thread_mutex_unlock( &so->s_mutex );
	if ( so->s_flags & PS_IS_DETACHED ) {
		filter_free( so->s_op->ors_filter );
		for ( ga = so->s_op->o_groups; ga; ga=gnext ) {
			gnext = ga->ga_next;
			ch_free( ga );
		}
		ch_free( so->s_op );
	}
	ch_free( so->s_base.bv_val );
	for ( sr=so->s_res; sr; sr=srnext ) {
		srnext = sr->s_next;
		ch_free( sr );
	}
	ldap_pvt_thread_mutex_destroy( &so->s_mutex );
	ch_free( so );
}

770
771
/* Send a persistent search response */
static int
772
773
syncprov_sendresp( Operation *op, opcookie *opc, syncops *so,
	Entry **e, int mode )
774
775
776
777
778
{
	slap_overinst *on = opc->son;

	SlapReply rs = { REP_SEARCH };
	LDAPControl *ctrls[2];
Quanah Gibson-Mount's avatar
Quanah Gibson-Mount committed
779
	struct berval cookie = BER_BVNULL, csns[2];
780
781
782
	Entry e_uuid = {0};
	Attribute a_uuid = {0};

Howard Chu's avatar
Howard Chu committed
783
784
785
	if ( so->s_op->o_abandon )
		return SLAPD_ABANDON;

786
	ctrls[1] = NULL;
Quanah Gibson-Mount's avatar
Quanah Gibson-Mount committed
787
788
789
790
791
	if ( !BER_BVISNULL( &opc->sctxcsn )) {
		csns[0] = opc->sctxcsn;
		BER_BVZERO( &csns[1] );
		slap_compose_sync_cookie( op, &cookie, csns, so->s_rid, slap_serverID ? slap_serverID : -1 );
	}
792

Quanah Gibson-Mount's avatar
Quanah Gibson-Mount committed
793
#ifdef LDAP_DEBUG
Quanah Gibson-Mount's avatar
Quanah Gibson-Mount committed
794
795
796
797
798
799
800
801
802
	if ( !BER_BVISNULL( &cookie )) {
		if ( so->s_sid > 0 ) {
			Debug( LDAP_DEBUG_SYNC, "syncprov_sendresp: to=%03x, cookie=%s\n",
				so->s_sid, cookie.bv_val , 0 );
		} else {
			Debug( LDAP_DEBUG_SYNC, "syncprov_sendresp: cookie=%s\n",
				cookie.bv_val, 0, 0 );
		}
	}		
Quanah Gibson-Mount's avatar
Quanah Gibson-Mount committed
803
#endif
Quanah Gibson-Mount's avatar
Quanah Gibson-Mount committed
804

805
806
807
	e_uuid.e_attrs = &a_uuid;
	a_uuid.a_desc = slap_schema.si_ad_entryUUID;
	a_uuid.a_nvals = &opc->suuid;
808
	rs.sr_err = syncprov_state_ctrl( op, &rs, &e_uuid,
809
		mode, ctrls, 0, 1, &cookie );
Quanah Gibson-Mount's avatar
Quanah Gibson-Mount committed
810
811
812
	if ( !BER_BVISNULL( &cookie )) {
		op->o_tmpfree( cookie.bv_val, op->o_tmpmemctx );
	}
813

814
	rs.sr_ctrls = ctrls;
815
	op->o_bd->bd_info = (BackendInfo *)on->on_info;
816
817
	switch( mode ) {
	case LDAP_SYNC_ADD:
818
819
820
		rs.sr_entry = *e;
		if ( rs.sr_entry->e_private )
			rs.sr_flags = REP_ENTRY_MUSTRELEASE;
Quanah Gibson-Mount's avatar
Quanah Gibson-Mount committed
821
		if ( opc->sreference && so->s_op->o_managedsait <= SLAP_CONTROL_IGNORED ) {
822
			rs.sr_ref = get_entry_referrals( op, rs.sr_entry );
823
			rs.sr_err = send_search_reference( op, &rs );
824
			ber_bvarray_free( rs.sr_ref );
825
826
			if ( !rs.sr_entry )
				*e = NULL;
827
828
829
830
			break;
		}
		/* fallthru */
	case LDAP_SYNC_MODIFY:
831
832
833
		rs.sr_entry = *e;
		if ( rs.sr_entry->e_private )
			rs.sr_flags = REP_ENTRY_MUSTRELEASE;
834
		rs.sr_attrs = op->ors_attrs;
835
		rs.sr_err = send_search_entry( op, &rs );
836
837
		if ( !rs.sr_entry )
			*e = NULL;
838
839
840
841
842
843
		break;
	case LDAP_SYNC_DELETE:
		e_uuid.e_attrs = NULL;
		e_uuid.e_name = opc->sdn;
		e_uuid.e_nname = opc->sndn;
		rs.sr_entry = &e_uuid;
Quanah Gibson-Mount's avatar
Quanah Gibson-Mount committed
844
		if ( opc->sreference && so->s_op->o_managedsait <= SLAP_CONTROL_IGNORED ) {
845
			struct berval bv = BER_BVNULL;
846
			rs.sr_ref = &bv;
847
			rs.sr_err = send_search_reference( op, &rs );
Howard Chu's avatar
Howard Chu committed
848
		} else {
849
			rs.sr_err = send_search_entry( op, &rs );
850
851
852
853
854
		}
		break;
	default:
		assert(0);
	}
855
856
857
858
859
	/* In case someone else freed it already? */
	if ( rs.sr_ctrls ) {
		op->o_tmpfree( rs.sr_ctrls[0], op->o_tmpmemctx );
		rs.sr_ctrls = NULL;
	}
860

861
862
863
	return rs.sr_err;
}

Quanah Gibson-Mount's avatar
Quanah Gibson-Mount committed
864
865
866
static void
syncprov_qstart( syncops *so );

867
868
/* Play back queued responses */
static int
Quanah Gibson-Mount's avatar
Quanah Gibson-Mount committed
869
syncprov_qplay( Operation *op, syncops *so )
870
{
Howard Chu's avatar
Howard Chu committed
871
	slap_overinst *on = LDAP_SLIST_FIRST(&so->s_op->o_extra)->oe_key;
872
873
874
	syncres *sr;
	Entry *e;
	opcookie opc;
875
	int rc = 0;
876
877
878

	opc.son = on;

Quanah Gibson-Mount's avatar
Quanah Gibson-Mount committed
879
	do {
880
		ldap_pvt_thread_mutex_lock( &so->s_mutex );
881
882
883
884
885
		sr = so->s_res;
		if ( sr )
			so->s_res = sr->s_next;
		if ( !so->s_res )
			so->s_restail = NULL;
Howard Chu's avatar
Howard Chu committed
886
		/* Exit loop with mutex held */
887
		if ( !sr || so->s_op->o_abandon )
888
			break;
Howard Chu's avatar
Howard Chu committed
889
		ldap_pvt_thread_mutex_unlock( &so->s_mutex );
890

Quanah Gibson-Mount's avatar
Quanah Gibson-Mount committed
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
		if ( sr->s_mode == LDAP_SYNC_NEW_COOKIE ) {
		    SlapReply rs = { REP_INTERMEDIATE };

		    rc = syncprov_sendinfo( op, &rs, LDAP_TAG_SYNC_NEW_COOKIE,
				&sr->s_csn, 0, NULL, 0 );
		} else {
			opc.sdn = sr->s_dn;
			opc.sndn = sr->s_ndn;
			opc.suuid = sr->s_uuid;
			opc.sctxcsn = sr->s_csn;
			opc.sreference = sr->s_isreference;
			e = NULL;

			if ( sr->s_mode != LDAP_SYNC_DELETE ) {
				rc = overlay_entry_get_ov( op, &opc.sndn, NULL, NULL, 0, &e, on );
				if ( rc ) {
					Debug( LDAP_DEBUG_SYNC, "syncprov_qplay: failed to get %s, "
						"error (%d), ignoring...\n", opc.sndn.bv_val, rc, 0 );
					ch_free( sr );
					rc = 0;
					continue;
				}
913
			}
Quanah Gibson-Mount's avatar
Quanah Gibson-Mount committed
914
			rc = syncprov_sendresp( op, &opc, so, &e, sr->s_mode );
915

Quanah Gibson-Mount's avatar
Quanah Gibson-Mount committed
916
917
918
			if ( e ) {
				overlay_entry_release_ov( op, e, 0, on );
			}
919
920
921
		}

		ch_free( sr );
922

Quanah Gibson-Mount's avatar
Quanah Gibson-Mount committed
923
924
		/* Exit loop with mutex held */
		ldap_pvt_thread_mutex_lock( &so->s_mutex );
Howard Chu's avatar
Howard Chu committed
925

Quanah Gibson-Mount's avatar
Quanah Gibson-Mount committed
926
927
928
929
930
931
	} while (0);

	/* We now only send one change at a time, to prevent one
	 * psearch from hogging all the CPU. Resubmit this task if
	 * there are more responses queued and no errors occurred.
	 */
Quanah Gibson-Mount's avatar
Quanah Gibson-Mount committed
932

Quanah Gibson-Mount's avatar
Quanah Gibson-Mount committed
933
934
935
936
	if ( rc == 0 && so->s_res ) {
		syncprov_qstart( so );
	} else {
		so->s_flags ^= PS_TASK_QUEUED;
937
	}
Quanah Gibson-Mount's avatar
Quanah Gibson-Mount committed
938

Howard Chu's avatar
Howard Chu committed
939
	ldap_pvt_thread_mutex_unlock( &so->s_mutex );
940
941
942
	return rc;
}

Quanah Gibson-Mount's avatar
Quanah Gibson-Mount committed
943
/* task for playing back queued responses */
944
945
946
static void *
syncprov_qtask( void *ctx, void *arg )
{
Quanah Gibson-Mount's avatar
Quanah Gibson-Mount committed
947
	syncops *so = arg;
948
	OperationBuffer opbuf;
949
950
	Operation *op;
	BackendDB be;
951
	int rc;
952

953
	op = &opbuf.ob_op;
954
	*op = *so->s_op;
955
956
957
	op->o_hdr = &opbuf.ob_hdr;
	op->o_controls = opbuf.ob_controls;
	memset( op->o_controls, 0, sizeof(opbuf.ob_controls) );
958
959
960

	*op->o_hdr = *so->s_op->o_hdr;

961
	op->o_tmpmemctx = slap_sl_mem_create(SLAP_SLAB_SIZE, SLAP_SLAB_STACK, ctx, 1);
962
963
964
965
966
967
968
	op->o_tmpmfuncs = &slap_sl_mfuncs;
	op->o_threadctx = ctx;

	/* syncprov_qplay expects a fake db */
	be = *so->s_op->o_bd;
	be.be_flags |= SLAP_DBFLAG_OVERLAY;
	op->o_bd = &be;
Howard Chu's avatar
Howard Chu committed
969
	LDAP_SLIST_FIRST(&op->o_extra) = NULL;
970
971
	op->o_callback = NULL;

Quanah Gibson-Mount's avatar
Quanah Gibson-Mount committed
972
	rc = syncprov_qplay( op, so );
973

974
975
976
	/* decrement use count... */
	syncprov_free_syncop( so );

977
978
979
	return NULL;
}

980
981
982
983
/* Start the task to play back queued psearch responses */
static void
syncprov_qstart( syncops *so )
{
Quanah Gibson-Mount's avatar
Quanah Gibson-Mount committed
984
985
986
987
	so->s_flags |= PS_TASK_QUEUED;
	so->s_inuse++;
	ldap_pvt_thread_pool_submit( &connection_pool, 
		syncprov_qtask, so );
988
989
}

990
991
992
993
994
/* Queue a persistent search response */
static int
syncprov_qresp( opcookie *opc, syncops *so, int mode )
{
	syncres *sr;
Quanah Gibson-Mount's avatar
Quanah Gibson-Mount committed
995
996
997
998
999
1000
	int srsize;
	struct berval cookie = opc->sctxcsn;

	if ( mode == LDAP_SYNC_NEW_COOKIE ) {
		syncprov_info_t	*si = opc->son->on_bi.bi_private;