operation.c 20.4 KB
Newer Older
Ondřej Kuzník's avatar
Ondřej Kuzník committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
/* $OpenLDAP$ */
/* This work is part of OpenLDAP Software <http://www.openldap.org/>.
 *
 * Copyright 1998-2020 The OpenLDAP Foundation.
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted only as authorized by the OpenLDAP
 * Public License.
 *
 * A copy of this license is available in the file LICENSE in the
 * top-level directory of the distribution or, alternatively, at
 * <http://www.OpenLDAP.org/license.html>.
 */

#include "portable.h"

#include "lutil.h"
19
#include "lload.h"
Ondřej Kuzník's avatar
Ondřej Kuzník committed
20

Ondřej Kuzník's avatar
Ondřej Kuzník committed
21
22
23
ldap_pvt_thread_mutex_t lload_pin_mutex;
unsigned long lload_next_pin = 1;

24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
ber_tag_t
slap_req2res( ber_tag_t tag )
{
    switch ( tag ) {
        case LDAP_REQ_ADD:
        case LDAP_REQ_BIND:
        case LDAP_REQ_COMPARE:
        case LDAP_REQ_EXTENDED:
        case LDAP_REQ_MODIFY:
        case LDAP_REQ_MODRDN:
            tag++;
            break;

        case LDAP_REQ_DELETE:
            tag = LDAP_RES_DELETE;
            break;

        case LDAP_REQ_ABANDON:
        case LDAP_REQ_UNBIND:
            tag = LBER_SEQUENCE;
            break;

        case LDAP_REQ_SEARCH:
            tag = LDAP_RES_SEARCH_RESULT;
            break;

        default:
            tag = LBER_SEQUENCE;
    }

    return tag;
}

const char *
58
lload_msgtype2str( ber_tag_t tag )
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
{
    switch ( tag ) {
        case LDAP_REQ_ABANDON: return "abandon request";
        case LDAP_REQ_ADD: return "add request";
        case LDAP_REQ_BIND: return "bind request";
        case LDAP_REQ_COMPARE: return "compare request";
        case LDAP_REQ_DELETE: return "delete request";
        case LDAP_REQ_EXTENDED: return "extended request";
        case LDAP_REQ_MODIFY: return "modify request";
        case LDAP_REQ_RENAME: return "rename request";
        case LDAP_REQ_SEARCH: return "search request";
        case LDAP_REQ_UNBIND: return "unbind request";

        case LDAP_RES_ADD: return "add result";
        case LDAP_RES_BIND: return "bind result";
        case LDAP_RES_COMPARE: return "compare result";
        case LDAP_RES_DELETE: return "delete result";
        case LDAP_RES_EXTENDED: return "extended result";
        case LDAP_RES_INTERMEDIATE: return "intermediate response";
        case LDAP_RES_MODIFY: return "modify result";
        case LDAP_RES_RENAME: return "rename result";
        case LDAP_RES_SEARCH_ENTRY: return "search-entry response";
        case LDAP_RES_SEARCH_REFERENCE: return "search-reference response";
        case LDAP_RES_SEARCH_RESULT: return "search result";
    }
    return "unknown message";
}

Ondřej Kuzník's avatar
Ondřej Kuzník committed
87
88
89
int
operation_client_cmp( const void *left, const void *right )
{
90
    const LloadOperation *l = left, *r = right;
Ondřej Kuzník's avatar
Ondřej Kuzník committed
91

92
    assert( l->o_client_connid == r->o_client_connid );
Ondřej Kuzník's avatar
Ondřej Kuzník committed
93
94
95
96
97
98
99
100
    if ( l->o_client_msgid || r->o_client_msgid ) {
        return ( l->o_client_msgid < r->o_client_msgid ) ?
                -1 :
                ( l->o_client_msgid > r->o_client_msgid );
    } else {
        return ( l->o_pin_id < r->o_pin_id ) ? -1 :
                ( l->o_pin_id > r->o_pin_id );
    }
Ondřej Kuzník's avatar
Ondřej Kuzník committed
101
102
103
104
105
}

int
operation_upstream_cmp( const void *left, const void *right )
{
106
    const LloadOperation *l = left, *r = right;
Ondřej Kuzník's avatar
Ondřej Kuzník committed
107

108
    assert( l->o_upstream_connid == r->o_upstream_connid );
Ondřej Kuzník's avatar
Ondřej Kuzník committed
109
110
111
112
113
114
115
116
    if ( l->o_upstream_msgid || r->o_upstream_msgid ) {
        return ( l->o_upstream_msgid < r->o_upstream_msgid ) ?
                -1 :
                ( l->o_upstream_msgid > r->o_upstream_msgid );
    } else {
        return ( l->o_pin_id < r->o_pin_id ) ? -1 :
                ( l->o_pin_id > r->o_pin_id );
    }
Ondřej Kuzník's avatar
Ondřej Kuzník committed
117
118
}

119
120
121
/*
 * Entered holding c_mutex for now.
 */
122
123
LloadOperation *
operation_init( LloadConnection *c, BerElement *ber )
Ondřej Kuzník's avatar
Ondřej Kuzník committed
124
{
125
    LloadOperation *op;
Ondřej Kuzník's avatar
Ondřej Kuzník committed
126
127
128
129
    ber_tag_t tag;
    ber_len_t len;
    int rc;

130
131
132
133
    if ( !IS_ALIVE( c, c_live ) ) {
        return NULL;
    }

134
    op = ch_calloc( 1, sizeof(LloadOperation) );
Ondřej Kuzník's avatar
Ondřej Kuzník committed
135
    op->o_client = c;
136
    op->o_client_connid = c->c_connid;
Ondřej Kuzník's avatar
Ondřej Kuzník committed
137
    op->o_ber = ber;
138
    op->o_start = slap_get_time();
Ondřej Kuzník's avatar
Ondřej Kuzník committed
139

Ondřej Kuzník's avatar
Ondřej Kuzník committed
140
    ldap_pvt_thread_mutex_init( &op->o_link_mutex );
141

142
    op->o_refcnt = 1;
Ondřej Kuzník's avatar
Ondřej Kuzník committed
143

Ondřej Kuzník's avatar
Ondřej Kuzník committed
144
145
146
147
148
149
150
151
    tag = ber_get_int( ber, &op->o_client_msgid );
    if ( tag != LDAP_TAG_MSGID ) {
        goto fail;
    }

    rc = tavl_insert( &c->c_ops, op, operation_client_cmp, avl_dup_error );
    if ( rc ) {
        Debug( LDAP_DEBUG_PACKETS, "operation_init: "
Ondřej Kuzník's avatar
Ondřej Kuzník committed
152
153
                "several operations with same msgid=%d in-flight "
                "from client connid=%lu\n",
154
                op->o_client_msgid, op->o_client_connid );
Ondřej Kuzník's avatar
Ondřej Kuzník committed
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
        goto fail;
    }

    tag = op->o_tag = ber_skip_element( ber, &op->o_request );
    switch ( tag ) {
        case LBER_ERROR:
            rc = -1;
            break;
    }
    if ( rc ) {
        tavl_delete( &c->c_ops, op, operation_client_cmp );
        goto fail;
    }

    tag = ber_peek_tag( ber, &len );
    if ( tag == LDAP_TAG_CONTROLS ) {
        ber_skip_element( ber, &op->o_ctrls );
    }

174
175
176
177
178
179
180
181
182
    switch ( op->o_tag ) {
        case LDAP_REQ_BIND:
            lload_stats.counters[LLOAD_STATS_OPS_BIND].lc_ops_received++;
            break;
        default:
            lload_stats.counters[LLOAD_STATS_OPS_OTHER].lc_ops_received++;
            break;
    }

Ondřej Kuzník's avatar
Ondřej Kuzník committed
183
184
185
    Debug( LDAP_DEBUG_STATS, "operation_init: "
            "received a new operation, %s with msgid=%d for client "
            "connid=%lu\n",
186
            lload_msgtype2str( op->o_tag ), op->o_client_msgid,
187
            op->o_client_connid );
Ondřej Kuzník's avatar
Ondřej Kuzník committed
188

189
    c->c_n_ops_executing++;
Ondřej Kuzník's avatar
Ondřej Kuzník committed
190
191
192
193
194
195
196
    return op;

fail:
    ch_free( op );
    return NULL;
}

197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
void
operation_destroy( LloadOperation *op )
{
    Debug( LDAP_DEBUG_TRACE, "operation_destroy: "
            "op=%p destroyed operation from client connid=%lu, "
            "client msgid=%d\n",
            op, op->o_client_connid, op->o_client_msgid );

    assert( op->o_refcnt == 0 );
    assert( op->o_client == NULL );
    assert( op->o_upstream == NULL );

    ber_free( op->o_ber, 1 );
    ldap_pvt_thread_mutex_destroy( &op->o_link_mutex );
    ch_free( op );
}

214
int
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
operation_unlink( LloadOperation *op )
{
    LloadConnection *client, *upstream;
    uintptr_t prev_refcnt;
    int result = 0;

    if ( !( prev_refcnt = try_release_ref(
                    &op->o_refcnt, op, (dispose_cb *)operation_destroy ) ) ) {
        return result;
    }

    assert( prev_refcnt == 1 );

    Debug( LDAP_DEBUG_TRACE, "operation_unlink: "
            "unlinking operation between client connid=%lu and upstream "
            "connid=%lu "
            "client msgid=%d\n",
            op->o_client_connid, op->o_upstream_connid, op->o_client_msgid );

    ldap_pvt_thread_mutex_lock( &op->o_link_mutex );
    client = op->o_client;
    upstream = op->o_upstream;

    op->o_client = NULL;
    op->o_upstream = NULL;
    ldap_pvt_thread_mutex_unlock( &op->o_link_mutex );

    assert( client || upstream );

    if ( client ) {
        result |= operation_unlink_client( op, client );
        operation_update_global_rejected( op );
    }

    if ( upstream ) {
        result |= operation_unlink_upstream( op, upstream );
    }

    return result;
}

int
operation_unlink_client( LloadOperation *op, LloadConnection *client )
{
    LloadOperation *removed;
    int result = 0;

    Debug( LDAP_DEBUG_TRACE, "operation_unlink_client: "
            "unlinking operation op=%p msgid=%d client connid=%lu\n",
            op, op->o_client_msgid, op->o_client_connid );

    CONNECTION_LOCK(client);
    if ( (removed = tavl_delete(
                   &client->c_ops, op, operation_client_cmp )) ) {
        result = LLOAD_OP_DETACHING_CLIENT;

        assert( op == removed );
        client->c_n_ops_executing--;

        if ( client->c_state == LLOAD_C_BINDING ) {
            client->c_state = LLOAD_C_READY;
            if ( !BER_BVISNULL( &client->c_auth ) ) {
                ber_memfree( client->c_auth.bv_val );
                BER_BVZERO( &client->c_auth );
            }
            if ( !BER_BVISNULL( &client->c_sasl_bind_mech ) ) {
                ber_memfree( client->c_sasl_bind_mech.bv_val );
                BER_BVZERO( &client->c_sasl_bind_mech );
            }
            if ( op->o_pin_id ) {
                client->c_pin_id = 0;
            }
        }
    }
    if ( client->c_state == LLOAD_C_CLOSING && !client->c_ops ) {
        CONNECTION_DESTROY(client);
    } else {
        CONNECTION_UNLOCK(client);
    }

    return result;
}

int
operation_unlink_upstream( LloadOperation *op, LloadConnection *upstream )
{
    LloadOperation *removed;
    LloadBackend *b = NULL;
    int result = 0;

    Debug( LDAP_DEBUG_TRACE, "operation_unlink_upstream: "
            "unlinking operation op=%p msgid=%d upstream connid=%lu\n",
            op, op->o_upstream_msgid, op->o_upstream_connid );

    CONNECTION_LOCK(upstream);
    if ( (removed = tavl_delete(
                   &upstream->c_ops, op, operation_upstream_cmp )) ) {
        result |= LLOAD_OP_DETACHING_UPSTREAM;

        assert( op == removed );
        upstream->c_n_ops_executing--;

        if ( upstream->c_state == LLOAD_C_BINDING ) {
            assert( op->o_tag == LDAP_REQ_BIND && upstream->c_ops == NULL );
            upstream->c_state = LLOAD_C_READY;
            if ( !BER_BVISNULL( &upstream->c_sasl_bind_mech ) ) {
                ber_memfree( upstream->c_sasl_bind_mech.bv_val );
                BER_BVZERO( &upstream->c_sasl_bind_mech );
            }
        }
        operation_update_conn_counters( op, upstream );
        b = (LloadBackend *)upstream->c_private;
    }
    if ( upstream->c_state == LLOAD_C_CLOSING && !upstream->c_ops ) {
        CONNECTION_DESTROY(upstream);
    } else {
        CONNECTION_UNLOCK(upstream);
    }

    if ( b ) {
        ldap_pvt_thread_mutex_lock( &b->b_mutex );
        b->b_n_ops_executing--;
        operation_update_backend_counters( op, b );
        ldap_pvt_thread_mutex_unlock( &b->b_mutex );
    }

    return result;
}

int
operation_send_abandon( LloadOperation *op, LloadConnection *upstream )
346
347
348
349
{
    BerElement *ber;
    int rc = -1;

350
    if ( !IS_ALIVE( upstream, c_live ) ) {
351
352
353
        return rc;
    }

354
355
356
357
358
359
360
361
362
    ldap_pvt_thread_mutex_lock( &upstream->c_io_mutex );
    ber = upstream->c_pendingber;
    if ( ber == NULL && (ber = ber_alloc()) == NULL ) {
        Debug( LDAP_DEBUG_ANY, "operation_send_abandon: "
                "ber_alloc failed\n" );
        goto done;
    }
    upstream->c_pendingber = ber;

363
364
365
366
367
368
369
370
371
372
373
374
375
376
    Debug( LDAP_DEBUG_TRACE, "operation_send_abandon: "
            "abandoning %s msgid=%d on connid=%lu\n",
            lload_msgtype2str( op->o_tag ), op->o_upstream_msgid,
            op->o_upstream_connid );

    if ( op->o_tag == LDAP_REQ_BIND ) {
        rc = ber_printf( ber, "t{tit{ist{s}}}", LDAP_TAG_MESSAGE,
                LDAP_TAG_MSGID, upstream->c_next_msgid++,
                LDAP_REQ_BIND, LDAP_VERSION3, "", LDAP_AUTH_SASL, "" );
    } else {
        rc = ber_printf( ber, "t{titi}", LDAP_TAG_MESSAGE,
                LDAP_TAG_MSGID, upstream->c_next_msgid++,
                LDAP_REQ_ABANDON, op->o_upstream_msgid );
    }
377
378
379
380
381
382
383
384
385
386
387
388
389

    if ( rc < 0 ) {
        ber_free( ber, 1 );
        upstream->c_pendingber = NULL;
        goto done;
    }
    rc = LDAP_SUCCESS;

done:
    ldap_pvt_thread_mutex_unlock( &upstream->c_io_mutex );
    return rc;
}

390
391
392
393
394
/*
 * Will remove the operation from its upstream and if it was still there,
 * sends an abandon request.
 *
 * Being called from client_reset or request_abandon, the following hold:
395
396
397
 * - noone else is processing the read part of the client connection (no new
 *   operations come in there - relevant for the c_state checks)
 * - op->o_client_refcnt > op->o_client_live (and it follows that op->o_client != NULL)
398
 */
Ondřej Kuzník's avatar
Ondřej Kuzník committed
399
void
400
operation_abandon( LloadOperation *op )
Ondřej Kuzník's avatar
Ondřej Kuzník committed
401
{
402
    LloadConnection *c;
Ondřej Kuzník's avatar
Ondřej Kuzník committed
403

Ondřej Kuzník's avatar
Ondřej Kuzník committed
404
    ldap_pvt_thread_mutex_lock( &op->o_link_mutex );
405
    c = op->o_upstream;
406
    ldap_pvt_thread_mutex_unlock( &op->o_link_mutex );
407
    if ( !c || !IS_ALIVE( c, c_live ) ) {
408
        goto done;
409
    }
Ondřej Kuzník's avatar
Ondřej Kuzník committed
410

Ondřej Kuzník's avatar
Ondřej Kuzník committed
411
412
413
    /* for now consider all abandoned operations completed,
     * perhaps add a separate counter later */
    op->o_res = LLOAD_OP_COMPLETED;
414
    if ( !operation_unlink_upstream( op, c ) ) {
415
        /* The operation has already been abandoned or finished */
416
417
418
419
420
        Debug( LDAP_DEBUG_TRACE, "operation_abandon: "
                "%s from connid=%lu msgid=%d not present in connid=%lu any "
                "more\n",
                lload_msgtype2str( op->o_tag ), op->o_client_connid,
                op->o_client_msgid, op->o_upstream_connid );
421
        goto done;
422
    }
423

424
    if ( operation_send_abandon( op, c ) == LDAP_SUCCESS ) {
425
        connection_write_cb( -1, 0, c );
Ondřej Kuzník's avatar
Ondřej Kuzník committed
426
427
    }

428
done:
429
    operation_unlink( op );
Ondřej Kuzník's avatar
Ondřej Kuzník committed
430
431
}

432
433
void
operation_send_reject(
434
        LloadOperation *op,
435
436
437
        int result,
        const char *msg,
        int send_anyway )
Ondřej Kuzník's avatar
Ondřej Kuzník committed
438
{
439
    LloadConnection *c;
Ondřej Kuzník's avatar
Ondřej Kuzník committed
440
441
442
    BerElement *ber;
    int found;

443
    Debug( LDAP_DEBUG_TRACE, "operation_send_reject: "
Ondřej Kuzník's avatar
Ondřej Kuzník committed
444
            "rejecting %s from client connid=%lu with message: \"%s\"\n",
445
446
447
448
449
            lload_msgtype2str( op->o_tag ), op->o_client_connid, msg );

    ldap_pvt_thread_mutex_lock( &op->o_link_mutex );
    c = op->o_client;
    ldap_pvt_thread_mutex_unlock( &op->o_link_mutex );
450
    if ( !c || !IS_ALIVE( c, c_live ) ) {
451
452
453
454
455
456
        Debug( LDAP_DEBUG_TRACE, "operation_send_reject: "
                "not sending msgid=%d, client connid=%lu is dead\n",
                op->o_client_msgid, op->o_client_connid );

        goto done;
    }
457

458
    found = operation_unlink_client( op, c );
459
    if ( !found && !send_anyway ) {
460
        Debug( LDAP_DEBUG_TRACE, "operation_send_reject: "
Ondřej Kuzník's avatar
Ondřej Kuzník committed
461
462
463
                "msgid=%d not scheduled for client connid=%lu anymore, "
                "not sending\n",
                op->o_client_msgid, c->c_connid );
Ondřej Kuzník's avatar
Ondřej Kuzník committed
464
        goto done;
Ondřej Kuzník's avatar
Ondřej Kuzník committed
465
466
    }

467
468
    if ( op->o_client_msgid == 0 ) {
        assert( op->o_saved_msgid == 0 && op->o_pin_id );
469
        Debug( LDAP_DEBUG_TRACE, "operation_send_reject: "
470
471
472
473
474
                "operation pin=%lu is just a pin, not sending\n",
                op->o_pin_id );
        goto done;
    }

Ondřej Kuzník's avatar
Ondřej Kuzník committed
475
476
477
478
    ldap_pvt_thread_mutex_lock( &c->c_io_mutex );
    ber = c->c_pendingber;
    if ( ber == NULL && (ber = ber_alloc()) == NULL ) {
        ldap_pvt_thread_mutex_unlock( &c->c_io_mutex );
479
        Debug( LDAP_DEBUG_ANY, "operation_send_reject: "
Ondřej Kuzník's avatar
Ondřej Kuzník committed
480
481
                "ber_alloc failed, closing connid=%lu\n",
                c->c_connid );
482
483
        CONNECTION_LOCK_DESTROY(c);
        goto done;
Ondřej Kuzník's avatar
Ondřej Kuzník committed
484
485
486
487
488
489
490
491
492
    }
    c->c_pendingber = ber;

    ber_printf( ber, "t{tit{ess}}", LDAP_TAG_MESSAGE,
            LDAP_TAG_MSGID, op->o_client_msgid,
            slap_req2res( op->o_tag ), result, "", msg );

    ldap_pvt_thread_mutex_unlock( &c->c_io_mutex );

493
    connection_write_cb( -1, 0, c );
Ondřej Kuzník's avatar
Ondřej Kuzník committed
494

Ondřej Kuzník's avatar
Ondřej Kuzník committed
495
done:
496
    operation_unlink( op );
Ondřej Kuzník's avatar
Ondřej Kuzník committed
497
498
}

499
500
501
502
503
504
505
/*
 * Upstream is shutting down, signal the client if necessary, but we have to
 * call operation_destroy_from_upstream ourselves to detach upstream from the
 * op.
 *
 * Only called from upstream_destroy.
 */
Ondřej Kuzník's avatar
Ondřej Kuzník committed
506
void
507
operation_lost_upstream( LloadOperation *op )
Ondřej Kuzník's avatar
Ondřej Kuzník committed
508
{
509
    operation_send_reject( op, LDAP_OTHER,
510
            "connection to the remote server has been severed", 0 );
Ondřej Kuzník's avatar
Ondřej Kuzník committed
511
}
512

Ondřej Kuzník's avatar
Ondřej Kuzník committed
513
514
int
connection_timeout( LloadConnection *upstream, void *arg )
515
{
516
    LloadOperation *op;
517
    TAvlnode *ops = NULL, *node, *next;
518
    LloadBackend *b = upstream->c_private;
Ondřej Kuzník's avatar
Ondřej Kuzník committed
519
    time_t threshold = *(time_t *)arg;
520
521
    int rc, nops = 0;

522
    CONNECTION_LOCK(upstream);
523
    for ( node = tavl_end( upstream->c_ops, TAVL_DIR_LEFT ); node &&
524
525
            ((LloadOperation *)node->avl_data)->o_start <
                    threshold; /* shortcut */
526
            node = next ) {
527
        LloadOperation *found_op;
528

529
        next = tavl_next( node, TAVL_DIR_RIGHT );
530
531
532
533
534
535
536
        op = node->avl_data;

        /* Have we received another response since? */
        if ( op->o_last_response && op->o_last_response >= threshold ) {
            continue;
        }

537
        op->o_res = LLOAD_OP_FAILED;
538
539
540
        found_op = tavl_delete( &upstream->c_ops, op, operation_upstream_cmp );
        assert( op == found_op );

541
542
543
544
545
546
547
548
549
        if ( upstream->c_state == LLOAD_C_BINDING ) {
            assert( op->o_tag == LDAP_REQ_BIND && upstream->c_ops == NULL );
            upstream->c_state = LLOAD_C_READY;
            if ( !BER_BVISNULL( &upstream->c_sasl_bind_mech ) ) {
                ber_memfree( upstream->c_sasl_bind_mech.bv_val );
                BER_BVZERO( &upstream->c_sasl_bind_mech );
            }
        }

550
551
552
553
554
555
        rc = tavl_insert( &ops, op, operation_upstream_cmp, avl_dup_error );
        assert( rc == LDAP_SUCCESS );

        Debug( LDAP_DEBUG_STATS2, "connection_timeout: "
                "timing out %s from connid=%lu msgid=%d sent to connid=%lu as "
                "msgid=%d\n",
556
                lload_msgtype2str( op->o_tag ), op->o_client_connid,
557
558
559
560
561
562
                op->o_client_msgid, op->o_upstream_connid,
                op->o_upstream_msgid );
        nops++;
    }

    if ( nops == 0 ) {
563
        CONNECTION_UNLOCK(upstream);
Ondřej Kuzník's avatar
Ondřej Kuzník committed
564
        return LDAP_SUCCESS;
565
566
    }
    upstream->c_n_ops_executing -= nops;
567
    upstream->c_counters.lc_ops_failed += nops;
568
569
570
    Debug( LDAP_DEBUG_STATS, "connection_timeout: "
            "timing out %d operations for connid=%lu\n",
            nops, upstream->c_connid );
571
    CONNECTION_UNLOCK(upstream);
572
573
574
575
576
577
578
579
580

    ldap_pvt_thread_mutex_lock( &b->b_mutex );
    b->b_n_ops_executing -= nops;
    ldap_pvt_thread_mutex_unlock( &b->b_mutex );

    for ( node = tavl_end( ops, TAVL_DIR_LEFT ); node;
            node = tavl_next( node, TAVL_DIR_RIGHT ) ) {
        op = node->avl_data;

581
582
583
584
        operation_send_reject( op,
                op->o_tag == LDAP_REQ_SEARCH ? LDAP_TIMELIMIT_EXCEEDED :
                                               LDAP_ADMINLIMIT_EXCEEDED,
                "upstream did not respond in time", 0 );
585
586

        if ( rc == LDAP_SUCCESS ) {
587
            rc = operation_send_abandon( op, upstream );
588
        }
589
        operation_unlink( op );
590
591
592
593
594
595
596
    }

    /* TODO: if operation_send_abandon failed, we need to kill the upstream */
    if ( rc == LDAP_SUCCESS ) {
        connection_write_cb( -1, 0, upstream );
    }

597
598
599
600
601
602
603
    CONNECTION_LOCK(upstream);
    if ( upstream->c_state == LLOAD_C_CLOSING && !upstream->c_ops ) {
        CONNECTION_DESTROY(upstream);
    } else {
        CONNECTION_UNLOCK(upstream);
    }

604
605
    /* just dispose of the AVL, most operations should already be gone */
    tavl_free( ops, NULL );
Ondřej Kuzník's avatar
Ondřej Kuzník committed
606
    return LDAP_SUCCESS;
607
608
609
610
611
}

void
operations_timeout( evutil_socket_t s, short what, void *arg )
{
612
    LloadBackend *b;
613
614
615
616
617
618
619
620
621
    time_t threshold;

    Debug( LDAP_DEBUG_TRACE, "operations_timeout: "
            "running timeout task\n" );
    if ( !lload_timeout_api ) goto done;

    threshold = slap_get_time() - lload_timeout_api->tv_sec;

    LDAP_CIRCLEQ_FOREACH ( b, &backend, b_next ) {
622
623
        epoch_t epoch;

Ondřej Kuzník's avatar
Ondřej Kuzník committed
624
625
626
627
628
        ldap_pvt_thread_mutex_lock( &b->b_mutex );
        if ( b->b_n_ops_executing == 0 ) {
            ldap_pvt_thread_mutex_unlock( &b->b_mutex );
            continue;
        }
629

630
631
        epoch = epoch_join();

632
633
634
        Debug( LDAP_DEBUG_TRACE, "operations_timeout: "
                "timing out binds for backend uri=%s\n",
                b->b_uri.bv_val );
Ondřej Kuzník's avatar
Ondřej Kuzník committed
635
636
        connections_walk_last( &b->b_mutex, &b->b_bindconns, b->b_last_bindconn,
                connection_timeout, &threshold );
637
638
639
640

        Debug( LDAP_DEBUG_TRACE, "operations_timeout: "
                "timing out other operations for backend uri=%s\n",
                b->b_uri.bv_val );
Ondřej Kuzník's avatar
Ondřej Kuzník committed
641
642
643
        connections_walk_last( &b->b_mutex, &b->b_conns, b->b_last_conn,
                connection_timeout, &threshold );

644
        epoch_leave( epoch );
Ondřej Kuzník's avatar
Ondřej Kuzník committed
645
        ldap_pvt_thread_mutex_unlock( &b->b_mutex );
646
647
648
649
    }
done:
    Debug( LDAP_DEBUG_TRACE, "operations_timeout: "
            "timeout task finished\n" );
Ondřej Kuzník's avatar
Ondřej Kuzník committed
650
    evtimer_add( lload_timeout_event, lload_timeout_api );
651
}
652
653
654
655

void
operation_update_global_rejected( LloadOperation *op )
{
Ondřej Kuzník's avatar
Ondřej Kuzník committed
656
657
    if ( op->o_res == LLOAD_OP_REJECTED ) {
        assert( op->o_upstream_connid == 0 );
658
659
660
661
662
663
664
665
666
667
668
669
        switch ( op->o_tag ) {
            case LDAP_REQ_BIND:
                lload_stats.counters[LLOAD_STATS_OPS_BIND].lc_ops_rejected++;
                break;
            default:
                lload_stats.counters[LLOAD_STATS_OPS_OTHER].lc_ops_rejected++;
                break;
        }
    }
}

void
670
operation_update_conn_counters( LloadOperation *op, LloadConnection *upstream )
671
672
{
    if ( op->o_res == LLOAD_OP_COMPLETED ) {
673
        upstream->c_counters.lc_ops_completed++;
674
    } else {
675
        upstream->c_counters.lc_ops_failed++;
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
    }
}

void
operation_update_backend_counters( LloadOperation *op, LloadBackend *b )
{
    int stat_type = op->o_tag == LDAP_REQ_BIND ? LLOAD_STATS_OPS_BIND :
                                                 LLOAD_STATS_OPS_OTHER;

    assert( b != NULL );
    if ( op->o_res == LLOAD_OP_COMPLETED ) {
        b->b_counters[stat_type].lc_ops_completed++;
    } else {
        b->b_counters[stat_type].lc_ops_failed++;
    }
}