backend.c 15.5 KB
Newer Older
Ondřej Kuzník's avatar
Ondřej Kuzník committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
/* $OpenLDAP$ */
/* This work is part of OpenLDAP Software <http://www.openldap.org/>.
 *
 * Copyright 1998-2020 The OpenLDAP Foundation.
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted only as authorized by the OpenLDAP
 * Public License.
 *
 * A copy of this license is available in the file LICENSE in the
 * top-level directory of the distribution or, alternatively, at
 * <http://www.OpenLDAP.org/license.html>.
 */

#include "portable.h"

#include <ac/socket.h>
#include <ac/errno.h>
#include <ac/string.h>
#include <ac/time.h>
#include <ac/unistd.h>

#include <event2/event.h>
#include <event2/dns.h>

#include "lutil.h"
28
#include "lload.h"
Ondřej Kuzník's avatar
Ondřej Kuzník committed
29

30
31
32
static void
upstream_connect_cb( evutil_socket_t s, short what, void *arg )
{
33
34
    LloadPendingConnection *conn = arg;
    LloadBackend *b = conn->backend;
Ondřej Kuzník's avatar
Ondřej Kuzník committed
35
    int error = 0, rc = -1;
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61

    ldap_pvt_thread_mutex_lock( &b->b_mutex );
    Debug( LDAP_DEBUG_CONNS, "upstream_connect_cb: "
            "fd=%d connection callback for backend uri='%s'\n",
            s, b->b_uri.bv_val );
    if ( what == EV_WRITE ) {
        socklen_t optlen = sizeof(error);

        if ( getsockopt( conn->fd, SOL_SOCKET, SO_ERROR, (void *)&error,
                     &optlen ) < 0 ) {
            goto done;
        }
        if ( error == EINTR || error == EINPROGRESS || error == EWOULDBLOCK ) {
            ldap_pvt_thread_mutex_unlock( &b->b_mutex );
            return;
        } else if ( error ) {
            goto done;
        } else if ( !upstream_init( s, conn->backend ) ) {
            goto done;
        }
        rc = LDAP_SUCCESS;
    }

done:
    if ( rc ) {
        evutil_closesocket( conn->fd );
Ondřej Kuzník's avatar
Ondřej Kuzník committed
62
        b->b_opening--;
63
        b->b_failed++;
64
65
66
67
68
69
70
71
72
73
74
        if ( what & EV_TIMEOUT ) {
            Debug( LDAP_DEBUG_ANY, "upstream_connect_cb: "
                    "fd=%d connection timed out\n",
                    s );
        } else {
            char ebuf[128];
            Debug( LDAP_DEBUG_ANY, "upstream_connect_cb: "
                    "fd=%d connection set up failed%s%s\n",
                    s, error ? ": " : "",
                    error ? sock_errstr( error, ebuf, sizeof(ebuf) ) : "" );
        }
75
76
77
    } else {
        b->b_failed = 0;
    }
Ondřej Kuzník's avatar
Ondřej Kuzník committed
78
    LDAP_LIST_REMOVE( conn, next );
79
80
81
82
83
    ldap_pvt_thread_mutex_unlock( &b->b_mutex );

    event_free( conn->event );
    ch_free( conn );

Ondřej Kuzník's avatar
Ondřej Kuzník committed
84
85
86
    if ( rc ) {
        backend_retry( b );
    }
87
88
}

Ondřej Kuzník's avatar
Ondřej Kuzník committed
89
90
91
static void
upstream_name_cb( int result, struct evutil_addrinfo *res, void *arg )
{
92
    LloadBackend *b = arg;
93
    ber_socket_t s = AC_SOCKET_INVALID;
Ondřej Kuzník's avatar
Ondřej Kuzník committed
94
95
    int rc;

96
97
    ldap_pvt_thread_mutex_lock( &b->b_mutex );

Ondřej Kuzník's avatar
Ondřej Kuzník committed
98
99
100
    if ( result || !res ) {
        Debug( LDAP_DEBUG_ANY, "upstream_name_cb: "
                "name resolution failed for backend '%s': %s\n",
101
                b->b_uri.bv_val, evutil_gai_strerror( result ) );
102
        goto fail;
Ondřej Kuzník's avatar
Ondřej Kuzník committed
103
104
    }

105
106
107
108
    /* TODO: if we get failures, try the other addrinfos */
    if ( (s = socket( res->ai_family, SOCK_STREAM, 0 )) ==
            AC_SOCKET_INVALID ) {
        goto fail;
Ondřej Kuzník's avatar
Ondřej Kuzník committed
109
110
    }

111
112
    if ( ber_pvt_socket_set_nonblock( s, 1 ) ) {
        goto fail;
Ondřej Kuzník's avatar
Ondřej Kuzník committed
113
114
115
116
117
118
119
120
121
122
123
    }

    if ( res->ai_family == PF_INET ) {
        struct sockaddr_in *ai = (struct sockaddr_in *)res->ai_addr;
        ai->sin_port = htons( b->b_port );
        rc = connect( s, (struct sockaddr *)ai, res->ai_addrlen );
    } else {
        struct sockaddr_in6 *ai = (struct sockaddr_in6 *)res->ai_addr;
        ai->sin6_port = htons( b->b_port );
        rc = connect( s, (struct sockaddr *)ai, res->ai_addrlen );
    }
124
125
    /* Asynchronous connect */
    if ( rc ) {
126
        LloadPendingConnection *conn;
127
128
129
130
131
132
133
134
135

        if ( errno != EINPROGRESS && errno != EWOULDBLOCK ) {
            Debug( LDAP_DEBUG_ANY, "upstream_name_cb: "
                    "failed to connect to server '%s'\n",
                    b->b_uri.bv_val );
            evutil_closesocket( s );
            goto fail;
        }

136
        conn = ch_calloc( 1, sizeof(LloadPendingConnection) );
137
138
139
140
        LDAP_LIST_ENTRY_INIT( conn, next );
        conn->backend = b;
        conn->fd = s;

141
        conn->event = event_new( lload_get_base( s ), s, EV_WRITE|EV_PERSIST,
142
143
144
145
146
147
148
149
150
151
                upstream_connect_cb, conn );
        if ( !conn->event ) {
            Debug( LDAP_DEBUG_ANY, "upstream_name_cb: "
                    "failed to acquire an event to finish upstream "
                    "connection setup.\n" );
            ch_free( conn );
            evutil_closesocket( s );
            goto fail;
        }

152
        event_add( conn->event, lload_timeout_net );
153
154
155
        LDAP_LIST_INSERT_HEAD( &b->b_connecting, conn, next );
        Debug( LDAP_DEBUG_CONNS, "upstream_name_cb: "
                "connection to backend uri=%s in progress\n",
156
                b->b_uri.bv_val );
Ondřej Kuzník's avatar
Ondřej Kuzník committed
157
    } else if ( !upstream_init( s, b ) ) {
158
159
        goto fail;
    }
Ondřej Kuzník's avatar
Ondřej Kuzník committed
160

Ondřej Kuzník's avatar
Ondřej Kuzník committed
161
    ldap_pvt_thread_mutex_unlock( &b->b_mutex );
Ondřej Kuzník's avatar
Ondřej Kuzník committed
162
    evutil_freeaddrinfo( res );
163
164
165
166
167
168
169
    return;

fail:
    if ( s != AC_SOCKET_INVALID ) {
        evutil_closesocket( s );
    }
    b->b_opening--;
170
    b->b_failed++;
171
172
    ldap_pvt_thread_mutex_unlock( &b->b_mutex );
    backend_retry( b );
Ondřej Kuzník's avatar
Ondřej Kuzník committed
173
174
175
    if ( res ) {
        evutil_freeaddrinfo( res );
    }
Ondřej Kuzník's avatar
Ondřej Kuzník committed
176
177
}

178
LloadConnection *
179
backend_select( LloadOperation *op, int *res )
Ondřej Kuzník's avatar
Ondřej Kuzník committed
180
{
181
    LloadBackend *b, *first, *next;
Ondřej Kuzník's avatar
Ondřej Kuzník committed
182
183
184
185
186

    ldap_pvt_thread_mutex_lock( &backend_mutex );
    first = b = current_backend;
    ldap_pvt_thread_mutex_unlock( &backend_mutex );

187
188
    *res = LDAP_UNAVAILABLE;

Ondřej Kuzník's avatar
Ondřej Kuzník committed
189
190
191
    if ( !first ) {
        return NULL;
    }
Ondřej Kuzník's avatar
Ondřej Kuzník committed
192

193
194
    /* TODO: Two runs, one with trylock, then one actually locked if we don't
     * find anything? */
Ondřej Kuzník's avatar
Ondřej Kuzník committed
195
    do {
196
197
        lload_c_head *head;
        LloadConnection *c;
Ondřej Kuzník's avatar
Ondřej Kuzník committed
198

Ondřej Kuzník's avatar
Ondřej Kuzník committed
199
        ldap_pvt_thread_mutex_lock( &b->b_mutex );
Ondřej Kuzník's avatar
Ondřej Kuzník committed
200
        next = LDAP_CIRCLEQ_LOOP_NEXT( &backend, b, b_next );
201
202
203
204

        if ( b->b_max_pending && b->b_n_ops_executing >= b->b_max_pending ) {
            Debug( LDAP_DEBUG_CONNS, "backend_select: "
                    "backend %s too busy\n",
205
                    b->b_uri.bv_val );
206
            ldap_pvt_thread_mutex_unlock( &b->b_mutex );
Ondřej Kuzník's avatar
Ondřej Kuzník committed
207
            b = next;
208
            *res = LDAP_BUSY;
209
210
211
            continue;
        }

212
213
214
215
216
        if ( op->o_tag == LDAP_REQ_BIND
#ifdef LDAP_API_FEATURE_VERIFY_CREDENTIALS
                && !(lload_features & LLOAD_FEATURE_VC)
#endif /* LDAP_API_FEATURE_VERIFY_CREDENTIALS */
        ) {
217
218
219
220
            head = &b->b_bindconns;
        } else {
            head = &b->b_conns;
        }
221
222
223
        if ( !LDAP_CIRCLEQ_EMPTY( head ) ) {
            *res = LDAP_BUSY;
        }
224

225
        LDAP_CIRCLEQ_FOREACH ( c, head, c_next ) {
226
            ldap_pvt_thread_mutex_lock( &c->c_io_mutex );
227
            CONNECTION_LOCK(c);
228
            if ( c->c_state == LLOAD_C_READY && !c->c_pendingber &&
229
230
                    ( b->b_max_conn_pending == 0 ||
                            c->c_n_ops_executing < b->b_max_conn_pending ) ) {
231
                Debug( LDAP_DEBUG_CONNS, "backend_select: "
Ondřej Kuzník's avatar
Ondřej Kuzník committed
232
233
                        "selected connection connid=%lu for client "
                        "connid=%lu msgid=%d\n",
234
235
                        c->c_connid, op->o_client_connid, op->o_client_msgid );

236
237
                /*
                 * Round-robin step:
Ondřej Kuzník's avatar
Ondřej Kuzník committed
238
239
                 * Rotate the queue to put this connection at the end, same for
                 * the backend.
240
241
242
                 */
                LDAP_CIRCLEQ_MAKE_TAIL( head, c, c_next );

Ondřej Kuzník's avatar
Ondřej Kuzník committed
243
244
245
246
                ldap_pvt_thread_mutex_lock( &backend_mutex );
                current_backend = next;
                ldap_pvt_thread_mutex_unlock( &backend_mutex );

247
                b->b_n_ops_executing++;
248
249
250
251
252
                if ( op->o_tag == LDAP_REQ_BIND ) {
                    b->b_counters[LLOAD_STATS_OPS_BIND].lc_ops_received++;
                } else {
                    b->b_counters[LLOAD_STATS_OPS_OTHER].lc_ops_received++;
                }
253
                c->c_n_ops_executing++;
254
                c->c_counters.lc_ops_received++;
255
                CONNECTION_UNLOCK_INCREF(c);
256

257
                ldap_pvt_thread_mutex_unlock( &b->b_mutex );
258
                *res = LDAP_SUCCESS;
259
260
                return c;
            }
261
            CONNECTION_UNLOCK(c);
262
            ldap_pvt_thread_mutex_unlock( &c->c_io_mutex );
Ondřej Kuzník's avatar
Ondřej Kuzník committed
263
        }
Ondřej Kuzník's avatar
Ondřej Kuzník committed
264
        ldap_pvt_thread_mutex_unlock( &b->b_mutex );
Ondřej Kuzník's avatar
Ondřej Kuzník committed
265
266
267

        b = next;
    } while ( b != first );
Ondřej Kuzník's avatar
Ondřej Kuzník committed
268
269
270
271

    return NULL;
}

272
void
273
backend_retry( LloadBackend *b )
274
275
276
{
    int rc, requested;

Ondřej Kuzník's avatar
Ondřej Kuzník committed
277
278
279
280
281
282
    if ( slapd_shutdown ) {
        Debug( LDAP_DEBUG_CONNS, "backend_retry: "
                "shutting down\n" );
        return;
    }

283
284
285
    ldap_pvt_thread_mutex_lock( &b->b_mutex );

    requested = b->b_numconns;
286
287
288
289
#ifdef LDAP_API_FEATURE_VERIFY_CREDENTIALS
    if ( !(lload_features & LLOAD_FEATURE_VC) )
#endif /* LDAP_API_FEATURE_VERIFY_CREDENTIALS */
    {
290
291
292
        requested += b->b_numbindconns;
    }
    if ( b->b_active + b->b_bindavail + b->b_opening < requested ) {
293
        if ( b->b_opening > 0 || b->b_failed > 0 ) {
294
295
            if ( b->b_failed > 0 &&
                    !event_pending( b->b_retry_event, EV_TIMEOUT, NULL ) ) {
296
297
298
299
300
301
302
303
304
                Debug( LDAP_DEBUG_CONNS, "backend_retry: "
                        "scheduling a retry in %d ms\n",
                        b->b_retry_timeout );
                b->b_opening++;
                event_add( b->b_retry_event, &b->b_retry_tv );
                ldap_pvt_thread_mutex_unlock( &b->b_mutex );
                return;
            } else {
                Debug( LDAP_DEBUG_CONNS, "backend_retry: "
305
                        "retry in progress already\n" );
306
307
308
309
310
311
312
313
314
315
316
317
            }
        } else {
            Debug( LDAP_DEBUG_CONNS, "backend_retry: "
                    "scheduling re-connection straight away\n" );
            b->b_opening++;
            rc = ldap_pvt_thread_pool_submit(
                    &connection_pool, backend_connect_task, b );
            if ( rc ) {
                ldap_pvt_thread_mutex_unlock( &b->b_mutex );
                backend_connect( -1, 0, b );
                return;
            }
318
        }
319
320
321
    } else {
        Debug( LDAP_DEBUG_CONNS, "backend_retry: "
                "no more connections needed for this backend\n" );
322
323
324
325
    }
    ldap_pvt_thread_mutex_unlock( &b->b_mutex );
}

326
327
void
backend_connect( evutil_socket_t s, short what, void *arg )
Ondřej Kuzník's avatar
Ondřej Kuzník committed
328
329
{
    struct evutil_addrinfo hints = {};
330
    LloadBackend *b = arg;
331
    char *hostname;
Ondřej Kuzník's avatar
Ondřej Kuzník committed
332

Ondřej Kuzník's avatar
Ondřej Kuzník committed
333
334
335
336
337
338
    if ( slapd_shutdown ) {
        Debug( LDAP_DEBUG_CONNS, "backend_connect: "
                "doing nothing, shutdown in progress\n" );
        return;
    }

339
    ldap_pvt_thread_mutex_lock( &b->b_mutex );
340
    Debug( LDAP_DEBUG_CONNS, "backend_connect: "
341
342
            "%sattempting connection to %s\n",
            (what & EV_TIMEOUT) ? "retry timeout finished, " : "",
343
344
            b->b_host );

Ondřej Kuzník's avatar
Ondřej Kuzník committed
345
346
347
348
349
350
351
#ifdef LDAP_PF_LOCAL
    if ( b->b_proto == LDAP_PROTO_IPC ) {
        struct sockaddr_un addr;
        ber_socket_t s = socket( PF_LOCAL, SOCK_STREAM, 0 );
        int rc;

        if ( s == AC_SOCKET_INVALID ) {
352
            goto fail;
Ondřej Kuzník's avatar
Ondřej Kuzník committed
353
354
355
356
357
        }

        rc = ber_pvt_socket_set_nonblock( s, 1 );
        if ( rc ) {
            evutil_closesocket( s );
358
            goto fail;
Ondřej Kuzník's avatar
Ondřej Kuzník committed
359
360
361
362
        }

        if ( strlen( b->b_host ) > ( sizeof(addr.sun_path) - 1 ) ) {
            evutil_closesocket( s );
363
            goto fail;
Ondřej Kuzník's avatar
Ondřej Kuzník committed
364
365
366
367
368
369
370
        }
        memset( &addr, '\0', sizeof(addr) );
        addr.sun_family = AF_LOCAL;
        strcpy( addr.sun_path, b->b_host );

        rc = connect(
                s, (struct sockaddr *)&addr, sizeof(struct sockaddr_un) );
371
372
        /* Asynchronous connect */
        if ( rc ) {
373
            LloadPendingConnection *conn;
374
375
376
377
378
379

            if ( errno != EINPROGRESS && errno != EWOULDBLOCK ) {
                evutil_closesocket( s );
                goto fail;
            }

380
            conn = ch_calloc( 1, sizeof(LloadPendingConnection) );
381
382
383
384
            LDAP_LIST_ENTRY_INIT( conn, next );
            conn->backend = b;
            conn->fd = s;

385
            conn->event = event_new( lload_get_base( s ), s,
386
387
388
389
390
391
392
393
394
395
                    EV_WRITE|EV_PERSIST, upstream_connect_cb, conn );
            if ( !conn->event ) {
                Debug( LDAP_DEBUG_ANY, "backend_connect: "
                        "failed to acquire an event to finish upstream "
                        "connection setup.\n" );
                ch_free( conn );
                evutil_closesocket( s );
                goto fail;
            }

396
            event_add( conn->event, lload_timeout_net );
397
398
399
400
401
            LDAP_LIST_INSERT_HEAD( &b->b_connecting, conn, next );
            Debug( LDAP_DEBUG_CONNS, "backend_connect: "
                    "connection to backend uri=%s in progress\n",
                    b->b_uri.bv_val );
        } else if ( !upstream_init( s, b ) ) {
402
403
            goto fail;
        }
404

405
        ldap_pvt_thread_mutex_unlock( &b->b_mutex );
406
        return;
Ondřej Kuzník's avatar
Ondřej Kuzník committed
407
408
409
410
411
412
413
414
    }
#endif /* LDAP_PF_LOCAL */

    hints.ai_family = AF_UNSPEC;
    hints.ai_flags = EVUTIL_AI_CANONNAME;
    hints.ai_socktype = SOCK_STREAM;
    hints.ai_protocol = IPPROTO_TCP;

415
416
417
418
    hostname = b->b_host;
    ldap_pvt_thread_mutex_unlock( &b->b_mutex );

    evdns_getaddrinfo( dnsbase, hostname, NULL, &hints, upstream_name_cb, b );
419
    return;
420
421
422

fail:
    b->b_opening--;
423
    b->b_failed++;
424
425
    ldap_pvt_thread_mutex_unlock( &b->b_mutex );
    backend_retry( b );
426
427
428
429
430
431
432
}

void *
backend_connect_task( void *ctx, void *arg )
{
    backend_connect( -1, 0, arg );
    return NULL;
Ondřej Kuzník's avatar
Ondřej Kuzník committed
433
}
Ondřej Kuzník's avatar
Ondřej Kuzník committed
434
435
436
437

void
backends_destroy( void )
{
438
    while ( !LDAP_CIRCLEQ_EMPTY( &backend ) ) {
439
        LloadBackend *b = LDAP_CIRCLEQ_FIRST( &backend );
Ondřej Kuzník's avatar
Ondřej Kuzník committed
440
441
442

        Debug( LDAP_DEBUG_CONNS, "backends_destroy: "
                "destroying backend uri='%s', numconns=%d, numbindconns=%d\n",
443
                b->b_uri.bv_val, b->b_numconns, b->b_numbindconns );
Ondřej Kuzník's avatar
Ondřej Kuzník committed
444

445
        while ( !LDAP_LIST_EMPTY( &b->b_connecting ) ) {
446
447
            LloadPendingConnection *pending =
                    LDAP_LIST_FIRST( &b->b_connecting );
448
449
450
451
452
453
454
455
456
457

            Debug( LDAP_DEBUG_CONNS, "backends_destroy: "
                    "destroying socket pending connect() fd=%d\n",
                    pending->fd );

            event_free( pending->event );
            evutil_closesocket( pending->fd );
            LDAP_LIST_REMOVE( pending, next );
            ch_free( pending );
        }
Ondřej Kuzník's avatar
Ondřej Kuzník committed
458
        while ( !LDAP_CIRCLEQ_EMPTY( &b->b_preparing ) ) {
459
            LloadConnection *c = LDAP_CIRCLEQ_FIRST( &b->b_preparing );
Ondřej Kuzník's avatar
Ondřej Kuzník committed
460
461
462
463
464
465
466
467
468

            CONNECTION_LOCK(c);
            Debug( LDAP_DEBUG_CONNS, "backends_destroy: "
                    "destroying connection being set up connid=%lu\n",
                    c->c_connid );

            assert( c->c_live );
            CONNECTION_DESTROY(c);
        }
469
        while ( !LDAP_CIRCLEQ_EMPTY( &b->b_bindconns ) ) {
470
            LloadConnection *c = LDAP_CIRCLEQ_FIRST( &b->b_bindconns );
471

Ondřej Kuzník's avatar
Ondřej Kuzník committed
472
            CONNECTION_LOCK(c);
473
474
475
476
477
            Debug( LDAP_DEBUG_CONNS, "backends_destroy: "
                    "destroying bind connection connid=%lu, pending ops=%ld\n",
                    c->c_connid, c->c_n_ops_executing );

            assert( c->c_live );
478
            CONNECTION_DESTROY(c);
Ondřej Kuzník's avatar
Ondřej Kuzník committed
479
        }
480
        while ( !LDAP_CIRCLEQ_EMPTY( &b->b_conns ) ) {
481
            LloadConnection *c = LDAP_CIRCLEQ_FIRST( &b->b_conns );
482

Ondřej Kuzník's avatar
Ondřej Kuzník committed
483
            CONNECTION_LOCK(c);
484
485
486
487
488
489
            Debug( LDAP_DEBUG_CONNS, "backends_destroy: "
                    "destroying regular connection connid=%lu, pending "
                    "ops=%ld\n",
                    c->c_connid, c->c_n_ops_executing );

            assert( c->c_live );
490
            CONNECTION_DESTROY(c);
Ondřej Kuzník's avatar
Ondřej Kuzník committed
491
492
        }

493
        LDAP_CIRCLEQ_REMOVE( &backend, b, b_next );
Ondřej Kuzník's avatar
Ondřej Kuzník committed
494
495
496
497
498
499
        ldap_pvt_thread_mutex_destroy( &b->b_mutex );

        event_del( b->b_retry_event );
        event_free( b->b_retry_event );

        ch_free( b->b_host );
500
        ch_free( b->b_uri.bv_val );
Ondřej Kuzník's avatar
Ondřej Kuzník committed
501
502
503
        ch_free( b );
    }
}