connection reference counting

servers/lloadd/backend.c

@@ -123,6 +123,7 @@ backend_select( Operation *op )
         LDAP_LIST_FOREACH( c, head, c_next )
         {
             ldap_pvt_thread_mutex_lock( &c->c_io_mutex );
+            CONNECTION_LOCK(c);
             if ( c->c_state == SLAP_C_READY && !c->c_pendingber &&
                     ( b->b_max_conn_pending == 0 ||
                             c->c_n_ops_executing < b->b_max_conn_pending ) ) {
@@ -132,9 +133,11 @@ backend_select( Operation *op )
 
                 b->b_n_ops_executing++;
                 c->c_n_ops_executing++;
+                CONNECTION_UNLOCK_INCREF(c);
                 ldap_pvt_thread_mutex_unlock( &b->b_mutex );
                 return c;
             }
+            CONNECTION_UNLOCK(c);
             ldap_pvt_thread_mutex_unlock( &c->c_io_mutex );
         }
         ldap_pvt_thread_mutex_unlock( &b->b_mutex );

servers/lloadd/bind.c

@@ -25,7 +25,8 @@
 #include "slap.h"
 
 /*
- * We hold op->o_upstream->c_io_mutex on entering the function.
+ * On entering the function, we've put a reference on both connections and hold
+ * upstream's c_io_mutex.
  */
 static int
 request_bind( Operation *op )
@@ -65,7 +66,7 @@ request_bind( Operation *op )
         goto fail;
     }
 
-    ldap_pvt_thread_mutex_lock( &client->c_mutex );
+    CONNECTION_LOCK(client);
     if ( !BER_BVISNULL( &client->c_auth ) ) {
         ch_free( client->c_auth.bv_val );
     }
@@ -81,9 +82,9 @@ request_bind( Operation *op )
     } else {
         BER_BVZERO( &client->c_auth );
     }
-    ldap_pvt_thread_mutex_unlock( &client->c_mutex );
+    CONNECTION_UNLOCK(client);
 
-    ldap_pvt_thread_mutex_lock( &upstream->c_mutex );
+    CONNECTION_LOCK(upstream);
     op->o_upstream_msgid = upstream->c_next_msgid++;
 
     ber_printf( ber, "t{titOtO}", LDAP_TAG_MESSAGE,
@@ -99,7 +100,7 @@ request_bind( Operation *op )
                  avl_dup_error ) ) {
         assert(0);
     }
-    ldap_pvt_thread_mutex_unlock( &upstream->c_mutex );
+    CONNECTION_UNLOCK(upstream);
 
     ldap_pvt_thread_mutex_unlock( &upstream->c_io_mutex );
 
@@ -112,13 +113,12 @@ fail:
         ber_free( copy, 0 );
     }
     ldap_pvt_thread_mutex_unlock( &upstream->c_io_mutex );
-    ldap_pvt_thread_mutex_lock( &op->o_client->c_mutex );
-    client_destroy( op->o_client );
     return 1;
 }
 
 /*
- * We hold op->o_upstream->c_io_mutex on entering the function.
+ * On entering the function, we've put a reference on both connections and hold
+ * upstream's c_io_mutex.
  */
 static int
 request_bind_as_vc( Operation *op )
@@ -166,7 +166,7 @@ request_bind_as_vc( Operation *op )
 
     op->o_upstream_msgid = upstream->c_next_msgid++;
 
-    ldap_pvt_thread_mutex_lock( &upstream->c_mutex );
+    CONNECTION_LOCK(upstream);
     ber_printf( ber, "t{tit{tst{{tOOtOtO}}}}", LDAP_TAG_MESSAGE,
             LDAP_TAG_MSGID, op->o_upstream_msgid,
             LDAP_REQ_EXTENDED,
@@ -175,23 +175,23 @@ request_bind_as_vc( Operation *op )
             LDAP_TAG_EXOP_VERIFY_CREDENTIALS_COOKIE, BER_BV_OPTIONAL( &upstream->c_vc_cookie ),
             &binddn, tag, &auth,
             LDAP_TAG_EXOP_VERIFY_CREDENTIALS_CONTROLS, BER_BV_OPTIONAL( &op->o_ctrls ) );
-    ldap_pvt_thread_mutex_unlock( &upstream->c_mutex );
+    CONNECTION_UNLOCK(upstream);
 
     tag = ber_peek_tag( copy, &len );
     switch ( tag ) {
         case LDAP_AUTH_SASL:
             ber_get_stringbv( copy, &mech, LBER_BV_NOTERM );
 
-            ldap_pvt_thread_mutex_lock( &client->c_mutex );
+            CONNECTION_LOCK(client);
             if ( ber_bvcmp( &mech, &client->c_sasl_bind_mech ) ) {
                 ber_memfree( client->c_sasl_bind_mech.bv_val );
                 ber_dupbv( &client->c_sasl_bind_mech, &mech );
             }
-            ldap_pvt_thread_mutex_unlock( &client->c_mutex );
+            CONNECTION_UNLOCK(client);
             /* TODO: extract authzdn from the message */
             break;
         case LDAP_AUTH_SIMPLE:
-            ldap_pvt_thread_mutex_lock( &client->c_mutex );
+            CONNECTION_LOCK(client);
             if ( !BER_BVISNULL( &client->c_auth ) ) {
                 ch_free( client->c_auth.bv_val );
             }
@@ -211,7 +211,7 @@ request_bind_as_vc( Operation *op )
                 ber_memfree( client->c_sasl_bind_mech.bv_val );
                 BER_BVZERO( &client->c_sasl_bind_mech );
             }
-            ldap_pvt_thread_mutex_unlock( &client->c_mutex );
+            CONNECTION_UNLOCK(client);
             break;
         default:
             result = LDAP_PROTOCOL_ERROR;
@@ -219,7 +219,7 @@ request_bind_as_vc( Operation *op )
             goto fail;
     }
 
-    ldap_pvt_thread_mutex_lock( &upstream->c_mutex );
+    CONNECTION_LOCK(upstream);
     Debug( LDAP_DEBUG_TRACE, "request_bind_as_vc: "
             "added bind from client connid=%lu to upstream connid=%lu as VC "
             "exop msgid=%d\n",
@@ -228,12 +228,13 @@ request_bind_as_vc( Operation *op )
                  avl_dup_error ) ) {
         assert(0);
     }
-    ldap_pvt_thread_mutex_unlock( &upstream->c_mutex );
+    CONNECTION_UNLOCK(upstream);
 
     ldap_pvt_thread_mutex_unlock( &upstream->c_io_mutex );
 
     ber_free( copy, 0 );
     upstream_write_cb( -1, 0, upstream );
+
     return 0;
 
 fail:
@@ -242,8 +243,6 @@ fail:
     }
     ldap_pvt_thread_mutex_unlock( &upstream->c_io_mutex );
     operation_send_reject( op, result, msg, 1 );
-    ldap_pvt_thread_mutex_lock( &client->c_mutex );
-    client_destroy( client );
     return 1;
 }
 
@@ -255,7 +254,7 @@ client_reset( void *ctx, void *arg )
     TAvlnode *root;
     int freed, destroy = 1;
 
-    ldap_pvt_thread_mutex_lock( &c->c_mutex );
+    CONNECTION_LOCK(c);
     root = c->c_ops;
     c->c_ops = NULL;
     c->c_state = SLAP_C_CLOSING;
@@ -271,7 +270,7 @@ client_reset( void *ctx, void *arg )
         ch_free( c->c_sasl_bind_mech.bv_val );
         BER_BVZERO( &c->c_sasl_bind_mech );
     }
-    ldap_pvt_thread_mutex_unlock( &c->c_mutex );
+    CONNECTION_UNLOCK_INCREF(c);
 
     tavl_delete( &root, op, operation_client_cmp );
     freed = tavl_free( root, (AVL_FREE)operation_abandon );
@@ -282,8 +281,10 @@ client_reset( void *ctx, void *arg )
 
     if ( destroy ) {
         operation_destroy( op );
-        ldap_pvt_thread_mutex_lock( &c->c_mutex );
-        client_destroy( c );
+        CLIENT_LOCK_DESTROY(c);
+    } else {
+        CONNECTION_LOCK_DECREF(c);
+        CLIENT_UNLOCK_OR_DESTROY(c);
     }
 
     return NULL;
@@ -296,6 +297,9 @@ client_bind( void *ctx, void *arg )
     Connection *upstream, *client = op->o_client;
     int rc = 0;
 
+    CONNECTION_LOCK(client);
+    CONNECTION_UNLOCK_INCREF(client);
+
     client_reset( ctx, arg );
 
     upstream = backend_select( op );
@@ -304,6 +308,8 @@ client_bind( void *ctx, void *arg )
                 "no available connection found\n" );
         operation_send_reject(
                 op, LDAP_UNAVAILABLE, "no connections available", 1 );
+        CONNECTION_LOCK_DECREF(client);
+        CLIENT_UNLOCK_OR_DESTROY(client);
         return NULL;
     }
 
@@ -315,15 +321,18 @@ client_bind( void *ctx, void *arg )
         rc = request_bind( op );
     }
 
+    CONNECTION_LOCK_DECREF(upstream);
+    UPSTREAM_UNLOCK_OR_DESTROY(upstream);
+
     if ( rc ) {
-        /* client doesn't exist anymore */
+        CLIENT_LOCK_DESTROY(client);
         return NULL;
     }
 
-    ldap_pvt_thread_mutex_lock( &client->c_mutex );
+    CONNECTION_LOCK_DECREF(client);
     rc = tavl_insert( &client->c_ops, op, operation_client_cmp, avl_dup_error );
     assert( rc == LDAP_SUCCESS );
-    ldap_pvt_thread_mutex_unlock( &client->c_mutex );
+    CLIENT_UNLOCK_OR_DESTROY(client);
 
     return NULL;
 }

servers/lloadd/client.c

@@ -34,7 +34,10 @@ client_read_cb( evutil_socket_t s, short what, void *arg )
     ber_len_t len;
     int rc = 0;
 
-    ldap_pvt_thread_mutex_lock( &c->c_mutex );
+    /* What if the shutdown is already in progress and we get to lock the
+     * connection? */
+    CONNECTION_LOCK(c);
+
     Debug( LDAP_DEBUG_CONNS, "client_read_cb: "
             "connection %lu ready to read\n",
             c->c_connid );
@@ -43,6 +46,7 @@ client_read_cb( evutil_socket_t s, short what, void *arg )
     if ( ber == NULL && (ber = ber_alloc()) == NULL ) {
         Debug( LDAP_DEBUG_ANY, "client_read_cb: "
                 "ber_alloc failed\n" );
+        CLIENT_DESTROY(c);
         goto fail;
     }
 
@@ -57,10 +61,11 @@ client_read_cb( evutil_socket_t s, short what, void *arg )
                     c->c_fd, err, sock_errstr( err, ebuf, sizeof(ebuf) ) );
 
             c->c_currentber = NULL;
+            CLIENT_DESTROY(c);
             goto fail;
         }
         c->c_currentber = ber;
-        ldap_pvt_thread_mutex_unlock( &c->c_mutex );
+        CONNECTION_UNLOCK(c);
         return;
     }
 
@@ -70,21 +75,21 @@ client_read_cb( evutil_socket_t s, short what, void *arg )
     if ( !op ) {
         Debug( LDAP_DEBUG_ANY, "client_read_cb: "
                 "operation_init failed\n" );
-        ber_free( ber, 1 );
+        CLIENT_DESTROY(c);
         goto fail;
     }
 
     switch ( op->o_tag ) {
         case LDAP_REQ_UNBIND:
-            /* We do not expect anything more from the client */
+            /* We do not expect anything more from the client. Also, we are the
+             * read event, so don't need to unlock */
             event_del( c->c_read_event );
 
             rc = ldap_pvt_thread_pool_submit(
                     &connection_pool, client_reset, op );
             if ( rc ) {
-                tavl_delete( &c->c_ops, op, operation_client_cmp );
-                operation_destroy( op );
-                client_destroy( c );
+                CONNECTION_UNLOCK(c);
+                client_reset( NULL, op );
                 return;
             }
             break;
@@ -94,7 +99,7 @@ client_read_cb( evutil_socket_t s, short what, void *arg )
             break;
         default:
             if ( c->c_state == SLAP_C_BINDING ) {
-                ldap_pvt_thread_mutex_unlock( &c->c_mutex );
+                CONNECTION_UNLOCK(c);
                 operation_send_reject(
                         op, LDAP_PROTOCOL_ERROR, "bind in progress", 0 );
                 return;
@@ -104,8 +109,11 @@ client_read_cb( evutil_socket_t s, short what, void *arg )
             break;
     }
 
+    /* FIXME: unlocks in this function need more thought when we refcount
+     * operations */
+    CONNECTION_UNLOCK(c);
+
     if ( !rc ) {
-        ldap_pvt_thread_mutex_unlock( &c->c_mutex );
         return;
     }
 
@@ -113,13 +121,11 @@ fail:
     if ( op ) {
         operation_send_reject(
                 op, LDAP_OTHER, "server error or overloaded", 1 );
-        op->o_client = NULL;
         operation_destroy( op );
     } else if ( ber ) {
         ber_free( ber, 1 );
     }
 
-    client_destroy( c );
     return;
 }
 
@@ -128,6 +134,11 @@ client_write_cb( evutil_socket_t s, short what, void *arg )
 {
     Connection *c = arg;
 
+    /* What if the shutdown is already in progress and we get to lock the
+     * connection? */
+    CONNECTION_LOCK(c);
+    CONNECTION_UNLOCK_INCREF(c);
+
     ldap_pvt_thread_mutex_lock( &c->c_io_mutex );
     Debug( LDAP_DEBUG_CONNS, "client_write_cb: "
             "have something to write to client %lu\n",
@@ -136,15 +147,17 @@ client_write_cb( evutil_socket_t s, short what, void *arg )
     if ( ber_flush( c->c_sb, c->c_pendingber, 1 ) ) {
         int err = sock_errno();
         if ( err != EWOULDBLOCK && err != EAGAIN ) {
-            ldap_pvt_thread_mutex_lock( &c->c_mutex );
             ldap_pvt_thread_mutex_unlock( &c->c_io_mutex );
-            client_destroy( c );
+            CLIENT_LOCK_DESTROY(c);
             return;
         }
         event_add( c->c_write_event, NULL );
     }
     c->c_pendingber = NULL;
     ldap_pvt_thread_mutex_unlock( &c->c_io_mutex );
+
+    CONNECTION_LOCK_DECREF(c);
+    CLIENT_UNLOCK_OR_DESTROY(c);
 }
 
 Connection *
@@ -183,7 +196,7 @@ client_init(
     c->c_write_event = event;
 
     c->c_private = listener;
-    ldap_pvt_thread_mutex_unlock( &c->c_mutex );
+    CONNECTION_UNLOCK(c);
 
     return c;
 fail:

servers/lloadd/connection.c

@@ -58,7 +58,10 @@ connection_destroy( Connection *c )
             "destroying connection %lu.\n",
             c->c_connid );
 
+    assert( c->c_live == 0 );
+    assert( c->c_refcnt == 0 );
     assert( c->c_state == SLAP_C_INVALID );
+
     ber_sockbuf_free( c->c_sb );
 
     if ( c->c_currentber ) {
@@ -68,7 +71,7 @@ connection_destroy( Connection *c )
         ber_free( c->c_pendingber, 1 );
     }
 
-    ldap_pvt_thread_mutex_unlock( &c->c_mutex );
+    CONNECTION_UNLOCK(c);
 
     ldap_pvt_thread_mutex_destroy( &c->c_io_mutex );
     ldap_pvt_thread_mutex_destroy( &c->c_mutex );
@@ -148,6 +151,7 @@ connection_init( ber_socket_t s, const char *peername, int flags )
 #endif
 
     c->c_next_msgid = 1;
+    c->c_refcnt = c->c_live = 1;
 
     ldap_pvt_thread_mutex_init( &c->c_mutex );
     ldap_pvt_thread_mutex_init( &c->c_io_mutex );
@@ -158,7 +162,7 @@ connection_init( ber_socket_t s, const char *peername, int flags )
             "connection connid=%lu allocated for socket fd=%d\n",
             c->c_connid, s );
 
-    ldap_pvt_thread_mutex_lock( &c->c_mutex );
+    CONNECTION_LOCK(c);
     c->c_state = SLAP_C_ACTIVE;
 
     return c;

servers/lloadd/operation.c

@@ -114,23 +114,23 @@ operation_destroy( Operation *op )
      * something in to test with until we implement the freelist */
     if ( op->o_client ) {
         c = op->o_client;
-        ldap_pvt_thread_mutex_lock( &c->c_mutex );
+        CONNECTION_LOCK(c);
         if ( tavl_delete( &c->c_ops, op, operation_client_cmp ) ) {
             c->c_n_ops_executing--;
         }
-        ldap_pvt_thread_mutex_unlock( &c->c_mutex );
+        CONNECTION_UNLOCK(c);
     }
 
     if ( op->o_upstream ) {
         Backend *b = NULL;
 
         c = op->o_upstream;
-        ldap_pvt_thread_mutex_lock( &c->c_mutex );
+        CONNECTION_LOCK(c);
         if ( tavl_delete( &c->c_ops, op, operation_upstream_cmp ) ) {
             c->c_n_ops_executing--;
             b = (Backend *)c->c_private;
         }
-        ldap_pvt_thread_mutex_unlock( &c->c_mutex );
+        CONNECTION_UNLOCK(c);
 
         if ( b ) {
             ldap_pvt_thread_mutex_lock( &b->b_mutex );
@@ -142,6 +142,9 @@ operation_destroy( Operation *op )
     ch_free( op );
 }
 
+/*
+ * Entered holding c_mutex for now.
+ */
 Operation *
 operation_init( Connection *c, BerElement *ber )
 {
@@ -208,13 +211,13 @@ operation_abandon( Operation *op )
         BerElement *ber;
         Backend *b;
 
-        ldap_pvt_thread_mutex_lock( &c->c_mutex );
+        CONNECTION_LOCK(c);
         rc = ( tavl_delete( &c->c_ops, op, operation_upstream_cmp ) == NULL );
         if ( !rc ) {
             c->c_n_ops_executing--;
         }
         b = (Backend *)c->c_private;
-        ldap_pvt_thread_mutex_unlock( &c->c_mutex );
+        CONNECTION_UNLOCK_INCREF(c);
 
         if ( rc ) {
             /* The operation has already been abandoned or finished */
@@ -231,7 +234,9 @@ operation_abandon( Operation *op )
         if ( ber == NULL && (ber = ber_alloc()) == NULL ) {
             Debug( LDAP_DEBUG_ANY, "operation_abandon: "
                     "ber_alloc failed\n" );
+            CONNECTION_LOCK_DECREF(c);
             ldap_pvt_thread_mutex_unlock( &c->c_io_mutex );
+            UPSTREAM_UNLOCK_OR_DESTROY(c);
             goto done;
         }
         c->c_pendingber = ber;
@@ -244,7 +249,9 @@ operation_abandon( Operation *op )
             ber_free( ber, 1 );
         }
 
+        CONNECTION_LOCK_DECREF(c);
         ldap_pvt_thread_mutex_unlock( &c->c_io_mutex );
+        UPSTREAM_UNLOCK_OR_DESTROY(c);
 
         if ( rc != -1 ) {
             upstream_write_cb( -1, 0, c );
@@ -270,20 +277,20 @@ operation_send_reject(
             "rejecting %s from client %lu with message: \"%s\"\n",
             slap_msgtype2str( op->o_tag ), op->o_client_connid, msg );
 
-    ldap_pvt_thread_mutex_lock( &c->c_mutex );
+    CONNECTION_LOCK(c);
     found = ( tavl_delete( &c->c_ops, op, operation_client_cmp ) == op );
-    ldap_pvt_thread_mutex_unlock( &c->c_mutex );
-
     if ( !found && !send_anyway ) {
+        CONNECTION_UNLOCK(c);
         return;
     }
 
+    CONNECTION_UNLOCK_INCREF(c);
     ldap_pvt_thread_mutex_lock( &c->c_io_mutex );
 
     ber = c->c_pendingber;
     if ( ber == NULL && (ber = ber_alloc()) == NULL ) {
         ldap_pvt_thread_mutex_unlock( &c->c_io_mutex );
-        client_destroy( c );
+        CLIENT_LOCK_DESTROY(c);
         return;
     }
     c->c_pendingber = ber;
@@ -297,6 +304,9 @@ operation_send_reject(
     client_write_cb( -1, 0, c );
 
     operation_destroy( op );
+
+    CONNECTION_LOCK_DECREF(c);
+    CLIENT_UNLOCK_OR_DESTROY(c);
 }
 
 void
@@ -304,6 +314,7 @@ operation_lost_upstream( Operation *op )
 {
     operation_send_reject( op, LDAP_UNAVAILABLE,
             "connection to the remote server has been severed", 0 );
+    operation_destroy( op );
 }
 
 void *
@@ -330,15 +341,15 @@ request_process( void *ctx, void *arg )
     }
     upstream->c_pendingber = output;
 
-    ldap_pvt_thread_mutex_unlock( &upstream->c_mutex );
+    CONNECTION_LOCK_DECREF(upstream);
     op->o_upstream_msgid = msgid = upstream->c_next_msgid++;
     rc = tavl_insert(
             &upstream->c_ops, op, operation_upstream_cmp, avl_dup_error );
-    ldap_pvt_thread_mutex_unlock( &upstream->c_mutex );
+    CONNECTION_UNLOCK_INCREF(upstream);
     assert( rc == LDAP_SUCCESS );
 
     if ( lload_features & LLOAD_FEATURE_PROXYAUTHZ ) {
-        ldap_pvt_thread_mutex_lock( &client->c_mutex );
+        CONNECTION_LOCK(client);
         Debug( LDAP_DEBUG_TRACE, "request_process: "
                 "proxying identity %s to upstream\n",
                 client->c_auth.bv_val );
@@ -347,7 +358,7 @@ request_process( void *ctx, void *arg )
                 op->o_tag, &op->o_request,
                 LDAP_TAG_CONTROLS,
                 LDAP_CONTROL_PROXY_AUTHZ, 1, &client->c_auth );
-        ldap_pvt_thread_mutex_unlock( &client->c_mutex );
+        CONNECTION_UNLOCK(client);
 
         if ( !BER_BVISNULL( &op->o_ctrls ) ) {
             BerElement *control_ber = ber_alloc();
@@ -373,11 +384,16 @@ request_process( void *ctx, void *arg )
 
     upstream_write_cb( -1, 0, upstream );
 
+    CONNECTION_LOCK_DECREF(upstream);
+    UPSTREAM_UNLOCK_OR_DESTROY(upstream);
+
     return NULL;
 
 fail:
     if ( upstream ) {
         ldap_pvt_thread_mutex_unlock( &upstream->c_io_mutex );
+        CONNECTION_LOCK_DECREF(upstream);
+        UPSTREAM_UNLOCK_OR_DESTROY(upstream);
     }
     operation_send_reject( op, LDAP_OTHER, "internal error", 0 );
     return NULL;

servers/lloadd/slap.h

@@ -283,8 +283,74 @@ struct Connection {
     enum sc_type c_type;
     ber_socket_t c_fd;
 
+/*
+ * Connection reference counting:
+ * - connection has a reference counter in c_refcnt
+ * - also a liveness/validity token is added to c_refcnt during
+ *   connection_init, its existence is tracked in c_live and is usually the
+ *   only one that prevents it from being destroyed
+ * - anyone who needs to be able to lock the connection after unlocking it has
+ *   to use CONNECTION_UNLOCK_INCREF, they are then responsible that
+ *   CONNECTION_LOCK_DECREF+CONNECTION_UNLOCK_OR_DESTROY is used when they are
+ *   done with it
+ * - when a connection is considered dead, use (UPSTREAM|CLIENT)_DESTROY on a
+ *   locked connection, it might get disposed of or if anyone still holds a
+ *   token, it just gets unlocked and it's the last token holder's
+ *   responsibility to run *_UNLOCK_OR_DESTROY
+ * - (UPSTREAM|CLIENT)_LOCK_DESTROY is a shorthand for locking, decreasing
+ *   refcount and (UPSTREAM|CLIENT)_DESTROY
+ */
     ldap_pvt_thread_mutex_t c_mutex; /* protect the connection */
-    Sockbuf *c_sb;                   /* ber connection stuff */
+    int c_refcnt, c_live;
+#define CONNECTION_LOCK(c) ldap_pvt_thread_mutex_lock( &(c)->c_mutex )
+#define CONNECTION_UNLOCK(c) ldap_pvt_thread_mutex_unlock( &(c)->c_mutex )
+#define CONNECTION_LOCK_DECREF(c) \
+    do { \
+        CONNECTION_LOCK(c); \
+        (c)->c_refcnt--; \
+    } while (0)
+#define CONNECTION_UNLOCK_INCREF(c) \
+    do { \
+        (c)->c_refcnt++; \
+        CONNECTION_UNLOCK(c); \
+    } while (0)
+#define CONNECTION_UNLOCK_OR_DESTROY(type, c) \
+    do { \
+        assert( (c)->c_refcnt >= 0 ); \
+        if ( !( c )->c_refcnt ) { \
+            Debug( LDAP_DEBUG_TRACE, "%s: destroying " #type " connection connid=%lu\n", \
+                    __func__, (c)->c_connid ); \
+            type##_destroy( (c) ); \
+            (c) = NULL; \
+        } else { \
+            CONNECTION_UNLOCK(c); \
+        } \
+    } while (0)
+#define CONNECTION_DESTROY(type, c) \
+    do { \
+        (c)->c_refcnt -= (c)->c_live; \
+        (c)->c_live = 0; \
+        CONNECTION_UNLOCK_OR_DESTROY(type, c); \
+    } while (0)
+
+#define UPSTREAM_UNLOCK_OR_DESTROY(c) \
+    CONNECTION_UNLOCK_OR_DESTROY(upstream, c);
+#define UPSTREAM_DESTROY(c) CONNECTION_DESTROY(upstream, c)
+#define UPSTREAM_LOCK_DESTROY(c) \
+    do { \
+        CONNECTION_LOCK_DECREF(c); \
+        UPSTREAM_DESTROY(c); \
+    } while (0);
+
+#define CLIENT_UNLOCK_OR_DESTROY(c) CONNECTION_UNLOCK_OR_DESTROY(client, c);
+#define CLIENT_DESTROY(c) CONNECTION_DESTROY(client, c)
+#define CLIENT_LOCK_DESTROY(c) \
+    do { \
+        CONNECTION_LOCK_DECREF(c); \
+        CLIENT_DESTROY(c); \
+    } while (0);
+
+    Sockbuf *c_sb; /* ber connection stuff */
 
     /* set by connection_init */
     unsigned long c_connid;    /* unique id of this connection */
@@ -294,6 +360,7 @@ struct Connection {
     time_t c_activitytime;  /* when the connection was last used */
     ber_int_t c_next_msgid; /* msgid of the next message */
 
+    /* must not be used while holding either mutex */