ITS#9599 Implement tiered load balancing

doc/man/man5/lloadd.conf.5

@@ -644,7 +644,13 @@ only valid when using GnuTLS and Mozilla NSS.
 .SH BACKEND CONFIGURATION
 Options in this section describe how the
 .B lloadd
-connects and authenticates to the backend servers.
+connects and authenticates to the backend servers. Backends are organised in groups
+.RB ( tiers ).
+Backends in the first tier are tried first, if none of them are reachable, the
+following tier is tried in the same way. If there is a backend in the tier that
+has suitable connections, but they are busy, no further tier is consulted. This
+is useful in high availability scenarios where a group of servers (e.g. the
+local environment) should be contacted if possible.
 
 It is assumed all backend servers serve the same data. On startup, the
 configured connections are set up and those not dedicated to handle bind
@@ -730,6 +736,33 @@ set on the upstream connections, overriding the operating system setting.
 Only some systems support the customization of this parameter, it is
 ignored otherwise and system-wide settings are used.
 
+.SH TIER OPTIONS
+
+.TP
+.B tier
+.B <tier type>
+
+Groups servers which should be considered in the same try. If a viable
+connection is found even if busy, the load balancer does not proceed to the
+next tier. The process of selection a connection within a tier depends on the
+tier's type.
+
+.RE
+Available types are:
+.TP
+.B roundrobin
+Servers are tried in order and if one is selected successfully, the following
+search will try from the one next on the list.
+.TP
+.B weighted
+Backend servers accept a new option
+.B weight=<int>
+which indicates how often it should be selected. If unspecified, weight
+defaults to 0 and such backends have a slight chance of being selected even
+when a non-zero weight backend is configured in the tier. The selection process
+is along the lines of
+.BR RFC2782 .
+
 .SH BACKEND OPTIONS
 
 .TP
@@ -879,6 +912,7 @@ bindconf
     binddn=cn=test
     credentials=pass
 
+tier weighted
 backend-server
     uri=ldap://ldap1.example.com
     numconns=3
@@ -886,6 +920,7 @@ backend-server
     retry=5000
     max-pending-ops=5
     conn-max-pending=3
+    weight=5
 
 backend-server
     uri=ldap://ldap2.example.com
@@ -894,6 +929,7 @@ backend-server
     retry=5000
     max-pending-ops=5
     conn-max-pending=3
+    weight=10
 .fi
 .RE
 .LP

servers/lloadd/Makefile.in

@@ -21,6 +21,7 @@ NT_OBJS = nt_svc.o ../../libraries/liblutil/slapdmsg.res
 
 SRCS	= backend.c bind.c config.c connection.c client.c \
 		  daemon.c epoch.c extended.c init.c operation.c \
+		  tier.c tier_roundrobin.c tier_weighted.c \
 		  upstream.c libevent_support.c \
 		  $(@PLAT@_SRCS)
 

servers/lloadd/backend.c

@@ -392,44 +392,17 @@ upstream_select(
         int *res,
         char **message )
 {
-    LloadBackend *b, *first, *next;
-    int rc = 0;
+    LloadTier *tier;
+    int finished = 0;
 
-    checked_lock( &backend_mutex );
-    first = b = current_backend;
-    checked_unlock( &backend_mutex );
-
-    *res = LDAP_UNAVAILABLE;
-
-    if ( !first ) {
-        return NULL;
-    }
-
-    /* TODO: Two runs, one with trylock, then one actually locked if we don't
-     * find anything? */
-    do {
-        checked_lock( &b->b_mutex );
-        next = LDAP_CIRCLEQ_LOOP_NEXT( &backend, b, b_next );
-
-        rc = backend_select( b, op, cp, res, message );
-        checked_unlock( &b->b_mutex );
-
-        if ( rc && *cp ) {
-            /*
-             * Round-robin step:
-             * Rotate the queue to put this backend at the end. The race here
-             * is acceptable.
-             */
-            checked_lock( &backend_mutex );
-            current_backend = next;
-            checked_unlock( &backend_mutex );
-            return rc;
+    LDAP_STAILQ_FOREACH( tier, &tiers, t_next ) {
+        if ( (finished = tier->t_type.tier_select(
+                       tier, op, cp, res, message )) ) {
+            break;
         }
+    }
 
-        b = next;
-    } while ( b != first );
-
-    return rc;
+    return finished;
 }
 
 /*
@@ -726,26 +699,41 @@ backend_reset( LloadBackend *b, int gentle )
     assert_locked( &b->b_mutex );
 }
 
+LloadBackend *
+lload_backend_new( void )
+{
+    LloadBackend *b;
+
+    b = ch_calloc( 1, sizeof(LloadBackend) );
+
+    LDAP_CIRCLEQ_INIT( &b->b_conns );
+    LDAP_CIRCLEQ_INIT( &b->b_bindconns );
+    LDAP_CIRCLEQ_INIT( &b->b_preparing );
+    LDAP_CIRCLEQ_ENTRY_INIT( b, b_next );
+
+    b->b_numconns = 1;
+    b->b_numbindconns = 1;
+    b->b_weight = 1;
+
+    b->b_retry_timeout = 5000;
+
+    ldap_pvt_thread_mutex_init( &b->b_mutex );
+
+    return b;
+}
+
 void
 lload_backend_destroy( LloadBackend *b )
 {
-    LloadBackend *next = LDAP_CIRCLEQ_LOOP_NEXT( &backend, b, b_next );
-
     Debug( LDAP_DEBUG_CONNS, "lload_backend_destroy: "
             "destroying backend uri='%s', numconns=%d, numbindconns=%d\n",
             b->b_uri.bv_val, b->b_numconns, b->b_numbindconns );
 
     checked_lock( &b->b_mutex );
+    b->b_tier->t_type.tier_remove_backend( b->b_tier, b );
     b->b_numconns = b->b_numbindconns = 0;
     backend_reset( b, 0 );
 
-    LDAP_CIRCLEQ_REMOVE( &backend, b, b_next );
-    if ( b == next ) {
-        current_backend = NULL;
-    } else {
-        current_backend = next;
-    }
-
 #ifdef BALANCER_MODULE
     if ( b->b_monitor ) {
         BackendDB *be;
@@ -760,6 +748,7 @@ lload_backend_destroy( LloadBackend *b )
         assert( rc == LDAP_SUCCESS );
     }
 #endif /* BALANCER_MODULE */
+
     checked_unlock( &b->b_mutex );
     ldap_pvt_thread_mutex_destroy( &b->b_mutex );
 
@@ -774,13 +763,3 @@ lload_backend_destroy( LloadBackend *b )
     ch_free( b->b_name.bv_val );
     ch_free( b );
 }
-
-void
-lload_backends_destroy( void )
-{
-    while ( !LDAP_CIRCLEQ_EMPTY( &backend ) ) {
-        LloadBackend *b = LDAP_CIRCLEQ_FIRST( &backend );
-
-        lload_backend_destroy( b );
-    }
-}

servers/lloadd/config.c

@@ -113,6 +113,7 @@ static ConfigFile *cfn;
 
 static ConfigDriver config_fname;
 static ConfigDriver config_generic;
+static ConfigDriver config_tier;
 static ConfigDriver config_backend;
 static ConfigDriver config_bindconf;
 static ConfigDriver config_restrict_oid;
@@ -132,10 +133,6 @@ static ConfigDriver config_share_tls_ctx;
 static ConfigDriver backend_cf_gen;
 #endif /* BALANCER_MODULE */
 
-lload_b_head backend = LDAP_CIRCLEQ_HEAD_INITIALIZER(backend);
-ldap_pvt_thread_mutex_t backend_mutex;
-LloadBackend *current_backend = NULL;
-
 struct slap_bindconf bindconf = {};
 struct berval lloadd_identity = BER_BVNULL;
 
@@ -182,6 +179,8 @@ enum {
     CFG_CLIENT_PENDING,
     CFG_RESTRICT_EXOP,
     CFG_RESTRICT_CONTROL,
+    CFG_TIER,
+    CFG_WEIGHT,
 
     CFG_LAST
 };
@@ -205,6 +204,17 @@ static ConfigTable config_back_cf_table[] = {
         &config_generic,
         NULL, NULL, NULL
     },
+    { "tier", "name", 2, 2, 0,
+        ARG_MAGIC|ARG_STRING|CFG_TIER,
+        &config_tier,
+        "( OLcfgBkAt:13.39 "
+            "NAME 'olcBkLloadTierType' "
+            "DESC 'Tier type' "
+            "EQUALITY caseIgnoreMatch "
+            "SYNTAX OMsDirectoryString "
+            "SINGLE-VALUE )",
+        NULL, NULL
+    },
     /* conf-file only option */
     { "backend-server", "backend options", 2, 0, 0,
         ARG_MAGIC|CFG_BACKEND,
@@ -747,6 +757,17 @@ static ConfigTable config_back_cf_table[] = {
             "SINGLE-VALUE )",
         NULL, NULL
     },
+    { "", NULL, 2, 2, 0,
+        ARG_MAGIC|ARG_UINT|CFG_WEIGHT,
+        &backend_cf_gen,
+        "( OLcfgBkAt:13.40 "
+            "NAME 'olcBkLloadWeight' "
+            "DESC 'Backend weight' "
+            "SYNTAX OMsInteger "
+            "SINGLE-VALUE )",
+        NULL,
+        { .v_uint = 0 },
+    },
 #endif /* BALANCER_MODULE */
 
     { NULL, NULL, 0, 0, 0, ARG_IGNORED, NULL }
@@ -754,9 +775,13 @@ static ConfigTable config_back_cf_table[] = {
 
 #ifdef BALANCER_MODULE
 static ConfigCfAdd lload_cfadd;
+
 static ConfigLDAPadd lload_backend_ldadd;
+static ConfigLDAPadd lload_tier_ldadd;
+
 #ifdef SLAP_CONFIG_DELETE
 static ConfigLDAPdel lload_backend_lddel;
+static ConfigLDAPdel lload_tier_lddel;
 #endif /* SLAP_CONFIG_DELETE */
 
 static ConfigOCs lloadocs[] = {
@@ -807,12 +832,27 @@ static ConfigOCs lloadocs[] = {
             "$ olcBkLloadMaxPendingOps "
             "$ olcBkLloadMaxPendingConns ) "
         "MAY ( olcBkLloadStartTLS "
+            "$ olcBkLloadWeight ) "
         ") )",
         Cft_Misc, config_back_cf_table,
         lload_backend_ldadd,
         NULL,
 #ifdef SLAP_CONFIG_DELETE
         lload_backend_lddel,
+#endif /* SLAP_CONFIG_DELETE */
+    },
+    { "( OLcfgBkOc:13.3 "
+        "NAME 'olcBkLloadTierConfig' "
+        "DESC 'Lload tier configuration' "
+        "SUP olcConfig STRUCTURAL "
+        "MUST ( cn "
+            "$ olcBkLloadTierType "
+        ") )",
+        Cft_Misc, config_back_cf_table,
+        lload_tier_ldadd,
+        NULL,
+#ifdef SLAP_CONFIG_DELETE
+        lload_tier_lddel,
 #endif /* SLAP_CONFIG_DELETE */
     },
     { NULL, 0, NULL }
@@ -1073,6 +1113,26 @@ lload_backend_finish( ConfigArgs *ca )
         b->b_retry_event = event;
     }
 
+    if ( BER_BVISEMPTY( &b->b_name ) ) {
+        struct berval bv;
+        LloadBackend *b2;
+        int i = 1;
+
+        LDAP_CIRCLEQ_FOREACH ( b2, &b->b_tier->t_backends, b_next ) {
+            i++;
+        }
+
+        bv.bv_val = ca->cr_msg;
+        bv.bv_len =
+                snprintf( ca->cr_msg, sizeof(ca->cr_msg), "server %d", i );
+
+        ber_dupbv( &b->b_name, &bv );
+    }
+
+    if ( b->b_tier->t_type.tier_add_backend( b->b_tier, b ) ) {
+        goto fail;
+    }
+
     return LDAP_SUCCESS;
 
 fail:
@@ -1085,28 +1145,6 @@ fail:
     return -1;
 }
 
-static LloadBackend *
-backend_alloc( void )
-{
-    LloadBackend *b;
-
-    b = ch_calloc( 1, sizeof(LloadBackend) );
-
-    LDAP_CIRCLEQ_INIT( &b->b_conns );
-    LDAP_CIRCLEQ_INIT( &b->b_bindconns );
-    LDAP_CIRCLEQ_INIT( &b->b_preparing );
-
-    b->b_numconns = 1;
-    b->b_numbindconns = 1;
-
-    b->b_retry_timeout = 5000;
-
-    ldap_pvt_thread_mutex_init( &b->b_mutex );
-
-    LDAP_CIRCLEQ_INSERT_TAIL( &backend, b, b_next );
-    return b;
-}
-
 static int
 backend_config_url( LloadBackend *b, struct berval *uri )
 {
@@ -1183,16 +1221,29 @@ static int
 config_backend( ConfigArgs *c )
 {
     LloadBackend *b;
+    LloadTier *tier;
     int i, rc = 0;
 
-    b = backend_alloc();
+    tier = LDAP_STAILQ_LAST( &tiers, LloadTier, t_next );
+    if ( !tier ) {
+        Debug( LDAP_DEBUG_ANY, "config_backend: "
+                "no tier configured yet\n" );
+        return -1;
+    }
+
+    /* FIXME: maybe tier_add_backend could allocate it? */
+    b = lload_backend_new();
+    b->b_tier = tier;
 
     for ( i = 1; i < c->argc; i++ ) {
         if ( lload_backend_parse( c->argv[i], b ) ) {
-            Debug( LDAP_DEBUG_ANY, "config_backend: "
-                    "error parsing backend configuration item '%s'\n",
-                    c->argv[i] );
-            return -1;
+            if ( !tier->t_type.tier_backend_config ||
+                    tier->t_type.tier_backend_config( tier, b, c->argv[i] ) ) {
+                Debug( LDAP_DEBUG_ANY, "config_backend: "
+                        "error parsing backend configuration item '%s'\n",
+                        c->argv[i] );
+                return -1;
+            }
         }
     }
 
@@ -1463,6 +1514,80 @@ done:
     return rc;
 }
 
+static int
+config_tier( ConfigArgs *c )
+{
+    int rc = LDAP_SUCCESS;
+    struct lload_tier_type *tier_impl;
+    LloadTier *tier = c->ca_private;
+    struct berval bv;
+    int i = 1;
+
+    if ( c->op == SLAP_CONFIG_EMIT ) {
+        switch ( c->type ) {
+            case CFG_TIER:
+                c->value_string = ch_strdup( tier->t_type.tier_name );
+                break;
+            default:
+                goto fail;
+                break;
+        }
+        return rc;
+
+    } else if ( c->op == LDAP_MOD_DELETE ) {
+        if ( lload_change.type != LLOAD_CHANGE_DEL ) {
+            /*
+             * TODO: Shouldn't really happen while this attribute is in the
+             * RDN, but we don't enforce it yet.
+             *
+             * How would we go about changing the backend type if we ever supported that?
+             */
+            goto fail;
+        }
+        return rc;
+    }
+
+    if ( CONFIG_ONLINE_ADD( c ) ) {
+        assert( tier );
+        lload_change.target = tier;
+        return rc;
+    }
+
+    tier_impl = lload_tier_find( c->value_string );
+    if ( !tier_impl ) {
+        goto fail;
+    }
+    tier = tier_impl->tier_init();
+    if ( !tier ) {
+        goto fail;
+    }
+
+    lload_change.target = tier;
+
+    if ( LDAP_STAILQ_EMPTY( &tiers ) ) {
+        LDAP_STAILQ_INSERT_HEAD( &tiers, tier, t_next );
+    } else {
+        LloadTier *tier2;
+        LDAP_STAILQ_FOREACH ( tier2, &tiers, t_next ) {
+            i++;
+        }
+        LDAP_STAILQ_INSERT_TAIL( &tiers, tier, t_next );
+    }
+
+    bv.bv_val = c->cr_msg;
+    bv.bv_len = snprintf( c->cr_msg, sizeof(c->cr_msg), "tier %d", i );
+    ber_dupbv( &tier->t_name, &bv );
+
+    return rc;
+
+fail:
+    if ( lload_change.type == LLOAD_CHANGE_ADD ) {
+        /* Abort the ADD */
+        lload_change.type = LLOAD_CHANGE_DEL;
+    }
+    return 1;
+}
+
 static int
 config_fname( ConfigArgs *c )
 {
@@ -2957,6 +3082,9 @@ static slap_cf_aux_table backendkey[] = {
     { BER_BVC("max-pending-ops="), offsetof(LloadBackend, b_max_pending), 'i', 0, NULL },
     { BER_BVC("conn-max-pending="), offsetof(LloadBackend, b_max_conn_pending), 'i', 0, NULL },
     { BER_BVC("starttls="), offsetof(LloadBackend, b_tls_conf), 'i', 0, tlskey },
+
+    { BER_BVC("weight="), offsetof(LloadBackend, b_weight), 'i', 0, NULL },
+
     { BER_BVNULL, 0, 0, 0, NULL }
 };
 
@@ -3803,6 +3931,9 @@ backend_cf_gen( ConfigArgs *c )
             case CFG_STARTTLS:
                 enum_to_verb( tlskey, b->b_tls_conf, &c->value_bv );
                 break;
+            case CFG_WEIGHT:
+                c->value_uint = b->b_weight;
+                break;
             default:
                 rc = 1;
                 break;
@@ -3884,6 +4015,9 @@ backend_cf_gen( ConfigArgs *c )
 #endif /* ! HAVE_TLS */
             b->b_tls_conf = tlskey[i].mask;
         } break;
+        case CFG_WEIGHT:
+            b->b_weight = c->value_uint;
+            break;
         default:
             rc = 1;
             break;
@@ -3922,9 +4056,82 @@ lload_back_init_cf( BackendInfo *bi )
     return config_register_schema( config_back_cf_table, lloadocs );
 }
 
+static int
+lload_tier_ldadd( CfEntryInfo *p, Entry *e, ConfigArgs *ca )
+{
+    LloadTier *tier;
+    Attribute *a;
+    AttributeDescription *ad = NULL;
+    struct lload_tier_type *tier_impl;
+    struct berval bv, type, rdn;
+    const char *text;
+    char *name;
+
+    Debug( LDAP_DEBUG_TRACE, "lload_tier_ldadd: "
+            "a new tier is being added\n" );
+
+    if ( p->ce_type != Cft_Backend || !p->ce_bi ||
+            p->ce_bi->bi_cf_ocs != lloadocs )
+        return LDAP_CONSTRAINT_VIOLATION;
+
+    dnRdn( &e->e_name, &rdn );
+    type.bv_len = strchr( rdn.bv_val, '=' ) - rdn.bv_val;
+    type.bv_val = rdn.bv_val;
+
+    /* Find attr */
+    slap_bv2ad( &type, &ad, &text );
+    if ( ad != slap_schema.si_ad_cn ) return LDAP_NAMING_VIOLATION;
+
+    a = attr_find( e->e_attrs, ad );
+    if ( !a || a->a_numvals != 1 ) return LDAP_NAMING_VIOLATION;
+    bv = a->a_vals[0];
+
+    if ( bv.bv_val[0] == '{' && ( name = strchr( bv.bv_val, '}' ) ) ) {
+        name++;
+        bv.bv_len -= name - bv.bv_val;
+        bv.bv_val = name;
+    }
+
+    ad = NULL;
+    slap_str2ad( "olcBkLloadTierType", &ad, &text );
+    assert( ad != NULL );
+
+    a = attr_find( e->e_attrs, ad );
+    if ( !a || a->a_numvals != 1 ) return LDAP_OBJECT_CLASS_VIOLATION;
+
+    tier_impl = lload_tier_find( a->a_vals[0].bv_val );
+    if ( !tier_impl ) {
+        Debug( LDAP_DEBUG_ANY, "lload_tier_ldadd: "
+                "tier type %s not recongnised\n",
+                bv.bv_val );
+        return LDAP_OTHER;
+    }
+
+    tier = tier_impl->tier_init();
+    if ( !tier ) {
+        return LDAP_OTHER;
+    }
+
+    ber_dupbv( &tier->t_name, &bv );
+
+    ca->bi = p->ce_bi;
+    ca->ca_private = tier;
+
+    /* ca cleanups are only run in the case of online config but we use it to
+     * save the new config when done with the entry */
+    ca->lineno = 0;
+
+    lload_change.type = LLOAD_CHANGE_ADD;
+    lload_change.object = LLOAD_TIER;
+    lload_change.target = tier;
+
+    return LDAP_SUCCESS;
+}
+
 static int
 lload_backend_ldadd( CfEntryInfo *p, Entry *e, ConfigArgs *ca )
 {
+    LloadTier *tier = p->ce_private;
     LloadBackend *b;
     Attribute *a;
     AttributeDescription *ad = NULL;
@@ -3935,7 +4142,7 @@ lload_backend_ldadd( CfEntryInfo *p, Entry *e, ConfigArgs *ca )
     Debug( LDAP_DEBUG_TRACE, "lload_backend_ldadd: "
             "a new backend-server is being added\n" );
 
-    if ( p->ce_type != Cft_Backend || !p->ce_bi ||
+    if ( p->ce_type != Cft_Misc || !p->ce_bi ||
             p->ce_bi->bi_cf_ocs != lloadocs )
         return LDAP_CONSTRAINT_VIOLATION;
 
@@ -3957,8 +4164,9 @@ lload_backend_ldadd( CfEntryInfo *p, Entry *e, ConfigArgs *ca )
         bv.bv_val = name;
     }
 
-    b = backend_alloc();
+    b = lload_backend_new();
     ber_dupbv( &b->b_name, &bv );
+    b->b_tier = tier;
 
     ca->bi = p->ce_bi;
     ca->ca_private = b;
@@ -3987,29 +4195,74 @@ lload_backend_lddel( CfEntryInfo *ce, Operation *op )
 
     return LDAP_SUCCESS;
 }
+
+static int
+lload_tier_lddel( CfEntryInfo *ce, Operation *op )
+{
+    LloadTier *tier = ce->ce_private;
+
+    lload_change.type = LLOAD_CHANGE_DEL;
+    lload_change.object = LLOAD_TIER;
+    lload_change.target = tier;
+
+    return LDAP_SUCCESS;
+}
 #endif /* SLAP_CONFIG_DELETE */
 
 static int