Skip to content
GitLab
Menu
Projects
Groups
Snippets
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Sign in / Register
Toggle navigation
Menu
Open sidebar
HAMANO Tsukasa
OpenLDAP
Commits
ab90bfd9
Commit
ab90bfd9
authored
Jul 24, 2018
by
Ondřej Kuzník
Browse files
ITS#9599 Implement tiered load balancing
parent
e98374fa
Changes
23
Hide whitespace changes
Inline
Side-by-side
doc/man/man5/lloadd.conf.5
View file @
ab90bfd9
...
...
@@ -644,7 +644,13 @@ only valid when using GnuTLS and Mozilla NSS.
.SH BACKEND CONFIGURATION
Options in this section describe how the
.B lloadd
connects and authenticates to the backend servers.
connects and authenticates to the backend servers. Backends are organised in groups
.RB ( tiers ).
Backends in the first tier are tried first, if none of them are reachable, the
following tier is tried in the same way. If there is a backend in the tier that
has suitable connections, but they are busy, no further tier is consulted. This
is useful in high availability scenarios where a group of servers (e.g. the
local environment) should be contacted if possible.
It is assumed all backend servers serve the same data. On startup, the
configured connections are set up and those not dedicated to handle bind
...
...
@@ -730,6 +736,33 @@ set on the upstream connections, overriding the operating system setting.
Only some systems support the customization of this parameter, it is
ignored otherwise and system-wide settings are used.
.SH TIER OPTIONS
.TP
.B tier
.B <tier type>
Groups servers which should be considered in the same try. If a viable
connection is found even if busy, the load balancer does not proceed to the
next tier. The process of selection a connection within a tier depends on the
tier's type.
.RE
Available types are:
.TP
.B roundrobin
Servers are tried in order and if one is selected successfully, the following
search will try from the one next on the list.
.TP
.B weighted
Backend servers accept a new option
.B weight=<int>
which indicates how often it should be selected. If unspecified, weight
defaults to 0 and such backends have a slight chance of being selected even
when a non-zero weight backend is configured in the tier. The selection process
is along the lines of
.BR RFC2782 .
.SH BACKEND OPTIONS
.TP
...
...
@@ -879,6 +912,7 @@ bindconf
binddn=cn=test
credentials=pass
tier weighted
backend-server
uri=ldap://ldap1.example.com
numconns=3
...
...
@@ -886,6 +920,7 @@ backend-server
retry=5000
max-pending-ops=5
conn-max-pending=3
weight=5
backend-server
uri=ldap://ldap2.example.com
...
...
@@ -894,6 +929,7 @@ backend-server
retry=5000
max-pending-ops=5
conn-max-pending=3
weight=10
.fi
.RE
.LP
...
...
servers/lloadd/Makefile.in
View file @
ab90bfd9
...
...
@@ -21,6 +21,7 @@ NT_OBJS = nt_svc.o ../../libraries/liblutil/slapdmsg.res
SRCS
=
backend.c bind.c config.c connection.c client.c
\
daemon.c epoch.c extended.c init.c operation.c
\
tier.c tier_roundrobin.c tier_weighted.c
\
upstream.c libevent_support.c
\
$
(
@PLAT@_SRCS
)
...
...
servers/lloadd/backend.c
View file @
ab90bfd9
...
...
@@ -392,44 +392,17 @@ upstream_select(
int
*
res
,
char
**
message
)
{
Lload
Backend
*
b
,
*
first
,
*
next
;
int
rc
=
0
;
Lload
Tier
*
tier
;
int
finished
=
0
;
checked_lock
(
&
backend_mutex
);
first
=
b
=
current_backend
;
checked_unlock
(
&
backend_mutex
);
*
res
=
LDAP_UNAVAILABLE
;
if
(
!
first
)
{
return
NULL
;
}
/* TODO: Two runs, one with trylock, then one actually locked if we don't
* find anything? */
do
{
checked_lock
(
&
b
->
b_mutex
);
next
=
LDAP_CIRCLEQ_LOOP_NEXT
(
&
backend
,
b
,
b_next
);
rc
=
backend_select
(
b
,
op
,
cp
,
res
,
message
);
checked_unlock
(
&
b
->
b_mutex
);
if
(
rc
&&
*
cp
)
{
/*
* Round-robin step:
* Rotate the queue to put this backend at the end. The race here
* is acceptable.
*/
checked_lock
(
&
backend_mutex
);
current_backend
=
next
;
checked_unlock
(
&
backend_mutex
);
return
rc
;
LDAP_STAILQ_FOREACH
(
tier
,
&
tiers
,
t_next
)
{
if
(
(
finished
=
tier
->
t_type
.
tier_select
(
tier
,
op
,
cp
,
res
,
message
))
)
{
break
;
}
}
b
=
next
;
}
while
(
b
!=
first
);
return
rc
;
return
finished
;
}
/*
...
...
@@ -726,26 +699,41 @@ backend_reset( LloadBackend *b, int gentle )
assert_locked
(
&
b
->
b_mutex
);
}
LloadBackend
*
lload_backend_new
(
void
)
{
LloadBackend
*
b
;
b
=
ch_calloc
(
1
,
sizeof
(
LloadBackend
)
);
LDAP_CIRCLEQ_INIT
(
&
b
->
b_conns
);
LDAP_CIRCLEQ_INIT
(
&
b
->
b_bindconns
);
LDAP_CIRCLEQ_INIT
(
&
b
->
b_preparing
);
LDAP_CIRCLEQ_ENTRY_INIT
(
b
,
b_next
);
b
->
b_numconns
=
1
;
b
->
b_numbindconns
=
1
;
b
->
b_weight
=
1
;
b
->
b_retry_timeout
=
5000
;
ldap_pvt_thread_mutex_init
(
&
b
->
b_mutex
);
return
b
;
}
void
lload_backend_destroy
(
LloadBackend
*
b
)
{
LloadBackend
*
next
=
LDAP_CIRCLEQ_LOOP_NEXT
(
&
backend
,
b
,
b_next
);
Debug
(
LDAP_DEBUG_CONNS
,
"lload_backend_destroy: "
"destroying backend uri='%s', numconns=%d, numbindconns=%d
\n
"
,
b
->
b_uri
.
bv_val
,
b
->
b_numconns
,
b
->
b_numbindconns
);
checked_lock
(
&
b
->
b_mutex
);
b
->
b_tier
->
t_type
.
tier_remove_backend
(
b
->
b_tier
,
b
);
b
->
b_numconns
=
b
->
b_numbindconns
=
0
;
backend_reset
(
b
,
0
);
LDAP_CIRCLEQ_REMOVE
(
&
backend
,
b
,
b_next
);
if
(
b
==
next
)
{
current_backend
=
NULL
;
}
else
{
current_backend
=
next
;
}
#ifdef BALANCER_MODULE
if
(
b
->
b_monitor
)
{
BackendDB
*
be
;
...
...
@@ -760,6 +748,7 @@ lload_backend_destroy( LloadBackend *b )
assert
(
rc
==
LDAP_SUCCESS
);
}
#endif
/* BALANCER_MODULE */
checked_unlock
(
&
b
->
b_mutex
);
ldap_pvt_thread_mutex_destroy
(
&
b
->
b_mutex
);
...
...
@@ -774,13 +763,3 @@ lload_backend_destroy( LloadBackend *b )
ch_free
(
b
->
b_name
.
bv_val
);
ch_free
(
b
);
}
void
lload_backends_destroy
(
void
)
{
while
(
!
LDAP_CIRCLEQ_EMPTY
(
&
backend
)
)
{
LloadBackend
*
b
=
LDAP_CIRCLEQ_FIRST
(
&
backend
);
lload_backend_destroy
(
b
);
}
}
servers/lloadd/config.c
View file @
ab90bfd9
...
...
@@ -113,6 +113,7 @@ static ConfigFile *cfn;
static
ConfigDriver
config_fname
;
static
ConfigDriver
config_generic
;
static
ConfigDriver
config_tier
;
static
ConfigDriver
config_backend
;
static
ConfigDriver
config_bindconf
;
static
ConfigDriver
config_restrict_oid
;
...
...
@@ -132,10 +133,6 @@ static ConfigDriver config_share_tls_ctx;
static
ConfigDriver
backend_cf_gen
;
#endif
/* BALANCER_MODULE */
lload_b_head
backend
=
LDAP_CIRCLEQ_HEAD_INITIALIZER
(
backend
);
ldap_pvt_thread_mutex_t
backend_mutex
;
LloadBackend
*
current_backend
=
NULL
;
struct
slap_bindconf
bindconf
=
{};
struct
berval
lloadd_identity
=
BER_BVNULL
;
...
...
@@ -182,6 +179,8 @@ enum {
CFG_CLIENT_PENDING
,
CFG_RESTRICT_EXOP
,
CFG_RESTRICT_CONTROL
,
CFG_TIER
,
CFG_WEIGHT
,
CFG_LAST
};
...
...
@@ -205,6 +204,17 @@ static ConfigTable config_back_cf_table[] = {
&
config_generic
,
NULL
,
NULL
,
NULL
},
{
"tier"
,
"name"
,
2
,
2
,
0
,
ARG_MAGIC
|
ARG_STRING
|
CFG_TIER
,
&
config_tier
,
"( OLcfgBkAt:13.39 "
"NAME 'olcBkLloadTierType' "
"DESC 'Tier type' "
"EQUALITY caseIgnoreMatch "
"SYNTAX OMsDirectoryString "
"SINGLE-VALUE )"
,
NULL
,
NULL
},
/* conf-file only option */
{
"backend-server"
,
"backend options"
,
2
,
0
,
0
,
ARG_MAGIC
|
CFG_BACKEND
,
...
...
@@ -747,6 +757,17 @@ static ConfigTable config_back_cf_table[] = {
"SINGLE-VALUE )"
,
NULL
,
NULL
},
{
""
,
NULL
,
2
,
2
,
0
,
ARG_MAGIC
|
ARG_UINT
|
CFG_WEIGHT
,
&
backend_cf_gen
,
"( OLcfgBkAt:13.40 "
"NAME 'olcBkLloadWeight' "
"DESC 'Backend weight' "
"SYNTAX OMsInteger "
"SINGLE-VALUE )"
,
NULL
,
{
.
v_uint
=
0
},
},
#endif
/* BALANCER_MODULE */
{
NULL
,
NULL
,
0
,
0
,
0
,
ARG_IGNORED
,
NULL
}
...
...
@@ -754,9 +775,13 @@ static ConfigTable config_back_cf_table[] = {
#ifdef BALANCER_MODULE
static
ConfigCfAdd
lload_cfadd
;
static
ConfigLDAPadd
lload_backend_ldadd
;
static
ConfigLDAPadd
lload_tier_ldadd
;
#ifdef SLAP_CONFIG_DELETE
static
ConfigLDAPdel
lload_backend_lddel
;
static
ConfigLDAPdel
lload_tier_lddel
;
#endif
/* SLAP_CONFIG_DELETE */
static
ConfigOCs
lloadocs
[]
=
{
...
...
@@ -807,12 +832,27 @@ static ConfigOCs lloadocs[] = {
"$ olcBkLloadMaxPendingOps "
"$ olcBkLloadMaxPendingConns ) "
"MAY ( olcBkLloadStartTLS "
"$ olcBkLloadWeight ) "
") )"
,
Cft_Misc
,
config_back_cf_table
,
lload_backend_ldadd
,
NULL
,
#ifdef SLAP_CONFIG_DELETE
lload_backend_lddel
,
#endif
/* SLAP_CONFIG_DELETE */
},
{
"( OLcfgBkOc:13.3 "
"NAME 'olcBkLloadTierConfig' "
"DESC 'Lload tier configuration' "
"SUP olcConfig STRUCTURAL "
"MUST ( cn "
"$ olcBkLloadTierType "
") )"
,
Cft_Misc
,
config_back_cf_table
,
lload_tier_ldadd
,
NULL
,
#ifdef SLAP_CONFIG_DELETE
lload_tier_lddel
,
#endif
/* SLAP_CONFIG_DELETE */
},
{
NULL
,
0
,
NULL
}
...
...
@@ -1073,6 +1113,26 @@ lload_backend_finish( ConfigArgs *ca )
b
->
b_retry_event
=
event
;
}
if
(
BER_BVISEMPTY
(
&
b
->
b_name
)
)
{
struct
berval
bv
;
LloadBackend
*
b2
;
int
i
=
1
;
LDAP_CIRCLEQ_FOREACH
(
b2
,
&
b
->
b_tier
->
t_backends
,
b_next
)
{
i
++
;
}
bv
.
bv_val
=
ca
->
cr_msg
;
bv
.
bv_len
=
snprintf
(
ca
->
cr_msg
,
sizeof
(
ca
->
cr_msg
),
"server %d"
,
i
);
ber_dupbv
(
&
b
->
b_name
,
&
bv
);
}
if
(
b
->
b_tier
->
t_type
.
tier_add_backend
(
b
->
b_tier
,
b
)
)
{
goto
fail
;
}
return
LDAP_SUCCESS
;
fail:
...
...
@@ -1085,28 +1145,6 @@ fail:
return
-
1
;
}
static
LloadBackend
*
backend_alloc
(
void
)
{
LloadBackend
*
b
;
b
=
ch_calloc
(
1
,
sizeof
(
LloadBackend
)
);
LDAP_CIRCLEQ_INIT
(
&
b
->
b_conns
);
LDAP_CIRCLEQ_INIT
(
&
b
->
b_bindconns
);
LDAP_CIRCLEQ_INIT
(
&
b
->
b_preparing
);
b
->
b_numconns
=
1
;
b
->
b_numbindconns
=
1
;
b
->
b_retry_timeout
=
5000
;
ldap_pvt_thread_mutex_init
(
&
b
->
b_mutex
);
LDAP_CIRCLEQ_INSERT_TAIL
(
&
backend
,
b
,
b_next
);
return
b
;
}
static
int
backend_config_url
(
LloadBackend
*
b
,
struct
berval
*
uri
)
{
...
...
@@ -1183,16 +1221,29 @@ static int
config_backend
(
ConfigArgs
*
c
)
{
LloadBackend
*
b
;
LloadTier
*
tier
;
int
i
,
rc
=
0
;
b
=
backend_alloc
();
tier
=
LDAP_STAILQ_LAST
(
&
tiers
,
LloadTier
,
t_next
);
if
(
!
tier
)
{
Debug
(
LDAP_DEBUG_ANY
,
"config_backend: "
"no tier configured yet
\n
"
);
return
-
1
;
}
/* FIXME: maybe tier_add_backend could allocate it? */
b
=
lload_backend_new
();
b
->
b_tier
=
tier
;
for
(
i
=
1
;
i
<
c
->
argc
;
i
++
)
{
if
(
lload_backend_parse
(
c
->
argv
[
i
],
b
)
)
{
Debug
(
LDAP_DEBUG_ANY
,
"config_backend: "
"error parsing backend configuration item '%s'
\n
"
,
c
->
argv
[
i
]
);
return
-
1
;
if
(
!
tier
->
t_type
.
tier_backend_config
||
tier
->
t_type
.
tier_backend_config
(
tier
,
b
,
c
->
argv
[
i
]
)
)
{
Debug
(
LDAP_DEBUG_ANY
,
"config_backend: "
"error parsing backend configuration item '%s'
\n
"
,
c
->
argv
[
i
]
);
return
-
1
;
}
}
}
...
...
@@ -1463,6 +1514,80 @@ done:
return
rc
;
}
static
int
config_tier
(
ConfigArgs
*
c
)
{
int
rc
=
LDAP_SUCCESS
;
struct
lload_tier_type
*
tier_impl
;
LloadTier
*
tier
=
c
->
ca_private
;
struct
berval
bv
;
int
i
=
1
;
if
(
c
->
op
==
SLAP_CONFIG_EMIT
)
{
switch
(
c
->
type
)
{
case
CFG_TIER
:
c
->
value_string
=
ch_strdup
(
tier
->
t_type
.
tier_name
);
break
;
default:
goto
fail
;
break
;
}
return
rc
;
}
else
if
(
c
->
op
==
LDAP_MOD_DELETE
)
{
if
(
lload_change
.
type
!=
LLOAD_CHANGE_DEL
)
{
/*
* TODO: Shouldn't really happen while this attribute is in the
* RDN, but we don't enforce it yet.
*
* How would we go about changing the backend type if we ever supported that?
*/
goto
fail
;
}
return
rc
;
}
if
(
CONFIG_ONLINE_ADD
(
c
)
)
{
assert
(
tier
);
lload_change
.
target
=
tier
;
return
rc
;
}
tier_impl
=
lload_tier_find
(
c
->
value_string
);
if
(
!
tier_impl
)
{
goto
fail
;
}
tier
=
tier_impl
->
tier_init
();
if
(
!
tier
)
{
goto
fail
;
}
lload_change
.
target
=
tier
;
if
(
LDAP_STAILQ_EMPTY
(
&
tiers
)
)
{
LDAP_STAILQ_INSERT_HEAD
(
&
tiers
,
tier
,
t_next
);
}
else
{
LloadTier
*
tier2
;
LDAP_STAILQ_FOREACH
(
tier2
,
&
tiers
,
t_next
)
{
i
++
;
}
LDAP_STAILQ_INSERT_TAIL
(
&
tiers
,
tier
,
t_next
);
}
bv
.
bv_val
=
c
->
cr_msg
;
bv
.
bv_len
=
snprintf
(
c
->
cr_msg
,
sizeof
(
c
->
cr_msg
),
"tier %d"
,
i
);
ber_dupbv
(
&
tier
->
t_name
,
&
bv
);
return
rc
;
fail:
if
(
lload_change
.
type
==
LLOAD_CHANGE_ADD
)
{
/* Abort the ADD */
lload_change
.
type
=
LLOAD_CHANGE_DEL
;
}
return
1
;
}
static
int
config_fname
(
ConfigArgs
*
c
)
{
...
...
@@ -2957,6 +3082,9 @@ static slap_cf_aux_table backendkey[] = {
{
BER_BVC
(
"max-pending-ops="
),
offsetof
(
LloadBackend
,
b_max_pending
),
'i'
,
0
,
NULL
},
{
BER_BVC
(
"conn-max-pending="
),
offsetof
(
LloadBackend
,
b_max_conn_pending
),
'i'
,
0
,
NULL
},
{
BER_BVC
(
"starttls="
),
offsetof
(
LloadBackend
,
b_tls_conf
),
'i'
,
0
,
tlskey
},
{
BER_BVC
(
"weight="
),
offsetof
(
LloadBackend
,
b_weight
),
'i'
,
0
,
NULL
},
{
BER_BVNULL
,
0
,
0
,
0
,
NULL
}
};
...
...
@@ -3803,6 +3931,9 @@ backend_cf_gen( ConfigArgs *c )
case
CFG_STARTTLS
:
enum_to_verb
(
tlskey
,
b
->
b_tls_conf
,
&
c
->
value_bv
);
break
;
case
CFG_WEIGHT
:
c
->
value_uint
=
b
->
b_weight
;
break
;
default:
rc
=
1
;
break
;
...
...
@@ -3884,6 +4015,9 @@ backend_cf_gen( ConfigArgs *c )
#endif
/* ! HAVE_TLS */
b
->
b_tls_conf
=
tlskey
[
i
].
mask
;
}
break
;
case
CFG_WEIGHT
:
b
->
b_weight
=
c
->
value_uint
;
break
;
default:
rc
=
1
;
break
;
...
...
@@ -3922,9 +4056,82 @@ lload_back_init_cf( BackendInfo *bi )
return
config_register_schema
(
config_back_cf_table
,
lloadocs
);
}
static
int
lload_tier_ldadd
(
CfEntryInfo
*
p
,
Entry
*
e
,
ConfigArgs
*
ca
)
{
LloadTier
*
tier
;
Attribute
*
a
;
AttributeDescription
*
ad
=
NULL
;
struct
lload_tier_type
*
tier_impl
;
struct
berval
bv
,
type
,
rdn
;
const
char
*
text
;
char
*
name
;
Debug
(
LDAP_DEBUG_TRACE
,
"lload_tier_ldadd: "
"a new tier is being added
\n
"
);
if
(
p
->
ce_type
!=
Cft_Backend
||
!
p
->
ce_bi
||
p
->
ce_bi
->
bi_cf_ocs
!=
lloadocs
)
return
LDAP_CONSTRAINT_VIOLATION
;
dnRdn
(
&
e
->
e_name
,
&
rdn
);
type
.
bv_len
=
strchr
(
rdn
.
bv_val
,
'='
)
-
rdn
.
bv_val
;
type
.
bv_val
=
rdn
.
bv_val
;
/* Find attr */
slap_bv2ad
(
&
type
,
&
ad
,
&
text
);
if
(
ad
!=
slap_schema
.
si_ad_cn
)
return
LDAP_NAMING_VIOLATION
;
a
=
attr_find
(
e
->
e_attrs
,
ad
);
if
(
!
a
||
a
->
a_numvals
!=
1
)
return
LDAP_NAMING_VIOLATION
;
bv
=
a
->
a_vals
[
0
];
if
(
bv
.
bv_val
[
0
]
==
'{'
&&
(
name
=
strchr
(
bv
.
bv_val
,
'}'
)
)
)
{
name
++
;
bv
.
bv_len
-=
name
-
bv
.
bv_val
;
bv
.
bv_val
=
name
;
}
ad
=
NULL
;
slap_str2ad
(
"olcBkLloadTierType"
,
&
ad
,
&
text
);
assert
(
ad
!=
NULL
);
a
=
attr_find
(
e
->
e_attrs
,
ad
);
if
(
!
a
||
a
->
a_numvals
!=
1
)
return
LDAP_OBJECT_CLASS_VIOLATION
;
tier_impl
=
lload_tier_find
(
a
->
a_vals
[
0
].
bv_val
);
if
(
!
tier_impl
)
{
Debug
(
LDAP_DEBUG_ANY
,
"lload_tier_ldadd: "
"tier type %s not recongnised
\n
"
,
bv
.
bv_val
);
return
LDAP_OTHER
;
}
tier
=
tier_impl
->
tier_init
();
if
(
!
tier
)
{
return
LDAP_OTHER
;
}
ber_dupbv
(
&
tier
->
t_name
,
&
bv
);
ca
->
bi
=
p
->
ce_bi
;
ca
->
ca_private
=
tier
;
/* ca cleanups are only run in the case of online config but we use it to
* save the new config when done with the entry */
ca
->
lineno
=
0
;
lload_change
.
type
=
LLOAD_CHANGE_ADD
;
lload_change
.
object
=
LLOAD_TIER
;
lload_change
.
target
=
tier
;
return
LDAP_SUCCESS
;
}
static
int
lload_backend_ldadd
(
CfEntryInfo
*
p
,
Entry
*
e
,
ConfigArgs
*
ca
)
{
LloadTier
*
tier
=
p
->
ce_private
;
LloadBackend
*
b
;
Attribute
*
a
;
AttributeDescription
*
ad
=
NULL
;
...
...
@@ -3935,7 +4142,7 @@ lload_backend_ldadd( CfEntryInfo *p, Entry *e, ConfigArgs *ca )
Debug
(
LDAP_DEBUG_TRACE
,
"lload_backend_ldadd: "
"a new backend-server is being added
\n
"
);
if
(
p
->
ce_type
!=
Cft_
Backend
||
!
p
->
ce_bi
||
if
(
p
->
ce_type
!=
Cft_
Misc
||
!
p
->
ce_bi
||
p
->
ce_bi
->
bi_cf_ocs
!=
lloadocs
)
return
LDAP_CONSTRAINT_VIOLATION
;
...
...
@@ -3957,8 +4164,9 @@ lload_backend_ldadd( CfEntryInfo *p, Entry *e, ConfigArgs *ca )
bv
.
bv_val
=
name
;
}
b
=
backend_
alloc
();
b
=
lload_
backend_
new
();
ber_dupbv
(
&
b
->
b_name
,
&
bv
);
b
->
b_tier
=
tier
;
ca
->
bi
=
p
->
ce_bi
;
ca
->
ca_private
=
b
;
...
...
@@ -3987,29 +4195,74 @@ lload_backend_lddel( CfEntryInfo *ce, Operation *op )
return
LDAP_SUCCESS
;
}
static
int
lload_tier_lddel
(
CfEntryInfo
*
ce
,
Operation
*
op
)
{
LloadTier
*
tier
=
ce
->
ce_private
;
lload_change
.
type
=
LLOAD_CHANGE_DEL
;
lload_change
.
object
=
LLOAD_TIER
;
lload_change
.
target
=
tier
;
return
LDAP_SUCCESS
;
}