Commit 61c95e76 authored by HAMANO Tsukasa's avatar HAMANO Tsukasa Committed by Howard Chu
Browse files

ITS#8114 OpenLDAP WiredTiger Backend

parent f385fd5a
......@@ -161,6 +161,7 @@ LTHREAD_LIBS = @LTHREAD_LIBS@
BDB_LIBS = @BDB_LIBS@
SLAPD_NDB_LIBS = @SLAPD_NDB_LIBS@
WT_LIBS = @WT_LIBS@
LDAP_LIBLBER_LA = $(LDAP_LIBDIR)/liblber/liblber.la
LDAP_LIBLDAP_LA = $(LDAP_LIBDIR)/libldap/libldap.la
......
......@@ -296,7 +296,8 @@ Backends="bdb \
relay \
shell \
sock \
sql"
sql \
wt"
AC_ARG_ENABLE(xxslapbackends,[
SLAPD Backend Options:])
......@@ -333,6 +334,8 @@ OL_ARG_ENABLE(sock,[ --enable-sock enable sock backend],
no, [no yes mod], ol_enable_backends)dnl
OL_ARG_ENABLE(sql,[ --enable-sql enable sql backend],
no, [no yes mod], ol_enable_backends)dnl
OL_ARG_ENABLE(wt,[ --enable-wt enable WiredTiger backend],
no, [no yes mod], ol_enable_backends)dnl
dnl ----------------------------------------------------------------
dnl SLAPD Overlay Options
......@@ -485,7 +488,8 @@ elif test $ol_enable_modules != yes &&
test $ol_enable_relay = no &&
test $ol_enable_shell = no &&
test $ol_enable_sock = no &&
test $ol_enable_sql = no ; then
test $ol_enable_sql = no &&
test $ol_enable_wt = no ; then
dnl no slapd backend
if test $ol_enable_slapd = yes ; then
......@@ -548,6 +552,7 @@ BUILD_RELAY=no
BUILD_SHELL=no
BUILD_SOCK=no
BUILD_SQL=no
BUILD_WT=no
BUILD_ACCESSLOG=no
BUILD_AUDITLOG=no
......@@ -2082,6 +2087,33 @@ if test $ol_enable_ndb != no ; then
fi
fi
dnl ----------------------------------------------------------------
dnl WiredTiger
ol_link_wt=no
if test $ol_enable_wt != no ; then
AC_CHECK_PROG(PKGCONFIG,pkg-config,yes)
if test "$PKGCONFIG" != yes ; then
AC_MSG_ERROR([could not locate pkg-config])
fi
WT_INCS=`pkg-config --cflags wiredtiger`
WT_LIBS=`pkg-config --libs wiredtiger`
save_CFLAGS="$CFLAGS"
save_LDFLAGS="$LDFLAGS"
CFLAGS="$WT_INCS"
CPPFLAGS="$WT_INCS"
LDFLAGS="$WT_LIBS"
AC_CHECK_HEADERS([wiredtiger.h])
AC_CHECK_LIB(wiredtiger,wiredtiger_version,[: ok],[
AC_MSG_ERROR([could not locate wiredtiger library])
])
CFLAGS="$save_CFLAGS"
CPPFLAGS="$save_CPPFLAGS"
LDFLAGS="$save_LDFLAGS"
SLAPD_LIBS="$SLAPD_LIBS \$(WT_LIBS)"
ol_link_wt=yes
fi
dnl ----------------------------------------------------------------
dnl International Components for Unicode
OL_ICU
......@@ -2811,6 +2843,19 @@ if test "$ol_link_sql" != no ; then
AC_DEFINE_UNQUOTED(SLAPD_SQL,$MFLAG,[define to support SQL backend])
fi
if test "$ol_link_wt" != no ; then
BUILD_SLAPD=yes
BUILD_WT=$ol_enable_wt
if test "$ol_enable_wt" = mod; then
SLAPD_DYNAMIC_BACKENDS="$SLAPD_DYNAMIC_BACKENDS back-wt"
MFLAG=SLAPD_MOD_DYNAMIC
else
SLAPD_STATIC_BACKENDS="$SLAPD_STATIC_BACKENDS back-wt"
MFLAG=SLAPD_MOD_STATIC
fi
AC_DEFINE_UNQUOTED(SLAPD_WT,$MFLAG,[define to support WiredTiger backend])
fi
if test "$ol_enable_accesslog" != no ; then
BUILD_ACCESSLOG=$ol_enable_accesslog
if test "$ol_enable_accesslog" = mod ; then
......@@ -3106,6 +3151,7 @@ dnl backends
AC_SUBST(BUILD_SHELL)
AC_SUBST(BUILD_SOCK)
AC_SUBST(BUILD_SQL)
AC_SUBST(BUILD_WT)
dnl overlays
AC_SUBST(BUILD_ACCESSLOG)
AC_SUBST(BUILD_AUDITLOG)
......@@ -3169,6 +3215,9 @@ AC_SUBST(SLAPD_SQL_LDFLAGS)
AC_SUBST(SLAPD_SQL_LIBS)
AC_SUBST(SLAPD_SQL_INCLUDES)
AC_SUBST(WT_INCS)
AC_SUBST(WT_LIBS)
dnl ----------------------------------------------------------------
dnl final help output
AC_ARG_WITH(xxinstall,[
......@@ -3213,6 +3262,7 @@ AC_CONFIG_FILES([Makefile:build/top.mk:Makefile.in:build/dir.mk]
[servers/slapd/back-shell/Makefile:build/top.mk:servers/slapd/back-shell/Makefile.in:build/mod.mk]
[servers/slapd/back-sock/Makefile:build/top.mk:servers/slapd/back-sock/Makefile.in:build/mod.mk]
[servers/slapd/back-sql/Makefile:build/top.mk:servers/slapd/back-sql/Makefile.in:build/mod.mk]
[servers/slapd/back-wt/Makefile:build/top.mk:servers/slapd/back-wt/Makefile.in:build/mod.mk]
[servers/slapd/shell-backends/Makefile:build/top.mk:servers/slapd/shell-backends/Makefile.in:build/srv.mk]
[servers/slapd/slapi/Makefile:build/top.mk:servers/slapd/slapi/Makefile.in:build/lib.mk:build/lib-shared.mk]
[servers/slapd/overlays/Makefile:build/top.mk:servers/slapd/overlays/Makefile.in:build/lib.mk]
......
.TH SLAPD-WT 5 "RELEASEDATE" "OpenLDAP LDVERSION"
.\" Copyright 2011-2015 The OpenLDAP Foundation All Rights Reserved.
.\" Copying restrictions apply. See COPYRIGHT/LICENSE.
.\" $OpenLDAP$
.SH NAME
slapd\-wt \- WiredTiger backend to slapd
.SH SYNOPSIS
.B ETCDIR/slapd.conf
.SH DESCRIPTION
The \fBwt\fP backend to
.BR slapd (8)
uses WiredTiger database library to store data.
.LP
The \fBwt\fP backend is experimental module that have potential high
write performance and high concurrency performance.
This backend have not some basic feature yet. Please backup data using
slapcat before update the module.
.SH CONFIGURATION
These
.B slapd.conf
options apply to the \fBwt\fP backend database.
That is, they must follow a "database wt" line and
come before any subsequent "backend" or "database" lines.
Other database options are described in the
.BR slapd.conf (5)
manual page.
.TP
.BI directory \ <directory>
Specify WiredTiger home directory that containing this database and
associated indexes live.
A separate directory must be specified for each database.
The default is
.BR LOCALSTATEDIR/openldap\-data .
.TP
\fBwtconfig \fR{\fBcreate\fR,\fBcache_size=512M\fR,\fBasync=(enabled)\fR}
Specify configuration for wiredtiger, This parameter is pass to
.BR wiredtiger_open (3).
.RS
.TP
.B create
create the database if it does not exist.
.RE
.RS
.TP
.B cache_size
maximum heap memory to allocate for the cache.
.RE
.RS
.TP
.B async
asynchronous operations configuration options. disabled by default.
.RE
.RS
.TP
\fBindex \fR{\fI<attrlist>\fR|\fBdefault\fR} [\fBpres\fR,\fBeq\fR,\fBapprox\fR,\fBsub\fR,\fI<special>\fR]
Specify the indexes to maintain for the given attribute (or
list of attributes).
Some attributes only support a subset of indexes.
If only an \fI<attr>\fP is given, the indices specified for \fBdefault\fR
are maintained.
Note that setting a default does not imply that all attributes will be
indexed. Also, for best performance, an
.B eq
index should always be configured for the
.B objectClass
attribute.
.SH ACCESS CONTROL
The
.B wt
backend honors access control semantics as indicated in
.BR slapd.access (5).
.SH FILES
.TP
.B ETCDIR/slapd.conf
default
.B slapd
configuration file
.SH SEE ALSO
.BR slapd.conf (5),
.BR slapd\-config (5),
.BR slapd (8),
.BR slapadd (8),
.BR slapcat (8),
.BR slapindex (8),
WiredTiger documentation.
.SH ACKNOWLEDGEMENTS
.so ../Project
Written by HAMANO Tsukasa <hamano@osstech.co.jp>.
# Makefile.in for back-wt
# $OpenLDAP$
## This work is part of OpenLDAP Software <http://www.openldap.org/>.
##
## Copyright 1998-2015 The OpenLDAP Foundation.
## All rights reserved.
##
## Redistribution and use in source and binary forms, with or without
## modification, are permitted only as authorized by the OpenLDAP
## Public License.
##
## A copy of this license is available in the file LICENSE in the
## top-level directory of the distribution or, alternatively, at
## <http://www.OpenLDAP.org/license.html>.
SRCS = init.c tools.c config.c \
add.c bind.c compare.c delete.c search.c \
operational.c \
attr.c index.c key.c filterindex.c \
dn2entry.c dn2id.c id2entry.c idl.c \
nextid.c ctx.c
OBJS = init.lo tools.lo config.lo \
add.lo bind.lo compare.lo delete.lo search.lo \
operational.lo \
attr.lo index.lo key.lo filterindex.lo \
dn2entry.lo dn2id.lo id2entry.lo idl.lo \
nextid.lo ctx.lo
LDAP_INCDIR= ../../../include
LDAP_LIBDIR= ../../../libraries
BUILD_OPT = "--enable-wt"
BUILD_MOD = @BUILD_WT@
mod_DEFS = -DSLAPD_IMPORT
MOD_DEFS = @WT_INCS@
MOD_LIBS = @WT_LIBS@
shared_LDAP_LIBS = $(LDAP_LIBLDAP_R_LA) $(LDAP_LIBLBER_LA)
NT_LINK_LIBS = -L.. -lslapd $(@BUILD_LIBS_DYNAMIC@_LDAP_LIBS)
UNIX_LINK_LIBS = $(@BUILD_LIBS_DYNAMIC@_LDAP_LIBS)
LIBBASE = back_wt
XINCPATH = -I.. -I$(srcdir)/..
XDEFS = $(MODULES_CPPFLAGS)
all-local-lib: ../.backend
../.backend: lib$(LIBBASE).a
@touch $@
/* OpenLDAP WiredTiger backend */
/* $OpenLDAP$ */
/* This work is part of OpenLDAP Software <http://www.openldap.org/>.
*
* Copyright 2002-2015 The OpenLDAP Foundation.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>.
*/
/* ACKNOWLEDGEMENTS:
* This work was developed by HAMANO Tsukasa <hamano@osstech.co.jp>
* based on back-bdb for inclusion in OpenLDAP Software.
* WiredTiger is a product of MongoDB Inc.
*/
#include "portable.h"
#include <stdio.h>
#include "back-wt.h"
#include "config.h"
int
wt_add( Operation *op, SlapReply *rs )
{
struct wt_info *wi = (struct wt_info *) op->o_bd->be_private;
struct berval pdn;
char textbuf[SLAP_TEXT_BUFLEN];
size_t textlen = sizeof textbuf;
AttributeDescription *children = slap_schema.si_ad_children;
AttributeDescription *entry = slap_schema.si_ad_entry;
ID eid;
int num_retries = 0;
int success;
LDAPControl **postread_ctrl = NULL;
LDAPControl *ctrls[SLAP_MAX_RESPONSE_CONTROLS];
int num_ctrls = 0;
wt_ctx *wc;
Entry *e = NULL;
Entry *p = NULL;
ID pid;
int rc;
Debug( LDAP_DEBUG_ARGS, "==> " LDAP_XSTRING(wt_add) ": %s\n",
op->ora_e->e_name.bv_val, 0, 0);
ctrls[num_ctrls] = 0;
/* check entry's schema */
rs->sr_err = entry_schema_check(
op, op->ora_e, NULL,
get_relax(op), 1, NULL, &rs->sr_text, textbuf, textlen );
if ( rs->sr_err != LDAP_SUCCESS ) {
Debug( LDAP_DEBUG_TRACE,
LDAP_XSTRING(wt_add)
": entry failed schema check: %s (%d)\n",
rs->sr_text, rs->sr_err, 0 );
goto return_results;
}
/* add opattrs to shadow as well, only missing attrs will actually
* be added; helps compatibility with older OL versions */
rs->sr_err = slap_add_opattrs( op, &rs->sr_text, textbuf, textlen, 1 );
if ( rs->sr_err != LDAP_SUCCESS ) {
Debug( LDAP_DEBUG_TRACE,
LDAP_XSTRING(wt_add)
": entry failed op attrs add: %s (%d)\n",
rs->sr_text, rs->sr_err, 0 );
goto return_results;
}
if ( get_assert( op ) &&
( test_filter( op, op->ora_e, get_assertion( op ))
!= LDAP_COMPARE_TRUE ))
{
rs->sr_err = LDAP_ASSERTION_FAILED;
goto return_results;
}
/* Not used
* subentry = is_entry_subentry( op->ora_e );
*/
/*
* Get the parent dn and see if the corresponding entry exists.
*/
if ( be_issuffix( op->o_bd, &op->ora_e->e_nname ) ) {
pdn = slap_empty_bv;
} else {
dnParent( &op->ora_e->e_nname, &pdn );
}
wc = wt_ctx_get(op, wi);
if( !wc ){
Debug( LDAP_DEBUG_ANY,
LDAP_XSTRING(wt_add)
": wt_ctx_get failed\n",
0, 0, 0 );
rs->sr_err = LDAP_OTHER;
rs->sr_text = "internal error";
send_ldap_result( op, rs );
return rs->sr_err;
}
rc = wt_dn2entry(op->o_bd, wc, &op->o_req_ndn, &e);
switch( rc ) {
case 0:
rs->sr_err = LDAP_ALREADY_EXISTS;
goto return_results;
break;
case WT_NOTFOUND:
break;
default:
/* TODO: retry handling */
Debug( LDAP_DEBUG_ANY,
LDAP_XSTRING(wt_add)
": error at wt_dn2entry() rc=%d\n",
rc, 0, 0 );
rs->sr_err = LDAP_OTHER;
rs->sr_text = "internal error";
goto return_results;
}
/* get parent entry */
rc = wt_dn2pentry(op->o_bd, wc, &op->o_req_ndn, &p);
switch( rc ){
case 0:
case WT_NOTFOUND:
break;
default:
Debug( LDAP_DEBUG_ANY,
LDAP_XSTRING(wt_add)
": error at wt_dn2pentry() rc=%d\n",
rc, 0, 0 );
rs->sr_err = LDAP_OTHER;
rs->sr_text = "internal error";
goto return_results;
}
if ( !p )
p = (Entry *)&slap_entry_root;
if ( !bvmatch( &pdn, &p->e_nname ) ) {
rs->sr_matched = ber_strdup_x( p->e_name.bv_val,
op->o_tmpmemctx );
if ( p != (Entry *)&slap_entry_root ) {
rs->sr_ref = is_entry_referral( p )
? get_entry_referrals( op, p )
: NULL;
wt_entry_return( p );
} else {
rs->sr_ref = NULL;
}
p = NULL;
Debug( LDAP_DEBUG_TRACE,
LDAP_XSTRING(wt_add) ": parent "
"does not exist\n", 0, 0, 0 );
rs->sr_err = LDAP_REFERRAL;
rs->sr_flags = REP_MATCHED_MUSTBEFREED | REP_REF_MUSTBEFREED;
goto return_results;
}
rs->sr_err = access_allowed( op, p,
children, NULL, ACL_WADD, NULL );
if ( ! rs->sr_err ) {
/*
if ( p != (Entry *)&slap_entry_root )
wt_entry_return( op, p );
*/
p = NULL;
Debug( LDAP_DEBUG_TRACE,
LDAP_XSTRING(wt_add) ": no write access to parent\n",
0, 0, 0 );
rs->sr_err = LDAP_INSUFFICIENT_ACCESS;
rs->sr_text = "no write access to parent";
goto return_results;;
}
if ( p != (Entry *)&slap_entry_root ) {
if ( is_entry_subentry( p ) ) {
wt_entry_return( p );
p = NULL;
/* parent is a subentry, don't allow add */
Debug( LDAP_DEBUG_TRACE,
LDAP_XSTRING(wt_add) ": parent is subentry\n",
0, 0, 0 );
rs->sr_err = LDAP_OBJECT_CLASS_VIOLATION;
rs->sr_text = "parent is a subentry";
goto return_results;;
}
if ( is_entry_alias( p ) ) {
wt_entry_return( p );
p = NULL;
/* parent is an alias, don't allow add */
Debug( LDAP_DEBUG_TRACE,
LDAP_XSTRING(wt_add) ": parent is alias\n",
0, 0, 0 );
rs->sr_err = LDAP_ALIAS_PROBLEM;
rs->sr_text = "parent is an alias";
goto return_results;;
}
if ( is_entry_referral( p ) ) {
BerVarray ref = get_entry_referrals( op, p );
/* parent is a referral, don't allow add */
rs->sr_matched = ber_strdup_x( p->e_name.bv_val,
op->o_tmpmemctx );
rs->sr_ref = referral_rewrite( ref, &p->e_name,
&op->o_req_dn, LDAP_SCOPE_DEFAULT );
ber_bvarray_free( ref );
wt_entry_return( p );
p = NULL;
Debug( LDAP_DEBUG_TRACE,
LDAP_XSTRING(wt_add) ": parent is referral\n",
0, 0, 0 );
rs->sr_err = LDAP_REFERRAL;
rs->sr_flags = REP_MATCHED_MUSTBEFREED | REP_REF_MUSTBEFREED;
goto return_results;
}
}
#if 0
if ( subentry ) {
/* FIXME: */
/* parent must be an administrative point of the required kind */
}
#endif
/* free parent */
if ( p != (Entry *)&slap_entry_root ) {
pid = p->e_id;
if ( p->e_nname.bv_len ) {
struct berval ppdn;
/* ITS#5326: use parent's DN if differs from provided one */
dnParent( &op->ora_e->e_name, &ppdn );
if ( !dn_match( &p->e_name, &ppdn ) ) {
struct berval rdn;
struct berval newdn;
dnRdn( &op->ora_e->e_name, &rdn );
build_new_dn( &newdn, &p->e_name, &rdn, NULL );
if ( op->ora_e->e_name.bv_val != op->o_req_dn.bv_val )
ber_memfree( op->ora_e->e_name.bv_val );
op->ora_e->e_name = newdn;
/* FIXME: should check whether
* dnNormalize(newdn) == e->e_nname ... */
}
}
wt_entry_return( p );
}
p = NULL;
rs->sr_err = access_allowed( op, op->ora_e,
entry, NULL, ACL_WADD, NULL );
if ( ! rs->sr_err ) {
Debug( LDAP_DEBUG_TRACE,
LDAP_XSTRING(wt_add) ": no write access to entry\n",
0, 0, 0 );
rs->sr_err = LDAP_INSUFFICIENT_ACCESS;
rs->sr_text = "no write access to entry";
goto return_results;
}
/*
* Check ACL for attribute write access
*/
if (!acl_check_modlist(op, op->ora_e, op->ora_modlist)) {
Debug( LDAP_DEBUG_TRACE,
LDAP_XSTRING(wt_add) ": no write access to attribute\n",
0, 0, 0 );
rs->sr_err = LDAP_INSUFFICIENT_ACCESS;
rs->sr_text = "no write access to attribute";
goto return_results;
}
rc = wc->session->begin_transaction(wc->session, NULL);
if( rc ) {
Debug( LDAP_DEBUG_TRACE,
LDAP_XSTRING(wt_add) ": begin_transaction failed: %s (%d)\n",
wiredtiger_strerror(rc), rc, 0 );
rs->sr_err = LDAP_OTHER;
rs->sr_text = "begin_transaction failed";
goto return_results;
}
Debug( LDAP_DEBUG_TRACE, LDAP_XSTRING(wt_add) ": session id: %p\n",
wc->session, 0, 0 );
wt_next_id( op->o_bd, &eid );
op->ora_e->e_id = eid;
rc = wt_dn2id_add( op, wc->session, pid, op->ora_e );
if( rc ){
Debug( LDAP_DEBUG_TRACE,
LDAP_XSTRING(wt_add)
": dn2id_add failed: %s (%d)\n",
wiredtiger_strerror(rc), rc, 0 );
switch( rc ) {
case WT_DUPLICATE_KEY:
rs->sr_err = LDAP_ALREADY_EXISTS;
break;
default:
rs->sr_err = LDAP_OTHER;
}
wc->session->rollback_transaction(wc->session, NULL);
goto return_results;
}
rc = wt_id2entry_add( op, wc->session, op->ora_e );
if ( rc ) {
Debug( LDAP_DEBUG_TRACE,
LDAP_XSTRING(wt_add)
": id2entry_add failed: %s (%d)\n",
wiredtiger_strerror(rc), rc, 0 );
if ( rc == LDAP_ADMINLIMIT_EXCEEDED ) {
rs->sr_err = LDAP_ADMINLIMIT_EXCEEDED;
rs->sr_text = "entry is too big";
} else {
rs->sr_err = LDAP_OTHER;
rs->sr_text = "entry store failed";
}
wc->session->rollback_transaction(wc->session, NULL);
goto return_results;
}
/* add indices */
rc = wt_index_entry_add( op, wc, op->ora_e );
if ( rc ) {
Debug(LDAP_DEBUG_TRACE,
"<== " LDAP_XSTRING(wt_add)
": index add failed: %s (%d)\n",
wiredtiger_strerror(rc), rc, 0 );
rs->sr_err = LDAP_OTHER;
rs->sr_text = "index add failed";
wc->session->rollback_transaction(wc->session, NULL);
goto return_results;
}
rc = wc->session->commit_transaction(wc->session, NULL);
if( rc ) {
Debug( LDAP_DEBUG_TRACE,
"<== " LDAP_XSTRING(wt_add)
": commit_transaction failed: %s (%d)\n",
wiredtiger_strerror(rc), rc, 0 );
rs->sr_err = LDAP_OTHER;
rs->sr_text = "commit_transaction failed";