Commit b73b0c61 authored by Kurt Zeilenga's avatar Kurt Zeilenga
Browse files

Enhance LDIF handling

ldapsearch:
    use draft guidelines for determining when to use
    -t only writes binary attributes to files
    -tt writes all attributes to files
    output now lists URL instead of path to such files
    -T dir can be used to specify directory to create temp files in
    -V urlprefix can be used to change the URL prefix
    LDIF is now commented (can be disabled using -LL)
    LDIF now contains version attribute (can be disabled with -LLL)
LDIF:
    put interface changed to allow caller to specify how to encode
    put interface uses draft guidelines for determine when to base64 encode
    wrapping kludged to match old off by one bug
Tests:
    removed trailing space from some attributes (to avoid base64 encoding)
    enabled -LLL in defines.sh (should sed output to remove
        wrapping/comments/redundant separators)
Misc:
    updated codes outputting LDIF to use new put interface
TODO:
    handling of version attribute on input
    handling of URLs on input (ie: fetch URL)
parent 18393e88
......@@ -340,7 +340,8 @@ static int
process_ldif_rec( char *rbuf )
{
char *line, *dn, *type, *value, *newrdn, *newsup, *p;
int rc, linenum, vlen, modop, replicaport;
int rc, linenum, modop, replicaport;
ber_len_t vlen;
int expect_modop, expect_sep, expect_ct, expect_newrdn, expect_newsup;
int expect_deleteoldrdn, deleteoldrdn;
int saw_replica, use_record, new_entry, delete_entry, got_all;
......
This diff is collapsed.
......@@ -7,6 +7,7 @@
* license is available at http://www.OpenLDAP.org/license.html or
* in file LICENSE in the top-level directory of the distribution.
*/
/* Portions
* Copyright (c) 1996 Regents of the University of Michigan.
* All rights reserved.
......@@ -32,10 +33,10 @@ extern int ldif_debug;
/*
* Macro to calculate maximum number of bytes that the base64 equivalent
* of an item that is "vlen" bytes long will take up. Base64 encoding
* of an item that is "len" bytes long will take up. Base64 encoding
* uses one byte for every six bits in the value plus up to two pad bytes.
*/
#define LDIF_BASE64_LEN(vlen) (((vlen) * 4 / 3 ) + 3)
#define LDIF_BASE64_LEN(len) (((len) * 4 / 3 ) + 3)
/*
* Macro to calculate maximum size that an LDIF-encoded type (length
......@@ -43,31 +44,49 @@ extern int ldif_debug;
* first newline + base64 value + continued lines. Each continued line
* needs room for a newline and a leading space character.
*/
#define LDIF_SIZE_NEEDED(tlen,vlen) \
((tlen) + 4 + LDIF_BASE64_LEN(vlen) \
+ ((LDIF_BASE64_LEN(vlen) + (tlen) + 3) / LDIF_LINE_WIDTH * 2 ))
#define LDIF_SIZE_NEEDED(nlen,vlen) \
((nlen) + 4 + LDIF_BASE64_LEN(vlen) \
+ ((LDIF_BASE64_LEN(vlen) + (nlen) + 3) / LDIF_LINE_WIDTH * 2 ))
LDAP_F( int )
ldif_parse_line LDAP_P((
LDAP_CONST char *line,
char **type, char **value, int *vlen));
char **name,
char **value,
ber_len_t *vlen));
LDAP_F( char * )
ldif_getline LDAP_P(( char **next ));
#define LDIF_PUT_NOVALUE 0x0000 /* no value */
#define LDIF_PUT_VALUE 0x0001 /* value w/ auto detection */
#define LDIF_PUT_TEXT 0x0002 /* assume text */
#define LDIF_PUT_BINARY 0x0004 /* assume binary (convert to base64) */
#define LDIF_PUT_B64 0x0008 /* pre-converted base64 value */
#define LDIF_PUT_COMMENT 0x0010 /* comment */
#define LDIF_PUT_URL 0x0020 /* url */
#define LDIF_PUT_SEP 0x0040 /* separator */
LDAP_F( void )
ldif_put_type_and_value LDAP_P((
ldif_sput LDAP_P((
char **out,
LDAP_CONST char *t,
int type,
LDAP_CONST char *name,
LDAP_CONST char *val,
int vlen ));
ber_len_t vlen ));
LDAP_F( char * )
ldif_type_and_value LDAP_P((
LDAP_CONST char *type,
ldif_put LDAP_P((
int type,
LDAP_CONST char *name,
LDAP_CONST char *val,
int vlen ));
ber_len_t vlen ));
LDAP_F( int )
ldif_is_not_printable LDAP_P((
LDAP_CONST char *val,
ber_len_t vlen ));
LDAP_END_DECL
......
......@@ -55,7 +55,7 @@ ldif_parse_line(
LDAP_CONST char *line,
char **type,
char **value,
int *vlen
ber_len_t *vlen
)
{
char *p, *s, *d, *byte, *stop;
......@@ -69,7 +69,7 @@ ldif_parse_line(
*type = line;
for ( s = line; *s && *s != ':'; s++ )
; /* NULL */
/* EMPTY */;
if ( *s == '\0' ) {
ber_pvt_log_printf( LDAP_DEBUG_PARSE, ldif_debug,
"ldif_parse_line missing ':'\n");
......@@ -167,9 +167,11 @@ ldif_parse_line(
* space character, it is a continuation of the previous line. the white
* space character (nb: only one char), and preceeding newline are changed
* into CONTINUED_LINE_MARKER chars, to be deleted later by the
* str_parse_line() routine above.
* ldif_parse_line() routine above.
*
* it takes a pointer to a pointer to the buffer on the first call,
* ldif_getline will skip over any line which starts '#'.
*
* ldif_getline takes a pointer to a pointer to the buffer on the first call,
* which it updates and must be supplied on subsequent calls.
*/
......@@ -202,99 +204,198 @@ ldif_getline( char **next )
return( line );
}
/* compatibility with U-Mich off by one bug */
#define LDIF_KLUDGE 1
void
ldif_put_type_and_value(
ldif_sput(
char **out,
LDAP_CONST char *t,
int type,
LDAP_CONST char *name,
LDAP_CONST char *val,
int vlen )
ber_len_t vlen )
{
unsigned char *byte, *p, *stop;
unsigned char *byte, *stop;
unsigned char buf[3];
unsigned long bits;
char *save;
int i, b64, pad, len, savelen;
len = 0;
int pad;
ber_len_t savelen;
ber_len_t len=0;
ber_len_t i;
/* prefix */
switch( type ) {
case LDIF_PUT_COMMENT:
if( name != NULL ) break;
*(*out)++ = '#';
len++;
if( vlen ) {
*(*out)++ = ' ';
len++;
break;
}
/* no value, fall-thru */
/* put the type + ": " */
for ( p = (unsigned char *) t; *p; p++, len++ ) {
*(*out)++ = *p;
case LDIF_PUT_SEP:
*(*out)++ = '\n';
return;
}
*(*out)++ = ':';
len++;
/* name (attribute type) */
if( name != NULL ) {
/* put the name + ":" */
for ( i=0 ; name[i]; i++ ) {
*(*out)++ = name[i];
len++;
}
if( type != LDIF_PUT_COMMENT ) {
*(*out)++ = ':';
len++;
}
#ifdef LDAP_DEBUG
} else {
assert( type == LDIF_PUT_COMMENT );
#endif
}
switch( type ) {
case LDIF_PUT_NOVALUE:
*(*out)++ = '\n';
return;
case LDIF_PUT_URL: /* url value */
*(*out)++ = '<';
len++;
break;
case LDIF_PUT_B64: /* base64 value */
*(*out)++ = ':';
len++;
break;
}
switch( type ) {
case LDIF_PUT_TEXT:
case LDIF_PUT_URL:
case LDIF_PUT_B64:
*(*out)++ = ' ';
len++;
/* fall-thru */
case LDIF_PUT_COMMENT:
/* pre-encoded names */
for ( i=0; i < vlen; i++ ) {
if ( len > LDIF_LINE_WIDTH ) {
*(*out)++ = '\n';
*(*out)++ = ' ';
len = 1;
}
*(*out)++ = val[i];
len++;
}
*(*out)++ = '\n';
return;
}
save = *out;
savelen = len;
*(*out)++ = ' ';
b64 = 0;
len++;
if( vlen == 0 ) {
*(*out)++ = '\n';
return;
}
stop = (unsigned char *) (val + vlen);
if ( isascii( val[0] ) && (isspace( val[0] ) || val[0] == ':') ) {
b64 = 1;
} else {
if ( type == LDIF_PUT_VALUE &&
isgraph( val[0] ) && val[0] != ':' && val[0] != '<' &&
isgraph( val[vlen-1] ) )
{
int b64 = 0;
for ( byte = (unsigned char *) val; byte < stop;
byte++, len++ ) {
byte++, len++ )
{
if ( !isascii( *byte ) || !isprint( *byte ) ) {
b64 = 1;
break;
}
if ( len > LDIF_LINE_WIDTH ) {
if ( len > LDIF_LINE_WIDTH+LDIF_KLUDGE ) {
*(*out)++ = '\n';
*(*out)++ = ' ';
len = 1;
}
*(*out)++ = *byte;
}
if( !b64 ) {
*(*out)++ = '\n';
return;
}
}
if ( b64 ) {
*out = save;
*(*out)++ = ':';
*(*out)++ = ' ';
len = savelen + 2;
/* convert to base 64 (3 bytes => 4 base 64 digits) */
for ( byte = (unsigned char *) val; byte < stop - 2;
byte += 3 ) {
bits = (byte[0] & 0xff) << 16;
bits |= (byte[1] & 0xff) << 8;
bits |= (byte[2] & 0xff);
for ( i = 0; i < 4; i++, len++, bits <<= 6 ) {
if ( len > LDIF_LINE_WIDTH ) {
*(*out)++ = '\n';
*(*out)++ = ' ';
len = 1;
}
/* get b64 digit from high order 6 bits */
*(*out)++ = nib2b64[ (bits & 0xfc0000L) >> 18 ];
*out = save;
*(*out)++ = ':';
*(*out)++ = ' ';
len = savelen + 2;
/* convert to base 64 (3 bytes => 4 base 64 digits) */
for ( byte = (unsigned char *) val;
byte < stop - 2;
byte += 3 )
{
bits = (byte[0] & 0xff) << 16;
bits |= (byte[1] & 0xff) << 8;
bits |= (byte[2] & 0xff);
for ( i = 0; i < 4; i++, len++, bits <<= 6 ) {
if ( len > LDIF_LINE_WIDTH+LDIF_KLUDGE ) {
*(*out)++ = '\n';
*(*out)++ = ' ';
len = 1;
}
/* get b64 digit from high order 6 bits */
*(*out)++ = nib2b64[ (bits & 0xfc0000L) >> 18 ];
}
}
/* add padding if necessary */
if ( byte < stop ) {
for ( i = 0; byte + i < stop; i++ ) {
buf[i] = byte[i];
}
for ( pad = 0; i < 3; i++, pad++ ) {
buf[i] = '\0';
/* add padding if necessary */
if ( byte < stop ) {
for ( i = 0; byte + i < stop; i++ ) {
buf[i] = byte[i];
}
for ( pad = 0; i < 3; i++, pad++ ) {
buf[i] = '\0';
}
byte = buf;
bits = (byte[0] & 0xff) << 16;
bits |= (byte[1] & 0xff) << 8;
bits |= (byte[2] & 0xff);
for ( i = 0; i < 4; i++, len++, bits <<= 6 ) {
if ( len > LDIF_LINE_WIDTH+LDIF_KLUDGE ) {
*(*out)++ = '\n';
*(*out)++ = ' ';
len = 1;
}
byte = buf;
bits = (byte[0] & 0xff) << 16;
bits |= (byte[1] & 0xff) << 8;
bits |= (byte[2] & 0xff);
for ( i = 0; i < 4; i++, len++, bits <<= 6 ) {
if ( len > LDIF_LINE_WIDTH ) {
*(*out)++ = '\n';
*(*out)++ = ' ';
len = 1;
}
if( i + pad < 4 ) {
/* get b64 digit from low order 6 bits */
*(*out)++ = nib2b64[ (bits & 0xfc0000L) >> 18 ];
} else {
*(*out)++ = '=';
}
if( i + pad < 4 ) {
/* get b64 digit from low order 6 bits */
*(*out)++ = nib2b64[ (bits & 0xfc0000L) >> 18 ];
} else {
*(*out)++ = '=';
}
}
}
......@@ -302,27 +403,57 @@ ldif_put_type_and_value(
}
char *
ldif_type_and_value( LDAP_CONST char *type, LDAP_CONST char *val, int vlen )
/*
* return BER malloc'd, zero-terminated LDIF line
* ldif_type_and_value return BER malloc'd, zero-terminated LDIF line
*/
char *
ldif_put(
int type,
LDAP_CONST char *name,
LDAP_CONST char *val,
ber_len_t vlen )
{
char *buf, *p;
int tlen;
ber_len_t nlen;
tlen = strlen( type );
if (( buf = (char *) ber_memalloc( LDIF_SIZE_NEEDED( tlen, vlen ) + 1 ))
== NULL )
{
nlen = ( name != NULL ) ? strlen( name ) : 0;
buf = (char *) ber_memalloc( LDIF_SIZE_NEEDED( nlen, vlen ) + 1 );
if ( buf == NULL ) {
ber_pvt_log_printf( LDAP_DEBUG_ANY, ldif_debug,
"ldif_type_and_value: malloc failed!" );
return NULL;
}
p = buf;
ldif_put_type_and_value( &p, type, val, vlen );
ldif_sput( &p, type, name, val, vlen );
*p = '\0';
return( buf );
}
int ldif_is_not_printable(
LDAP_CONST char *val,
ber_len_t vlen )
{
if( vlen == 0 || val == NULL ) {
return -1;
}
if( isgraph( val[0] ) && val[0] != ':' && val[0] != '<' &&
isgraph( val[vlen-1] ) )
{
ber_len_t i;
for ( i = 0; val[i]; i++ ) {
if ( !isascii( val[i] ) || !isprint( val[i] ) ) {
return 1;
}
}
return 0;
}
return 1;
}
......@@ -24,7 +24,8 @@ str2entry( char *s )
char *type;
char *value;
char *next;
int vlen, nvals, maxvals;
ber_len_t vlen;
int nvals, maxvals;
struct berval bval;
struct berval *vals[2];
char ptype[64];
......@@ -194,7 +195,7 @@ entry2str(
/* put "dn: <dn>" */
tmplen = strlen( e->e_dn );
MAKE_SPACE( LDIF_SIZE_NEEDED( 2, tmplen ));
ldif_put_type_and_value( (char **) &ecur, "dn", e->e_dn, tmplen );
ldif_sput( (char **) &ecur, LDIF_PUT_VALUE, "dn", e->e_dn, tmplen );
}
/* put the attributes */
......@@ -204,7 +205,7 @@ entry2str(
bv = a->a_vals[i];
tmplen = strlen( a->a_type );
MAKE_SPACE( LDIF_SIZE_NEEDED( tmplen, bv->bv_len ));
ldif_put_type_and_value( (char **) &ecur, a->a_type,
ldif_sput( (char **) &ecur, LDIF_PUT_VALUE, a->a_type,
bv->bv_val, bv->bv_len );
}
}
......
......@@ -77,7 +77,8 @@ replog(
buf = (char *) ch_malloc( len );
bufp = buf;
ldif_put_type_and_value( &bufp, ml->ml_type,
ldif_sput( &bufp, LDIF_PUT_VALUE,
ml->ml_type,
ml->ml_bvalues[i]->bv_val,
ml->ml_bvalues[i]->bv_len );
*bufp = '\0';
......
......@@ -71,7 +71,7 @@ main( int argc, char **argv )
cur += nread;
}
if (( out = ldif_type_and_value( type, val, cur )) == NULL ) {
if (( out = ldif_put( LDIF_PUT_BINARY, type, val, cur )) == NULL ) {
perror( "ldif_type_and_value" );
exit( 1 );
}
......@@ -86,7 +86,7 @@ main( int argc, char **argv )
while ( fgets( buf, sizeof(buf), stdin ) != NULL ) {
if( buf[len=strlen(buf)] == '\n') buf[len] = '\0';
if (( out = ldif_type_and_value( type, buf, strlen( buf ) ))
if (( out = ldif_put( LDIF_PUT_VALUE, type, buf, strlen( buf ) ))
== NULL ) {
perror( "ldif_type_and_value" );
exit( 1 );
......
......@@ -130,7 +130,7 @@ main( int argc, char **argv )
vals[1] = NULL;
while ( ! stop ) {
char *type, *val, *s;
int vlen;
ber_len_t vlen;
Datum key, data;
ldbm_datum_init( key );
......
......@@ -132,7 +132,7 @@ main( int argc, char **argv )
vals[1] = NULL;
while ( ! stop ) {
char *type, *val, *s;
int vlen;
ber_len_t vlen;
Datum key, data;
ldbm_datum_init( key );
......@@ -219,7 +219,7 @@ main( int argc, char **argv )
vals[1] = NULL;
while ( ! stop ) {
char *type, *val, *s, *dn;
int vlen;
ber_len_t vlen;
ID pid;
char buf2[20];
Datum key, data;
......
......@@ -121,7 +121,7 @@ main( int argc, char **argv )
vals[1] = NULL;
while ( ! stop ) {
char *type, *val, *s;
int vlen;
ber_len_t vlen;
if ( fgets( line, sizeof(line), stdin ) != NULL ) {
int len;
......
......@@ -123,7 +123,7 @@ main( int argc, char **argv )
vals[1] = NULL;
while ( ! stop ) {
char *type, *val, *s;
int vlen;
ber_len_t vlen;
if ( fgets( line, sizeof(line), stdin ) != NULL ) {
int len;
......
......@@ -214,7 +214,8 @@ main( int argc, char **argv )
vals[1] = NULL;
while ( ! stop ) {
char *type, *val, *s;
int vlen, indexmask, syntaxmask;
ber_len_t vlen;
int indexmask, syntaxmask;
if ( fgets( line, sizeof(line), stdin ) != NULL ) {
int len;
......
......@@ -222,7 +222,8 @@ main( int argc, char **argv )
vals[1] = NULL;
while ( ! stop ) {
char *type, *val, *s;
int vlen, indexmask, syntaxmask;
ber_len_t vlen;
int indexmask, syntaxmask;
if ( fgets( line, sizeof(line), stdin ) != NULL ) {
int len;
......
......@@ -125,9 +125,9 @@ Re_parse(
int state;
int nml;
char *buf, *rp, *p;
long buflen;
size_t buflen;
char *type, *value;
int len;
ber_len_t len;
int nreplicas;
if ( re == NULL ) {
......@@ -158,7 +158,7 @@ Re_parse(
if ( strncmp( buf, ERROR_STR, strlen( ERROR_STR )) == 0 ) {
continue;
}
buflen = ( long ) strlen( buf );
buflen = strlen( buf );
if ( ldif_parse_line( buf, &type, &value, &len ) < 0 ) {
Debug( LDAP_DEBUG_ANY,
"Error: Re_parse: malformed replog file\n",
......@@ -208,7 +208,7 @@ Re_parse(
if (( buf = ldif_getline( &rp )) == NULL ) {
break;
}
buflen = ( long ) strlen( buf );
buflen = strlen( buf );
if (( buflen == 1 ) && ( buf[ 0 ] == '-' )) {
type = "-";
value = NULL;
......@@ -259,7 +259,8 @@ get_repl_hosts(
char buf[ LDIF_LINE_WIDTH + 1 ];