mdb_load.c 11.3 KB
Newer Older
Howard Chu's avatar
Howard Chu committed
1
2
/* mdb_load.c - memory-mapped database load tool */
/*
Howard Chu's avatar
Howard Chu committed
3
 * Copyright 2011-2017 Howard Chu, Symas Corp.
Howard Chu's avatar
Howard Chu committed
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted only as authorized by the OpenLDAP
 * Public License.
 *
 * A copy of this license is available in the file LICENSE in the
 * top-level directory of the distribution or, alternatively, at
 * <http://www.OpenLDAP.org/license.html>.
 */
#include <stdio.h>
#include <stdlib.h>
#include <errno.h>
#include <string.h>
#include <ctype.h>
#include <unistd.h>
#include "lmdb.h"

#define PRINT	1
#define NOHDR	2
static int mode;

static char *subname = NULL;

28
static mdb_size_t lineno;
Howard Chu's avatar
Howard Chu committed
29
30
31
32
33
34
static int version;

static int flags;

static char *prog;

Howard Chu's avatar
Howard Chu committed
35
static int Eof;
Howard Chu's avatar
Howard Chu committed
36

Howard Chu's avatar
Howard Chu committed
37
38
static MDB_envinfo info;

Howard Chu's avatar
Howard Chu committed
39
static MDB_val kbuf, dbuf;
Howard Chu's avatar
Howard Chu committed
40
static MDB_val k0buf;
Howard Chu's avatar
Howard Chu committed
41

42
#define Yu	MDB_PRIy(u)
Howard Chu's avatar
Howard Chu committed
43

Howard Chu's avatar
Howard Chu committed
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
#define STRLENOF(s)	(sizeof(s)-1)

typedef struct flagbit {
	int bit;
	char *name;
	int len;
} flagbit;

#define S(s)	s, STRLENOF(s)

flagbit dbflags[] = {
	{ MDB_REVERSEKEY, S("reversekey") },
	{ MDB_DUPSORT, S("dupsort") },
	{ MDB_INTEGERKEY, S("integerkey") },
	{ MDB_DUPFIXED, S("dupfixed") },
	{ MDB_INTEGERDUP, S("integerdup") },
	{ MDB_REVERSEDUP, S("reversedup") },
	{ 0, NULL, 0 }
};

Howard Chu's avatar
Howard Chu committed
64
static void readhdr(void)
Howard Chu's avatar
Howard Chu committed
65
66
67
68
69
70
{
	char *ptr;

	while (fgets(dbuf.mv_data, dbuf.mv_size, stdin) != NULL) {
		lineno++;
		if (!strncmp(dbuf.mv_data, "VERSION=", STRLENOF("VERSION="))) {
Howard Chu's avatar
Howard Chu committed
71
			version=atoi((char *)dbuf.mv_data+STRLENOF("VERSION="));
Howard Chu's avatar
Howard Chu committed
72
			if (version > 3) {
73
				fprintf(stderr, "%s: line %"Yu": unsupported VERSION %d\n",
Howard Chu's avatar
Howard Chu committed
74
75
76
77
78
79
					prog, lineno, version);
				exit(EXIT_FAILURE);
			}
		} else if (!strncmp(dbuf.mv_data, "HEADER=END", STRLENOF("HEADER=END"))) {
			break;
		} else if (!strncmp(dbuf.mv_data, "format=", STRLENOF("format="))) {
Howard Chu's avatar
Howard Chu committed
80
			if (!strncmp((char *)dbuf.mv_data+STRLENOF("FORMAT="), "print", STRLENOF("print")))
Howard Chu's avatar
Howard Chu committed
81
				mode |= PRINT;
Howard Chu's avatar
Howard Chu committed
82
			else if (strncmp((char *)dbuf.mv_data+STRLENOF("FORMAT="), "bytevalue", STRLENOF("bytevalue"))) {
83
				fprintf(stderr, "%s: line %"Yu": unsupported FORMAT %s\n",
Howard Chu's avatar
Howard Chu committed
84
85
86
87
88
89
90
					prog, lineno, (char *)dbuf.mv_data+STRLENOF("FORMAT="));
				exit(EXIT_FAILURE);
			}
		} else if (!strncmp(dbuf.mv_data, "database=", STRLENOF("database="))) {
			ptr = memchr(dbuf.mv_data, '\n', dbuf.mv_size);
			if (ptr) *ptr = '\0';
			if (subname) free(subname);
Howard Chu's avatar
Howard Chu committed
91
			subname = strdup((char *)dbuf.mv_data+STRLENOF("database="));
Howard Chu's avatar
Howard Chu committed
92
		} else if (!strncmp(dbuf.mv_data, "type=", STRLENOF("type="))) {
Howard Chu's avatar
Howard Chu committed
93
			if (strncmp((char *)dbuf.mv_data+STRLENOF("type="), "btree", STRLENOF("btree")))  {
94
				fprintf(stderr, "%s: line %"Yu": unsupported type %s\n",
Howard Chu's avatar
Howard Chu committed
95
96
97
					prog, lineno, (char *)dbuf.mv_data+STRLENOF("type="));
				exit(EXIT_FAILURE);
			}
Howard Chu's avatar
Howard Chu committed
98
99
100
101
102
103
		} else if (!strncmp(dbuf.mv_data, "mapaddr=", STRLENOF("mapaddr="))) {
			int i;
			ptr = memchr(dbuf.mv_data, '\n', dbuf.mv_size);
			if (ptr) *ptr = '\0';
			i = sscanf((char *)dbuf.mv_data+STRLENOF("mapaddr="), "%p", &info.me_mapaddr);
			if (i != 1) {
104
				fprintf(stderr, "%s: line %"Yu": invalid mapaddr %s\n",
Howard Chu's avatar
Howard Chu committed
105
106
107
108
109
110
111
					prog, lineno, (char *)dbuf.mv_data+STRLENOF("mapaddr="));
				exit(EXIT_FAILURE);
			}
		} else if (!strncmp(dbuf.mv_data, "mapsize=", STRLENOF("mapsize="))) {
			int i;
			ptr = memchr(dbuf.mv_data, '\n', dbuf.mv_size);
			if (ptr) *ptr = '\0';
112
113
			i = sscanf((char *)dbuf.mv_data+STRLENOF("mapsize="),
				"%" MDB_SCNy(u), &info.me_mapsize);
Howard Chu's avatar
Howard Chu committed
114
			if (i != 1) {
115
				fprintf(stderr, "%s: line %"Yu": invalid mapsize %s\n",
Howard Chu's avatar
Howard Chu committed
116
117
118
119
120
121
122
123
124
					prog, lineno, (char *)dbuf.mv_data+STRLENOF("mapsize="));
				exit(EXIT_FAILURE);
			}
		} else if (!strncmp(dbuf.mv_data, "maxreaders=", STRLENOF("maxreaders="))) {
			int i;
			ptr = memchr(dbuf.mv_data, '\n', dbuf.mv_size);
			if (ptr) *ptr = '\0';
			i = sscanf((char *)dbuf.mv_data+STRLENOF("maxreaders="), "%u", &info.me_maxreaders);
			if (i != 1) {
125
				fprintf(stderr, "%s: line %"Yu": invalid maxreaders %s\n",
Howard Chu's avatar
Howard Chu committed
126
127
128
					prog, lineno, (char *)dbuf.mv_data+STRLENOF("maxreaders="));
				exit(EXIT_FAILURE);
			}
Howard Chu's avatar
Howard Chu committed
129
130
131
132
133
134
135
136
137
138
139
140
		} else {
			int i;
			for (i=0; dbflags[i].bit; i++) {
				if (!strncmp(dbuf.mv_data, dbflags[i].name, dbflags[i].len) &&
					((char *)dbuf.mv_data)[dbflags[i].len] == '=') {
					flags |= dbflags[i].bit;
					break;
				}
			}
			if (!dbflags[i].bit) {
				ptr = memchr(dbuf.mv_data, '=', dbuf.mv_size);
				if (!ptr) {
141
					fprintf(stderr, "%s: line %"Yu": unexpected format\n",
Howard Chu's avatar
Howard Chu committed
142
143
144
145
						prog, lineno);
					exit(EXIT_FAILURE);
				} else {
					*ptr = '\0';
146
					fprintf(stderr, "%s: line %"Yu": unrecognized keyword ignored: %s\n",
Howard Chu's avatar
Howard Chu committed
147
148
149
150
151
152
153
						prog, lineno, (char *)dbuf.mv_data);
				}
			}
		}
	}
}

Howard Chu's avatar
Howard Chu committed
154
static void badend(void)
Howard Chu's avatar
Howard Chu committed
155
{
156
	fprintf(stderr, "%s: line %"Yu": unexpected end of input\n",
Howard Chu's avatar
Howard Chu committed
157
158
159
160
161
162
163
164
		prog, lineno);
}

static int unhex(unsigned char *c2)
{
	int x, c;
	x = *c2++ & 0x4f;
	if (x & 0x40)
Howard Chu's avatar
Howard Chu committed
165
		x -= 55;
Howard Chu's avatar
Howard Chu committed
166
167
168
	c = x << 4;
	x = *c2 & 0x4f;
	if (x & 0x40)
Howard Chu's avatar
Howard Chu committed
169
		x -= 55;
Howard Chu's avatar
Howard Chu committed
170
171
172
173
174
175
176
	c |= x;
	return c;
}

static int readline(MDB_val *out, MDB_val *buf)
{
	unsigned char *c1, *c2, *end;
177
	size_t len, l2;
Howard Chu's avatar
Howard Chu committed
178
179
180
181
182
	int c;

	if (!(mode & NOHDR)) {
		c = fgetc(stdin);
		if (c == EOF) {
Howard Chu's avatar
Howard Chu committed
183
			Eof = 1;
Howard Chu's avatar
Howard Chu committed
184
185
186
			return EOF;
		}
		if (c != ' ') {
Howard Chu's avatar
Howard Chu committed
187
			lineno++;
Howard Chu's avatar
Howard Chu committed
188
189
			if (fgets(buf->mv_data, buf->mv_size, stdin) == NULL) {
badend:
Howard Chu's avatar
Howard Chu committed
190
				Eof = 1;
Howard Chu's avatar
Howard Chu committed
191
192
193
194
195
196
197
198
199
				badend();
				return EOF;
			}
			if (c == 'D' && !strncmp(buf->mv_data, "ATA=END", STRLENOF("ATA=END")))
				return EOF;
			goto badend;
		}
	}
	if (fgets(buf->mv_data, buf->mv_size, stdin) == NULL) {
Howard Chu's avatar
Howard Chu committed
200
		Eof = 1;
Howard Chu's avatar
Howard Chu committed
201
202
203
204
205
206
		return EOF;
	}
	lineno++;

	c1 = buf->mv_data;
	len = strlen((char *)c1);
207
	l2 = len;
Howard Chu's avatar
Howard Chu committed
208
209
210
211
212

	/* Is buffer too short? */
	while (c1[len-1] != '\n') {
		buf->mv_data = realloc(buf->mv_data, buf->mv_size*2);
		if (!buf->mv_data) {
Howard Chu's avatar
Howard Chu committed
213
			Eof = 1;
214
			fprintf(stderr, "%s: line %"Yu": out of memory, line too long\n",
Howard Chu's avatar
Howard Chu committed
215
216
217
218
				prog, lineno);
			return EOF;
		}
		c1 = buf->mv_data;
219
220
		c1 += l2;
		if (fgets((char *)c1, buf->mv_size+1, stdin) == NULL) {
Howard Chu's avatar
Howard Chu committed
221
			Eof = 1;
Howard Chu's avatar
Howard Chu committed
222
223
224
225
226
			badend();
			return EOF;
		}
		buf->mv_size *= 2;
		len = strlen((char *)c1);
227
		l2 += len;
Howard Chu's avatar
Howard Chu committed
228
229
	}
	c1 = c2 = buf->mv_data;
230
	len = l2;
Howard Chu's avatar
Howard Chu committed
231
232
233
234
235
236
237
238
239
	c1[--len] = '\0';
	end = c1 + len;

	if (mode & PRINT) {
		while (c2 < end) {
			if (*c2 == '\\') {
				if (c2[1] == '\\') {
					c1++; c2 += 2;
				} else {
Howard Chu's avatar
Howard Chu committed
240
					if (c2+3 > end || !isxdigit(c2[1]) || !isxdigit(c2[2])) {
Howard Chu's avatar
Howard Chu committed
241
						Eof = 1;
Howard Chu's avatar
Howard Chu committed
242
243
244
245
246
247
248
						badend();
						return EOF;
					}
					*c1++ = unhex(++c2);
					c2 += 2;
				}
			} else {
249
250
				/* copies are redundant when no escapes were used */
				*c1++ = *c2++;
Howard Chu's avatar
Howard Chu committed
251
252
253
254
255
			}
		}
	} else {
		/* odd length not allowed */
		if (len & 1) {
Howard Chu's avatar
Howard Chu committed
256
			Eof = 1;
Howard Chu's avatar
Howard Chu committed
257
258
259
260
261
			badend();
			return EOF;
		}
		while (c2 < end) {
			if (!isxdigit(*c2) || !isxdigit(c2[1])) {
Howard Chu's avatar
Howard Chu committed
262
				Eof = 1;
Howard Chu's avatar
Howard Chu committed
263
264
265
266
267
268
269
270
271
272
273
274
275
				badend();
				return EOF;
			}
			*c1++ = unhex(c2);
			c2 += 2;
		}
	}
	c2 = out->mv_data = buf->mv_data;
	out->mv_size = c1 - c2;

	return 0;
}

Howard Chu's avatar
Howard Chu committed
276
static void usage(void)
Howard Chu's avatar
Howard Chu committed
277
{
Howard Chu's avatar
Howard Chu committed
278
	fprintf(stderr, "usage: %s [-V] [-a] [-f input] [-n] [-s name] [-N] [-T] dbpath\n", prog);
Howard Chu's avatar
Howard Chu committed
279
280
281
	exit(EXIT_FAILURE);
}

Howard Chu's avatar
Howard Chu committed
282
283
284
285
286
static int greater(const MDB_val *a, const MDB_val *b)
{
	return 1;
}

Howard Chu's avatar
Howard Chu committed
287
288
289
290
291
292
293
294
int main(int argc, char *argv[])
{
	int i, rc;
	MDB_env *env;
	MDB_txn *txn;
	MDB_cursor *mc;
	MDB_dbi dbi;
	char *envname;
Howard Chu's avatar
Howard Chu committed
295
296
297
	int envflags = MDB_NOSYNC, putflags = 0;
	int dohdr = 0, append = 0;
	MDB_val prevk;
Howard Chu's avatar
Howard Chu committed
298
299
300
301

	prog = argv[0];

	if (argc < 2) {
Howard Chu's avatar
Howard Chu committed
302
		usage();
Howard Chu's avatar
Howard Chu committed
303
304
	}

Howard Chu's avatar
Howard Chu committed
305
306
	/* -a: append records in input order
	 * -f: load file instead of stdin
Howard Chu's avatar
Howard Chu committed
307
308
309
310
311
312
	 * -n: use NOSUBDIR flag on env_open
	 * -s: load into named subDB
	 * -N: use NOOVERWRITE on puts
	 * -T: read plaintext
	 * -V: print version and exit
	 */
Howard Chu's avatar
Howard Chu committed
313
	while ((i = getopt(argc, argv, "af:ns:NTV")) != EOF) {
Howard Chu's avatar
Howard Chu committed
314
315
316
317
318
		switch(i) {
		case 'V':
			printf("%s\n", MDB_VERSION_STRING);
			exit(0);
			break;
Howard Chu's avatar
Howard Chu committed
319
320
321
		case 'a':
			append = 1;
			break;
Howard Chu's avatar
Howard Chu committed
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
		case 'f':
			if (freopen(optarg, "r", stdin) == NULL) {
				fprintf(stderr, "%s: %s: reopen: %s\n",
					prog, optarg, strerror(errno));
				exit(EXIT_FAILURE);
			}
			break;
		case 'n':
			envflags |= MDB_NOSUBDIR;
			break;
		case 's':
			subname = strdup(optarg);
			break;
		case 'N':
			putflags = MDB_NOOVERWRITE|MDB_NODUPDATA;
			break;
		case 'T':
339
			mode |= NOHDR | PRINT;
Howard Chu's avatar
Howard Chu committed
340
341
			break;
		default:
Howard Chu's avatar
Howard Chu committed
342
			usage();
Howard Chu's avatar
Howard Chu committed
343
344
345
346
		}
	}

	if (optind != argc - 1)
Howard Chu's avatar
Howard Chu committed
347
		usage();
Howard Chu's avatar
Howard Chu committed
348

Howard Chu's avatar
Howard Chu committed
349
350
351
352
353
354
	dbuf.mv_size = 4096;
	dbuf.mv_data = malloc(dbuf.mv_size);

	if (!(mode & NOHDR))
		readhdr();

Howard Chu's avatar
Howard Chu committed
355
356
	envname = argv[optind];
	rc = mdb_env_create(&env);
Howard Chu's avatar
Howard Chu committed
357
358
359
360
	if (rc) {
		fprintf(stderr, "mdb_env_create failed, error %d %s\n", rc, mdb_strerror(rc));
		return EXIT_FAILURE;
	}
Howard Chu's avatar
Howard Chu committed
361

Howard Chu's avatar
Howard Chu committed
362
	mdb_env_set_maxdbs(env, 2);
Howard Chu's avatar
Howard Chu committed
363

Howard Chu's avatar
Howard Chu committed
364
365
366
367
368
369
370
371
372
	if (info.me_maxreaders)
		mdb_env_set_maxreaders(env, info.me_maxreaders);

	if (info.me_mapsize)
		mdb_env_set_mapsize(env, info.me_mapsize);

	if (info.me_mapaddr)
		envflags |= MDB_FIXEDMAP;

Howard Chu's avatar
Howard Chu committed
373
374
	rc = mdb_env_open(env, envname, envflags, 0664);
	if (rc) {
375
		fprintf(stderr, "mdb_env_open failed, error %d %s\n", rc, mdb_strerror(rc));
Howard Chu's avatar
Howard Chu committed
376
377
378
379
		goto env_close;
	}

	kbuf.mv_size = mdb_env_get_maxkeysize(env) * 2 + 2;
Howard Chu's avatar
Howard Chu committed
380
381
382
383
	kbuf.mv_data = malloc(kbuf.mv_size * 2);
	k0buf.mv_size = kbuf.mv_size;
	k0buf.mv_data = (char *)kbuf.mv_data + kbuf.mv_size;
	prevk.mv_data = k0buf.mv_data;
Howard Chu's avatar
Howard Chu committed
384

Howard Chu's avatar
Howard Chu committed
385
	while(!Eof) {
Howard Chu's avatar
Howard Chu committed
386
387
388
		MDB_val key, data;
		int batch = 0;
		flags = 0;
Howard Chu's avatar
Howard Chu committed
389
		int appflag;
Howard Chu's avatar
Howard Chu committed
390

Howard Chu's avatar
Howard Chu committed
391
392
393
		if (!dohdr) {
			dohdr = 1;
		} else if (!(mode & NOHDR))
Howard Chu's avatar
Howard Chu committed
394
395
396
397
			readhdr();
		
		rc = mdb_txn_begin(env, NULL, 0, &txn);
		if (rc) {
398
			fprintf(stderr, "mdb_txn_begin failed, error %d %s\n", rc, mdb_strerror(rc));
Howard Chu's avatar
Howard Chu committed
399
400
401
			goto env_close;
		}

Howard Chu's avatar
Howard Chu committed
402
		rc = mdb_dbi_open(txn, subname, flags|MDB_CREATE, &dbi);
Howard Chu's avatar
Howard Chu committed
403
		if (rc) {
Howard Chu's avatar
Howard Chu committed
404
			fprintf(stderr, "mdb_dbi_open failed, error %d %s\n", rc, mdb_strerror(rc));
Howard Chu's avatar
Howard Chu committed
405
406
			goto txn_abort;
		}
Howard Chu's avatar
Howard Chu committed
407
408
409
410
411
412
		prevk.mv_size = 0;
		if (append) {
			mdb_set_compare(txn, dbi, greater);
			if (flags & MDB_DUPSORT)
				mdb_set_dupsort(txn, dbi, greater);
		}
Howard Chu's avatar
Howard Chu committed
413
414
415

		rc = mdb_cursor_open(txn, dbi, &mc);
		if (rc) {
416
			fprintf(stderr, "mdb_cursor_open failed, error %d %s\n", rc, mdb_strerror(rc));
Howard Chu's avatar
Howard Chu committed
417
418
419
420
421
			goto txn_abort;
		}

		while(1) {
			rc = readline(&key, &kbuf);
422
			if (rc)  /* rc == EOF */
Howard Chu's avatar
Howard Chu committed
423
424
425
				break;

			rc = readline(&data, &dbuf);
426
			if (rc) {
427
				fprintf(stderr, "%s: line %"Yu": failed to read key value\n", prog, lineno);
Howard Chu's avatar
Howard Chu committed
428
				goto txn_abort;
429
430
			}

Howard Chu's avatar
Howard Chu committed
431
432
433
434
435
436
437
438
439
440
441
442
443
444
			if (append) {
				appflag = MDB_APPEND;
				if (flags & MDB_DUPSORT) {
					if (prevk.mv_size == key.mv_size && !memcmp(prevk.mv_data, key.mv_data, key.mv_size))
						appflag = MDB_CURRENT|MDB_APPENDDUP;
					else {
						memcpy(prevk.mv_data, key.mv_data, key.mv_size);
						prevk.mv_size = key.mv_size;
					}
				}
			} else {
				appflag = 0;
			}
			rc = mdb_cursor_put(mc, &key, &data, putflags|appflag);
Howard Chu's avatar
Howard Chu committed
445
446
			if (rc == MDB_KEYEXIST && putflags)
				continue;
447
448
			if (rc) {
				fprintf(stderr, "mdb_cursor_put failed, error %d %s\n", rc, mdb_strerror(rc));
Howard Chu's avatar
Howard Chu committed
449
				goto txn_abort;
450
			}
Howard Chu's avatar
Howard Chu committed
451
452
453
454
			batch++;
			if (batch == 100) {
				rc = mdb_txn_commit(txn);
				if (rc) {
455
					fprintf(stderr, "%s: line %"Yu": txn_commit: %s\n",
Howard Chu's avatar
Howard Chu committed
456
457
458
459
460
						prog, lineno, mdb_strerror(rc));
					goto env_close;
				}
				rc = mdb_txn_begin(env, NULL, 0, &txn);
				if (rc) {
461
					fprintf(stderr, "mdb_txn_begin failed, error %d %s\n", rc, mdb_strerror(rc));
Howard Chu's avatar
Howard Chu committed
462
463
464
465
					goto env_close;
				}
				rc = mdb_cursor_open(txn, dbi, &mc);
				if (rc) {
466
					fprintf(stderr, "mdb_cursor_open failed, error %d %s\n", rc, mdb_strerror(rc));
Howard Chu's avatar
Howard Chu committed
467
468
					goto txn_abort;
				}
Howard Chu's avatar
Howard Chu committed
469
470
471
472
				if (appflag & MDB_APPENDDUP) {
					MDB_val k, d;
					mdb_cursor_get(mc, &k, &d, MDB_LAST);
				}
Howard Chu's avatar
Howard Chu committed
473
474
475
476
477
478
				batch = 0;
			}
		}
		rc = mdb_txn_commit(txn);
		txn = NULL;
		if (rc) {
479
			fprintf(stderr, "%s: line %"Yu": txn_commit: %s\n",
Howard Chu's avatar
Howard Chu committed
480
481
482
483
484
485
486
487
488
489
490
491
492
				prog, lineno, mdb_strerror(rc));
			goto env_close;
		}
		mdb_dbi_close(env, dbi);
	}

txn_abort:
	mdb_txn_abort(txn);
env_close:
	mdb_env_close(env);

	return rc ? EXIT_FAILURE : EXIT_SUCCESS;
}