mdb_load.c 10.4 KB
Newer Older
Howard Chu's avatar
Howard Chu committed
1
2
/* mdb_load.c - memory-mapped database load tool */
/*
Quanah Gibson-Mount's avatar
Quanah Gibson-Mount committed
3
 * Copyright 2011-2015 Howard Chu, Symas Corp.
Howard Chu's avatar
Howard Chu committed
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted only as authorized by the OpenLDAP
 * Public License.
 *
 * A copy of this license is available in the file LICENSE in the
 * top-level directory of the distribution or, alternatively, at
 * <http://www.OpenLDAP.org/license.html>.
 */
#include <stdio.h>
#include <stdlib.h>
#include <errno.h>
#include <string.h>
#include <ctype.h>
#include <unistd.h>
#include "lmdb.h"

#define PRINT	1
#define NOHDR	2
static int mode;

static char *subname = NULL;

static size_t lineno;
static int version;

static int flags;

static char *prog;

Howard Chu's avatar
Howard Chu committed
35
static int Eof;
Howard Chu's avatar
Howard Chu committed
36

Howard Chu's avatar
Howard Chu committed
37
38
static MDB_envinfo info;

Howard Chu's avatar
Howard Chu committed
39
40
static MDB_val kbuf, dbuf;

Howard Chu's avatar
Howard Chu committed
41
42
43
44
45
#ifdef _WIN32
#define Z	"I"
#else
#define Z	"z"
#endif
Howard Chu's avatar
Howard Chu committed
46
47
48
49
50
51
52
53
54
#ifdef MDB_VL32
#ifdef _WIN32
#define	Y	"I64"
#else
#define	Y	"ll"
#endif
#else
#define Y	Z
#endif
Howard Chu's avatar
Howard Chu committed
55

Howard Chu's avatar
Howard Chu committed
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
#define STRLENOF(s)	(sizeof(s)-1)

typedef struct flagbit {
	int bit;
	char *name;
	int len;
} flagbit;

#define S(s)	s, STRLENOF(s)

flagbit dbflags[] = {
	{ MDB_REVERSEKEY, S("reversekey") },
	{ MDB_DUPSORT, S("dupsort") },
	{ MDB_INTEGERKEY, S("integerkey") },
	{ MDB_DUPFIXED, S("dupfixed") },
	{ MDB_INTEGERDUP, S("integerdup") },
	{ MDB_REVERSEDUP, S("reversedup") },
	{ 0, NULL, 0 }
};

Howard Chu's avatar
Howard Chu committed
76
static void readhdr(void)
Howard Chu's avatar
Howard Chu committed
77
78
79
80
81
82
{
	char *ptr;

	while (fgets(dbuf.mv_data, dbuf.mv_size, stdin) != NULL) {
		lineno++;
		if (!strncmp(dbuf.mv_data, "VERSION=", STRLENOF("VERSION="))) {
Howard Chu's avatar
Howard Chu committed
83
			version=atoi((char *)dbuf.mv_data+STRLENOF("VERSION="));
Howard Chu's avatar
Howard Chu committed
84
			if (version > 3) {
Howard Chu's avatar
Howard Chu committed
85
				fprintf(stderr, "%s: line %" Z "d: unsupported VERSION %d\n",
Howard Chu's avatar
Howard Chu committed
86
87
88
89
90
91
					prog, lineno, version);
				exit(EXIT_FAILURE);
			}
		} else if (!strncmp(dbuf.mv_data, "HEADER=END", STRLENOF("HEADER=END"))) {
			break;
		} else if (!strncmp(dbuf.mv_data, "format=", STRLENOF("format="))) {
Howard Chu's avatar
Howard Chu committed
92
			if (!strncmp((char *)dbuf.mv_data+STRLENOF("FORMAT="), "print", STRLENOF("print")))
Howard Chu's avatar
Howard Chu committed
93
				mode |= PRINT;
Howard Chu's avatar
Howard Chu committed
94
			else if (strncmp((char *)dbuf.mv_data+STRLENOF("FORMAT="), "bytevalue", STRLENOF("bytevalue"))) {
Howard Chu's avatar
Howard Chu committed
95
				fprintf(stderr, "%s: line %" Z "d: unsupported FORMAT %s\n",
Howard Chu's avatar
Howard Chu committed
96
97
98
99
100
101
102
					prog, lineno, (char *)dbuf.mv_data+STRLENOF("FORMAT="));
				exit(EXIT_FAILURE);
			}
		} else if (!strncmp(dbuf.mv_data, "database=", STRLENOF("database="))) {
			ptr = memchr(dbuf.mv_data, '\n', dbuf.mv_size);
			if (ptr) *ptr = '\0';
			if (subname) free(subname);
Howard Chu's avatar
Howard Chu committed
103
			subname = strdup((char *)dbuf.mv_data+STRLENOF("database="));
Howard Chu's avatar
Howard Chu committed
104
		} else if (!strncmp(dbuf.mv_data, "type=", STRLENOF("type="))) {
Howard Chu's avatar
Howard Chu committed
105
			if (strncmp((char *)dbuf.mv_data+STRLENOF("type="), "btree", STRLENOF("btree")))  {
Howard Chu's avatar
Howard Chu committed
106
				fprintf(stderr, "%s: line %" Z "d: unsupported type %s\n",
Howard Chu's avatar
Howard Chu committed
107
108
109
					prog, lineno, (char *)dbuf.mv_data+STRLENOF("type="));
				exit(EXIT_FAILURE);
			}
Howard Chu's avatar
Howard Chu committed
110
111
112
113
114
115
		} else if (!strncmp(dbuf.mv_data, "mapaddr=", STRLENOF("mapaddr="))) {
			int i;
			ptr = memchr(dbuf.mv_data, '\n', dbuf.mv_size);
			if (ptr) *ptr = '\0';
			i = sscanf((char *)dbuf.mv_data+STRLENOF("mapaddr="), "%p", &info.me_mapaddr);
			if (i != 1) {
Howard Chu's avatar
Howard Chu committed
116
				fprintf(stderr, "%s: line %" Z "d: invalid mapaddr %s\n",
Howard Chu's avatar
Howard Chu committed
117
118
119
120
121
122
123
					prog, lineno, (char *)dbuf.mv_data+STRLENOF("mapaddr="));
				exit(EXIT_FAILURE);
			}
		} else if (!strncmp(dbuf.mv_data, "mapsize=", STRLENOF("mapsize="))) {
			int i;
			ptr = memchr(dbuf.mv_data, '\n', dbuf.mv_size);
			if (ptr) *ptr = '\0';
Howard Chu's avatar
Howard Chu committed
124
			i = sscanf((char *)dbuf.mv_data+STRLENOF("mapsize="), "%" Y "u", &info.me_mapsize);
Howard Chu's avatar
Howard Chu committed
125
			if (i != 1) {
Howard Chu's avatar
Howard Chu committed
126
				fprintf(stderr, "%s: line %" Z "d: invalid mapsize %s\n",
Howard Chu's avatar
Howard Chu committed
127
128
129
130
131
132
133
134
135
					prog, lineno, (char *)dbuf.mv_data+STRLENOF("mapsize="));
				exit(EXIT_FAILURE);
			}
		} else if (!strncmp(dbuf.mv_data, "maxreaders=", STRLENOF("maxreaders="))) {
			int i;
			ptr = memchr(dbuf.mv_data, '\n', dbuf.mv_size);
			if (ptr) *ptr = '\0';
			i = sscanf((char *)dbuf.mv_data+STRLENOF("maxreaders="), "%u", &info.me_maxreaders);
			if (i != 1) {
Howard Chu's avatar
Howard Chu committed
136
				fprintf(stderr, "%s: line %" Z "d: invalid maxreaders %s\n",
Howard Chu's avatar
Howard Chu committed
137
138
139
					prog, lineno, (char *)dbuf.mv_data+STRLENOF("maxreaders="));
				exit(EXIT_FAILURE);
			}
Howard Chu's avatar
Howard Chu committed
140
141
142
143
144
145
146
147
148
149
150
151
		} else {
			int i;
			for (i=0; dbflags[i].bit; i++) {
				if (!strncmp(dbuf.mv_data, dbflags[i].name, dbflags[i].len) &&
					((char *)dbuf.mv_data)[dbflags[i].len] == '=') {
					flags |= dbflags[i].bit;
					break;
				}
			}
			if (!dbflags[i].bit) {
				ptr = memchr(dbuf.mv_data, '=', dbuf.mv_size);
				if (!ptr) {
Howard Chu's avatar
Howard Chu committed
152
					fprintf(stderr, "%s: line %" Z "d: unexpected format\n",
Howard Chu's avatar
Howard Chu committed
153
154
155
156
						prog, lineno);
					exit(EXIT_FAILURE);
				} else {
					*ptr = '\0';
Howard Chu's avatar
Howard Chu committed
157
					fprintf(stderr, "%s: line %" Z "d: unrecognized keyword ignored: %s\n",
Howard Chu's avatar
Howard Chu committed
158
159
160
161
162
163
164
						prog, lineno, (char *)dbuf.mv_data);
				}
			}
		}
	}
}

Howard Chu's avatar
Howard Chu committed
165
static void badend(void)
Howard Chu's avatar
Howard Chu committed
166
{
Howard Chu's avatar
Howard Chu committed
167
	fprintf(stderr, "%s: line %" Z "d: unexpected end of input\n",
Howard Chu's avatar
Howard Chu committed
168
169
170
171
172
173
174
175
		prog, lineno);
}

static int unhex(unsigned char *c2)
{
	int x, c;
	x = *c2++ & 0x4f;
	if (x & 0x40)
Howard Chu's avatar
Howard Chu committed
176
		x -= 55;
Howard Chu's avatar
Howard Chu committed
177
178
179
	c = x << 4;
	x = *c2 & 0x4f;
	if (x & 0x40)
Howard Chu's avatar
Howard Chu committed
180
		x -= 55;
Howard Chu's avatar
Howard Chu committed
181
182
183
184
185
186
187
	c |= x;
	return c;
}

static int readline(MDB_val *out, MDB_val *buf)
{
	unsigned char *c1, *c2, *end;
188
	size_t len, l2;
Howard Chu's avatar
Howard Chu committed
189
190
191
192
193
	int c;

	if (!(mode & NOHDR)) {
		c = fgetc(stdin);
		if (c == EOF) {
Howard Chu's avatar
Howard Chu committed
194
			Eof = 1;
Howard Chu's avatar
Howard Chu committed
195
196
197
			return EOF;
		}
		if (c != ' ') {
Howard Chu's avatar
Howard Chu committed
198
			lineno++;
Howard Chu's avatar
Howard Chu committed
199
200
			if (fgets(buf->mv_data, buf->mv_size, stdin) == NULL) {
badend:
Howard Chu's avatar
Howard Chu committed
201
				Eof = 1;
Howard Chu's avatar
Howard Chu committed
202
203
204
205
206
207
208
209
210
				badend();
				return EOF;
			}
			if (c == 'D' && !strncmp(buf->mv_data, "ATA=END", STRLENOF("ATA=END")))
				return EOF;
			goto badend;
		}
	}
	if (fgets(buf->mv_data, buf->mv_size, stdin) == NULL) {
Howard Chu's avatar
Howard Chu committed
211
		Eof = 1;
Howard Chu's avatar
Howard Chu committed
212
213
214
215
216
217
		return EOF;
	}
	lineno++;

	c1 = buf->mv_data;
	len = strlen((char *)c1);
218
	l2 = len;
Howard Chu's avatar
Howard Chu committed
219
220
221
222
223

	/* Is buffer too short? */
	while (c1[len-1] != '\n') {
		buf->mv_data = realloc(buf->mv_data, buf->mv_size*2);
		if (!buf->mv_data) {
Howard Chu's avatar
Howard Chu committed
224
225
			Eof = 1;
			fprintf(stderr, "%s: line %" Z "d: out of memory, line too long\n",
Howard Chu's avatar
Howard Chu committed
226
227
228
229
				prog, lineno);
			return EOF;
		}
		c1 = buf->mv_data;
230
231
		c1 += l2;
		if (fgets((char *)c1, buf->mv_size+1, stdin) == NULL) {
Howard Chu's avatar
Howard Chu committed
232
			Eof = 1;
Howard Chu's avatar
Howard Chu committed
233
234
235
236
237
			badend();
			return EOF;
		}
		buf->mv_size *= 2;
		len = strlen((char *)c1);
238
		l2 += len;
Howard Chu's avatar
Howard Chu committed
239
240
	}
	c1 = c2 = buf->mv_data;
241
	len = l2;
Howard Chu's avatar
Howard Chu committed
242
243
244
245
246
247
248
249
250
	c1[--len] = '\0';
	end = c1 + len;

	if (mode & PRINT) {
		while (c2 < end) {
			if (*c2 == '\\') {
				if (c2[1] == '\\') {
					c1++; c2 += 2;
				} else {
Howard Chu's avatar
Howard Chu committed
251
					if (c2+3 > end || !isxdigit(c2[1]) || !isxdigit(c2[2])) {
Howard Chu's avatar
Howard Chu committed
252
						Eof = 1;
Howard Chu's avatar
Howard Chu committed
253
254
255
256
257
258
259
260
261
262
263
264
265
						badend();
						return EOF;
					}
					*c1++ = unhex(++c2);
					c2 += 2;
				}
			} else {
				c1++; c2++;
			}
		}
	} else {
		/* odd length not allowed */
		if (len & 1) {
Howard Chu's avatar
Howard Chu committed
266
			Eof = 1;
Howard Chu's avatar
Howard Chu committed
267
268
269
270
271
			badend();
			return EOF;
		}
		while (c2 < end) {
			if (!isxdigit(*c2) || !isxdigit(c2[1])) {
Howard Chu's avatar
Howard Chu committed
272
				Eof = 1;
Howard Chu's avatar
Howard Chu committed
273
274
275
276
277
278
279
280
281
282
283
284
285
				badend();
				return EOF;
			}
			*c1++ = unhex(c2);
			c2 += 2;
		}
	}
	c2 = out->mv_data = buf->mv_data;
	out->mv_size = c1 - c2;

	return 0;
}

Howard Chu's avatar
Howard Chu committed
286
static void usage(void)
Howard Chu's avatar
Howard Chu committed
287
{
Howard Chu's avatar
Howard Chu committed
288
	fprintf(stderr, "usage: %s [-V] [-f input] [-n] [-s name] [-N] [-T] dbpath\n", prog);
Howard Chu's avatar
Howard Chu committed
289
290
291
292
293
294
295
296
297
298
299
300
	exit(EXIT_FAILURE);
}

int main(int argc, char *argv[])
{
	int i, rc;
	MDB_env *env;
	MDB_txn *txn;
	MDB_cursor *mc;
	MDB_dbi dbi;
	char *envname;
	int envflags = 0, putflags = 0;
Howard Chu's avatar
Howard Chu committed
301
	int dohdr = 0;
Howard Chu's avatar
Howard Chu committed
302
303
304
305

	prog = argv[0];

	if (argc < 2) {
Howard Chu's avatar
Howard Chu committed
306
		usage();
Howard Chu's avatar
Howard Chu committed
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
	}

	/* -f: load file instead of stdin
	 * -n: use NOSUBDIR flag on env_open
	 * -s: load into named subDB
	 * -N: use NOOVERWRITE on puts
	 * -T: read plaintext
	 * -V: print version and exit
	 */
	while ((i = getopt(argc, argv, "f:ns:NTV")) != EOF) {
		switch(i) {
		case 'V':
			printf("%s\n", MDB_VERSION_STRING);
			exit(0);
			break;
		case 'f':
			if (freopen(optarg, "r", stdin) == NULL) {
				fprintf(stderr, "%s: %s: reopen: %s\n",
					prog, optarg, strerror(errno));
				exit(EXIT_FAILURE);
			}
			break;
		case 'n':
			envflags |= MDB_NOSUBDIR;
			break;
		case 's':
			subname = strdup(optarg);
			break;
		case 'N':
			putflags = MDB_NOOVERWRITE|MDB_NODUPDATA;
			break;
		case 'T':
339
			mode |= NOHDR | PRINT;
Howard Chu's avatar
Howard Chu committed
340
341
			break;
		default:
Howard Chu's avatar
Howard Chu committed
342
			usage();
Howard Chu's avatar
Howard Chu committed
343
344
345
346
		}
	}

	if (optind != argc - 1)
Howard Chu's avatar
Howard Chu committed
347
		usage();
Howard Chu's avatar
Howard Chu committed
348

Howard Chu's avatar
Howard Chu committed
349
350
351
352
353
354
	dbuf.mv_size = 4096;
	dbuf.mv_data = malloc(dbuf.mv_size);

	if (!(mode & NOHDR))
		readhdr();

Howard Chu's avatar
Howard Chu committed
355
356
	envname = argv[optind];
	rc = mdb_env_create(&env);
Howard Chu's avatar
Howard Chu committed
357
358
359
360
	if (rc) {
		fprintf(stderr, "mdb_env_create failed, error %d %s\n", rc, mdb_strerror(rc));
		return EXIT_FAILURE;
	}
Howard Chu's avatar
Howard Chu committed
361

Howard Chu's avatar
Howard Chu committed
362
	mdb_env_set_maxdbs(env, 2);
Howard Chu's avatar
Howard Chu committed
363

Howard Chu's avatar
Howard Chu committed
364
365
366
367
368
369
370
371
372
	if (info.me_maxreaders)
		mdb_env_set_maxreaders(env, info.me_maxreaders);

	if (info.me_mapsize)
		mdb_env_set_mapsize(env, info.me_mapsize);

	if (info.me_mapaddr)
		envflags |= MDB_FIXEDMAP;

Howard Chu's avatar
Howard Chu committed
373
374
	rc = mdb_env_open(env, envname, envflags, 0664);
	if (rc) {
375
		fprintf(stderr, "mdb_env_open failed, error %d %s\n", rc, mdb_strerror(rc));
Howard Chu's avatar
Howard Chu committed
376
377
378
379
380
381
		goto env_close;
	}

	kbuf.mv_size = mdb_env_get_maxkeysize(env) * 2 + 2;
	kbuf.mv_data = malloc(kbuf.mv_size);

Howard Chu's avatar
Howard Chu committed
382
	while(!Eof) {
Howard Chu's avatar
Howard Chu committed
383
384
385
386
		MDB_val key, data;
		int batch = 0;
		flags = 0;

Howard Chu's avatar
Howard Chu committed
387
388
389
		if (!dohdr) {
			dohdr = 1;
		} else if (!(mode & NOHDR))
Howard Chu's avatar
Howard Chu committed
390
391
392
393
			readhdr();
		
		rc = mdb_txn_begin(env, NULL, 0, &txn);
		if (rc) {
394
			fprintf(stderr, "mdb_txn_begin failed, error %d %s\n", rc, mdb_strerror(rc));
Howard Chu's avatar
Howard Chu committed
395
396
397
			goto env_close;
		}

Howard Chu's avatar
Howard Chu committed
398
		rc = mdb_open(txn, subname, flags|MDB_CREATE, &dbi);
Howard Chu's avatar
Howard Chu committed
399
		if (rc) {
400
			fprintf(stderr, "mdb_open failed, error %d %s\n", rc, mdb_strerror(rc));
Howard Chu's avatar
Howard Chu committed
401
402
403
404
405
			goto txn_abort;
		}

		rc = mdb_cursor_open(txn, dbi, &mc);
		if (rc) {
406
			fprintf(stderr, "mdb_cursor_open failed, error %d %s\n", rc, mdb_strerror(rc));
Howard Chu's avatar
Howard Chu committed
407
408
409
410
411
			goto txn_abort;
		}

		while(1) {
			rc = readline(&key, &kbuf);
412
			if (rc)  /* rc == EOF */
Howard Chu's avatar
Howard Chu committed
413
414
415
				break;

			rc = readline(&data, &dbuf);
416
417
			if (rc) {
				fprintf(stderr, "%s: line %" Z "d: failed to read key value\n", prog, lineno);
Howard Chu's avatar
Howard Chu committed
418
				goto txn_abort;
419
420
			}

Howard Chu's avatar
Howard Chu committed
421
422
423
			rc = mdb_cursor_put(mc, &key, &data, putflags);
			if (rc == MDB_KEYEXIST && putflags)
				continue;
424
425
			if (rc) {
				fprintf(stderr, "mdb_cursor_put failed, error %d %s\n", rc, mdb_strerror(rc));
Howard Chu's avatar
Howard Chu committed
426
				goto txn_abort;
427
			}
Howard Chu's avatar
Howard Chu committed
428
429
430
431
			batch++;
			if (batch == 100) {
				rc = mdb_txn_commit(txn);
				if (rc) {
Howard Chu's avatar
Howard Chu committed
432
					fprintf(stderr, "%s: line %" Z "d: txn_commit: %s\n",
Howard Chu's avatar
Howard Chu committed
433
434
435
436
437
						prog, lineno, mdb_strerror(rc));
					goto env_close;
				}
				rc = mdb_txn_begin(env, NULL, 0, &txn);
				if (rc) {
438
					fprintf(stderr, "mdb_txn_begin failed, error %d %s\n", rc, mdb_strerror(rc));
Howard Chu's avatar
Howard Chu committed
439
440
441
442
					goto env_close;
				}
				rc = mdb_cursor_open(txn, dbi, &mc);
				if (rc) {
443
					fprintf(stderr, "mdb_cursor_open failed, error %d %s\n", rc, mdb_strerror(rc));
Howard Chu's avatar
Howard Chu committed
444
445
446
447
448
449
450
451
					goto txn_abort;
				}
				batch = 0;
			}
		}
		rc = mdb_txn_commit(txn);
		txn = NULL;
		if (rc) {
Howard Chu's avatar
Howard Chu committed
452
			fprintf(stderr, "%s: line %" Z "d: txn_commit: %s\n",
Howard Chu's avatar
Howard Chu committed
453
454
455
456
457
458
459
460
461
462
463
464
465
				prog, lineno, mdb_strerror(rc));
			goto env_close;
		}
		mdb_dbi_close(env, dbi);
	}

txn_abort:
	mdb_txn_abort(txn);
env_close:
	mdb_env_close(env);

	return rc ? EXIT_FAILURE : EXIT_SUCCESS;
}