mdb_load.c 10.3 KB
Newer Older
Howard Chu's avatar
Howard Chu committed
1
2
/* mdb_load.c - memory-mapped database load tool */
/*
Howard Chu's avatar
Howard Chu committed
3
 * Copyright 2011-2016 Howard Chu, Symas Corp.
Howard Chu's avatar
Howard Chu committed
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted only as authorized by the OpenLDAP
 * Public License.
 *
 * A copy of this license is available in the file LICENSE in the
 * top-level directory of the distribution or, alternatively, at
 * <http://www.OpenLDAP.org/license.html>.
 */
#include <stdio.h>
#include <stdlib.h>
#include <errno.h>
#include <string.h>
#include <ctype.h>
#include <unistd.h>
#include "lmdb.h"

#define PRINT	1
#define NOHDR	2
static int mode;

static char *subname = NULL;

static size_t lineno;
static int version;

static int flags;

static char *prog;

Howard Chu's avatar
Howard Chu committed
35
static int Eof;
Howard Chu's avatar
Howard Chu committed
36

Howard Chu's avatar
Howard Chu committed
37
38
static MDB_envinfo info;

Howard Chu's avatar
Howard Chu committed
39
40
static MDB_val kbuf, dbuf;

41
42
#define Z	MDB_FMT_Z
#define Y	MDB_FMT_Y
Howard Chu's avatar
Howard Chu committed
43

Howard Chu's avatar
Howard Chu committed
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
#define STRLENOF(s)	(sizeof(s)-1)

typedef struct flagbit {
	int bit;
	char *name;
	int len;
} flagbit;

#define S(s)	s, STRLENOF(s)

flagbit dbflags[] = {
	{ MDB_REVERSEKEY, S("reversekey") },
	{ MDB_DUPSORT, S("dupsort") },
	{ MDB_INTEGERKEY, S("integerkey") },
	{ MDB_DUPFIXED, S("dupfixed") },
	{ MDB_INTEGERDUP, S("integerdup") },
	{ MDB_REVERSEDUP, S("reversedup") },
	{ 0, NULL, 0 }
};

Howard Chu's avatar
Howard Chu committed
64
static void readhdr(void)
Howard Chu's avatar
Howard Chu committed
65
66
67
68
69
70
{
	char *ptr;

	while (fgets(dbuf.mv_data, dbuf.mv_size, stdin) != NULL) {
		lineno++;
		if (!strncmp(dbuf.mv_data, "VERSION=", STRLENOF("VERSION="))) {
Howard Chu's avatar
Howard Chu committed
71
			version=atoi((char *)dbuf.mv_data+STRLENOF("VERSION="));
Howard Chu's avatar
Howard Chu committed
72
			if (version > 3) {
Howard Chu's avatar
Howard Chu committed
73
				fprintf(stderr, "%s: line %" Z "d: unsupported VERSION %d\n",
Howard Chu's avatar
Howard Chu committed
74
75
76
77
78
79
					prog, lineno, version);
				exit(EXIT_FAILURE);
			}
		} else if (!strncmp(dbuf.mv_data, "HEADER=END", STRLENOF("HEADER=END"))) {
			break;
		} else if (!strncmp(dbuf.mv_data, "format=", STRLENOF("format="))) {
Howard Chu's avatar
Howard Chu committed
80
			if (!strncmp((char *)dbuf.mv_data+STRLENOF("FORMAT="), "print", STRLENOF("print")))
Howard Chu's avatar
Howard Chu committed
81
				mode |= PRINT;
Howard Chu's avatar
Howard Chu committed
82
			else if (strncmp((char *)dbuf.mv_data+STRLENOF("FORMAT="), "bytevalue", STRLENOF("bytevalue"))) {
Howard Chu's avatar
Howard Chu committed
83
				fprintf(stderr, "%s: line %" Z "d: unsupported FORMAT %s\n",
Howard Chu's avatar
Howard Chu committed
84
85
86
87
88
89
90
					prog, lineno, (char *)dbuf.mv_data+STRLENOF("FORMAT="));
				exit(EXIT_FAILURE);
			}
		} else if (!strncmp(dbuf.mv_data, "database=", STRLENOF("database="))) {
			ptr = memchr(dbuf.mv_data, '\n', dbuf.mv_size);
			if (ptr) *ptr = '\0';
			if (subname) free(subname);
Howard Chu's avatar
Howard Chu committed
91
			subname = strdup((char *)dbuf.mv_data+STRLENOF("database="));
Howard Chu's avatar
Howard Chu committed
92
		} else if (!strncmp(dbuf.mv_data, "type=", STRLENOF("type="))) {
Howard Chu's avatar
Howard Chu committed
93
			if (strncmp((char *)dbuf.mv_data+STRLENOF("type="), "btree", STRLENOF("btree")))  {
Howard Chu's avatar
Howard Chu committed
94
				fprintf(stderr, "%s: line %" Z "d: unsupported type %s\n",
Howard Chu's avatar
Howard Chu committed
95
96
97
					prog, lineno, (char *)dbuf.mv_data+STRLENOF("type="));
				exit(EXIT_FAILURE);
			}
Howard Chu's avatar
Howard Chu committed
98
99
100
101
102
103
		} else if (!strncmp(dbuf.mv_data, "mapaddr=", STRLENOF("mapaddr="))) {
			int i;
			ptr = memchr(dbuf.mv_data, '\n', dbuf.mv_size);
			if (ptr) *ptr = '\0';
			i = sscanf((char *)dbuf.mv_data+STRLENOF("mapaddr="), "%p", &info.me_mapaddr);
			if (i != 1) {
Howard Chu's avatar
Howard Chu committed
104
				fprintf(stderr, "%s: line %" Z "d: invalid mapaddr %s\n",
Howard Chu's avatar
Howard Chu committed
105
106
107
108
109
110
111
					prog, lineno, (char *)dbuf.mv_data+STRLENOF("mapaddr="));
				exit(EXIT_FAILURE);
			}
		} else if (!strncmp(dbuf.mv_data, "mapsize=", STRLENOF("mapsize="))) {
			int i;
			ptr = memchr(dbuf.mv_data, '\n', dbuf.mv_size);
			if (ptr) *ptr = '\0';
Howard Chu's avatar
Howard Chu committed
112
			i = sscanf((char *)dbuf.mv_data+STRLENOF("mapsize="), "%" Y "u", &info.me_mapsize);
Howard Chu's avatar
Howard Chu committed
113
			if (i != 1) {
Howard Chu's avatar
Howard Chu committed
114
				fprintf(stderr, "%s: line %" Z "d: invalid mapsize %s\n",
Howard Chu's avatar
Howard Chu committed
115
116
117
118
119
120
121
122
123
					prog, lineno, (char *)dbuf.mv_data+STRLENOF("mapsize="));
				exit(EXIT_FAILURE);
			}
		} else if (!strncmp(dbuf.mv_data, "maxreaders=", STRLENOF("maxreaders="))) {
			int i;
			ptr = memchr(dbuf.mv_data, '\n', dbuf.mv_size);
			if (ptr) *ptr = '\0';
			i = sscanf((char *)dbuf.mv_data+STRLENOF("maxreaders="), "%u", &info.me_maxreaders);
			if (i != 1) {
Howard Chu's avatar
Howard Chu committed
124
				fprintf(stderr, "%s: line %" Z "d: invalid maxreaders %s\n",
Howard Chu's avatar
Howard Chu committed
125
126
127
					prog, lineno, (char *)dbuf.mv_data+STRLENOF("maxreaders="));
				exit(EXIT_FAILURE);
			}
Howard Chu's avatar
Howard Chu committed
128
129
130
131
132
133
134
135
136
137
138
139
		} else {
			int i;
			for (i=0; dbflags[i].bit; i++) {
				if (!strncmp(dbuf.mv_data, dbflags[i].name, dbflags[i].len) &&
					((char *)dbuf.mv_data)[dbflags[i].len] == '=') {
					flags |= dbflags[i].bit;
					break;
				}
			}
			if (!dbflags[i].bit) {
				ptr = memchr(dbuf.mv_data, '=', dbuf.mv_size);
				if (!ptr) {
Howard Chu's avatar
Howard Chu committed
140
					fprintf(stderr, "%s: line %" Z "d: unexpected format\n",
Howard Chu's avatar
Howard Chu committed
141
142
143
144
						prog, lineno);
					exit(EXIT_FAILURE);
				} else {
					*ptr = '\0';
Howard Chu's avatar
Howard Chu committed
145
					fprintf(stderr, "%s: line %" Z "d: unrecognized keyword ignored: %s\n",
Howard Chu's avatar
Howard Chu committed
146
147
148
149
150
151
152
						prog, lineno, (char *)dbuf.mv_data);
				}
			}
		}
	}
}

Howard Chu's avatar
Howard Chu committed
153
static void badend(void)
Howard Chu's avatar
Howard Chu committed
154
{
Howard Chu's avatar
Howard Chu committed
155
	fprintf(stderr, "%s: line %" Z "d: unexpected end of input\n",
Howard Chu's avatar
Howard Chu committed
156
157
158
159
160
161
162
163
		prog, lineno);
}

static int unhex(unsigned char *c2)
{
	int x, c;
	x = *c2++ & 0x4f;
	if (x & 0x40)
Howard Chu's avatar
Howard Chu committed
164
		x -= 55;
Howard Chu's avatar
Howard Chu committed
165
166
167
	c = x << 4;
	x = *c2 & 0x4f;
	if (x & 0x40)
Howard Chu's avatar
Howard Chu committed
168
		x -= 55;
Howard Chu's avatar
Howard Chu committed
169
170
171
172
173
174
175
	c |= x;
	return c;
}

static int readline(MDB_val *out, MDB_val *buf)
{
	unsigned char *c1, *c2, *end;
176
	size_t len, l2;
Howard Chu's avatar
Howard Chu committed
177
178
179
180
181
	int c;

	if (!(mode & NOHDR)) {
		c = fgetc(stdin);
		if (c == EOF) {
Howard Chu's avatar
Howard Chu committed
182
			Eof = 1;
Howard Chu's avatar
Howard Chu committed
183
184
185
			return EOF;
		}
		if (c != ' ') {
Howard Chu's avatar
Howard Chu committed
186
			lineno++;
Howard Chu's avatar
Howard Chu committed
187
188
			if (fgets(buf->mv_data, buf->mv_size, stdin) == NULL) {
badend:
Howard Chu's avatar
Howard Chu committed
189
				Eof = 1;
Howard Chu's avatar
Howard Chu committed
190
191
192
193
194
195
196
197
198
				badend();
				return EOF;
			}
			if (c == 'D' && !strncmp(buf->mv_data, "ATA=END", STRLENOF("ATA=END")))
				return EOF;
			goto badend;
		}
	}
	if (fgets(buf->mv_data, buf->mv_size, stdin) == NULL) {
Howard Chu's avatar
Howard Chu committed
199
		Eof = 1;
Howard Chu's avatar
Howard Chu committed
200
201
202
203
204
205
		return EOF;
	}
	lineno++;

	c1 = buf->mv_data;
	len = strlen((char *)c1);
206
	l2 = len;
Howard Chu's avatar
Howard Chu committed
207
208
209
210
211

	/* Is buffer too short? */
	while (c1[len-1] != '\n') {
		buf->mv_data = realloc(buf->mv_data, buf->mv_size*2);
		if (!buf->mv_data) {
Howard Chu's avatar
Howard Chu committed
212
213
			Eof = 1;
			fprintf(stderr, "%s: line %" Z "d: out of memory, line too long\n",
Howard Chu's avatar
Howard Chu committed
214
215
216
217
				prog, lineno);
			return EOF;
		}
		c1 = buf->mv_data;
218
219
		c1 += l2;
		if (fgets((char *)c1, buf->mv_size+1, stdin) == NULL) {
Howard Chu's avatar
Howard Chu committed
220
			Eof = 1;
Howard Chu's avatar
Howard Chu committed
221
222
223
224
225
			badend();
			return EOF;
		}
		buf->mv_size *= 2;
		len = strlen((char *)c1);
226
		l2 += len;
Howard Chu's avatar
Howard Chu committed
227
228
	}
	c1 = c2 = buf->mv_data;
229
	len = l2;
Howard Chu's avatar
Howard Chu committed
230
231
232
233
234
235
236
237
238
	c1[--len] = '\0';
	end = c1 + len;

	if (mode & PRINT) {
		while (c2 < end) {
			if (*c2 == '\\') {
				if (c2[1] == '\\') {
					c1++; c2 += 2;
				} else {
Howard Chu's avatar
Howard Chu committed
239
					if (c2+3 > end || !isxdigit(c2[1]) || !isxdigit(c2[2])) {
Howard Chu's avatar
Howard Chu committed
240
						Eof = 1;
Howard Chu's avatar
Howard Chu committed
241
242
243
244
245
246
247
248
249
250
251
252
253
						badend();
						return EOF;
					}
					*c1++ = unhex(++c2);
					c2 += 2;
				}
			} else {
				c1++; c2++;
			}
		}
	} else {
		/* odd length not allowed */
		if (len & 1) {
Howard Chu's avatar
Howard Chu committed
254
			Eof = 1;
Howard Chu's avatar
Howard Chu committed
255
256
257
258
259
			badend();
			return EOF;
		}
		while (c2 < end) {
			if (!isxdigit(*c2) || !isxdigit(c2[1])) {
Howard Chu's avatar
Howard Chu committed
260
				Eof = 1;
Howard Chu's avatar
Howard Chu committed
261
262
263
264
265
266
267
268
269
270
271
272
273
				badend();
				return EOF;
			}
			*c1++ = unhex(c2);
			c2 += 2;
		}
	}
	c2 = out->mv_data = buf->mv_data;
	out->mv_size = c1 - c2;

	return 0;
}

Howard Chu's avatar
Howard Chu committed
274
static void usage(void)
Howard Chu's avatar
Howard Chu committed
275
{
Howard Chu's avatar
Howard Chu committed
276
	fprintf(stderr, "usage: %s [-V] [-f input] [-n] [-s name] [-N] [-T] dbpath\n", prog);
Howard Chu's avatar
Howard Chu committed
277
278
279
280
281
282
283
284
285
286
287
288
	exit(EXIT_FAILURE);
}

int main(int argc, char *argv[])
{
	int i, rc;
	MDB_env *env;
	MDB_txn *txn;
	MDB_cursor *mc;
	MDB_dbi dbi;
	char *envname;
	int envflags = 0, putflags = 0;
Howard Chu's avatar
Howard Chu committed
289
	int dohdr = 0;
Howard Chu's avatar
Howard Chu committed
290
291
292
293

	prog = argv[0];

	if (argc < 2) {
Howard Chu's avatar
Howard Chu committed
294
		usage();
Howard Chu's avatar
Howard Chu committed
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
	}

	/* -f: load file instead of stdin
	 * -n: use NOSUBDIR flag on env_open
	 * -s: load into named subDB
	 * -N: use NOOVERWRITE on puts
	 * -T: read plaintext
	 * -V: print version and exit
	 */
	while ((i = getopt(argc, argv, "f:ns:NTV")) != EOF) {
		switch(i) {
		case 'V':
			printf("%s\n", MDB_VERSION_STRING);
			exit(0);
			break;
		case 'f':
			if (freopen(optarg, "r", stdin) == NULL) {
				fprintf(stderr, "%s: %s: reopen: %s\n",
					prog, optarg, strerror(errno));
				exit(EXIT_FAILURE);
			}
			break;
		case 'n':
			envflags |= MDB_NOSUBDIR;
			break;
		case 's':
			subname = strdup(optarg);
			break;
		case 'N':
			putflags = MDB_NOOVERWRITE|MDB_NODUPDATA;
			break;
		case 'T':
327
			mode |= NOHDR | PRINT;
Howard Chu's avatar
Howard Chu committed
328
329
			break;
		default:
Howard Chu's avatar
Howard Chu committed
330
			usage();
Howard Chu's avatar
Howard Chu committed
331
332
333
334
		}
	}

	if (optind != argc - 1)
Howard Chu's avatar
Howard Chu committed
335
		usage();
Howard Chu's avatar
Howard Chu committed
336

Howard Chu's avatar
Howard Chu committed
337
338
339
340
341
342
	dbuf.mv_size = 4096;
	dbuf.mv_data = malloc(dbuf.mv_size);

	if (!(mode & NOHDR))
		readhdr();

Howard Chu's avatar
Howard Chu committed
343
344
	envname = argv[optind];
	rc = mdb_env_create(&env);
Howard Chu's avatar
Howard Chu committed
345
346
347
348
	if (rc) {
		fprintf(stderr, "mdb_env_create failed, error %d %s\n", rc, mdb_strerror(rc));
		return EXIT_FAILURE;
	}
Howard Chu's avatar
Howard Chu committed
349

Howard Chu's avatar
Howard Chu committed
350
	mdb_env_set_maxdbs(env, 2);
Howard Chu's avatar
Howard Chu committed
351

Howard Chu's avatar
Howard Chu committed
352
353
354
355
356
357
358
359
360
	if (info.me_maxreaders)
		mdb_env_set_maxreaders(env, info.me_maxreaders);

	if (info.me_mapsize)
		mdb_env_set_mapsize(env, info.me_mapsize);

	if (info.me_mapaddr)
		envflags |= MDB_FIXEDMAP;

Howard Chu's avatar
Howard Chu committed
361
362
	rc = mdb_env_open(env, envname, envflags, 0664);
	if (rc) {
363
		fprintf(stderr, "mdb_env_open failed, error %d %s\n", rc, mdb_strerror(rc));
Howard Chu's avatar
Howard Chu committed
364
365
366
367
368
369
		goto env_close;
	}

	kbuf.mv_size = mdb_env_get_maxkeysize(env) * 2 + 2;
	kbuf.mv_data = malloc(kbuf.mv_size);

Howard Chu's avatar
Howard Chu committed
370
	while(!Eof) {
Howard Chu's avatar
Howard Chu committed
371
372
373
374
		MDB_val key, data;
		int batch = 0;
		flags = 0;

Howard Chu's avatar
Howard Chu committed
375
376
377
		if (!dohdr) {
			dohdr = 1;
		} else if (!(mode & NOHDR))
Howard Chu's avatar
Howard Chu committed
378
379
380
381
			readhdr();
		
		rc = mdb_txn_begin(env, NULL, 0, &txn);
		if (rc) {
382
			fprintf(stderr, "mdb_txn_begin failed, error %d %s\n", rc, mdb_strerror(rc));
Howard Chu's avatar
Howard Chu committed
383
384
385
			goto env_close;
		}

Howard Chu's avatar
Howard Chu committed
386
		rc = mdb_open(txn, subname, flags|MDB_CREATE, &dbi);
Howard Chu's avatar
Howard Chu committed
387
		if (rc) {
388
			fprintf(stderr, "mdb_open failed, error %d %s\n", rc, mdb_strerror(rc));
Howard Chu's avatar
Howard Chu committed
389
390
391
392
393
			goto txn_abort;
		}

		rc = mdb_cursor_open(txn, dbi, &mc);
		if (rc) {
394
			fprintf(stderr, "mdb_cursor_open failed, error %d %s\n", rc, mdb_strerror(rc));
Howard Chu's avatar
Howard Chu committed
395
396
397
398
399
			goto txn_abort;
		}

		while(1) {
			rc = readline(&key, &kbuf);
400
			if (rc)  /* rc == EOF */
Howard Chu's avatar
Howard Chu committed
401
402
403
				break;

			rc = readline(&data, &dbuf);
404
405
			if (rc) {
				fprintf(stderr, "%s: line %" Z "d: failed to read key value\n", prog, lineno);
Howard Chu's avatar
Howard Chu committed
406
				goto txn_abort;
407
408
			}

Howard Chu's avatar
Howard Chu committed
409
410
411
			rc = mdb_cursor_put(mc, &key, &data, putflags);
			if (rc == MDB_KEYEXIST && putflags)
				continue;
412
413
			if (rc) {
				fprintf(stderr, "mdb_cursor_put failed, error %d %s\n", rc, mdb_strerror(rc));
Howard Chu's avatar
Howard Chu committed
414
				goto txn_abort;
415
			}
Howard Chu's avatar
Howard Chu committed
416
417
418
419
			batch++;
			if (batch == 100) {
				rc = mdb_txn_commit(txn);
				if (rc) {
Howard Chu's avatar
Howard Chu committed
420
					fprintf(stderr, "%s: line %" Z "d: txn_commit: %s\n",
Howard Chu's avatar
Howard Chu committed
421
422
423
424
425
						prog, lineno, mdb_strerror(rc));
					goto env_close;
				}
				rc = mdb_txn_begin(env, NULL, 0, &txn);
				if (rc) {
426
					fprintf(stderr, "mdb_txn_begin failed, error %d %s\n", rc, mdb_strerror(rc));
Howard Chu's avatar
Howard Chu committed
427
428
429
430
					goto env_close;
				}
				rc = mdb_cursor_open(txn, dbi, &mc);
				if (rc) {
431
					fprintf(stderr, "mdb_cursor_open failed, error %d %s\n", rc, mdb_strerror(rc));
Howard Chu's avatar
Howard Chu committed
432
433
434
435
436
437
438
439
					goto txn_abort;
				}
				batch = 0;
			}
		}
		rc = mdb_txn_commit(txn);
		txn = NULL;
		if (rc) {
Howard Chu's avatar
Howard Chu committed
440
			fprintf(stderr, "%s: line %" Z "d: txn_commit: %s\n",
Howard Chu's avatar
Howard Chu committed
441
442
443
444
445
446
447
448
449
450
451
452
453
				prog, lineno, mdb_strerror(rc));
			goto env_close;
		}
		mdb_dbi_close(env, dbi);
	}

txn_abort:
	mdb_txn_abort(txn);
env_close:
	mdb_env_close(env);

	return rc ? EXIT_FAILURE : EXIT_SUCCESS;
}