diff options
| -rw-r--r-- | Documentation/config.txt | 2 | ||||
| -rw-r--r-- | Documentation/config/reftable.txt | 48 | ||||
| -rw-r--r-- | refs/reftable-backend.c | 49 | ||||
| -rw-r--r-- | reftable/block.h | 2 | ||||
| -rw-r--r-- | reftable/constants.h | 1 | ||||
| -rw-r--r-- | reftable/dump.c | 12 | ||||
| -rw-r--r-- | reftable/reader.c | 63 | ||||
| -rw-r--r-- | reftable/reftable-reader.h | 2 | ||||
| -rw-r--r-- | reftable/reftable-stack.h | 2 | ||||
| -rw-r--r-- | reftable/reftable-writer.h | 10 | ||||
| -rw-r--r-- | reftable/stack.c | 58 | ||||
| -rw-r--r-- | reftable/stack.h | 5 | ||||
| -rw-r--r-- | reftable/stack_test.c | 118 | ||||
| -rw-r--r-- | reftable/writer.c | 23 | ||||
| -rwxr-xr-x | t/t0613-reftable-write-options.sh | 286 |
15 files changed, 566 insertions, 115 deletions
diff --git a/Documentation/config.txt b/Documentation/config.txt index 6f649c997c..cbf0b99c44 100644 --- a/Documentation/config.txt +++ b/Documentation/config.txt @@ -498,6 +498,8 @@ include::config/rebase.txt[] include::config/receive.txt[] +include::config/reftable.txt[] + include::config/remote.txt[] include::config/remotes.txt[] diff --git a/Documentation/config/reftable.txt b/Documentation/config/reftable.txt new file mode 100644 index 0000000000..0515727977 --- /dev/null +++ b/Documentation/config/reftable.txt @@ -0,0 +1,48 @@ +reftable.blockSize:: + The size in bytes used by the reftable backend when writing blocks. + The block size is determined by the writer, and does not have to be a + power of 2. The block size must be larger than the longest reference + name or log entry used in the repository, as references cannot span + blocks. ++ +Powers of two that are friendly to the virtual memory system or +filesystem (such as 4kB or 8kB) are recommended. Larger sizes (64kB) can +yield better compression, with a possible increased cost incurred by +readers during access. ++ +The largest block size is `16777215` bytes (15.99 MiB). The default value is +`4096` bytes (4kB). A value of `0` will use the default value. + +reftable.restartInterval:: + The interval at which to create restart points. The reftable backend + determines the restart points at file creation. Every 16 may be + more suitable for smaller block sizes (4k or 8k), every 64 for larger + block sizes (64k). ++ +More frequent restart points reduces prefix compression and increases +space consumed by the restart table, both of which increase file size. ++ +Less frequent restart points makes prefix compression more effective, +decreasing overall file size, with increased penalties for readers +walking through more records after the binary search step. ++ +A maximum of `65535` restart points per block is supported. ++ +The default value is to create restart points every 16 records. A value of `0` +will use the default value. + +reftable.indexObjects:: + Whether the reftable backend shall write object blocks. Object blocks + are a reverse mapping of object ID to the references pointing to them. ++ +The default value is `true`. + +reftable.geometricFactor:: + Whenever the reftable backend appends a new table to the stack, it + performs auto compaction to ensure that there is only a handful of + tables. The backend does this by ensuring that tables form a geometric + sequence regarding the respective sizes of each table. ++ +By default, the geometric sequence uses a factor of 2, meaning that for any +table, the next-biggest table must at least be twice as big. A maximum factor +of 256 is supported. diff --git a/refs/reftable-backend.c b/refs/reftable-backend.c index 1af86bbdec..543afa3f0f 100644 --- a/refs/reftable-backend.c +++ b/refs/reftable-backend.c @@ -1,6 +1,7 @@ #include "../git-compat-util.h" #include "../abspath.h" #include "../chdir-notify.h" +#include "../config.h" #include "../environment.h" #include "../gettext.h" #include "../hash.h" @@ -129,7 +130,7 @@ static struct reftable_stack *stack_for(struct reftable_ref_store *store, store->base.repo->commondir, wtname_buf.buf); store->err = reftable_new_stack(&stack, wt_dir.buf, - store->write_options); + &store->write_options); assert(store->err != REFTABLE_API_ERROR); strmap_put(&store->worktree_stacks, wtname_buf.buf, stack); } @@ -228,6 +229,34 @@ done: return ret; } +static int reftable_be_config(const char *var, const char *value, + const struct config_context *ctx, + void *_opts) +{ + struct reftable_write_options *opts = _opts; + + if (!strcmp(var, "reftable.blocksize")) { + unsigned long block_size = git_config_ulong(var, value, ctx->kvi); + if (block_size > 16777215) + die("reftable block size cannot exceed 16MB"); + opts->block_size = block_size; + } else if (!strcmp(var, "reftable.restartinterval")) { + unsigned long restart_interval = git_config_ulong(var, value, ctx->kvi); + if (restart_interval > UINT16_MAX) + die("reftable block size cannot exceed %u", (unsigned)UINT16_MAX); + opts->restart_interval = restart_interval; + } else if (!strcmp(var, "reftable.indexobjects")) { + opts->skip_index_objects = !git_config_bool(var, value); + } else if (!strcmp(var, "reftable.geometricfactor")) { + unsigned long factor = git_config_ulong(var, value, ctx->kvi); + if (factor > UINT8_MAX) + die("reftable geometric factor cannot exceed %u", (unsigned)UINT8_MAX); + opts->auto_compaction_factor = factor; + } + + return 0; +} + static struct ref_store *reftable_be_init(struct repository *repo, const char *gitdir, unsigned int store_flags) @@ -243,12 +272,24 @@ static struct ref_store *reftable_be_init(struct repository *repo, base_ref_store_init(&refs->base, repo, gitdir, &refs_be_reftable); strmap_init(&refs->worktree_stacks); refs->store_flags = store_flags; - refs->write_options.block_size = 4096; + refs->write_options.hash_id = repo->hash_algo->format_id; refs->write_options.default_permissions = calc_shared_perm(0666 & ~mask); refs->write_options.disable_auto_compact = !git_env_bool("GIT_TEST_REFTABLE_AUTOCOMPACTION", 1); + git_config(reftable_be_config, &refs->write_options); + + /* + * It is somewhat unfortunate that we have to mirror the default block + * size of the reftable library here. But given that the write options + * wouldn't be updated by the library here, and given that we require + * the proper block size to trim reflog message so that they fit, we + * must set up a proper value here. + */ + if (!refs->write_options.block_size) + refs->write_options.block_size = 4096; + /* * Set up the main reftable stack that is hosted in GIT_COMMON_DIR. * This stack contains both the shared and the main worktree refs. @@ -263,7 +304,7 @@ static struct ref_store *reftable_be_init(struct repository *repo, } strbuf_addstr(&path, "/reftable"); refs->err = reftable_new_stack(&refs->main_stack, path.buf, - refs->write_options); + &refs->write_options); if (refs->err) goto done; @@ -280,7 +321,7 @@ static struct ref_store *reftable_be_init(struct repository *repo, strbuf_addf(&path, "%s/reftable", gitdir); refs->err = reftable_new_stack(&refs->worktree_stack, path.buf, - refs->write_options); + &refs->write_options); if (refs->err) goto done; } diff --git a/reftable/block.h b/reftable/block.h index e91f3d2790..1c8f25ee6e 100644 --- a/reftable/block.h +++ b/reftable/block.h @@ -29,7 +29,7 @@ struct block_writer { uint32_t header_off; /* How often to restart keys. */ - int restart_interval; + uint16_t restart_interval; int hash_size; /* Offset of next uint8_t to write. */ diff --git a/reftable/constants.h b/reftable/constants.h index 5eee72c4c1..f6beb843eb 100644 --- a/reftable/constants.h +++ b/reftable/constants.h @@ -17,5 +17,6 @@ https://developers.google.com/open-source/licenses/bsd #define MAX_RESTARTS ((1 << 16) - 1) #define DEFAULT_BLOCK_SIZE 4096 +#define DEFAULT_GEOMETRIC_FACTOR 2 #endif diff --git a/reftable/dump.c b/reftable/dump.c index 26e0393c7d..41abbb8ecf 100644 --- a/reftable/dump.c +++ b/reftable/dump.c @@ -27,9 +27,9 @@ https://developers.google.com/open-source/licenses/bsd static int compact_stack(const char *stackdir) { struct reftable_stack *stack = NULL; - struct reftable_write_options cfg = { 0 }; + struct reftable_write_options opts = { 0 }; - int err = reftable_new_stack(&stack, stackdir, cfg); + int err = reftable_new_stack(&stack, stackdir, &opts); if (err < 0) goto done; @@ -48,6 +48,7 @@ static void print_help(void) printf("usage: dump [-cst] arg\n\n" "options: \n" " -c compact\n" + " -b dump blocks\n" " -t dump table\n" " -s dump stack\n" " -6 sha256 hash format\n" @@ -58,6 +59,7 @@ static void print_help(void) int reftable_dump_main(int argc, char *const *argv) { int err = 0; + int opt_dump_blocks = 0; int opt_dump_table = 0; int opt_dump_stack = 0; int opt_compact = 0; @@ -67,6 +69,8 @@ int reftable_dump_main(int argc, char *const *argv) for (; argc > 1; argv++, argc--) if (*argv[1] != '-') break; + else if (!strcmp("-b", argv[1])) + opt_dump_blocks = 1; else if (!strcmp("-t", argv[1])) opt_dump_table = 1; else if (!strcmp("-6", argv[1])) @@ -88,7 +92,9 @@ int reftable_dump_main(int argc, char *const *argv) arg = argv[1]; - if (opt_dump_table) { + if (opt_dump_blocks) { + err = reftable_reader_print_blocks(arg); + } else if (opt_dump_table) { err = reftable_reader_print_file(arg); } else if (opt_dump_stack) { err = reftable_stack_print_directory(arg, opt_hash_id); diff --git a/reftable/reader.c b/reftable/reader.c index 481dff10d4..f23c8523db 100644 --- a/reftable/reader.c +++ b/reftable/reader.c @@ -856,3 +856,66 @@ done: reftable_reader_free(r); return err; } + +int reftable_reader_print_blocks(const char *tablename) +{ + struct { + const char *name; + int type; + } sections[] = { + { + .name = "ref", + .type = BLOCK_TYPE_REF, + }, + { + .name = "obj", + .type = BLOCK_TYPE_OBJ, + }, + { + .name = "log", + .type = BLOCK_TYPE_LOG, + }, + }; + struct reftable_block_source src = { 0 }; + struct table_iter ti = TABLE_ITER_INIT; + struct reftable_reader *r = NULL; + size_t i; + int err; + + err = reftable_block_source_from_file(&src, tablename); + if (err < 0) + goto done; + + err = reftable_new_reader(&r, &src, tablename); + if (err < 0) + goto done; + + printf("header:\n"); + printf(" block_size: %d\n", r->block_size); + + for (i = 0; i < ARRAY_SIZE(sections); i++) { + err = reader_start(r, &ti, sections[i].type, 0); + if (err < 0) + goto done; + if (err > 0) + continue; + + printf("%s:\n", sections[i].name); + + while (1) { + printf(" - length: %u\n", ti.br.block_len); + printf(" restarts: %u\n", ti.br.restart_count); + + err = table_iter_next_block(&ti); + if (err < 0) + goto done; + if (err > 0) + break; + } + } + +done: + reftable_reader_free(r); + table_iter_close(&ti); + return err; +} diff --git a/reftable/reftable-reader.h b/reftable/reftable-reader.h index 4a4bc2fdf8..4a04857773 100644 --- a/reftable/reftable-reader.h +++ b/reftable/reftable-reader.h @@ -97,5 +97,7 @@ void reftable_table_from_reader(struct reftable_table *tab, /* print table onto stdout for debugging. */ int reftable_reader_print_file(const char *tablename); +/* print blocks onto stdout for debugging. */ +int reftable_reader_print_blocks(const char *tablename); #endif diff --git a/reftable/reftable-stack.h b/reftable/reftable-stack.h index 1b602dda58..c15632c401 100644 --- a/reftable/reftable-stack.h +++ b/reftable/reftable-stack.h @@ -29,7 +29,7 @@ struct reftable_stack; * stored in 'dir'. Typically, this should be .git/reftables. */ int reftable_new_stack(struct reftable_stack **dest, const char *dir, - struct reftable_write_options config); + const struct reftable_write_options *opts); /* returns the update_index at which a next table should be written. */ uint64_t reftable_stack_next_update_index(struct reftable_stack *st); diff --git a/reftable/reftable-writer.h b/reftable/reftable-writer.h index b601a69a40..189b1f4144 100644 --- a/reftable/reftable-writer.h +++ b/reftable/reftable-writer.h @@ -28,7 +28,7 @@ struct reftable_write_options { unsigned skip_index_objects : 1; /* how often to write complete keys in each block. */ - int restart_interval; + uint16_t restart_interval; /* 4-byte identifier ("sha1", "s256") of the hash. * Defaults to SHA1 if unset @@ -45,6 +45,12 @@ struct reftable_write_options { /* boolean: Prevent auto-compaction of tables. */ unsigned disable_auto_compact : 1; + + /* + * Geometric sequence factor used by auto-compaction to decide which + * tables to compact. Defaults to 2 if unset. + */ + uint8_t auto_compaction_factor; }; /* reftable_block_stats holds statistics for a single block type */ @@ -88,7 +94,7 @@ struct reftable_stats { struct reftable_writer * reftable_new_writer(ssize_t (*writer_func)(void *, const void *, size_t), int (*flush_func)(void *), - void *writer_arg, struct reftable_write_options *opts); + void *writer_arg, const struct reftable_write_options *opts); /* Set the range of update indices for the records we will add. When writing a table into a stack, the min should be at least diff --git a/reftable/stack.c b/reftable/stack.c index a59ebe038d..0ebe69e81d 100644 --- a/reftable/stack.c +++ b/reftable/stack.c @@ -10,6 +10,7 @@ https://developers.google.com/open-source/licenses/bsd #include "../write-or-die.h" #include "system.h" +#include "constants.h" #include "merged.h" #include "reader.h" #include "reftable-error.h" @@ -54,15 +55,17 @@ static int reftable_fd_flush(void *arg) } int reftable_new_stack(struct reftable_stack **dest, const char *dir, - struct reftable_write_options config) + const struct reftable_write_options *_opts) { struct reftable_stack *p = reftable_calloc(1, sizeof(*p)); struct strbuf list_file_name = STRBUF_INIT; + struct reftable_write_options opts = {0}; int err = 0; - if (config.hash_id == 0) { - config.hash_id = GIT_SHA1_FORMAT_ID; - } + if (_opts) + opts = *_opts; + if (opts.hash_id == 0) + opts.hash_id = GIT_SHA1_FORMAT_ID; *dest = NULL; @@ -73,7 +76,7 @@ int reftable_new_stack(struct reftable_stack **dest, const char *dir, p->list_file = strbuf_detach(&list_file_name, NULL); p->list_fd = -1; p->reftable_dir = xstrdup(dir); - p->config = config; + p->opts = opts; err = reftable_stack_reload_maybe_reuse(p, 1); if (err < 0) { @@ -255,7 +258,7 @@ static int reftable_stack_reload_once(struct reftable_stack *st, char **names, /* success! */ err = reftable_new_merged_table(&new_merged, new_tables, - new_readers_len, st->config.hash_id); + new_readers_len, st->opts.hash_id); if (err < 0) goto done; @@ -578,8 +581,8 @@ static int reftable_stack_init_addition(struct reftable_addition *add, } goto done; } - if (st->config.default_permissions) { - if (chmod(add->lock_file->filename.buf, st->config.default_permissions) < 0) { + if (st->opts.default_permissions) { + if (chmod(add->lock_file->filename.buf, st->opts.default_permissions) < 0) { err = REFTABLE_IO_ERROR; goto done; } @@ -678,7 +681,7 @@ int reftable_addition_commit(struct reftable_addition *add) if (err) goto done; - if (!add->stack->config.disable_auto_compact) { + if (!add->stack->opts.disable_auto_compact) { /* * Auto-compact the stack to keep the number of tables in * control. It is possible that a concurrent writer is already @@ -756,9 +759,9 @@ int reftable_addition_add(struct reftable_addition *add, err = REFTABLE_IO_ERROR; goto done; } - if (add->stack->config.default_permissions) { + if (add->stack->opts.default_permissions) { if (chmod(get_tempfile_path(tab_file), - add->stack->config.default_permissions)) { + add->stack->opts.default_permissions)) { err = REFTABLE_IO_ERROR; goto done; } @@ -766,7 +769,7 @@ int reftable_addition_add(struct reftable_addition *add, tab_fd = get_tempfile_fd(tab_file); wr = reftable_new_writer(reftable_fd_write, reftable_fd_flush, &tab_fd, - &add->stack->config); + &add->stack->opts); err = write_table(wr, arg); if (err < 0) goto done; @@ -849,14 +852,14 @@ static int stack_compact_locked(struct reftable_stack *st, } tab_fd = get_tempfile_fd(tab_file); - if (st->config.default_permissions && - chmod(get_tempfile_path(tab_file), st->config.default_permissions) < 0) { + if (st->opts.default_permissions && + chmod(get_tempfile_path(tab_file), st->opts.default_permissions) < 0) { err = REFTABLE_IO_ERROR; goto done; } wr = reftable_new_writer(reftable_fd_write, reftable_fd_flush, - &tab_fd, &st->config); + &tab_fd, &st->opts); err = stack_write_compact(st, wr, first, last, config); if (err < 0) goto done; @@ -904,7 +907,7 @@ static int stack_write_compact(struct reftable_stack *st, st->readers[last]->max_update_index); err = reftable_new_merged_table(&mt, subtabs, subtabs_len, - st->config.hash_id); + st->opts.hash_id); if (err < 0) { reftable_free(subtabs); goto done; @@ -1094,9 +1097,9 @@ static int stack_compact_range(struct reftable_stack *st, goto done; } - if (st->config.default_permissions) { + if (st->opts.default_permissions) { if (chmod(get_lock_file_path(&tables_list_lock), - st->config.default_permissions) < 0) { + st->opts.default_permissions) < 0) { err = REFTABLE_IO_ERROR; goto done; } @@ -1210,12 +1213,16 @@ static int segment_size(struct segment *s) return s->end - s->start; } -struct segment suggest_compaction_segment(uint64_t *sizes, size_t n) +struct segment suggest_compaction_segment(uint64_t *sizes, size_t n, + uint8_t factor) { struct segment seg = { 0 }; uint64_t bytes; size_t i; + if (!factor) + factor = DEFAULT_GEOMETRIC_FACTOR; + /* * If there are no tables or only a single one then we don't have to * compact anything. The sequence is geometric by definition already. @@ -1247,7 +1254,7 @@ struct segment suggest_compaction_segment(uint64_t *sizes, size_t n) * 64, 32, 16, 8, 4, 3, 1 */ for (i = n - 1; i > 0; i--) { - if (sizes[i - 1] < sizes[i] * 2) { + if (sizes[i - 1] < sizes[i] * factor) { seg.end = i + 1; bytes = sizes[i]; break; @@ -1273,7 +1280,7 @@ struct segment suggest_compaction_segment(uint64_t *sizes, size_t n) uint64_t curr = bytes; bytes += sizes[i - 1]; - if (sizes[i - 1] < curr * 2) { + if (sizes[i - 1] < curr * factor) { seg.start = i - 1; seg.bytes = bytes; } @@ -1286,7 +1293,7 @@ static uint64_t *stack_table_sizes_for_compaction(struct reftable_stack *st) { uint64_t *sizes = reftable_calloc(st->merged->stack_len, sizeof(*sizes)); - int version = (st->config.hash_id == GIT_SHA1_FORMAT_ID) ? 1 : 2; + int version = (st->opts.hash_id == GIT_SHA1_FORMAT_ID) ? 1 : 2; int overhead = header_size(version) - 1; int i = 0; for (i = 0; i < st->merged->stack_len; i++) { @@ -1299,7 +1306,8 @@ int reftable_stack_auto_compact(struct reftable_stack *st) { uint64_t *sizes = stack_table_sizes_for_compaction(st); struct segment seg = - suggest_compaction_segment(sizes, st->merged->stack_len); + suggest_compaction_segment(sizes, st->merged->stack_len, + st->opts.auto_compaction_factor); reftable_free(sizes); if (segment_size(&seg) > 0) return stack_compact_range_stats(st, seg.start, seg.end - 1, @@ -1435,11 +1443,11 @@ done: int reftable_stack_print_directory(const char *stackdir, uint32_t hash_id) { struct reftable_stack *stack = NULL; - struct reftable_write_options cfg = { .hash_id = hash_id }; + struct reftable_write_options opts = { .hash_id = hash_id }; struct reftable_merged_table *merged = NULL; struct reftable_table table = { NULL }; - int err = reftable_new_stack(&stack, stackdir, cfg); + int err = reftable_new_stack(&stack, stackdir, &opts); if (err < 0) goto done; diff --git a/reftable/stack.h b/reftable/stack.h index d43efa4760..5b45cff4f7 100644 --- a/reftable/stack.h +++ b/reftable/stack.h @@ -20,7 +20,7 @@ struct reftable_stack { char *reftable_dir; - struct reftable_write_options config; + struct reftable_write_options opts; struct reftable_reader **readers; size_t readers_len; @@ -35,6 +35,7 @@ struct segment { uint64_t bytes; }; -struct segment suggest_compaction_segment(uint64_t *sizes, size_t n); +struct segment suggest_compaction_segment(uint64_t *sizes, size_t n, + uint8_t factor); #endif diff --git a/reftable/stack_test.c b/reftable/stack_test.c index 7889f818d1..0f7b1453e6 100644 --- a/reftable/stack_test.c +++ b/reftable/stack_test.c @@ -150,7 +150,7 @@ static void test_reftable_stack_add_one(void) char *dir = get_tmp_dir(__LINE__); struct strbuf scratch = STRBUF_INIT; int mask = umask(002); - struct reftable_write_options cfg = { + struct reftable_write_options opts = { .default_permissions = 0660, }; struct reftable_stack *st = NULL; @@ -163,7 +163,7 @@ static void test_reftable_stack_add_one(void) }; struct reftable_ref_record dest = { NULL }; struct stat stat_result = { 0 }; - err = reftable_new_stack(&st, dir, cfg); + err = reftable_new_stack(&st, dir, &opts); EXPECT_ERR(err); err = reftable_stack_add(st, &write_test_ref, &ref); @@ -186,7 +186,7 @@ static void test_reftable_stack_add_one(void) strbuf_addstr(&scratch, "/tables.list"); err = stat(scratch.buf, &stat_result); EXPECT(!err); - EXPECT((stat_result.st_mode & 0777) == cfg.default_permissions); + EXPECT((stat_result.st_mode & 0777) == opts.default_permissions); strbuf_reset(&scratch); strbuf_addstr(&scratch, dir); @@ -195,7 +195,7 @@ static void test_reftable_stack_add_one(void) strbuf_addstr(&scratch, st->readers[0]->name); err = stat(scratch.buf, &stat_result); EXPECT(!err); - EXPECT((stat_result.st_mode & 0777) == cfg.default_permissions); + EXPECT((stat_result.st_mode & 0777) == opts.default_permissions); #else (void) stat_result; #endif @@ -209,7 +209,7 @@ static void test_reftable_stack_add_one(void) static void test_reftable_stack_uptodate(void) { - struct reftable_write_options cfg = { 0 }; + struct reftable_write_options opts = { 0 }; struct reftable_stack *st1 = NULL; struct reftable_stack *st2 = NULL; char *dir = get_tmp_dir(__LINE__); @@ -232,10 +232,10 @@ static void test_reftable_stack_uptodate(void) /* simulate multi-process access to the same stack by creating two stacks for the same directory. */ - err = reftable_new_stack(&st1, dir, cfg); + err = reftable_new_stack(&st1, dir, &opts); EXPECT_ERR(err); - err = reftable_new_stack(&st2, dir, cfg); + err = reftable_new_stack(&st2, dir, &opts); EXPECT_ERR(err); err = reftable_stack_add(st1, &write_test_ref, &ref1); @@ -257,8 +257,7 @@ static void test_reftable_stack_uptodate(void) static void test_reftable_stack_transaction_api(void) { char *dir = get_tmp_dir(__LINE__); - - struct reftable_write_options cfg = { 0 }; + struct reftable_write_options opts = { 0 }; struct reftable_stack *st = NULL; int err; struct reftable_addition *add = NULL; @@ -271,8 +270,7 @@ static void test_reftable_stack_transaction_api(void) }; struct reftable_ref_record dest = { NULL }; - - err = reftable_new_stack(&st, dir, cfg); + err = reftable_new_stack(&st, dir, &opts); EXPECT_ERR(err); reftable_addition_destroy(add); @@ -301,12 +299,12 @@ static void test_reftable_stack_transaction_api(void) static void test_reftable_stack_transaction_api_performs_auto_compaction(void) { char *dir = get_tmp_dir(__LINE__); - struct reftable_write_options cfg = {0}; + struct reftable_write_options opts = {0}; struct reftable_addition *add = NULL; struct reftable_stack *st = NULL; int i, n = 20, err; - err = reftable_new_stack(&st, dir, cfg); + err = reftable_new_stack(&st, dir, &opts); EXPECT_ERR(err); for (i = 0; i <= n; i++) { @@ -325,7 +323,7 @@ static void test_reftable_stack_transaction_api_performs_auto_compaction(void) * we can ensure that we indeed honor this setting and have * better control over when exactly auto compaction runs. */ - st->config.disable_auto_compact = i != n; + st->opts.disable_auto_compact = i != n; err = reftable_stack_new_addition(&add, st); EXPECT_ERR(err); @@ -361,13 +359,13 @@ static void test_reftable_stack_auto_compaction_fails_gracefully(void) .value_type = REFTABLE_REF_VAL1, .value.val1 = {0x01}, }; - struct reftable_write_options cfg = {0}; + struct reftable_write_options opts = {0}; struct reftable_stack *st; struct strbuf table_path = STRBUF_INIT; char *dir = get_tmp_dir(__LINE__); int err; - err = reftable_new_stack(&st, dir, cfg); + err = reftable_new_stack(&st, dir, &opts); EXPECT_ERR(err); err = reftable_stack_add(st, write_test_ref, &ref); @@ -404,8 +402,7 @@ static int write_error(struct reftable_writer *wr, void *arg) static void test_reftable_stack_update_index_check(void) { char *dir = get_tmp_dir(__LINE__); - - struct reftable_write_options cfg = { 0 }; + struct reftable_write_options opts = { 0 }; struct reftable_stack *st = NULL; int err; struct reftable_ref_record ref1 = { @@ -421,7 +418,7 @@ static void test_reftable_stack_update_index_check(void) .value.symref = "master", }; - err = reftable_new_stack(&st, dir, cfg); + err = reftable_new_stack(&st, dir, &opts); EXPECT_ERR(err); err = reftable_stack_add(st, &write_test_ref, &ref1); @@ -436,12 +433,11 @@ static void test_reftable_stack_update_index_check(void) static void test_reftable_stack_lock_failure(void) { char *dir = get_tmp_dir(__LINE__); - - struct reftable_write_options cfg = { 0 }; + struct reftable_write_options opts = { 0 }; struct reftable_stack *st = NULL; int err, i; - err = reftable_new_stack(&st, dir, cfg); + err = reftable_new_stack(&st, dir, &opts); EXPECT_ERR(err); for (i = -1; i != REFTABLE_EMPTY_TABLE_ERROR; i--) { err = reftable_stack_add(st, &write_error, &i); @@ -456,7 +452,7 @@ static void test_reftable_stack_add(void) { int i = 0; int err = 0; - struct reftable_write_options cfg = { + struct reftable_write_options opts = { .exact_log_message = 1, .default_permissions = 0660, .disable_auto_compact = 1, @@ -469,7 +465,7 @@ static void test_reftable_stack_add(void) struct stat stat_result; int N = ARRAY_SIZE(refs); - err = reftable_new_stack(&st, dir, cfg); + err = reftable_new_stack(&st, dir, &opts); EXPECT_ERR(err); for (i = 0; i < N; i++) { @@ -528,7 +524,7 @@ static void test_reftable_stack_add(void) strbuf_addstr(&path, "/tables.list"); err = stat(path.buf, &stat_result); EXPECT(!err); - EXPECT((stat_result.st_mode & 0777) == cfg.default_permissions); + EXPECT((stat_result.st_mode & 0777) == opts.default_permissions); strbuf_reset(&path); strbuf_addstr(&path, dir); @@ -537,7 +533,7 @@ static void test_reftable_stack_add(void) strbuf_addstr(&path, st->readers[0]->name); err = stat(path.buf, &stat_result); EXPECT(!err); - EXPECT((stat_result.st_mode & 0777) == cfg.default_permissions); + EXPECT((stat_result.st_mode & 0777) == opts.default_permissions); #else (void) stat_result; #endif @@ -555,7 +551,7 @@ static void test_reftable_stack_add(void) static void test_reftable_stack_log_normalize(void) { int err = 0; - struct reftable_write_options cfg = { + struct reftable_write_options opts = { 0, }; struct reftable_stack *st = NULL; @@ -579,7 +575,7 @@ static void test_reftable_stack_log_normalize(void) .update_index = 1, }; - err = reftable_new_stack(&st, dir, cfg); + err = reftable_new_stack(&st, dir, &opts); EXPECT_ERR(err); input.value.update.message = "one\ntwo"; @@ -612,8 +608,7 @@ static void test_reftable_stack_tombstone(void) { int i = 0; char *dir = get_tmp_dir(__LINE__); - - struct reftable_write_options cfg = { 0 }; + struct reftable_write_options opts = { 0 }; struct reftable_stack *st = NULL; int err; struct reftable_ref_record refs[2] = { { NULL } }; @@ -622,8 +617,7 @@ static void test_reftable_stack_tombstone(void) struct reftable_ref_record dest = { NULL }; struct reftable_log_record log_dest = { NULL }; - - err = reftable_new_stack(&st, dir, cfg); + err = reftable_new_stack(&st, dir, &opts); EXPECT_ERR(err); /* even entries add the refs, odd entries delete them. */ @@ -691,8 +685,7 @@ static void test_reftable_stack_tombstone(void) static void test_reftable_stack_hash_id(void) { char *dir = get_tmp_dir(__LINE__); - - struct reftable_write_options cfg = { 0 }; + struct reftable_write_options opts = { 0 }; struct reftable_stack *st = NULL; int err; @@ -702,24 +695,24 @@ static void test_reftable_stack_hash_id(void) .value.symref = "target", .update_index = 1, }; - struct reftable_write_options cfg32 = { .hash_id = GIT_SHA256_FORMAT_ID }; + struct reftable_write_options opts32 = { .hash_id = GIT_SHA256_FORMAT_ID }; struct reftable_stack *st32 = NULL; - struct reftable_write_options cfg_default = { 0 }; + struct reftable_write_options opts_default = { 0 }; struct reftable_stack *st_default = NULL; struct reftable_ref_record dest = { NULL }; - err = reftable_new_stack(&st, dir, cfg); + err = reftable_new_stack(&st, dir, &opts); EXPECT_ERR(err); err = reftable_stack_add(st, &write_test_ref, &ref); EXPECT_ERR(err); /* can't read it with the wrong hash ID. */ - err = reftable_new_stack(&st32, dir, cfg32); + err = reftable_new_stack(&st32, dir, &opts32); EXPECT(err == REFTABLE_FORMAT_ERROR); - /* check that we can read it back with default config too. */ - err = reftable_new_stack(&st_default, dir, cfg_default); + /* check that we can read it back with default opts too. */ + err = reftable_new_stack(&st_default, dir, &opts_default); EXPECT_ERR(err); err = reftable_stack_read_ref(st_default, "master", &dest); @@ -736,7 +729,7 @@ static void test_suggest_compaction_segment(void) { uint64_t sizes[] = { 512, 64, 17, 16, 9, 9, 9, 16, 2, 16 }; struct segment min = - suggest_compaction_segment(sizes, ARRAY_SIZE(sizes)); + suggest_compaction_segment(sizes, ARRAY_SIZE(sizes), 2); EXPECT(min.start == 1); EXPECT(min.end == 10); } @@ -745,15 +738,14 @@ static void test_suggest_compaction_segment_nothing(void) { uint64_t sizes[] = { 64, 32, 16, 8, 4, 2 }; struct segment result = - suggest_compaction_segment(sizes, ARRAY_SIZE(sizes)); + suggest_compaction_segment(sizes, ARRAY_SIZE(sizes), 2); EXPECT(result.start == result.end); } static void test_reflog_expire(void) { char *dir = get_tmp_dir(__LINE__); - - struct reftable_write_options cfg = { 0 }; + struct reftable_write_options opts = { 0 }; struct reftable_stack *st = NULL; struct reftable_log_record logs[20] = { { NULL } }; int N = ARRAY_SIZE(logs) - 1; @@ -764,8 +756,7 @@ static void test_reflog_expire(void) }; struct reftable_log_record log = { NULL }; - - err = reftable_new_stack(&st, dir, cfg); + err = reftable_new_stack(&st, dir, &opts); EXPECT_ERR(err); for (i = 1; i <= N; i++) { @@ -828,21 +819,19 @@ static int write_nothing(struct reftable_writer *wr, void *arg) static void test_empty_add(void) { - struct reftable_write_options cfg = { 0 }; + struct reftable_write_options opts = { 0 }; struct reftable_stack *st = NULL; int err; char *dir = get_tmp_dir(__LINE__); - struct reftable_stack *st2 = NULL; - - err = reftable_new_stack(&st, dir, cfg); + err = reftable_new_stack(&st, dir, &opts); EXPECT_ERR(err); err = reftable_stack_add(st, &write_nothing, NULL); EXPECT_ERR(err); - err = reftable_new_stack(&st2, dir, cfg); + err = reftable_new_stack(&st2, dir, &opts); EXPECT_ERR(err); clear_dir(dir); reftable_stack_destroy(st); @@ -861,16 +850,15 @@ static int fastlog2(uint64_t sz) static void test_reftable_stack_auto_compaction(void) { - struct reftable_write_options cfg = { + struct reftable_write_options opts = { .disable_auto_compact = 1, }; struct reftable_stack *st = NULL; char *dir = get_tmp_dir(__LINE__); - int err, i; int N = 100; - err = reftable_new_stack(&st, dir, cfg); + err = reftable_new_stack(&st, dir, &opts); EXPECT_ERR(err); for (i = 0; i < N; i++) { @@ -900,13 +888,13 @@ static void test_reftable_stack_auto_compaction(void) static void test_reftable_stack_add_performs_auto_compaction(void) { - struct reftable_write_options cfg = { 0 }; + struct reftable_write_options opts = { 0 }; struct reftable_stack *st = NULL; struct strbuf refname = STRBUF_INIT; char *dir = get_tmp_dir(__LINE__); int err, i, n = 20; - err = reftable_new_stack(&st, dir, cfg); + err = reftable_new_stack(&st, dir, &opts); EXPECT_ERR(err); for (i = 0; i <= n; i++) { @@ -921,7 +909,7 @@ static void test_reftable_stack_add_performs_auto_compaction(void) * we can ensure that we indeed honor this setting and have * better control over when exactly auto compaction runs. */ - st->config.disable_auto_compact = i != n; + st->opts.disable_auto_compact = i != n; strbuf_reset(&refname); strbuf_addf(&refname, "branch-%04d", i); @@ -948,14 +936,13 @@ static void test_reftable_stack_add_performs_auto_compaction(void) static void test_reftable_stack_compaction_concurrent(void) { - struct reftable_write_options cfg = { 0 }; + struct reftable_write_options opts = { 0 }; struct reftable_stack *st1 = NULL, *st2 = NULL; char *dir = get_tmp_dir(__LINE__); - int err, i; int N = 3; - err = reftable_new_stack(&st1, dir, cfg); + err = reftable_new_stack(&st1, dir, &opts); EXPECT_ERR(err); for (i = 0; i < N; i++) { @@ -972,7 +959,7 @@ static void test_reftable_stack_compaction_concurrent(void) EXPECT_ERR(err); } - err = reftable_new_stack(&st2, dir, cfg); + err = reftable_new_stack(&st2, dir, &opts); EXPECT_ERR(err); err = reftable_stack_compact_all(st1, NULL); @@ -998,14 +985,13 @@ static void unclean_stack_close(struct reftable_stack *st) static void test_reftable_stack_compaction_concurrent_clean(void) { - struct reftable_write_options cfg = { 0 }; + struct reftable_write_options opts = { 0 }; struct reftable_stack *st1 = NULL, *st2 = NULL, *st3 = NULL; char *dir = get_tmp_dir(__LINE__); - int err, i; int N = 3; - err = reftable_new_stack(&st1, dir, cfg); + err = reftable_new_stack(&st1, dir, &opts); EXPECT_ERR(err); for (i = 0; i < N; i++) { @@ -1022,7 +1008,7 @@ static void test_reftable_stack_compaction_concurrent_clean(void) EXPECT_ERR(err); } - err = reftable_new_stack(&st2, dir, cfg); + err = reftable_new_stack(&st2, dir, &opts); EXPECT_ERR(err); err = reftable_stack_compact_all(st1, NULL); @@ -1031,7 +1017,7 @@ static void test_reftable_stack_compaction_concurrent_clean(void) unclean_stack_close(st1); unclean_stack_close(st2); - err = reftable_new_stack(&st3, dir, cfg); + err = reftable_new_stack(&st3, dir, &opts); EXPECT_ERR(err); err = reftable_stack_clean(st3); diff --git a/reftable/writer.c b/reftable/writer.c index 10eccaaa07..45b3e9ce1f 100644 --- a/reftable/writer.c +++ b/reftable/writer.c @@ -117,25 +117,26 @@ static void writer_reinit_block_writer(struct reftable_writer *w, uint8_t typ) w->block_writer->restart_interval = w->opts.restart_interval; } -static struct strbuf reftable_empty_strbuf = STRBUF_INIT; - struct reftable_writer * reftable_new_writer(ssize_t (*writer_func)(void *, const void *, size_t), int (*flush_func)(void *), - void *writer_arg, struct reftable_write_options *opts) + void *writer_arg, const struct reftable_write_options *_opts) { struct reftable_writer *wp = reftable_calloc(1, sizeof(*wp)); + struct reftable_write_options opts = {0}; + + if (_opts) + opts = *_opts; + options_set_defaults(&opts); + if (opts.block_size >= (1 << 24)) + BUG("configured block size exceeds 16MB"); + strbuf_init(&wp->block_writer_data.last_key, 0); - options_set_defaults(opts); - if (opts->block_size >= (1 << 24)) { - /* TODO - error return? */ - abort(); - } - wp->last_key = reftable_empty_strbuf; - REFTABLE_CALLOC_ARRAY(wp->block, opts->block_size); + strbuf_init(&wp->last_key, 0); + REFTABLE_CALLOC_ARRAY(wp->block, opts.block_size); wp->write = writer_func; wp->write_arg = writer_arg; - wp->opts = *opts; + wp->opts = opts; wp->flush = flush_func; writer_reinit_block_writer(wp, BLOCK_TYPE_REF); diff --git a/t/t0613-reftable-write-options.sh b/t/t0613-reftable-write-options.sh new file mode 100755 index 0000000000..e2708e11d5 --- /dev/null +++ b/t/t0613-reftable-write-options.sh @@ -0,0 +1,286 @@ +#!/bin/sh + +test_description='reftable write options' + +GIT_TEST_DEFAULT_REF_FORMAT=reftable +export GIT_TEST_DEFAULT_REF_FORMAT +# Disable auto-compaction for all tests as we explicitly control repacking of +# refs. +GIT_TEST_REFTABLE_AUTOCOMPACTION=false +export GIT_TEST_REFTABLE_AUTOCOMPACTION +# Block sizes depend on the hash function, so we force SHA1 here. +GIT_TEST_DEFAULT_HASH=sha1 +export GIT_TEST_DEFAULT_HASH +# Block sizes also depend on the actual refs we write, so we force "master" to +# be the default initial branch name. +GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=master +export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME + +. ./test-lib.sh + +test_expect_success 'default write options' ' + test_when_finished "rm -rf repo" && + git init repo && + ( + cd repo && + test_commit initial && + git pack-refs && + cat >expect <<-EOF && + header: + block_size: 4096 + ref: + - length: 129 + restarts: 2 + log: + - length: 262 + restarts: 2 + EOF + test-tool dump-reftable -b .git/reftable/*.ref >actual && + test_cmp expect actual + ) +' + +test_expect_success 'disabled reflog writes no log blocks' ' + test_config_global core.logAllRefUpdates false && + test_when_finished "rm -rf repo" && + git init repo && + ( + cd repo && + test_commit initial && + git pack-refs && + cat >expect <<-EOF && + header: + block_size: 4096 + ref: + - length: 129 + restarts: 2 + EOF + test-tool dump-reftable -b .git/reftable/*.ref >actual && + test_cmp expect actual + ) +' + +test_expect_success 'many refs results in multiple blocks' ' + test_when_finished "rm -rf repo" && + git init repo && + ( + cd repo && + test_commit initial && + for i in $(test_seq 200) + do + printf "update refs/heads/branch-%d HEAD\n" "$i" || + return 1 + done >input && + git update-ref --stdin <input && + git pack-refs && + + cat >expect <<-EOF && + header: + block_size: 4096 + ref: + - length: 4049 + restarts: 11 + - length: 1136 + restarts: 3 + log: + - length: 4041 + restarts: 4 + - length: 4015 + restarts: 3 + - length: 4014 + restarts: 3 + - length: 4012 + restarts: 3 + - length: 3289 + restarts: 3 + EOF + test-tool dump-reftable -b .git/reftable/*.ref >actual && + test_cmp expect actual + ) +' + +test_expect_success 'tiny block size leads to error' ' + test_when_finished "rm -rf repo" && + git init repo && + ( + cd repo && + test_commit initial && + cat >expect <<-EOF && + error: unable to compact stack: entry too large + EOF + test_must_fail git -c reftable.blockSize=50 pack-refs 2>err && + test_cmp expect err + ) +' + +test_expect_success 'small block size leads to multiple ref blocks' ' + test_config_global core.logAllRefUpdates false && + test_when_finished "rm -rf repo" && + git init repo && + ( + cd repo && + test_commit A && + test_commit B && + git -c reftable.blockSize=100 pack-refs && + + cat >expect <<-EOF && + header: + block_size: 100 + ref: + - length: 53 + restarts: 1 + - length: 74 + restarts: 1 + - length: 38 + restarts: 1 + EOF + test-tool dump-reftable -b .git/reftable/*.ref >actual && + test_cmp expect actual + ) +' + +test_expect_success 'small block size fails with large reflog message' ' + test_when_finished "rm -rf repo" && + git init repo && + ( + cd repo && + test_commit A && + perl -e "print \"a\" x 500" >logmsg && + cat >expect <<-EOF && + fatal: update_ref failed for ref ${SQ}refs/heads/logme${SQ}: reftable: transaction failure: entry too large + EOF + test_must_fail git -c reftable.blockSize=100 \ + update-ref -m "$(cat logmsg)" refs/heads/logme HEAD 2>err && + test_cmp expect err + ) +' + +test_expect_success 'block size exceeding maximum supported size' ' + test_config_global core.logAllRefUpdates false && + test_when_finished "rm -rf repo" && + git init repo && + ( + cd repo && + test_commit A && + test_commit B && + cat >expect <<-EOF && + fatal: reftable block size cannot exceed 16MB + EOF + test_must_fail git -c reftable.blockSize=16777216 pack-refs 2>err && + test_cmp expect err + ) +' + +test_expect_success 'restart interval at every single record' ' + test_when_finished "rm -rf repo" && + git init repo && + ( + cd repo && + test_commit initial && + for i in $(test_seq 10) + do + printf "update refs/heads/branch-%d HEAD\n" "$i" || + return 1 + done >input && + git update-ref --stdin <input && + git -c reftable.restartInterval=1 pack-refs && + + cat >expect <<-EOF && + header: + block_size: 4096 + ref: + - length: 566 + restarts: 13 + log: + - length: 1393 + restarts: 12 + EOF + test-tool dump-reftable -b .git/reftable/*.ref >actual && + test_cmp expect actual + ) +' + +test_expect_success 'restart interval exceeding maximum supported interval' ' + test_when_finished "rm -rf repo" && + git init repo && + ( + cd repo && + test_commit initial && + cat >expect <<-EOF && + fatal: reftable block size cannot exceed 65535 + EOF + test_must_fail git -c reftable.restartInterval=65536 pack-refs 2>err && + test_cmp expect err + ) +' + +test_expect_success 'object index gets written by default with ref index' ' + test_config_global core.logAllRefUpdates false && + test_when_finished "rm -rf repo" && + git init repo && + ( + cd repo && + test_commit initial && + for i in $(test_seq 5) + do + printf "update refs/heads/branch-%d HEAD\n" "$i" || + return 1 + done >input && + git update-ref --stdin <input && + git -c reftable.blockSize=100 pack-refs && + + cat >expect <<-EOF && + header: + block_size: 100 + ref: + - length: 53 + restarts: 1 + - length: 95 + restarts: 1 + - length: 71 + restarts: 1 + - length: 80 + restarts: 1 + obj: + - length: 11 + restarts: 1 + EOF + test-tool dump-reftable -b .git/reftable/*.ref >actual && + test_cmp expect actual + ) +' + +test_expect_success 'object index can be disabled' ' + test_config_global core.logAllRefUpdates false && + test_when_finished "rm -rf repo" && + git init repo && + ( + cd repo && + test_commit initial && + for i in $(test_seq 5) + do + printf "update refs/heads/branch-%d HEAD\n" "$i" || + return 1 + done >input && + git update-ref --stdin <input && + git -c reftable.blockSize=100 -c reftable.indexObjects=false pack-refs && + + cat >expect <<-EOF && + header: + block_size: 100 + ref: + - length: 53 + restarts: 1 + - length: 95 + restarts: 1 + - length: 71 + restarts: 1 + - length: 80 + restarts: 1 + EOF + test-tool dump-reftable -b .git/reftable/*.ref >actual && + test_cmp expect actual + ) +' + +test_done |
