diff options
| author | Junio C Hamano <gitster@pobox.com> | 2022-06-03 14:30:37 -0700 |
|---|---|---|
| committer | Junio C Hamano <gitster@pobox.com> | 2022-06-03 14:30:37 -0700 |
| commit | a50036da1a39806a8ae1aba2e2f2fea6f7fb8e08 (patch) | |
| tree | 1e6c184852ee5a5e5860073c04ea73e034b0060f /builtin/repack.c | |
| parent | 37d4ae58efcc9f716435f327c39d5552aedb4b7c (diff) | |
| parent | a613164257b46700ca583bdcab160c712ad392fe (diff) | |
| download | git-a50036da1a39806a8ae1aba2e2f2fea6f7fb8e08.tar.gz | |
Merge branch 'tb/cruft-packs'
A mechanism to pack unreachable objects into a "cruft pack",
instead of ejecting them into loose form to be reclaimed later, has
been introduced.
* tb/cruft-packs:
sha1-file.c: don't freshen cruft packs
builtin/gc.c: conditionally avoid pruning objects via loose
builtin/repack.c: add cruft packs to MIDX during geometric repack
builtin/repack.c: use named flags for existing_packs
builtin/repack.c: allow configuring cruft pack generation
builtin/repack.c: support generating a cruft pack
builtin/pack-objects.c: --cruft with expiration
reachable: report precise timestamps from objects in cruft packs
reachable: add options to add_unseen_recent_objects_to_traversal
builtin/pack-objects.c: --cruft without expiration
builtin/pack-objects.c: return from create_object_entry()
t/helper: add 'pack-mtimes' test-tool
pack-mtimes: support writing pack .mtimes files
chunk-format.h: extract oid_version()
pack-write: pass 'struct packing_data' to 'stage_tmp_packfiles'
pack-mtimes: support reading .mtimes files
Documentation/technical: add cruft-packs.txt
Diffstat (limited to 'builtin/repack.c')
| -rw-r--r-- | builtin/repack.c | 185 |
1 files changed, 161 insertions, 24 deletions
diff --git a/builtin/repack.c b/builtin/repack.c index 0e4aae80c0..c957b2959f 100644 --- a/builtin/repack.c +++ b/builtin/repack.c @@ -18,12 +18,21 @@ #include "pack-bitmap.h" #include "refs.h" +#define ALL_INTO_ONE 1 +#define LOOSEN_UNREACHABLE 2 +#define PACK_CRUFT 4 + +#define DELETE_PACK 1 +#define CRUFT_PACK 2 + +static int pack_everything; static int delta_base_offset = 1; static int pack_kept_objects = -1; static int write_bitmaps = -1; static int use_delta_islands; static int run_update_server_info = 1; static char *packdir, *packtmp_name, *packtmp; +static char *cruft_expiration; static const char *const git_repack_usage[] = { N_("git repack [<options>]"), @@ -35,9 +44,21 @@ static const char incremental_bitmap_conflict_error[] = N_( "--no-write-bitmap-index or disable the pack.writebitmaps configuration." ); +struct pack_objects_args { + const char *window; + const char *window_memory; + const char *depth; + const char *threads; + const char *max_pack_size; + int no_reuse_delta; + int no_reuse_object; + int quiet; + int local; +}; static int repack_config(const char *var, const char *value, void *cb) { + struct pack_objects_args *cruft_po_args = cb; if (!strcmp(var, "repack.usedeltabaseoffset")) { delta_base_offset = git_config_bool(var, value); return 0; @@ -59,6 +80,14 @@ static int repack_config(const char *var, const char *value, void *cb) run_update_server_info = git_config_bool(var, value); return 0; } + if (!strcmp(var, "repack.cruftwindow")) + return git_config_string(&cruft_po_args->window, var, value); + if (!strcmp(var, "repack.cruftwindowmemory")) + return git_config_string(&cruft_po_args->window_memory, var, value); + if (!strcmp(var, "repack.cruftdepth")) + return git_config_string(&cruft_po_args->depth, var, value); + if (!strcmp(var, "repack.cruftthreads")) + return git_config_string(&cruft_po_args->threads, var, value); return git_default_config(var, value, cb); } @@ -131,10 +160,15 @@ static void collect_pack_filenames(struct string_list *fname_nonkept_list, fname = xmemdupz(e->d_name, len); if ((extra_keep->nr > 0 && i < extra_keep->nr) || - (file_exists(mkpath("%s/%s.keep", packdir, fname)))) + (file_exists(mkpath("%s/%s.keep", packdir, fname)))) { string_list_append_nodup(fname_kept_list, fname); - else - string_list_append_nodup(fname_nonkept_list, fname); + } else { + struct string_list_item *item; + item = string_list_append_nodup(fname_nonkept_list, + fname); + if (file_exists(mkpath("%s/%s.mtimes", packdir, fname))) + item->util = (void*)(uintptr_t)CRUFT_PACK; + } } closedir(dir); @@ -153,18 +187,6 @@ static void remove_redundant_pack(const char *dir_name, const char *base_name) strbuf_release(&buf); } -struct pack_objects_args { - const char *window; - const char *window_memory; - const char *depth; - const char *threads; - const char *max_pack_size; - int no_reuse_delta; - int no_reuse_object; - int quiet; - int local; -}; - static void prepare_pack_objects(struct child_process *cmd, const struct pack_objects_args *args) { @@ -219,6 +241,7 @@ static struct { } exts[] = { {".pack"}, {".rev", 1}, + {".mtimes", 1}, {".bitmap", 1}, {".promisor", 1}, {".idx"}, @@ -306,9 +329,6 @@ static void repack_promisor_objects(const struct pack_objects_args *args, die(_("could not finish pack-objects to repack promisor objects")); } -#define ALL_INTO_ONE 1 -#define LOOSEN_UNREACHABLE 2 - struct pack_geometry { struct packed_git **pack; uint32_t pack_nr, pack_alloc; @@ -366,6 +386,8 @@ static void init_pack_geometry(struct pack_geometry **geometry_p, if (string_list_has_string(existing_kept_packs, buf.buf)) continue; } + if (p->is_cruft) + continue; ALLOC_GROW(geometry->pack, geometry->pack_nr + 1, @@ -572,9 +594,20 @@ static void midx_included_packs(struct string_list *include, string_list_insert(include, strbuf_detach(&buf, NULL)); } + + for_each_string_list_item(item, existing_nonkept_packs) { + if (!((uintptr_t)item->util & CRUFT_PACK)) { + /* + * no need to check DELETE_PACK, since we're not + * doing an ALL_INTO_ONE repack + */ + continue; + } + string_list_insert(include, xstrfmt("%s.idx", item->string)); + } } else { for_each_string_list_item(item, existing_nonkept_packs) { - if (item->util) + if ((uintptr_t)item->util & DELETE_PACK) continue; string_list_insert(include, xstrfmt("%s.idx", item->string)); } @@ -628,6 +661,67 @@ static int write_midx_included_packs(struct string_list *include, return finish_command(&cmd); } +static int write_cruft_pack(const struct pack_objects_args *args, + const char *pack_prefix, + struct string_list *names, + struct string_list *existing_packs, + struct string_list *existing_kept_packs) +{ + struct child_process cmd = CHILD_PROCESS_INIT; + struct strbuf line = STRBUF_INIT; + struct string_list_item *item; + FILE *in, *out; + int ret; + + prepare_pack_objects(&cmd, args); + + strvec_push(&cmd.args, "--cruft"); + if (cruft_expiration) + strvec_pushf(&cmd.args, "--cruft-expiration=%s", + cruft_expiration); + + strvec_push(&cmd.args, "--honor-pack-keep"); + strvec_push(&cmd.args, "--non-empty"); + strvec_push(&cmd.args, "--max-pack-size=0"); + + cmd.in = -1; + + ret = start_command(&cmd); + if (ret) + return ret; + + /* + * names has a confusing double use: it both provides the list + * of just-written new packs, and accepts the name of the cruft + * pack we are writing. + * + * By the time it is read here, it contains only the pack(s) + * that were just written, which is exactly the set of packs we + * want to consider kept. + */ + in = xfdopen(cmd.in, "w"); + for_each_string_list_item(item, names) + fprintf(in, "%s-%s.pack\n", pack_prefix, item->string); + for_each_string_list_item(item, existing_packs) + fprintf(in, "-%s.pack\n", item->string); + for_each_string_list_item(item, existing_kept_packs) + fprintf(in, "%s.pack\n", item->string); + fclose(in); + + out = xfdopen(cmd.out, "r"); + while (strbuf_getline_lf(&line, out) != EOF) { + if (line.len != the_hash_algo->hexsz) + die(_("repack: Expecting full hex object ID lines only " + "from pack-objects.")); + string_list_append(names, line.buf); + } + fclose(out); + + strbuf_release(&line); + + return finish_command(&cmd); +} + int cmd_repack(int argc, const char **argv, const char *prefix) { struct child_process cmd = CHILD_PROCESS_INIT; @@ -644,12 +738,12 @@ int cmd_repack(int argc, const char **argv, const char *prefix) int show_progress; /* variables to be filled by option parsing */ - int pack_everything = 0; int delete_redundant = 0; const char *unpack_unreachable = NULL; int keep_unreachable = 0; struct string_list keep_pack_list = STRING_LIST_INIT_NODUP; struct pack_objects_args po_args = {NULL}; + struct pack_objects_args cruft_po_args = {NULL}; int geometric_factor = 0; int write_midx = 0; @@ -659,6 +753,11 @@ int cmd_repack(int argc, const char **argv, const char *prefix) OPT_BIT('A', NULL, &pack_everything, N_("same as -a, and turn unreachable objects loose"), LOOSEN_UNREACHABLE | ALL_INTO_ONE), + OPT_BIT(0, "cruft", &pack_everything, + N_("same as -a, pack unreachable cruft objects separately"), + PACK_CRUFT), + OPT_STRING(0, "cruft-expiration", &cruft_expiration, N_("approxidate"), + N_("with -C, expire objects older than this")), OPT_BOOL('d', NULL, &delete_redundant, N_("remove redundant packs, and run git-prune-packed")), OPT_BOOL('f', NULL, &po_args.no_reuse_delta, @@ -699,7 +798,7 @@ int cmd_repack(int argc, const char **argv, const char *prefix) OPT_END() }; - git_config(repack_config, NULL); + git_config(repack_config, &cruft_po_args); argc = parse_options(argc, argv, prefix, builtin_repack_options, git_repack_usage, 0); @@ -711,6 +810,15 @@ int cmd_repack(int argc, const char **argv, const char *prefix) (unpack_unreachable || (pack_everything & LOOSEN_UNREACHABLE))) die(_("options '%s' and '%s' cannot be used together"), "--keep-unreachable", "-A"); + if (pack_everything & PACK_CRUFT) { + pack_everything |= ALL_INTO_ONE; + + if (unpack_unreachable || (pack_everything & LOOSEN_UNREACHABLE)) + die(_("options '%s' and '%s' cannot be used together"), "--cruft", "-A"); + if (keep_unreachable) + die(_("options '%s' and '%s' cannot be used together"), "--cruft", "-k"); + } + if (write_bitmaps < 0) { if (!write_midx && (!(pack_everything & ALL_INTO_ONE) || !is_bare_repository())) @@ -794,7 +902,8 @@ int cmd_repack(int argc, const char **argv, const char *prefix) if (pack_everything & ALL_INTO_ONE) { repack_promisor_objects(&po_args, &names); - if (existing_nonkept_packs.nr && delete_redundant) { + if (existing_nonkept_packs.nr && delete_redundant && + !(pack_everything & PACK_CRUFT)) { for_each_string_list_item(item, &names) { strvec_pushf(&cmd.args, "--keep-pack=%s-%s.pack", packtmp_name, item->string); @@ -856,6 +965,33 @@ int cmd_repack(int argc, const char **argv, const char *prefix) if (!names.nr && !po_args.quiet) printf_ln(_("Nothing new to pack.")); + if (pack_everything & PACK_CRUFT) { + const char *pack_prefix; + if (!skip_prefix(packtmp, packdir, &pack_prefix)) + die(_("pack prefix %s does not begin with objdir %s"), + packtmp, packdir); + if (*pack_prefix == '/') + pack_prefix++; + + if (!cruft_po_args.window) + cruft_po_args.window = po_args.window; + if (!cruft_po_args.window_memory) + cruft_po_args.window_memory = po_args.window_memory; + if (!cruft_po_args.depth) + cruft_po_args.depth = po_args.depth; + if (!cruft_po_args.threads) + cruft_po_args.threads = po_args.threads; + + cruft_po_args.local = po_args.local; + cruft_po_args.quiet = po_args.quiet; + + ret = write_cruft_pack(&cruft_po_args, pack_prefix, &names, + &existing_nonkept_packs, + &existing_kept_packs); + if (ret) + return ret; + } + string_list_sort(&names); for_each_string_list_item(item, &names) { @@ -910,7 +1046,8 @@ int cmd_repack(int argc, const char **argv, const char *prefix) * was given) and that we will actually delete this pack * (if `-d` was given). */ - item->util = (void*)(intptr_t)!string_list_has_string(&names, sha1); + if (!string_list_has_string(&names, sha1)) + item->util = (void*)(uintptr_t)((size_t)item->util | DELETE_PACK); } } @@ -934,7 +1071,7 @@ int cmd_repack(int argc, const char **argv, const char *prefix) if (delete_redundant) { int opts = 0; for_each_string_list_item(item, &existing_nonkept_packs) { - if (!item->util) + if (!((uintptr_t)item->util & DELETE_PACK)) continue; remove_redundant_pack(packdir, item->string); } |