development/git
development/git
1
0
Fork 0
mirror of https://github.com/git/git synced 2024-07-02 15:48:44 +00:00
Code Issues Projects Releases Wiki Activity

Merge branch 'tb/repack-existing-packs-cleanup'

Browse Source 
The code to keep track of existing packs in the repository while
repacking has been refactored.

* tb/repack-existing-packs-cleanup:
  builtin/repack.c: extract common cruft pack loop
  builtin/repack.c: avoid directly inspecting "util"
  builtin/repack.c: store existing cruft packs separately
  builtin/repack.c: extract `has_existing_non_kept_packs()`
  builtin/repack.c: extract redundant pack cleanup for existing packs
  builtin/repack.c: extract redundant pack cleanup for --geometric
  builtin/repack.c: extract marking packs for deletion
  builtin/repack.c: extract structure to store existing packs
This commit is contained in:
Junio C Hamano 2023-09-22 17:01:36 -07:00
commit 5c0f9933ec
1 changed files with 180 additions and 113 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

293
builtin/repack.c
View File

@ -27,7 +27,6 @@
#define PACK_CRUFT 4
#define DELETE_PACK 1
#define CRUFT_PACK 2
static int pack_everything;
static int delta_base_offset = 1;
@ -95,14 +94,106 @@ static int repack_config(const char *var, const char *value,
return git_default_config(var, value, ctx, cb);
}
struct existing_packs {
struct string_list kept_packs;
struct string_list non_kept_packs;
struct string_list cruft_packs;
};
#define EXISTING_PACKS_INIT { \
.kept_packs = STRING_LIST_INIT_DUP, \
.non_kept_packs = STRING_LIST_INIT_DUP, \
.cruft_packs = STRING_LIST_INIT_DUP, \
}
static int has_existing_non_kept_packs(const struct existing_packs *existing)
{
return existing->non_kept_packs.nr || existing->cruft_packs.nr;
}
static void pack_mark_for_deletion(struct string_list_item *item)
{
item->util = (void*)((uintptr_t)item->util | DELETE_PACK);
}
static int pack_is_marked_for_deletion(struct string_list_item *item)
{
return (uintptr_t)item->util & DELETE_PACK;
}
static void mark_packs_for_deletion_1(struct string_list *names,
struct string_list *list)
{
struct string_list_item *item;
const int hexsz = the_hash_algo->hexsz;
for_each_string_list_item(item, list) {
char *sha1;
size_t len = strlen(item->string);
if (len < hexsz)
continue;
sha1 = item->string + len - hexsz;
/*
* Mark this pack for deletion, which ensures that this
* pack won't be included in a MIDX (if `--write-midx`
* was given) and that we will actually delete this pack
* (if `-d` was given).
*/
if (!string_list_has_string(names, sha1))
pack_mark_for_deletion(item);
}
}
static void mark_packs_for_deletion(struct existing_packs *existing,
struct string_list *names)
{
mark_packs_for_deletion_1(names, &existing->non_kept_packs);
mark_packs_for_deletion_1(names, &existing->cruft_packs);
}
static void remove_redundant_pack(const char *dir_name, const char *base_name)
{
struct strbuf buf = STRBUF_INIT;
struct multi_pack_index *m = get_local_multi_pack_index(the_repository);
strbuf_addf(&buf, "%s.pack", base_name);
if (m && midx_contains_pack(m, buf.buf))
clear_midx_file(the_repository);
strbuf_insertf(&buf, 0, "%s/", dir_name);
unlink_pack_path(buf.buf, 1);
strbuf_release(&buf);
}
static void remove_redundant_packs_1(struct string_list *packs)
{
struct string_list_item *item;
for_each_string_list_item(item, packs) {
if (!pack_is_marked_for_deletion(item))
continue;
remove_redundant_pack(packdir, item->string);
}
}
static void remove_redundant_existing_packs(struct existing_packs *existing)
{
remove_redundant_packs_1(&existing->non_kept_packs);
remove_redundant_packs_1(&existing->cruft_packs);
}
static void existing_packs_release(struct existing_packs *existing)
{
string_list_clear(&existing->kept_packs, 0);
string_list_clear(&existing->non_kept_packs, 0);
string_list_clear(&existing->cruft_packs, 0);
}
/*
* Adds all packs hex strings (pack-$HASH) to either fname_nonkept_list
* or fname_kept_list based on whether each pack has a corresponding
* Adds all packs hex strings (pack-$HASH) to either packs->non_kept
* or packs->kept based on whether each pack has a corresponding
* .keep file or not. Packs without a .keep file are not to be kept
* if we are going to pack everything into one file.
*/
static void collect_pack_filenames(struct string_list *fname_nonkept_list,
struct string_list *fname_kept_list,
static void collect_pack_filenames(struct existing_packs *existing,
const struct string_list *extra_keep)
{
struct packed_git *p;
@ -126,28 +217,14 @@ static void collect_pack_filenames(struct string_list *fname_nonkept_list,
strbuf_strip_suffix(&buf, ".pack");
if ((extra_keep->nr > 0 && i < extra_keep->nr) || p->pack_keep)
string_list_append(fname_kept_list, buf.buf);
else {
struct string_list_item *item;
item = string_list_append(fname_nonkept_list, buf.buf);
if (p->is_cruft)
item->util = (void*)(uintptr_t)CRUFT_PACK;
}
string_list_append(&existing->kept_packs, buf.buf);
else if (p->is_cruft)
string_list_append(&existing->cruft_packs, buf.buf);
else
string_list_append(&existing->non_kept_packs, buf.buf);
}
string_list_sort(fname_kept_list);
strbuf_release(&buf);
}
static void remove_redundant_pack(const char *dir_name, const char *base_name)
{
struct strbuf buf = STRBUF_INIT;
struct multi_pack_index *m = get_local_multi_pack_index(the_repository);
strbuf_addf(&buf, "%s.pack", base_name);
if (m && midx_contains_pack(m, buf.buf))
clear_midx_file(the_repository);
strbuf_insertf(&buf, 0, "%s/", dir_name);
unlink_pack_path(buf.buf, 1);
string_list_sort(&existing->kept_packs);
strbuf_release(&buf);
}
@ -327,7 +404,7 @@ static int geometry_cmp(const void *va, const void *vb)
}
static void init_pack_geometry(struct pack_geometry *geometry,
struct string_list *existing_kept_packs,
struct existing_packs *existing,
const struct pack_objects_args *args)
{
struct packed_git *p;
@ -344,23 +421,24 @@ static void init_pack_geometry(struct pack_geometry *geometry,
if (!pack_kept_objects) {
/*
* Any pack that has its pack_keep bit set will appear
* in existing_kept_packs below, but this saves us from
* doing a more expensive check.
* Any pack that has its pack_keep bit set will
* appear in existing->kept_packs below, but
* this saves us from doing a more expensive
* check.
*/
if (p->pack_keep)
continue;
/*
* The pack may be kept via the --keep-pack option;
* check 'existing_kept_packs' to determine whether to
* ignore it.
* The pack may be kept via the --keep-pack
* option; check 'existing->kept_packs' to
* determine whether to ignore it.
*/
strbuf_reset(&buf);
strbuf_addstr(&buf, pack_basename(p));
strbuf_strip_suffix(&buf, ".pack");
if (string_list_has_string(existing_kept_packs, buf.buf))
if (string_list_has_string(&existing->kept_packs, buf.buf))
continue;
}
if (p->is_cruft)
@ -494,6 +572,32 @@ static struct packed_git *get_preferred_pack(struct pack_geometry *geometry)
return NULL;
}
static void geometry_remove_redundant_packs(struct pack_geometry *geometry,
struct string_list *names,
struct existing_packs *existing)
{
struct strbuf buf = STRBUF_INIT;
uint32_t i;
for (i = 0; i < geometry->split; i++) {
struct packed_git *p = geometry->pack[i];
if (string_list_has_string(names, hash_to_hex(p->hash)))
continue;
strbuf_reset(&buf);
strbuf_addstr(&buf, pack_basename(p));
strbuf_strip_suffix(&buf, ".pack");
if ((p->pack_keep) ||
(string_list_has_string(&existing->kept_packs, buf.buf)))
continue;
remove_redundant_pack(packdir, buf.buf);
}
strbuf_release(&buf);
}
static void free_pack_geometry(struct pack_geometry *geometry)
{
if (!geometry)
@ -565,14 +669,13 @@ static void midx_snapshot_refs(struct tempfile *f)
}
static void midx_included_packs(struct string_list *include,
struct string_list *existing_nonkept_packs,
struct string_list *existing_kept_packs,
struct existing_packs *existing,
struct string_list *names,
struct pack_geometry *geometry)
{
struct string_list_item *item;
for_each_string_list_item(item, existing_kept_packs)
for_each_string_list_item(item, &existing->kept_packs)
string_list_insert(include, xstrfmt("%s.idx", item->string));
for_each_string_list_item(item, names)
string_list_insert(include, xstrfmt("pack-%s.idx", item->string));
@ -599,24 +702,32 @@ static void midx_included_packs(struct string_list *include,
string_list_insert(include, strbuf_detach(&buf, NULL));
}
for_each_string_list_item(item, existing_nonkept_packs) {
if (!((uintptr_t)item->util & CRUFT_PACK)) {
/*
* no need to check DELETE_PACK, since we're not
* doing an ALL_INTO_ONE repack
*/
continue;
}
string_list_insert(include, xstrfmt("%s.idx", item->string));
}
} else {
for_each_string_list_item(item, existing_nonkept_packs) {
if ((uintptr_t)item->util & DELETE_PACK)
for_each_string_list_item(item, &existing->non_kept_packs) {
if (pack_is_marked_for_deletion(item))
continue;
string_list_insert(include, xstrfmt("%s.idx", item->string));
}
}
for_each_string_list_item(item, &existing->cruft_packs) {
/*
* When doing a --geometric repack, there is no need to check
* for deleted packs, since we're by definition not doing an
* ALL_INTO_ONE repack (hence no packs will be deleted).
* Otherwise we must check for and exclude any packs which are
* enqueued for deletion.
*
* So we could omit the conditional below in the --geometric
* case, but doing so is unnecessary since no packs are marked
* as pending deletion (since we only call
* `mark_packs_for_deletion()` when doing an all-into-one
* repack).
*/
if (pack_is_marked_for_deletion(item))
continue;
string_list_insert(include, xstrfmt("%s.idx", item->string));
}
}
static int write_midx_included_packs(struct string_list *include,
@ -700,8 +811,7 @@ static int write_cruft_pack(const struct pack_objects_args *args,
const char *pack_prefix,
const char *cruft_expiration,
struct string_list *names,
struct string_list *existing_packs,
struct string_list *existing_kept_packs)
struct existing_packs *existing)
{
struct child_process cmd = CHILD_PROCESS_INIT;
struct strbuf line = STRBUF_INIT;
@ -743,9 +853,11 @@ static int write_cruft_pack(const struct pack_objects_args *args,
in = xfdopen(cmd.in, "w");
for_each_string_list_item(item, names)
fprintf(in, "%s-%s.pack\n", pack_prefix, item->string);
for_each_string_list_item(item, existing_packs)
for_each_string_list_item(item, &existing->non_kept_packs)
fprintf(in, "-%s.pack\n", item->string);
for_each_string_list_item(item, existing_kept_packs)
for_each_string_list_item(item, &existing->cruft_packs)
fprintf(in, "-%s.pack\n", item->string);
for_each_string_list_item(item, &existing->kept_packs)
fprintf(in, "%s.pack\n", item->string);
fclose(in);
@ -777,8 +889,7 @@ int cmd_repack(int argc, const char **argv, const char *prefix)
struct child_process cmd = CHILD_PROCESS_INIT;
struct string_list_item *item;
struct string_list names = STRING_LIST_INIT_DUP;
struct string_list existing_nonkept_packs = STRING_LIST_INIT_DUP;
struct string_list existing_kept_packs = STRING_LIST_INIT_DUP;
struct existing_packs existing = EXISTING_PACKS_INIT;
struct pack_geometry geometry = { 0 };
struct strbuf line = STRBUF_INIT;
struct tempfile *refs_snapshot = NULL;
@ -914,13 +1025,12 @@ int cmd_repack(int argc, const char **argv, const char *prefix)
packtmp_name = xstrfmt(".tmp-%d-pack", (int)getpid());
packtmp = mkpathdup("%s/%s", packdir, packtmp_name);
collect_pack_filenames(&existing_nonkept_packs, &existing_kept_packs,
&keep_pack_list);
collect_pack_filenames(&existing, &keep_pack_list);
if (geometry.split_factor) {
if (pack_everything)
die(_("options '%s' and '%s' cannot be used together"), "--geometric", "-A/-a");
init_pack_geometry(&geometry, &existing_kept_packs, &po_args);
init_pack_geometry(&geometry, &existing, &po_args);
split_pack_geometry(&geometry);
}
@ -964,7 +1074,8 @@ int cmd_repack(int argc, const char **argv, const char *prefix)
if (pack_everything & ALL_INTO_ONE) {
repack_promisor_objects(&po_args, &names);
if (existing_nonkept_packs.nr && delete_redundant &&
if (has_existing_non_kept_packs(&existing) &&
delete_redundant &&
!(pack_everything & PACK_CRUFT)) {
for_each_string_list_item(item, &names) {
strvec_pushf(&cmd.args, "--keep-pack=%s-%s.pack",
@ -1055,8 +1166,7 @@ int cmd_repack(int argc, const char **argv, const char *prefix)
ret = write_cruft_pack(&cruft_po_args, packtmp, pack_prefix,
cruft_expiration, &names,
&existing_nonkept_packs,
&existing_kept_packs);
&existing);
if (ret)
goto cleanup;
@ -1087,8 +1197,7 @@ int cmd_repack(int argc, const char **argv, const char *prefix)
pack_prefix,
NULL,
&names,
&existing_nonkept_packs,
&existing_kept_packs);
&existing);
if (ret)
goto cleanup;
}
@ -1132,29 +1241,12 @@ int cmd_repack(int argc, const char **argv, const char *prefix)
}
/* End of pack replacement. */
if (delete_redundant && pack_everything & ALL_INTO_ONE) {
const int hexsz = the_hash_algo->hexsz;
for_each_string_list_item(item, &existing_nonkept_packs) {
char *sha1;
size_t len = strlen(item->string);
if (len < hexsz)
continue;
sha1 = item->string + len - hexsz;
/*
* Mark this pack for deletion, which ensures that this
* pack won't be included in a MIDX (if `--write-midx`
* was given) and that we will actually delete this pack
* (if `-d` was given).
*/
if (!string_list_has_string(&names, sha1))
item->util = (void*)(uintptr_t)((size_t)item->util | DELETE_PACK);
}
}
if (delete_redundant && pack_everything & ALL_INTO_ONE)
mark_packs_for_deletion(&existing, &names);
if (write_midx) {
struct string_list include = STRING_LIST_INIT_NODUP;
midx_included_packs(&include, &existing_nonkept_packs,
&existing_kept_packs, &names, &geometry);
midx_included_packs(&include, &existing, &names, &geometry);
ret = write_midx_included_packs(&include, &geometry,
refs_snapshot ? get_tempfile_path(refs_snapshot) : NULL,
@ -1173,35 +1265,11 @@ int cmd_repack(int argc, const char **argv, const char *prefix)
if (delete_redundant) {
int opts = 0;
for_each_string_list_item(item, &existing_nonkept_packs) {
if (!((uintptr_t)item->util & DELETE_PACK))
continue;
remove_redundant_pack(packdir, item->string);
}
remove_redundant_existing_packs(&existing);
if (geometry.split_factor) {
struct strbuf buf = STRBUF_INIT;
uint32_t i;
for (i = 0; i < geometry.split; i++) {
struct packed_git *p = geometry.pack[i];
if (string_list_has_string(&names,
hash_to_hex(p->hash)))
continue;
strbuf_reset(&buf);
strbuf_addstr(&buf, pack_basename(p));
strbuf_strip_suffix(&buf, ".pack");
if ((p->pack_keep) ||
(string_list_has_string(&existing_kept_packs,
buf.buf)))
continue;
remove_redundant_pack(packdir, buf.buf);
}
strbuf_release(&buf);
}
if (geometry.split_factor)
geometry_remove_redundant_packs(&geometry, &names,
&existing);
if (show_progress)
opts |= PRUNE_PACKED_VERBOSE;
prune_packed_objects(opts);
@ -1225,8 +1293,7 @@ int cmd_repack(int argc, const char **argv, const char *prefix)
cleanup:
string_list_clear(&names, 1);
string_list_clear(&existing_nonkept_packs, 0);
string_list_clear(&existing_kept_packs, 0);
existing_packs_release(&existing);
free_pack_geometry(&geometry);
return ret;