mirror of
https://github.com/git/git
synced 2024-08-27 19:49:21 +00:00
Merge branch 'tb/midx-write-propagate-namehash'
"git multi-pack-index write --bitmap" learns to propagate the hashcache from original bitmap to resulting bitmap. * tb/midx-write-propagate-namehash: t5326: test propagating hashcache values p5326: generate pack bitmaps before writing the MIDX bitmap p5326: don't set core.multiPackIndex unnecessarily p5326: create missing 'perf-tag' tag midx.c: respect 'pack.writeBitmapHashcache' when writing bitmaps pack-bitmap.c: propagate namehash values from existing bitmaps t/helper/test-bitmap.c: add 'dump-hashes' mode
This commit is contained in:
commit
9567a670d2
|
@ -159,6 +159,10 @@ pack.writeBitmapHashCache::
|
||||||
between an older, bitmapped pack and objects that have been
|
between an older, bitmapped pack and objects that have been
|
||||||
pushed since the last gc). The downside is that it consumes 4
|
pushed since the last gc). The downside is that it consumes 4
|
||||||
bytes per object of disk space. Defaults to true.
|
bytes per object of disk space. Defaults to true.
|
||||||
|
+
|
||||||
|
When writing a multi-pack reachability bitmap, no new namehashes are
|
||||||
|
computed; instead, any namehashes stored in an existing bitmap are
|
||||||
|
permuted into their appropriate location when writing a new bitmap.
|
||||||
|
|
||||||
pack.writeReverseIndex::
|
pack.writeReverseIndex::
|
||||||
When true, git will write a corresponding .rev file (see:
|
When true, git will write a corresponding .rev file (see:
|
||||||
|
|
|
@ -60,6 +60,23 @@ static struct option *add_common_options(struct option *prev)
|
||||||
return parse_options_concat(common_opts, prev);
|
return parse_options_concat(common_opts, prev);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static int git_multi_pack_index_write_config(const char *var, const char *value,
|
||||||
|
void *cb)
|
||||||
|
{
|
||||||
|
if (!strcmp(var, "pack.writebitmaphashcache")) {
|
||||||
|
if (git_config_bool(var, value))
|
||||||
|
opts.flags |= MIDX_WRITE_BITMAP_HASH_CACHE;
|
||||||
|
else
|
||||||
|
opts.flags &= ~MIDX_WRITE_BITMAP_HASH_CACHE;
|
||||||
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
* We should never make a fall-back call to 'git_default_config', since
|
||||||
|
* this was already called in 'cmd_multi_pack_index()'.
|
||||||
|
*/
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
static int cmd_multi_pack_index_write(int argc, const char **argv)
|
static int cmd_multi_pack_index_write(int argc, const char **argv)
|
||||||
{
|
{
|
||||||
struct option *options;
|
struct option *options;
|
||||||
|
@ -74,6 +91,10 @@ static int cmd_multi_pack_index_write(int argc, const char **argv)
|
||||||
OPT_END(),
|
OPT_END(),
|
||||||
};
|
};
|
||||||
|
|
||||||
|
opts.flags |= MIDX_WRITE_BITMAP_HASH_CACHE;
|
||||||
|
|
||||||
|
git_config(git_multi_pack_index_write_config, NULL);
|
||||||
|
|
||||||
options = add_common_options(builtin_multi_pack_index_write_options);
|
options = add_common_options(builtin_multi_pack_index_write_options);
|
||||||
|
|
||||||
trace2_cmd_mode(argv[0]);
|
trace2_cmd_mode(argv[0]);
|
||||||
|
|
6
midx.c
6
midx.c
|
@ -993,9 +993,13 @@ static int write_midx_bitmap(char *midx_name, unsigned char *midx_hash,
|
||||||
struct pack_idx_entry **index;
|
struct pack_idx_entry **index;
|
||||||
struct commit **commits = NULL;
|
struct commit **commits = NULL;
|
||||||
uint32_t i, commits_nr;
|
uint32_t i, commits_nr;
|
||||||
|
uint16_t options = 0;
|
||||||
char *bitmap_name = xstrfmt("%s-%s.bitmap", midx_name, hash_to_hex(midx_hash));
|
char *bitmap_name = xstrfmt("%s-%s.bitmap", midx_name, hash_to_hex(midx_hash));
|
||||||
int ret;
|
int ret;
|
||||||
|
|
||||||
|
if (flags & MIDX_WRITE_BITMAP_HASH_CACHE)
|
||||||
|
options |= BITMAP_OPT_HASH_CACHE;
|
||||||
|
|
||||||
prepare_midx_packing_data(&pdata, ctx);
|
prepare_midx_packing_data(&pdata, ctx);
|
||||||
|
|
||||||
commits = find_commits_for_midx_bitmap(&commits_nr, ctx);
|
commits = find_commits_for_midx_bitmap(&commits_nr, ctx);
|
||||||
|
@ -1034,7 +1038,7 @@ static int write_midx_bitmap(char *midx_name, unsigned char *midx_hash,
|
||||||
goto cleanup;
|
goto cleanup;
|
||||||
|
|
||||||
bitmap_writer_set_checksum(midx_hash);
|
bitmap_writer_set_checksum(midx_hash);
|
||||||
bitmap_writer_finish(index, pdata.nr_objects, bitmap_name, 0);
|
bitmap_writer_finish(index, pdata.nr_objects, bitmap_name, options);
|
||||||
|
|
||||||
cleanup:
|
cleanup:
|
||||||
free(index);
|
free(index);
|
||||||
|
|
1
midx.h
1
midx.h
|
@ -44,6 +44,7 @@ struct multi_pack_index {
|
||||||
#define MIDX_PROGRESS (1 << 0)
|
#define MIDX_PROGRESS (1 << 0)
|
||||||
#define MIDX_WRITE_REV_INDEX (1 << 1)
|
#define MIDX_WRITE_REV_INDEX (1 << 1)
|
||||||
#define MIDX_WRITE_BITMAP (1 << 2)
|
#define MIDX_WRITE_BITMAP (1 << 2)
|
||||||
|
#define MIDX_WRITE_BITMAP_HASH_CACHE (1 << 3)
|
||||||
|
|
||||||
const unsigned char *get_midx_checksum(struct multi_pack_index *m);
|
const unsigned char *get_midx_checksum(struct multi_pack_index *m);
|
||||||
char *get_midx_filename(const char *object_dir);
|
char *get_midx_filename(const char *object_dir);
|
||||||
|
|
|
@ -1742,6 +1742,33 @@ int test_bitmap_commits(struct repository *r)
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
int test_bitmap_hashes(struct repository *r)
|
||||||
|
{
|
||||||
|
struct bitmap_index *bitmap_git = prepare_bitmap_git(r);
|
||||||
|
struct object_id oid;
|
||||||
|
uint32_t i, index_pos;
|
||||||
|
|
||||||
|
if (!bitmap_git->hashes)
|
||||||
|
goto cleanup;
|
||||||
|
|
||||||
|
for (i = 0; i < bitmap_num_objects(bitmap_git); i++) {
|
||||||
|
if (bitmap_is_midx(bitmap_git))
|
||||||
|
index_pos = pack_pos_to_midx(bitmap_git->midx, i);
|
||||||
|
else
|
||||||
|
index_pos = pack_pos_to_index(bitmap_git->pack, i);
|
||||||
|
|
||||||
|
nth_bitmap_object_oid(bitmap_git, &oid, index_pos);
|
||||||
|
|
||||||
|
printf("%s %"PRIu32"\n",
|
||||||
|
oid_to_hex(&oid), get_be32(bitmap_git->hashes + index_pos));
|
||||||
|
}
|
||||||
|
|
||||||
|
cleanup:
|
||||||
|
free_bitmap_index(bitmap_git);
|
||||||
|
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
int rebuild_bitmap(const uint32_t *reposition,
|
int rebuild_bitmap(const uint32_t *reposition,
|
||||||
struct ewah_bitmap *source,
|
struct ewah_bitmap *source,
|
||||||
struct bitmap *dest)
|
struct bitmap *dest)
|
||||||
|
@ -1791,18 +1818,20 @@ uint32_t *create_bitmap_mapping(struct bitmap_index *bitmap_git,
|
||||||
for (i = 0; i < num_objects; ++i) {
|
for (i = 0; i < num_objects; ++i) {
|
||||||
struct object_id oid;
|
struct object_id oid;
|
||||||
struct object_entry *oe;
|
struct object_entry *oe;
|
||||||
|
uint32_t index_pos;
|
||||||
|
|
||||||
if (bitmap_is_midx(bitmap_git))
|
if (bitmap_is_midx(bitmap_git))
|
||||||
nth_midxed_object_oid(&oid,
|
index_pos = pack_pos_to_midx(bitmap_git->midx, i);
|
||||||
bitmap_git->midx,
|
|
||||||
pack_pos_to_midx(bitmap_git->midx, i));
|
|
||||||
else
|
else
|
||||||
nth_packed_object_id(&oid, bitmap_git->pack,
|
index_pos = pack_pos_to_index(bitmap_git->pack, i);
|
||||||
pack_pos_to_index(bitmap_git->pack, i));
|
nth_bitmap_object_oid(bitmap_git, &oid, index_pos);
|
||||||
oe = packlist_find(mapping, &oid);
|
oe = packlist_find(mapping, &oid);
|
||||||
|
|
||||||
if (oe)
|
if (oe) {
|
||||||
reposition[i] = oe_in_pack_pos(mapping, oe) + 1;
|
reposition[i] = oe_in_pack_pos(mapping, oe) + 1;
|
||||||
|
if (bitmap_git->hashes && !oe->hash)
|
||||||
|
oe->hash = get_be32(bitmap_git->hashes + index_pos);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return reposition;
|
return reposition;
|
||||||
|
|
|
@ -52,6 +52,7 @@ void traverse_bitmap_commit_list(struct bitmap_index *,
|
||||||
show_reachable_fn show_reachable);
|
show_reachable_fn show_reachable);
|
||||||
void test_bitmap_walk(struct rev_info *revs);
|
void test_bitmap_walk(struct rev_info *revs);
|
||||||
int test_bitmap_commits(struct repository *r);
|
int test_bitmap_commits(struct repository *r);
|
||||||
|
int test_bitmap_hashes(struct repository *r);
|
||||||
struct bitmap_index *prepare_bitmap_walk(struct rev_info *revs,
|
struct bitmap_index *prepare_bitmap_walk(struct rev_info *revs,
|
||||||
struct list_objects_filter_options *filter,
|
struct list_objects_filter_options *filter,
|
||||||
int filter_provided_objects);
|
int filter_provided_objects);
|
||||||
|
|
|
@ -7,6 +7,11 @@ static int bitmap_list_commits(void)
|
||||||
return test_bitmap_commits(the_repository);
|
return test_bitmap_commits(the_repository);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static int bitmap_dump_hashes(void)
|
||||||
|
{
|
||||||
|
return test_bitmap_hashes(the_repository);
|
||||||
|
}
|
||||||
|
|
||||||
int cmd__bitmap(int argc, const char **argv)
|
int cmd__bitmap(int argc, const char **argv)
|
||||||
{
|
{
|
||||||
setup_git_directory();
|
setup_git_directory();
|
||||||
|
@ -16,9 +21,12 @@ int cmd__bitmap(int argc, const char **argv)
|
||||||
|
|
||||||
if (!strcmp(argv[1], "list-commits"))
|
if (!strcmp(argv[1], "list-commits"))
|
||||||
return bitmap_list_commits();
|
return bitmap_list_commits();
|
||||||
|
if (!strcmp(argv[1], "dump-hashes"))
|
||||||
|
return bitmap_dump_hashes();
|
||||||
|
|
||||||
usage:
|
usage:
|
||||||
usage("\ttest-tool bitmap list-commits");
|
usage("\ttest-tool bitmap list-commits\n"
|
||||||
|
"\ttest-tool bitmap dump-hashes");
|
||||||
|
|
||||||
return -1;
|
return -1;
|
||||||
}
|
}
|
||||||
|
|
|
@ -6,15 +6,24 @@ test_description='Tests performance using midx bitmaps'
|
||||||
|
|
||||||
test_perf_large_repo
|
test_perf_large_repo
|
||||||
|
|
||||||
test_expect_success 'enable multi-pack index' '
|
# we need to create the tag up front such that it is covered by the repack and
|
||||||
git config core.multiPackIndex true
|
# thus by generated bitmaps.
|
||||||
|
test_expect_success 'create tags' '
|
||||||
|
git tag --message="tag pointing to HEAD" perf-tag HEAD
|
||||||
|
'
|
||||||
|
|
||||||
|
test_expect_success 'start with bitmapped pack' '
|
||||||
|
git repack -adb
|
||||||
'
|
'
|
||||||
|
|
||||||
test_perf 'setup multi-pack index' '
|
test_perf 'setup multi-pack index' '
|
||||||
git repack -ad &&
|
|
||||||
git multi-pack-index write --bitmap
|
git multi-pack-index write --bitmap
|
||||||
'
|
'
|
||||||
|
|
||||||
|
test_expect_success 'drop pack bitmap' '
|
||||||
|
rm -f .git/objects/pack/pack-*.bitmap
|
||||||
|
'
|
||||||
|
|
||||||
test_full_bitmap
|
test_full_bitmap
|
||||||
|
|
||||||
test_expect_success 'create partial bitmap state' '
|
test_expect_success 'create partial bitmap state' '
|
||||||
|
|
|
@ -283,4 +283,34 @@ test_expect_success 'pack.preferBitmapTips' '
|
||||||
)
|
)
|
||||||
'
|
'
|
||||||
|
|
||||||
|
test_expect_success 'hash-cache values are propagated from pack bitmaps' '
|
||||||
|
rm -fr repo &&
|
||||||
|
git init repo &&
|
||||||
|
test_when_finished "rm -fr repo" &&
|
||||||
|
(
|
||||||
|
cd repo &&
|
||||||
|
|
||||||
|
test_commit base &&
|
||||||
|
test_commit base2 &&
|
||||||
|
git repack -adb &&
|
||||||
|
|
||||||
|
test-tool bitmap dump-hashes >pack.raw &&
|
||||||
|
test_file_not_empty pack.raw &&
|
||||||
|
sort pack.raw >pack.hashes &&
|
||||||
|
|
||||||
|
test_commit new &&
|
||||||
|
git repack &&
|
||||||
|
git multi-pack-index write --bitmap &&
|
||||||
|
|
||||||
|
test-tool bitmap dump-hashes >midx.raw &&
|
||||||
|
sort midx.raw >midx.hashes &&
|
||||||
|
|
||||||
|
# ensure that every namehash in the pack bitmap can be found in
|
||||||
|
# the midx bitmap (i.e., that there are no oid-namehash pairs
|
||||||
|
# unique to the pack bitmap).
|
||||||
|
comm -23 pack.hashes midx.hashes >dropped.hashes &&
|
||||||
|
test_must_be_empty dropped.hashes
|
||||||
|
)
|
||||||
|
'
|
||||||
|
|
||||||
test_done
|
test_done
|
||||||
|
|
Loading…
Reference in a new issue