Merge branch 'jx/bundle'

"git bundle" learns "--stdin" option to read its refs from the
standard input.  Also, it now does not lose refs whey they point
at the same object.

* jx/bundle:
  bundle: arguments can be read from stdin
  bundle: lost objects when removing duplicate pendings
  test: add helper functions for git-bundle
This commit is contained in:
Junio C Hamano 2021-01-25 14:19:19 -08:00
commit 8b48981987
6 changed files with 576 additions and 78 deletions

109
bundle.c
View file

@ -338,48 +338,6 @@ static int write_pack_data(int bundle_fd, struct rev_info *revs, struct strvec *
return 0;
}
static int compute_and_write_prerequisites(int bundle_fd,
struct rev_info *revs,
int argc, const char **argv)
{
struct child_process rls = CHILD_PROCESS_INIT;
struct strbuf buf = STRBUF_INIT;
FILE *rls_fout;
int i;
strvec_pushl(&rls.args,
"rev-list", "--boundary", "--pretty=oneline",
NULL);
for (i = 1; i < argc; i++)
strvec_push(&rls.args, argv[i]);
rls.out = -1;
rls.git_cmd = 1;
if (start_command(&rls))
return -1;
rls_fout = xfdopen(rls.out, "r");
while (strbuf_getwholeline(&buf, rls_fout, '\n') != EOF) {
struct object_id oid;
if (buf.len > 0 && buf.buf[0] == '-') {
write_or_die(bundle_fd, buf.buf, buf.len);
if (!get_oid_hex(buf.buf + 1, &oid)) {
struct object *object = parse_object_or_die(&oid,
buf.buf);
object->flags |= UNINTERESTING;
add_pending_object(revs, object, buf.buf);
}
} else if (!get_oid_hex(buf.buf, &oid)) {
struct object *object = parse_object_or_die(&oid,
buf.buf);
object->flags |= SHOWN;
}
}
strbuf_release(&buf);
fclose(rls_fout);
if (finish_command(&rls))
return error(_("rev-list died"));
return 0;
}
/*
* Write out bundle refs based on the tips already
* parsed into revs.pending. As a side effect, may
@ -474,6 +432,38 @@ static int write_bundle_refs(int bundle_fd, struct rev_info *revs)
return ref_count;
}
struct bundle_prerequisites_info {
struct object_array *pending;
int fd;
};
static void write_bundle_prerequisites(struct commit *commit, void *data)
{
struct bundle_prerequisites_info *bpi = data;
struct object *object;
struct pretty_print_context ctx = { 0 };
struct strbuf buf = STRBUF_INIT;
if (!(commit->object.flags & BOUNDARY))
return;
strbuf_addf(&buf, "-%s ", oid_to_hex(&commit->object.oid));
write_or_die(bpi->fd, buf.buf, buf.len);
ctx.fmt = CMIT_FMT_ONELINE;
ctx.output_encoding = get_log_output_encoding();
strbuf_reset(&buf);
pretty_print_commit(&ctx, commit, &buf);
strbuf_trim(&buf);
object = (struct object *)commit;
object->flags |= UNINTERESTING;
add_object_array_with_path(object, buf.buf, bpi->pending, S_IFINVALID,
NULL);
strbuf_addch(&buf, '\n');
write_or_die(bpi->fd, buf.buf, buf.len);
strbuf_release(&buf);
}
int create_bundle(struct repository *r, const char *path,
int argc, const char **argv, struct strvec *pack_options, int version)
{
@ -481,8 +471,10 @@ int create_bundle(struct repository *r, const char *path,
int bundle_fd = -1;
int bundle_to_stdout;
int ref_count = 0;
struct rev_info revs;
struct rev_info revs, revs_copy;
int min_version = the_hash_algo == &hash_algos[GIT_HASH_SHA1] ? 2 : 3;
struct bundle_prerequisites_info bpi;
int i;
bundle_to_stdout = !strcmp(path, "-");
if (bundle_to_stdout)
@ -512,10 +504,6 @@ int create_bundle(struct repository *r, const char *path,
save_commit_buffer = 0;
repo_init_revisions(r, &revs, NULL);
/* write prerequisites */
if (compute_and_write_prerequisites(bundle_fd, &revs, argc, argv))
goto err;
argc = setup_revisions(argc, argv, &revs, NULL);
if (argc > 1) {
@ -523,16 +511,37 @@ int create_bundle(struct repository *r, const char *path,
goto err;
}
object_array_remove_duplicates(&revs.pending);
/* save revs.pending in revs_copy for later use */
memcpy(&revs_copy, &revs, sizeof(revs));
revs_copy.pending.nr = 0;
revs_copy.pending.alloc = 0;
revs_copy.pending.objects = NULL;
for (i = 0; i < revs.pending.nr; i++) {
struct object_array_entry *e = revs.pending.objects + i;
if (e)
add_object_array_with_path(e->item, e->name,
&revs_copy.pending,
e->mode, e->path);
}
ref_count = write_bundle_refs(bundle_fd, &revs);
/* write prerequisites */
revs.boundary = 1;
if (prepare_revision_walk(&revs))
die("revision walk setup failed");
bpi.fd = bundle_fd;
bpi.pending = &revs_copy.pending;
traverse_commit_list(&revs, write_bundle_prerequisites, NULL, &bpi);
object_array_remove_duplicates(&revs_copy.pending);
/* write bundle refs */
ref_count = write_bundle_refs(bundle_fd, &revs_copy);
if (!ref_count)
die(_("Refusing to create empty bundle."));
else if (ref_count < 0)
goto err;
/* write pack */
if (write_pack_data(bundle_fd, &revs, pack_options))
if (write_pack_data(bundle_fd, &revs_copy, pack_options))
goto err;
if (!bundle_to_stdout) {

View file

@ -412,15 +412,16 @@ void object_array_clear(struct object_array *array)
}
/*
* Return true iff array already contains an entry with name.
* Return true if array already contains an entry.
*/
static int contains_name(struct object_array *array, const char *name)
static int contains_object(struct object_array *array,
const struct object *item, const char *name)
{
unsigned nr = array->nr, i;
struct object_array_entry *object = array->objects;
for (i = 0; i < nr; i++, object++)
if (!strcmp(object->name, name))
if (item == object->item && !strcmp(object->name, name))
return 1;
return 0;
}
@ -432,7 +433,8 @@ void object_array_remove_duplicates(struct object_array *array)
array->nr = 0;
for (src = 0; src < nr; src++) {
if (!contains_name(array, objects[src].name)) {
if (!contains_object(array, objects[src].item,
objects[src].name)) {
if (src != array->nr)
objects[array->nr] = objects[src];
array->nr++;

View file

@ -9,22 +9,10 @@ GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=main
export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME
. ./test-lib.sh
. "$TEST_DIRECTORY"/test-bundle-functions.sh
D=$(pwd)
test_bundle_object_count () {
git verify-pack -v "$1" >verify.out &&
test "$2" = $(grep "^$OID_REGEX " verify.out | wc -l)
}
convert_bundle_to_pack () {
while read x && test -n "$x"
do
:;
done
cat
}
test_expect_success setup '
echo >file original &&
git add file &&
@ -483,9 +471,7 @@ test_expect_success 'unbundle 1' '
test_expect_success 'bundle 1 has only 3 files ' '
cd "$D" &&
convert_bundle_to_pack <bundle1 >bundle.pack &&
git index-pack bundle.pack &&
test_bundle_object_count bundle.pack 3
test_bundle_object_count bundle1 3
'
test_expect_success 'unbundle 2' '
@ -500,9 +486,7 @@ test_expect_success 'bundle does not prerequisite objects' '
git add file2 &&
git commit -m add.file2 file2 &&
git bundle create bundle3 -1 HEAD &&
convert_bundle_to_pack <bundle3 >bundle.pack &&
git index-pack bundle.pack &&
test_bundle_object_count bundle.pack 3
test_bundle_object_count bundle3 3
'
test_expect_success 'bundle should be able to create a full history' '
@ -1055,9 +1039,7 @@ test_expect_success 'all boundary commits are excluded' '
git merge otherside &&
ad=$(git log --no-walk --format=%ad HEAD) &&
git bundle create twoside-boundary.bdl main --since="$ad" &&
convert_bundle_to_pack <twoside-boundary.bdl >twoside-boundary.pack &&
pack=$(git index-pack --fix-thin --stdin <twoside-boundary.pack) &&
test_bundle_object_count .git/objects/pack/pack-${pack##pack }.pack 3
test_bundle_object_count --thin twoside-boundary.bdl 3
'
test_expect_success 'fetch --prune prints the remotes url' '

View file

@ -41,13 +41,13 @@ test_expect_success 'die if bundle file cannot be created' '
test_must_fail git bundle create adir --all
'
test_expect_failure 'bundle --stdin' '
test_expect_success 'bundle --stdin' '
echo main | git bundle create stdin-bundle.bdl --stdin &&
git ls-remote stdin-bundle.bdl >output &&
grep main output
'
test_expect_failure 'bundle --stdin <rev-list options>' '
test_expect_success 'bundle --stdin <rev-list options>' '
echo main | git bundle create hybrid-bundle.bdl --stdin tag &&
git ls-remote hybrid-bundle.bdl >output &&
grep main output

463
t/t6020-bundle-misc.sh Executable file
View file

@ -0,0 +1,463 @@
#!/bin/sh
#
# Copyright (c) 2021 Jiang Xin
#
test_description='Test git-bundle'
GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=main
export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME
. ./test-lib.sh
. "$TEST_DIRECTORY"/test-bundle-functions.sh
# Create a commit or tag and set the variable with the object ID.
test_commit_setvar () {
notick=
signoff=
indir=
merge=
tag=
var=
while test $# != 0
do
case "$1" in
--merge)
merge=t
;;
--tag)
tag=t
;;
--notick)
notick=t
;;
--signoff)
signoff="$1"
;;
-C)
shift
indir="$1"
;;
-*)
echo >&2 "error: unknown option $1"
return 1
;;
*)
break
;;
esac
shift
done
if test $# -lt 2
then
echo >&2 "error: test_commit_setvar must have at least 2 arguments"
return 1
fi
var=$1
shift
indir=${indir:+"$indir"/}
if test -z "$notick"
then
test_tick
fi &&
if test -n "$merge"
then
git ${indir:+ -C "$indir"} merge --no-edit --no-ff \
${2:+-m "$2"} "$1" &&
oid=$(git ${indir:+ -C "$indir"} rev-parse HEAD)
elif test -n "$tag"
then
git ${indir:+ -C "$indir"} tag -m "$1" "$1" "${2:-HEAD}" &&
oid=$(git ${indir:+ -C "$indir"} rev-parse "$1")
else
file=${2:-"$1.t"} &&
echo "${3-$1}" >"$indir$file" &&
git ${indir:+ -C "$indir"} add "$file" &&
git ${indir:+ -C "$indir"} commit $signoff -m "$1" &&
oid=$(git ${indir:+ -C "$indir"} rev-parse HEAD)
fi &&
eval $var=$oid
}
# Format the output of git commands to make a user-friendly and stable
# text. We can easily prepare the expect text without having to worry
# about future changes of the commit ID and spaces of the output.
make_user_friendly_and_stable_output () {
sed \
-e "s/${A%${A#???????}}[0-9a-f]*/<COMMIT-A>/g" \
-e "s/${B%${B#???????}}[0-9a-f]*/<COMMIT-B>/g" \
-e "s/${C%${C#???????}}[0-9a-f]*/<COMMIT-C>/g" \
-e "s/${D%${D#???????}}[0-9a-f]*/<COMMIT-D>/g" \
-e "s/${E%${E#???????}}[0-9a-f]*/<COMMIT-E>/g" \
-e "s/${F%${F#???????}}[0-9a-f]*/<COMMIT-F>/g" \
-e "s/${G%${G#???????}}[0-9a-f]*/<COMMIT-G>/g" \
-e "s/${H%${H#???????}}[0-9a-f]*/<COMMIT-H>/g" \
-e "s/${I%${I#???????}}[0-9a-f]*/<COMMIT-I>/g" \
-e "s/${J%${J#???????}}[0-9a-f]*/<COMMIT-J>/g" \
-e "s/${K%${K#???????}}[0-9a-f]*/<COMMIT-K>/g" \
-e "s/${L%${L#???????}}[0-9a-f]*/<COMMIT-L>/g" \
-e "s/${M%${M#???????}}[0-9a-f]*/<COMMIT-M>/g" \
-e "s/${N%${N#???????}}[0-9a-f]*/<COMMIT-N>/g" \
-e "s/${O%${O#???????}}[0-9a-f]*/<COMMIT-O>/g" \
-e "s/${P%${P#???????}}[0-9a-f]*/<COMMIT-P>/g" \
-e "s/${TAG1%${TAG1#???????}}[0-9a-f]*/<TAG-1>/g" \
-e "s/${TAG2%${TAG2#???????}}[0-9a-f]*/<TAG-2>/g" \
-e "s/${TAG3%${TAG3#???????}}[0-9a-f]*/<TAG-3>/g" \
-e "s/ *\$//"
}
# (C) (D, pull/1/head, topic/1)
# o --- o
# / \ (L)
# / \ o (H, topic/2) (M, tag:v2)
# / (F) \ / (N, tag:v3)
# / o --------- o (G, pull/2/head) o --- o --- o (release)
# / / \ \ / \
# o --- o --- o -------- o -- o ------------------ o ------- o --- o (main)
# (A) (B) (E, tag:v1) (I) (J) (K) (O) (P)
#
test_expect_success 'setup' '
# Try to make a stable fixed width for abbreviated commit ID,
# this fixed-width oid will be replaced with "<OID>".
git config core.abbrev 7 &&
# branch main: commit A & B
test_commit_setvar A "Commit A" main.txt &&
test_commit_setvar B "Commit B" main.txt &&
# branch topic/1: commit C & D, refs/pull/1/head
git checkout -b topic/1 &&
test_commit_setvar C "Commit C" topic-1.txt &&
test_commit_setvar D "Commit D" topic-1.txt &&
git update-ref refs/pull/1/head HEAD &&
# branch topic/1: commit E, tag v1
git checkout main &&
test_commit_setvar E "Commit E" main.txt &&
test_commit_setvar --tag TAG1 v1 &&
# branch topic/2: commit F & G, refs/pull/2/head
git checkout -b topic/2 &&
test_commit_setvar F "Commit F" topic-2.txt &&
test_commit_setvar G "Commit G" topic-2.txt &&
git update-ref refs/pull/2/head HEAD &&
test_commit_setvar H "Commit H" topic-2.txt &&
# branch main: merge commit I & J
git checkout main &&
test_commit_setvar --merge I topic/1 "Merge commit I" &&
test_commit_setvar --merge J refs/pull/2/head "Merge commit J" &&
# branch main: commit K
git checkout main &&
test_commit_setvar K "Commit K" main.txt &&
# branch release:
git checkout -b release &&
test_commit_setvar L "Commit L" release.txt &&
test_commit_setvar M "Commit M" release.txt &&
test_commit_setvar --tag TAG2 v2 &&
test_commit_setvar N "Commit N" release.txt &&
test_commit_setvar --tag TAG3 v3 &&
# branch main: merge commit O, commit P
git checkout main &&
test_commit_setvar --merge O tags/v2 "Merge commit O" &&
test_commit_setvar P "Commit P" main.txt
'
test_expect_success 'create bundle from special rev: main^!' '
git bundle create special-rev.bdl "main^!" &&
git bundle list-heads special-rev.bdl |
make_user_friendly_and_stable_output >actual &&
cat >expect <<-\EOF &&
<COMMIT-P> refs/heads/main
EOF
test_i18ncmp expect actual &&
git bundle verify special-rev.bdl |
make_user_friendly_and_stable_output >actual &&
cat >expect <<-\EOF &&
The bundle contains this ref:
<COMMIT-P> refs/heads/main
The bundle requires this ref:
<COMMIT-O>
EOF
test_i18ncmp expect actual &&
test_bundle_object_count special-rev.bdl 3
'
test_expect_success 'create bundle with --max-count option' '
git bundle create max-count.bdl --max-count 1 \
main \
"^release" \
refs/tags/v1 \
refs/pull/1/head \
refs/pull/2/head &&
git bundle verify max-count.bdl |
make_user_friendly_and_stable_output >actual &&
cat >expect <<-\EOF &&
The bundle contains these 2 refs:
<COMMIT-P> refs/heads/main
<TAG-1> refs/tags/v1
The bundle requires this ref:
<COMMIT-O>
EOF
test_i18ncmp expect actual &&
test_bundle_object_count max-count.bdl 4
'
test_expect_success 'create bundle with --since option' '
git log -1 --pretty="%ad" $M >actual &&
cat >expect <<-\EOF &&
Thu Apr 7 15:26:13 2005 -0700
EOF
test_cmp expect actual &&
git bundle create since.bdl \
--since "Thu Apr 7 15:27:00 2005 -0700" \
--all &&
git bundle verify since.bdl |
make_user_friendly_and_stable_output >actual &&
cat >expect <<-\EOF &&
The bundle contains these 5 refs:
<COMMIT-P> refs/heads/main
<COMMIT-N> refs/heads/release
<TAG-2> refs/tags/v2
<TAG-3> refs/tags/v3
<COMMIT-P> HEAD
The bundle requires these 2 refs:
<COMMIT-M>
<COMMIT-K>
EOF
test_i18ncmp expect actual &&
test_bundle_object_count --thin since.bdl 13
'
test_expect_success 'create bundle 1 - no prerequisites' '
# create bundle from args
git bundle create 1.bdl topic/1 topic/2 &&
# create bundle from stdin
cat >input <<-\EOF &&
topic/1
topic/2
EOF
git bundle create stdin-1.bdl --stdin <input &&
cat >expect <<-\EOF &&
The bundle contains these 2 refs:
<COMMIT-D> refs/heads/topic/1
<COMMIT-H> refs/heads/topic/2
The bundle records a complete history.
EOF
# verify bundle, which has no prerequisites
git bundle verify 1.bdl |
make_user_friendly_and_stable_output >actual &&
test_i18ncmp expect actual &&
git bundle verify stdin-1.bdl |
make_user_friendly_and_stable_output >actual &&
test_i18ncmp expect actual &&
test_bundle_object_count 1.bdl 24 &&
test_bundle_object_count stdin-1.bdl 24
'
test_expect_success 'create bundle 2 - has prerequisites' '
# create bundle from args
git bundle create 2.bdl \
--ignore-missing \
^topic/deleted \
^$D \
^topic/2 \
release &&
# create bundle from stdin
# input has a non-exist reference: "topic/deleted"
cat >input <<-EOF &&
^topic/deleted
^$D
^topic/2
EOF
git bundle create stdin-2.bdl \
--ignore-missing \
--stdin \
release <input &&
cat >expect <<-\EOF &&
The bundle contains this ref:
<COMMIT-N> refs/heads/release
The bundle requires these 3 refs:
<COMMIT-D>
<COMMIT-E>
<COMMIT-G>
EOF
git bundle verify 2.bdl |
make_user_friendly_and_stable_output >actual &&
test_i18ncmp expect actual &&
git bundle verify stdin-2.bdl |
make_user_friendly_and_stable_output >actual &&
test_i18ncmp expect actual &&
test_bundle_object_count 2.bdl 16 &&
test_bundle_object_count stdin-2.bdl 16
'
test_expect_success 'fail to verify bundle without prerequisites' '
git init --bare test1.git &&
cat >expect <<-\EOF &&
error: Repository lacks these prerequisite commits:
error: <COMMIT-D>
error: <COMMIT-E>
error: <COMMIT-G>
EOF
test_must_fail git -C test1.git bundle verify ../2.bdl 2>&1 |
make_user_friendly_and_stable_output >actual &&
test_i18ncmp expect actual &&
test_must_fail git -C test1.git bundle verify ../stdin-2.bdl 2>&1 |
make_user_friendly_and_stable_output >actual &&
test_i18ncmp expect actual
'
test_expect_success 'create bundle 3 - two refs, same object' '
# create bundle from args
git bundle create --version=3 3.bdl \
^release \
^topic/1 \
^topic/2 \
main \
HEAD &&
# create bundle from stdin
cat >input <<-\EOF &&
^release
^topic/1
^topic/2
EOF
git bundle create --version=3 stdin-3.bdl \
--stdin \
main HEAD <input &&
cat >expect <<-\EOF &&
The bundle contains these 2 refs:
<COMMIT-P> refs/heads/main
<COMMIT-P> HEAD
The bundle requires these 2 refs:
<COMMIT-M>
<COMMIT-K>
EOF
git bundle verify 3.bdl |
make_user_friendly_and_stable_output >actual &&
test_i18ncmp expect actual &&
git bundle verify stdin-3.bdl |
make_user_friendly_and_stable_output >actual &&
test_i18ncmp expect actual &&
test_bundle_object_count 3.bdl 4 &&
test_bundle_object_count stdin-3.bdl 4
'
test_expect_success 'create bundle 4 - with tags' '
# create bundle from args
git bundle create 4.bdl \
^main \
^release \
^topic/1 \
^topic/2 \
--all &&
# create bundle from stdin
cat >input <<-\EOF &&
^main
^release
^topic/1
^topic/2
EOF
git bundle create stdin-4.bdl \
--ignore-missing \
--stdin \
--all <input &&
cat >expect <<-\EOF &&
The bundle contains these 3 refs:
<TAG-1> refs/tags/v1
<TAG-2> refs/tags/v2
<TAG-3> refs/tags/v3
The bundle records a complete history.
EOF
git bundle verify 4.bdl |
make_user_friendly_and_stable_output >actual &&
test_i18ncmp expect actual &&
git bundle verify stdin-4.bdl |
make_user_friendly_and_stable_output >actual &&
test_i18ncmp expect actual &&
test_bundle_object_count 4.bdl 3 &&
test_bundle_object_count stdin-4.bdl 3
'
test_expect_success 'clone from bundle' '
git clone --mirror 1.bdl mirror.git &&
git -C mirror.git show-ref |
make_user_friendly_and_stable_output >actual &&
cat >expect <<-\EOF &&
<COMMIT-D> refs/heads/topic/1
<COMMIT-H> refs/heads/topic/2
EOF
test_cmp expect actual &&
git -C mirror.git fetch ../2.bdl "+refs/*:refs/*" &&
git -C mirror.git show-ref |
make_user_friendly_and_stable_output >actual &&
cat >expect <<-\EOF &&
<COMMIT-N> refs/heads/release
<COMMIT-D> refs/heads/topic/1
<COMMIT-H> refs/heads/topic/2
EOF
test_cmp expect actual &&
git -C mirror.git fetch ../3.bdl "+refs/*:refs/*" &&
git -C mirror.git show-ref |
make_user_friendly_and_stable_output >actual &&
cat >expect <<-\EOF &&
<COMMIT-P> refs/heads/main
<COMMIT-N> refs/heads/release
<COMMIT-D> refs/heads/topic/1
<COMMIT-H> refs/heads/topic/2
EOF
test_cmp expect actual &&
git -C mirror.git fetch ../4.bdl "+refs/*:refs/*" &&
git -C mirror.git show-ref |
make_user_friendly_and_stable_output >actual &&
cat >expect <<-\EOF &&
<COMMIT-P> refs/heads/main
<COMMIT-N> refs/heads/release
<COMMIT-D> refs/heads/topic/1
<COMMIT-H> refs/heads/topic/2
<TAG-1> refs/tags/v1
<TAG-2> refs/tags/v2
<TAG-3> refs/tags/v3
EOF
test_cmp expect actual
'
test_done

View file

@ -0,0 +1,42 @@
# Library of git-bundle related functions.
# Display the pack data contained in the bundle file, bypassing the
# header that contains the signature, prerequisites and references.
convert_bundle_to_pack () {
while read x && test -n "$x"
do
:;
done
cat
}
# Check count of objects in a bundle file.
# We can use "--thin" opiton to check thin pack, which must be fixed by
# command `git-index-pack --fix-thin --stdin`.
test_bundle_object_count () {
thin=
if test "$1" = "--thin"
then
thin=t
shift
fi
if test $# -ne 2
then
echo >&2 "args should be: <bundle> <count>"
return 1
fi
bundle=$1
pack=$bundle.pack
convert_bundle_to_pack <"$bundle" >"$pack" &&
if test -n "$thin"
then
mv "$pack" "$bundle.thin.pack" &&
git index-pack --stdin --fix-thin "$pack" <"$bundle.thin.pack"
else
git index-pack "$pack"
fi || return 1
count=$(git show-index <"${pack%pack}idx" | wc -l) &&
test $2 = $count && return 0
echo >&2 "error: object count for $bundle is $count, not $2"
return 1
}