git/t/t1092-sparse-checkout-compatibility.sh
Derrick Stolee daa1acefc5 commit: integrate with sparse-index
Update 'git commit' to allow using the sparse-index in memory without
expanding to a full one. The only place that had an ensure_full_index()
call was in cache_tree_update(). The recursive algorithm for
update_one() was already updated in 2de37c536 (cache-tree: integrate
with sparse directory entries, 2021-03-03) to handle sparse directory
entries in the index.

Most of this change involves testing different command-line options that
allow specifying which on-disk changes should be included in the commit.
This includes no options (only take currently-staged changes), -a (take
all tracked changes), and --include (take a list of specific changes).
To simplify testing that these options do not expand the index, update
the test that previously verified that 'git status' does not expand the
index with a helper method, ensure_not_expanded().

This allows 'git commit' to operate much faster when the sparse-checkout
cone is much smaller than the full list of files at HEAD.

Here are the relevant lines from p2000-sparse-operations.sh:

Test                                      HEAD~1           HEAD
----------------------------------------------------------------------------------
2000.14: git commit -a -m A (full-v3)     0.35(0.26+0.06)  0.36(0.28+0.07) +2.9%
2000.15: git commit -a -m A (full-v4)     0.32(0.26+0.05)  0.34(0.28+0.06) +6.3%
2000.16: git commit -a -m A (sparse-v3)   0.63(0.59+0.06)  0.04(0.05+0.05) -93.7%
2000.17: git commit -a -m A (sparse-v4)   0.64(0.59+0.08)  0.04(0.04+0.04) -93.8%

It is important to compare the full-index case to the sparse-index case,
so the improvement for index version v4 is actually an 88% improvement in
this synthetic example.

In a real repository with over two million files at HEAD and 60,000
files in the sparse-checkout definition, the time for 'git commit -a'
went from 2.61 seconds to 134ms. I compared this to the result if the
index only contained the paths in the sparse-checkout definition and
found the theoretical optimum to be 120ms, so the out-of-cone paths only
add a 12% overhead.

Signed-off-by: Derrick Stolee <dstolee@microsoft.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2021-07-14 15:05:53 -07:00

601 lines
17 KiB
Bash
Executable file

#!/bin/sh
test_description='compare full workdir to sparse workdir'
GIT_TEST_SPLIT_INDEX=0
GIT_TEST_SPARSE_INDEX=
. ./test-lib.sh
test_expect_success 'setup' '
git init initial-repo &&
(
GIT_TEST_SPARSE_INDEX=0 &&
cd initial-repo &&
echo a >a &&
echo "after deep" >e &&
echo "after folder1" >g &&
echo "after x" >z &&
mkdir folder1 folder2 deep x &&
mkdir deep/deeper1 deep/deeper2 deep/before deep/later &&
mkdir deep/deeper1/deepest &&
echo "after deeper1" >deep/e &&
echo "after deepest" >deep/deeper1/e &&
cp a folder1 &&
cp a folder2 &&
cp a x &&
cp a deep &&
cp a deep/before &&
cp a deep/deeper1 &&
cp a deep/deeper2 &&
cp a deep/later &&
cp a deep/deeper1/deepest &&
cp -r deep/deeper1/deepest deep/deeper2 &&
mkdir deep/deeper1/0 &&
mkdir deep/deeper1/0/0 &&
touch deep/deeper1/0/1 &&
touch deep/deeper1/0/0/0 &&
>folder1- &&
>folder1.x &&
>folder10 &&
cp -r deep/deeper1/0 folder1 &&
cp -r deep/deeper1/0 folder2 &&
echo >>folder1/0/0/0 &&
echo >>folder2/0/1 &&
git add . &&
git commit -m "initial commit" &&
git checkout -b base &&
for dir in folder1 folder2 deep
do
git checkout -b update-$dir &&
echo "updated $dir" >$dir/a &&
git commit -a -m "update $dir" || return 1
done &&
git checkout -b rename-base base &&
cat >folder1/larger-content <<-\EOF &&
matching
lines
help
inexact
renames
EOF
cp folder1/larger-content folder2/ &&
cp folder1/larger-content deep/deeper1/ &&
git add . &&
git commit -m "add interesting rename content" &&
git checkout -b rename-out-to-out rename-base &&
mv folder1/a folder2/b &&
mv folder1/larger-content folder2/edited-content &&
echo >>folder2/edited-content &&
echo >>folder2/0/1 &&
echo stuff >>deep/deeper1/a &&
git add . &&
git commit -m "rename folder1/... to folder2/..." &&
git checkout -b rename-out-to-in rename-base &&
mv folder1/a deep/deeper1/b &&
echo more stuff >>deep/deeper1/a &&
rm folder2/0/1 &&
mkdir folder2/0/1 &&
echo >>folder2/0/1/1 &&
mv folder1/larger-content deep/deeper1/edited-content &&
echo >>deep/deeper1/edited-content &&
git add . &&
git commit -m "rename folder1/... to deep/deeper1/..." &&
git checkout -b rename-in-to-out rename-base &&
mv deep/deeper1/a folder1/b &&
echo >>folder2/0/1 &&
rm -rf folder1/0/0 &&
echo >>folder1/0/0 &&
mv deep/deeper1/larger-content folder1/edited-content &&
echo >>folder1/edited-content &&
git add . &&
git commit -m "rename deep/deeper1/... to folder1/..." &&
git checkout -b deepest base &&
echo "updated deepest" >deep/deeper1/deepest/a &&
git commit -a -m "update deepest" &&
git checkout -f base &&
git reset --hard
)
'
init_repos () {
rm -rf full-checkout sparse-checkout sparse-index &&
# create repos in initial state
cp -r initial-repo full-checkout &&
git -C full-checkout reset --hard &&
cp -r initial-repo sparse-checkout &&
git -C sparse-checkout reset --hard &&
cp -r initial-repo sparse-index &&
git -C sparse-index reset --hard &&
# initialize sparse-checkout definitions
git -C sparse-checkout sparse-checkout init --cone &&
git -C sparse-checkout sparse-checkout set deep &&
git -C sparse-index sparse-checkout init --cone --sparse-index &&
test_cmp_config -C sparse-index true index.sparse &&
git -C sparse-index sparse-checkout set deep
}
run_on_sparse () {
(
cd sparse-checkout &&
GIT_PROGRESS_DELAY=100000 "$@" >../sparse-checkout-out 2>../sparse-checkout-err
) &&
(
cd sparse-index &&
GIT_PROGRESS_DELAY=100000 "$@" >../sparse-index-out 2>../sparse-index-err
)
}
run_on_all () {
(
cd full-checkout &&
GIT_PROGRESS_DELAY=100000 "$@" >../full-checkout-out 2>../full-checkout-err
) &&
run_on_sparse "$@"
}
test_all_match () {
run_on_all "$@" &&
test_cmp full-checkout-out sparse-checkout-out &&
test_cmp full-checkout-out sparse-index-out &&
test_cmp full-checkout-err sparse-checkout-err &&
test_cmp full-checkout-err sparse-index-err
}
test_sparse_match () {
run_on_sparse "$@" &&
test_cmp sparse-checkout-out sparse-index-out &&
test_cmp sparse-checkout-err sparse-index-err
}
test_expect_success 'sparse-index contents' '
init_repos &&
test-tool -C sparse-index read-cache --table >cache &&
for dir in folder1 folder2 x
do
TREE=$(git -C sparse-index rev-parse HEAD:$dir) &&
grep "040000 tree $TREE $dir/" cache \
|| return 1
done &&
git -C sparse-index sparse-checkout set folder1 &&
test-tool -C sparse-index read-cache --table >cache &&
for dir in deep folder2 x
do
TREE=$(git -C sparse-index rev-parse HEAD:$dir) &&
grep "040000 tree $TREE $dir/" cache \
|| return 1
done &&
git -C sparse-index sparse-checkout set deep/deeper1 &&
test-tool -C sparse-index read-cache --table >cache &&
for dir in deep/deeper2 folder1 folder2 x
do
TREE=$(git -C sparse-index rev-parse HEAD:$dir) &&
grep "040000 tree $TREE $dir/" cache \
|| return 1
done &&
# Disabling the sparse-index removes tree entries with full ones
git -C sparse-index sparse-checkout init --no-sparse-index &&
test-tool -C sparse-index read-cache --table >cache &&
! grep "040000 tree" cache &&
test_sparse_match test-tool read-cache --table
'
test_expect_success 'expanded in-memory index matches full index' '
init_repos &&
test_sparse_match test-tool read-cache --expand --table
'
test_expect_success 'status with options' '
init_repos &&
test_sparse_match ls &&
test_all_match git status --porcelain=v2 &&
test_all_match git status --porcelain=v2 -z -u &&
test_all_match git status --porcelain=v2 -uno &&
run_on_all touch README.md &&
test_all_match git status --porcelain=v2 &&
test_all_match git status --porcelain=v2 -z -u &&
test_all_match git status --porcelain=v2 -uno &&
test_all_match git add README.md &&
test_all_match git status --porcelain=v2 &&
test_all_match git status --porcelain=v2 -z -u &&
test_all_match git status --porcelain=v2 -uno
'
test_expect_success 'status reports sparse-checkout' '
init_repos &&
git -C sparse-checkout status >full &&
git -C sparse-index status >sparse &&
test_i18ngrep "You are in a sparse checkout with " full &&
test_i18ngrep "You are in a sparse checkout." sparse
'
test_expect_success 'add, commit, checkout' '
init_repos &&
write_script edit-contents <<-\EOF &&
echo text >>$1
EOF
run_on_all ../edit-contents README.md &&
test_all_match git add README.md &&
test_all_match git status --porcelain=v2 &&
test_all_match git commit -m "Add README.md" &&
test_all_match git checkout HEAD~1 &&
test_all_match git checkout - &&
run_on_all ../edit-contents README.md &&
test_all_match git add -A &&
test_all_match git status --porcelain=v2 &&
test_all_match git commit -m "Extend README.md" &&
test_all_match git checkout HEAD~1 &&
test_all_match git checkout - &&
run_on_all ../edit-contents deep/newfile &&
test_all_match git status --porcelain=v2 -uno &&
test_all_match git status --porcelain=v2 &&
test_all_match git add . &&
test_all_match git status --porcelain=v2 &&
test_all_match git commit -m "add deep/newfile" &&
test_all_match git checkout HEAD~1 &&
test_all_match git checkout -
'
test_expect_success 'commit including unstaged changes' '
init_repos &&
write_script edit-file <<-\EOF &&
echo $1 >$2
EOF
run_on_all ../edit-file 1 a &&
run_on_all ../edit-file 1 deep/a &&
test_all_match git commit -m "-a" -a &&
test_all_match git status --porcelain=v2 &&
run_on_all ../edit-file 2 a &&
run_on_all ../edit-file 2 deep/a &&
test_all_match git commit -m "--include" --include deep/a &&
test_all_match git status --porcelain=v2 &&
test_all_match git commit -m "--include" --include a &&
test_all_match git status --porcelain=v2 &&
run_on_all ../edit-file 3 a &&
run_on_all ../edit-file 3 deep/a &&
test_all_match git commit -m "--amend" -a --amend &&
test_all_match git status --porcelain=v2
'
test_expect_success 'status/add: outside sparse cone' '
init_repos &&
# adding a "missing" file outside the cone should fail
test_sparse_match test_must_fail git add folder1/a &&
# folder1 is at HEAD, but outside the sparse cone
run_on_sparse mkdir folder1 &&
cp initial-repo/folder1/a sparse-checkout/folder1/a &&
cp initial-repo/folder1/a sparse-index/folder1/a &&
test_sparse_match git status &&
write_script edit-contents <<-\EOF &&
echo text >>$1
EOF
run_on_sparse ../edit-contents folder1/a &&
run_on_all ../edit-contents folder1/new &&
test_sparse_match git status --porcelain=v2 &&
# This "git add folder1/a" fails with a warning
# in the sparse repos, differing from the full
# repo. This is intentional.
test_sparse_match test_must_fail git add folder1/a &&
test_sparse_match test_must_fail git add --refresh folder1/a &&
test_all_match git status --porcelain=v2 &&
test_all_match git add . &&
test_all_match git status --porcelain=v2 &&
test_all_match git commit -m folder1/new &&
run_on_all ../edit-contents folder1/newer &&
test_all_match git add folder1/ &&
test_all_match git status --porcelain=v2 &&
test_all_match git commit -m folder1/newer
'
test_expect_success 'checkout and reset --hard' '
init_repos &&
test_all_match git checkout update-folder1 &&
test_all_match git status --porcelain=v2 &&
test_all_match git checkout update-deep &&
test_all_match git status --porcelain=v2 &&
test_all_match git checkout -b reset-test &&
test_all_match git reset --hard deepest &&
test_all_match git reset --hard update-folder1 &&
test_all_match git reset --hard update-folder2
'
test_expect_success 'diff --staged' '
init_repos &&
write_script edit-contents <<-\EOF &&
echo text >>README.md
EOF
run_on_all ../edit-contents &&
test_all_match git diff &&
test_all_match git diff --staged &&
test_all_match git add README.md &&
test_all_match git diff &&
test_all_match git diff --staged
'
test_expect_success 'diff with renames and conflicts' '
init_repos &&
for branch in rename-out-to-out rename-out-to-in rename-in-to-out
do
test_all_match git checkout rename-base &&
test_all_match git checkout $branch -- . &&
test_all_match git status --porcelain=v2 &&
test_all_match git diff --staged --no-renames &&
test_all_match git diff --staged --find-renames || return 1
done
'
test_expect_success 'diff with directory/file conflicts' '
init_repos &&
for branch in rename-out-to-out rename-out-to-in rename-in-to-out
do
git -C full-checkout reset --hard &&
test_sparse_match git reset --hard &&
test_all_match git checkout $branch &&
test_all_match git checkout rename-base -- . &&
test_all_match git status --porcelain=v2 &&
test_all_match git diff --staged --no-renames &&
test_all_match git diff --staged --find-renames || return 1
done
'
test_expect_success 'log with pathspec outside sparse definition' '
init_repos &&
test_all_match git log -- a &&
test_all_match git log -- folder1/a &&
test_all_match git log -- folder2/a &&
test_all_match git log -- deep/a &&
test_all_match git log -- deep/deeper1/a &&
test_all_match git log -- deep/deeper1/deepest/a &&
test_all_match git checkout update-folder1 &&
test_all_match git log -- folder1/a
'
test_expect_success 'blame with pathspec inside sparse definition' '
init_repos &&
test_all_match git blame a &&
test_all_match git blame deep/a &&
test_all_match git blame deep/deeper1/a &&
test_all_match git blame deep/deeper1/deepest/a
'
# TODO: blame currently does not support blaming files outside of the
# sparse definition. It complains that the file doesn't exist locally.
test_expect_failure 'blame with pathspec outside sparse definition' '
init_repos &&
test_all_match git blame folder1/a &&
test_all_match git blame folder2/a &&
test_all_match git blame deep/deeper2/a &&
test_all_match git blame deep/deeper2/deepest/a
'
# NEEDSWORK: a sparse-checkout behaves differently from a full checkout
# in this scenario, but it shouldn't.
test_expect_failure 'checkout and reset (mixed)' '
init_repos &&
test_all_match git checkout -b reset-test update-deep &&
test_all_match git reset deepest &&
test_all_match git reset update-folder1 &&
test_all_match git reset update-folder2
'
# NEEDSWORK: a sparse-checkout behaves differently from a full checkout
# in this scenario, but it shouldn't.
test_expect_success 'checkout and reset (mixed) [sparse]' '
init_repos &&
test_sparse_match git checkout -b reset-test update-deep &&
test_sparse_match git reset deepest &&
test_sparse_match git reset update-folder1 &&
test_sparse_match git reset update-folder2
'
test_expect_success 'merge' '
init_repos &&
test_all_match git checkout -b merge update-deep &&
test_all_match git merge -m "folder1" update-folder1 &&
test_all_match git rev-parse HEAD^{tree} &&
test_all_match git merge -m "folder2" update-folder2 &&
test_all_match git rev-parse HEAD^{tree}
'
test_expect_success 'merge with outside renames' '
init_repos &&
for type in out-to-out out-to-in in-to-out
do
test_all_match git reset --hard &&
test_all_match git checkout -f -b merge-$type update-deep &&
test_all_match git merge -m "$type" rename-$type &&
test_all_match git rev-parse HEAD^{tree} || return 1
done
'
# Sparse-index fails to convert the index in the
# final 'git cherry-pick' command.
test_expect_success 'cherry-pick with conflicts' '
init_repos &&
write_script edit-conflict <<-\EOF &&
echo $1 >conflict
EOF
test_all_match git checkout -b to-cherry-pick &&
run_on_all ../edit-conflict ABC &&
test_all_match git add conflict &&
test_all_match git commit -m "conflict to pick" &&
test_all_match git checkout -B base HEAD~1 &&
run_on_all ../edit-conflict DEF &&
test_all_match git add conflict &&
test_all_match git commit -m "conflict in base" &&
test_all_match test_must_fail git cherry-pick to-cherry-pick
'
test_expect_success 'clean' '
init_repos &&
echo bogus >>.gitignore &&
run_on_all cp ../.gitignore . &&
test_all_match git add .gitignore &&
test_all_match git commit -m "ignore bogus files" &&
run_on_sparse mkdir folder1 &&
run_on_all touch folder1/bogus &&
test_all_match git status --porcelain=v2 &&
test_all_match git clean -f &&
test_all_match git status --porcelain=v2 &&
test_sparse_match ls &&
test_sparse_match ls folder1 &&
test_all_match git clean -xf &&
test_all_match git status --porcelain=v2 &&
test_sparse_match ls &&
test_sparse_match ls folder1 &&
test_all_match git clean -xdf &&
test_all_match git status --porcelain=v2 &&
test_sparse_match ls &&
test_sparse_match ls folder1 &&
test_sparse_match test_path_is_dir folder1
'
test_expect_success 'submodule handling' '
init_repos &&
test_all_match mkdir modules &&
test_all_match touch modules/a &&
test_all_match git add modules &&
test_all_match git commit -m "add modules directory" &&
run_on_all git submodule add "$(pwd)/initial-repo" modules/sub &&
test_all_match git commit -m "add submodule" &&
# having a submodule prevents "modules" from collapse
test-tool -C sparse-index read-cache --table >cache &&
grep "100644 blob .* modules/a" cache &&
grep "160000 commit $(git -C initial-repo rev-parse HEAD) modules/sub" cache
'
test_expect_success 'sparse-index is expanded and converted back' '
init_repos &&
GIT_TRACE2_EVENT="$(pwd)/trace2.txt" GIT_TRACE2_EVENT_NESTING=10 \
git -C sparse-index -c core.fsmonitor="" reset --hard &&
test_region index convert_to_sparse trace2.txt &&
test_region index ensure_full_index trace2.txt
'
ensure_not_expanded () {
rm -f trace2.txt &&
echo >>sparse-index/untracked.txt &&
GIT_TRACE2_EVENT="$(pwd)/trace2.txt" GIT_TRACE2_EVENT_NESTING=10 \
git -C sparse-index "$@" &&
test_region ! index ensure_full_index trace2.txt
}
test_expect_success 'sparse-index is not expanded' '
init_repos &&
ensure_not_expanded status &&
ensure_not_expanded commit --allow-empty -m empty &&
echo >>sparse-index/a &&
ensure_not_expanded commit -a -m a &&
echo >>sparse-index/a &&
ensure_not_expanded commit --include a -m a &&
echo >>sparse-index/deep/deeper1/a &&
ensure_not_expanded commit --include deep/deeper1/a -m deeper
'
# NEEDSWORK: a sparse-checkout behaves differently from a full checkout
# in this scenario, but it shouldn't.
test_expect_success 'reset mixed and checkout orphan' '
init_repos &&
test_all_match git checkout rename-out-to-in &&
# Sparse checkouts do not agree with full checkouts about
# how to report a directory/file conflict during a reset.
# This command would fail with test_all_match because the
# full checkout reports "T folder1/0/1" while a sparse
# checkout reports "D folder1/0/1". This matches because
# the sparse checkouts skip "adding" the other side of
# the conflict.
test_sparse_match git reset --mixed HEAD~1 &&
test_sparse_match test-tool read-cache --table --expand &&
test_sparse_match git status --porcelain=v2 &&
# At this point, sparse-checkouts behave differently
# from the full-checkout.
test_sparse_match git checkout --orphan new-branch &&
test_sparse_match test-tool read-cache --table --expand &&
test_sparse_match git status --porcelain=v2
'
test_expect_success 'add everything with deep new file' '
init_repos &&
run_on_sparse git sparse-checkout set deep/deeper1/deepest &&
run_on_all touch deep/deeper1/x &&
test_all_match git add . &&
test_all_match git status --porcelain=v2
'
test_done