Merge branch 'ds/reachable-topo-order'

The revision walker machinery learned to take advantage of the
commit generation numbers stored in the commit-graph file.

* ds/reachable-topo-order:
  t6012: make rev-list tests more interesting
  revision.c: generation-based topo-order algorithm
  commit/revisions: bookkeeping before refactoring
  revision.c: begin refactoring --topo-order logic
  test-reach: add rev-list tests
  test-reach: add run_three_modes method
  prio-queue: add 'peek' operation
This commit is contained in:
Junio C Hamano 2018-11-18 18:23:52 +09:00
commit 62ca33e02a
11 changed files with 427 additions and 39 deletions

View file

@ -657,11 +657,10 @@ struct commit *pop_commit(struct commit_list **stack)
/* count number of children that have not been emitted */
define_commit_slab(indegree_slab, int);
/* record author-date for each commit object */
define_commit_slab(author_date_slab, timestamp_t);
static void record_author_date(struct author_date_slab *author_date,
struct commit *commit)
void record_author_date(struct author_date_slab *author_date,
struct commit *commit)
{
const char *buffer = get_commit_buffer(commit, NULL);
struct ident_split ident;
@ -686,8 +685,8 @@ static void record_author_date(struct author_date_slab *author_date,
unuse_commit_buffer(commit, buffer);
}
static int compare_commits_by_author_date(const void *a_, const void *b_,
void *cb_data)
int compare_commits_by_author_date(const void *a_, const void *b_,
void *cb_data)
{
const struct commit *a = a_, *b = b_;
struct author_date_slab *author_date = cb_data;

View file

@ -8,6 +8,7 @@
#include "gpg-interface.h"
#include "string-list.h"
#include "pretty.h"
#include "commit-slab.h"
#define COMMIT_NOT_FROM_GRAPH 0xFFFFFFFF
#define GENERATION_NUMBER_INFINITY 0xFFFFFFFF
@ -333,6 +334,12 @@ extern int remove_signature(struct strbuf *buf);
*/
extern int check_commit_signature(const struct commit *commit, struct signature_check *sigc);
/* record author-date for each commit object */
struct author_date_slab;
void record_author_date(struct author_date_slab *author_date,
struct commit *commit);
int compare_commits_by_author_date(const void *a_, const void *b_, void *unused);
int compare_commits_by_commit_date(const void *a_, const void *b_, void *unused);
int compare_commits_by_gen_then_commit_date(const void *a_, const void *b_, void *unused);

View file

@ -59,7 +59,7 @@ struct object_array {
/*
* object flag allocation:
* revision.h: 0---------10 2526
* revision.h: 0---------10 25----28
* fetch-pack.c: 01
* negotiator/default.c: 2--5
* walker.c: 0-2
@ -78,7 +78,7 @@ struct object_array {
* builtin/show-branch.c: 0-------------------------------------------26
* builtin/unpack-objects.c: 2021
*/
#define FLAG_BITS 27
#define FLAG_BITS 29
/*
* The object type is stored in 3 bits.

View file

@ -85,3 +85,12 @@ void *prio_queue_get(struct prio_queue *queue)
}
return result;
}
void *prio_queue_peek(struct prio_queue *queue)
{
if (!queue->nr)
return NULL;
if (!queue->compare)
return queue->array[queue->nr - 1].data;
return queue->array[0].data;
}

View file

@ -46,6 +46,12 @@ extern void prio_queue_put(struct prio_queue *, void *thing);
*/
extern void *prio_queue_get(struct prio_queue *);
/*
* Gain access to the "thing" that would be returned by
* prio_queue_get, but do not remove it from the queue.
*/
extern void *prio_queue_peek(struct prio_queue *);
extern void clear_prio_queue(struct prio_queue *);
/* Reverse the LIFO elements */

View file

@ -25,6 +25,8 @@
#include "worktree.h"
#include "argv-array.h"
#include "commit-reach.h"
#include "commit-graph.h"
#include "prio-queue.h"
volatile show_early_output_fn_t show_early_output;
@ -767,8 +769,8 @@ static void commit_list_insert_by_date_cached(struct commit *p, struct commit_li
*cache = new_entry;
}
static int add_parents_to_list(struct rev_info *revs, struct commit *commit,
struct commit_list **list, struct commit_list **cache_ptr)
static int process_parents(struct rev_info *revs, struct commit *commit,
struct commit_list **list, struct commit_list **cache_ptr)
{
struct commit_list *parent = commit->parents;
unsigned left_flag;
@ -807,7 +809,8 @@ static int add_parents_to_list(struct rev_info *revs, struct commit *commit,
if (p->object.flags & SEEN)
continue;
p->object.flags |= SEEN;
commit_list_insert_by_date_cached(p, list, cached_base, cache_ptr);
if (list)
commit_list_insert_by_date_cached(p, list, cached_base, cache_ptr);
}
return 0;
}
@ -846,7 +849,8 @@ static int add_parents_to_list(struct rev_info *revs, struct commit *commit,
p->object.flags |= left_flag;
if (!(p->object.flags & SEEN)) {
p->object.flags |= SEEN;
commit_list_insert_by_date_cached(p, list, cached_base, cache_ptr);
if (list)
commit_list_insert_by_date_cached(p, list, cached_base, cache_ptr);
}
if (revs->first_parent_only)
break;
@ -1090,7 +1094,7 @@ static int limit_list(struct rev_info *revs)
if (revs->max_age != -1 && (commit->date < revs->max_age))
obj->flags |= UNINTERESTING;
if (add_parents_to_list(revs, commit, &list, NULL) < 0)
if (process_parents(revs, commit, &list, NULL) < 0)
return -1;
if (obj->flags & UNINTERESTING) {
mark_parents_uninteresting(commit);
@ -2468,7 +2472,7 @@ int setup_revisions(int argc, const char **argv, struct rev_info *revs, struct s
if (revs->diffopt.objfind)
revs->simplify_history = 0;
if (revs->topo_order)
if (revs->topo_order && !generation_numbers_enabled(the_repository))
revs->limited = 1;
if (revs->prune_data.nr) {
@ -2907,6 +2911,217 @@ static int mark_uninteresting(const struct object_id *oid,
return 0;
}
define_commit_slab(indegree_slab, int);
define_commit_slab(author_date_slab, timestamp_t);
struct topo_walk_info {
uint32_t min_generation;
struct prio_queue explore_queue;
struct prio_queue indegree_queue;
struct prio_queue topo_queue;
struct indegree_slab indegree;
struct author_date_slab author_date;
};
static inline void test_flag_and_insert(struct prio_queue *q, struct commit *c, int flag)
{
if (c->object.flags & flag)
return;
c->object.flags |= flag;
prio_queue_put(q, c);
}
static void explore_walk_step(struct rev_info *revs)
{
struct topo_walk_info *info = revs->topo_walk_info;
struct commit_list *p;
struct commit *c = prio_queue_get(&info->explore_queue);
if (!c)
return;
if (parse_commit_gently(c, 1) < 0)
return;
if (revs->sort_order == REV_SORT_BY_AUTHOR_DATE)
record_author_date(&info->author_date, c);
if (revs->max_age != -1 && (c->date < revs->max_age))
c->object.flags |= UNINTERESTING;
if (process_parents(revs, c, NULL, NULL) < 0)
return;
if (c->object.flags & UNINTERESTING)
mark_parents_uninteresting(c);
for (p = c->parents; p; p = p->next)
test_flag_and_insert(&info->explore_queue, p->item, TOPO_WALK_EXPLORED);
}
static void explore_to_depth(struct rev_info *revs,
uint32_t gen_cutoff)
{
struct topo_walk_info *info = revs->topo_walk_info;
struct commit *c;
while ((c = prio_queue_peek(&info->explore_queue)) &&
c->generation >= gen_cutoff)
explore_walk_step(revs);
}
static void indegree_walk_step(struct rev_info *revs)
{
struct commit_list *p;
struct topo_walk_info *info = revs->topo_walk_info;
struct commit *c = prio_queue_get(&info->indegree_queue);
if (!c)
return;
if (parse_commit_gently(c, 1) < 0)
return;
explore_to_depth(revs, c->generation);
for (p = c->parents; p; p = p->next) {
struct commit *parent = p->item;
int *pi = indegree_slab_at(&info->indegree, parent);
if (*pi)
(*pi)++;
else
*pi = 2;
test_flag_and_insert(&info->indegree_queue, parent, TOPO_WALK_INDEGREE);
if (revs->first_parent_only)
return;
}
}
static void compute_indegrees_to_depth(struct rev_info *revs,
uint32_t gen_cutoff)
{
struct topo_walk_info *info = revs->topo_walk_info;
struct commit *c;
while ((c = prio_queue_peek(&info->indegree_queue)) &&
c->generation >= gen_cutoff)
indegree_walk_step(revs);
}
static void init_topo_walk(struct rev_info *revs)
{
struct topo_walk_info *info;
struct commit_list *list;
revs->topo_walk_info = xmalloc(sizeof(struct topo_walk_info));
info = revs->topo_walk_info;
memset(info, 0, sizeof(struct topo_walk_info));
init_indegree_slab(&info->indegree);
memset(&info->explore_queue, 0, sizeof(info->explore_queue));
memset(&info->indegree_queue, 0, sizeof(info->indegree_queue));
memset(&info->topo_queue, 0, sizeof(info->topo_queue));
switch (revs->sort_order) {
default: /* REV_SORT_IN_GRAPH_ORDER */
info->topo_queue.compare = NULL;
break;
case REV_SORT_BY_COMMIT_DATE:
info->topo_queue.compare = compare_commits_by_commit_date;
break;
case REV_SORT_BY_AUTHOR_DATE:
init_author_date_slab(&info->author_date);
info->topo_queue.compare = compare_commits_by_author_date;
info->topo_queue.cb_data = &info->author_date;
break;
}
info->explore_queue.compare = compare_commits_by_gen_then_commit_date;
info->indegree_queue.compare = compare_commits_by_gen_then_commit_date;
info->min_generation = GENERATION_NUMBER_INFINITY;
for (list = revs->commits; list; list = list->next) {
struct commit *c = list->item;
if (parse_commit_gently(c, 1))
continue;
test_flag_and_insert(&info->explore_queue, c, TOPO_WALK_EXPLORED);
test_flag_and_insert(&info->indegree_queue, c, TOPO_WALK_INDEGREE);
if (c->generation < info->min_generation)
info->min_generation = c->generation;
*(indegree_slab_at(&info->indegree, c)) = 1;
if (revs->sort_order == REV_SORT_BY_AUTHOR_DATE)
record_author_date(&info->author_date, c);
}
compute_indegrees_to_depth(revs, info->min_generation);
for (list = revs->commits; list; list = list->next) {
struct commit *c = list->item;
if (*(indegree_slab_at(&info->indegree, c)) == 1)
prio_queue_put(&info->topo_queue, c);
}
/*
* This is unfortunate; the initial tips need to be shown
* in the order given from the revision traversal machinery.
*/
if (revs->sort_order == REV_SORT_IN_GRAPH_ORDER)
prio_queue_reverse(&info->topo_queue);
}
static struct commit *next_topo_commit(struct rev_info *revs)
{
struct commit *c;
struct topo_walk_info *info = revs->topo_walk_info;
/* pop next off of topo_queue */
c = prio_queue_get(&info->topo_queue);
if (c)
*(indegree_slab_at(&info->indegree, c)) = 0;
return c;
}
static void expand_topo_walk(struct rev_info *revs, struct commit *commit)
{
struct commit_list *p;
struct topo_walk_info *info = revs->topo_walk_info;
if (process_parents(revs, commit, NULL, NULL) < 0) {
if (!revs->ignore_missing_links)
die("Failed to traverse parents of commit %s",
oid_to_hex(&commit->object.oid));
}
for (p = commit->parents; p; p = p->next) {
struct commit *parent = p->item;
int *pi;
if (parse_commit_gently(parent, 1) < 0)
continue;
if (parent->generation < info->min_generation) {
info->min_generation = parent->generation;
compute_indegrees_to_depth(revs, info->min_generation);
}
pi = indegree_slab_at(&info->indegree, parent);
(*pi)--;
if (*pi == 1)
prio_queue_put(&info->topo_queue, parent);
if (revs->first_parent_only)
return;
}
}
int prepare_revision_walk(struct rev_info *revs)
{
int i;
@ -2943,11 +3158,13 @@ int prepare_revision_walk(struct rev_info *revs)
commit_list_sort_by_date(&revs->commits);
if (revs->no_walk)
return 0;
if (revs->limited)
if (revs->limited) {
if (limit_list(revs) < 0)
return -1;
if (revs->topo_order)
sort_in_topological_order(&revs->commits, revs->sort_order);
if (revs->topo_order)
sort_in_topological_order(&revs->commits, revs->sort_order);
} else if (revs->topo_order)
init_topo_walk(revs);
if (revs->line_level_traverse)
line_log_filter(revs);
if (revs->simplify_merges)
@ -2964,7 +3181,7 @@ static enum rewrite_result rewrite_one(struct rev_info *revs, struct commit **pp
for (;;) {
struct commit *p = *pp;
if (!revs->limited)
if (add_parents_to_list(revs, p, &revs->commits, &cache) < 0)
if (process_parents(revs, p, &revs->commits, &cache) < 0)
return rewrite_one_error;
if (p->object.flags & UNINTERESTING)
return rewrite_one_ok;
@ -3272,6 +3489,8 @@ static struct commit *get_revision_1(struct rev_info *revs)
if (revs->reflog_info)
commit = next_reflog_entry(revs->reflog_info);
else if (revs->topo_walk_info)
commit = next_topo_commit(revs);
else
commit = pop_commit(&revs->commits);
@ -3293,7 +3512,9 @@ static struct commit *get_revision_1(struct rev_info *revs)
if (revs->reflog_info)
try_to_simplify_commit(revs, commit);
else if (add_parents_to_list(revs, commit, &revs->commits, NULL) < 0) {
else if (revs->topo_walk_info)
expand_topo_walk(revs, commit);
else if (process_parents(revs, commit, &revs->commits, NULL) < 0) {
if (!revs->ignore_missing_links)
die("Failed to traverse parents of commit %s",
oid_to_hex(&commit->object.oid));

View file

@ -32,6 +32,9 @@
#define TRACK_LINEAR (1u<<26)
#define ALL_REV_FLAGS (((1u<<11)-1) | NOT_USER_GIVEN | TRACK_LINEAR)
#define TOPO_WALK_EXPLORED (1u<<27)
#define TOPO_WALK_INDEGREE (1u<<28)
#define DECORATE_SHORT_REFS 1
#define DECORATE_FULL_REFS 2
@ -64,6 +67,8 @@ struct rev_cmdline_info {
#define REVISION_WALK_NO_WALK_SORTED 1
#define REVISION_WALK_NO_WALK_UNSORTED 2
struct topo_walk_info;
struct rev_info {
/* Starting list */
struct commit_list *commits;
@ -270,6 +275,8 @@ struct rev_info {
const char *break_bar;
struct revision_sources *sources;
struct topo_walk_info *topo_walk_info;
};
int ref_excluded(struct string_list *, const char *path);

View file

@ -22,14 +22,24 @@ int cmd__prio_queue(int argc, const char **argv)
struct prio_queue pq = { intcmp };
while (*++argv) {
if (!strcmp(*argv, "get"))
show(prio_queue_get(&pq));
else if (!strcmp(*argv, "dump")) {
int *v;
while ((v = prio_queue_get(&pq)))
show(v);
}
else {
if (!strcmp(*argv, "get")) {
void *peek = prio_queue_peek(&pq);
void *get = prio_queue_get(&pq);
if (peek != get)
BUG("peek and get results do not match");
show(get);
} else if (!strcmp(*argv, "dump")) {
void *peek;
void *get;
while ((peek = prio_queue_peek(&pq))) {
get = prio_queue_get(&pq);
if (peek != get)
BUG("peek and get results do not match");
show(get);
}
} else if (!strcmp(*argv, "stack")) {
pq.compare = NULL;
} else {
int *v = malloc(sizeof(*v));
*v = atoi(*argv);
prio_queue_put(&pq, v);

View file

@ -47,4 +47,18 @@ test_expect_success 'notice empty queue' '
test_cmp expect actual
'
cat >expect <<'EOF'
3
2
6
4
5
1
8
EOF
test_expect_success 'stack order' '
test-tool prio-queue stack 8 1 5 4 6 2 3 dump >actual &&
test_cmp expect actual
'
test_done

View file

@ -12,6 +12,22 @@ unnote () {
git name-rev --tags --stdin | sed -e "s|$OID_REGEX (tags/\([^)]*\)) |\1 |g"
}
#
# Create a test repo with interesting commit graph:
#
# A--B----------G--H--I--K--L
# \ \ / /
# \ \ / /
# C------E---F J
# \_/
#
# The commits are laid out from left-to-right starting with
# the root commit A and terminating at the tip commit L.
#
# There are a few places where we adjust the commit date or
# author date to make the --topo-order, --date-order, and
# --author-date-order flags produce different output.
test_expect_success setup '
echo "Hi there" >file &&
echo "initial" >lost &&
@ -21,10 +37,18 @@ test_expect_success setup '
git branch other-branch &&
git symbolic-ref HEAD refs/heads/unrelated &&
git rm -f "*" &&
echo "Unrelated branch" >side &&
git add side &&
test_tick && git commit -m "Side root" &&
note J &&
git checkout master &&
echo "Hello" >file &&
echo "second" >lost &&
git add file lost &&
test_tick && git commit -m "Modified file and lost" &&
test_tick && GIT_AUTHOR_DATE=$(($test_tick + 120)) git commit -m "Modified file and lost" &&
note B &&
git checkout other-branch &&
@ -63,13 +87,6 @@ test_expect_success setup '
test_tick && git commit -a -m "Final change" &&
note I &&
git symbolic-ref HEAD refs/heads/unrelated &&
git rm -f "*" &&
echo "Unrelated branch" >side &&
git add side &&
test_tick && git commit -m "Side root" &&
note J &&
git checkout master &&
test_tick && git merge --allow-unrelated-histories -m "Coolest" unrelated &&
note K &&
@ -103,14 +120,24 @@ check_result () {
check_outcome success "$@"
}
check_result 'L K J I H G F E D C B A' --full-history
check_result 'L K J I H F E D C G B A' --full-history --topo-order
check_result 'L K I H G F E D C B J A' --full-history
check_result 'L K I H G F E D C B J A' --full-history --date-order
check_result 'L K I H G F E D B C J A' --full-history --author-date-order
check_result 'K I H E C B A' --full-history -- file
check_result 'K I H E C B A' --full-history --topo-order -- file
check_result 'K I H E C B A' --full-history --date-order -- file
check_result 'K I H E B C A' --full-history --author-date-order -- file
check_result 'I E C B A' --simplify-merges -- file
check_result 'I E C B A' --simplify-merges --topo-order -- file
check_result 'I E C B A' --simplify-merges --date-order -- file
check_result 'I E B C A' --simplify-merges --author-date-order -- file
check_result 'I B A' -- file
check_result 'I B A' --topo-order -- file
check_result 'I B A' --date-order -- file
check_result 'I B A' --author-date-order -- file
check_result 'H' --first-parent -- another-file
check_result 'H' --first-parent --topo-order -- another-file
check_result 'E C B A' --full-history E -- lost
test_expect_success 'full history simplification without parent' '

View file

@ -56,18 +56,22 @@ test_expect_success 'setup' '
git config core.commitGraph true
'
test_three_modes () {
run_three_modes () {
test_when_finished rm -rf .git/objects/info/commit-graph &&
test-tool reach $1 <input >actual &&
"$@" <input >actual &&
test_cmp expect actual &&
cp commit-graph-full .git/objects/info/commit-graph &&
test-tool reach $1 <input >actual &&
"$@" <input >actual &&
test_cmp expect actual &&
cp commit-graph-half .git/objects/info/commit-graph &&
test-tool reach $1 <input >actual &&
"$@" <input >actual &&
test_cmp expect actual
}
test_three_modes () {
run_three_modes test-tool reach "$@"
}
test_expect_success 'ref_newer:miss' '
cat >input <<-\EOF &&
A:commit-5-7
@ -265,6 +269,90 @@ test_expect_success 'commit_contains:miss' '
test_three_modes commit_contains --tag
'
test_expect_success 'rev-list: basic topo-order' '
git rev-parse \
commit-6-6 commit-5-6 commit-4-6 commit-3-6 commit-2-6 commit-1-6 \
commit-6-5 commit-5-5 commit-4-5 commit-3-5 commit-2-5 commit-1-5 \
commit-6-4 commit-5-4 commit-4-4 commit-3-4 commit-2-4 commit-1-4 \
commit-6-3 commit-5-3 commit-4-3 commit-3-3 commit-2-3 commit-1-3 \
commit-6-2 commit-5-2 commit-4-2 commit-3-2 commit-2-2 commit-1-2 \
commit-6-1 commit-5-1 commit-4-1 commit-3-1 commit-2-1 commit-1-1 \
>expect &&
run_three_modes git rev-list --topo-order commit-6-6
'
test_expect_success 'rev-list: first-parent topo-order' '
git rev-parse \
commit-6-6 \
commit-6-5 \
commit-6-4 \
commit-6-3 \
commit-6-2 \
commit-6-1 commit-5-1 commit-4-1 commit-3-1 commit-2-1 commit-1-1 \
>expect &&
run_three_modes git rev-list --first-parent --topo-order commit-6-6
'
test_expect_success 'rev-list: range topo-order' '
git rev-parse \
commit-6-6 commit-5-6 commit-4-6 commit-3-6 commit-2-6 commit-1-6 \
commit-6-5 commit-5-5 commit-4-5 commit-3-5 commit-2-5 commit-1-5 \
commit-6-4 commit-5-4 commit-4-4 commit-3-4 commit-2-4 commit-1-4 \
commit-6-3 commit-5-3 commit-4-3 \
commit-6-2 commit-5-2 commit-4-2 \
commit-6-1 commit-5-1 commit-4-1 \
>expect &&
run_three_modes git rev-list --topo-order commit-3-3..commit-6-6
'
test_expect_success 'rev-list: range topo-order' '
git rev-parse \
commit-6-6 commit-5-6 commit-4-6 \
commit-6-5 commit-5-5 commit-4-5 \
commit-6-4 commit-5-4 commit-4-4 \
commit-6-3 commit-5-3 commit-4-3 \
commit-6-2 commit-5-2 commit-4-2 \
commit-6-1 commit-5-1 commit-4-1 \
>expect &&
run_three_modes git rev-list --topo-order commit-3-8..commit-6-6
'
test_expect_success 'rev-list: first-parent range topo-order' '
git rev-parse \
commit-6-6 \
commit-6-5 \
commit-6-4 \
commit-6-3 \
commit-6-2 \
commit-6-1 commit-5-1 commit-4-1 \
>expect &&
run_three_modes git rev-list --first-parent --topo-order commit-3-8..commit-6-6
'
test_expect_success 'rev-list: ancestry-path topo-order' '
git rev-parse \
commit-6-6 commit-5-6 commit-4-6 commit-3-6 \
commit-6-5 commit-5-5 commit-4-5 commit-3-5 \
commit-6-4 commit-5-4 commit-4-4 commit-3-4 \
commit-6-3 commit-5-3 commit-4-3 \
>expect &&
run_three_modes git rev-list --topo-order --ancestry-path commit-3-3..commit-6-6
'
test_expect_success 'rev-list: symmetric difference topo-order' '
git rev-parse \
commit-6-6 commit-5-6 commit-4-6 \
commit-6-5 commit-5-5 commit-4-5 \
commit-6-4 commit-5-4 commit-4-4 \
commit-6-3 commit-5-3 commit-4-3 \
commit-6-2 commit-5-2 commit-4-2 \
commit-6-1 commit-5-1 commit-4-1 \
commit-3-8 commit-2-8 commit-1-8 \
commit-3-7 commit-2-7 commit-1-7 \
>expect &&
run_three_modes git rev-list --topo-order commit-3-8...commit-6-6
'
test_expect_success 'get_reachable_subset:all' '
cat >input <<-\EOF &&
X:commit-9-1