2018-04-02 20:34:18 +00:00
|
|
|
#include "builtin.h"
|
|
|
|
#include "config.h"
|
2018-04-02 20:34:20 +00:00
|
|
|
#include "dir.h"
|
|
|
|
#include "lockfile.h"
|
2018-04-02 20:34:18 +00:00
|
|
|
#include "parse-options.h"
|
2018-06-27 13:24:32 +00:00
|
|
|
#include "repository.h"
|
2018-04-02 20:34:20 +00:00
|
|
|
#include "commit-graph.h"
|
2018-04-02 20:34:18 +00:00
|
|
|
|
|
|
|
static char const * const builtin_commit_graph_usage[] = {
|
|
|
|
N_("git commit-graph [--object-dir <objdir>]"),
|
2018-04-10 12:56:02 +00:00
|
|
|
N_("git commit-graph read [--object-dir <objdir>]"),
|
2018-06-27 13:24:32 +00:00
|
|
|
N_("git commit-graph verify [--object-dir <objdir>]"),
|
2018-06-27 13:24:45 +00:00
|
|
|
N_("git commit-graph write [--object-dir <objdir>] [--append] [--reachable|--stdin-packs|--stdin-commits]"),
|
2018-04-02 20:34:20 +00:00
|
|
|
NULL
|
|
|
|
};
|
|
|
|
|
2018-06-27 13:24:32 +00:00
|
|
|
static const char * const builtin_commit_graph_verify_usage[] = {
|
|
|
|
N_("git commit-graph verify [--object-dir <objdir>]"),
|
|
|
|
NULL
|
|
|
|
};
|
|
|
|
|
2018-04-10 12:56:02 +00:00
|
|
|
static const char * const builtin_commit_graph_read_usage[] = {
|
|
|
|
N_("git commit-graph read [--object-dir <objdir>]"),
|
|
|
|
NULL
|
|
|
|
};
|
|
|
|
|
2018-04-02 20:34:20 +00:00
|
|
|
static const char * const builtin_commit_graph_write_usage[] = {
|
2018-06-27 13:24:45 +00:00
|
|
|
N_("git commit-graph write [--object-dir <objdir>] [--append] [--reachable|--stdin-packs|--stdin-commits]"),
|
2018-04-02 20:34:18 +00:00
|
|
|
NULL
|
|
|
|
};
|
|
|
|
|
|
|
|
static struct opts_commit_graph {
|
|
|
|
const char *obj_dir;
|
2018-06-27 13:24:45 +00:00
|
|
|
int reachable;
|
2018-04-10 12:56:06 +00:00
|
|
|
int stdin_packs;
|
2018-04-10 12:56:07 +00:00
|
|
|
int stdin_commits;
|
2018-04-10 12:56:08 +00:00
|
|
|
int append;
|
2018-04-02 20:34:18 +00:00
|
|
|
} opts;
|
|
|
|
|
2018-06-27 13:24:32 +00:00
|
|
|
|
|
|
|
static int graph_verify(int argc, const char **argv)
|
|
|
|
{
|
|
|
|
struct commit_graph *graph = NULL;
|
|
|
|
char *graph_name;
|
2019-03-25 12:08:30 +00:00
|
|
|
int open_ok;
|
|
|
|
int fd;
|
|
|
|
struct stat st;
|
2018-06-27 13:24:32 +00:00
|
|
|
|
|
|
|
static struct option builtin_commit_graph_verify_options[] = {
|
|
|
|
OPT_STRING(0, "object-dir", &opts.obj_dir,
|
|
|
|
N_("dir"),
|
|
|
|
N_("The object directory to store the graph")),
|
|
|
|
OPT_END(),
|
|
|
|
};
|
|
|
|
|
|
|
|
argc = parse_options(argc, argv, NULL,
|
|
|
|
builtin_commit_graph_verify_options,
|
|
|
|
builtin_commit_graph_verify_usage, 0);
|
|
|
|
|
|
|
|
if (!opts.obj_dir)
|
|
|
|
opts.obj_dir = get_object_directory();
|
|
|
|
|
|
|
|
graph_name = get_commit_graph_filename(opts.obj_dir);
|
2019-03-25 12:08:30 +00:00
|
|
|
open_ok = open_commit_graph(graph_name, &fd, &st);
|
2019-03-25 12:08:32 +00:00
|
|
|
if (!open_ok && errno == ENOENT)
|
2019-03-25 12:08:30 +00:00
|
|
|
return 0;
|
2019-03-25 12:08:32 +00:00
|
|
|
if (!open_ok)
|
|
|
|
die_errno(_("Could not open commit-graph '%s'"), graph_name);
|
2019-03-25 12:08:31 +00:00
|
|
|
graph = load_commit_graph_one_fd_st(fd, &st);
|
2018-06-27 13:24:32 +00:00
|
|
|
FREE_AND_NULL(graph_name);
|
|
|
|
|
|
|
|
if (!graph)
|
2019-03-25 12:08:30 +00:00
|
|
|
return 1;
|
2018-06-27 13:24:32 +00:00
|
|
|
|
2018-10-03 17:12:17 +00:00
|
|
|
UNLEAK(graph);
|
2018-06-27 13:24:32 +00:00
|
|
|
return verify_commit_graph(the_repository, graph);
|
|
|
|
}
|
|
|
|
|
2018-04-10 12:56:02 +00:00
|
|
|
static int graph_read(int argc, const char **argv)
|
|
|
|
{
|
|
|
|
struct commit_graph *graph = NULL;
|
|
|
|
char *graph_name;
|
2019-03-25 12:08:30 +00:00
|
|
|
int open_ok;
|
|
|
|
int fd;
|
|
|
|
struct stat st;
|
2018-04-10 12:56:02 +00:00
|
|
|
|
|
|
|
static struct option builtin_commit_graph_read_options[] = {
|
|
|
|
OPT_STRING(0, "object-dir", &opts.obj_dir,
|
|
|
|
N_("dir"),
|
|
|
|
N_("The object directory to store the graph")),
|
|
|
|
OPT_END(),
|
|
|
|
};
|
|
|
|
|
|
|
|
argc = parse_options(argc, argv, NULL,
|
|
|
|
builtin_commit_graph_read_options,
|
|
|
|
builtin_commit_graph_read_usage, 0);
|
|
|
|
|
|
|
|
if (!opts.obj_dir)
|
|
|
|
opts.obj_dir = get_object_directory();
|
|
|
|
|
|
|
|
graph_name = get_commit_graph_filename(opts.obj_dir);
|
|
|
|
|
2019-03-25 12:08:30 +00:00
|
|
|
open_ok = open_commit_graph(graph_name, &fd, &st);
|
|
|
|
if (!open_ok)
|
|
|
|
die_errno(_("Could not open commit-graph '%s'"), graph_name);
|
|
|
|
|
2019-03-25 12:08:31 +00:00
|
|
|
graph = load_commit_graph_one_fd_st(fd, &st);
|
2018-10-03 17:12:17 +00:00
|
|
|
if (!graph)
|
2019-03-25 12:08:30 +00:00
|
|
|
return 1;
|
2018-06-27 13:24:27 +00:00
|
|
|
|
2018-04-10 12:56:02 +00:00
|
|
|
FREE_AND_NULL(graph_name);
|
|
|
|
|
|
|
|
printf("header: %08x %d %d %d %d\n",
|
|
|
|
ntohl(*(uint32_t*)graph->data),
|
|
|
|
*(unsigned char*)(graph->data + 4),
|
|
|
|
*(unsigned char*)(graph->data + 5),
|
|
|
|
*(unsigned char*)(graph->data + 6),
|
|
|
|
*(unsigned char*)(graph->data + 7));
|
|
|
|
printf("num_commits: %u\n", graph->num_commits);
|
|
|
|
printf("chunks:");
|
|
|
|
|
|
|
|
if (graph->chunk_oid_fanout)
|
|
|
|
printf(" oid_fanout");
|
|
|
|
if (graph->chunk_oid_lookup)
|
|
|
|
printf(" oid_lookup");
|
|
|
|
if (graph->chunk_commit_data)
|
|
|
|
printf(" commit_metadata");
|
commit-graph: rename "large edges" to "extra edges"
The optional 'Large Edge List' chunk of the commit graph file stores
parent information for commits with more than two parents, and the
names of most of the macros, variables, struct fields, and functions
related to this chunk contain the term "large edges", e.g.
write_graph_chunk_large_edges(). However, it's not a really great
term, as the edges to the second and subsequent parents stored in this
chunk are not any larger than the edges to the first and second
parents stored in the "main" 'Commit Data' chunk. It's the number of
edges, IOW number of parents, that is larger compared to non-merge and
"regular" two-parent merge commits. And indeed, two functions in
'commit-graph.c' have a local variable called 'num_extra_edges' that
refer to the same thing, and this "extra edges" term is much better at
describing these edges.
So let's rename all these references to "large edges" in macro,
variable, function, etc. names to "extra edges". There is a
GRAPH_OCTOPUS_EDGES_NEEDED macro as well; for the sake of consistency
rename it to GRAPH_EXTRA_EDGES_NEEDED.
We can do so safely without causing any incompatibility issues,
because the term "large edges" doesn't come up in the file format
itself in any form (the chunk's magic is {'E', 'D', 'G', 'E'}, there
is no 'L' in there), but only in the specification text. The string
"large edges", however, does come up in the output of 'git
commit-graph read' and in tests looking at its input, but that command
is explicitly documented as debugging aid, so we can change its output
and the affected tests safely.
Signed-off-by: SZEDER Gábor <szeder.dev@gmail.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2019-01-19 20:21:13 +00:00
|
|
|
if (graph->chunk_extra_edges)
|
|
|
|
printf(" extra_edges");
|
2018-04-10 12:56:02 +00:00
|
|
|
printf("\n");
|
|
|
|
|
2018-10-03 17:12:17 +00:00
|
|
|
UNLEAK(graph);
|
2018-07-11 22:42:40 +00:00
|
|
|
|
2018-04-10 12:56:02 +00:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2018-08-20 18:24:27 +00:00
|
|
|
extern int read_replace_refs;
|
|
|
|
|
2018-04-02 20:34:20 +00:00
|
|
|
static int graph_write(int argc, const char **argv)
|
|
|
|
{
|
2018-06-27 13:24:44 +00:00
|
|
|
struct string_list *pack_indexes = NULL;
|
|
|
|
struct string_list *commit_hex = NULL;
|
|
|
|
struct string_list lines;
|
2018-04-10 12:56:06 +00:00
|
|
|
|
2018-04-02 20:34:20 +00:00
|
|
|
static struct option builtin_commit_graph_write_options[] = {
|
|
|
|
OPT_STRING(0, "object-dir", &opts.obj_dir,
|
|
|
|
N_("dir"),
|
|
|
|
N_("The object directory to store the graph")),
|
2018-06-27 13:24:45 +00:00
|
|
|
OPT_BOOL(0, "reachable", &opts.reachable,
|
|
|
|
N_("start walk at all refs")),
|
2018-04-10 12:56:06 +00:00
|
|
|
OPT_BOOL(0, "stdin-packs", &opts.stdin_packs,
|
|
|
|
N_("scan pack-indexes listed by stdin for commits")),
|
2018-04-10 12:56:07 +00:00
|
|
|
OPT_BOOL(0, "stdin-commits", &opts.stdin_commits,
|
|
|
|
N_("start walk at commits listed by stdin")),
|
2018-04-10 12:56:08 +00:00
|
|
|
OPT_BOOL(0, "append", &opts.append,
|
|
|
|
N_("include all commits already in the commit-graph file")),
|
2018-04-02 20:34:20 +00:00
|
|
|
OPT_END(),
|
|
|
|
};
|
|
|
|
|
|
|
|
argc = parse_options(argc, argv, NULL,
|
|
|
|
builtin_commit_graph_write_options,
|
|
|
|
builtin_commit_graph_write_usage, 0);
|
|
|
|
|
2018-06-27 13:24:45 +00:00
|
|
|
if (opts.reachable + opts.stdin_packs + opts.stdin_commits > 1)
|
|
|
|
die(_("use at most one of --reachable, --stdin-commits, or --stdin-packs"));
|
2018-04-02 20:34:20 +00:00
|
|
|
if (!opts.obj_dir)
|
|
|
|
opts.obj_dir = get_object_directory();
|
|
|
|
|
2018-08-20 18:24:27 +00:00
|
|
|
read_replace_refs = 0;
|
|
|
|
|
2018-06-27 13:24:45 +00:00
|
|
|
if (opts.reachable) {
|
commit-graph write: add progress output
Before this change the "commit-graph write" command didn't report any
progress. On my machine this command takes more than 10 seconds to
write the graph for linux.git, and around 1m30s on the
2015-04-03-1M-git.git[1] test repository (a test case for a large
monorepository).
Furthermore, since the gc.writeCommitGraph setting was added in
d5d5d7b641 ("gc: automatically write commit-graph files", 2018-06-27),
there was no indication at all from a "git gc" run that anything was
different. This why one of the progress bars being added here uses
start_progress() instead of start_delayed_progress(), so that it's
guaranteed to be seen. E.g. on my tiny 867 commit dotfiles.git
repository:
$ git -c gc.writeCommitGraph=true gc
Enumerating objects: 2821, done.
[...]
Computing commit graph generation numbers: 100% (867/867), done.
On larger repositories, such as linux.git the delayed progress bar(s)
will kick in, and we'll show what's going on instead of, as was
previously happening, printing nothing while we write the graph:
$ git -c gc.writeCommitGraph=true gc
[...]
Annotating commits in commit graph: 1565573, done.
Computing commit graph generation numbers: 100% (782484/782484), done.
Note that here we don't show "Finding commits for commit graph", this
is because under "git gc" we seed the search with the commit
references in the repository, and that set is too small to show any
progress, but would e.g. on a smaller repo such as git.git with
--stdin-commits:
$ git rev-list --all | git -c gc.writeCommitGraph=true write --stdin-commits
Finding commits for commit graph: 100% (162576/162576), done.
Computing commit graph generation numbers: 100% (162576/162576), done.
With --stdin-packs we don't show any estimation of how much is left to
do. This is because we might be processing more than one pack. We
could be less lazy here and show progress, either by detecting that
we're only processing one pack, or by first looping over the packs to
discover how many commits they have. I don't see the point in doing
that work. So instead we get (on 2015-04-03-1M-git.git):
$ echo pack-<HASH>.idx | git -c gc.writeCommitGraph=true --exec-path=$PWD commit-graph write --stdin-packs
Finding commits for commit graph: 13064614, done.
Annotating commits in commit graph: 3001341, done.
Computing commit graph generation numbers: 100% (1000447/1000447), done.
No GC mode uses --stdin-packs. It's what they use at Microsoft to
manually compute the generation numbers for their collection of large
packs which are never coalesced.
The reason we need a "report_progress" variable passed down from "git
gc" is so that we don't report this output when we're running in the
process "git gc --auto" detaches from the terminal.
Since we write the commit graph from the "git gc" process itself (as
opposed to what we do with say the "git repack" phase), we'd end up
writing the output to .git/gc.log and reporting it to the user next
time as part of the "The last gc run reported the following[...]"
error, see 329e6e8794 ("gc: save log from daemonized gc --auto and
print it next time", 2015-09-19).
So we must keep track of whether or not we're running in that
demonized mode, and if so print no progress.
See [2] and subsequent replies for a discussion of an approach not
taken in compute_generation_numbers(). I.e. we're saying "Computing
commit graph generation numbers", even though on an established
history we're mostly skipping over all the work we did in the
past. This is similar to the white lie we tell in the "Writing
objects" phase (not all are objects being written).
Always showing progress is considered more important than
accuracy. I.e. on a repository like 2015-04-03-1M-git.git we'd hang
for 6 seconds with no output on the second "git gc" if no changes were
made to any objects in the interim if we'd take the approach in [2].
1. https://github.com/avar/2015-04-03-1M-git
2. <c6960252-c095-fb2b-e0bc-b1e6bb261614@gmail.com>
(https://public-inbox.org/git/c6960252-c095-fb2b-e0bc-b1e6bb261614@gmail.com/)
Signed-off-by: Ævar Arnfjörð Bjarmason <avarab@gmail.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2018-09-17 15:33:35 +00:00
|
|
|
write_commit_graph_reachable(opts.obj_dir, opts.append, 1);
|
2018-06-27 13:24:45 +00:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2018-06-27 13:24:44 +00:00
|
|
|
string_list_init(&lines, 0);
|
2018-04-10 12:56:07 +00:00
|
|
|
if (opts.stdin_packs || opts.stdin_commits) {
|
2018-04-10 12:56:06 +00:00
|
|
|
struct strbuf buf = STRBUF_INIT;
|
2018-06-27 13:24:44 +00:00
|
|
|
|
|
|
|
while (strbuf_getline(&buf, stdin) != EOF)
|
|
|
|
string_list_append(&lines, strbuf_detach(&buf, NULL));
|
|
|
|
|
|
|
|
if (opts.stdin_packs)
|
|
|
|
pack_indexes = &lines;
|
|
|
|
if (opts.stdin_commits)
|
|
|
|
commit_hex = &lines;
|
2018-10-03 17:12:17 +00:00
|
|
|
|
|
|
|
UNLEAK(buf);
|
2018-04-10 12:56:06 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
write_commit_graph(opts.obj_dir,
|
|
|
|
pack_indexes,
|
2018-04-10 12:56:07 +00:00
|
|
|
commit_hex,
|
commit-graph write: add progress output
Before this change the "commit-graph write" command didn't report any
progress. On my machine this command takes more than 10 seconds to
write the graph for linux.git, and around 1m30s on the
2015-04-03-1M-git.git[1] test repository (a test case for a large
monorepository).
Furthermore, since the gc.writeCommitGraph setting was added in
d5d5d7b641 ("gc: automatically write commit-graph files", 2018-06-27),
there was no indication at all from a "git gc" run that anything was
different. This why one of the progress bars being added here uses
start_progress() instead of start_delayed_progress(), so that it's
guaranteed to be seen. E.g. on my tiny 867 commit dotfiles.git
repository:
$ git -c gc.writeCommitGraph=true gc
Enumerating objects: 2821, done.
[...]
Computing commit graph generation numbers: 100% (867/867), done.
On larger repositories, such as linux.git the delayed progress bar(s)
will kick in, and we'll show what's going on instead of, as was
previously happening, printing nothing while we write the graph:
$ git -c gc.writeCommitGraph=true gc
[...]
Annotating commits in commit graph: 1565573, done.
Computing commit graph generation numbers: 100% (782484/782484), done.
Note that here we don't show "Finding commits for commit graph", this
is because under "git gc" we seed the search with the commit
references in the repository, and that set is too small to show any
progress, but would e.g. on a smaller repo such as git.git with
--stdin-commits:
$ git rev-list --all | git -c gc.writeCommitGraph=true write --stdin-commits
Finding commits for commit graph: 100% (162576/162576), done.
Computing commit graph generation numbers: 100% (162576/162576), done.
With --stdin-packs we don't show any estimation of how much is left to
do. This is because we might be processing more than one pack. We
could be less lazy here and show progress, either by detecting that
we're only processing one pack, or by first looping over the packs to
discover how many commits they have. I don't see the point in doing
that work. So instead we get (on 2015-04-03-1M-git.git):
$ echo pack-<HASH>.idx | git -c gc.writeCommitGraph=true --exec-path=$PWD commit-graph write --stdin-packs
Finding commits for commit graph: 13064614, done.
Annotating commits in commit graph: 3001341, done.
Computing commit graph generation numbers: 100% (1000447/1000447), done.
No GC mode uses --stdin-packs. It's what they use at Microsoft to
manually compute the generation numbers for their collection of large
packs which are never coalesced.
The reason we need a "report_progress" variable passed down from "git
gc" is so that we don't report this output when we're running in the
process "git gc --auto" detaches from the terminal.
Since we write the commit graph from the "git gc" process itself (as
opposed to what we do with say the "git repack" phase), we'd end up
writing the output to .git/gc.log and reporting it to the user next
time as part of the "The last gc run reported the following[...]"
error, see 329e6e8794 ("gc: save log from daemonized gc --auto and
print it next time", 2015-09-19).
So we must keep track of whether or not we're running in that
demonized mode, and if so print no progress.
See [2] and subsequent replies for a discussion of an approach not
taken in compute_generation_numbers(). I.e. we're saying "Computing
commit graph generation numbers", even though on an established
history we're mostly skipping over all the work we did in the
past. This is similar to the white lie we tell in the "Writing
objects" phase (not all are objects being written).
Always showing progress is considered more important than
accuracy. I.e. on a repository like 2015-04-03-1M-git.git we'd hang
for 6 seconds with no output on the second "git gc" if no changes were
made to any objects in the interim if we'd take the approach in [2].
1. https://github.com/avar/2015-04-03-1M-git
2. <c6960252-c095-fb2b-e0bc-b1e6bb261614@gmail.com>
(https://public-inbox.org/git/c6960252-c095-fb2b-e0bc-b1e6bb261614@gmail.com/)
Signed-off-by: Ævar Arnfjörð Bjarmason <avarab@gmail.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2018-09-17 15:33:35 +00:00
|
|
|
opts.append,
|
|
|
|
1);
|
2018-04-10 12:56:06 +00:00
|
|
|
|
2018-10-03 17:12:17 +00:00
|
|
|
UNLEAK(lines);
|
2018-04-02 20:34:20 +00:00
|
|
|
return 0;
|
|
|
|
}
|
2018-04-02 20:34:18 +00:00
|
|
|
|
|
|
|
int cmd_commit_graph(int argc, const char **argv, const char *prefix)
|
|
|
|
{
|
|
|
|
static struct option builtin_commit_graph_options[] = {
|
|
|
|
OPT_STRING(0, "object-dir", &opts.obj_dir,
|
|
|
|
N_("dir"),
|
|
|
|
N_("The object directory to store the graph")),
|
|
|
|
OPT_END(),
|
|
|
|
};
|
|
|
|
|
|
|
|
if (argc == 2 && !strcmp(argv[1], "-h"))
|
|
|
|
usage_with_options(builtin_commit_graph_usage,
|
|
|
|
builtin_commit_graph_options);
|
|
|
|
|
|
|
|
git_config(git_default_config, NULL);
|
|
|
|
argc = parse_options(argc, argv, prefix,
|
|
|
|
builtin_commit_graph_options,
|
|
|
|
builtin_commit_graph_usage,
|
|
|
|
PARSE_OPT_STOP_AT_NON_OPTION);
|
|
|
|
|
2018-04-02 20:34:20 +00:00
|
|
|
if (argc > 0) {
|
2018-04-10 12:56:02 +00:00
|
|
|
if (!strcmp(argv[0], "read"))
|
|
|
|
return graph_read(argc, argv);
|
2018-06-27 13:24:32 +00:00
|
|
|
if (!strcmp(argv[0], "verify"))
|
|
|
|
return graph_verify(argc, argv);
|
2018-04-02 20:34:20 +00:00
|
|
|
if (!strcmp(argv[0], "write"))
|
|
|
|
return graph_write(argc, argv);
|
|
|
|
}
|
|
|
|
|
2018-04-02 20:34:18 +00:00
|
|
|
usage_with_options(builtin_commit_graph_usage,
|
|
|
|
builtin_commit_graph_options);
|
|
|
|
}
|