git/builtin/multi-pack-index.c

292 lines
7.8 KiB
C
Raw Normal View History

#include "builtin.h"
#include "abspath.h"
#include "config.h"
#include "environment.h"
#include "gettext.h"
#include "parse-options.h"
#include "midx.h"
#include "strbuf.h"
#include "trace2.h"
#include "object-store-ll.h"
#define BUILTIN_MIDX_WRITE_USAGE \
midx: preliminary support for `--refs-snapshot` To figure out which commits we can write a bitmap for, the multi-pack index/bitmap code does a reachability traversal, marking any commit which can be found in the MIDX as eligible to receive a bitmap. This approach will cause a problem when multi-pack bitmaps are able to be generated from `git repack`, since the reference tips can change during the repack. Even though we ignore commits that don't exist in the MIDX (when doing a scan of the ref tips), it's possible that a commit in the MIDX reaches something that isn't. This can happen when a multi-pack index contains some pack which refers to loose objects (e.g., if a pack was pushed after starting the repack but before generating the MIDX which depends on an object which is stored as loose in the repository, and by definition isn't included in the multi-pack index). By taking a snapshot of the references before we start repacking, we can close that race window. In the above scenario (where we have a packed object pointing at a loose one), we'll either (a) take a snapshot of the references before seeing the packed one, or (b) take it after, at which point we can guarantee that the loose object will be packed and included in the MIDX. This patch does just that. It writes a temporary "reference snapshot", which is a list of OIDs that are at the ref tips before writing a multi-pack bitmap. References that are "preferred" (i.e,. are a suffix of at least one value of the 'pack.preferBitmapTips' configuration) are marked with a special '+'. The format is simple: one line per commit at each tip, with an optional '+' at the beginning (for preferred references, as described above). When provided, the reference snapshot is used to drive bitmap selection instead of the MIDX code doing its own traversal. When it isn't provided, the usual traversal takes place instead. Signed-off-by: Taylor Blau <me@ttaylorr.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>
2021-09-29 01:55:07 +00:00
N_("git multi-pack-index [<options>] write [--preferred-pack=<pack>]" \
"[--refs-snapshot=<path>]")
#define BUILTIN_MIDX_VERIFY_USAGE \
N_("git multi-pack-index [<options>] verify")
#define BUILTIN_MIDX_EXPIRE_USAGE \
N_("git multi-pack-index [<options>] expire")
#define BUILTIN_MIDX_REPACK_USAGE \
N_("git multi-pack-index [<options>] repack [--batch-size=<size>]")
static char const * const builtin_multi_pack_index_write_usage[] = {
BUILTIN_MIDX_WRITE_USAGE,
NULL
};
static char const * const builtin_multi_pack_index_verify_usage[] = {
BUILTIN_MIDX_VERIFY_USAGE,
NULL
};
static char const * const builtin_multi_pack_index_expire_usage[] = {
BUILTIN_MIDX_EXPIRE_USAGE,
NULL
};
static char const * const builtin_multi_pack_index_repack_usage[] = {
BUILTIN_MIDX_REPACK_USAGE,
NULL
};
static char const * const builtin_multi_pack_index_usage[] = {
BUILTIN_MIDX_WRITE_USAGE,
BUILTIN_MIDX_VERIFY_USAGE,
BUILTIN_MIDX_EXPIRE_USAGE,
BUILTIN_MIDX_REPACK_USAGE,
NULL
};
static struct opts_multi_pack_index {
multi-pack-index: use --object-dir real path The --object-dir argument to 'git multi-pack-index' allows a user to specify an alternate to use instead of the local $GITDIR. This is used by third-party tools like VFS for Git to maintain the pack-files in a "shared object cache" used by multiple clones. On Windows, the user can specify a path using a Windows-style file path with backslashes such as "C:\Path\To\ObjectDir". This same path style is used in the .git/objects/info/alternates file, so it already matches the path of that alternate. However, find_odb() converts these paths to real-paths for the comparison, which use forward slashes. As of the previous change, lookup_multi_pack_index() uses real-paths, so it correctly finds the target multi-pack-index when given these paths. Some commands such as 'git multi-pack-index repack' call child processes using the object_dir value, so it can be helpful to convert the path to the real-path before sending it to those locations. Add a callback to convert the real path immediately upon parsing the argument. We need to be careful that we don't store the exact value out of get_object_directory() and free it, or we could corrupt a later use of the_repository->objects->odb->path. We don't use get_object_directory() for the initial instantiation in cmd_multi_pack_index() because we need 'git multi-pack-index -h' to work without a Git repository. Signed-off-by: Derrick Stolee <derrickstolee@github.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>
2022-04-25 18:27:13 +00:00
char *object_dir;
const char *preferred_pack;
midx: preliminary support for `--refs-snapshot` To figure out which commits we can write a bitmap for, the multi-pack index/bitmap code does a reachability traversal, marking any commit which can be found in the MIDX as eligible to receive a bitmap. This approach will cause a problem when multi-pack bitmaps are able to be generated from `git repack`, since the reference tips can change during the repack. Even though we ignore commits that don't exist in the MIDX (when doing a scan of the ref tips), it's possible that a commit in the MIDX reaches something that isn't. This can happen when a multi-pack index contains some pack which refers to loose objects (e.g., if a pack was pushed after starting the repack but before generating the MIDX which depends on an object which is stored as loose in the repository, and by definition isn't included in the multi-pack index). By taking a snapshot of the references before we start repacking, we can close that race window. In the above scenario (where we have a packed object pointing at a loose one), we'll either (a) take a snapshot of the references before seeing the packed one, or (b) take it after, at which point we can guarantee that the loose object will be packed and included in the MIDX. This patch does just that. It writes a temporary "reference snapshot", which is a list of OIDs that are at the ref tips before writing a multi-pack bitmap. References that are "preferred" (i.e,. are a suffix of at least one value of the 'pack.preferBitmapTips' configuration) are marked with a special '+'. The format is simple: one line per commit at each tip, with an optional '+' at the beginning (for preferred references, as described above). When provided, the reference snapshot is used to drive bitmap selection instead of the MIDX code doing its own traversal. When it isn't provided, the usual traversal takes place instead. Signed-off-by: Taylor Blau <me@ttaylorr.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>
2021-09-29 01:55:07 +00:00
const char *refs_snapshot;
multi-pack-index: prepare 'repack' subcommand In an environment where the multi-pack-index is useful, it is due to many pack-files and an inability to repack the object store into a single pack-file. However, it is likely that many of these pack-files are rather small, and could be repacked into a slightly larger pack-file without too much effort. It may also be important to ensure the object store is highly available and the repack operation does not interrupt concurrent git commands. Introduce a 'repack' subcommand to 'git multi-pack-index' that takes a '--batch-size' option. The subcommand will inspect the multi-pack-index for referenced pack-files whose size is smaller than the batch size, until collecting a list of pack-files whose sizes sum to larger than the batch size. Then, a new pack-file will be created containing the objects from those pack-files that are referenced by the multi-pack-index. The resulting pack is likely to actually be smaller than the batch size due to compression and the fact that there may be objects in the pack- files that have duplicate copies in other pack-files. The current change introduces the command-line arguments, and we add a test that ensures we parse these options properly. Since we specify a small batch size, we will guarantee that future implementations do not change the list of pack-files. In addition, we hard-code the modified times of the packs in the pack directory to ensure the list of packs sorted by modified time matches the order if sorted by size (ascending). This will be important in a future test. Signed-off-by: Derrick Stolee <dstolee@microsoft.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>
2019-06-10 23:35:26 +00:00
unsigned long batch_size;
unsigned flags;
int stdin_packs;
} opts;
multi-pack-index: use --object-dir real path The --object-dir argument to 'git multi-pack-index' allows a user to specify an alternate to use instead of the local $GITDIR. This is used by third-party tools like VFS for Git to maintain the pack-files in a "shared object cache" used by multiple clones. On Windows, the user can specify a path using a Windows-style file path with backslashes such as "C:\Path\To\ObjectDir". This same path style is used in the .git/objects/info/alternates file, so it already matches the path of that alternate. However, find_odb() converts these paths to real-paths for the comparison, which use forward slashes. As of the previous change, lookup_multi_pack_index() uses real-paths, so it correctly finds the target multi-pack-index when given these paths. Some commands such as 'git multi-pack-index repack' call child processes using the object_dir value, so it can be helpful to convert the path to the real-path before sending it to those locations. Add a callback to convert the real path immediately upon parsing the argument. We need to be careful that we don't store the exact value out of get_object_directory() and free it, or we could corrupt a later use of the_repository->objects->odb->path. We don't use get_object_directory() for the initial instantiation in cmd_multi_pack_index() because we need 'git multi-pack-index -h' to work without a Git repository. Signed-off-by: Derrick Stolee <derrickstolee@github.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>
2022-04-25 18:27:13 +00:00
static int parse_object_dir(const struct option *opt, const char *arg,
int unset)
{
char **value = opt->value;
free(*value);
multi-pack-index: use --object-dir real path The --object-dir argument to 'git multi-pack-index' allows a user to specify an alternate to use instead of the local $GITDIR. This is used by third-party tools like VFS for Git to maintain the pack-files in a "shared object cache" used by multiple clones. On Windows, the user can specify a path using a Windows-style file path with backslashes such as "C:\Path\To\ObjectDir". This same path style is used in the .git/objects/info/alternates file, so it already matches the path of that alternate. However, find_odb() converts these paths to real-paths for the comparison, which use forward slashes. As of the previous change, lookup_multi_pack_index() uses real-paths, so it correctly finds the target multi-pack-index when given these paths. Some commands such as 'git multi-pack-index repack' call child processes using the object_dir value, so it can be helpful to convert the path to the real-path before sending it to those locations. Add a callback to convert the real path immediately upon parsing the argument. We need to be careful that we don't store the exact value out of get_object_directory() and free it, or we could corrupt a later use of the_repository->objects->odb->path. We don't use get_object_directory() for the initial instantiation in cmd_multi_pack_index() because we need 'git multi-pack-index -h' to work without a Git repository. Signed-off-by: Derrick Stolee <derrickstolee@github.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>
2022-04-25 18:27:13 +00:00
if (unset)
*value = xstrdup(get_object_directory());
multi-pack-index: use --object-dir real path The --object-dir argument to 'git multi-pack-index' allows a user to specify an alternate to use instead of the local $GITDIR. This is used by third-party tools like VFS for Git to maintain the pack-files in a "shared object cache" used by multiple clones. On Windows, the user can specify a path using a Windows-style file path with backslashes such as "C:\Path\To\ObjectDir". This same path style is used in the .git/objects/info/alternates file, so it already matches the path of that alternate. However, find_odb() converts these paths to real-paths for the comparison, which use forward slashes. As of the previous change, lookup_multi_pack_index() uses real-paths, so it correctly finds the target multi-pack-index when given these paths. Some commands such as 'git multi-pack-index repack' call child processes using the object_dir value, so it can be helpful to convert the path to the real-path before sending it to those locations. Add a callback to convert the real path immediately upon parsing the argument. We need to be careful that we don't store the exact value out of get_object_directory() and free it, or we could corrupt a later use of the_repository->objects->odb->path. We don't use get_object_directory() for the initial instantiation in cmd_multi_pack_index() because we need 'git multi-pack-index -h' to work without a Git repository. Signed-off-by: Derrick Stolee <derrickstolee@github.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>
2022-04-25 18:27:13 +00:00
else
*value = real_pathdup(arg, 1);
multi-pack-index: use --object-dir real path The --object-dir argument to 'git multi-pack-index' allows a user to specify an alternate to use instead of the local $GITDIR. This is used by third-party tools like VFS for Git to maintain the pack-files in a "shared object cache" used by multiple clones. On Windows, the user can specify a path using a Windows-style file path with backslashes such as "C:\Path\To\ObjectDir". This same path style is used in the .git/objects/info/alternates file, so it already matches the path of that alternate. However, find_odb() converts these paths to real-paths for the comparison, which use forward slashes. As of the previous change, lookup_multi_pack_index() uses real-paths, so it correctly finds the target multi-pack-index when given these paths. Some commands such as 'git multi-pack-index repack' call child processes using the object_dir value, so it can be helpful to convert the path to the real-path before sending it to those locations. Add a callback to convert the real path immediately upon parsing the argument. We need to be careful that we don't store the exact value out of get_object_directory() and free it, or we could corrupt a later use of the_repository->objects->odb->path. We don't use get_object_directory() for the initial instantiation in cmd_multi_pack_index() because we need 'git multi-pack-index -h' to work without a Git repository. Signed-off-by: Derrick Stolee <derrickstolee@github.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>
2022-04-25 18:27:13 +00:00
return 0;
}
static struct option common_opts[] = {
multi-pack-index: use --object-dir real path The --object-dir argument to 'git multi-pack-index' allows a user to specify an alternate to use instead of the local $GITDIR. This is used by third-party tools like VFS for Git to maintain the pack-files in a "shared object cache" used by multiple clones. On Windows, the user can specify a path using a Windows-style file path with backslashes such as "C:\Path\To\ObjectDir". This same path style is used in the .git/objects/info/alternates file, so it already matches the path of that alternate. However, find_odb() converts these paths to real-paths for the comparison, which use forward slashes. As of the previous change, lookup_multi_pack_index() uses real-paths, so it correctly finds the target multi-pack-index when given these paths. Some commands such as 'git multi-pack-index repack' call child processes using the object_dir value, so it can be helpful to convert the path to the real-path before sending it to those locations. Add a callback to convert the real path immediately upon parsing the argument. We need to be careful that we don't store the exact value out of get_object_directory() and free it, or we could corrupt a later use of the_repository->objects->odb->path. We don't use get_object_directory() for the initial instantiation in cmd_multi_pack_index() because we need 'git multi-pack-index -h' to work without a Git repository. Signed-off-by: Derrick Stolee <derrickstolee@github.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>
2022-04-25 18:27:13 +00:00
OPT_CALLBACK(0, "object-dir", &opts.object_dir,
N_("directory"),
N_("object directory containing set of packfile and pack-index pairs"),
parse_object_dir),
OPT_END(),
};
static struct option *add_common_options(struct option *prev)
{
return parse_options_concat(common_opts, prev);
}
static int git_multi_pack_index_write_config(const char *var, const char *value,
config: add ctx arg to config_fn_t Add a new "const struct config_context *ctx" arg to config_fn_t to hold additional information about the config iteration operation. config_context has a "struct key_value_info kvi" member that holds metadata about the config source being read (e.g. what kind of config source it is, the filename, etc). In this series, we're only interested in .kvi, so we could have just used "struct key_value_info" as an arg, but config_context makes it possible to add/adjust members in the future without changing the config_fn_t signature. We could also consider other ways of organizing the args (e.g. moving the config name and value into config_context or key_value_info), but in my experiments, the incremental benefit doesn't justify the added complexity (e.g. a config_fn_t will sometimes invoke another config_fn_t but with a different config value). In subsequent commits, the .kvi member will replace the global "struct config_reader" in config.c, making config iteration a global-free operation. It requires much more work for the machinery to provide meaningful values of .kvi, so for now, merely change the signature and call sites, pass NULL as a placeholder value, and don't rely on the arg in any meaningful way. Most of the changes are performed by contrib/coccinelle/config_fn_ctx.pending.cocci, which, for every config_fn_t: - Modifies the signature to accept "const struct config_context *ctx" - Passes "ctx" to any inner config_fn_t, if needed - Adds UNUSED attributes to "ctx", if needed Most config_fn_t instances are easily identified by seeing if they are called by the various config functions. Most of the remaining ones are manually named in the .cocci patch. Manual cleanups are still needed, but the majority of it is trivial; it's either adjusting config_fn_t that the .cocci patch didn't catch, or adding forward declarations of "struct config_context ctx" to make the signatures make sense. The non-trivial changes are in cases where we are invoking a config_fn_t outside of config machinery, and we now need to decide what value of "ctx" to pass. These cases are: - trace2/tr2_cfg.c:tr2_cfg_set_fl() This is indirectly called by git_config_set() so that the trace2 machinery can notice the new config values and update its settings using the tr2 config parsing function, i.e. tr2_cfg_cb(). - builtin/checkout.c:checkout_main() This calls git_xmerge_config() as a shorthand for parsing a CLI arg. This might be worth refactoring away in the future, since git_xmerge_config() can call git_default_config(), which can do much more than just parsing. Handle them by creating a KVI_INIT macro that initializes "struct key_value_info" to a reasonable default, and use that to construct the "ctx" arg. Signed-off-by: Glen Choo <chooglen@google.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>
2023-06-28 19:26:22 +00:00
const struct config_context *ctx UNUSED,
void *cb UNUSED)
{
if (!strcmp(var, "pack.writebitmaphashcache")) {
if (git_config_bool(var, value))
opts.flags |= MIDX_WRITE_BITMAP_HASH_CACHE;
else
opts.flags &= ~MIDX_WRITE_BITMAP_HASH_CACHE;
}
if (!strcmp(var, "pack.writebitmaplookuptable")) {
if (git_config_bool(var, value))
opts.flags |= MIDX_WRITE_BITMAP_LOOKUP_TABLE;
else
opts.flags &= ~MIDX_WRITE_BITMAP_LOOKUP_TABLE;
}
/*
* We should never make a fall-back call to 'git_default_config', since
* this was already called in 'cmd_multi_pack_index()'.
*/
return 0;
}
static void read_packs_from_stdin(struct string_list *to)
{
struct strbuf buf = STRBUF_INIT;
while (strbuf_getline(&buf, stdin) != EOF)
string_list_append(to, buf.buf);
string_list_sort(to);
strbuf_release(&buf);
}
static int cmd_multi_pack_index_write(int argc, const char **argv,
const char *prefix)
{
struct option *options;
static struct option builtin_multi_pack_index_write_options[] = {
OPT_STRING(0, "preferred-pack", &opts.preferred_pack,
N_("preferred-pack"),
N_("pack for reuse when computing a multi-pack bitmap")),
OPT_BIT(0, "bitmap", &opts.flags, N_("write multi-pack bitmap"),
MIDX_WRITE_BITMAP | MIDX_WRITE_REV_INDEX),
OPT_BIT(0, "progress", &opts.flags,
N_("force progress reporting"), MIDX_PROGRESS),
OPT_BOOL(0, "stdin-packs", &opts.stdin_packs,
N_("write multi-pack index containing only given indexes")),
midx: preliminary support for `--refs-snapshot` To figure out which commits we can write a bitmap for, the multi-pack index/bitmap code does a reachability traversal, marking any commit which can be found in the MIDX as eligible to receive a bitmap. This approach will cause a problem when multi-pack bitmaps are able to be generated from `git repack`, since the reference tips can change during the repack. Even though we ignore commits that don't exist in the MIDX (when doing a scan of the ref tips), it's possible that a commit in the MIDX reaches something that isn't. This can happen when a multi-pack index contains some pack which refers to loose objects (e.g., if a pack was pushed after starting the repack but before generating the MIDX which depends on an object which is stored as loose in the repository, and by definition isn't included in the multi-pack index). By taking a snapshot of the references before we start repacking, we can close that race window. In the above scenario (where we have a packed object pointing at a loose one), we'll either (a) take a snapshot of the references before seeing the packed one, or (b) take it after, at which point we can guarantee that the loose object will be packed and included in the MIDX. This patch does just that. It writes a temporary "reference snapshot", which is a list of OIDs that are at the ref tips before writing a multi-pack bitmap. References that are "preferred" (i.e,. are a suffix of at least one value of the 'pack.preferBitmapTips' configuration) are marked with a special '+'. The format is simple: one line per commit at each tip, with an optional '+' at the beginning (for preferred references, as described above). When provided, the reference snapshot is used to drive bitmap selection instead of the MIDX code doing its own traversal. When it isn't provided, the usual traversal takes place instead. Signed-off-by: Taylor Blau <me@ttaylorr.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>
2021-09-29 01:55:07 +00:00
OPT_FILENAME(0, "refs-snapshot", &opts.refs_snapshot,
N_("refs snapshot for selecting bitmap commits")),
OPT_END(),
};
opts.flags |= MIDX_WRITE_BITMAP_HASH_CACHE;
git_config(git_multi_pack_index_write_config, NULL);
options = add_common_options(builtin_multi_pack_index_write_options);
trace2_cmd_mode(argv[0]);
if (isatty(2))
opts.flags |= MIDX_PROGRESS;
pass subcommand "prefix" arguments to parse_options() Recent commits such as bf0a6b65fc (builtin/multi-pack-index.c: let parse-options parse subcommands, 2022-08-19) converted a few functions to match our usual argc/argv/prefix conventions, but the prefix argument remains unused. However, there is a good use for it: they should pass it to their own parse_options() functions, where it may be used to adjust the value of any filename options. In all but one of these functions, there's no behavior change, since they don't use OPT_FILENAME. But this is an actual fix for one option, which you can see by modifying the test suite like so: diff --git a/t/t5326-multi-pack-bitmaps.sh b/t/t5326-multi-pack-bitmaps.sh index 4fe57414c1..d0974d4371 100755 --- a/t/t5326-multi-pack-bitmaps.sh +++ b/t/t5326-multi-pack-bitmaps.sh @@ -186,7 +186,11 @@ test_expect_success 'writing a bitmap with --refs-snapshot' ' # Then again, but with a refs snapshot which only sees # refs/tags/one. - git multi-pack-index write --bitmap --refs-snapshot=snapshot && + ( + mkdir subdir && + cd subdir && + git multi-pack-index write --bitmap --refs-snapshot=../snapshot + ) && test_path_is_file $midx && test_path_is_file $midx-$(midx_checksum $objdir).bitmap && I'd emphasize that this wasn't broken by bf0a6b65fc; it has been broken all along, because the sub-function never got to see the prefix. It is that commit which is actually enabling us to fix it (and which also brought attention to the problem because it triggers -Wunused-parameter!) The other functions changed here don't use OPT_FILENAME at all. In their cases this isn't fixing anything visible, but it's following the usual pattern and future-proofing them against somebody adding new options and being surprised. I didn't include a test for the one visible case above. We don't generally test routine parse-options behavior for individual options. The challenge here was finding the problem, and now that this has been done, it's not likely to regress. Likewise, we could apply the patch above to cover it "for free" but it makes reading the rest of the test unnecessarily complicated. Signed-off-by: Jeff King <peff@peff.net> Signed-off-by: Junio C Hamano <gitster@pobox.com>
2022-08-25 10:47:00 +00:00
argc = parse_options(argc, argv, prefix,
options, builtin_multi_pack_index_write_usage,
0);
if (argc)
usage_with_options(builtin_multi_pack_index_write_usage,
options);
FREE_AND_NULL(options);
if (opts.stdin_packs) {
struct string_list packs = STRING_LIST_INIT_DUP;
int ret;
read_packs_from_stdin(&packs);
ret = write_midx_file_only(opts.object_dir, &packs,
midx: preliminary support for `--refs-snapshot` To figure out which commits we can write a bitmap for, the multi-pack index/bitmap code does a reachability traversal, marking any commit which can be found in the MIDX as eligible to receive a bitmap. This approach will cause a problem when multi-pack bitmaps are able to be generated from `git repack`, since the reference tips can change during the repack. Even though we ignore commits that don't exist in the MIDX (when doing a scan of the ref tips), it's possible that a commit in the MIDX reaches something that isn't. This can happen when a multi-pack index contains some pack which refers to loose objects (e.g., if a pack was pushed after starting the repack but before generating the MIDX which depends on an object which is stored as loose in the repository, and by definition isn't included in the multi-pack index). By taking a snapshot of the references before we start repacking, we can close that race window. In the above scenario (where we have a packed object pointing at a loose one), we'll either (a) take a snapshot of the references before seeing the packed one, or (b) take it after, at which point we can guarantee that the loose object will be packed and included in the MIDX. This patch does just that. It writes a temporary "reference snapshot", which is a list of OIDs that are at the ref tips before writing a multi-pack bitmap. References that are "preferred" (i.e,. are a suffix of at least one value of the 'pack.preferBitmapTips' configuration) are marked with a special '+'. The format is simple: one line per commit at each tip, with an optional '+' at the beginning (for preferred references, as described above). When provided, the reference snapshot is used to drive bitmap selection instead of the MIDX code doing its own traversal. When it isn't provided, the usual traversal takes place instead. Signed-off-by: Taylor Blau <me@ttaylorr.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>
2021-09-29 01:55:07 +00:00
opts.preferred_pack,
opts.refs_snapshot, opts.flags);
string_list_clear(&packs, 0);
return ret;
}
return write_midx_file(opts.object_dir, opts.preferred_pack,
midx: preliminary support for `--refs-snapshot` To figure out which commits we can write a bitmap for, the multi-pack index/bitmap code does a reachability traversal, marking any commit which can be found in the MIDX as eligible to receive a bitmap. This approach will cause a problem when multi-pack bitmaps are able to be generated from `git repack`, since the reference tips can change during the repack. Even though we ignore commits that don't exist in the MIDX (when doing a scan of the ref tips), it's possible that a commit in the MIDX reaches something that isn't. This can happen when a multi-pack index contains some pack which refers to loose objects (e.g., if a pack was pushed after starting the repack but before generating the MIDX which depends on an object which is stored as loose in the repository, and by definition isn't included in the multi-pack index). By taking a snapshot of the references before we start repacking, we can close that race window. In the above scenario (where we have a packed object pointing at a loose one), we'll either (a) take a snapshot of the references before seeing the packed one, or (b) take it after, at which point we can guarantee that the loose object will be packed and included in the MIDX. This patch does just that. It writes a temporary "reference snapshot", which is a list of OIDs that are at the ref tips before writing a multi-pack bitmap. References that are "preferred" (i.e,. are a suffix of at least one value of the 'pack.preferBitmapTips' configuration) are marked with a special '+'. The format is simple: one line per commit at each tip, with an optional '+' at the beginning (for preferred references, as described above). When provided, the reference snapshot is used to drive bitmap selection instead of the MIDX code doing its own traversal. When it isn't provided, the usual traversal takes place instead. Signed-off-by: Taylor Blau <me@ttaylorr.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>
2021-09-29 01:55:07 +00:00
opts.refs_snapshot, opts.flags);
}
static int cmd_multi_pack_index_verify(int argc, const char **argv,
const char *prefix)
{
struct option *options;
static struct option builtin_multi_pack_index_verify_options[] = {
OPT_BIT(0, "progress", &opts.flags,
N_("force progress reporting"), MIDX_PROGRESS),
OPT_END(),
};
options = add_common_options(builtin_multi_pack_index_verify_options);
trace2_cmd_mode(argv[0]);
if (isatty(2))
opts.flags |= MIDX_PROGRESS;
pass subcommand "prefix" arguments to parse_options() Recent commits such as bf0a6b65fc (builtin/multi-pack-index.c: let parse-options parse subcommands, 2022-08-19) converted a few functions to match our usual argc/argv/prefix conventions, but the prefix argument remains unused. However, there is a good use for it: they should pass it to their own parse_options() functions, where it may be used to adjust the value of any filename options. In all but one of these functions, there's no behavior change, since they don't use OPT_FILENAME. But this is an actual fix for one option, which you can see by modifying the test suite like so: diff --git a/t/t5326-multi-pack-bitmaps.sh b/t/t5326-multi-pack-bitmaps.sh index 4fe57414c1..d0974d4371 100755 --- a/t/t5326-multi-pack-bitmaps.sh +++ b/t/t5326-multi-pack-bitmaps.sh @@ -186,7 +186,11 @@ test_expect_success 'writing a bitmap with --refs-snapshot' ' # Then again, but with a refs snapshot which only sees # refs/tags/one. - git multi-pack-index write --bitmap --refs-snapshot=snapshot && + ( + mkdir subdir && + cd subdir && + git multi-pack-index write --bitmap --refs-snapshot=../snapshot + ) && test_path_is_file $midx && test_path_is_file $midx-$(midx_checksum $objdir).bitmap && I'd emphasize that this wasn't broken by bf0a6b65fc; it has been broken all along, because the sub-function never got to see the prefix. It is that commit which is actually enabling us to fix it (and which also brought attention to the problem because it triggers -Wunused-parameter!) The other functions changed here don't use OPT_FILENAME at all. In their cases this isn't fixing anything visible, but it's following the usual pattern and future-proofing them against somebody adding new options and being surprised. I didn't include a test for the one visible case above. We don't generally test routine parse-options behavior for individual options. The challenge here was finding the problem, and now that this has been done, it's not likely to regress. Likewise, we could apply the patch above to cover it "for free" but it makes reading the rest of the test unnecessarily complicated. Signed-off-by: Jeff King <peff@peff.net> Signed-off-by: Junio C Hamano <gitster@pobox.com>
2022-08-25 10:47:00 +00:00
argc = parse_options(argc, argv, prefix,
options, builtin_multi_pack_index_verify_usage,
0);
if (argc)
usage_with_options(builtin_multi_pack_index_verify_usage,
options);
FREE_AND_NULL(options);
return verify_midx_file(the_repository, opts.object_dir, opts.flags);
}
static int cmd_multi_pack_index_expire(int argc, const char **argv,
const char *prefix)
{
struct option *options;
static struct option builtin_multi_pack_index_expire_options[] = {
OPT_BIT(0, "progress", &opts.flags,
N_("force progress reporting"), MIDX_PROGRESS),
OPT_END(),
};
options = add_common_options(builtin_multi_pack_index_expire_options);
trace2_cmd_mode(argv[0]);
if (isatty(2))
opts.flags |= MIDX_PROGRESS;
pass subcommand "prefix" arguments to parse_options() Recent commits such as bf0a6b65fc (builtin/multi-pack-index.c: let parse-options parse subcommands, 2022-08-19) converted a few functions to match our usual argc/argv/prefix conventions, but the prefix argument remains unused. However, there is a good use for it: they should pass it to their own parse_options() functions, where it may be used to adjust the value of any filename options. In all but one of these functions, there's no behavior change, since they don't use OPT_FILENAME. But this is an actual fix for one option, which you can see by modifying the test suite like so: diff --git a/t/t5326-multi-pack-bitmaps.sh b/t/t5326-multi-pack-bitmaps.sh index 4fe57414c1..d0974d4371 100755 --- a/t/t5326-multi-pack-bitmaps.sh +++ b/t/t5326-multi-pack-bitmaps.sh @@ -186,7 +186,11 @@ test_expect_success 'writing a bitmap with --refs-snapshot' ' # Then again, but with a refs snapshot which only sees # refs/tags/one. - git multi-pack-index write --bitmap --refs-snapshot=snapshot && + ( + mkdir subdir && + cd subdir && + git multi-pack-index write --bitmap --refs-snapshot=../snapshot + ) && test_path_is_file $midx && test_path_is_file $midx-$(midx_checksum $objdir).bitmap && I'd emphasize that this wasn't broken by bf0a6b65fc; it has been broken all along, because the sub-function never got to see the prefix. It is that commit which is actually enabling us to fix it (and which also brought attention to the problem because it triggers -Wunused-parameter!) The other functions changed here don't use OPT_FILENAME at all. In their cases this isn't fixing anything visible, but it's following the usual pattern and future-proofing them against somebody adding new options and being surprised. I didn't include a test for the one visible case above. We don't generally test routine parse-options behavior for individual options. The challenge here was finding the problem, and now that this has been done, it's not likely to regress. Likewise, we could apply the patch above to cover it "for free" but it makes reading the rest of the test unnecessarily complicated. Signed-off-by: Jeff King <peff@peff.net> Signed-off-by: Junio C Hamano <gitster@pobox.com>
2022-08-25 10:47:00 +00:00
argc = parse_options(argc, argv, prefix,
options, builtin_multi_pack_index_expire_usage,
0);
if (argc)
usage_with_options(builtin_multi_pack_index_expire_usage,
options);
FREE_AND_NULL(options);
return expire_midx_packs(the_repository, opts.object_dir, opts.flags);
}
static int cmd_multi_pack_index_repack(int argc, const char **argv,
const char *prefix)
{
struct option *options;
static struct option builtin_multi_pack_index_repack_options[] = {
multi-pack-index: prepare 'repack' subcommand In an environment where the multi-pack-index is useful, it is due to many pack-files and an inability to repack the object store into a single pack-file. However, it is likely that many of these pack-files are rather small, and could be repacked into a slightly larger pack-file without too much effort. It may also be important to ensure the object store is highly available and the repack operation does not interrupt concurrent git commands. Introduce a 'repack' subcommand to 'git multi-pack-index' that takes a '--batch-size' option. The subcommand will inspect the multi-pack-index for referenced pack-files whose size is smaller than the batch size, until collecting a list of pack-files whose sizes sum to larger than the batch size. Then, a new pack-file will be created containing the objects from those pack-files that are referenced by the multi-pack-index. The resulting pack is likely to actually be smaller than the batch size due to compression and the fact that there may be objects in the pack- files that have duplicate copies in other pack-files. The current change introduces the command-line arguments, and we add a test that ensures we parse these options properly. Since we specify a small batch size, we will guarantee that future implementations do not change the list of pack-files. In addition, we hard-code the modified times of the packs in the pack directory to ensure the list of packs sorted by modified time matches the order if sorted by size (ascending). This will be important in a future test. Signed-off-by: Derrick Stolee <dstolee@microsoft.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>
2019-06-10 23:35:26 +00:00
OPT_MAGNITUDE(0, "batch-size", &opts.batch_size,
N_("during repack, collect pack-files of smaller size into a batch that is larger than this size")),
OPT_BIT(0, "progress", &opts.flags,
N_("force progress reporting"), MIDX_PROGRESS),
OPT_END(),
};
options = add_common_options(builtin_multi_pack_index_repack_options);
trace2_cmd_mode(argv[0]);
if (isatty(2))
opts.flags |= MIDX_PROGRESS;
pass subcommand "prefix" arguments to parse_options() Recent commits such as bf0a6b65fc (builtin/multi-pack-index.c: let parse-options parse subcommands, 2022-08-19) converted a few functions to match our usual argc/argv/prefix conventions, but the prefix argument remains unused. However, there is a good use for it: they should pass it to their own parse_options() functions, where it may be used to adjust the value of any filename options. In all but one of these functions, there's no behavior change, since they don't use OPT_FILENAME. But this is an actual fix for one option, which you can see by modifying the test suite like so: diff --git a/t/t5326-multi-pack-bitmaps.sh b/t/t5326-multi-pack-bitmaps.sh index 4fe57414c1..d0974d4371 100755 --- a/t/t5326-multi-pack-bitmaps.sh +++ b/t/t5326-multi-pack-bitmaps.sh @@ -186,7 +186,11 @@ test_expect_success 'writing a bitmap with --refs-snapshot' ' # Then again, but with a refs snapshot which only sees # refs/tags/one. - git multi-pack-index write --bitmap --refs-snapshot=snapshot && + ( + mkdir subdir && + cd subdir && + git multi-pack-index write --bitmap --refs-snapshot=../snapshot + ) && test_path_is_file $midx && test_path_is_file $midx-$(midx_checksum $objdir).bitmap && I'd emphasize that this wasn't broken by bf0a6b65fc; it has been broken all along, because the sub-function never got to see the prefix. It is that commit which is actually enabling us to fix it (and which also brought attention to the problem because it triggers -Wunused-parameter!) The other functions changed here don't use OPT_FILENAME at all. In their cases this isn't fixing anything visible, but it's following the usual pattern and future-proofing them against somebody adding new options and being surprised. I didn't include a test for the one visible case above. We don't generally test routine parse-options behavior for individual options. The challenge here was finding the problem, and now that this has been done, it's not likely to regress. Likewise, we could apply the patch above to cover it "for free" but it makes reading the rest of the test unnecessarily complicated. Signed-off-by: Jeff King <peff@peff.net> Signed-off-by: Junio C Hamano <gitster@pobox.com>
2022-08-25 10:47:00 +00:00
argc = parse_options(argc, argv, prefix,
options,
builtin_multi_pack_index_repack_usage,
0);
if (argc)
usage_with_options(builtin_multi_pack_index_repack_usage,
options);
FREE_AND_NULL(options);
return midx_repack(the_repository, opts.object_dir,
(size_t)opts.batch_size, opts.flags);
}
int cmd_multi_pack_index(int argc, const char **argv,
const char *prefix)
{
multi-pack-index: use --object-dir real path The --object-dir argument to 'git multi-pack-index' allows a user to specify an alternate to use instead of the local $GITDIR. This is used by third-party tools like VFS for Git to maintain the pack-files in a "shared object cache" used by multiple clones. On Windows, the user can specify a path using a Windows-style file path with backslashes such as "C:\Path\To\ObjectDir". This same path style is used in the .git/objects/info/alternates file, so it already matches the path of that alternate. However, find_odb() converts these paths to real-paths for the comparison, which use forward slashes. As of the previous change, lookup_multi_pack_index() uses real-paths, so it correctly finds the target multi-pack-index when given these paths. Some commands such as 'git multi-pack-index repack' call child processes using the object_dir value, so it can be helpful to convert the path to the real-path before sending it to those locations. Add a callback to convert the real path immediately upon parsing the argument. We need to be careful that we don't store the exact value out of get_object_directory() and free it, or we could corrupt a later use of the_repository->objects->odb->path. We don't use get_object_directory() for the initial instantiation in cmd_multi_pack_index() because we need 'git multi-pack-index -h' to work without a Git repository. Signed-off-by: Derrick Stolee <derrickstolee@github.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>
2022-04-25 18:27:13 +00:00
int res;
parse_opt_subcommand_fn *fn = NULL;
struct option builtin_multi_pack_index_options[] = {
OPT_SUBCOMMAND("repack", &fn, cmd_multi_pack_index_repack),
OPT_SUBCOMMAND("write", &fn, cmd_multi_pack_index_write),
OPT_SUBCOMMAND("verify", &fn, cmd_multi_pack_index_verify),
OPT_SUBCOMMAND("expire", &fn, cmd_multi_pack_index_expire),
OPT_END(),
};
struct option *options = parse_options_concat(builtin_multi_pack_index_options, common_opts);
git_config(git_default_config, NULL);
multi-pack-index: use --object-dir real path The --object-dir argument to 'git multi-pack-index' allows a user to specify an alternate to use instead of the local $GITDIR. This is used by third-party tools like VFS for Git to maintain the pack-files in a "shared object cache" used by multiple clones. On Windows, the user can specify a path using a Windows-style file path with backslashes such as "C:\Path\To\ObjectDir". This same path style is used in the .git/objects/info/alternates file, so it already matches the path of that alternate. However, find_odb() converts these paths to real-paths for the comparison, which use forward slashes. As of the previous change, lookup_multi_pack_index() uses real-paths, so it correctly finds the target multi-pack-index when given these paths. Some commands such as 'git multi-pack-index repack' call child processes using the object_dir value, so it can be helpful to convert the path to the real-path before sending it to those locations. Add a callback to convert the real path immediately upon parsing the argument. We need to be careful that we don't store the exact value out of get_object_directory() and free it, or we could corrupt a later use of the_repository->objects->odb->path. We don't use get_object_directory() for the initial instantiation in cmd_multi_pack_index() because we need 'git multi-pack-index -h' to work without a Git repository. Signed-off-by: Derrick Stolee <derrickstolee@github.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>
2022-04-25 18:27:13 +00:00
if (the_repository &&
the_repository->objects &&
the_repository->objects->odb)
opts.object_dir = xstrdup(the_repository->objects->odb->path);
argc = parse_options(argc, argv, prefix, options,
builtin_multi_pack_index_usage, 0);
FREE_AND_NULL(options);
res = fn(argc, argv, prefix);
multi-pack-index: use --object-dir real path The --object-dir argument to 'git multi-pack-index' allows a user to specify an alternate to use instead of the local $GITDIR. This is used by third-party tools like VFS for Git to maintain the pack-files in a "shared object cache" used by multiple clones. On Windows, the user can specify a path using a Windows-style file path with backslashes such as "C:\Path\To\ObjectDir". This same path style is used in the .git/objects/info/alternates file, so it already matches the path of that alternate. However, find_odb() converts these paths to real-paths for the comparison, which use forward slashes. As of the previous change, lookup_multi_pack_index() uses real-paths, so it correctly finds the target multi-pack-index when given these paths. Some commands such as 'git multi-pack-index repack' call child processes using the object_dir value, so it can be helpful to convert the path to the real-path before sending it to those locations. Add a callback to convert the real path immediately upon parsing the argument. We need to be careful that we don't store the exact value out of get_object_directory() and free it, or we could corrupt a later use of the_repository->objects->odb->path. We don't use get_object_directory() for the initial instantiation in cmd_multi_pack_index() because we need 'git multi-pack-index -h' to work without a Git repository. Signed-off-by: Derrick Stolee <derrickstolee@github.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>
2022-04-25 18:27:13 +00:00
free(opts.object_dir);
return res;
}