git/builtin/reflog.c

468 lines
12 KiB
C
Raw Normal View History

#include "builtin.h"
#include "config.h"
#include "gettext.h"
#include "repository.h"
#include "revision.h"
#include "reachable.h"
#include "wildmatch.h"
#include "worktree.h"
#include "reflog.h"
#include "refs.h"
#include "parse-options.h"
#define BUILTIN_REFLOG_SHOW_USAGE \
N_("git reflog [show] [<log-options>] [<ref>]")
#define BUILTIN_REFLOG_LIST_USAGE \
N_("git reflog list")
#define BUILTIN_REFLOG_EXPIRE_USAGE \
N_("git reflog expire [--expire=<time>] [--expire-unreachable=<time>]\n" \
" [--rewrite] [--updateref] [--stale-fix]\n" \
" [--dry-run | -n] [--verbose] [--all [--single-worktree] | <refs>...]")
#define BUILTIN_REFLOG_DELETE_USAGE \
N_("git reflog delete [--rewrite] [--updateref]\n" \
" [--dry-run | -n] [--verbose] <ref>@{<specifier>}...")
#define BUILTIN_REFLOG_EXISTS_USAGE \
N_("git reflog exists <ref>")
static const char *const reflog_show_usage[] = {
BUILTIN_REFLOG_SHOW_USAGE,
NULL,
};
static const char *const reflog_list_usage[] = {
BUILTIN_REFLOG_LIST_USAGE,
NULL,
};
static const char *const reflog_expire_usage[] = {
BUILTIN_REFLOG_EXPIRE_USAGE,
NULL
};
static const char *const reflog_delete_usage[] = {
BUILTIN_REFLOG_DELETE_USAGE,
NULL
};
static const char *const reflog_exists_usage[] = {
BUILTIN_REFLOG_EXISTS_USAGE,
NULL,
};
static const char *const reflog_usage[] = {
BUILTIN_REFLOG_SHOW_USAGE,
BUILTIN_REFLOG_LIST_USAGE,
BUILTIN_REFLOG_EXPIRE_USAGE,
BUILTIN_REFLOG_DELETE_USAGE,
BUILTIN_REFLOG_EXISTS_USAGE,
NULL
};
static timestamp_t default_reflog_expire;
static timestamp_t default_reflog_expire_unreachable;
struct worktree_reflogs {
struct worktree *worktree;
struct string_list reflogs;
};
static int collect_reflog(const char *ref, void *cb_data)
{
struct worktree_reflogs *cb = cb_data;
struct worktree *worktree = cb->worktree;
struct strbuf newref = STRBUF_INIT;
/*
* Avoid collecting the same shared ref multiple times because
* they are available via all worktrees.
*/
if (!worktree->is_current &&
parse_worktree_ref(ref, NULL, NULL, NULL) == REF_WORKTREE_SHARED)
return 0;
strbuf_worktree_ref(worktree, &newref, ref);
string_list_append_nodup(&cb->reflogs, strbuf_detach(&newref, NULL));
return 0;
}
static struct reflog_expire_cfg {
struct reflog_expire_cfg *next;
timestamp_t expire_total;
timestamp_t expire_unreachable;
char pattern[FLEX_ARRAY];
} *reflog_expire_cfg, **reflog_expire_cfg_tail;
static struct reflog_expire_cfg *find_cfg_ent(const char *pattern, size_t len)
{
struct reflog_expire_cfg *ent;
if (!reflog_expire_cfg_tail)
reflog_expire_cfg_tail = &reflog_expire_cfg;
for (ent = reflog_expire_cfg; ent; ent = ent->next)
if (!xstrncmpz(ent->pattern, pattern, len))
return ent;
FLEX_ALLOC_MEM(ent, pattern, pattern, len);
*reflog_expire_cfg_tail = ent;
reflog_expire_cfg_tail = &(ent->next);
return ent;
}
/* expiry timer slot */
#define EXPIRE_TOTAL 01
#define EXPIRE_UNREACH 02
config: add ctx arg to config_fn_t Add a new "const struct config_context *ctx" arg to config_fn_t to hold additional information about the config iteration operation. config_context has a "struct key_value_info kvi" member that holds metadata about the config source being read (e.g. what kind of config source it is, the filename, etc). In this series, we're only interested in .kvi, so we could have just used "struct key_value_info" as an arg, but config_context makes it possible to add/adjust members in the future without changing the config_fn_t signature. We could also consider other ways of organizing the args (e.g. moving the config name and value into config_context or key_value_info), but in my experiments, the incremental benefit doesn't justify the added complexity (e.g. a config_fn_t will sometimes invoke another config_fn_t but with a different config value). In subsequent commits, the .kvi member will replace the global "struct config_reader" in config.c, making config iteration a global-free operation. It requires much more work for the machinery to provide meaningful values of .kvi, so for now, merely change the signature and call sites, pass NULL as a placeholder value, and don't rely on the arg in any meaningful way. Most of the changes are performed by contrib/coccinelle/config_fn_ctx.pending.cocci, which, for every config_fn_t: - Modifies the signature to accept "const struct config_context *ctx" - Passes "ctx" to any inner config_fn_t, if needed - Adds UNUSED attributes to "ctx", if needed Most config_fn_t instances are easily identified by seeing if they are called by the various config functions. Most of the remaining ones are manually named in the .cocci patch. Manual cleanups are still needed, but the majority of it is trivial; it's either adjusting config_fn_t that the .cocci patch didn't catch, or adding forward declarations of "struct config_context ctx" to make the signatures make sense. The non-trivial changes are in cases where we are invoking a config_fn_t outside of config machinery, and we now need to decide what value of "ctx" to pass. These cases are: - trace2/tr2_cfg.c:tr2_cfg_set_fl() This is indirectly called by git_config_set() so that the trace2 machinery can notice the new config values and update its settings using the tr2 config parsing function, i.e. tr2_cfg_cb(). - builtin/checkout.c:checkout_main() This calls git_xmerge_config() as a shorthand for parsing a CLI arg. This might be worth refactoring away in the future, since git_xmerge_config() can call git_default_config(), which can do much more than just parsing. Handle them by creating a KVI_INIT macro that initializes "struct key_value_info" to a reasonable default, and use that to construct the "ctx" arg. Signed-off-by: Glen Choo <chooglen@google.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>
2023-06-28 19:26:22 +00:00
static int reflog_expire_config(const char *var, const char *value,
const struct config_context *ctx, void *cb)
{
const char *pattern, *key;
size_t pattern_len;
timestamp_t expire;
int slot;
struct reflog_expire_cfg *ent;
if (parse_config_key(var, "gc", &pattern, &pattern_len, &key) < 0)
config: add ctx arg to config_fn_t Add a new "const struct config_context *ctx" arg to config_fn_t to hold additional information about the config iteration operation. config_context has a "struct key_value_info kvi" member that holds metadata about the config source being read (e.g. what kind of config source it is, the filename, etc). In this series, we're only interested in .kvi, so we could have just used "struct key_value_info" as an arg, but config_context makes it possible to add/adjust members in the future without changing the config_fn_t signature. We could also consider other ways of organizing the args (e.g. moving the config name and value into config_context or key_value_info), but in my experiments, the incremental benefit doesn't justify the added complexity (e.g. a config_fn_t will sometimes invoke another config_fn_t but with a different config value). In subsequent commits, the .kvi member will replace the global "struct config_reader" in config.c, making config iteration a global-free operation. It requires much more work for the machinery to provide meaningful values of .kvi, so for now, merely change the signature and call sites, pass NULL as a placeholder value, and don't rely on the arg in any meaningful way. Most of the changes are performed by contrib/coccinelle/config_fn_ctx.pending.cocci, which, for every config_fn_t: - Modifies the signature to accept "const struct config_context *ctx" - Passes "ctx" to any inner config_fn_t, if needed - Adds UNUSED attributes to "ctx", if needed Most config_fn_t instances are easily identified by seeing if they are called by the various config functions. Most of the remaining ones are manually named in the .cocci patch. Manual cleanups are still needed, but the majority of it is trivial; it's either adjusting config_fn_t that the .cocci patch didn't catch, or adding forward declarations of "struct config_context ctx" to make the signatures make sense. The non-trivial changes are in cases where we are invoking a config_fn_t outside of config machinery, and we now need to decide what value of "ctx" to pass. These cases are: - trace2/tr2_cfg.c:tr2_cfg_set_fl() This is indirectly called by git_config_set() so that the trace2 machinery can notice the new config values and update its settings using the tr2 config parsing function, i.e. tr2_cfg_cb(). - builtin/checkout.c:checkout_main() This calls git_xmerge_config() as a shorthand for parsing a CLI arg. This might be worth refactoring away in the future, since git_xmerge_config() can call git_default_config(), which can do much more than just parsing. Handle them by creating a KVI_INIT macro that initializes "struct key_value_info" to a reasonable default, and use that to construct the "ctx" arg. Signed-off-by: Glen Choo <chooglen@google.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>
2023-06-28 19:26:22 +00:00
return git_default_config(var, value, ctx, cb);
if (!strcmp(key, "reflogexpire")) {
slot = EXPIRE_TOTAL;
if (git_config_expiry_date(&expire, var, value))
return -1;
} else if (!strcmp(key, "reflogexpireunreachable")) {
slot = EXPIRE_UNREACH;
if (git_config_expiry_date(&expire, var, value))
return -1;
} else
config: add ctx arg to config_fn_t Add a new "const struct config_context *ctx" arg to config_fn_t to hold additional information about the config iteration operation. config_context has a "struct key_value_info kvi" member that holds metadata about the config source being read (e.g. what kind of config source it is, the filename, etc). In this series, we're only interested in .kvi, so we could have just used "struct key_value_info" as an arg, but config_context makes it possible to add/adjust members in the future without changing the config_fn_t signature. We could also consider other ways of organizing the args (e.g. moving the config name and value into config_context or key_value_info), but in my experiments, the incremental benefit doesn't justify the added complexity (e.g. a config_fn_t will sometimes invoke another config_fn_t but with a different config value). In subsequent commits, the .kvi member will replace the global "struct config_reader" in config.c, making config iteration a global-free operation. It requires much more work for the machinery to provide meaningful values of .kvi, so for now, merely change the signature and call sites, pass NULL as a placeholder value, and don't rely on the arg in any meaningful way. Most of the changes are performed by contrib/coccinelle/config_fn_ctx.pending.cocci, which, for every config_fn_t: - Modifies the signature to accept "const struct config_context *ctx" - Passes "ctx" to any inner config_fn_t, if needed - Adds UNUSED attributes to "ctx", if needed Most config_fn_t instances are easily identified by seeing if they are called by the various config functions. Most of the remaining ones are manually named in the .cocci patch. Manual cleanups are still needed, but the majority of it is trivial; it's either adjusting config_fn_t that the .cocci patch didn't catch, or adding forward declarations of "struct config_context ctx" to make the signatures make sense. The non-trivial changes are in cases where we are invoking a config_fn_t outside of config machinery, and we now need to decide what value of "ctx" to pass. These cases are: - trace2/tr2_cfg.c:tr2_cfg_set_fl() This is indirectly called by git_config_set() so that the trace2 machinery can notice the new config values and update its settings using the tr2 config parsing function, i.e. tr2_cfg_cb(). - builtin/checkout.c:checkout_main() This calls git_xmerge_config() as a shorthand for parsing a CLI arg. This might be worth refactoring away in the future, since git_xmerge_config() can call git_default_config(), which can do much more than just parsing. Handle them by creating a KVI_INIT macro that initializes "struct key_value_info" to a reasonable default, and use that to construct the "ctx" arg. Signed-off-by: Glen Choo <chooglen@google.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>
2023-06-28 19:26:22 +00:00
return git_default_config(var, value, ctx, cb);
if (!pattern) {
switch (slot) {
case EXPIRE_TOTAL:
default_reflog_expire = expire;
break;
case EXPIRE_UNREACH:
default_reflog_expire_unreachable = expire;
break;
}
return 0;
}
ent = find_cfg_ent(pattern, pattern_len);
if (!ent)
return -1;
switch (slot) {
case EXPIRE_TOTAL:
ent->expire_total = expire;
break;
case EXPIRE_UNREACH:
ent->expire_unreachable = expire;
break;
}
return 0;
}
static void set_reflog_expiry_param(struct cmd_reflog_expire_cb *cb, const char *ref)
{
struct reflog_expire_cfg *ent;
if (cb->explicit_expiry == (EXPIRE_TOTAL|EXPIRE_UNREACH))
return; /* both given explicitly -- nothing to tweak */
for (ent = reflog_expire_cfg; ent; ent = ent->next) {
if (!wildmatch(ent->pattern, ref, 0)) {
if (!(cb->explicit_expiry & EXPIRE_TOTAL))
cb->expire_total = ent->expire_total;
if (!(cb->explicit_expiry & EXPIRE_UNREACH))
cb->expire_unreachable = ent->expire_unreachable;
return;
}
}
/*
* If unconfigured, make stash never expire
*/
if (!strcmp(ref, "refs/stash")) {
if (!(cb->explicit_expiry & EXPIRE_TOTAL))
cb->expire_total = 0;
if (!(cb->explicit_expiry & EXPIRE_UNREACH))
cb->expire_unreachable = 0;
return;
}
/* Nothing matched -- use the default value */
if (!(cb->explicit_expiry & EXPIRE_TOTAL))
cb->expire_total = default_reflog_expire;
if (!(cb->explicit_expiry & EXPIRE_UNREACH))
cb->expire_unreachable = default_reflog_expire_unreachable;
}
static int expire_unreachable_callback(const struct option *opt,
const char *arg,
int unset)
{
struct cmd_reflog_expire_cb *cmd = opt->value;
BUG_ON_OPT_NEG(unset);
if (parse_expiry_date(arg, &cmd->expire_unreachable))
die(_("invalid timestamp '%s' given to '--%s'"),
arg, opt->long_name);
cmd->explicit_expiry |= EXPIRE_UNREACH;
return 0;
}
static int expire_total_callback(const struct option *opt,
const char *arg,
int unset)
{
struct cmd_reflog_expire_cb *cmd = opt->value;
BUG_ON_OPT_NEG(unset);
if (parse_expiry_date(arg, &cmd->expire_total))
die(_("invalid timestamp '%s' given to '--%s'"),
arg, opt->long_name);
cmd->explicit_expiry |= EXPIRE_TOTAL;
return 0;
}
static int cmd_reflog_show(int argc, const char **argv, const char *prefix)
{
struct option options[] = {
OPT_END()
};
parse_options(argc, argv, prefix, options, reflog_show_usage,
PARSE_OPT_KEEP_DASHDASH | PARSE_OPT_KEEP_ARGV0 |
PARSE_OPT_KEEP_UNKNOWN_OPT);
return cmd_log_reflog(argc, argv, prefix);
}
static int show_reflog(const char *refname, void *cb_data UNUSED)
{
printf("%s\n", refname);
return 0;
}
static int cmd_reflog_list(int argc, const char **argv, const char *prefix)
{
struct option options[] = {
OPT_END()
};
struct ref_store *ref_store;
argc = parse_options(argc, argv, prefix, options, reflog_list_usage, 0);
if (argc)
return error(_("%s does not accept arguments: '%s'"),
"list", argv[0]);
ref_store = get_main_ref_store(the_repository);
return refs_for_each_reflog(ref_store, show_reflog, NULL);
}
static int cmd_reflog_expire(int argc, const char **argv, const char *prefix)
{
struct cmd_reflog_expire_cb cmd = { 0 };
timestamp_t now = time(NULL);
int i, status, do_all, single_worktree = 0;
unsigned int flags = 0;
int verbose = 0;
reflog_expiry_should_prune_fn *should_prune_fn = should_expire_reflog_ent;
const struct option options[] = {
OPT_BIT('n', "dry-run", &flags, N_("do not actually prune any entries"),
EXPIRE_REFLOGS_DRY_RUN),
OPT_BIT(0, "rewrite", &flags,
N_("rewrite the old SHA1 with the new SHA1 of the entry that now precedes it"),
EXPIRE_REFLOGS_REWRITE),
OPT_BIT(0, "updateref", &flags,
N_("update the reference to the value of the top reflog entry"),
EXPIRE_REFLOGS_UPDATE_REF),
OPT_BOOL(0, "verbose", &verbose, N_("print extra information on screen")),
OPT_CALLBACK_F(0, "expire", &cmd, N_("timestamp"),
N_("prune entries older than the specified time"),
PARSE_OPT_NONEG,
expire_total_callback),
OPT_CALLBACK_F(0, "expire-unreachable", &cmd, N_("timestamp"),
N_("prune entries older than <time> that are not reachable from the current tip of the branch"),
PARSE_OPT_NONEG,
expire_unreachable_callback),
OPT_BOOL(0, "stale-fix", &cmd.stalefix,
N_("prune any reflog entries that point to broken commits")),
OPT_BOOL(0, "all", &do_all, N_("process the reflogs of all references")),
OPT_BOOL(0, "single-worktree", &single_worktree,
N_("limits processing to reflogs from the current worktree only")),
OPT_END()
};
default_reflog_expire_unreachable = now - 30 * 24 * 3600;
default_reflog_expire = now - 90 * 24 * 3600;
git_config(reflog_expire_config, NULL);
save_commit_buffer = 0;
do_all = status = 0;
cmd.explicit_expiry = 0;
cmd.expire_total = default_reflog_expire;
cmd.expire_unreachable = default_reflog_expire_unreachable;
argc = parse_options(argc, argv, prefix, options, reflog_expire_usage, 0);
if (verbose)
should_prune_fn = should_expire_reflog_ent_verbose;
/*
* We can trust the commits and objects reachable from refs
* even in older repository. We cannot trust what's reachable
* from reflog if the repository was pruned with older git.
*/
if (cmd.stalefix) {
struct rev_info revs;
repo_init_revisions(the_repository, &revs, prefix);
revs.do_not_die_on_missing_objects = 1;
revs.ignore_missing = 1;
revs.ignore_missing_links = 1;
if (verbose)
printf(_("Marking reachable objects..."));
mark_reachable_objects(&revs, 0, 0, NULL);
release_revisions(&revs);
if (verbose)
putchar('\n');
}
if (do_all) {
struct worktree_reflogs collected = {
.reflogs = STRING_LIST_INIT_DUP,
};
struct string_list_item *item;
struct worktree **worktrees, **p;
worktrees = get_worktrees();
for (p = worktrees; *p; p++) {
if (single_worktree && !(*p)->is_current)
continue;
collected.worktree = *p;
refs_for_each_reflog(get_worktree_ref_store(*p),
collect_reflog, &collected);
}
free_worktrees(worktrees);
for_each_string_list_item(item, &collected.reflogs) {
struct expire_reflog_policy_cb cb = {
.cmd = cmd,
.dry_run = !!(flags & EXPIRE_REFLOGS_DRY_RUN),
};
reflog expire: don't lock reflogs using previously seen OID During reflog expiry, the cmd_reflog_expire() function first iterates over all reflogs in logs/*, and then one-by-one acquires the lock for each one and expires it. This behavior has been with us since this command was implemented in 4264dc15e1 ("git reflog expire", 2006-12-19). Change this to stop calling lock_ref_oid_basic() with the OID we saw when we looped over the logs, instead have it pass the OID it managed to lock. This mostly mitigates a race condition where e.g. "git gc" will fail in a concurrently updated repository because the branch moved since "git reflog expire --all" was started. I.e. with: error: cannot lock ref '<refname>': ref '<refname>' is at <OID-A> but expected <OID-B> This behavior of passing in an "oid" was needed for an edge-case that I've untangled in this and preceding commits though, namely that we needed this OID because we'd: 1. Lookup the reflog name/OID via dwim_log() 2. With that OID, lock the reflog 3. Later in builtin/reflog.c we use the OID we looked as input to lookup_commit_reference_gently(), assured that it's equal to the OID we got from dwim_log(). We can be sure that this change is safe to make because between dwim_log (step #1) and lock_ref_oid_basic (step #2) there was no other logic relevant to the OID or expiry run in the cmd_reflog_expire() caller. We can thus treat that code as a black box, before and after this change it would get an OID that's been locked, the only difference is that now we mostly won't be failing to get the lock due to the TOCTOU race[0]. That failure was purely an implementation detail in how the "current OID" was looked up, it was divorced from the locking mechanism. What do we mean with "mostly"? It mostly mitigates it because we'll still run into cases where the ref is locked and being updated as we want to expire it, and other git processes wanting to update the refs will in turn race with us as we expire the reflog. That remaining race can in turn be mitigated with the core.filesRefLockTimeout setting, see 4ff0f01cb7 ("refs: retry acquiring reference locks for 100ms", 2017-08-21). In practice if that value is high enough we'll probably never have ref updates or reflog expiry failing, since the clients involved will retry for far longer than the time any of those operations could take. See [1] for an initial report of how this impacted "git gc" and a large discussion about this change in early 2019. In particular patch looked good to Michael Haggerty, see his[2]. That message seems to not have made it to the ML archive, its content is quoted in full in my [3]. I'm leaving behind now-unused code the refs API etc. that takes the now-NULL "unused_oid" argument, and other code that can be simplified now that we never have on OID in that context, that'll be cleaned up in subsequent commits, but for now let's narrowly focus on fixing the "git gc" issue. As the modified assert() shows we always pass a NULL oid to reflog_expire() now. Unfortunately this sort of probabilistic contention is hard to turn into a test. I've tested this by running the following three subshells in concurrent terminals: ( rm -rf /tmp/git && git init /tmp/git && while true do head -c 10 /dev/urandom | hexdump >/tmp/git/out && git -C /tmp/git add out && git -C /tmp/git commit -m"out" done ) ( rm -rf /tmp/git-clone && git clone file:///tmp/git /tmp/git-clone && while git -C /tmp/git-clone pull do date done ) ( while git -C /tmp/git-clone reflog expire --all do date done ) Before this change the "reflog expire" would fail really quickly with the "but expected" error noted above. After this change both the "pull" and "reflog expire" will run for a while, but eventually fail because I get unlucky with core.filesRefLockTimeout (the "reflog expire" is in a really tight loop). As noted above that can in turn be mitigated with higher values of core.filesRefLockTimeout than the 100ms default. As noted in the commentary added in the preceding commit there's also the case of branches being racily deleted, that can be tested by adding this to the above: ( while git -C /tmp/git-clone branch topic master && git -C /tmp/git-clone branch -D topic do date done ) With core.filesRefLockTimeout set to 10 seconds (it can probably be a lot lower) I managed to run all four of these concurrently for about an hour, and accumulated ~125k commits, auto-gc's and all, and didn't have a single failure. The loops visibly stall while waiting for the lock, but that's expected and desired behavior. 0. https://en.wikipedia.org/wiki/Time-of-check_to_time-of-use 1. https://lore.kernel.org/git/87tvg7brlm.fsf@evledraar.gmail.com/ 2. http://lore.kernel.org/git/b870a17d-2103-41b8-3cbc-7389d5fff33a@alum.mit.edu 3. https://lore.kernel.org/git/87pnqkco8v.fsf@evledraar.gmail.com/ Signed-off-by: Ævar Arnfjörð Bjarmason <avarab@gmail.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>
2021-08-23 11:36:10 +00:00
set_reflog_expiry_param(&cb.cmd, item->string);
status |= reflog_expire(item->string, flags,
reflog_expiry_prepare,
should_prune_fn,
reflog_expiry_cleanup,
&cb);
}
string_list_clear(&collected.reflogs, 0);
}
for (i = 0; i < argc; i++) {
char *ref;
struct expire_reflog_policy_cb cb = { .cmd = cmd };
reflog expire: don't lock reflogs using previously seen OID During reflog expiry, the cmd_reflog_expire() function first iterates over all reflogs in logs/*, and then one-by-one acquires the lock for each one and expires it. This behavior has been with us since this command was implemented in 4264dc15e1 ("git reflog expire", 2006-12-19). Change this to stop calling lock_ref_oid_basic() with the OID we saw when we looped over the logs, instead have it pass the OID it managed to lock. This mostly mitigates a race condition where e.g. "git gc" will fail in a concurrently updated repository because the branch moved since "git reflog expire --all" was started. I.e. with: error: cannot lock ref '<refname>': ref '<refname>' is at <OID-A> but expected <OID-B> This behavior of passing in an "oid" was needed for an edge-case that I've untangled in this and preceding commits though, namely that we needed this OID because we'd: 1. Lookup the reflog name/OID via dwim_log() 2. With that OID, lock the reflog 3. Later in builtin/reflog.c we use the OID we looked as input to lookup_commit_reference_gently(), assured that it's equal to the OID we got from dwim_log(). We can be sure that this change is safe to make because between dwim_log (step #1) and lock_ref_oid_basic (step #2) there was no other logic relevant to the OID or expiry run in the cmd_reflog_expire() caller. We can thus treat that code as a black box, before and after this change it would get an OID that's been locked, the only difference is that now we mostly won't be failing to get the lock due to the TOCTOU race[0]. That failure was purely an implementation detail in how the "current OID" was looked up, it was divorced from the locking mechanism. What do we mean with "mostly"? It mostly mitigates it because we'll still run into cases where the ref is locked and being updated as we want to expire it, and other git processes wanting to update the refs will in turn race with us as we expire the reflog. That remaining race can in turn be mitigated with the core.filesRefLockTimeout setting, see 4ff0f01cb7 ("refs: retry acquiring reference locks for 100ms", 2017-08-21). In practice if that value is high enough we'll probably never have ref updates or reflog expiry failing, since the clients involved will retry for far longer than the time any of those operations could take. See [1] for an initial report of how this impacted "git gc" and a large discussion about this change in early 2019. In particular patch looked good to Michael Haggerty, see his[2]. That message seems to not have made it to the ML archive, its content is quoted in full in my [3]. I'm leaving behind now-unused code the refs API etc. that takes the now-NULL "unused_oid" argument, and other code that can be simplified now that we never have on OID in that context, that'll be cleaned up in subsequent commits, but for now let's narrowly focus on fixing the "git gc" issue. As the modified assert() shows we always pass a NULL oid to reflog_expire() now. Unfortunately this sort of probabilistic contention is hard to turn into a test. I've tested this by running the following three subshells in concurrent terminals: ( rm -rf /tmp/git && git init /tmp/git && while true do head -c 10 /dev/urandom | hexdump >/tmp/git/out && git -C /tmp/git add out && git -C /tmp/git commit -m"out" done ) ( rm -rf /tmp/git-clone && git clone file:///tmp/git /tmp/git-clone && while git -C /tmp/git-clone pull do date done ) ( while git -C /tmp/git-clone reflog expire --all do date done ) Before this change the "reflog expire" would fail really quickly with the "but expected" error noted above. After this change both the "pull" and "reflog expire" will run for a while, but eventually fail because I get unlucky with core.filesRefLockTimeout (the "reflog expire" is in a really tight loop). As noted above that can in turn be mitigated with higher values of core.filesRefLockTimeout than the 100ms default. As noted in the commentary added in the preceding commit there's also the case of branches being racily deleted, that can be tested by adding this to the above: ( while git -C /tmp/git-clone branch topic master && git -C /tmp/git-clone branch -D topic do date done ) With core.filesRefLockTimeout set to 10 seconds (it can probably be a lot lower) I managed to run all four of these concurrently for about an hour, and accumulated ~125k commits, auto-gc's and all, and didn't have a single failure. The loops visibly stall while waiting for the lock, but that's expected and desired behavior. 0. https://en.wikipedia.org/wiki/Time-of-check_to_time-of-use 1. https://lore.kernel.org/git/87tvg7brlm.fsf@evledraar.gmail.com/ 2. http://lore.kernel.org/git/b870a17d-2103-41b8-3cbc-7389d5fff33a@alum.mit.edu 3. https://lore.kernel.org/git/87pnqkco8v.fsf@evledraar.gmail.com/ Signed-off-by: Ævar Arnfjörð Bjarmason <avarab@gmail.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>
2021-08-23 11:36:10 +00:00
if (!dwim_log(argv[i], strlen(argv[i]), NULL, &ref)) {
status |= error(_("%s points nowhere!"), argv[i]);
continue;
}
set_reflog_expiry_param(&cb.cmd, ref);
status |= reflog_expire(ref, flags,
reflog_expiry_prepare,
should_prune_fn,
reflog_expiry_cleanup,
&cb);
free(ref);
}
return status;
}
static int cmd_reflog_delete(int argc, const char **argv, const char *prefix)
{
int i, status = 0;
unsigned int flags = 0;
int verbose = 0;
const struct option options[] = {
OPT_BIT('n', "dry-run", &flags, N_("do not actually prune any entries"),
EXPIRE_REFLOGS_DRY_RUN),
OPT_BIT(0, "rewrite", &flags,
N_("rewrite the old SHA1 with the new SHA1 of the entry that now precedes it"),
EXPIRE_REFLOGS_REWRITE),
OPT_BIT(0, "updateref", &flags,
N_("update the reference to the value of the top reflog entry"),
EXPIRE_REFLOGS_UPDATE_REF),
OPT_BOOL(0, "verbose", &verbose, N_("print extra information on screen")),
OPT_END()
};
argc = parse_options(argc, argv, prefix, options, reflog_delete_usage, 0);
if (argc < 1)
return error(_("no reflog specified to delete"));
for (i = 0; i < argc; i++)
status |= reflog_delete(argv[i], flags, verbose);
return status;
}
static int cmd_reflog_exists(int argc, const char **argv, const char *prefix)
{
struct option options[] = {
OPT_END()
};
const char *refname;
argc = parse_options(argc, argv, prefix, options, reflog_exists_usage,
0);
if (!argc)
usage_with_options(reflog_exists_usage, options);
refname = argv[0];
if (check_refname_format(refname, REFNAME_ALLOW_ONELEVEL))
die(_("invalid ref format: %s"), refname);
return !reflog_exists(refname);
}
/*
* main "reflog"
*/
int cmd_reflog(int argc, const char **argv, const char *prefix)
{
parse_opt_subcommand_fn *fn = NULL;
struct option options[] = {
OPT_SUBCOMMAND("show", &fn, cmd_reflog_show),
OPT_SUBCOMMAND("list", &fn, cmd_reflog_list),
OPT_SUBCOMMAND("expire", &fn, cmd_reflog_expire),
OPT_SUBCOMMAND("delete", &fn, cmd_reflog_delete),
OPT_SUBCOMMAND("exists", &fn, cmd_reflog_exists),
OPT_END()
};
argc = parse_options(argc, argv, prefix, options, reflog_usage,
PARSE_OPT_SUBCOMMAND_OPTIONAL |
PARSE_OPT_KEEP_DASHDASH | PARSE_OPT_KEEP_ARGV0 |
PARSE_OPT_KEEP_UNKNOWN_OPT);
if (fn)
return fn(argc - 1, argv + 1, prefix);
else
return cmd_log_reflog(argc, argv, prefix);
}