From 25a0023f28600102f54e7529c20da5928c3e9c75 Mon Sep 17 00:00:00 2001 From: Patrick Steinhardt Date: Thu, 6 Jun 2024 07:29:49 +0200 Subject: [PATCH] builtin/refs: new command to migrate ref storage formats Introduce a new command that allows the user to migrate a repository between ref storage formats. This new command is implemented as part of a new git-refs(1) executable. This is due to two reasons: - There is no good place to put the migration logic in existing commands. git-maintenance(1) felt unwieldy, and git-pack-refs(1) is not the correct place to put it, either. - I had it in my mind to create a new low-level command for accessing refs for quite a while already. git-refs(1) is that command and can over time grow more functionality relating to refs. This should help discoverability by consolidating low-level access to refs into a single executable. As mentioned in the preceding commit that introduces the ref storage format migration logic, the new `git refs migrate` command still has a bunch of restrictions. These restrictions are documented accordingly. Signed-off-by: Patrick Steinhardt Signed-off-by: Junio C Hamano --- .gitignore | 1 + Documentation/git-refs.txt | 61 ++++++++++ Makefile | 1 + builtin.h | 1 + builtin/refs.c | 75 ++++++++++++ command-list.txt | 1 + git.c | 1 + t/t1460-refs-migrate.sh | 243 +++++++++++++++++++++++++++++++++++++ 8 files changed, 384 insertions(+) create mode 100644 Documentation/git-refs.txt create mode 100644 builtin/refs.c create mode 100755 t/t1460-refs-migrate.sh diff --git a/.gitignore b/.gitignore index 612c0f6a0f..8caf3700c2 100644 --- a/.gitignore +++ b/.gitignore @@ -126,6 +126,7 @@ /git-rebase /git-receive-pack /git-reflog +/git-refs /git-remote /git-remote-http /git-remote-https diff --git a/Documentation/git-refs.txt b/Documentation/git-refs.txt new file mode 100644 index 0000000000..5b99e04385 --- /dev/null +++ b/Documentation/git-refs.txt @@ -0,0 +1,61 @@ +git-refs(1) +=========== + +NAME +---- +git-refs - Low-level access to refs + + +SYNOPSIS +-------- +[verse] +'git refs migrate' --ref-format= [--dry-run] + +DESCRIPTION +----------- + +This command provides low-level access to refs. + +COMMANDS +-------- + +migrate:: + Migrate ref store between different formats. + +OPTIONS +------- + +The following options are specific to 'git refs migrate': + +--ref-format=:: + The ref format to migrate the ref store to. Can be one of: ++ +include::ref-storage-format.txt[] + +--dry-run:: + Perform the migration, but do not modify the repository. The migrated + refs will be written into a separate directory that can be inspected + separately. The name of the directory will be reported on stdout. This + can be used to double check that the migration works as expected before + performing the actual migration. + +KNOWN LIMITATIONS +----------------- + +The ref format migration has several known limitations in its current form: + +* It is not possible to migrate repositories that have reflogs. + +* It is not possible to migrate repositories that have worktrees. + +* There is no way to block concurrent writes to the repository during an + ongoing migration. Concurrent writes can lead to an inconsistent migrated + state. Users are expected to block writes on a higher level. If your + repository is registered for scheduled maintenance, it is recommended to + unregister it first with git-maintenance(1). + +These limitations may eventually be lifted. + +GIT +--- +Part of the linkgit:git[1] suite diff --git a/Makefile b/Makefile index cf504963c2..2d702b552c 100644 --- a/Makefile +++ b/Makefile @@ -1283,6 +1283,7 @@ BUILTIN_OBJS += builtin/read-tree.o BUILTIN_OBJS += builtin/rebase.o BUILTIN_OBJS += builtin/receive-pack.o BUILTIN_OBJS += builtin/reflog.o +BUILTIN_OBJS += builtin/refs.o BUILTIN_OBJS += builtin/remote-ext.o BUILTIN_OBJS += builtin/remote-fd.o BUILTIN_OBJS += builtin/remote.o diff --git a/builtin.h b/builtin.h index 28280636da..7eda9b2486 100644 --- a/builtin.h +++ b/builtin.h @@ -207,6 +207,7 @@ int cmd_rebase(int argc, const char **argv, const char *prefix); int cmd_rebase__interactive(int argc, const char **argv, const char *prefix); int cmd_receive_pack(int argc, const char **argv, const char *prefix); int cmd_reflog(int argc, const char **argv, const char *prefix); +int cmd_refs(int argc, const char **argv, const char *prefix); int cmd_remote(int argc, const char **argv, const char *prefix); int cmd_remote_ext(int argc, const char **argv, const char *prefix); int cmd_remote_fd(int argc, const char **argv, const char *prefix); diff --git a/builtin/refs.c b/builtin/refs.c new file mode 100644 index 0000000000..46dcd150d4 --- /dev/null +++ b/builtin/refs.c @@ -0,0 +1,75 @@ +#include "builtin.h" +#include "parse-options.h" +#include "refs.h" +#include "repository.h" +#include "strbuf.h" + +#define REFS_MIGRATE_USAGE \ + N_("git refs migrate --ref-format= [--dry-run]") + +static int cmd_refs_migrate(int argc, const char **argv, const char *prefix) +{ + const char * const migrate_usage[] = { + REFS_MIGRATE_USAGE, + NULL, + }; + const char *format_str = NULL; + enum ref_storage_format format; + unsigned int flags = 0; + struct option options[] = { + OPT_STRING_F(0, "ref-format", &format_str, N_("format"), + N_("specify the reference format to convert to"), + PARSE_OPT_NONEG), + OPT_BIT(0, "dry-run", &flags, + N_("perform a non-destructive dry-run"), + REPO_MIGRATE_REF_STORAGE_FORMAT_DRYRUN), + OPT_END(), + }; + struct strbuf errbuf = STRBUF_INIT; + int err; + + argc = parse_options(argc, argv, prefix, options, migrate_usage, 0); + if (argc) + usage(_("too many arguments")); + if (!format_str) + usage(_("missing --ref-format=")); + + format = ref_storage_format_by_name(format_str); + if (format == REF_STORAGE_FORMAT_UNKNOWN) { + err = error(_("unknown ref storage format '%s'"), format_str); + goto out; + } + + if (the_repository->ref_storage_format == format) { + err = error(_("repository already uses '%s' format"), + ref_storage_format_to_name(format)); + goto out; + } + + if (repo_migrate_ref_storage_format(the_repository, format, flags, &errbuf) < 0) { + err = error("%s", errbuf.buf); + goto out; + } + + err = 0; + +out: + strbuf_release(&errbuf); + return err; +} + +int cmd_refs(int argc, const char **argv, const char *prefix) +{ + const char * const refs_usage[] = { + REFS_MIGRATE_USAGE, + NULL, + }; + parse_opt_subcommand_fn *fn = NULL; + struct option opts[] = { + OPT_SUBCOMMAND("migrate", &fn, cmd_refs_migrate), + OPT_END(), + }; + + argc = parse_options(argc, argv, prefix, opts, refs_usage, 0); + return fn(argc, argv, prefix); +} diff --git a/command-list.txt b/command-list.txt index c4cd0f352b..e0bb87b3b5 100644 --- a/command-list.txt +++ b/command-list.txt @@ -157,6 +157,7 @@ git-read-tree plumbingmanipulators git-rebase mainporcelain history git-receive-pack synchelpers git-reflog ancillarymanipulators complete +git-refs ancillarymanipulators complete git-remote ancillarymanipulators complete git-repack ancillarymanipulators complete git-replace ancillarymanipulators complete diff --git a/git.c b/git.c index 637c61ca9c..683bb69194 100644 --- a/git.c +++ b/git.c @@ -594,6 +594,7 @@ static struct cmd_struct commands[] = { { "rebase", cmd_rebase, RUN_SETUP | NEED_WORK_TREE }, { "receive-pack", cmd_receive_pack }, { "reflog", cmd_reflog, RUN_SETUP }, + { "refs", cmd_refs, RUN_SETUP }, { "remote", cmd_remote, RUN_SETUP }, { "remote-ext", cmd_remote_ext, NO_PARSEOPT }, { "remote-fd", cmd_remote_fd, NO_PARSEOPT }, diff --git a/t/t1460-refs-migrate.sh b/t/t1460-refs-migrate.sh new file mode 100755 index 0000000000..f7c0783d30 --- /dev/null +++ b/t/t1460-refs-migrate.sh @@ -0,0 +1,243 @@ +#!/bin/sh + +test_description='migration of ref storage backends' + +GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=main +export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME + +TEST_PASSES_SANITIZE_LEAK=true +. ./test-lib.sh + +test_migration () { + git -C "$1" for-each-ref --include-root-refs \ + --format='%(refname) %(objectname) %(symref)' >expect && + git -C "$1" refs migrate --ref-format="$2" && + git -C "$1" for-each-ref --include-root-refs \ + --format='%(refname) %(objectname) %(symref)' >actual && + test_cmp expect actual && + + git -C "$1" rev-parse --show-ref-format >actual && + echo "$2" >expect && + test_cmp expect actual +} + +test_expect_success 'setup' ' + rm -rf .git && + # The migration does not yet support reflogs. + git config --global core.logAllRefUpdates false +' + +test_expect_success "superfluous arguments" ' + test_when_finished "rm -rf repo" && + git init repo && + test_must_fail git -C repo refs migrate foo 2>err && + cat >expect <<-EOF && + usage: too many arguments + EOF + test_cmp expect err +' + +test_expect_success "missing ref storage format" ' + test_when_finished "rm -rf repo" && + git init repo && + test_must_fail git -C repo refs migrate 2>err && + cat >expect <<-EOF && + usage: missing --ref-format= + EOF + test_cmp expect err +' + +test_expect_success "unknown ref storage format" ' + test_when_finished "rm -rf repo" && + git init repo && + test_must_fail git -C repo refs migrate \ + --ref-format=unknown 2>err && + cat >expect <<-EOF && + error: unknown ref storage format ${SQ}unknown${SQ} + EOF + test_cmp expect err +' + +ref_formats="files reftable" +for from_format in $ref_formats +do + for to_format in $ref_formats + do + if test "$from_format" = "$to_format" + then + continue + fi + + test_expect_success "$from_format: migration to same format fails" ' + test_when_finished "rm -rf repo" && + git init --ref-format=$from_format repo && + test_must_fail git -C repo refs migrate \ + --ref-format=$from_format 2>err && + cat >expect <<-EOF && + error: repository already uses ${SQ}$from_format${SQ} format + EOF + test_cmp expect err + ' + + test_expect_success "$from_format -> $to_format: migration with reflog fails" ' + test_when_finished "rm -rf repo" && + git init --ref-format=$from_format repo && + test_config -C repo core.logAllRefUpdates true && + test_commit -C repo logged && + test_must_fail git -C repo refs migrate \ + --ref-format=$to_format 2>err && + cat >expect <<-EOF && + error: migrating reflogs is not supported yet + EOF + test_cmp expect err + ' + + test_expect_success "$from_format -> $to_format: migration with worktree fails" ' + test_when_finished "rm -rf repo" && + git init --ref-format=$from_format repo && + git -C repo worktree add wt && + test_must_fail git -C repo refs migrate \ + --ref-format=$to_format 2>err && + cat >expect <<-EOF && + error: migrating repositories with worktrees is not supported yet + EOF + test_cmp expect err + ' + + test_expect_success "$from_format -> $to_format: unborn HEAD" ' + test_when_finished "rm -rf repo" && + git init --ref-format=$from_format repo && + test_migration repo "$to_format" + ' + + test_expect_success "$from_format -> $to_format: single ref" ' + test_when_finished "rm -rf repo" && + git init --ref-format=$from_format repo && + test_commit -C repo initial && + test_migration repo "$to_format" + ' + + test_expect_success "$from_format -> $to_format: bare repository" ' + test_when_finished "rm -rf repo repo.git" && + git init --ref-format=$from_format repo && + test_commit -C repo initial && + git clone --ref-format=$from_format --mirror repo repo.git && + test_migration repo.git "$to_format" + ' + + test_expect_success "$from_format -> $to_format: dangling symref" ' + test_when_finished "rm -rf repo" && + git init --ref-format=$from_format repo && + test_commit -C repo initial && + git -C repo symbolic-ref BROKEN_HEAD refs/heads/nonexistent && + test_migration repo "$to_format" && + echo refs/heads/nonexistent >expect && + git -C repo symbolic-ref BROKEN_HEAD >actual && + test_cmp expect actual + ' + + test_expect_success "$from_format -> $to_format: broken ref" ' + test_when_finished "rm -rf repo" && + git init --ref-format=$from_format repo && + test_commit -C repo initial && + test-tool -C repo ref-store main update-ref "" refs/heads/broken \ + "$(test_oid 001)" "$ZERO_OID" REF_SKIP_CREATE_REFLOG,REF_SKIP_OID_VERIFICATION && + test_migration repo "$to_format" && + test_oid 001 >expect && + git -C repo rev-parse refs/heads/broken >actual && + test_cmp expect actual + ' + + test_expect_success "$from_format -> $to_format: pseudo-refs" ' + test_when_finished "rm -rf repo" && + git init --ref-format=$from_format repo && + test_commit -C repo initial && + git -C repo update-ref FOO_HEAD HEAD && + test_migration repo "$to_format" + ' + + test_expect_success "$from_format -> $to_format: special refs are left alone" ' + test_when_finished "rm -rf repo" && + git init --ref-format=$from_format repo && + test_commit -C repo initial && + git -C repo rev-parse HEAD >repo/.git/MERGE_HEAD && + git -C repo rev-parse MERGE_HEAD && + test_migration repo "$to_format" && + test_path_is_file repo/.git/MERGE_HEAD + ' + + test_expect_success "$from_format -> $to_format: a bunch of refs" ' + test_when_finished "rm -rf repo" && + git init --ref-format=$from_format repo && + + test_commit -C repo initial && + cat >input <<-EOF && + create FOO_HEAD HEAD + create refs/heads/branch-1 HEAD + create refs/heads/branch-2 HEAD + create refs/heads/branch-3 HEAD + create refs/heads/branch-4 HEAD + create refs/tags/tag-1 HEAD + create refs/tags/tag-2 HEAD + EOF + git -C repo update-ref --stdin $to_format: dry-run migration does not modify repository" ' + test_when_finished "rm -rf repo" && + git init --ref-format=$from_format repo && + test_commit -C repo initial && + git -C repo refs migrate --dry-run \ + --ref-format=$to_format >output && + grep "Finished dry-run migration of refs" output && + test_path_is_dir repo/.git/ref_migration.* && + echo $from_format >expect && + git -C repo rev-parse --show-ref-format >actual && + test_cmp expect actual + ' + done +done + +test_expect_success 'migrating from files format deletes backend files' ' + test_when_finished "rm -rf repo" && + git init --ref-format=files repo && + test_commit -C repo first && + git -C repo pack-refs --all && + test_commit -C repo second && + git -C repo update-ref ORIG_HEAD HEAD && + git -C repo rev-parse HEAD >repo/.git/FETCH_HEAD && + + test_path_is_file repo/.git/HEAD && + test_path_is_file repo/.git/ORIG_HEAD && + test_path_is_file repo/.git/refs/heads/main && + test_path_is_file repo/.git/packed-refs && + + test_migration repo reftable && + + echo "ref: refs/heads/.invalid" >expect && + test_cmp expect repo/.git/HEAD && + echo "this repository uses the reftable format" >expect && + test_cmp expect repo/.git/refs/heads && + test_path_is_file repo/.git/FETCH_HEAD && + test_path_is_missing repo/.git/ORIG_HEAD && + test_path_is_missing repo/.git/refs/heads/main && + test_path_is_missing repo/.git/logs && + test_path_is_missing repo/.git/packed-refs +' + +test_expect_success 'migrating from reftable format deletes backend files' ' + test_when_finished "rm -rf repo" && + git init --ref-format=reftable repo && + test_commit -C repo first && + + test_path_is_dir repo/.git/reftable && + test_migration repo files && + + test_path_is_missing repo/.git/reftable && + echo "ref: refs/heads/main" >expect && + test_cmp expect repo/.git/HEAD && + test_path_is_file repo/.git/refs/heads/main +' + +test_done