development/git
development/git
1
0
Fork 0
mirror of https://github.com/git/git synced 2024-10-29 17:08:46 +00:00
Code Issues Projects Releases Packages Wiki Activity

Merge branch 'gc/recursive-fetch-with-unused-submodules'

Browse source 
When "git fetch --recurse-submodules" grabbed submodule commits
that would be needed to recursively check out newly fetched commits
in the superproject, it only paid attention to submodules that are
in the current checkout of the superproject.  We now do so for all
submodules that have been run "git submodule init" on.

* gc/recursive-fetch-with-unused-submodules:
  submodule: fix latent check_has_commit() bug
  fetch: fetch unpopulated, changed submodules
  submodule: move logic into fetch_task_create()
  submodule: extract get_fetch_task()
  submodule: store new submodule commits oid_array in a struct
  submodule: inline submodule_commits() into caller
  submodule: make static functions read submodules from commits
  t5526: create superproject commits with test helper
  t5526: stop asserting on stderr literally
  t5526: introduce test helper to assert on fetches
This commit is contained in:
Junio C Hamano 2022-03-25 16:38:25 -07:00
commit dd9ff30dff
6 changed files with 761 additions and 312 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

26
Documentation/fetch-options.txt
View file

@ -186,15 +186,23 @@ endif::git-pull[]
ifndef::git-pull[] ifndef::git-pull[]
--recurse-submodules[=yes|on-demand|no]:: --recurse-submodules[=yes|on-demand|no]::
This option controls if and under what conditions new commits of This option controls if and under what conditions new commits of
populated submodules should be fetched too. It can be used as a submodules should be fetched too. When recursing through submodules,
boolean option to completely disable recursion when set to 'no' or to `git fetch` always attempts to fetch "changed" submodules, that is, a
unconditionally recurse into all populated submodules when set to submodule that has commits that are referenced by a newly fetched
'yes', which is the default when this option is used without any superproject commit but are missing in the local submodule clone. A
value. Use 'on-demand' to only recurse into a populated submodule changed submodule can be fetched as long as it is present locally e.g.
when the superproject retrieves a commit that updates the submodule's in `$GIT_DIR/modules/` (see linkgit:gitsubmodules[7]); if the upstream
reference to a commit that isn't already in the local submodule adds a new submodule, that submodule cannot be fetched until it is
clone. By default, 'on-demand' is used, unless cloned e.g. by `git submodule update`.
`fetch.recurseSubmodules` is set (see linkgit:git-config[1]). +
When set to 'on-demand', only changed submodules are fetched. When set
to 'yes', all populated submodules are fetched and submodules that are
both unpopulated and changed are fetched. When set to 'no', submodules
are never fetched.
+
When unspecified, this uses the value of `fetch.recurseSubmodules` if it
is set (see linkgit:git-config[1]), defaulting to 'on-demand' if unset.
When this option is used without any value, it defaults to 'yes'.
endif::git-pull[] endif::git-pull[]
-j:: -j::

10
Documentation/git-fetch.txt
View file

@ -287,12 +287,10 @@ include::transfer-data-leaks.txt[]
BUGS BUGS
---- ----
Using --recurse-submodules can only fetch new commits in already checked Using --recurse-submodules can only fetch new commits in submodules that are
out submodules right now. When e.g. upstream added a new submodule in the present locally e.g. in `$GIT_DIR/modules/`. If the upstream adds a new
just fetched commits of the superproject the submodule itself cannot be submodule, that submodule cannot be fetched until it is cloned e.g. by `git
fetched, making it impossible to check out that submodule later without submodule update`. This is expected to be fixed in a future Git version.
having to do a fetch again. This is expected to be fixed in a future Git
version.
SEE ALSO SEE ALSO
-------- --------

14
builtin/fetch.c
View file

@ -2258,13 +2258,13 @@ int cmd_fetch(int argc, const char **argv, const char *prefix)
max_children = fetch_parallel_config; max_children = fetch_parallel_config;
add_options_to_argv(&options); add_options_to_argv(&options);
result = fetch_populated_submodules(the_repository, result = fetch_submodules(the_repository,
&options, &options,
submodule_prefix, submodule_prefix,
recurse_submodules, recurse_submodules,
recurse_submodules_default, recurse_submodules_default,
verbosity < 0, verbosity < 0,
max_children); max_children);
strvec_clear(&options); strvec_clear(&options);
} }

457
submodule.c
View file

@ -167,26 +167,6 @@ void stage_updated_gitmodules(struct index_state *istate)
static struct string_list added_submodule_odb_paths = STRING_LIST_INIT_NODUP; static struct string_list added_submodule_odb_paths = STRING_LIST_INIT_NODUP;
/* TODO: remove this function, use repo_submodule_init instead. */
int add_submodule_odb(const char *path)
{
struct strbuf objects_directory = STRBUF_INIT;
int ret = 0;
ret = strbuf_git_path_submodule(&objects_directory, path, "objects/");
if (ret)
goto done;
if (!is_directory(objects_directory.buf)) {
ret = -1;
goto done;
}
string_list_insert(&added_submodule_odb_paths,
strbuf_detach(&objects_directory, NULL));
done:
strbuf_release(&objects_directory);
return ret;
}
void add_submodule_odb_by_path(const char *path) void add_submodule_odb_by_path(const char *path)
{ {
string_list_insert(&added_submodule_odb_paths, xstrdup(path)); string_list_insert(&added_submodule_odb_paths, xstrdup(path));
@ -782,19 +762,6 @@ const struct submodule *submodule_from_ce(const struct cache_entry *ce)
return submodule_from_path(the_repository, null_oid(), ce->name); return submodule_from_path(the_repository, null_oid(), ce->name);
} }
static struct oid_array *submodule_commits(struct string_list *submodules,
const char *name)
{
struct string_list_item *item;
item = string_list_insert(submodules, name);
if (item->util)
return (struct oid_array *) item->util;
/* NEEDSWORK: should we have oid_array_init()? */
item->util = xcalloc(1, sizeof(struct oid_array));
return (struct oid_array *) item->util;
}
struct collect_changed_submodules_cb_data { struct collect_changed_submodules_cb_data {
struct repository *repo; struct repository *repo;
@ -819,6 +786,52 @@ static const char *default_name_or_path(const char *path_or_name)
return path_or_name; return path_or_name;
} }
/*
* Holds relevant information for a changed submodule. Used as the .util
* member of the changed submodule name string_list_item.
*
* (super_oid, path) allows the submodule config to be read from _some_
* .gitmodules file. We store this information the first time we find a
* superproject commit that points to the submodule, but this is
* arbitrary - we can choose any (super_oid, path) that matches the
* submodule's name.
*
* NEEDSWORK: Storing an arbitrary commit is undesirable because we can't
* guarantee that we're reading the commit that the user would expect. A better
* scheme would be to just fetch a submodule by its name. This requires two
* steps:
* - Create a function that behaves like repo_submodule_init(), but accepts a
* submodule name instead of treeish_name and path. This should be easy
* because repo_submodule_init() internally uses the submodule's name.
*
* - Replace most instances of 'struct submodule' (which is the .gitmodules
* config) with just the submodule name. This is OK because we expect
* submodule settings to be stored in .git/config (via "git submodule init"),
* not .gitmodules. This also lets us delete get_non_gitmodules_submodule(),
* which constructs a bogus 'struct submodule' for the sake of giving a
* placeholder name to a gitlink.
*/
struct changed_submodule_data {
/*
* The first superproject commit in the rev walk that points to
* the submodule.
*/
const struct object_id *super_oid;
/*
* Path to the submodule in the superproject commit referenced
* by 'super_oid'.
*/
char *path;
/* The submodule commits that have changed in the rev walk. */
struct oid_array new_commits;
};
static void changed_submodule_data_clear(struct changed_submodule_data *cs_data)
{
oid_array_clear(&cs_data->new_commits);
free(cs_data->path);
}
static void collect_changed_submodules_cb(struct diff_queue_struct *q, static void collect_changed_submodules_cb(struct diff_queue_struct *q,
struct diff_options *options, struct diff_options *options,
void *data) void *data)
@ -830,9 +843,10 @@ static void collect_changed_submodules_cb(struct diff_queue_struct *q,
for (i = 0; i < q->nr; i++) { for (i = 0; i < q->nr; i++) {
struct diff_filepair *p = q->queue[i]; struct diff_filepair *p = q->queue[i];
struct oid_array *commits;
const struct submodule *submodule; const struct submodule *submodule;
const char *name; const char *name;
struct string_list_item *item;
struct changed_submodule_data *cs_data;
if (!S_ISGITLINK(p->two->mode)) if (!S_ISGITLINK(p->two->mode))
continue; continue;
@ -859,8 +873,16 @@ static void collect_changed_submodules_cb(struct diff_queue_struct *q,
if (!name) if (!name)
continue; continue;
commits = submodule_commits(changed, name); item = string_list_insert(changed, name);
oid_array_append(commits, &p->two->oid); if (item->util)
cs_data = item->util;
else {
item->util = xcalloc(1, sizeof(struct changed_submodule_data));
cs_data = item->util;
cs_data->super_oid = commit_oid;
cs_data->path = xstrdup(p->two->path);
}
oid_array_append(&cs_data->new_commits, &p->two->oid);
} }
} }
@ -907,11 +929,12 @@ static void collect_changed_submodules(struct repository *r,
reset_revision_walk(); reset_revision_walk();
} }
static void free_submodules_oids(struct string_list *submodules) static void free_submodules_data(struct string_list *submodules)
{ {
struct string_list_item *item; struct string_list_item *item;
for_each_string_list_item(item, submodules) for_each_string_list_item(item, submodules)
oid_array_clear((struct oid_array *) item->util); changed_submodule_data_clear(item->util);
string_list_clear(submodules, 1); string_list_clear(submodules, 1);
} }
@ -932,6 +955,7 @@ struct has_commit_data {
struct repository *repo; struct repository *repo;
int result; int result;
const char *path; const char *path;
const struct object_id *super_oid;
}; };
static int check_has_commit(const struct object_id *oid, void *data) static int check_has_commit(const struct object_id *oid, void *data)
@ -940,9 +964,10 @@ static int check_has_commit(const struct object_id *oid, void *data)
struct repository subrepo; struct repository subrepo;
enum object_type type; enum object_type type;
if (repo_submodule_init(&subrepo, cb->repo, cb->path, null_oid())) { if (repo_submodule_init(&subrepo, cb->repo, cb->path, cb->super_oid)) {
cb->result = 0; cb->result = 0;
goto cleanup; /* subrepo failed to init, so don't clean it up. */
return 0;
} }
type = oid_object_info(&subrepo, oid, NULL); type = oid_object_info(&subrepo, oid, NULL);
@ -968,21 +993,15 @@ static int check_has_commit(const struct object_id *oid, void *data)
static int submodule_has_commits(struct repository *r, static int submodule_has_commits(struct repository *r,
const char *path, const char *path,
const struct object_id *super_oid,
struct oid_array *commits) struct oid_array *commits)
{ {
struct has_commit_data has_commit = { r, 1, path }; struct has_commit_data has_commit = {
.repo = r,
/* .result = 1,
* Perform a cheap, but incorrect check for the existence of 'commits'. .path = path,
* This is done by adding the submodule's object store to the in-core .super_oid = super_oid
* object store, and then querying for each commit's existence. If we };
* do not have the commit object anywhere, there is no chance we have
* it in the object store of the correct submodule and have it
* reachable from a ref, so we can fail early without spawning rev-list
* which is expensive.
*/
if (add_submodule_odb(path))
return 0;
oid_array_for_each_unique(commits, check_has_commit, &has_commit); oid_array_for_each_unique(commits, check_has_commit, &has_commit);
@ -1017,7 +1036,7 @@ static int submodule_needs_pushing(struct repository *r,
const char *path, const char *path,
struct oid_array *commits) struct oid_array *commits)
{ {
if (!submodule_has_commits(r, path, commits)) if (!submodule_has_commits(r, path, null_oid(), commits))
/* /*
* NOTE: We do consider it safe to return "no" here. The * NOTE: We do consider it safe to return "no" here. The
* correct answer would be "We do not know" instead of * correct answer would be "We do not know" instead of
@ -1077,7 +1096,7 @@ int find_unpushed_submodules(struct repository *r,
collect_changed_submodules(r, &submodules, &argv); collect_changed_submodules(r, &submodules, &argv);
for_each_string_list_item(name, &submodules) { for_each_string_list_item(name, &submodules) {
struct oid_array *commits = name->util; struct changed_submodule_data *cs_data = name->util;
const struct submodule *submodule; const struct submodule *submodule;
const char *path = NULL; const char *path = NULL;
@ -1090,11 +1109,11 @@ int find_unpushed_submodules(struct repository *r,
if (!path) if (!path)
continue; continue;
if (submodule_needs_pushing(r, path, commits)) if (submodule_needs_pushing(r, path, &cs_data->new_commits))
string_list_insert(needs_pushing, path); string_list_insert(needs_pushing, path);
} }
free_submodules_oids(&submodules); free_submodules_data(&submodules);
strvec_clear(&argv); strvec_clear(&argv);
return needs_pushing->nr; return needs_pushing->nr;
@ -1240,14 +1259,36 @@ void check_for_new_submodule_commits(struct object_id *oid)
oid_array_append(&ref_tips_after_fetch, oid); oid_array_append(&ref_tips_after_fetch, oid);
} }
/*
* Returns 1 if there is at least one submodule gitdir in
* $GIT_DIR/modules and 0 otherwise. This follows
* submodule_name_to_gitdir(), which looks for submodules in
* $GIT_DIR/modules, not $GIT_COMMON_DIR.
*
* A submodule can be moved to $GIT_DIR/modules manually by running "git
* submodule absorbgitdirs", or it may be initialized there by "git
* submodule update".
*/
static int repo_has_absorbed_submodules(struct repository *r)
{
int ret;
struct strbuf buf = STRBUF_INIT;
strbuf_repo_git_path(&buf, r, "modules/");
ret = file_exists(buf.buf) && !is_empty_dir(buf.buf);
strbuf_release(&buf);
return ret;
}
static void calculate_changed_submodule_paths(struct repository *r, static void calculate_changed_submodule_paths(struct repository *r,
struct string_list *changed_submodule_names) struct string_list *changed_submodule_names)
{ {
struct strvec argv = STRVEC_INIT; struct strvec argv = STRVEC_INIT;
struct string_list_item *name; struct string_list_item *name;
/* No need to check if there are no submodules configured */ /* No need to check if no submodules would be fetched */
if (!submodule_from_path(r, NULL, NULL)) if (!submodule_from_path(r, NULL, NULL) &&
!repo_has_absorbed_submodules(r))
return; return;
strvec_push(&argv, "--"); /* argv[0] program name */ strvec_push(&argv, "--"); /* argv[0] program name */
@ -1264,7 +1305,7 @@ static void calculate_changed_submodule_paths(struct repository *r,
collect_changed_submodules(r, changed_submodule_names, &argv); collect_changed_submodules(r, changed_submodule_names, &argv);
for_each_string_list_item(name, changed_submodule_names) { for_each_string_list_item(name, changed_submodule_names) {
struct oid_array *commits = name->util; struct changed_submodule_data *cs_data = name->util;
const struct submodule *submodule; const struct submodule *submodule;
const char *path = NULL; const char *path = NULL;
@ -1277,8 +1318,8 @@ static void calculate_changed_submodule_paths(struct repository *r,
if (!path) if (!path)
continue; continue;
if (submodule_has_commits(r, path, commits)) { if (submodule_has_commits(r, path, null_oid(), &cs_data->new_commits)) {
oid_array_clear(commits); changed_submodule_data_clear(cs_data);
*name->string = '\0'; *name->string = '\0';
} }
} }
@ -1315,12 +1356,21 @@ int submodule_touches_in_range(struct repository *r,
strvec_clear(&args); strvec_clear(&args);
free_submodules_oids(&subs); free_submodules_data(&subs);
return ret; return ret;
} }
struct submodule_parallel_fetch { struct submodule_parallel_fetch {
int count; /*
* The index of the last index entry processed by
* get_fetch_task_from_index().
*/
int index_count;
/*
* The index of the last string_list entry processed by
* get_fetch_task_from_changed().
*/
int changed_count;
struct strvec args; struct strvec args;
struct repository *r; struct repository *r;
const char *prefix; const char *prefix;
@ -1329,7 +1379,16 @@ struct submodule_parallel_fetch {
int quiet; int quiet;
int result; int result;
/*
* Names of submodules that have new commits. Generated by
* walking the newly fetched superproject commits.
*/
struct string_list changed_submodule_names; struct string_list changed_submodule_names;
/*
* Names of submodules that have already been processed. Lets us
* avoid fetching the same submodule more than once.
*/
struct string_list seen_submodule_names;
/* Pending fetches by OIDs */ /* Pending fetches by OIDs */
struct fetch_task **oid_fetch_tasks; struct fetch_task **oid_fetch_tasks;
@ -1340,6 +1399,7 @@ struct submodule_parallel_fetch {
#define SPF_INIT { \ #define SPF_INIT { \
.args = STRVEC_INIT, \ .args = STRVEC_INIT, \
.changed_submodule_names = STRING_LIST_INIT_DUP, \ .changed_submodule_names = STRING_LIST_INIT_DUP, \
.seen_submodule_names = STRING_LIST_INIT_DUP, \
.submodules_with_errors = STRBUF_INIT, \ .submodules_with_errors = STRBUF_INIT, \
} }
@ -1376,6 +1436,8 @@ struct fetch_task {
struct repository *repo; struct repository *repo;
const struct submodule *sub; const struct submodule *sub;
unsigned free_sub : 1; /* Do we need to free the submodule? */ unsigned free_sub : 1; /* Do we need to free the submodule? */
const char *default_argv; /* The default fetch mode. */
struct strvec git_args; /* Args for the child git process. */
struct oid_array *commits; /* Ensure these commits are fetched */ struct oid_array *commits; /* Ensure these commits are fetched */
}; };
@ -1401,31 +1463,6 @@ static const struct submodule *get_non_gitmodules_submodule(const char *path)
return (const struct submodule *) ret; return (const struct submodule *) ret;
} }
static struct fetch_task *fetch_task_create(struct repository *r,
const char *path)
{
struct fetch_task *task = xmalloc(sizeof(*task));
memset(task, 0, sizeof(*task));
task->sub = submodule_from_path(r, null_oid(), path);
if (!task->sub) {
/*
* No entry in .gitmodules? Technically not a submodule,
* but historically we supported repositories that happen to be
* in-place where a gitlink is. Keep supporting them.
*/
task->sub = get_non_gitmodules_submodule(path);
if (!task->sub) {
free(task);
return NULL;
}
task->free_sub = 1;
}
return task;
}
static void fetch_task_release(struct fetch_task *p) static void fetch_task_release(struct fetch_task *p)
{ {
if (p->free_sub) if (p->free_sub)
@ -1436,14 +1473,17 @@ static void fetch_task_release(struct fetch_task *p)
if (p->repo) if (p->repo)
repo_clear(p->repo); repo_clear(p->repo);
FREE_AND_NULL(p->repo); FREE_AND_NULL(p->repo);
strvec_clear(&p->git_args);
} }
static struct repository *get_submodule_repo_for(struct repository *r, static struct repository *get_submodule_repo_for(struct repository *r,
const char *path) const char *path,
const struct object_id *treeish_name)
{ {
struct repository *ret = xmalloc(sizeof(*ret)); struct repository *ret = xmalloc(sizeof(*ret));
if (repo_submodule_init(ret, r, path, null_oid())) { if (repo_submodule_init(ret, r, path, treeish_name)) {
free(ret); free(ret);
return NULL; return NULL;
} }
@ -1451,67 +1491,83 @@ static struct repository *get_submodule_repo_for(struct repository *r,
return ret; return ret;
} }
static int get_next_submodule(struct child_process *cp, static struct fetch_task *fetch_task_create(struct submodule_parallel_fetch *spf,
struct strbuf *err, void *data, void **task_cb) const char *path,
const struct object_id *treeish_name)
{ {
struct submodule_parallel_fetch *spf = data; struct fetch_task *task = xmalloc(sizeof(*task));
memset(task, 0, sizeof(*task));
for (; spf->count < spf->r->index->cache_nr; spf->count++) { task->sub = submodule_from_path(spf->r, treeish_name, path);
const struct cache_entry *ce = spf->r->index->cache[spf->count];
const char *default_argv; if (!task->sub) {
/*
* No entry in .gitmodules? Technically not a submodule,
* but historically we supported repositories that happen to be
* in-place where a gitlink is. Keep supporting them.
*/
task->sub = get_non_gitmodules_submodule(path);
if (!task->sub)
goto cleanup;
task->free_sub = 1;
}
if (string_list_lookup(&spf->seen_submodule_names, task->sub->name))
goto cleanup;
switch (get_fetch_recurse_config(task->sub, spf))
{
default:
case RECURSE_SUBMODULES_DEFAULT:
case RECURSE_SUBMODULES_ON_DEMAND:
if (!task->sub ||
!string_list_lookup(
&spf->changed_submodule_names,
task->sub->name))
goto cleanup;
task->default_argv = "on-demand";
break;
case RECURSE_SUBMODULES_ON:
task->default_argv = "yes";
break;
case RECURSE_SUBMODULES_OFF:
goto cleanup;
}
task->repo = get_submodule_repo_for(spf->r, path, treeish_name);
return task;
cleanup:
fetch_task_release(task);
free(task);
return NULL;
}
static struct fetch_task *
get_fetch_task_from_index(struct submodule_parallel_fetch *spf,
struct strbuf *err)
{
for (; spf->index_count < spf->r->index->cache_nr; spf->index_count++) {
const struct cache_entry *ce =
spf->r->index->cache[spf->index_count];
struct fetch_task *task; struct fetch_task *task;
if (!S_ISGITLINK(ce->ce_mode)) if (!S_ISGITLINK(ce->ce_mode))
continue; continue;
task = fetch_task_create(spf->r, ce->name); task = fetch_task_create(spf, ce->name, null_oid());
if (!task) if (!task)
continue; continue;
switch (get_fetch_recurse_config(task->sub, spf))
{
default:
case RECURSE_SUBMODULES_DEFAULT:
case RECURSE_SUBMODULES_ON_DEMAND:
if (!task->sub ||
!string_list_lookup(
&spf->changed_submodule_names,
task->sub->name))
continue;
default_argv = "on-demand";
break;
case RECURSE_SUBMODULES_ON:
default_argv = "yes";
break;
case RECURSE_SUBMODULES_OFF:
continue;
}
task->repo = get_submodule_repo_for(spf->r, task->sub->path);
if (task->repo) { if (task->repo) {
struct strbuf submodule_prefix = STRBUF_INIT;
child_process_init(cp);
cp->dir = task->repo->gitdir;
prepare_submodule_repo_env_in_gitdir(&cp->env_array);
cp->git_cmd = 1;
if (!spf->quiet) if (!spf->quiet)
strbuf_addf(err, _("Fetching submodule %s%s\n"), strbuf_addf(err, _("Fetching submodule %s%s\n"),
spf->prefix, ce->name); spf->prefix, ce->name);
strvec_init(&cp->args);
strvec_pushv(&cp->args, spf->args.v);
strvec_push(&cp->args, default_argv);
strvec_push(&cp->args, "--submodule-prefix");
strbuf_addf(&submodule_prefix, "%s%s/", spf->index_count++;
spf->prefix, return task;
task->sub->path);
strvec_push(&cp->args, submodule_prefix.buf);
spf->count++;
*task_cb = task;
strbuf_release(&submodule_prefix);
return 1;
} else { } else {
struct strbuf empty_submodule_path = STRBUF_INIT; struct strbuf empty_submodule_path = STRBUF_INIT;
@ -1535,6 +1591,111 @@ static int get_next_submodule(struct child_process *cp,
strbuf_release(&empty_submodule_path); strbuf_release(&empty_submodule_path);
} }
} }
return NULL;
}
static struct fetch_task *
get_fetch_task_from_changed(struct submodule_parallel_fetch *spf,
struct strbuf *err)
{
for (; spf->changed_count < spf->changed_submodule_names.nr;
spf->changed_count++) {
struct string_list_item item =
spf->changed_submodule_names.items[spf->changed_count];
struct changed_submodule_data *cs_data = item.util;
struct fetch_task *task;
if (!is_tree_submodule_active(spf->r, cs_data->super_oid,cs_data->path))
continue;
task = fetch_task_create(spf, cs_data->path,
cs_data->super_oid);
if (!task)
continue;
if (!task->repo) {
strbuf_addf(err, _("Could not access submodule '%s' at commit %s\n"),
cs_data->path,
find_unique_abbrev(cs_data->super_oid, DEFAULT_ABBREV));
fetch_task_release(task);
free(task);
continue;
}
if (!spf->quiet)
strbuf_addf(err,
_("Fetching submodule %s%s at commit %s\n"),
spf->prefix, task->sub->path,
find_unique_abbrev(cs_data->super_oid,
DEFAULT_ABBREV));
spf->changed_count++;
/*
* NEEDSWORK: Submodules set/unset a value for
* core.worktree when they are populated/unpopulated by
* "git checkout" (and similar commands, see
* submodule_move_head() and
* connect_work_tree_and_git_dir()), but if the
* submodule is unpopulated in another way (e.g. "git
* rm", "rm -r"), core.worktree will still be set even
* though the directory doesn't exist, and the child
* process will crash while trying to chdir into the
* nonexistent directory.
*
* In this case, we know that the submodule has no
* working tree, so we can work around this by
* setting "--work-tree=." (--bare does not work because
* worktree settings take precedence over bare-ness).
* However, this is not necessarily true in other cases,
* so a generalized solution is still necessary.
*
* Possible solutions:
* - teach "git [add|rm]" to unset core.worktree and
* discourage users from removing submodules without
* using a Git command.
* - teach submodule child processes to ignore stale
* core.worktree values.
*/
strvec_push(&task->git_args, "--work-tree=.");
return task;
}
return NULL;
}
static int get_next_submodule(struct child_process *cp, struct strbuf *err,
void *data, void **task_cb)
{
struct submodule_parallel_fetch *spf = data;
struct fetch_task *task =
get_fetch_task_from_index(spf, err);
if (!task)
task = get_fetch_task_from_changed(spf, err);
if (task) {
struct strbuf submodule_prefix = STRBUF_INIT;
child_process_init(cp);
cp->dir = task->repo->gitdir;
prepare_submodule_repo_env_in_gitdir(&cp->env_array);
cp->git_cmd = 1;
strvec_init(&cp->args);
if (task->git_args.nr)
strvec_pushv(&cp->args, task->git_args.v);
strvec_pushv(&cp->args, spf->args.v);
strvec_push(&cp->args, task->default_argv);
strvec_push(&cp->args, "--submodule-prefix");
strbuf_addf(&submodule_prefix, "%s%s/",
spf->prefix,
task->sub->path);
strvec_push(&cp->args, submodule_prefix.buf);
*task_cb = task;
strbuf_release(&submodule_prefix);
string_list_insert(&spf->seen_submodule_names, task->sub->name);
return 1;
}
if (spf->oid_fetch_tasks_nr) { if (spf->oid_fetch_tasks_nr) {
struct fetch_task *task = struct fetch_task *task =
@ -1597,7 +1758,7 @@ static int fetch_finish(int retvalue, struct strbuf *err,
struct fetch_task *task = task_cb; struct fetch_task *task = task_cb;
struct string_list_item *it; struct string_list_item *it;
struct oid_array *commits; struct changed_submodule_data *cs_data;
if (!task || !task->sub) if (!task || !task->sub)
BUG("callback cookie bogus"); BUG("callback cookie bogus");
@ -1625,14 +1786,14 @@ static int fetch_finish(int retvalue, struct strbuf *err,
/* Could be an unchanged submodule, not contained in the list */ /* Could be an unchanged submodule, not contained in the list */
goto out; goto out;
commits = it->util; cs_data = it->util;
oid_array_filter(commits, oid_array_filter(&cs_data->new_commits,
commit_missing_in_sub, commit_missing_in_sub,
task->repo); task->repo);
/* Are there commits we want, but do not exist? */ /* Are there commits we want, but do not exist? */
if (commits->nr) { if (cs_data->new_commits.nr) {
task->commits = commits; task->commits = &cs_data->new_commits;
ALLOC_GROW(spf->oid_fetch_tasks, ALLOC_GROW(spf->oid_fetch_tasks,
spf->oid_fetch_tasks_nr + 1, spf->oid_fetch_tasks_nr + 1,
spf->oid_fetch_tasks_alloc); spf->oid_fetch_tasks_alloc);
@ -1647,11 +1808,11 @@ static int fetch_finish(int retvalue, struct strbuf *err,
return 0; return 0;
} }
int fetch_populated_submodules(struct repository *r, int fetch_submodules(struct repository *r,
const struct strvec *options, const struct strvec *options,
const char *prefix, int command_line_option, const char *prefix, int command_line_option,
int default_option, int default_option,
int quiet, int max_parallel_jobs) int quiet, int max_parallel_jobs)
{ {
int i; int i;
struct submodule_parallel_fetch spf = SPF_INIT; struct submodule_parallel_fetch spf = SPF_INIT;
@ -1690,7 +1851,7 @@ int fetch_populated_submodules(struct repository *r,
strvec_clear(&spf.args); strvec_clear(&spf.args);
out: out:
free_submodules_oids(&spf.changed_submodule_names); free_submodules_data(&spf.changed_submodule_names);
return spf.result; return spf.result;
} }

21
submodule.h
View file

@ -88,12 +88,12 @@ int should_update_submodules(void);
*/ */
const struct submodule *submodule_from_ce(const struct cache_entry *ce); const struct submodule *submodule_from_ce(const struct cache_entry *ce);
void check_for_new_submodule_commits(struct object_id *oid); void check_for_new_submodule_commits(struct object_id *oid);
int fetch_populated_submodules(struct repository *r, int fetch_submodules(struct repository *r,
const struct strvec *options, const struct strvec *options,
const char *prefix, const char *prefix,
int command_line_option, int command_line_option,
int default_option, int default_option,
int quiet, int max_parallel_jobs); int quiet, int max_parallel_jobs);
unsigned is_submodule_modified(const char *path, int ignore_untracked); unsigned is_submodule_modified(const char *path, int ignore_untracked);
int submodule_uses_gitfile(const char *path); int submodule_uses_gitfile(const char *path);
@ -103,12 +103,11 @@ int submodule_uses_gitfile(const char *path);
int bad_to_remove_submodule(const char *path, unsigned flags); int bad_to_remove_submodule(const char *path, unsigned flags);
/* /*
* Call add_submodule_odb() to add the submodule at the given path to a list. * Call add_submodule_odb_by_path() to add the submodule at the given
* When register_all_submodule_odb_as_alternates() is called, the object stores * path to a list. When register_all_submodule_odb_as_alternates() is
* of all submodules in that list will be added as alternates in * called, the object stores of all submodules in that list will be
* the_repository. * added as alternates in the_repository.
*/ */
int add_submodule_odb(const char *path);
void add_submodule_odb_by_path(const char *path); void add_submodule_odb_by_path(const char *path);
int register_all_submodule_odb_as_alternates(void); int register_all_submodule_odb_as_alternates(void);

545
t/t5526-fetch-submodules.sh
View file

@ -10,33 +10,122 @@ export GIT_TEST_FATAL_REGISTER_SUBMODULE_ODB
pwd=$(pwd) pwd=$(pwd)
add_upstream_commit() { write_expected_sub () {
NEW_HEAD=$1 &&
SUPER_HEAD=$2 &&
cat >"$pwd/expect.err.sub" <<-EOF
Fetching submodule submodule${SUPER_HEAD:+ at commit $SUPER_HEAD}
From $pwd/submodule
OLD_HEAD..$NEW_HEAD sub -> origin/sub
EOF
}
write_expected_sub2 () {
NEW_HEAD=$1 &&
SUPER_HEAD=$2 &&
cat >"$pwd/expect.err.sub2" <<-EOF
Fetching submodule submodule2${SUPER_HEAD:+ at commit $SUPER_HEAD}
From $pwd/submodule2
OLD_HEAD..$NEW_HEAD sub2 -> origin/sub2
EOF
}
write_expected_deep () {
NEW_HEAD=$1 &&
SUB_HEAD=$2 &&
cat >"$pwd/expect.err.deep" <<-EOF
Fetching submodule submodule/subdir/deepsubmodule${SUB_HEAD:+ at commit $SUB_HEAD}
From $pwd/deepsubmodule
OLD_HEAD..$NEW_HEAD deep -> origin/deep
EOF
}
write_expected_super () {
NEW_HEAD=$1 &&
cat >"$pwd/expect.err.super" <<-EOF
From $pwd/.
OLD_HEAD..$NEW_HEAD super -> origin/super
EOF
}
# For each submodule in the test setup, this creates a commit and writes
# a file that contains the expected err if that new commit were fetched.
# These output files get concatenated in the right order by
# verify_fetch_result().
add_submodule_commits () {
( (
cd submodule && cd submodule &&
head1=$(git rev-parse --short HEAD) &&
echo new >> subfile && echo new >> subfile &&
test_tick && test_tick &&
git add subfile && git add subfile &&
git commit -m new subfile && git commit -m new subfile &&
head2=$(git rev-parse --short HEAD) && new_head=$(git rev-parse --short HEAD) &&
echo "Fetching submodule submodule" > ../expect.err && write_expected_sub $new_head
echo "From $pwd/submodule" >> ../expect.err &&
echo " $head1..$head2 sub -> origin/sub" >> ../expect.err
) && ) &&
( (
cd deepsubmodule && cd deepsubmodule &&
head1=$(git rev-parse --short HEAD) &&
echo new >> deepsubfile && echo new >> deepsubfile &&
test_tick && test_tick &&
git add deepsubfile && git add deepsubfile &&
git commit -m new deepsubfile && git commit -m new deepsubfile &&
head2=$(git rev-parse --short HEAD) && new_head=$(git rev-parse --short HEAD) &&
echo "Fetching submodule submodule/subdir/deepsubmodule" >> ../expect.err write_expected_deep $new_head
echo "From $pwd/deepsubmodule" >> ../expect.err &&
echo " $head1..$head2 deep -> origin/deep" >> ../expect.err
) )
} }
# For each superproject in the test setup, update its submodule, add the
# submodule and create a new commit with the submodule change.
#
# This requires add_submodule_commits() to be called first, otherwise
# the submodules will not have changed and cannot be "git add"-ed.
add_superproject_commits () {
(
cd submodule &&
(
cd subdir/deepsubmodule &&
git fetch &&
git checkout -q FETCH_HEAD
) &&
git add subdir/deepsubmodule &&
git commit -m "new deep submodule"
) &&
git add submodule &&
git commit -m "new submodule" &&
super_head=$(git rev-parse --short HEAD) &&
sub_head=$(git -C submodule rev-parse --short HEAD) &&
write_expected_super $super_head &&
write_expected_sub $sub_head
}
# Verifies that the expected repositories were fetched. This is done by
# concatenating the files expect.err.[super|sub|deep] in the correct
# order and comparing it to the actual stderr.
#
# If a repo should not be fetched in the test, its corresponding
# expect.err file should be rm-ed.
verify_fetch_result () {
ACTUAL_ERR=$1 &&
rm -f expect.err.combined &&
if test -f expect.err.super
then
cat expect.err.super >>expect.err.combined
fi &&
if test -f expect.err.sub
then
cat expect.err.sub >>expect.err.combined
fi &&
if test -f expect.err.deep
then
cat expect.err.deep >>expect.err.combined
fi &&
if test -f expect.err.sub2
then
cat expect.err.sub2 >>expect.err.combined
fi &&
sed -e 's/[0-9a-f][0-9a-f]*\.\./OLD_HEAD\.\./' "$ACTUAL_ERR" >actual.err.cmp &&
test_cmp expect.err.combined actual.err.cmp
}
test_expect_success setup ' test_expect_success setup '
mkdir deepsubmodule && mkdir deepsubmodule &&
( (
@ -68,38 +157,38 @@ test_expect_success setup '
' '
test_expect_success "fetch --recurse-submodules recurses into submodules" ' test_expect_success "fetch --recurse-submodules recurses into submodules" '
add_upstream_commit && add_submodule_commits &&
( (
cd downstream && cd downstream &&
git fetch --recurse-submodules >../actual.out 2>../actual.err git fetch --recurse-submodules >../actual.out 2>../actual.err
) && ) &&
test_must_be_empty actual.out && test_must_be_empty actual.out &&
test_cmp expect.err actual.err verify_fetch_result actual.err
' '
test_expect_success "submodule.recurse option triggers recursive fetch" ' test_expect_success "submodule.recurse option triggers recursive fetch" '
add_upstream_commit && add_submodule_commits &&
( (
cd downstream && cd downstream &&
git -c submodule.recurse fetch >../actual.out 2>../actual.err git -c submodule.recurse fetch >../actual.out 2>../actual.err
) && ) &&
test_must_be_empty actual.out && test_must_be_empty actual.out &&
test_cmp expect.err actual.err verify_fetch_result actual.err
' '
test_expect_success "fetch --recurse-submodules -j2 has the same output behaviour" ' test_expect_success "fetch --recurse-submodules -j2 has the same output behaviour" '
add_upstream_commit && add_submodule_commits &&
( (
cd downstream && cd downstream &&
GIT_TRACE="$TRASH_DIRECTORY/trace.out" git fetch --recurse-submodules -j2 2>../actual.err GIT_TRACE="$TRASH_DIRECTORY/trace.out" git fetch --recurse-submodules -j2 2>../actual.err
) && ) &&
test_must_be_empty actual.out && test_must_be_empty actual.out &&
test_cmp expect.err actual.err && verify_fetch_result actual.err &&
grep "2 tasks" trace.out grep "2 tasks" trace.out
' '
test_expect_success "fetch alone only fetches superproject" ' test_expect_success "fetch alone only fetches superproject" '
add_upstream_commit && add_submodule_commits &&
( (
cd downstream && cd downstream &&
git fetch >../actual.out 2>../actual.err git fetch >../actual.out 2>../actual.err
@ -124,11 +213,11 @@ test_expect_success "using fetchRecurseSubmodules=true in .gitmodules recurses i
git fetch >../actual.out 2>../actual.err git fetch >../actual.out 2>../actual.err
) && ) &&
test_must_be_empty actual.out && test_must_be_empty actual.out &&
test_cmp expect.err actual.err verify_fetch_result actual.err
' '
test_expect_success "--no-recurse-submodules overrides .gitmodules config" ' test_expect_success "--no-recurse-submodules overrides .gitmodules config" '
add_upstream_commit && add_submodule_commits &&
( (
cd downstream && cd downstream &&
git fetch --no-recurse-submodules >../actual.out 2>../actual.err git fetch --no-recurse-submodules >../actual.out 2>../actual.err
@ -155,7 +244,7 @@ test_expect_success "--recurse-submodules overrides fetchRecurseSubmodules setti
git config --unset submodule.submodule.fetchRecurseSubmodules git config --unset submodule.submodule.fetchRecurseSubmodules
) && ) &&
test_must_be_empty actual.out && test_must_be_empty actual.out &&
test_cmp expect.err actual.err verify_fetch_result actual.err
' '
test_expect_success "--quiet propagates to submodules" ' test_expect_success "--quiet propagates to submodules" '
@ -177,13 +266,13 @@ test_expect_success "--quiet propagates to parallel submodules" '
' '
test_expect_success "--dry-run propagates to submodules" ' test_expect_success "--dry-run propagates to submodules" '
add_upstream_commit && add_submodule_commits &&
( (
cd downstream && cd downstream &&
git fetch --recurse-submodules --dry-run >../actual.out 2>../actual.err git fetch --recurse-submodules --dry-run >../actual.out 2>../actual.err
) && ) &&
test_must_be_empty actual.out && test_must_be_empty actual.out &&
test_cmp expect.err actual.err verify_fetch_result actual.err
' '
test_expect_success "Without --dry-run propagates to submodules" ' test_expect_success "Without --dry-run propagates to submodules" '
@ -192,22 +281,22 @@ test_expect_success "Without --dry-run propagates to submodules" '
git fetch --recurse-submodules >../actual.out 2>../actual.err git fetch --recurse-submodules >../actual.out 2>../actual.err
) && ) &&
test_must_be_empty actual.out && test_must_be_empty actual.out &&
test_cmp expect.err actual.err verify_fetch_result actual.err
' '
test_expect_success "recurseSubmodules=true propagates into submodules" ' test_expect_success "recurseSubmodules=true propagates into submodules" '
add_upstream_commit && add_submodule_commits &&
( (
cd downstream && cd downstream &&
git config fetch.recurseSubmodules true && git config fetch.recurseSubmodules true &&
git fetch >../actual.out 2>../actual.err git fetch >../actual.out 2>../actual.err
) && ) &&
test_must_be_empty actual.out && test_must_be_empty actual.out &&
test_cmp expect.err actual.err verify_fetch_result actual.err
' '
test_expect_success "--recurse-submodules overrides config in submodule" ' test_expect_success "--recurse-submodules overrides config in submodule" '
add_upstream_commit && add_submodule_commits &&
( (
cd downstream && cd downstream &&
( (
@ -217,11 +306,11 @@ test_expect_success "--recurse-submodules overrides config in submodule" '
git fetch --recurse-submodules >../actual.out 2>../actual.err git fetch --recurse-submodules >../actual.out 2>../actual.err
) && ) &&
test_must_be_empty actual.out && test_must_be_empty actual.out &&
test_cmp expect.err actual.err verify_fetch_result actual.err
' '
test_expect_success "--no-recurse-submodules overrides config setting" ' test_expect_success "--no-recurse-submodules overrides config setting" '
add_upstream_commit && add_submodule_commits &&
( (
cd downstream && cd downstream &&
git config fetch.recurseSubmodules true && git config fetch.recurseSubmodules true &&
@ -246,36 +335,34 @@ test_expect_success "Recursion doesn't happen when no new commits are fetched in
' '
test_expect_success "Recursion stops when no new submodule commits are fetched" ' test_expect_success "Recursion stops when no new submodule commits are fetched" '
head1=$(git rev-parse --short HEAD) &&
git add submodule && git add submodule &&
git commit -m "new submodule" && git commit -m "new submodule" &&
head2=$(git rev-parse --short HEAD) && new_head=$(git rev-parse --short HEAD) &&
echo "From $pwd/." > expect.err.sub && write_expected_super $new_head &&
echo " $head1..$head2 super -> origin/super" >>expect.err.sub && rm expect.err.deep &&
head -3 expect.err >> expect.err.sub &&
( (
cd downstream && cd downstream &&
git fetch >../actual.out 2>../actual.err git fetch >../actual.out 2>../actual.err
) && ) &&
test_cmp expect.err.sub actual.err && verify_fetch_result actual.err &&
test_must_be_empty actual.out test_must_be_empty actual.out
' '
test_expect_success "Recursion doesn't happen when new superproject commits don't change any submodules" ' test_expect_success "Recursion doesn't happen when new superproject commits don't change any submodules" '
add_upstream_commit && add_submodule_commits &&
head1=$(git rev-parse --short HEAD) &&
echo a > file && echo a > file &&
git add file && git add file &&
git commit -m "new file" && git commit -m "new file" &&
head2=$(git rev-parse --short HEAD) && new_head=$(git rev-parse --short HEAD) &&
echo "From $pwd/." > expect.err.file && write_expected_super $new_head &&
echo " $head1..$head2 super -> origin/super" >> expect.err.file && rm expect.err.sub &&
rm expect.err.deep &&
( (
cd downstream && cd downstream &&
git fetch >../actual.out 2>../actual.err git fetch >../actual.out 2>../actual.err
) && ) &&
test_must_be_empty actual.out && test_must_be_empty actual.out &&
test_cmp expect.err.file actual.err verify_fetch_result actual.err
' '
test_expect_success "Recursion picks up config in submodule" ' test_expect_success "Recursion picks up config in submodule" '
@ -287,14 +374,11 @@ test_expect_success "Recursion picks up config in submodule" '
git config fetch.recurseSubmodules true git config fetch.recurseSubmodules true
) )
) && ) &&
add_upstream_commit && add_submodule_commits &&
head1=$(git rev-parse --short HEAD) &&
git add submodule && git add submodule &&
git commit -m "new submodule" && git commit -m "new submodule" &&
head2=$(git rev-parse --short HEAD) && new_head=$(git rev-parse --short HEAD) &&
echo "From $pwd/." > expect.err.sub && write_expected_super $new_head &&
echo " $head1..$head2 super -> origin/super" >> expect.err.sub &&
cat expect.err >> expect.err.sub &&
( (
cd downstream && cd downstream &&
git fetch >../actual.out 2>../actual.err && git fetch >../actual.out 2>../actual.err &&
@ -303,60 +387,23 @@ test_expect_success "Recursion picks up config in submodule" '
git config --unset fetch.recurseSubmodules git config --unset fetch.recurseSubmodules
) )
) && ) &&
test_cmp expect.err.sub actual.err && verify_fetch_result actual.err &&
test_must_be_empty actual.out test_must_be_empty actual.out
' '
test_expect_success "Recursion picks up all submodules when necessary" ' test_expect_success "Recursion picks up all submodules when necessary" '
add_upstream_commit && add_submodule_commits &&
( add_superproject_commits &&
cd submodule &&
(
cd subdir/deepsubmodule &&
git fetch &&
git checkout -q FETCH_HEAD
) &&
head1=$(git rev-parse --short HEAD^) &&
git add subdir/deepsubmodule &&
git commit -m "new deepsubmodule" &&
head2=$(git rev-parse --short HEAD) &&
echo "Fetching submodule submodule" > ../expect.err.sub &&
echo "From $pwd/submodule" >> ../expect.err.sub &&
echo " $head1..$head2 sub -> origin/sub" >> ../expect.err.sub
) &&
head1=$(git rev-parse --short HEAD) &&
git add submodule &&
git commit -m "new submodule" &&
head2=$(git rev-parse --short HEAD) &&
echo "From $pwd/." > expect.err.2 &&
echo " $head1..$head2 super -> origin/super" >> expect.err.2 &&
cat expect.err.sub >> expect.err.2 &&
tail -3 expect.err >> expect.err.2 &&
( (
cd downstream && cd downstream &&
git fetch >../actual.out 2>../actual.err git fetch >../actual.out 2>../actual.err
) && ) &&
test_cmp expect.err.2 actual.err && verify_fetch_result actual.err &&
test_must_be_empty actual.out test_must_be_empty actual.out
' '
test_expect_success "'--recurse-submodules=on-demand' doesn't recurse when no new commits are fetched in the superproject (and ignores config)" ' test_expect_success "'--recurse-submodules=on-demand' doesn't recurse when no new commits are fetched in the superproject (and ignores config)" '
add_upstream_commit && add_submodule_commits &&
(
cd submodule &&
(
cd subdir/deepsubmodule &&
git fetch &&
git checkout -q FETCH_HEAD
) &&
head1=$(git rev-parse --short HEAD^) &&
git add subdir/deepsubmodule &&
git commit -m "new deepsubmodule" &&
head2=$(git rev-parse --short HEAD) &&
echo Fetching submodule submodule > ../expect.err.sub &&
echo "From $pwd/submodule" >> ../expect.err.sub &&
echo " $head1..$head2 sub -> origin/sub" >> ../expect.err.sub
) &&
( (
cd downstream && cd downstream &&
git config fetch.recurseSubmodules true && git config fetch.recurseSubmodules true &&
@ -368,15 +415,8 @@ test_expect_success "'--recurse-submodules=on-demand' doesn't recurse when no ne
' '
test_expect_success "'--recurse-submodules=on-demand' recurses as deep as necessary (and ignores config)" ' test_expect_success "'--recurse-submodules=on-demand' recurses as deep as necessary (and ignores config)" '
head1=$(git rev-parse --short HEAD) && add_submodule_commits &&
git add submodule && add_superproject_commits &&
git commit -m "new submodule" &&
head2=$(git rev-parse --short HEAD) &&
tail -3 expect.err > expect.err.deepsub &&
echo "From $pwd/." > expect.err &&
echo " $head1..$head2 super -> origin/super" >>expect.err &&
cat expect.err.sub >> expect.err &&
cat expect.err.deepsub >> expect.err &&
( (
cd downstream && cd downstream &&
git config fetch.recurseSubmodules false && git config fetch.recurseSubmodules false &&
@ -392,24 +432,165 @@ test_expect_success "'--recurse-submodules=on-demand' recurses as deep as necess
) )
) && ) &&
test_must_be_empty actual.out && test_must_be_empty actual.out &&
test_cmp expect.err actual.err verify_fetch_result actual.err
'
# These tests verify that we can fetch submodules that aren't in the
# index.
#
# First, test the simple case where the index is empty and we only fetch
# submodules that are not in the index.
test_expect_success 'setup downstream branch without submodules' '
(
cd downstream &&
git checkout --recurse-submodules -b no-submodules &&
git rm .gitmodules &&
git rm submodule &&
git commit -m "no submodules" &&
git checkout --recurse-submodules super
)
'
test_expect_success "'--recurse-submodules=on-demand' should fetch submodule commits if the submodule is changed but the index has no submodules" '
add_submodule_commits &&
add_superproject_commits &&
# Fetch the new superproject commit
(
cd downstream &&
git switch --recurse-submodules no-submodules &&
git fetch --recurse-submodules=on-demand >../actual.out 2>../actual.err
) &&
super_head=$(git rev-parse --short HEAD) &&
sub_head=$(git -C submodule rev-parse --short HEAD) &&
deep_head=$(git -C submodule/subdir/deepsubmodule rev-parse --short HEAD) &&
# assert that these are fetched from commits, not the index
write_expected_sub $sub_head $super_head &&
write_expected_deep $deep_head $sub_head &&
test_must_be_empty actual.out &&
verify_fetch_result actual.err
'
test_expect_success "'--recurse-submodules' should fetch submodule commits if the submodule is changed but the index has no submodules" '
add_submodule_commits &&
add_superproject_commits &&
# Fetch the new superproject commit
(
cd downstream &&
git switch --recurse-submodules no-submodules &&
git fetch --recurse-submodules >../actual.out 2>../actual.err
) &&
super_head=$(git rev-parse --short HEAD) &&
sub_head=$(git -C submodule rev-parse --short HEAD) &&
deep_head=$(git -C submodule/subdir/deepsubmodule rev-parse --short HEAD) &&
# assert that these are fetched from commits, not the index
write_expected_sub $sub_head $super_head &&
write_expected_deep $deep_head $sub_head &&
test_must_be_empty actual.out &&
verify_fetch_result actual.err
'
test_expect_success "'--recurse-submodules' should ignore changed, inactive submodules" '
add_submodule_commits &&
add_superproject_commits &&
# Fetch the new superproject commit
(
cd downstream &&
git switch --recurse-submodules no-submodules &&
git -c submodule.submodule.active=false fetch --recurse-submodules >../actual.out 2>../actual.err
) &&
test_must_be_empty actual.out &&
super_head=$(git rev-parse --short HEAD) &&
write_expected_super $super_head &&
# Neither should be fetched because the submodule is inactive
rm expect.err.sub &&
rm expect.err.deep &&
verify_fetch_result actual.err
'
# Now that we know we can fetch submodules that are not in the index,
# test that we can fetch index and non-index submodules in the same
# operation.
test_expect_success 'setup downstream branch with other submodule' '
mkdir submodule2 &&
(
cd submodule2 &&
git init &&
echo sub2content >sub2file &&
git add sub2file &&
git commit -a -m new &&
git branch -M sub2
) &&
git checkout -b super-sub2-only &&
git submodule add "$pwd/submodule2" submodule2 &&
git commit -m "add sub2" &&
git checkout super &&
(
cd downstream &&
git fetch --recurse-submodules origin &&
git checkout super-sub2-only &&
# Explicitly run "git submodule update" because sub2 is new
# and has not been cloned.
git submodule update --init &&
git checkout --recurse-submodules super
)
'
test_expect_success "'--recurse-submodules' should fetch submodule commits in changed submodules and the index" '
test_when_finished "rm expect.err.sub2" &&
# Create new commit in origin/super
add_submodule_commits &&
add_superproject_commits &&
# Create new commit in origin/super-sub2-only
git checkout super-sub2-only &&
(
cd submodule2 &&
test_commit --no-tag foo
) &&
git add submodule2 &&
git commit -m "new submodule2" &&
git checkout super &&
(
cd downstream &&
git fetch --recurse-submodules >../actual.out 2>../actual.err
) &&
test_must_be_empty actual.out &&
sub2_head=$(git -C submodule2 rev-parse --short HEAD) &&
super_head=$(git rev-parse --short super) &&
super_sub2_only_head=$(git rev-parse --short super-sub2-only) &&
write_expected_sub2 $sub2_head $super_sub2_only_head &&
# write_expected_super cannot handle >1 branch. Since this is a
# one-off, construct expect.err.super manually.
cat >"$pwd/expect.err.super" <<-EOF &&
From $pwd/.
OLD_HEAD..$super_head super -> origin/super
OLD_HEAD..$super_sub2_only_head super-sub2-only -> origin/super-sub2-only
EOF
verify_fetch_result actual.err
' '
test_expect_success "'--recurse-submodules=on-demand' stops when no new submodule commits are found in the superproject (and ignores config)" ' test_expect_success "'--recurse-submodules=on-demand' stops when no new submodule commits are found in the superproject (and ignores config)" '
add_upstream_commit && add_submodule_commits &&
head1=$(git rev-parse --short HEAD) &&
echo a >> file && echo a >> file &&
git add file && git add file &&
git commit -m "new file" && git commit -m "new file" &&
head2=$(git rev-parse --short HEAD) && new_head=$(git rev-parse --short HEAD) &&
echo "From $pwd/." > expect.err.file && write_expected_super $new_head &&
echo " $head1..$head2 super -> origin/super" >> expect.err.file && rm expect.err.sub &&
rm expect.err.deep &&
( (
cd downstream && cd downstream &&
git fetch --recurse-submodules=on-demand >../actual.out 2>../actual.err git fetch --recurse-submodules=on-demand >../actual.out 2>../actual.err
) && ) &&
test_must_be_empty actual.out && test_must_be_empty actual.out &&
test_cmp expect.err.file actual.err verify_fetch_result actual.err
' '
test_expect_success "'fetch.recurseSubmodules=on-demand' overrides global config" ' test_expect_success "'fetch.recurseSubmodules=on-demand' overrides global config" '
@ -417,15 +598,13 @@ test_expect_success "'fetch.recurseSubmodules=on-demand' overrides global config
cd downstream && cd downstream &&
git fetch --recurse-submodules git fetch --recurse-submodules
) && ) &&
add_upstream_commit && add_submodule_commits &&
git config --global fetch.recurseSubmodules false && git config --global fetch.recurseSubmodules false &&
head1=$(git rev-parse --short HEAD) &&
git add submodule && git add submodule &&
git commit -m "new submodule" && git commit -m "new submodule" &&
head2=$(git rev-parse --short HEAD) && new_head=$(git rev-parse --short HEAD) &&
echo "From $pwd/." > expect.err.2 && write_expected_super $new_head &&
echo " $head1..$head2 super -> origin/super" >>expect.err.2 && rm expect.err.deep &&
head -3 expect.err >> expect.err.2 &&
( (
cd downstream && cd downstream &&
git config fetch.recurseSubmodules on-demand && git config fetch.recurseSubmodules on-demand &&
@ -437,7 +616,7 @@ test_expect_success "'fetch.recurseSubmodules=on-demand' overrides global config
git config --unset fetch.recurseSubmodules git config --unset fetch.recurseSubmodules
) && ) &&
test_must_be_empty actual.out && test_must_be_empty actual.out &&
test_cmp expect.err.2 actual.err verify_fetch_result actual.err
' '
test_expect_success "'submodule.<sub>.fetchRecurseSubmodules=on-demand' overrides fetch.recurseSubmodules" ' test_expect_success "'submodule.<sub>.fetchRecurseSubmodules=on-demand' overrides fetch.recurseSubmodules" '
@ -445,15 +624,13 @@ test_expect_success "'submodule.<sub>.fetchRecurseSubmodules=on-demand' override
cd downstream && cd downstream &&
git fetch --recurse-submodules git fetch --recurse-submodules
) && ) &&
add_upstream_commit && add_submodule_commits &&
git config fetch.recurseSubmodules false && git config fetch.recurseSubmodules false &&
head1=$(git rev-parse --short HEAD) &&
git add submodule && git add submodule &&
git commit -m "new submodule" && git commit -m "new submodule" &&
head2=$(git rev-parse --short HEAD) && new_head=$(git rev-parse --short HEAD) &&
echo "From $pwd/." > expect.err.2 && write_expected_super $new_head &&
echo " $head1..$head2 super -> origin/super" >>expect.err.2 && rm expect.err.deep &&
head -3 expect.err >> expect.err.2 &&
( (
cd downstream && cd downstream &&
git config submodule.submodule.fetchRecurseSubmodules on-demand && git config submodule.submodule.fetchRecurseSubmodules on-demand &&
@ -465,7 +642,7 @@ test_expect_success "'submodule.<sub>.fetchRecurseSubmodules=on-demand' override
git config --unset submodule.submodule.fetchRecurseSubmodules git config --unset submodule.submodule.fetchRecurseSubmodules
) && ) &&
test_must_be_empty actual.out && test_must_be_empty actual.out &&
test_cmp expect.err.2 actual.err verify_fetch_result actual.err
' '
test_expect_success "don't fetch submodule when newly recorded commits are already present" ' test_expect_success "don't fetch submodule when newly recorded commits are already present" '
@ -473,18 +650,19 @@ test_expect_success "don't fetch submodule when newly recorded commits are alrea
cd submodule && cd submodule &&
git checkout -q HEAD^^ git checkout -q HEAD^^
) && ) &&
head1=$(git rev-parse --short HEAD) &&
git add submodule && git add submodule &&
git commit -m "submodule rewound" && git commit -m "submodule rewound" &&
head2=$(git rev-parse --short HEAD) && new_head=$(git rev-parse --short HEAD) &&
echo "From $pwd/." > expect.err && write_expected_super $new_head &&
echo " $head1..$head2 super -> origin/super" >> expect.err && rm expect.err.sub &&
# This file does not exist, but rm -f for readability
rm -f expect.err.deep &&
( (
cd downstream && cd downstream &&
git fetch >../actual.out 2>../actual.err git fetch >../actual.out 2>../actual.err
) && ) &&
test_must_be_empty actual.out && test_must_be_empty actual.out &&
test_cmp expect.err actual.err && verify_fetch_result actual.err &&
( (
cd submodule && cd submodule &&
git checkout -q sub git checkout -q sub
@ -496,15 +674,13 @@ test_expect_success "'fetch.recurseSubmodules=on-demand' works also without .git
cd downstream && cd downstream &&
git fetch --recurse-submodules git fetch --recurse-submodules
) && ) &&
add_upstream_commit && add_submodule_commits &&
head1=$(git rev-parse --short HEAD) &&
git add submodule && git add submodule &&
git rm .gitmodules && git rm .gitmodules &&
git commit -m "new submodule without .gitmodules" && git commit -m "new submodule without .gitmodules" &&
head2=$(git rev-parse --short HEAD) && new_head=$(git rev-parse --short HEAD) &&
echo "From $pwd/." >expect.err.2 && write_expected_super $new_head &&
echo " $head1..$head2 super -> origin/super" >>expect.err.2 && rm expect.err.deep &&
head -3 expect.err >>expect.err.2 &&
( (
cd downstream && cd downstream &&
rm .gitmodules && rm .gitmodules &&
@ -520,7 +696,7 @@ test_expect_success "'fetch.recurseSubmodules=on-demand' works also without .git
git reset --hard git reset --hard
) && ) &&
test_must_be_empty actual.out && test_must_be_empty actual.out &&
test_cmp expect.err.2 actual.err && verify_fetch_result actual.err &&
git checkout HEAD^ -- .gitmodules && git checkout HEAD^ -- .gitmodules &&
git add .gitmodules && git add .gitmodules &&
git commit -m "new submodule restored .gitmodules" git commit -m "new submodule restored .gitmodules"
@ -842,4 +1018,111 @@ test_expect_success 'recursive fetch after deinit a submodule' '
test_cmp expect actual test_cmp expect actual
' '
test_expect_success 'setup repo with upstreams that share a submodule name' '
mkdir same-name-1 &&
(
cd same-name-1 &&
git init -b main &&
test_commit --no-tag a
) &&
git clone same-name-1 same-name-2 &&
# same-name-1 and same-name-2 both add a submodule with the
# name "submodule"
(
cd same-name-1 &&
mkdir submodule &&
git -C submodule init -b main &&
test_commit -C submodule --no-tag a1 &&
git submodule add "$pwd/same-name-1/submodule" &&
git add submodule &&
git commit -m "super-a1"
) &&
(
cd same-name-2 &&
mkdir submodule &&
git -C submodule init -b main &&
test_commit -C submodule --no-tag a2 &&
git submodule add "$pwd/same-name-2/submodule" &&
git add submodule &&
git commit -m "super-a2"
) &&
git clone same-name-1 -o same-name-1 same-name-downstream &&
(
cd same-name-downstream &&
git remote add same-name-2 ../same-name-2 &&
git fetch --all &&
# init downstream with same-name-1
git submodule update --init
)
'
test_expect_success 'fetch --recurse-submodules updates name-conflicted, populated submodule' '
test_when_finished "git -C same-name-downstream checkout main" &&
(
cd same-name-1 &&
test_commit -C submodule --no-tag b1 &&
git add submodule &&
git commit -m "super-b1"
) &&
(
cd same-name-2 &&
test_commit -C submodule --no-tag b2 &&
git add submodule &&
git commit -m "super-b2"
) &&
(
cd same-name-downstream &&
# even though the .gitmodules is correct, we cannot
# fetch from same-name-2
git checkout same-name-2/main &&
git fetch --recurse-submodules same-name-1 &&
test_must_fail git fetch --recurse-submodules same-name-2
) &&
super_head1=$(git -C same-name-1 rev-parse HEAD) &&
git -C same-name-downstream cat-file -e $super_head1 &&
super_head2=$(git -C same-name-2 rev-parse HEAD) &&
git -C same-name-downstream cat-file -e $super_head2 &&
sub_head1=$(git -C same-name-1/submodule rev-parse HEAD) &&
git -C same-name-downstream/submodule cat-file -e $sub_head1 &&
sub_head2=$(git -C same-name-2/submodule rev-parse HEAD) &&
test_must_fail git -C same-name-downstream/submodule cat-file -e $sub_head2
'
test_expect_success 'fetch --recurse-submodules updates name-conflicted, unpopulated submodule' '
(
cd same-name-1 &&
test_commit -C submodule --no-tag c1 &&
git add submodule &&
git commit -m "super-c1"
) &&
(
cd same-name-2 &&
test_commit -C submodule --no-tag c2 &&
git add submodule &&
git commit -m "super-c2"
) &&
(
cd same-name-downstream &&
git checkout main &&
git rm .gitmodules &&
git rm submodule &&
git commit -m "no submodules" &&
git fetch --recurse-submodules same-name-1
) &&
head1=$(git -C same-name-1/submodule rev-parse HEAD) &&
head2=$(git -C same-name-2/submodule rev-parse HEAD) &&
(
cd same-name-downstream/.git/modules/submodule &&
# The submodule has core.worktree pointing to the "git
# rm"-ed directory, overwrite the invalid value. See
# comment in get_fetch_task_from_changed() for more
# information.
git --work-tree=. cat-file -e $head1 &&
test_must_fail git --work-tree=. cat-file -e $head2
)
'
test_done test_done