Merge branch 'sb/pull-rebase-submodule'

"git pull --rebase --recurse-submodules" learns to rebase the
branch in the submodules to an updated base.

* sb/pull-rebase-submodule:
  builtin/fetch cleanup: always set default value for submodule recursing
  pull: optionally rebase submodules (remote submodule changes only)
  builtin/fetch: parse recurse-submodules-default at default options parsing
  builtin/fetch: factor submodule recurse parsing out to submodule config

Documentation/git-pull.txt

@@ -86,12 +86,12 @@ OPTIONS
 
 --[no-]recurse-submodules[=yes|on-demand|no]::
 	This option controls if new commits of all populated submodules should
-	be fetched too (see linkgit:git-config[1] and linkgit:gitmodules[5]).
-	That might be necessary to get the data needed for merging submodule
-	commits, a feature Git learned in 1.7.3. Notice that the result of a
-	merge will not be checked out in the submodule, "git submodule update"
-	has to be called afterwards to bring the work tree up to date with the
-	merge result.
+	be fetched and updated, too (see linkgit:git-config[1] and
+	linkgit:gitmodules[5]).
++
+If the checkout is done via rebase, local submodule commits are rebased as well.
++
+If the update is done via merge, the submodule conflicts are resolved and checked out.
 
 Options related to merging
 ~~~~~~~~~~~~~~~~~~~~~~~~~~

builtin/fetch.c

@@ -37,7 +37,7 @@ static int prune = -1; /* unspecified */
 #define PRUNE_BY_DEFAULT 0 /* do we prune by default? */
 
 static int all, append, dry_run, force, keep, multiple, update_head_ok, verbosity, deepen_relative;
-static int progress = -1, recurse_submodules = RECURSE_SUBMODULES_DEFAULT;
+static int progress = -1;
 static int tags = TAGS_DEFAULT, unshallow, update_shallow, deepen;
 static int max_children = -1;
 static enum transport_family family;
@@ -49,25 +49,12 @@ static struct strbuf default_rla = STRBUF_INIT;
 static struct transport *gtransport;
 static struct transport *gsecondary;
 static const char *submodule_prefix = "";
-static const char *recurse_submodules_default;
+static int recurse_submodules = RECURSE_SUBMODULES_DEFAULT;
+static int recurse_submodules_default = RECURSE_SUBMODULES_ON_DEMAND;
 static int shown_url = 0;
 static int refmap_alloc, refmap_nr;
 static const char **refmap_array;
 
-static int option_parse_recurse_submodules(const struct option *opt,
-				   const char *arg, int unset)
-{
-	if (unset) {
-		recurse_submodules = RECURSE_SUBMODULES_OFF;
-	} else {
-		if (arg)
-			recurse_submodules = parse_fetch_recurse_submodules_arg(opt->long_name, arg);
-		else
-			recurse_submodules = RECURSE_SUBMODULES_ON;
-	}
-	return 0;
-}
-
 static int git_fetch_config(const char *k, const char *v, void *cb)
 {
 	if (!strcmp(k, "fetch.prune")) {
@@ -116,9 +103,9 @@ static struct option builtin_fetch_options[] = {
 		    N_("number of submodules fetched in parallel")),
 	OPT_BOOL('p', "prune", &prune,
 		 N_("prune remote-tracking branches no longer on remote")),
-	{ OPTION_CALLBACK, 0, "recurse-submodules", NULL, N_("on-demand"),
+	{ OPTION_CALLBACK, 0, "recurse-submodules", &recurse_submodules, N_("on-demand"),
 		    N_("control recursive fetching of submodules"),
-		    PARSE_OPT_OPTARG, option_parse_recurse_submodules },
+		    PARSE_OPT_OPTARG, option_fetch_parse_recurse_submodules },
 	OPT_BOOL(0, "dry-run", &dry_run,
 		 N_("dry run")),
 	OPT_BOOL('k', "keep", &keep, N_("keep downloaded pack")),
@@ -138,9 +125,11 @@ static struct option builtin_fetch_options[] = {
 		   PARSE_OPT_NONEG | PARSE_OPT_NOARG, NULL, 1 },
 	{ OPTION_STRING, 0, "submodule-prefix", &submodule_prefix, N_("dir"),
 		   N_("prepend this to submodule path output"), PARSE_OPT_HIDDEN },
-	{ OPTION_STRING, 0, "recurse-submodules-default",
-		   &recurse_submodules_default, NULL,
-		   N_("default mode for recursion"), PARSE_OPT_HIDDEN },
+	{ OPTION_CALLBACK, 0, "recurse-submodules-default",
+		   &recurse_submodules_default, N_("on-demand"),
+		   N_("default for recursive fetching of submodules "
+		      "(lower priority than config files)"),
+		   PARSE_OPT_HIDDEN, option_fetch_parse_recurse_submodules },
 	OPT_BOOL(0, "update-shallow", &update_shallow,
 		 N_("accept refs that update .git/shallow")),
 	{ OPTION_CALLBACK, 0, "refmap", NULL, N_("refmap"),
@@ -1350,10 +1339,7 @@ int cmd_fetch(int argc, const char **argv, const char *prefix)
 		deepen = 1;
 
 	if (recurse_submodules != RECURSE_SUBMODULES_OFF) {
-		if (recurse_submodules_default) {
-			int arg = parse_fetch_recurse_submodules_arg("--recurse-submodules-default", recurse_submodules_default);
-			set_config_fetch_recurse_submodules(arg);
-		}
+		set_config_fetch_recurse_submodules(recurse_submodules_default);
 		gitmodules_config();
 		git_config(submodule_config, NULL);
 	}

builtin/pull.c

@@ -16,6 +16,8 @@
 #include "dir.h"
 #include "refs.h"
 #include "revision.h"
+#include "submodule.h"
+#include "submodule-config.h"
 #include "tempfile.h"
 #include "lockfile.h"
 #include "wt-status.h"
@@ -78,6 +80,7 @@ static const char * const pull_usage[] = {
 /* Shared options */
 static int opt_verbosity;
 static char *opt_progress;
+static int recurse_submodules = RECURSE_SUBMODULES_DEFAULT;
 
 /* Options passed to git-merge or git-rebase */
 static enum rebase_type opt_rebase = -1;
@@ -102,7 +105,6 @@ static char *opt_upload_pack;
 static int opt_force;
 static char *opt_tags;
 static char *opt_prune;
-static char *opt_recurse_submodules;
 static char *max_children;
 static int opt_dry_run;
 static char *opt_keep;
@@ -117,6 +119,10 @@ static struct option pull_options[] = {
 	OPT_PASSTHRU(0, "progress", &opt_progress, NULL,
 		N_("force progress reporting"),
 		PARSE_OPT_NOARG),
+	{ OPTION_CALLBACK, 0, "recurse-submodules",
+		   &recurse_submodules, N_("on-demand"),
+		   N_("control for recursive fetching of submodules"),
+		   PARSE_OPT_OPTARG, option_fetch_parse_recurse_submodules },
 
 	/* Options passed to git-merge or git-rebase */
 	OPT_GROUP(N_("Options related to merging")),
@@ -188,10 +194,6 @@ static struct option pull_options[] = {
 	OPT_PASSTHRU('p', "prune", &opt_prune, NULL,
 		N_("prune remote-tracking branches no longer on remote"),
 		PARSE_OPT_NOARG),
-	OPT_PASSTHRU(0, "recurse-submodules", &opt_recurse_submodules,
-		N_("on-demand"),
-		N_("control recursive fetching of submodules"),
-		PARSE_OPT_OPTARG),
 	OPT_PASSTHRU('j', "jobs", &max_children, N_("n"),
 		N_("number of submodules pulled in parallel"),
 		PARSE_OPT_OPTARG),
@@ -484,8 +486,20 @@ static int run_fetch(const char *repo, const char **refspecs)
 		argv_array_push(&args, opt_tags);
 	if (opt_prune)
 		argv_array_push(&args, opt_prune);
-	if (opt_recurse_submodules)
-		argv_array_push(&args, opt_recurse_submodules);
+	if (recurse_submodules != RECURSE_SUBMODULES_DEFAULT)
+		switch (recurse_submodules) {
+		case RECURSE_SUBMODULES_ON:
+			argv_array_push(&args, "--recurse-submodules=on");
+			break;
+		case RECURSE_SUBMODULES_OFF:
+			argv_array_push(&args, "--recurse-submodules=no");
+			break;
+		case RECURSE_SUBMODULES_ON_DEMAND:
+			argv_array_push(&args, "--recurse-submodules=on-demand");
+			break;
+		default:
+			BUG("submodule recursion option not understood");
+		}
 	if (max_children)
 		argv_array_push(&args, max_children);
 	if (opt_dry_run)
@@ -532,6 +546,30 @@ static int pull_into_void(const struct object_id *merge_head,
 	return 0;
 }
 
+static int rebase_submodules(void)
+{
+	struct child_process cp = CHILD_PROCESS_INIT;
+
+	cp.git_cmd = 1;
+	cp.no_stdin = 1;
+	argv_array_pushl(&cp.args, "submodule", "update",
+				   "--recursive", "--rebase", NULL);
+
+	return run_command(&cp);
+}
+
+static int update_submodules(void)
+{
+	struct child_process cp = CHILD_PROCESS_INIT;
+
+	cp.git_cmd = 1;
+	cp.no_stdin = 1;
+	argv_array_pushl(&cp.args, "submodule", "update",
+				   "--recursive", "--checkout", NULL);
+
+	return run_command(&cp);
+}
+
 /**
  * Runs git-merge, returning its exit status.
  */
@@ -863,6 +901,11 @@ int cmd_pull(int argc, const char **argv, const char *prefix)
 		die(_("Cannot rebase onto multiple branches."));
 
 	if (opt_rebase) {
+		int ret = 0;
+		if ((recurse_submodules == RECURSE_SUBMODULES_ON ||
+		     recurse_submodules == RECURSE_SUBMODULES_ON_DEMAND) &&
+		    submodule_touches_in_range(&rebase_fork_point, &curr_head))
+			die(_("cannot rebase with locally recorded submodule modifications"));
 		if (!autostash) {
 			struct commit_list *list = NULL;
 			struct commit *merge_head, *head;
@@ -873,11 +916,21 @@ int cmd_pull(int argc, const char **argv, const char *prefix)
 			if (is_descendant_of(merge_head, list)) {
 				/* we can fast-forward this without invoking rebase */
 				opt_ff = "--ff-only";
-				return run_merge();
+				ret = run_merge();
 			}
 		}
-		return run_rebase(&curr_head, merge_heads.oid, &rebase_fork_point);
+		ret = run_rebase(&curr_head, merge_heads.oid, &rebase_fork_point);
+
+		if (!ret && (recurse_submodules == RECURSE_SUBMODULES_ON ||
+			     recurse_submodules == RECURSE_SUBMODULES_ON_DEMAND))
+			ret = rebase_submodules();
+
+		return ret;
 	} else {
-		return run_merge();
+		int ret = run_merge();
+		if (!ret && (recurse_submodules == RECURSE_SUBMODULES_ON ||
+			     recurse_submodules == RECURSE_SUBMODULES_ON_DEMAND))
+			ret = update_submodules();
+		return ret;
 	}
 }

submodule-config.c

@@ -4,6 +4,7 @@
 #include "submodule-config.h"
 #include "submodule.h"
 #include "strbuf.h"
+#include "parse-options.h"
 
 /*
  * submodule cache lookup structure
@@ -252,6 +253,27 @@ int parse_fetch_recurse_submodules_arg(const char *opt, const char *arg)
 	return parse_fetch_recurse(opt, arg, 1);
 }
 
+int option_fetch_parse_recurse_submodules(const struct option *opt,
+					  const char *arg, int unset)
+{
+	int *v;
+
+	if (!opt->value)
+		return -1;
+
+	v = opt->value;
+
+	if (unset) {
+		*v = RECURSE_SUBMODULES_OFF;
+	} else {
+		if (arg)
+			*v = parse_fetch_recurse_submodules_arg(opt->long_name, arg);
+		else
+			*v = RECURSE_SUBMODULES_ON;
+	}
+	return 0;
+}
+
 static int parse_update_recurse(const char *opt, const char *arg,
 				int die_on_error)
 {

submodule-config.h

@@ -28,6 +28,9 @@ struct repository;
 extern void submodule_cache_free(struct submodule_cache *cache);
 
 extern int parse_fetch_recurse_submodules_arg(const char *opt, const char *arg);
+struct option;
+extern int option_fetch_parse_recurse_submodules(const struct option *opt,
+						 const char *arg, int unset);
 extern int parse_update_recurse_submodules_arg(const char *opt, const char *arg);
 extern int parse_push_recurse_submodules_arg(const char *opt, const char *arg);
 extern int parse_submodule_config_option(const char *var, const char *value);

submodule.c

@@ -1138,6 +1138,32 @@ static void calculate_changed_submodule_paths(void)
 	initialized_fetch_ref_tips = 0;
 }
 
+int submodule_touches_in_range(struct object_id *excl_oid,
+			       struct object_id *incl_oid)
+{
+	struct string_list subs = STRING_LIST_INIT_DUP;
+	struct argv_array args = ARGV_ARRAY_INIT;
+	int ret;
+
+	gitmodules_config();
+	/* No need to check if there are no submodules configured */
+	if (!submodule_from_path(NULL, NULL))
+		return 0;
+
+	argv_array_push(&args, "--"); /* args[0] program name */
+	argv_array_push(&args, oid_to_hex(incl_oid));
+	argv_array_push(&args, "--not");
+	argv_array_push(&args, oid_to_hex(excl_oid));
+
+	collect_changed_submodules(&subs, &args);
+	ret = subs.nr;
+
+	argv_array_clear(&args);
+
+	free_submodules_oids(&subs);
+	return ret;
+}
+
 struct submodule_parallel_fetch {
 	int count;
 	struct argv_array args;

submodule.h

@@ -99,6 +99,10 @@ extern int merge_submodule(struct object_id *result, const char *path,
 			   const struct object_id *base,
 			   const struct object_id *a,
 			   const struct object_id *b, int search);
+
+/* Checks if there are submodule changes in a..b. */
+extern int submodule_touches_in_range(struct object_id *a,
+				      struct object_id *b);
 extern int find_unpushed_submodules(struct oid_array *commits,
 				    const char *remotes_name,
 				    struct string_list *needs_pushing);

t/t5572-pull-submodule.sh

@@ -42,4 +42,62 @@ KNOWN_FAILURE_NOFF_MERGE_DOESNT_CREATE_EMPTY_SUBMODULE_DIR=1
 KNOWN_FAILURE_NOFF_MERGE_ATTEMPTS_TO_MERGE_REMOVED_SUBMODULE_FILES=1
 test_submodule_switch "git_pull_noff"
 
+test_expect_success 'pull --recurse-submodule setup' '
+	test_create_repo child &&
+	test_commit -C child bar &&
+
+	test_create_repo parent &&
+	test_commit -C child foo &&
+
+	git -C parent submodule add ../child sub &&
+	git -C parent commit -m "add submodule" &&
+
+	git clone --recurse-submodules parent super
+'
+
+test_expect_success 'recursive pull updates working tree' '
+	test_commit -C child merge_strategy &&
+	git -C parent submodule update --remote &&
+	git -C parent add sub &&
+	git -C parent commit -m "update submodule" &&
+
+	git -C super pull --no-rebase --recurse-submodules &&
+	test_path_is_file super/sub/merge_strategy.t
+'
+
+test_expect_success 'recursive rebasing pull' '
+	# change upstream
+	test_commit -C child rebase_strategy &&
+	git -C parent submodule update --remote &&
+	git -C parent add sub &&
+	git -C parent commit -m "update submodule" &&
+
+	# also have local commits
+	test_commit -C super/sub local_stuff &&
+
+	git -C super pull --rebase --recurse-submodules &&
+	test_path_is_file super/sub/rebase_strategy.t &&
+	test_path_is_file super/sub/local_stuff.t
+'
+
+test_expect_success 'pull rebase recursing fails with conflicts' '
+
+	# local changes in submodule recorded in superproject:
+	test_commit -C super/sub local_stuff_2 &&
+	git -C super add sub &&
+	git -C super commit -m "local update submodule" &&
+
+	# and in the remote as well:
+	test_commit -C child important_upstream_work &&
+	git -C parent submodule update --remote &&
+	git -C parent add sub &&
+	git -C parent commit -m "remote update submodule" &&
+
+	# Unfortunately we fail here, despite no conflict in the
+	# submodule itself, but the merge strategy in submodules
+	# does not support rebase:
+	test_must_fail git -C super pull --rebase --recurse-submodules 2>err &&
+	test_i18ngrep "locally recorded submodule modifications" err
+'
+
 test_done