Merge branch 'it/fetch-pack-many-refs' into maint

When "git fetch" encounters repositories with too many references, the
command line of "fetch-pack" that is run by a helper e.g. remote-curl, may
fail to hold all of them. Now such an internal invocation can feed the
references through the standard input of "fetch-pack".

By Ivan Todoroski
* it/fetch-pack-many-refs:
  remote-curl: main test case for the OS command line overflow
  fetch-pack: test cases for the new --stdin option
  remote-curl: send the refs to fetch-pack on stdin
  fetch-pack: new --stdin option to read refs from stdin

Conflicts:
	t/t5500-fetch-pack.sh

Documentation/git-fetch-pack.txt

@@ -32,6 +32,16 @@ OPTIONS
 --all::
 	Fetch all remote refs.
 
+--stdin::
+	Take the list of refs from stdin, one per line. If there
+	are refs specified on the command line in addition to this
+	option, then the refs from stdin are processed after those
+	on the command line.
++
+If '--stateless-rpc' is specified together with this option then
+the list of refs must be in packet format (pkt-line). Each ref must
+be in a separate packet, and the list must end with a flush packet.
+
 -q::
 --quiet::
 	Pass '-q' flag to 'git unpack-objects'; this makes the

builtin/fetch-pack.c

@@ -23,7 +23,9 @@ static struct fetch_pack_args args = {
 };
 
 static const char fetch_pack_usage[] =
-"git fetch-pack [--all] [--quiet|-q] [--keep|-k] [--thin] [--include-tag] [--upload-pack=<git-upload-pack>] [--depth=<n>] [--no-progress] [-v] [<host>:]<directory> [<refs>...]";
+"git fetch-pack [--all] [--stdin] [--quiet|-q] [--keep|-k] [--thin] "
+"[--include-tag] [--upload-pack=<git-upload-pack>] [--depth=<n>] "
+"[--no-progress] [-v] [<host>:]<directory> [<refs>...]";
 
 #define COMPLETE	(1U << 0)
 #define COMMON		(1U << 1)
@@ -942,6 +944,10 @@ int cmd_fetch_pack(int argc, const char **argv, const char *prefix)
 				args.fetch_all = 1;
 				continue;
 			}
+			if (!strcmp("--stdin", arg)) {
+				args.stdin_refs = 1;
+				continue;
+			}
 			if (!strcmp("-v", arg)) {
 				args.verbose = 1;
 				continue;
@@ -973,6 +979,40 @@ int cmd_fetch_pack(int argc, const char **argv, const char *prefix)
 	if (!dest)
 		usage(fetch_pack_usage);
 
+	if (args.stdin_refs) {
+		/*
+		 * Copy refs from cmdline to new growable list, then
+		 * append the refs from the standard input.
+		 */
+		int alloc_heads = nr_heads;
+		int size = nr_heads * sizeof(*heads);
+		heads = memcpy(xmalloc(size), heads, size);
+		if (args.stateless_rpc) {
+			/* in stateless RPC mode we use pkt-line to read
+			 * from stdin, until we get a flush packet
+			 */
+			static char line[1000];
+			for (;;) {
+				int n = packet_read_line(0, line, sizeof(line));
+				if (!n)
+					break;
+				if (line[n-1] == '\n')
+					n--;
+				ALLOC_GROW(heads, nr_heads + 1, alloc_heads);
+				heads[nr_heads++] = xmemdupz(line, n);
+			}
+		}
+		else {
+			/* read from stdin one ref per line, until EOF */
+			struct strbuf line = STRBUF_INIT;
+			while (strbuf_getline(&line, stdin, '\n') != EOF) {
+				ALLOC_GROW(heads, nr_heads + 1, alloc_heads);
+				heads[nr_heads++] = strbuf_detach(&line, NULL);
+			}
+			strbuf_release(&line);
+		}
+	}
+
 	if (args.stateless_rpc) {
 		conn = NULL;
 		fd[0] = 0;

fetch-pack.h

@@ -10,6 +10,7 @@ struct fetch_pack_args {
 		lock_pack:1,
 		use_thin_pack:1,
 		fetch_all:1,
+		stdin_refs:1,
 		verbose:1,
 		no_progress:1,
 		include_tag:1,

remote-curl.c

@@ -290,6 +290,7 @@ static void output_refs(struct ref *refs)
 struct rpc_state {
 	const char *service_name;
 	const char **argv;
+	struct strbuf *stdin_preamble;
 	char *service_url;
 	char *hdr_content_type;
 	char *hdr_accept;
@@ -535,6 +536,7 @@ static int rpc_service(struct rpc_state *rpc, struct discovery *heads)
 {
 	const char *svc = rpc->service_name;
 	struct strbuf buf = STRBUF_INIT;
+	struct strbuf *preamble = rpc->stdin_preamble;
 	struct child_process client;
 	int err = 0;
 
@@ -545,6 +547,8 @@ static int rpc_service(struct rpc_state *rpc, struct discovery *heads)
 	client.argv = rpc->argv;
 	if (start_command(&client))
 		exit(1);
+	if (preamble)
+		write_or_die(client.in, preamble->buf, preamble->len);
 	if (heads)
 		write_or_die(client.in, heads->buf, heads->len);
 
@@ -626,13 +630,14 @@ static int fetch_git(struct discovery *heads,
 	int nr_heads, struct ref **to_fetch)
 {
 	struct rpc_state rpc;
+	struct strbuf preamble = STRBUF_INIT;
 	char *depth_arg = NULL;
-	const char **argv;
 	int argc = 0, i, err;
+	const char *argv[15];
 
-	argv = xmalloc((15 + nr_heads) * sizeof(char*));
 	argv[argc++] = "fetch-pack";
 	argv[argc++] = "--stateless-rpc";
+	argv[argc++] = "--stdin";
 	argv[argc++] = "--lock-pack";
 	if (options.followtags)
 		argv[argc++] = "--include-tag";
@@ -651,24 +656,27 @@ static int fetch_git(struct discovery *heads,
 		argv[argc++] = depth_arg;
 	}
 	argv[argc++] = url;
+	argv[argc++] = NULL;
+
 	for (i = 0; i < nr_heads; i++) {
 		struct ref *ref = to_fetch[i];
 		if (!ref->name || !*ref->name)
 			die("cannot fetch by sha1 over smart http");
-		argv[argc++] = ref->name;
+		packet_buf_write(&preamble, "%s\n", ref->name);
 	}
-	argv[argc++] = NULL;
+	packet_buf_flush(&preamble);
 
 	memset(&rpc, 0, sizeof(rpc));
 	rpc.service_name = "git-upload-pack",
 	rpc.argv = argv;
+	rpc.stdin_preamble = &preamble;
 	rpc.gzip_request = 1;
 
 	err = rpc_service(&rpc, heads);
 	if (rpc.result.len)
 		safe_write(1, rpc.result.buf, rpc.result.len);
 	strbuf_release(&rpc.result);
-	free(argv);
+	strbuf_release(&preamble);
 	free(depth_arg);
 	return err;
 }

t/t5500-fetch-pack.sh

@@ -326,4 +326,70 @@ EOF
 	test_cmp count7.expected count7.actual
 '
 
+test_expect_success 'setup tests for the --stdin parameter' '
+	for head in C D E F
+	do
+		add $head
+	done &&
+	for head in A B C D E F
+	do
+		git tag $head $head
+	done &&
+	cat >input <<-\EOF
+	refs/heads/C
+	refs/heads/A
+	refs/heads/D
+	refs/tags/C
+	refs/heads/B
+	refs/tags/A
+	refs/heads/E
+	refs/tags/B
+	refs/tags/E
+	refs/tags/D
+	EOF
+	sort <input >expect &&
+	(
+		echo refs/heads/E &&
+		echo refs/tags/E &&
+		cat input
+	) >input.dup
+'
+
+test_expect_success 'fetch refs from cmdline' '
+	(
+		cd client &&
+		git fetch-pack --no-progress .. $(cat ../input)
+	) >output &&
+	cut -d " " -f 2 <output | sort >actual &&
+	test_cmp expect actual
+'
+
+test_expect_success 'fetch refs from stdin' '
+	(
+		cd client &&
+		git fetch-pack --stdin --no-progress .. <../input
+	) >output &&
+	cut -d " " -f 2 <output | sort >actual &&
+	test_cmp expect actual
+'
+
+test_expect_success 'fetch mixed refs from cmdline and stdin' '
+	(
+		cd client &&
+		tail -n +5 ../input |
+		git fetch-pack --stdin --no-progress .. $(head -n 4 ../input)
+	) >output &&
+	cut -d " " -f 2 <output | sort >actual &&
+	test_cmp expect actual
+'
+
+test_expect_success 'test duplicate refs from stdin' '
+	(
+	cd client &&
+	test_must_fail git fetch-pack --stdin --no-progress .. <../input.dup
+	) >output &&
+	cut -d " " -f 2 <output | sort >actual &&
+	test_cmp expect actual
+'
+
 test_done

t/t5551-http-fetch.sh

@@ -109,5 +109,36 @@ test_expect_success 'follow redirects (302)' '
 	git clone $HTTPD_URL/smart-redir-temp/repo.git --quiet repo-t
 '
 
+test -n "$GIT_TEST_LONG" && test_set_prereq EXPENSIVE
+
+test_expect_success EXPENSIVE 'create 50,000 tags in the repo' '
+	(
+	cd "$HTTPD_DOCUMENT_ROOT_PATH/repo.git" &&
+	for i in `seq 50000`
+	do
+		echo "commit refs/heads/too-many-refs"
+		echo "mark :$i"
+		echo "committer git <git@example.com> $i +0000"
+		echo "data 0"
+		echo "M 644 inline bla.txt"
+		echo "data 4"
+		echo "bla"
+		# make every commit dangling by always
+		# rewinding the branch after each commit
+		echo "reset refs/heads/too-many-refs"
+		echo "from :1"
+	done | git fast-import --export-marks=marks &&
+
+	# now assign tags to all the dangling commits we created above
+	tag=$(perl -e "print \"bla\" x 30") &&
+	sed -e "s/^:\(.\+\) \(.\+\)$/\2 refs\/tags\/$tag-\1/" <marks >>packed-refs
+	)
+'
+
+test_expect_success EXPENSIVE 'clone the 50,000 tag repo to check OS command line overflow' '
+	git clone $HTTPD_URL/smart/repo.git too-many-refs 2>err &&
+	test_line_count = 0 err
+'
+
 stop_httpd
 test_done