diff --git a/builtin/fetch.c b/builtin/fetch.c
index 8470850415..3bfc081c6d 100644
--- a/builtin/fetch.c
+++ b/builtin/fetch.c
@@ -146,7 +146,9 @@ static struct ref *get_ref_map(struct transport *transport,
 		struct remote *remote = transport->remote;
 		struct branch *branch = branch_get(NULL);
 		int has_merge = branch_has_merge_config(branch);
-		if (remote && (remote->fetch_refspec_nr || has_merge)) {
+		if (remote &&
+		    (remote->fetch_refspec_nr ||
+		     (has_merge && !strcmp(branch->remote_name, remote->name)))) {
 			for (i = 0; i < remote->fetch_refspec_nr; i++) {
 				get_fetch_map(remote_refs, &remote->fetch[i], &tail, 0);
 				if (remote->fetch[i].dst &&
diff --git a/t/t5510-fetch.sh b/t/t5510-fetch.sh
index 950b7720e3..6a83771334 100755
--- a/t/t5510-fetch.sh
+++ b/t/t5510-fetch.sh
@@ -240,7 +240,7 @@ test_expect_success 'fetch with a non-applying branch.<name>.merge' '
 	git fetch blub
 '
 
-test_expect_failure 'fetch from GIT URL with a non-applying branch.<name>.merge' '
+test_expect_success 'fetch from GIT URL with a non-applying branch.<name>.merge' '
 	git update-ref -d FETCH_HEAD &&
 	git fetch one &&
 	git rev-parse --verify FETCH_HEAD