diff --git a/builtin/remote.c b/builtin/remote.c
index 2b415911bf..addf97ad29 100644
--- a/builtin/remote.c
+++ b/builtin/remote.c
@@ -769,7 +769,9 @@ static int rm(int argc, const char **argv)
 				strbuf_reset(&buf);
 				strbuf_addf(&buf, "branch.%s.%s",
 						item->string, *k);
-				git_config_set(buf.buf, NULL);
+				result = git_config_set_gently(buf.buf, NULL);
+				if (result && result != CONFIG_NOTHING_SET)
+					die(_("could not unset '%s'"), buf.buf);
 			}
 		}
 	}
diff --git a/t/t5505-remote.sh b/t/t5505-remote.sh
index 535d53fa63..a6c0178f3a 100755
--- a/t/t5505-remote.sh
+++ b/t/t5505-remote.sh
@@ -153,6 +153,25 @@ test_expect_success 'remove errors out early when deleting non-existent branch'
 	)
 '
 
+test_expect_success 'remove remote with a branch without configured merge' '
+	test_when_finished "(
+		git -C test checkout master;
+		git -C test branch -D two;
+		git -C test config --remove-section remote.two;
+		git -C test config --remove-section branch.second;
+		true
+	)" &&
+	(
+		cd test &&
+		git remote add two ../two &&
+		git fetch two &&
+		git checkout -b second two/master^0 &&
+		git config branch.second.remote two &&
+		git checkout master &&
+		git remote rm two
+	)
+'
+
 test_expect_success 'rename errors out early when deleting non-existent branch' '
 	(
 		cd test &&