diff --git a/builtin/config.c b/builtin/config.c
index e8e1c0a456..8cd08da991 100644
--- a/builtin/config.c
+++ b/builtin/config.c
@@ -387,12 +387,20 @@ int cmd_config(int argc, const char **argv, const char *prefix)
 
 		home_config_paths(&user_config, &xdg_config, "config");
 
-		if (access(user_config, R_OK) && !access(xdg_config, R_OK))
-			given_config_file = xdg_config;
-		else if (user_config)
-			given_config_file = user_config;
-		else
+		if (!user_config)
+			/*
+			 * It is unknown if HOME/.gitconfig exists, so
+			 * we do not know if we should write to XDG
+			 * location; error out even if XDG_CONFIG_HOME
+			 * is set and points at a sane location.
+			 */
 			die("$HOME not set");
+
+		if (access(user_config, R_OK) &&
+		    xdg_config && !access(xdg_config, R_OK))
+			given_config_file = xdg_config;
+		else
+			given_config_file = user_config;
 	}
 	else if (use_system_config)
 		given_config_file = git_etc_gitconfig();
diff --git a/config.c b/config.c
index d28a499b0b..6b97503b06 100644
--- a/config.c
+++ b/config.c
@@ -940,12 +940,12 @@ int git_config_early(config_fn_t fn, void *data, const char *repo_config)
 		found += 1;
 	}
 
-	if (!access(xdg_config, R_OK)) {
+	if (xdg_config && !access(xdg_config, R_OK)) {
 		ret += git_config_from_file(fn, xdg_config, data);
 		found += 1;
 	}
 
-	if (!access(user_config, R_OK)) {
+	if (user_config && !access(user_config, R_OK)) {
 		ret += git_config_from_file(fn, user_config, data);
 		found += 1;
 	}