From 02253b6b91472e251418bd0545afb2b653b5385c Mon Sep 17 00:00:00 2001
From: Danil Alexeev <danil@alexeev.xyz>
Date: Thu, 14 Mar 2024 12:33:56 +0300
Subject: [PATCH] GDScript: Fix continuation lines in `GDScriptTokenizerBuffer`

---
 modules/gdscript/gdscript_tokenizer.cpp       |  3 ++-
 .../gdscript/gdscript_tokenizer_buffer.cpp    |  6 ++---
 .../gdscript/tests/gdscript_test_runner.cpp   | 23 +++++++++++++------
 .../continuation_lines_comments.bin.gd        | 12 ++++++++++
 .../continuation_lines_comments.bin.out       |  2 ++
 5 files changed, 35 insertions(+), 11 deletions(-)
 create mode 100644 modules/gdscript/tests/scripts/parser/features/continuation_lines_comments.bin.gd
 create mode 100644 modules/gdscript/tests/scripts/parser/features/continuation_lines_comments.bin.out

diff --git a/modules/gdscript/gdscript_tokenizer.cpp b/modules/gdscript/gdscript_tokenizer.cpp
index 2940af585dcb..5b1639e2505a 100644
--- a/modules/gdscript/gdscript_tokenizer.cpp
+++ b/modules/gdscript/gdscript_tokenizer.cpp
@@ -1455,10 +1455,11 @@ GDScriptTokenizer::Token GDScriptTokenizerText::scan() {
 		if (_peek() != '\n') {
 			return make_error("Expected new line after \"\\\".");
 		}
-		continuation_lines.push_back(line);
 		_advance();
 		newline(false);
 		line_continuation = true;
+		_skip_whitespace(); // Skip whitespace/comment lines after `\`. See GH-89403.
+		continuation_lines.push_back(line);
 		return scan(); // Recurse to get next token.
 	}
 
diff --git a/modules/gdscript/gdscript_tokenizer_buffer.cpp b/modules/gdscript/gdscript_tokenizer_buffer.cpp
index db523ea9419b..e53bc5bc41ae 100644
--- a/modules/gdscript/gdscript_tokenizer_buffer.cpp
+++ b/modules/gdscript/gdscript_tokenizer_buffer.cpp
@@ -285,9 +285,9 @@ Vector<uint8_t> GDScriptTokenizerBuffer::parse_code_string(const String &p_code,
 
 	// Remove continuation lines from map.
 	for (int line : tokenizer.get_continuation_lines()) {
-		if (rev_token_lines.has(line + 1)) {
-			token_lines.erase(rev_token_lines[line + 1]);
-			token_columns.erase(rev_token_lines[line + 1]);
+		if (rev_token_lines.has(line)) {
+			token_lines.erase(rev_token_lines[line]);
+			token_columns.erase(rev_token_lines[line]);
 		}
 	}
 
diff --git a/modules/gdscript/tests/gdscript_test_runner.cpp b/modules/gdscript/tests/gdscript_test_runner.cpp
index a0329eb8d231..e3d16eaf4267 100644
--- a/modules/gdscript/tests/gdscript_test_runner.cpp
+++ b/modules/gdscript/tests/gdscript_test_runner.cpp
@@ -300,14 +300,23 @@ bool GDScriptTestRunner::make_tests_for_dir(const String &p_dir) {
 #endif
 
 				String out_file = next.get_basename() + ".out";
-				if (!is_generating && !dir->file_exists(out_file)) {
-					ERR_FAIL_V_MSG(false, "Could not find output file for " + next);
+				ERR_FAIL_COND_V_MSG(!is_generating && !dir->file_exists(out_file), false, "Could not find output file for " + next);
+
+				if (next.ends_with(".bin.gd")) {
+					// Test text mode first.
+					GDScriptTest text_test(current_dir.path_join(next), current_dir.path_join(out_file), source_dir);
+					tests.push_back(text_test);
+					// Test binary mode even without `--use-binary-tokens`.
+					GDScriptTest bin_test(current_dir.path_join(next), current_dir.path_join(out_file), source_dir);
+					bin_test.set_tokenizer_mode(GDScriptTest::TOKENIZER_BUFFER);
+					tests.push_back(bin_test);
+				} else {
+					GDScriptTest test(current_dir.path_join(next), current_dir.path_join(out_file), source_dir);
+					if (binary_tokens) {
+						test.set_tokenizer_mode(GDScriptTest::TOKENIZER_BUFFER);
+					}
+					tests.push_back(test);
 				}
-				GDScriptTest test(current_dir.path_join(next), current_dir.path_join(out_file), source_dir);
-				if (binary_tokens) {
-					test.set_tokenizer_mode(GDScriptTest::TOKENIZER_BUFFER);
-				}
-				tests.push_back(test);
 			}
 		}
 
diff --git a/modules/gdscript/tests/scripts/parser/features/continuation_lines_comments.bin.gd b/modules/gdscript/tests/scripts/parser/features/continuation_lines_comments.bin.gd
new file mode 100644
index 000000000000..cb0bc94d2e13
--- /dev/null
+++ b/modules/gdscript/tests/scripts/parser/features/continuation_lines_comments.bin.gd
@@ -0,0 +1,12 @@
+# GH-89403
+
+func test():
+	var x := 1
+	if x == 0 \
+			# Comment.
+			# Comment.
+			and (x < 1 or x > 2) \
+			# Comment.
+			and x != 3:
+		pass
+	print("Ok")
diff --git a/modules/gdscript/tests/scripts/parser/features/continuation_lines_comments.bin.out b/modules/gdscript/tests/scripts/parser/features/continuation_lines_comments.bin.out
new file mode 100644
index 000000000000..0e9f482af48f
--- /dev/null
+++ b/modules/gdscript/tests/scripts/parser/features/continuation_lines_comments.bin.out
@@ -0,0 +1,2 @@
+GDTEST_OK
+Ok