system/serenity
system/serenity
1
0
Fork 0
mirror of https://github.com/SerenityOS/serenity synced 2024-09-19 07:51:39 +00:00
Code Issues Packages Projects Releases Wiki Activity

LibWeb+LibSyntax: Implement nested syntax highlighters

Browse source 
And use them to highlight javascript in HTML source.
This commit also changes how TextDocumentSpan::data is interpreted,
as it used to be an opaque pointer, but everyone stuffed an enum value
inside it, which made the values not unique to each highlighter;
that field is now a u64 serial id.
The syntax highlighters don't need to change their ways of stuffing
token types into that field, but a highlighter that calls another
nested highlighter needs to register the nested types for use with
token pairs.
This commit is contained in:
Ali Mohammad Pur 2021-06-07 12:03:09 +04:30 committed by Ali Mohammad Pur
parent 3bac14e19e
commit 71b4433b0d
18 changed files with 219 additions and 104 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
Userland/Applications/Spreadsheet/CellSyntaxHighlighter.cpp
View file

@ -31,7 +31,7 @@ void CellSyntaxHighlighter::rehighlight(const Palette& palette)
false,
false,
},
nullptr,
(u64)-1,
false);
if (m_cell && m_cell->exception()) {
@ -47,7 +47,7 @@ void CellSyntaxHighlighter::rehighlight(const Palette& palette)
false,
false,
},
nullptr,
(u64)-1,
false });
}
m_client->do_update();

22
Userland/Libraries/LibCpp/SyntaxHighlighter.cpp
View file

@ -43,15 +43,15 @@ static Syntax::TextStyle style_for_token_type(Gfx::Palette const& palette, Cpp::
}
}
bool SyntaxHighlighter::is_identifier(void* token) const
bool SyntaxHighlighter::is_identifier(u64 token) const
{
auto cpp_token = static_cast<Cpp::Token::Type>(reinterpret_cast<size_t>(token));
auto cpp_token = static_cast<Cpp::Token::Type>(token);
return cpp_token == Cpp::Token::Type::Identifier;
}
bool SyntaxHighlighter::is_navigatable(void* token) const
bool SyntaxHighlighter::is_navigatable(u64 token) const
{
auto cpp_token = static_cast<Cpp::Token::Type>(reinterpret_cast<size_t>(token));
auto cpp_token = static_cast<Cpp::Token::Type>(token);
return cpp_token == Cpp::Token::Type::IncludePath;
}
@ -72,7 +72,7 @@ void SyntaxHighlighter::rehighlight(Palette const& palette)
span.attributes.color = style.color;
span.attributes.bold = style.bold;
span.is_skippable = token.type() == Cpp::Token::Type::Whitespace;
span.data = reinterpret_cast<void*>(token.type());
span.data = static_cast<u64>(token.type());
spans.append(span);
}
m_client->do_set_spans(move(spans));
@ -83,20 +83,20 @@ void SyntaxHighlighter::rehighlight(Palette const& palette)
m_client->do_update();
}
Vector<SyntaxHighlighter::MatchingTokenPair> SyntaxHighlighter::matching_token_pairs() const
Vector<SyntaxHighlighter::MatchingTokenPair> SyntaxHighlighter::matching_token_pairs_impl() const
{
static Vector<SyntaxHighlighter::MatchingTokenPair> pairs;
if (pairs.is_empty()) {
pairs.append({ reinterpret_cast<void*>(Cpp::Token::Type::LeftCurly), reinterpret_cast<void*>(Cpp::Token::Type::RightCurly) });
pairs.append({ reinterpret_cast<void*>(Cpp::Token::Type::LeftParen), reinterpret_cast<void*>(Cpp::Token::Type::RightParen) });
pairs.append({ reinterpret_cast<void*>(Cpp::Token::Type::LeftBracket), reinterpret_cast<void*>(Cpp::Token::Type::RightBracket) });
pairs.append({ static_cast<u64>(Cpp::Token::Type::LeftCurly), static_cast<u64>(Cpp::Token::Type::RightCurly) });
pairs.append({ static_cast<u64>(Cpp::Token::Type::LeftParen), static_cast<u64>(Cpp::Token::Type::RightParen) });
pairs.append({ static_cast<u64>(Cpp::Token::Type::LeftBracket), static_cast<u64>(Cpp::Token::Type::RightBracket) });
}
return pairs;
}
bool SyntaxHighlighter::token_types_equal(void* token1, void* token2) const
bool SyntaxHighlighter::token_types_equal(u64 token1, u64 token2) const
{
return static_cast<Cpp::Token::Type>(reinterpret_cast<size_t>(token1)) == static_cast<Cpp::Token::Type>(reinterpret_cast<size_t>(token2));
return static_cast<Cpp::Token::Type>(token1) == static_cast<Cpp::Token::Type>(token2);
}
SyntaxHighlighter::~SyntaxHighlighter()

8
Userland/Libraries/LibCpp/SyntaxHighlighter.h
View file

@ -15,15 +15,15 @@ public:
SyntaxHighlighter() { }
virtual ~SyntaxHighlighter() override;
virtual bool is_identifier(void*) const override;
virtual bool is_navigatable(void*) const override;
virtual bool is_identifier(u64) const override;
virtual bool is_navigatable(u64) const override;
virtual Syntax::Language language() const override { return Syntax::Language::Cpp; }
virtual void rehighlight(Palette const&) override;
protected:
virtual Vector<MatchingTokenPair> matching_token_pairs() const override;
virtual bool token_types_equal(void*, void*) const override;
virtual Vector<MatchingTokenPair> matching_token_pairs_impl() const override;
virtual bool token_types_equal(u64, u64) const override;
};
}

14
Userland/Libraries/LibGUI/GMLSyntaxHighlighter.cpp
View file

@ -33,9 +33,9 @@ static Syntax::TextStyle style_for_token_type(const Gfx::Palette& palette, GMLTo
}
}
bool GMLSyntaxHighlighter::is_identifier(void* token) const
bool GMLSyntaxHighlighter::is_identifier(u64 token) const
{
auto ini_token = static_cast<GUI::GMLToken::Type>(reinterpret_cast<size_t>(token));
auto ini_token = static_cast<GUI::GMLToken::Type>(token);
return ini_token == GUI::GMLToken::Type::Identifier;
}
@ -54,7 +54,7 @@ void GMLSyntaxHighlighter::rehighlight(const Palette& palette)
span.attributes.color = style.color;
span.attributes.bold = style.bold;
span.is_skippable = false;
span.data = reinterpret_cast<void*>(token.m_type);
span.data = static_cast<u64>(token.m_type);
spans.append(span);
}
m_client->do_set_spans(move(spans));
@ -65,18 +65,18 @@ void GMLSyntaxHighlighter::rehighlight(const Palette& palette)
m_client->do_update();
}
Vector<GMLSyntaxHighlighter::MatchingTokenPair> GMLSyntaxHighlighter::matching_token_pairs() const
Vector<GMLSyntaxHighlighter::MatchingTokenPair> GMLSyntaxHighlighter::matching_token_pairs_impl() const
{
static Vector<MatchingTokenPair> pairs;
if (pairs.is_empty()) {
pairs.append({ reinterpret_cast<void*>(GMLToken::Type::LeftCurly), reinterpret_cast<void*>(GMLToken::Type::RightCurly) });
pairs.append({ static_cast<u64>(GMLToken::Type::LeftCurly), static_cast<u64>(GMLToken::Type::RightCurly) });
}
return pairs;
}
bool GMLSyntaxHighlighter::token_types_equal(void* token1, void* token2) const
bool GMLSyntaxHighlighter::token_types_equal(u64 token1, u64 token2) const
{
return static_cast<GUI::GMLToken::Type>(reinterpret_cast<size_t>(token1)) == static_cast<GUI::GMLToken::Type>(reinterpret_cast<size_t>(token2));
return static_cast<GUI::GMLToken::Type>(token1) == static_cast<GUI::GMLToken::Type>(token2);
}
GMLSyntaxHighlighter::~GMLSyntaxHighlighter()

6
Userland/Libraries/LibGUI/GMLSyntaxHighlighter.h
View file

@ -15,14 +15,14 @@ public:
GMLSyntaxHighlighter() { }
virtual ~GMLSyntaxHighlighter() override;
virtual bool is_identifier(void*) const override;
virtual bool is_identifier(u64) const override;
virtual Syntax::Language language() const override { return Syntax::Language::GML; }
virtual void rehighlight(const Palette&) override;
protected:
virtual Vector<MatchingTokenPair> matching_token_pairs() const override;
virtual bool token_types_equal(void*, void*) const override;
virtual Vector<MatchingTokenPair> matching_token_pairs_impl() const override;
virtual bool token_types_equal(u64, u64) const override;
};
}

14
Userland/Libraries/LibGUI/INISyntaxHighlighter.cpp
View file

@ -32,9 +32,9 @@ static Syntax::TextStyle style_for_token_type(const Gfx::Palette& palette, IniTo
}
}
bool IniSyntaxHighlighter::is_identifier(void* token) const
bool IniSyntaxHighlighter::is_identifier(u64 token) const
{
auto ini_token = static_cast<GUI::IniToken::Type>(reinterpret_cast<size_t>(token));
auto ini_token = static_cast<GUI::IniToken::Type>(token);
return ini_token == GUI::IniToken::Type::Name;
}
@ -53,7 +53,7 @@ void IniSyntaxHighlighter::rehighlight(const Palette& palette)
span.attributes.color = style.color;
span.attributes.bold = style.bold;
span.is_skippable = token.m_type == IniToken::Type::Whitespace;
span.data = reinterpret_cast<void*>(token.m_type);
span.data = static_cast<u64>(token.m_type);
spans.append(span);
}
m_client->do_set_spans(move(spans));
@ -64,18 +64,18 @@ void IniSyntaxHighlighter::rehighlight(const Palette& palette)
m_client->do_update();
}
Vector<IniSyntaxHighlighter::MatchingTokenPair> IniSyntaxHighlighter::matching_token_pairs() const
Vector<IniSyntaxHighlighter::MatchingTokenPair> IniSyntaxHighlighter::matching_token_pairs_impl() const
{
static Vector<MatchingTokenPair> pairs;
if (pairs.is_empty()) {
pairs.append({ reinterpret_cast<void*>(IniToken::Type::LeftBracket), reinterpret_cast<void*>(IniToken::Type::RightBracket) });
pairs.append({ static_cast<u64>(IniToken::Type::LeftBracket), static_cast<u64>(IniToken::Type::RightBracket) });
}
return pairs;
}
bool IniSyntaxHighlighter::token_types_equal(void* token1, void* token2) const
bool IniSyntaxHighlighter::token_types_equal(u64 token1, u64 token2) const
{
return static_cast<GUI::IniToken::Type>(reinterpret_cast<size_t>(token1)) == static_cast<GUI::IniToken::Type>(reinterpret_cast<size_t>(token2));
return static_cast<GUI::IniToken::Type>(token1) == static_cast<GUI::IniToken::Type>(token2);
}
IniSyntaxHighlighter::~IniSyntaxHighlighter()

6
Userland/Libraries/LibGUI/INISyntaxHighlighter.h
View file

@ -15,14 +15,14 @@ public:
IniSyntaxHighlighter() { }
virtual ~IniSyntaxHighlighter() override;
virtual bool is_identifier(void*) const override;
virtual bool is_identifier(u64) const override;
virtual Syntax::Language language() const override { return Syntax::Language::INI; }
virtual void rehighlight(const Palette&) override;
protected:
virtual Vector<MatchingTokenPair> matching_token_pairs() const override;
virtual bool token_types_equal(void*, void*) const override;
virtual Vector<MatchingTokenPair> matching_token_pairs_impl() const override;
virtual bool token_types_equal(u64, u64) const override;
};
}

2
Userland/Libraries/LibGUI/TextDocument.h
View file

@ -26,7 +26,7 @@ namespace GUI {
struct TextDocumentSpan {
TextRange range;
Gfx::TextAttributes attributes;
void* data { nullptr };
u64 data { 0 };
bool is_skippable { false };
};

20
Userland/Libraries/LibJS/SyntaxHighlighter.cpp
View file

@ -38,13 +38,13 @@ static Syntax::TextStyle style_for_token_type(const Gfx::Palette& palette, JS::T
}
}
bool SyntaxHighlighter::is_identifier(void* token) const
bool SyntaxHighlighter::is_identifier(u64 token) const
{
auto js_token = static_cast<JS::TokenType>(reinterpret_cast<size_t>(token));
auto js_token = static_cast<JS::TokenType>(static_cast<size_t>(token));
return js_token == JS::TokenType::Identifier;
}
bool SyntaxHighlighter::is_navigatable([[maybe_unused]] void* token) const
bool SyntaxHighlighter::is_navigatable([[maybe_unused]] u64 token) const
{
return false;
}
@ -83,7 +83,7 @@ void SyntaxHighlighter::rehighlight(const Palette& palette)
span.attributes.color = style.color;
span.attributes.bold = style.bold;
span.is_skippable = is_trivia;
span.data = reinterpret_cast<void*>(static_cast<size_t>(type));
span.data = static_cast<u64>(type);
spans.append(span);
dbgln_if(SYNTAX_HIGHLIGHTING_DEBUG, "{}{} @ '{}' {}:{} - {}:{}",
@ -111,20 +111,20 @@ void SyntaxHighlighter::rehighlight(const Palette& palette)
m_client->do_update();
}
Vector<Syntax::Highlighter::MatchingTokenPair> SyntaxHighlighter::matching_token_pairs() const
Vector<Syntax::Highlighter::MatchingTokenPair> SyntaxHighlighter::matching_token_pairs_impl() const
{
static Vector<Syntax::Highlighter::MatchingTokenPair> pairs;
if (pairs.is_empty()) {
pairs.append({ reinterpret_cast<void*>(JS::TokenType::CurlyOpen), reinterpret_cast<void*>(JS::TokenType::CurlyClose) });
pairs.append({ reinterpret_cast<void*>(JS::TokenType::ParenOpen), reinterpret_cast<void*>(JS::TokenType::ParenClose) });
pairs.append({ reinterpret_cast<void*>(JS::TokenType::BracketOpen), reinterpret_cast<void*>(JS::TokenType::BracketClose) });
pairs.append({ static_cast<u64>(JS::TokenType::CurlyOpen), static_cast<u64>(JS::TokenType::CurlyClose) });
pairs.append({ static_cast<u64>(JS::TokenType::ParenOpen), static_cast<u64>(JS::TokenType::ParenClose) });
pairs.append({ static_cast<u64>(JS::TokenType::BracketOpen), static_cast<u64>(JS::TokenType::BracketClose) });
}
return pairs;
}
bool SyntaxHighlighter::token_types_equal(void* token1, void* token2) const
bool SyntaxHighlighter::token_types_equal(u64 token1, u64 token2) const
{
return static_cast<JS::TokenType>(reinterpret_cast<size_t>(token1)) == static_cast<JS::TokenType>(reinterpret_cast<size_t>(token2));
return static_cast<JS::TokenType>(token1) == static_cast<JS::TokenType>(token2);
}
SyntaxHighlighter::~SyntaxHighlighter()

8
Userland/Libraries/LibJS/SyntaxHighlighter.h
View file

@ -15,15 +15,15 @@ public:
SyntaxHighlighter() { }
virtual ~SyntaxHighlighter() override;
virtual bool is_identifier(void*) const override;
virtual bool is_navigatable(void*) const override;
virtual bool is_identifier(u64) const override;
virtual bool is_navigatable(u64) const override;
virtual Syntax::Language language() const override { return Syntax::Language::JavaScript; }
virtual void rehighlight(const Palette&) override;
protected:
virtual Vector<MatchingTokenPair> matching_token_pairs() const override;
virtual bool token_types_equal(void*, void*) const override;
virtual Vector<MatchingTokenPair> matching_token_pairs_impl() const override;
virtual bool token_types_equal(u64, u64) const override;
};
}

14
Userland/Libraries/LibSQL/SyntaxHighlighter.cpp
View file

@ -33,9 +33,9 @@ static Syntax::TextStyle style_for_token_type(Gfx::Palette const& palette, Token
}
}
bool SyntaxHighlighter::is_identifier(void* token) const
bool SyntaxHighlighter::is_identifier(u64 token) const
{
auto sql_token = static_cast<SQL::TokenType>(reinterpret_cast<size_t>(token));
auto sql_token = static_cast<SQL::TokenType>(static_cast<size_t>(token));
return sql_token == SQL::TokenType::Identifier;
}
@ -66,7 +66,7 @@ void SyntaxHighlighter::rehighlight(Palette const& palette)
auto style = style_for_token_type(palette, token.type());
span.attributes.color = style.color;
span.attributes.bold = style.bold;
span.data = reinterpret_cast<void*>(static_cast<size_t>(token.type()));
span.data = static_cast<u64>(token.type());
spans.append(span);
dbgln_if(SYNTAX_HIGHLIGHTING_DEBUG, "{} @ '{}' {}:{} - {}:{}",
@ -91,18 +91,18 @@ void SyntaxHighlighter::rehighlight(Palette const& palette)
m_client->do_update();
}
Vector<SyntaxHighlighter::MatchingTokenPair> SyntaxHighlighter::matching_token_pairs() const
Vector<SyntaxHighlighter::MatchingTokenPair> SyntaxHighlighter::matching_token_pairs_impl() const
{
static Vector<SyntaxHighlighter::MatchingTokenPair> pairs;
if (pairs.is_empty()) {
pairs.append({ reinterpret_cast<void*>(TokenType::ParenOpen), reinterpret_cast<void*>(TokenType::ParenClose) });
pairs.append({ static_cast<u64>(TokenType::ParenOpen), static_cast<u64>(TokenType::ParenClose) });
}
return pairs;
}
bool SyntaxHighlighter::token_types_equal(void* token1, void* token2) const
bool SyntaxHighlighter::token_types_equal(u64 token1, u64 token2) const
{
return static_cast<TokenType>(reinterpret_cast<size_t>(token1)) == static_cast<TokenType>(reinterpret_cast<size_t>(token2));
return static_cast<TokenType>(token1) == static_cast<TokenType>(token2);
}
SyntaxHighlighter::~SyntaxHighlighter()

6
Userland/Libraries/LibSQL/SyntaxHighlighter.h
View file

@ -15,14 +15,14 @@ public:
SyntaxHighlighter() { }
virtual ~SyntaxHighlighter() override;
virtual bool is_identifier(void*) const override;
virtual bool is_identifier(u64) const override;
virtual Syntax::Language language() const override { return Syntax::Language::SQL; }
virtual void rehighlight(Palette const&) override;
protected:
virtual Vector<MatchingTokenPair> matching_token_pairs() const override;
virtual bool token_types_equal(void*, void*) const override;
virtual Vector<MatchingTokenPair> matching_token_pairs_impl() const override;
virtual bool token_types_equal(u64, u64) const override;
};
}

17
Userland/Libraries/LibSyntax/Highlighter.cpp
View file

@ -23,7 +23,7 @@ void Highlighter::highlight_matching_token_pair()
Backward,
};
auto find_span_of_type = [&](auto i, void* type, void* not_type, Direction direction) -> Optional<size_t> {
auto find_span_of_type = [&](auto i, u64 type, u64 not_type, Direction direction) -> Optional<size_t> {
size_t nesting_level = 0;
bool forward = direction == Direction::Forward;
@ -130,4 +130,19 @@ void Highlighter::cursor_did_change()
highlight_matching_token_pair();
}
Vector<Highlighter::MatchingTokenPair> Highlighter::matching_token_pairs() const
{
auto own_pairs = matching_token_pairs_impl();
own_pairs.ensure_capacity(own_pairs.size() + m_nested_token_pairs.size());
for (auto& nested_pair : m_nested_token_pairs)
own_pairs.append(nested_pair);
return own_pairs;
}
void Highlighter::register_nested_token_pairs(Vector<MatchingTokenPair> pairs)
{
for (auto& pair : pairs)
m_nested_token_pairs.set(pair);
}
}

91
Userland/Libraries/LibSyntax/Highlighter.h
View file

@ -41,26 +41,30 @@ public:
virtual void rehighlight(const Palette&) = 0;
virtual void highlight_matching_token_pair();
virtual bool is_identifier(void*) const { return false; };
virtual bool is_navigatable(void*) const { return false; };
virtual bool is_identifier(u64) const { return false; };
virtual bool is_navigatable(u64) const { return false; };
void attach(HighlighterClient&);
void detach();
void cursor_did_change();
struct MatchingTokenPair {
u64 open;
u64 close;
};
Vector<MatchingTokenPair> matching_token_pairs() const;
protected:
Highlighter() { }
// FIXME: This should be WeakPtr somehow
HighlighterClient* m_client { nullptr };
struct MatchingTokenPair {
void* open;
void* close;
};
virtual Vector<MatchingTokenPair> matching_token_pairs() const = 0;
virtual bool token_types_equal(void*, void*) const = 0;
virtual Vector<MatchingTokenPair> matching_token_pairs_impl() const = 0;
virtual bool token_types_equal(u64, u64) const = 0;
void register_nested_token_pairs(Vector<MatchingTokenPair>);
void clear_nested_token_pairs() { m_nested_token_pairs.clear(); }
size_t first_free_token_kind_serial_value() const { return m_nested_token_pairs.size(); }
struct BuddySpan {
int index { -1 };
@ -69,6 +73,75 @@ protected:
bool m_has_brace_buddies { false };
BuddySpan m_brace_buddies[2];
HashTable<MatchingTokenPair> m_nested_token_pairs;
};
class ProxyHighlighterClient final : public Syntax::HighlighterClient {
public:
ProxyHighlighterClient(Syntax::HighlighterClient& client, GUI::TextPosition start, u64 nested_kind_start_value, StringView source)
: m_document(client.get_document())
, m_text(source)
, m_start(start)
, m_nested_kind_start_value(nested_kind_start_value)
{
}
Vector<GUI::TextDocumentSpan> corrected_spans() const
{
Vector<GUI::TextDocumentSpan> spans { m_spans };
for (auto& entry : spans) {
entry.range.start() = {
entry.range.start().line() + m_start.line(),
entry.range.start().line() == 0 ? entry.range.start().column() + m_start.column() : entry.range.start().column(),
};
entry.range.end() = {
entry.range.end().line() + m_start.line(),
entry.range.end().line() == 0 ? entry.range.end().column() + m_start.column() : entry.range.end().column(),
};
if (entry.data != (u64)-1)
entry.data += m_nested_kind_start_value;
}
return spans;
}
Vector<Syntax::Highlighter::MatchingTokenPair> corrected_token_pairs(Vector<Syntax::Highlighter::MatchingTokenPair> pairs) const
{
for (auto& pair : pairs) {
pair.close += m_nested_kind_start_value;
pair.open += m_nested_kind_start_value;
}
return pairs;
}
private:
virtual Vector<GUI::TextDocumentSpan>& spans() override { return m_spans; }
virtual const Vector<GUI::TextDocumentSpan>& spans() const override { return m_spans; }
virtual void set_span_at_index(size_t index, GUI::TextDocumentSpan span) override { m_spans.at(index) = move(span); }
virtual String highlighter_did_request_text() const override { return m_text; }
virtual void highlighter_did_request_update() override { }
virtual GUI::TextDocument& highlighter_did_request_document() override { return m_document; }
virtual GUI::TextPosition highlighter_did_request_cursor() const override { return {}; }
virtual void highlighter_did_set_spans(Vector<GUI::TextDocumentSpan> spans) override { m_spans = move(spans); }
Vector<GUI::TextDocumentSpan> m_spans;
GUI::TextDocument& m_document;
StringView m_text;
GUI::TextPosition m_start;
u64 m_nested_kind_start_value { 0 };
};
}
template<>
struct AK::Traits<Syntax::Highlighter::MatchingTokenPair> : public AK::GenericTraits<Syntax::Highlighter::MatchingTokenPair> {
static unsigned hash(Syntax::Highlighter::MatchingTokenPair const& pair)
{
return pair_int_hash(u64_hash(pair.open), u64_hash(pair.close));
}
static bool equals(Syntax::Highlighter::MatchingTokenPair const& a, Syntax::Highlighter::MatchingTokenPair const& b)
{
return a.open == b.open && a.close == b.close;
}
};

45
Userland/Libraries/LibWeb/HTML/SyntaxHighlighter/SyntaxHighlighter.cpp
View file

@ -6,6 +6,7 @@
*/
#include <AK/Debug.h>
#include <LibJS/SyntaxHighlighter.h>
#include <LibWeb/HTML/Parser/HTMLTokenizer.h>
#include <LibWeb/HTML/SyntaxHighlighter/SyntaxHighlighter.h>
@ -18,16 +19,17 @@ enum class AugmentedTokenKind : u32 {
CloseTag,
Comment,
Doctype,
__Count,
};
bool SyntaxHighlighter::is_identifier(void* token) const
bool SyntaxHighlighter::is_identifier(u64 token) const
{
if (!token)
return false;
return false;
}
bool SyntaxHighlighter::is_navigatable(void*) const
bool SyntaxHighlighter::is_navigatable(u64) const
{
return false;
}
@ -35,8 +37,8 @@ bool SyntaxHighlighter::is_navigatable(void*) const
void SyntaxHighlighter::rehighlight(Palette const& palette)
{
dbgln_if(SYNTAX_HIGHLIGHTING_DEBUG, "(HTML::SyntaxHighlighter) starting rehighlight");
(void)palette;
auto text = m_client->get_text();
clear_nested_token_pairs();
Vector<GUI::TextDocumentSpan> spans;
auto highlight = [&](auto start_line, auto start_column, auto end_line, auto end_column, Gfx::TextAttributes attributes, AugmentedTokenKind kind) {
@ -51,7 +53,7 @@ void SyntaxHighlighter::rehighlight(Palette const& palette)
{ end_line, end_column },
},
move(attributes),
(void*)kind,
static_cast<u64>(kind),
false);
};
@ -61,6 +63,9 @@ void SyntaxHighlighter::rehighlight(Palette const& palette)
Javascript,
CSS,
} state { State::HTML };
StringBuilder substring_builder;
GUI::TextPosition substring_start_position;
for (;;) {
auto token = tokenizer.next_token();
if (!token.has_value() || token.value().is_end_of_file())
@ -71,19 +76,41 @@ void SyntaxHighlighter::rehighlight(Palette const& palette)
if (token->tag_name() == "script"sv) {
tokenizer.switch_to(HTMLTokenizer::State::ScriptData);
state = State::Javascript;
substring_start_position = { token->end_position().line, token->end_position().column };
} else if (token->tag_name() == "style"sv) {
tokenizer.switch_to(HTMLTokenizer::State::RAWTEXT);
state = State::CSS;
substring_start_position = { token->end_position().line, token->end_position().column };
}
} else if (token->is_end_tag()) {
if (token->tag_name().is_one_of("script"sv, "style"sv)) {
if (state == State::Javascript) {
// FIXME: Highlight javascript code here instead.
Syntax::ProxyHighlighterClient proxy_client {
*m_client,
substring_start_position,
static_cast<u64>(AugmentedTokenKind::__Count) + first_free_token_kind_serial_value(),
substring_builder.string_view()
};
{
JS::SyntaxHighlighter highlighter;
highlighter.attach(proxy_client);
highlighter.rehighlight(palette);
highlighter.detach();
register_nested_token_pairs(proxy_client.corrected_token_pairs(highlighter.matching_token_pairs()));
}
spans.append(proxy_client.corrected_spans());
substring_builder.clear();
} else if (state == State::CSS) {
// FIXME: Highlight CSS code here instead.
substring_builder.clear();
}
state = State::HTML;
}
} else if (state != State::HTML) {
VERIFY(token->is_character());
substring_builder.append_code_point(token->code_point());
continue;
}
size_t token_start_offset = token->is_end_tag() ? 1 : 0;
@ -135,7 +162,7 @@ void SyntaxHighlighter::rehighlight(Palette const& palette)
if constexpr (SYNTAX_HIGHLIGHTING_DEBUG) {
dbgln("(HTML::SyntaxHighlighter) list of all spans:");
for (auto& span : spans)
dbgln("{}, {}", span.range, span.attributes.color);
dbgln("{}, {} - {}", span.range, span.attributes.color, span.data);
dbgln("(HTML::SyntaxHighlighter) end of list");
}
@ -145,16 +172,16 @@ void SyntaxHighlighter::rehighlight(Palette const& palette)
m_client->do_update();
}
Vector<Syntax::Highlighter::MatchingTokenPair> SyntaxHighlighter::matching_token_pairs() const
Vector<Syntax::Highlighter::MatchingTokenPair> SyntaxHighlighter::matching_token_pairs_impl() const
{
static Vector<MatchingTokenPair> pairs;
if (pairs.is_empty()) {
pairs.append({ (void*)AugmentedTokenKind::OpenTag, (void*)AugmentedTokenKind::CloseTag });
pairs.append({ static_cast<u64>(AugmentedTokenKind::OpenTag), static_cast<u64>(AugmentedTokenKind::CloseTag) });
}
return pairs;
}
bool SyntaxHighlighter::token_types_equal(void* token0, void* token1) const
bool SyntaxHighlighter::token_types_equal(u64 token0, u64 token1) const
{
return token0 == token1;
}

8
Userland/Libraries/LibWeb/HTML/SyntaxHighlighter/SyntaxHighlighter.h
View file

@ -15,15 +15,15 @@ public:
SyntaxHighlighter() = default;
virtual ~SyntaxHighlighter() override = default;
virtual bool is_identifier(void*) const override;
virtual bool is_navigatable(void*) const override;
virtual bool is_identifier(u64) const override;
virtual bool is_navigatable(u64) const override;
virtual Syntax::Language language() const override { return Syntax::Language::HTML; }
virtual void rehighlight(Palette const&) override;
protected:
virtual Vector<MatchingTokenPair> matching_token_pairs() const override;
virtual bool token_types_equal(void*, void*) const override;
virtual Vector<MatchingTokenPair> matching_token_pairs_impl() const override;
virtual bool token_types_equal(u64, u64) const override;
size_t m_line { 1 };
size_t m_column { 0 };

30
Userland/Shell/SyntaxHighlighter.cpp
View file

@ -68,7 +68,7 @@ private:
GUI::TextDocumentSpan span;
set_offset_range_start(span.range, node->position().start_line);
set_offset_range_end(span.range, node->position().end_line);
span.data = (void*)static_cast<size_t>(node->kind());
span.data = static_cast<u64>(node->kind());
span.is_skippable = false;
m_spans.append(move(span));
@ -143,12 +143,12 @@ private:
auto& start_span = span_for_node(node);
start_span.attributes.color = m_palette.syntax_punctuation();
start_span.range.set_end({ node->position().start_line.line_number, node->position().start_line.line_column + 2 });
start_span.data = (void*)static_cast<size_t>(AugmentedTokenKind::OpenParen);
start_span.data = static_cast<u64>(AugmentedTokenKind::OpenParen);
auto& end_span = span_for_node(node);
end_span.attributes.color = m_palette.syntax_punctuation();
set_offset_range_start(end_span.range, node->position().end_line, 1);
end_span.data = (void*)static_cast<size_t>(AugmentedTokenKind::CloseParen);
end_span.data = static_cast<u64>(AugmentedTokenKind::CloseParen);
}
virtual void visit(const AST::CloseFdRedirection* node) override
{
@ -289,12 +289,12 @@ private:
auto& start_span = span_for_node(node);
start_span.attributes.color = m_palette.syntax_punctuation();
start_span.range.set_end({ node->position().start_line.line_number, node->position().start_line.line_column + 2 });
start_span.data = (void*)static_cast<size_t>(AugmentedTokenKind::OpenParen);
start_span.data = static_cast<u64>(AugmentedTokenKind::OpenParen);
auto& end_span = span_for_node(node);
end_span.attributes.color = m_palette.syntax_punctuation();
set_offset_range_start(end_span.range, node->position().end_line, 1);
end_span.data = (void*)static_cast<size_t>(AugmentedTokenKind::CloseParen);
end_span.data = static_cast<u64>(AugmentedTokenKind::CloseParen);
}
}
virtual void visit(const AST::IfCond* node) override
@ -328,7 +328,7 @@ private:
auto& start_span = span_for_node(node);
start_span.attributes.color = m_palette.syntax_punctuation();
start_span.range.set_end({ node->position().start_line.line_number, node->position().start_line.line_column + 2 });
start_span.data = (void*)static_cast<size_t>(AugmentedTokenKind::OpenParen);
start_span.data = static_cast<u64>(AugmentedTokenKind::OpenParen);
// Function name
auto& name_span = span_for_node(node);
@ -340,7 +340,7 @@ private:
auto& end_span = span_for_node(node);
end_span.attributes.color = m_palette.syntax_punctuation();
set_offset_range_start(end_span.range, node->position().end_line, 1);
end_span.data = (void*)static_cast<size_t>(AugmentedTokenKind::CloseParen);
end_span.data = static_cast<u64>(AugmentedTokenKind::CloseParen);
}
virtual void visit(const AST::Join* node) override
@ -500,7 +500,7 @@ private:
start_span.range.set_start({ decl.name->position().end_line.line_number, decl.name->position().end_line.line_column });
start_span.range.set_end({ decl.value->position().start_line.line_number, decl.value->position().start_line.line_column + 1 });
start_span.attributes.color = m_palette.syntax_punctuation();
start_span.data = (void*)static_cast<size_t>(AugmentedTokenKind::OpenParen);
start_span.data = static_cast<u64>(AugmentedTokenKind::OpenParen);
}
}
virtual void visit(const AST::WriteAppendRedirection* node) override
@ -518,7 +518,7 @@ private:
bool m_is_first_in_command { false };
};
bool SyntaxHighlighter::is_identifier(void* token) const
bool SyntaxHighlighter::is_identifier(u64 token) const
{
if (!token)
return false;
@ -529,7 +529,7 @@ bool SyntaxHighlighter::is_identifier(void* token) const
|| kind == (size_t)AST::Node::Kind::Tilde;
}
bool SyntaxHighlighter::is_navigatable(void*) const
bool SyntaxHighlighter::is_navigatable(u64) const
{
return false;
}
@ -551,7 +551,7 @@ void SyntaxHighlighter::rehighlight(const Palette& palette)
if constexpr (SYNTAX_HIGHLIGHTING_DEBUG) {
for (auto& span : spans) {
dbgln("Kind {}, range {}.", reinterpret_cast<size_t>(span.data), span.range);
dbgln("Kind {}, range {}.", span.data, span.range);
}
}
@ -561,19 +561,19 @@ void SyntaxHighlighter::rehighlight(const Palette& palette)
m_client->do_update();
}
Vector<Syntax::Highlighter::MatchingTokenPair> SyntaxHighlighter::matching_token_pairs() const
Vector<Syntax::Highlighter::MatchingTokenPair> SyntaxHighlighter::matching_token_pairs_impl() const
{
static Vector<MatchingTokenPair> pairs;
if (pairs.is_empty()) {
pairs.append({
(void*)static_cast<size_t>(AugmentedTokenKind::OpenParen),
(void*)static_cast<size_t>(AugmentedTokenKind::CloseParen),
static_cast<u64>(AugmentedTokenKind::OpenParen),
static_cast<u64>(AugmentedTokenKind::CloseParen),
});
}
return pairs;
}
bool SyntaxHighlighter::token_types_equal(void* token0, void* token1) const
bool SyntaxHighlighter::token_types_equal(u64 token0, u64 token1) const
{
return token0 == token1;
}

8
Userland/Shell/SyntaxHighlighter.h
View file

@ -15,15 +15,15 @@ public:
SyntaxHighlighter() { }
virtual ~SyntaxHighlighter() override;
virtual bool is_identifier(void*) const override;
virtual bool is_navigatable(void*) const override;
virtual bool is_identifier(u64) const override;
virtual bool is_navigatable(u64) const override;
virtual Syntax::Language language() const override { return Syntax::Language::Shell; }
virtual void rehighlight(const Palette&) override;
protected:
virtual Vector<MatchingTokenPair> matching_token_pairs() const override;
virtual bool token_types_equal(void*, void*) const override;
virtual Vector<MatchingTokenPair> matching_token_pairs_impl() const override;
virtual bool token_types_equal(u64, u64) const override;
};
}