HackStudio: Add tokens_info_result() and tokens_info_result() IPC calls

These IPC calls are used in the communication with the language server
to fetch semantic information about the tokens in a code document.

Userland/DevTools/HackStudio/LanguageClient.cpp

@@ -43,6 +43,16 @@ void ServerConnection::parameters_hint_result(Vector<String> const& params, int
     m_current_language_client->parameters_hint_result(params, static_cast<size_t>(argument_index));
 }
 
+void ServerConnection::tokens_info_result(Vector<GUI::AutocompleteProvider::TokenInfo> const& tokens_info)
+{
+    if (!m_current_language_client) {
+        dbgln("Language Server connection has no attached language client");
+        return;
+    }
+    VERIFY(m_current_language_client->on_tokens_info_result);
+    m_current_language_client->on_tokens_info_result(tokens_info);
+}
+
 void ServerConnection::die()
 {
     VERIFY(m_wrapper);
@@ -68,7 +78,6 @@ void LanguageClient::insert_text(const String& path, const String& text, size_t
 {
     if (!m_connection_wrapper.connection())
         return;
-    //    set_active_client();
     m_connection_wrapper.connection()->async_file_edit_insert_text(path, text, line, column);
 }
 
@@ -102,6 +111,13 @@ void LanguageClient::set_active_client()
     m_connection_wrapper.set_active_client(*this);
 }
 
+bool LanguageClient::is_active_client() const
+{
+    if (!m_connection_wrapper.connection())
+        return false;
+    return m_connection_wrapper.connection()->active_client() == this;
+}
+
 HashMap<String, NonnullOwnPtr<ServerConnectionWrapper>> ServerConnectionInstances::s_instance_for_language;
 
 void ServerConnection::declarations_in_document(const String& filename, const Vector<GUI::AutocompleteProvider::Declaration>& declarations)
@@ -130,6 +146,14 @@ void LanguageClient::get_parameters_hint(const String& path, size_t line, size_t
     m_connection_wrapper.connection()->async_get_parameters_hint(GUI::AutocompleteProvider::ProjectLocation { path, line, column });
 }
 
+void LanguageClient::get_tokens_info(const String& filename)
+{
+    if (!m_connection_wrapper.connection())
+        return;
+    VERIFY(is_active_client());
+    m_connection_wrapper.connection()->async_get_tokens_info(filename);
+}
+
 void LanguageClient::declaration_found(const String& file, size_t line, size_t column) const
 {
     if (!on_declaration_found) {

Userland/DevTools/HackStudio/LanguageClient.h

@@ -43,12 +43,15 @@ public:
 
     virtual void die() override;
 
+    const LanguageClient* active_client() const { return !m_current_language_client ? nullptr : m_current_language_client.ptr(); }
+
 protected:
     virtual void auto_complete_suggestions(Vector<GUI::AutocompleteProvider::Entry> const&) override;
     virtual void declaration_location(GUI::AutocompleteProvider::ProjectLocation const&) override;
     virtual void declarations_in_document(String const&, Vector<GUI::AutocompleteProvider::Declaration> const&) override;
     virtual void todo_entries_in_document(String const&, Vector<Cpp::Parser::TodoEntry> const&) override;
     virtual void parameters_hint_result(Vector<String> const&, int index) override;
+    virtual void tokens_info_result(Vector<GUI::AutocompleteProvider::TokenInfo> const&) override;
     void set_wrapper(ServerConnectionWrapper& wrapper) { m_wrapper = &wrapper; }
 
     String m_project_path;
@@ -124,6 +127,7 @@ public:
 
     Language language() const { return m_connection_wrapper.language(); }
     void set_active_client();
+    bool is_active_client() const;
     virtual void open_file(const String& path, int fd);
     virtual void set_file_content(const String& path, const String& content);
     virtual void insert_text(const String& path, const String& text, size_t line, size_t column);
@@ -131,6 +135,7 @@ public:
     virtual void request_autocomplete(const String& path, size_t cursor_line, size_t cursor_column);
     virtual void search_declaration(const String& path, size_t line, size_t column);
     virtual void get_parameters_hint(const String& path, size_t line, size_t column);
+    virtual void get_tokens_info(const String& filename);
 
     void provide_autocomplete_suggestions(const Vector<GUI::AutocompleteProvider::Entry>&) const;
     void declaration_found(const String& file, size_t line, size_t column) const;
@@ -140,6 +145,7 @@ public:
     Function<void(Vector<GUI::AutocompleteProvider::Entry>)> on_autocomplete_suggestions;
     Function<void(const String&, size_t, size_t)> on_declaration_found;
     Function<void(Vector<String> const&, size_t)> on_function_parameters_hint_result;
+    Function<void(Vector<GUI::AutocompleteProvider::TokenInfo> const&)> on_tokens_info_result;
 
 private:
     ServerConnectionWrapper& m_connection_wrapper;

Userland/DevTools/HackStudio/LanguageServers/ClientConnection.cpp

@@ -120,7 +120,7 @@ void ClientConnection::find_declaration(GUI::AutocompleteProvider::ProjectLocati
 
 void ClientConnection::get_parameters_hint(GUI::AutocompleteProvider::ProjectLocation const& location)
 {
-    dbgln_if(LANGUAGE_SERVER_DEBUG, "GetFunctionParams: {} {}:{}", location.file, location.line, location.column);
+    dbgln_if(LANGUAGE_SERVER_DEBUG, "GetParametersHint: {} {}:{}", location.file, location.line, location.column);
     auto document = m_filedb.get(location.file);
     if (!document) {
         dbgln("file {} has not been opened", location.file);
@@ -143,4 +143,17 @@ void ClientConnection::get_parameters_hint(GUI::AutocompleteProvider::ProjectLoc
     async_parameters_hint_result(params->params, params->current_index);
 }
 
+void ClientConnection::get_tokens_info(String const& filename)
+{
+    dbgln_if(LANGUAGE_SERVER_DEBUG, "GetTokenInfo: {}", filename);
+    auto document = m_filedb.get(filename);
+    if (!document) {
+        dbgln("file {} has not been opened", filename);
+        return;
+    }
+
+    auto token_info = m_autocomplete_engine->get_tokens_info(filename);
+    async_tokens_info_result(move(token_info));
+}
+
 }

Userland/DevTools/HackStudio/LanguageServers/ClientConnection.h

@@ -34,6 +34,7 @@ protected:
     virtual void auto_complete_suggestions(GUI::AutocompleteProvider::ProjectLocation const&) override;
     virtual void find_declaration(GUI::AutocompleteProvider::ProjectLocation const&) override;
     virtual void get_parameters_hint(GUI::AutocompleteProvider::ProjectLocation const&) override;
+    virtual void get_tokens_info(String const&) override;
 
     FileDB m_filedb;
     OwnPtr<CodeComprehensionEngine> m_autocomplete_engine;

Userland/DevTools/HackStudio/LanguageServers/CodeComprehensionEngine.h

@@ -34,6 +34,8 @@ public:
     };
     virtual Optional<FunctionParamsHint> get_function_params_hint(const String&, const GUI::TextPosition&) { return {}; }
 
+    virtual Vector<GUI::AutocompleteProvider::TokenInfo> get_tokens_info(const String&) { return {}; }
+
 public:
     Function<void(const String&, Vector<GUI::AutocompleteProvider::Declaration>&&)> set_declarations_of_document_callback;
     Function<void(String const&, Vector<Cpp::Parser::TodoEntry>&&)> set_todo_entries_of_document_callback;

Userland/DevTools/HackStudio/LanguageServers/Cpp/CppComprehensionEngine.cpp

@@ -894,4 +894,79 @@ Optional<CppComprehensionEngine::FunctionParamsHint> CppComprehensionEngine::get
     return hint;
 }
 
+Vector<GUI::AutocompleteProvider::TokenInfo> CppComprehensionEngine::get_tokens_info(const String& filename)
+{
+    dbgln_if(CPP_LANGUAGE_SERVER_DEBUG, "CppComprehensionEngine::get_tokens_info: {}", filename);
+
+    const auto* document_ptr = get_or_create_document_data(filename);
+    if (!document_ptr)
+        return {};
+
+    const auto& document = *document_ptr;
+
+    Vector<GUI::AutocompleteProvider::TokenInfo> tokens_info;
+    size_t i = 0;
+    for (auto const& token : document.preprocessor().unprocessed_tokens()) {
+
+        tokens_info.append({ get_token_semantic_type(document, token),
+            token.start().line, token.start().column, token.end().line, token.end().column });
+        ++i;
+    }
+    return tokens_info;
+}
+
+GUI::AutocompleteProvider::TokenInfo::SemanticType CppComprehensionEngine::get_token_semantic_type(DocumentData const& document, Token const& token)
+{
+    using GUI::AutocompleteProvider;
+    switch (token.type()) {
+    case Cpp::Token::Type::Identifier:
+        return get_semantic_type_for_identifier(document, token.start());
+    case Cpp::Token::Type::Keyword:
+        return AutocompleteProvider::TokenInfo::SemanticType::Keyword;
+    case Cpp::Token::Type::KnownType:
+        return AutocompleteProvider::TokenInfo::SemanticType::Type;
+    case Cpp::Token::Type::DoubleQuotedString:
+    case Cpp::Token::Type::SingleQuotedString:
+    case Cpp::Token::Type::RawString:
+        return AutocompleteProvider::TokenInfo::SemanticType::String;
+    case Cpp::Token::Type::Integer:
+    case Cpp::Token::Type::Float:
+        return AutocompleteProvider::TokenInfo::SemanticType::Number;
+    case Cpp::Token::Type::IncludePath:
+        return AutocompleteProvider::TokenInfo::SemanticType::IncludePath;
+    case Cpp::Token::Type::EscapeSequence:
+        return AutocompleteProvider::TokenInfo::SemanticType::Keyword;
+    case Cpp::Token::Type::PreprocessorStatement:
+    case Cpp::Token::Type::IncludeStatement:
+        return AutocompleteProvider::TokenInfo::SemanticType::PreprocessorStatement;
+    case Cpp::Token::Type::Comment:
+        return AutocompleteProvider::TokenInfo::SemanticType::Comment;
+    default:
+        return AutocompleteProvider::TokenInfo::SemanticType::Unknown;
+    }
+}
+
+GUI::AutocompleteProvider::TokenInfo::SemanticType CppComprehensionEngine::get_semantic_type_for_identifier(DocumentData const& document, Position position)
+{
+    auto decl = find_declaration_of(document, GUI::TextPosition { position.line, position.column });
+    if (!decl)
+        return GUI::AutocompleteProvider::TokenInfo::SemanticType::Identifier;
+
+    if (decl->is_function())
+        return GUI::AutocompleteProvider::TokenInfo::SemanticType::Function;
+    if (decl->is_parameter())
+        return GUI::AutocompleteProvider::TokenInfo::SemanticType::Parameter;
+    if (decl->is_variable_declaration()) {
+        if (decl->is_member())
+            return GUI::AutocompleteProvider::TokenInfo::SemanticType::Member;
+        return GUI::AutocompleteProvider::TokenInfo::SemanticType::Variable;
+    }
+    if (decl->is_struct_or_class())
+        return GUI::AutocompleteProvider::TokenInfo::SemanticType::CustomType;
+    if (decl->is_namespace())
+        return GUI::AutocompleteProvider::TokenInfo::SemanticType::Namespace;
+
+    return GUI::AutocompleteProvider::TokenInfo::SemanticType::Identifier;
+}
+
 }

Userland/DevTools/HackStudio/LanguageServers/Cpp/CppComprehensionEngine.h

@@ -30,6 +30,7 @@ public:
     virtual void file_opened([[maybe_unused]] const String& file) override;
     virtual Optional<GUI::AutocompleteProvider::ProjectLocation> find_declaration_of(const String& filename, const GUI::TextPosition& identifier_position) override;
     virtual Optional<FunctionParamsHint> get_function_params_hint(const String&, const GUI::TextPosition&) override;
+    virtual Vector<GUI::AutocompleteProvider::TokenInfo> get_tokens_info(const String& filename) override;
 
 private:
     struct SymbolName {
@@ -140,6 +141,9 @@ private:
     template<typename Func>
     void for_each_included_document_recursive(const DocumentData&, Func) const;
 
+    GUI::AutocompleteProvider::TokenInfo::SemanticType get_token_semantic_type(DocumentData const&, Token const&);
+    GUI::AutocompleteProvider::TokenInfo::SemanticType get_semantic_type_for_identifier(DocumentData const&, Position);
+
     HashMap<String, OwnPtr<DocumentData>> m_documents;
 
     // A document's path will be in this set if we're currently processing it.

Userland/DevTools/HackStudio/LanguageServers/LanguageClient.ipc

@@ -5,4 +5,5 @@ endpoint LanguageClient
     declarations_in_document(String filename, Vector<GUI::AutocompleteProvider::Declaration> declarations) =|
     todo_entries_in_document(String filename, Vector<Cpp::Parser::TodoEntry> todo_entries) =|
     parameters_hint_result(Vector<String> params, int current_index) =|
+    tokens_info_result(Vector<GUI::AutocompleteProvider::TokenInfo> tokens_info) =|
 }

Userland/DevTools/HackStudio/LanguageServers/LanguageServer.ipc

@@ -10,4 +10,6 @@ endpoint LanguageServer
     auto_complete_suggestions(GUI::AutocompleteProvider::ProjectLocation location) =|
     find_declaration(GUI::AutocompleteProvider::ProjectLocation location) =|
     get_parameters_hint(GUI::AutocompleteProvider::ProjectLocation location) =|
+    get_tokens_info(String filename) =|
+
 }