HackStudio/LanguageServers: Move some components out of Cpp

This makes them available for use by other language servers.
Also as a bonus, update the Shell language server to discover some
symbols and add go-to-definition functionality :^)

AK/Debug.h.in

@@ -238,6 +238,10 @@
 #cmakedefine01 KEYBOARD_SHORTCUTS_DEBUG
 #endif
 
+#ifndef LANGUAGE_SERVER_DEBUG
+#cmakedefine01 LANGUAGE_SERVER_DEBUG
+#endif
+
 #ifndef LEXER_DEBUG
 #cmakedefine01 LEXER_DEBUG
 #endif

Meta/CMake/all_the_debug_macros.cmake

@@ -170,6 +170,7 @@ set(FILE_WATCHER_DEBUG ON)
 set(SYSCALL_1_DEBUG ON)
 set(RSA_PARSE_DEBUG ON)
 set(LINE_EDITOR_DEBUG ON)
+set(LANGUAGE_SERVER_DEBUG ON)
 
 # False positive: DEBUG is a flag but it works differently.
 # set(DEBUG ON)

Userland/DevTools/HackStudio/LanguageServers/AutoCompleteEngine.cpp

@@ -26,11 +26,12 @@
 
 #include "AutoCompleteEngine.h"
 
-namespace LanguageServers::Cpp {
+namespace LanguageServers {
 
-AutoCompleteEngine::AutoCompleteEngine(ClientConnection& connection, const FileDB& filedb)
+AutoCompleteEngine::AutoCompleteEngine(ClientConnection& connection, const FileDB& filedb, bool should_store_all_declarations)
     : m_connection(connection)
     , m_filedb(filedb)
+    , m_store_all_declarations(should_store_all_declarations)
 {
 }
 
@@ -40,6 +41,8 @@ AutoCompleteEngine::~AutoCompleteEngine()
 void AutoCompleteEngine::set_declarations_of_document(const String& filename, Vector<GUI::AutocompleteProvider::Declaration>&& declarations)
 {
     VERIFY(set_declarations_of_document_callback);
+    if (m_store_all_declarations)
+        m_all_declarations.set(filename, declarations);
     set_declarations_of_document_callback(m_connection, filename, move(declarations));
 }
 }

Userland/DevTools/HackStudio/LanguageServers/AutoCompleteEngine.h

@@ -26,18 +26,18 @@
 
 #pragma once
 
+#include "../AutoCompleteResponse.h"
 #include "FileDB.h"
-#include <DevTools/HackStudio/AutoCompleteResponse.h>
 #include <LibGUI/AutocompleteProvider.h>
 #include <LibGUI/TextPosition.h>
 
-namespace LanguageServers::Cpp {
+namespace LanguageServers {
 
 class ClientConnection;
 
 class AutoCompleteEngine {
 public:
-    AutoCompleteEngine(ClientConnection&, const FileDB& filedb);
+    AutoCompleteEngine(ClientConnection&, const FileDB& filedb, bool store_all_declarations = false);
     virtual ~AutoCompleteEngine();
 
     virtual Vector<GUI::AutocompleteProvider::Entry> get_suggestions(const String& file, const GUI::TextPosition& autocomplete_position) = 0;
@@ -54,9 +54,12 @@ public:
 protected:
     const FileDB& filedb() const { return m_filedb; }
     void set_declarations_of_document(const String&, Vector<GUI::AutocompleteProvider::Declaration>&&);
+    const HashMap<String, Vector<GUI::AutocompleteProvider::Declaration>>& all_declarations() const { return m_all_declarations; }
 
 private:
     ClientConnection& m_connection;
+    HashMap<String, Vector<GUI::AutocompleteProvider::Declaration>> m_all_declarations;
     const FileDB& m_filedb;
+    bool m_store_all_declarations { false };
 };
 }

Userland/DevTools/HackStudio/LanguageServers/CMakeLists.txt

@@ -1,5 +1,16 @@
 compile_ipc(LanguageServer.ipc LanguageServerEndpoint.h)
 compile_ipc(LanguageClient.ipc LanguageClientEndpoint.h)
 
+set(SOURCES
+        AutoCompleteEngine.cpp
+        ClientConnection.cpp
+        FileDB.cpp)
+set(GENERATED_SOURCES
+        LanguageClientEndpoint.h
+        LanguageServerEndpoint.h)
+
+serenity_lib(LibLanguageServer language_server)
+target_link_libraries(LibLanguageServer LibC)
+
 add_subdirectory(Cpp)
 add_subdirectory(Shell)

Userland/DevTools/HackStudio/LanguageServers/ClientConnection.cpp

@@ -25,14 +25,12 @@
  */
 
 #include "ClientConnection.h"
-#include "LexerAutoComplete.h"
-#include "ParserAutoComplete.h"
 #include <AK/Debug.h>
 #include <AK/HashMap.h>
 #include <LibCore/File.h>
 #include <LibGUI/TextDocument.h>
 
-namespace LanguageServers::Cpp {
+namespace LanguageServers {
 
 static HashMap<int, RefPtr<ClientConnection>> s_connections;
 
@@ -40,8 +38,6 @@ ClientConnection::ClientConnection(NonnullRefPtr<Core::LocalSocket> socket, int
     : IPC::ClientConnection<LanguageClientEndpoint, LanguageServerEndpoint>(*this, move(socket), client_id)
 {
     s_connections.set(client_id, *this);
-    m_autocomplete_engine = make<ParserAutoComplete>(*this, m_filedb);
-    m_autocomplete_engine->set_declarations_of_document_callback = &ClientConnection::set_declarations_of_document_callback;
 }
 
 ClientConnection::~ClientConnection()
@@ -127,20 +123,9 @@ void ClientConnection::handle(const Messages::LanguageServer::SetFileContent& me
     m_autocomplete_engine->on_edit(message.file_name());
 }
 
-void ClientConnection::handle(const Messages::LanguageServer::SetAutoCompleteMode& message)
-{
-#ifdef CPP_LANGUAGE_SERVER_DEBUG
-    dbgln("SetAutoCompleteMode: {}", message.mode());
-#endif
-    if (message.mode() == "Parser")
-        m_autocomplete_engine = make<ParserAutoComplete>(*this, m_filedb);
-    else
-        m_autocomplete_engine = make<LexerAutoComplete>(*this, m_filedb);
-}
-
 void ClientConnection::handle(const Messages::LanguageServer::FindDeclaration& message)
 {
-    dbgln_if(CPP_LANGUAGE_SERVER_DEBUG, "FindDeclaration: {} {}:{}", message.location().file, message.location().line, message.location().column);
+    dbgln_if(LANGUAGE_SERVER_DEBUG, "FindDeclaration: {} {}:{}", message.location().file, message.location().line, message.location().column);
     auto document = m_filedb.get(message.location().file);
     if (!document) {
         dbgln("file {} has not been opened", message.location().file);
@@ -153,7 +138,7 @@ void ClientConnection::handle(const Messages::LanguageServer::FindDeclaration& m
         dbgln("could not find declaration");
         return;
     }
-    dbgln_if(CPP_LANGUAGE_SERVER_DEBUG, "declaration location: {} {}:{}", location.value().file, location.value().line, location.value().column);
+    dbgln_if(LANGUAGE_SERVER_DEBUG, "declaration location: {} {}:{}", location.value().file, location.value().line, location.value().column);
     post_message(Messages::LanguageClient::DeclarationLocation(GUI::AutocompleteProvider::ProjectLocation { location.value().file, location.value().line, location.value().column }));
 }
 

Userland/DevTools/HackStudio/LanguageServers/ClientConnection.h

@@ -0,0 +1,67 @@
+/*
+ * Copyright (c) 2020, Itamar S. <itamar8910@gmail.com>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice, this
+ *    list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#pragma once
+
+#include "../AutoCompleteResponse.h"
+#include "AutoCompleteEngine.h"
+#include "FileDB.h"
+#include <AK/HashMap.h>
+#include <AK/LexicalPath.h>
+#include <LibIPC/ClientConnection.h>
+
+#include <Userland/DevTools/HackStudio/LanguageServers/LanguageClientEndpoint.h>
+#include <Userland/DevTools/HackStudio/LanguageServers/LanguageServerEndpoint.h>
+
+namespace LanguageServers {
+
+class ClientConnection
+    : public IPC::ClientConnection<LanguageClientEndpoint, LanguageServerEndpoint>
+    , public LanguageServerEndpoint {
+
+public:
+    explicit ClientConnection(NonnullRefPtr<Core::LocalSocket>, int client_id);
+    ~ClientConnection() override;
+
+    virtual void die() override;
+
+protected:
+    virtual OwnPtr<Messages::LanguageServer::GreetResponse> handle(const Messages::LanguageServer::Greet&) override;
+    virtual void handle(const Messages::LanguageServer::FileOpened&) override;
+    virtual void handle(const Messages::LanguageServer::FileEditInsertText&) override;
+    virtual void handle(const Messages::LanguageServer::FileEditRemoveText&) override;
+    virtual void handle(const Messages::LanguageServer::SetFileContent&) override;
+    virtual void handle(const Messages::LanguageServer::AutoCompleteSuggestions&) override;
+    virtual void handle(const Messages::LanguageServer::FindDeclaration&) override;
+    virtual void handle(const Messages::LanguageServer::SetAutoCompleteMode&) override = 0;
+
+    static void set_declarations_of_document_callback(ClientConnection&, const String&, Vector<GUI::AutocompleteProvider::Declaration>&&);
+
+    FileDB m_filedb;
+    OwnPtr<AutoCompleteEngine> m_autocomplete_engine;
+};
+
+}

Userland/DevTools/HackStudio/LanguageServers/Cpp/CMakeLists.txt

@@ -1,7 +1,4 @@
 set(SOURCES
-    AutoCompleteEngine.cpp
-    ClientConnection.cpp
-    FileDB.cpp
     LexerAutoComplete.cpp
     ParserAutoComplete.cpp
     main.cpp
@@ -15,4 +12,4 @@ serenity_bin(CppLanguageServer)
 
 # We link with LibGUI because we use GUI::TextDocument to update
 # the content of files according to the edit actions we receive over IPC.
-target_link_libraries(CppLanguageServer LibIPC LibCpp LibGUI)
+target_link_libraries(CppLanguageServer LibIPC LibCpp LibGUI LibLanguageServer)

Userland/DevTools/HackStudio/LanguageServers/Cpp/ClientConnection.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2020, Itamar S. <itamar8910@gmail.com>
+ * Copyright (c) 2020, the SerenityOS developers.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -26,43 +26,33 @@
 
 #pragma once
 
-#include "AutoCompleteEngine.h"
-#include "FileDB.h"
-#include <AK/HashMap.h>
-#include <AK/LexicalPath.h>
-#include <DevTools/HackStudio/AutoCompleteResponse.h>
-#include <LibIPC/ClientConnection.h>
-
-#include <DevTools/HackStudio/LanguageServers/LanguageClientEndpoint.h>
-#include <DevTools/HackStudio/LanguageServers/LanguageServerEndpoint.h>
+#include "LexerAutoComplete.h"
+#include "ParserAutoComplete.h"
+#include <DevTools/HackStudio/LanguageServers/ClientConnection.h>
 
 namespace LanguageServers::Cpp {
 
-class ClientConnection final
-    : public IPC::ClientConnection<LanguageClientEndpoint, LanguageServerEndpoint>
-    , public LanguageServerEndpoint {
+class ClientConnection final : public LanguageServers::ClientConnection {
     C_OBJECT(ClientConnection);
 
 public:
-    explicit ClientConnection(NonnullRefPtr<Core::LocalSocket>, int client_id);
-    ~ClientConnection() override;
+    ClientConnection(NonnullRefPtr<Core::LocalSocket> socket, int client_id)
+        : LanguageServers::ClientConnection(move(socket), client_id)
+    {
+        m_autocomplete_engine = make<ParserAutoComplete>(*this, m_filedb);
+        m_autocomplete_engine->set_declarations_of_document_callback = &ClientConnection::set_declarations_of_document_callback;
+    }
 
-    virtual void die() override;
+    virtual ~ClientConnection() override = default;
 
 private:
-    virtual OwnPtr<Messages::LanguageServer::GreetResponse> handle(const Messages::LanguageServer::Greet&) override;
-    virtual void handle(const Messages::LanguageServer::FileOpened&) override;
-    virtual void handle(const Messages::LanguageServer::FileEditInsertText&) override;
-    virtual void handle(const Messages::LanguageServer::FileEditRemoveText&) override;
-    virtual void handle(const Messages::LanguageServer::SetFileContent&) override;
-    virtual void handle(const Messages::LanguageServer::AutoCompleteSuggestions&) override;
-    virtual void handle(const Messages::LanguageServer::SetAutoCompleteMode&) override;
-    virtual void handle(const Messages::LanguageServer::FindDeclaration&) override;
-
-    static void set_declarations_of_document_callback(ClientConnection&, const String&, Vector<GUI::AutocompleteProvider::Declaration>&&);
-
-    FileDB m_filedb;
-    OwnPtr<AutoCompleteEngine> m_autocomplete_engine;
+    virtual void handle(const Messages::LanguageServer::SetAutoCompleteMode& message) override
+    {
+        dbgln_if(CPP_LANGUAGE_SERVER_DEBUG, "SetAutoCompleteMode: {}", message.mode());
+        if (message.mode() == "Parser")
+            m_autocomplete_engine = make<ParserAutoComplete>(*this, m_filedb);
+        else
+            m_autocomplete_engine = make<LexerAutoComplete>(*this, m_filedb);
+    }
 };
-
 }

Userland/DevTools/HackStudio/LanguageServers/Cpp/LexerAutoComplete.h

@@ -26,10 +26,10 @@
 
 #pragma once
 
-#include "AutoCompleteEngine.h"
 #include <AK/String.h>
 #include <AK/Vector.h>
 #include <DevTools/HackStudio/AutoCompleteResponse.h>
+#include <DevTools/HackStudio/LanguageServers/AutoCompleteEngine.h>
 #include <LibCpp/Lexer.h>
 #include <LibGUI/TextPosition.h>
 

Userland/DevTools/HackStudio/LanguageServers/Cpp/ParserAutoComplete.cpp

@@ -27,12 +27,12 @@
 #include "ParserAutoComplete.h"
 #include <AK/Assertions.h>
 #include <AK/HashTable.h>
-#include <DevTools/HackStudio/LanguageServers/Cpp/ClientConnection.h>
 #include <LibCpp/AST.h>
 #include <LibCpp/Lexer.h>
 #include <LibCpp/Parser.h>
 #include <LibCpp/Preprocessor.h>
 #include <LibRegex/Regex.h>
+#include <Userland/DevTools/HackStudio/LanguageServers/ClientConnection.h>
 
 namespace LanguageServers::Cpp {
 

Userland/DevTools/HackStudio/LanguageServers/Cpp/ParserAutoComplete.h

@@ -26,12 +26,12 @@
 
 #pragma once
 
-#include "AutoCompleteEngine.h"
-#include "FileDB.h"
 #include <AK/Function.h>
 #include <AK/String.h>
 #include <AK/Vector.h>
 #include <DevTools/HackStudio/AutoCompleteResponse.h>
+#include <DevTools/HackStudio/LanguageServers/AutoCompleteEngine.h>
+#include <DevTools/HackStudio/LanguageServers/FileDB.h>
 #include <LibCpp/AST.h>
 #include <LibCpp/Parser.h>
 #include <LibCpp/Preprocessor.h>

Userland/DevTools/HackStudio/LanguageServers/FileDB.cpp

@@ -29,6 +29,8 @@
 #include <AK/LexicalPath.h>
 #include <LibCore/File.h>
 
+namespace LanguageServers {
+
 RefPtr<const GUI::TextDocument> FileDB::get(const String& file_name) const
 {
     auto absolute_path = to_absolute_path(file_name);
@@ -126,9 +128,7 @@ void FileDB::on_file_edit_insert_text(const String& file_name, const String& ins
     GUI::TextPosition start_position { start_line, start_column };
     document->insert_at(start_position, inserted_text, &s_default_document_client);
 
-#if FILE_CONTENT_DEBUG
-    dbgln("{}", document->text());
-#endif
+    dbgln_if(FILE_CONTENT_DEBUG, "{}", document->text());
 }
 
 void FileDB::on_file_edit_remove_text(const String& file_name, size_t start_line, size_t start_column, size_t end_line, size_t end_column)
@@ -145,7 +145,7 @@ void FileDB::on_file_edit_remove_text(const String& file_name, size_t start_line
     };
 
     document->remove(range);
-#if FILE_CONTENT_DEBUG
-    dbgln("{}", document->text());
-#endif
+    dbgln_if(FILE_CONTENT_DEBUG, "{}", document->text());
+}
+
 }

Userland/DevTools/HackStudio/LanguageServers/FileDB.h

@@ -31,6 +31,8 @@
 #include <AK/String.h>
 #include <LibGUI/TextDocument.h>
 
+namespace LanguageServers {
+
 class FileDB final {
 public:
     RefPtr<const GUI::TextDocument> get(const String& file_name) const;
@@ -53,3 +55,5 @@ private:
     HashMap<String, NonnullRefPtr<GUI::TextDocument>> m_open_files;
     String m_project_root;
 };
+
+}

Userland/DevTools/HackStudio/LanguageServers/Shell/AutoComplete.cpp

@@ -25,37 +25,232 @@
  */
 
 #include "AutoComplete.h"
+#include <AK/Assertions.h>
 #include <AK/HashTable.h>
-#include <LibLine/SuggestionManager.h>
-#include <Shell/AST.h>
-#include <Shell/Parser.h>
-#include <Shell/Shell.h>
+#include <LibRegex/Regex.h>
+#include <Userland/DevTools/HackStudio/LanguageServers/ClientConnection.h>
 
 namespace LanguageServers::Shell {
 
-Vector<GUI::AutocompleteProvider::Entry> AutoComplete::get_suggestions(const String& code, size_t offset)
+RefPtr<::Shell::Shell> AutoComplete::s_shell {};
+
+AutoComplete::AutoComplete(ClientConnection& connection, const FileDB& filedb)
+    : AutoCompleteEngine(connection, filedb, true)
 {
-    // FIXME: No need to reparse this every time!
-    auto ast = ::Shell::Parser { code }.parse();
-    if (!ast)
+}
+
+const AutoComplete::DocumentData& AutoComplete::get_or_create_document_data(const String& file)
+{
+    auto absolute_path = filedb().to_absolute_path(file);
+    if (!m_documents.contains(absolute_path)) {
+        set_document_data(absolute_path, create_document_data_for(absolute_path));
+    }
+    return get_document_data(absolute_path);
+}
+
+const AutoComplete::DocumentData& AutoComplete::get_document_data(const String& file) const
+{
+    auto absolute_path = filedb().to_absolute_path(file);
+    auto document_data = m_documents.get(absolute_path);
+    VERIFY(document_data.has_value());
+    return *document_data.value();
+}
+
+OwnPtr<AutoComplete::DocumentData> AutoComplete::create_document_data_for(const String& file)
+{
+    auto document = filedb().get(file);
+    if (!document)
         return {};
+    auto content = document->text();
+    auto document_data = make<DocumentData>(document->text(), file);
+    for (auto& path : document_data->sourced_paths())
+        get_or_create_document_data(path);
 
-#if AUTOCOMPLETE_DEBUG
-    dbgln("Complete '{}'", code);
-    ast->dump(1);
-    dbgln("At offset {}", offset);
-#endif
+    update_declared_symbols(*document_data);
+    return move(document_data);
+}
 
-    auto result = ast->complete_for_editor(m_shell, offset);
-    Vector<GUI::AutocompleteProvider::Entry> completions;
-    for (auto& entry : result) {
-#if AUTOCOMPLETE_DEBUG
-        dbgln("Suggestion: '{}' starting at {}", entry.text_string, entry.input_offset);
-#endif
-        completions.append({ entry.text_string, entry.input_offset });
+void AutoComplete::set_document_data(const String& file, OwnPtr<DocumentData>&& data)
+{
+    m_documents.set(filedb().to_absolute_path(file), move(data));
+}
+
+AutoComplete::DocumentData::DocumentData(String&& _text, String _filename)
+    : filename(move(_filename))
+    , text(move(_text))
+    , node(parse())
+{
+}
+
+const Vector<String>& AutoComplete::DocumentData::sourced_paths() const
+{
+    if (all_sourced_paths.has_value())
+        return all_sourced_paths.value();
+
+    struct : public ::Shell::AST::NodeVisitor {
+        void visit(const ::Shell::AST::CastToCommand* node) override
+        {
+            auto& inner = node->inner();
+            if (inner->is_list()) {
+                if (auto* list = dynamic_cast<const ::Shell::AST::ListConcatenate*>(inner.ptr())) {
+                    auto& entries = list->list();
+                    if (entries.size() == 2 && entries.first()->is_bareword() && static_ptr_cast<::Shell::AST::BarewordLiteral>(entries.first())->text() == "source") {
+                        auto& filename = entries[1];
+                        if (filename->would_execute())
+                            return;
+                        auto name_list = const_cast<::Shell::AST::Node*>(filename.ptr())->run(nullptr)->resolve_as_list(nullptr);
+                        StringBuilder builder;
+                        builder.join(" ", name_list);
+                        sourced_files.set(builder.build());
+                    }
+                }
+            }
+            ::Shell::AST::NodeVisitor::visit(node);
+        }
+
+        HashTable<String> sourced_files;
+    } visitor;
+
+    node->visit(visitor);
+
+    Vector<String> sourced_paths;
+    for (auto& entry : visitor.sourced_files)
+        sourced_paths.append(move(entry));
+
+    all_sourced_paths = move(sourced_paths);
+    return all_sourced_paths.value();
+}
+
+NonnullRefPtr<::Shell::AST::Node> AutoComplete::DocumentData::parse() const
+{
+    ::Shell::Parser parser { text };
+    if (auto node = parser.parse())
+        return node.release_nonnull();
+
+    return ::Shell::AST::create<::Shell::AST::SyntaxError>(::Shell::AST::Position {}, "Unable to parse file");
+}
+
+size_t AutoComplete::resolve(const AutoComplete::DocumentData& document, const GUI::TextPosition& position)
+{
+    size_t offset = 0;
+
+    if (position.line() > 0) {
+        auto first = true;
+        size_t line = 0;
+        for (auto& line_view : document.text.split_limit('\n', position.line() + 1, true)) {
+            if (line == position.line())
+                break;
+            if (first)
+                first = false;
+            else
+                ++offset; // For the newline.
+            offset += line_view.length();
+            ++line;
+        }
     }
 
-    return completions;
+    offset += position.column() + 1;
+    return offset;
 }
 
+Vector<GUI::AutocompleteProvider::Entry> AutoComplete::get_suggestions(const String& file, const GUI::TextPosition& position)
+{
+    dbgln_if(SH_LANGUAGE_SERVER_DEBUG, "AutoComplete position {}:{}", position.line(), position.column());
+
+    const auto& document = get_or_create_document_data(file);
+    size_t offset_in_file = resolve(document, position);
+
+    ::Shell::AST::HitTestResult hit_test = document.node->hit_test_position(offset_in_file);
+    if (!hit_test.matching_node) {
+        dbgln_if(SH_LANGUAGE_SERVER_DEBUG, "no node at position {}:{}", position.line(), position.column());
+        return {};
+    }
+
+    auto completions = const_cast<::Shell::AST::Node*>(document.node.ptr())->complete_for_editor(shell(), offset_in_file, hit_test);
+    Vector<GUI::AutocompleteProvider::Entry> entries;
+    for (auto& completion : completions)
+        entries.append({ completion.text_string, completion.input_offset });
+
+    return entries;
+}
+
+void AutoComplete::on_edit(const String& file)
+{
+    set_document_data(file, create_document_data_for(file));
+}
+
+void AutoComplete::file_opened([[maybe_unused]] const String& file)
+{
+    set_document_data(file, create_document_data_for(file));
+}
+
+Optional<GUI::AutocompleteProvider::ProjectLocation> AutoComplete::find_declaration_of(const String& file_name, const GUI::TextPosition& identifier_position)
+{
+    dbgln_if(SH_LANGUAGE_SERVER_DEBUG, "find_declaration_of({}, {}:{})", file_name, identifier_position.line(), identifier_position.column());
+    const auto& document = get_or_create_document_data(file_name);
+    auto position = resolve(document, identifier_position);
+    auto result = document.node->hit_test_position(position);
+    if (!result.matching_node) {
+        dbgln_if(SH_LANGUAGE_SERVER_DEBUG, "no node at position {}:{}", identifier_position.line(), identifier_position.column());
+        return {};
+    }
+
+    if (!result.matching_node->is_bareword()) {
+        dbgln_if(SH_LANGUAGE_SERVER_DEBUG, "no bareword at position {}:{}", identifier_position.line(), identifier_position.column());
+        return {};
+    }
+
+    auto name = static_ptr_cast<::Shell::AST::BarewordLiteral>(result.matching_node)->text();
+    auto& declarations = all_declarations();
+    for (auto& entry : declarations) {
+        for (auto& declaration : entry.value) {
+            if (declaration.name == name)
+                return declaration.position;
+        }
+    }
+
+    return {};
+}
+
+void AutoComplete::update_declared_symbols(const DocumentData& document)
+{
+    struct Visitor : public ::Shell::AST::NodeVisitor {
+        explicit Visitor(const String& filename)
+            : filename(filename)
+        {
+        }
+
+        void visit(const ::Shell::AST::VariableDeclarations* node) override
+        {
+            for (auto& entry : node->variables()) {
+                auto literal = entry.name->leftmost_trivial_literal();
+                if (!literal)
+                    continue;
+
+                String name;
+                if (literal->is_bareword())
+                    name = static_ptr_cast<::Shell::AST::BarewordLiteral>(literal)->text();
+
+                if (!name.is_empty()) {
+                    dbgln("Found variable {}", name);
+                    declarations.append({ move(name), { filename, entry.name->position().start_line.line_number, entry.name->position().start_line.line_column }, GUI::AutocompleteProvider::DeclarationType::Variable });
+                }
+            }
+            ::Shell::AST::NodeVisitor::visit(node);
+        }
+
+        void visit(const ::Shell::AST::FunctionDeclaration* node) override
+        {
+            dbgln("Found function {}", node->name().name);
+            declarations.append({ node->name().name, { filename, node->position().start_line.line_number, node->position().start_line.line_column }, GUI::AutocompleteProvider::DeclarationType::Function });
+        }
+
+        const String& filename;
+        Vector<GUI::AutocompleteProvider::Declaration> declarations;
+    } visitor { document.filename };
+
+    document.node->visit(visitor);
+
+    set_declarations_of_document(document.filename, move(visitor.declarations));
+}
 }

Userland/DevTools/HackStudio/LanguageServers/Shell/AutoComplete.h

@@ -26,25 +26,53 @@
 
 #pragma once
 
-#include <AK/String.h>
-#include <AK/Vector.h>
-#include <DevTools/HackStudio/AutoCompleteResponse.h>
-#include <LibGUI/TextPosition.h>
+#include <DevTools/HackStudio/LanguageServers/AutoCompleteEngine.h>
 #include <Shell/Shell.h>
 
 namespace LanguageServers::Shell {
 
-class AutoComplete {
+class AutoComplete : public AutoCompleteEngine {
 public:
-    AutoComplete()
-        : m_shell(::Shell::Shell::construct())
-    {
-    }
-
-    Vector<GUI::AutocompleteProvider::Entry> get_suggestions(const String& code, size_t autocomplete_position);
+    AutoComplete(ClientConnection&, const FileDB& filedb);
+    virtual Vector<GUI::AutocompleteProvider::Entry> get_suggestions(const String& file, const GUI::TextPosition& position) override;
+    virtual void on_edit(const String& file) override;
+    virtual void file_opened([[maybe_unused]] const String& file) override;
+    virtual Optional<GUI::AutocompleteProvider::ProjectLocation> find_declaration_of(const String& file_name, const GUI::TextPosition& identifier_position) override;
 
 private:
-    NonnullRefPtr<::Shell::Shell> m_shell;
-};
+    struct DocumentData {
+        DocumentData(String&& text, String filename);
+        String filename;
+        String text;
+        NonnullRefPtr<::Shell::AST::Node> node;
 
+        const Vector<String>& sourced_paths() const;
+
+    private:
+        NonnullRefPtr<::Shell::AST::Node> parse() const;
+
+        mutable Optional<Vector<String>> all_sourced_paths {};
+    };
+
+    const DocumentData& get_document_data(const String& file) const;
+    const DocumentData& get_or_create_document_data(const String& file);
+    void set_document_data(const String& file, OwnPtr<DocumentData>&& data);
+
+    OwnPtr<DocumentData> create_document_data_for(const String& file);
+    String document_path_from_include_path(const StringView& include_path) const;
+    void update_declared_symbols(const DocumentData&);
+
+    static size_t resolve(const AutoComplete::DocumentData& document, const GUI::TextPosition& position);
+
+    ::Shell::Shell& shell()
+    {
+        if (s_shell)
+            return *s_shell;
+        s_shell = ::Shell::Shell::construct();
+        return *s_shell;
+    }
+
+    HashMap<String, OwnPtr<DocumentData>> m_documents;
+    static RefPtr<::Shell::Shell> s_shell;
+};
 }

Userland/DevTools/HackStudio/LanguageServers/Shell/CMakeLists.txt

@@ -1,7 +1,6 @@
 set(SOURCES
-    ClientConnection.cpp
-    main.cpp
     AutoComplete.cpp
+    main.cpp
 )
 
 set(GENERATED_SOURCES
@@ -12,4 +11,4 @@ serenity_bin(ShellLanguageServer)
 
 # We link with LibGUI because we use GUI::TextDocument to update
 # the content of files according to the edit actions we receive over IPC.
-target_link_libraries(ShellLanguageServer LibIPC LibShell LibGUI)
+target_link_libraries(ShellLanguageServer LibIPC LibShell LibGUI LibLanguageServer)

Userland/DevTools/HackStudio/LanguageServers/Shell/ClientConnection.cpp

@@ -1,180 +0,0 @@
-/*
- * Copyright (c) 2020, the SerenityOS developers.
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- * 1. Redistributions of source code must retain the above copyright notice, this
- *    list of conditions and the following disclaimer.
- *
- * 2. Redistributions in binary form must reproduce the above copyright notice,
- *    this list of conditions and the following disclaimer in the documentation
- *    and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
- * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
- * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
- * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */
-
-#include "ClientConnection.h"
-#include "AutoComplete.h"
-#include <AK/Debug.h>
-#include <AK/HashMap.h>
-#include <LibCore/File.h>
-#include <LibGUI/TextDocument.h>
-
-namespace LanguageServers::Shell {
-
-static HashMap<int, RefPtr<ClientConnection>> s_connections;
-
-ClientConnection::ClientConnection(NonnullRefPtr<Core::LocalSocket> socket, int client_id)
-    : IPC::ClientConnection<LanguageClientEndpoint, LanguageServerEndpoint>(*this, move(socket), client_id)
-{
-    s_connections.set(client_id, *this);
-}
-
-ClientConnection::~ClientConnection()
-{
-}
-
-void ClientConnection::die()
-{
-    s_connections.remove(client_id());
-    exit(0);
-}
-
-OwnPtr<Messages::LanguageServer::GreetResponse> ClientConnection::handle(const Messages::LanguageServer::Greet&)
-{
-    return make<Messages::LanguageServer::GreetResponse>();
-}
-
-class DefaultDocumentClient final : public GUI::TextDocument::Client {
-public:
-    virtual ~DefaultDocumentClient() override = default;
-    virtual void document_did_append_line() override {};
-    virtual void document_did_insert_line(size_t) override {};
-    virtual void document_did_remove_line(size_t) override {};
-    virtual void document_did_remove_all_lines() override {};
-    virtual void document_did_change() override {};
-    virtual void document_did_set_text() override {};
-    virtual void document_did_set_cursor(const GUI::TextPosition&) override {};
-
-    virtual bool is_automatic_indentation_enabled() const override { return false; }
-    virtual int soft_tab_width() const override { return 4; }
-};
-
-static DefaultDocumentClient s_default_document_client;
-
-void ClientConnection::handle(const Messages::LanguageServer::FileOpened& message)
-{
-    auto file = Core::File::construct(this);
-    if (!file->open(message.file().take_fd(), Core::IODevice::ReadOnly, Core::File::ShouldCloseFileDescriptor::Yes)) {
-        errno = file->error();
-        perror("open");
-        dbgln("Failed to open project file");
-        return;
-    }
-    auto content = file->read_all();
-    StringView content_view(content);
-    auto document = GUI::TextDocument::create(&s_default_document_client);
-    document->set_text(content_view);
-    m_open_files.set(message.file_name(), document);
-    dbgln_if(FILE_CONTENT_DEBUG, "{}", document->text());
-}
-
-void ClientConnection::handle(const Messages::LanguageServer::FileEditInsertText& message)
-{
-#if SH_LANGUAGE_SERVER_DEBUG
-    dbgln("InsertText for file: {}", message.file_name());
-    dbgln("Text: {}", message.text());
-    dbgln("[{}:{}]", message.start_line(), message.start_column());
-#endif
-    auto document = document_for(message.file_name());
-    if (!document) {
-        dbgln("file {} has not been opened", message.file_name());
-        return;
-    }
-    GUI::TextPosition start_position { (size_t)message.start_line(), (size_t)message.start_column() };
-    document->insert_at(start_position, message.text(), &s_default_document_client);
-#if FILE_CONTENT_DEBUG
-    dbgln("{}", document->text());
-#endif
-}
-
-void ClientConnection::handle(const Messages::LanguageServer::FileEditRemoveText& message)
-{
-#if SH_LANGUAGE_SERVER_DEBUG
-    dbgln("RemoveText for file: {}", message.file_name());
-    dbgln("[{}:{} - {}:{}]", message.start_line(), message.start_column(), message.end_line(), message.end_column());
-#endif
-    auto document = document_for(message.file_name());
-    if (!document) {
-        dbgln("file {} has not been opened", message.file_name());
-        return;
-    }
-    GUI::TextPosition start_position { (size_t)message.start_line(), (size_t)message.start_column() };
-    GUI::TextRange range {
-        GUI::TextPosition { (size_t)message.start_line(),
-            (size_t)message.start_column() },
-        GUI::TextPosition { (size_t)message.end_line(),
-            (size_t)message.end_column() }
-    };
-
-    document->remove(range);
-    dbgln_if(FILE_CONTENT_DEBUG, "{}", document->text());
-}
-
-void ClientConnection::handle(const Messages::LanguageServer::AutoCompleteSuggestions& message)
-{
-#if SH_LANGUAGE_SERVER_DEBUG
-    dbgln("AutoCompleteSuggestions for: {} {}:{}", message.location().file, message.location().line, message.location().column);
-#endif
-
-    auto document = document_for(message.location().file);
-    if (!document) {
-        dbgln("file {} has not been opened", message.location().file);
-        return;
-    }
-
-    auto& lines = document->lines();
-    size_t offset = 0;
-
-    if (message.location().line > 0) {
-        for (size_t i = 0; i < message.location().line; ++i)
-            offset += lines[i].length() + 1;
-    }
-    offset += message.location().column;
-
-    auto suggestions = m_autocomplete.get_suggestions(document->text(), offset);
-    post_message(Messages::LanguageClient::AutoCompleteSuggestions(move(suggestions)));
-}
-
-RefPtr<GUI::TextDocument> ClientConnection::document_for(const String& file_name)
-{
-    auto document_optional = m_open_files.get(file_name);
-    if (!document_optional.has_value())
-        return nullptr;
-
-    return document_optional.value();
-}
-
-void ClientConnection::handle(const Messages::LanguageServer::SetFileContent& message)
-{
-    auto document = document_for(message.file_name());
-    if (!document) {
-        dbgln("file {} has not been opened", message.file_name());
-        return;
-    }
-    auto content = message.content();
-    document->set_text(content.view());
-}
-
-}

Userland/DevTools/HackStudio/LanguageServers/Shell/ClientConnection.h

@@ -27,43 +27,22 @@
 #pragma once
 
 #include "AutoComplete.h"
-#include <AK/HashMap.h>
-#include <AK/LexicalPath.h>
-#include <DevTools/HackStudio/AutoCompleteResponse.h>
-#include <LibGUI/TextDocument.h>
-#include <LibIPC/ClientConnection.h>
-
-#include <DevTools/HackStudio/LanguageServers/LanguageClientEndpoint.h>
-#include <DevTools/HackStudio/LanguageServers/LanguageServerEndpoint.h>
+#include <DevTools/HackStudio/LanguageServers/ClientConnection.h>
 
 namespace LanguageServers::Shell {
 
-class ClientConnection final
-    : public IPC::ClientConnection<LanguageClientEndpoint, LanguageServerEndpoint>
-    , public LanguageServerEndpoint {
+class ClientConnection final : public LanguageServers::ClientConnection {
     C_OBJECT(ClientConnection);
 
-public:
-    explicit ClientConnection(NonnullRefPtr<Core::LocalSocket>, int client_id);
-    ~ClientConnection() override;
-
-    virtual void die() override;
+    ClientConnection(NonnullRefPtr<Core::LocalSocket> socket, int client_id)
+        : LanguageServers::ClientConnection(move(socket), client_id)
+    {
+        m_autocomplete_engine = make<AutoComplete>(*this, m_filedb);
+        m_autocomplete_engine->set_declarations_of_document_callback = &ClientConnection::set_declarations_of_document_callback;
+    }
+    virtual ~ClientConnection() override = default;
 
 private:
-    virtual OwnPtr<Messages::LanguageServer::GreetResponse> handle(const Messages::LanguageServer::Greet&) override;
-    virtual void handle(const Messages::LanguageServer::FileOpened&) override;
-    virtual void handle(const Messages::LanguageServer::FileEditInsertText&) override;
-    virtual void handle(const Messages::LanguageServer::FileEditRemoveText&) override;
-    virtual void handle(const Messages::LanguageServer::SetFileContent&) override;
-    virtual void handle(const Messages::LanguageServer::AutoCompleteSuggestions&) override;
     virtual void handle(const Messages::LanguageServer::SetAutoCompleteMode&) override { }
-    virtual void handle(const Messages::LanguageServer::FindDeclaration&) override {};
-
-    RefPtr<GUI::TextDocument> document_for(const String& file_name);
-
-    HashMap<String, NonnullRefPtr<GUI::TextDocument>> m_open_files;
-
-    AutoComplete m_autocomplete;
 };
-
 }

Userland/DevTools/HackStudio/LanguageServers/Shell/main.cpp

@@ -55,9 +55,5 @@ int main(int, char**)
         perror("unveil");
         return 1;
     }
-    if (unveil(nullptr, nullptr) < 0) {
-        perror("unveil");
-        return 1;
-    }
     return event_loop.exec();
 }