LibC+LibELF: Implement more fully-features dlfcn functionality

This implements more of the dlfcn functionality. Most notably:

* It's now possible to dlopen() libraries which were already
  loaded at program startup time. This does not cause those
  libraries to be loaded twice.
* Errors are reported via dlerror() rather than by crashing
  the program.
* Calls to the dl*() functions are thread-safe.

Userland/Libraries/CMakeLists.txt

@@ -11,6 +11,7 @@ add_subdirectory(LibCrypto)
 add_subdirectory(LibDebug)
 add_subdirectory(LibDesktop)
 add_subdirectory(LibDiff)
+add_subdirectory(LibDl)
 add_subdirectory(LibELF)
 add_subdirectory(LibGemini)
 add_subdirectory(LibGfx)

Userland/Libraries/LibC/dlfcn.cpp

@@ -1,95 +1,12 @@
 /*
- * Copyright (c) 2018-2020, Andreas Kling <kling@serenityos.org>
+ * Copyright (c) 2021, Gunnar Beutner <gunnar@beutner.name>
  *
  * SPDX-License-Identifier: BSD-2-Clause
  */
 
-#include <AK/HashMap.h>
-#include <AK/LexicalPath.h>
-#include <AK/RefPtr.h>
-#include <AK/ScopeGuard.h>
-#include <AK/String.h>
-#include <LibELF/DynamicLoader.h>
-#include <assert.h>
-#include <dlfcn.h>
-#include <fcntl.h>
-#include <unistd.h>
+#include <LibDl/dlfcn_integration.h>
 
-// NOTE: The string here should never include a trailing newline (according to POSIX)
-String g_dlerror_msg;
-
-HashMap<String, RefPtr<ELF::DynamicLoader>> g_elf_objects;
-
-extern "C" {
-
-int dlclose(void*)
-{
-    g_dlerror_msg = "dlclose not implemented!";
-    return -1;
-}
-
-char* dlerror()
-{
-    return const_cast<char*>(g_dlerror_msg.characters());
-}
-
-void* dlopen(const char* filename, int flags)
-{
-    // FIXME: Create a global mutex/semaphore/lock for dlopen/dlclose/dlsym and (?) dlerror
-    // FIXME: refcount?
-
-    if (!filename) {
-        // FIXME: Return the handle for "the main executable"
-        //     The Serenity Kernel will keep a mapping of the main elf binary resident in memory,
-        //     But a future dynamic loader might have a different idea/way of letting us access this information
-        TODO();
-    }
-
-    auto basename = LexicalPath(filename).basename();
-
-    auto existing_elf_object = g_elf_objects.get(basename);
-    if (existing_elf_object.has_value()) {
-        return const_cast<ELF::DynamicLoader*>(existing_elf_object.value());
-    }
-
-    int fd = open(filename, O_RDONLY);
-    if (fd < 0) {
-        g_dlerror_msg = String::formatted("Unable to open file {}", filename);
-        return nullptr;
-    }
-
-    ScopeGuard close_fd_guard([fd]() { close(fd); });
-
-    auto loader = ELF::DynamicLoader::try_create(fd, filename);
-    if (!loader || !loader->is_valid()) {
-        g_dlerror_msg = String::formatted("{} is not a valid ELF dynamic shared object!", filename);
-        return nullptr;
-    }
-
-    auto object = loader->map();
-    if (!object || !loader->link(flags, /* total_tls_size (FIXME) */ 0)) {
-        g_dlerror_msg = String::formatted("Failed to load ELF object {}", filename);
-        return nullptr;
-    }
-
-    g_elf_objects.set(basename, move(loader));
-    g_dlerror_msg = "Successfully loaded ELF object.";
-
-    // we have one refcount already
-    return const_cast<ELF::DynamicLoader*>(g_elf_objects.get(basename).value());
-}
-
-void* dlsym(void* handle, const char* symbol_name)
-{
-    // FIXME: When called with a NULL handle we're supposed to search every dso in the process... that'll get expensive
-    VERIFY(handle);
-    auto* dso = reinterpret_cast<ELF::DynamicLoader*>(handle);
-    void* symbol = dso->symbol_for_name(symbol_name);
-    if (!symbol) {
-        g_dlerror_msg = "Symbol not found";
-        return nullptr;
-    }
-    return symbol;
-}
-
-} // extern "C"
+// These are used by libdl and are filled in by the dynamic loader.
+DlCloseFunction __dlclose;
+DlOpenFunction __dlopen;
+DlSymFunction __dlsym;

Userland/Libraries/LibDl/CMakeLists.txt

@@ -0,0 +1,6 @@
+set(SOURCES
+    dlfcn.cpp
+)
+
+serenity_libc(LibDl dl)
+target_link_libraries(LibDl LibC)

Userland/Libraries/LibDl/dlfcn.cpp

@@ -0,0 +1,62 @@
+/*
+ * Copyright (c) 2021, Gunnar Beutner <gunnar@beutner.name>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#include <AK/String.h>
+#include <AK/Types.h>
+#include <LibDl/dlfcn.h>
+#include <LibDl/dlfcn_integration.h>
+#include <string.h>
+
+// FIXME: use thread_local and a String once TLS works
+__thread char* s_dlerror_text = NULL;
+__thread bool s_dlerror_retrieved = false;
+
+static void store_error(const String& error)
+{
+    free(s_dlerror_text);
+    s_dlerror_text = strdup(error.characters());
+    s_dlerror_retrieved = false;
+}
+
+int dlclose(void* handle)
+{
+    auto result = __dlclose(handle);
+    if (result.is_error()) {
+        store_error(result.error().text);
+        return -1;
+    }
+    return 0;
+}
+
+char* dlerror()
+{
+    if (s_dlerror_retrieved) {
+        free(s_dlerror_text);
+        s_dlerror_text = nullptr;
+    }
+    s_dlerror_retrieved = true;
+    return const_cast<char*>(s_dlerror_text);
+}
+
+void* dlopen(const char* file_name, int flags)
+{
+    auto result = __dlopen(file_name, flags);
+    if (result.is_error()) {
+        store_error(result.error().text);
+        return nullptr;
+    }
+    return result.value();
+}
+
+void* dlsym(void* handle, const char* symbol_name)
+{
+    auto result = __dlsym(handle, symbol_name);
+    if (result.is_error()) {
+        store_error(result.error().text);
+        return nullptr;
+    }
+    return result.value();
+}

Userland/Libraries/LibDl/dlfcn_integration.h

@@ -0,0 +1,34 @@
+/*
+ * Copyright (c) 2021, Gunnar Beutner <gunnar@beutner.name>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#pragma once
+
+#include <AK/Result.h>
+#include <AK/String.h>
+
+struct DlErrorMessage {
+    DlErrorMessage(String&& other)
+        : text(move(other))
+    {
+    }
+
+    // The virtual destructor is required because we're passing this
+    // struct to the dynamic loader - whose operator delete differs
+    // from the one in libc.so
+    virtual ~DlErrorMessage() { }
+
+    String text;
+};
+
+typedef Result<void, DlErrorMessage> (*DlCloseFunction)(void*);
+typedef Result<void*, DlErrorMessage> (*DlOpenFunction)(const char*, int);
+typedef Result<void*, DlErrorMessage> (*DlSymFunction)(void*, const char*);
+
+extern "C" {
+extern DlCloseFunction __dlclose;
+extern DlOpenFunction __dlopen;
+extern DlSymFunction __dlsym;
+}

Userland/Libraries/LibELF/DynamicLinker.cpp

@@ -12,38 +12,44 @@
 #include <AK/LexicalPath.h>
 #include <AK/NonnullRefPtrVector.h>
 #include <AK/ScopeGuard.h>
+#include <LibC/bits/pthread_integration.h>
 #include <LibC/link.h>
 #include <LibC/mman.h>
 #include <LibC/unistd.h>
+#include <LibDl/dlfcn.h>
+#include <LibDl/dlfcn_integration.h>
 #include <LibELF/AuxiliaryVector.h>
 #include <LibELF/DynamicLinker.h>
 #include <LibELF/DynamicLoader.h>
 #include <LibELF/DynamicObject.h>
 #include <LibELF/Hashes.h>
-#include <dlfcn.h>
 #include <fcntl.h>
 #include <sys/types.h>
 #include <syscall.h>
 
 namespace ELF {
 
-namespace {
-HashMap<String, NonnullRefPtr<ELF::DynamicLoader>> g_loaders;
-Vector<NonnullRefPtr<ELF::DynamicObject>> g_global_objects;
+static HashMap<String, NonnullRefPtr<ELF::DynamicLoader>> s_loaders;
+static String s_main_program_name;
+static HashMap<String, NonnullRefPtr<ELF::DynamicObject>> s_global_objects;
 
 using EntryPointFunction = int (*)(int, char**, char**);
 using LibCExitFunction = void (*)(int);
 using DlIteratePhdrCallbackFunction = int (*)(struct dl_phdr_info*, size_t, void*);
 using DlIteratePhdrFunction = int (*)(DlIteratePhdrCallbackFunction, void*);
 
-size_t g_current_tls_offset = 0;
-size_t g_total_tls_size = 0;
-char** g_envp = nullptr;
-LibCExitFunction g_libc_exit = nullptr;
+static size_t s_current_tls_offset = 0;
+static size_t s_total_tls_size = 0;
+static char** s_envp = nullptr;
+static LibCExitFunction s_libc_exit = nullptr;
+static __pthread_mutex_t s_loader_lock = __PTHREAD_MUTEX_INITIALIZER;
 
-bool g_allowed_to_check_environment_variables { false };
-bool g_do_breakpoint_trap_before_entry { false };
-}
+static bool s_allowed_to_check_environment_variables { false };
+static bool s_do_breakpoint_trap_before_entry { false };
+
+static Result<void, DlErrorMessage> __dlclose(void* handle);
+static Result<void*, DlErrorMessage> __dlopen(const char* filename, int flags);
+static Result<void*, DlErrorMessage> __dlsym(void* handle, const char* symbol_name);
 
 Optional<DynamicObject::SymbolLookupResult> DynamicLinker::lookup_global_symbol(const StringView& name)
 {
@@ -51,8 +57,8 @@ Optional<DynamicObject::SymbolLookupResult> DynamicLinker::lookup_global_symbol(
 
     auto symbol = DynamicObject::HashSymbol { name };
 
-    for (auto& lib : g_global_objects) {
-        auto res = lib->lookup_symbol(symbol);
+    for (auto& lib : s_global_objects) {
+        auto res = lib.value->lookup_symbol(symbol);
         if (!res.has_value())
             continue;
         if (res.value().bind == STB_GLOBAL)
@@ -64,22 +70,39 @@ Optional<DynamicObject::SymbolLookupResult> DynamicLinker::lookup_global_symbol(
     return weak_result;
 }
 
-static void map_library(const String& name, int fd)
+static String get_library_name(String path)
 {
-    auto loader = ELF::DynamicLoader::try_create(fd, name);
-    if (!loader) {
-        dbgln("Failed to create ELF::DynamicLoader for fd={}, name={}", fd, name);
-        VERIFY_NOT_REACHED();
-    }
-    loader->set_tls_offset(g_current_tls_offset);
-
-    g_loaders.set(name, *loader);
-
-    g_current_tls_offset += loader->tls_size();
+    return LexicalPath(move(path)).basename();
 }
 
-static void map_library(const String& name)
+static Result<NonnullRefPtr<DynamicLoader>, DlErrorMessage> map_library(const String& filename, int fd, bool is_program)
 {
+    auto result = ELF::DynamicLoader::try_create(fd, filename);
+    if (result.is_error()) {
+        return result;
+    }
+
+    auto& loader = result.value();
+
+    s_loaders.set(get_library_name(filename), *loader);
+
+    if (is_program) {
+        loader->set_tls_offset(s_current_tls_offset);
+        s_current_tls_offset += loader->tls_size();
+    }
+
+    return loader;
+}
+
+static Result<NonnullRefPtr<DynamicLoader>, DlErrorMessage> map_library(const String& name, bool is_program)
+{
+    if (name.contains("/")) {
+        int fd = open(name.characters(), O_RDONLY);
+        if (fd < 0)
+            return DlErrorMessage { String::formatted("Could not open shared library: {}", name) };
+        return map_library(name, fd, is_program);
+    }
+
     // TODO: Do we want to also look for libs in other paths too?
     const char* search_paths[] = { "/usr/lib/{}", "/usr/local/lib/{}" };
     for (auto& search_path : search_paths) {
@@ -87,22 +110,15 @@ static void map_library(const String& name)
         int fd = open(path.characters(), O_RDONLY);
         if (fd < 0)
             continue;
-        map_library(name, fd);
-        return;
+        return map_library(name, fd, is_program);
     }
 
-    fprintf(stderr, "Could not find required shared library: %s\n", name.characters());
-    VERIFY_NOT_REACHED();
-}
-
-static String get_library_name(String path)
-{
-    return LexicalPath(move(path)).basename();
+    return DlErrorMessage { String::formatted("Could not find required shared library: {}", name) };
 }
 
 static Vector<String> get_dependencies(const String& name)
 {
-    auto lib = g_loaders.get(name).value();
+    auto lib = s_loaders.get(name).value();
     Vector<String> dependencies;
 
     lib->for_each_needed_library([&dependencies, &name](auto needed_name) {
@@ -114,7 +130,7 @@ static Vector<String> get_dependencies(const String& name)
     return dependencies;
 }
 
-static void map_dependencies(const String& name)
+static Result<void, DlErrorMessage> map_dependencies(const String& name)
 {
     dbgln_if(DYNAMIC_LOAD_DEBUG, "mapping dependencies for: {}", name);
 
@@ -122,18 +138,25 @@ static void map_dependencies(const String& name)
         dbgln_if(DYNAMIC_LOAD_DEBUG, "needed library: {}", needed_name.characters());
         String library_name = get_library_name(needed_name);
 
-        if (!g_loaders.contains(library_name)) {
-            map_library(library_name);
-            map_dependencies(library_name);
+        if (!s_loaders.contains(library_name) && !s_global_objects.contains(library_name)) {
+            auto result1 = map_library(needed_name, false);
+            if (result1.is_error()) {
+                return result1.error();
+            }
+            auto result2 = map_dependencies(library_name);
+            if (result2.is_error()) {
+                return result2.error();
+            }
         }
     }
     dbgln_if(DYNAMIC_LOAD_DEBUG, "mapped dependencies for {}", name);
+    return {};
 }
 
 static void allocate_tls()
 {
     size_t total_tls_size = 0;
-    for (const auto& data : g_loaders) {
+    for (const auto& data : s_loaders) {
         dbgln_if(DYNAMIC_LOAD_DEBUG, "{}: TLS Size: {}", data.key, data.value->tls_size());
         total_tls_size += data.value->tls_size();
     }
@@ -141,12 +164,16 @@ static void allocate_tls()
         [[maybe_unused]] void* tls_address = ::allocate_tls(total_tls_size);
         dbgln_if(DYNAMIC_LOAD_DEBUG, "from userspace, tls_address: {:p}", tls_address);
     }
-    g_total_tls_size = total_tls_size;
+    s_total_tls_size = total_tls_size;
 }
 
 static int __dl_iterate_phdr(DlIteratePhdrCallbackFunction callback, void* data)
 {
-    for (auto& object : g_global_objects) {
+    __pthread_mutex_lock(&s_loader_lock);
+    ScopeGuard unlock_guard = [] { __pthread_mutex_unlock(&s_loader_lock); };
+
+    for (auto& it : s_global_objects) {
+        auto& object = it.value;
         auto info = dl_phdr_info {
             .dlpi_addr = (ElfW(Addr))object->base_address().as_ptr(),
             .dlpi_name = object->filename().characters(),
@@ -171,7 +198,7 @@ static void initialize_libc(DynamicObject& libc)
     // because it uses getenv() internally, so `environ` has to be initialized before we call `__libc_init`.
     auto res = libc.lookup_symbol("environ"sv);
     VERIFY(res.has_value());
-    *((char***)res.value().address.as_ptr()) = g_envp;
+    *((char***)res.value().address.as_ptr()) = s_envp;
 
     res = libc.lookup_symbol("__environ_is_malloced"sv);
     VERIFY(res.has_value());
@@ -179,12 +206,24 @@ static void initialize_libc(DynamicObject& libc)
 
     res = libc.lookup_symbol("exit"sv);
     VERIFY(res.has_value());
-    g_libc_exit = (LibCExitFunction)res.value().address.as_ptr();
+    s_libc_exit = (LibCExitFunction)res.value().address.as_ptr();
 
     res = libc.lookup_symbol("__dl_iterate_phdr"sv);
     VERIFY(res.has_value());
     *((DlIteratePhdrFunction*)res.value().address.as_ptr()) = __dl_iterate_phdr;
 
+    res = libc.lookup_symbol("__dlclose"sv);
+    VERIFY(res.has_value());
+    *((DlCloseFunction*)res.value().address.as_ptr()) = __dlclose;
+
+    res = libc.lookup_symbol("__dlopen"sv);
+    VERIFY(res.has_value());
+    *((DlOpenFunction*)res.value().address.as_ptr()) = __dlopen;
+
+    res = libc.lookup_symbol("__dlsym"sv);
+    VERIFY(res.has_value());
+    *((DlSymFunction*)res.value().address.as_ptr()) = __dlsym;
+
     res = libc.lookup_symbol("__libc_init"sv);
     VERIFY(res.has_value());
     typedef void libc_init_func();
@@ -192,52 +231,56 @@ static void initialize_libc(DynamicObject& libc)
 }
 
 template<typename Callback>
-static void for_each_dependency_of(const String& name, HashTable<String>& seen_names, Callback callback)
+static void for_each_unfinished_dependency_of(const String& name, HashTable<String>& seen_names, Callback callback)
 {
+    if (!s_loaders.contains(name))
+        return;
+
     if (seen_names.contains(name))
         return;
     seen_names.set(name);
 
     for (const auto& needed_name : get_dependencies(name))
-        for_each_dependency_of(get_library_name(needed_name), seen_names, callback);
+        for_each_unfinished_dependency_of(get_library_name(needed_name), seen_names, callback);
 
-    callback(*g_loaders.get(name).value());
+    callback(*s_loaders.get(name).value());
 }
 
-static NonnullRefPtrVector<DynamicLoader> collect_loaders_for_executable(const String& name)
+static NonnullRefPtrVector<DynamicLoader> collect_loaders_for_library(const String& name)
 {
     HashTable<String> seen_names;
     NonnullRefPtrVector<DynamicLoader> loaders;
-    for_each_dependency_of(name, seen_names, [&](auto& loader) {
+    for_each_unfinished_dependency_of(name, seen_names, [&](auto& loader) {
         loaders.append(loader);
     });
     return loaders;
 }
 
-static NonnullRefPtr<DynamicLoader> load_main_executable(const String& name)
+static Result<NonnullRefPtr<DynamicLoader>, DlErrorMessage> load_main_library(const String& name, int flags)
 {
-    // NOTE: We always map the main executable first, since it may require
-    //       placement at a specific address.
-    auto& main_executable_loader = *g_loaders.get(name).value();
-    auto main_executable_object = main_executable_loader.map();
-    g_global_objects.append(*main_executable_object);
+    auto main_library_loader = *s_loaders.get(name);
+    auto main_library_object = main_library_loader->map();
+    s_global_objects.set(name, *main_library_object);
 
-    auto loaders = collect_loaders_for_executable(name);
+    auto loaders = collect_loaders_for_library(name);
 
     for (auto& loader : loaders) {
         auto dynamic_object = loader.map();
         if (dynamic_object)
-            g_global_objects.append(*dynamic_object);
+            s_global_objects.set(dynamic_object->filename(), *dynamic_object);
     }
 
     for (auto& loader : loaders) {
-        bool success = loader.link(RTLD_GLOBAL | RTLD_LAZY, g_total_tls_size);
-        VERIFY(success);
+        bool success = loader.link(flags, s_total_tls_size);
+        if (!success) {
+            return DlErrorMessage { String::formatted("Failed to link library {}", loader.filename()) };
+        }
     }
 
     for (auto& loader : loaders) {
-        auto object = loader.load_stage_3(RTLD_GLOBAL | RTLD_LAZY, g_total_tls_size);
-        VERIFY(object);
+        auto result = loader.load_stage_3(flags, s_total_tls_size);
+        VERIFY(!result.is_error());
+        auto& object = result.value();
 
         if (loader.filename() == "libsystem.so") {
             if (syscall(SC_msyscall, object->base_address().as_ptr())) {
@@ -254,45 +297,150 @@ static NonnullRefPtr<DynamicLoader> load_main_executable(const String& name)
         loader.load_stage_4();
     }
 
-    return main_executable_loader;
+    return main_library_loader;
+}
+
+static Result<void, DlErrorMessage> __dlclose(void* handle)
+{
+    dbgln_if(DYNAMIC_LOAD_DEBUG, "__dlclose: {}", handle);
+
+    __pthread_mutex_lock(&s_loader_lock);
+    ScopeGuard unlock_guard = [] { __pthread_mutex_unlock(&s_loader_lock); };
+
+    // FIXME: this will not currently destroy the dynamic object
+    // because we're intentionally holding a strong reference to it
+    // via s_global_objects until there's proper unload support.
+    auto object = static_cast<ELF::DynamicObject*>(handle);
+    object->unref();
+    return {};
+}
+
+static Result<void*, DlErrorMessage> __dlopen(const char* filename, int flags)
+{
+    // FIXME: RTLD_NOW and RTLD_LOCAL are not supported
+    flags &= ~RTLD_NOW;
+    flags |= RTLD_LAZY;
+    flags &= ~RTLD_LOCAL;
+    flags |= RTLD_GLOBAL;
+
+    dbgln_if(DYNAMIC_LOAD_DEBUG, "__dlopen invoked, filename={}, flags={}", filename, flags);
+
+    auto library_name = get_library_name(filename ? filename : s_main_program_name);
+
+    if (__pthread_mutex_trylock(&s_loader_lock) != 0)
+        return DlErrorMessage { "Nested calls to dlopen() are not permitted." };
+    ScopeGuard unlock_guard = [] { __pthread_mutex_unlock(&s_loader_lock); };
+
+    auto existing_elf_object = s_global_objects.get(library_name);
+    if (existing_elf_object.has_value()) {
+        // It's up to the caller to release the ref with dlclose().
+        return &existing_elf_object->leak_ref();
+    }
+
+    VERIFY(!library_name.is_empty());
+
+    auto result1 = map_library(filename, false);
+    if (result1.is_error()) {
+        return result1.error();
+    }
+
+    auto result2 = map_dependencies(library_name);
+    if (result2.is_error()) {
+        return result2.error();
+    }
+
+    auto result = load_main_library(library_name, flags);
+    if (result.is_error())
+        return result.error();
+
+    auto object = s_global_objects.get(library_name);
+    if (!object.has_value())
+        return DlErrorMessage { "Could not load ELF object." };
+
+    // It's up to the caller to release the ref with dlclose().
+    return &object->leak_ref();
+}
+
+static Result<void*, DlErrorMessage> __dlsym(void* handle, const char* symbol_name)
+{
+    dbgln_if(DYNAMIC_LOAD_DEBUG, "__dlsym: {}, {}", handle, symbol_name);
+
+    __pthread_mutex_lock(&s_loader_lock);
+    ScopeGuard unlock_guard = [] { __pthread_mutex_unlock(&s_loader_lock); };
+
+    auto object = static_cast<DynamicObject*>(handle);
+    if (!handle) {
+        auto library_name = get_library_name(s_main_program_name);
+        auto global_object = s_global_objects.get(library_name);
+        object = *global_object;
+    }
+    auto symbol = object->lookup_symbol(symbol_name);
+    if (!symbol.has_value()) {
+        return DlErrorMessage { String::formatted("Symbol {} not found", symbol_name) };
+    }
+    return symbol.value().address.as_ptr();
 }
 
 static void read_environment_variables()
 {
-    for (char** env = g_envp; *env; ++env) {
+    for (char** env = s_envp; *env; ++env) {
         if (StringView { *env } == "_LOADER_BREAKPOINT=1") {
-            g_do_breakpoint_trap_before_entry = true;
+            s_do_breakpoint_trap_before_entry = true;
         }
     }
 }
 
 void ELF::DynamicLinker::linker_main(String&& main_program_name, int main_program_fd, bool is_secure, int argc, char** argv, char** envp)
 {
-    g_envp = envp;
+    s_envp = envp;
 
-    g_allowed_to_check_environment_variables = !is_secure;
-    if (g_allowed_to_check_environment_variables)
+    s_allowed_to_check_environment_variables = !is_secure;
+    if (s_allowed_to_check_environment_variables)
         read_environment_variables();
 
-    map_library(main_program_name, main_program_fd);
-    map_dependencies(main_program_name);
+    s_main_program_name = main_program_name;
+
+    auto library_name = get_library_name(main_program_name);
+
+    // NOTE: We always map the main library first, since it may require
+    //       placement at a specific address.
+    auto result1 = map_library(main_program_name, main_program_fd, true);
+    if (result1.is_error()) {
+        warnln("{}", result1.error().text);
+        fflush(stderr);
+        _exit(1);
+    }
+    result1.release_value();
+
+    auto result2 = map_dependencies(library_name);
+    if (result2.is_error()) {
+        warnln("{}", result2.error().text);
+        fflush(stderr);
+        _exit(1);
+    }
 
     dbgln_if(DYNAMIC_LOAD_DEBUG, "loaded all dependencies");
-    for ([[maybe_unused]] auto& lib : g_loaders) {
+    for ([[maybe_unused]] auto& lib : s_loaders) {
         dbgln_if(DYNAMIC_LOAD_DEBUG, "{} - tls size: {}, tls offset: {}", lib.key, lib.value->tls_size(), lib.value->tls_offset());
     }
 
     allocate_tls();
 
     auto entry_point_function = [&main_program_name] {
-        auto main_executable_loader = load_main_executable(main_program_name);
+        auto library_name = get_library_name(main_program_name);
+        auto result = load_main_library(library_name, RTLD_GLOBAL | RTLD_LAZY);
+        if (result.is_error()) {
+            warnln("{}", result.error().text);
+            _exit(1);
+        }
+        auto& main_executable_loader = result.value();
         auto entry_point = main_executable_loader->image().entry();
         if (main_executable_loader->is_dynamic())
             entry_point = entry_point.offset(main_executable_loader->base_address().get());
         return (EntryPointFunction)(entry_point.as_ptr());
     }();
 
-    g_loaders.clear();
+    s_loaders.clear();
 
     int rc = syscall(SC_msyscall, nullptr);
     if (rc < 0) {
@@ -300,13 +448,13 @@ void ELF::DynamicLinker::linker_main(String&& main_program_name, int main_progra
     }
 
     dbgln_if(DYNAMIC_LOAD_DEBUG, "Jumping to entry point: {:p}", entry_point_function);
-    if (g_do_breakpoint_trap_before_entry) {
+    if (s_do_breakpoint_trap_before_entry) {
         asm("int3");
     }
     rc = entry_point_function(argc, argv, envp);
     dbgln_if(DYNAMIC_LOAD_DEBUG, "rc: {}", rc);
-    if (g_libc_exit != nullptr) {
-        g_libc_exit(rc);
+    if (s_libc_exit != nullptr) {
+        s_libc_exit(rc);
     } else {
         _exit(rc);
     }

Userland/Libraries/LibELF/DynamicLoader.cpp

@@ -10,12 +10,14 @@
 #include <AK/Optional.h>
 #include <AK/QuickSort.h>
 #include <AK/StringBuilder.h>
+#include <LibDl/dlfcn.h>
+#include <LibDl/dlfcn_integration.h>
 #include <LibELF/DynamicLinker.h>
 #include <LibELF/DynamicLoader.h>
 #include <LibELF/Hashes.h>
 #include <LibELF/Validation.h>
 #include <assert.h>
-#include <dlfcn.h>
+#include <errno.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
@@ -34,24 +36,22 @@ static void* mmap_with_name(void* addr, size_t length, int prot, int flags, int
 
 namespace ELF {
 
-RefPtr<DynamicLoader> DynamicLoader::try_create(int fd, String filename)
+Result<NonnullRefPtr<DynamicLoader>, DlErrorMessage> DynamicLoader::try_create(int fd, String filename)
 {
     struct stat stat;
     if (fstat(fd, &stat) < 0) {
-        perror("DynamicLoader::try_create fstat");
-        return {};
+        return DlErrorMessage { "DynamicLoader::try_create fstat" };
     }
 
     VERIFY(stat.st_size >= 0);
     auto size = static_cast<size_t>(stat.st_size);
     if (size < sizeof(Elf32_Ehdr))
-        return {};
+        return DlErrorMessage { String::formatted("File {} has invalid ELF header", filename) };
 
     String file_mmap_name = String::formatted("ELF_DYN: {}", filename);
     auto* data = mmap_with_name(nullptr, size, PROT_READ, MAP_PRIVATE, fd, 0, file_mmap_name.characters());
     if (data == MAP_FAILED) {
-        perror("DynamicLoader::try_create mmap");
-        return {};
+        return DlErrorMessage { "DynamicLoader::try_create mmap" };
     }
 
     return adopt_ref(*new DynamicLoader(fd, move(filename), data, size));
@@ -123,17 +123,6 @@ bool DynamicLoader::validate()
     return true;
 }
 
-void* DynamicLoader::symbol_for_name(const StringView& name)
-{
-    auto result = m_dynamic_object->hash_section().lookup_symbol(name);
-    if (!result.has_value())
-        return nullptr;
-    auto symbol = result.value();
-    if (symbol.is_undefined())
-        return nullptr;
-    return m_dynamic_object->base_address().offset(symbol.value()).as_ptr();
-}
-
 RefPtr<DynamicObject> DynamicLoader::map()
 {
     if (m_dynamic_object) {
@@ -207,7 +196,7 @@ void DynamicLoader::do_main_relocations(size_t total_tls_size)
     m_dynamic_object->plt_relocation_section().for_each_relocation(do_single_relocation);
 }
 
-RefPtr<DynamicObject> DynamicLoader::load_stage_3(unsigned flags, size_t total_tls_size)
+Result<NonnullRefPtr<DynamicObject>, DlErrorMessage> DynamicLoader::load_stage_3(unsigned flags, size_t total_tls_size)
 {
     do_lazy_relocations(total_tls_size);
     if (flags & RTLD_LAZY) {
@@ -217,26 +206,23 @@ RefPtr<DynamicObject> DynamicLoader::load_stage_3(unsigned flags, size_t total_t
 
     for (auto& text_segment : m_text_segments) {
         if (mprotect(text_segment.address().as_ptr(), text_segment.size(), PROT_READ | PROT_EXEC) < 0) {
-            perror("mprotect .text: PROT_READ | PROT_EXEC"); // FIXME: dlerror?
-            return nullptr;
+            return DlErrorMessage { String::formatted("mprotect .text: PROT_READ | PROT_EXEC: {}", strerror(errno)) };
         }
     }
 
     if (m_relro_segment_size) {
         if (mprotect(m_relro_segment_address.as_ptr(), m_relro_segment_size, PROT_READ) < 0) {
-            perror("mprotect .relro: PROT_READ");
-            return nullptr;
+            return DlErrorMessage { String::formatted("mprotect .text: PROT_READ: {}", strerror(errno)) };
         }
 
 #if __serenity__
         if (set_mmap_name(m_relro_segment_address.as_ptr(), m_relro_segment_size, String::formatted("{}: .relro", m_filename).characters()) < 0) {
-            perror("set_mmap_name .relro");
-            return nullptr;
+            return DlErrorMessage { String::formatted("set_mmap_name .relro: {}", strerror(errno)) };
         }
 #endif
     }
 
-    return m_dynamic_object;
+    return NonnullRefPtr<DynamicObject> { *m_dynamic_object };
 }
 
 void DynamicLoader::load_stage_4()

Userland/Libraries/LibELF/DynamicLoader.h

@@ -12,6 +12,7 @@
 #include <AK/RefCounted.h>
 #include <AK/String.h>
 #include <LibC/elf.h>
+#include <LibDl/dlfcn_integration.h>
 #include <LibELF/DynamicObject.h>
 #include <LibELF/Image.h>
 #include <sys/mman.h>
@@ -41,7 +42,7 @@ enum class ShouldInitializeWeak {
 
 class DynamicLoader : public RefCounted<DynamicLoader> {
 public:
-    static RefPtr<DynamicLoader> try_create(int fd, String filename);
+    static Result<NonnullRefPtr<DynamicLoader>, DlErrorMessage> try_create(int fd, String filename);
     ~DynamicLoader();
 
     const String& filename() const { return m_filename; }
@@ -59,14 +60,11 @@ public:
     bool load_stage_2(unsigned flags, size_t total_tls_size);
 
     // Stage 3 of loading: lazy relocations
-    RefPtr<DynamicObject> load_stage_3(unsigned flags, size_t total_tls_size);
+    Result<NonnullRefPtr<DynamicObject>, DlErrorMessage> load_stage_3(unsigned flags, size_t total_tls_size);
 
     // Stage 4 of loading: initializers
     void load_stage_4();
 
-    // Intended for use by dlsym or other internal methods
-    void* symbol_for_name(const StringView&);
-
     void set_tls_offset(size_t offset) { m_tls_offset = offset; };
     size_t tls_size() const { return m_tls_size; }
     size_t tls_offset() const { return m_tls_offset; }

Userland/Libraries/LibELF/DynamicObject.cpp

@@ -36,6 +36,7 @@ DynamicObject::DynamicObject(const String& filename, VirtualAddress base_address
 
 DynamicObject::~DynamicObject()
 {
+    // TODO: unmap the object
 }
 
 void DynamicObject::dump() const
@@ -482,4 +483,15 @@ u32 DynamicObject::HashSymbol::sysv_hash() const
         m_sysv_hash = compute_sysv_hash(m_name);
     return m_sysv_hash.value();
 }
+
+void* DynamicObject::symbol_for_name(const StringView& name)
+{
+    auto result = hash_section().lookup_symbol(name);
+    if (!result.has_value())
+        return nullptr;
+    auto symbol = result.value();
+    if (symbol.is_undefined())
+        return nullptr;
+    return base_address().offset(symbol.value()).as_ptr();
+}
 } // end namespace ELF

Userland/Libraries/LibELF/DynamicObject.h

@@ -277,6 +277,8 @@ public:
 
     bool elf_is_dynamic() const { return m_is_elf_dynamic; }
 
+    void* symbol_for_name(const StringView& name);
+
 private:
     explicit DynamicObject(const String& filename, VirtualAddress base_address, VirtualAddress dynamic_section_address);
 

Userland/Utilities/readelf.cpp

@@ -452,19 +452,24 @@ int main(int argc, char** argv)
 
         int fd = open(path, O_RDONLY);
         if (fd < 0) {
-            outln(String::formatted("Unable to open file {}", path).characters());
+            outln("Unable to open file {}", path);
             return 1;
         }
 
-        auto loader = ELF::DynamicLoader::try_create(fd, path);
-        if (!loader || !loader->is_valid()) {
-            outln(String::formatted("{} is not a valid ELF dynamic shared object!", path));
+        auto result = ELF::DynamicLoader::try_create(fd, path);
+        if (result.is_error()) {
+            outln("{}", result.error().text);
+            return 1;
+        }
+        auto& loader = result.value();
+        if (!loader->is_valid()) {
+            outln("{} is not a valid ELF dynamic shared object!", path);
             return 1;
         }
 
         object = loader->map();
         if (!object) {
-            outln(String::formatted("Failed to map dynamic ELF object {}", path));
+            outln("Failed to map dynamic ELF object {}", path);
             return 1;
         }
     }