From 273a6eeb66ed1ff2fcc1ee3a6eaae4eedf437875 Mon Sep 17 00:00:00 2001 From: Juan Linietsky Date: Fri, 28 Apr 2023 13:15:36 +0200 Subject: [PATCH] Redo how the remote filesystem works Instead of reading files over the network, the new version uses a local file cache and only updates files when it changes. The original remote filesystem was created 14 years ago, when ethernet was faster than hard drives or even flash. Also, mobile devices have a very small amount of storage. Nowadays, this is no longer the case so the approach is changed to using a persistent cache in the target device. Co-authored-by: m4gr3d --- core/config/project_settings.cpp | 12 - core/io/file_access_network.cpp | 498 ---------------------- core/io/file_access_network.h | 167 -------- core/io/remote_filesystem_client.cpp | 329 ++++++++++++++ core/io/remote_filesystem_client.h | 65 +++ core/os/os.cpp | 16 +- core/os/os.h | 6 + core/string/print_string.cpp | 6 +- core/string/print_string.h | 10 +- core/variant/variant_utility.cpp | 2 + doc/classes/ProjectSettings.xml | 6 - editor/debugger/editor_file_server.cpp | 446 +++++++++---------- editor/debugger/editor_file_server.h | 30 +- editor/export/editor_export_platform.cpp | 108 +++-- editor/export/editor_export_platform.h | 2 + editor/plugins/debugger_editor_plugin.cpp | 6 + main/main.cpp | 14 +- platform/android/export/export_plugin.cpp | 6 +- platform/android/os_android.cpp | 16 +- platform/android/os_android.h | 3 + platform/linuxbsd/os_linuxbsd.cpp | 4 + platform/linuxbsd/os_linuxbsd.h | 1 + 22 files changed, 714 insertions(+), 1039 deletions(-) delete mode 100644 core/io/file_access_network.cpp delete mode 100644 core/io/file_access_network.h create mode 100644 core/io/remote_filesystem_client.cpp create mode 100644 core/io/remote_filesystem_client.h diff --git a/core/config/project_settings.cpp b/core/config/project_settings.cpp index 9fd7ef998876..264259eb2f32 100644 --- a/core/config/project_settings.cpp +++ b/core/config/project_settings.cpp @@ -36,7 +36,6 @@ #include "core/io/config_file.h" #include "core/io/dir_access.h" #include "core/io/file_access.h" -#include "core/io/file_access_network.h" #include "core/io/file_access_pack.h" #include "core/io/marshalls.h" #include "core/os/keyboard.h" @@ -502,17 +501,6 @@ Error ProjectSettings::_setup(const String &p_path, const String &p_main_pack, b } } - // If looking for files in a network client, use it directly - - if (FileAccessNetworkClient::get_singleton()) { - Error err = _load_settings_text_or_binary("res://project.godot", "res://project.binary"); - if (err == OK && !p_ignore_override) { - // Optional, we don't mind if it fails - _load_settings_text("res://override.cfg"); - } - return err; - } - // Attempt with a user-defined main pack first if (!p_main_pack.is_empty()) { diff --git a/core/io/file_access_network.cpp b/core/io/file_access_network.cpp deleted file mode 100644 index 7fabff26ac55..000000000000 --- a/core/io/file_access_network.cpp +++ /dev/null @@ -1,498 +0,0 @@ -/**************************************************************************/ -/* file_access_network.cpp */ -/**************************************************************************/ -/* This file is part of: */ -/* GODOT ENGINE */ -/* https://godotengine.org */ -/**************************************************************************/ -/* Copyright (c) 2014-present Godot Engine contributors (see AUTHORS.md). */ -/* Copyright (c) 2007-2014 Juan Linietsky, Ariel Manzur. */ -/* */ -/* Permission is hereby granted, free of charge, to any person obtaining */ -/* a copy of this software and associated documentation files (the */ -/* "Software"), to deal in the Software without restriction, including */ -/* without limitation the rights to use, copy, modify, merge, publish, */ -/* distribute, sublicense, and/or sell copies of the Software, and to */ -/* permit persons to whom the Software is furnished to do so, subject to */ -/* the following conditions: */ -/* */ -/* The above copyright notice and this permission notice shall be */ -/* included in all copies or substantial portions of the Software. */ -/* */ -/* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, */ -/* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF */ -/* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. */ -/* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY */ -/* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, */ -/* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE */ -/* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -/**************************************************************************/ - -#include "file_access_network.h" - -#include "core/config/project_settings.h" -#include "core/io/ip.h" -#include "core/io/marshalls.h" -#include "core/os/os.h" - -//#define DEBUG_PRINT(m_p) print_line(m_p) -//#define DEBUG_TIME(m_what) printf("MS: %s - %lli\n",m_what,OS::get_singleton()->get_ticks_usec()); -#define DEBUG_PRINT(m_p) -#define DEBUG_TIME(m_what) - -void FileAccessNetworkClient::lock_mutex() { - mutex.lock(); - lockcount++; -} - -void FileAccessNetworkClient::unlock_mutex() { - lockcount--; - mutex.unlock(); -} - -void FileAccessNetworkClient::put_32(int p_32) { - uint8_t buf[4]; - encode_uint32(p_32, buf); - client->put_data(buf, 4); - DEBUG_PRINT("put32: " + itos(p_32)); -} - -void FileAccessNetworkClient::put_64(int64_t p_64) { - uint8_t buf[8]; - encode_uint64(p_64, buf); - client->put_data(buf, 8); - DEBUG_PRINT("put64: " + itos(p_64)); -} - -int FileAccessNetworkClient::get_32() { - uint8_t buf[4]; - client->get_data(buf, 4); - return decode_uint32(buf); -} - -int64_t FileAccessNetworkClient::get_64() { - uint8_t buf[8]; - client->get_data(buf, 8); - return decode_uint64(buf); -} - -void FileAccessNetworkClient::_thread_func() { - client->set_no_delay(true); - while (!quit) { - DEBUG_PRINT("SEM WAIT - " + itos(sem->get())); - sem.wait(); - DEBUG_TIME("sem_unlock"); - //DEBUG_PRINT("semwait returned "+itos(werr)); - DEBUG_PRINT("MUTEX LOCK " + itos(lockcount)); - lock_mutex(); - DEBUG_PRINT("MUTEX PASS"); - - { - MutexLock lock(blockrequest_mutex); - while (block_requests.size()) { - put_32(block_requests.front()->get().id); - put_32(FileAccessNetwork::COMMAND_READ_BLOCK); - put_64(block_requests.front()->get().offset); - put_32(block_requests.front()->get().size); - block_requests.pop_front(); - } - } - - DEBUG_PRINT("THREAD ITER"); - - DEBUG_TIME("sem_read"); - int id = get_32(); - - int response = get_32(); - DEBUG_PRINT("GET RESPONSE: " + itos(response)); - - FileAccessNetwork *fa = nullptr; - - if (response != FileAccessNetwork::RESPONSE_DATA) { - if (!accesses.has(id)) { - unlock_mutex(); - ERR_FAIL_COND(!accesses.has(id)); - } - } - - if (accesses.has(id)) { - fa = accesses[id]; - } - - switch (response) { - case FileAccessNetwork::RESPONSE_OPEN: { - DEBUG_TIME("sem_open"); - int status = get_32(); - if (status != OK) { - fa->_respond(0, Error(status)); - } else { - int64_t len = get_64(); - fa->_respond(len, Error(status)); - } - - fa->sem.post(); - - } break; - case FileAccessNetwork::RESPONSE_DATA: { - int64_t offset = get_64(); - int32_t len = get_32(); - - Vector resp_block; - resp_block.resize(len); - client->get_data(resp_block.ptrw(), len); - - if (fa) { //may have been queued - fa->_set_block(offset, resp_block); - } - - } break; - case FileAccessNetwork::RESPONSE_FILE_EXISTS: { - int status = get_32(); - fa->exists_modtime = status != 0; - fa->sem.post(); - - } break; - case FileAccessNetwork::RESPONSE_GET_MODTIME: { - uint64_t status = get_64(); - fa->exists_modtime = status; - fa->sem.post(); - - } break; - } - - unlock_mutex(); - } -} - -void FileAccessNetworkClient::_thread_func(void *s) { - FileAccessNetworkClient *self = static_cast(s); - - self->_thread_func(); -} - -Error FileAccessNetworkClient::connect(const String &p_host, int p_port, const String &p_password) { - IPAddress ip; - - if (p_host.is_valid_ip_address()) { - ip = p_host; - } else { - ip = IP::get_singleton()->resolve_hostname(p_host); - } - - DEBUG_PRINT("IP: " + String(ip) + " port " + itos(p_port)); - Error err = client->connect_to_host(ip, p_port); - ERR_FAIL_COND_V_MSG(err != OK, err, "Cannot connect to host with IP: " + String(ip) + " and port: " + itos(p_port)); - while (client->get_status() == StreamPeerTCP::STATUS_CONNECTING) { - //DEBUG_PRINT("trying to connect...."); - OS::get_singleton()->delay_usec(1000); - } - - if (client->get_status() != StreamPeerTCP::STATUS_CONNECTED) { - return ERR_CANT_CONNECT; - } - - CharString cs = p_password.utf8(); - put_32(cs.length()); - client->put_data((const uint8_t *)cs.ptr(), cs.length()); - - int e = get_32(); - - if (e != OK) { - return ERR_INVALID_PARAMETER; - } - - thread.start(_thread_func, this); - - return OK; -} - -FileAccessNetworkClient *FileAccessNetworkClient::singleton = nullptr; - -FileAccessNetworkClient::FileAccessNetworkClient() { - singleton = this; - client.instantiate(); -} - -FileAccessNetworkClient::~FileAccessNetworkClient() { - quit = true; - sem.post(); - thread.wait_to_finish(); -} - -void FileAccessNetwork::_set_block(uint64_t p_offset, const Vector &p_block) { - int32_t page = p_offset / page_size; - ERR_FAIL_INDEX(page, pages.size()); - if (page < pages.size() - 1) { - ERR_FAIL_COND(p_block.size() != page_size); - } else { - ERR_FAIL_COND((uint64_t)p_block.size() != total_size % page_size); - } - - { - MutexLock lock(buffer_mutex); - pages.write[page].buffer = p_block; - pages.write[page].queued = false; - } - - if (waiting_on_page == page) { - waiting_on_page = -1; - page_sem.post(); - } -} - -void FileAccessNetwork::_respond(uint64_t p_len, Error p_status) { - DEBUG_PRINT("GOT RESPONSE - len: " + itos(p_len) + " status: " + itos(p_status)); - response = p_status; - if (response != OK) { - return; - } - opened = true; - total_size = p_len; - int32_t pc = ((total_size - 1) / page_size) + 1; - pages.resize(pc); -} - -Error FileAccessNetwork::open_internal(const String &p_path, int p_mode_flags) { - ERR_FAIL_COND_V(p_mode_flags != READ, ERR_UNAVAILABLE); - _close(); - - FileAccessNetworkClient *nc = FileAccessNetworkClient::singleton; - DEBUG_PRINT("open: " + p_path); - - DEBUG_TIME("open_begin"); - - nc->lock_mutex(); - nc->put_32(id); - nc->accesses[id] = this; - nc->put_32(COMMAND_OPEN_FILE); - CharString cs = p_path.utf8(); - nc->put_32(cs.length()); - nc->client->put_data((const uint8_t *)cs.ptr(), cs.length()); - pos = 0; - eof_flag = false; - last_page = -1; - last_page_buff = nullptr; - - //buffers.clear(); - nc->unlock_mutex(); - DEBUG_PRINT("OPEN POST"); - DEBUG_TIME("open_post"); - nc->sem.post(); //awaiting answer - DEBUG_PRINT("WAIT..."); - sem.wait(); - DEBUG_TIME("open_end"); - DEBUG_PRINT("WAIT ENDED..."); - - return response; -} - -void FileAccessNetwork::_close() { - if (!opened) { - return; - } - - FileAccessNetworkClient *nc = FileAccessNetworkClient::singleton; - - DEBUG_PRINT("CLOSE"); - nc->lock_mutex(); - nc->put_32(id); - nc->put_32(COMMAND_CLOSE); - pages.clear(); - opened = false; - nc->unlock_mutex(); -} - -bool FileAccessNetwork::is_open() const { - return opened; -} - -void FileAccessNetwork::seek(uint64_t p_position) { - ERR_FAIL_COND_MSG(!opened, "File must be opened before use."); - - eof_flag = p_position > total_size; - - if (p_position >= total_size) { - p_position = total_size; - } - - pos = p_position; -} - -void FileAccessNetwork::seek_end(int64_t p_position) { - seek(total_size + p_position); -} - -uint64_t FileAccessNetwork::get_position() const { - ERR_FAIL_COND_V_MSG(!opened, 0, "File must be opened before use."); - return pos; -} - -uint64_t FileAccessNetwork::get_length() const { - ERR_FAIL_COND_V_MSG(!opened, 0, "File must be opened before use."); - return total_size; -} - -bool FileAccessNetwork::eof_reached() const { - ERR_FAIL_COND_V_MSG(!opened, false, "File must be opened before use."); - return eof_flag; -} - -uint8_t FileAccessNetwork::get_8() const { - uint8_t v; - get_buffer(&v, 1); - return v; -} - -void FileAccessNetwork::_queue_page(int32_t p_page) const { - if (p_page >= pages.size()) { - return; - } - if (pages[p_page].buffer.is_empty() && !pages[p_page].queued) { - FileAccessNetworkClient *nc = FileAccessNetworkClient::singleton; - { - MutexLock lock(nc->blockrequest_mutex); - - FileAccessNetworkClient::BlockRequest br; - br.id = id; - br.offset = (uint64_t)p_page * page_size; - br.size = page_size; - nc->block_requests.push_back(br); - pages.write[p_page].queued = true; - } - DEBUG_PRINT("QUEUE PAGE POST"); - nc->sem.post(); - DEBUG_PRINT("queued " + itos(p_page)); - } -} - -uint64_t FileAccessNetwork::get_buffer(uint8_t *p_dst, uint64_t p_length) const { - ERR_FAIL_COND_V(!p_dst && p_length > 0, -1); - - if (pos + p_length > total_size) { - eof_flag = true; - } - if (pos + p_length >= total_size) { - p_length = total_size - pos; - } - - uint8_t *buff = last_page_buff; - - for (uint64_t i = 0; i < p_length; i++) { - int32_t page = pos / page_size; - - if (page != last_page) { - buffer_mutex.lock(); - if (pages[page].buffer.is_empty()) { - waiting_on_page = page; - for (int32_t j = 0; j < read_ahead; j++) { - _queue_page(page + j); - } - buffer_mutex.unlock(); - DEBUG_PRINT("wait"); - page_sem.wait(); - DEBUG_PRINT("done"); - } else { - for (int32_t j = 0; j < read_ahead; j++) { - _queue_page(page + j); - } - buffer_mutex.unlock(); - } - - buff = pages.write[page].buffer.ptrw(); - last_page_buff = buff; - last_page = page; - } - - p_dst[i] = buff[pos - uint64_t(page) * page_size]; - pos++; - } - - return p_length; -} - -Error FileAccessNetwork::get_error() const { - return pos == total_size ? ERR_FILE_EOF : OK; -} - -void FileAccessNetwork::flush() { - ERR_FAIL(); -} - -void FileAccessNetwork::store_8(uint8_t p_dest) { - ERR_FAIL(); -} - -bool FileAccessNetwork::file_exists(const String &p_path) { - FileAccessNetworkClient *nc = FileAccessNetworkClient::singleton; - nc->lock_mutex(); - nc->put_32(id); - nc->put_32(COMMAND_FILE_EXISTS); - CharString cs = p_path.utf8(); - nc->put_32(cs.length()); - nc->client->put_data((const uint8_t *)cs.ptr(), cs.length()); - nc->unlock_mutex(); - DEBUG_PRINT("FILE EXISTS POST"); - nc->sem.post(); - sem.wait(); - - return exists_modtime != 0; -} - -uint64_t FileAccessNetwork::_get_modified_time(const String &p_file) { - FileAccessNetworkClient *nc = FileAccessNetworkClient::singleton; - nc->lock_mutex(); - nc->put_32(id); - nc->put_32(COMMAND_GET_MODTIME); - CharString cs = p_file.utf8(); - nc->put_32(cs.length()); - nc->client->put_data((const uint8_t *)cs.ptr(), cs.length()); - nc->unlock_mutex(); - DEBUG_PRINT("MODTIME POST"); - nc->sem.post(); - sem.wait(); - - return exists_modtime; -} - -uint32_t FileAccessNetwork::_get_unix_permissions(const String &p_file) { - ERR_PRINT("Getting UNIX permissions from network drives is not implemented yet"); - return 0; -} - -Error FileAccessNetwork::_set_unix_permissions(const String &p_file, uint32_t p_permissions) { - ERR_PRINT("Setting UNIX permissions on network drives is not implemented yet"); - return ERR_UNAVAILABLE; -} - -void FileAccessNetwork::configure() { - GLOBAL_DEF(PropertyInfo(Variant::INT, "network/remote_fs/page_size", PROPERTY_HINT_RANGE, "1,65536,1,or_greater"), 65536); // Is used as denominator and can't be zero - GLOBAL_DEF(PropertyInfo(Variant::INT, "network/remote_fs/page_read_ahead", PROPERTY_HINT_RANGE, "0,8,1,or_greater"), 4); -} - -void FileAccessNetwork::close() { - _close(); - - FileAccessNetworkClient *nc = FileAccessNetworkClient::singleton; - nc->lock_mutex(); - nc->accesses.erase(id); - nc->unlock_mutex(); -} - -FileAccessNetwork::FileAccessNetwork() { - FileAccessNetworkClient *nc = FileAccessNetworkClient::singleton; - nc->lock_mutex(); - id = nc->last_id++; - nc->accesses[id] = this; - nc->unlock_mutex(); - page_size = GLOBAL_GET("network/remote_fs/page_size"); - read_ahead = GLOBAL_GET("network/remote_fs/page_read_ahead"); -} - -FileAccessNetwork::~FileAccessNetwork() { - _close(); - - FileAccessNetworkClient *nc = FileAccessNetworkClient::singleton; - nc->lock_mutex(); - nc->accesses.erase(id); - nc->unlock_mutex(); -} diff --git a/core/io/file_access_network.h b/core/io/file_access_network.h deleted file mode 100644 index 78c19347ce3b..000000000000 --- a/core/io/file_access_network.h +++ /dev/null @@ -1,167 +0,0 @@ -/**************************************************************************/ -/* file_access_network.h */ -/**************************************************************************/ -/* This file is part of: */ -/* GODOT ENGINE */ -/* https://godotengine.org */ -/**************************************************************************/ -/* Copyright (c) 2014-present Godot Engine contributors (see AUTHORS.md). */ -/* Copyright (c) 2007-2014 Juan Linietsky, Ariel Manzur. */ -/* */ -/* Permission is hereby granted, free of charge, to any person obtaining */ -/* a copy of this software and associated documentation files (the */ -/* "Software"), to deal in the Software without restriction, including */ -/* without limitation the rights to use, copy, modify, merge, publish, */ -/* distribute, sublicense, and/or sell copies of the Software, and to */ -/* permit persons to whom the Software is furnished to do so, subject to */ -/* the following conditions: */ -/* */ -/* The above copyright notice and this permission notice shall be */ -/* included in all copies or substantial portions of the Software. */ -/* */ -/* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, */ -/* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF */ -/* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. */ -/* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY */ -/* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, */ -/* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE */ -/* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -/**************************************************************************/ - -#ifndef FILE_ACCESS_NETWORK_H -#define FILE_ACCESS_NETWORK_H - -#include "core/io/file_access.h" -#include "core/io/stream_peer_tcp.h" -#include "core/os/semaphore.h" -#include "core/os/thread.h" - -class FileAccessNetwork; - -class FileAccessNetworkClient { - struct BlockRequest { - int32_t id; - uint64_t offset; - int32_t size; - }; - - List block_requests; - - Semaphore sem; - Thread thread; - bool quit = false; - Mutex mutex; - Mutex blockrequest_mutex; - HashMap accesses; - Ref client; - int32_t last_id = 0; - int32_t lockcount = 0; - - Vector block; - - void _thread_func(); - static void _thread_func(void *s); - - void put_32(int32_t p_32); - void put_64(int64_t p_64); - int32_t get_32(); - int64_t get_64(); - void lock_mutex(); - void unlock_mutex(); - - friend class FileAccessNetwork; - static FileAccessNetworkClient *singleton; - -public: - static FileAccessNetworkClient *get_singleton() { return singleton; } - - Error connect(const String &p_host, int p_port, const String &p_password = ""); - - FileAccessNetworkClient(); - ~FileAccessNetworkClient(); -}; - -class FileAccessNetwork : public FileAccess { - Semaphore sem; - Semaphore page_sem; - Mutex buffer_mutex; - bool opened = false; - uint64_t total_size = 0; - mutable uint64_t pos = 0; - int32_t id = -1; - mutable bool eof_flag = false; - mutable int32_t last_page = -1; - mutable uint8_t *last_page_buff = nullptr; - - int32_t page_size = 0; - int32_t read_ahead = 0; - - mutable int waiting_on_page = -1; - - struct Page { - int activity = 0; - bool queued = false; - Vector buffer; - }; - - mutable Vector pages; - - mutable Error response; - - uint64_t exists_modtime = 0; - - friend class FileAccessNetworkClient; - void _queue_page(int32_t p_page) const; - void _respond(uint64_t p_len, Error p_status); - void _set_block(uint64_t p_offset, const Vector &p_block); - void _close(); - -public: - enum Command { - COMMAND_OPEN_FILE, - COMMAND_READ_BLOCK, - COMMAND_CLOSE, - COMMAND_FILE_EXISTS, - COMMAND_GET_MODTIME, - }; - - enum Response { - RESPONSE_OPEN, - RESPONSE_DATA, - RESPONSE_FILE_EXISTS, - RESPONSE_GET_MODTIME, - }; - - virtual Error open_internal(const String &p_path, int p_mode_flags) override; ///< open a file - virtual bool is_open() const override; ///< true when file is open - - virtual void seek(uint64_t p_position) override; ///< seek to a given position - virtual void seek_end(int64_t p_position = 0) override; ///< seek from the end of file - virtual uint64_t get_position() const override; ///< get position in the file - virtual uint64_t get_length() const override; ///< get size of the file - - virtual bool eof_reached() const override; ///< reading passed EOF - - virtual uint8_t get_8() const override; ///< get a byte - virtual uint64_t get_buffer(uint8_t *p_dst, uint64_t p_length) const override; - - virtual Error get_error() const override; ///< get last error - - virtual void flush() override; - virtual void store_8(uint8_t p_dest) override; ///< store a byte - - virtual bool file_exists(const String &p_path) override; ///< return true if a file exists - - virtual uint64_t _get_modified_time(const String &p_file) override; - virtual uint32_t _get_unix_permissions(const String &p_file) override; - virtual Error _set_unix_permissions(const String &p_file, uint32_t p_permissions) override; - - virtual void close() override; - - static void configure(); - - FileAccessNetwork(); - ~FileAccessNetwork(); -}; - -#endif // FILE_ACCESS_NETWORK_H diff --git a/core/io/remote_filesystem_client.cpp b/core/io/remote_filesystem_client.cpp new file mode 100644 index 000000000000..f22e442a3440 --- /dev/null +++ b/core/io/remote_filesystem_client.cpp @@ -0,0 +1,329 @@ +/**************************************************************************/ +/* remote_filesystem_client.cpp */ +/**************************************************************************/ +/* This file is part of: */ +/* GODOT ENGINE */ +/* https://godotengine.org */ +/**************************************************************************/ +/* Copyright (c) 2014-present Godot Engine contributors (see AUTHORS.md). */ +/* Copyright (c) 2007-2014 Juan Linietsky, Ariel Manzur. */ +/* */ +/* Permission is hereby granted, free of charge, to any person obtaining */ +/* a copy of this software and associated documentation files (the */ +/* "Software"), to deal in the Software without restriction, including */ +/* without limitation the rights to use, copy, modify, merge, publish, */ +/* distribute, sublicense, and/or sell copies of the Software, and to */ +/* permit persons to whom the Software is furnished to do so, subject to */ +/* the following conditions: */ +/* */ +/* The above copyright notice and this permission notice shall be */ +/* included in all copies or substantial portions of the Software. */ +/* */ +/* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, */ +/* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF */ +/* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. */ +/* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY */ +/* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, */ +/* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE */ +/* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ +/**************************************************************************/ + +#include "remote_filesystem_client.h" + +#include "core/io/dir_access.h" +#include "core/io/file_access.h" +#include "core/io/stream_peer_tcp.h" +#include "core/string/string_builder.h" + +#define FILESYSTEM_CACHE_VERSION 1 +#define FILESYSTEM_PROTOCOL_VERSION 1 +#define PASSWORD_LENGTH 32 + +#define FILES_SUBFOLDER "remote_filesystem_files" +#define FILES_CACHE_FILE "remote_filesystem.cache" + +Vector RemoteFilesystemClient::_load_cache_file() { + Ref fa = FileAccess::open(cache_path.path_join(FILES_CACHE_FILE), FileAccess::READ); + if (!fa.is_valid()) { + return Vector(); // No cache, return empty + } + + int version = fa->get_line().to_int(); + if (version != FILESYSTEM_CACHE_VERSION) { + return Vector(); // Version mismatch, ignore everything. + } + + String file_path = cache_path.path_join(FILES_SUBFOLDER); + + Vector file_cache; + + while (!fa->eof_reached()) { + String l = fa->get_line(); + Vector fields = l.split("::"); + if (fields.size() != 3) { + break; + } + FileCache fc; + fc.path = fields[0]; + fc.server_modified_time = fields[1].to_int(); + fc.modified_time = fields[2].to_int(); + + String full_path = file_path.path_join(fc.path); + if (!FileAccess::exists(full_path)) { + continue; // File is gone. + } + + if (FileAccess::get_modified_time(full_path) != fc.modified_time) { + DirAccess::remove_absolute(full_path); // Take the chance to remove this file and assume we no longer have it. + continue; + } + + file_cache.push_back(fc); + } + + return file_cache; +} + +Error RemoteFilesystemClient::_store_file(const String &p_path, const LocalVector &p_file, uint64_t &modified_time) { + modified_time = 0; + String full_path = cache_path.path_join(FILES_SUBFOLDER).path_join(p_path); + String base_file_dir = full_path.get_base_dir(); + + if (!validated_directories.has(base_file_dir)) { + // Verify that path exists before writing file, but only verify once for performance. + DirAccess::make_dir_recursive_absolute(base_file_dir); + validated_directories.insert(base_file_dir); + } + + Ref f = FileAccess::open(full_path, FileAccess::WRITE); + ERR_FAIL_COND_V_MSG(f.is_null(), ERR_FILE_CANT_OPEN, "Unable to open file for writing to remote filesystem cache: " + p_path); + f->store_buffer(p_file.ptr(), p_file.size()); + Error err = f->get_error(); + if (err) { + return err; + } + f.unref(); // Unref to ensure file is not locked and modified time can be obtained. + + modified_time = FileAccess::get_modified_time(full_path); + return OK; +} + +Error RemoteFilesystemClient::_remove_file(const String &p_path) { + return DirAccess::remove_absolute(cache_path.path_join(FILES_SUBFOLDER).path_join(p_path)); +} +Error RemoteFilesystemClient::_store_cache_file(const Vector &p_cache) { + String full_path = cache_path.path_join(FILES_CACHE_FILE); + String base_file_dir = full_path.get_base_dir(); + Error err = DirAccess::make_dir_recursive_absolute(base_file_dir); + ERR_FAIL_COND_V_MSG(err != OK && err != ERR_ALREADY_EXISTS, err, "Unable to create base directory to store cache file: " + base_file_dir); + + Ref f = FileAccess::open(full_path, FileAccess::WRITE); + ERR_FAIL_COND_V_MSG(f.is_null(), ERR_FILE_CANT_OPEN, "Unable to open the remote cache file for writing: " + full_path); + f->store_line(itos(FILESYSTEM_CACHE_VERSION)); + for (int i = 0; i < p_cache.size(); i++) { + String l = p_cache[i].path + "::" + itos(p_cache[i].server_modified_time) + "::" + itos(p_cache[i].modified_time); + f->store_line(l); + } + return OK; +} + +Error RemoteFilesystemClient::synchronize_with_server(const String &p_host, int p_port, const String &p_password, String &r_cache_path) { + Error err = _synchronize_with_server(p_host, p_port, p_password, r_cache_path); + // Ensure no memory is kept + validated_directories.reset(); + cache_path = String(); + return err; +} + +void RemoteFilesystemClient::_update_cache_path(String &r_cache_path) { + r_cache_path = cache_path.path_join(FILES_SUBFOLDER); +} + +Error RemoteFilesystemClient::_synchronize_with_server(const String &p_host, int p_port, const String &p_password, String &r_cache_path) { + cache_path = r_cache_path; + { + Ref dir = DirAccess::create(DirAccess::ACCESS_FILESYSTEM); + dir->change_dir(cache_path); + cache_path = dir->get_current_dir(); + } + + Ref tcp_client; + tcp_client.instantiate(); + + IPAddress ip = p_host.is_valid_ip_address() ? IPAddress(p_host) : IP::get_singleton()->resolve_hostname(p_host); + ERR_FAIL_COND_V_MSG(!ip.is_valid(), ERR_INVALID_PARAMETER, "Unable to resolve remote filesystem server hostname: " + p_host); + print_verbose(vformat("Remote Filesystem: Connecting to host %s, port %d.", ip, p_port)); + Error err = tcp_client->connect_to_host(ip, p_port); + ERR_FAIL_COND_V_MSG(err != OK, err, "Unable to open connection to remote file server (" + String(p_host) + ", port " + itos(p_port) + ") failed."); + + while (tcp_client->get_status() == StreamPeerTCP::STATUS_CONNECTING) { + tcp_client->poll(); + OS::get_singleton()->delay_usec(100); + } + + if (tcp_client->get_status() != StreamPeerTCP::STATUS_CONNECTED) { + ERR_FAIL_V_MSG(ERR_CANT_CONNECT, "Connection to remote file server (" + String(p_host) + ", port " + itos(p_port) + ") failed."); + } + + // Connection OK, now send the current file state. + print_verbose("Remote Filesystem: Connection OK."); + + // Header (GRFS) - Godot Remote File System + print_verbose("Remote Filesystem: Sending header"); + tcp_client->put_u8('G'); + tcp_client->put_u8('R'); + tcp_client->put_u8('F'); + tcp_client->put_u8('S'); + // Protocol version + tcp_client->put_32(FILESYSTEM_PROTOCOL_VERSION); + print_verbose("Remote Filesystem: Sending password"); + uint8_t password[PASSWORD_LENGTH]; // Send fixed size password, since it's easier and safe to validate. + for (int i = 0; i < PASSWORD_LENGTH; i++) { + if (i < p_password.length()) { + password[i] = p_password[i]; + } else { + password[i] = 0; + } + } + tcp_client->put_data(password, PASSWORD_LENGTH); + print_verbose("Remote Filesystem: Tags."); + Vector tags; + { + tags.push_back(OS::get_singleton()->get_identifier()); + switch (OS::get_singleton()->get_preferred_texture_format()) { + case OS::PREFERRED_TEXTURE_FORMAT_S3TC_BPTC: { + tags.push_back("bptc"); + tags.push_back("s3tc"); + } break; + case OS::PREFERRED_TEXTURE_FORMAT_ETC2_ASTC: { + tags.push_back("etc2"); + tags.push_back("astc"); + } break; + } + } + + tcp_client->put_32(tags.size()); + for (int i = 0; i < tags.size(); i++) { + tcp_client->put_utf8_string(tags[i]); + } + // Size of compressed list of files + print_verbose("Remote Filesystem: Sending file list"); + + Vector file_cache = _load_cache_file(); + + // Encode file cache to send it via network. + Vector file_cache_buffer; + if (file_cache.size()) { + StringBuilder sbuild; + for (int i = 0; i < file_cache.size(); i++) { + sbuild.append(file_cache[i].path); + sbuild.append("::"); + sbuild.append(itos(file_cache[i].server_modified_time)); + sbuild.append("\n"); + } + String s = sbuild.as_string(); + CharString cs = s.utf8(); + file_cache_buffer.resize(Compression::get_max_compressed_buffer_size(cs.length(), Compression::MODE_ZSTD)); + int res_len = Compression::compress(file_cache_buffer.ptrw(), (const uint8_t *)cs.ptr(), cs.length(), Compression::MODE_ZSTD); + file_cache_buffer.resize(res_len); + + tcp_client->put_32(cs.length()); // Size of buffer uncompressed + tcp_client->put_32(file_cache_buffer.size()); // Size of buffer compressed + tcp_client->put_data(file_cache_buffer.ptr(), file_cache_buffer.size()); // Buffer + } else { + tcp_client->put_32(0); // No file cache buffer + } + + tcp_client->poll(); + ERR_FAIL_COND_V_MSG(tcp_client->get_status() != StreamPeerTCP::STATUS_CONNECTED, ERR_CONNECTION_ERROR, "Remote filesystem server disconnected after sending header."); + + uint32_t file_count = tcp_client->get_32(); + + ERR_FAIL_COND_V_MSG(tcp_client->get_status() != StreamPeerTCP::STATUS_CONNECTED, ERR_CONNECTION_ERROR, "Remote filesystem server disconnected while waiting for file list"); + + LocalVector file_buffer; + + Vector temp_file_cache; + + HashSet files_processed; + for (uint32_t i = 0; i < file_count; i++) { + String file = tcp_client->get_utf8_string(); + ERR_FAIL_COND_V_MSG(file == String(), ERR_CONNECTION_ERROR, "Invalid file name received from remote filesystem."); + uint64_t server_modified_time = tcp_client->get_u64(); + ERR_FAIL_COND_V_MSG(tcp_client->get_status() != StreamPeerTCP::STATUS_CONNECTED, ERR_CONNECTION_ERROR, "Remote filesystem server disconnected while waiting for file info."); + + FileCache fc; + fc.path = file; + fc.server_modified_time = server_modified_time; + temp_file_cache.push_back(fc); + + files_processed.insert(file); + } + + Vector new_file_cache; + + // Get the actual files. As a robustness measure, if the connection is interrupted here, any file not yet received will be considered removed. + // Since the file changed anyway, this makes it the easiest way to keep robustness. + + bool server_disconnected = false; + for (uint32_t i = 0; i < file_count; i++) { + String file = temp_file_cache[i].path; + + if (temp_file_cache[i].server_modified_time == 0 || server_disconnected) { + // File was removed, or server disconnected before tranferring it. Since it's no longer valid, remove anyway. + _remove_file(file); + continue; + } + + uint64_t file_size = tcp_client->get_u64(); + file_buffer.resize(file_size); + + err = tcp_client->get_data(file_buffer.ptr(), file_size); + if (err != OK) { + ERR_PRINT("Error retrieving file from remote filesystem: " + file); + server_disconnected = true; + } + + if (tcp_client->get_status() != StreamPeerTCP::STATUS_CONNECTED) { + // Early disconnect, stop accepting files. + server_disconnected = true; + } + + if (server_disconnected) { + // No more server, transfer is invalid, remove this file. + _remove_file(file); + continue; + } + + uint64_t modified_time = 0; + err = _store_file(file, file_buffer, modified_time); + if (err != OK) { + server_disconnected = true; + continue; + } + FileCache fc = temp_file_cache[i]; + fc.modified_time = modified_time; + new_file_cache.push_back(fc); + } + + print_verbose("Remote Filesystem: Updating the cache file."); + + // Go through the list of local files read initially (file_cache) and see which ones are + // unchanged (not sent again from the server). + // These need to be re-saved in the new list (new_file_cache). + + for (int i = 0; i < file_cache.size(); i++) { + if (files_processed.has(file_cache[i].path)) { + continue; // This was either added or removed, so skip. + } + new_file_cache.push_back(file_cache[i]); + } + + err = _store_cache_file(new_file_cache); + ERR_FAIL_COND_V_MSG(err != OK, ERR_FILE_CANT_OPEN, "Error writing the remote filesystem file cache."); + + print_verbose("Remote Filesystem: Update success."); + + _update_cache_path(r_cache_path); + return OK; +} diff --git a/core/io/remote_filesystem_client.h b/core/io/remote_filesystem_client.h new file mode 100644 index 000000000000..42eba98eb100 --- /dev/null +++ b/core/io/remote_filesystem_client.h @@ -0,0 +1,65 @@ +/**************************************************************************/ +/* remote_filesystem_client.h */ +/**************************************************************************/ +/* This file is part of: */ +/* GODOT ENGINE */ +/* https://godotengine.org */ +/**************************************************************************/ +/* Copyright (c) 2014-present Godot Engine contributors (see AUTHORS.md). */ +/* Copyright (c) 2007-2014 Juan Linietsky, Ariel Manzur. */ +/* */ +/* Permission is hereby granted, free of charge, to any person obtaining */ +/* a copy of this software and associated documentation files (the */ +/* "Software"), to deal in the Software without restriction, including */ +/* without limitation the rights to use, copy, modify, merge, publish, */ +/* distribute, sublicense, and/or sell copies of the Software, and to */ +/* permit persons to whom the Software is furnished to do so, subject to */ +/* the following conditions: */ +/* */ +/* The above copyright notice and this permission notice shall be */ +/* included in all copies or substantial portions of the Software. */ +/* */ +/* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, */ +/* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF */ +/* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. */ +/* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY */ +/* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, */ +/* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE */ +/* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ +/**************************************************************************/ + +#ifndef REMOTE_FILESYSTEM_CLIENT_H +#define REMOTE_FILESYSTEM_CLIENT_H + +#include "core/io/ip_address.h" +#include "core/string/ustring.h" +#include "core/templates/hash_set.h" +#include "core/templates/local_vector.h" + +class RemoteFilesystemClient { + String cache_path; + HashSet validated_directories; + +protected: + String _get_cache_path() { return cache_path; } + struct FileCache { + String path; // Local path (as in "folder/to/file.png") + uint64_t server_modified_time; // MD5 checksum. + uint64_t modified_time; + }; + virtual bool _is_configured() { return !cache_path.is_empty(); } + // Can be re-implemented per platform. If so, feel free to ignore get_cache_path() + virtual Vector _load_cache_file(); + virtual Error _store_file(const String &p_path, const LocalVector &p_file, uint64_t &modified_time); + virtual Error _remove_file(const String &p_path); + virtual Error _store_cache_file(const Vector &p_cache); + virtual Error _synchronize_with_server(const String &p_host, int p_port, const String &p_password, String &r_cache_path); + + virtual void _update_cache_path(String &r_cache_path); + +public: + Error synchronize_with_server(const String &p_host, int p_port, const String &p_password, String &r_cache_path); + virtual ~RemoteFilesystemClient() {} +}; + +#endif // REMOTE_FILESYSTEM_CLIENT_H diff --git a/core/os/os.cpp b/core/os/os.cpp index c82f87c731d5..4123a1d60255 100644 --- a/core/os/os.cpp +++ b/core/os/os.cpp @@ -72,6 +72,10 @@ void OS::add_logger(Logger *p_logger) { } } +String OS::get_identifier() const { + return get_name().to_lower(); +} + void OS::print_error(const char *p_function, const char *p_file, int p_line, const char *p_code, const char *p_rationale, bool p_editor_notify, Logger::ErrorType p_type) { if (!_stderr_enabled) { return; @@ -357,13 +361,7 @@ void OS::set_has_server_feature_callback(HasServerFeatureCallback p_callback) { bool OS::has_feature(const String &p_feature) { // Feature tags are always lowercase for consistency. - if (p_feature == get_name().to_lower()) { - return true; - } - - // Catch-all `linuxbsd` feature tag that matches on both Linux and BSD. - // This is the one exposed in the project settings dialog. - if (p_feature == "linuxbsd" && (get_name() == "Linux" || get_name() == "FreeBSD" || get_name() == "NetBSD" || get_name() == "OpenBSD" || get_name() == "BSD")) { + if (p_feature == get_identifier()) { return true; } @@ -569,6 +567,10 @@ void OS::add_frame_delay(bool p_can_draw) { } } +Error OS::setup_remote_filesystem(const String &p_server_host, int p_port, const String &p_password, String &r_project_path) { + return default_rfs.synchronize_with_server(p_server_host, p_port, p_password, r_project_path); +} + OS::PreferredTextureFormat OS::get_preferred_texture_format() const { #if defined(__arm__) || defined(__aarch64__) || defined(_M_ARM) || defined(_M_ARM64) return PREFERRED_TEXTURE_FORMAT_ETC2_ASTC; // By rule, ARM hardware uses ETC texture compression. diff --git a/core/os/os.h b/core/os/os.h index 3248330c74c0..db1f1f13c9cd 100644 --- a/core/os/os.h +++ b/core/os/os.h @@ -34,6 +34,7 @@ #include "core/config/engine.h" #include "core/io/image.h" #include "core/io/logger.h" +#include "core/io/remote_filesystem_client.h" #include "core/os/time_enums.h" #include "core/string/ustring.h" #include "core/templates/list.h" @@ -72,6 +73,8 @@ class OS { String _current_rendering_driver_name; String _current_rendering_method; + RemoteFilesystemClient default_rfs; + protected: void _set_logger(CompositeLogger *p_logger); @@ -172,6 +175,7 @@ public: virtual void unset_environment(const String &p_var) const = 0; virtual String get_name() const = 0; + virtual String get_identifier() const; virtual String get_distribution_name() const = 0; virtual String get_version() const = 0; virtual List get_cmdline_args() const { return _cmdline; } @@ -292,6 +296,8 @@ public: virtual void process_and_drop_events() {} + virtual Error setup_remote_filesystem(const String &p_server_host, int p_port, const String &p_password, String &r_project_path); + enum PreferredTextureFormat { PREFERRED_TEXTURE_FORMAT_S3TC_BPTC, PREFERRED_TEXTURE_FORMAT_ETC2_ASTC diff --git a/core/string/print_string.cpp b/core/string/print_string.cpp index 7b894d83bf0e..7b90710308cd 100644 --- a/core/string/print_string.cpp +++ b/core/string/print_string.cpp @@ -193,10 +193,8 @@ void print_error(String p_string) { _global_unlock(); } -void print_verbose(String p_string) { - if (OS::get_singleton()->is_stdout_verbose()) { - print_line(p_string); - } +bool is_print_verbose_enabled() { + return OS::get_singleton()->is_stdout_verbose(); } String stringify_variants(Variant p_var) { diff --git a/core/string/print_string.h b/core/string/print_string.h index 6496384b3f6a..7656e9bfa11d 100644 --- a/core/string/print_string.h +++ b/core/string/print_string.h @@ -59,7 +59,15 @@ void remove_print_handler(const PrintHandlerList *p_handler); extern void __print_line(String p_string); extern void __print_line_rich(String p_string); extern void print_error(String p_string); -extern void print_verbose(String p_string); +extern bool is_print_verbose_enabled(); + +// This version avoids processing the text to be printed until it actually has to be printed, saving some CPU usage. +#define print_verbose(m_text) \ + { \ + if (is_print_verbose_enabled()) { \ + print_line(m_text); \ + } \ + } inline void print_line(Variant v) { __print_line(stringify_variants(v)); diff --git a/core/variant/variant_utility.cpp b/core/variant/variant_utility.cpp index fd079dbeeaaa..545825011a95 100644 --- a/core/variant/variant_utility.cpp +++ b/core/variant/variant_utility.cpp @@ -804,6 +804,8 @@ struct VariantUtilityFunctions { r_error.error = Callable::CallError::CALL_OK; } +#undef print_verbose + static inline void print_verbose(const Variant **p_args, int p_arg_count, Callable::CallError &r_error) { if (OS::get_singleton()->is_stdout_verbose()) { String s; diff --git a/doc/classes/ProjectSettings.xml b/doc/classes/ProjectSettings.xml index 4a6368ca6c75..b46f395a0a84 100644 --- a/doc/classes/ProjectSettings.xml +++ b/doc/classes/ProjectSettings.xml @@ -1784,12 +1784,6 @@ Maximum size (in kiB) for the [WebRTCDataChannel] input buffer. - - Amount of read ahead used by remote filesystem. Higher values decrease the effects of latency at the cost of higher bandwidth usage. - - - Page size used by remote filesystem (in bytes). - The CA certificates bundle to use for TLS connections. If this is set to a non-empty value, this will [i]override[/i] Godot's default [url=https://github.com/godotengine/godot/blob/master/thirdparty/certs/ca-certificates.crt]Mozilla certificate bundle[/url]. If left empty, the default certificate bundle will be used. If in doubt, leave this setting empty. diff --git a/editor/debugger/editor_file_server.cpp b/editor/debugger/editor_file_server.cpp index ba5dd9e6e2c3..df317f810656 100644 --- a/editor/debugger/editor_file_server.cpp +++ b/editor/debugger/editor_file_server.cpp @@ -32,273 +32,230 @@ #include "../editor_settings.h" #include "core/io/marshalls.h" +#include "editor/editor_node.h" +#include "editor/export/editor_export_platform.h" -//#define DEBUG_PRINT(m_p) print_line(m_p) -//#define DEBUG_TIME(m_what) printf("MS: %s - %lu\n", m_what, OS::get_singleton()->get_ticks_usec()); +#define FILESYSTEM_PROTOCOL_VERSION 1 +#define PASSWORD_LENGTH 32 +#define MAX_FILE_BUFFER_SIZE 100 * 1024 * 1024 // 100mb max file buffer size (description of files to update, compressed). -#define DEBUG_PRINT(m_what) -#define DEBUG_TIME(m_what) - -void EditorFileServer::_close_client(ClientData *cd) { - cd->connection->disconnect_from_host(); - { - MutexLock lock(cd->efs->wait_mutex); - cd->efs->to_wait.insert(cd->thread); +static void _add_file(String f, const uint64_t &p_modified_time, HashMap &files_to_send, HashMap &cached_files) { + f = f.replace_first("res://", ""); // remove res:// + const uint64_t *cached_mt = cached_files.getptr(f); + if (cached_mt && *cached_mt == p_modified_time) { + // File is good, skip it. + cached_files.erase(f); // Erase to mark this file as existing. Remaning files not added to files_to_send will be considered erased here, so they need to be erased in the client too. + return; } - while (cd->files.size()) { - cd->files.remove(cd->files.begin()); - } - memdelete(cd); + files_to_send.insert(f, p_modified_time); } -void EditorFileServer::_subthread_start(void *s) { - ClientData *cd = static_cast(s); +void EditorFileServer::_scan_files_changed(EditorFileSystemDirectory *efd, const Vector &p_tags, HashMap &files_to_send, HashMap &cached_files) { + for (int i = 0; i < efd->get_file_count(); i++) { + String f = efd->get_file_path(i); + if (FileAccess::exists(f + ".import")) { + // is imported, determine what to do + // Todo the modified times of remapped files should most likely be kept in EditorFileSystem to speed this up in the future. + Ref cf; + cf.instantiate(); + Error err = cf->load(f + ".import"); - cd->connection->set_no_delay(true); - uint8_t buf4[8]; - Error err = cd->connection->get_data(buf4, 4); - if (err != OK) { - _close_client(cd); - ERR_FAIL_COND(err != OK); + ERR_CONTINUE(err != OK); + { + uint64_t mt = FileAccess::get_modified_time(f + ".import"); + _add_file(f + ".import", mt, files_to_send, cached_files); + } + + if (!cf->has_section("remap")) { + continue; + } + + List remaps; + cf->get_section_keys("remap", &remaps); + + for (const String &remap : remaps) { + if (remap == "path") { + String remapped_path = cf->get_value("remap", remap); + uint64_t mt = FileAccess::get_modified_time(remapped_path); + _add_file(remapped_path, mt, files_to_send, cached_files); + } else if (remap.begins_with("path.")) { + String feature = remap.get_slice(".", 1); + if (p_tags.find(feature) != -1) { + String remapped_path = cf->get_value("remap", remap); + uint64_t mt = FileAccess::get_modified_time(remapped_path); + _add_file(remapped_path, mt, files_to_send, cached_files); + } + } + } + } else { + uint64_t mt = efd->get_file_modified_time(i); + _add_file(f, mt, files_to_send, cached_files); + } } - int passlen = decode_uint32(buf4); + for (int i = 0; i < efd->get_subdir_count(); i++) { + _scan_files_changed(efd->get_subdir(i), p_tags, files_to_send, cached_files); + } +} - if (passlen > 512) { - _close_client(cd); - ERR_FAIL_COND(passlen > 512); - } else if (passlen > 0) { - Vector passutf8; - passutf8.resize(passlen + 1); - err = cd->connection->get_data((uint8_t *)passutf8.ptr(), passlen); - if (err != OK) { - _close_client(cd); - ERR_FAIL_COND(err != OK); - } - passutf8.write[passlen] = 0; - String s2; - s2.parse_utf8(passutf8.ptr()); - if (s2 != cd->efs->password) { - encode_uint32(ERR_INVALID_DATA, buf4); - cd->connection->put_data(buf4, 4); - OS::get_singleton()->delay_usec(1000000); - _close_client(cd); - ERR_PRINT("CLIENT PASSWORD MISMATCH"); - ERR_FAIL(); +static void _add_custom_file(const String f, HashMap &files_to_send, HashMap &cached_files) { + if (!FileAccess::exists(f)) { + return; + } + _add_file(f, FileAccess::get_modified_time(f), files_to_send, cached_files); +} + +void EditorFileServer::poll() { + if (!active) { + return; + } + + if (!server->is_connection_available()) { + return; + } + + Ref tcp_peer = server->take_connection(); + ERR_FAIL_COND(tcp_peer.is_null()); + + // Got a connection! + EditorProgress pr("updating_remote_file_system", TTR("Updating assets on target device:"), 105); + + pr.step(TTR("Syncinc headers"), 0, true); + print_verbose("EFS: Connecting taken!"); + char header[4]; + Error err = tcp_peer->get_data((uint8_t *)&header, 4); + ERR_FAIL_COND(err != OK); + ERR_FAIL_COND(header[0] != 'G'); + ERR_FAIL_COND(header[1] != 'R'); + ERR_FAIL_COND(header[2] != 'F'); + ERR_FAIL_COND(header[3] != 'S'); + + uint32_t protocol_version = tcp_peer->get_u32(); + ERR_FAIL_COND(protocol_version != FILESYSTEM_PROTOCOL_VERSION); + + char cpassword[PASSWORD_LENGTH + 1]; + err = tcp_peer->get_data((uint8_t *)cpassword, PASSWORD_LENGTH); + cpassword[PASSWORD_LENGTH] = 0; + ERR_FAIL_COND(err != OK); + print_verbose("EFS: Got password: " + String(cpassword)); + ERR_FAIL_COND_MSG(password != cpassword, "Client disconnected because password mismatch."); + + uint32_t tag_count = tcp_peer->get_u32(); + print_verbose("EFS: Getting tags: " + itos(tag_count)); + + ERR_FAIL_COND(tcp_peer->get_status() != StreamPeerTCP::STATUS_CONNECTED); + Vector tags; + for (uint32_t i = 0; i < tag_count; i++) { + String tag = tcp_peer->get_utf8_string(); + print_verbose("EFS: tag #" + itos(i) + ": " + tag); + ERR_FAIL_COND(tcp_peer->get_status() != StreamPeerTCP::STATUS_CONNECTED); + tags.push_back(tag); + } + + uint32_t file_buffer_decompressed_size = tcp_peer->get_32(); + HashMap cached_files; + + if (file_buffer_decompressed_size > 0) { + pr.step(TTR("Getting remote file system"), 1, true); + + // Got files cached by client. + uint32_t file_buffer_size = tcp_peer->get_32(); + print_verbose("EFS: Getting file buffer: compressed - " + String::humanize_size(file_buffer_size) + " decompressed: " + String::humanize_size(file_buffer_decompressed_size)); + + ERR_FAIL_COND(tcp_peer->get_status() != StreamPeerTCP::STATUS_CONNECTED); + ERR_FAIL_COND(file_buffer_size > MAX_FILE_BUFFER_SIZE); + LocalVector file_buffer; + file_buffer.resize(file_buffer_size); + LocalVector file_buffer_decompressed; + file_buffer_decompressed.resize(file_buffer_decompressed_size); + + err = tcp_peer->get_data(file_buffer.ptr(), file_buffer_size); + + pr.step(TTR("Decompressing remote file system"), 2, true); + + ERR_FAIL_COND(err != OK); + // Decompress the text with all the files + Compression::decompress(file_buffer_decompressed.ptr(), file_buffer_decompressed.size(), file_buffer.ptr(), file_buffer.size(), Compression::MODE_ZSTD); + String files_text = String::utf8((const char *)file_buffer_decompressed.ptr(), file_buffer_decompressed.size()); + Vector files = files_text.split("\n"); + + print_verbose("EFS: Total cached files received: " + itos(files.size())); + for (int i = 0; i < files.size(); i++) { + if (files[i].get_slice_count("::") != 2) { + continue; + } + String file = files[i].get_slice("::", 0); + uint64_t modified_time = files[i].get_slice("::", 1).to_int(); + + cached_files.insert(file, modified_time); } } else { - if (!cd->efs->password.is_empty()) { - encode_uint32(ERR_INVALID_DATA, buf4); - cd->connection->put_data(buf4, 4); - OS::get_singleton()->delay_usec(1000000); - _close_client(cd); - ERR_PRINT("CLIENT PASSWORD MISMATCH (should be empty!)"); - ERR_FAIL(); + // Client does not have any files stored. + } + + pr.step(TTR("Scanning for local changes"), 3, true); + + print_verbose("EFS: Scanning changes:"); + + HashMap files_to_send; + // Scan files to send. + _scan_files_changed(EditorFileSystem::get_singleton()->get_filesystem(), tags, files_to_send, cached_files); + // Add forced export files + Vector forced_export = EditorExportPlatform::get_forced_export_files(); + for (int i = 0; i < forced_export.size(); i++) { + _add_custom_file(forced_export[i], files_to_send, cached_files); + } + + _add_custom_file("res://project.godot", files_to_send, cached_files); + // Check which files were removed and also add them + for (KeyValue K : cached_files) { + if (!files_to_send.has(K.key)) { + files_to_send.insert(K.key, 0); //0 means removed } } - encode_uint32(OK, buf4); - cd->connection->put_data(buf4, 4); + tcp_peer->put_32(files_to_send.size()); - while (!cd->quit) { - //wait for ID - err = cd->connection->get_data(buf4, 4); - DEBUG_TIME("get_data") + print_verbose("EFS: Sending list of changed files."); + pr.step(TTR("Sending list of changed files:"), 4, true); - if (err != OK) { - _close_client(cd); - ERR_FAIL_COND(err != OK); - } - int id = decode_uint32(buf4); - - //wait for command - err = cd->connection->get_data(buf4, 4); - if (err != OK) { - _close_client(cd); - ERR_FAIL_COND(err != OK); - } - int cmd = decode_uint32(buf4); - - switch (cmd) { - case FileAccessNetwork::COMMAND_FILE_EXISTS: - case FileAccessNetwork::COMMAND_GET_MODTIME: - case FileAccessNetwork::COMMAND_OPEN_FILE: { - DEBUG_TIME("open_file") - err = cd->connection->get_data(buf4, 4); - if (err != OK) { - _close_client(cd); - ERR_FAIL_COND(err != OK); - } - - int namelen = decode_uint32(buf4); - Vector fileutf8; - fileutf8.resize(namelen + 1); - err = cd->connection->get_data((uint8_t *)fileutf8.ptr(), namelen); - if (err != OK) { - _close_client(cd); - ERR_FAIL_COND(err != OK); - } - fileutf8.write[namelen] = 0; - String s2; - s2.parse_utf8(fileutf8.ptr()); - - if (cmd == FileAccessNetwork::COMMAND_FILE_EXISTS) { - print_verbose("FILE EXISTS: " + s2); - } - if (cmd == FileAccessNetwork::COMMAND_GET_MODTIME) { - print_verbose("MOD TIME: " + s2); - } - if (cmd == FileAccessNetwork::COMMAND_OPEN_FILE) { - print_verbose("OPEN: " + s2); - } - - if (!s2.begins_with("res://")) { - _close_client(cd); - ERR_FAIL_COND(!s2.begins_with("res://")); - } - ERR_CONTINUE(cd->files.has(id)); - - if (cmd == FileAccessNetwork::COMMAND_FILE_EXISTS) { - encode_uint32(id, buf4); - cd->connection->put_data(buf4, 4); - encode_uint32(FileAccessNetwork::RESPONSE_FILE_EXISTS, buf4); - cd->connection->put_data(buf4, 4); - encode_uint32(FileAccess::exists(s2), buf4); - cd->connection->put_data(buf4, 4); - DEBUG_TIME("open_file_end") - break; - } - - if (cmd == FileAccessNetwork::COMMAND_GET_MODTIME) { - encode_uint32(id, buf4); - cd->connection->put_data(buf4, 4); - encode_uint32(FileAccessNetwork::RESPONSE_GET_MODTIME, buf4); - cd->connection->put_data(buf4, 4); - encode_uint64(FileAccess::get_modified_time(s2), buf4); - cd->connection->put_data(buf4, 8); - DEBUG_TIME("open_file_end") - break; - } - - Ref fa = FileAccess::open(s2, FileAccess::READ); - if (fa.is_null()) { - //not found, continue - encode_uint32(id, buf4); - cd->connection->put_data(buf4, 4); - encode_uint32(FileAccessNetwork::RESPONSE_OPEN, buf4); - cd->connection->put_data(buf4, 4); - encode_uint32(ERR_FILE_NOT_FOUND, buf4); - cd->connection->put_data(buf4, 4); - DEBUG_TIME("open_file_end") - break; - } - - encode_uint32(id, buf4); - cd->connection->put_data(buf4, 4); - encode_uint32(FileAccessNetwork::RESPONSE_OPEN, buf4); - cd->connection->put_data(buf4, 4); - encode_uint32(OK, buf4); - cd->connection->put_data(buf4, 4); - encode_uint64(fa->get_length(), buf4); - cd->connection->put_data(buf4, 8); - - cd->files[id] = fa; - DEBUG_TIME("open_file_end") - - } break; - case FileAccessNetwork::COMMAND_READ_BLOCK: { - err = cd->connection->get_data(buf4, 8); - if (err != OK) { - _close_client(cd); - ERR_FAIL_COND(err != OK); - } - - ERR_CONTINUE(!cd->files.has(id)); - - uint64_t offset = decode_uint64(buf4); - - err = cd->connection->get_data(buf4, 4); - if (err != OK) { - _close_client(cd); - ERR_FAIL_COND(err != OK); - } - - int blocklen = decode_uint32(buf4); - ERR_CONTINUE(blocklen > (16 * 1024 * 1024)); - - cd->files[id]->seek(offset); - Vector buf; - buf.resize(blocklen); - uint32_t read = cd->files[id]->get_buffer(buf.ptrw(), blocklen); - - print_verbose("GET BLOCK - offset: " + itos(offset) + ", blocklen: " + itos(blocklen)); - - //not found, continue - encode_uint32(id, buf4); - cd->connection->put_data(buf4, 4); - encode_uint32(FileAccessNetwork::RESPONSE_DATA, buf4); - cd->connection->put_data(buf4, 4); - encode_uint64(offset, buf4); - cd->connection->put_data(buf4, 8); - encode_uint32(read, buf4); - cd->connection->put_data(buf4, 4); - cd->connection->put_data(buf.ptr(), read); - - } break; - case FileAccessNetwork::COMMAND_CLOSE: { - print_verbose("CLOSED"); - ERR_CONTINUE(!cd->files.has(id)); - cd->files.erase(id); - } break; - } + // Send list of changed files first, to ensure that if connecting breaks, the client is not found in a broken state. + for (KeyValue K : files_to_send) { + tcp_peer->put_utf8_string(K.key); + tcp_peer->put_64(K.value); } - _close_client(cd); -} + print_verbose("EFS: Sending " + itos(files_to_send.size()) + " files."); -void EditorFileServer::_thread_start(void *s) { - EditorFileServer *self = static_cast(s); - while (!self->quit) { - if (self->cmd == CMD_ACTIVATE) { - self->server->listen(self->port); - self->active = true; - self->cmd = CMD_NONE; - } else if (self->cmd == CMD_STOP) { - self->server->stop(); - self->active = false; - self->cmd = CMD_NONE; + int idx = 0; + for (KeyValue K : files_to_send) { + pr.step(TTR("Sending file: ") + K.key.get_file(), 5 + idx * 100 / files_to_send.size(), false); + idx++; + + if (K.value == 0 || !FileAccess::exists("res://" + K.key)) { // File was removed + continue; } - if (self->active) { - if (self->server->is_connection_available()) { - ClientData *cd = memnew(ClientData); - cd->connection = self->server->take_connection(); - cd->efs = self; - cd->quit = false; - cd->thread = memnew(Thread); - cd->thread->start(_subthread_start, cd); - } - } - - self->wait_mutex.lock(); - while (self->to_wait.size()) { - Thread *w = *self->to_wait.begin(); - self->to_wait.erase(w); - self->wait_mutex.unlock(); - w->wait_to_finish(); - self->wait_mutex.lock(); - } - self->wait_mutex.unlock(); - - OS::get_singleton()->delay_usec(100000); + Vector array = FileAccess::_get_file_as_bytes("res://" + K.key); + tcp_peer->put_64(array.size()); + tcp_peer->put_data(array.ptr(), array.size()); + ERR_FAIL_COND(tcp_peer->get_status() != StreamPeerTCP::STATUS_CONNECTED); } + + tcp_peer->put_data((const uint8_t *)"GEND", 4); // End marker. + + print_verbose("EFS: Done."); } void EditorFileServer::start() { - stop(); + if (active) { + stop(); + } port = EDITOR_GET("filesystem/file_server/port"); password = EDITOR_GET("filesystem/file_server/password"); - cmd = CMD_ACTIVATE; + Error err = server->listen(port); + ERR_FAIL_COND_MSG(err != OK, "EditorFileServer: Unable to listen on port " + itos(port)); + active = true; } bool EditorFileServer::is_active() const { @@ -306,18 +263,19 @@ bool EditorFileServer::is_active() const { } void EditorFileServer::stop() { - cmd = CMD_STOP; + if (active) { + server->stop(); + active = false; + } } EditorFileServer::EditorFileServer() { server.instantiate(); - thread.start(_thread_start, this); EDITOR_DEF("filesystem/file_server/port", 6010); EDITOR_DEF("filesystem/file_server/password", ""); } EditorFileServer::~EditorFileServer() { - quit = true; - thread.wait_to_finish(); + stop(); } diff --git a/editor/debugger/editor_file_server.h b/editor/debugger/editor_file_server.h index ff2742e73fea..4374f508ee36 100644 --- a/editor/debugger/editor_file_server.h +++ b/editor/debugger/editor_file_server.h @@ -31,46 +31,24 @@ #ifndef EDITOR_FILE_SERVER_H #define EDITOR_FILE_SERVER_H -#include "core/io/file_access_network.h" #include "core/io/packet_peer.h" #include "core/io/tcp_server.h" #include "core/object/class_db.h" #include "core/os/thread.h" +#include "editor/editor_file_system.h" class EditorFileServer : public Object { GDCLASS(EditorFileServer, Object); - enum Command { - CMD_NONE, - CMD_ACTIVATE, - CMD_STOP, - }; - - struct ClientData { - Thread *thread = nullptr; - Ref connection; - HashMap> files; - EditorFileServer *efs = nullptr; - bool quit = false; - }; - Ref server; - HashSet to_wait; - - static void _close_client(ClientData *cd); - static void _subthread_start(void *s); - - Mutex wait_mutex; - Thread thread; - static void _thread_start(void *); - bool quit = false; - Command cmd = CMD_NONE; - String password; int port = 0; bool active = false; + void _scan_files_changed(EditorFileSystemDirectory *efd, const Vector &p_tags, HashMap &files_to_send, HashMap &cached_files); public: + void poll(); + void start(); void stop(); diff --git a/editor/export/editor_export_platform.cpp b/editor/export/editor_export_platform.cpp index 121088f27a51..670f5af713b0 100644 --- a/editor/export/editor_export_platform.cpp +++ b/editor/export/editor_export_platform.cpp @@ -818,12 +818,56 @@ String EditorExportPlatform::_export_customize(const String &p_path, LocalVector return save_path.is_empty() ? p_path : save_path; } +Vector EditorExportPlatform::get_forced_export_files() { + Vector files; + + files.push_back(ProjectSettings::get_singleton()->get_global_class_list_path()); + + String icon = GLOBAL_GET("application/config/icon"); + String splash = GLOBAL_GET("application/boot_splash/image"); + if (!icon.is_empty() && FileAccess::exists(icon)) { + files.push_back(icon); + } + if (!splash.is_empty() && FileAccess::exists(splash) && icon != splash) { + files.push_back(splash); + } + String resource_cache_file = ResourceUID::get_cache_file(); + if (FileAccess::exists(resource_cache_file)) { + files.push_back(resource_cache_file); + } + + String extension_list_config_file = GDExtension::get_extension_list_config_file(); + if (FileAccess::exists(extension_list_config_file)) { + files.push_back(extension_list_config_file); + } + + // Store text server data if it is supported. + if (TS->has_feature(TextServer::FEATURE_USE_SUPPORT_DATA)) { + bool use_data = GLOBAL_GET("internationalization/locale/include_text_server_data"); + if (use_data) { + // Try using user provided data file. + String ts_data = "res://" + TS->get_support_data_filename(); + if (FileAccess::exists(ts_data)) { + files.push_back(ts_data); + } else { + // Use default text server data. + String icu_data_file = EditorPaths::get_singleton()->get_cache_dir().path_join("tmp_icu_data"); + ERR_FAIL_COND_V(!TS->save_support_data(icu_data_file), files); + files.push_back(icu_data_file); + // Remove the file later. + MessageQueue::get_singleton()->push_callable(callable_mp_static(DirAccess::remove_absolute), icu_data_file); + } + } + } + + return files; +} + Error EditorExportPlatform::export_project_files(const Ref &p_preset, bool p_debug, EditorExportSaveFunction p_func, void *p_udata, EditorExportSaveSharedObject p_so_func) { //figure out paths of files that will be exported HashSet paths; Vector path_remaps; - paths.insert(ProjectSettings::get_singleton()->get_global_class_list_path()); if (p_preset->get_export_filter() == EditorExportPreset::EXPORT_ALL_RESOURCES) { //find stuff _export_find_resources(EditorFileSystem::get_singleton()->get_filesystem(), paths); @@ -1295,68 +1339,14 @@ Error EditorExportPlatform::export_project_files(const Ref & } } - // Store icon and splash images directly, they need to bypass the import system and be loaded as images - String icon = GLOBAL_GET("application/config/icon"); - String splash = GLOBAL_GET("application/boot_splash/image"); - if (!icon.is_empty() && FileAccess::exists(icon)) { - Vector array = FileAccess::get_file_as_bytes(icon); - err = p_func(p_udata, icon, array, idx, total, enc_in_filters, enc_ex_filters, key); + Vector forced_export = get_forced_export_files(); + for (int i = 0; i < forced_export.size(); i++) { + Vector array = FileAccess::get_file_as_bytes(forced_export[i]); + err = p_func(p_udata, forced_export[i], array, idx, total, enc_in_filters, enc_ex_filters, key); if (err != OK) { return err; } } - if (!splash.is_empty() && FileAccess::exists(splash) && icon != splash) { - Vector array = FileAccess::get_file_as_bytes(splash); - err = p_func(p_udata, splash, array, idx, total, enc_in_filters, enc_ex_filters, key); - if (err != OK) { - return err; - } - } - String resource_cache_file = ResourceUID::get_cache_file(); - if (FileAccess::exists(resource_cache_file)) { - Vector array = FileAccess::get_file_as_bytes(resource_cache_file); - err = p_func(p_udata, resource_cache_file, array, idx, total, enc_in_filters, enc_ex_filters, key); - if (err != OK) { - return err; - } - } - - String extension_list_config_file = GDExtension::get_extension_list_config_file(); - if (FileAccess::exists(extension_list_config_file)) { - Vector array = FileAccess::get_file_as_bytes(extension_list_config_file); - err = p_func(p_udata, extension_list_config_file, array, idx, total, enc_in_filters, enc_ex_filters, key); - if (err != OK) { - return err; - } - } - - // Store text server data if it is supported. - if (TS->has_feature(TextServer::FEATURE_USE_SUPPORT_DATA)) { - bool use_data = GLOBAL_GET("internationalization/locale/include_text_server_data"); - if (use_data) { - // Try using user provided data file. - String ts_data = "res://" + TS->get_support_data_filename(); - if (FileAccess::exists(ts_data)) { - Vector array = FileAccess::get_file_as_bytes(ts_data); - err = p_func(p_udata, ts_data, array, idx, total, enc_in_filters, enc_ex_filters, key); - if (err != OK) { - return err; - } - } else { - // Use default text server data. - String icu_data_file = EditorPaths::get_singleton()->get_cache_dir().path_join("tmp_icu_data"); - if (!TS->save_support_data(icu_data_file)) { - return ERR_INVALID_DATA; - } - Vector array = FileAccess::get_file_as_bytes(icu_data_file); - err = p_func(p_udata, ts_data, array, idx, total, enc_in_filters, enc_ex_filters, key); - DirAccess::remove_file_or_error(icu_data_file); - if (err != OK) { - return err; - } - } - } - } String config_file = "project.binary"; String engine_cfb = EditorPaths::get_singleton()->get_cache_dir().path_join("tmp" + config_file); diff --git a/editor/export/editor_export_platform.h b/editor/export/editor_export_platform.h index 21d871990224..df5a66f099a7 100644 --- a/editor/export/editor_export_platform.h +++ b/editor/export/editor_export_platform.h @@ -196,6 +196,8 @@ public: return worst_type; } + static Vector get_forced_export_files(); + virtual bool fill_log_messages(RichTextLabel *p_log, Error p_err); virtual void get_export_options(List *r_options) const = 0; diff --git a/editor/plugins/debugger_editor_plugin.cpp b/editor/plugins/debugger_editor_plugin.cpp index 7863e6d19e4f..aecf3d295caf 100644 --- a/editor/plugins/debugger_editor_plugin.cpp +++ b/editor/plugins/debugger_editor_plugin.cpp @@ -126,8 +126,10 @@ void DebuggerEditorPlugin::_menu_option(int p_option) { if (ischecked) { file_server->stop(); + set_process(false); } else { file_server->start(); + set_process(true); } debug_menu->set_item_checked(debug_menu->get_item_index(RUN_FILE_SERVER), !ischecked); @@ -190,6 +192,10 @@ void DebuggerEditorPlugin::_notification(int p_what) { case NOTIFICATION_READY: { _update_debug_options(); } break; + + case NOTIFICATION_PROCESS: { + file_server->poll(); + } break; } } diff --git a/main/main.cpp b/main/main.cpp index 5e0187cc7fd9..707ef6ec3d00 100644 --- a/main/main.cpp +++ b/main/main.cpp @@ -41,7 +41,6 @@ #include "core/input/input.h" #include "core/input/input_map.h" #include "core/io/dir_access.h" -#include "core/io/file_access_network.h" #include "core/io/file_access_pack.h" #include "core/io/file_access_zip.h" #include "core/io/image_loader.h" @@ -129,7 +128,6 @@ static Time *time_singleton = nullptr; #ifdef MINIZIP_ENABLED static ZipArchive *zip_packed_data = nullptr; #endif -static FileAccessNetworkClient *file_access_network_client = nullptr; static MessageQueue *message_queue = nullptr; // Initialized in setup2() @@ -1384,9 +1382,7 @@ Error Main::setup(const char *execpath, int argc, char *argv[], bool p_second_ph // Network file system needs to be configured before globals, since globals are based on the // 'project.godot' file which will only be available through the network if this is enabled - FileAccessNetwork::configure(); if (!remotefs.is_empty()) { - file_access_network_client = memnew(FileAccessNetworkClient); int port; if (remotefs.contains(":")) { port = remotefs.get_slicec(':', 1).to_int(); @@ -1394,14 +1390,12 @@ Error Main::setup(const char *execpath, int argc, char *argv[], bool p_second_ph } else { port = 6010; } + Error err = OS::get_singleton()->setup_remote_filesystem(remotefs, port, remotefs_pass, project_path); - Error err = file_access_network_client->connect(remotefs, port, remotefs_pass); if (err) { OS::get_singleton()->printerr("Could not connect to remotefs: %s:%i.\n", remotefs.utf8().get_data(), port); goto error; } - - FileAccess::make_default(FileAccess::ACCESS_RESOURCES); } if (globals->setup(project_path, main_pack, upwards, editor) == OK) { @@ -1937,9 +1931,6 @@ error: if (packed_data) { memdelete(packed_data); } - if (file_access_network_client) { - memdelete(file_access_network_client); - } unregister_core_driver_types(); unregister_core_extensions(); @@ -3448,9 +3439,6 @@ void Main::cleanup(bool p_force) { if (packed_data) { memdelete(packed_data); } - if (file_access_network_client) { - memdelete(file_access_network_client); - } if (performance) { memdelete(performance); } diff --git a/platform/android/export/export_plugin.cpp b/platform/android/export/export_plugin.cpp index f52edf2b6106..87f4cf2de2d7 100644 --- a/platform/android/export/export_plugin.cpp +++ b/platform/android/export/export_plugin.cpp @@ -2791,7 +2791,11 @@ Error EditorExportPlatformAndroid::export_project_helper(const Ref(FileAccess::ACCESS_RESOURCES); + } + return err; +} + OS_Android::~OS_Android() { } diff --git a/platform/android/os_android.h b/platform/android/os_android.h index 53910b14981f..902712d69eec 100644 --- a/platform/android/os_android.h +++ b/platform/android/os_android.h @@ -57,6 +57,7 @@ private: mutable String data_dir_cache; mutable String cache_dir_cache; + mutable String remote_fs_dir; AudioDriverOpenSL audio_driver_android; @@ -160,6 +161,8 @@ public: virtual Error create_instance(const List &p_arguments, ProcessID *r_child_id = nullptr) override; virtual Error kill(const ProcessID &p_pid) override; + virtual Error setup_remote_filesystem(const String &p_server_host, int p_port, const String &p_password, String &r_project_path) override; + virtual bool _check_internal_feature_support(const String &p_feature) override; OS_Android(GodotJavaWrapper *p_godot_java, GodotIOJavaWrapper *p_godot_io_java, bool p_use_apk_expansion); ~OS_Android(); diff --git a/platform/linuxbsd/os_linuxbsd.cpp b/platform/linuxbsd/os_linuxbsd.cpp index 11c81be4a953..82500f83cb58 100644 --- a/platform/linuxbsd/os_linuxbsd.cpp +++ b/platform/linuxbsd/os_linuxbsd.cpp @@ -195,6 +195,10 @@ void OS_LinuxBSD::set_main_loop(MainLoop *p_main_loop) { main_loop = p_main_loop; } +String OS_LinuxBSD::get_identifier() const { + return "linuxbsd"; +} + String OS_LinuxBSD::get_name() const { #ifdef __linux__ return "Linux"; diff --git a/platform/linuxbsd/os_linuxbsd.h b/platform/linuxbsd/os_linuxbsd.h index 942351494478..8dda06b6df68 100644 --- a/platform/linuxbsd/os_linuxbsd.h +++ b/platform/linuxbsd/os_linuxbsd.h @@ -96,6 +96,7 @@ protected: virtual void set_main_loop(MainLoop *p_main_loop) override; public: + virtual String get_identifier() const override; virtual String get_name() const override; virtual String get_distribution_name() const override; virtual String get_version() const override;