Ability to change a resource UID from API

* Works for text, binary and imported resources
* Allows better clean up of duplicate files.

TODO (future PRs):

* Use this API for assigning new UIDs to copied files.
* Use this API for UID conflict on FS scanning (if more than one file has the same UID, the newer one(s) should get assigned a different UID).
This commit is contained in:
Juan Linietsky 2022-12-05 19:01:59 +01:00
parent 964fc6e15d
commit 07a964fce3
11 changed files with 256 additions and 8 deletions

View file

@ -1045,10 +1045,10 @@ void ResourceLoaderBinary::open(Ref<FileAccess> p_f, bool p_no_resources, bool p
#ifdef TOOLS_ENABLED
// Silence a warning that can happen during the initial filesystem scan due to cache being regenerated.
if (ResourceLoader::get_resource_uid(res_path) != er.uid) {
WARN_PRINT(String(res_path + ": In external resource #" + itos(i) + ", invalid UUID: " + ResourceUID::get_singleton()->id_to_text(er.uid) + " - using text path instead: " + er.path).utf8().get_data());
WARN_PRINT(String(res_path + ": In external resource #" + itos(i) + ", invalid UID: " + ResourceUID::get_singleton()->id_to_text(er.uid) + " - using text path instead: " + er.path).utf8().get_data());
}
#else
WARN_PRINT(String(res_path + ": In external resource #" + itos(i) + ", invalid UUID: " + ResourceUID::get_singleton()->id_to_text(er.uid) + " - using text path instead: " + er.path).utf8().get_data());
WARN_PRINT(String(res_path + ": In external resource #" + itos(i) + ", invalid UID: " + ResourceUID::get_singleton()->id_to_text(er.uid) + " - using text path instead: " + er.path).utf8().get_data());
#endif
}
}
@ -2209,12 +2209,130 @@ Error ResourceFormatSaverBinaryInstance::save(const String &p_path, const Ref<Re
return OK;
}
Error ResourceFormatSaverBinaryInstance::set_uid(const String &p_path, ResourceUID::ID p_uid) {
Ref<FileAccess> f = FileAccess::open(p_path, FileAccess::READ);
ERR_FAIL_COND_V_MSG(f.is_null(), ERR_CANT_OPEN, "Cannot open file '" + p_path + "'.");
Ref<FileAccess> fw;
local_path = p_path.get_base_dir();
uint8_t header[4];
f->get_buffer(header, 4);
if (header[0] == 'R' && header[1] == 'S' && header[2] == 'C' && header[3] == 'C') {
// Compressed.
Ref<FileAccessCompressed> fac;
fac.instantiate();
Error err = fac->open_after_magic(f);
ERR_FAIL_COND_V_MSG(err != OK, err, "Cannot open file '" + p_path + "'.");
f = fac;
Ref<FileAccessCompressed> facw;
facw.instantiate();
facw->configure("RSCC");
err = facw->open_internal(p_path + ".uidren", FileAccess::WRITE);
ERR_FAIL_COND_V_MSG(err, ERR_FILE_CORRUPT, "Cannot create file '" + p_path + ".uidren'.");
fw = facw;
} else if (header[0] != 'R' || header[1] != 'S' || header[2] != 'R' || header[3] != 'C') {
// Not a binary resource.
return ERR_FILE_UNRECOGNIZED;
} else {
fw = FileAccess::open(p_path + ".uidren", FileAccess::WRITE);
ERR_FAIL_COND_V_MSG(fw.is_null(), ERR_CANT_CREATE, "Cannot create file '" + p_path + ".uidren'.");
uint8_t magich[4] = { 'R', 'S', 'R', 'C' };
fw->store_buffer(magich, 4);
}
big_endian = f->get_32();
bool use_real64 = f->get_32();
f->set_big_endian(big_endian != 0); //read big endian if saved as big endian
#ifdef BIG_ENDIAN_ENABLED
fw->store_32(!big_endian);
#else
fw->store_32(big_endian);
#endif
fw->set_big_endian(big_endian != 0);
fw->store_32(use_real64); //use real64
uint32_t ver_major = f->get_32();
uint32_t ver_minor = f->get_32();
uint32_t ver_format = f->get_32();
if (ver_format < FORMAT_VERSION_CAN_RENAME_DEPS) {
fw.unref();
{
Ref<DirAccess> da = DirAccess::create(DirAccess::ACCESS_FILESYSTEM);
da->remove(p_path + ".uidren");
}
// Use the old approach.
WARN_PRINT("This file is old, so it does not support UIDs, opening and resaving '" + p_path + "'.");
return ERR_UNAVAILABLE;
}
if (ver_format > FORMAT_VERSION || ver_major > VERSION_MAJOR) {
ERR_FAIL_V_MSG(ERR_FILE_UNRECOGNIZED,
vformat("File '%s' can't be loaded, as it uses a format version (%d) or engine version (%d.%d) which are not supported by your engine version (%s).",
local_path, ver_format, ver_major, ver_minor, VERSION_BRANCH));
}
// Since we're not actually converting the file contents, leave the version
// numbers in the file untouched.
fw->store_32(ver_major);
fw->store_32(ver_minor);
fw->store_32(ver_format);
save_ustring(fw, get_ustring(f)); //type
fw->store_64(f->get_64()); //metadata offset
uint32_t flags = f->get_32();
flags |= ResourceFormatSaverBinaryInstance::FORMAT_FLAG_UIDS;
f->get_64(); // Skip previous UID
fw->store_32(flags);
fw->store_64(p_uid);
//rest of file
uint8_t b = f->get_8();
while (!f->eof_reached()) {
fw->store_8(b);
b = f->get_8();
}
f.unref();
bool all_ok = fw->get_error() == OK;
if (!all_ok) {
return ERR_CANT_CREATE;
}
fw.unref();
Ref<DirAccess> da = DirAccess::create(DirAccess::ACCESS_RESOURCES);
da->remove(p_path);
da->rename(p_path + ".uidren", p_path);
return OK;
}
Error ResourceFormatSaverBinary::save(const Ref<Resource> &p_resource, const String &p_path, uint32_t p_flags) {
String local_path = ProjectSettings::get_singleton()->localize_path(p_path);
ResourceFormatSaverBinaryInstance saver;
return saver.save(local_path, p_resource, p_flags);
}
Error ResourceFormatSaverBinary::set_uid(const String &p_path, ResourceUID::ID p_uid) {
String local_path = ProjectSettings::get_singleton()->localize_path(p_path);
ResourceFormatSaverBinaryInstance saver;
return saver.set_uid(local_path, p_uid);
}
bool ResourceFormatSaverBinary::recognize(const Ref<Resource> &p_resource) const {
return true; //all recognized
}

View file

@ -170,6 +170,7 @@ public:
RESERVED_FIELDS = 11
};
Error save(const String &p_path, const Ref<Resource> &p_resource, uint32_t p_flags = 0);
Error set_uid(const String &p_path, ResourceUID::ID p_uid);
static void write_variant(Ref<FileAccess> f, const Variant &p_property, HashMap<Ref<Resource>, int> &resource_map, HashMap<Ref<Resource>, int> &external_resources, HashMap<StringName, int> &string_map, const PropertyInfo &p_hint = PropertyInfo());
};
@ -177,6 +178,7 @@ class ResourceFormatSaverBinary : public ResourceFormatSaver {
public:
static ResourceFormatSaverBinary *singleton;
virtual Error save(const Ref<Resource> &p_resource, const String &p_path, uint32_t p_flags = 0);
virtual Error set_uid(const String &p_path, ResourceUID::ID p_uid);
virtual bool recognize(const Ref<Resource> &p_resource) const;
virtual void get_recognized_extensions(const Ref<Resource> &p_resource, List<String> *p_extensions) const;

View file

@ -31,6 +31,7 @@
#include "resource_importer.h"
#include "core/config/project_settings.h"
#include "core/io/config_file.h"
#include "core/os/os.h"
#include "core/variant/variant_parser.h"
@ -484,3 +485,18 @@ void ResourceFormatImporter::add_importer(const Ref<ResourceImporter> &p_importe
importers.push_back(p_importer);
}
}
/////
Error ResourceFormatImporterSaver::set_uid(const String &p_path, ResourceUID::ID p_uid) {
Ref<ConfigFile> cf;
cf.instantiate();
Error err = cf->load(p_path + ".import");
if (err != OK) {
return err;
}
cf->set_value("remap", "uid", ResourceUID::get_singleton()->id_to_text(p_uid));
cf->save(p_path + ".import");
return OK;
}

View file

@ -32,6 +32,7 @@
#define RESOURCE_IMPORTER_H
#include "core/io/resource_loader.h"
#include "core/io/resource_saver.h"
class ResourceImporter;
@ -149,4 +150,11 @@ public:
VARIANT_ENUM_CAST(ResourceImporter::ImportOrder);
class ResourceFormatImporterSaver : public ResourceFormatSaver {
GDCLASS(ResourceFormatImporterSaver, ResourceFormatSaver)
public:
virtual Error set_uid(const String &p_path, ResourceUID::ID p_uid) override;
};
#endif // RESOURCE_IMPORTER_H

View file

@ -47,6 +47,12 @@ Error ResourceFormatSaver::save(const Ref<Resource> &p_resource, const String &p
return (Error)res;
}
Error ResourceFormatSaver::set_uid(const String &p_path, ResourceUID::ID p_uid) {
Error err = ERR_FILE_UNRECOGNIZED;
GDVIRTUAL_CALL(_set_uid, p_path, p_uid, err);
return err;
}
bool ResourceFormatSaver::recognize(const Ref<Resource> &p_resource) const {
bool success = false;
GDVIRTUAL_CALL(_recognize, p_resource, success);
@ -85,6 +91,7 @@ bool ResourceFormatSaver::recognize_path(const Ref<Resource> &p_resource, const
void ResourceFormatSaver::_bind_methods() {
GDVIRTUAL_BIND(_save, "resource", "path", "flags");
GDVIRTUAL_BIND(_set_uid, "path", "uid");
GDVIRTUAL_BIND(_recognize, "resource");
GDVIRTUAL_BIND(_get_recognized_extensions, "resource");
GDVIRTUAL_BIND(_recognize_path, "resource", "path");
@ -146,6 +153,23 @@ Error ResourceSaver::save(const Ref<Resource> &p_resource, const String &p_path,
return err;
}
Error ResourceSaver::set_uid(const String &p_path, ResourceUID::ID p_uid) {
String path = p_path;
ERR_FAIL_COND_V_MSG(path.is_empty(), ERR_INVALID_PARAMETER, "Can't update UID to empty path. Provide non-empty path.");
Error err = ERR_FILE_UNRECOGNIZED;
for (int i = 0; i < saver_count; i++) {
err = saver[i]->set_uid(path, p_uid);
if (err == OK) {
break;
}
}
return err;
}
void ResourceSaver::set_save_callback(ResourceSavedCallback p_callback) {
save_callback = p_callback;
}

View file

@ -42,12 +42,14 @@ protected:
static void _bind_methods();
GDVIRTUAL3R(int64_t, _save, Ref<Resource>, String, uint32_t)
GDVIRTUAL2R(Error, _set_uid, String, ResourceUID::ID)
GDVIRTUAL1RC(bool, _recognize, Ref<Resource>)
GDVIRTUAL1RC(Vector<String>, _get_recognized_extensions, Ref<Resource>)
GDVIRTUAL2RC(bool, _recognize_path, Ref<Resource>, String)
public:
virtual Error save(const Ref<Resource> &p_resource, const String &p_path, uint32_t p_flags = 0);
virtual Error set_uid(const String &p_path, ResourceUID::ID p_uid);
virtual bool recognize(const Ref<Resource> &p_resource) const;
virtual void get_recognized_extensions(const Ref<Resource> &p_resource, List<String> *p_extensions) const;
virtual bool recognize_path(const Ref<Resource> &p_resource, const String &p_path) const;
@ -88,6 +90,8 @@ public:
static void add_resource_format_saver(Ref<ResourceFormatSaver> p_format_saver, bool p_at_front = false);
static void remove_resource_format_saver(Ref<ResourceFormatSaver> p_format_saver);
static Error set_uid(const String &p_path, ResourceUID::ID p_uid);
static void set_timestamp_on_save(bool p_timestamp) { timestamp_on_save = p_timestamp; }
static bool get_timestamp_on_save() { return timestamp_on_save; }

View file

@ -84,6 +84,7 @@
static Ref<ResourceFormatSaverBinary> resource_saver_binary;
static Ref<ResourceFormatLoaderBinary> resource_loader_binary;
static Ref<ResourceFormatImporter> resource_format_importer;
static Ref<ResourceFormatImporterSaver> resource_format_importer_saver;
static Ref<ResourceFormatLoaderImage> resource_format_image;
static Ref<TranslationLoaderPO> resource_format_po;
static Ref<ResourceFormatSaverCrypto> resource_format_saver_crypto;
@ -144,6 +145,9 @@ void register_core_types() {
resource_format_importer.instantiate();
ResourceLoader::add_resource_format_loader(resource_format_importer);
resource_format_importer_saver.instantiate();
ResourceSaver::add_resource_format_saver(resource_format_importer_saver);
resource_format_image.instantiate();
ResourceLoader::add_resource_format_loader(resource_format_image);
@ -389,6 +393,9 @@ void unregister_core_types() {
ResourceLoader::remove_resource_format_loader(resource_format_importer);
resource_format_importer.unref();
ResourceSaver::remove_resource_format_saver(resource_format_importer_saver);
resource_format_importer_saver.unref();
ResourceLoader::remove_resource_format_loader(resource_format_po);
resource_format_po.unref();

View file

@ -43,5 +43,13 @@
Returns [constant OK] on success, or an [enum Error] constant in case of failure.
</description>
</method>
<method name="_set_uid" qualifiers="virtual">
<return type="int" enum="Error" />
<param index="0" name="path" type="String" />
<param index="1" name="uid" type="int" />
<description>
Sets a new UID for the resource at the given [param path]. Returns [constant OK] on success, or an [enum Error] constant in case of failure.
</description>
</method>
</methods>
</class>

View file

@ -433,7 +433,7 @@ bool EditorFileSystem::_test_for_reimport(const String &p_path, bool p_only_impo
}
if (!found_uid) {
return true; //UUID not found, old format, reimport.
return true; //UID not found, old format, reimport.
}
Ref<ResourceImporter> importer = ResourceFormatImporter::get_singleton()->get_importer_by_name(importer_name);
@ -868,7 +868,7 @@ void EditorFileSystem::_scan_new_dir(EditorFileSystemDirectory *p_dir, Ref<DirAc
}
if (fc->uid == ResourceUID::INVALID_ID) {
// imported files should always have a UUID, so attempt to fetch it.
// imported files should always have a UID, so attempt to fetch it.
fi->uid = ResourceLoader::get_resource_uid(path);
}
@ -2319,14 +2319,14 @@ ResourceUID::ID EditorFileSystem::_resource_saver_get_resource_id_for_path(const
}
if (p_generate) {
return ResourceUID::get_singleton()->create_id(); // Just create a new one, we will be notified of save anyway and fetch the right UUID at that time, to keep things simple.
return ResourceUID::get_singleton()->create_id(); // Just create a new one, we will be notified of save anyway and fetch the right UID at that time, to keep things simple.
} else {
return ResourceUID::INVALID_ID;
}
} else if (fs->files[cpos]->uid != ResourceUID::INVALID_ID) {
return fs->files[cpos]->uid;
} else if (p_generate) {
return ResourceUID::get_singleton()->create_id(); // Just create a new one, we will be notified of save anyway and fetch the right UUID at that time, to keep things simple.
return ResourceUID::get_singleton()->create_id(); // Just create a new one, we will be notified of save anyway and fetch the right UID at that time, to keep things simple.
} else {
return ResourceUID::INVALID_ID;
}

View file

@ -449,10 +449,10 @@ Error ResourceLoaderText::load() {
#ifdef TOOLS_ENABLED
// Silence a warning that can happen during the initial filesystem scan due to cache being regenerated.
if (ResourceLoader::get_resource_uid(path) != uid) {
WARN_PRINT(String(res_path + ":" + itos(lines) + " - ext_resource, invalid UUID: " + uidt + " - using text path instead: " + path).utf8().get_data());
WARN_PRINT(String(res_path + ":" + itos(lines) + " - ext_resource, invalid UID: " + uidt + " - using text path instead: " + path).utf8().get_data());
}
#else
WARN_PRINT(String(res_path + ":" + itos(lines) + " - ext_resource, invalid UUID: " + uidt + " - using text path instead: " + path).utf8().get_data());
WARN_PRINT(String(res_path + ":" + itos(lines) + " - ext_resource, invalid UID: " + uidt + " - using text path instead: " + path).utf8().get_data());
#endif
}
}
@ -2237,6 +2237,35 @@ Error ResourceFormatSaverTextInstance::save(const String &p_path, const Ref<Reso
return OK;
}
Error ResourceLoaderText::set_uid(Ref<FileAccess> p_f, ResourceUID::ID p_uid) {
open(p_f, true);
ERR_FAIL_COND_V(error != OK, error);
ignore_resource_parsing = true;
Ref<FileAccess> fw;
fw = FileAccess::open(local_path + ".uidren", FileAccess::WRITE);
if (is_scene) {
fw->store_string("[gd_scene load_steps=" + itos(resources_total) + " format=" + itos(FORMAT_VERSION) + " uid=\"" + ResourceUID::get_singleton()->id_to_text(p_uid) + "\"]");
} else {
fw->store_string("[gd_resource type=\"" + res_type + "\" load_steps=" + itos(resources_total) + " format=" + itos(FORMAT_VERSION) + " uid=\"" + ResourceUID::get_singleton()->id_to_text(p_uid) + "\"]");
}
uint8_t c = f->get_8();
while (!f->eof_reached()) {
fw->store_8(c);
c = f->get_8();
}
bool all_ok = fw->get_error() == OK;
if (!all_ok) {
return ERR_CANT_CREATE;
}
return OK;
}
Error ResourceFormatSaverText::save(const Ref<Resource> &p_resource, const String &p_path, uint32_t p_flags) {
if (p_path.ends_with(".tscn") && !Ref<PackedScene>(p_resource).is_valid()) {
return ERR_FILE_UNRECOGNIZED;
@ -2246,6 +2275,35 @@ Error ResourceFormatSaverText::save(const Ref<Resource> &p_resource, const Strin
return saver.save(p_path, p_resource, p_flags);
}
Error ResourceFormatSaverText::set_uid(const String &p_path, ResourceUID::ID p_uid) {
String lc = p_path.to_lower();
if (!lc.ends_with(".tscn") && !lc.ends_with(".tres")) {
return ERR_FILE_UNRECOGNIZED;
}
String local_path = ProjectSettings::get_singleton()->localize_path(p_path);
Error err = OK;
{
Ref<FileAccess> fo = FileAccess::open(p_path, FileAccess::READ);
if (fo.is_null()) {
ERR_FAIL_V(ERR_CANT_OPEN);
}
ResourceLoaderText loader;
loader.local_path = local_path;
loader.res_path = loader.local_path;
err = loader.set_uid(fo, p_uid);
}
if (err == OK) {
Ref<DirAccess> da = DirAccess::create(DirAccess::ACCESS_RESOURCES);
da->remove(local_path);
da->rename(local_path + ".uidren", local_path);
}
return err;
}
bool ResourceFormatSaverText::recognize(const Ref<Resource> &p_resource) const {
return true; // All resources recognized!
}

View file

@ -106,6 +106,7 @@ class ResourceLoaderText {
VariantParser::ResourceParser rp;
friend class ResourceFormatLoaderText;
friend class ResourceFormatSaverText;
Error error = OK;
@ -117,6 +118,7 @@ public:
void set_local_path(const String &p_local_path);
Ref<Resource> get_resource();
Error load();
Error set_uid(Ref<FileAccess> p_f, ResourceUID::ID p_uid);
int get_stage() const;
int get_stage_count() const;
void set_translation_remapped(bool p_remapped);
@ -195,6 +197,7 @@ class ResourceFormatSaverText : public ResourceFormatSaver {
public:
static ResourceFormatSaverText *singleton;
virtual Error save(const Ref<Resource> &p_resource, const String &p_path, uint32_t p_flags = 0);
virtual Error set_uid(const String &p_path, ResourceUID::ID p_uid);
virtual bool recognize(const Ref<Resource> &p_resource) const;
virtual void get_recognized_extensions(const Ref<Resource> &p_resource, List<String> *p_extensions) const;