system/serenity
system/serenity
1
0
Fork 0
mirror of https://github.com/SerenityOS/serenity synced 2024-07-23 02:55:15 +00:00
Code Issues Actions 4 Packages Projects Releases Wiki Activity

LibUnicode: Do not limit language display names to known locales

Browse source 
Currently, the UnicodeLocale generator collects a list of known locales
from the CLDR before processing language display names. For each locale,
the identifier is broken into language, script, and region subtags, and
we create a list of seen languages. When processing display names, we
skip languages we hadn't seen in that first step.

This is insufficient for language display names like "en-GB", which do
not have an locale entry in the CLDR, and thus are skipped. So instead,
create the list of known languages by actually reading through the list
of languages which have a display name.
This commit is contained in:
Timothy Flynn 2022-01-13 14:02:12 -05:00 committed by Linus Groh
parent b0671ceb74
commit c86f7a675d
2 changed files with 35 additions and 7 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

40
Meta/Lagom/Tools/CodeGenerators/LibUnicode/GenerateUnicodeLocale.cpp
View file

@ -21,8 +21,8 @@
#include <LibCore/DirIterator.h>
#include <LibCore/File.h>
using StringIndexType = u16;
constexpr auto s_string_index_type = "u16"sv;
using StringIndexType = u32;
constexpr auto s_string_index_type = "u32"sv;
using DisplayPatternIndexType = u8;
constexpr auto s_display_pattern_index_type = "u8"sv;
@ -319,8 +319,6 @@ static ErrorOr<void> parse_identity(String locale_path, UnicodeLocaleData& local
auto const& variant_string = identity_object.as_object().get("variant"sv);
locale.language = language_string.as_string();
if (!locale_data.languages.contains_slow(locale.language))
locale_data.languages.append(locale.language);
if (territory_string.is_string()) {
locale.territory = territory_string.as_string();
@ -366,6 +364,27 @@ static ErrorOr<void> parse_locale_display_patterns(String locale_path, UnicodeLo
return {};
}
static ErrorOr<void> preprocess_languages(String locale_path, UnicodeLocaleData& locale_data)
{
LexicalPath languages_path(move(locale_path));
languages_path = languages_path.append("languages.json"sv);
auto languages_file = TRY(Core::File::open(languages_path.string(), Core::OpenMode::ReadOnly));
auto locale_languages = TRY(JsonValue::from_string(languages_file->read_all()));
auto const& main_object = locale_languages.as_object().get("main"sv);
auto const& locale_object = main_object.as_object().get(languages_path.parent().basename());
auto const& locale_display_names_object = locale_object.as_object().get("localeDisplayNames"sv);
auto const& languages_object = locale_display_names_object.as_object().get("languages"sv);
languages_object.as_object().for_each_member([&](auto const& key, auto const&) {
if (!key.contains("-alt-"sv) && !locale_data.languages.contains_slow(key))
locale_data.languages.append(key);
});
return {};
}
static ErrorOr<void> parse_locale_languages(String locale_path, UnicodeLocaleData& locale_data, Locale& locale)
{
LexicalPath languages_path(move(locale_path));
@ -383,8 +402,11 @@ static ErrorOr<void> parse_locale_languages(String locale_path, UnicodeLocaleDat
languages.resize(locale_data.languages.size());
languages_object.as_object().for_each_member([&](auto const& key, JsonValue const& value) {
if (auto index = locale_data.languages.find_first_index(key); index.has_value())
languages[*index] = locale_data.unique_strings.ensure(value.as_string());
if (key.contains("-alt-"sv))
return;
auto index = locale_data.languages.find_first_index(key).value();
languages[index] = locale_data.unique_strings.ensure(value.as_string());
});
locale.languages = locale_data.unique_language_lists.ensure(move(languages));
@ -802,6 +824,7 @@ static ErrorOr<void> define_aliases_without_scripts(UnicodeLocaleData& locale_da
static ErrorOr<void> parse_all_locales(String core_path, String locale_names_path, String misc_path, String numbers_path, String dates_path, UnicodeLocaleData& locale_data)
{
auto identity_iterator = TRY(path_to_dir_iterator(locale_names_path));
auto preprocess_iterator = TRY(path_to_dir_iterator(locale_names_path));
auto locale_names_iterator = TRY(path_to_dir_iterator(move(locale_names_path)));
auto misc_iterator = TRY(path_to_dir_iterator(move(misc_path)));
auto numbers_iterator = TRY(path_to_dir_iterator(move(numbers_path)));
@ -835,6 +858,11 @@ static ErrorOr<void> parse_all_locales(String core_path, String locale_names_pat
TRY(parse_identity(locale_path, locale_data, locale));
}
while (preprocess_iterator.has_next()) {
auto locale_path = TRY(next_path_from_dir_iterator(preprocess_iterator));
TRY(preprocess_languages(locale_path, locale_data));
}
quick_sort(locale_data.languages);
quick_sort(locale_data.territories);
quick_sort(locale_data.scripts);

2
Userland/Libraries/LibUnicode/Forward.h
View file

@ -25,7 +25,7 @@ enum class GeneralCategory : u8;
enum class HourCycle : u8;
enum class HourCycleRegion : u8;
enum class Key : u8;
enum class Language : u8;
enum class Language : u16;
enum class ListPatternStyle : u8;
enum class ListPatternType : u8;
enum class Locale : u16;