localed: Run locale-gen if available to generate missing locale

This change improves integration with distributions using locale-gen to
generate missing locale on-demand, like Debian-based distributions
(Debian/Ubuntu/PureOS/Tanglu/...) and Arch Linux.
We only ever enable new locales for generation, and never disable them.
Furthermore, we only generate UTF-8 locale.

This feature is only used if explicitly enabled at compile-time, and
will also be inert at runtime if the locale-gen binary is missing.

meson.build

@@ -832,6 +832,14 @@ if default_locale == ''
 endif
 conf.set_quoted('SYSTEMD_DEFAULT_LOCALE', default_locale)
 
+localegen_path = get_option('localegen-path')
+have = false
+if localegen_path != ''
+        conf.set_quoted('LOCALEGEN_PATH', localegen_path)
+        have = true
+endif
+conf.set10('HAVE_LOCALEGEN', have)
+
 conf.set_quoted('GETTEXT_PACKAGE', meson.project_name())
 
 service_watchdog = get_option('service-watchdog')

meson_options.txt

@@ -240,6 +240,8 @@ option('gshadow', type : 'boolean',
        description : 'support for shadow group')
 option('default-locale', type : 'string', value : '',
        description : 'default locale used when /etc/locale.conf does not exist')
+option('localegen-path', type : 'string', value : '',
+       description : 'absolute path to the locale-gen binary in case the system is using locale-gen')
 option('service-watchdog', type : 'string', value : '3min',
        description : 'default watchdog setting for systemd services')
 

src/locale/keymap-util.c

@@ -6,18 +6,21 @@
 #include <unistd.h>
 
 #include "bus-polkit.h"
+#include "copy.h"
 #include "env-file-label.h"
 #include "env-file.h"
 #include "env-util.h"
 #include "fd-util.h"
 #include "fileio-label.h"
 #include "fileio.h"
+#include "fs-util.h"
 #include "kbd-util.h"
 #include "keymap-util.h"
 #include "locale-util.h"
 #include "macro.h"
 #include "mkdir.h"
 #include "nulstr-util.h"
+#include "process-util.h"
 #include "string-util.h"
 #include "strv.h"
 #include "tmpfile-util.h"
@@ -780,3 +783,211 @@ int x11_convert_to_vconsole(Context *c) {
 
         return modified;
 }
+
+bool locale_gen_check_available(void) {
+#if HAVE_LOCALEGEN
+        if (access(LOCALEGEN_PATH, X_OK) < 0) {
+                if (errno != ENOENT)
+                        log_warning_errno(errno, "Unable to determine whether " LOCALEGEN_PATH " exists and is executable, assuming it is not: %m");
+                return false;
+        }
+        if (access("/etc/locale.gen", F_OK) < 0) {
+                if (errno != ENOENT)
+                        log_warning_errno(errno, "Unable to determine whether /etc/locale.gen exists, assuming it does not: %m");
+                return false;
+        }
+        return true;
+#else
+        return false;
+#endif
+}
+
+#if HAVE_LOCALEGEN
+static bool locale_encoding_is_utf8_or_unspecified(const char *locale) {
+        const char *c = strchr(locale, '.');
+        return !c || strcaseeq(c, ".UTF-8") || strcasestr(locale, ".UTF-8@");
+}
+
+static int locale_gen_locale_supported(const char *locale_entry) {
+        /* Returns an error valus <= 0 if the locale-gen entry is invalid or unsupported,
+         * 1 in case the locale entry is valid, and -EOPNOTSUPP specifically in case
+         * the distributor has not provided us with a SUPPORTED file to check
+         * locale for validity. */
+
+        _cleanup_fclose_ FILE *f = NULL;
+        int r;
+
+        assert(locale_entry);
+
+        /* Locale templates without country code are never supported */
+        if (!strstr(locale_entry, "_"))
+                return -EINVAL;
+
+        f = fopen("/usr/share/i18n/SUPPORTED", "re");
+        if (!f) {
+                if (errno == ENOENT)
+                        return log_debug_errno(SYNTHETIC_ERRNO(EOPNOTSUPP),
+                                               "Unable to check validity of locale entry %s: /usr/share/i18n/SUPPORTED does not exist",
+                                               locale_entry);
+                return -errno;
+        }
+
+        for (;;) {
+                _cleanup_free_ char *line = NULL;
+
+                r = read_line(f, LONG_LINE_MAX, &line);
+                if (r < 0)
+                        return log_debug_errno(r, "Failed to read /usr/share/i18n/SUPPORTED: %m");
+                if (r == 0)
+                        return 0;
+
+                line = strstrip(line);
+                if (strcaseeq_ptr(line, locale_entry))
+                        return 1;
+        }
+}
+#endif
+
+int locale_gen_enable_locale(const char *locale) {
+#if HAVE_LOCALEGEN
+        _cleanup_fclose_ FILE *fr = NULL, *fw = NULL;
+        _cleanup_(unlink_and_freep) char *temp_path = NULL;
+        _cleanup_free_ char *locale_entry = NULL;
+        bool locale_enabled = false, first_line = false;
+        bool write_new = false;
+        int r;
+
+        if (isempty(locale))
+                return 0;
+
+        if (locale_encoding_is_utf8_or_unspecified(locale)) {
+                locale_entry = strjoin(locale, " UTF-8");
+                if (!locale_entry)
+                        return -ENOMEM;
+        } else
+                return -ENOEXEC; /* We do not process non-UTF-8 locale */
+
+        r = locale_gen_locale_supported(locale_entry);
+        if (r == 0)
+                return -EINVAL;
+        if (r < 0 && r != -EOPNOTSUPP)
+                return r;
+
+        fr = fopen("/etc/locale.gen", "re");
+        if (!fr) {
+                if (errno != ENOENT)
+                        return -errno;
+                write_new = true;
+        }
+
+        r = fopen_temporary("/etc/locale.gen", &fw, &temp_path);
+        if (r < 0)
+                return r;
+
+        if (write_new)
+                (void) fchmod(fileno(fw), 0644);
+        else {
+                /* apply mode & xattrs of the original file to new file */
+                r = copy_access(fileno(fr), fileno(fw));
+                if (r < 0)
+                        return r;
+                r = copy_xattr(fileno(fr), fileno(fw));
+                if (r < 0)
+                        return r;
+        }
+
+        if (!write_new) {
+                /* The config file ends with a line break, which we do not want to include before potentially appending a new locale
+                * instead of uncommenting an existing line. By prepending linebreaks, we can avoid buffering this file but can still write
+                * a nice config file without empty lines */
+                first_line = true;
+                for (;;) {
+                        _cleanup_free_ char *line = NULL;
+                        char *line_locale;
+
+                        r = read_line(fr, LONG_LINE_MAX, &line);
+                        if (r < 0)
+                                return r;
+                        if (r == 0)
+                                break;
+
+                        if (locale_enabled) {
+                                /* Just complete writing the file if the new locale was already enabled */
+                                if (!first_line)
+                                        fputc('\n', fw);
+                                fputs(line, fw);
+                                first_line = false;
+                                continue;
+                        }
+
+                        line = strstrip(line);
+                        if (isempty(line)) {
+                                fputc('\n', fw);
+                                first_line = false;
+                                continue;
+                        }
+
+                        line_locale = line;
+                        if (line_locale[0] == '#')
+                                line_locale = strstrip(line_locale + 1);
+                        else if (strcaseeq_ptr(line_locale, locale_entry))
+                                return 0; /* the file already had our locale activated, so skip updating it */
+
+                        if (strcaseeq_ptr(line_locale, locale_entry)) {
+                                /* Uncomment existing line for new locale */
+                                if (!first_line)
+                                        fputc('\n', fw);
+                                fputs(locale_entry, fw);
+                                locale_enabled = true;
+                                first_line = false;
+                                continue;
+                        }
+
+                        /* The line was not for the locale we want to enable, just copy it */
+                        if (!first_line)
+                                fputc('\n', fw);
+                        fputs(line, fw);
+                        first_line = false;
+                }
+        }
+
+        /* Add locale to enable to the end of the file if it was not found as commented line */
+        if (!locale_enabled) {
+                if (!write_new)
+                        fputc('\n', fw);
+                fputs(locale_entry, fw);
+        }
+        fputc('\n', fw);
+
+        r = fflush_sync_and_check(fw);
+        if (r < 0)
+                return r;
+
+        if (rename(temp_path, "/etc/locale.gen") < 0)
+                return -errno;
+        temp_path = mfree(temp_path);
+
+        return 0;
+#else
+        return -EOPNOTSUPP;
+#endif
+}
+
+int locale_gen_run(void) {
+#if HAVE_LOCALEGEN
+        pid_t pid;
+        int r;
+
+        r = safe_fork("(sd-localegen)", FORK_RESET_SIGNALS|FORK_RLIMIT_NOFILE_SAFE|FORK_CLOSE_ALL_FDS|FORK_LOG|FORK_WAIT, &pid);
+        if (r < 0)
+                return r;
+        if (r == 0) {
+                execl(LOCALEGEN_PATH, LOCALEGEN_PATH, NULL);
+                _exit(EXIT_FAILURE);
+        }
+
+        return 0;
+#else
+        return -EOPNOTSUPP;
+#endif
+}

src/locale/keymap-util.h

@@ -42,3 +42,7 @@ int x11_convert_to_vconsole(Context *c);
 int x11_write_data(Context *c);
 void locale_simplify(char *locale[_VARIABLE_LC_MAX]);
 int locale_write_data(Context *c, char ***settings);
+
+bool locale_gen_check_available(void);
+int locale_gen_enable_locale(const char *locale);
+int locale_gen_run(void);

src/locale/localectl.c

@@ -26,6 +26,9 @@
 #include "verbs.h"
 #include "virt.h"
 
+/* Enough time for locale-gen to finish server-side (in case it is in use) */
+#define LOCALE_SLOW_BUS_CALL_TIMEOUT_USEC (2*USEC_PER_MINUTE)
+
 static PagerFlags arg_pager_flags = 0;
 static bool arg_ask_password = true;
 static BusTransport arg_transport = BUS_TRANSPORT_LOCAL;
@@ -176,7 +179,8 @@ static int set_locale(int argc, char **argv, void *userdata) {
         if (r < 0)
                 return bus_log_create_error(r);
 
-        r = sd_bus_call(bus, m, 0, &error, NULL);
+        /* We use a longer timeout for the method call in case localed is running locale-gen */
+        r = sd_bus_call(bus, m, LOCALE_SLOW_BUS_CALL_TIMEOUT_USEC, &error, NULL);
         if (r < 0)
                 return log_error_errno(r, "Failed to issue method call: %s", bus_error_message(&error, r));
 

src/locale/localed.c

@@ -262,6 +262,7 @@ static int property_get_xkb(
 static int process_locale_list_item(
                 const char *assignment,
                 char *new_locale[static _VARIABLE_LC_MAX],
+                bool use_localegen,
                 sd_bus_error *error) {
 
         assert(assignment);
@@ -283,7 +284,7 @@ static int process_locale_list_item(
 
                 if (!locale_is_valid(e))
                         return sd_bus_error_setf(error, SD_BUS_ERROR_INVALID_ARGS, "Locale %s is not valid, refusing.", e);
-                if (locale_is_installed(e) <= 0)
+                if (!use_localegen && locale_is_installed(e) <= 0)
                         return sd_bus_error_setf(error, SD_BUS_ERROR_INVALID_ARGS, "Locale %s not installed, refusing.", e);
                 if (new_locale[p])
                         return sd_bus_error_setf(error, SD_BUS_ERROR_INVALID_ARGS, "Locale variable %s set twice, refusing.", name);
@@ -298,6 +299,47 @@ static int process_locale_list_item(
         return sd_bus_error_setf(error, SD_BUS_ERROR_INVALID_ARGS, "Locale assignment %s not valid, refusing.", assignment);
 }
 
+static int locale_gen_process_locale(char *new_locale[static _VARIABLE_LC_MAX],
+                                     sd_bus_error *error) {
+        int r;
+        assert(new_locale);
+
+        for (LocaleVariable p = 0; p < _VARIABLE_LC_MAX; p++) {
+                if (p == VARIABLE_LANGUAGE)
+                        continue;
+                if (isempty(new_locale[p]))
+                        continue;
+                if (locale_is_installed(new_locale[p]))
+                        continue;
+
+                r = locale_gen_enable_locale(new_locale[p]);
+                if (r == -ENOEXEC) {
+                        log_error_errno(r, "Refused to enable locale for generation: %m");
+                        return sd_bus_error_setf(error,
+                                                 SD_BUS_ERROR_INVALID_ARGS,
+                                                 "Specified locale is not installed and non-UTF-8 locale will not be auto-generated: %s",
+                                                 new_locale[p]);
+                } else if (r == -EINVAL) {
+                        log_error_errno(r, "Failed to enable invalid locale %s for generation.", new_locale[p]);
+                        return sd_bus_error_setf(error,
+                                                 SD_BUS_ERROR_INVALID_ARGS,
+                                                 "Can not enable locale generation for invalid locale: %s",
+                                                 new_locale[p]);
+                } else if (r < 0) {
+                        log_error_errno(r, "Failed to enable locale for generation: %m");
+                        return sd_bus_error_set_errnof(error, r, "Failed to enable locale generation: %m");
+                }
+
+                r = locale_gen_run();
+                if (r < 0) {
+                        log_error_errno(r, "Failed to generate locale: %m");
+                        return sd_bus_error_set_errnof(error, r, "Failed to generate locale: %m");
+                }
+        }
+
+        return 0;
+}
+
 static int method_set_locale(sd_bus_message *m, void *userdata, sd_bus_error *error) {
         _cleanup_(locale_variables_freep) char *new_locale[_VARIABLE_LC_MAX] = {};
         _cleanup_strv_free_ char **settings = NULL, **l = NULL;
@@ -305,6 +347,7 @@ static int method_set_locale(sd_bus_message *m, void *userdata, sd_bus_error *er
         bool modified = false;
         int interactive, r;
         char **i;
+        bool use_localegen;
 
         assert(m);
         assert(c);
@@ -317,11 +360,13 @@ static int method_set_locale(sd_bus_message *m, void *userdata, sd_bus_error *er
         if (r < 0)
                 return r;
 
+        use_localegen = locale_gen_check_available();
+
         /* If single locale without variable name is provided, then we assume it is LANG=. */
         if (strv_length(l) == 1 && !strchr(l[0], '=')) {
                 if (!locale_is_valid(l[0]))
                         return sd_bus_error_setf(error, SD_BUS_ERROR_INVALID_ARGS, "Invalid locale specification: %s", l[0]);
-                if (locale_is_installed(l[0]) <= 0)
+                if (!use_localegen && locale_is_installed(l[0]) <= 0)
                         return sd_bus_error_setf(error, SD_BUS_ERROR_INVALID_ARGS, "Specified locale is not installed: %s", l[0]);
 
                 new_locale[VARIABLE_LANG] = strdup(l[0]);
@@ -333,7 +378,7 @@ static int method_set_locale(sd_bus_message *m, void *userdata, sd_bus_error *er
 
         /* Check whether a variable is valid */
         STRV_FOREACH(i, l) {
-                r = process_locale_list_item(*i, new_locale, error);
+                r = process_locale_list_item(*i, new_locale, use_localegen, error);
                 if (r < 0)
                         return r;
         }
@@ -392,9 +437,17 @@ static int method_set_locale(sd_bus_message *m, void *userdata, sd_bus_error *er
         if (r == 0)
                 return 1; /* No authorization for now, but the async polkit stuff will call us again when it has it */
 
+        /* Generate locale in case it is missing and the system is using locale-gen */
+        if (use_localegen) {
+                r = locale_gen_process_locale(new_locale, error);
+                if (r < 0)
+                        return r;
+        }
+
         for (LocaleVariable p = 0; p < _VARIABLE_LC_MAX; p++)
                 free_and_replace(c->locale[p], new_locale[p]);
 
+        /* Write locale configuration */
         r = locale_write_data(c, &settings);
         if (r < 0) {
                 log_error_errno(r, "Failed to set locale: %m");