manager: add option to rate limit daemon-reload

Reloading is a heavy-weight operation, and currently it is not
possible to stop an orchestrator from spamming reload requests.
Add configuration options to allow rate-limiting.

man/kernel-command-line.xml

@@ -72,6 +72,8 @@
         <term><varname>systemd.machine_id=</varname></term>
         <term><varname>systemd.set_credential=</varname></term>
         <term><varname>systemd.import_credentials=</varname></term>
+        <term><varname>systemd.reload_limit_interval_sec=</varname></term>
+        <term><varname>systemd.reload_limit_burst=</varname></term>
         <listitem>
           <para>Parameters understood by the system and service
           manager to control system behavior. For details, see

man/systemd-system.conf.xml

@@ -550,6 +550,16 @@
         <para>If the value is <literal>/</literal>, only labels specified with <varname>SmackProcessLabel=</varname>
         are assigned and the compile-time default is ignored.</para></listitem>
       </varlistentry>
+
+      <varlistentry>
+        <term><varname>ReloadLimitIntervalSec=</varname></term>
+        <term><varname>ReloadLimitBurst=</varname></term>
+
+        <listitem><para>Rate limiting for daemon-reload requests. Default to unset, and any number of daemon-reload
+        operations can be requested at any time. <varname>ReloadLimitIntervalSec=</varname> takes a value in seconds
+        to configure the rate limit window, and <varname>ReloadLimitBurst=</varname> takes a positive integer to
+        configure the maximum allowed number of reloads within the configured time window.</para></listitem>
+      </varlistentry>
     </variablelist>
   </refsect1>
 

src/core/dbus-manager.c

@@ -1497,6 +1497,14 @@ static int method_reload(sd_bus_message *message, void *userdata, sd_bus_error *
         /* Write a log message noting the unit or process who requested the Reload() */
         log_reload_caller(message, m);
 
+        /* Check the rate limit after the authorization succeeds, to avoid denial-of-service issues. */
+        if (!ratelimit_below(&m->reload_ratelimit)) {
+                log_warning("Reloading request rejected due to rate limit.");
+                return sd_bus_error_setf(error,
+                                         SD_BUS_ERROR_LIMITS_EXCEEDED,
+                                         "Reload() request rejected due to rate limit.");
+        }
+
         /* Instead of sending the reply back right away, we just
          * remember that we need to and then send it after the reload
          * is finished. That way the caller knows when the reload

src/core/main.c

@@ -173,6 +173,8 @@ static size_t arg_random_seed_size;
 static int arg_default_oom_score_adjust;
 static bool arg_default_oom_score_adjust_set;
 static char *arg_default_smack_process_label;
+static usec_t arg_reload_limit_interval_sec;
+static unsigned arg_reload_limit_burst;
 
 /* A copy of the original environment block */
 static char **saved_env = NULL;
@@ -483,6 +485,28 @@ static int parse_proc_cmdline_item(const char *key, const char *value, void *dat
                 arg_random_seed = sz > 0 ? p : mfree(p);
                 arg_random_seed_size = sz;
 
+        } else if (proc_cmdline_key_streq(key, "systemd.reload_limit_interval_sec")) {
+
+                if (proc_cmdline_value_missing(key, value))
+                        return 0;
+
+                r = parse_sec(value, &arg_reload_limit_interval_sec);
+                if (r < 0) {
+                        log_warning_errno(r, "Failed to parse systemd.reload_limit_interval_sec= argument '%s', ignoring: %m", value);
+                        return 0;
+                }
+
+        } else if (proc_cmdline_key_streq(key, "systemd.reload_limit_burst")) {
+
+                if (proc_cmdline_value_missing(key, value))
+                        return 0;
+
+                r = safe_atou(value, &arg_reload_limit_burst);
+                if (r < 0) {
+                        log_warning_errno(r, "Failed to parse systemd.reload_limit_burst= argument '%s', ignoring: %m", value);
+                        return 0;
+                }
+
         } else if (streq(key, "quiet") && !value) {
 
                 if (arg_show_status == _SHOW_STATUS_INVALID)
@@ -662,6 +686,8 @@ static int parse_config_file(void) {
                 { "Manager", "CtrlAltDelBurstAction",        config_parse_emergency_action,      0,                        &arg_cad_burst_action             },
                 { "Manager", "DefaultOOMPolicy",             config_parse_oom_policy,            0,                        &arg_default_oom_policy           },
                 { "Manager", "DefaultOOMScoreAdjust",        config_parse_oom_score_adjust,      0,                        NULL                              },
+                { "Manager", "ReloadLimitIntervalSec",       config_parse_sec,                   0,                        &arg_reload_limit_interval_sec    },
+                { "Manager", "ReloadLimitBurst",             config_parse_unsigned,              0,                        &arg_reload_limit_burst           },
 #if ENABLE_SMACK
                 { "Manager", "DefaultSmackProcessLabel",     config_parse_string,                0,                        &arg_default_smack_process_label  },
 #else
@@ -762,6 +788,10 @@ static void set_manager_settings(Manager *m) {
         m->confirm_spawn = arg_confirm_spawn;
         m->service_watchdogs = arg_service_watchdogs;
         m->cad_burst_action = arg_cad_burst_action;
+        /* Note that we don't do structured initialization here, otherwise it will reset the rate limit
+         * counter on every daemon-reload. */
+        m->reload_ratelimit.interval = arg_reload_limit_interval_sec;
+        m->reload_ratelimit.burst = arg_reload_limit_burst;
 
         manager_set_watchdog(m, WATCHDOG_RUNTIME, arg_runtime_watchdog);
         manager_set_watchdog(m, WATCHDOG_REBOOT, arg_reboot_watchdog);
@@ -2451,6 +2481,9 @@ static void reset_arguments(void) {
 
         arg_default_oom_score_adjust_set = false;
         arg_default_smack_process_label = mfree(arg_default_smack_process_label);
+
+        arg_reload_limit_interval_sec = 0;
+        arg_reload_limit_burst = 0;
 }
 
 static void determine_default_oom_score_adjust(void) {

src/core/manager.h

@@ -461,6 +461,9 @@ struct Manager {
         struct restrict_fs_bpf *restrict_fs;
 
         char *default_smack_process_label;
+
+        /* Allow users to configure a rate limit for Reload() operations */
+        RateLimit reload_ratelimit;
 };
 
 static inline usec_t manager_default_timeout_abort_usec(Manager *m) {

src/core/system.conf.in

@@ -75,3 +75,5 @@
 #DefaultLimitRTTIME=
 #DefaultOOMPolicy=stop
 #DefaultSmackProcessLabel=
+#ReloadLimitIntervalSec=
+#ReloadLimitBurst=

src/core/user.conf.in

@@ -48,3 +48,5 @@
 #DefaultLimitRTPRIO=
 #DefaultLimitRTTIME=
 #DefaultSmackProcessLabel=
+#ReloadLimitIntervalSec=
+#ReloadLimitBurst

test/units/testsuite-59.sh

@@ -85,6 +85,25 @@ wait_on_state_or_fail "testservice-abort-restart-59.service" "failed" "30"
 
 systemd-analyze log-level info
 
+# Test that rate-limiting daemon-reload works
+mkdir -p /run/systemd/system.conf.d/
+cat >/run/systemd/system.conf.d/50-test-59-reload.conf <<EOF
+[Manager]
+ReloadLimitIntervalSec=9
+ReloadLimitBurst=3
+EOF
+
+# Pick up the new config
+systemctl daemon-reload
+
+# The timeout will hit (and the test will fail) if the reloads are not rate-limited
+timeout 15 bash -c 'while systemctl daemon-reload --no-block; do true; done'
+
+# Rate limit should reset after 9s
+sleep 10
+
+systemctl daemon-reload
+
 echo OK >/testok
 
 exit 0