Merge pull request #30263 from msizanoen1/fix-onboot-rotate-2

journal: Reset runtime seqnum data when flushing to system journal

src/journal/journald-server.c

@@ -42,6 +42,7 @@
 #include "journald-stream.h"
 #include "journald-syslog.h"
 #include "log.h"
+#include "memory-util.h"
 #include "missing_audit.h"
 #include "mkdir.h"
 #include "parse-util.h"
@@ -1274,6 +1275,10 @@ int server_flush_to_var(Server *s, bool require_flag_file) {
         if (!s->system_journal)
                 return 0;
 
+        /* Reset current seqnum data to avoid unnecessary rotation when switching to system journal.
+         * See issue #30092. */
+        zero(*s->seqnum);
+
         log_debug("Flushing to %s...", s->system_storage.path);
 
         start = now(CLOCK_MONOTONIC);
@@ -1347,12 +1352,29 @@ finish:
         if (s->system_journal)
                 journal_file_post_change(s->system_journal);
 
+        /* Save parent directories of runtime journals before closing runtime journals. */
+        _cleanup_strv_free_ char **dirs = NULL;
+        (void) journal_get_directories(j, &dirs);
+
+        /* First, close all runtime journals opened in the above. */
+        sd_journal_close(j);
+
+        /* Offline and close the 'main' runtime journal file. */
         s->runtime_journal = journal_file_offline_close(s->runtime_journal);
 
-        if (r >= 0)
+        /* Remove the runtime directory if the all entries are successfully flushed to /var/. */
+        if (r >= 0) {
                 (void) rm_rf(s->runtime_storage.path, REMOVE_ROOT);
 
-        sd_journal_close(j);
+                /* The initrd may have a different machine ID from the host's one. Typically, that happens
+                 * when our tests running on qemu, as the host's initrd is picked as is without updating
+                 * the machine ID in the initrd with the one used in the image. Even in such the case, the
+                 * runtime journals in the subdirectory named with the initrd's machine ID are flushed to
+                 * the persistent journal. To make not the runtime journal flushed multiple times, let's
+                 * also remove the runtime directories. */
+                STRV_FOREACH(p, dirs)
+                        (void) rm_rf(*p, REMOVE_ROOT);
+        }
 
         server_driver_message(s, 0, NULL,
                               LOG_MESSAGE("Time spent on flushing to %s is %s for %u entries.",

src/libsystemd/sd-journal/journal-internal.h

@@ -138,6 +138,7 @@ struct sd_journal {
 
 char *journal_make_match_string(sd_journal *j);
 void journal_print_header(sd_journal *j);
+int journal_get_directories(sd_journal *j, char ***ret);
 
 #define JOURNAL_FOREACH_DATA_RETVAL(j, data, l, retval)                     \
         for (sd_journal_restart_data(j); ((retval) = sd_journal_enumerate_data((j), &(data), &(l))) > 0; )

src/libsystemd/sd-journal/sd-journal.c

@@ -1594,6 +1594,41 @@ error:
         return r;
 }
 
+int journal_get_directories(sd_journal *j, char ***ret) {
+        _cleanup_strv_free_ char **paths = NULL;
+        JournalFile *f;
+        const char *p;
+        size_t n = SIZE_MAX;
+        int r;
+
+        assert(j);
+        assert(ret);
+
+        /* This returns parent directories of opened journal files. */
+
+        ORDERED_HASHMAP_FOREACH_KEY(f, p, j->files) {
+                _cleanup_free_ char *d = NULL;
+
+                /* Ignore paths generated from fd. */
+                if (path_startswith(p, "/proc/"))
+                        continue;
+
+                r = path_extract_directory(p, &d);
+                if (r < 0)
+                        return r;
+
+                if (path_strv_contains(paths, d))
+                        continue;
+
+                r = strv_extend_with_size(&paths, &n, d);
+                if (r < 0)
+                        return r;
+        }
+
+        *ret = TAKE_PTR(paths);
+        return 0;
+}
+
 static int add_file_by_name(
                 sd_journal *j,
                 const char *prefix,

test/units/testsuite-09.journal.sh

@@ -70,3 +70,47 @@ journalctl --list-boots -o json | jq -r '.[] | [.index, .boot_id, .first_entry,
             assert_eq "$entry_ts" "$last_ts"
         fi
     done
+
+verify_seqnum() {
+    if [[ "$REBOOT_COUNT" -ne "$NUM_REBOOT" ]]; then
+        return 0
+    fi
+
+    journalctl --flush
+    journalctl --sync
+
+    ls -lR /var/log/journal/
+    ls -lR /run/log/journal/
+
+    journalctl --system --header
+
+    (! journalctl --system -q -o short-monotonic -u systemd-journald.service --grep 'rotating')
+
+    set +x
+    previous_seqnum=0
+    previous_seqnum_id=
+    previous_boot_id=
+    journalctl --system -q -o json | jq -r '[.__SEQNUM, .__SEQNUM_ID, ._BOOT_ID] | @tsv' |
+        while read -r seqnum seqnum_id boot_id; do
+
+            if [[ -n "$previous_seqnum_id" ]]; then
+                if ! test "$seqnum" -gt "$previous_seqnum"; then
+                    echo "seqnum=$seqnum is not greater than previous_seqnum=$previous_seqnum"
+                    echo "seqnum_id=$seqnum_id, previous_seqnum_id=$previous_seqnum_id"
+                    echo "boot_id=$boot_id, previous_boot_id=$previous_boot_id"
+                    return 1
+                fi
+
+                assert_eq "$seqnum_id" "$previous_seqnum_id"
+            fi
+
+            previous_seqnum="$seqnum"
+            previous_seqnum_id="$seqnum_id"
+            previous_boot_id="$boot_id"
+        done
+    set -x
+
+    return 0
+}
+
+verify_seqnum

test/units/testsuite-09.sh

@@ -3,7 +3,7 @@
 set -eux
 set -o pipefail
 
-NUM_REBOOT=4
+export NUM_REBOOT=4
 
 # shellcheck source=test/units/test-control.sh
 . "$(dirname "$0")"/test-control.sh