2021-10-07 20:25:13 +00:00
|
|
|
/*
|
|
|
|
Copyright 2020 Google LLC
|
|
|
|
|
|
|
|
Use of this source code is governed by a BSD-style
|
|
|
|
license that can be found in the LICENSE file or at
|
|
|
|
https://developers.google.com/open-source/licenses/bsd
|
|
|
|
*/
|
|
|
|
|
|
|
|
#include "stack.h"
|
|
|
|
|
|
|
|
#include "system.h"
|
|
|
|
|
|
|
|
#include "reftable-reader.h"
|
|
|
|
#include "merged.h"
|
|
|
|
#include "basics.h"
|
|
|
|
#include "record.h"
|
|
|
|
#include "test_framework.h"
|
|
|
|
#include "reftable-tests.h"
|
2021-12-23 19:29:50 +00:00
|
|
|
#include "reader.h"
|
2021-10-07 20:25:13 +00:00
|
|
|
|
|
|
|
#include <sys/types.h>
|
|
|
|
#include <dirent.h>
|
|
|
|
|
|
|
|
static void clear_dir(const char *dirname)
|
|
|
|
{
|
|
|
|
struct strbuf path = STRBUF_INIT;
|
|
|
|
strbuf_addstr(&path, dirname);
|
|
|
|
remove_dir_recursively(&path, 0);
|
|
|
|
strbuf_release(&path);
|
|
|
|
}
|
|
|
|
|
|
|
|
static int count_dir_entries(const char *dirname)
|
|
|
|
{
|
|
|
|
DIR *dir = opendir(dirname);
|
|
|
|
int len = 0;
|
|
|
|
struct dirent *d;
|
2022-05-02 17:09:21 +00:00
|
|
|
if (!dir)
|
2021-10-07 20:25:13 +00:00
|
|
|
return 0;
|
|
|
|
|
|
|
|
while ((d = readdir(dir))) {
|
reftable: fix tests being broken by NFS' delete-after-close semantics
It was reported that the reftable unit tests in t0032 fail with the
following assertion when running on top of NFS:
running test_reftable_stack_compaction_concurrent_clean
reftable/stack_test.c: 1063: failed assertion count_dir_entries(dir) == 2
Aborted
Setting a breakpoint immediately before the assertion in fact shows the
following list of files:
./stack_test-1027.QJBpnd
./stack_test-1027.QJBpnd/0x000000000001-0x000000000003-dad7ac80.ref
./stack_test-1027.QJBpnd/.nfs000000000001729f00001e11
./stack_test-1027.QJBpnd/tables.list
Note the weird ".nfs*" file? This file is maintained by NFS clients in
order to emulate delete-after-last-close semantics that we rely on in
the reftable code [1]. Instead of unlinking the file right away and
keeping it open in the client, the NFS client will rename it to ".nfs*"
and then delete that temporary file when the last reference to it gets
dropped. Quoting the NFS FAQ:
> D2. What is a "silly rename"? Why do these .nfsXXXXX files keep
> showing up?
>
> A. Unix applications often open a scratch file and then unlink it.
> They do this so that the file is not visible in the file system name
> space to any other applications, and so that the system will
> automatically clean up (delete) the file when the application exits.
> This is known as "delete on last close", and is a tradition among
> Unix applications.
>
> Because of the design of the NFS protocol, there is no way for a
> file to be deleted from the name space but still remain in use by an
> application. Thus NFS clients have to emulate this using what
> already exists in the protocol. If an open file is unlinked, an NFS
> client renames it to a special name that looks like ".nfsXXXXX".
> This "hides" the file while it remains in use. This is known as a
> "silly rename." Note that NFS servers have nothing to do with this
> behavior.
This of course throws off the assertion that we got exactly two files in
that directory.
The test in question triggers this behaviour by holding two open file
descriptors to the "tables.list" file. One of the references is because
we are about to append to the stack, whereas the other reference is
because we want to compact it. As the compaction has just finished we
already rewrote "tables.list" to point to the new contents, but the
other file descriptor pointing to the old version is still open. Thus we
trigger the delete-after-last-close emulation.
Furthermore, it was reported that this behaviour only triggers with
4f36b8597c (reftable/stack: fix race in up-to-date check, 2024-01-18).
This is expected as well because it is the first point in time where we
actually keep the "tables.list" file descriptor open for the stat cache.
Fix this bug by skipping over any files that start with a leading dot
when counting files. While we could explicitly check for a prefix of
".nfs", other network file systems like SMB for example do the same
trickery but with a ".smb" prefix. In any case though, this loosening of
the assertion should be fine given that the reftable library would never
write files with leading dots by itself.
[1]: https://nfs.sourceforge.net/#faq_d2
Reported-by: Chuck Lever <chuck.lever@oracle.com>
Signed-off-by: Patrick Steinhardt <ps@pks.im>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2024-03-21 15:39:52 +00:00
|
|
|
/*
|
|
|
|
* Besides skipping over "." and "..", we also need to
|
|
|
|
* skip over other files that have a leading ".". This
|
|
|
|
* is due to behaviour of NFS, which will rename files
|
|
|
|
* to ".nfs*" to emulate delete-on-last-close.
|
|
|
|
*
|
|
|
|
* In any case this should be fine as the reftable
|
|
|
|
* library will never write files with leading dots
|
|
|
|
* anyway.
|
|
|
|
*/
|
|
|
|
if (starts_with(d->d_name, "."))
|
2021-10-07 20:25:13 +00:00
|
|
|
continue;
|
|
|
|
len++;
|
|
|
|
}
|
|
|
|
closedir(dir);
|
|
|
|
return len;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Work linenumber into the tempdir, so we can see which tests forget to
|
|
|
|
* cleanup.
|
|
|
|
*/
|
|
|
|
static char *get_tmp_template(int linenumber)
|
|
|
|
{
|
|
|
|
const char *tmp = getenv("TMPDIR");
|
|
|
|
static char template[1024];
|
|
|
|
snprintf(template, sizeof(template) - 1, "%s/stack_test-%d.XXXXXX",
|
|
|
|
tmp ? tmp : "/tmp", linenumber);
|
|
|
|
return template;
|
|
|
|
}
|
|
|
|
|
|
|
|
static char *get_tmp_dir(int linenumber)
|
|
|
|
{
|
|
|
|
char *dir = get_tmp_template(linenumber);
|
|
|
|
EXPECT(mkdtemp(dir));
|
|
|
|
return dir;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void test_read_file(void)
|
|
|
|
{
|
|
|
|
char *fn = get_tmp_template(__LINE__);
|
|
|
|
int fd = mkstemp(fn);
|
|
|
|
char out[1024] = "line1\n\nline2\nline3";
|
|
|
|
int n, err;
|
|
|
|
char **names = NULL;
|
2024-06-07 06:37:39 +00:00
|
|
|
const char *want[] = { "line1", "line2", "line3" };
|
2021-10-07 20:25:13 +00:00
|
|
|
int i = 0;
|
|
|
|
|
|
|
|
EXPECT(fd > 0);
|
2023-12-11 09:07:38 +00:00
|
|
|
n = write_in_full(fd, out, strlen(out));
|
2021-10-07 20:25:13 +00:00
|
|
|
EXPECT(n == strlen(out));
|
|
|
|
err = close(fd);
|
|
|
|
EXPECT(err >= 0);
|
|
|
|
|
|
|
|
err = read_lines(fn, &names);
|
|
|
|
EXPECT_ERR(err);
|
|
|
|
|
|
|
|
for (i = 0; names[i]; i++) {
|
|
|
|
EXPECT(0 == strcmp(want[i], names[i]));
|
|
|
|
}
|
|
|
|
free_names(names);
|
2022-01-20 15:12:04 +00:00
|
|
|
(void) remove(fn);
|
2021-10-07 20:25:13 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static void test_parse_names(void)
|
|
|
|
{
|
|
|
|
char buf[] = "line\n";
|
|
|
|
char **names = NULL;
|
|
|
|
parse_names(buf, strlen(buf), &names);
|
|
|
|
|
|
|
|
EXPECT(NULL != names[0]);
|
|
|
|
EXPECT(0 == strcmp(names[0], "line"));
|
|
|
|
EXPECT(NULL == names[1]);
|
|
|
|
free_names(names);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void test_names_equal(void)
|
|
|
|
{
|
2024-06-07 06:37:39 +00:00
|
|
|
const char *a[] = { "a", "b", "c", NULL };
|
|
|
|
const char *b[] = { "a", "b", "d", NULL };
|
|
|
|
const char *c[] = { "a", "b", NULL };
|
2021-10-07 20:25:13 +00:00
|
|
|
|
|
|
|
EXPECT(names_equal(a, a));
|
|
|
|
EXPECT(!names_equal(a, b));
|
|
|
|
EXPECT(!names_equal(a, c));
|
|
|
|
}
|
|
|
|
|
|
|
|
static int write_test_ref(struct reftable_writer *wr, void *arg)
|
|
|
|
{
|
|
|
|
struct reftable_ref_record *ref = arg;
|
|
|
|
reftable_writer_set_limits(wr, ref->update_index, ref->update_index);
|
|
|
|
return reftable_writer_add_ref(wr, ref);
|
|
|
|
}
|
|
|
|
|
|
|
|
struct write_log_arg {
|
|
|
|
struct reftable_log_record *log;
|
|
|
|
uint64_t update_index;
|
|
|
|
};
|
|
|
|
|
|
|
|
static int write_test_log(struct reftable_writer *wr, void *arg)
|
|
|
|
{
|
|
|
|
struct write_log_arg *wla = arg;
|
|
|
|
|
|
|
|
reftable_writer_set_limits(wr, wla->update_index, wla->update_index);
|
|
|
|
return reftable_writer_add_log(wr, wla->log);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void test_reftable_stack_add_one(void)
|
|
|
|
{
|
|
|
|
char *dir = get_tmp_dir(__LINE__);
|
2021-12-23 19:29:50 +00:00
|
|
|
struct strbuf scratch = STRBUF_INIT;
|
|
|
|
int mask = umask(002);
|
|
|
|
struct reftable_write_options cfg = {
|
|
|
|
.default_permissions = 0660,
|
|
|
|
};
|
2021-10-07 20:25:13 +00:00
|
|
|
struct reftable_stack *st = NULL;
|
|
|
|
int err;
|
|
|
|
struct reftable_ref_record ref = {
|
|
|
|
.refname = "HEAD",
|
|
|
|
.update_index = 1,
|
|
|
|
.value_type = REFTABLE_REF_SYMREF,
|
|
|
|
.value.symref = "master",
|
|
|
|
};
|
|
|
|
struct reftable_ref_record dest = { NULL };
|
2021-12-23 19:29:50 +00:00
|
|
|
struct stat stat_result = { 0 };
|
2021-10-07 20:25:13 +00:00
|
|
|
err = reftable_new_stack(&st, dir, cfg);
|
|
|
|
EXPECT_ERR(err);
|
|
|
|
|
|
|
|
err = reftable_stack_add(st, &write_test_ref, &ref);
|
|
|
|
EXPECT_ERR(err);
|
|
|
|
|
|
|
|
err = reftable_stack_read_ref(st, ref.refname, &dest);
|
|
|
|
EXPECT_ERR(err);
|
|
|
|
EXPECT(0 == strcmp("master", dest.value.symref));
|
2021-12-23 19:29:50 +00:00
|
|
|
EXPECT(st->readers_len > 0);
|
2021-10-07 20:25:13 +00:00
|
|
|
|
|
|
|
printf("testing print functionality:\n");
|
|
|
|
err = reftable_stack_print_directory(dir, GIT_SHA1_FORMAT_ID);
|
|
|
|
EXPECT_ERR(err);
|
|
|
|
|
|
|
|
err = reftable_stack_print_directory(dir, GIT_SHA256_FORMAT_ID);
|
|
|
|
EXPECT(err == REFTABLE_FORMAT_ERROR);
|
|
|
|
|
2021-12-23 19:29:50 +00:00
|
|
|
#ifndef GIT_WINDOWS_NATIVE
|
|
|
|
strbuf_addstr(&scratch, dir);
|
|
|
|
strbuf_addstr(&scratch, "/tables.list");
|
|
|
|
err = stat(scratch.buf, &stat_result);
|
|
|
|
EXPECT(!err);
|
|
|
|
EXPECT((stat_result.st_mode & 0777) == cfg.default_permissions);
|
|
|
|
|
|
|
|
strbuf_reset(&scratch);
|
|
|
|
strbuf_addstr(&scratch, dir);
|
|
|
|
strbuf_addstr(&scratch, "/");
|
|
|
|
/* do not try at home; not an external API for reftable. */
|
|
|
|
strbuf_addstr(&scratch, st->readers[0]->name);
|
|
|
|
err = stat(scratch.buf, &stat_result);
|
|
|
|
EXPECT(!err);
|
|
|
|
EXPECT((stat_result.st_mode & 0777) == cfg.default_permissions);
|
|
|
|
#else
|
|
|
|
(void) stat_result;
|
|
|
|
#endif
|
|
|
|
|
2021-10-07 20:25:13 +00:00
|
|
|
reftable_ref_record_release(&dest);
|
|
|
|
reftable_stack_destroy(st);
|
2021-12-23 19:29:50 +00:00
|
|
|
strbuf_release(&scratch);
|
2021-10-07 20:25:13 +00:00
|
|
|
clear_dir(dir);
|
2021-12-23 19:29:50 +00:00
|
|
|
umask(mask);
|
2021-10-07 20:25:13 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static void test_reftable_stack_uptodate(void)
|
|
|
|
{
|
|
|
|
struct reftable_write_options cfg = { 0 };
|
|
|
|
struct reftable_stack *st1 = NULL;
|
|
|
|
struct reftable_stack *st2 = NULL;
|
|
|
|
char *dir = get_tmp_dir(__LINE__);
|
|
|
|
|
|
|
|
int err;
|
|
|
|
struct reftable_ref_record ref1 = {
|
|
|
|
.refname = "HEAD",
|
|
|
|
.update_index = 1,
|
|
|
|
.value_type = REFTABLE_REF_SYMREF,
|
|
|
|
.value.symref = "master",
|
|
|
|
};
|
|
|
|
struct reftable_ref_record ref2 = {
|
|
|
|
.refname = "branch2",
|
|
|
|
.update_index = 2,
|
|
|
|
.value_type = REFTABLE_REF_SYMREF,
|
|
|
|
.value.symref = "master",
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
|
|
/* simulate multi-process access to the same stack
|
|
|
|
by creating two stacks for the same directory.
|
|
|
|
*/
|
|
|
|
err = reftable_new_stack(&st1, dir, cfg);
|
|
|
|
EXPECT_ERR(err);
|
|
|
|
|
|
|
|
err = reftable_new_stack(&st2, dir, cfg);
|
|
|
|
EXPECT_ERR(err);
|
|
|
|
|
|
|
|
err = reftable_stack_add(st1, &write_test_ref, &ref1);
|
|
|
|
EXPECT_ERR(err);
|
|
|
|
|
|
|
|
err = reftable_stack_add(st2, &write_test_ref, &ref2);
|
2024-03-25 10:02:42 +00:00
|
|
|
EXPECT(err == REFTABLE_OUTDATED_ERROR);
|
2021-10-07 20:25:13 +00:00
|
|
|
|
|
|
|
err = reftable_stack_reload(st2);
|
|
|
|
EXPECT_ERR(err);
|
|
|
|
|
|
|
|
err = reftable_stack_add(st2, &write_test_ref, &ref2);
|
|
|
|
EXPECT_ERR(err);
|
|
|
|
reftable_stack_destroy(st1);
|
|
|
|
reftable_stack_destroy(st2);
|
|
|
|
clear_dir(dir);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void test_reftable_stack_transaction_api(void)
|
|
|
|
{
|
|
|
|
char *dir = get_tmp_dir(__LINE__);
|
|
|
|
|
|
|
|
struct reftable_write_options cfg = { 0 };
|
|
|
|
struct reftable_stack *st = NULL;
|
|
|
|
int err;
|
|
|
|
struct reftable_addition *add = NULL;
|
|
|
|
|
|
|
|
struct reftable_ref_record ref = {
|
|
|
|
.refname = "HEAD",
|
|
|
|
.update_index = 1,
|
|
|
|
.value_type = REFTABLE_REF_SYMREF,
|
|
|
|
.value.symref = "master",
|
|
|
|
};
|
|
|
|
struct reftable_ref_record dest = { NULL };
|
|
|
|
|
|
|
|
|
|
|
|
err = reftable_new_stack(&st, dir, cfg);
|
|
|
|
EXPECT_ERR(err);
|
|
|
|
|
|
|
|
reftable_addition_destroy(add);
|
|
|
|
|
|
|
|
err = reftable_stack_new_addition(&add, st);
|
|
|
|
EXPECT_ERR(err);
|
|
|
|
|
|
|
|
err = reftable_addition_add(add, &write_test_ref, &ref);
|
|
|
|
EXPECT_ERR(err);
|
|
|
|
|
|
|
|
err = reftable_addition_commit(add);
|
|
|
|
EXPECT_ERR(err);
|
|
|
|
|
|
|
|
reftable_addition_destroy(add);
|
|
|
|
|
|
|
|
err = reftable_stack_read_ref(st, ref.refname, &dest);
|
|
|
|
EXPECT_ERR(err);
|
|
|
|
EXPECT(REFTABLE_REF_SYMREF == dest.value_type);
|
|
|
|
EXPECT(0 == strcmp("master", dest.value.symref));
|
|
|
|
|
|
|
|
reftable_ref_record_release(&dest);
|
|
|
|
reftable_stack_destroy(st);
|
|
|
|
clear_dir(dir);
|
|
|
|
}
|
|
|
|
|
2023-12-11 09:07:46 +00:00
|
|
|
static void test_reftable_stack_transaction_api_performs_auto_compaction(void)
|
|
|
|
{
|
|
|
|
char *dir = get_tmp_dir(__LINE__);
|
|
|
|
struct reftable_write_options cfg = {0};
|
|
|
|
struct reftable_addition *add = NULL;
|
|
|
|
struct reftable_stack *st = NULL;
|
|
|
|
int i, n = 20, err;
|
|
|
|
|
|
|
|
err = reftable_new_stack(&st, dir, cfg);
|
|
|
|
EXPECT_ERR(err);
|
|
|
|
|
|
|
|
for (i = 0; i <= n; i++) {
|
|
|
|
struct reftable_ref_record ref = {
|
|
|
|
.update_index = reftable_stack_next_update_index(st),
|
|
|
|
.value_type = REFTABLE_REF_SYMREF,
|
|
|
|
.value.symref = "master",
|
|
|
|
};
|
|
|
|
char name[100];
|
|
|
|
|
|
|
|
snprintf(name, sizeof(name), "branch%04d", i);
|
|
|
|
ref.refname = name;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Disable auto-compaction for all but the last runs. Like this
|
|
|
|
* we can ensure that we indeed honor this setting and have
|
|
|
|
* better control over when exactly auto compaction runs.
|
|
|
|
*/
|
2024-04-08 16:16:53 +00:00
|
|
|
st->config.disable_auto_compact = i != n;
|
2023-12-11 09:07:46 +00:00
|
|
|
|
|
|
|
err = reftable_stack_new_addition(&add, st);
|
|
|
|
EXPECT_ERR(err);
|
|
|
|
|
|
|
|
err = reftable_addition_add(add, &write_test_ref, &ref);
|
|
|
|
EXPECT_ERR(err);
|
|
|
|
|
|
|
|
err = reftable_addition_commit(add);
|
|
|
|
EXPECT_ERR(err);
|
|
|
|
|
|
|
|
reftable_addition_destroy(add);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* The stack length should grow continuously for all runs where
|
|
|
|
* auto compaction is disabled. When enabled, we should merge
|
|
|
|
* all tables in the stack.
|
|
|
|
*/
|
|
|
|
if (i != n)
|
|
|
|
EXPECT(st->merged->stack_len == i + 1);
|
|
|
|
else
|
|
|
|
EXPECT(st->merged->stack_len == 1);
|
|
|
|
}
|
|
|
|
|
|
|
|
reftable_stack_destroy(st);
|
|
|
|
clear_dir(dir);
|
|
|
|
}
|
|
|
|
|
2024-03-25 10:02:50 +00:00
|
|
|
static void test_reftable_stack_auto_compaction_fails_gracefully(void)
|
|
|
|
{
|
|
|
|
struct reftable_ref_record ref = {
|
|
|
|
.refname = "refs/heads/master",
|
|
|
|
.update_index = 1,
|
|
|
|
.value_type = REFTABLE_REF_VAL1,
|
|
|
|
.value.val1 = {0x01},
|
|
|
|
};
|
|
|
|
struct reftable_write_options cfg = {0};
|
|
|
|
struct reftable_stack *st;
|
|
|
|
struct strbuf table_path = STRBUF_INIT;
|
|
|
|
char *dir = get_tmp_dir(__LINE__);
|
|
|
|
int err;
|
|
|
|
|
|
|
|
err = reftable_new_stack(&st, dir, cfg);
|
|
|
|
EXPECT_ERR(err);
|
|
|
|
|
|
|
|
err = reftable_stack_add(st, write_test_ref, &ref);
|
|
|
|
EXPECT_ERR(err);
|
|
|
|
EXPECT(st->merged->stack_len == 1);
|
|
|
|
EXPECT(st->stats.attempts == 0);
|
|
|
|
EXPECT(st->stats.failures == 0);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Lock the newly written table such that it cannot be compacted.
|
|
|
|
* Adding a new table to the stack should not be impacted by this, even
|
|
|
|
* though auto-compaction will now fail.
|
|
|
|
*/
|
|
|
|
strbuf_addf(&table_path, "%s/%s.lock", dir, st->readers[0]->name);
|
|
|
|
write_file_buf(table_path.buf, "", 0);
|
|
|
|
|
|
|
|
ref.update_index = 2;
|
|
|
|
err = reftable_stack_add(st, write_test_ref, &ref);
|
|
|
|
EXPECT_ERR(err);
|
|
|
|
EXPECT(st->merged->stack_len == 2);
|
|
|
|
EXPECT(st->stats.attempts == 1);
|
|
|
|
EXPECT(st->stats.failures == 1);
|
|
|
|
|
|
|
|
reftable_stack_destroy(st);
|
|
|
|
strbuf_release(&table_path);
|
|
|
|
clear_dir(dir);
|
|
|
|
}
|
|
|
|
|
2021-10-07 20:25:13 +00:00
|
|
|
static int write_error(struct reftable_writer *wr, void *arg)
|
|
|
|
{
|
|
|
|
return *((int *)arg);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void test_reftable_stack_update_index_check(void)
|
|
|
|
{
|
|
|
|
char *dir = get_tmp_dir(__LINE__);
|
|
|
|
|
|
|
|
struct reftable_write_options cfg = { 0 };
|
|
|
|
struct reftable_stack *st = NULL;
|
|
|
|
int err;
|
|
|
|
struct reftable_ref_record ref1 = {
|
|
|
|
.refname = "name1",
|
|
|
|
.update_index = 1,
|
|
|
|
.value_type = REFTABLE_REF_SYMREF,
|
|
|
|
.value.symref = "master",
|
|
|
|
};
|
|
|
|
struct reftable_ref_record ref2 = {
|
|
|
|
.refname = "name2",
|
|
|
|
.update_index = 1,
|
|
|
|
.value_type = REFTABLE_REF_SYMREF,
|
|
|
|
.value.symref = "master",
|
|
|
|
};
|
|
|
|
|
|
|
|
err = reftable_new_stack(&st, dir, cfg);
|
|
|
|
EXPECT_ERR(err);
|
|
|
|
|
|
|
|
err = reftable_stack_add(st, &write_test_ref, &ref1);
|
|
|
|
EXPECT_ERR(err);
|
|
|
|
|
|
|
|
err = reftable_stack_add(st, &write_test_ref, &ref2);
|
|
|
|
EXPECT(err == REFTABLE_API_ERROR);
|
|
|
|
reftable_stack_destroy(st);
|
|
|
|
clear_dir(dir);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void test_reftable_stack_lock_failure(void)
|
|
|
|
{
|
|
|
|
char *dir = get_tmp_dir(__LINE__);
|
|
|
|
|
|
|
|
struct reftable_write_options cfg = { 0 };
|
|
|
|
struct reftable_stack *st = NULL;
|
|
|
|
int err, i;
|
|
|
|
|
|
|
|
err = reftable_new_stack(&st, dir, cfg);
|
|
|
|
EXPECT_ERR(err);
|
|
|
|
for (i = -1; i != REFTABLE_EMPTY_TABLE_ERROR; i--) {
|
|
|
|
err = reftable_stack_add(st, &write_error, &i);
|
|
|
|
EXPECT(err == i);
|
|
|
|
}
|
|
|
|
|
|
|
|
reftable_stack_destroy(st);
|
|
|
|
clear_dir(dir);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void test_reftable_stack_add(void)
|
|
|
|
{
|
|
|
|
int i = 0;
|
|
|
|
int err = 0;
|
|
|
|
struct reftable_write_options cfg = {
|
|
|
|
.exact_log_message = 1,
|
2024-01-26 10:09:10 +00:00
|
|
|
.default_permissions = 0660,
|
2024-04-08 16:16:53 +00:00
|
|
|
.disable_auto_compact = 1,
|
2021-10-07 20:25:13 +00:00
|
|
|
};
|
|
|
|
struct reftable_stack *st = NULL;
|
|
|
|
char *dir = get_tmp_dir(__LINE__);
|
|
|
|
struct reftable_ref_record refs[2] = { { NULL } };
|
|
|
|
struct reftable_log_record logs[2] = { { NULL } };
|
2024-01-26 10:09:10 +00:00
|
|
|
struct strbuf path = STRBUF_INIT;
|
|
|
|
struct stat stat_result;
|
2021-10-07 20:25:13 +00:00
|
|
|
int N = ARRAY_SIZE(refs);
|
|
|
|
|
|
|
|
err = reftable_new_stack(&st, dir, cfg);
|
|
|
|
EXPECT_ERR(err);
|
|
|
|
|
|
|
|
for (i = 0; i < N; i++) {
|
|
|
|
char buf[256];
|
|
|
|
snprintf(buf, sizeof(buf), "branch%02d", i);
|
|
|
|
refs[i].refname = xstrdup(buf);
|
|
|
|
refs[i].update_index = i + 1;
|
|
|
|
refs[i].value_type = REFTABLE_REF_VAL1;
|
|
|
|
set_test_hash(refs[i].value.val1, i);
|
|
|
|
|
|
|
|
logs[i].refname = xstrdup(buf);
|
|
|
|
logs[i].update_index = N + i + 1;
|
|
|
|
logs[i].value_type = REFTABLE_LOG_UPDATE;
|
|
|
|
logs[i].value.update.email = xstrdup("identity@invalid");
|
|
|
|
set_test_hash(logs[i].value.update.new_hash, i);
|
|
|
|
}
|
|
|
|
|
|
|
|
for (i = 0; i < N; i++) {
|
|
|
|
int err = reftable_stack_add(st, &write_test_ref, &refs[i]);
|
|
|
|
EXPECT_ERR(err);
|
|
|
|
}
|
|
|
|
|
|
|
|
for (i = 0; i < N; i++) {
|
|
|
|
struct write_log_arg arg = {
|
|
|
|
.log = &logs[i],
|
|
|
|
.update_index = reftable_stack_next_update_index(st),
|
|
|
|
};
|
|
|
|
int err = reftable_stack_add(st, &write_test_log, &arg);
|
|
|
|
EXPECT_ERR(err);
|
|
|
|
}
|
|
|
|
|
|
|
|
err = reftable_stack_compact_all(st, NULL);
|
|
|
|
EXPECT_ERR(err);
|
|
|
|
|
|
|
|
for (i = 0; i < N; i++) {
|
|
|
|
struct reftable_ref_record dest = { NULL };
|
|
|
|
|
|
|
|
int err = reftable_stack_read_ref(st, refs[i].refname, &dest);
|
|
|
|
EXPECT_ERR(err);
|
|
|
|
EXPECT(reftable_ref_record_equal(&dest, refs + i,
|
|
|
|
GIT_SHA1_RAWSZ));
|
|
|
|
reftable_ref_record_release(&dest);
|
|
|
|
}
|
|
|
|
|
|
|
|
for (i = 0; i < N; i++) {
|
|
|
|
struct reftable_log_record dest = { NULL };
|
|
|
|
int err = reftable_stack_read_log(st, refs[i].refname, &dest);
|
|
|
|
EXPECT_ERR(err);
|
|
|
|
EXPECT(reftable_log_record_equal(&dest, logs + i,
|
|
|
|
GIT_SHA1_RAWSZ));
|
|
|
|
reftable_log_record_release(&dest);
|
|
|
|
}
|
|
|
|
|
2024-01-26 10:09:10 +00:00
|
|
|
#ifndef GIT_WINDOWS_NATIVE
|
|
|
|
strbuf_addstr(&path, dir);
|
|
|
|
strbuf_addstr(&path, "/tables.list");
|
|
|
|
err = stat(path.buf, &stat_result);
|
|
|
|
EXPECT(!err);
|
|
|
|
EXPECT((stat_result.st_mode & 0777) == cfg.default_permissions);
|
|
|
|
|
|
|
|
strbuf_reset(&path);
|
|
|
|
strbuf_addstr(&path, dir);
|
|
|
|
strbuf_addstr(&path, "/");
|
|
|
|
/* do not try at home; not an external API for reftable. */
|
|
|
|
strbuf_addstr(&path, st->readers[0]->name);
|
|
|
|
err = stat(path.buf, &stat_result);
|
|
|
|
EXPECT(!err);
|
|
|
|
EXPECT((stat_result.st_mode & 0777) == cfg.default_permissions);
|
|
|
|
#else
|
|
|
|
(void) stat_result;
|
|
|
|
#endif
|
|
|
|
|
2021-10-07 20:25:13 +00:00
|
|
|
/* cleanup */
|
|
|
|
reftable_stack_destroy(st);
|
|
|
|
for (i = 0; i < N; i++) {
|
|
|
|
reftable_ref_record_release(&refs[i]);
|
|
|
|
reftable_log_record_release(&logs[i]);
|
|
|
|
}
|
2024-01-26 10:09:10 +00:00
|
|
|
strbuf_release(&path);
|
2021-10-07 20:25:13 +00:00
|
|
|
clear_dir(dir);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void test_reftable_stack_log_normalize(void)
|
|
|
|
{
|
|
|
|
int err = 0;
|
|
|
|
struct reftable_write_options cfg = {
|
|
|
|
0,
|
|
|
|
};
|
|
|
|
struct reftable_stack *st = NULL;
|
|
|
|
char *dir = get_tmp_dir(__LINE__);
|
reftable/record: convert old and new object IDs to arrays
In 7af607c58d (reftable/record: store "val1" hashes as static arrays,
2024-01-03) and b31e3cc620 (reftable/record: store "val2" hashes as
static arrays, 2024-01-03) we have converted ref records to store their
object IDs in a static array. Convert log records to do the same so that
their old and new object IDs are arrays, too.
This change results in two allocations less per log record that we're
iterating over. Before:
HEAP SUMMARY:
in use at exit: 13,473 bytes in 122 blocks
total heap usage: 8,068,495 allocs, 8,068,373 frees, 401,011,862 bytes allocated
After:
HEAP SUMMARY:
in use at exit: 13,473 bytes in 122 blocks
total heap usage: 6,068,489 allocs, 6,068,367 frees, 361,011,822 bytes allocated
Signed-off-by: Patrick Steinhardt <ps@pks.im>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2024-03-05 12:10:59 +00:00
|
|
|
struct reftable_log_record input = {
|
|
|
|
.refname = "branch",
|
|
|
|
.update_index = 1,
|
|
|
|
.value_type = REFTABLE_LOG_UPDATE,
|
|
|
|
.value = {
|
|
|
|
.update = {
|
|
|
|
.new_hash = { 1 },
|
|
|
|
.old_hash = { 2 },
|
|
|
|
},
|
|
|
|
},
|
|
|
|
};
|
2021-10-07 20:25:13 +00:00
|
|
|
struct reftable_log_record dest = {
|
|
|
|
.update_index = 0,
|
|
|
|
};
|
|
|
|
struct write_log_arg arg = {
|
|
|
|
.log = &input,
|
|
|
|
.update_index = 1,
|
|
|
|
};
|
|
|
|
|
|
|
|
err = reftable_new_stack(&st, dir, cfg);
|
|
|
|
EXPECT_ERR(err);
|
|
|
|
|
|
|
|
input.value.update.message = "one\ntwo";
|
|
|
|
err = reftable_stack_add(st, &write_test_log, &arg);
|
|
|
|
EXPECT(err == REFTABLE_API_ERROR);
|
|
|
|
|
|
|
|
input.value.update.message = "one";
|
|
|
|
err = reftable_stack_add(st, &write_test_log, &arg);
|
|
|
|
EXPECT_ERR(err);
|
|
|
|
|
|
|
|
err = reftable_stack_read_log(st, input.refname, &dest);
|
|
|
|
EXPECT_ERR(err);
|
|
|
|
EXPECT(0 == strcmp(dest.value.update.message, "one\n"));
|
|
|
|
|
|
|
|
input.value.update.message = "two\n";
|
|
|
|
arg.update_index = 2;
|
|
|
|
err = reftable_stack_add(st, &write_test_log, &arg);
|
|
|
|
EXPECT_ERR(err);
|
|
|
|
err = reftable_stack_read_log(st, input.refname, &dest);
|
|
|
|
EXPECT_ERR(err);
|
|
|
|
EXPECT(0 == strcmp(dest.value.update.message, "two\n"));
|
|
|
|
|
|
|
|
/* cleanup */
|
|
|
|
reftable_stack_destroy(st);
|
|
|
|
reftable_log_record_release(&dest);
|
|
|
|
clear_dir(dir);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void test_reftable_stack_tombstone(void)
|
|
|
|
{
|
|
|
|
int i = 0;
|
|
|
|
char *dir = get_tmp_dir(__LINE__);
|
|
|
|
|
|
|
|
struct reftable_write_options cfg = { 0 };
|
|
|
|
struct reftable_stack *st = NULL;
|
|
|
|
int err;
|
|
|
|
struct reftable_ref_record refs[2] = { { NULL } };
|
|
|
|
struct reftable_log_record logs[2] = { { NULL } };
|
|
|
|
int N = ARRAY_SIZE(refs);
|
|
|
|
struct reftable_ref_record dest = { NULL };
|
|
|
|
struct reftable_log_record log_dest = { NULL };
|
|
|
|
|
|
|
|
|
|
|
|
err = reftable_new_stack(&st, dir, cfg);
|
|
|
|
EXPECT_ERR(err);
|
|
|
|
|
|
|
|
/* even entries add the refs, odd entries delete them. */
|
|
|
|
for (i = 0; i < N; i++) {
|
|
|
|
const char *buf = "branch";
|
|
|
|
refs[i].refname = xstrdup(buf);
|
|
|
|
refs[i].update_index = i + 1;
|
|
|
|
if (i % 2 == 0) {
|
|
|
|
refs[i].value_type = REFTABLE_REF_VAL1;
|
|
|
|
set_test_hash(refs[i].value.val1, i);
|
|
|
|
}
|
|
|
|
|
|
|
|
logs[i].refname = xstrdup(buf);
|
|
|
|
/* update_index is part of the key. */
|
|
|
|
logs[i].update_index = 42;
|
|
|
|
if (i % 2 == 0) {
|
|
|
|
logs[i].value_type = REFTABLE_LOG_UPDATE;
|
|
|
|
set_test_hash(logs[i].value.update.new_hash, i);
|
|
|
|
logs[i].value.update.email =
|
|
|
|
xstrdup("identity@invalid");
|
|
|
|
}
|
|
|
|
}
|
|
|
|
for (i = 0; i < N; i++) {
|
|
|
|
int err = reftable_stack_add(st, &write_test_ref, &refs[i]);
|
|
|
|
EXPECT_ERR(err);
|
|
|
|
}
|
|
|
|
|
|
|
|
for (i = 0; i < N; i++) {
|
|
|
|
struct write_log_arg arg = {
|
|
|
|
.log = &logs[i],
|
|
|
|
.update_index = reftable_stack_next_update_index(st),
|
|
|
|
};
|
|
|
|
int err = reftable_stack_add(st, &write_test_log, &arg);
|
|
|
|
EXPECT_ERR(err);
|
|
|
|
}
|
|
|
|
|
|
|
|
err = reftable_stack_read_ref(st, "branch", &dest);
|
|
|
|
EXPECT(err == 1);
|
|
|
|
reftable_ref_record_release(&dest);
|
|
|
|
|
|
|
|
err = reftable_stack_read_log(st, "branch", &log_dest);
|
|
|
|
EXPECT(err == 1);
|
|
|
|
reftable_log_record_release(&log_dest);
|
|
|
|
|
|
|
|
err = reftable_stack_compact_all(st, NULL);
|
|
|
|
EXPECT_ERR(err);
|
|
|
|
|
|
|
|
err = reftable_stack_read_ref(st, "branch", &dest);
|
|
|
|
EXPECT(err == 1);
|
|
|
|
|
|
|
|
err = reftable_stack_read_log(st, "branch", &log_dest);
|
|
|
|
EXPECT(err == 1);
|
|
|
|
reftable_ref_record_release(&dest);
|
|
|
|
reftable_log_record_release(&log_dest);
|
|
|
|
|
|
|
|
/* cleanup */
|
|
|
|
reftable_stack_destroy(st);
|
|
|
|
for (i = 0; i < N; i++) {
|
|
|
|
reftable_ref_record_release(&refs[i]);
|
|
|
|
reftable_log_record_release(&logs[i]);
|
|
|
|
}
|
|
|
|
clear_dir(dir);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void test_reftable_stack_hash_id(void)
|
|
|
|
{
|
|
|
|
char *dir = get_tmp_dir(__LINE__);
|
|
|
|
|
|
|
|
struct reftable_write_options cfg = { 0 };
|
|
|
|
struct reftable_stack *st = NULL;
|
|
|
|
int err;
|
|
|
|
|
|
|
|
struct reftable_ref_record ref = {
|
|
|
|
.refname = "master",
|
|
|
|
.value_type = REFTABLE_REF_SYMREF,
|
|
|
|
.value.symref = "target",
|
|
|
|
.update_index = 1,
|
|
|
|
};
|
|
|
|
struct reftable_write_options cfg32 = { .hash_id = GIT_SHA256_FORMAT_ID };
|
|
|
|
struct reftable_stack *st32 = NULL;
|
|
|
|
struct reftable_write_options cfg_default = { 0 };
|
|
|
|
struct reftable_stack *st_default = NULL;
|
|
|
|
struct reftable_ref_record dest = { NULL };
|
|
|
|
|
|
|
|
err = reftable_new_stack(&st, dir, cfg);
|
|
|
|
EXPECT_ERR(err);
|
|
|
|
|
|
|
|
err = reftable_stack_add(st, &write_test_ref, &ref);
|
|
|
|
EXPECT_ERR(err);
|
|
|
|
|
|
|
|
/* can't read it with the wrong hash ID. */
|
|
|
|
err = reftable_new_stack(&st32, dir, cfg32);
|
|
|
|
EXPECT(err == REFTABLE_FORMAT_ERROR);
|
|
|
|
|
|
|
|
/* check that we can read it back with default config too. */
|
|
|
|
err = reftable_new_stack(&st_default, dir, cfg_default);
|
|
|
|
EXPECT_ERR(err);
|
|
|
|
|
|
|
|
err = reftable_stack_read_ref(st_default, "master", &dest);
|
|
|
|
EXPECT_ERR(err);
|
|
|
|
|
|
|
|
EXPECT(reftable_ref_record_equal(&ref, &dest, GIT_SHA1_RAWSZ));
|
|
|
|
reftable_ref_record_release(&dest);
|
|
|
|
reftable_stack_destroy(st);
|
|
|
|
reftable_stack_destroy(st_default);
|
|
|
|
clear_dir(dir);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void test_suggest_compaction_segment(void)
|
|
|
|
{
|
reftable/stack: use geometric table compaction
To reduce the number of on-disk reftables, compaction is performed.
Contiguous tables with the same binary log value of size are grouped
into segments. The segment that has both the lowest binary log value and
contains more than one table is set as the starting point when
identifying the compaction segment.
Since segments containing a single table are not initially considered
for compaction, if the table appended to the list does not match the
previous table log value, no compaction occurs for the new table. It is
therefore possible for unbounded growth of the table list. This can be
demonstrated by repeating the following sequence:
git branch -f foo
git branch -d foo
Each operation results in a new table being written with no compaction
occurring until a separate operation produces a table matching the
previous table log value.
Instead, to avoid unbounded growth of the table list, the compaction
strategy is updated to ensure tables follow a geometric sequence after
each operation by individually evaluating each table in reverse index
order. This strategy results in a much simpler and more robust algorithm
compared to the previous one while also maintaining a minimal ordered
set of tables on-disk.
When creating 10 thousand references, the new strategy has no
performance impact:
Benchmark 1: update-ref: create refs sequentially (revision = HEAD~)
Time (mean ± σ): 26.516 s ± 0.047 s [User: 17.864 s, System: 8.491 s]
Range (min … max): 26.447 s … 26.569 s 10 runs
Benchmark 2: update-ref: create refs sequentially (revision = HEAD)
Time (mean ± σ): 26.417 s ± 0.028 s [User: 17.738 s, System: 8.500 s]
Range (min … max): 26.366 s … 26.444 s 10 runs
Summary
update-ref: create refs sequentially (revision = HEAD) ran
1.00 ± 0.00 times faster than update-ref: create refs sequentially (revision = HEAD~)
Some tests in `t0610-reftable-basics.sh` assert the on-disk state of
tables and are therefore updated to specify the correct new table count.
Since compaction is more aggressive in ensuring tables maintain a
geometric sequence, the expected table count is reduced in these tests.
In `reftable/stack_test.c` tests related to `sizes_to_segments()` are
removed because the function is no longer needed. Also, the
`test_suggest_compaction_segment()` test is updated to better showcase
and reflect the new geometric compaction behavior.
Signed-off-by: Justin Tobler <jltobler@gmail.com>
Acked-by: Patrick Steinhardt <ps@pks.im>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2024-04-08 16:16:55 +00:00
|
|
|
uint64_t sizes[] = { 512, 64, 17, 16, 9, 9, 9, 16, 2, 16 };
|
2021-10-07 20:25:13 +00:00
|
|
|
struct segment min =
|
|
|
|
suggest_compaction_segment(sizes, ARRAY_SIZE(sizes));
|
reftable/stack: use geometric table compaction
To reduce the number of on-disk reftables, compaction is performed.
Contiguous tables with the same binary log value of size are grouped
into segments. The segment that has both the lowest binary log value and
contains more than one table is set as the starting point when
identifying the compaction segment.
Since segments containing a single table are not initially considered
for compaction, if the table appended to the list does not match the
previous table log value, no compaction occurs for the new table. It is
therefore possible for unbounded growth of the table list. This can be
demonstrated by repeating the following sequence:
git branch -f foo
git branch -d foo
Each operation results in a new table being written with no compaction
occurring until a separate operation produces a table matching the
previous table log value.
Instead, to avoid unbounded growth of the table list, the compaction
strategy is updated to ensure tables follow a geometric sequence after
each operation by individually evaluating each table in reverse index
order. This strategy results in a much simpler and more robust algorithm
compared to the previous one while also maintaining a minimal ordered
set of tables on-disk.
When creating 10 thousand references, the new strategy has no
performance impact:
Benchmark 1: update-ref: create refs sequentially (revision = HEAD~)
Time (mean ± σ): 26.516 s ± 0.047 s [User: 17.864 s, System: 8.491 s]
Range (min … max): 26.447 s … 26.569 s 10 runs
Benchmark 2: update-ref: create refs sequentially (revision = HEAD)
Time (mean ± σ): 26.417 s ± 0.028 s [User: 17.738 s, System: 8.500 s]
Range (min … max): 26.366 s … 26.444 s 10 runs
Summary
update-ref: create refs sequentially (revision = HEAD) ran
1.00 ± 0.00 times faster than update-ref: create refs sequentially (revision = HEAD~)
Some tests in `t0610-reftable-basics.sh` assert the on-disk state of
tables and are therefore updated to specify the correct new table count.
Since compaction is more aggressive in ensuring tables maintain a
geometric sequence, the expected table count is reduced in these tests.
In `reftable/stack_test.c` tests related to `sizes_to_segments()` are
removed because the function is no longer needed. Also, the
`test_suggest_compaction_segment()` test is updated to better showcase
and reflect the new geometric compaction behavior.
Signed-off-by: Justin Tobler <jltobler@gmail.com>
Acked-by: Patrick Steinhardt <ps@pks.im>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2024-04-08 16:16:55 +00:00
|
|
|
EXPECT(min.start == 1);
|
|
|
|
EXPECT(min.end == 10);
|
2021-10-07 20:25:13 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static void test_suggest_compaction_segment_nothing(void)
|
|
|
|
{
|
|
|
|
uint64_t sizes[] = { 64, 32, 16, 8, 4, 2 };
|
|
|
|
struct segment result =
|
|
|
|
suggest_compaction_segment(sizes, ARRAY_SIZE(sizes));
|
|
|
|
EXPECT(result.start == result.end);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void test_reflog_expire(void)
|
|
|
|
{
|
|
|
|
char *dir = get_tmp_dir(__LINE__);
|
|
|
|
|
|
|
|
struct reftable_write_options cfg = { 0 };
|
|
|
|
struct reftable_stack *st = NULL;
|
|
|
|
struct reftable_log_record logs[20] = { { NULL } };
|
|
|
|
int N = ARRAY_SIZE(logs) - 1;
|
|
|
|
int i = 0;
|
|
|
|
int err;
|
|
|
|
struct reftable_log_expiry_config expiry = {
|
|
|
|
.time = 10,
|
|
|
|
};
|
|
|
|
struct reftable_log_record log = { NULL };
|
|
|
|
|
|
|
|
|
|
|
|
err = reftable_new_stack(&st, dir, cfg);
|
|
|
|
EXPECT_ERR(err);
|
|
|
|
|
|
|
|
for (i = 1; i <= N; i++) {
|
|
|
|
char buf[256];
|
|
|
|
snprintf(buf, sizeof(buf), "branch%02d", i);
|
|
|
|
|
|
|
|
logs[i].refname = xstrdup(buf);
|
|
|
|
logs[i].update_index = i;
|
|
|
|
logs[i].value_type = REFTABLE_LOG_UPDATE;
|
|
|
|
logs[i].value.update.time = i;
|
|
|
|
logs[i].value.update.email = xstrdup("identity@invalid");
|
|
|
|
set_test_hash(logs[i].value.update.new_hash, i);
|
|
|
|
}
|
|
|
|
|
|
|
|
for (i = 1; i <= N; i++) {
|
|
|
|
struct write_log_arg arg = {
|
|
|
|
.log = &logs[i],
|
|
|
|
.update_index = reftable_stack_next_update_index(st),
|
|
|
|
};
|
|
|
|
int err = reftable_stack_add(st, &write_test_log, &arg);
|
|
|
|
EXPECT_ERR(err);
|
|
|
|
}
|
|
|
|
|
|
|
|
err = reftable_stack_compact_all(st, NULL);
|
|
|
|
EXPECT_ERR(err);
|
|
|
|
|
|
|
|
err = reftable_stack_compact_all(st, &expiry);
|
|
|
|
EXPECT_ERR(err);
|
|
|
|
|
|
|
|
err = reftable_stack_read_log(st, logs[9].refname, &log);
|
|
|
|
EXPECT(err == 1);
|
|
|
|
|
|
|
|
err = reftable_stack_read_log(st, logs[11].refname, &log);
|
|
|
|
EXPECT_ERR(err);
|
|
|
|
|
|
|
|
expiry.min_update_index = 15;
|
|
|
|
err = reftable_stack_compact_all(st, &expiry);
|
|
|
|
EXPECT_ERR(err);
|
|
|
|
|
|
|
|
err = reftable_stack_read_log(st, logs[14].refname, &log);
|
|
|
|
EXPECT(err == 1);
|
|
|
|
|
|
|
|
err = reftable_stack_read_log(st, logs[16].refname, &log);
|
|
|
|
EXPECT_ERR(err);
|
|
|
|
|
|
|
|
/* cleanup */
|
|
|
|
reftable_stack_destroy(st);
|
|
|
|
for (i = 0; i <= N; i++) {
|
|
|
|
reftable_log_record_release(&logs[i]);
|
|
|
|
}
|
|
|
|
clear_dir(dir);
|
|
|
|
reftable_log_record_release(&log);
|
|
|
|
}
|
|
|
|
|
|
|
|
static int write_nothing(struct reftable_writer *wr, void *arg)
|
|
|
|
{
|
|
|
|
reftable_writer_set_limits(wr, 1, 1);
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void test_empty_add(void)
|
|
|
|
{
|
|
|
|
struct reftable_write_options cfg = { 0 };
|
|
|
|
struct reftable_stack *st = NULL;
|
|
|
|
int err;
|
|
|
|
char *dir = get_tmp_dir(__LINE__);
|
|
|
|
|
|
|
|
struct reftable_stack *st2 = NULL;
|
|
|
|
|
|
|
|
|
|
|
|
err = reftable_new_stack(&st, dir, cfg);
|
|
|
|
EXPECT_ERR(err);
|
|
|
|
|
|
|
|
err = reftable_stack_add(st, &write_nothing, NULL);
|
|
|
|
EXPECT_ERR(err);
|
|
|
|
|
|
|
|
err = reftable_new_stack(&st2, dir, cfg);
|
|
|
|
EXPECT_ERR(err);
|
|
|
|
clear_dir(dir);
|
|
|
|
reftable_stack_destroy(st);
|
|
|
|
reftable_stack_destroy(st2);
|
|
|
|
}
|
|
|
|
|
reftable/stack: use geometric table compaction
To reduce the number of on-disk reftables, compaction is performed.
Contiguous tables with the same binary log value of size are grouped
into segments. The segment that has both the lowest binary log value and
contains more than one table is set as the starting point when
identifying the compaction segment.
Since segments containing a single table are not initially considered
for compaction, if the table appended to the list does not match the
previous table log value, no compaction occurs for the new table. It is
therefore possible for unbounded growth of the table list. This can be
demonstrated by repeating the following sequence:
git branch -f foo
git branch -d foo
Each operation results in a new table being written with no compaction
occurring until a separate operation produces a table matching the
previous table log value.
Instead, to avoid unbounded growth of the table list, the compaction
strategy is updated to ensure tables follow a geometric sequence after
each operation by individually evaluating each table in reverse index
order. This strategy results in a much simpler and more robust algorithm
compared to the previous one while also maintaining a minimal ordered
set of tables on-disk.
When creating 10 thousand references, the new strategy has no
performance impact:
Benchmark 1: update-ref: create refs sequentially (revision = HEAD~)
Time (mean ± σ): 26.516 s ± 0.047 s [User: 17.864 s, System: 8.491 s]
Range (min … max): 26.447 s … 26.569 s 10 runs
Benchmark 2: update-ref: create refs sequentially (revision = HEAD)
Time (mean ± σ): 26.417 s ± 0.028 s [User: 17.738 s, System: 8.500 s]
Range (min … max): 26.366 s … 26.444 s 10 runs
Summary
update-ref: create refs sequentially (revision = HEAD) ran
1.00 ± 0.00 times faster than update-ref: create refs sequentially (revision = HEAD~)
Some tests in `t0610-reftable-basics.sh` assert the on-disk state of
tables and are therefore updated to specify the correct new table count.
Since compaction is more aggressive in ensuring tables maintain a
geometric sequence, the expected table count is reduced in these tests.
In `reftable/stack_test.c` tests related to `sizes_to_segments()` are
removed because the function is no longer needed. Also, the
`test_suggest_compaction_segment()` test is updated to better showcase
and reflect the new geometric compaction behavior.
Signed-off-by: Justin Tobler <jltobler@gmail.com>
Acked-by: Patrick Steinhardt <ps@pks.im>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2024-04-08 16:16:55 +00:00
|
|
|
static int fastlog2(uint64_t sz)
|
|
|
|
{
|
|
|
|
int l = 0;
|
|
|
|
if (sz == 0)
|
|
|
|
return 0;
|
|
|
|
for (; sz; sz /= 2)
|
|
|
|
l++;
|
|
|
|
return l - 1;
|
|
|
|
}
|
|
|
|
|
2021-10-07 20:25:13 +00:00
|
|
|
static void test_reftable_stack_auto_compaction(void)
|
|
|
|
{
|
2024-04-08 16:16:53 +00:00
|
|
|
struct reftable_write_options cfg = {
|
|
|
|
.disable_auto_compact = 1,
|
|
|
|
};
|
2021-10-07 20:25:13 +00:00
|
|
|
struct reftable_stack *st = NULL;
|
|
|
|
char *dir = get_tmp_dir(__LINE__);
|
|
|
|
|
|
|
|
int err, i;
|
|
|
|
int N = 100;
|
|
|
|
|
|
|
|
err = reftable_new_stack(&st, dir, cfg);
|
|
|
|
EXPECT_ERR(err);
|
|
|
|
|
|
|
|
for (i = 0; i < N; i++) {
|
|
|
|
char name[100];
|
|
|
|
struct reftable_ref_record ref = {
|
|
|
|
.refname = name,
|
|
|
|
.update_index = reftable_stack_next_update_index(st),
|
|
|
|
.value_type = REFTABLE_REF_SYMREF,
|
|
|
|
.value.symref = "master",
|
|
|
|
};
|
|
|
|
snprintf(name, sizeof(name), "branch%04d", i);
|
|
|
|
|
|
|
|
err = reftable_stack_add(st, &write_test_ref, &ref);
|
|
|
|
EXPECT_ERR(err);
|
|
|
|
|
|
|
|
err = reftable_stack_auto_compact(st);
|
2022-01-20 15:12:03 +00:00
|
|
|
EXPECT_ERR(err);
|
2021-10-07 20:25:13 +00:00
|
|
|
EXPECT(i < 3 || st->merged->stack_len < 2 * fastlog2(i));
|
|
|
|
}
|
|
|
|
|
|
|
|
EXPECT(reftable_stack_compaction_stats(st)->entries_written <
|
|
|
|
(uint64_t)(N * fastlog2(N)));
|
|
|
|
|
|
|
|
reftable_stack_destroy(st);
|
|
|
|
clear_dir(dir);
|
|
|
|
}
|
|
|
|
|
2023-12-11 09:07:42 +00:00
|
|
|
static void test_reftable_stack_add_performs_auto_compaction(void)
|
|
|
|
{
|
|
|
|
struct reftable_write_options cfg = { 0 };
|
|
|
|
struct reftable_stack *st = NULL;
|
|
|
|
struct strbuf refname = STRBUF_INIT;
|
|
|
|
char *dir = get_tmp_dir(__LINE__);
|
|
|
|
int err, i, n = 20;
|
|
|
|
|
|
|
|
err = reftable_new_stack(&st, dir, cfg);
|
|
|
|
EXPECT_ERR(err);
|
|
|
|
|
|
|
|
for (i = 0; i <= n; i++) {
|
|
|
|
struct reftable_ref_record ref = {
|
|
|
|
.update_index = reftable_stack_next_update_index(st),
|
|
|
|
.value_type = REFTABLE_REF_SYMREF,
|
|
|
|
.value.symref = "master",
|
|
|
|
};
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Disable auto-compaction for all but the last runs. Like this
|
|
|
|
* we can ensure that we indeed honor this setting and have
|
|
|
|
* better control over when exactly auto compaction runs.
|
|
|
|
*/
|
2024-04-08 16:16:53 +00:00
|
|
|
st->config.disable_auto_compact = i != n;
|
2023-12-11 09:07:42 +00:00
|
|
|
|
|
|
|
strbuf_reset(&refname);
|
|
|
|
strbuf_addf(&refname, "branch-%04d", i);
|
|
|
|
ref.refname = refname.buf;
|
|
|
|
|
|
|
|
err = reftable_stack_add(st, &write_test_ref, &ref);
|
|
|
|
EXPECT_ERR(err);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* The stack length should grow continuously for all runs where
|
|
|
|
* auto compaction is disabled. When enabled, we should merge
|
|
|
|
* all tables in the stack.
|
|
|
|
*/
|
|
|
|
if (i != n)
|
|
|
|
EXPECT(st->merged->stack_len == i + 1);
|
|
|
|
else
|
|
|
|
EXPECT(st->merged->stack_len == 1);
|
|
|
|
}
|
|
|
|
|
|
|
|
reftable_stack_destroy(st);
|
|
|
|
strbuf_release(&refname);
|
|
|
|
clear_dir(dir);
|
|
|
|
}
|
|
|
|
|
2021-10-07 20:25:13 +00:00
|
|
|
static void test_reftable_stack_compaction_concurrent(void)
|
|
|
|
{
|
|
|
|
struct reftable_write_options cfg = { 0 };
|
|
|
|
struct reftable_stack *st1 = NULL, *st2 = NULL;
|
|
|
|
char *dir = get_tmp_dir(__LINE__);
|
|
|
|
|
|
|
|
int err, i;
|
|
|
|
int N = 3;
|
|
|
|
|
|
|
|
err = reftable_new_stack(&st1, dir, cfg);
|
|
|
|
EXPECT_ERR(err);
|
|
|
|
|
|
|
|
for (i = 0; i < N; i++) {
|
|
|
|
char name[100];
|
|
|
|
struct reftable_ref_record ref = {
|
|
|
|
.refname = name,
|
|
|
|
.update_index = reftable_stack_next_update_index(st1),
|
|
|
|
.value_type = REFTABLE_REF_SYMREF,
|
|
|
|
.value.symref = "master",
|
|
|
|
};
|
|
|
|
snprintf(name, sizeof(name), "branch%04d", i);
|
|
|
|
|
|
|
|
err = reftable_stack_add(st1, &write_test_ref, &ref);
|
|
|
|
EXPECT_ERR(err);
|
|
|
|
}
|
|
|
|
|
|
|
|
err = reftable_new_stack(&st2, dir, cfg);
|
|
|
|
EXPECT_ERR(err);
|
|
|
|
|
|
|
|
err = reftable_stack_compact_all(st1, NULL);
|
|
|
|
EXPECT_ERR(err);
|
|
|
|
|
|
|
|
reftable_stack_destroy(st1);
|
|
|
|
reftable_stack_destroy(st2);
|
|
|
|
|
|
|
|
EXPECT(count_dir_entries(dir) == 2);
|
|
|
|
clear_dir(dir);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void unclean_stack_close(struct reftable_stack *st)
|
|
|
|
{
|
|
|
|
/* break abstraction boundary to simulate unclean shutdown. */
|
|
|
|
int i = 0;
|
|
|
|
for (; i < st->readers_len; i++) {
|
|
|
|
reftable_reader_free(st->readers[i]);
|
|
|
|
}
|
|
|
|
st->readers_len = 0;
|
|
|
|
FREE_AND_NULL(st->readers);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void test_reftable_stack_compaction_concurrent_clean(void)
|
|
|
|
{
|
|
|
|
struct reftable_write_options cfg = { 0 };
|
|
|
|
struct reftable_stack *st1 = NULL, *st2 = NULL, *st3 = NULL;
|
|
|
|
char *dir = get_tmp_dir(__LINE__);
|
|
|
|
|
|
|
|
int err, i;
|
|
|
|
int N = 3;
|
|
|
|
|
|
|
|
err = reftable_new_stack(&st1, dir, cfg);
|
|
|
|
EXPECT_ERR(err);
|
|
|
|
|
|
|
|
for (i = 0; i < N; i++) {
|
|
|
|
char name[100];
|
|
|
|
struct reftable_ref_record ref = {
|
|
|
|
.refname = name,
|
|
|
|
.update_index = reftable_stack_next_update_index(st1),
|
|
|
|
.value_type = REFTABLE_REF_SYMREF,
|
|
|
|
.value.symref = "master",
|
|
|
|
};
|
|
|
|
snprintf(name, sizeof(name), "branch%04d", i);
|
|
|
|
|
|
|
|
err = reftable_stack_add(st1, &write_test_ref, &ref);
|
|
|
|
EXPECT_ERR(err);
|
|
|
|
}
|
|
|
|
|
|
|
|
err = reftable_new_stack(&st2, dir, cfg);
|
|
|
|
EXPECT_ERR(err);
|
|
|
|
|
|
|
|
err = reftable_stack_compact_all(st1, NULL);
|
|
|
|
EXPECT_ERR(err);
|
|
|
|
|
|
|
|
unclean_stack_close(st1);
|
|
|
|
unclean_stack_close(st2);
|
|
|
|
|
|
|
|
err = reftable_new_stack(&st3, dir, cfg);
|
|
|
|
EXPECT_ERR(err);
|
|
|
|
|
|
|
|
err = reftable_stack_clean(st3);
|
|
|
|
EXPECT_ERR(err);
|
|
|
|
EXPECT(count_dir_entries(dir) == 2);
|
|
|
|
|
|
|
|
reftable_stack_destroy(st1);
|
|
|
|
reftable_stack_destroy(st2);
|
|
|
|
reftable_stack_destroy(st3);
|
|
|
|
|
|
|
|
clear_dir(dir);
|
|
|
|
}
|
|
|
|
|
|
|
|
int stack_test_main(int argc, const char *argv[])
|
|
|
|
{
|
|
|
|
RUN_TEST(test_empty_add);
|
|
|
|
RUN_TEST(test_names_equal);
|
|
|
|
RUN_TEST(test_parse_names);
|
|
|
|
RUN_TEST(test_read_file);
|
|
|
|
RUN_TEST(test_reflog_expire);
|
|
|
|
RUN_TEST(test_reftable_stack_add);
|
|
|
|
RUN_TEST(test_reftable_stack_add_one);
|
|
|
|
RUN_TEST(test_reftable_stack_auto_compaction);
|
2023-12-11 09:07:42 +00:00
|
|
|
RUN_TEST(test_reftable_stack_add_performs_auto_compaction);
|
2021-10-07 20:25:13 +00:00
|
|
|
RUN_TEST(test_reftable_stack_compaction_concurrent);
|
|
|
|
RUN_TEST(test_reftable_stack_compaction_concurrent_clean);
|
|
|
|
RUN_TEST(test_reftable_stack_hash_id);
|
|
|
|
RUN_TEST(test_reftable_stack_lock_failure);
|
|
|
|
RUN_TEST(test_reftable_stack_log_normalize);
|
|
|
|
RUN_TEST(test_reftable_stack_tombstone);
|
|
|
|
RUN_TEST(test_reftable_stack_transaction_api);
|
2023-12-11 09:07:46 +00:00
|
|
|
RUN_TEST(test_reftable_stack_transaction_api_performs_auto_compaction);
|
2024-03-25 10:02:50 +00:00
|
|
|
RUN_TEST(test_reftable_stack_auto_compaction_fails_gracefully);
|
2021-10-07 20:25:13 +00:00
|
|
|
RUN_TEST(test_reftable_stack_update_index_check);
|
|
|
|
RUN_TEST(test_reftable_stack_uptodate);
|
|
|
|
RUN_TEST(test_suggest_compaction_segment);
|
|
|
|
RUN_TEST(test_suggest_compaction_segment_nothing);
|
|
|
|
return 0;
|
|
|
|
}
|