mirror of
https://github.com/git/git
synced 2024-08-27 19:49:21 +00:00
reftable: add blocksource, an abstraction for random access reads
The reftable format is usually used with files for storage. However, we abstract away this using the blocksource data structure. This has two advantages: * log blocks are zlib compressed, and handling them is simplified if we can discard byte segments from within the block layer. * for unittests, it is useful to read and write in-memory. The blocksource allows us to abstract the data away from on-disk files. Signed-off-by: Han-Wen Nienhuys <hanwen@google.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>
This commit is contained in:
parent
ef8a6c6268
commit
1214aa841b
1
Makefile
1
Makefile
|
@ -2451,6 +2451,7 @@ xdiff-objs: $(XDIFF_OBJS)
|
|||
|
||||
REFTABLE_OBJS += reftable/basics.o
|
||||
REFTABLE_OBJS += reftable/error.o
|
||||
REFTABLE_OBJS += reftable/blocksource.o
|
||||
REFTABLE_OBJS += reftable/publicbasics.o
|
||||
|
||||
REFTABLE_TEST_OBJS += reftable/test_framework.o
|
||||
|
|
148
reftable/blocksource.c
Normal file
148
reftable/blocksource.c
Normal file
|
@ -0,0 +1,148 @@
|
|||
/*
|
||||
Copyright 2020 Google LLC
|
||||
|
||||
Use of this source code is governed by a BSD-style
|
||||
license that can be found in the LICENSE file or at
|
||||
https://developers.google.com/open-source/licenses/bsd
|
||||
*/
|
||||
|
||||
#include "system.h"
|
||||
|
||||
#include "basics.h"
|
||||
#include "blocksource.h"
|
||||
#include "reftable-blocksource.h"
|
||||
#include "reftable-error.h"
|
||||
|
||||
static void strbuf_return_block(void *b, struct reftable_block *dest)
|
||||
{
|
||||
memset(dest->data, 0xff, dest->len);
|
||||
reftable_free(dest->data);
|
||||
}
|
||||
|
||||
static void strbuf_close(void *b)
|
||||
{
|
||||
}
|
||||
|
||||
static int strbuf_read_block(void *v, struct reftable_block *dest, uint64_t off,
|
||||
uint32_t size)
|
||||
{
|
||||
struct strbuf *b = v;
|
||||
assert(off + size <= b->len);
|
||||
dest->data = reftable_calloc(size);
|
||||
memcpy(dest->data, b->buf + off, size);
|
||||
dest->len = size;
|
||||
return size;
|
||||
}
|
||||
|
||||
static uint64_t strbuf_size(void *b)
|
||||
{
|
||||
return ((struct strbuf *)b)->len;
|
||||
}
|
||||
|
||||
static struct reftable_block_source_vtable strbuf_vtable = {
|
||||
.size = &strbuf_size,
|
||||
.read_block = &strbuf_read_block,
|
||||
.return_block = &strbuf_return_block,
|
||||
.close = &strbuf_close,
|
||||
};
|
||||
|
||||
void block_source_from_strbuf(struct reftable_block_source *bs,
|
||||
struct strbuf *buf)
|
||||
{
|
||||
assert(!bs->ops);
|
||||
bs->ops = &strbuf_vtable;
|
||||
bs->arg = buf;
|
||||
}
|
||||
|
||||
static void malloc_return_block(void *b, struct reftable_block *dest)
|
||||
{
|
||||
memset(dest->data, 0xff, dest->len);
|
||||
reftable_free(dest->data);
|
||||
}
|
||||
|
||||
static struct reftable_block_source_vtable malloc_vtable = {
|
||||
.return_block = &malloc_return_block,
|
||||
};
|
||||
|
||||
static struct reftable_block_source malloc_block_source_instance = {
|
||||
.ops = &malloc_vtable,
|
||||
};
|
||||
|
||||
struct reftable_block_source malloc_block_source(void)
|
||||
{
|
||||
return malloc_block_source_instance;
|
||||
}
|
||||
|
||||
struct file_block_source {
|
||||
int fd;
|
||||
uint64_t size;
|
||||
};
|
||||
|
||||
static uint64_t file_size(void *b)
|
||||
{
|
||||
return ((struct file_block_source *)b)->size;
|
||||
}
|
||||
|
||||
static void file_return_block(void *b, struct reftable_block *dest)
|
||||
{
|
||||
memset(dest->data, 0xff, dest->len);
|
||||
reftable_free(dest->data);
|
||||
}
|
||||
|
||||
static void file_close(void *b)
|
||||
{
|
||||
int fd = ((struct file_block_source *)b)->fd;
|
||||
if (fd > 0) {
|
||||
close(fd);
|
||||
((struct file_block_source *)b)->fd = 0;
|
||||
}
|
||||
|
||||
reftable_free(b);
|
||||
}
|
||||
|
||||
static int file_read_block(void *v, struct reftable_block *dest, uint64_t off,
|
||||
uint32_t size)
|
||||
{
|
||||
struct file_block_source *b = v;
|
||||
assert(off + size <= b->size);
|
||||
dest->data = reftable_malloc(size);
|
||||
if (pread(b->fd, dest->data, size, off) != size)
|
||||
return -1;
|
||||
dest->len = size;
|
||||
return size;
|
||||
}
|
||||
|
||||
static struct reftable_block_source_vtable file_vtable = {
|
||||
.size = &file_size,
|
||||
.read_block = &file_read_block,
|
||||
.return_block = &file_return_block,
|
||||
.close = &file_close,
|
||||
};
|
||||
|
||||
int reftable_block_source_from_file(struct reftable_block_source *bs,
|
||||
const char *name)
|
||||
{
|
||||
struct stat st = { 0 };
|
||||
int err = 0;
|
||||
int fd = open(name, O_RDONLY);
|
||||
struct file_block_source *p = NULL;
|
||||
if (fd < 0) {
|
||||
if (errno == ENOENT) {
|
||||
return REFTABLE_NOT_EXIST_ERROR;
|
||||
}
|
||||
return -1;
|
||||
}
|
||||
|
||||
err = fstat(fd, &st);
|
||||
if (err < 0)
|
||||
return -1;
|
||||
|
||||
p = reftable_calloc(sizeof(struct file_block_source));
|
||||
p->size = st.st_size;
|
||||
p->fd = fd;
|
||||
|
||||
assert(!bs->ops);
|
||||
bs->ops = &file_vtable;
|
||||
bs->arg = p;
|
||||
return 0;
|
||||
}
|
22
reftable/blocksource.h
Normal file
22
reftable/blocksource.h
Normal file
|
@ -0,0 +1,22 @@
|
|||
/*
|
||||
Copyright 2020 Google LLC
|
||||
|
||||
Use of this source code is governed by a BSD-style
|
||||
license that can be found in the LICENSE file or at
|
||||
https://developers.google.com/open-source/licenses/bsd
|
||||
*/
|
||||
|
||||
#ifndef BLOCKSOURCE_H
|
||||
#define BLOCKSOURCE_H
|
||||
|
||||
#include "system.h"
|
||||
|
||||
struct reftable_block_source;
|
||||
|
||||
/* Create an in-memory block source for reading reftables */
|
||||
void block_source_from_strbuf(struct reftable_block_source *bs,
|
||||
struct strbuf *buf);
|
||||
|
||||
struct reftable_block_source malloc_block_source(void);
|
||||
|
||||
#endif
|
49
reftable/reftable-blocksource.h
Normal file
49
reftable/reftable-blocksource.h
Normal file
|
@ -0,0 +1,49 @@
|
|||
/*
|
||||
Copyright 2020 Google LLC
|
||||
|
||||
Use of this source code is governed by a BSD-style
|
||||
license that can be found in the LICENSE file or at
|
||||
https://developers.google.com/open-source/licenses/bsd
|
||||
*/
|
||||
|
||||
#ifndef REFTABLE_BLOCKSOURCE_H
|
||||
#define REFTABLE_BLOCKSOURCE_H
|
||||
|
||||
#include <stdint.h>
|
||||
|
||||
/* block_source is a generic wrapper for a seekable readable file.
|
||||
*/
|
||||
struct reftable_block_source {
|
||||
struct reftable_block_source_vtable *ops;
|
||||
void *arg;
|
||||
};
|
||||
|
||||
/* a contiguous segment of bytes. It keeps track of its generating block_source
|
||||
* so it can return itself into the pool. */
|
||||
struct reftable_block {
|
||||
uint8_t *data;
|
||||
int len;
|
||||
struct reftable_block_source source;
|
||||
};
|
||||
|
||||
/* block_source_vtable are the operations that make up block_source */
|
||||
struct reftable_block_source_vtable {
|
||||
/* returns the size of a block source */
|
||||
uint64_t (*size)(void *source);
|
||||
|
||||
/* reads a segment from the block source. It is an error to read
|
||||
beyond the end of the block */
|
||||
int (*read_block)(void *source, struct reftable_block *dest,
|
||||
uint64_t off, uint32_t size);
|
||||
/* mark the block as read; may return the data back to malloc */
|
||||
void (*return_block)(void *source, struct reftable_block *blockp);
|
||||
|
||||
/* release all resources associated with the block source */
|
||||
void (*close)(void *source);
|
||||
};
|
||||
|
||||
/* opens a file on the file system as a block_source */
|
||||
int reftable_block_source_from_file(struct reftable_block_source *block_src,
|
||||
const char *name);
|
||||
|
||||
#endif
|
Loading…
Reference in a new issue