2021-02-18 14:07:24 +00:00
|
|
|
#include "cache.h"
|
|
|
|
|
#include "chunk-format.h"
|
|
|
|
|
#include "csum-file.h"
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* When writing a chunk-based file format, collect the chunks in
|
|
|
|
|
* an array of chunk_info structs. The size stores the _expected_
|
|
|
|
|
* amount of data that will be written by write_fn.
|
|
|
|
|
*/
|
|
|
|
|
struct chunk_info {
|
|
|
|
|
uint32_t id;
|
|
|
|
|
uint64_t size;
|
|
|
|
|
chunk_write_fn write_fn;
|
2021-02-18 14:07:34 +00:00
|
|
|
|
|
|
|
|
const void *start;
|
2021-02-18 14:07:24 +00:00
|
|
|
};
|
|
|
|
|
|
|
|
|
|
struct chunkfile {
|
|
|
|
|
struct hashfile *f;
|
|
|
|
|
|
|
|
|
|
struct chunk_info *chunks;
|
|
|
|
|
size_t chunks_nr;
|
|
|
|
|
size_t chunks_alloc;
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
struct chunkfile *init_chunkfile(struct hashfile *f)
|
|
|
|
|
{
|
|
|
|
|
struct chunkfile *cf = xcalloc(1, sizeof(*cf));
|
|
|
|
|
cf->f = f;
|
|
|
|
|
return cf;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
void free_chunkfile(struct chunkfile *cf)
|
|
|
|
|
{
|
|
|
|
|
if (!cf)
|
|
|
|
|
return;
|
|
|
|
|
free(cf->chunks);
|
|
|
|
|
free(cf);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
int get_num_chunks(struct chunkfile *cf)
|
|
|
|
|
{
|
|
|
|
|
return cf->chunks_nr;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
void add_chunk(struct chunkfile *cf,
|
|
|
|
|
uint32_t id,
|
|
|
|
|
size_t size,
|
|
|
|
|
chunk_write_fn fn)
|
|
|
|
|
{
|
|
|
|
|
ALLOC_GROW(cf->chunks, cf->chunks_nr + 1, cf->chunks_alloc);
|
|
|
|
|
|
|
|
|
|
cf->chunks[cf->chunks_nr].id = id;
|
|
|
|
|
cf->chunks[cf->chunks_nr].write_fn = fn;
|
|
|
|
|
cf->chunks[cf->chunks_nr].size = size;
|
|
|
|
|
cf->chunks_nr++;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
int write_chunkfile(struct chunkfile *cf, void *data)
|
|
|
|
|
{
|
csum-file.h: increase hashfile buffer size
The hashfile API uses a hard-coded buffer size of 8KB and has ever since
it was introduced in c38138c (git-pack-objects: write the pack files
with a SHA1 csum, 2005-06-26). It performs a similar function to the
hashing buffers in read-cache.c, but that code was updated from 8KB to
128KB in f279894 (read-cache: make the index write buffer size 128K,
2021-02-18). The justification there was that do_write_index() improves
from 1.02s to 0.72s. Since our end goal is to have the index writing
code use the hashfile API, we need to unify this buffer size to avoid a
performance regression.
There is a buffer, 'check_buffer', that is used to verify the check_fd
file descriptor. When this buffer increases to 128K to fit the data
being flushed, it causes the stack to overflow the limits placed in the
test suite. To avoid issues with stack size, move both 'buffer' and
'check_buffer' to be heap pointers within 'struct hashfile'. The
'check_buffer' member is left as NULL unless check_fd is set in
hashfd_check(). Both buffers are cleared as part of finalize_hashfile()
which also frees the full structure.
Since these buffers are now on the heap, we can adjust their size based
on the needs of the consumer. In particular, callers to
hashfd_throughput() are expecting to report progress indicators as the
buffer flushes. These callers would prefer the smaller 8k buffer to
avoid large delays between updates, especially for users with slower
networks. When the progress indicator is not used, the larger buffer is
preferrable.
By adding a new trace2 region in the chunk-format API, we can see that
the writing portion of 'git multi-pack-index write' lowers from ~1.49s
to ~1.47s on a Linux machine. These effects may be more pronounced or
diminished on other filesystems. The end-to-end timing is too noisy to
have a definitive change either way.
Signed-off-by: Derrick Stolee <dstolee@microsoft.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2021-05-18 18:32:46 +00:00
|
|
|
int i, result = 0;
|
2021-02-18 14:07:24 +00:00
|
|
|
uint64_t cur_offset = hashfile_total(cf->f);
|
|
|
|
|
|
csum-file.h: increase hashfile buffer size
The hashfile API uses a hard-coded buffer size of 8KB and has ever since
it was introduced in c38138c (git-pack-objects: write the pack files
with a SHA1 csum, 2005-06-26). It performs a similar function to the
hashing buffers in read-cache.c, but that code was updated from 8KB to
128KB in f279894 (read-cache: make the index write buffer size 128K,
2021-02-18). The justification there was that do_write_index() improves
from 1.02s to 0.72s. Since our end goal is to have the index writing
code use the hashfile API, we need to unify this buffer size to avoid a
performance regression.
There is a buffer, 'check_buffer', that is used to verify the check_fd
file descriptor. When this buffer increases to 128K to fit the data
being flushed, it causes the stack to overflow the limits placed in the
test suite. To avoid issues with stack size, move both 'buffer' and
'check_buffer' to be heap pointers within 'struct hashfile'. The
'check_buffer' member is left as NULL unless check_fd is set in
hashfd_check(). Both buffers are cleared as part of finalize_hashfile()
which also frees the full structure.
Since these buffers are now on the heap, we can adjust their size based
on the needs of the consumer. In particular, callers to
hashfd_throughput() are expecting to report progress indicators as the
buffer flushes. These callers would prefer the smaller 8k buffer to
avoid large delays between updates, especially for users with slower
networks. When the progress indicator is not used, the larger buffer is
preferrable.
By adding a new trace2 region in the chunk-format API, we can see that
the writing portion of 'git multi-pack-index write' lowers from ~1.49s
to ~1.47s on a Linux machine. These effects may be more pronounced or
diminished on other filesystems. The end-to-end timing is too noisy to
have a definitive change either way.
Signed-off-by: Derrick Stolee <dstolee@microsoft.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2021-05-18 18:32:46 +00:00
|
|
|
trace2_region_enter("chunkfile", "write", the_repository);
|
|
|
|
|
|
2021-02-18 14:07:24 +00:00
|
|
|
/* Add the table of contents to the current offset */
|
|
|
|
|
cur_offset += (cf->chunks_nr + 1) * CHUNK_TOC_ENTRY_SIZE;
|
|
|
|
|
|
|
|
|
|
for (i = 0; i < cf->chunks_nr; i++) {
|
|
|
|
|
hashwrite_be32(cf->f, cf->chunks[i].id);
|
|
|
|
|
hashwrite_be64(cf->f, cur_offset);
|
|
|
|
|
|
|
|
|
|
cur_offset += cf->chunks[i].size;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/* Trailing entry marks the end of the chunks */
|
|
|
|
|
hashwrite_be32(cf->f, 0);
|
|
|
|
|
hashwrite_be64(cf->f, cur_offset);
|
|
|
|
|
|
|
|
|
|
for (i = 0; i < cf->chunks_nr; i++) {
|
|
|
|
|
off_t start_offset = hashfile_total(cf->f);
|
csum-file.h: increase hashfile buffer size
The hashfile API uses a hard-coded buffer size of 8KB and has ever since
it was introduced in c38138c (git-pack-objects: write the pack files
with a SHA1 csum, 2005-06-26). It performs a similar function to the
hashing buffers in read-cache.c, but that code was updated from 8KB to
128KB in f279894 (read-cache: make the index write buffer size 128K,
2021-02-18). The justification there was that do_write_index() improves
from 1.02s to 0.72s. Since our end goal is to have the index writing
code use the hashfile API, we need to unify this buffer size to avoid a
performance regression.
There is a buffer, 'check_buffer', that is used to verify the check_fd
file descriptor. When this buffer increases to 128K to fit the data
being flushed, it causes the stack to overflow the limits placed in the
test suite. To avoid issues with stack size, move both 'buffer' and
'check_buffer' to be heap pointers within 'struct hashfile'. The
'check_buffer' member is left as NULL unless check_fd is set in
hashfd_check(). Both buffers are cleared as part of finalize_hashfile()
which also frees the full structure.
Since these buffers are now on the heap, we can adjust their size based
on the needs of the consumer. In particular, callers to
hashfd_throughput() are expecting to report progress indicators as the
buffer flushes. These callers would prefer the smaller 8k buffer to
avoid large delays between updates, especially for users with slower
networks. When the progress indicator is not used, the larger buffer is
preferrable.
By adding a new trace2 region in the chunk-format API, we can see that
the writing portion of 'git multi-pack-index write' lowers from ~1.49s
to ~1.47s on a Linux machine. These effects may be more pronounced or
diminished on other filesystems. The end-to-end timing is too noisy to
have a definitive change either way.
Signed-off-by: Derrick Stolee <dstolee@microsoft.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2021-05-18 18:32:46 +00:00
|
|
|
result = cf->chunks[i].write_fn(cf->f, data);
|
2021-02-18 14:07:24 +00:00
|
|
|
|
|
|
|
|
if (result)
|
csum-file.h: increase hashfile buffer size
The hashfile API uses a hard-coded buffer size of 8KB and has ever since
it was introduced in c38138c (git-pack-objects: write the pack files
with a SHA1 csum, 2005-06-26). It performs a similar function to the
hashing buffers in read-cache.c, but that code was updated from 8KB to
128KB in f279894 (read-cache: make the index write buffer size 128K,
2021-02-18). The justification there was that do_write_index() improves
from 1.02s to 0.72s. Since our end goal is to have the index writing
code use the hashfile API, we need to unify this buffer size to avoid a
performance regression.
There is a buffer, 'check_buffer', that is used to verify the check_fd
file descriptor. When this buffer increases to 128K to fit the data
being flushed, it causes the stack to overflow the limits placed in the
test suite. To avoid issues with stack size, move both 'buffer' and
'check_buffer' to be heap pointers within 'struct hashfile'. The
'check_buffer' member is left as NULL unless check_fd is set in
hashfd_check(). Both buffers are cleared as part of finalize_hashfile()
which also frees the full structure.
Since these buffers are now on the heap, we can adjust their size based
on the needs of the consumer. In particular, callers to
hashfd_throughput() are expecting to report progress indicators as the
buffer flushes. These callers would prefer the smaller 8k buffer to
avoid large delays between updates, especially for users with slower
networks. When the progress indicator is not used, the larger buffer is
preferrable.
By adding a new trace2 region in the chunk-format API, we can see that
the writing portion of 'git multi-pack-index write' lowers from ~1.49s
to ~1.47s on a Linux machine. These effects may be more pronounced or
diminished on other filesystems. The end-to-end timing is too noisy to
have a definitive change either way.
Signed-off-by: Derrick Stolee <dstolee@microsoft.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2021-05-18 18:32:46 +00:00
|
|
|
goto cleanup;
|
2021-02-18 14:07:24 +00:00
|
|
|
|
|
|
|
|
if (hashfile_total(cf->f) - start_offset != cf->chunks[i].size)
|
|
|
|
|
BUG("expected to write %"PRId64" bytes to chunk %"PRIx32", but wrote %"PRId64" instead",
|
|
|
|
|
cf->chunks[i].size, cf->chunks[i].id,
|
|
|
|
|
hashfile_total(cf->f) - start_offset);
|
|
|
|
|
}
|
|
|
|
|
|
csum-file.h: increase hashfile buffer size
The hashfile API uses a hard-coded buffer size of 8KB and has ever since
it was introduced in c38138c (git-pack-objects: write the pack files
with a SHA1 csum, 2005-06-26). It performs a similar function to the
hashing buffers in read-cache.c, but that code was updated from 8KB to
128KB in f279894 (read-cache: make the index write buffer size 128K,
2021-02-18). The justification there was that do_write_index() improves
from 1.02s to 0.72s. Since our end goal is to have the index writing
code use the hashfile API, we need to unify this buffer size to avoid a
performance regression.
There is a buffer, 'check_buffer', that is used to verify the check_fd
file descriptor. When this buffer increases to 128K to fit the data
being flushed, it causes the stack to overflow the limits placed in the
test suite. To avoid issues with stack size, move both 'buffer' and
'check_buffer' to be heap pointers within 'struct hashfile'. The
'check_buffer' member is left as NULL unless check_fd is set in
hashfd_check(). Both buffers are cleared as part of finalize_hashfile()
which also frees the full structure.
Since these buffers are now on the heap, we can adjust their size based
on the needs of the consumer. In particular, callers to
hashfd_throughput() are expecting to report progress indicators as the
buffer flushes. These callers would prefer the smaller 8k buffer to
avoid large delays between updates, especially for users with slower
networks. When the progress indicator is not used, the larger buffer is
preferrable.
By adding a new trace2 region in the chunk-format API, we can see that
the writing portion of 'git multi-pack-index write' lowers from ~1.49s
to ~1.47s on a Linux machine. These effects may be more pronounced or
diminished on other filesystems. The end-to-end timing is too noisy to
have a definitive change either way.
Signed-off-by: Derrick Stolee <dstolee@microsoft.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2021-05-18 18:32:46 +00:00
|
|
|
cleanup:
|
|
|
|
|
trace2_region_leave("chunkfile", "write", the_repository);
|
|
|
|
|
return result;
|
2021-02-18 14:07:24 +00:00
|
|
|
}
|
2021-02-18 14:07:34 +00:00
|
|
|
|
|
|
|
|
int read_table_of_contents(struct chunkfile *cf,
|
|
|
|
|
const unsigned char *mfile,
|
|
|
|
|
size_t mfile_size,
|
|
|
|
|
uint64_t toc_offset,
|
|
|
|
|
int toc_length)
|
|
|
|
|
{
|
2021-02-18 14:07:38 +00:00
|
|
|
int i;
|
2021-02-18 14:07:34 +00:00
|
|
|
uint32_t chunk_id;
|
|
|
|
|
const unsigned char *table_of_contents = mfile + toc_offset;
|
|
|
|
|
|
|
|
|
|
ALLOC_GROW(cf->chunks, toc_length, cf->chunks_alloc);
|
|
|
|
|
|
|
|
|
|
while (toc_length--) {
|
|
|
|
|
uint64_t chunk_offset, next_chunk_offset;
|
|
|
|
|
|
|
|
|
|
chunk_id = get_be32(table_of_contents);
|
|
|
|
|
chunk_offset = get_be64(table_of_contents + 4);
|
|
|
|
|
|
|
|
|
|
if (!chunk_id) {
|
|
|
|
|
error(_("terminating chunk id appears earlier than expected"));
|
|
|
|
|
return 1;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
table_of_contents += CHUNK_TOC_ENTRY_SIZE;
|
|
|
|
|
next_chunk_offset = get_be64(table_of_contents + 4);
|
|
|
|
|
|
|
|
|
|
if (next_chunk_offset < chunk_offset ||
|
|
|
|
|
next_chunk_offset > mfile_size - the_hash_algo->rawsz) {
|
|
|
|
|
error(_("improper chunk offset(s) %"PRIx64" and %"PRIx64""),
|
|
|
|
|
chunk_offset, next_chunk_offset);
|
|
|
|
|
return -1;
|
|
|
|
|
}
|
|
|
|
|
|
2021-02-18 14:07:38 +00:00
|
|
|
for (i = 0; i < cf->chunks_nr; i++) {
|
|
|
|
|
if (cf->chunks[i].id == chunk_id) {
|
|
|
|
|
error(_("duplicate chunk ID %"PRIx32" found"),
|
|
|
|
|
chunk_id);
|
|
|
|
|
return -1;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2021-02-18 14:07:34 +00:00
|
|
|
cf->chunks[cf->chunks_nr].id = chunk_id;
|
|
|
|
|
cf->chunks[cf->chunks_nr].start = mfile + chunk_offset;
|
|
|
|
|
cf->chunks[cf->chunks_nr].size = next_chunk_offset - chunk_offset;
|
|
|
|
|
cf->chunks_nr++;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
chunk_id = get_be32(table_of_contents);
|
|
|
|
|
if (chunk_id) {
|
|
|
|
|
error(_("final chunk has non-zero id %"PRIx32""), chunk_id);
|
|
|
|
|
return -1;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static int pair_chunk_fn(const unsigned char *chunk_start,
|
|
|
|
|
size_t chunk_size,
|
|
|
|
|
void *data)
|
|
|
|
|
{
|
|
|
|
|
const unsigned char **p = data;
|
|
|
|
|
*p = chunk_start;
|
|
|
|
|
return 0;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
int pair_chunk(struct chunkfile *cf,
|
|
|
|
|
uint32_t chunk_id,
|
|
|
|
|
const unsigned char **p)
|
|
|
|
|
{
|
|
|
|
|
return read_chunk(cf, chunk_id, pair_chunk_fn, p);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
int read_chunk(struct chunkfile *cf,
|
|
|
|
|
uint32_t chunk_id,
|
|
|
|
|
chunk_read_fn fn,
|
|
|
|
|
void *data)
|
|
|
|
|
{
|
|
|
|
|
int i;
|
|
|
|
|
|
|
|
|
|
for (i = 0; i < cf->chunks_nr; i++) {
|
|
|
|
|
if (cf->chunks[i].id == chunk_id)
|
|
|
|
|
return fn(cf->chunks[i].start, cf->chunks[i].size, data);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return CHUNK_NOT_FOUND;
|
|
|
|
|
}
|