mirror of
https://github.com/git/git
synced 2024-08-25 02:35:46 +00:00
![Andrzej Hunt](/assets/img/avatar_default.png)
ibuf can be reused for multiple iterations of the loop. Specifically: deflate() overwrites s.avail_in to show how much of the input buffer has not been processed yet - and sometimes leaves 'avail_in > 0', in which case ibuf will be processed again during the loop's subsequent iteration. But if we declare ibuf within the loop, then (in theory) we get a new (and uninitialised) buffer for every iteration. In practice, my compiler seems to resue the same buffer - meaning that this code does work - but it doesn't seem safe to rely on this behaviour. MSAN correctly catches this issue - as soon as we hit the 's.avail_in > 0' condition, we end up reading from what seems to be uninitialised memory. Therefore, we move ibuf out of the loop, making this reuse safe. See MSAN output from t1050-large below - the interesting part is the ibuf creation at the end, although there's a lot of indirection before we reach the read from unitialised memory: ==11294==WARNING: MemorySanitizer: use-of-uninitialized-value #0 0x7f75db58fb1c in crc32_little crc32.c:283:9 #1 0x7f75db58d5b3 in crc32_z crc32.c:220:20 #2 0x7f75db59668c in crc32 crc32.c:242:12 #3 0x8c94f8 in hashwrite csum-file.c:101:15 #4 0x825faf in stream_to_pack bulk-checkin.c:154:5 #5 0x82467b in deflate_to_pack bulk-checkin.c:225:8 #6 0x823ff1 in index_bulk_checkin bulk-checkin.c:264:15 #7 0xa7cff2 in index_stream object-file.c:2234:9 #8 0xa7bff7 in index_fd object-file.c:2256:9 #9 0xa7d22d in index_path object-file.c:2274:7 #10 0xb3c8c9 in add_to_index read-cache.c:802:7 #11 0xb3e039 in add_file_to_index read-cache.c:835:9 #12 0x4a99c3 in add_files add.c:458:7 #13 0x4a7276 in cmd_add add.c:670:18 #14 0x4a1e76 in run_builtin git.c:461:11 #15 0x49e1e7 in handle_builtin git.c:714:3 #16 0x4a0c08 in run_argv git.c:781:4 #17 0x49d5a8 in cmd_main git.c:912:19 #18 0x7974da in main common-main.c:52:11 #19 0x7f75da66f349 in __libc_start_main (/lib64/libc.so.6+0x24349) #20 0x421bd9 in _start start.S:120 Uninitialized value was stored to memory at #0 0x7f75db58fa6b in crc32_little crc32.c:283:9 #1 0x7f75db58d5b3 in crc32_z crc32.c:220:20 #2 0x7f75db59668c in crc32 crc32.c:242:12 #3 0x8c94f8 in hashwrite csum-file.c:101:15 #4 0x825faf in stream_to_pack bulk-checkin.c:154:5 #5 0x82467b in deflate_to_pack bulk-checkin.c:225:8 #6 0x823ff1 in index_bulk_checkin bulk-checkin.c:264:15 #7 0xa7cff2 in index_stream object-file.c:2234:9 #8 0xa7bff7 in index_fd object-file.c:2256:9 #9 0xa7d22d in index_path object-file.c:2274:7 #10 0xb3c8c9 in add_to_index read-cache.c:802:7 #11 0xb3e039 in add_file_to_index read-cache.c:835:9 #12 0x4a99c3 in add_files add.c:458:7 #13 0x4a7276 in cmd_add add.c:670:18 #14 0x4a1e76 in run_builtin git.c:461:11 #15 0x49e1e7 in handle_builtin git.c:714:3 #16 0x4a0c08 in run_argv git.c:781:4 #17 0x49d5a8 in cmd_main git.c:912:19 #18 0x7974da in main common-main.c:52:11 #19 0x7f75da66f349 in __libc_start_main (/lib64/libc.so.6+0x24349) Uninitialized value was stored to memory at #0 0x447eb9 in __msan_memcpy msan_interceptors.cpp:1558:3 #1 0x7f75db5c2011 in flush_pending deflate.c:746:5 #2 0x7f75db5cafa0 in deflate_stored deflate.c:1815:9 #3 0x7f75db5bb7d2 in deflate deflate.c:1005:34 #4 0xd80b7f in git_deflate zlib.c:244:12 #5 0x825dff in stream_to_pack bulk-checkin.c:140:12 #6 0x82467b in deflate_to_pack bulk-checkin.c:225:8 #7 0x823ff1 in index_bulk_checkin bulk-checkin.c:264:15 #8 0xa7cff2 in index_stream object-file.c:2234:9 #9 0xa7bff7 in index_fd object-file.c:2256:9 #10 0xa7d22d in index_path object-file.c:2274:7 #11 0xb3c8c9 in add_to_index read-cache.c:802:7 #12 0xb3e039 in add_file_to_index read-cache.c:835:9 #13 0x4a99c3 in add_files add.c:458:7 #14 0x4a7276 in cmd_add add.c:670:18 #15 0x4a1e76 in run_builtin git.c:461:11 #16 0x49e1e7 in handle_builtin git.c:714:3 #17 0x4a0c08 in run_argv git.c:781:4 #18 0x49d5a8 in cmd_main git.c:912:19 #19 0x7974da in main common-main.c:52:11 Uninitialized value was stored to memory at #0 0x447eb9 in __msan_memcpy msan_interceptors.cpp:1558:3 #1 0x7f75db644241 in _tr_stored_block trees.c:873:5 #2 0x7f75db5cad7c in deflate_stored deflate.c:1813:9 #3 0x7f75db5bb7d2 in deflate deflate.c:1005:34 #4 0xd80b7f in git_deflate zlib.c:244:12 #5 0x825dff in stream_to_pack bulk-checkin.c:140:12 #6 0x82467b in deflate_to_pack bulk-checkin.c:225:8 #7 0x823ff1 in index_bulk_checkin bulk-checkin.c:264:15 #8 0xa7cff2 in index_stream object-file.c:2234:9 #9 0xa7bff7 in index_fd object-file.c:2256:9 #10 0xa7d22d in index_path object-file.c:2274:7 #11 0xb3c8c9 in add_to_index read-cache.c:802:7 #12 0xb3e039 in add_file_to_index read-cache.c:835:9 #13 0x4a99c3 in add_files add.c:458:7 #14 0x4a7276 in cmd_add add.c:670:18 #15 0x4a1e76 in run_builtin git.c:461:11 #16 0x49e1e7 in handle_builtin git.c:714:3 #17 0x4a0c08 in run_argv git.c:781:4 #18 0x49d5a8 in cmd_main git.c:912:19 #19 0x7974da in main common-main.c:52:11 Uninitialized value was stored to memory at #0 0x447eb9 in __msan_memcpy msan_interceptors.cpp:1558:3 #1 0x7f75db5c8fcf in deflate_stored deflate.c:1783:9 #2 0x7f75db5bb7d2 in deflate deflate.c:1005:34 #3 0xd80b7f in git_deflate zlib.c:244:12 #4 0x825dff in stream_to_pack bulk-checkin.c:140:12 #5 0x82467b in deflate_to_pack bulk-checkin.c:225:8 #6 0x823ff1 in index_bulk_checkin bulk-checkin.c:264:15 #7 0xa7cff2 in index_stream object-file.c:2234:9 #8 0xa7bff7 in index_fd object-file.c:2256:9 #9 0xa7d22d in index_path object-file.c:2274:7 #10 0xb3c8c9 in add_to_index read-cache.c:802:7 #11 0xb3e039 in add_file_to_index read-cache.c:835:9 #12 0x4a99c3 in add_files add.c:458:7 #13 0x4a7276 in cmd_add add.c:670:18 #14 0x4a1e76 in run_builtin git.c:461:11 #15 0x49e1e7 in handle_builtin git.c:714:3 #16 0x4a0c08 in run_argv git.c:781:4 #17 0x49d5a8 in cmd_main git.c:912:19 #18 0x7974da in main common-main.c:52:11 #19 0x7f75da66f349 in __libc_start_main (/lib64/libc.so.6+0x24349) Uninitialized value was stored to memory at #0 0x447eb9 in __msan_memcpy msan_interceptors.cpp:1558:3 #1 0x7f75db5ea545 in read_buf deflate.c:1181:5 #2 0x7f75db5c97f7 in deflate_stored deflate.c:1791:9 #3 0x7f75db5bb7d2 in deflate deflate.c:1005:34 #4 0xd80b7f in git_deflate zlib.c:244:12 #5 0x825dff in stream_to_pack bulk-checkin.c:140:12 #6 0x82467b in deflate_to_pack bulk-checkin.c:225:8 #7 0x823ff1 in index_bulk_checkin bulk-checkin.c:264:15 #8 0xa7cff2 in index_stream object-file.c:2234:9 #9 0xa7bff7 in index_fd object-file.c:2256:9 #10 0xa7d22d in index_path object-file.c:2274:7 #11 0xb3c8c9 in add_to_index read-cache.c:802:7 #12 0xb3e039 in add_file_to_index read-cache.c:835:9 #13 0x4a99c3 in add_files add.c:458:7 #14 0x4a7276 in cmd_add add.c:670:18 #15 0x4a1e76 in run_builtin git.c:461:11 #16 0x49e1e7 in handle_builtin git.c:714:3 #17 0x4a0c08 in run_argv git.c:781:4 #18 0x49d5a8 in cmd_main git.c:912:19 #19 0x7974da in main common-main.c:52:11 Uninitialized value was created by an allocation of 'ibuf' in the stack frame of function 'stream_to_pack' #0 0x825710 in stream_to_pack bulk-checkin.c:101 SUMMARY: MemorySanitizer: use-of-uninitialized-value crc32.c:283:9 in crc32_little Exiting Signed-off-by: Andrzej Hunt <andrzej@ahunt.org> Signed-off-by: Junio C Hamano <gitster@pobox.com>
281 lines
7.3 KiB
C
281 lines
7.3 KiB
C
/*
|
|
* Copyright (c) 2011, Google Inc.
|
|
*/
|
|
#include "cache.h"
|
|
#include "bulk-checkin.h"
|
|
#include "repository.h"
|
|
#include "csum-file.h"
|
|
#include "pack.h"
|
|
#include "strbuf.h"
|
|
#include "packfile.h"
|
|
#include "object-store.h"
|
|
|
|
static struct bulk_checkin_state {
|
|
unsigned plugged:1;
|
|
|
|
char *pack_tmp_name;
|
|
struct hashfile *f;
|
|
off_t offset;
|
|
struct pack_idx_option pack_idx_opts;
|
|
|
|
struct pack_idx_entry **written;
|
|
uint32_t alloc_written;
|
|
uint32_t nr_written;
|
|
} state;
|
|
|
|
static void finish_bulk_checkin(struct bulk_checkin_state *state)
|
|
{
|
|
struct object_id oid;
|
|
struct strbuf packname = STRBUF_INIT;
|
|
int i;
|
|
|
|
if (!state->f)
|
|
return;
|
|
|
|
if (state->nr_written == 0) {
|
|
close(state->f->fd);
|
|
unlink(state->pack_tmp_name);
|
|
goto clear_exit;
|
|
} else if (state->nr_written == 1) {
|
|
finalize_hashfile(state->f, oid.hash, CSUM_HASH_IN_STREAM | CSUM_FSYNC | CSUM_CLOSE);
|
|
} else {
|
|
int fd = finalize_hashfile(state->f, oid.hash, 0);
|
|
fixup_pack_header_footer(fd, oid.hash, state->pack_tmp_name,
|
|
state->nr_written, oid.hash,
|
|
state->offset);
|
|
close(fd);
|
|
}
|
|
|
|
strbuf_addf(&packname, "%s/pack/pack-", get_object_directory());
|
|
finish_tmp_packfile(&packname, state->pack_tmp_name,
|
|
state->written, state->nr_written,
|
|
&state->pack_idx_opts, oid.hash);
|
|
for (i = 0; i < state->nr_written; i++)
|
|
free(state->written[i]);
|
|
|
|
clear_exit:
|
|
free(state->written);
|
|
memset(state, 0, sizeof(*state));
|
|
|
|
strbuf_release(&packname);
|
|
/* Make objects we just wrote available to ourselves */
|
|
reprepare_packed_git(the_repository);
|
|
}
|
|
|
|
static int already_written(struct bulk_checkin_state *state, struct object_id *oid)
|
|
{
|
|
int i;
|
|
|
|
/* The object may already exist in the repository */
|
|
if (has_object_file(oid))
|
|
return 1;
|
|
|
|
/* Might want to keep the list sorted */
|
|
for (i = 0; i < state->nr_written; i++)
|
|
if (oideq(&state->written[i]->oid, oid))
|
|
return 1;
|
|
|
|
/* This is a new object we need to keep */
|
|
return 0;
|
|
}
|
|
|
|
/*
|
|
* Read the contents from fd for size bytes, streaming it to the
|
|
* packfile in state while updating the hash in ctx. Signal a failure
|
|
* by returning a negative value when the resulting pack would exceed
|
|
* the pack size limit and this is not the first object in the pack,
|
|
* so that the caller can discard what we wrote from the current pack
|
|
* by truncating it and opening a new one. The caller will then call
|
|
* us again after rewinding the input fd.
|
|
*
|
|
* The already_hashed_to pointer is kept untouched by the caller to
|
|
* make sure we do not hash the same byte when we are called
|
|
* again. This way, the caller does not have to checkpoint its hash
|
|
* status before calling us just in case we ask it to call us again
|
|
* with a new pack.
|
|
*/
|
|
static int stream_to_pack(struct bulk_checkin_state *state,
|
|
git_hash_ctx *ctx, off_t *already_hashed_to,
|
|
int fd, size_t size, enum object_type type,
|
|
const char *path, unsigned flags)
|
|
{
|
|
git_zstream s;
|
|
unsigned char ibuf[16384];
|
|
unsigned char obuf[16384];
|
|
unsigned hdrlen;
|
|
int status = Z_OK;
|
|
int write_object = (flags & HASH_WRITE_OBJECT);
|
|
off_t offset = 0;
|
|
|
|
git_deflate_init(&s, pack_compression_level);
|
|
|
|
hdrlen = encode_in_pack_object_header(obuf, sizeof(obuf), type, size);
|
|
s.next_out = obuf + hdrlen;
|
|
s.avail_out = sizeof(obuf) - hdrlen;
|
|
|
|
while (status != Z_STREAM_END) {
|
|
if (size && !s.avail_in) {
|
|
ssize_t rsize = size < sizeof(ibuf) ? size : sizeof(ibuf);
|
|
ssize_t read_result = read_in_full(fd, ibuf, rsize);
|
|
if (read_result < 0)
|
|
die_errno("failed to read from '%s'", path);
|
|
if (read_result != rsize)
|
|
die("failed to read %d bytes from '%s'",
|
|
(int)rsize, path);
|
|
offset += rsize;
|
|
if (*already_hashed_to < offset) {
|
|
size_t hsize = offset - *already_hashed_to;
|
|
if (rsize < hsize)
|
|
hsize = rsize;
|
|
if (hsize)
|
|
the_hash_algo->update_fn(ctx, ibuf, hsize);
|
|
*already_hashed_to = offset;
|
|
}
|
|
s.next_in = ibuf;
|
|
s.avail_in = rsize;
|
|
size -= rsize;
|
|
}
|
|
|
|
status = git_deflate(&s, size ? 0 : Z_FINISH);
|
|
|
|
if (!s.avail_out || status == Z_STREAM_END) {
|
|
if (write_object) {
|
|
size_t written = s.next_out - obuf;
|
|
|
|
/* would we bust the size limit? */
|
|
if (state->nr_written &&
|
|
pack_size_limit_cfg &&
|
|
pack_size_limit_cfg < state->offset + written) {
|
|
git_deflate_abort(&s);
|
|
return -1;
|
|
}
|
|
|
|
hashwrite(state->f, obuf, written);
|
|
state->offset += written;
|
|
}
|
|
s.next_out = obuf;
|
|
s.avail_out = sizeof(obuf);
|
|
}
|
|
|
|
switch (status) {
|
|
case Z_OK:
|
|
case Z_BUF_ERROR:
|
|
case Z_STREAM_END:
|
|
continue;
|
|
default:
|
|
die("unexpected deflate failure: %d", status);
|
|
}
|
|
}
|
|
git_deflate_end(&s);
|
|
return 0;
|
|
}
|
|
|
|
/* Lazily create backing packfile for the state */
|
|
static void prepare_to_stream(struct bulk_checkin_state *state,
|
|
unsigned flags)
|
|
{
|
|
if (!(flags & HASH_WRITE_OBJECT) || state->f)
|
|
return;
|
|
|
|
state->f = create_tmp_packfile(&state->pack_tmp_name);
|
|
reset_pack_idx_option(&state->pack_idx_opts);
|
|
|
|
/* Pretend we are going to write only one object */
|
|
state->offset = write_pack_header(state->f, 1);
|
|
if (!state->offset)
|
|
die_errno("unable to write pack header");
|
|
}
|
|
|
|
static int deflate_to_pack(struct bulk_checkin_state *state,
|
|
struct object_id *result_oid,
|
|
int fd, size_t size,
|
|
enum object_type type, const char *path,
|
|
unsigned flags)
|
|
{
|
|
off_t seekback, already_hashed_to;
|
|
git_hash_ctx ctx;
|
|
unsigned char obuf[16384];
|
|
unsigned header_len;
|
|
struct hashfile_checkpoint checkpoint = {0};
|
|
struct pack_idx_entry *idx = NULL;
|
|
|
|
seekback = lseek(fd, 0, SEEK_CUR);
|
|
if (seekback == (off_t) -1)
|
|
return error("cannot find the current offset");
|
|
|
|
header_len = xsnprintf((char *)obuf, sizeof(obuf), "%s %" PRIuMAX,
|
|
type_name(type), (uintmax_t)size) + 1;
|
|
the_hash_algo->init_fn(&ctx);
|
|
the_hash_algo->update_fn(&ctx, obuf, header_len);
|
|
|
|
/* Note: idx is non-NULL when we are writing */
|
|
if ((flags & HASH_WRITE_OBJECT) != 0)
|
|
CALLOC_ARRAY(idx, 1);
|
|
|
|
already_hashed_to = 0;
|
|
|
|
while (1) {
|
|
prepare_to_stream(state, flags);
|
|
if (idx) {
|
|
hashfile_checkpoint(state->f, &checkpoint);
|
|
idx->offset = state->offset;
|
|
crc32_begin(state->f);
|
|
}
|
|
if (!stream_to_pack(state, &ctx, &already_hashed_to,
|
|
fd, size, type, path, flags))
|
|
break;
|
|
/*
|
|
* Writing this object to the current pack will make
|
|
* it too big; we need to truncate it, start a new
|
|
* pack, and write into it.
|
|
*/
|
|
if (!idx)
|
|
BUG("should not happen");
|
|
hashfile_truncate(state->f, &checkpoint);
|
|
state->offset = checkpoint.offset;
|
|
finish_bulk_checkin(state);
|
|
if (lseek(fd, seekback, SEEK_SET) == (off_t) -1)
|
|
return error("cannot seek back");
|
|
}
|
|
the_hash_algo->final_oid_fn(result_oid, &ctx);
|
|
if (!idx)
|
|
return 0;
|
|
|
|
idx->crc32 = crc32_end(state->f);
|
|
if (already_written(state, result_oid)) {
|
|
hashfile_truncate(state->f, &checkpoint);
|
|
state->offset = checkpoint.offset;
|
|
free(idx);
|
|
} else {
|
|
oidcpy(&idx->oid, result_oid);
|
|
ALLOC_GROW(state->written,
|
|
state->nr_written + 1,
|
|
state->alloc_written);
|
|
state->written[state->nr_written++] = idx;
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
int index_bulk_checkin(struct object_id *oid,
|
|
int fd, size_t size, enum object_type type,
|
|
const char *path, unsigned flags)
|
|
{
|
|
int status = deflate_to_pack(&state, oid, fd, size, type,
|
|
path, flags);
|
|
if (!state.plugged)
|
|
finish_bulk_checkin(&state);
|
|
return status;
|
|
}
|
|
|
|
void plug_bulk_checkin(void)
|
|
{
|
|
state.plugged = 1;
|
|
}
|
|
|
|
void unplug_bulk_checkin(void)
|
|
{
|
|
state.plugged = 0;
|
|
if (state.f)
|
|
finish_bulk_checkin(&state);
|
|
}
|