2005-06-29 09:51:27 +00:00
|
|
|
#include "cache.h"
|
2018-04-25 18:21:04 +00:00
|
|
|
#include "repository.h"
|
2005-06-29 09:51:27 +00:00
|
|
|
#include "pack.h"
|
2008-02-28 05:25:19 +00:00
|
|
|
#include "pack-revindex.h"
|
2011-11-07 02:59:26 +00:00
|
|
|
#include "progress.h"
|
2017-08-18 22:20:19 +00:00
|
|
|
#include "packfile.h"
|
2018-03-23 17:20:59 +00:00
|
|
|
#include "object-store.h"
|
2005-06-29 09:51:27 +00:00
|
|
|
|
2011-03-16 07:08:34 +00:00
|
|
|
struct idx_entry {
|
2007-06-03 18:21:41 +00:00
|
|
|
off_t offset;
|
2008-06-25 03:19:02 +00:00
|
|
|
unsigned int nr;
|
2007-06-03 18:21:41 +00:00
|
|
|
};
|
|
|
|
|
|
|
|
static int compare_entries(const void *e1, const void *e2)
|
|
|
|
{
|
|
|
|
const struct idx_entry *entry1 = e1;
|
|
|
|
const struct idx_entry *entry2 = e2;
|
|
|
|
if (entry1->offset < entry2->offset)
|
|
|
|
return -1;
|
|
|
|
if (entry1->offset > entry2->offset)
|
|
|
|
return 1;
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2008-06-25 03:19:02 +00:00
|
|
|
int check_pack_crc(struct packed_git *p, struct pack_window **w_curs,
|
|
|
|
off_t offset, off_t len, unsigned int nr)
|
|
|
|
{
|
|
|
|
const uint32_t *index_crc;
|
2011-04-03 07:06:54 +00:00
|
|
|
uint32_t data_crc = crc32(0, NULL, 0);
|
2008-06-25 03:19:02 +00:00
|
|
|
|
|
|
|
do {
|
2011-06-10 18:52:15 +00:00
|
|
|
unsigned long avail;
|
2008-06-25 03:19:02 +00:00
|
|
|
void *data = use_pack(p, w_curs, offset, &avail);
|
|
|
|
if (avail > len)
|
|
|
|
avail = len;
|
|
|
|
data_crc = crc32(data_crc, data, avail);
|
|
|
|
offset += avail;
|
|
|
|
len -= avail;
|
|
|
|
} while (len);
|
|
|
|
|
|
|
|
index_crc = p->index_data;
|
compute pack .idx byte offsets using size_t
A pack and its matching .idx file are limited to 2^32 objects, because
the pack format contains a 32-bit field to store the number of objects.
Hence we use uint32_t in the code.
But the byte count of even a .idx file can be much larger than that,
because it stores at least a hash and an offset for each object. So
using SHA-1, a v2 .idx file will cross the 4GB boundary at 153,391,650
objects. This confuses load_idx(), which computes the minimum size like
this:
unsigned long min_size = 8 + 4*256 + nr*(hashsz + 4 + 4) + hashsz + hashsz;
Even though min_size will be big enough on most 64-bit platforms, the
actual arithmetic is done as a uint32_t, resulting in a truncation. We
actually exceed that min_size, but then we do:
unsigned long max_size = min_size;
if (nr)
max_size += (nr - 1)*8;
to account for the variable-sized table. That computation doesn't
overflow quite so low, but with the truncation for min_size, we end up
with a max_size that is much smaller than our actual size. So we
complain that the idx is invalid, and can't find any of its objects.
We can fix this case by casting "nr" to a size_t, which will do the
multiplication in 64-bits (assuming you're on a 64-bit platform; this
will never work on a 32-bit system since we couldn't map the whole .idx
anyway). Likewise, we don't have to worry about further additions,
because adding a smaller number to a size_t will convert the other side
to a size_t.
A few notes:
- obviously we could just declare "nr" as a size_t in the first place
(and likewise, packed_git.num_objects). But it's conceptually a
uint32_t because of the on-disk format, and we correctly treat it
that way in other contexts that don't need to compute byte offsets
(e.g., iterating over the set of objects should and generally does
use a uint32_t). Switching to size_t would make all of those other
cases look wrong.
- it could be argued that the proper type is off_t to represent the
file offset. But in practice the .idx file must fit within memory,
because we mmap the whole thing. And the rest of the code (including
the idx_size variable we're comparing against) uses size_t.
- we'll add the same cast to the max_size arithmetic line. Even though
we're adding to a larger type, which will convert our result, the
multiplication is still done as a 32-bit value and can itself
overflow. I didn't check this with my test case, since it would need
an even larger pack (~530M objects), but looking at compiler output
shows that it works this way. The standard should agree, but I
couldn't find anything explicit in 6.3.1.8 ("usual arithmetic
conversions").
The case in load_idx() was the most immediate one that I was able to
trigger. After fixing it, looking up actual objects (including the very
last one in sha1 order) works in a test repo with 153,725,110 objects.
That's because bsearch_hash() works with uint32_t entry indices, and the
actual byte access:
int cmp = hashcmp(table + mi * stride, sha1);
is done with "stride" as a size_t, causing the uint32_t "mi" to be
promoted to a size_t. This is the way most code will access the index
data.
However, I audited all of the other byte-wise accesses of
packed_git.index_data, and many of the others are suspect (they are
similar to the max_size one, where we are adding to a properly sized
offset or directly to a pointer, but the multiplication in the
sub-expression can overflow). I didn't trigger any of these in practice,
but I believe they're potential problems, and certainly adding in the
cast is not going to hurt anything here.
Signed-off-by: Jeff King <peff@peff.net>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2020-11-13 05:06:48 +00:00
|
|
|
index_crc += 2 + 256 + (size_t)p->num_objects * (the_hash_algo->rawsz/4) + nr;
|
2008-06-25 03:19:02 +00:00
|
|
|
|
|
|
|
return data_crc != ntohl(*index_crc);
|
|
|
|
}
|
|
|
|
|
2018-11-10 05:49:07 +00:00
|
|
|
static int verify_packfile(struct repository *r,
|
|
|
|
struct packed_git *p,
|
2011-11-07 02:59:25 +00:00
|
|
|
struct pack_window **w_curs,
|
2011-11-07 02:59:26 +00:00
|
|
|
verify_fn fn,
|
|
|
|
struct progress *progress, uint32_t base_count)
|
|
|
|
|
2005-06-29 09:51:27 +00:00
|
|
|
{
|
2007-03-07 01:44:30 +00:00
|
|
|
off_t index_size = p->index_size;
|
2007-03-16 20:42:50 +00:00
|
|
|
const unsigned char *index_base = p->index_data;
|
2018-02-01 02:18:43 +00:00
|
|
|
git_hash_ctx ctx;
|
2017-05-06 22:10:20 +00:00
|
|
|
unsigned char hash[GIT_MAX_RAWSZ], *pack_sig;
|
2009-01-18 08:04:26 +00:00
|
|
|
off_t offset = 0, pack_sig_ofs = 0;
|
2007-03-07 01:44:19 +00:00
|
|
|
uint32_t nr_objects, i;
|
2008-05-29 21:34:50 +00:00
|
|
|
int err = 0;
|
2007-06-03 18:21:41 +00:00
|
|
|
struct idx_entry *entries;
|
2005-06-29 09:51:27 +00:00
|
|
|
|
verify_packfile: check pack validity before accessing data
The verify_packfile() does not explicitly open the packfile;
instead, it starts with a sha1 checksum over the whole pack,
and relies on use_pack() to open the packfile as a side
effect.
If the pack cannot be opened for whatever reason (either
because its header information is corrupted, or perhaps
because a simultaneous repack deleted it), then use_pack()
will die(), as it has no way to return an error. This is not
ideal, as verify_packfile() otherwise tries to gently return
an error (this lets programs like git-fsck go on to check
other packs).
Instead, let's check is_pack_valid() up front, and return an
error if it fails. This will open the pack as a side effect,
and then use_pack() will later rely on our cached
descriptor, and avoid calling die().
Signed-off-by: Jeff King <peff@peff.net>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2016-09-22 03:49:05 +00:00
|
|
|
if (!is_pack_valid(p))
|
|
|
|
return error("packfile %s cannot be accessed", p->pack_name);
|
2005-06-29 09:51:27 +00:00
|
|
|
|
2020-01-30 20:32:19 +00:00
|
|
|
r->hash_algo->init_fn(&ctx);
|
2009-01-18 08:04:26 +00:00
|
|
|
do {
|
2011-06-10 18:52:15 +00:00
|
|
|
unsigned long remaining;
|
2006-12-23 07:34:13 +00:00
|
|
|
unsigned char *in = use_pack(p, w_curs, offset, &remaining);
|
|
|
|
offset += remaining;
|
2009-01-18 08:04:26 +00:00
|
|
|
if (!pack_sig_ofs)
|
2020-01-30 20:32:19 +00:00
|
|
|
pack_sig_ofs = p->pack_size - r->hash_algo->rawsz;
|
2008-05-29 21:34:50 +00:00
|
|
|
if (offset > pack_sig_ofs)
|
|
|
|
remaining -= (unsigned int)(offset - pack_sig_ofs);
|
2020-01-30 20:32:19 +00:00
|
|
|
r->hash_algo->update_fn(&ctx, in, remaining);
|
2009-01-18 08:04:26 +00:00
|
|
|
} while (offset < pack_sig_ofs);
|
2020-01-30 20:32:19 +00:00
|
|
|
r->hash_algo->final_fn(hash, &ctx);
|
2008-05-29 21:34:50 +00:00
|
|
|
pack_sig = use_pack(p, w_curs, pack_sig_ofs, NULL);
|
2018-08-28 21:22:52 +00:00
|
|
|
if (!hasheq(hash, pack_sig))
|
2018-02-01 02:18:43 +00:00
|
|
|
err = error("%s pack checksum mismatch",
|
2008-05-29 21:34:50 +00:00
|
|
|
p->pack_name);
|
2020-01-30 20:32:19 +00:00
|
|
|
if (!hasheq(index_base + index_size - r->hash_algo->hexsz, pack_sig))
|
2018-02-01 02:18:43 +00:00
|
|
|
err = error("%s pack checksum does not match its index",
|
2008-05-29 21:34:50 +00:00
|
|
|
p->pack_name);
|
2006-12-23 07:34:13 +00:00
|
|
|
unuse_pack(w_curs);
|
2005-07-01 00:15:39 +00:00
|
|
|
|
|
|
|
/* Make sure everything reachable from idx is valid. Since we
|
|
|
|
* have verified that nr_objects matches between idx and pack,
|
|
|
|
* we do not do scan-streaming check on the pack file.
|
|
|
|
*/
|
2007-04-09 05:06:28 +00:00
|
|
|
nr_objects = p->num_objects;
|
2016-02-22 22:44:25 +00:00
|
|
|
ALLOC_ARRAY(entries, nr_objects + 1);
|
2008-06-25 03:19:02 +00:00
|
|
|
entries[nr_objects].offset = pack_sig_ofs;
|
2007-06-03 18:21:41 +00:00
|
|
|
/* first sort entries by pack offset, since unpacking them is more efficient that way */
|
|
|
|
for (i = 0; i < nr_objects; i++) {
|
2008-06-25 03:17:12 +00:00
|
|
|
entries[i].offset = nth_packed_object_offset(p, i);
|
2008-06-25 03:19:02 +00:00
|
|
|
entries[i].nr = i;
|
2007-06-03 18:21:41 +00:00
|
|
|
}
|
2016-09-29 15:27:31 +00:00
|
|
|
QSORT(entries, nr_objects, compare_entries);
|
2007-06-03 18:21:41 +00:00
|
|
|
|
2008-05-29 21:34:50 +00:00
|
|
|
for (i = 0; i < nr_objects; i++) {
|
2005-07-01 00:15:39 +00:00
|
|
|
void *data;
|
pack-check: push oid lookup into loop
When we're checking a pack with fsck or verify-pack, we first sort the
idx entries by offset, since accessing them in pack order is more
efficient. To do so, we loop over them and fill in an array of structs
with the offset, object_id, and index position of each, sort the result,
and only then do we iterate over the sorted array and process each
entry.
In order to avoid the memory cost of storing the hash of each object, we
just store a pointer into the copy in the mmap'd pack index file. To
keep that property even as the rest of the code converted to "struct
object_id", commit 9fd750461b (Convert the verify_pack callback to
struct object_id, 2017-05-06) introduced a union in order to type-pun
the pointer-to-hash into an object_id struct.
But we can make this even simpler by observing that the sort operation
doesn't need the object id at all! We only need them one at a time while
we actually process each entry. So we can just omit the oid from the
struct entirely and load it on the fly into a local variable in the
second loop.
This gets rid of the type-punning, and lets us directly use the more
type-safe nth_packed_object_id(), simplifying the code. And as a bonus,
it saves 8 bytes of memory per object.
Note that this does mean we'll do the offset lookup for each object
before the oid lookup. The oid lookup has more safety checks in it
(e.g., for looking past p->num_objects) which in theory protected the
offset lookup. But since violating those checks was already a BUG()
condition (as described in the previous commit), it's not worth worrying
about.
Signed-off-by: Jeff King <peff@peff.net>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2020-02-24 04:36:31 +00:00
|
|
|
struct object_id oid;
|
2007-02-26 19:55:59 +00:00
|
|
|
enum object_type type;
|
2007-03-07 01:44:30 +00:00
|
|
|
unsigned long size;
|
fsck: use streaming interface for large blobs in pack
For blobs, we want to make sure the on-disk data is not corrupted
(i.e. can be inflated and produce the expected SHA-1). Blob content is
opaque, there's nothing else inside to check for.
For really large blobs, we may want to avoid unpacking the entire blob
in memory, just to check whether it produces the same SHA-1. On 32-bit
systems, we may not have enough virtual address space for such memory
allocation. And even on 64-bit where it's not a problem, allocating a
lot more memory could result in kicking other parts of systems to swap
file, generating lots of I/O and slowing everything down.
For this particular operation, not unpacking the blob and letting
check_sha1_signature, which supports streaming interface, do the job
is sufficient. check_sha1_signature() is not shown in the diff,
unfortunately. But if will be called when "data_valid && !data" is
false.
We will call the callback function "fn" with NULL as "data". The only
callback of this function is fsck_obj_buffer(), which does not touch
"data" at all if it's a blob.
Signed-off-by: Nguyễn Thái Ngọc Duy <pclouds@gmail.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2016-07-13 15:44:04 +00:00
|
|
|
off_t curpos;
|
|
|
|
int data_valid;
|
2005-07-01 00:15:39 +00:00
|
|
|
|
pack-check: push oid lookup into loop
When we're checking a pack with fsck or verify-pack, we first sort the
idx entries by offset, since accessing them in pack order is more
efficient. To do so, we loop over them and fill in an array of structs
with the offset, object_id, and index position of each, sort the result,
and only then do we iterate over the sorted array and process each
entry.
In order to avoid the memory cost of storing the hash of each object, we
just store a pointer into the copy in the mmap'd pack index file. To
keep that property even as the rest of the code converted to "struct
object_id", commit 9fd750461b (Convert the verify_pack callback to
struct object_id, 2017-05-06) introduced a union in order to type-pun
the pointer-to-hash into an object_id struct.
But we can make this even simpler by observing that the sort operation
doesn't need the object id at all! We only need them one at a time while
we actually process each entry. So we can just omit the oid from the
struct entirely and load it on the fly into a local variable in the
second loop.
This gets rid of the type-punning, and lets us directly use the more
type-safe nth_packed_object_id(), simplifying the code. And as a bonus,
it saves 8 bytes of memory per object.
Note that this does mean we'll do the offset lookup for each object
before the oid lookup. The oid lookup has more safety checks in it
(e.g., for looking past p->num_objects) which in theory protected the
offset lookup. But since violating those checks was already a BUG()
condition (as described in the previous commit), it's not worth worrying
about.
Signed-off-by: Jeff King <peff@peff.net>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2020-02-24 04:36:31 +00:00
|
|
|
if (nth_packed_object_id(&oid, p, entries[i].nr) < 0)
|
|
|
|
BUG("unable to get oid of object %lu from %s",
|
|
|
|
(unsigned long)entries[i].nr, p->pack_name);
|
|
|
|
|
2008-06-25 03:19:02 +00:00
|
|
|
if (p->index_version > 1) {
|
|
|
|
off_t offset = entries[i].offset;
|
|
|
|
off_t len = entries[i+1].offset - offset;
|
|
|
|
unsigned int nr = entries[i].nr;
|
|
|
|
if (check_pack_crc(p, w_curs, offset, len, nr))
|
|
|
|
err = error("index CRC mismatch for object %s "
|
|
|
|
"from %s at offset %"PRIuMAX"",
|
pack-check: push oid lookup into loop
When we're checking a pack with fsck or verify-pack, we first sort the
idx entries by offset, since accessing them in pack order is more
efficient. To do so, we loop over them and fill in an array of structs
with the offset, object_id, and index position of each, sort the result,
and only then do we iterate over the sorted array and process each
entry.
In order to avoid the memory cost of storing the hash of each object, we
just store a pointer into the copy in the mmap'd pack index file. To
keep that property even as the rest of the code converted to "struct
object_id", commit 9fd750461b (Convert the verify_pack callback to
struct object_id, 2017-05-06) introduced a union in order to type-pun
the pointer-to-hash into an object_id struct.
But we can make this even simpler by observing that the sort operation
doesn't need the object id at all! We only need them one at a time while
we actually process each entry. So we can just omit the oid from the
struct entirely and load it on the fly into a local variable in the
second loop.
This gets rid of the type-punning, and lets us directly use the more
type-safe nth_packed_object_id(), simplifying the code. And as a bonus,
it saves 8 bytes of memory per object.
Note that this does mean we'll do the offset lookup for each object
before the oid lookup. The oid lookup has more safety checks in it
(e.g., for looking past p->num_objects) which in theory protected the
offset lookup. But since violating those checks was already a BUG()
condition (as described in the previous commit), it's not worth worrying
about.
Signed-off-by: Jeff King <peff@peff.net>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2020-02-24 04:36:31 +00:00
|
|
|
oid_to_hex(&oid),
|
2008-06-25 03:19:02 +00:00
|
|
|
p->pack_name, (uintmax_t)offset);
|
|
|
|
}
|
fsck: use streaming interface for large blobs in pack
For blobs, we want to make sure the on-disk data is not corrupted
(i.e. can be inflated and produce the expected SHA-1). Blob content is
opaque, there's nothing else inside to check for.
For really large blobs, we may want to avoid unpacking the entire blob
in memory, just to check whether it produces the same SHA-1. On 32-bit
systems, we may not have enough virtual address space for such memory
allocation. And even on 64-bit where it's not a problem, allocating a
lot more memory could result in kicking other parts of systems to swap
file, generating lots of I/O and slowing everything down.
For this particular operation, not unpacking the blob and letting
check_sha1_signature, which supports streaming interface, do the job
is sufficient. check_sha1_signature() is not shown in the diff,
unfortunately. But if will be called when "data_valid && !data" is
false.
We will call the callback function "fn" with NULL as "data". The only
callback of this function is fsck_obj_buffer(), which does not touch
"data" at all if it's a blob.
Signed-off-by: Nguyễn Thái Ngọc Duy <pclouds@gmail.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2016-07-13 15:44:04 +00:00
|
|
|
|
|
|
|
curpos = entries[i].offset;
|
|
|
|
type = unpack_object_header(p, w_curs, &curpos, &size);
|
|
|
|
unuse_pack(w_curs);
|
|
|
|
|
|
|
|
if (type == OBJ_BLOB && big_file_threshold <= size) {
|
|
|
|
/*
|
2018-03-12 02:27:39 +00:00
|
|
|
* Let check_object_signature() check it with
|
fsck: use streaming interface for large blobs in pack
For blobs, we want to make sure the on-disk data is not corrupted
(i.e. can be inflated and produce the expected SHA-1). Blob content is
opaque, there's nothing else inside to check for.
For really large blobs, we may want to avoid unpacking the entire blob
in memory, just to check whether it produces the same SHA-1. On 32-bit
systems, we may not have enough virtual address space for such memory
allocation. And even on 64-bit where it's not a problem, allocating a
lot more memory could result in kicking other parts of systems to swap
file, generating lots of I/O and slowing everything down.
For this particular operation, not unpacking the blob and letting
check_sha1_signature, which supports streaming interface, do the job
is sufficient. check_sha1_signature() is not shown in the diff,
unfortunately. But if will be called when "data_valid && !data" is
false.
We will call the callback function "fn" with NULL as "data". The only
callback of this function is fsck_obj_buffer(), which does not touch
"data" at all if it's a blob.
Signed-off-by: Nguyễn Thái Ngọc Duy <pclouds@gmail.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2016-07-13 15:44:04 +00:00
|
|
|
* the streaming interface; no point slurping
|
|
|
|
* the data in-core only to discard.
|
|
|
|
*/
|
|
|
|
data = NULL;
|
|
|
|
data_valid = 0;
|
|
|
|
} else {
|
2018-11-10 05:49:07 +00:00
|
|
|
data = unpack_entry(r, p, entries[i].offset, &type, &size);
|
fsck: use streaming interface for large blobs in pack
For blobs, we want to make sure the on-disk data is not corrupted
(i.e. can be inflated and produce the expected SHA-1). Blob content is
opaque, there's nothing else inside to check for.
For really large blobs, we may want to avoid unpacking the entire blob
in memory, just to check whether it produces the same SHA-1. On 32-bit
systems, we may not have enough virtual address space for such memory
allocation. And even on 64-bit where it's not a problem, allocating a
lot more memory could result in kicking other parts of systems to swap
file, generating lots of I/O and slowing everything down.
For this particular operation, not unpacking the blob and letting
check_sha1_signature, which supports streaming interface, do the job
is sufficient. check_sha1_signature() is not shown in the diff,
unfortunately. But if will be called when "data_valid && !data" is
false.
We will call the callback function "fn" with NULL as "data". The only
callback of this function is fsck_obj_buffer(), which does not touch
"data" at all if it's a blob.
Signed-off-by: Nguyễn Thái Ngọc Duy <pclouds@gmail.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2016-07-13 15:44:04 +00:00
|
|
|
data_valid = 1;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (data_valid && !data)
|
2008-05-29 21:34:50 +00:00
|
|
|
err = error("cannot unpack %s from %s at offset %"PRIuMAX"",
|
pack-check: push oid lookup into loop
When we're checking a pack with fsck or verify-pack, we first sort the
idx entries by offset, since accessing them in pack order is more
efficient. To do so, we loop over them and fill in an array of structs
with the offset, object_id, and index position of each, sort the result,
and only then do we iterate over the sorted array and process each
entry.
In order to avoid the memory cost of storing the hash of each object, we
just store a pointer into the copy in the mmap'd pack index file. To
keep that property even as the rest of the code converted to "struct
object_id", commit 9fd750461b (Convert the verify_pack callback to
struct object_id, 2017-05-06) introduced a union in order to type-pun
the pointer-to-hash into an object_id struct.
But we can make this even simpler by observing that the sort operation
doesn't need the object id at all! We only need them one at a time while
we actually process each entry. So we can just omit the oid from the
struct entirely and load it on the fly into a local variable in the
second loop.
This gets rid of the type-punning, and lets us directly use the more
type-safe nth_packed_object_id(), simplifying the code. And as a bonus,
it saves 8 bytes of memory per object.
Note that this does mean we'll do the offset lookup for each object
before the oid lookup. The oid lookup has more safety checks in it
(e.g., for looking past p->num_objects) which in theory protected the
offset lookup. But since violating those checks was already a BUG()
condition (as described in the previous commit), it's not worth worrying
about.
Signed-off-by: Jeff King <peff@peff.net>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2020-02-24 04:36:31 +00:00
|
|
|
oid_to_hex(&oid), p->pack_name,
|
2008-05-29 21:34:50 +00:00
|
|
|
(uintmax_t)entries[i].offset);
|
fsck: report invalid object type-path combinations
Improve the error that's emitted in cases where we find a loose object
we parse, but which isn't at the location we expect it to be.
Before this change we'd prefix the error with a not-a-OID derived from
the path at which the object was found, due to an emergent behavior in
how we'd end up with an "OID" in these codepaths.
Now we'll instead say what object we hashed, and what path it was
found at. Before this patch series e.g.:
$ git hash-object --stdin -w -t blob </dev/null
e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
$ mv objects/e6/ objects/e7
Would emit ("[...]" used to abbreviate the OIDs):
git fsck
error: hash mismatch for ./objects/e7/9d[...] (expected e79d[...])
error: e79d[...]: object corrupt or missing: ./objects/e7/9d[...]
Now we'll instead emit:
error: e69d[...]: hash-path mismatch, found at: ./objects/e7/9d[...]
Furthermore, we'll do the right thing when the object type and its
location are bad. I.e. this case:
$ git hash-object --stdin -w -t garbage --literally </dev/null
8315a83d2acc4c174aed59430f9a9c4ed926440f
$ mv objects/83 objects/84
As noted in an earlier commits we'd simply die early in those cases,
until preceding commits fixed the hard die on invalid object type:
$ git fsck
fatal: invalid object type
Now we'll instead emit sensible error messages:
$ git fsck
error: 8315[...]: hash-path mismatch, found at: ./objects/84/15[...]
error: 8315[...]: object is of unknown type 'garbage': ./objects/84/15[...]
In both fsck.c and object-file.c we're using null_oid as a sentinel
value for checking whether we got far enough to be certain that the
issue was indeed this OID mismatch.
We need to add the "object corrupt or missing" special-case to deal
with cases where read_loose_object() will return an error before
completing check_object_signature(), e.g. if we have an error in
unpack_loose_rest() because we find garbage after the valid gzip
content:
$ git hash-object --stdin -w -t blob </dev/null
e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
$ chmod 755 objects/e6/9de29bb2d1d6434b8b29ae775ad8c2e48c5391
$ echo garbage >>objects/e6/9de29bb2d1d6434b8b29ae775ad8c2e48c5391
$ git fsck
error: garbage at end of loose object 'e69d[...]'
error: unable to unpack contents of ./objects/e6/9d[...]
error: e69d[...]: object corrupt or missing: ./objects/e6/9d[...]
There is currently some weird messaging in the edge case when the two
are combined, i.e. because we're not explicitly passing along an error
state about this specific scenario from check_stream_oid() via
read_loose_object() we'll end up printing the null OID if an object is
of an unknown type *and* it can't be unpacked by zlib, e.g.:
$ git hash-object --stdin -w -t garbage --literally </dev/null
8315a83d2acc4c174aed59430f9a9c4ed926440f
$ chmod 755 objects/83/15a83d2acc4c174aed59430f9a9c4ed926440f
$ echo garbage >>objects/83/15a83d2acc4c174aed59430f9a9c4ed926440f
$ /usr/bin/git fsck
fatal: invalid object type
$ ~/g/git/git fsck
error: garbage at end of loose object '8315a83d2acc4c174aed59430f9a9c4ed926440f'
error: unable to unpack contents of ./objects/83/15a83d2acc4c174aed59430f9a9c4ed926440f
error: 8315a83d2acc4c174aed59430f9a9c4ed926440f: object corrupt or missing: ./objects/83/15a83d2acc4c174aed59430f9a9c4ed926440f
error: 0000000000000000000000000000000000000000: object is of unknown type 'garbage': ./objects/83/15a83d2acc4c174aed59430f9a9c4ed926440f
[...]
I think it's OK to leave that for future improvements, which would
involve enum-ifying more error state as we've done with "enum
unpack_loose_header_result" in preceding commits. In these
increasingly more obscure cases the worst that can happen is that
we'll get slightly nonsensical or inapplicable error messages.
There's other such potential edge cases, all of which might produce
some confusing messaging, but still be handled correctly as far as
passing along errors goes. E.g. if check_object_signature() returns
and oideq(real_oid, null_oid()) is true, which could happen if it
returns -1 due to the read_istream() call having failed.
Signed-off-by: Ævar Arnfjörð Bjarmason <avarab@gmail.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2021-10-01 09:16:53 +00:00
|
|
|
else if (check_object_signature(r, &oid, data, size,
|
2022-02-04 23:48:29 +00:00
|
|
|
type_name(type), NULL) < 0)
|
2005-07-07 22:12:20 +00:00
|
|
|
err = error("packed %s from %s is corrupt",
|
pack-check: push oid lookup into loop
When we're checking a pack with fsck or verify-pack, we first sort the
idx entries by offset, since accessing them in pack order is more
efficient. To do so, we loop over them and fill in an array of structs
with the offset, object_id, and index position of each, sort the result,
and only then do we iterate over the sorted array and process each
entry.
In order to avoid the memory cost of storing the hash of each object, we
just store a pointer into the copy in the mmap'd pack index file. To
keep that property even as the rest of the code converted to "struct
object_id", commit 9fd750461b (Convert the verify_pack callback to
struct object_id, 2017-05-06) introduced a union in order to type-pun
the pointer-to-hash into an object_id struct.
But we can make this even simpler by observing that the sort operation
doesn't need the object id at all! We only need them one at a time while
we actually process each entry. So we can just omit the oid from the
struct entirely and load it on the fly into a local variable in the
second loop.
This gets rid of the type-punning, and lets us directly use the more
type-safe nth_packed_object_id(), simplifying the code. And as a bonus,
it saves 8 bytes of memory per object.
Note that this does mean we'll do the offset lookup for each object
before the oid lookup. The oid lookup has more safety checks in it
(e.g., for looking past p->num_objects) which in theory protected the
offset lookup. But since violating those checks was already a BUG()
condition (as described in the previous commit), it's not worth worrying
about.
Signed-off-by: Jeff King <peff@peff.net>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2020-02-24 04:36:31 +00:00
|
|
|
oid_to_hex(&oid), p->pack_name);
|
2011-11-07 02:59:25 +00:00
|
|
|
else if (fn) {
|
|
|
|
int eaten = 0;
|
pack-check: push oid lookup into loop
When we're checking a pack with fsck or verify-pack, we first sort the
idx entries by offset, since accessing them in pack order is more
efficient. To do so, we loop over them and fill in an array of structs
with the offset, object_id, and index position of each, sort the result,
and only then do we iterate over the sorted array and process each
entry.
In order to avoid the memory cost of storing the hash of each object, we
just store a pointer into the copy in the mmap'd pack index file. To
keep that property even as the rest of the code converted to "struct
object_id", commit 9fd750461b (Convert the verify_pack callback to
struct object_id, 2017-05-06) introduced a union in order to type-pun
the pointer-to-hash into an object_id struct.
But we can make this even simpler by observing that the sort operation
doesn't need the object id at all! We only need them one at a time while
we actually process each entry. So we can just omit the oid from the
struct entirely and load it on the fly into a local variable in the
second loop.
This gets rid of the type-punning, and lets us directly use the more
type-safe nth_packed_object_id(), simplifying the code. And as a bonus,
it saves 8 bytes of memory per object.
Note that this does mean we'll do the offset lookup for each object
before the oid lookup. The oid lookup has more safety checks in it
(e.g., for looking past p->num_objects) which in theory protected the
offset lookup. But since violating those checks was already a BUG()
condition (as described in the previous commit), it's not worth worrying
about.
Signed-off-by: Jeff King <peff@peff.net>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2020-02-24 04:36:31 +00:00
|
|
|
err |= fn(&oid, type, size, data, &eaten);
|
2011-11-07 02:59:25 +00:00
|
|
|
if (eaten)
|
|
|
|
data = NULL;
|
|
|
|
}
|
2011-11-07 02:59:26 +00:00
|
|
|
if (((base_count + i) & 1023) == 0)
|
|
|
|
display_progress(progress, base_count + i);
|
2005-07-01 00:15:39 +00:00
|
|
|
free(data);
|
2011-11-07 02:59:26 +00:00
|
|
|
|
2005-07-01 00:15:39 +00:00
|
|
|
}
|
2011-11-07 02:59:26 +00:00
|
|
|
display_progress(progress, base_count + i);
|
2007-06-03 18:21:41 +00:00
|
|
|
free(entries);
|
2005-07-01 00:15:39 +00:00
|
|
|
|
|
|
|
return err;
|
2005-06-29 09:51:27 +00:00
|
|
|
}
|
|
|
|
|
2010-04-19 14:23:07 +00:00
|
|
|
int verify_pack_index(struct packed_git *p)
|
2005-06-29 09:51:27 +00:00
|
|
|
{
|
2008-05-29 21:34:50 +00:00
|
|
|
int err = 0;
|
2005-06-29 09:51:27 +00:00
|
|
|
|
2007-05-26 05:24:19 +00:00
|
|
|
if (open_pack_index(p))
|
|
|
|
return error("packfile %s index not opened", p->pack_name);
|
|
|
|
|
2005-06-29 09:51:27 +00:00
|
|
|
/* Verify SHA1 sum of the index file */
|
2021-06-23 18:39:07 +00:00
|
|
|
if (!hashfile_checksum_valid(p->index_data, p->index_size))
|
2018-02-01 02:18:43 +00:00
|
|
|
err = error("Packfile index for %s hash mismatch",
|
2005-07-01 00:15:39 +00:00
|
|
|
p->pack_name);
|
2010-04-19 14:23:07 +00:00
|
|
|
return err;
|
|
|
|
}
|
|
|
|
|
2018-11-10 05:49:07 +00:00
|
|
|
int verify_pack(struct repository *r, struct packed_git *p, verify_fn fn,
|
2011-11-07 02:59:26 +00:00
|
|
|
struct progress *progress, uint32_t base_count)
|
2010-04-19 14:23:07 +00:00
|
|
|
{
|
|
|
|
int err = 0;
|
|
|
|
struct pack_window *w_curs = NULL;
|
|
|
|
|
|
|
|
err |= verify_pack_index(p);
|
|
|
|
if (!p->index_data)
|
|
|
|
return -1;
|
2005-07-01 00:15:39 +00:00
|
|
|
|
2018-11-10 05:49:07 +00:00
|
|
|
err |= verify_packfile(r, p, &w_curs, fn, progress, base_count);
|
2008-05-29 21:34:50 +00:00
|
|
|
unuse_pack(&w_curs);
|
2005-07-01 00:15:39 +00:00
|
|
|
|
2008-05-29 21:34:50 +00:00
|
|
|
return err;
|
2005-06-29 09:51:27 +00:00
|
|
|
}
|