mirror of
https://github.com/git/git
synced 2024-08-24 18:26:02 +00:00
git-repack --max-pack-size: write_{object,one}() respect pack limit
With --max-pack-size, generate the appropriate write limit for each object and check against it before each group of writes. Update delta usability rules to handle base being in a previously- written pack. Inline sha1write_compress() so we know the exact size of the written data when it needs to be compressed. Detect and return write "failure". Signed-off-by: Dana L. How <danahow@gmail.com> Signed-off-by: Junio C Hamano <junkio@cox.net>
This commit is contained in:
Dana L. How
Junio C Hamano
parent
d01fb92f8d
commit
17b08f2cd0
|
|
@ -352,16 +352,31 @@ static void copy_pack_data(struct sha1file *f,
|
|||
}
|
||||
|
||||
static unsigned long write_object(struct sha1file *f,
|
||||
struct object_entry *entry)
|
||||
struct object_entry *entry,
|
||||
off_t write_offset)
|
||||
{
|
||||
unsigned long size;
|
||||
enum object_type type;
|
||||
void *buf;
|
||||
unsigned char header[10];
|
||||
unsigned char dheader[10];
|
||||
unsigned hdrlen;
|
||||
off_t datalen;
|
||||
enum object_type obj_type;
|
||||
int to_reuse = 0;
|
||||
/* write limit if limited packsize and not first object */
|
||||
unsigned long limit = pack_size_limit && nr_written ?
|
||||
pack_size_limit - write_offset : 0;
|
||||
/* no if no delta */
|
||||
int usable_delta = !entry->delta ? 0 :
|
||||
/* yes if unlimited packfile */
|
||||
!pack_size_limit ? 1 :
|
||||
/* no if base written to previous pack */
|
||||
entry->delta->offset == (off_t)-1 ? 0 :
|
||||
/* otherwise double-check written to this
|
||||
* pack, like we do below
|
||||
*/
|
||||
entry->delta->offset ? 1 : 0;
|
||||
|
||||
if (!pack_to_stdout)
|
||||
crc32_begin(f);
|
||||
|
|
@ -372,7 +387,9 @@ static unsigned long write_object(struct sha1file *f,
|
|||
else if (!entry->in_pack)
|
||||
to_reuse = 0; /* can't reuse what we don't have */
|
||||
else if (obj_type == OBJ_REF_DELTA || obj_type == OBJ_OFS_DELTA)
|
||||
to_reuse = 1; /* check_object() decided it for us */
|
||||
/* check_object() decided it for us ... */
|
||||
to_reuse = usable_delta;
|
||||
/* ... but pack split may override that */
|
||||
else if (obj_type != entry->in_pack_type)
|
||||
to_reuse = 0; /* pack has delta which is unusable */
|
||||
else if (entry->delta)
|
||||
|
|
@ -383,24 +400,48 @@ static unsigned long write_object(struct sha1file *f,
|
|||
*/
|
||||
|
||||
if (!to_reuse) {
|
||||
z_stream stream;
|
||||
unsigned long maxsize;
|
||||
void *out;
|
||||
buf = read_sha1_file(entry->sha1, &type, &size);
|
||||
if (!buf)
|
||||
die("unable to read %s", sha1_to_hex(entry->sha1));
|
||||
if (size != entry->size)
|
||||
die("object %s size inconsistency (%lu vs %lu)",
|
||||
sha1_to_hex(entry->sha1), size, entry->size);
|
||||
if (entry->delta) {
|
||||
if (usable_delta) {
|
||||
buf = delta_against(buf, size, entry);
|
||||
size = entry->delta_size;
|
||||
obj_type = (allow_ofs_delta && entry->delta->offset) ?
|
||||
OBJ_OFS_DELTA : OBJ_REF_DELTA;
|
||||
} else {
|
||||
/*
|
||||
* recover real object type in case
|
||||
* check_object() wanted to re-use a delta,
|
||||
* but we couldn't since base was in previous split pack
|
||||
*/
|
||||
obj_type = type;
|
||||
}
|
||||
/* compress the data to store and put compressed length in datalen */
|
||||
memset(&stream, 0, sizeof(stream));
|
||||
deflateInit(&stream, pack_compression_level);
|
||||
maxsize = deflateBound(&stream, size);
|
||||
out = xmalloc(maxsize);
|
||||
/* Compress it */
|
||||
stream.next_in = buf;
|
||||
stream.avail_in = size;
|
||||
stream.next_out = out;
|
||||
stream.avail_out = maxsize;
|
||||
while (deflate(&stream, Z_FINISH) == Z_OK)
|
||||
/* nothing */;
|
||||
deflateEnd(&stream);
|
||||
datalen = stream.total_out;
|
||||
deflateEnd(&stream);
|
||||
/*
|
||||
* The object header is a byte of 'type' followed by zero or
|
||||
* more bytes of length.
|
||||
*/
|
||||
hdrlen = encode_header(obj_type, size, header);
|
||||
sha1write(f, header, hdrlen);
|
||||
|
||||
if (obj_type == OBJ_OFS_DELTA) {
|
||||
/*
|
||||
|
|
@ -409,21 +450,41 @@ static unsigned long write_object(struct sha1file *f,
|
|||
* base from this object's position in the pack.
|
||||
*/
|
||||
off_t ofs = entry->offset - entry->delta->offset;
|
||||
unsigned pos = sizeof(header) - 1;
|
||||
header[pos] = ofs & 127;
|
||||
unsigned pos = sizeof(dheader) - 1;
|
||||
dheader[pos] = ofs & 127;
|
||||
while (ofs >>= 7)
|
||||
header[--pos] = 128 | (--ofs & 127);
|
||||
sha1write(f, header + pos, sizeof(header) - pos);
|
||||
hdrlen += sizeof(header) - pos;
|
||||
dheader[--pos] = 128 | (--ofs & 127);
|
||||
if (limit && hdrlen + sizeof(dheader) - pos + datalen + 20 >= limit) {
|
||||
free(out);
|
||||
free(buf);
|
||||
return 0;
|
||||
}
|
||||
sha1write(f, header, hdrlen);
|
||||
sha1write(f, dheader + pos, sizeof(dheader) - pos);
|
||||
hdrlen += sizeof(dheader) - pos;
|
||||
} else if (obj_type == OBJ_REF_DELTA) {
|
||||
/*
|
||||
* Deltas with a base reference contain
|
||||
* an additional 20 bytes for the base sha1.
|
||||
*/
|
||||
if (limit && hdrlen + 20 + datalen + 20 >= limit) {
|
||||
free(out);
|
||||
free(buf);
|
||||
return 0;
|
||||
}
|
||||
sha1write(f, header, hdrlen);
|
||||
sha1write(f, entry->delta->sha1, 20);
|
||||
hdrlen += 20;
|
||||
} else {
|
||||
if (limit && hdrlen + datalen + 20 >= limit) {
|
||||
free(out);
|
||||
free(buf);
|
||||
return 0;
|
||||
}
|
||||
sha1write(f, header, hdrlen);
|
||||
}
|
||||
datalen = sha1write_compressed(f, buf, size, pack_compression_level);
|
||||
sha1write(f, out, datalen);
|
||||
free(out);
|
||||
free(buf);
|
||||
}
|
||||
else {
|
||||
|
|
@ -438,20 +499,6 @@ static unsigned long write_object(struct sha1file *f,
|
|||
reused_delta++;
|
||||
}
|
||||
hdrlen = encode_header(obj_type, entry->size, header);
|
||||
sha1write(f, header, hdrlen);
|
||||
if (obj_type == OBJ_OFS_DELTA) {
|
||||
off_t ofs = entry->offset - entry->delta->offset;
|
||||
unsigned pos = sizeof(header) - 1;
|
||||
header[pos] = ofs & 127;
|
||||
while (ofs >>= 7)
|
||||
header[--pos] = 128 | (--ofs & 127);
|
||||
sha1write(f, header + pos, sizeof(header) - pos);
|
||||
hdrlen += sizeof(header) - pos;
|
||||
} else if (obj_type == OBJ_REF_DELTA) {
|
||||
sha1write(f, entry->delta->sha1, 20);
|
||||
hdrlen += 20;
|
||||
}
|
||||
|
||||
offset = entry->in_pack_offset;
|
||||
revidx = find_packed_object(p, offset);
|
||||
datalen = revidx[1].offset - offset;
|
||||
|
|
@ -460,6 +507,29 @@ static unsigned long write_object(struct sha1file *f,
|
|||
die("bad packed object CRC for %s", sha1_to_hex(entry->sha1));
|
||||
offset += entry->in_pack_header_size;
|
||||
datalen -= entry->in_pack_header_size;
|
||||
if (obj_type == OBJ_OFS_DELTA) {
|
||||
off_t ofs = entry->offset - entry->delta->offset;
|
||||
unsigned pos = sizeof(dheader) - 1;
|
||||
dheader[pos] = ofs & 127;
|
||||
while (ofs >>= 7)
|
||||
dheader[--pos] = 128 | (--ofs & 127);
|
||||
if (limit && hdrlen + sizeof(dheader) - pos + datalen + 20 >= limit)
|
||||
return 0;
|
||||
sha1write(f, header, hdrlen);
|
||||
sha1write(f, dheader + pos, sizeof(dheader) - pos);
|
||||
hdrlen += sizeof(dheader) - pos;
|
||||
} else if (obj_type == OBJ_REF_DELTA) {
|
||||
if (limit && hdrlen + 20 + datalen + 20 >= limit)
|
||||
return 0;
|
||||
sha1write(f, header, hdrlen);
|
||||
sha1write(f, entry->delta->sha1, 20);
|
||||
hdrlen += 20;
|
||||
} else {
|
||||
if (limit && hdrlen + datalen + 20 >= limit)
|
||||
return 0;
|
||||
sha1write(f, header, hdrlen);
|
||||
}
|
||||
|
||||
if (!pack_to_stdout && p->index_version == 1 &&
|
||||
check_pack_inflate(p, &w_curs, offset, datalen, entry->size))
|
||||
die("corrupt packed object for %s", sha1_to_hex(entry->sha1));
|
||||
|
|
@ -467,7 +537,7 @@ static unsigned long write_object(struct sha1file *f,
|
|||
unuse_pack(&w_curs);
|
||||
reused++;
|
||||
}
|
||||
if (entry->delta)
|
||||
if (usable_delta)
|
||||
written_delta++;
|
||||
written++;
|
||||
if (!pack_to_stdout)
|
||||
|
|
@ -486,11 +556,18 @@ static off_t write_one(struct sha1file *f,
|
|||
return offset;
|
||||
|
||||
/* if we are deltified, write out base object first. */
|
||||
if (e->delta)
|
||||
if (e->delta) {
|
||||
offset = write_one(f, e->delta, offset);
|
||||
if (!offset)
|
||||
return 0;
|
||||
}
|
||||
|
||||
e->offset = offset;
|
||||
size = write_object(f, e);
|
||||
size = write_object(f, e, offset);
|
||||
if (!size) {
|
||||
e->offset = 0;
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* make sure off_t is sufficiently large not to wrap */
|
||||
if (offset > offset + size)
|
||||
|
|
|
|||
Loading…
Reference in a new issue