mirror of
https://github.com/git/git
synced 2024-08-28 12:09:19 +00:00
Fix object packing/unpacking.
This actually successfully packed and unpacked a git archive down to 1.3MB (17MB unpacked). Right now unpacking is way too noisy, lots of debug messages left.
This commit is contained in:
parent
8ee378a0f0
commit
c4fb06c0d0
|
@ -96,7 +96,7 @@ static unsigned long write_object(FILE *f, struct object_entry *entry)
|
||||||
unsigned long size;
|
unsigned long size;
|
||||||
char type[10];
|
char type[10];
|
||||||
void *buf = read_sha1_file(entry->sha1, type, &size);
|
void *buf = read_sha1_file(entry->sha1, type, &size);
|
||||||
char header[21];
|
char header[25];
|
||||||
unsigned hdrlen, datalen;
|
unsigned hdrlen, datalen;
|
||||||
|
|
||||||
if (!buf)
|
if (!buf)
|
||||||
|
@ -110,16 +110,16 @@ static unsigned long write_object(FILE *f, struct object_entry *entry)
|
||||||
* instead.
|
* instead.
|
||||||
*/
|
*/
|
||||||
header[0] = ".CTB"[entry->type];
|
header[0] = ".CTB"[entry->type];
|
||||||
datalen = htonl(size);
|
|
||||||
memcpy(header+1, &datalen, 4);
|
|
||||||
hdrlen = 5;
|
hdrlen = 5;
|
||||||
if (entry->delta) {
|
if (entry->delta) {
|
||||||
header[0] = 'D';
|
header[0] = 'D';
|
||||||
memcpy(header+1, entry->delta, 20);
|
memcpy(header+5, entry->delta, 20);
|
||||||
buf = delta_against(buf, size, entry);
|
buf = delta_against(buf, size, entry);
|
||||||
size = entry->delta_size;
|
size = entry->delta_size;
|
||||||
hdrlen = 21;
|
hdrlen = 25;
|
||||||
}
|
}
|
||||||
|
datalen = htonl(size);
|
||||||
|
memcpy(header+1, &datalen, 4);
|
||||||
fwrite(header, hdrlen, 1, f);
|
fwrite(header, hdrlen, 1, f);
|
||||||
datalen = fwrite_compressed(buf, size, f);
|
datalen = fwrite_compressed(buf, size, f);
|
||||||
free(buf);
|
free(buf);
|
||||||
|
|
101
unpack-objects.c
101
unpack-objects.c
|
@ -88,26 +88,26 @@ static int check_index(void)
|
||||||
}
|
}
|
||||||
|
|
||||||
static int unpack_non_delta_entry(struct pack_entry *entry,
|
static int unpack_non_delta_entry(struct pack_entry *entry,
|
||||||
unsigned char *pack)
|
int kind,
|
||||||
|
unsigned char *data,
|
||||||
|
unsigned long size,
|
||||||
|
unsigned long left)
|
||||||
{
|
{
|
||||||
int st, kind;
|
int st;
|
||||||
unsigned long size;
|
|
||||||
z_stream stream;
|
z_stream stream;
|
||||||
char *buffer;
|
char *buffer;
|
||||||
unsigned char sha1[20];
|
unsigned char sha1[20];
|
||||||
char *type_s;
|
char *type_s;
|
||||||
unsigned long offset = ntohl(entry->offset);
|
|
||||||
|
|
||||||
kind = pack[0];
|
|
||||||
size = (pack[1] << 24) + (pack[2] << 16) + (pack[3] << 8) + pack[4];
|
|
||||||
printf("%s %c %lu\n", sha1_to_hex(entry->sha1), kind, size);
|
printf("%s %c %lu\n", sha1_to_hex(entry->sha1), kind, size);
|
||||||
pack += 5;
|
if (dry_run)
|
||||||
|
return 0;
|
||||||
|
|
||||||
buffer = xmalloc(size + 1);
|
buffer = xmalloc(size + 1);
|
||||||
buffer[size] = 0;
|
buffer[size] = 0;
|
||||||
memset(&stream, 0, sizeof(stream));
|
memset(&stream, 0, sizeof(stream));
|
||||||
stream.next_in = pack;
|
stream.next_in = data;
|
||||||
stream.avail_in = pack_size - offset; /* sheesh. */
|
stream.avail_in = left;
|
||||||
stream.next_out = buffer;
|
stream.next_out = buffer;
|
||||||
stream.avail_out = size;
|
stream.avail_out = size;
|
||||||
|
|
||||||
|
@ -148,14 +148,15 @@ static int find_pack_entry(unsigned char *sha1, struct pack_entry **ent)
|
||||||
do {
|
do {
|
||||||
int mi = (lo + hi) / 2;
|
int mi = (lo + hi) / 2;
|
||||||
int cmp = memcmp(index + 24 * mi + 4, sha1, 20);
|
int cmp = memcmp(index + 24 * mi + 4, sha1, 20);
|
||||||
|
printf("lo=%d mi=%d hi=%d cmp=%d\n", lo, mi, hi, cmp);
|
||||||
if (!cmp) {
|
if (!cmp) {
|
||||||
*ent = index + 24 * mi;
|
*ent = index + 24 * mi;
|
||||||
return 1;
|
return 1;
|
||||||
}
|
}
|
||||||
if (cmp < 0)
|
if (cmp > 0)
|
||||||
hi = mi;
|
hi = mi;
|
||||||
else
|
else
|
||||||
lo = mi;
|
lo = mi+1;
|
||||||
} while (lo < hi);
|
} while (lo < hi);
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
@ -163,70 +164,55 @@ static int find_pack_entry(unsigned char *sha1, struct pack_entry **ent)
|
||||||
/* forward declaration for a mutually recursive function */
|
/* forward declaration for a mutually recursive function */
|
||||||
static void unpack_entry(struct pack_entry *);
|
static void unpack_entry(struct pack_entry *);
|
||||||
|
|
||||||
static int unpack_delta_entry(struct pack_entry *entry, unsigned char *pack)
|
static int unpack_delta_entry(struct pack_entry *entry,
|
||||||
|
unsigned char *base_sha1,
|
||||||
|
unsigned long delta_size,
|
||||||
|
unsigned long left)
|
||||||
{
|
{
|
||||||
void *delta_data, *result, *base;
|
void *data, *delta_data, *result, *base;
|
||||||
unsigned long delta_alloc, delta_size, result_size, base_size;
|
unsigned long data_size, result_size, base_size;
|
||||||
z_stream stream;
|
z_stream stream;
|
||||||
int st;
|
int st;
|
||||||
char type[20];
|
char type[20];
|
||||||
unsigned char sha1[20];
|
unsigned char sha1[20];
|
||||||
|
|
||||||
printf("%s D", sha1_to_hex(entry->sha1));
|
if (left < 20)
|
||||||
printf(" %s\n", sha1_to_hex(pack+1));
|
die("truncated pack file");
|
||||||
|
data = base_sha1 + 20;
|
||||||
|
data_size = left - 20;
|
||||||
|
printf("%s D %lu", sha1_to_hex(entry->sha1), delta_size);
|
||||||
|
printf(" %s\n", sha1_to_hex(base_sha1));
|
||||||
|
|
||||||
/* pack+1 is the base sha1, unless we have it, we need to
|
if (dry_run)
|
||||||
|
return 0;
|
||||||
|
|
||||||
|
/* pack+5 is the base sha1, unless we have it, we need to
|
||||||
* unpack it first.
|
* unpack it first.
|
||||||
*/
|
*/
|
||||||
if (!has_sha1_file(pack+1)) {
|
if (!has_sha1_file(base_sha1)) {
|
||||||
struct pack_entry *base;
|
struct pack_entry *base;
|
||||||
if (!find_pack_entry(pack+1, &base))
|
if (!find_pack_entry(base_sha1, &base))
|
||||||
die("cannot find delta-pack base object");
|
die("cannot find delta-pack base object");
|
||||||
unpack_entry(base);
|
unpack_entry(base);
|
||||||
}
|
}
|
||||||
|
delta_data = xmalloc(delta_size);
|
||||||
/* pack+1 thru pack+20 is the base sha1 and
|
|
||||||
* pack+21 thru unknown number is the delta data.
|
|
||||||
* we do not even have size of the delta data uncompressed.
|
|
||||||
* sheesh!
|
|
||||||
*/
|
|
||||||
delta_alloc = 1024;
|
|
||||||
delta_data = xmalloc(delta_alloc);
|
|
||||||
|
|
||||||
memset(&stream, 0, sizeof(stream));
|
memset(&stream, 0, sizeof(stream));
|
||||||
|
|
||||||
stream.next_in = pack + 21;
|
stream.next_in = data;
|
||||||
stream.avail_in = pack_size - ntohl(entry->offset); /* sheesh. */
|
stream.avail_in = data_size;
|
||||||
stream.next_out = delta_data;
|
stream.next_out = delta_data;
|
||||||
stream.avail_out = delta_alloc;
|
stream.avail_out = delta_size;
|
||||||
delta_size = 0;
|
|
||||||
|
|
||||||
inflateInit(&stream);
|
inflateInit(&stream);
|
||||||
while (1) {
|
st = inflate(&stream, Z_FINISH);
|
||||||
st = inflate(&stream, Z_FINISH);
|
|
||||||
if (st == Z_STREAM_END) {
|
|
||||||
delta_size = stream.total_out;
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
if (st < 0)
|
|
||||||
break;
|
|
||||||
|
|
||||||
if (delta_alloc <= stream.total_out) {
|
|
||||||
delta_alloc = (delta_alloc +1024) * 3 / 2;
|
|
||||||
delta_data = xrealloc(delta_data, delta_alloc);
|
|
||||||
stream.next_out = delta_data + stream.total_out;
|
|
||||||
stream.avail_out = delta_alloc - stream.total_out;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
inflateEnd(&stream);
|
inflateEnd(&stream);
|
||||||
if (st != Z_STREAM_END) {
|
if ((st != Z_STREAM_END) || stream.total_out != delta_size)
|
||||||
free(delta_data);
|
die("delta data unpack failed");
|
||||||
return -1;
|
|
||||||
}
|
|
||||||
|
|
||||||
base = read_sha1_file(pack+1, type, &base_size);
|
base = read_sha1_file(base_sha1, type, &base_size);
|
||||||
if (!base)
|
if (!base)
|
||||||
die("failed to read delta-pack base object");
|
die("failed to read delta-pack base object %s", sha1_to_hex(base_sha1));
|
||||||
result = patch_delta(base, base_size,
|
result = patch_delta(base, base_size,
|
||||||
delta_data, delta_size,
|
delta_data, delta_size,
|
||||||
&result_size);
|
&result_size);
|
||||||
|
@ -246,7 +232,7 @@ static int unpack_delta_entry(struct pack_entry *entry, unsigned char *pack)
|
||||||
|
|
||||||
static void unpack_entry(struct pack_entry *entry)
|
static void unpack_entry(struct pack_entry *entry)
|
||||||
{
|
{
|
||||||
unsigned long offset;
|
unsigned long offset, size, left;
|
||||||
unsigned char *pack;
|
unsigned char *pack;
|
||||||
|
|
||||||
/* Have we done this one already due to deltas based on it? */
|
/* Have we done this one already due to deltas based on it? */
|
||||||
|
@ -257,13 +243,14 @@ static void unpack_entry(struct pack_entry *entry)
|
||||||
if (offset > pack_size - 5)
|
if (offset > pack_size - 5)
|
||||||
die("object offset outside of pack file");
|
die("object offset outside of pack file");
|
||||||
pack = pack_base + offset;
|
pack = pack_base + offset;
|
||||||
offset = pack_size - offset;
|
size = (pack[1] << 24) + (pack[2] << 16) + (pack[3] << 8) + pack[4];
|
||||||
|
left = pack_size - offset - 5;
|
||||||
switch (*pack) {
|
switch (*pack) {
|
||||||
case 'C': case 'T': case 'B':
|
case 'C': case 'T': case 'B':
|
||||||
unpack_non_delta_entry(entry, pack);
|
unpack_non_delta_entry(entry, *pack, pack+5, size, left);
|
||||||
break;
|
break;
|
||||||
case 'D':
|
case 'D':
|
||||||
unpack_delta_entry(entry, pack);
|
unpack_delta_entry(entry, pack+5, size, left);
|
||||||
break;
|
break;
|
||||||
default:
|
default:
|
||||||
die("corrupted pack file");
|
die("corrupted pack file");
|
||||||
|
|
Loading…
Reference in a new issue