From 1a507fc1121865009fd961ef08cb03700650cbd4 Mon Sep 17 00:00:00 2001 From: Junio C Hamano Date: Fri, 10 Jun 2011 10:31:34 -0700 Subject: [PATCH 1/7] zlib wrapper: refactor error message formatter Before refactoring the main part of the wrappers, first move the logic to convert error status that come back from zlib to string to a helper function. Signed-off-by: Junio C Hamano --- zlib.c | 75 ++++++++++++++++++++++++++++++---------------------------- 1 file changed, 39 insertions(+), 36 deletions(-) diff --git a/zlib.c b/zlib.c index c4d58da4e9..be9d7e963b 100644 --- a/zlib.c +++ b/zlib.c @@ -4,58 +4,61 @@ */ #include "cache.h" +static const char *zerr_to_string(int status) +{ + switch (status) { + case Z_MEM_ERROR: + return "out of memory"; + case Z_VERSION_ERROR: + return "wrong version"; + case Z_NEED_DICT: + return "needs dictionary"; + case Z_DATA_ERROR: + return "data stream error"; + case Z_STREAM_ERROR: + return "stream consistency error"; + default: + return "unknown error"; + } +} + void git_inflate_init(z_streamp strm) { - const char *err; + int status = inflateInit(strm); - switch (inflateInit(strm)) { - case Z_OK: + if (status == Z_OK) return; - - case Z_MEM_ERROR: - err = "out of memory"; - break; - case Z_VERSION_ERROR: - err = "wrong version"; - break; - default: - err = "error"; - } - die("inflateInit: %s (%s)", err, strm->msg ? strm->msg : "no message"); + die("inflateInit: %s (%s)", zerr_to_string(status), + strm->msg ? strm->msg : "no message"); } void git_inflate_end(z_streamp strm) { - if (inflateEnd(strm) != Z_OK) - error("inflateEnd: %s", strm->msg ? strm->msg : "failed"); + int status = inflateEnd(strm); + + if (status == Z_OK) + return; + error("inflateEnd: %s (%s)", zerr_to_string(status), + strm->msg ? strm->msg : "no message"); } int git_inflate(z_streamp strm, int flush) { - int ret = inflate(strm, flush); - const char *err; - - switch (ret) { - /* Out of memory is fatal. */ - case Z_MEM_ERROR: - die("inflate: out of memory"); - - /* Data corruption errors: we may want to recover from them (fsck) */ - case Z_NEED_DICT: - err = "needs dictionary"; break; - case Z_DATA_ERROR: - err = "data stream error"; break; - case Z_STREAM_ERROR: - err = "stream consistency error"; break; - default: - err = "unknown error"; break; + int status = inflate(strm, flush); + switch (status) { /* Z_BUF_ERROR: normal, needs more space in the output buffer */ case Z_BUF_ERROR: case Z_OK: case Z_STREAM_END: - return ret; + return status; + + case Z_MEM_ERROR: + die("inflate: out of memory"); + default: + break; } - error("inflate: %s (%s)", err, strm->msg ? strm->msg : "no message"); - return ret; + error("inflate: %s (%s)", zerr_to_string(status), + strm->msg ? strm->msg : "no message"); + return status; } From 9e7e5ca372a0242069ff4e7eccecc7d5fe7ec385 Mon Sep 17 00:00:00 2001 From: Junio C Hamano Date: Fri, 10 Jun 2011 10:39:27 -0700 Subject: [PATCH 2/7] zlib: wrap remaining calls to direct inflate/inflateEnd Two callsites in http-backend.c to inflate() and inflateEnd() were not using git_ prefixed versions. After this, running $ find all objects -print | xargs nm -ugo | grep inflate shows only zlib.c makes direct calls to zlib for inflate operation, except for a singlecall to inflateInit2 in http-backend.c Signed-off-by: Junio C Hamano --- http-backend.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/http-backend.c b/http-backend.c index 85015048dd..c74cb03a70 100644 --- a/http-backend.c +++ b/http-backend.c @@ -296,7 +296,7 @@ static void inflate_request(const char *prog_name, int out) stream.next_out = out_buf; stream.avail_out = sizeof(out_buf); - ret = inflate(&stream, Z_NO_FLUSH); + ret = git_inflate(&stream, Z_NO_FLUSH); if (ret != Z_OK && ret != Z_STREAM_END) die("zlib error inflating request, result %d", ret); @@ -311,7 +311,7 @@ static void inflate_request(const char *prog_name, int out) } done: - inflateEnd(&stream); + git_inflate_end(&stream); close(out); } From 5e86c1fb866ca4bc8d6e015ccbdafd114fd616fa Mon Sep 17 00:00:00 2001 From: Junio C Hamano Date: Fri, 10 Jun 2011 10:45:29 -0700 Subject: [PATCH 3/7] zlib: wrap inflateInit2 used to accept only for gzip format http-backend.c uses inflateInit2() to tell the library that it wants to accept only gzip format. Wrap it in a helper function so that readers do not have to wonder what the magic numbers 15 and 16 are for. Signed-off-by: Junio C Hamano --- cache.h | 1 + http-backend.c | 5 +---- zlib.c | 15 +++++++++++++++ 3 files changed, 17 insertions(+), 4 deletions(-) diff --git a/cache.h b/cache.h index ce73e1f09d..50f09d05d4 100644 --- a/cache.h +++ b/cache.h @@ -21,6 +21,7 @@ #endif void git_inflate_init(z_streamp strm); +void git_inflate_init_gzip_only(z_streamp strm); void git_inflate_end(z_streamp strm); int git_inflate(z_streamp strm, int flush); diff --git a/http-backend.c b/http-backend.c index c74cb03a70..ab5015d9d9 100644 --- a/http-backend.c +++ b/http-backend.c @@ -275,12 +275,9 @@ static void inflate_request(const char *prog_name, int out) unsigned char in_buf[8192]; unsigned char out_buf[8192]; unsigned long cnt = 0; - int ret; memset(&stream, 0, sizeof(stream)); - ret = inflateInit2(&stream, (15 + 16)); - if (ret != Z_OK) - die("cannot start zlib inflater, zlib err %d", ret); + git_inflate_init_gzip_only(&stream); while (1) { ssize_t n = xread(0, in_buf, sizeof(in_buf)); diff --git a/zlib.c b/zlib.c index be9d7e963b..b613cbd750 100644 --- a/zlib.c +++ b/zlib.c @@ -32,6 +32,21 @@ void git_inflate_init(z_streamp strm) strm->msg ? strm->msg : "no message"); } +void git_inflate_init_gzip_only(z_streamp strm) +{ + /* + * Use default 15 bits, +16 is to accept only gzip and to + * yield Z_DATA_ERROR when fed zlib format. + */ + const int windowBits = 15 + 16; + int status = inflateInit2(strm, windowBits); + + if (status == Z_OK) + return; + die("inflateInit2: %s (%s)", zerr_to_string(status), + strm->msg ? strm->msg : "no message"); +} + void git_inflate_end(z_streamp strm) { int status = inflateEnd(strm); From 55bb5c9147a209eba44c3e9866704ece424c3d31 Mon Sep 17 00:00:00 2001 From: Junio C Hamano Date: Fri, 10 Jun 2011 10:55:10 -0700 Subject: [PATCH 4/7] zlib: wrap deflate side of the API Wrap deflateInit, deflate, and deflateEnd for everybody, and the sole use of deflateInit2 in remote-curl.c to tell the library to use gzip header and trailer in git_deflate_init_gzip(). There is only one caller that cares about the status from deflateEnd(). Introduce git_deflate_end_gently() to let that sole caller retrieve the status and act on it (i.e. die) for now, but we would probably want to make inflate_end/deflate_end die when they ran out of memory and get rid of the _gently() kind. Signed-off-by: Junio C Hamano --- archive-zip.c | 6 ++-- builtin/index-pack.c | 6 ++-- builtin/pack-objects.c | 6 ++-- cache.h | 6 ++++ diff.c | 6 ++-- fast-import.c | 22 +++++++-------- http-push.c | 12 ++++---- remote-curl.c | 10 ++----- sha1_file.c | 10 +++---- zlib.c | 62 ++++++++++++++++++++++++++++++++++++++++++ 10 files changed, 105 insertions(+), 41 deletions(-) diff --git a/archive-zip.c b/archive-zip.c index cf285044e3..081ff830c8 100644 --- a/archive-zip.c +++ b/archive-zip.c @@ -96,7 +96,7 @@ static void *zlib_deflate(void *data, unsigned long size, int result; memset(&stream, 0, sizeof(stream)); - deflateInit(&stream, compression_level); + git_deflate_init(&stream, compression_level); maxsize = deflateBound(&stream, size); buffer = xmalloc(maxsize); @@ -106,7 +106,7 @@ static void *zlib_deflate(void *data, unsigned long size, stream.avail_out = maxsize; do { - result = deflate(&stream, Z_FINISH); + result = git_deflate(&stream, Z_FINISH); } while (result == Z_OK); if (result != Z_STREAM_END) { @@ -114,7 +114,7 @@ static void *zlib_deflate(void *data, unsigned long size, return NULL; } - deflateEnd(&stream); + git_deflate_end(&stream); *compressed_size = stream.total_out; return buffer; diff --git a/builtin/index-pack.c b/builtin/index-pack.c index 31f001f105..dbfb5da52a 100644 --- a/builtin/index-pack.c +++ b/builtin/index-pack.c @@ -671,21 +671,21 @@ static int write_compressed(struct sha1file *f, void *in, unsigned int size) unsigned char outbuf[4096]; memset(&stream, 0, sizeof(stream)); - deflateInit(&stream, zlib_compression_level); + git_deflate_init(&stream, zlib_compression_level); stream.next_in = in; stream.avail_in = size; do { stream.next_out = outbuf; stream.avail_out = sizeof(outbuf); - status = deflate(&stream, Z_FINISH); + status = git_deflate(&stream, Z_FINISH); sha1write(f, outbuf, sizeof(outbuf) - stream.avail_out); } while (status == Z_OK); if (status != Z_STREAM_END) die("unable to deflate appended object (%d)", status); size = stream.total_out; - deflateEnd(&stream); + git_deflate_end(&stream); return size; } diff --git a/builtin/pack-objects.c b/builtin/pack-objects.c index f402a843bb..61f975585d 100644 --- a/builtin/pack-objects.c +++ b/builtin/pack-objects.c @@ -131,7 +131,7 @@ static unsigned long do_compress(void **pptr, unsigned long size) unsigned long maxsize; memset(&stream, 0, sizeof(stream)); - deflateInit(&stream, pack_compression_level); + git_deflate_init(&stream, pack_compression_level); maxsize = deflateBound(&stream, size); in = *pptr; @@ -142,9 +142,9 @@ static unsigned long do_compress(void **pptr, unsigned long size) stream.avail_in = size; stream.next_out = out; stream.avail_out = maxsize; - while (deflate(&stream, Z_FINISH) == Z_OK) + while (git_deflate(&stream, Z_FINISH) == Z_OK) ; /* nothing */ - deflateEnd(&stream); + git_deflate_end(&stream); free(in); return stream.total_out; diff --git a/cache.h b/cache.h index 50f09d05d4..5eb61ac04e 100644 --- a/cache.h +++ b/cache.h @@ -25,6 +25,12 @@ void git_inflate_init_gzip_only(z_streamp strm); void git_inflate_end(z_streamp strm); int git_inflate(z_streamp strm, int flush); +void git_deflate_init(z_streamp strm, int level); +void git_deflate_init_gzip(z_streamp strm, int level); +void git_deflate_end(z_streamp strm); +int git_deflate_end_gently(z_streamp strm); +int git_deflate(z_streamp strm, int flush); + #if defined(DT_UNKNOWN) && !defined(NO_D_TYPE_IN_DIRENT) #define DTYPE(de) ((de)->d_type) #else diff --git a/diff.c b/diff.c index 559bf574a8..c15c70f977 100644 --- a/diff.c +++ b/diff.c @@ -1732,7 +1732,7 @@ static unsigned char *deflate_it(char *data, z_stream stream; memset(&stream, 0, sizeof(stream)); - deflateInit(&stream, zlib_compression_level); + git_deflate_init(&stream, zlib_compression_level); bound = deflateBound(&stream, size); deflated = xmalloc(bound); stream.next_out = deflated; @@ -1740,9 +1740,9 @@ static unsigned char *deflate_it(char *data, stream.next_in = (unsigned char *)data; stream.avail_in = size; - while (deflate(&stream, Z_FINISH) == Z_OK) + while (git_deflate(&stream, Z_FINISH) == Z_OK) ; /* nothing */ - deflateEnd(&stream); + git_deflate_end(&stream); *result_size = stream.total_out; return deflated; } diff --git a/fast-import.c b/fast-import.c index 78d978684d..42979e6db0 100644 --- a/fast-import.c +++ b/fast-import.c @@ -1050,7 +1050,7 @@ static int store_object( delta = NULL; memset(&s, 0, sizeof(s)); - deflateInit(&s, pack_compression_level); + git_deflate_init(&s, pack_compression_level); if (delta) { s.next_in = delta; s.avail_in = deltalen; @@ -1060,9 +1060,9 @@ static int store_object( } s.avail_out = deflateBound(&s, s.avail_in); s.next_out = out = xmalloc(s.avail_out); - while (deflate(&s, Z_FINISH) == Z_OK) - /* nothing */; - deflateEnd(&s); + while (git_deflate(&s, Z_FINISH) == Z_OK) + ; /* nothing */ + git_deflate_end(&s); /* Determine if we should auto-checkpoint. */ if ((max_packsize && (pack_size + 60 + s.total_out) > max_packsize) @@ -1078,14 +1078,14 @@ static int store_object( delta = NULL; memset(&s, 0, sizeof(s)); - deflateInit(&s, pack_compression_level); + git_deflate_init(&s, pack_compression_level); s.next_in = (void *)dat->buf; s.avail_in = dat->len; s.avail_out = deflateBound(&s, s.avail_in); s.next_out = out = xrealloc(out, s.avail_out); - while (deflate(&s, Z_FINISH) == Z_OK) - /* nothing */; - deflateEnd(&s); + while (git_deflate(&s, Z_FINISH) == Z_OK) + ; /* nothing */ + git_deflate_end(&s); } } @@ -1187,7 +1187,7 @@ static void stream_blob(uintmax_t len, unsigned char *sha1out, uintmax_t mark) crc32_begin(pack_file); memset(&s, 0, sizeof(s)); - deflateInit(&s, pack_compression_level); + git_deflate_init(&s, pack_compression_level); hdrlen = encode_in_pack_object_header(OBJ_BLOB, len, out_buf); if (out_sz <= hdrlen) @@ -1209,7 +1209,7 @@ static void stream_blob(uintmax_t len, unsigned char *sha1out, uintmax_t mark) len -= n; } - status = deflate(&s, len ? 0 : Z_FINISH); + status = git_deflate(&s, len ? 0 : Z_FINISH); if (!s.avail_out || status == Z_STREAM_END) { size_t n = s.next_out - out_buf; @@ -1228,7 +1228,7 @@ static void stream_blob(uintmax_t len, unsigned char *sha1out, uintmax_t mark) die("unexpected deflate failure: %d", status); } } - deflateEnd(&s); + git_deflate_end(&s); git_SHA1_Final(sha1, &c); if (sha1out) diff --git a/http-push.c b/http-push.c index d18346c0f5..46e68c80e5 100644 --- a/http-push.c +++ b/http-push.c @@ -359,7 +359,7 @@ static void start_put(struct transfer_request *request) /* Set it up */ memset(&stream, 0, sizeof(stream)); - deflateInit(&stream, zlib_compression_level); + git_deflate_init(&stream, zlib_compression_level); size = deflateBound(&stream, len + hdrlen); strbuf_init(&request->buffer.buf, size); request->buffer.posn = 0; @@ -371,15 +371,15 @@ static void start_put(struct transfer_request *request) /* First header.. */ stream.next_in = (void *)hdr; stream.avail_in = hdrlen; - while (deflate(&stream, 0) == Z_OK) - /* nothing */; + while (git_deflate(&stream, 0) == Z_OK) + ; /* nothing */ /* Then the data itself.. */ stream.next_in = unpacked; stream.avail_in = len; - while (deflate(&stream, Z_FINISH) == Z_OK) - /* nothing */; - deflateEnd(&stream); + while (git_deflate(&stream, Z_FINISH) == Z_OK) + ; /* nothing */ + git_deflate_end(&stream); free(unpacked); request->buffer.buf.len = stream.total_out; diff --git a/remote-curl.c b/remote-curl.c index 775d614303..3c7621aa06 100644 --- a/remote-curl.c +++ b/remote-curl.c @@ -475,11 +475,7 @@ static int post_rpc(struct rpc_state *rpc) int ret; memset(&stream, 0, sizeof(stream)); - ret = deflateInit2(&stream, Z_BEST_COMPRESSION, - Z_DEFLATED, (15 + 16), - 8, Z_DEFAULT_STRATEGY); - if (ret != Z_OK) - die("cannot deflate request; zlib init error %d", ret); + git_deflate_init_gzip(&stream, Z_BEST_COMPRESSION); size = deflateBound(&stream, rpc->len); gzip_body = xmalloc(size); @@ -488,11 +484,11 @@ static int post_rpc(struct rpc_state *rpc) stream.next_out = (unsigned char *)gzip_body; stream.avail_out = size; - ret = deflate(&stream, Z_FINISH); + ret = git_deflate(&stream, Z_FINISH); if (ret != Z_STREAM_END) die("cannot deflate request; zlib deflate error %d", ret); - ret = deflateEnd(&stream); + ret = git_deflate_end_gently(&stream); if (ret != Z_OK) die("cannot deflate request; zlib end error %d", ret); diff --git a/sha1_file.c b/sha1_file.c index 1a7e41070e..0eefb6194e 100644 --- a/sha1_file.c +++ b/sha1_file.c @@ -2442,7 +2442,7 @@ static int write_loose_object(const unsigned char *sha1, char *hdr, int hdrlen, /* Set it up */ memset(&stream, 0, sizeof(stream)); - deflateInit(&stream, zlib_compression_level); + git_deflate_init(&stream, zlib_compression_level); stream.next_out = compressed; stream.avail_out = sizeof(compressed); git_SHA1_Init(&c); @@ -2450,8 +2450,8 @@ static int write_loose_object(const unsigned char *sha1, char *hdr, int hdrlen, /* First header.. */ stream.next_in = (unsigned char *)hdr; stream.avail_in = hdrlen; - while (deflate(&stream, 0) == Z_OK) - /* nothing */; + while (git_deflate(&stream, 0) == Z_OK) + ; /* nothing */ git_SHA1_Update(&c, hdr, hdrlen); /* Then the data itself.. */ @@ -2459,7 +2459,7 @@ static int write_loose_object(const unsigned char *sha1, char *hdr, int hdrlen, stream.avail_in = len; do { unsigned char *in0 = stream.next_in; - ret = deflate(&stream, Z_FINISH); + ret = git_deflate(&stream, Z_FINISH); git_SHA1_Update(&c, in0, stream.next_in - in0); if (write_buffer(fd, compressed, stream.next_out - compressed) < 0) die("unable to write sha1 file"); @@ -2469,7 +2469,7 @@ static int write_loose_object(const unsigned char *sha1, char *hdr, int hdrlen, if (ret != Z_STREAM_END) die("unable to deflate new object %s (%d)", sha1_to_hex(sha1), ret); - ret = deflateEnd(&stream); + ret = git_deflate_end_gently(&stream); if (ret != Z_OK) die("deflateEnd on object %s failed (%d)", sha1_to_hex(sha1), ret); git_SHA1_Final(parano_sha1, &c); diff --git a/zlib.c b/zlib.c index b613cbd750..ee47a3a4a9 100644 --- a/zlib.c +++ b/zlib.c @@ -77,3 +77,65 @@ int git_inflate(z_streamp strm, int flush) strm->msg ? strm->msg : "no message"); return status; } + +void git_deflate_init(z_streamp strm, int level) +{ + int status = deflateInit(strm, level); + + if (status == Z_OK) + return; + die("deflateInit: %s (%s)", zerr_to_string(status), + strm->msg ? strm->msg : "no message"); +} + +void git_deflate_init_gzip(z_streamp strm, int level) +{ + /* + * Use default 15 bits, +16 is to generate gzip header/trailer + * instead of the zlib wrapper. + */ + const int windowBits = 15 + 16; + int status = deflateInit2(strm, level, + Z_DEFLATED, windowBits, + 8, Z_DEFAULT_STRATEGY); + if (status == Z_OK) + return; + die("deflateInit2: %s (%s)", zerr_to_string(status), + strm->msg ? strm->msg : "no message"); +} + +void git_deflate_end(z_streamp strm) +{ + int status = deflateEnd(strm); + + if (status == Z_OK) + return; + error("deflateEnd: %s (%s)", zerr_to_string(status), + strm->msg ? strm->msg : "no message"); +} + +int git_deflate_end_gently(z_streamp strm) +{ + return deflateEnd(strm); +} + +int git_deflate(z_streamp strm, int flush) +{ + int status = deflate(strm, flush); + + switch (status) { + /* Z_BUF_ERROR: normal, needs more space in the output buffer */ + case Z_BUF_ERROR: + case Z_OK: + case Z_STREAM_END: + return status; + + case Z_MEM_ERROR: + die("deflate: out of memory"); + default: + break; + } + error("deflate: %s (%s)", zerr_to_string(status), + strm->msg ? strm->msg : "no message"); + return status; +} From 225a6f1068f71723a910e8565db4e252b3ca21fa Mon Sep 17 00:00:00 2001 From: Junio C Hamano Date: Fri, 10 Jun 2011 11:18:17 -0700 Subject: [PATCH 5/7] zlib: wrap deflateBound() too Signed-off-by: Junio C Hamano --- archive-zip.c | 2 +- builtin/pack-objects.c | 2 +- cache.h | 4 +--- diff.c | 2 +- fast-import.c | 4 ++-- http-push.c | 2 +- remote-curl.c | 2 +- zlib.c | 9 +++++++++ 8 files changed, 17 insertions(+), 10 deletions(-) diff --git a/archive-zip.c b/archive-zip.c index 081ff830c8..8fbac2741a 100644 --- a/archive-zip.c +++ b/archive-zip.c @@ -97,7 +97,7 @@ static void *zlib_deflate(void *data, unsigned long size, memset(&stream, 0, sizeof(stream)); git_deflate_init(&stream, compression_level); - maxsize = deflateBound(&stream, size); + maxsize = git_deflate_bound(&stream, size); buffer = xmalloc(maxsize); stream.next_in = data; diff --git a/builtin/pack-objects.c b/builtin/pack-objects.c index 61f975585d..8981dd6066 100644 --- a/builtin/pack-objects.c +++ b/builtin/pack-objects.c @@ -132,7 +132,7 @@ static unsigned long do_compress(void **pptr, unsigned long size) memset(&stream, 0, sizeof(stream)); git_deflate_init(&stream, pack_compression_level); - maxsize = deflateBound(&stream, size); + maxsize = git_deflate_bound(&stream, size); in = *pptr; out = xmalloc(maxsize); diff --git a/cache.h b/cache.h index 5eb61ac04e..85ac5ec17c 100644 --- a/cache.h +++ b/cache.h @@ -16,9 +16,6 @@ #endif #include -#if defined(NO_DEFLATE_BOUND) || ZLIB_VERNUM < 0x1200 -#define deflateBound(c,s) ((s) + (((s) + 7) >> 3) + (((s) + 63) >> 6) + 11) -#endif void git_inflate_init(z_streamp strm); void git_inflate_init_gzip_only(z_streamp strm); @@ -30,6 +27,7 @@ void git_deflate_init_gzip(z_streamp strm, int level); void git_deflate_end(z_streamp strm); int git_deflate_end_gently(z_streamp strm); int git_deflate(z_streamp strm, int flush); +unsigned long git_deflate_bound(z_streamp, unsigned long); #if defined(DT_UNKNOWN) && !defined(NO_D_TYPE_IN_DIRENT) #define DTYPE(de) ((de)->d_type) diff --git a/diff.c b/diff.c index c15c70f977..bac9a4bc22 100644 --- a/diff.c +++ b/diff.c @@ -1733,7 +1733,7 @@ static unsigned char *deflate_it(char *data, memset(&stream, 0, sizeof(stream)); git_deflate_init(&stream, zlib_compression_level); - bound = deflateBound(&stream, size); + bound = git_deflate_bound(&stream, size); deflated = xmalloc(bound); stream.next_out = deflated; stream.avail_out = bound; diff --git a/fast-import.c b/fast-import.c index 42979e6db0..e4116a4ace 100644 --- a/fast-import.c +++ b/fast-import.c @@ -1058,7 +1058,7 @@ static int store_object( s.next_in = (void *)dat->buf; s.avail_in = dat->len; } - s.avail_out = deflateBound(&s, s.avail_in); + s.avail_out = git_deflate_bound(&s, s.avail_in); s.next_out = out = xmalloc(s.avail_out); while (git_deflate(&s, Z_FINISH) == Z_OK) ; /* nothing */ @@ -1081,7 +1081,7 @@ static int store_object( git_deflate_init(&s, pack_compression_level); s.next_in = (void *)dat->buf; s.avail_in = dat->len; - s.avail_out = deflateBound(&s, s.avail_in); + s.avail_out = git_deflate_bound(&s, s.avail_in); s.next_out = out = xrealloc(out, s.avail_out); while (git_deflate(&s, Z_FINISH) == Z_OK) ; /* nothing */ diff --git a/http-push.c b/http-push.c index 46e68c80e5..ecd67cf40a 100644 --- a/http-push.c +++ b/http-push.c @@ -360,7 +360,7 @@ static void start_put(struct transfer_request *request) /* Set it up */ memset(&stream, 0, sizeof(stream)); git_deflate_init(&stream, zlib_compression_level); - size = deflateBound(&stream, len + hdrlen); + size = git_deflate_bound(&stream, len + hdrlen); strbuf_init(&request->buffer.buf, size); request->buffer.posn = 0; diff --git a/remote-curl.c b/remote-curl.c index 3c7621aa06..13d8ceede3 100644 --- a/remote-curl.c +++ b/remote-curl.c @@ -476,7 +476,7 @@ static int post_rpc(struct rpc_state *rpc) memset(&stream, 0, sizeof(stream)); git_deflate_init_gzip(&stream, Z_BEST_COMPRESSION); - size = deflateBound(&stream, rpc->len); + size = git_deflate_bound(&stream, rpc->len); gzip_body = xmalloc(size); stream.next_in = (unsigned char *)rpc->buf; diff --git a/zlib.c b/zlib.c index ee47a3a4a9..8f19d2fe38 100644 --- a/zlib.c +++ b/zlib.c @@ -78,6 +78,15 @@ int git_inflate(z_streamp strm, int flush) return status; } +#if defined(NO_DEFLATE_BOUND) || ZLIB_VERNUM < 0x1200 +#define deflateBound(c,s) ((s) + (((s) + 7) >> 3) + (((s) + 63) >> 6) + 11) +#endif + +unsigned long git_deflate_bound(z_streamp strm, unsigned long size) +{ + return deflateBound(strm, size); +} + void git_deflate_init(z_streamp strm, int level) { int status = deflateInit(strm, level); From ef49a7a0126d64359c974b4b3b71d7ad42ee3bca Mon Sep 17 00:00:00 2001 From: Junio C Hamano Date: Fri, 10 Jun 2011 11:52:15 -0700 Subject: [PATCH 6/7] zlib: zlib can only process 4GB at a time The size of objects we read from the repository and data we try to put into the repository are represented in "unsigned long", so that on larger architectures we can handle objects that weigh more than 4GB. But the interface defined in zlib.h to communicate with inflate/deflate limits avail_in (how many bytes of input are we calling zlib with) and avail_out (how many bytes of output from zlib are we ready to accept) fields effectively to 4GB by defining their type to be uInt. In many places in our code, we allocate a large buffer (e.g. mmap'ing a large loose object file) and tell zlib its size by assigning the size to avail_in field of the stream, but that will truncate the high octets of the real size. The worst part of this story is that we often pass around z_stream (the state object used by zlib) to keep track of the number of used bytes in input/output buffer by inspecting these two fields, which practically limits our callchain to the same 4GB limit. Wrap z_stream in another structure git_zstream that can express avail_in and avail_out in unsigned long. For now, just die() when the caller gives a size that cannot be given to a single zlib call. In later patches in the series, we would make git_inflate() and git_deflate() internally loop to give callers an illusion that our "improved" version of zlib interface can operate on a buffer larger than 4GB in one go. Signed-off-by: Junio C Hamano --- archive-zip.c | 2 +- builtin/apply.c | 2 +- builtin/index-pack.c | 6 +- builtin/pack-objects.c | 10 ++-- builtin/unpack-objects.c | 2 +- cache.h | 31 ++++++---- diff.c | 2 +- fast-import.c | 4 +- http-backend.c | 2 +- http-push.c | 2 +- http.h | 2 +- pack-check.c | 4 +- remote-curl.c | 2 +- sha1_file.c | 18 +++--- zlib.c | 119 ++++++++++++++++++++++++++++++--------- 15 files changed, 140 insertions(+), 68 deletions(-) diff --git a/archive-zip.c b/archive-zip.c index 8fbac2741a..72d55a58ac 100644 --- a/archive-zip.c +++ b/archive-zip.c @@ -90,7 +90,7 @@ static void copy_le32(unsigned char *dest, unsigned int n) static void *zlib_deflate(void *data, unsigned long size, int compression_level, unsigned long *compressed_size) { - z_stream stream; + git_zstream stream; unsigned long maxsize; void *buffer; int result; diff --git a/builtin/apply.c b/builtin/apply.c index 530d4bb7e7..f2edc52818 100644 --- a/builtin/apply.c +++ b/builtin/apply.c @@ -1634,7 +1634,7 @@ static inline int metadata_changes(struct patch *patch) static char *inflate_it(const void *data, unsigned long size, unsigned long inflated_size) { - z_stream stream; + git_zstream stream; void *out; int st; diff --git a/builtin/index-pack.c b/builtin/index-pack.c index dbfb5da52a..f65cb373bf 100644 --- a/builtin/index-pack.c +++ b/builtin/index-pack.c @@ -265,7 +265,7 @@ static void unlink_base_data(struct base_data *c) static void *unpack_entry_data(unsigned long offset, unsigned long size) { int status; - z_stream stream; + git_zstream stream; void *buf = xmalloc(size); memset(&stream, 0, sizeof(stream)); @@ -355,7 +355,7 @@ static void *get_data_from_pack(struct object_entry *obj) off_t from = obj[0].idx.offset + obj[0].hdr_size; unsigned long len = obj[1].idx.offset - from; unsigned char *data, *inbuf; - z_stream stream; + git_zstream stream; int status; data = xmalloc(obj->size); @@ -666,7 +666,7 @@ static void parse_pack_objects(unsigned char *sha1) static int write_compressed(struct sha1file *f, void *in, unsigned int size) { - z_stream stream; + git_zstream stream; int status; unsigned char outbuf[4096]; diff --git a/builtin/pack-objects.c b/builtin/pack-objects.c index 8981dd6066..c6e2d8766b 100644 --- a/builtin/pack-objects.c +++ b/builtin/pack-objects.c @@ -126,7 +126,7 @@ static void *get_delta(struct object_entry *entry) static unsigned long do_compress(void **pptr, unsigned long size) { - z_stream stream; + git_zstream stream; void *in, *out; unsigned long maxsize; @@ -160,7 +160,7 @@ static int check_pack_inflate(struct packed_git *p, off_t len, unsigned long expect) { - z_stream stream; + git_zstream stream; unsigned char fakebuf[4096], *in; int st; @@ -187,12 +187,12 @@ static void copy_pack_data(struct sha1file *f, off_t len) { unsigned char *in; - unsigned int avail; + unsigned long avail; while (len) { in = use_pack(p, w_curs, offset, &avail); if (avail > len) - avail = (unsigned int)len; + avail = (unsigned long)len; sha1write(f, in, avail); offset += avail; len -= avail; @@ -994,7 +994,7 @@ static void check_object(struct object_entry *entry) const unsigned char *base_ref = NULL; struct object_entry *base_entry; unsigned long used, used_0; - unsigned int avail; + unsigned long avail; off_t ofs; unsigned char *buf, c; diff --git a/builtin/unpack-objects.c b/builtin/unpack-objects.c index f63973c914..14e04e6795 100644 --- a/builtin/unpack-objects.c +++ b/builtin/unpack-objects.c @@ -90,7 +90,7 @@ static void use(int bytes) static void *get_data(unsigned long size) { - z_stream stream; + git_zstream stream; void *buf = xmalloc(size); memset(&stream, 0, sizeof(stream)); diff --git a/cache.h b/cache.h index 85ac5ec17c..1784bab7d5 100644 --- a/cache.h +++ b/cache.h @@ -16,18 +16,27 @@ #endif #include +typedef struct git_zstream { + z_stream z; + unsigned long avail_in; + unsigned long avail_out; + unsigned long total_in; + unsigned long total_out; + unsigned char *next_in; + unsigned char *next_out; +} git_zstream; -void git_inflate_init(z_streamp strm); -void git_inflate_init_gzip_only(z_streamp strm); -void git_inflate_end(z_streamp strm); -int git_inflate(z_streamp strm, int flush); +void git_inflate_init(git_zstream *); +void git_inflate_init_gzip_only(git_zstream *); +void git_inflate_end(git_zstream *); +int git_inflate(git_zstream *, int flush); -void git_deflate_init(z_streamp strm, int level); -void git_deflate_init_gzip(z_streamp strm, int level); -void git_deflate_end(z_streamp strm); -int git_deflate_end_gently(z_streamp strm); -int git_deflate(z_streamp strm, int flush); -unsigned long git_deflate_bound(z_streamp, unsigned long); +void git_deflate_init(git_zstream *, int level); +void git_deflate_init_gzip(git_zstream *, int level); +void git_deflate_end(git_zstream *); +int git_deflate_end_gently(git_zstream *); +int git_deflate(git_zstream *, int flush); +unsigned long git_deflate_bound(git_zstream *, unsigned long); #if defined(DT_UNKNOWN) && !defined(NO_D_TYPE_IN_DIRENT) #define DTYPE(de) ((de)->d_type) @@ -991,7 +1000,7 @@ extern struct packed_git *find_sha1_pack(const unsigned char *sha1, extern void pack_report(void); extern int open_pack_index(struct packed_git *); extern void close_pack_index(struct packed_git *); -extern unsigned char *use_pack(struct packed_git *, struct pack_window **, off_t, unsigned int *); +extern unsigned char *use_pack(struct packed_git *, struct pack_window **, off_t, unsigned long *); extern void close_pack_windows(struct packed_git *); extern void unuse_pack(struct pack_window **); extern void free_pack_by_name(const char *); diff --git a/diff.c b/diff.c index bac9a4bc22..431873d43f 100644 --- a/diff.c +++ b/diff.c @@ -1729,7 +1729,7 @@ static unsigned char *deflate_it(char *data, { int bound; unsigned char *deflated; - z_stream stream; + git_zstream stream; memset(&stream, 0, sizeof(stream)); git_deflate_init(&stream, zlib_compression_level); diff --git a/fast-import.c b/fast-import.c index e4116a4ace..1d5e3336a5 100644 --- a/fast-import.c +++ b/fast-import.c @@ -1017,7 +1017,7 @@ static int store_object( unsigned char sha1[20]; unsigned long hdrlen, deltalen; git_SHA_CTX c; - z_stream s; + git_zstream s; hdrlen = sprintf((char *)hdr,"%s %lu", typename(type), (unsigned long)dat->len) + 1; @@ -1163,7 +1163,7 @@ static void stream_blob(uintmax_t len, unsigned char *sha1out, uintmax_t mark) off_t offset; git_SHA_CTX c; git_SHA_CTX pack_file_ctx; - z_stream s; + git_zstream s; int status = Z_OK; /* Determine if we should auto-checkpoint. */ diff --git a/http-backend.c b/http-backend.c index ab5015d9d9..59ad7da605 100644 --- a/http-backend.c +++ b/http-backend.c @@ -271,7 +271,7 @@ static struct rpc_service *select_service(const char *name) static void inflate_request(const char *prog_name, int out) { - z_stream stream; + git_zstream stream; unsigned char in_buf[8192]; unsigned char out_buf[8192]; unsigned long cnt = 0; diff --git a/http-push.c b/http-push.c index ecd67cf40a..1e8a830937 100644 --- a/http-push.c +++ b/http-push.c @@ -352,7 +352,7 @@ static void start_put(struct transfer_request *request) unsigned long len; int hdrlen; ssize_t size; - z_stream stream; + git_zstream stream; unpacked = read_sha1_file(request->obj->sha1, &type, &len); hdrlen = sprintf(hdr, "%s %lu", typename(type), len) + 1; diff --git a/http.h b/http.h index e9ed3c2e82..a39304a7ab 100644 --- a/http.h +++ b/http.h @@ -172,7 +172,7 @@ struct http_object_request { unsigned char sha1[20]; unsigned char real_sha1[20]; git_SHA_CTX c; - z_stream stream; + git_zstream stream; int zret; int rename; struct active_request_slot *slot; diff --git a/pack-check.c b/pack-check.c index a1a521648d..0c19b6e5a5 100644 --- a/pack-check.c +++ b/pack-check.c @@ -26,7 +26,7 @@ int check_pack_crc(struct packed_git *p, struct pack_window **w_curs, uint32_t data_crc = crc32(0, NULL, 0); do { - unsigned int avail; + unsigned long avail; void *data = use_pack(p, w_curs, offset, &avail); if (avail > len) avail = len; @@ -61,7 +61,7 @@ static int verify_packfile(struct packed_git *p, git_SHA1_Init(&ctx); do { - unsigned int remaining; + unsigned long remaining; unsigned char *in = use_pack(p, w_curs, offset, &remaining); offset += remaining; if (!pack_sig_ofs) diff --git a/remote-curl.c b/remote-curl.c index 13d8ceede3..bc48a36ef5 100644 --- a/remote-curl.c +++ b/remote-curl.c @@ -471,7 +471,7 @@ static int post_rpc(struct rpc_state *rpc) * the transfer time. */ size_t size; - z_stream stream; + git_zstream stream; int ret; memset(&stream, 0, sizeof(stream)); diff --git a/sha1_file.c b/sha1_file.c index 0eefb6194e..94d431907c 100644 --- a/sha1_file.c +++ b/sha1_file.c @@ -833,7 +833,7 @@ static int in_window(struct pack_window *win, off_t offset) unsigned char *use_pack(struct packed_git *p, struct pack_window **w_cursor, off_t offset, - unsigned int *left) + unsigned long *left) { struct pack_window *win = *w_cursor; @@ -1244,7 +1244,7 @@ unsigned long unpack_object_header_buffer(const unsigned char *buf, return used; } -static int unpack_sha1_header(z_stream *stream, unsigned char *map, unsigned long mapsize, void *buffer, unsigned long bufsiz) +static int unpack_sha1_header(git_zstream *stream, unsigned char *map, unsigned long mapsize, void *buffer, unsigned long bufsiz) { unsigned long size, used; static const char valid_loose_object_type[8] = { @@ -1291,7 +1291,7 @@ static int unpack_sha1_header(z_stream *stream, unsigned char *map, unsigned lon return 0; } -static void *unpack_sha1_rest(z_stream *stream, void *buffer, unsigned long size, const unsigned char *sha1) +static void *unpack_sha1_rest(git_zstream *stream, void *buffer, unsigned long size, const unsigned char *sha1) { int bytes = strlen(buffer) + 1; unsigned char *buf = xmallocz(size); @@ -1390,7 +1390,7 @@ static int parse_sha1_header(const char *hdr, unsigned long *sizep) static void *unpack_sha1_file(void *map, unsigned long mapsize, enum object_type *type, unsigned long *size, const unsigned char *sha1) { int ret; - z_stream stream; + git_zstream stream; char hdr[8192]; ret = unpack_sha1_header(&stream, map, mapsize, hdr, sizeof(hdr)); @@ -1406,7 +1406,7 @@ unsigned long get_size_from_delta(struct packed_git *p, { const unsigned char *data; unsigned char delta_head[20], *in; - z_stream stream; + git_zstream stream; int st; memset(&stream, 0, sizeof(stream)); @@ -1528,7 +1528,7 @@ static int unpack_object_header(struct packed_git *p, unsigned long *sizep) { unsigned char *base; - unsigned int left; + unsigned long left; unsigned long used; enum object_type type; @@ -1641,7 +1641,7 @@ static void *unpack_compressed_entry(struct packed_git *p, unsigned long size) { int st; - z_stream stream; + git_zstream stream; unsigned char *buffer, *in; buffer = xmallocz(size); @@ -2074,7 +2074,7 @@ static int sha1_loose_object_info(const unsigned char *sha1, unsigned long *size int status; unsigned long mapsize, size; void *map; - z_stream stream; + git_zstream stream; char hdr[32]; map = map_sha1_file(sha1, &mapsize); @@ -2425,7 +2425,7 @@ static int write_loose_object(const unsigned char *sha1, char *hdr, int hdrlen, { int fd, ret; unsigned char compressed[4096]; - z_stream stream; + git_zstream stream; git_SHA_CTX c; unsigned char parano_sha1[20]; char *filename; diff --git a/zlib.c b/zlib.c index 8f19d2fe38..fe537e3ff6 100644 --- a/zlib.c +++ b/zlib.c @@ -22,45 +22,90 @@ static const char *zerr_to_string(int status) } } -void git_inflate_init(z_streamp strm) +/* + * avail_in and avail_out in zlib are counted in uInt, which typically + * limits the size of the buffer we can use to 4GB when interacting + * with zlib in a single call to inflate/deflate. + */ +#define ZLIB_BUF_MAX ((uInt)-1) +static inline uInt zlib_buf_cap(unsigned long len) { - int status = inflateInit(strm); + if (ZLIB_BUF_MAX < len) + die("working buffer for zlib too large"); + return len; +} +static void zlib_pre_call(git_zstream *s) +{ + s->z.next_in = s->next_in; + s->z.next_out = s->next_out; + s->z.total_in = s->total_in; + s->z.total_out = s->total_out; + s->z.avail_in = zlib_buf_cap(s->avail_in); + s->z.avail_out = zlib_buf_cap(s->avail_out); +} + +static void zlib_post_call(git_zstream *s) +{ + s->next_in = s->z.next_in; + s->next_out = s->z.next_out; + s->total_in = s->z.total_in; + s->total_out = s->z.total_out; + s->avail_in = s->z.avail_in; + s->avail_out = s->z.avail_out; +} + +void git_inflate_init(git_zstream *strm) +{ + int status; + + zlib_pre_call(strm); + status = inflateInit(&strm->z); + zlib_post_call(strm); if (status == Z_OK) return; die("inflateInit: %s (%s)", zerr_to_string(status), - strm->msg ? strm->msg : "no message"); + strm->z.msg ? strm->z.msg : "no message"); } -void git_inflate_init_gzip_only(z_streamp strm) +void git_inflate_init_gzip_only(git_zstream *strm) { /* * Use default 15 bits, +16 is to accept only gzip and to * yield Z_DATA_ERROR when fed zlib format. */ const int windowBits = 15 + 16; - int status = inflateInit2(strm, windowBits); + int status; + zlib_pre_call(strm); + status = inflateInit2(&strm->z, windowBits); + zlib_post_call(strm); if (status == Z_OK) return; die("inflateInit2: %s (%s)", zerr_to_string(status), - strm->msg ? strm->msg : "no message"); + strm->z.msg ? strm->z.msg : "no message"); } -void git_inflate_end(z_streamp strm) +void git_inflate_end(git_zstream *strm) { - int status = inflateEnd(strm); + int status; + zlib_pre_call(strm); + status = inflateEnd(&strm->z); + zlib_post_call(strm); if (status == Z_OK) return; error("inflateEnd: %s (%s)", zerr_to_string(status), - strm->msg ? strm->msg : "no message"); + strm->z.msg ? strm->z.msg : "no message"); } -int git_inflate(z_streamp strm, int flush) +int git_inflate(git_zstream *strm, int flush) { - int status = inflate(strm, flush); + int status; + zlib_pre_call(strm); + status = inflate(&strm->z, flush); + zlib_post_call(strm); switch (status) { /* Z_BUF_ERROR: normal, needs more space in the output buffer */ case Z_BUF_ERROR: @@ -74,7 +119,7 @@ int git_inflate(z_streamp strm, int flush) break; } error("inflate: %s (%s)", zerr_to_string(status), - strm->msg ? strm->msg : "no message"); + strm->z.msg ? strm->z.msg : "no message"); return status; } @@ -82,56 +127,74 @@ int git_inflate(z_streamp strm, int flush) #define deflateBound(c,s) ((s) + (((s) + 7) >> 3) + (((s) + 63) >> 6) + 11) #endif -unsigned long git_deflate_bound(z_streamp strm, unsigned long size) +unsigned long git_deflate_bound(git_zstream *strm, unsigned long size) { - return deflateBound(strm, size); + return deflateBound(&strm->z, size); } -void git_deflate_init(z_streamp strm, int level) +void git_deflate_init(git_zstream *strm, int level) { - int status = deflateInit(strm, level); + int status; + zlib_pre_call(strm); + status = deflateInit(&strm->z, level); + zlib_post_call(strm); if (status == Z_OK) return; die("deflateInit: %s (%s)", zerr_to_string(status), - strm->msg ? strm->msg : "no message"); + strm->z.msg ? strm->z.msg : "no message"); } -void git_deflate_init_gzip(z_streamp strm, int level) +void git_deflate_init_gzip(git_zstream *strm, int level) { /* * Use default 15 bits, +16 is to generate gzip header/trailer * instead of the zlib wrapper. */ const int windowBits = 15 + 16; - int status = deflateInit2(strm, level, + int status; + + zlib_pre_call(strm); + status = deflateInit2(&strm->z, level, Z_DEFLATED, windowBits, 8, Z_DEFAULT_STRATEGY); + zlib_post_call(strm); if (status == Z_OK) return; die("deflateInit2: %s (%s)", zerr_to_string(status), - strm->msg ? strm->msg : "no message"); + strm->z.msg ? strm->z.msg : "no message"); } -void git_deflate_end(z_streamp strm) +void git_deflate_end(git_zstream *strm) { - int status = deflateEnd(strm); + int status; + zlib_pre_call(strm); + status = deflateEnd(&strm->z); + zlib_post_call(strm); if (status == Z_OK) return; error("deflateEnd: %s (%s)", zerr_to_string(status), - strm->msg ? strm->msg : "no message"); + strm->z.msg ? strm->z.msg : "no message"); } -int git_deflate_end_gently(z_streamp strm) +int git_deflate_end_gently(git_zstream *strm) { - return deflateEnd(strm); + int status; + + zlib_pre_call(strm); + status = deflateEnd(&strm->z); + zlib_post_call(strm); + return status; } -int git_deflate(z_streamp strm, int flush) +int git_deflate(git_zstream *strm, int flush) { - int status = deflate(strm, flush); + int status; + zlib_pre_call(strm); + status = deflate(&strm->z, flush); + zlib_post_call(strm); switch (status) { /* Z_BUF_ERROR: normal, needs more space in the output buffer */ case Z_BUF_ERROR: @@ -145,6 +208,6 @@ int git_deflate(z_streamp strm, int flush) break; } error("deflate: %s (%s)", zerr_to_string(status), - strm->msg ? strm->msg : "no message"); + strm->z.msg ? strm->z.msg : "no message"); return status; } From e01503b523e79748ac91d876f506811c597d03cb Mon Sep 17 00:00:00 2001 From: Junio C Hamano Date: Fri, 10 Jun 2011 12:15:17 -0700 Subject: [PATCH 7/7] zlib: allow feeding more than 4GB in one go Update zlib_post_call() that adjusts the wrapper's notion of avail_in and avail_out to what came back from zlib, so that the callers can feed buffers larger than than 4GB to the API. When underlying inflate/deflate stopped processing because we fed a buffer larger than 4GB limit, detect that case, update the state variables, and let the zlib function work another round. Signed-off-by: Junio C Hamano --- zlib.c | 78 +++++++++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 58 insertions(+), 20 deletions(-) diff --git a/zlib.c b/zlib.c index fe537e3ff6..3c63d480c7 100644 --- a/zlib.c +++ b/zlib.c @@ -27,12 +27,11 @@ static const char *zerr_to_string(int status) * limits the size of the buffer we can use to 4GB when interacting * with zlib in a single call to inflate/deflate. */ -#define ZLIB_BUF_MAX ((uInt)-1) +/* #define ZLIB_BUF_MAX ((uInt)-1) */ +#define ZLIB_BUF_MAX ((uInt) 1024 * 1024 * 1024) /* 1GB */ static inline uInt zlib_buf_cap(unsigned long len) { - if (ZLIB_BUF_MAX < len) - die("working buffer for zlib too large"); - return len; + return (ZLIB_BUF_MAX < len) ? ZLIB_BUF_MAX : len; } static void zlib_pre_call(git_zstream *s) @@ -47,12 +46,22 @@ static void zlib_pre_call(git_zstream *s) static void zlib_post_call(git_zstream *s) { + unsigned long bytes_consumed; + unsigned long bytes_produced; + + bytes_consumed = s->z.next_in - s->next_in; + bytes_produced = s->z.next_out - s->next_out; + if (s->z.total_out != s->total_out + bytes_produced) + die("BUG: total_out mismatch"); + if (s->z.total_in != s->total_in + bytes_consumed) + die("BUG: total_in mismatch"); + + s->total_out = s->z.total_out; + s->total_in = s->z.total_in; s->next_in = s->z.next_in; s->next_out = s->z.next_out; - s->total_in = s->z.total_in; - s->total_out = s->z.total_out; - s->avail_in = s->z.avail_in; - s->avail_out = s->z.avail_out; + s->avail_in -= bytes_consumed; + s->avail_out -= bytes_produced; } void git_inflate_init(git_zstream *strm) @@ -103,18 +112,32 @@ int git_inflate(git_zstream *strm, int flush) { int status; - zlib_pre_call(strm); - status = inflate(&strm->z, flush); - zlib_post_call(strm); + for (;;) { + zlib_pre_call(strm); + /* Never say Z_FINISH unless we are feeding everything */ + status = inflate(&strm->z, + (strm->z.avail_in != strm->avail_in) + ? 0 : flush); + if (status == Z_MEM_ERROR) + die("inflate: out of memory"); + zlib_post_call(strm); + + /* + * Let zlib work another round, while we can still + * make progress. + */ + if ((strm->avail_out && !strm->z.avail_out) && + (status == Z_OK || status == Z_BUF_ERROR)) + continue; + break; + } + switch (status) { /* Z_BUF_ERROR: normal, needs more space in the output buffer */ case Z_BUF_ERROR: case Z_OK: case Z_STREAM_END: return status; - - case Z_MEM_ERROR: - die("inflate: out of memory"); default: break; } @@ -192,18 +215,33 @@ int git_deflate(git_zstream *strm, int flush) { int status; - zlib_pre_call(strm); - status = deflate(&strm->z, flush); - zlib_post_call(strm); + for (;;) { + zlib_pre_call(strm); + + /* Never say Z_FINISH unless we are feeding everything */ + status = deflate(&strm->z, + (strm->z.avail_in != strm->avail_in) + ? 0 : flush); + if (status == Z_MEM_ERROR) + die("deflate: out of memory"); + zlib_post_call(strm); + + /* + * Let zlib work another round, while we can still + * make progress. + */ + if ((strm->avail_out && !strm->z.avail_out) && + (status == Z_OK || status == Z_BUF_ERROR)) + continue; + break; + } + switch (status) { /* Z_BUF_ERROR: normal, needs more space in the output buffer */ case Z_BUF_ERROR: case Z_OK: case Z_STREAM_END: return status; - - case Z_MEM_ERROR: - die("deflate: out of memory"); default: break; }