git/zlib.c
Junio C Hamano ef49a7a012 zlib: zlib can only process 4GB at a time
The size of objects we read from the repository and data we try to put
into the repository are represented in "unsigned long", so that on larger
architectures we can handle objects that weigh more than 4GB.

But the interface defined in zlib.h to communicate with inflate/deflate
limits avail_in (how many bytes of input are we calling zlib with) and
avail_out (how many bytes of output from zlib are we ready to accept)
fields effectively to 4GB by defining their type to be uInt.

In many places in our code, we allocate a large buffer (e.g. mmap'ing a
large loose object file) and tell zlib its size by assigning the size to
avail_in field of the stream, but that will truncate the high octets of
the real size. The worst part of this story is that we often pass around
z_stream (the state object used by zlib) to keep track of the number of
used bytes in input/output buffer by inspecting these two fields, which
practically limits our callchain to the same 4GB limit.

Wrap z_stream in another structure git_zstream that can express avail_in
and avail_out in unsigned long. For now, just die() when the caller gives
a size that cannot be given to a single zlib call. In later patches in the
series, we would make git_inflate() and git_deflate() internally loop to
give callers an illusion that our "improved" version of zlib interface can
operate on a buffer larger than 4GB in one go.

Signed-off-by: Junio C Hamano <gitster@pobox.com>
2011-06-10 11:52:15 -07:00

214 lines
4.6 KiB
C

/*
* zlib wrappers to make sure we don't silently miss errors
* at init time.
*/
#include "cache.h"
static const char *zerr_to_string(int status)
{
switch (status) {
case Z_MEM_ERROR:
return "out of memory";
case Z_VERSION_ERROR:
return "wrong version";
case Z_NEED_DICT:
return "needs dictionary";
case Z_DATA_ERROR:
return "data stream error";
case Z_STREAM_ERROR:
return "stream consistency error";
default:
return "unknown error";
}
}
/*
* avail_in and avail_out in zlib are counted in uInt, which typically
* limits the size of the buffer we can use to 4GB when interacting
* with zlib in a single call to inflate/deflate.
*/
#define ZLIB_BUF_MAX ((uInt)-1)
static inline uInt zlib_buf_cap(unsigned long len)
{
if (ZLIB_BUF_MAX < len)
die("working buffer for zlib too large");
return len;
}
static void zlib_pre_call(git_zstream *s)
{
s->z.next_in = s->next_in;
s->z.next_out = s->next_out;
s->z.total_in = s->total_in;
s->z.total_out = s->total_out;
s->z.avail_in = zlib_buf_cap(s->avail_in);
s->z.avail_out = zlib_buf_cap(s->avail_out);
}
static void zlib_post_call(git_zstream *s)
{
s->next_in = s->z.next_in;
s->next_out = s->z.next_out;
s->total_in = s->z.total_in;
s->total_out = s->z.total_out;
s->avail_in = s->z.avail_in;
s->avail_out = s->z.avail_out;
}
void git_inflate_init(git_zstream *strm)
{
int status;
zlib_pre_call(strm);
status = inflateInit(&strm->z);
zlib_post_call(strm);
if (status == Z_OK)
return;
die("inflateInit: %s (%s)", zerr_to_string(status),
strm->z.msg ? strm->z.msg : "no message");
}
void git_inflate_init_gzip_only(git_zstream *strm)
{
/*
* Use default 15 bits, +16 is to accept only gzip and to
* yield Z_DATA_ERROR when fed zlib format.
*/
const int windowBits = 15 + 16;
int status;
zlib_pre_call(strm);
status = inflateInit2(&strm->z, windowBits);
zlib_post_call(strm);
if (status == Z_OK)
return;
die("inflateInit2: %s (%s)", zerr_to_string(status),
strm->z.msg ? strm->z.msg : "no message");
}
void git_inflate_end(git_zstream *strm)
{
int status;
zlib_pre_call(strm);
status = inflateEnd(&strm->z);
zlib_post_call(strm);
if (status == Z_OK)
return;
error("inflateEnd: %s (%s)", zerr_to_string(status),
strm->z.msg ? strm->z.msg : "no message");
}
int git_inflate(git_zstream *strm, int flush)
{
int status;
zlib_pre_call(strm);
status = inflate(&strm->z, flush);
zlib_post_call(strm);
switch (status) {
/* Z_BUF_ERROR: normal, needs more space in the output buffer */
case Z_BUF_ERROR:
case Z_OK:
case Z_STREAM_END:
return status;
case Z_MEM_ERROR:
die("inflate: out of memory");
default:
break;
}
error("inflate: %s (%s)", zerr_to_string(status),
strm->z.msg ? strm->z.msg : "no message");
return status;
}
#if defined(NO_DEFLATE_BOUND) || ZLIB_VERNUM < 0x1200
#define deflateBound(c,s) ((s) + (((s) + 7) >> 3) + (((s) + 63) >> 6) + 11)
#endif
unsigned long git_deflate_bound(git_zstream *strm, unsigned long size)
{
return deflateBound(&strm->z, size);
}
void git_deflate_init(git_zstream *strm, int level)
{
int status;
zlib_pre_call(strm);
status = deflateInit(&strm->z, level);
zlib_post_call(strm);
if (status == Z_OK)
return;
die("deflateInit: %s (%s)", zerr_to_string(status),
strm->z.msg ? strm->z.msg : "no message");
}
void git_deflate_init_gzip(git_zstream *strm, int level)
{
/*
* Use default 15 bits, +16 is to generate gzip header/trailer
* instead of the zlib wrapper.
*/
const int windowBits = 15 + 16;
int status;
zlib_pre_call(strm);
status = deflateInit2(&strm->z, level,
Z_DEFLATED, windowBits,
8, Z_DEFAULT_STRATEGY);
zlib_post_call(strm);
if (status == Z_OK)
return;
die("deflateInit2: %s (%s)", zerr_to_string(status),
strm->z.msg ? strm->z.msg : "no message");
}
void git_deflate_end(git_zstream *strm)
{
int status;
zlib_pre_call(strm);
status = deflateEnd(&strm->z);
zlib_post_call(strm);
if (status == Z_OK)
return;
error("deflateEnd: %s (%s)", zerr_to_string(status),
strm->z.msg ? strm->z.msg : "no message");
}
int git_deflate_end_gently(git_zstream *strm)
{
int status;
zlib_pre_call(strm);
status = deflateEnd(&strm->z);
zlib_post_call(strm);
return status;
}
int git_deflate(git_zstream *strm, int flush)
{
int status;
zlib_pre_call(strm);
status = deflate(&strm->z, flush);
zlib_post_call(strm);
switch (status) {
/* Z_BUF_ERROR: normal, needs more space in the output buffer */
case Z_BUF_ERROR:
case Z_OK:
case Z_STREAM_END:
return status;
case Z_MEM_ERROR:
die("deflate: out of memory");
default:
break;
}
error("deflate: %s (%s)", zerr_to_string(status),
strm->z.msg ? strm->z.msg : "no message");
return status;
}