2006-08-03 15:24:36 +00:00
|
|
|
#include "builtin.h"
|
2005-06-25 21:42:43 +00:00
|
|
|
#include "cache.h"
|
2007-05-19 07:39:31 +00:00
|
|
|
#include "attr.h"
|
2005-06-25 21:42:43 +00:00
|
|
|
#include "object.h"
|
2006-04-02 12:44:09 +00:00
|
|
|
#include "blob.h"
|
|
|
|
#include "commit.h"
|
|
|
|
#include "tag.h"
|
|
|
|
#include "tree.h"
|
2005-06-25 21:42:43 +00:00
|
|
|
#include "delta.h"
|
2005-06-28 21:21:02 +00:00
|
|
|
#include "pack.h"
|
2008-02-28 05:25:17 +00:00
|
|
|
#include "pack-revindex.h"
|
2005-06-27 03:27:56 +00:00
|
|
|
#include "csum-file.h"
|
2006-03-30 06:55:43 +00:00
|
|
|
#include "tree-walk.h"
|
2006-09-05 06:47:39 +00:00
|
|
|
#include "diff.h"
|
|
|
|
#include "revision.h"
|
|
|
|
#include "list-objects.h"
|
2007-04-18 18:27:45 +00:00
|
|
|
#include "progress.h"
|
2008-03-04 03:27:20 +00:00
|
|
|
#include "refs.h"
|
2005-06-25 21:42:43 +00:00
|
|
|
|
2007-09-06 06:13:11 +00:00
|
|
|
#ifdef THREADED_DELTA_SEARCH
|
2008-02-23 02:11:56 +00:00
|
|
|
#include "thread-utils.h"
|
2007-09-06 06:13:11 +00:00
|
|
|
#include <pthread.h>
|
|
|
|
#endif
|
|
|
|
|
2006-11-07 15:51:23 +00:00
|
|
|
static const char pack_usage[] = "\
|
2008-09-13 17:18:36 +00:00
|
|
|
git pack-objects [{ -q | --progress | --all-progress }] \n\
|
2007-07-12 13:07:46 +00:00
|
|
|
[--max-pack-size=N] [--local] [--incremental] \n\
|
|
|
|
[--window=N] [--window-memory=N] [--depth=N] \n\
|
2007-05-09 16:31:28 +00:00
|
|
|
[--no-reuse-delta] [--no-reuse-object] [--delta-base-offset] \n\
|
2007-09-10 04:06:11 +00:00
|
|
|
[--threads=N] [--non-empty] [--revs [--unpacked | --all]*] [--reflog] \n\
|
2008-05-14 05:33:53 +00:00
|
|
|
[--stdout | base-name] [--include-tag] \n\
|
|
|
|
[--keep-unreachable | --unpack-unreachable] \n\
|
2008-03-04 03:27:20 +00:00
|
|
|
[<ref-list | <object-list]";
|
2005-06-25 21:42:43 +00:00
|
|
|
|
|
|
|
struct object_entry {
|
2007-06-01 19:18:05 +00:00
|
|
|
struct pack_idx_entry idx;
|
pack-objects: reuse data from existing packs.
When generating a new pack, notice if we have already needed
objects in existing packs. If an object is stored deltified,
and its base object is also what we are going to pack, then
reuse the existing deltified representation unconditionally,
bypassing all the expensive find_deltas() and try_deltas()
calls.
Also, notice if what we are going to write out exactly match
what is already in an existing pack (either deltified or just
compressed). In such a case, we can just copy it instead of
going through the usual uncompressing & recompressing cycle.
Without this patch, in linux-2.6 repository with about 1500
loose objects and a single mega pack:
$ git-rev-list --objects v2.6.16-rc3 >RL
$ wc -l RL
184141 RL
$ time git-pack-objects p <RL
Generating pack...
Done counting 184141 objects.
Packing 184141 objects....................
a1fc7b3e537fcb9b3c46b7505df859f0a11e79d2
real 12m4.323s
user 11m2.560s
sys 0m55.950s
With this patch, the same input:
$ time ../git.junio/git-pack-objects q <RL
Generating pack...
Done counting 184141 objects.
Packing 184141 objects.....................
a1fc7b3e537fcb9b3c46b7505df859f0a11e79d2
Total 184141, written 184141, reused 182441
real 1m2.608s
user 0m55.090s
sys 0m1.830s
Signed-off-by: Junio C Hamano <junkio@cox.net>
2006-02-16 01:34:29 +00:00
|
|
|
unsigned long size; /* uncompressed size */
|
|
|
|
struct packed_git *in_pack; /* already in pack */
|
2007-03-07 01:44:34 +00:00
|
|
|
off_t in_pack_offset;
|
2007-04-16 16:31:31 +00:00
|
|
|
struct object_entry *delta; /* delta base object */
|
2006-07-10 05:50:18 +00:00
|
|
|
struct object_entry *delta_child; /* deltified objects who bases me */
|
2006-02-18 04:58:45 +00:00
|
|
|
struct object_entry *delta_sibling; /* other deltified objects who
|
|
|
|
* uses the same base as me
|
|
|
|
*/
|
2007-05-28 21:20:58 +00:00
|
|
|
void *delta_data; /* cached delta (uncompressed) */
|
2007-04-16 16:31:31 +00:00
|
|
|
unsigned long delta_size; /* delta data size (uncompressed) */
|
2008-05-02 19:11:50 +00:00
|
|
|
unsigned long z_delta_size; /* delta data size (compressed) */
|
2007-07-12 21:07:59 +00:00
|
|
|
unsigned int hash; /* name hint hash */
|
2007-04-16 16:31:31 +00:00
|
|
|
enum object_type type;
|
|
|
|
enum object_type in_pack_type; /* could be delta */
|
|
|
|
unsigned char in_pack_header_size;
|
|
|
|
unsigned char preferred_base; /* we do not pack this, but is available
|
2007-05-19 07:39:31 +00:00
|
|
|
* to be used as the base object to delta
|
2007-04-16 16:31:31 +00:00
|
|
|
* objects against.
|
|
|
|
*/
|
2007-05-19 07:39:31 +00:00
|
|
|
unsigned char no_try_delta;
|
2005-06-25 21:42:43 +00:00
|
|
|
};
|
|
|
|
|
pack-objects: reuse data from existing packs.
When generating a new pack, notice if we have already needed
objects in existing packs. If an object is stored deltified,
and its base object is also what we are going to pack, then
reuse the existing deltified representation unconditionally,
bypassing all the expensive find_deltas() and try_deltas()
calls.
Also, notice if what we are going to write out exactly match
what is already in an existing pack (either deltified or just
compressed). In such a case, we can just copy it instead of
going through the usual uncompressing & recompressing cycle.
Without this patch, in linux-2.6 repository with about 1500
loose objects and a single mega pack:
$ git-rev-list --objects v2.6.16-rc3 >RL
$ wc -l RL
184141 RL
$ time git-pack-objects p <RL
Generating pack...
Done counting 184141 objects.
Packing 184141 objects....................
a1fc7b3e537fcb9b3c46b7505df859f0a11e79d2
real 12m4.323s
user 11m2.560s
sys 0m55.950s
With this patch, the same input:
$ time ../git.junio/git-pack-objects q <RL
Generating pack...
Done counting 184141 objects.
Packing 184141 objects.....................
a1fc7b3e537fcb9b3c46b7505df859f0a11e79d2
Total 184141, written 184141, reused 182441
real 1m2.608s
user 0m55.090s
sys 0m1.830s
Signed-off-by: Junio C Hamano <junkio@cox.net>
2006-02-16 01:34:29 +00:00
|
|
|
/*
|
2006-07-10 05:50:18 +00:00
|
|
|
* Objects we are going to pack are collected in objects array (dynamically
|
pack-objects: reuse data from existing packs.
When generating a new pack, notice if we have already needed
objects in existing packs. If an object is stored deltified,
and its base object is also what we are going to pack, then
reuse the existing deltified representation unconditionally,
bypassing all the expensive find_deltas() and try_deltas()
calls.
Also, notice if what we are going to write out exactly match
what is already in an existing pack (either deltified or just
compressed). In such a case, we can just copy it instead of
going through the usual uncompressing & recompressing cycle.
Without this patch, in linux-2.6 repository with about 1500
loose objects and a single mega pack:
$ git-rev-list --objects v2.6.16-rc3 >RL
$ wc -l RL
184141 RL
$ time git-pack-objects p <RL
Generating pack...
Done counting 184141 objects.
Packing 184141 objects....................
a1fc7b3e537fcb9b3c46b7505df859f0a11e79d2
real 12m4.323s
user 11m2.560s
sys 0m55.950s
With this patch, the same input:
$ time ../git.junio/git-pack-objects q <RL
Generating pack...
Done counting 184141 objects.
Packing 184141 objects.....................
a1fc7b3e537fcb9b3c46b7505df859f0a11e79d2
Total 184141, written 184141, reused 182441
real 1m2.608s
user 0m55.090s
sys 0m1.830s
Signed-off-by: Junio C Hamano <junkio@cox.net>
2006-02-16 01:34:29 +00:00
|
|
|
* expanded). nr_objects & nr_alloc controls this array. They are stored
|
|
|
|
* in the order we see -- typically rev-list --objects order that gives us
|
|
|
|
* nice "minimum seek" order.
|
|
|
|
*/
|
2007-04-20 02:28:02 +00:00
|
|
|
static struct object_entry *objects;
|
2007-11-02 03:43:24 +00:00
|
|
|
static struct pack_idx_entry **written_list;
|
2007-05-13 18:34:56 +00:00
|
|
|
static uint32_t nr_objects, nr_alloc, nr_result, nr_written;
|
pack-objects: reuse data from existing packs.
When generating a new pack, notice if we have already needed
objects in existing packs. If an object is stored deltified,
and its base object is also what we are going to pack, then
reuse the existing deltified representation unconditionally,
bypassing all the expensive find_deltas() and try_deltas()
calls.
Also, notice if what we are going to write out exactly match
what is already in an existing pack (either deltified or just
compressed). In such a case, we can just copy it instead of
going through the usual uncompressing & recompressing cycle.
Without this patch, in linux-2.6 repository with about 1500
loose objects and a single mega pack:
$ git-rev-list --objects v2.6.16-rc3 >RL
$ wc -l RL
184141 RL
$ time git-pack-objects p <RL
Generating pack...
Done counting 184141 objects.
Packing 184141 objects....................
a1fc7b3e537fcb9b3c46b7505df859f0a11e79d2
real 12m4.323s
user 11m2.560s
sys 0m55.950s
With this patch, the same input:
$ time ../git.junio/git-pack-objects q <RL
Generating pack...
Done counting 184141 objects.
Packing 184141 objects.....................
a1fc7b3e537fcb9b3c46b7505df859f0a11e79d2
Total 184141, written 184141, reused 182441
real 1m2.608s
user 0m55.090s
sys 0m1.830s
Signed-off-by: Junio C Hamano <junkio@cox.net>
2006-02-16 01:34:29 +00:00
|
|
|
|
2006-08-15 17:23:48 +00:00
|
|
|
static int non_empty;
|
2008-05-02 19:11:46 +00:00
|
|
|
static int reuse_delta = 1, reuse_object = 1;
|
2008-05-23 23:06:01 +00:00
|
|
|
static int keep_unreachable, unpack_unreachable, include_tag;
|
2006-08-15 17:23:48 +00:00
|
|
|
static int local;
|
|
|
|
static int incremental;
|
2008-11-12 17:59:04 +00:00
|
|
|
static int ignore_packed_keep;
|
2006-09-21 04:09:44 +00:00
|
|
|
static int allow_ofs_delta;
|
2007-05-13 18:34:56 +00:00
|
|
|
static const char *base_name;
|
2006-02-12 21:01:54 +00:00
|
|
|
static int progress = 1;
|
2006-07-23 05:50:30 +00:00
|
|
|
static int window = 10;
|
2008-02-05 14:25:04 +00:00
|
|
|
static uint32_t pack_size_limit, pack_size_limit_cfg;
|
2007-05-08 13:28:26 +00:00
|
|
|
static int depth = 50;
|
2008-12-11 20:36:47 +00:00
|
|
|
static int delta_search_threads;
|
2006-09-01 22:05:12 +00:00
|
|
|
static int pack_to_stdout;
|
2006-09-06 08:42:23 +00:00
|
|
|
static int num_preferred_base;
|
2007-10-30 18:57:32 +00:00
|
|
|
static struct progress *progress_state;
|
Custom compression levels for objects and packs
Add config variables pack.compression and core.loosecompression ,
and switch --compression=level to pack-objects.
Loose objects will be compressed using core.loosecompression if set,
else core.compression if set, else Z_BEST_SPEED.
Packed objects will be compressed using --compression=level if seen,
else pack.compression if set, else core.compression if set,
else Z_DEFAULT_COMPRESSION. This is the "pack compression level".
Loose objects added to a pack undeltified will be recompressed
to the pack compression level if it is unequal to the current
loose compression level by the preceding rules, or if the loose
object was written while core.legacyheaders = true. Newly
deltified loose objects are always compressed to the current
pack compression level.
Previously packed objects added to a pack are recompressed
to the current pack compression level exactly when their
deltification status changes, since the previous pack data
cannot be reused.
In either case, the --no-reuse-object switch from the first
patch below will always force recompression to the current pack
compression level, instead of assuming the pack compression level
hasn't changed and pack data can be reused when possible.
This applies on top of the following patches from Nicolas Pitre:
[PATCH] allow for undeltified objects not to be reused
[PATCH] make "repack -f" imply "pack-objects --no-reuse-object"
Signed-off-by: Dana L. How <danahow@gmail.com>
Signed-off-by: Junio C Hamano <junkio@cox.net>
2007-05-09 20:56:50 +00:00
|
|
|
static int pack_compression_level = Z_DEFAULT_COMPRESSION;
|
|
|
|
static int pack_compression_seen;
|
2005-06-25 21:42:43 +00:00
|
|
|
|
2007-05-28 21:20:58 +00:00
|
|
|
static unsigned long delta_cache_size = 0;
|
2009-08-05 20:55:07 +00:00
|
|
|
static unsigned long max_delta_cache_size = 256 * 1024 * 1024;
|
2007-05-28 21:20:59 +00:00
|
|
|
static unsigned long cache_max_small_delta_size = 1000;
|
2007-05-28 21:20:58 +00:00
|
|
|
|
2007-07-12 13:07:46 +00:00
|
|
|
static unsigned long window_memory_limit = 0;
|
|
|
|
|
pack-objects: reuse data from existing packs.
When generating a new pack, notice if we have already needed
objects in existing packs. If an object is stored deltified,
and its base object is also what we are going to pack, then
reuse the existing deltified representation unconditionally,
bypassing all the expensive find_deltas() and try_deltas()
calls.
Also, notice if what we are going to write out exactly match
what is already in an existing pack (either deltified or just
compressed). In such a case, we can just copy it instead of
going through the usual uncompressing & recompressing cycle.
Without this patch, in linux-2.6 repository with about 1500
loose objects and a single mega pack:
$ git-rev-list --objects v2.6.16-rc3 >RL
$ wc -l RL
184141 RL
$ time git-pack-objects p <RL
Generating pack...
Done counting 184141 objects.
Packing 184141 objects....................
a1fc7b3e537fcb9b3c46b7505df859f0a11e79d2
real 12m4.323s
user 11m2.560s
sys 0m55.950s
With this patch, the same input:
$ time ../git.junio/git-pack-objects q <RL
Generating pack...
Done counting 184141 objects.
Packing 184141 objects.....................
a1fc7b3e537fcb9b3c46b7505df859f0a11e79d2
Total 184141, written 184141, reused 182441
real 1m2.608s
user 0m55.090s
sys 0m1.830s
Signed-off-by: Junio C Hamano <junkio@cox.net>
2006-02-16 01:34:29 +00:00
|
|
|
/*
|
|
|
|
* The object names in objects array are hashed with this hashtable,
|
2007-04-20 02:28:02 +00:00
|
|
|
* to help looking up the entry by object name.
|
pack-objects: reuse data from existing packs.
When generating a new pack, notice if we have already needed
objects in existing packs. If an object is stored deltified,
and its base object is also what we are going to pack, then
reuse the existing deltified representation unconditionally,
bypassing all the expensive find_deltas() and try_deltas()
calls.
Also, notice if what we are going to write out exactly match
what is already in an existing pack (either deltified or just
compressed). In such a case, we can just copy it instead of
going through the usual uncompressing & recompressing cycle.
Without this patch, in linux-2.6 repository with about 1500
loose objects and a single mega pack:
$ git-rev-list --objects v2.6.16-rc3 >RL
$ wc -l RL
184141 RL
$ time git-pack-objects p <RL
Generating pack...
Done counting 184141 objects.
Packing 184141 objects....................
a1fc7b3e537fcb9b3c46b7505df859f0a11e79d2
real 12m4.323s
user 11m2.560s
sys 0m55.950s
With this patch, the same input:
$ time ../git.junio/git-pack-objects q <RL
Generating pack...
Done counting 184141 objects.
Packing 184141 objects.....................
a1fc7b3e537fcb9b3c46b7505df859f0a11e79d2
Total 184141, written 184141, reused 182441
real 1m2.608s
user 0m55.090s
sys 0m1.830s
Signed-off-by: Junio C Hamano <junkio@cox.net>
2006-02-16 01:34:29 +00:00
|
|
|
* This hashtable is built after all the objects are seen.
|
|
|
|
*/
|
2006-08-15 17:23:48 +00:00
|
|
|
static int *object_ix;
|
|
|
|
static int object_ix_hashsz;
|
pack-objects: reuse data from existing packs.
When generating a new pack, notice if we have already needed
objects in existing packs. If an object is stored deltified,
and its base object is also what we are going to pack, then
reuse the existing deltified representation unconditionally,
bypassing all the expensive find_deltas() and try_deltas()
calls.
Also, notice if what we are going to write out exactly match
what is already in an existing pack (either deltified or just
compressed). In such a case, we can just copy it instead of
going through the usual uncompressing & recompressing cycle.
Without this patch, in linux-2.6 repository with about 1500
loose objects and a single mega pack:
$ git-rev-list --objects v2.6.16-rc3 >RL
$ wc -l RL
184141 RL
$ time git-pack-objects p <RL
Generating pack...
Done counting 184141 objects.
Packing 184141 objects....................
a1fc7b3e537fcb9b3c46b7505df859f0a11e79d2
real 12m4.323s
user 11m2.560s
sys 0m55.950s
With this patch, the same input:
$ time ../git.junio/git-pack-objects q <RL
Generating pack...
Done counting 184141 objects.
Packing 184141 objects.....................
a1fc7b3e537fcb9b3c46b7505df859f0a11e79d2
Total 184141, written 184141, reused 182441
real 1m2.608s
user 0m55.090s
sys 0m1.830s
Signed-off-by: Junio C Hamano <junkio@cox.net>
2006-02-16 01:34:29 +00:00
|
|
|
|
|
|
|
/*
|
|
|
|
* stats
|
|
|
|
*/
|
2007-03-07 01:44:24 +00:00
|
|
|
static uint32_t written, written_delta;
|
|
|
|
static uint32_t reused, reused_delta;
|
pack-objects: reuse data from existing packs.
When generating a new pack, notice if we have already needed
objects in existing packs. If an object is stored deltified,
and its base object is also what we are going to pack, then
reuse the existing deltified representation unconditionally,
bypassing all the expensive find_deltas() and try_deltas()
calls.
Also, notice if what we are going to write out exactly match
what is already in an existing pack (either deltified or just
compressed). In such a case, we can just copy it instead of
going through the usual uncompressing & recompressing cycle.
Without this patch, in linux-2.6 repository with about 1500
loose objects and a single mega pack:
$ git-rev-list --objects v2.6.16-rc3 >RL
$ wc -l RL
184141 RL
$ time git-pack-objects p <RL
Generating pack...
Done counting 184141 objects.
Packing 184141 objects....................
a1fc7b3e537fcb9b3c46b7505df859f0a11e79d2
real 12m4.323s
user 11m2.560s
sys 0m55.950s
With this patch, the same input:
$ time ../git.junio/git-pack-objects q <RL
Generating pack...
Done counting 184141 objects.
Packing 184141 objects.....................
a1fc7b3e537fcb9b3c46b7505df859f0a11e79d2
Total 184141, written 184141, reused 182441
real 1m2.608s
user 0m55.090s
sys 0m1.830s
Signed-off-by: Junio C Hamano <junkio@cox.net>
2006-02-16 01:34:29 +00:00
|
|
|
|
2006-09-23 01:25:04 +00:00
|
|
|
|
2008-05-02 19:11:45 +00:00
|
|
|
static void *get_delta(struct object_entry *entry)
|
2005-06-25 21:42:43 +00:00
|
|
|
{
|
2008-05-02 19:11:45 +00:00
|
|
|
unsigned long size, base_size, delta_size;
|
|
|
|
void *buf, *base_buf, *delta_buf;
|
2007-02-26 19:55:59 +00:00
|
|
|
enum object_type type;
|
2005-06-25 21:42:43 +00:00
|
|
|
|
2008-05-02 19:11:45 +00:00
|
|
|
buf = read_sha1_file(entry->idx.sha1, &type, &size);
|
|
|
|
if (!buf)
|
|
|
|
die("unable to read %s", sha1_to_hex(entry->idx.sha1));
|
|
|
|
base_buf = read_sha1_file(entry->delta->idx.sha1, &type, &base_size);
|
|
|
|
if (!base_buf)
|
2007-06-01 19:18:05 +00:00
|
|
|
die("unable to read %s", sha1_to_hex(entry->delta->idx.sha1));
|
2008-05-02 19:11:45 +00:00
|
|
|
delta_buf = diff_delta(base_buf, base_size,
|
2005-06-29 06:49:56 +00:00
|
|
|
buf, size, &delta_size, 0);
|
2008-05-02 19:11:45 +00:00
|
|
|
if (!delta_buf || delta_size != entry->delta_size)
|
2007-06-07 07:04:01 +00:00
|
|
|
die("delta size changed");
|
2008-05-02 19:11:45 +00:00
|
|
|
free(buf);
|
|
|
|
free(base_buf);
|
2005-06-25 21:42:43 +00:00
|
|
|
return delta_buf;
|
|
|
|
}
|
|
|
|
|
2008-05-02 19:11:49 +00:00
|
|
|
static unsigned long do_compress(void **pptr, unsigned long size)
|
|
|
|
{
|
|
|
|
z_stream stream;
|
|
|
|
void *in, *out;
|
|
|
|
unsigned long maxsize;
|
|
|
|
|
|
|
|
memset(&stream, 0, sizeof(stream));
|
|
|
|
deflateInit(&stream, pack_compression_level);
|
|
|
|
maxsize = deflateBound(&stream, size);
|
|
|
|
|
|
|
|
in = *pptr;
|
|
|
|
out = xmalloc(maxsize);
|
|
|
|
*pptr = out;
|
|
|
|
|
|
|
|
stream.next_in = in;
|
|
|
|
stream.avail_in = size;
|
|
|
|
stream.next_out = out;
|
|
|
|
stream.avail_out = maxsize;
|
|
|
|
while (deflate(&stream, Z_FINISH) == Z_OK)
|
|
|
|
; /* nothing */
|
|
|
|
deflateEnd(&stream);
|
|
|
|
|
|
|
|
free(in);
|
|
|
|
return stream.total_out;
|
|
|
|
}
|
|
|
|
|
2005-06-28 21:21:02 +00:00
|
|
|
/*
|
|
|
|
* The per-object header is a pretty dense thing, which is
|
|
|
|
* - first byte: low four bits are "size", then three bits of "type",
|
|
|
|
* and the high bit is "size continues".
|
|
|
|
* - each byte afterwards: low seven bits are size continuation,
|
|
|
|
* with the high bit being "size continues"
|
|
|
|
*/
|
|
|
|
static int encode_header(enum object_type type, unsigned long size, unsigned char *hdr)
|
|
|
|
{
|
2005-06-29 05:15:57 +00:00
|
|
|
int n = 1;
|
2005-06-28 21:21:02 +00:00
|
|
|
unsigned char c;
|
|
|
|
|
2006-09-21 04:06:49 +00:00
|
|
|
if (type < OBJ_COMMIT || type > OBJ_REF_DELTA)
|
2005-06-28 21:21:02 +00:00
|
|
|
die("bad type %d", type);
|
|
|
|
|
2005-06-29 05:15:57 +00:00
|
|
|
c = (type << 4) | (size & 15);
|
|
|
|
size >>= 4;
|
|
|
|
while (size) {
|
2005-06-28 21:21:02 +00:00
|
|
|
*hdr++ = c | 0x80;
|
2005-06-29 05:15:57 +00:00
|
|
|
c = size & 0x7f;
|
|
|
|
size >>= 7;
|
|
|
|
n++;
|
2005-06-28 21:21:02 +00:00
|
|
|
}
|
|
|
|
*hdr = c;
|
|
|
|
return n;
|
|
|
|
}
|
|
|
|
|
2006-09-23 01:25:04 +00:00
|
|
|
/*
|
|
|
|
* we are going to reuse the existing object data as is. make
|
|
|
|
* sure it is not corrupt.
|
|
|
|
*/
|
2006-12-23 07:34:13 +00:00
|
|
|
static int check_pack_inflate(struct packed_git *p,
|
|
|
|
struct pack_window **w_curs,
|
2007-03-07 01:44:34 +00:00
|
|
|
off_t offset,
|
|
|
|
off_t len,
|
2006-12-23 07:34:13 +00:00
|
|
|
unsigned long expect)
|
|
|
|
{
|
|
|
|
z_stream stream;
|
|
|
|
unsigned char fakebuf[4096], *in;
|
|
|
|
int st;
|
|
|
|
|
|
|
|
memset(&stream, 0, sizeof(stream));
|
2009-01-08 03:54:47 +00:00
|
|
|
git_inflate_init(&stream);
|
2006-12-23 07:34:13 +00:00
|
|
|
do {
|
|
|
|
in = use_pack(p, w_curs, offset, &stream.avail_in);
|
|
|
|
stream.next_in = in;
|
|
|
|
stream.next_out = fakebuf;
|
|
|
|
stream.avail_out = sizeof(fakebuf);
|
2009-01-08 03:54:47 +00:00
|
|
|
st = git_inflate(&stream, Z_FINISH);
|
2006-12-23 07:34:13 +00:00
|
|
|
offset += stream.next_in - in;
|
|
|
|
} while (st == Z_OK || st == Z_BUF_ERROR);
|
2009-01-08 03:54:47 +00:00
|
|
|
git_inflate_end(&stream);
|
2006-12-23 07:34:13 +00:00
|
|
|
return (st == Z_STREAM_END &&
|
|
|
|
stream.total_out == expect &&
|
|
|
|
stream.total_in == len) ? 0 : -1;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void copy_pack_data(struct sha1file *f,
|
|
|
|
struct packed_git *p,
|
|
|
|
struct pack_window **w_curs,
|
2007-03-07 01:44:34 +00:00
|
|
|
off_t offset,
|
|
|
|
off_t len)
|
2006-12-23 07:34:13 +00:00
|
|
|
{
|
|
|
|
unsigned char *in;
|
|
|
|
unsigned int avail;
|
|
|
|
|
|
|
|
while (len) {
|
|
|
|
in = use_pack(p, w_curs, offset, &avail);
|
|
|
|
if (avail > len)
|
2007-03-07 01:44:34 +00:00
|
|
|
avail = (unsigned int)len;
|
2006-12-23 07:34:13 +00:00
|
|
|
sha1write(f, in, avail);
|
|
|
|
offset += avail;
|
|
|
|
len -= avail;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2007-04-09 05:06:30 +00:00
|
|
|
static unsigned long write_object(struct sha1file *f,
|
2007-05-13 19:06:18 +00:00
|
|
|
struct object_entry *entry,
|
|
|
|
off_t write_offset)
|
2005-06-25 21:42:43 +00:00
|
|
|
{
|
2008-05-02 19:11:49 +00:00
|
|
|
unsigned long size, limit, datalen;
|
pack-objects: reuse data from existing packs.
When generating a new pack, notice if we have already needed
objects in existing packs. If an object is stored deltified,
and its base object is also what we are going to pack, then
reuse the existing deltified representation unconditionally,
bypassing all the expensive find_deltas() and try_deltas()
calls.
Also, notice if what we are going to write out exactly match
what is already in an existing pack (either deltified or just
compressed). In such a case, we can just copy it instead of
going through the usual uncompressing & recompressing cycle.
Without this patch, in linux-2.6 repository with about 1500
loose objects and a single mega pack:
$ git-rev-list --objects v2.6.16-rc3 >RL
$ wc -l RL
184141 RL
$ time git-pack-objects p <RL
Generating pack...
Done counting 184141 objects.
Packing 184141 objects....................
a1fc7b3e537fcb9b3c46b7505df859f0a11e79d2
real 12m4.323s
user 11m2.560s
sys 0m55.950s
With this patch, the same input:
$ time ../git.junio/git-pack-objects q <RL
Generating pack...
Done counting 184141 objects.
Packing 184141 objects.....................
a1fc7b3e537fcb9b3c46b7505df859f0a11e79d2
Total 184141, written 184141, reused 182441
real 1m2.608s
user 0m55.090s
sys 0m1.830s
Signed-off-by: Junio C Hamano <junkio@cox.net>
2006-02-16 01:34:29 +00:00
|
|
|
void *buf;
|
2008-05-02 19:11:48 +00:00
|
|
|
unsigned char header[10], dheader[10];
|
2007-03-07 01:44:34 +00:00
|
|
|
unsigned hdrlen;
|
2008-05-02 19:11:48 +00:00
|
|
|
enum object_type type;
|
|
|
|
int usable_delta, to_reuse;
|
2005-06-25 21:42:43 +00:00
|
|
|
|
compute a CRC32 for each object as stored in a pack
The most important optimization for performance when repacking is the
ability to reuse data from a previous pack as is and bypass any delta
or even SHA1 computation by simply copying the raw data from one pack
to another directly.
The problem with this is that any data corruption within a copied object
would go unnoticed and the new (repacked) pack would be self-consistent
with its own checksum despite containing a corrupted object. This is a
real issue that already happened at least once in the past.
In some attempt to prevent this, we validate the copied data by inflating
it and making sure no error is signaled by zlib. But this is still not
perfect as a significant portion of a pack content is made of object
headers and references to delta base objects which are not deflated and
therefore not validated when repacking actually making the pack data reuse
still not as safe as it could be.
Of course a full SHA1 validation could be performed, but that implies
full data inflating and delta replaying which is extremely costly, which
cost the data reuse optimization was designed to avoid in the first place.
So the best solution to this is simply to store a CRC32 of the raw pack
data for each object in the pack index. This way any object in a pack can
be validated before being copied as is in another pack, including header
and any other non deflated data.
Why CRC32 instead of a faster checksum like Adler32? Quoting Wikipedia:
Jonathan Stone discovered in 2001 that Adler-32 has a weakness for very
short messages. He wrote "Briefly, the problem is that, for very short
packets, Adler32 is guaranteed to give poor coverage of the available
bits. Don't take my word for it, ask Mark Adler. :-)" The problem is
that sum A does not wrap for short messages. The maximum value of A for
a 128-byte message is 32640, which is below the value 65521 used by the
modulo operation. An extended explanation can be found in RFC 3309,
which mandates the use of CRC32 instead of Adler-32 for SCTP, the
Stream Control Transmission Protocol.
In the context of a GIT pack, we have lots of small objects, especially
deltas, which are likely to be quite small and in a size range for which
Adler32 is dimed not to be sufficient. Another advantage of CRC32 is the
possibility for recovery from certain types of small corruptions like
single bit errors which are the most probable type of corruptions.
OK what this patch does is to compute the CRC32 of each object written to
a pack within pack-objects. It is not written to the index yet and it is
obviously not validated when reusing pack data yet either.
Signed-off-by: Nicolas Pitre <nico@cam.org>
Signed-off-by: Junio C Hamano <junkio@cox.net>
2007-04-09 05:06:31 +00:00
|
|
|
if (!pack_to_stdout)
|
|
|
|
crc32_begin(f);
|
|
|
|
|
2008-05-02 19:11:48 +00:00
|
|
|
type = entry->type;
|
|
|
|
|
|
|
|
/* write limit if limited packsize and not first object */
|
2008-11-12 18:23:58 +00:00
|
|
|
if (!pack_size_limit || !nr_written)
|
|
|
|
limit = 0;
|
|
|
|
else if (pack_size_limit <= write_offset)
|
|
|
|
/*
|
|
|
|
* the earlier object did not fit the limit; avoid
|
|
|
|
* mistaking this with unlimited (i.e. limit = 0).
|
|
|
|
*/
|
|
|
|
limit = 1;
|
|
|
|
else
|
|
|
|
limit = pack_size_limit - write_offset;
|
2008-05-02 19:11:48 +00:00
|
|
|
|
|
|
|
if (!entry->delta)
|
|
|
|
usable_delta = 0; /* no delta */
|
|
|
|
else if (!pack_size_limit)
|
|
|
|
usable_delta = 1; /* unlimited packfile */
|
|
|
|
else if (entry->delta->idx.offset == (off_t)-1)
|
|
|
|
usable_delta = 0; /* base was written to another pack */
|
|
|
|
else if (entry->delta->idx.offset)
|
|
|
|
usable_delta = 1; /* base already exists in this pack */
|
|
|
|
else
|
|
|
|
usable_delta = 0; /* base could end up in another pack */
|
|
|
|
|
2008-05-02 19:11:46 +00:00
|
|
|
if (!reuse_object)
|
2007-05-09 16:31:28 +00:00
|
|
|
to_reuse = 0; /* explicit */
|
|
|
|
else if (!entry->in_pack)
|
pack-objects: finishing touches.
This introduces --no-reuse-delta option to disable reusing of
existing delta, which is a large part of the optimization
introduced by this series. This may become necessary if
repeated repacking makes delta chain too long. With this, the
output of the command becomes identical to that of the older
implementation. But the performance suffers greatly.
It still allows reusing non-deltified representations; there is
no point uncompressing and recompressing the whole text.
It also adds a couple more statistics output, while squelching
it under -q flag, which the last round forgot to do.
$ time old-git-pack-objects --stdout >/dev/null <RL
Generating pack...
Done counting 184141 objects.
Packing 184141 objects....................
real 12m8.530s user 11m1.450s sys 0m57.920s
$ time git-pack-objects --stdout >/dev/null <RL
Generating pack...
Done counting 184141 objects.
Packing 184141 objects.....................
Total 184141, written 184141 (delta 138297), reused 178833 (delta 134081)
real 0m59.549s user 0m56.670s sys 0m2.400s
$ time git-pack-objects --stdout --no-reuse-delta >/dev/null <RL
Generating pack...
Done counting 184141 objects.
Packing 184141 objects.....................
Total 184141, written 184141 (delta 134833), reused 47904 (delta 0)
real 11m13.830s user 9m45.240s sys 0m44.330s
There is one remaining issue when --no-reuse-delta option is not
used. It can create delta chains that are deeper than specified.
A<--B<--C<--D E F G
Suppose we have a delta chain A to D (A is stored in full either
in a pack or as a loose object. B is depth1 delta relative to A,
C is depth2 delta relative to B...) with loose objects E, F, G.
And we are going to pack all of them.
B, C and D are left as delta against A, B and C respectively.
So A, E, F, and G are examined for deltification, and let's say
we decided to keep E expanded, and store the rest as deltas like
this:
E<--F<--G<--A
Oops. We ended up making D a bit too deep, didn't we? B, C and
D form a chain on top of A!
This is because we did not know what the final depth of A would
be, when we checked objects and decided to keep the existing
delta. Unfortunately, deferring the decision until just before
the deltification is not an option. To be able to make B, C,
and D candidates for deltification with the rest, we need to
know the type and final unexpanded size of them, but the major
part of the optimization comes from the fact that we do not read
the delta data to do so -- getting the final size is quite an
expensive operation.
To prevent this from happening, we should keep A from being
deltified. But how would we tell that, cheaply?
To do this most precisely, after check_object() runs, each
object that is used as the base object of some existing delta
needs to be marked with the maximum depth of the objects we
decided to keep deltified (in this case, D is depth 3 relative
to A, so if no other delta chain that is longer than 3 based on
A exists, mark A with 3). Then when attempting to deltify A, we
would take that number into account to see if the final delta
chain that leads to D becomes too deep.
However, this is a bit cumbersome to compute, so we would cheat
and reduce the maximum depth for A arbitrarily to depth/4 in
this implementation.
Signed-off-by: Junio C Hamano <junkio@cox.net>
2006-02-16 19:55:51 +00:00
|
|
|
to_reuse = 0; /* can't reuse what we don't have */
|
2008-05-02 19:11:48 +00:00
|
|
|
else if (type == OBJ_REF_DELTA || type == OBJ_OFS_DELTA)
|
2007-05-13 19:06:18 +00:00
|
|
|
/* check_object() decided it for us ... */
|
|
|
|
to_reuse = usable_delta;
|
|
|
|
/* ... but pack split may override that */
|
2008-05-02 19:11:48 +00:00
|
|
|
else if (type != entry->in_pack_type)
|
pack-objects: finishing touches.
This introduces --no-reuse-delta option to disable reusing of
existing delta, which is a large part of the optimization
introduced by this series. This may become necessary if
repeated repacking makes delta chain too long. With this, the
output of the command becomes identical to that of the older
implementation. But the performance suffers greatly.
It still allows reusing non-deltified representations; there is
no point uncompressing and recompressing the whole text.
It also adds a couple more statistics output, while squelching
it under -q flag, which the last round forgot to do.
$ time old-git-pack-objects --stdout >/dev/null <RL
Generating pack...
Done counting 184141 objects.
Packing 184141 objects....................
real 12m8.530s user 11m1.450s sys 0m57.920s
$ time git-pack-objects --stdout >/dev/null <RL
Generating pack...
Done counting 184141 objects.
Packing 184141 objects.....................
Total 184141, written 184141 (delta 138297), reused 178833 (delta 134081)
real 0m59.549s user 0m56.670s sys 0m2.400s
$ time git-pack-objects --stdout --no-reuse-delta >/dev/null <RL
Generating pack...
Done counting 184141 objects.
Packing 184141 objects.....................
Total 184141, written 184141 (delta 134833), reused 47904 (delta 0)
real 11m13.830s user 9m45.240s sys 0m44.330s
There is one remaining issue when --no-reuse-delta option is not
used. It can create delta chains that are deeper than specified.
A<--B<--C<--D E F G
Suppose we have a delta chain A to D (A is stored in full either
in a pack or as a loose object. B is depth1 delta relative to A,
C is depth2 delta relative to B...) with loose objects E, F, G.
And we are going to pack all of them.
B, C and D are left as delta against A, B and C respectively.
So A, E, F, and G are examined for deltification, and let's say
we decided to keep E expanded, and store the rest as deltas like
this:
E<--F<--G<--A
Oops. We ended up making D a bit too deep, didn't we? B, C and
D form a chain on top of A!
This is because we did not know what the final depth of A would
be, when we checked objects and decided to keep the existing
delta. Unfortunately, deferring the decision until just before
the deltification is not an option. To be able to make B, C,
and D candidates for deltification with the rest, we need to
know the type and final unexpanded size of them, but the major
part of the optimization comes from the fact that we do not read
the delta data to do so -- getting the final size is quite an
expensive operation.
To prevent this from happening, we should keep A from being
deltified. But how would we tell that, cheaply?
To do this most precisely, after check_object() runs, each
object that is used as the base object of some existing delta
needs to be marked with the maximum depth of the objects we
decided to keep deltified (in this case, D is depth 3 relative
to A, so if no other delta chain that is longer than 3 based on
A exists, mark A with 3). Then when attempting to deltify A, we
would take that number into account to see if the final delta
chain that leads to D becomes too deep.
However, this is a bit cumbersome to compute, so we would cheat
and reduce the maximum depth for A arbitrarily to depth/4 in
this implementation.
Signed-off-by: Junio C Hamano <junkio@cox.net>
2006-02-16 19:55:51 +00:00
|
|
|
to_reuse = 0; /* pack has delta which is unusable */
|
|
|
|
else if (entry->delta)
|
|
|
|
to_reuse = 0; /* we want to pack afresh */
|
|
|
|
else
|
|
|
|
to_reuse = 1; /* we have it in-pack undeltified,
|
|
|
|
* and we do not need to deltify it.
|
|
|
|
*/
|
|
|
|
|
2006-09-01 22:05:12 +00:00
|
|
|
if (!to_reuse) {
|
2008-10-29 23:02:50 +00:00
|
|
|
no_reuse:
|
2007-05-31 01:43:12 +00:00
|
|
|
if (!usable_delta) {
|
2008-05-02 19:11:48 +00:00
|
|
|
buf = read_sha1_file(entry->idx.sha1, &type, &size);
|
2007-05-31 01:43:12 +00:00
|
|
|
if (!buf)
|
2007-06-01 19:18:05 +00:00
|
|
|
die("unable to read %s", sha1_to_hex(entry->idx.sha1));
|
2008-05-02 19:11:50 +00:00
|
|
|
/*
|
|
|
|
* make sure no cached delta data remains from a
|
2009-04-17 18:13:30 +00:00
|
|
|
* previous attempt before a pack split occurred.
|
2008-05-02 19:11:50 +00:00
|
|
|
*/
|
|
|
|
free(entry->delta_data);
|
|
|
|
entry->delta_data = NULL;
|
|
|
|
entry->z_delta_size = 0;
|
2007-05-31 01:43:12 +00:00
|
|
|
} else if (entry->delta_data) {
|
pack-objects: reuse data from existing packs.
When generating a new pack, notice if we have already needed
objects in existing packs. If an object is stored deltified,
and its base object is also what we are going to pack, then
reuse the existing deltified representation unconditionally,
bypassing all the expensive find_deltas() and try_deltas()
calls.
Also, notice if what we are going to write out exactly match
what is already in an existing pack (either deltified or just
compressed). In such a case, we can just copy it instead of
going through the usual uncompressing & recompressing cycle.
Without this patch, in linux-2.6 repository with about 1500
loose objects and a single mega pack:
$ git-rev-list --objects v2.6.16-rc3 >RL
$ wc -l RL
184141 RL
$ time git-pack-objects p <RL
Generating pack...
Done counting 184141 objects.
Packing 184141 objects....................
a1fc7b3e537fcb9b3c46b7505df859f0a11e79d2
real 12m4.323s
user 11m2.560s
sys 0m55.950s
With this patch, the same input:
$ time ../git.junio/git-pack-objects q <RL
Generating pack...
Done counting 184141 objects.
Packing 184141 objects.....................
a1fc7b3e537fcb9b3c46b7505df859f0a11e79d2
Total 184141, written 184141, reused 182441
real 1m2.608s
user 0m55.090s
sys 0m1.830s
Signed-off-by: Junio C Hamano <junkio@cox.net>
2006-02-16 01:34:29 +00:00
|
|
|
size = entry->delta_size;
|
2007-05-31 01:43:12 +00:00
|
|
|
buf = entry->delta_data;
|
|
|
|
entry->delta_data = NULL;
|
2008-05-02 19:11:48 +00:00
|
|
|
type = (allow_ofs_delta && entry->delta->idx.offset) ?
|
2006-09-21 04:09:44 +00:00
|
|
|
OBJ_OFS_DELTA : OBJ_REF_DELTA;
|
2007-05-13 19:06:18 +00:00
|
|
|
} else {
|
2008-05-02 19:11:45 +00:00
|
|
|
buf = get_delta(entry);
|
2007-05-31 01:43:12 +00:00
|
|
|
size = entry->delta_size;
|
2008-05-02 19:11:48 +00:00
|
|
|
type = (allow_ofs_delta && entry->delta->idx.offset) ?
|
2007-05-31 01:43:12 +00:00
|
|
|
OBJ_OFS_DELTA : OBJ_REF_DELTA;
|
pack-objects: reuse data from existing packs.
When generating a new pack, notice if we have already needed
objects in existing packs. If an object is stored deltified,
and its base object is also what we are going to pack, then
reuse the existing deltified representation unconditionally,
bypassing all the expensive find_deltas() and try_deltas()
calls.
Also, notice if what we are going to write out exactly match
what is already in an existing pack (either deltified or just
compressed). In such a case, we can just copy it instead of
going through the usual uncompressing & recompressing cycle.
Without this patch, in linux-2.6 repository with about 1500
loose objects and a single mega pack:
$ git-rev-list --objects v2.6.16-rc3 >RL
$ wc -l RL
184141 RL
$ time git-pack-objects p <RL
Generating pack...
Done counting 184141 objects.
Packing 184141 objects....................
a1fc7b3e537fcb9b3c46b7505df859f0a11e79d2
real 12m4.323s
user 11m2.560s
sys 0m55.950s
With this patch, the same input:
$ time ../git.junio/git-pack-objects q <RL
Generating pack...
Done counting 184141 objects.
Packing 184141 objects.....................
a1fc7b3e537fcb9b3c46b7505df859f0a11e79d2
Total 184141, written 184141, reused 182441
real 1m2.608s
user 0m55.090s
sys 0m1.830s
Signed-off-by: Junio C Hamano <junkio@cox.net>
2006-02-16 01:34:29 +00:00
|
|
|
}
|
2008-05-02 19:11:50 +00:00
|
|
|
|
|
|
|
if (entry->z_delta_size)
|
|
|
|
datalen = entry->z_delta_size;
|
|
|
|
else
|
|
|
|
datalen = do_compress(&buf, size);
|
2008-01-11 07:26:09 +00:00
|
|
|
|
pack-objects: reuse data from existing packs.
When generating a new pack, notice if we have already needed
objects in existing packs. If an object is stored deltified,
and its base object is also what we are going to pack, then
reuse the existing deltified representation unconditionally,
bypassing all the expensive find_deltas() and try_deltas()
calls.
Also, notice if what we are going to write out exactly match
what is already in an existing pack (either deltified or just
compressed). In such a case, we can just copy it instead of
going through the usual uncompressing & recompressing cycle.
Without this patch, in linux-2.6 repository with about 1500
loose objects and a single mega pack:
$ git-rev-list --objects v2.6.16-rc3 >RL
$ wc -l RL
184141 RL
$ time git-pack-objects p <RL
Generating pack...
Done counting 184141 objects.
Packing 184141 objects....................
a1fc7b3e537fcb9b3c46b7505df859f0a11e79d2
real 12m4.323s
user 11m2.560s
sys 0m55.950s
With this patch, the same input:
$ time ../git.junio/git-pack-objects q <RL
Generating pack...
Done counting 184141 objects.
Packing 184141 objects.....................
a1fc7b3e537fcb9b3c46b7505df859f0a11e79d2
Total 184141, written 184141, reused 182441
real 1m2.608s
user 0m55.090s
sys 0m1.830s
Signed-off-by: Junio C Hamano <junkio@cox.net>
2006-02-16 01:34:29 +00:00
|
|
|
/*
|
|
|
|
* The object header is a byte of 'type' followed by zero or
|
2006-09-21 04:09:44 +00:00
|
|
|
* more bytes of length.
|
pack-objects: reuse data from existing packs.
When generating a new pack, notice if we have already needed
objects in existing packs. If an object is stored deltified,
and its base object is also what we are going to pack, then
reuse the existing deltified representation unconditionally,
bypassing all the expensive find_deltas() and try_deltas()
calls.
Also, notice if what we are going to write out exactly match
what is already in an existing pack (either deltified or just
compressed). In such a case, we can just copy it instead of
going through the usual uncompressing & recompressing cycle.
Without this patch, in linux-2.6 repository with about 1500
loose objects and a single mega pack:
$ git-rev-list --objects v2.6.16-rc3 >RL
$ wc -l RL
184141 RL
$ time git-pack-objects p <RL
Generating pack...
Done counting 184141 objects.
Packing 184141 objects....................
a1fc7b3e537fcb9b3c46b7505df859f0a11e79d2
real 12m4.323s
user 11m2.560s
sys 0m55.950s
With this patch, the same input:
$ time ../git.junio/git-pack-objects q <RL
Generating pack...
Done counting 184141 objects.
Packing 184141 objects.....................
a1fc7b3e537fcb9b3c46b7505df859f0a11e79d2
Total 184141, written 184141, reused 182441
real 1m2.608s
user 0m55.090s
sys 0m1.830s
Signed-off-by: Junio C Hamano <junkio@cox.net>
2006-02-16 01:34:29 +00:00
|
|
|
*/
|
2008-05-02 19:11:48 +00:00
|
|
|
hdrlen = encode_header(type, size, header);
|
pack-objects: reuse data from existing packs.
When generating a new pack, notice if we have already needed
objects in existing packs. If an object is stored deltified,
and its base object is also what we are going to pack, then
reuse the existing deltified representation unconditionally,
bypassing all the expensive find_deltas() and try_deltas()
calls.
Also, notice if what we are going to write out exactly match
what is already in an existing pack (either deltified or just
compressed). In such a case, we can just copy it instead of
going through the usual uncompressing & recompressing cycle.
Without this patch, in linux-2.6 repository with about 1500
loose objects and a single mega pack:
$ git-rev-list --objects v2.6.16-rc3 >RL
$ wc -l RL
184141 RL
$ time git-pack-objects p <RL
Generating pack...
Done counting 184141 objects.
Packing 184141 objects....................
a1fc7b3e537fcb9b3c46b7505df859f0a11e79d2
real 12m4.323s
user 11m2.560s
sys 0m55.950s
With this patch, the same input:
$ time ../git.junio/git-pack-objects q <RL
Generating pack...
Done counting 184141 objects.
Packing 184141 objects.....................
a1fc7b3e537fcb9b3c46b7505df859f0a11e79d2
Total 184141, written 184141, reused 182441
real 1m2.608s
user 0m55.090s
sys 0m1.830s
Signed-off-by: Junio C Hamano <junkio@cox.net>
2006-02-16 01:34:29 +00:00
|
|
|
|
2008-05-02 19:11:48 +00:00
|
|
|
if (type == OBJ_OFS_DELTA) {
|
2006-09-21 04:09:44 +00:00
|
|
|
/*
|
|
|
|
* Deltas with relative base contain an additional
|
|
|
|
* encoding of the relative offset for the delta
|
|
|
|
* base from this object's position in the pack.
|
|
|
|
*/
|
2007-06-01 19:18:05 +00:00
|
|
|
off_t ofs = entry->idx.offset - entry->delta->idx.offset;
|
2007-05-13 19:06:18 +00:00
|
|
|
unsigned pos = sizeof(dheader) - 1;
|
|
|
|
dheader[pos] = ofs & 127;
|
2006-09-21 04:09:44 +00:00
|
|
|
while (ofs >>= 7)
|
2007-05-13 19:06:18 +00:00
|
|
|
dheader[--pos] = 128 | (--ofs & 127);
|
|
|
|
if (limit && hdrlen + sizeof(dheader) - pos + datalen + 20 >= limit) {
|
|
|
|
free(buf);
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
sha1write(f, header, hdrlen);
|
|
|
|
sha1write(f, dheader + pos, sizeof(dheader) - pos);
|
|
|
|
hdrlen += sizeof(dheader) - pos;
|
2008-05-02 19:11:48 +00:00
|
|
|
} else if (type == OBJ_REF_DELTA) {
|
2006-09-21 04:09:44 +00:00
|
|
|
/*
|
|
|
|
* Deltas with a base reference contain
|
|
|
|
* an additional 20 bytes for the base sha1.
|
|
|
|
*/
|
2007-05-13 19:06:18 +00:00
|
|
|
if (limit && hdrlen + 20 + datalen + 20 >= limit) {
|
|
|
|
free(buf);
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
sha1write(f, header, hdrlen);
|
2007-06-01 19:18:05 +00:00
|
|
|
sha1write(f, entry->delta->idx.sha1, 20);
|
pack-objects: reuse data from existing packs.
When generating a new pack, notice if we have already needed
objects in existing packs. If an object is stored deltified,
and its base object is also what we are going to pack, then
reuse the existing deltified representation unconditionally,
bypassing all the expensive find_deltas() and try_deltas()
calls.
Also, notice if what we are going to write out exactly match
what is already in an existing pack (either deltified or just
compressed). In such a case, we can just copy it instead of
going through the usual uncompressing & recompressing cycle.
Without this patch, in linux-2.6 repository with about 1500
loose objects and a single mega pack:
$ git-rev-list --objects v2.6.16-rc3 >RL
$ wc -l RL
184141 RL
$ time git-pack-objects p <RL
Generating pack...
Done counting 184141 objects.
Packing 184141 objects....................
a1fc7b3e537fcb9b3c46b7505df859f0a11e79d2
real 12m4.323s
user 11m2.560s
sys 0m55.950s
With this patch, the same input:
$ time ../git.junio/git-pack-objects q <RL
Generating pack...
Done counting 184141 objects.
Packing 184141 objects.....................
a1fc7b3e537fcb9b3c46b7505df859f0a11e79d2
Total 184141, written 184141, reused 182441
real 1m2.608s
user 0m55.090s
sys 0m1.830s
Signed-off-by: Junio C Hamano <junkio@cox.net>
2006-02-16 01:34:29 +00:00
|
|
|
hdrlen += 20;
|
2007-05-13 19:06:18 +00:00
|
|
|
} else {
|
|
|
|
if (limit && hdrlen + datalen + 20 >= limit) {
|
|
|
|
free(buf);
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
sha1write(f, header, hdrlen);
|
pack-objects: reuse data from existing packs.
When generating a new pack, notice if we have already needed
objects in existing packs. If an object is stored deltified,
and its base object is also what we are going to pack, then
reuse the existing deltified representation unconditionally,
bypassing all the expensive find_deltas() and try_deltas()
calls.
Also, notice if what we are going to write out exactly match
what is already in an existing pack (either deltified or just
compressed). In such a case, we can just copy it instead of
going through the usual uncompressing & recompressing cycle.
Without this patch, in linux-2.6 repository with about 1500
loose objects and a single mega pack:
$ git-rev-list --objects v2.6.16-rc3 >RL
$ wc -l RL
184141 RL
$ time git-pack-objects p <RL
Generating pack...
Done counting 184141 objects.
Packing 184141 objects....................
a1fc7b3e537fcb9b3c46b7505df859f0a11e79d2
real 12m4.323s
user 11m2.560s
sys 0m55.950s
With this patch, the same input:
$ time ../git.junio/git-pack-objects q <RL
Generating pack...
Done counting 184141 objects.
Packing 184141 objects.....................
a1fc7b3e537fcb9b3c46b7505df859f0a11e79d2
Total 184141, written 184141, reused 182441
real 1m2.608s
user 0m55.090s
sys 0m1.830s
Signed-off-by: Junio C Hamano <junkio@cox.net>
2006-02-16 01:34:29 +00:00
|
|
|
}
|
2008-05-02 19:11:49 +00:00
|
|
|
sha1write(f, buf, datalen);
|
pack-objects: reuse data from existing packs.
When generating a new pack, notice if we have already needed
objects in existing packs. If an object is stored deltified,
and its base object is also what we are going to pack, then
reuse the existing deltified representation unconditionally,
bypassing all the expensive find_deltas() and try_deltas()
calls.
Also, notice if what we are going to write out exactly match
what is already in an existing pack (either deltified or just
compressed). In such a case, we can just copy it instead of
going through the usual uncompressing & recompressing cycle.
Without this patch, in linux-2.6 repository with about 1500
loose objects and a single mega pack:
$ git-rev-list --objects v2.6.16-rc3 >RL
$ wc -l RL
184141 RL
$ time git-pack-objects p <RL
Generating pack...
Done counting 184141 objects.
Packing 184141 objects....................
a1fc7b3e537fcb9b3c46b7505df859f0a11e79d2
real 12m4.323s
user 11m2.560s
sys 0m55.950s
With this patch, the same input:
$ time ../git.junio/git-pack-objects q <RL
Generating pack...
Done counting 184141 objects.
Packing 184141 objects.....................
a1fc7b3e537fcb9b3c46b7505df859f0a11e79d2
Total 184141, written 184141, reused 182441
real 1m2.608s
user 0m55.090s
sys 0m1.830s
Signed-off-by: Junio C Hamano <junkio@cox.net>
2006-02-16 01:34:29 +00:00
|
|
|
free(buf);
|
2005-06-25 21:42:43 +00:00
|
|
|
}
|
pack-objects: reuse data from existing packs.
When generating a new pack, notice if we have already needed
objects in existing packs. If an object is stored deltified,
and its base object is also what we are going to pack, then
reuse the existing deltified representation unconditionally,
bypassing all the expensive find_deltas() and try_deltas()
calls.
Also, notice if what we are going to write out exactly match
what is already in an existing pack (either deltified or just
compressed). In such a case, we can just copy it instead of
going through the usual uncompressing & recompressing cycle.
Without this patch, in linux-2.6 repository with about 1500
loose objects and a single mega pack:
$ git-rev-list --objects v2.6.16-rc3 >RL
$ wc -l RL
184141 RL
$ time git-pack-objects p <RL
Generating pack...
Done counting 184141 objects.
Packing 184141 objects....................
a1fc7b3e537fcb9b3c46b7505df859f0a11e79d2
real 12m4.323s
user 11m2.560s
sys 0m55.950s
With this patch, the same input:
$ time ../git.junio/git-pack-objects q <RL
Generating pack...
Done counting 184141 objects.
Packing 184141 objects.....................
a1fc7b3e537fcb9b3c46b7505df859f0a11e79d2
Total 184141, written 184141, reused 182441
real 1m2.608s
user 0m55.090s
sys 0m1.830s
Signed-off-by: Junio C Hamano <junkio@cox.net>
2006-02-16 01:34:29 +00:00
|
|
|
else {
|
|
|
|
struct packed_git *p = entry->in_pack;
|
2006-12-23 07:34:08 +00:00
|
|
|
struct pack_window *w_curs = NULL;
|
2007-04-10 04:15:41 +00:00
|
|
|
struct revindex_entry *revidx;
|
2007-03-07 01:44:34 +00:00
|
|
|
off_t offset;
|
pack-objects: reuse data from existing packs.
When generating a new pack, notice if we have already needed
objects in existing packs. If an object is stored deltified,
and its base object is also what we are going to pack, then
reuse the existing deltified representation unconditionally,
bypassing all the expensive find_deltas() and try_deltas()
calls.
Also, notice if what we are going to write out exactly match
what is already in an existing pack (either deltified or just
compressed). In such a case, we can just copy it instead of
going through the usual uncompressing & recompressing cycle.
Without this patch, in linux-2.6 repository with about 1500
loose objects and a single mega pack:
$ git-rev-list --objects v2.6.16-rc3 >RL
$ wc -l RL
184141 RL
$ time git-pack-objects p <RL
Generating pack...
Done counting 184141 objects.
Packing 184141 objects....................
a1fc7b3e537fcb9b3c46b7505df859f0a11e79d2
real 12m4.323s
user 11m2.560s
sys 0m55.950s
With this patch, the same input:
$ time ../git.junio/git-pack-objects q <RL
Generating pack...
Done counting 184141 objects.
Packing 184141 objects.....................
a1fc7b3e537fcb9b3c46b7505df859f0a11e79d2
Total 184141, written 184141, reused 182441
real 1m2.608s
user 0m55.090s
sys 0m1.830s
Signed-off-by: Junio C Hamano <junkio@cox.net>
2006-02-16 01:34:29 +00:00
|
|
|
|
2008-10-29 23:02:50 +00:00
|
|
|
if (entry->delta)
|
2008-05-02 19:11:48 +00:00
|
|
|
type = (allow_ofs_delta && entry->delta->idx.offset) ?
|
2006-09-23 01:25:04 +00:00
|
|
|
OBJ_OFS_DELTA : OBJ_REF_DELTA;
|
2008-05-02 19:11:48 +00:00
|
|
|
hdrlen = encode_header(type, entry->size, header);
|
2008-10-29 23:02:50 +00:00
|
|
|
|
2007-04-10 04:15:41 +00:00
|
|
|
offset = entry->in_pack_offset;
|
2008-02-28 05:25:17 +00:00
|
|
|
revidx = find_pack_revindex(p, offset);
|
2007-04-10 04:15:41 +00:00
|
|
|
datalen = revidx[1].offset - offset;
|
|
|
|
if (!pack_to_stdout && p->index_version > 1 &&
|
2008-10-29 23:02:50 +00:00
|
|
|
check_pack_crc(p, &w_curs, offset, datalen, revidx->nr)) {
|
|
|
|
error("bad packed object CRC for %s", sha1_to_hex(entry->idx.sha1));
|
|
|
|
unuse_pack(&w_curs);
|
|
|
|
goto no_reuse;
|
|
|
|
}
|
|
|
|
|
2007-04-10 04:15:41 +00:00
|
|
|
offset += entry->in_pack_header_size;
|
|
|
|
datalen -= entry->in_pack_header_size;
|
2008-10-29 23:02:50 +00:00
|
|
|
if (!pack_to_stdout && p->index_version == 1 &&
|
|
|
|
check_pack_inflate(p, &w_curs, offset, datalen, entry->size)) {
|
|
|
|
error("corrupt packed object for %s", sha1_to_hex(entry->idx.sha1));
|
|
|
|
unuse_pack(&w_curs);
|
|
|
|
goto no_reuse;
|
|
|
|
}
|
|
|
|
|
2008-05-02 19:11:48 +00:00
|
|
|
if (type == OBJ_OFS_DELTA) {
|
2007-06-01 19:18:05 +00:00
|
|
|
off_t ofs = entry->idx.offset - entry->delta->idx.offset;
|
2007-05-13 19:06:18 +00:00
|
|
|
unsigned pos = sizeof(dheader) - 1;
|
|
|
|
dheader[pos] = ofs & 127;
|
|
|
|
while (ofs >>= 7)
|
|
|
|
dheader[--pos] = 128 | (--ofs & 127);
|
2008-10-29 23:02:52 +00:00
|
|
|
if (limit && hdrlen + sizeof(dheader) - pos + datalen + 20 >= limit) {
|
|
|
|
unuse_pack(&w_curs);
|
2007-05-13 19:06:18 +00:00
|
|
|
return 0;
|
2008-10-29 23:02:52 +00:00
|
|
|
}
|
2007-05-13 19:06:18 +00:00
|
|
|
sha1write(f, header, hdrlen);
|
|
|
|
sha1write(f, dheader + pos, sizeof(dheader) - pos);
|
|
|
|
hdrlen += sizeof(dheader) - pos;
|
2008-10-29 23:02:50 +00:00
|
|
|
reused_delta++;
|
2008-05-02 19:11:48 +00:00
|
|
|
} else if (type == OBJ_REF_DELTA) {
|
2008-10-29 23:02:52 +00:00
|
|
|
if (limit && hdrlen + 20 + datalen + 20 >= limit) {
|
|
|
|
unuse_pack(&w_curs);
|
2007-05-13 19:06:18 +00:00
|
|
|
return 0;
|
2008-10-29 23:02:52 +00:00
|
|
|
}
|
2007-05-13 19:06:18 +00:00
|
|
|
sha1write(f, header, hdrlen);
|
2007-06-01 19:18:05 +00:00
|
|
|
sha1write(f, entry->delta->idx.sha1, 20);
|
2007-05-13 19:06:18 +00:00
|
|
|
hdrlen += 20;
|
2008-10-29 23:02:50 +00:00
|
|
|
reused_delta++;
|
2007-05-13 19:06:18 +00:00
|
|
|
} else {
|
2008-10-29 23:02:52 +00:00
|
|
|
if (limit && hdrlen + datalen + 20 >= limit) {
|
|
|
|
unuse_pack(&w_curs);
|
2007-05-13 19:06:18 +00:00
|
|
|
return 0;
|
2008-10-29 23:02:52 +00:00
|
|
|
}
|
2007-05-13 19:06:18 +00:00
|
|
|
sha1write(f, header, hdrlen);
|
|
|
|
}
|
2006-12-23 07:34:13 +00:00
|
|
|
copy_pack_data(f, p, &w_curs, offset, datalen);
|
2006-12-23 07:34:08 +00:00
|
|
|
unuse_pack(&w_curs);
|
pack-objects: reuse data from existing packs.
When generating a new pack, notice if we have already needed
objects in existing packs. If an object is stored deltified,
and its base object is also what we are going to pack, then
reuse the existing deltified representation unconditionally,
bypassing all the expensive find_deltas() and try_deltas()
calls.
Also, notice if what we are going to write out exactly match
what is already in an existing pack (either deltified or just
compressed). In such a case, we can just copy it instead of
going through the usual uncompressing & recompressing cycle.
Without this patch, in linux-2.6 repository with about 1500
loose objects and a single mega pack:
$ git-rev-list --objects v2.6.16-rc3 >RL
$ wc -l RL
184141 RL
$ time git-pack-objects p <RL
Generating pack...
Done counting 184141 objects.
Packing 184141 objects....................
a1fc7b3e537fcb9b3c46b7505df859f0a11e79d2
real 12m4.323s
user 11m2.560s
sys 0m55.950s
With this patch, the same input:
$ time ../git.junio/git-pack-objects q <RL
Generating pack...
Done counting 184141 objects.
Packing 184141 objects.....................
a1fc7b3e537fcb9b3c46b7505df859f0a11e79d2
Total 184141, written 184141, reused 182441
real 1m2.608s
user 0m55.090s
sys 0m1.830s
Signed-off-by: Junio C Hamano <junkio@cox.net>
2006-02-16 01:34:29 +00:00
|
|
|
reused++;
|
2005-06-28 21:21:02 +00:00
|
|
|
}
|
2007-05-13 19:06:18 +00:00
|
|
|
if (usable_delta)
|
pack-objects: finishing touches.
This introduces --no-reuse-delta option to disable reusing of
existing delta, which is a large part of the optimization
introduced by this series. This may become necessary if
repeated repacking makes delta chain too long. With this, the
output of the command becomes identical to that of the older
implementation. But the performance suffers greatly.
It still allows reusing non-deltified representations; there is
no point uncompressing and recompressing the whole text.
It also adds a couple more statistics output, while squelching
it under -q flag, which the last round forgot to do.
$ time old-git-pack-objects --stdout >/dev/null <RL
Generating pack...
Done counting 184141 objects.
Packing 184141 objects....................
real 12m8.530s user 11m1.450s sys 0m57.920s
$ time git-pack-objects --stdout >/dev/null <RL
Generating pack...
Done counting 184141 objects.
Packing 184141 objects.....................
Total 184141, written 184141 (delta 138297), reused 178833 (delta 134081)
real 0m59.549s user 0m56.670s sys 0m2.400s
$ time git-pack-objects --stdout --no-reuse-delta >/dev/null <RL
Generating pack...
Done counting 184141 objects.
Packing 184141 objects.....................
Total 184141, written 184141 (delta 134833), reused 47904 (delta 0)
real 11m13.830s user 9m45.240s sys 0m44.330s
There is one remaining issue when --no-reuse-delta option is not
used. It can create delta chains that are deeper than specified.
A<--B<--C<--D E F G
Suppose we have a delta chain A to D (A is stored in full either
in a pack or as a loose object. B is depth1 delta relative to A,
C is depth2 delta relative to B...) with loose objects E, F, G.
And we are going to pack all of them.
B, C and D are left as delta against A, B and C respectively.
So A, E, F, and G are examined for deltification, and let's say
we decided to keep E expanded, and store the rest as deltas like
this:
E<--F<--G<--A
Oops. We ended up making D a bit too deep, didn't we? B, C and
D form a chain on top of A!
This is because we did not know what the final depth of A would
be, when we checked objects and decided to keep the existing
delta. Unfortunately, deferring the decision until just before
the deltification is not an option. To be able to make B, C,
and D candidates for deltification with the rest, we need to
know the type and final unexpanded size of them, but the major
part of the optimization comes from the fact that we do not read
the delta data to do so -- getting the final size is quite an
expensive operation.
To prevent this from happening, we should keep A from being
deltified. But how would we tell that, cheaply?
To do this most precisely, after check_object() runs, each
object that is used as the base object of some existing delta
needs to be marked with the maximum depth of the objects we
decided to keep deltified (in this case, D is depth 3 relative
to A, so if no other delta chain that is longer than 3 based on
A exists, mark A with 3). Then when attempting to deltify A, we
would take that number into account to see if the final delta
chain that leads to D becomes too deep.
However, this is a bit cumbersome to compute, so we would cheat
and reduce the maximum depth for A arbitrarily to depth/4 in
this implementation.
Signed-off-by: Junio C Hamano <junkio@cox.net>
2006-02-16 19:55:51 +00:00
|
|
|
written_delta++;
|
pack-objects: reuse data from existing packs.
When generating a new pack, notice if we have already needed
objects in existing packs. If an object is stored deltified,
and its base object is also what we are going to pack, then
reuse the existing deltified representation unconditionally,
bypassing all the expensive find_deltas() and try_deltas()
calls.
Also, notice if what we are going to write out exactly match
what is already in an existing pack (either deltified or just
compressed). In such a case, we can just copy it instead of
going through the usual uncompressing & recompressing cycle.
Without this patch, in linux-2.6 repository with about 1500
loose objects and a single mega pack:
$ git-rev-list --objects v2.6.16-rc3 >RL
$ wc -l RL
184141 RL
$ time git-pack-objects p <RL
Generating pack...
Done counting 184141 objects.
Packing 184141 objects....................
a1fc7b3e537fcb9b3c46b7505df859f0a11e79d2
real 12m4.323s
user 11m2.560s
sys 0m55.950s
With this patch, the same input:
$ time ../git.junio/git-pack-objects q <RL
Generating pack...
Done counting 184141 objects.
Packing 184141 objects.....................
a1fc7b3e537fcb9b3c46b7505df859f0a11e79d2
Total 184141, written 184141, reused 182441
real 1m2.608s
user 0m55.090s
sys 0m1.830s
Signed-off-by: Junio C Hamano <junkio@cox.net>
2006-02-16 01:34:29 +00:00
|
|
|
written++;
|
compute a CRC32 for each object as stored in a pack
The most important optimization for performance when repacking is the
ability to reuse data from a previous pack as is and bypass any delta
or even SHA1 computation by simply copying the raw data from one pack
to another directly.
The problem with this is that any data corruption within a copied object
would go unnoticed and the new (repacked) pack would be self-consistent
with its own checksum despite containing a corrupted object. This is a
real issue that already happened at least once in the past.
In some attempt to prevent this, we validate the copied data by inflating
it and making sure no error is signaled by zlib. But this is still not
perfect as a significant portion of a pack content is made of object
headers and references to delta base objects which are not deflated and
therefore not validated when repacking actually making the pack data reuse
still not as safe as it could be.
Of course a full SHA1 validation could be performed, but that implies
full data inflating and delta replaying which is extremely costly, which
cost the data reuse optimization was designed to avoid in the first place.
So the best solution to this is simply to store a CRC32 of the raw pack
data for each object in the pack index. This way any object in a pack can
be validated before being copied as is in another pack, including header
and any other non deflated data.
Why CRC32 instead of a faster checksum like Adler32? Quoting Wikipedia:
Jonathan Stone discovered in 2001 that Adler-32 has a weakness for very
short messages. He wrote "Briefly, the problem is that, for very short
packets, Adler32 is guaranteed to give poor coverage of the available
bits. Don't take my word for it, ask Mark Adler. :-)" The problem is
that sum A does not wrap for short messages. The maximum value of A for
a 128-byte message is 32640, which is below the value 65521 used by the
modulo operation. An extended explanation can be found in RFC 3309,
which mandates the use of CRC32 instead of Adler-32 for SCTP, the
Stream Control Transmission Protocol.
In the context of a GIT pack, we have lots of small objects, especially
deltas, which are likely to be quite small and in a size range for which
Adler32 is dimed not to be sufficient. Another advantage of CRC32 is the
possibility for recovery from certain types of small corruptions like
single bit errors which are the most probable type of corruptions.
OK what this patch does is to compute the CRC32 of each object written to
a pack within pack-objects. It is not written to the index yet and it is
obviously not validated when reusing pack data yet either.
Signed-off-by: Nicolas Pitre <nico@cam.org>
Signed-off-by: Junio C Hamano <junkio@cox.net>
2007-04-09 05:06:31 +00:00
|
|
|
if (!pack_to_stdout)
|
2007-06-01 19:18:05 +00:00
|
|
|
entry->idx.crc32 = crc32_end(f);
|
2005-06-25 21:42:43 +00:00
|
|
|
return hdrlen + datalen;
|
|
|
|
}
|
|
|
|
|
2008-08-29 20:07:58 +00:00
|
|
|
static int write_one(struct sha1file *f,
|
2005-06-29 00:49:27 +00:00
|
|
|
struct object_entry *e,
|
2008-08-29 20:07:58 +00:00
|
|
|
off_t *offset)
|
2005-06-29 00:49:27 +00:00
|
|
|
{
|
2007-04-09 05:06:30 +00:00
|
|
|
unsigned long size;
|
|
|
|
|
|
|
|
/* offset is non zero if object is written already. */
|
2007-06-01 19:18:05 +00:00
|
|
|
if (e->idx.offset || e->preferred_base)
|
2008-08-29 20:07:58 +00:00
|
|
|
return 1;
|
2007-04-09 05:06:30 +00:00
|
|
|
|
|
|
|
/* if we are deltified, write out base object first. */
|
2008-08-29 20:07:58 +00:00
|
|
|
if (e->delta && !write_one(f, e->delta, offset))
|
|
|
|
return 0;
|
2007-04-09 05:06:30 +00:00
|
|
|
|
2008-08-29 20:07:58 +00:00
|
|
|
e->idx.offset = *offset;
|
|
|
|
size = write_object(f, e, *offset);
|
2007-05-13 19:06:18 +00:00
|
|
|
if (!size) {
|
2007-06-01 19:18:05 +00:00
|
|
|
e->idx.offset = 0;
|
2007-05-13 19:06:18 +00:00
|
|
|
return 0;
|
|
|
|
}
|
2007-11-02 03:43:24 +00:00
|
|
|
written_list[nr_written++] = &e->idx;
|
2007-04-09 05:06:30 +00:00
|
|
|
|
|
|
|
/* make sure off_t is sufficiently large not to wrap */
|
2008-08-29 20:07:58 +00:00
|
|
|
if (*offset > *offset + size)
|
2007-04-09 05:06:30 +00:00
|
|
|
die("pack too large for current definition of off_t");
|
2008-08-29 20:07:58 +00:00
|
|
|
*offset += size;
|
|
|
|
return 1;
|
2005-06-29 00:49:27 +00:00
|
|
|
}
|
|
|
|
|
2007-06-01 19:18:05 +00:00
|
|
|
/* forward declaration for write_pack_file */
|
2007-05-13 18:34:56 +00:00
|
|
|
static int adjust_perm(const char *path, mode_t mode);
|
|
|
|
|
|
|
|
static void write_pack_file(void)
|
2005-06-25 21:42:43 +00:00
|
|
|
{
|
2007-05-13 19:09:16 +00:00
|
|
|
uint32_t i = 0, j;
|
2005-06-28 18:10:48 +00:00
|
|
|
struct sha1file *f;
|
2008-08-29 20:07:58 +00:00
|
|
|
off_t offset;
|
2005-06-28 21:21:02 +00:00
|
|
|
struct pack_header hdr;
|
2007-05-13 19:09:16 +00:00
|
|
|
uint32_t nr_remaining = nr_result;
|
2008-03-13 18:59:29 +00:00
|
|
|
time_t last_mtime = 0;
|
2007-04-16 16:31:05 +00:00
|
|
|
|
2008-05-02 19:11:47 +00:00
|
|
|
if (progress > pack_to_stdout)
|
2007-10-30 18:57:32 +00:00
|
|
|
progress_state = start_progress("Writing objects", nr_result);
|
2007-11-02 03:43:24 +00:00
|
|
|
written_list = xmalloc(nr_objects * sizeof(*written_list));
|
2006-02-23 00:02:59 +00:00
|
|
|
|
2007-05-13 19:09:16 +00:00
|
|
|
do {
|
2007-06-01 19:18:05 +00:00
|
|
|
unsigned char sha1[20];
|
2007-10-17 01:55:48 +00:00
|
|
|
char *pack_tmp_name = NULL;
|
2007-06-01 19:18:05 +00:00
|
|
|
|
2007-05-13 19:09:16 +00:00
|
|
|
if (pack_to_stdout) {
|
2007-10-30 21:06:21 +00:00
|
|
|
f = sha1fd_throughput(1, "<stdout>", progress_state);
|
2007-05-13 19:09:16 +00:00
|
|
|
} else {
|
2007-10-17 01:55:48 +00:00
|
|
|
char tmpname[PATH_MAX];
|
|
|
|
int fd;
|
2009-02-25 07:11:29 +00:00
|
|
|
fd = odb_mkstemp(tmpname, sizeof(tmpname),
|
|
|
|
"pack/tmp_pack_XXXXXX");
|
2007-05-13 19:09:16 +00:00
|
|
|
pack_tmp_name = xstrdup(tmpname);
|
|
|
|
f = sha1fd(fd, pack_tmp_name);
|
|
|
|
}
|
|
|
|
|
|
|
|
hdr.hdr_signature = htonl(PACK_SIGNATURE);
|
|
|
|
hdr.hdr_version = htonl(PACK_VERSION);
|
|
|
|
hdr.hdr_entries = htonl(nr_remaining);
|
|
|
|
sha1write(f, &hdr, sizeof(hdr));
|
|
|
|
offset = sizeof(hdr);
|
|
|
|
nr_written = 0;
|
|
|
|
for (; i < nr_objects; i++) {
|
2008-08-29 20:07:58 +00:00
|
|
|
if (!write_one(f, objects + i, &offset))
|
2007-05-13 19:09:16 +00:00
|
|
|
break;
|
2007-10-30 18:57:33 +00:00
|
|
|
display_progress(progress_state, written);
|
2007-05-13 19:09:16 +00:00
|
|
|
}
|
2007-04-09 05:06:33 +00:00
|
|
|
|
2007-05-13 19:09:16 +00:00
|
|
|
/*
|
|
|
|
* Did we write the wrong # entries in the header?
|
|
|
|
* If so, rewrite it like in fast-import
|
|
|
|
*/
|
2008-05-30 15:54:46 +00:00
|
|
|
if (pack_to_stdout) {
|
|
|
|
sha1close(f, sha1, CSUM_CLOSE);
|
|
|
|
} else if (nr_written == nr_remaining) {
|
|
|
|
sha1close(f, sha1, CSUM_FSYNC);
|
2007-05-13 19:09:16 +00:00
|
|
|
} else {
|
2008-08-29 20:08:00 +00:00
|
|
|
int fd = sha1close(f, sha1, 0);
|
2008-08-29 20:07:59 +00:00
|
|
|
fixup_pack_header_footer(fd, sha1, pack_tmp_name,
|
2008-08-29 20:08:00 +00:00
|
|
|
nr_written, sha1, offset);
|
2007-10-17 01:55:48 +00:00
|
|
|
close(fd);
|
2007-05-13 19:09:16 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
if (!pack_to_stdout) {
|
2007-05-13 18:34:56 +00:00
|
|
|
mode_t mode = umask(0);
|
2008-03-13 18:59:29 +00:00
|
|
|
struct stat st;
|
2007-10-17 01:55:48 +00:00
|
|
|
char *idx_tmp_name, tmpname[PATH_MAX];
|
2007-05-13 18:34:56 +00:00
|
|
|
|
|
|
|
umask(mode);
|
|
|
|
mode = 0444 & ~mode;
|
|
|
|
|
2007-11-02 03:43:24 +00:00
|
|
|
idx_tmp_name = write_idx_file(NULL, written_list,
|
|
|
|
nr_written, sha1);
|
2008-03-13 18:59:29 +00:00
|
|
|
|
2007-05-13 18:34:56 +00:00
|
|
|
snprintf(tmpname, sizeof(tmpname), "%s-%s.pack",
|
2007-06-01 19:18:05 +00:00
|
|
|
base_name, sha1_to_hex(sha1));
|
2008-12-09 19:26:52 +00:00
|
|
|
free_pack_by_name(tmpname);
|
2007-05-13 18:34:56 +00:00
|
|
|
if (adjust_perm(pack_tmp_name, mode))
|
2009-06-27 15:58:46 +00:00
|
|
|
die_errno("unable to make temporary pack file readable");
|
2007-05-13 18:34:56 +00:00
|
|
|
if (rename(pack_tmp_name, tmpname))
|
2009-06-27 15:58:46 +00:00
|
|
|
die_errno("unable to rename temporary pack file");
|
2008-03-13 18:59:29 +00:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Packs are runtime accessed in their mtime
|
|
|
|
* order since newer packs are more likely to contain
|
|
|
|
* younger objects. So if we are creating multiple
|
|
|
|
* packs then we should modify the mtime of later ones
|
|
|
|
* to preserve this property.
|
|
|
|
*/
|
|
|
|
if (stat(tmpname, &st) < 0) {
|
|
|
|
warning("failed to stat %s: %s",
|
|
|
|
tmpname, strerror(errno));
|
|
|
|
} else if (!last_mtime) {
|
|
|
|
last_mtime = st.st_mtime;
|
|
|
|
} else {
|
|
|
|
struct utimbuf utb;
|
|
|
|
utb.actime = st.st_atime;
|
|
|
|
utb.modtime = --last_mtime;
|
|
|
|
if (utime(tmpname, &utb) < 0)
|
|
|
|
warning("failed utime() on %s: %s",
|
|
|
|
tmpname, strerror(errno));
|
|
|
|
}
|
|
|
|
|
2007-05-13 18:34:56 +00:00
|
|
|
snprintf(tmpname, sizeof(tmpname), "%s-%s.idx",
|
2007-06-01 19:18:05 +00:00
|
|
|
base_name, sha1_to_hex(sha1));
|
2007-05-13 18:34:56 +00:00
|
|
|
if (adjust_perm(idx_tmp_name, mode))
|
2009-06-27 15:58:46 +00:00
|
|
|
die_errno("unable to make temporary index file readable");
|
2007-05-13 18:34:56 +00:00
|
|
|
if (rename(idx_tmp_name, tmpname))
|
2009-06-27 15:58:46 +00:00
|
|
|
die_errno("unable to rename temporary index file");
|
2008-03-13 18:59:29 +00:00
|
|
|
|
2007-10-17 01:55:48 +00:00
|
|
|
free(idx_tmp_name);
|
|
|
|
free(pack_tmp_name);
|
2007-06-01 19:18:05 +00:00
|
|
|
puts(sha1_to_hex(sha1));
|
2007-05-13 19:09:16 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
/* mark written objects as written to previous pack */
|
|
|
|
for (j = 0; j < nr_written; j++) {
|
2007-11-02 03:43:24 +00:00
|
|
|
written_list[j]->offset = (off_t)-1;
|
2007-05-13 19:09:16 +00:00
|
|
|
}
|
|
|
|
nr_remaining -= nr_written;
|
|
|
|
} while (nr_remaining && i < nr_objects);
|
|
|
|
|
|
|
|
free(written_list);
|
2007-10-30 18:57:33 +00:00
|
|
|
stop_progress(&progress_state);
|
2006-11-29 22:15:48 +00:00
|
|
|
if (written != nr_result)
|
2008-07-03 15:52:09 +00:00
|
|
|
die("wrote %"PRIu32" objects while expecting %"PRIu32,
|
|
|
|
written, nr_result);
|
2007-05-23 17:11:33 +00:00
|
|
|
/*
|
|
|
|
* We have scanned through [0 ... i). Since we have written
|
|
|
|
* the correct number of objects, the remaining [i ... nr_objects)
|
|
|
|
* items must be either already written (due to out-of-order delta base)
|
|
|
|
* or a preferred base. Count those which are neither and complain if any.
|
|
|
|
*/
|
2007-05-13 19:09:16 +00:00
|
|
|
for (j = 0; i < nr_objects; i++) {
|
|
|
|
struct object_entry *e = objects + i;
|
2007-06-01 19:18:05 +00:00
|
|
|
j += !e->idx.offset && !e->preferred_base;
|
2007-05-13 18:34:56 +00:00
|
|
|
}
|
2007-05-13 19:09:16 +00:00
|
|
|
if (j)
|
2008-07-03 15:52:09 +00:00
|
|
|
die("wrote %"PRIu32" objects as expected but %"PRIu32
|
|
|
|
" unwritten", written, j);
|
2005-06-25 21:42:43 +00:00
|
|
|
}
|
|
|
|
|
2006-02-19 22:47:21 +00:00
|
|
|
static int locate_object_entry_hash(const unsigned char *sha1)
|
|
|
|
{
|
|
|
|
int i;
|
|
|
|
unsigned int ui;
|
|
|
|
memcpy(&ui, sha1, sizeof(unsigned int));
|
|
|
|
i = ui % object_ix_hashsz;
|
|
|
|
while (0 < object_ix[i]) {
|
2007-06-01 19:18:05 +00:00
|
|
|
if (!hashcmp(sha1, objects[object_ix[i] - 1].idx.sha1))
|
2006-02-19 22:47:21 +00:00
|
|
|
return i;
|
|
|
|
if (++i == object_ix_hashsz)
|
|
|
|
i = 0;
|
|
|
|
}
|
|
|
|
return -1 - i;
|
|
|
|
}
|
|
|
|
|
|
|
|
static struct object_entry *locate_object_entry(const unsigned char *sha1)
|
|
|
|
{
|
|
|
|
int i;
|
|
|
|
|
|
|
|
if (!object_ix_hashsz)
|
|
|
|
return NULL;
|
|
|
|
|
|
|
|
i = locate_object_entry_hash(sha1);
|
|
|
|
if (0 <= i)
|
|
|
|
return &objects[object_ix[i]-1];
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void rehash_objects(void)
|
2005-06-25 21:42:43 +00:00
|
|
|
{
|
2007-03-07 01:44:24 +00:00
|
|
|
uint32_t i;
|
2006-02-19 22:47:21 +00:00
|
|
|
struct object_entry *oe;
|
|
|
|
|
|
|
|
object_ix_hashsz = nr_objects * 3;
|
|
|
|
if (object_ix_hashsz < 1024)
|
|
|
|
object_ix_hashsz = 1024;
|
|
|
|
object_ix = xrealloc(object_ix, sizeof(int) * object_ix_hashsz);
|
2006-04-06 06:24:57 +00:00
|
|
|
memset(object_ix, 0, sizeof(int) * object_ix_hashsz);
|
2006-02-19 22:47:21 +00:00
|
|
|
for (i = 0, oe = objects; i < nr_objects; i++, oe++) {
|
2007-06-01 19:18:05 +00:00
|
|
|
int ix = locate_object_entry_hash(oe->idx.sha1);
|
2006-02-19 22:47:21 +00:00
|
|
|
if (0 <= ix)
|
|
|
|
continue;
|
|
|
|
ix = -1 - ix;
|
|
|
|
object_ix[ix] = i + 1;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
pack-objects: improve path grouping heuristics.
This trivial patch not only simplifies the name hashing, it actually
improves packing for both git and the kernel.
The git archive pack shrinks from 6824090->6622627 bytes (a 3%
improvement), and the kernel pack shrinks from 108756213 to 108219021 (a
mere 0.5% improvement, but still, it's an improvement from making the
hashing much simpler!)
We just create a 32-bit hash, where we "age" previous characters by two
bits, so the last characters in a filename count most. So when we then
compare the hashes in the sort routine, filenames that end the same way
sort the same way.
It takes the subdirectory into account (unless the filename is > 16
characters), but files with the same name within the same subdirectory
will obviously sort closer than files in different subdirectories.
And, incidentally (which is why I tried the hash change in the first
place, of course) builtin-rev-list.c will sort fairly close to rev-list.c.
And no, it's not a "good hash" in the sense of being secure or unique, but
that's not what we're looking for. The whole "hash" thing is misnamed
here. It's not so much a hash as a "sorting number".
[jc: rolled in simplification for computing the sorting number
computation for thin pack base objects]
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
Signed-off-by: Junio C Hamano <junkio@cox.net>
2006-06-05 19:03:31 +00:00
|
|
|
static unsigned name_hash(const char *name)
|
2006-02-23 06:10:24 +00:00
|
|
|
{
|
Fix big left-shifts of unsigned char
Shifting 'unsigned char' or 'unsigned short' left can result in sign
extension errors, since the C integer promotion rules means that the
unsigned char/short will get implicitly promoted to a signed 'int' due to
the shift (or due to other operations).
This normally doesn't matter, but if you shift things up sufficiently, it
will now set the sign bit in 'int', and a subsequent cast to a bigger type
(eg 'long' or 'unsigned long') will now sign-extend the value despite the
original expression being unsigned.
One example of this would be something like
unsigned long size;
unsigned char c;
size += c << 24;
where despite all the variables being unsigned, 'c << 24' ends up being a
signed entity, and will get sign-extended when then doing the addition in
an 'unsigned long' type.
Since git uses 'unsigned char' pointers extensively, we actually have this
bug in a couple of places.
I may have missed some, but this is the result of looking at
git grep '[^0-9 ][ ]*<<[ ][a-z]' -- '*.c' '*.h'
git grep '<<[ ]*24'
which catches at least the common byte cases (shifting variables by a
variable amount, and shifting by 24 bits).
I also grepped for just 'unsigned char' variables in general, and
converted the ones that most obviously ended up getting implicitly cast
immediately anyway (eg hash_name(), encode_85()).
In addition to just avoiding 'unsigned char', this patch also tries to use
a common idiom for the delta header size thing. We had three different
variations on it: "& 0x7fUL" in one place (getting the sign extension
right), and "& ~0x80" and "& 0x7f" in two other places (not getting it
right). Apart from making them all just avoid using "unsigned char" at
all, I also unified them to then use a simple "& 0x7f".
I considered making a sparse extension which warns about doing implicit
casts from unsigned types to signed types, but it gets rather complex very
quickly, so this is just a hack.
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2009-06-18 00:22:27 +00:00
|
|
|
unsigned c, hash = 0;
|
pack-objects: improve path grouping heuristics.
This trivial patch not only simplifies the name hashing, it actually
improves packing for both git and the kernel.
The git archive pack shrinks from 6824090->6622627 bytes (a 3%
improvement), and the kernel pack shrinks from 108756213 to 108219021 (a
mere 0.5% improvement, but still, it's an improvement from making the
hashing much simpler!)
We just create a 32-bit hash, where we "age" previous characters by two
bits, so the last characters in a filename count most. So when we then
compare the hashes in the sort routine, filenames that end the same way
sort the same way.
It takes the subdirectory into account (unless the filename is > 16
characters), but files with the same name within the same subdirectory
will obviously sort closer than files in different subdirectories.
And, incidentally (which is why I tried the hash change in the first
place, of course) builtin-rev-list.c will sort fairly close to rev-list.c.
And no, it's not a "good hash" in the sense of being secure or unique, but
that's not what we're looking for. The whole "hash" thing is misnamed
here. It's not so much a hash as a "sorting number".
[jc: rolled in simplification for computing the sorting number
computation for thin pack base objects]
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
Signed-off-by: Junio C Hamano <junkio@cox.net>
2006-06-05 19:03:31 +00:00
|
|
|
|
2007-05-19 07:19:23 +00:00
|
|
|
if (!name)
|
|
|
|
return 0;
|
|
|
|
|
2006-02-24 07:27:49 +00:00
|
|
|
/*
|
pack-objects: improve path grouping heuristics.
This trivial patch not only simplifies the name hashing, it actually
improves packing for both git and the kernel.
The git archive pack shrinks from 6824090->6622627 bytes (a 3%
improvement), and the kernel pack shrinks from 108756213 to 108219021 (a
mere 0.5% improvement, but still, it's an improvement from making the
hashing much simpler!)
We just create a 32-bit hash, where we "age" previous characters by two
bits, so the last characters in a filename count most. So when we then
compare the hashes in the sort routine, filenames that end the same way
sort the same way.
It takes the subdirectory into account (unless the filename is > 16
characters), but files with the same name within the same subdirectory
will obviously sort closer than files in different subdirectories.
And, incidentally (which is why I tried the hash change in the first
place, of course) builtin-rev-list.c will sort fairly close to rev-list.c.
And no, it's not a "good hash" in the sense of being secure or unique, but
that's not what we're looking for. The whole "hash" thing is misnamed
here. It's not so much a hash as a "sorting number".
[jc: rolled in simplification for computing the sorting number
computation for thin pack base objects]
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
Signed-off-by: Junio C Hamano <junkio@cox.net>
2006-06-05 19:03:31 +00:00
|
|
|
* This effectively just creates a sortable number from the
|
|
|
|
* last sixteen non-whitespace characters. Last characters
|
|
|
|
* count "most", so things that end in ".c" sort together.
|
2006-02-24 07:27:49 +00:00
|
|
|
*/
|
pack-objects: improve path grouping heuristics.
This trivial patch not only simplifies the name hashing, it actually
improves packing for both git and the kernel.
The git archive pack shrinks from 6824090->6622627 bytes (a 3%
improvement), and the kernel pack shrinks from 108756213 to 108219021 (a
mere 0.5% improvement, but still, it's an improvement from making the
hashing much simpler!)
We just create a 32-bit hash, where we "age" previous characters by two
bits, so the last characters in a filename count most. So when we then
compare the hashes in the sort routine, filenames that end the same way
sort the same way.
It takes the subdirectory into account (unless the filename is > 16
characters), but files with the same name within the same subdirectory
will obviously sort closer than files in different subdirectories.
And, incidentally (which is why I tried the hash change in the first
place, of course) builtin-rev-list.c will sort fairly close to rev-list.c.
And no, it's not a "good hash" in the sense of being secure or unique, but
that's not what we're looking for. The whole "hash" thing is misnamed
here. It's not so much a hash as a "sorting number".
[jc: rolled in simplification for computing the sorting number
computation for thin pack base objects]
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
Signed-off-by: Junio C Hamano <junkio@cox.net>
2006-06-05 19:03:31 +00:00
|
|
|
while ((c = *name++) != 0) {
|
|
|
|
if (isspace(c))
|
|
|
|
continue;
|
|
|
|
hash = (hash >> 2) + (c << 24);
|
|
|
|
}
|
2006-02-23 06:10:24 +00:00
|
|
|
return hash;
|
|
|
|
}
|
|
|
|
|
2007-05-19 07:39:31 +00:00
|
|
|
static void setup_delta_attr_check(struct git_attr_check *check)
|
|
|
|
{
|
|
|
|
static struct git_attr *attr_delta;
|
|
|
|
|
|
|
|
if (!attr_delta)
|
|
|
|
attr_delta = git_attr("delta", 5);
|
|
|
|
|
|
|
|
check[0].attr = attr_delta;
|
|
|
|
}
|
|
|
|
|
|
|
|
static int no_try_delta(const char *path)
|
|
|
|
{
|
|
|
|
struct git_attr_check check[1];
|
|
|
|
|
|
|
|
setup_delta_attr_check(check);
|
|
|
|
if (git_checkattr(path, ARRAY_SIZE(check), check))
|
|
|
|
return 0;
|
|
|
|
if (ATTR_FALSE(check->value))
|
|
|
|
return 1;
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2007-04-16 16:32:13 +00:00
|
|
|
static int add_object_entry(const unsigned char *sha1, enum object_type type,
|
2007-05-19 07:19:23 +00:00
|
|
|
const char *name, int exclude)
|
2005-06-25 21:42:43 +00:00
|
|
|
{
|
|
|
|
struct object_entry *entry;
|
2007-04-11 02:54:36 +00:00
|
|
|
struct packed_git *p, *found_pack = NULL;
|
2007-03-07 01:44:34 +00:00
|
|
|
off_t found_offset = 0;
|
2007-04-11 02:54:36 +00:00
|
|
|
int ix;
|
2007-05-19 07:19:23 +00:00
|
|
|
unsigned hash = name_hash(name);
|
2007-04-11 02:54:36 +00:00
|
|
|
|
|
|
|
ix = nr_objects ? locate_object_entry_hash(sha1) : -1;
|
|
|
|
if (ix >= 0) {
|
|
|
|
if (exclude) {
|
|
|
|
entry = objects + object_ix[ix] - 1;
|
2007-04-16 16:31:05 +00:00
|
|
|
if (!entry->preferred_base)
|
|
|
|
nr_result--;
|
2007-04-11 02:54:36 +00:00
|
|
|
entry->preferred_base = 1;
|
|
|
|
}
|
|
|
|
return 0;
|
|
|
|
}
|
2005-06-25 21:42:43 +00:00
|
|
|
|
2008-11-10 05:59:58 +00:00
|
|
|
if (!exclude && local && has_loose_object_nonlocal(sha1))
|
|
|
|
return 0;
|
|
|
|
|
2007-04-16 16:32:13 +00:00
|
|
|
for (p = packed_git; p; p = p->next) {
|
|
|
|
off_t offset = find_pack_entry_one(sha1, p);
|
|
|
|
if (offset) {
|
|
|
|
if (!found_pack) {
|
|
|
|
found_offset = offset;
|
|
|
|
found_pack = p;
|
2005-10-13 22:38:28 +00:00
|
|
|
}
|
2007-04-16 16:32:13 +00:00
|
|
|
if (exclude)
|
|
|
|
break;
|
|
|
|
if (incremental)
|
|
|
|
return 0;
|
|
|
|
if (local && !p->pack_local)
|
|
|
|
return 0;
|
2008-11-12 17:59:04 +00:00
|
|
|
if (ignore_packed_keep && p->pack_local && p->pack_keep)
|
|
|
|
return 0;
|
2005-10-13 22:38:28 +00:00
|
|
|
}
|
|
|
|
}
|
2005-07-03 20:08:40 +00:00
|
|
|
|
2007-04-11 02:54:36 +00:00
|
|
|
if (nr_objects >= nr_alloc) {
|
|
|
|
nr_alloc = (nr_alloc + 1024) * 3 / 2;
|
2007-03-07 01:44:24 +00:00
|
|
|
objects = xrealloc(objects, nr_alloc * sizeof(*entry));
|
2005-06-25 21:42:43 +00:00
|
|
|
}
|
2007-04-11 02:54:36 +00:00
|
|
|
|
|
|
|
entry = objects + nr_objects++;
|
2005-06-25 21:42:43 +00:00
|
|
|
memset(entry, 0, sizeof(*entry));
|
2007-06-01 19:18:05 +00:00
|
|
|
hashcpy(entry->idx.sha1, sha1);
|
2005-06-26 22:27:28 +00:00
|
|
|
entry->hash = hash;
|
2007-04-16 16:32:13 +00:00
|
|
|
if (type)
|
|
|
|
entry->type = type;
|
2007-04-11 02:54:36 +00:00
|
|
|
if (exclude)
|
|
|
|
entry->preferred_base = 1;
|
2007-04-16 16:31:05 +00:00
|
|
|
else
|
|
|
|
nr_result++;
|
2007-04-11 02:54:36 +00:00
|
|
|
if (found_pack) {
|
|
|
|
entry->in_pack = found_pack;
|
|
|
|
entry->in_pack_offset = found_offset;
|
|
|
|
}
|
2006-02-19 22:47:21 +00:00
|
|
|
|
|
|
|
if (object_ix_hashsz * 3 <= nr_objects * 4)
|
|
|
|
rehash_objects();
|
2007-04-11 02:54:36 +00:00
|
|
|
else
|
|
|
|
object_ix[-1 - ix] = nr_objects;
|
2006-02-19 22:47:21 +00:00
|
|
|
|
2007-10-30 18:57:33 +00:00
|
|
|
display_progress(progress_state, nr_objects);
|
2007-04-11 02:54:36 +00:00
|
|
|
|
2007-05-19 07:39:31 +00:00
|
|
|
if (name && no_try_delta(name))
|
|
|
|
entry->no_try_delta = 1;
|
|
|
|
|
2007-04-11 02:54:36 +00:00
|
|
|
return 1;
|
2005-06-25 21:42:43 +00:00
|
|
|
}
|
|
|
|
|
2006-04-06 06:24:57 +00:00
|
|
|
struct pbase_tree_cache {
|
|
|
|
unsigned char sha1[20];
|
|
|
|
int ref;
|
|
|
|
int temporary;
|
|
|
|
void *tree_data;
|
|
|
|
unsigned long tree_size;
|
|
|
|
};
|
|
|
|
|
|
|
|
static struct pbase_tree_cache *(pbase_tree_cache[256]);
|
|
|
|
static int pbase_tree_cache_ix(const unsigned char *sha1)
|
|
|
|
{
|
|
|
|
return sha1[0] % ARRAY_SIZE(pbase_tree_cache);
|
|
|
|
}
|
|
|
|
static int pbase_tree_cache_ix_incr(int ix)
|
|
|
|
{
|
|
|
|
return (ix+1) % ARRAY_SIZE(pbase_tree_cache);
|
|
|
|
}
|
|
|
|
|
|
|
|
static struct pbase_tree {
|
|
|
|
struct pbase_tree *next;
|
|
|
|
/* This is a phony "cache" entry; we are not
|
|
|
|
* going to evict it nor find it through _get()
|
|
|
|
* mechanism -- this is for the toplevel node that
|
|
|
|
* would almost always change with any commit.
|
|
|
|
*/
|
|
|
|
struct pbase_tree_cache pcache;
|
|
|
|
} *pbase_tree;
|
|
|
|
|
|
|
|
static struct pbase_tree_cache *pbase_tree_get(const unsigned char *sha1)
|
|
|
|
{
|
|
|
|
struct pbase_tree_cache *ent, *nent;
|
|
|
|
void *data;
|
|
|
|
unsigned long size;
|
2007-02-26 19:55:59 +00:00
|
|
|
enum object_type type;
|
2006-04-06 06:24:57 +00:00
|
|
|
int neigh;
|
|
|
|
int my_ix = pbase_tree_cache_ix(sha1);
|
|
|
|
int available_ix = -1;
|
|
|
|
|
|
|
|
/* pbase-tree-cache acts as a limited hashtable.
|
|
|
|
* your object will be found at your index or within a few
|
|
|
|
* slots after that slot if it is cached.
|
|
|
|
*/
|
|
|
|
for (neigh = 0; neigh < 8; neigh++) {
|
|
|
|
ent = pbase_tree_cache[my_ix];
|
2006-08-17 18:54:57 +00:00
|
|
|
if (ent && !hashcmp(ent->sha1, sha1)) {
|
2006-04-06 06:24:57 +00:00
|
|
|
ent->ref++;
|
|
|
|
return ent;
|
|
|
|
}
|
|
|
|
else if (((available_ix < 0) && (!ent || !ent->ref)) ||
|
|
|
|
((0 <= available_ix) &&
|
|
|
|
(!ent && pbase_tree_cache[available_ix])))
|
|
|
|
available_ix = my_ix;
|
|
|
|
if (!ent)
|
|
|
|
break;
|
|
|
|
my_ix = pbase_tree_cache_ix_incr(my_ix);
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Did not find one. Either we got a bogus request or
|
|
|
|
* we need to read and perhaps cache.
|
|
|
|
*/
|
2007-02-26 19:55:59 +00:00
|
|
|
data = read_sha1_file(sha1, &type, &size);
|
2006-04-06 06:24:57 +00:00
|
|
|
if (!data)
|
|
|
|
return NULL;
|
2007-02-26 19:55:59 +00:00
|
|
|
if (type != OBJ_TREE) {
|
2006-04-06 06:24:57 +00:00
|
|
|
free(data);
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* We need to either cache or return a throwaway copy */
|
|
|
|
|
|
|
|
if (available_ix < 0)
|
|
|
|
ent = NULL;
|
|
|
|
else {
|
|
|
|
ent = pbase_tree_cache[available_ix];
|
|
|
|
my_ix = available_ix;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (!ent) {
|
|
|
|
nent = xmalloc(sizeof(*nent));
|
|
|
|
nent->temporary = (available_ix < 0);
|
|
|
|
}
|
|
|
|
else {
|
|
|
|
/* evict and reuse */
|
|
|
|
free(ent->tree_data);
|
|
|
|
nent = ent;
|
|
|
|
}
|
2006-08-23 06:49:00 +00:00
|
|
|
hashcpy(nent->sha1, sha1);
|
2006-04-06 06:24:57 +00:00
|
|
|
nent->tree_data = data;
|
|
|
|
nent->tree_size = size;
|
|
|
|
nent->ref = 1;
|
|
|
|
if (!nent->temporary)
|
|
|
|
pbase_tree_cache[my_ix] = nent;
|
|
|
|
return nent;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void pbase_tree_put(struct pbase_tree_cache *cache)
|
|
|
|
{
|
|
|
|
if (!cache->temporary) {
|
|
|
|
cache->ref--;
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
free(cache->tree_data);
|
|
|
|
free(cache);
|
|
|
|
}
|
|
|
|
|
|
|
|
static int name_cmp_len(const char *name)
|
|
|
|
{
|
|
|
|
int i;
|
|
|
|
for (i = 0; name[i] && name[i] != '\n' && name[i] != '/'; i++)
|
|
|
|
;
|
|
|
|
return i;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void add_pbase_object(struct tree_desc *tree,
|
|
|
|
const char *name,
|
pack-objects: improve path grouping heuristics.
This trivial patch not only simplifies the name hashing, it actually
improves packing for both git and the kernel.
The git archive pack shrinks from 6824090->6622627 bytes (a 3%
improvement), and the kernel pack shrinks from 108756213 to 108219021 (a
mere 0.5% improvement, but still, it's an improvement from making the
hashing much simpler!)
We just create a 32-bit hash, where we "age" previous characters by two
bits, so the last characters in a filename count most. So when we then
compare the hashes in the sort routine, filenames that end the same way
sort the same way.
It takes the subdirectory into account (unless the filename is > 16
characters), but files with the same name within the same subdirectory
will obviously sort closer than files in different subdirectories.
And, incidentally (which is why I tried the hash change in the first
place, of course) builtin-rev-list.c will sort fairly close to rev-list.c.
And no, it's not a "good hash" in the sense of being secure or unique, but
that's not what we're looking for. The whole "hash" thing is misnamed
here. It's not so much a hash as a "sorting number".
[jc: rolled in simplification for computing the sorting number
computation for thin pack base objects]
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
Signed-off-by: Junio C Hamano <junkio@cox.net>
2006-06-05 19:03:31 +00:00
|
|
|
int cmplen,
|
|
|
|
const char *fullname)
|
pack-objects: reuse data from existing packs.
When generating a new pack, notice if we have already needed
objects in existing packs. If an object is stored deltified,
and its base object is also what we are going to pack, then
reuse the existing deltified representation unconditionally,
bypassing all the expensive find_deltas() and try_deltas()
calls.
Also, notice if what we are going to write out exactly match
what is already in an existing pack (either deltified or just
compressed). In such a case, we can just copy it instead of
going through the usual uncompressing & recompressing cycle.
Without this patch, in linux-2.6 repository with about 1500
loose objects and a single mega pack:
$ git-rev-list --objects v2.6.16-rc3 >RL
$ wc -l RL
184141 RL
$ time git-pack-objects p <RL
Generating pack...
Done counting 184141 objects.
Packing 184141 objects....................
a1fc7b3e537fcb9b3c46b7505df859f0a11e79d2
real 12m4.323s
user 11m2.560s
sys 0m55.950s
With this patch, the same input:
$ time ../git.junio/git-pack-objects q <RL
Generating pack...
Done counting 184141 objects.
Packing 184141 objects.....................
a1fc7b3e537fcb9b3c46b7505df859f0a11e79d2
Total 184141, written 184141, reused 182441
real 1m2.608s
user 0m55.090s
sys 0m1.830s
Signed-off-by: Junio C Hamano <junkio@cox.net>
2006-02-16 01:34:29 +00:00
|
|
|
{
|
tree_entry(): new tree-walking helper function
This adds a "tree_entry()" function that combines the common operation of
doing a "tree_entry_extract()" + "update_tree_entry()".
It also has a simplified calling convention, designed for simple loops
that traverse over a whole tree: the arguments are pointers to the tree
descriptor and a name_entry structure to fill in, and it returns a boolean
"true" if there was an entry left to be gotten in the tree.
This allows tree traversal with
struct tree_desc desc;
struct name_entry entry;
desc.buf = tree->buffer;
desc.size = tree->size;
while (tree_entry(&desc, &entry) {
... use "entry.{path, sha1, mode, pathlen}" ...
}
which is not only shorter than writing it out in full, it's hopefully less
error prone too.
[ It's actually a tad faster too - we don't need to recalculate the entry
pathlength in both extract and update, but need to do it only once.
Also, some callers can avoid doing a "strlen()" on the result, since
it's returned as part of the name_entry structure.
However, by now we're talking just 1% speedup on "git-rev-list --objects
--all", and we're definitely at the point where tree walking is no
longer the issue any more. ]
NOTE! Not everybody wants to use this new helper function, since some of
the tree walkers very much on purpose do the descriptor update separately
from the entry extraction. So the "extract + update" sequence still
remains as the core sequence, this is just a simplified interface.
We should probably add a silly two-line inline helper function for
initializing the descriptor from the "struct tree" too, just to cut down
on the noise from that common "desc" initializer.
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
Signed-off-by: Junio C Hamano <junkio@cox.net>
2006-05-30 16:45:45 +00:00
|
|
|
struct name_entry entry;
|
2007-04-16 16:28:10 +00:00
|
|
|
int cmp;
|
tree_entry(): new tree-walking helper function
This adds a "tree_entry()" function that combines the common operation of
doing a "tree_entry_extract()" + "update_tree_entry()".
It also has a simplified calling convention, designed for simple loops
that traverse over a whole tree: the arguments are pointers to the tree
descriptor and a name_entry structure to fill in, and it returns a boolean
"true" if there was an entry left to be gotten in the tree.
This allows tree traversal with
struct tree_desc desc;
struct name_entry entry;
desc.buf = tree->buffer;
desc.size = tree->size;
while (tree_entry(&desc, &entry) {
... use "entry.{path, sha1, mode, pathlen}" ...
}
which is not only shorter than writing it out in full, it's hopefully less
error prone too.
[ It's actually a tad faster too - we don't need to recalculate the entry
pathlength in both extract and update, but need to do it only once.
Also, some callers can avoid doing a "strlen()" on the result, since
it's returned as part of the name_entry structure.
However, by now we're talking just 1% speedup on "git-rev-list --objects
--all", and we're definitely at the point where tree walking is no
longer the issue any more. ]
NOTE! Not everybody wants to use this new helper function, since some of
the tree walkers very much on purpose do the descriptor update separately
from the entry extraction. So the "extract + update" sequence still
remains as the core sequence, this is just a simplified interface.
We should probably add a silly two-line inline helper function for
initializing the descriptor from the "struct tree" too, just to cut down
on the noise from that common "desc" initializer.
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
Signed-off-by: Junio C Hamano <junkio@cox.net>
2006-05-30 16:45:45 +00:00
|
|
|
|
|
|
|
while (tree_entry(tree,&entry)) {
|
2007-08-17 16:56:54 +00:00
|
|
|
if (S_ISGITLINK(entry.mode))
|
|
|
|
continue;
|
2007-04-16 16:28:10 +00:00
|
|
|
cmp = tree_entry_len(entry.path, entry.sha1) != cmplen ? 1 :
|
|
|
|
memcmp(name, entry.path, cmplen);
|
|
|
|
if (cmp > 0)
|
2006-02-19 22:47:21 +00:00
|
|
|
continue;
|
2007-04-16 16:28:10 +00:00
|
|
|
if (cmp < 0)
|
|
|
|
return;
|
2006-04-06 06:24:57 +00:00
|
|
|
if (name[cmplen] != '/') {
|
2007-04-16 16:32:13 +00:00
|
|
|
add_object_entry(entry.sha1,
|
2007-11-11 23:35:23 +00:00
|
|
|
object_type(entry.mode),
|
2007-05-19 07:19:23 +00:00
|
|
|
fullname, 1);
|
2006-04-06 06:24:57 +00:00
|
|
|
return;
|
|
|
|
}
|
2007-04-16 16:28:10 +00:00
|
|
|
if (S_ISDIR(entry.mode)) {
|
2006-02-19 22:47:21 +00:00
|
|
|
struct tree_desc sub;
|
2006-04-06 06:24:57 +00:00
|
|
|
struct pbase_tree_cache *tree;
|
|
|
|
const char *down = name+cmplen+1;
|
|
|
|
int downlen = name_cmp_len(down);
|
|
|
|
|
tree_entry(): new tree-walking helper function
This adds a "tree_entry()" function that combines the common operation of
doing a "tree_entry_extract()" + "update_tree_entry()".
It also has a simplified calling convention, designed for simple loops
that traverse over a whole tree: the arguments are pointers to the tree
descriptor and a name_entry structure to fill in, and it returns a boolean
"true" if there was an entry left to be gotten in the tree.
This allows tree traversal with
struct tree_desc desc;
struct name_entry entry;
desc.buf = tree->buffer;
desc.size = tree->size;
while (tree_entry(&desc, &entry) {
... use "entry.{path, sha1, mode, pathlen}" ...
}
which is not only shorter than writing it out in full, it's hopefully less
error prone too.
[ It's actually a tad faster too - we don't need to recalculate the entry
pathlength in both extract and update, but need to do it only once.
Also, some callers can avoid doing a "strlen()" on the result, since
it's returned as part of the name_entry structure.
However, by now we're talking just 1% speedup on "git-rev-list --objects
--all", and we're definitely at the point where tree walking is no
longer the issue any more. ]
NOTE! Not everybody wants to use this new helper function, since some of
the tree walkers very much on purpose do the descriptor update separately
from the entry extraction. So the "extract + update" sequence still
remains as the core sequence, this is just a simplified interface.
We should probably add a silly two-line inline helper function for
initializing the descriptor from the "struct tree" too, just to cut down
on the noise from that common "desc" initializer.
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
Signed-off-by: Junio C Hamano <junkio@cox.net>
2006-05-30 16:45:45 +00:00
|
|
|
tree = pbase_tree_get(entry.sha1);
|
2006-04-06 06:24:57 +00:00
|
|
|
if (!tree)
|
|
|
|
return;
|
2007-03-21 17:08:25 +00:00
|
|
|
init_tree_desc(&sub, tree->tree_data, tree->tree_size);
|
2006-04-06 06:24:57 +00:00
|
|
|
|
pack-objects: improve path grouping heuristics.
This trivial patch not only simplifies the name hashing, it actually
improves packing for both git and the kernel.
The git archive pack shrinks from 6824090->6622627 bytes (a 3%
improvement), and the kernel pack shrinks from 108756213 to 108219021 (a
mere 0.5% improvement, but still, it's an improvement from making the
hashing much simpler!)
We just create a 32-bit hash, where we "age" previous characters by two
bits, so the last characters in a filename count most. So when we then
compare the hashes in the sort routine, filenames that end the same way
sort the same way.
It takes the subdirectory into account (unless the filename is > 16
characters), but files with the same name within the same subdirectory
will obviously sort closer than files in different subdirectories.
And, incidentally (which is why I tried the hash change in the first
place, of course) builtin-rev-list.c will sort fairly close to rev-list.c.
And no, it's not a "good hash" in the sense of being secure or unique, but
that's not what we're looking for. The whole "hash" thing is misnamed
here. It's not so much a hash as a "sorting number".
[jc: rolled in simplification for computing the sorting number
computation for thin pack base objects]
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
Signed-off-by: Junio C Hamano <junkio@cox.net>
2006-06-05 19:03:31 +00:00
|
|
|
add_pbase_object(&sub, down, downlen, fullname);
|
2006-04-06 06:24:57 +00:00
|
|
|
pbase_tree_put(tree);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
2006-02-23 06:10:24 +00:00
|
|
|
|
2006-04-06 06:24:57 +00:00
|
|
|
static unsigned *done_pbase_paths;
|
|
|
|
static int done_pbase_paths_num;
|
|
|
|
static int done_pbase_paths_alloc;
|
|
|
|
static int done_pbase_path_pos(unsigned hash)
|
|
|
|
{
|
|
|
|
int lo = 0;
|
|
|
|
int hi = done_pbase_paths_num;
|
|
|
|
while (lo < hi) {
|
|
|
|
int mi = (hi + lo) / 2;
|
|
|
|
if (done_pbase_paths[mi] == hash)
|
|
|
|
return mi;
|
|
|
|
if (done_pbase_paths[mi] < hash)
|
|
|
|
hi = mi;
|
|
|
|
else
|
|
|
|
lo = mi + 1;
|
|
|
|
}
|
|
|
|
return -lo-1;
|
|
|
|
}
|
|
|
|
|
|
|
|
static int check_pbase_path(unsigned hash)
|
|
|
|
{
|
|
|
|
int pos = (!done_pbase_paths) ? -1 : done_pbase_path_pos(hash);
|
|
|
|
if (0 <= pos)
|
|
|
|
return 1;
|
|
|
|
pos = -pos - 1;
|
|
|
|
if (done_pbase_paths_alloc <= done_pbase_paths_num) {
|
|
|
|
done_pbase_paths_alloc = alloc_nr(done_pbase_paths_alloc);
|
|
|
|
done_pbase_paths = xrealloc(done_pbase_paths,
|
|
|
|
done_pbase_paths_alloc *
|
|
|
|
sizeof(unsigned));
|
|
|
|
}
|
|
|
|
done_pbase_paths_num++;
|
|
|
|
if (pos < done_pbase_paths_num)
|
|
|
|
memmove(done_pbase_paths + pos + 1,
|
|
|
|
done_pbase_paths + pos,
|
|
|
|
(done_pbase_paths_num - pos - 1) * sizeof(unsigned));
|
|
|
|
done_pbase_paths[pos] = hash;
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2007-05-19 07:19:23 +00:00
|
|
|
static void add_preferred_base_object(const char *name)
|
2006-04-06 06:24:57 +00:00
|
|
|
{
|
|
|
|
struct pbase_tree *it;
|
2007-04-16 16:28:10 +00:00
|
|
|
int cmplen;
|
2007-05-19 07:19:23 +00:00
|
|
|
unsigned hash = name_hash(name);
|
2006-04-06 06:24:57 +00:00
|
|
|
|
2007-04-16 16:28:10 +00:00
|
|
|
if (!num_preferred_base || check_pbase_path(hash))
|
2006-04-06 06:24:57 +00:00
|
|
|
return;
|
|
|
|
|
2007-04-16 16:28:10 +00:00
|
|
|
cmplen = name_cmp_len(name);
|
2006-04-06 06:24:57 +00:00
|
|
|
for (it = pbase_tree; it; it = it->next) {
|
|
|
|
if (cmplen == 0) {
|
2007-05-19 07:19:23 +00:00
|
|
|
add_object_entry(it->pcache.sha1, OBJ_TREE, NULL, 1);
|
2006-04-06 06:24:57 +00:00
|
|
|
}
|
|
|
|
else {
|
|
|
|
struct tree_desc tree;
|
2007-03-21 17:08:25 +00:00
|
|
|
init_tree_desc(&tree, it->pcache.tree_data, it->pcache.tree_size);
|
pack-objects: improve path grouping heuristics.
This trivial patch not only simplifies the name hashing, it actually
improves packing for both git and the kernel.
The git archive pack shrinks from 6824090->6622627 bytes (a 3%
improvement), and the kernel pack shrinks from 108756213 to 108219021 (a
mere 0.5% improvement, but still, it's an improvement from making the
hashing much simpler!)
We just create a 32-bit hash, where we "age" previous characters by two
bits, so the last characters in a filename count most. So when we then
compare the hashes in the sort routine, filenames that end the same way
sort the same way.
It takes the subdirectory into account (unless the filename is > 16
characters), but files with the same name within the same subdirectory
will obviously sort closer than files in different subdirectories.
And, incidentally (which is why I tried the hash change in the first
place, of course) builtin-rev-list.c will sort fairly close to rev-list.c.
And no, it's not a "good hash" in the sense of being secure or unique, but
that's not what we're looking for. The whole "hash" thing is misnamed
here. It's not so much a hash as a "sorting number".
[jc: rolled in simplification for computing the sorting number
computation for thin pack base objects]
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
Signed-off-by: Junio C Hamano <junkio@cox.net>
2006-06-05 19:03:31 +00:00
|
|
|
add_pbase_object(&tree, name, cmplen, name);
|
2006-02-19 22:47:21 +00:00
|
|
|
}
|
pack-objects: reuse data from existing packs.
When generating a new pack, notice if we have already needed
objects in existing packs. If an object is stored deltified,
and its base object is also what we are going to pack, then
reuse the existing deltified representation unconditionally,
bypassing all the expensive find_deltas() and try_deltas()
calls.
Also, notice if what we are going to write out exactly match
what is already in an existing pack (either deltified or just
compressed). In such a case, we can just copy it instead of
going through the usual uncompressing & recompressing cycle.
Without this patch, in linux-2.6 repository with about 1500
loose objects and a single mega pack:
$ git-rev-list --objects v2.6.16-rc3 >RL
$ wc -l RL
184141 RL
$ time git-pack-objects p <RL
Generating pack...
Done counting 184141 objects.
Packing 184141 objects....................
a1fc7b3e537fcb9b3c46b7505df859f0a11e79d2
real 12m4.323s
user 11m2.560s
sys 0m55.950s
With this patch, the same input:
$ time ../git.junio/git-pack-objects q <RL
Generating pack...
Done counting 184141 objects.
Packing 184141 objects.....................
a1fc7b3e537fcb9b3c46b7505df859f0a11e79d2
Total 184141, written 184141, reused 182441
real 1m2.608s
user 0m55.090s
sys 0m1.830s
Signed-off-by: Junio C Hamano <junkio@cox.net>
2006-02-16 01:34:29 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2006-02-19 22:47:21 +00:00
|
|
|
static void add_preferred_base(unsigned char *sha1)
|
pack-objects: reuse data from existing packs.
When generating a new pack, notice if we have already needed
objects in existing packs. If an object is stored deltified,
and its base object is also what we are going to pack, then
reuse the existing deltified representation unconditionally,
bypassing all the expensive find_deltas() and try_deltas()
calls.
Also, notice if what we are going to write out exactly match
what is already in an existing pack (either deltified or just
compressed). In such a case, we can just copy it instead of
going through the usual uncompressing & recompressing cycle.
Without this patch, in linux-2.6 repository with about 1500
loose objects and a single mega pack:
$ git-rev-list --objects v2.6.16-rc3 >RL
$ wc -l RL
184141 RL
$ time git-pack-objects p <RL
Generating pack...
Done counting 184141 objects.
Packing 184141 objects....................
a1fc7b3e537fcb9b3c46b7505df859f0a11e79d2
real 12m4.323s
user 11m2.560s
sys 0m55.950s
With this patch, the same input:
$ time ../git.junio/git-pack-objects q <RL
Generating pack...
Done counting 184141 objects.
Packing 184141 objects.....................
a1fc7b3e537fcb9b3c46b7505df859f0a11e79d2
Total 184141, written 184141, reused 182441
real 1m2.608s
user 0m55.090s
sys 0m1.830s
Signed-off-by: Junio C Hamano <junkio@cox.net>
2006-02-16 01:34:29 +00:00
|
|
|
{
|
2006-04-06 06:24:57 +00:00
|
|
|
struct pbase_tree *it;
|
|
|
|
void *data;
|
|
|
|
unsigned long size;
|
|
|
|
unsigned char tree_sha1[20];
|
2006-02-23 06:10:24 +00:00
|
|
|
|
2006-09-06 08:42:23 +00:00
|
|
|
if (window <= num_preferred_base++)
|
|
|
|
return;
|
|
|
|
|
2006-04-06 06:24:57 +00:00
|
|
|
data = read_object_with_reference(sha1, tree_type, &size, tree_sha1);
|
|
|
|
if (!data)
|
2006-02-19 22:47:21 +00:00
|
|
|
return;
|
2006-04-06 06:24:57 +00:00
|
|
|
|
|
|
|
for (it = pbase_tree; it; it = it->next) {
|
2006-08-17 18:54:57 +00:00
|
|
|
if (!hashcmp(it->pcache.sha1, tree_sha1)) {
|
2006-04-06 06:24:57 +00:00
|
|
|
free(data);
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
it = xcalloc(1, sizeof(*it));
|
|
|
|
it->next = pbase_tree;
|
|
|
|
pbase_tree = it;
|
|
|
|
|
2006-08-23 06:49:00 +00:00
|
|
|
hashcpy(it->pcache.sha1, tree_sha1);
|
2006-04-06 06:24:57 +00:00
|
|
|
it->pcache.tree_data = data;
|
|
|
|
it->pcache.tree_size = size;
|
pack-objects: reuse data from existing packs.
When generating a new pack, notice if we have already needed
objects in existing packs. If an object is stored deltified,
and its base object is also what we are going to pack, then
reuse the existing deltified representation unconditionally,
bypassing all the expensive find_deltas() and try_deltas()
calls.
Also, notice if what we are going to write out exactly match
what is already in an existing pack (either deltified or just
compressed). In such a case, we can just copy it instead of
going through the usual uncompressing & recompressing cycle.
Without this patch, in linux-2.6 repository with about 1500
loose objects and a single mega pack:
$ git-rev-list --objects v2.6.16-rc3 >RL
$ wc -l RL
184141 RL
$ time git-pack-objects p <RL
Generating pack...
Done counting 184141 objects.
Packing 184141 objects....................
a1fc7b3e537fcb9b3c46b7505df859f0a11e79d2
real 12m4.323s
user 11m2.560s
sys 0m55.950s
With this patch, the same input:
$ time ../git.junio/git-pack-objects q <RL
Generating pack...
Done counting 184141 objects.
Packing 184141 objects.....................
a1fc7b3e537fcb9b3c46b7505df859f0a11e79d2
Total 184141, written 184141, reused 182441
real 1m2.608s
user 0m55.090s
sys 0m1.830s
Signed-off-by: Junio C Hamano <junkio@cox.net>
2006-02-16 01:34:29 +00:00
|
|
|
}
|
|
|
|
|
2005-06-25 21:42:43 +00:00
|
|
|
static void check_object(struct object_entry *entry)
|
|
|
|
{
|
2007-04-16 16:32:13 +00:00
|
|
|
if (entry->in_pack) {
|
2006-09-23 01:25:04 +00:00
|
|
|
struct packed_git *p = entry->in_pack;
|
2006-12-23 07:34:08 +00:00
|
|
|
struct pack_window *w_curs = NULL;
|
2007-04-16 16:32:13 +00:00
|
|
|
const unsigned char *base_ref = NULL;
|
|
|
|
struct object_entry *base_entry;
|
|
|
|
unsigned long used, used_0;
|
2007-03-07 01:44:34 +00:00
|
|
|
unsigned int avail;
|
2007-04-16 16:32:13 +00:00
|
|
|
off_t ofs;
|
|
|
|
unsigned char *buf, c;
|
2006-09-23 01:25:04 +00:00
|
|
|
|
2007-03-07 01:44:34 +00:00
|
|
|
buf = use_pack(p, &w_curs, entry->in_pack_offset, &avail);
|
pack-objects: finishing touches.
This introduces --no-reuse-delta option to disable reusing of
existing delta, which is a large part of the optimization
introduced by this series. This may become necessary if
repeated repacking makes delta chain too long. With this, the
output of the command becomes identical to that of the older
implementation. But the performance suffers greatly.
It still allows reusing non-deltified representations; there is
no point uncompressing and recompressing the whole text.
It also adds a couple more statistics output, while squelching
it under -q flag, which the last round forgot to do.
$ time old-git-pack-objects --stdout >/dev/null <RL
Generating pack...
Done counting 184141 objects.
Packing 184141 objects....................
real 12m8.530s user 11m1.450s sys 0m57.920s
$ time git-pack-objects --stdout >/dev/null <RL
Generating pack...
Done counting 184141 objects.
Packing 184141 objects.....................
Total 184141, written 184141 (delta 138297), reused 178833 (delta 134081)
real 0m59.549s user 0m56.670s sys 0m2.400s
$ time git-pack-objects --stdout --no-reuse-delta >/dev/null <RL
Generating pack...
Done counting 184141 objects.
Packing 184141 objects.....................
Total 184141, written 184141 (delta 134833), reused 47904 (delta 0)
real 11m13.830s user 9m45.240s sys 0m44.330s
There is one remaining issue when --no-reuse-delta option is not
used. It can create delta chains that are deeper than specified.
A<--B<--C<--D E F G
Suppose we have a delta chain A to D (A is stored in full either
in a pack or as a loose object. B is depth1 delta relative to A,
C is depth2 delta relative to B...) with loose objects E, F, G.
And we are going to pack all of them.
B, C and D are left as delta against A, B and C respectively.
So A, E, F, and G are examined for deltification, and let's say
we decided to keep E expanded, and store the rest as deltas like
this:
E<--F<--G<--A
Oops. We ended up making D a bit too deep, didn't we? B, C and
D form a chain on top of A!
This is because we did not know what the final depth of A would
be, when we checked objects and decided to keep the existing
delta. Unfortunately, deferring the decision until just before
the deltification is not an option. To be able to make B, C,
and D candidates for deltification with the rest, we need to
know the type and final unexpanded size of them, but the major
part of the optimization comes from the fact that we do not read
the delta data to do so -- getting the final size is quite an
expensive operation.
To prevent this from happening, we should keep A from being
deltified. But how would we tell that, cheaply?
To do this most precisely, after check_object() runs, each
object that is used as the base object of some existing delta
needs to be marked with the maximum depth of the objects we
decided to keep deltified (in this case, D is depth 3 relative
to A, so if no other delta chain that is longer than 3 based on
A exists, mark A with 3). Then when attempting to deltify A, we
would take that number into account to see if the final delta
chain that leads to D becomes too deep.
However, this is a bit cumbersome to compute, so we would cheat
and reduce the maximum depth for A arbitrarily to depth/4 in
this implementation.
Signed-off-by: Junio C Hamano <junkio@cox.net>
2006-02-16 19:55:51 +00:00
|
|
|
|
2007-04-16 16:32:13 +00:00
|
|
|
/*
|
2007-05-09 16:31:28 +00:00
|
|
|
* We want in_pack_type even if we do not reuse delta
|
|
|
|
* since non-delta representations could still be reused.
|
pack-objects: finishing touches.
This introduces --no-reuse-delta option to disable reusing of
existing delta, which is a large part of the optimization
introduced by this series. This may become necessary if
repeated repacking makes delta chain too long. With this, the
output of the command becomes identical to that of the older
implementation. But the performance suffers greatly.
It still allows reusing non-deltified representations; there is
no point uncompressing and recompressing the whole text.
It also adds a couple more statistics output, while squelching
it under -q flag, which the last round forgot to do.
$ time old-git-pack-objects --stdout >/dev/null <RL
Generating pack...
Done counting 184141 objects.
Packing 184141 objects....................
real 12m8.530s user 11m1.450s sys 0m57.920s
$ time git-pack-objects --stdout >/dev/null <RL
Generating pack...
Done counting 184141 objects.
Packing 184141 objects.....................
Total 184141, written 184141 (delta 138297), reused 178833 (delta 134081)
real 0m59.549s user 0m56.670s sys 0m2.400s
$ time git-pack-objects --stdout --no-reuse-delta >/dev/null <RL
Generating pack...
Done counting 184141 objects.
Packing 184141 objects.....................
Total 184141, written 184141 (delta 134833), reused 47904 (delta 0)
real 11m13.830s user 9m45.240s sys 0m44.330s
There is one remaining issue when --no-reuse-delta option is not
used. It can create delta chains that are deeper than specified.
A<--B<--C<--D E F G
Suppose we have a delta chain A to D (A is stored in full either
in a pack or as a loose object. B is depth1 delta relative to A,
C is depth2 delta relative to B...) with loose objects E, F, G.
And we are going to pack all of them.
B, C and D are left as delta against A, B and C respectively.
So A, E, F, and G are examined for deltification, and let's say
we decided to keep E expanded, and store the rest as deltas like
this:
E<--F<--G<--A
Oops. We ended up making D a bit too deep, didn't we? B, C and
D form a chain on top of A!
This is because we did not know what the final depth of A would
be, when we checked objects and decided to keep the existing
delta. Unfortunately, deferring the decision until just before
the deltification is not an option. To be able to make B, C,
and D candidates for deltification with the rest, we need to
know the type and final unexpanded size of them, but the major
part of the optimization comes from the fact that we do not read
the delta data to do so -- getting the final size is quite an
expensive operation.
To prevent this from happening, we should keep A from being
deltified. But how would we tell that, cheaply?
To do this most precisely, after check_object() runs, each
object that is used as the base object of some existing delta
needs to be marked with the maximum depth of the objects we
decided to keep deltified (in this case, D is depth 3 relative
to A, so if no other delta chain that is longer than 3 based on
A exists, mark A with 3). Then when attempting to deltify A, we
would take that number into account to see if the final delta
chain that leads to D becomes too deep.
However, this is a bit cumbersome to compute, so we would cheat
and reduce the maximum depth for A arbitrarily to depth/4 in
this implementation.
Signed-off-by: Junio C Hamano <junkio@cox.net>
2006-02-16 19:55:51 +00:00
|
|
|
*/
|
2008-10-29 23:02:46 +00:00
|
|
|
used = unpack_object_header_buffer(buf, avail,
|
2007-04-16 16:32:13 +00:00
|
|
|
&entry->in_pack_type,
|
|
|
|
&entry->size);
|
2008-10-29 23:02:48 +00:00
|
|
|
if (used == 0)
|
|
|
|
goto give_up;
|
pack-objects: finishing touches.
This introduces --no-reuse-delta option to disable reusing of
existing delta, which is a large part of the optimization
introduced by this series. This may become necessary if
repeated repacking makes delta chain too long. With this, the
output of the command becomes identical to that of the older
implementation. But the performance suffers greatly.
It still allows reusing non-deltified representations; there is
no point uncompressing and recompressing the whole text.
It also adds a couple more statistics output, while squelching
it under -q flag, which the last round forgot to do.
$ time old-git-pack-objects --stdout >/dev/null <RL
Generating pack...
Done counting 184141 objects.
Packing 184141 objects....................
real 12m8.530s user 11m1.450s sys 0m57.920s
$ time git-pack-objects --stdout >/dev/null <RL
Generating pack...
Done counting 184141 objects.
Packing 184141 objects.....................
Total 184141, written 184141 (delta 138297), reused 178833 (delta 134081)
real 0m59.549s user 0m56.670s sys 0m2.400s
$ time git-pack-objects --stdout --no-reuse-delta >/dev/null <RL
Generating pack...
Done counting 184141 objects.
Packing 184141 objects.....................
Total 184141, written 184141 (delta 134833), reused 47904 (delta 0)
real 11m13.830s user 9m45.240s sys 0m44.330s
There is one remaining issue when --no-reuse-delta option is not
used. It can create delta chains that are deeper than specified.
A<--B<--C<--D E F G
Suppose we have a delta chain A to D (A is stored in full either
in a pack or as a loose object. B is depth1 delta relative to A,
C is depth2 delta relative to B...) with loose objects E, F, G.
And we are going to pack all of them.
B, C and D are left as delta against A, B and C respectively.
So A, E, F, and G are examined for deltification, and let's say
we decided to keep E expanded, and store the rest as deltas like
this:
E<--F<--G<--A
Oops. We ended up making D a bit too deep, didn't we? B, C and
D form a chain on top of A!
This is because we did not know what the final depth of A would
be, when we checked objects and decided to keep the existing
delta. Unfortunately, deferring the decision until just before
the deltification is not an option. To be able to make B, C,
and D candidates for deltification with the rest, we need to
know the type and final unexpanded size of them, but the major
part of the optimization comes from the fact that we do not read
the delta data to do so -- getting the final size is quite an
expensive operation.
To prevent this from happening, we should keep A from being
deltified. But how would we tell that, cheaply?
To do this most precisely, after check_object() runs, each
object that is used as the base object of some existing delta
needs to be marked with the maximum depth of the objects we
decided to keep deltified (in this case, D is depth 3 relative
to A, so if no other delta chain that is longer than 3 based on
A exists, mark A with 3). Then when attempting to deltify A, we
would take that number into account to see if the final delta
chain that leads to D becomes too deep.
However, this is a bit cumbersome to compute, so we would cheat
and reduce the maximum depth for A arbitrarily to depth/4 in
this implementation.
Signed-off-by: Junio C Hamano <junkio@cox.net>
2006-02-16 19:55:51 +00:00
|
|
|
|
2007-04-16 16:32:13 +00:00
|
|
|
/*
|
|
|
|
* Determine if this is a delta and if so whether we can
|
|
|
|
* reuse it or not. Otherwise let's find out as cheaply as
|
|
|
|
* possible what the actual type and size for this object is.
|
pack-objects: reuse data from existing packs.
When generating a new pack, notice if we have already needed
objects in existing packs. If an object is stored deltified,
and its base object is also what we are going to pack, then
reuse the existing deltified representation unconditionally,
bypassing all the expensive find_deltas() and try_deltas()
calls.
Also, notice if what we are going to write out exactly match
what is already in an existing pack (either deltified or just
compressed). In such a case, we can just copy it instead of
going through the usual uncompressing & recompressing cycle.
Without this patch, in linux-2.6 repository with about 1500
loose objects and a single mega pack:
$ git-rev-list --objects v2.6.16-rc3 >RL
$ wc -l RL
184141 RL
$ time git-pack-objects p <RL
Generating pack...
Done counting 184141 objects.
Packing 184141 objects....................
a1fc7b3e537fcb9b3c46b7505df859f0a11e79d2
real 12m4.323s
user 11m2.560s
sys 0m55.950s
With this patch, the same input:
$ time ../git.junio/git-pack-objects q <RL
Generating pack...
Done counting 184141 objects.
Packing 184141 objects.....................
a1fc7b3e537fcb9b3c46b7505df859f0a11e79d2
Total 184141, written 184141, reused 182441
real 1m2.608s
user 0m55.090s
sys 0m1.830s
Signed-off-by: Junio C Hamano <junkio@cox.net>
2006-02-16 01:34:29 +00:00
|
|
|
*/
|
2007-04-16 16:32:13 +00:00
|
|
|
switch (entry->in_pack_type) {
|
|
|
|
default:
|
|
|
|
/* Not a delta hence we've already got all we need. */
|
|
|
|
entry->type = entry->in_pack_type;
|
|
|
|
entry->in_pack_header_size = used;
|
2008-10-29 23:02:48 +00:00
|
|
|
if (entry->type < OBJ_COMMIT || entry->type > OBJ_BLOB)
|
|
|
|
goto give_up;
|
2007-04-16 16:32:13 +00:00
|
|
|
unuse_pack(&w_curs);
|
|
|
|
return;
|
|
|
|
case OBJ_REF_DELTA:
|
2008-05-02 19:11:46 +00:00
|
|
|
if (reuse_delta && !entry->preferred_base)
|
2007-04-16 16:32:13 +00:00
|
|
|
base_ref = use_pack(p, &w_curs,
|
|
|
|
entry->in_pack_offset + used, NULL);
|
|
|
|
entry->in_pack_header_size = used + 20;
|
|
|
|
break;
|
|
|
|
case OBJ_OFS_DELTA:
|
|
|
|
buf = use_pack(p, &w_curs,
|
|
|
|
entry->in_pack_offset + used, NULL);
|
|
|
|
used_0 = 0;
|
|
|
|
c = buf[used_0++];
|
|
|
|
ofs = c & 127;
|
|
|
|
while (c & 128) {
|
|
|
|
ofs += 1;
|
2008-10-29 23:02:48 +00:00
|
|
|
if (!ofs || MSB(ofs, 7)) {
|
|
|
|
error("delta base offset overflow in pack for %s",
|
|
|
|
sha1_to_hex(entry->idx.sha1));
|
|
|
|
goto give_up;
|
|
|
|
}
|
2007-04-16 16:32:13 +00:00
|
|
|
c = buf[used_0++];
|
|
|
|
ofs = (ofs << 7) + (c & 127);
|
2006-09-23 01:25:04 +00:00
|
|
|
}
|
2007-04-16 16:32:13 +00:00
|
|
|
ofs = entry->in_pack_offset - ofs;
|
2008-10-29 23:02:48 +00:00
|
|
|
if (ofs <= 0 || ofs >= entry->in_pack_offset) {
|
|
|
|
error("delta base offset out of bound for %s",
|
|
|
|
sha1_to_hex(entry->idx.sha1));
|
|
|
|
goto give_up;
|
|
|
|
}
|
2008-05-02 19:11:46 +00:00
|
|
|
if (reuse_delta && !entry->preferred_base) {
|
2008-02-28 05:25:17 +00:00
|
|
|
struct revindex_entry *revidx;
|
|
|
|
revidx = find_pack_revindex(p, ofs);
|
2008-10-29 23:02:49 +00:00
|
|
|
if (!revidx)
|
|
|
|
goto give_up;
|
2008-02-28 05:25:17 +00:00
|
|
|
base_ref = nth_packed_object_sha1(p, revidx->nr);
|
|
|
|
}
|
2007-04-16 16:32:13 +00:00
|
|
|
entry->in_pack_header_size = used + used_0;
|
|
|
|
break;
|
2006-09-23 01:25:04 +00:00
|
|
|
}
|
|
|
|
|
2007-04-16 16:32:13 +00:00
|
|
|
if (base_ref && (base_entry = locate_object_entry(base_ref))) {
|
|
|
|
/*
|
|
|
|
* If base_ref was set above that means we wish to
|
|
|
|
* reuse delta data, and we even found that base
|
|
|
|
* in the list of objects we want to pack. Goodie!
|
|
|
|
*
|
|
|
|
* Depth value does not matter - find_deltas() will
|
|
|
|
* never consider reused delta as the base object to
|
|
|
|
* deltify other objects against, in order to avoid
|
|
|
|
* circular deltas.
|
pack-objects: reuse data from existing packs.
When generating a new pack, notice if we have already needed
objects in existing packs. If an object is stored deltified,
and its base object is also what we are going to pack, then
reuse the existing deltified representation unconditionally,
bypassing all the expensive find_deltas() and try_deltas()
calls.
Also, notice if what we are going to write out exactly match
what is already in an existing pack (either deltified or just
compressed). In such a case, we can just copy it instead of
going through the usual uncompressing & recompressing cycle.
Without this patch, in linux-2.6 repository with about 1500
loose objects and a single mega pack:
$ git-rev-list --objects v2.6.16-rc3 >RL
$ wc -l RL
184141 RL
$ time git-pack-objects p <RL
Generating pack...
Done counting 184141 objects.
Packing 184141 objects....................
a1fc7b3e537fcb9b3c46b7505df859f0a11e79d2
real 12m4.323s
user 11m2.560s
sys 0m55.950s
With this patch, the same input:
$ time ../git.junio/git-pack-objects q <RL
Generating pack...
Done counting 184141 objects.
Packing 184141 objects.....................
a1fc7b3e537fcb9b3c46b7505df859f0a11e79d2
Total 184141, written 184141, reused 182441
real 1m2.608s
user 0m55.090s
sys 0m1.830s
Signed-off-by: Junio C Hamano <junkio@cox.net>
2006-02-16 01:34:29 +00:00
|
|
|
*/
|
2006-09-23 01:25:04 +00:00
|
|
|
entry->type = entry->in_pack_type;
|
2007-04-16 16:32:13 +00:00
|
|
|
entry->delta = base_entry;
|
2008-10-29 23:02:50 +00:00
|
|
|
entry->delta_size = entry->size;
|
2006-02-18 04:58:45 +00:00
|
|
|
entry->delta_sibling = base_entry->delta_child;
|
|
|
|
base_entry->delta_child = entry;
|
2007-04-16 16:32:13 +00:00
|
|
|
unuse_pack(&w_curs);
|
|
|
|
return;
|
|
|
|
}
|
pack-objects: finishing touches.
This introduces --no-reuse-delta option to disable reusing of
existing delta, which is a large part of the optimization
introduced by this series. This may become necessary if
repeated repacking makes delta chain too long. With this, the
output of the command becomes identical to that of the older
implementation. But the performance suffers greatly.
It still allows reusing non-deltified representations; there is
no point uncompressing and recompressing the whole text.
It also adds a couple more statistics output, while squelching
it under -q flag, which the last round forgot to do.
$ time old-git-pack-objects --stdout >/dev/null <RL
Generating pack...
Done counting 184141 objects.
Packing 184141 objects....................
real 12m8.530s user 11m1.450s sys 0m57.920s
$ time git-pack-objects --stdout >/dev/null <RL
Generating pack...
Done counting 184141 objects.
Packing 184141 objects.....................
Total 184141, written 184141 (delta 138297), reused 178833 (delta 134081)
real 0m59.549s user 0m56.670s sys 0m2.400s
$ time git-pack-objects --stdout --no-reuse-delta >/dev/null <RL
Generating pack...
Done counting 184141 objects.
Packing 184141 objects.....................
Total 184141, written 184141 (delta 134833), reused 47904 (delta 0)
real 11m13.830s user 9m45.240s sys 0m44.330s
There is one remaining issue when --no-reuse-delta option is not
used. It can create delta chains that are deeper than specified.
A<--B<--C<--D E F G
Suppose we have a delta chain A to D (A is stored in full either
in a pack or as a loose object. B is depth1 delta relative to A,
C is depth2 delta relative to B...) with loose objects E, F, G.
And we are going to pack all of them.
B, C and D are left as delta against A, B and C respectively.
So A, E, F, and G are examined for deltification, and let's say
we decided to keep E expanded, and store the rest as deltas like
this:
E<--F<--G<--A
Oops. We ended up making D a bit too deep, didn't we? B, C and
D form a chain on top of A!
This is because we did not know what the final depth of A would
be, when we checked objects and decided to keep the existing
delta. Unfortunately, deferring the decision until just before
the deltification is not an option. To be able to make B, C,
and D candidates for deltification with the rest, we need to
know the type and final unexpanded size of them, but the major
part of the optimization comes from the fact that we do not read
the delta data to do so -- getting the final size is quite an
expensive operation.
To prevent this from happening, we should keep A from being
deltified. But how would we tell that, cheaply?
To do this most precisely, after check_object() runs, each
object that is used as the base object of some existing delta
needs to be marked with the maximum depth of the objects we
decided to keep deltified (in this case, D is depth 3 relative
to A, so if no other delta chain that is longer than 3 based on
A exists, mark A with 3). Then when attempting to deltify A, we
would take that number into account to see if the final delta
chain that leads to D becomes too deep.
However, this is a bit cumbersome to compute, so we would cheat
and reduce the maximum depth for A arbitrarily to depth/4 in
this implementation.
Signed-off-by: Junio C Hamano <junkio@cox.net>
2006-02-16 19:55:51 +00:00
|
|
|
|
2007-04-16 16:32:13 +00:00
|
|
|
if (entry->type) {
|
|
|
|
/*
|
|
|
|
* This must be a delta and we already know what the
|
|
|
|
* final object type is. Let's extract the actual
|
|
|
|
* object size from the delta header.
|
|
|
|
*/
|
|
|
|
entry->size = get_size_from_delta(p, &w_curs,
|
|
|
|
entry->in_pack_offset + entry->in_pack_header_size);
|
2008-10-29 23:02:48 +00:00
|
|
|
if (entry->size == 0)
|
|
|
|
goto give_up;
|
2007-04-16 16:32:13 +00:00
|
|
|
unuse_pack(&w_curs);
|
pack-objects: reuse data from existing packs.
When generating a new pack, notice if we have already needed
objects in existing packs. If an object is stored deltified,
and its base object is also what we are going to pack, then
reuse the existing deltified representation unconditionally,
bypassing all the expensive find_deltas() and try_deltas()
calls.
Also, notice if what we are going to write out exactly match
what is already in an existing pack (either deltified or just
compressed). In such a case, we can just copy it instead of
going through the usual uncompressing & recompressing cycle.
Without this patch, in linux-2.6 repository with about 1500
loose objects and a single mega pack:
$ git-rev-list --objects v2.6.16-rc3 >RL
$ wc -l RL
184141 RL
$ time git-pack-objects p <RL
Generating pack...
Done counting 184141 objects.
Packing 184141 objects....................
a1fc7b3e537fcb9b3c46b7505df859f0a11e79d2
real 12m4.323s
user 11m2.560s
sys 0m55.950s
With this patch, the same input:
$ time ../git.junio/git-pack-objects q <RL
Generating pack...
Done counting 184141 objects.
Packing 184141 objects.....................
a1fc7b3e537fcb9b3c46b7505df859f0a11e79d2
Total 184141, written 184141, reused 182441
real 1m2.608s
user 0m55.090s
sys 0m1.830s
Signed-off-by: Junio C Hamano <junkio@cox.net>
2006-02-16 01:34:29 +00:00
|
|
|
return;
|
|
|
|
}
|
2007-04-16 16:32:13 +00:00
|
|
|
|
|
|
|
/*
|
|
|
|
* No choice but to fall back to the recursive delta walk
|
|
|
|
* with sha1_object_info() to find about the object type
|
|
|
|
* at this point...
|
|
|
|
*/
|
2008-10-29 23:02:48 +00:00
|
|
|
give_up:
|
2007-04-16 16:32:13 +00:00
|
|
|
unuse_pack(&w_curs);
|
2005-06-27 10:34:06 +00:00
|
|
|
}
|
pack-objects: reuse data from existing packs.
When generating a new pack, notice if we have already needed
objects in existing packs. If an object is stored deltified,
and its base object is also what we are going to pack, then
reuse the existing deltified representation unconditionally,
bypassing all the expensive find_deltas() and try_deltas()
calls.
Also, notice if what we are going to write out exactly match
what is already in an existing pack (either deltified or just
compressed). In such a case, we can just copy it instead of
going through the usual uncompressing & recompressing cycle.
Without this patch, in linux-2.6 repository with about 1500
loose objects and a single mega pack:
$ git-rev-list --objects v2.6.16-rc3 >RL
$ wc -l RL
184141 RL
$ time git-pack-objects p <RL
Generating pack...
Done counting 184141 objects.
Packing 184141 objects....................
a1fc7b3e537fcb9b3c46b7505df859f0a11e79d2
real 12m4.323s
user 11m2.560s
sys 0m55.950s
With this patch, the same input:
$ time ../git.junio/git-pack-objects q <RL
Generating pack...
Done counting 184141 objects.
Packing 184141 objects.....................
a1fc7b3e537fcb9b3c46b7505df859f0a11e79d2
Total 184141, written 184141, reused 182441
real 1m2.608s
user 0m55.090s
sys 0m1.830s
Signed-off-by: Junio C Hamano <junkio@cox.net>
2006-02-16 01:34:29 +00:00
|
|
|
|
2007-06-01 19:18:05 +00:00
|
|
|
entry->type = sha1_object_info(entry->idx.sha1, &entry->size);
|
2008-08-12 18:31:06 +00:00
|
|
|
/*
|
|
|
|
* The error condition is checked in prepare_pack(). This is
|
|
|
|
* to permit a missing preferred base object to be ignored
|
|
|
|
* as a preferred base. Doing so can result in a larger
|
|
|
|
* pack file, but the transfer will still take place.
|
|
|
|
*/
|
pack-objects: reuse data from existing packs.
When generating a new pack, notice if we have already needed
objects in existing packs. If an object is stored deltified,
and its base object is also what we are going to pack, then
reuse the existing deltified representation unconditionally,
bypassing all the expensive find_deltas() and try_deltas()
calls.
Also, notice if what we are going to write out exactly match
what is already in an existing pack (either deltified or just
compressed). In such a case, we can just copy it instead of
going through the usual uncompressing & recompressing cycle.
Without this patch, in linux-2.6 repository with about 1500
loose objects and a single mega pack:
$ git-rev-list --objects v2.6.16-rc3 >RL
$ wc -l RL
184141 RL
$ time git-pack-objects p <RL
Generating pack...
Done counting 184141 objects.
Packing 184141 objects....................
a1fc7b3e537fcb9b3c46b7505df859f0a11e79d2
real 12m4.323s
user 11m2.560s
sys 0m55.950s
With this patch, the same input:
$ time ../git.junio/git-pack-objects q <RL
Generating pack...
Done counting 184141 objects.
Packing 184141 objects.....................
a1fc7b3e537fcb9b3c46b7505df859f0a11e79d2
Total 184141, written 184141, reused 182441
real 1m2.608s
user 0m55.090s
sys 0m1.830s
Signed-off-by: Junio C Hamano <junkio@cox.net>
2006-02-16 01:34:29 +00:00
|
|
|
}
|
|
|
|
|
2007-04-16 16:32:13 +00:00
|
|
|
static int pack_offset_sort(const void *_a, const void *_b)
|
|
|
|
{
|
|
|
|
const struct object_entry *a = *(struct object_entry **)_a;
|
|
|
|
const struct object_entry *b = *(struct object_entry **)_b;
|
|
|
|
|
|
|
|
/* avoid filesystem trashing with loose objects */
|
|
|
|
if (!a->in_pack && !b->in_pack)
|
2007-06-01 19:18:05 +00:00
|
|
|
return hashcmp(a->idx.sha1, b->idx.sha1);
|
2007-04-16 16:32:13 +00:00
|
|
|
|
|
|
|
if (a->in_pack < b->in_pack)
|
|
|
|
return -1;
|
|
|
|
if (a->in_pack > b->in_pack)
|
|
|
|
return 1;
|
|
|
|
return a->in_pack_offset < b->in_pack_offset ? -1 :
|
|
|
|
(a->in_pack_offset > b->in_pack_offset);
|
|
|
|
}
|
|
|
|
|
2005-06-25 21:42:43 +00:00
|
|
|
static void get_object_details(void)
|
|
|
|
{
|
2007-03-07 01:44:24 +00:00
|
|
|
uint32_t i;
|
2007-04-16 16:32:13 +00:00
|
|
|
struct object_entry **sorted_by_offset;
|
|
|
|
|
|
|
|
sorted_by_offset = xcalloc(nr_objects, sizeof(struct object_entry *));
|
|
|
|
for (i = 0; i < nr_objects; i++)
|
|
|
|
sorted_by_offset[i] = objects + i;
|
|
|
|
qsort(sorted_by_offset, nr_objects, sizeof(*sorted_by_offset), pack_offset_sort);
|
2005-06-25 21:42:43 +00:00
|
|
|
|
2007-04-16 16:32:13 +00:00
|
|
|
for (i = 0; i < nr_objects; i++)
|
|
|
|
check_object(sorted_by_offset[i]);
|
2008-02-28 05:25:17 +00:00
|
|
|
|
2007-04-16 16:32:13 +00:00
|
|
|
free(sorted_by_offset);
|
2005-06-25 21:42:43 +00:00
|
|
|
}
|
|
|
|
|
2007-12-08 05:00:08 +00:00
|
|
|
/*
|
|
|
|
* We search for deltas in a list sorted by type, by filename hash, and then
|
|
|
|
* by size, so that we see progressively smaller and smaller files.
|
|
|
|
* That's because we prefer deltas to be from the bigger file
|
|
|
|
* to the smaller -- deletes are potentially cheaper, but perhaps
|
|
|
|
* more importantly, the bigger file is likely the more recent
|
|
|
|
* one. The deepest deltas are therefore the oldest objects which are
|
|
|
|
* less susceptible to be accessed often.
|
|
|
|
*/
|
2007-04-16 16:29:54 +00:00
|
|
|
static int type_size_sort(const void *_a, const void *_b)
|
2005-06-25 21:42:43 +00:00
|
|
|
{
|
2007-04-16 16:29:54 +00:00
|
|
|
const struct object_entry *a = *(struct object_entry **)_a;
|
|
|
|
const struct object_entry *b = *(struct object_entry **)_b;
|
|
|
|
|
2005-06-25 21:42:43 +00:00
|
|
|
if (a->type > b->type)
|
2005-06-26 22:27:28 +00:00
|
|
|
return -1;
|
2007-12-08 05:00:08 +00:00
|
|
|
if (a->type < b->type)
|
2005-06-26 22:27:28 +00:00
|
|
|
return 1;
|
2007-12-08 05:00:08 +00:00
|
|
|
if (a->hash > b->hash)
|
2006-02-19 22:47:21 +00:00
|
|
|
return -1;
|
2007-12-08 05:00:08 +00:00
|
|
|
if (a->hash < b->hash)
|
2006-02-19 22:47:21 +00:00
|
|
|
return 1;
|
2007-12-08 05:00:08 +00:00
|
|
|
if (a->preferred_base > b->preferred_base)
|
2005-06-25 21:42:43 +00:00
|
|
|
return -1;
|
2007-12-08 05:00:08 +00:00
|
|
|
if (a->preferred_base < b->preferred_base)
|
|
|
|
return 1;
|
2005-06-25 21:42:43 +00:00
|
|
|
if (a->size > b->size)
|
2007-12-08 05:00:08 +00:00
|
|
|
return -1;
|
|
|
|
if (a->size < b->size)
|
2005-06-25 21:42:43 +00:00
|
|
|
return 1;
|
2007-12-08 05:00:08 +00:00
|
|
|
return a < b ? -1 : (a > b); /* newest first */
|
2005-06-25 21:42:43 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
struct unpacked {
|
|
|
|
struct object_entry *entry;
|
|
|
|
void *data;
|
2006-04-27 03:58:00 +00:00
|
|
|
struct delta_index *index;
|
2007-07-12 21:07:59 +00:00
|
|
|
unsigned depth;
|
2005-06-25 21:42:43 +00:00
|
|
|
};
|
|
|
|
|
2007-08-16 02:46:01 +00:00
|
|
|
static int delta_cacheable(unsigned long src_size, unsigned long trg_size,
|
|
|
|
unsigned long delta_size)
|
2007-05-28 21:20:58 +00:00
|
|
|
{
|
|
|
|
if (max_delta_cache_size && delta_cache_size + delta_size > max_delta_cache_size)
|
|
|
|
return 0;
|
|
|
|
|
2007-05-28 21:20:59 +00:00
|
|
|
if (delta_size < cache_max_small_delta_size)
|
|
|
|
return 1;
|
|
|
|
|
2007-05-28 21:20:58 +00:00
|
|
|
/* cache delta, if objects are large enough compared to delta size */
|
|
|
|
if ((src_size >> 20) + (trg_size >> 21) > (delta_size >> 10))
|
|
|
|
return 1;
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2007-09-06 06:13:11 +00:00
|
|
|
#ifdef THREADED_DELTA_SEARCH
|
|
|
|
|
|
|
|
static pthread_mutex_t read_mutex = PTHREAD_MUTEX_INITIALIZER;
|
|
|
|
#define read_lock() pthread_mutex_lock(&read_mutex)
|
|
|
|
#define read_unlock() pthread_mutex_unlock(&read_mutex)
|
|
|
|
|
2007-09-10 15:10:11 +00:00
|
|
|
static pthread_mutex_t cache_mutex = PTHREAD_MUTEX_INITIALIZER;
|
|
|
|
#define cache_lock() pthread_mutex_lock(&cache_mutex)
|
|
|
|
#define cache_unlock() pthread_mutex_unlock(&cache_mutex)
|
|
|
|
|
2007-09-06 06:13:11 +00:00
|
|
|
static pthread_mutex_t progress_mutex = PTHREAD_MUTEX_INITIALIZER;
|
|
|
|
#define progress_lock() pthread_mutex_lock(&progress_mutex)
|
|
|
|
#define progress_unlock() pthread_mutex_unlock(&progress_mutex)
|
|
|
|
|
|
|
|
#else
|
|
|
|
|
2007-09-15 05:30:20 +00:00
|
|
|
#define read_lock() (void)0
|
|
|
|
#define read_unlock() (void)0
|
|
|
|
#define cache_lock() (void)0
|
|
|
|
#define cache_unlock() (void)0
|
|
|
|
#define progress_lock() (void)0
|
|
|
|
#define progress_unlock() (void)0
|
2007-09-06 06:13:11 +00:00
|
|
|
|
|
|
|
#endif
|
|
|
|
|
2006-04-27 03:58:00 +00:00
|
|
|
static int try_delta(struct unpacked *trg, struct unpacked *src,
|
2007-09-06 06:13:09 +00:00
|
|
|
unsigned max_depth, unsigned long *mem_usage)
|
2005-06-25 21:42:43 +00:00
|
|
|
{
|
2006-04-27 03:58:00 +00:00
|
|
|
struct object_entry *trg_entry = trg->entry;
|
|
|
|
struct object_entry *src_entry = src->entry;
|
2006-07-01 02:55:30 +00:00
|
|
|
unsigned long trg_size, src_size, delta_size, sizediff, max_size, sz;
|
2007-07-12 18:33:21 +00:00
|
|
|
unsigned ref_depth;
|
2007-02-26 19:55:59 +00:00
|
|
|
enum object_type type;
|
2005-06-25 21:42:43 +00:00
|
|
|
void *delta_buf;
|
|
|
|
|
|
|
|
/* Don't bother doing diffs between different types */
|
2006-04-27 03:58:00 +00:00
|
|
|
if (trg_entry->type != src_entry->type)
|
2005-06-25 21:42:43 +00:00
|
|
|
return -1;
|
|
|
|
|
2006-06-29 21:04:01 +00:00
|
|
|
/*
|
|
|
|
* We do not bother to try a delta that we discarded
|
2006-06-30 03:44:52 +00:00
|
|
|
* on an earlier try, but only when reusing delta data.
|
2006-06-29 21:04:01 +00:00
|
|
|
*/
|
2008-05-02 19:11:46 +00:00
|
|
|
if (reuse_delta && trg_entry->in_pack &&
|
2006-11-15 06:18:31 +00:00
|
|
|
trg_entry->in_pack == src_entry->in_pack &&
|
|
|
|
trg_entry->in_pack_type != OBJ_REF_DELTA &&
|
|
|
|
trg_entry->in_pack_type != OBJ_OFS_DELTA)
|
2006-06-29 21:04:01 +00:00
|
|
|
return 0;
|
|
|
|
|
2007-04-16 16:29:16 +00:00
|
|
|
/* Let's not bust the allowed depth. */
|
2007-07-12 21:07:59 +00:00
|
|
|
if (src->depth >= max_depth)
|
2005-06-26 03:17:59 +00:00
|
|
|
return 0;
|
2005-06-25 21:42:43 +00:00
|
|
|
|
2006-05-16 20:29:14 +00:00
|
|
|
/* Now some size filtering heuristics. */
|
2006-07-01 02:55:30 +00:00
|
|
|
trg_size = trg_entry->size;
|
2007-07-12 18:33:21 +00:00
|
|
|
if (!trg_entry->delta) {
|
|
|
|
max_size = trg_size/2 - 20;
|
|
|
|
ref_depth = 1;
|
|
|
|
} else {
|
|
|
|
max_size = trg_entry->delta_size;
|
2007-07-12 21:07:59 +00:00
|
|
|
ref_depth = trg->depth;
|
2007-07-12 18:33:21 +00:00
|
|
|
}
|
2009-03-24 19:56:12 +00:00
|
|
|
max_size = (uint64_t)max_size * (max_depth - src->depth) /
|
2007-07-12 18:33:21 +00:00
|
|
|
(max_depth - ref_depth + 1);
|
2006-05-16 20:29:14 +00:00
|
|
|
if (max_size == 0)
|
|
|
|
return 0;
|
2006-04-27 03:58:00 +00:00
|
|
|
src_size = src_entry->size;
|
2006-07-01 02:55:30 +00:00
|
|
|
sizediff = src_size < trg_size ? trg_size - src_size : 0;
|
2005-06-26 22:27:28 +00:00
|
|
|
if (sizediff >= max_size)
|
2006-04-21 06:36:22 +00:00
|
|
|
return 0;
|
2007-07-12 12:55:47 +00:00
|
|
|
if (trg_size < src_size / 32)
|
|
|
|
return 0;
|
2006-04-27 03:58:00 +00:00
|
|
|
|
2006-07-01 02:55:30 +00:00
|
|
|
/* Load data if not already done */
|
|
|
|
if (!trg->data) {
|
2007-09-06 06:13:11 +00:00
|
|
|
read_lock();
|
2007-06-01 19:18:05 +00:00
|
|
|
trg->data = read_sha1_file(trg_entry->idx.sha1, &type, &sz);
|
2007-09-06 06:13:11 +00:00
|
|
|
read_unlock();
|
2007-08-25 08:26:47 +00:00
|
|
|
if (!trg->data)
|
|
|
|
die("object %s cannot be read",
|
|
|
|
sha1_to_hex(trg_entry->idx.sha1));
|
2006-07-01 02:55:30 +00:00
|
|
|
if (sz != trg_size)
|
|
|
|
die("object %s inconsistent object length (%lu vs %lu)",
|
2007-06-01 19:18:05 +00:00
|
|
|
sha1_to_hex(trg_entry->idx.sha1), sz, trg_size);
|
2007-09-06 06:13:09 +00:00
|
|
|
*mem_usage += sz;
|
2006-07-01 02:55:30 +00:00
|
|
|
}
|
|
|
|
if (!src->data) {
|
2007-09-06 06:13:11 +00:00
|
|
|
read_lock();
|
2007-06-01 19:18:05 +00:00
|
|
|
src->data = read_sha1_file(src_entry->idx.sha1, &type, &sz);
|
2007-09-06 06:13:11 +00:00
|
|
|
read_unlock();
|
2007-08-25 08:26:47 +00:00
|
|
|
if (!src->data)
|
|
|
|
die("object %s cannot be read",
|
|
|
|
sha1_to_hex(src_entry->idx.sha1));
|
2006-07-01 02:55:30 +00:00
|
|
|
if (sz != src_size)
|
|
|
|
die("object %s inconsistent object length (%lu vs %lu)",
|
2007-06-01 19:18:05 +00:00
|
|
|
sha1_to_hex(src_entry->idx.sha1), sz, src_size);
|
2007-09-06 06:13:09 +00:00
|
|
|
*mem_usage += sz;
|
2006-07-01 02:55:30 +00:00
|
|
|
}
|
|
|
|
if (!src->index) {
|
|
|
|
src->index = create_delta_index(src->data, src_size);
|
2007-05-28 21:20:57 +00:00
|
|
|
if (!src->index) {
|
|
|
|
static int warned = 0;
|
|
|
|
if (!warned++)
|
|
|
|
warning("suboptimal pack - out of memory");
|
|
|
|
return 0;
|
|
|
|
}
|
2007-09-06 06:13:09 +00:00
|
|
|
*mem_usage += sizeof_delta_index(src->index);
|
2006-07-01 02:55:30 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
delta_buf = create_delta(src->index, trg->data, trg_size, &delta_size, max_size);
|
2005-06-25 21:42:43 +00:00
|
|
|
if (!delta_buf)
|
2005-06-26 02:30:20 +00:00
|
|
|
return 0;
|
2006-04-27 03:58:00 +00:00
|
|
|
|
2007-08-30 01:17:17 +00:00
|
|
|
if (trg_entry->delta) {
|
2007-07-09 04:45:21 +00:00
|
|
|
/* Prefer only shallower same-sized deltas. */
|
|
|
|
if (delta_size == trg_entry->delta_size &&
|
2007-07-12 21:07:59 +00:00
|
|
|
src->depth + 1 >= trg->depth) {
|
2007-07-09 04:45:21 +00:00
|
|
|
free(delta_buf);
|
|
|
|
return 0;
|
|
|
|
}
|
2007-05-28 21:20:58 +00:00
|
|
|
}
|
2007-08-30 01:17:17 +00:00
|
|
|
|
2007-09-10 15:10:11 +00:00
|
|
|
/*
|
|
|
|
* Handle memory allocation outside of the cache
|
|
|
|
* accounting lock. Compiler will optimize the strangeness
|
|
|
|
* away when THREADED_DELTA_SEARCH is not defined.
|
|
|
|
*/
|
Avoid unnecessary "if-before-free" tests.
This change removes all obvious useless if-before-free tests.
E.g., it replaces code like this:
if (some_expression)
free (some_expression);
with the now-equivalent:
free (some_expression);
It is equivalent not just because POSIX has required free(NULL)
to work for a long time, but simply because it has worked for
so long that no reasonable porting target fails the test.
Here's some evidence from nearly 1.5 years ago:
http://www.winehq.org/pipermail/wine-patches/2006-October/031544.html
FYI, the change below was prepared by running the following:
git ls-files -z | xargs -0 \
perl -0x3b -pi -e \
's/\bif\s*\(\s*(\S+?)(?:\s*!=\s*NULL)?\s*\)\s+(free\s*\(\s*\1\s*\))/$2/s'
Note however, that it doesn't handle brace-enclosed blocks like
"if (x) { free (x); }". But that's ok, since there were none like
that in git sources.
Beware: if you do use the above snippet, note that it can
produce syntactically invalid C code. That happens when the
affected "if"-statement has a matching "else".
E.g., it would transform this
if (x)
free (x);
else
foo ();
into this:
free (x);
else
foo ();
There were none of those here, either.
If you're interested in automating detection of the useless
tests, you might like the useless-if-before-free script in gnulib:
[it *does* detect brace-enclosed free statements, and has a --name=S
option to make it detect free-like functions with different names]
http://git.sv.gnu.org/gitweb/?p=gnulib.git;a=blob;f=build-aux/useless-if-before-free
Addendum:
Remove one more (in imap-send.c), spotted by Jean-Luc Herren <jlh@gmx.ch>.
Signed-off-by: Jim Meyering <meyering@redhat.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2008-01-31 17:26:32 +00:00
|
|
|
free(trg_entry->delta_data);
|
2007-09-10 15:10:11 +00:00
|
|
|
cache_lock();
|
2007-08-30 01:17:17 +00:00
|
|
|
if (trg_entry->delta_data) {
|
|
|
|
delta_cache_size -= trg_entry->delta_size;
|
|
|
|
trg_entry->delta_data = NULL;
|
|
|
|
}
|
2007-08-16 02:46:01 +00:00
|
|
|
if (delta_cacheable(src_size, trg_size, delta_size)) {
|
2007-12-08 01:27:52 +00:00
|
|
|
delta_cache_size += delta_size;
|
2007-09-10 15:10:11 +00:00
|
|
|
cache_unlock();
|
|
|
|
trg_entry->delta_data = xrealloc(delta_buf, delta_size);
|
|
|
|
} else {
|
|
|
|
cache_unlock();
|
2007-05-28 21:20:58 +00:00
|
|
|
free(delta_buf);
|
2007-09-10 15:10:11 +00:00
|
|
|
}
|
|
|
|
|
2007-12-08 01:27:52 +00:00
|
|
|
trg_entry->delta = src_entry;
|
|
|
|
trg_entry->delta_size = delta_size;
|
|
|
|
trg->depth = src->depth + 1;
|
|
|
|
|
2006-04-27 03:58:00 +00:00
|
|
|
return 1;
|
2005-06-25 21:42:43 +00:00
|
|
|
}
|
|
|
|
|
2007-04-16 16:29:16 +00:00
|
|
|
static unsigned int check_delta_limit(struct object_entry *me, unsigned int n)
|
2006-02-22 21:00:08 +00:00
|
|
|
{
|
2007-04-16 16:29:16 +00:00
|
|
|
struct object_entry *child = me->delta_child;
|
|
|
|
unsigned int m = n;
|
|
|
|
while (child) {
|
|
|
|
unsigned int c = check_delta_limit(child, n + 1);
|
|
|
|
if (m < c)
|
|
|
|
m = c;
|
|
|
|
child = child->delta_sibling;
|
|
|
|
}
|
|
|
|
return m;
|
2006-02-22 21:00:08 +00:00
|
|
|
}
|
|
|
|
|
2008-02-13 07:39:03 +00:00
|
|
|
static unsigned long free_unpacked(struct unpacked *n)
|
2007-07-12 13:07:46 +00:00
|
|
|
{
|
2007-09-06 06:13:09 +00:00
|
|
|
unsigned long freed_mem = sizeof_delta_index(n->index);
|
2007-07-12 13:07:46 +00:00
|
|
|
free_delta_index(n->index);
|
|
|
|
n->index = NULL;
|
|
|
|
if (n->data) {
|
2007-09-06 06:13:09 +00:00
|
|
|
freed_mem += n->entry->size;
|
2007-07-12 13:07:46 +00:00
|
|
|
free(n->data);
|
|
|
|
n->data = NULL;
|
|
|
|
}
|
|
|
|
n->entry = NULL;
|
2007-07-13 02:27:12 +00:00
|
|
|
n->depth = 0;
|
2007-09-06 06:13:09 +00:00
|
|
|
return freed_mem;
|
2007-07-12 13:07:46 +00:00
|
|
|
}
|
|
|
|
|
2007-12-08 05:03:17 +00:00
|
|
|
static void find_deltas(struct object_entry **list, unsigned *list_size,
|
2007-09-06 06:13:10 +00:00
|
|
|
int window, int depth, unsigned *processed)
|
2005-06-25 21:42:43 +00:00
|
|
|
{
|
2007-12-08 05:03:17 +00:00
|
|
|
uint32_t i, idx = 0, count = 0;
|
2007-03-07 01:44:24 +00:00
|
|
|
struct unpacked *array;
|
2007-09-06 06:13:09 +00:00
|
|
|
unsigned long mem_usage = 0;
|
2005-06-25 21:42:43 +00:00
|
|
|
|
2008-10-06 23:39:10 +00:00
|
|
|
array = xcalloc(window, sizeof(struct unpacked));
|
2006-02-12 01:54:18 +00:00
|
|
|
|
2007-12-08 05:03:17 +00:00
|
|
|
for (;;) {
|
pack-objects: avoid reading uninitalized data
In the main loop of find_deltas, we do:
struct object_entry *entry = *list++;
...
if (!*list_size)
...
break
Because we look at and increment *list _before_ the check of
list_size, in the very last iteration of the loop we will
look at uninitialized data, and increment the pointer beyond
one past the end of the allocated space. Since we don't
actually do anything with the data until after the check,
this is not a problem in practice.
But since it technically violates the C standard, and
because it provokes a spurious valgrind warning, let's just
move the initialization of entry to a safe place.
This fixes valgrind errors in t5300, t5301, t5302, t303, and
t9400.
Signed-off-by: Jeff King <peff@peff.net>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2008-10-23 04:31:03 +00:00
|
|
|
struct object_entry *entry;
|
2005-06-25 21:42:43 +00:00
|
|
|
struct unpacked *n = array + idx;
|
2007-09-06 06:13:09 +00:00
|
|
|
int j, max_depth, best_base = -1;
|
2005-06-25 21:42:43 +00:00
|
|
|
|
2007-12-08 05:03:17 +00:00
|
|
|
progress_lock();
|
|
|
|
if (!*list_size) {
|
|
|
|
progress_unlock();
|
|
|
|
break;
|
|
|
|
}
|
pack-objects: avoid reading uninitalized data
In the main loop of find_deltas, we do:
struct object_entry *entry = *list++;
...
if (!*list_size)
...
break
Because we look at and increment *list _before_ the check of
list_size, in the very last iteration of the loop we will
look at uninitialized data, and increment the pointer beyond
one past the end of the allocated space. Since we don't
actually do anything with the data until after the check,
this is not a problem in practice.
But since it technically violates the C standard, and
because it provokes a spurious valgrind warning, let's just
move the initialization of entry to a safe place.
This fixes valgrind errors in t5300, t5301, t5302, t303, and
t9400.
Signed-off-by: Jeff King <peff@peff.net>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2008-10-23 04:31:03 +00:00
|
|
|
entry = *list++;
|
2007-12-08 05:03:17 +00:00
|
|
|
(*list_size)--;
|
|
|
|
if (!entry->preferred_base) {
|
|
|
|
(*processed)++;
|
|
|
|
display_progress(progress_state, *processed);
|
|
|
|
}
|
|
|
|
progress_unlock();
|
|
|
|
|
2007-09-06 06:13:09 +00:00
|
|
|
mem_usage -= free_unpacked(n);
|
2005-06-25 21:42:43 +00:00
|
|
|
n->entry = entry;
|
pack-objects: finishing touches.
This introduces --no-reuse-delta option to disable reusing of
existing delta, which is a large part of the optimization
introduced by this series. This may become necessary if
repeated repacking makes delta chain too long. With this, the
output of the command becomes identical to that of the older
implementation. But the performance suffers greatly.
It still allows reusing non-deltified representations; there is
no point uncompressing and recompressing the whole text.
It also adds a couple more statistics output, while squelching
it under -q flag, which the last round forgot to do.
$ time old-git-pack-objects --stdout >/dev/null <RL
Generating pack...
Done counting 184141 objects.
Packing 184141 objects....................
real 12m8.530s user 11m1.450s sys 0m57.920s
$ time git-pack-objects --stdout >/dev/null <RL
Generating pack...
Done counting 184141 objects.
Packing 184141 objects.....................
Total 184141, written 184141 (delta 138297), reused 178833 (delta 134081)
real 0m59.549s user 0m56.670s sys 0m2.400s
$ time git-pack-objects --stdout --no-reuse-delta >/dev/null <RL
Generating pack...
Done counting 184141 objects.
Packing 184141 objects.....................
Total 184141, written 184141 (delta 134833), reused 47904 (delta 0)
real 11m13.830s user 9m45.240s sys 0m44.330s
There is one remaining issue when --no-reuse-delta option is not
used. It can create delta chains that are deeper than specified.
A<--B<--C<--D E F G
Suppose we have a delta chain A to D (A is stored in full either
in a pack or as a loose object. B is depth1 delta relative to A,
C is depth2 delta relative to B...) with loose objects E, F, G.
And we are going to pack all of them.
B, C and D are left as delta against A, B and C respectively.
So A, E, F, and G are examined for deltification, and let's say
we decided to keep E expanded, and store the rest as deltas like
this:
E<--F<--G<--A
Oops. We ended up making D a bit too deep, didn't we? B, C and
D form a chain on top of A!
This is because we did not know what the final depth of A would
be, when we checked objects and decided to keep the existing
delta. Unfortunately, deferring the decision until just before
the deltification is not an option. To be able to make B, C,
and D candidates for deltification with the rest, we need to
know the type and final unexpanded size of them, but the major
part of the optimization comes from the fact that we do not read
the delta data to do so -- getting the final size is quite an
expensive operation.
To prevent this from happening, we should keep A from being
deltified. But how would we tell that, cheaply?
To do this most precisely, after check_object() runs, each
object that is used as the base object of some existing delta
needs to be marked with the maximum depth of the objects we
decided to keep deltified (in this case, D is depth 3 relative
to A, so if no other delta chain that is longer than 3 based on
A exists, mark A with 3). Then when attempting to deltify A, we
would take that number into account to see if the final delta
chain that leads to D becomes too deep.
However, this is a bit cumbersome to compute, so we would cheat
and reduce the maximum depth for A arbitrarily to depth/4 in
this implementation.
Signed-off-by: Junio C Hamano <junkio@cox.net>
2006-02-16 19:55:51 +00:00
|
|
|
|
2007-07-12 13:07:46 +00:00
|
|
|
while (window_memory_limit &&
|
2007-09-06 06:13:09 +00:00
|
|
|
mem_usage > window_memory_limit &&
|
2007-07-12 13:07:46 +00:00
|
|
|
count > 1) {
|
|
|
|
uint32_t tail = (idx + window - count) % window;
|
2008-02-13 07:39:03 +00:00
|
|
|
mem_usage -= free_unpacked(array + tail);
|
2007-07-12 13:07:46 +00:00
|
|
|
count--;
|
|
|
|
}
|
|
|
|
|
2007-09-06 06:13:08 +00:00
|
|
|
/* We do not compute delta to *create* objects we are not
|
|
|
|
* going to pack.
|
|
|
|
*/
|
|
|
|
if (entry->preferred_base)
|
|
|
|
goto next;
|
|
|
|
|
2007-04-16 16:29:16 +00:00
|
|
|
/*
|
|
|
|
* If the current object is at pack edge, take the depth the
|
|
|
|
* objects that depend on the current object into account
|
|
|
|
* otherwise they would become too deep.
|
|
|
|
*/
|
|
|
|
max_depth = depth;
|
|
|
|
if (entry->delta_child) {
|
|
|
|
max_depth -= check_delta_limit(entry, 0);
|
|
|
|
if (max_depth <= 0)
|
|
|
|
goto next;
|
|
|
|
}
|
|
|
|
|
2005-06-26 01:29:23 +00:00
|
|
|
j = window;
|
|
|
|
while (--j > 0) {
|
2007-09-02 06:53:47 +00:00
|
|
|
int ret;
|
2007-03-07 01:44:24 +00:00
|
|
|
uint32_t other_idx = idx + j;
|
2005-06-25 21:42:43 +00:00
|
|
|
struct unpacked *m;
|
2005-06-26 01:29:23 +00:00
|
|
|
if (other_idx >= window)
|
|
|
|
other_idx -= window;
|
2005-06-25 21:42:43 +00:00
|
|
|
m = array + other_idx;
|
|
|
|
if (!m->entry)
|
|
|
|
break;
|
2007-09-06 06:13:09 +00:00
|
|
|
ret = try_delta(n, m, max_depth, &mem_usage);
|
2007-09-02 06:53:47 +00:00
|
|
|
if (ret < 0)
|
2005-06-25 21:42:43 +00:00
|
|
|
break;
|
2007-09-02 06:53:47 +00:00
|
|
|
else if (ret > 0)
|
|
|
|
best_base = other_idx;
|
2005-06-25 21:42:43 +00:00
|
|
|
}
|
2007-04-16 16:29:16 +00:00
|
|
|
|
2008-05-02 19:11:50 +00:00
|
|
|
/*
|
|
|
|
* If we decided to cache the delta data, then it is best
|
|
|
|
* to compress it right away. First because we have to do
|
|
|
|
* it anyway, and doing it here while we're threaded will
|
|
|
|
* save a lot of time in the non threaded write phase,
|
|
|
|
* as well as allow for caching more deltas within
|
|
|
|
* the same cache size limit.
|
|
|
|
* ...
|
|
|
|
* But only if not writing to stdout, since in that case
|
|
|
|
* the network is most likely throttling writes anyway,
|
|
|
|
* and therefore it is best to go to the write phase ASAP
|
|
|
|
* instead, as we can afford spending more time compressing
|
|
|
|
* between writes at that moment.
|
|
|
|
*/
|
|
|
|
if (entry->delta_data && !pack_to_stdout) {
|
|
|
|
entry->z_delta_size = do_compress(&entry->delta_data,
|
|
|
|
entry->delta_size);
|
|
|
|
cache_lock();
|
|
|
|
delta_cache_size -= entry->delta_size;
|
|
|
|
delta_cache_size += entry->z_delta_size;
|
|
|
|
cache_unlock();
|
|
|
|
}
|
|
|
|
|
2006-03-05 19:22:57 +00:00
|
|
|
/* if we made n a delta, and if n is already at max
|
|
|
|
* depth, leaving it in the window is pointless. we
|
|
|
|
* should evict it first.
|
|
|
|
*/
|
2008-05-02 19:11:51 +00:00
|
|
|
if (entry->delta && max_depth <= n->depth)
|
2006-03-05 19:22:57 +00:00
|
|
|
continue;
|
2006-05-15 17:47:16 +00:00
|
|
|
|
2007-09-02 06:53:47 +00:00
|
|
|
/*
|
|
|
|
* Move the best delta base up in the window, after the
|
|
|
|
* currently deltified object, to keep it longer. It will
|
|
|
|
* be the first base object to be attempted next.
|
|
|
|
*/
|
|
|
|
if (entry->delta) {
|
|
|
|
struct unpacked swap = array[best_base];
|
|
|
|
int dist = (window + idx - best_base) % window;
|
|
|
|
int dst = best_base;
|
|
|
|
while (dist--) {
|
|
|
|
int src = (dst + 1) % window;
|
|
|
|
array[dst] = array[src];
|
|
|
|
dst = src;
|
|
|
|
}
|
|
|
|
array[dst] = swap;
|
|
|
|
}
|
|
|
|
|
2007-04-16 16:29:16 +00:00
|
|
|
next:
|
2005-06-26 20:43:41 +00:00
|
|
|
idx++;
|
2007-07-12 13:07:46 +00:00
|
|
|
if (count + 1 < window)
|
|
|
|
count++;
|
2005-06-26 20:43:41 +00:00
|
|
|
if (idx >= window)
|
|
|
|
idx = 0;
|
2007-12-08 05:03:17 +00:00
|
|
|
}
|
2005-08-08 18:46:58 +00:00
|
|
|
|
2006-04-27 03:58:00 +00:00
|
|
|
for (i = 0; i < window; ++i) {
|
2006-05-15 17:47:16 +00:00
|
|
|
free_delta_index(array[i].index);
|
2005-08-08 18:46:58 +00:00
|
|
|
free(array[i].data);
|
2006-04-27 03:58:00 +00:00
|
|
|
}
|
2005-08-08 18:46:58 +00:00
|
|
|
free(array);
|
2005-06-25 21:42:43 +00:00
|
|
|
}
|
|
|
|
|
2007-09-06 06:13:11 +00:00
|
|
|
#ifdef THREADED_DELTA_SEARCH
|
|
|
|
|
2007-12-16 19:45:34 +00:00
|
|
|
/*
|
|
|
|
* The main thread waits on the condition that (at least) one of the workers
|
|
|
|
* has stopped working (which is indicated in the .working member of
|
|
|
|
* struct thread_params).
|
|
|
|
* When a work thread has completed its work, it sets .working to 0 and
|
|
|
|
* signals the main thread and waits on the condition that .data_ready
|
|
|
|
* becomes 1.
|
|
|
|
*/
|
|
|
|
|
2007-09-06 06:13:11 +00:00
|
|
|
struct thread_params {
|
|
|
|
pthread_t thread;
|
|
|
|
struct object_entry **list;
|
|
|
|
unsigned list_size;
|
2007-12-08 05:03:17 +00:00
|
|
|
unsigned remaining;
|
2007-09-06 06:13:11 +00:00
|
|
|
int window;
|
|
|
|
int depth;
|
2007-12-16 19:45:34 +00:00
|
|
|
int working;
|
|
|
|
int data_ready;
|
|
|
|
pthread_mutex_t mutex;
|
|
|
|
pthread_cond_t cond;
|
2007-09-06 06:13:11 +00:00
|
|
|
unsigned *processed;
|
|
|
|
};
|
|
|
|
|
2007-12-16 19:45:34 +00:00
|
|
|
static pthread_cond_t progress_cond = PTHREAD_COND_INITIALIZER;
|
2007-09-10 04:06:09 +00:00
|
|
|
|
2007-09-06 06:13:11 +00:00
|
|
|
static void *threaded_find_deltas(void *arg)
|
|
|
|
{
|
2007-09-10 04:06:09 +00:00
|
|
|
struct thread_params *me = arg;
|
|
|
|
|
2007-12-16 19:45:34 +00:00
|
|
|
while (me->remaining) {
|
2007-12-08 05:03:17 +00:00
|
|
|
find_deltas(me->list, &me->remaining,
|
2007-09-10 04:06:09 +00:00
|
|
|
me->window, me->depth, me->processed);
|
2007-12-16 19:45:34 +00:00
|
|
|
|
|
|
|
progress_lock();
|
|
|
|
me->working = 0;
|
|
|
|
pthread_cond_signal(&progress_cond);
|
|
|
|
progress_unlock();
|
|
|
|
|
|
|
|
/*
|
|
|
|
* We must not set ->data_ready before we wait on the
|
|
|
|
* condition because the main thread may have set it to 1
|
|
|
|
* before we get here. In order to be sure that new
|
|
|
|
* work is available if we see 1 in ->data_ready, it
|
|
|
|
* was initialized to 0 before this thread was spawned
|
|
|
|
* and we reset it to 0 right away.
|
|
|
|
*/
|
|
|
|
pthread_mutex_lock(&me->mutex);
|
|
|
|
while (!me->data_ready)
|
|
|
|
pthread_cond_wait(&me->cond, &me->mutex);
|
|
|
|
me->data_ready = 0;
|
|
|
|
pthread_mutex_unlock(&me->mutex);
|
2007-09-10 04:06:09 +00:00
|
|
|
}
|
2007-12-16 19:45:34 +00:00
|
|
|
/* leave ->working 1 so that this doesn't get more work assigned */
|
|
|
|
return NULL;
|
2007-09-06 06:13:11 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static void ll_find_deltas(struct object_entry **list, unsigned list_size,
|
|
|
|
int window, int depth, unsigned *processed)
|
|
|
|
{
|
2007-12-16 19:45:34 +00:00
|
|
|
struct thread_params p[delta_search_threads];
|
2007-12-08 05:03:17 +00:00
|
|
|
int i, ret, active_threads = 0;
|
2007-09-10 04:06:09 +00:00
|
|
|
|
2007-09-10 04:06:11 +00:00
|
|
|
if (delta_search_threads <= 1) {
|
2007-12-08 05:03:17 +00:00
|
|
|
find_deltas(list, &list_size, window, depth, processed);
|
2007-09-10 04:06:11 +00:00
|
|
|
return;
|
|
|
|
}
|
2008-12-11 20:36:47 +00:00
|
|
|
if (progress > pack_to_stdout)
|
2009-04-09 15:45:39 +00:00
|
|
|
fprintf(stderr, "Delta compression using up to %d threads.\n",
|
2008-12-11 20:36:47 +00:00
|
|
|
delta_search_threads);
|
2007-09-10 04:06:11 +00:00
|
|
|
|
2007-12-16 19:45:34 +00:00
|
|
|
/* Partition the work amongst work threads. */
|
2007-09-10 04:06:11 +00:00
|
|
|
for (i = 0; i < delta_search_threads; i++) {
|
2007-12-16 19:45:34 +00:00
|
|
|
unsigned sub_size = list_size / (delta_search_threads - i);
|
|
|
|
|
2008-12-13 20:06:40 +00:00
|
|
|
/* don't use too small segments or no deltas will be found */
|
|
|
|
if (sub_size < 2*window && i+1 < delta_search_threads)
|
|
|
|
sub_size = 0;
|
|
|
|
|
2007-09-06 06:13:11 +00:00
|
|
|
p[i].window = window;
|
|
|
|
p[i].depth = depth;
|
|
|
|
p[i].processed = processed;
|
2007-12-16 19:45:34 +00:00
|
|
|
p[i].working = 1;
|
|
|
|
p[i].data_ready = 0;
|
2007-09-10 04:06:09 +00:00
|
|
|
|
2007-09-10 04:06:10 +00:00
|
|
|
/* try to split chunks on "path" boundaries */
|
2008-01-21 16:07:15 +00:00
|
|
|
while (sub_size && sub_size < list_size &&
|
|
|
|
list[sub_size]->hash &&
|
2007-12-08 05:03:17 +00:00
|
|
|
list[sub_size]->hash == list[sub_size-1]->hash)
|
|
|
|
sub_size++;
|
|
|
|
|
2007-12-16 19:45:34 +00:00
|
|
|
p[i].list = list;
|
|
|
|
p[i].list_size = sub_size;
|
|
|
|
p[i].remaining = sub_size;
|
2007-09-10 04:06:10 +00:00
|
|
|
|
2007-12-08 05:03:17 +00:00
|
|
|
list += sub_size;
|
|
|
|
list_size -= sub_size;
|
|
|
|
}
|
|
|
|
|
2007-12-16 19:45:34 +00:00
|
|
|
/* Start work threads. */
|
|
|
|
for (i = 0; i < delta_search_threads; i++) {
|
|
|
|
if (!p[i].list_size)
|
|
|
|
continue;
|
2007-12-17 19:12:52 +00:00
|
|
|
pthread_mutex_init(&p[i].mutex, NULL);
|
|
|
|
pthread_cond_init(&p[i].cond, NULL);
|
2007-12-16 19:45:34 +00:00
|
|
|
ret = pthread_create(&p[i].thread, NULL,
|
|
|
|
threaded_find_deltas, &p[i]);
|
|
|
|
if (ret)
|
|
|
|
die("unable to create thread: %s", strerror(ret));
|
|
|
|
active_threads++;
|
|
|
|
}
|
|
|
|
|
2007-12-08 05:03:17 +00:00
|
|
|
/*
|
|
|
|
* Now let's wait for work completion. Each time a thread is done
|
|
|
|
* with its work, we steal half of the remaining work from the
|
|
|
|
* thread with the largest number of unprocessed objects and give
|
|
|
|
* it to that newly idle thread. This ensure good load balancing
|
|
|
|
* until the remaining object list segments are simply too short
|
|
|
|
* to be worth splitting anymore.
|
|
|
|
*/
|
2007-12-16 19:45:34 +00:00
|
|
|
while (active_threads) {
|
|
|
|
struct thread_params *target = NULL;
|
2007-12-08 05:03:17 +00:00
|
|
|
struct thread_params *victim = NULL;
|
|
|
|
unsigned sub_size = 0;
|
|
|
|
|
|
|
|
progress_lock();
|
2007-12-16 19:45:34 +00:00
|
|
|
for (;;) {
|
|
|
|
for (i = 0; !target && i < delta_search_threads; i++)
|
|
|
|
if (!p[i].working)
|
|
|
|
target = &p[i];
|
|
|
|
if (target)
|
|
|
|
break;
|
|
|
|
pthread_cond_wait(&progress_cond, &progress_mutex);
|
|
|
|
}
|
|
|
|
|
2007-12-08 05:03:17 +00:00
|
|
|
for (i = 0; i < delta_search_threads; i++)
|
|
|
|
if (p[i].remaining > 2*window &&
|
|
|
|
(!victim || victim->remaining < p[i].remaining))
|
|
|
|
victim = &p[i];
|
|
|
|
if (victim) {
|
|
|
|
sub_size = victim->remaining / 2;
|
|
|
|
list = victim->list + victim->list_size - sub_size;
|
|
|
|
while (sub_size && list[0]->hash &&
|
|
|
|
list[0]->hash == list[-1]->hash) {
|
|
|
|
list++;
|
|
|
|
sub_size--;
|
|
|
|
}
|
2007-12-10 19:19:32 +00:00
|
|
|
if (!sub_size) {
|
|
|
|
/*
|
|
|
|
* It is possible for some "paths" to have
|
|
|
|
* so many objects that no hash boundary
|
|
|
|
* might be found. Let's just steal the
|
|
|
|
* exact half in that case.
|
|
|
|
*/
|
|
|
|
sub_size = victim->remaining / 2;
|
|
|
|
list -= sub_size;
|
|
|
|
}
|
2007-12-08 05:03:17 +00:00
|
|
|
target->list = list;
|
|
|
|
victim->list_size -= sub_size;
|
|
|
|
victim->remaining -= sub_size;
|
|
|
|
}
|
|
|
|
target->list_size = sub_size;
|
|
|
|
target->remaining = sub_size;
|
2007-12-16 19:45:34 +00:00
|
|
|
target->working = 1;
|
|
|
|
progress_unlock();
|
|
|
|
|
|
|
|
pthread_mutex_lock(&target->mutex);
|
|
|
|
target->data_ready = 1;
|
|
|
|
pthread_cond_signal(&target->cond);
|
|
|
|
pthread_mutex_unlock(&target->mutex);
|
2007-09-10 04:06:09 +00:00
|
|
|
|
2007-12-08 05:03:17 +00:00
|
|
|
if (!sub_size) {
|
2007-09-10 12:40:44 +00:00
|
|
|
pthread_join(target->thread, NULL);
|
2007-12-16 19:45:34 +00:00
|
|
|
pthread_cond_destroy(&target->cond);
|
|
|
|
pthread_mutex_destroy(&target->mutex);
|
2007-12-08 05:03:17 +00:00
|
|
|
active_threads--;
|
2007-09-10 04:06:09 +00:00
|
|
|
}
|
2007-12-16 19:45:34 +00:00
|
|
|
}
|
2007-09-06 06:13:11 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
#else
|
2007-12-08 05:03:17 +00:00
|
|
|
#define ll_find_deltas(l, s, w, d, p) find_deltas(l, &s, w, d, p)
|
2007-09-06 06:13:11 +00:00
|
|
|
#endif
|
|
|
|
|
2008-03-04 03:27:20 +00:00
|
|
|
static int add_ref_tag(const char *path, const unsigned char *sha1, int flag, void *cb_data)
|
|
|
|
{
|
|
|
|
unsigned char peeled[20];
|
|
|
|
|
|
|
|
if (!prefixcmp(path, "refs/tags/") && /* is a tag? */
|
|
|
|
!peel_ref(path, peeled) && /* peelable? */
|
|
|
|
!is_null_sha1(peeled) && /* annotated tag? */
|
|
|
|
locate_object_entry(peeled)) /* object packed? */
|
|
|
|
add_object_entry(sha1, OBJ_TAG, NULL, 0);
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2005-10-22 08:28:13 +00:00
|
|
|
static void prepare_pack(int window, int depth)
|
|
|
|
{
|
2007-04-16 16:29:54 +00:00
|
|
|
struct object_entry **delta_list;
|
2008-07-03 15:52:09 +00:00
|
|
|
uint32_t i, nr_deltas;
|
|
|
|
unsigned n;
|
2007-04-16 16:29:54 +00:00
|
|
|
|
pack-objects: reuse data from existing packs.
When generating a new pack, notice if we have already needed
objects in existing packs. If an object is stored deltified,
and its base object is also what we are going to pack, then
reuse the existing deltified representation unconditionally,
bypassing all the expensive find_deltas() and try_deltas()
calls.
Also, notice if what we are going to write out exactly match
what is already in an existing pack (either deltified or just
compressed). In such a case, we can just copy it instead of
going through the usual uncompressing & recompressing cycle.
Without this patch, in linux-2.6 repository with about 1500
loose objects and a single mega pack:
$ git-rev-list --objects v2.6.16-rc3 >RL
$ wc -l RL
184141 RL
$ time git-pack-objects p <RL
Generating pack...
Done counting 184141 objects.
Packing 184141 objects....................
a1fc7b3e537fcb9b3c46b7505df859f0a11e79d2
real 12m4.323s
user 11m2.560s
sys 0m55.950s
With this patch, the same input:
$ time ../git.junio/git-pack-objects q <RL
Generating pack...
Done counting 184141 objects.
Packing 184141 objects.....................
a1fc7b3e537fcb9b3c46b7505df859f0a11e79d2
Total 184141, written 184141, reused 182441
real 1m2.608s
user 0m55.090s
sys 0m1.830s
Signed-off-by: Junio C Hamano <junkio@cox.net>
2006-02-16 01:34:29 +00:00
|
|
|
get_object_details();
|
2007-04-16 16:29:54 +00:00
|
|
|
|
close another possibility for propagating pack corruption
Abstract
--------
With index v2 we have a per object CRC to allow quick and safe reuse of
pack data when repacking. This, however, doesn't currently prevent a
stealth corruption from being propagated into a new pack when _not_
reusing pack data as demonstrated by the modification to t5302 included
here.
The Context
-----------
The Git database is all checksummed with SHA1 hashes. Any kind of
corruption can be confirmed by verifying this per object hash against
corresponding data. However this can be costly to perform systematically
and therefore this check is often not performed at run time when
accessing the object database.
First, the loose object format is entirely compressed with zlib which
already provide a CRC verification of its own when inflating data. Any
disk corruption would be caught already in this case.
Then, packed objects are also compressed with zlib but only for their
actual payload. The object headers and delta base references are not
deflated for obvious performance reasons, however this leave them
vulnerable to potentially undetected disk corruptions. Object types
are often validated against the expected type when they're requested,
and deflated size must always match the size recorded in the object header,
so those cases are pretty much covered as well.
Where corruptions could go unnoticed is in the delta base reference.
Of course, in the OBJ_REF_DELTA case, the odds for a SHA1 reference to
get corrupted so it actually matches the SHA1 of another object with the
same size (the delta header stores the expected size of the base object
to apply against) are virtually zero. In the OBJ_OFS_DELTA case, the
reference is a pack offset which would have to match the start boundary
of a different base object but still with the same size, and although this
is relatively much more "probable" than in the OBJ_REF_DELTA case, the
probability is also about zero in absolute terms. Still, the possibility
exists as demonstrated in t5302 and is certainly greater than a SHA1
collision, especially in the OBJ_OFS_DELTA case which is now the default
when repacking.
Again, repacking by reusing existing pack data is OK since the per object
CRC provided by index v2 guards against any such corruptions. What t5302
failed to test is a full repack in such case.
The Solution
------------
As unlikely as this kind of stealth corruption can be in practice, it
certainly isn't acceptable to propagate it into a freshly created pack.
But, because this is so unlikely, we don't want to pay the run time cost
associated with extra validation checks all the time either. Furthermore,
consequences of such corruption in anything but repacking should be rather
visible, and even if it could be quite unpleasant, it still has far less
severe consequences than actively creating bad packs.
So the best compromize is to check packed object CRC when unpacking
objects, and only during the compression/writing phase of a repack, and
only when not streaming the result. The cost of this is minimal (less
than 1% CPU time), and visible only with a full repack.
Someone with a stats background could provide an objective evaluation of
this, but I suspect that it's bad RAM that has more potential for data
corruptions at this point, even in those cases where this extra check
is not performed. Still, it is best to prevent a known hole for
corruption when recreating object data into a new pack.
What about the streamed pack case? Well, any client receiving a pack
must always consider that pack as untrusty and perform full validation
anyway, hence no such stealth corruption could be propagated to remote
repositoryes already. It is therefore worthless doing local validation
in that case.
Signed-off-by: Nicolas Pitre <nico@cam.org>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2008-10-31 15:31:08 +00:00
|
|
|
/*
|
|
|
|
* If we're locally repacking then we need to be doubly careful
|
|
|
|
* from now on in order to make sure no stealth corruption gets
|
|
|
|
* propagated to the new pack. Clients receiving streamed packs
|
|
|
|
* should validate everything they get anyway so no need to incur
|
|
|
|
* the additional cost here in that case.
|
|
|
|
*/
|
|
|
|
if (!pack_to_stdout)
|
|
|
|
do_check_packed_object_crc = 1;
|
|
|
|
|
2007-09-06 06:13:08 +00:00
|
|
|
if (!nr_objects || !window || !depth)
|
2007-04-16 16:29:54 +00:00
|
|
|
return;
|
|
|
|
|
|
|
|
delta_list = xmalloc(nr_objects * sizeof(*delta_list));
|
2007-09-06 06:13:08 +00:00
|
|
|
nr_deltas = n = 0;
|
|
|
|
|
|
|
|
for (i = 0; i < nr_objects; i++) {
|
|
|
|
struct object_entry *entry = objects + i;
|
|
|
|
|
|
|
|
if (entry->delta)
|
|
|
|
/* This happens if we decided to reuse existing
|
2008-05-02 19:11:46 +00:00
|
|
|
* delta from a pack. "reuse_delta &&" is implied.
|
2007-09-06 06:13:08 +00:00
|
|
|
*/
|
|
|
|
continue;
|
|
|
|
|
|
|
|
if (entry->size < 50)
|
|
|
|
continue;
|
|
|
|
|
|
|
|
if (entry->no_try_delta)
|
|
|
|
continue;
|
|
|
|
|
2008-08-12 18:31:06 +00:00
|
|
|
if (!entry->preferred_base) {
|
2007-09-06 06:13:08 +00:00
|
|
|
nr_deltas++;
|
2008-08-12 18:31:06 +00:00
|
|
|
if (entry->type < 0)
|
|
|
|
die("unable to get type of object %s",
|
|
|
|
sha1_to_hex(entry->idx.sha1));
|
2008-09-02 14:22:21 +00:00
|
|
|
} else {
|
|
|
|
if (entry->type < 0) {
|
|
|
|
/*
|
|
|
|
* This object is not found, but we
|
|
|
|
* don't have to include it anyway.
|
|
|
|
*/
|
|
|
|
continue;
|
|
|
|
}
|
2008-08-12 18:31:06 +00:00
|
|
|
}
|
2007-09-06 06:13:08 +00:00
|
|
|
|
|
|
|
delta_list[n++] = entry;
|
|
|
|
}
|
|
|
|
|
2007-10-17 01:55:47 +00:00
|
|
|
if (nr_deltas && n > 1) {
|
2007-09-06 06:13:10 +00:00
|
|
|
unsigned nr_done = 0;
|
|
|
|
if (progress)
|
2007-10-30 18:57:32 +00:00
|
|
|
progress_state = start_progress("Compressing objects",
|
|
|
|
nr_deltas);
|
2007-09-06 06:13:08 +00:00
|
|
|
qsort(delta_list, n, sizeof(*delta_list), type_size_sort);
|
2007-09-06 06:13:11 +00:00
|
|
|
ll_find_deltas(delta_list, n, window+1, depth, &nr_done);
|
2007-10-30 18:57:33 +00:00
|
|
|
stop_progress(&progress_state);
|
2007-09-06 06:13:10 +00:00
|
|
|
if (nr_done != nr_deltas)
|
|
|
|
die("inconsistency with delta count");
|
2007-09-06 06:13:08 +00:00
|
|
|
}
|
2007-04-16 16:29:54 +00:00
|
|
|
free(delta_list);
|
2005-10-22 08:28:13 +00:00
|
|
|
}
|
|
|
|
|
2008-05-14 17:46:53 +00:00
|
|
|
static int git_pack_config(const char *k, const char *v, void *cb)
|
2006-07-23 05:50:30 +00:00
|
|
|
{
|
|
|
|
if(!strcmp(k, "pack.window")) {
|
|
|
|
window = git_config_int(k, v);
|
|
|
|
return 0;
|
|
|
|
}
|
2007-07-12 13:07:46 +00:00
|
|
|
if (!strcmp(k, "pack.windowmemory")) {
|
|
|
|
window_memory_limit = git_config_ulong(k, v);
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
if (!strcmp(k, "pack.depth")) {
|
2007-05-08 13:28:26 +00:00
|
|
|
depth = git_config_int(k, v);
|
|
|
|
return 0;
|
|
|
|
}
|
Custom compression levels for objects and packs
Add config variables pack.compression and core.loosecompression ,
and switch --compression=level to pack-objects.
Loose objects will be compressed using core.loosecompression if set,
else core.compression if set, else Z_BEST_SPEED.
Packed objects will be compressed using --compression=level if seen,
else pack.compression if set, else core.compression if set,
else Z_DEFAULT_COMPRESSION. This is the "pack compression level".
Loose objects added to a pack undeltified will be recompressed
to the pack compression level if it is unequal to the current
loose compression level by the preceding rules, or if the loose
object was written while core.legacyheaders = true. Newly
deltified loose objects are always compressed to the current
pack compression level.
Previously packed objects added to a pack are recompressed
to the current pack compression level exactly when their
deltification status changes, since the previous pack data
cannot be reused.
In either case, the --no-reuse-object switch from the first
patch below will always force recompression to the current pack
compression level, instead of assuming the pack compression level
hasn't changed and pack data can be reused when possible.
This applies on top of the following patches from Nicolas Pitre:
[PATCH] allow for undeltified objects not to be reused
[PATCH] make "repack -f" imply "pack-objects --no-reuse-object"
Signed-off-by: Dana L. How <danahow@gmail.com>
Signed-off-by: Junio C Hamano <junkio@cox.net>
2007-05-09 20:56:50 +00:00
|
|
|
if (!strcmp(k, "pack.compression")) {
|
|
|
|
int level = git_config_int(k, v);
|
|
|
|
if (level == -1)
|
|
|
|
level = Z_DEFAULT_COMPRESSION;
|
|
|
|
else if (level < 0 || level > Z_BEST_COMPRESSION)
|
|
|
|
die("bad pack compression level %d", level);
|
|
|
|
pack_compression_level = level;
|
|
|
|
pack_compression_seen = 1;
|
|
|
|
return 0;
|
|
|
|
}
|
2007-05-28 21:20:58 +00:00
|
|
|
if (!strcmp(k, "pack.deltacachesize")) {
|
|
|
|
max_delta_cache_size = git_config_int(k, v);
|
|
|
|
return 0;
|
|
|
|
}
|
2007-05-28 21:20:59 +00:00
|
|
|
if (!strcmp(k, "pack.deltacachelimit")) {
|
|
|
|
cache_max_small_delta_size = git_config_int(k, v);
|
|
|
|
return 0;
|
|
|
|
}
|
2007-09-10 15:51:34 +00:00
|
|
|
if (!strcmp(k, "pack.threads")) {
|
|
|
|
delta_search_threads = git_config_int(k, v);
|
2008-02-23 02:11:56 +00:00
|
|
|
if (delta_search_threads < 0)
|
2007-09-10 15:51:34 +00:00
|
|
|
die("invalid number of threads specified (%d)",
|
|
|
|
delta_search_threads);
|
|
|
|
#ifndef THREADED_DELTA_SEARCH
|
2008-02-23 02:11:56 +00:00
|
|
|
if (delta_search_threads != 1)
|
2007-09-10 15:51:34 +00:00
|
|
|
warning("no threads support, ignoring %s", k);
|
|
|
|
#endif
|
|
|
|
return 0;
|
|
|
|
}
|
2007-11-02 03:26:04 +00:00
|
|
|
if (!strcmp(k, "pack.indexversion")) {
|
|
|
|
pack_idx_default_version = git_config_int(k, v);
|
|
|
|
if (pack_idx_default_version > 2)
|
2008-07-03 15:52:09 +00:00
|
|
|
die("bad pack.indexversion=%"PRIu32,
|
|
|
|
pack_idx_default_version);
|
2007-11-02 03:26:04 +00:00
|
|
|
return 0;
|
|
|
|
}
|
2008-02-05 14:25:04 +00:00
|
|
|
if (!strcmp(k, "pack.packsizelimit")) {
|
|
|
|
pack_size_limit_cfg = git_config_ulong(k, v);
|
|
|
|
return 0;
|
|
|
|
}
|
2008-05-14 17:46:53 +00:00
|
|
|
return git_default_config(k, v, cb);
|
2006-07-23 05:50:30 +00:00
|
|
|
}
|
|
|
|
|
2006-09-05 06:47:39 +00:00
|
|
|
static void read_object_list_from_stdin(void)
|
2005-06-25 21:42:43 +00:00
|
|
|
{
|
2006-09-05 06:47:39 +00:00
|
|
|
char line[40 + 1 + PATH_MAX + 2];
|
|
|
|
unsigned char sha1[20];
|
2006-02-22 21:00:08 +00:00
|
|
|
|
2006-04-02 20:31:54 +00:00
|
|
|
for (;;) {
|
|
|
|
if (!fgets(line, sizeof(line), stdin)) {
|
|
|
|
if (feof(stdin))
|
|
|
|
break;
|
|
|
|
if (!ferror(stdin))
|
|
|
|
die("fgets returned NULL, not EOF, not error!");
|
2006-04-04 06:41:09 +00:00
|
|
|
if (errno != EINTR)
|
2009-06-27 15:58:46 +00:00
|
|
|
die_errno("fgets");
|
2006-04-04 06:41:09 +00:00
|
|
|
clearerr(stdin);
|
|
|
|
continue;
|
2006-04-02 20:31:54 +00:00
|
|
|
}
|
2006-02-19 22:47:21 +00:00
|
|
|
if (line[0] == '-') {
|
|
|
|
if (get_sha1_hex(line+1, sha1))
|
|
|
|
die("expected edge sha1, got garbage:\n %s",
|
2006-09-05 06:47:39 +00:00
|
|
|
line);
|
2006-09-06 08:42:23 +00:00
|
|
|
add_preferred_base(sha1);
|
2006-02-19 22:47:21 +00:00
|
|
|
continue;
|
2006-02-12 01:54:18 +00:00
|
|
|
}
|
2005-06-25 21:42:43 +00:00
|
|
|
if (get_sha1_hex(line, sha1))
|
2005-11-21 20:38:31 +00:00
|
|
|
die("expected sha1, got garbage:\n %s", line);
|
2006-09-05 06:47:39 +00:00
|
|
|
|
2007-05-19 07:19:23 +00:00
|
|
|
add_preferred_base_object(line+41);
|
|
|
|
add_object_entry(sha1, 0, line+41, 0);
|
2005-06-25 21:42:43 +00:00
|
|
|
}
|
2006-09-05 06:47:39 +00:00
|
|
|
}
|
|
|
|
|
2007-09-17 06:20:07 +00:00
|
|
|
#define OBJECT_ADDED (1u<<20)
|
|
|
|
|
2009-04-06 19:28:36 +00:00
|
|
|
static void show_commit(struct commit *commit, void *data)
|
2006-09-05 06:47:39 +00:00
|
|
|
{
|
2007-05-19 07:19:23 +00:00
|
|
|
add_object_entry(commit->object.sha1, OBJ_COMMIT, NULL, 0);
|
2007-09-17 06:20:07 +00:00
|
|
|
commit->object.flags |= OBJECT_ADDED;
|
2006-09-05 06:47:39 +00:00
|
|
|
}
|
|
|
|
|
show_object(): push path_name() call further down
In particular, pushing the "path_name()" call _into_ the show() function
would seem to allow
- more clarity into who "owns" the name (ie now when we free the name in
the show_object callback, it's because we generated it ourselves by
calling path_name())
- not calling path_name() at all, either because we don't care about the
name in the first place, or because we are actually happy walking the
linked list of "struct name_path *" and the last component.
Now, I didn't do that latter optimization, because it would require some
more coding, but especially looking at "builtin-pack-objects.c", we really
don't even want the whole pathname, we really would be better off with the
list of path components.
Why? We use that name for two things:
- add_preferred_base_object(), which actually _wants_ to traverse the
path, and now does it by looking for '/' characters!
- for 'name_hash()', which only cares about the last 16 characters of a
name, so again, generating the full name seems to be just unnecessary
work.
Anyway, so I didn't look any closer at those things, but it did convince
me that the "show_object()" calling convention was crazy, and we're
actually better off doing _less_ in list-objects.c, and giving people
access to the internal data structures so that they can decide whether
they want to generate a path-name or not.
This patch does that, and then for people who did use the name (even if
they might do something more clever in the future), it just does the
straightforward "name = path_name(path, component); .. free(name);" thing.
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2009-04-11 01:15:26 +00:00
|
|
|
static void show_object(struct object *obj, const struct name_path *path, const char *last)
|
2006-09-05 06:47:39 +00:00
|
|
|
{
|
show_object(): push path_name() call further down
In particular, pushing the "path_name()" call _into_ the show() function
would seem to allow
- more clarity into who "owns" the name (ie now when we free the name in
the show_object callback, it's because we generated it ourselves by
calling path_name())
- not calling path_name() at all, either because we don't care about the
name in the first place, or because we are actually happy walking the
linked list of "struct name_path *" and the last component.
Now, I didn't do that latter optimization, because it would require some
more coding, but especially looking at "builtin-pack-objects.c", we really
don't even want the whole pathname, we really would be better off with the
list of path components.
Why? We use that name for two things:
- add_preferred_base_object(), which actually _wants_ to traverse the
path, and now does it by looking for '/' characters!
- for 'name_hash()', which only cares about the last 16 characters of a
name, so again, generating the full name seems to be just unnecessary
work.
Anyway, so I didn't look any closer at those things, but it did convince
me that the "show_object()" calling convention was crazy, and we're
actually better off doing _less_ in list-objects.c, and giving people
access to the internal data structures so that they can decide whether
they want to generate a path-name or not.
This patch does that, and then for people who did use the name (even if
they might do something more clever in the future), it just does the
straightforward "name = path_name(path, component); .. free(name);" thing.
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2009-04-11 01:15:26 +00:00
|
|
|
char *name = path_name(path, last);
|
|
|
|
|
process_{tree,blob}: show objects without buffering
Here's a less trivial thing, and slightly more dubious one.
I was looking at that "struct object_array objects", and wondering why we
do that. I have honestly totally forgotten. Why not just call the "show()"
function as we encounter the objects? Rather than add the objects to the
object_array, and then at the very end going through the array and doing a
'show' on all, just do things more incrementally.
Now, there are possible downsides to this:
- the "buffer using object_array" _can_ in theory result in at least
better I-cache usage (two tight loops rather than one more spread out
one). I don't think this is a real issue, but in theory..
- this _does_ change the order of the objects printed. Instead of doing a
"process_tree(revs, commit->tree, &objects, NULL, "");" in the loop
over the commits (which puts all the root trees _first_ in the object
list, this patch just adds them to the list of pending objects, and
then we'll traverse them in that order (and thus show each root tree
object together with the objects we discover under it)
I _think_ the new ordering actually makes more sense, but the object
ordering is actually a subtle thing when it comes to packing
efficiency, so any change in order is going to have implications for
packing. Good or bad, I dunno.
- There may be some reason why we did it that odd way with the object
array, that I have simply forgotten.
Anyway, now that we don't buffer up the objects before showing them
that may actually result in lower memory usage during that whole
traverse_commit_list() phase.
This is seriously not very deeply tested. It makes sense to me, it seems
to pass all the tests, it looks ok, but...
Does anybody remember why we did that "object_array" thing? It used to be
an "object_list" a long long time ago, but got changed into the array due
to better memory usage patterns (those linked lists of obejcts are
horrible from a memory allocation standpoint). But I wonder why we didn't
do this back then. Maybe there's a reason for it.
Or maybe there _used_ to be a reason, and no longer is.
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2009-04-11 00:27:58 +00:00
|
|
|
add_preferred_base_object(name);
|
|
|
|
add_object_entry(obj->sha1, obj->type, name, 0);
|
|
|
|
obj->flags |= OBJECT_ADDED;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* We will have generated the hash from the name,
|
|
|
|
* but not saved a pointer to it - we can free it
|
|
|
|
*/
|
|
|
|
free((char *)name);
|
2006-09-05 06:47:39 +00:00
|
|
|
}
|
|
|
|
|
2006-09-06 08:42:23 +00:00
|
|
|
static void show_edge(struct commit *commit)
|
|
|
|
{
|
|
|
|
add_preferred_base(commit->object.sha1);
|
|
|
|
}
|
|
|
|
|
2007-09-17 06:20:07 +00:00
|
|
|
struct in_pack_object {
|
|
|
|
off_t offset;
|
|
|
|
struct object *object;
|
|
|
|
};
|
|
|
|
|
|
|
|
struct in_pack {
|
|
|
|
int alloc;
|
|
|
|
int nr;
|
|
|
|
struct in_pack_object *array;
|
|
|
|
};
|
|
|
|
|
|
|
|
static void mark_in_pack_object(struct object *object, struct packed_git *p, struct in_pack *in_pack)
|
|
|
|
{
|
|
|
|
in_pack->array[in_pack->nr].offset = find_pack_entry_one(object->sha1, p);
|
|
|
|
in_pack->array[in_pack->nr].object = object;
|
|
|
|
in_pack->nr++;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Compare the objects in the offset order, in order to emulate the
|
2008-09-13 17:18:36 +00:00
|
|
|
* "git rev-list --objects" output that produced the pack originally.
|
2007-09-17 06:20:07 +00:00
|
|
|
*/
|
|
|
|
static int ofscmp(const void *a_, const void *b_)
|
|
|
|
{
|
|
|
|
struct in_pack_object *a = (struct in_pack_object *)a_;
|
|
|
|
struct in_pack_object *b = (struct in_pack_object *)b_;
|
|
|
|
|
|
|
|
if (a->offset < b->offset)
|
|
|
|
return -1;
|
|
|
|
else if (a->offset > b->offset)
|
|
|
|
return 1;
|
|
|
|
else
|
|
|
|
return hashcmp(a->object->sha1, b->object->sha1);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void add_objects_in_unpacked_packs(struct rev_info *revs)
|
|
|
|
{
|
|
|
|
struct packed_git *p;
|
|
|
|
struct in_pack in_pack;
|
|
|
|
uint32_t i;
|
|
|
|
|
|
|
|
memset(&in_pack, 0, sizeof(in_pack));
|
|
|
|
|
|
|
|
for (p = packed_git; p; p = p->next) {
|
|
|
|
const unsigned char *sha1;
|
|
|
|
struct object *o;
|
|
|
|
|
2009-03-20 03:47:52 +00:00
|
|
|
if (!p->pack_local || p->pack_keep)
|
2007-09-17 06:20:07 +00:00
|
|
|
continue;
|
|
|
|
if (open_pack_index(p))
|
|
|
|
die("cannot open pack index");
|
|
|
|
|
|
|
|
ALLOC_GROW(in_pack.array,
|
|
|
|
in_pack.nr + p->num_objects,
|
|
|
|
in_pack.alloc);
|
|
|
|
|
|
|
|
for (i = 0; i < p->num_objects; i++) {
|
|
|
|
sha1 = nth_packed_object_sha1(p, i);
|
|
|
|
o = lookup_unknown_object(sha1);
|
|
|
|
if (!(o->flags & OBJECT_ADDED))
|
|
|
|
mark_in_pack_object(o, p, &in_pack);
|
|
|
|
o->flags |= OBJECT_ADDED;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if (in_pack.nr) {
|
|
|
|
qsort(in_pack.array, in_pack.nr, sizeof(in_pack.array[0]),
|
|
|
|
ofscmp);
|
|
|
|
for (i = 0; i < in_pack.nr; i++) {
|
|
|
|
struct object *o = in_pack.array[i].object;
|
|
|
|
add_object_entry(o->sha1, o->type, "", 0);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
free(in_pack.array);
|
|
|
|
}
|
|
|
|
|
2009-03-21 22:26:11 +00:00
|
|
|
static int has_sha1_pack_kept_or_nonlocal(const unsigned char *sha1)
|
|
|
|
{
|
|
|
|
static struct packed_git *last_found = (void *)1;
|
|
|
|
struct packed_git *p;
|
|
|
|
|
|
|
|
p = (last_found != (void *)1) ? last_found : packed_git;
|
|
|
|
|
|
|
|
while (p) {
|
|
|
|
if ((!p->pack_local || p->pack_keep) &&
|
|
|
|
find_pack_entry_one(sha1, p)) {
|
|
|
|
last_found = p;
|
|
|
|
return 1;
|
|
|
|
}
|
|
|
|
if (p == last_found)
|
|
|
|
p = packed_git;
|
|
|
|
else
|
|
|
|
p = p->next;
|
|
|
|
if (p == last_found)
|
|
|
|
p = p->next;
|
|
|
|
}
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2008-05-14 05:33:53 +00:00
|
|
|
static void loosen_unused_packed_objects(struct rev_info *revs)
|
|
|
|
{
|
|
|
|
struct packed_git *p;
|
|
|
|
uint32_t i;
|
|
|
|
const unsigned char *sha1;
|
|
|
|
|
|
|
|
for (p = packed_git; p; p = p->next) {
|
2009-03-20 03:47:52 +00:00
|
|
|
if (!p->pack_local || p->pack_keep)
|
2008-05-14 05:33:53 +00:00
|
|
|
continue;
|
|
|
|
|
|
|
|
if (open_pack_index(p))
|
|
|
|
die("cannot open pack index");
|
|
|
|
|
|
|
|
for (i = 0; i < p->num_objects; i++) {
|
|
|
|
sha1 = nth_packed_object_sha1(p, i);
|
2009-03-21 22:26:11 +00:00
|
|
|
if (!locate_object_entry(sha1) &&
|
|
|
|
!has_sha1_pack_kept_or_nonlocal(sha1))
|
2008-05-14 05:33:53 +00:00
|
|
|
if (force_object_loose(sha1, p->mtime))
|
|
|
|
die("unable to force loose object");
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2006-09-06 08:42:23 +00:00
|
|
|
static void get_object_list(int ac, const char **av)
|
2006-09-05 06:47:39 +00:00
|
|
|
{
|
|
|
|
struct rev_info revs;
|
|
|
|
char line[1000];
|
|
|
|
int flags = 0;
|
|
|
|
|
|
|
|
init_revisions(&revs, NULL);
|
|
|
|
save_commit_buffer = 0;
|
|
|
|
setup_revisions(ac, av, &revs, NULL);
|
|
|
|
|
|
|
|
while (fgets(line, sizeof(line), stdin) != NULL) {
|
|
|
|
int len = strlen(line);
|
2008-01-04 17:37:41 +00:00
|
|
|
if (len && line[len - 1] == '\n')
|
2006-09-05 06:47:39 +00:00
|
|
|
line[--len] = 0;
|
|
|
|
if (!len)
|
|
|
|
break;
|
|
|
|
if (*line == '-') {
|
|
|
|
if (!strcmp(line, "--not")) {
|
|
|
|
flags ^= UNINTERESTING;
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
die("not a rev '%s'", line);
|
|
|
|
}
|
|
|
|
if (handle_revision_arg(line, &revs, flags, 1))
|
|
|
|
die("bad revision '%s'", line);
|
|
|
|
}
|
|
|
|
|
2008-02-18 07:31:56 +00:00
|
|
|
if (prepare_revision_walk(&revs))
|
|
|
|
die("revision walk setup failed");
|
2006-09-06 08:42:23 +00:00
|
|
|
mark_edges_uninteresting(revs.commits, &revs, show_edge);
|
2009-04-06 19:28:36 +00:00
|
|
|
traverse_commit_list(&revs, show_commit, show_object, NULL);
|
2007-09-17 06:20:07 +00:00
|
|
|
|
|
|
|
if (keep_unreachable)
|
|
|
|
add_objects_in_unpacked_packs(&revs);
|
2008-05-14 05:33:53 +00:00
|
|
|
if (unpack_unreachable)
|
|
|
|
loosen_unused_packed_objects(&revs);
|
2006-09-05 06:47:39 +00:00
|
|
|
}
|
|
|
|
|
2007-04-22 19:28:34 +00:00
|
|
|
static int adjust_perm(const char *path, mode_t mode)
|
|
|
|
{
|
|
|
|
if (chmod(path, mode))
|
|
|
|
return -1;
|
|
|
|
return adjust_shared_perm(path);
|
|
|
|
}
|
|
|
|
|
2006-09-05 06:47:39 +00:00
|
|
|
int cmd_pack_objects(int argc, const char **argv, const char *prefix)
|
|
|
|
{
|
|
|
|
int use_internal_rev_list = 0;
|
2006-09-06 08:42:23 +00:00
|
|
|
int thin = 0;
|
2007-03-07 01:44:24 +00:00
|
|
|
uint32_t i;
|
2007-02-25 17:34:27 +00:00
|
|
|
const char **rp_av;
|
|
|
|
int rp_ac_alloc = 64;
|
2006-09-06 08:42:23 +00:00
|
|
|
int rp_ac;
|
|
|
|
|
2009-01-23 09:07:46 +00:00
|
|
|
read_replace_refs = 0;
|
|
|
|
|
2007-02-25 17:34:27 +00:00
|
|
|
rp_av = xcalloc(rp_ac_alloc, sizeof(*rp_av));
|
|
|
|
|
2006-09-06 08:42:23 +00:00
|
|
|
rp_av[0] = "pack-objects";
|
|
|
|
rp_av[1] = "--objects"; /* --thin will make it --objects-edge */
|
|
|
|
rp_ac = 2;
|
2006-09-05 06:47:39 +00:00
|
|
|
|
2008-05-14 17:46:53 +00:00
|
|
|
git_config(git_pack_config, NULL);
|
Custom compression levels for objects and packs
Add config variables pack.compression and core.loosecompression ,
and switch --compression=level to pack-objects.
Loose objects will be compressed using core.loosecompression if set,
else core.compression if set, else Z_BEST_SPEED.
Packed objects will be compressed using --compression=level if seen,
else pack.compression if set, else core.compression if set,
else Z_DEFAULT_COMPRESSION. This is the "pack compression level".
Loose objects added to a pack undeltified will be recompressed
to the pack compression level if it is unequal to the current
loose compression level by the preceding rules, or if the loose
object was written while core.legacyheaders = true. Newly
deltified loose objects are always compressed to the current
pack compression level.
Previously packed objects added to a pack are recompressed
to the current pack compression level exactly when their
deltification status changes, since the previous pack data
cannot be reused.
In either case, the --no-reuse-object switch from the first
patch below will always force recompression to the current pack
compression level, instead of assuming the pack compression level
hasn't changed and pack data can be reused when possible.
This applies on top of the following patches from Nicolas Pitre:
[PATCH] allow for undeltified objects not to be reused
[PATCH] make "repack -f" imply "pack-objects --no-reuse-object"
Signed-off-by: Dana L. How <danahow@gmail.com>
Signed-off-by: Junio C Hamano <junkio@cox.net>
2007-05-09 20:56:50 +00:00
|
|
|
if (!pack_compression_seen && core_compression_seen)
|
|
|
|
pack_compression_level = core_compression_level;
|
2006-09-05 06:47:39 +00:00
|
|
|
|
|
|
|
progress = isatty(2);
|
|
|
|
for (i = 1; i < argc; i++) {
|
|
|
|
const char *arg = argv[i];
|
|
|
|
|
|
|
|
if (*arg != '-')
|
|
|
|
break;
|
|
|
|
|
|
|
|
if (!strcmp("--non-empty", arg)) {
|
|
|
|
non_empty = 1;
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
if (!strcmp("--local", arg)) {
|
|
|
|
local = 1;
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
if (!strcmp("--incremental", arg)) {
|
|
|
|
incremental = 1;
|
|
|
|
continue;
|
|
|
|
}
|
2008-11-12 17:59:04 +00:00
|
|
|
if (!strcmp("--honor-pack-keep", arg)) {
|
|
|
|
ignore_packed_keep = 1;
|
|
|
|
continue;
|
|
|
|
}
|
Custom compression levels for objects and packs
Add config variables pack.compression and core.loosecompression ,
and switch --compression=level to pack-objects.
Loose objects will be compressed using core.loosecompression if set,
else core.compression if set, else Z_BEST_SPEED.
Packed objects will be compressed using --compression=level if seen,
else pack.compression if set, else core.compression if set,
else Z_DEFAULT_COMPRESSION. This is the "pack compression level".
Loose objects added to a pack undeltified will be recompressed
to the pack compression level if it is unequal to the current
loose compression level by the preceding rules, or if the loose
object was written while core.legacyheaders = true. Newly
deltified loose objects are always compressed to the current
pack compression level.
Previously packed objects added to a pack are recompressed
to the current pack compression level exactly when their
deltification status changes, since the previous pack data
cannot be reused.
In either case, the --no-reuse-object switch from the first
patch below will always force recompression to the current pack
compression level, instead of assuming the pack compression level
hasn't changed and pack data can be reused when possible.
This applies on top of the following patches from Nicolas Pitre:
[PATCH] allow for undeltified objects not to be reused
[PATCH] make "repack -f" imply "pack-objects --no-reuse-object"
Signed-off-by: Dana L. How <danahow@gmail.com>
Signed-off-by: Junio C Hamano <junkio@cox.net>
2007-05-09 20:56:50 +00:00
|
|
|
if (!prefixcmp(arg, "--compression=")) {
|
|
|
|
char *end;
|
|
|
|
int level = strtoul(arg+14, &end, 0);
|
|
|
|
if (!arg[14] || *end)
|
|
|
|
usage(pack_usage);
|
|
|
|
if (level == -1)
|
|
|
|
level = Z_DEFAULT_COMPRESSION;
|
|
|
|
else if (level < 0 || level > Z_BEST_COMPRESSION)
|
|
|
|
die("bad pack compression level %d", level);
|
|
|
|
pack_compression_level = level;
|
|
|
|
continue;
|
|
|
|
}
|
2007-05-13 19:47:09 +00:00
|
|
|
if (!prefixcmp(arg, "--max-pack-size=")) {
|
|
|
|
char *end;
|
2008-02-05 14:25:04 +00:00
|
|
|
pack_size_limit_cfg = 0;
|
2007-05-13 19:47:09 +00:00
|
|
|
pack_size_limit = strtoul(arg+16, &end, 0) * 1024 * 1024;
|
|
|
|
if (!arg[16] || *end)
|
|
|
|
usage(pack_usage);
|
|
|
|
continue;
|
|
|
|
}
|
2007-02-20 09:54:00 +00:00
|
|
|
if (!prefixcmp(arg, "--window=")) {
|
2006-09-05 06:47:39 +00:00
|
|
|
char *end;
|
|
|
|
window = strtoul(arg+9, &end, 0);
|
|
|
|
if (!arg[9] || *end)
|
|
|
|
usage(pack_usage);
|
|
|
|
continue;
|
|
|
|
}
|
2007-07-12 13:07:46 +00:00
|
|
|
if (!prefixcmp(arg, "--window-memory=")) {
|
|
|
|
if (!git_parse_ulong(arg+16, &window_memory_limit))
|
|
|
|
usage(pack_usage);
|
|
|
|
continue;
|
|
|
|
}
|
2007-09-10 04:06:11 +00:00
|
|
|
if (!prefixcmp(arg, "--threads=")) {
|
|
|
|
char *end;
|
|
|
|
delta_search_threads = strtoul(arg+10, &end, 0);
|
2008-02-23 02:11:56 +00:00
|
|
|
if (!arg[10] || *end || delta_search_threads < 0)
|
2007-09-10 04:06:11 +00:00
|
|
|
usage(pack_usage);
|
|
|
|
#ifndef THREADED_DELTA_SEARCH
|
2008-02-23 02:11:56 +00:00
|
|
|
if (delta_search_threads != 1)
|
2007-09-10 04:06:11 +00:00
|
|
|
warning("no threads support, "
|
|
|
|
"ignoring %s", arg);
|
|
|
|
#endif
|
|
|
|
continue;
|
|
|
|
}
|
2007-02-20 09:54:00 +00:00
|
|
|
if (!prefixcmp(arg, "--depth=")) {
|
2006-09-05 06:47:39 +00:00
|
|
|
char *end;
|
|
|
|
depth = strtoul(arg+8, &end, 0);
|
|
|
|
if (!arg[8] || *end)
|
|
|
|
usage(pack_usage);
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
if (!strcmp("--progress", arg)) {
|
|
|
|
progress = 1;
|
|
|
|
continue;
|
|
|
|
}
|
2006-11-07 15:51:23 +00:00
|
|
|
if (!strcmp("--all-progress", arg)) {
|
|
|
|
progress = 2;
|
|
|
|
continue;
|
|
|
|
}
|
2006-09-05 06:47:39 +00:00
|
|
|
if (!strcmp("-q", arg)) {
|
|
|
|
progress = 0;
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
if (!strcmp("--no-reuse-delta", arg)) {
|
2008-05-02 19:11:46 +00:00
|
|
|
reuse_delta = 0;
|
2006-09-05 06:47:39 +00:00
|
|
|
continue;
|
|
|
|
}
|
2007-05-09 16:31:28 +00:00
|
|
|
if (!strcmp("--no-reuse-object", arg)) {
|
2008-05-02 19:11:46 +00:00
|
|
|
reuse_object = reuse_delta = 0;
|
2007-05-09 16:31:28 +00:00
|
|
|
continue;
|
|
|
|
}
|
2006-09-21 04:09:44 +00:00
|
|
|
if (!strcmp("--delta-base-offset", arg)) {
|
2006-09-23 01:25:04 +00:00
|
|
|
allow_ofs_delta = 1;
|
2006-09-21 04:09:44 +00:00
|
|
|
continue;
|
|
|
|
}
|
2006-09-05 06:47:39 +00:00
|
|
|
if (!strcmp("--stdout", arg)) {
|
|
|
|
pack_to_stdout = 1;
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
if (!strcmp("--revs", arg)) {
|
|
|
|
use_internal_rev_list = 1;
|
|
|
|
continue;
|
|
|
|
}
|
2007-09-17 06:20:07 +00:00
|
|
|
if (!strcmp("--keep-unreachable", arg)) {
|
|
|
|
keep_unreachable = 1;
|
|
|
|
continue;
|
|
|
|
}
|
2008-05-14 05:33:53 +00:00
|
|
|
if (!strcmp("--unpack-unreachable", arg)) {
|
|
|
|
unpack_unreachable = 1;
|
|
|
|
continue;
|
|
|
|
}
|
2008-03-04 03:27:20 +00:00
|
|
|
if (!strcmp("--include-tag", arg)) {
|
|
|
|
include_tag = 1;
|
|
|
|
continue;
|
|
|
|
}
|
2006-09-06 08:42:23 +00:00
|
|
|
if (!strcmp("--unpacked", arg) ||
|
2006-12-19 01:25:28 +00:00
|
|
|
!strcmp("--reflog", arg) ||
|
2006-09-06 08:42:23 +00:00
|
|
|
!strcmp("--all", arg)) {
|
|
|
|
use_internal_rev_list = 1;
|
2007-02-25 17:34:27 +00:00
|
|
|
if (rp_ac >= rp_ac_alloc - 1) {
|
|
|
|
rp_ac_alloc = alloc_nr(rp_ac_alloc);
|
|
|
|
rp_av = xrealloc(rp_av,
|
|
|
|
rp_ac_alloc * sizeof(*rp_av));
|
|
|
|
}
|
2006-09-06 08:42:23 +00:00
|
|
|
rp_av[rp_ac++] = arg;
|
2006-09-05 06:47:39 +00:00
|
|
|
continue;
|
|
|
|
}
|
2006-09-06 08:42:23 +00:00
|
|
|
if (!strcmp("--thin", arg)) {
|
|
|
|
use_internal_rev_list = 1;
|
|
|
|
thin = 1;
|
|
|
|
rp_av[1] = "--objects-edge";
|
2006-09-05 06:47:39 +00:00
|
|
|
continue;
|
2007-04-09 21:32:03 +00:00
|
|
|
}
|
|
|
|
if (!prefixcmp(arg, "--index-version=")) {
|
|
|
|
char *c;
|
2007-06-01 19:18:05 +00:00
|
|
|
pack_idx_default_version = strtoul(arg + 16, &c, 10);
|
|
|
|
if (pack_idx_default_version > 2)
|
2007-04-09 21:32:03 +00:00
|
|
|
die("bad %s", arg);
|
|
|
|
if (*c == ',')
|
2007-06-01 19:18:05 +00:00
|
|
|
pack_idx_off32_limit = strtoul(c+1, &c, 0);
|
|
|
|
if (*c || pack_idx_off32_limit & 0x80000000)
|
2007-04-09 21:32:03 +00:00
|
|
|
die("bad %s", arg);
|
|
|
|
continue;
|
2006-09-05 06:47:39 +00:00
|
|
|
}
|
2009-07-23 15:33:49 +00:00
|
|
|
if (!strcmp(arg, "--keep-true-parents")) {
|
|
|
|
grafts_replace_parents = 0;
|
|
|
|
continue;
|
|
|
|
}
|
2006-09-05 06:47:39 +00:00
|
|
|
usage(pack_usage);
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Traditionally "pack-objects [options] base extra" failed;
|
|
|
|
* we would however want to take refs parameter that would
|
|
|
|
* have been given to upstream rev-list ourselves, which means
|
|
|
|
* we somehow want to say what the base name is. So the
|
|
|
|
* syntax would be:
|
|
|
|
*
|
|
|
|
* pack-objects [options] base <refs...>
|
|
|
|
*
|
|
|
|
* in other words, we would treat the first non-option as the
|
|
|
|
* base_name and send everything else to the internal revision
|
|
|
|
* walker.
|
|
|
|
*/
|
|
|
|
|
|
|
|
if (!pack_to_stdout)
|
|
|
|
base_name = argv[i++];
|
|
|
|
|
|
|
|
if (pack_to_stdout != !base_name)
|
|
|
|
usage(pack_usage);
|
|
|
|
|
2008-02-05 14:25:04 +00:00
|
|
|
if (!pack_to_stdout && !pack_size_limit)
|
|
|
|
pack_size_limit = pack_size_limit_cfg;
|
|
|
|
|
2007-05-23 17:11:33 +00:00
|
|
|
if (pack_to_stdout && pack_size_limit)
|
|
|
|
die("--max-pack-size cannot be used to build a pack for transfer.");
|
|
|
|
|
2006-09-06 08:42:23 +00:00
|
|
|
if (!pack_to_stdout && thin)
|
|
|
|
die("--thin cannot be used to build an indexable pack.");
|
2006-09-05 06:47:39 +00:00
|
|
|
|
2008-05-14 05:33:53 +00:00
|
|
|
if (keep_unreachable && unpack_unreachable)
|
|
|
|
die("--keep-unreachable and --unpack-unreachable are incompatible.");
|
|
|
|
|
2008-02-23 02:11:56 +00:00
|
|
|
#ifdef THREADED_DELTA_SEARCH
|
|
|
|
if (!delta_search_threads) /* --threads=0 means autodetect */
|
|
|
|
delta_search_threads = online_cpus();
|
|
|
|
#endif
|
|
|
|
|
2006-09-05 06:47:39 +00:00
|
|
|
prepare_packed_git();
|
|
|
|
|
2007-04-20 18:10:07 +00:00
|
|
|
if (progress)
|
2007-10-30 18:57:32 +00:00
|
|
|
progress_state = start_progress("Counting objects", 0);
|
2006-09-05 06:47:39 +00:00
|
|
|
if (!use_internal_rev_list)
|
|
|
|
read_object_list_from_stdin();
|
2006-09-06 08:42:23 +00:00
|
|
|
else {
|
|
|
|
rp_av[rp_ac] = NULL;
|
|
|
|
get_object_list(rp_ac, rp_av);
|
|
|
|
}
|
2008-03-04 03:27:20 +00:00
|
|
|
if (include_tag && nr_result)
|
|
|
|
for_each_ref(add_ref_tag, NULL);
|
2007-10-30 18:57:33 +00:00
|
|
|
stop_progress(&progress_state);
|
2007-04-18 18:27:45 +00:00
|
|
|
|
2006-02-25 05:55:23 +00:00
|
|
|
if (non_empty && !nr_result)
|
2005-07-03 20:36:58 +00:00
|
|
|
return 0;
|
2007-04-16 16:30:15 +00:00
|
|
|
if (nr_result)
|
|
|
|
prepare_pack(window, depth);
|
2007-05-13 18:34:56 +00:00
|
|
|
write_pack_file();
|
pack-objects: finishing touches.
This introduces --no-reuse-delta option to disable reusing of
existing delta, which is a large part of the optimization
introduced by this series. This may become necessary if
repeated repacking makes delta chain too long. With this, the
output of the command becomes identical to that of the older
implementation. But the performance suffers greatly.
It still allows reusing non-deltified representations; there is
no point uncompressing and recompressing the whole text.
It also adds a couple more statistics output, while squelching
it under -q flag, which the last round forgot to do.
$ time old-git-pack-objects --stdout >/dev/null <RL
Generating pack...
Done counting 184141 objects.
Packing 184141 objects....................
real 12m8.530s user 11m1.450s sys 0m57.920s
$ time git-pack-objects --stdout >/dev/null <RL
Generating pack...
Done counting 184141 objects.
Packing 184141 objects.....................
Total 184141, written 184141 (delta 138297), reused 178833 (delta 134081)
real 0m59.549s user 0m56.670s sys 0m2.400s
$ time git-pack-objects --stdout --no-reuse-delta >/dev/null <RL
Generating pack...
Done counting 184141 objects.
Packing 184141 objects.....................
Total 184141, written 184141 (delta 134833), reused 47904 (delta 0)
real 11m13.830s user 9m45.240s sys 0m44.330s
There is one remaining issue when --no-reuse-delta option is not
used. It can create delta chains that are deeper than specified.
A<--B<--C<--D E F G
Suppose we have a delta chain A to D (A is stored in full either
in a pack or as a loose object. B is depth1 delta relative to A,
C is depth2 delta relative to B...) with loose objects E, F, G.
And we are going to pack all of them.
B, C and D are left as delta against A, B and C respectively.
So A, E, F, and G are examined for deltification, and let's say
we decided to keep E expanded, and store the rest as deltas like
this:
E<--F<--G<--A
Oops. We ended up making D a bit too deep, didn't we? B, C and
D form a chain on top of A!
This is because we did not know what the final depth of A would
be, when we checked objects and decided to keep the existing
delta. Unfortunately, deferring the decision until just before
the deltification is not an option. To be able to make B, C,
and D candidates for deltification with the rest, we need to
know the type and final unexpanded size of them, but the major
part of the optimization comes from the fact that we do not read
the delta data to do so -- getting the final size is quite an
expensive operation.
To prevent this from happening, we should keep A from being
deltified. But how would we tell that, cheaply?
To do this most precisely, after check_object() runs, each
object that is used as the base object of some existing delta
needs to be marked with the maximum depth of the objects we
decided to keep deltified (in this case, D is depth 3 relative
to A, so if no other delta chain that is longer than 3 based on
A exists, mark A with 3). Then when attempting to deltify A, we
would take that number into account to see if the final delta
chain that leads to D becomes too deep.
However, this is a bit cumbersome to compute, so we would cheat
and reduce the maximum depth for A arbitrarily to depth/4 in
this implementation.
Signed-off-by: Junio C Hamano <junkio@cox.net>
2006-02-16 19:55:51 +00:00
|
|
|
if (progress)
|
2008-07-03 15:52:09 +00:00
|
|
|
fprintf(stderr, "Total %"PRIu32" (delta %"PRIu32"),"
|
|
|
|
" reused %"PRIu32" (delta %"PRIu32")\n",
|
2006-11-29 22:15:48 +00:00
|
|
|
written, written_delta, reused, reused_delta);
|
2005-06-25 21:42:43 +00:00
|
|
|
return 0;
|
|
|
|
}
|