mirror of
https://github.com/git/git
synced 2024-08-24 18:26:02 +00:00
basic threaded delta search
this is still rough, hence it is disabled by default. You need to compile with "make THREADED_DELTA_SEARCH=1 ..." at the moment. Threading is done on different portions of the object list to be deltified. This is currently done by spliting the list into n parts and then a thread is spawned for each of them. A better method would consist of spliting the list into more smaller parts and have the n threads pick the next part available. Signed-off-by: Nicolas Pitre <nico@cam.org> Signed-off-by: Junio C Hamano <gitster@pobox.com>
This commit is contained in:
parent
e334977dfa
commit
8ecce684a3
8
Makefile
8
Makefile
|
@ -122,6 +122,9 @@ all::
|
|||
# If not set it defaults to the bare 'wish'. If it is set to the empty
|
||||
# string then NO_TCLTK will be forced (this is used by configure script).
|
||||
#
|
||||
# Define THREADED_DELTA_SEARCH if you have pthreads and wish to exploit
|
||||
# parallel delta searching when packing objects.
|
||||
#
|
||||
|
||||
GIT-VERSION-FILE: .FORCE-GIT-VERSION-FILE
|
||||
@$(SHELL_PATH) ./GIT-VERSION-GEN
|
||||
|
@ -662,6 +665,11 @@ ifdef NO_HSTRERROR
|
|||
COMPAT_OBJS += compat/hstrerror.o
|
||||
endif
|
||||
|
||||
ifdef THREADED_DELTA_SEARCH
|
||||
BASIC_CFLAGS += -DTHREADED_DELTA_SEARCH
|
||||
EXTLIBS += -lpthread
|
||||
endif
|
||||
|
||||
ifeq ($(TCLTK_PATH),)
|
||||
NO_TCLTK=NoThanks
|
||||
endif
|
||||
|
|
|
@ -15,6 +15,10 @@
|
|||
#include "list-objects.h"
|
||||
#include "progress.h"
|
||||
|
||||
#ifdef THREADED_DELTA_SEARCH
|
||||
#include <pthread.h>
|
||||
#endif
|
||||
|
||||
static const char pack_usage[] = "\
|
||||
git-pack-objects [{ -q | --progress | --all-progress }] \n\
|
||||
[--max-pack-size=N] [--local] [--incremental] \n\
|
||||
|
@ -1290,6 +1294,25 @@ static int delta_cacheable(unsigned long src_size, unsigned long trg_size,
|
|||
return 0;
|
||||
}
|
||||
|
||||
#ifdef THREADED_DELTA_SEARCH
|
||||
|
||||
static pthread_mutex_t read_mutex = PTHREAD_MUTEX_INITIALIZER;
|
||||
#define read_lock() pthread_mutex_lock(&read_mutex)
|
||||
#define read_unlock() pthread_mutex_unlock(&read_mutex)
|
||||
|
||||
static pthread_mutex_t progress_mutex = PTHREAD_MUTEX_INITIALIZER;
|
||||
#define progress_lock() pthread_mutex_lock(&progress_mutex)
|
||||
#define progress_unlock() pthread_mutex_unlock(&progress_mutex)
|
||||
|
||||
#else
|
||||
|
||||
#define read_lock() 0
|
||||
#define read_unlock() 0
|
||||
#define progress_lock() 0
|
||||
#define progress_unlock() 0
|
||||
|
||||
#endif
|
||||
|
||||
/*
|
||||
* We search for deltas _backwards_ in a list sorted by type and
|
||||
* by size, so that we see progressively smaller and smaller files.
|
||||
|
@ -1348,7 +1371,9 @@ static int try_delta(struct unpacked *trg, struct unpacked *src,
|
|||
|
||||
/* Load data if not already done */
|
||||
if (!trg->data) {
|
||||
read_lock();
|
||||
trg->data = read_sha1_file(trg_entry->idx.sha1, &type, &sz);
|
||||
read_unlock();
|
||||
if (!trg->data)
|
||||
die("object %s cannot be read",
|
||||
sha1_to_hex(trg_entry->idx.sha1));
|
||||
|
@ -1358,7 +1383,9 @@ static int try_delta(struct unpacked *trg, struct unpacked *src,
|
|||
*mem_usage += sz;
|
||||
}
|
||||
if (!src->data) {
|
||||
read_lock();
|
||||
src->data = read_sha1_file(src_entry->idx.sha1, &type, &sz);
|
||||
read_unlock();
|
||||
if (!src->data)
|
||||
die("object %s cannot be read",
|
||||
sha1_to_hex(src_entry->idx.sha1));
|
||||
|
@ -1470,9 +1497,11 @@ static void find_deltas(struct object_entry **list, unsigned list_size,
|
|||
if (entry->preferred_base)
|
||||
goto next;
|
||||
|
||||
progress_lock();
|
||||
(*processed)++;
|
||||
if (progress)
|
||||
display_progress(&progress_state, *processed);
|
||||
progress_unlock();
|
||||
|
||||
/*
|
||||
* If the current object is at pack edge, take the depth the
|
||||
|
@ -1542,6 +1571,58 @@ static void find_deltas(struct object_entry **list, unsigned list_size,
|
|||
free(array);
|
||||
}
|
||||
|
||||
#ifdef THREADED_DELTA_SEARCH
|
||||
|
||||
struct thread_params {
|
||||
pthread_t thread;
|
||||
struct object_entry **list;
|
||||
unsigned list_size;
|
||||
int window;
|
||||
int depth;
|
||||
unsigned *processed;
|
||||
};
|
||||
|
||||
static void *threaded_find_deltas(void *arg)
|
||||
{
|
||||
struct thread_params *p = arg;
|
||||
if (p->list_size)
|
||||
find_deltas(p->list, p->list_size,
|
||||
p->window, p->depth, p->processed);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
#define NR_THREADS 8
|
||||
|
||||
static void ll_find_deltas(struct object_entry **list, unsigned list_size,
|
||||
int window, int depth, unsigned *processed)
|
||||
{
|
||||
struct thread_params p[NR_THREADS];
|
||||
int i, ret;
|
||||
|
||||
for (i = 0; i < NR_THREADS; i++) {
|
||||
unsigned sublist_size = list_size / (NR_THREADS - i);
|
||||
p[i].list = list;
|
||||
p[i].list_size = sublist_size;
|
||||
p[i].window = window;
|
||||
p[i].depth = depth;
|
||||
p[i].processed = processed;
|
||||
ret = pthread_create(&p[i].thread, NULL,
|
||||
threaded_find_deltas, &p[i]);
|
||||
if (ret)
|
||||
die("unable to create thread: %s", strerror(ret));
|
||||
list += sublist_size;
|
||||
list_size -= sublist_size;
|
||||
}
|
||||
|
||||
for (i = 0; i < NR_THREADS; i++) {
|
||||
pthread_join(p[i].thread, NULL);
|
||||
}
|
||||
}
|
||||
|
||||
#else
|
||||
#define ll_find_deltas find_deltas
|
||||
#endif
|
||||
|
||||
static void prepare_pack(int window, int depth)
|
||||
{
|
||||
struct object_entry **delta_list;
|
||||
|
@ -1583,7 +1664,7 @@ static void prepare_pack(int window, int depth)
|
|||
"Deltifying %u objects...", "",
|
||||
nr_deltas);
|
||||
qsort(delta_list, n, sizeof(*delta_list), type_size_sort);
|
||||
find_deltas(delta_list, n, window+1, depth, &nr_done);
|
||||
ll_find_deltas(delta_list, n, window+1, depth, &nr_done);
|
||||
if (progress)
|
||||
stop_progress(&progress_state);
|
||||
if (nr_done != nr_deltas)
|
||||
|
|
Loading…
Reference in a new issue