mirror of
https://github.com/git/git
synced 2024-08-24 18:26:02 +00:00
![Linus Torvalds](/assets/img/avatar_default.png)
Now that the exact rename detection is linear-time (with a very small constant factor to boot), there is no longer any reason to limit it by the number of files involved. In some trivial testing, I created a repository with a directory that had a hundred thousand files in it (all with different contents), and then moved that directory to show the effects of renaming 100,000 files. With the new code, that resulted in [torvalds@woody big-rename]$ time ~/git/git show -C | wc -l 400006 real 0m2.071s user 0m1.520s sys 0m0.576s ie the code can correctly detect the hundred thousand renames in about 2 seconds (the number "400006" comes from four lines for each rename: diff --git a/really-big-dir/file-1-1-1-1-1 b/moved-big-dir/file-1-1-1-1-1 similarity index 100% rename from really-big-dir/file-1-1-1-1-1 rename to moved-big-dir/file-1-1-1-1-1 and the extra six lines is from a one-liner commit message and all the commit information and spacing). Most of those two seconds weren't even really the rename detection, it's really all the other stuff needed to get there. With the old code, this wouldn't have been practically possible. Doing a pairwise check of the ten billion possible pairs would have been prohibitively expensive. In fact, even with the rename limiter in place, the old code would waste a lot of time just on the diff_filespec checks, and despite not even trying to find renames, it used to look like: [torvalds@woody big-rename]$ time git show -C | wc -l 1400006 real 0m12.337s user 0m12.285s sys 0m0.192s ie we used to take 12 seconds for this load and not even do any rename detection! (The number 1400006 comes from fourteen lines per file moved: seven lines each for the delete and the create of a one-liner file, and the same extra six lines of commit information). Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org> Signed-off-by: Junio C Hamano <gitster@pobox.com>
589 lines
15 KiB
C
589 lines
15 KiB
C
/*
|
|
* Copyright (C) 2005 Junio C Hamano
|
|
*/
|
|
#include "cache.h"
|
|
#include "diff.h"
|
|
#include "diffcore.h"
|
|
#include "hash.h"
|
|
|
|
/* Table of rename/copy destinations */
|
|
|
|
static struct diff_rename_dst {
|
|
struct diff_filespec *two;
|
|
struct diff_filepair *pair;
|
|
} *rename_dst;
|
|
static int rename_dst_nr, rename_dst_alloc;
|
|
|
|
static struct diff_rename_dst *locate_rename_dst(struct diff_filespec *two,
|
|
int insert_ok)
|
|
{
|
|
int first, last;
|
|
|
|
first = 0;
|
|
last = rename_dst_nr;
|
|
while (last > first) {
|
|
int next = (last + first) >> 1;
|
|
struct diff_rename_dst *dst = &(rename_dst[next]);
|
|
int cmp = strcmp(two->path, dst->two->path);
|
|
if (!cmp)
|
|
return dst;
|
|
if (cmp < 0) {
|
|
last = next;
|
|
continue;
|
|
}
|
|
first = next+1;
|
|
}
|
|
/* not found */
|
|
if (!insert_ok)
|
|
return NULL;
|
|
/* insert to make it at "first" */
|
|
if (rename_dst_alloc <= rename_dst_nr) {
|
|
rename_dst_alloc = alloc_nr(rename_dst_alloc);
|
|
rename_dst = xrealloc(rename_dst,
|
|
rename_dst_alloc * sizeof(*rename_dst));
|
|
}
|
|
rename_dst_nr++;
|
|
if (first < rename_dst_nr)
|
|
memmove(rename_dst + first + 1, rename_dst + first,
|
|
(rename_dst_nr - first - 1) * sizeof(*rename_dst));
|
|
rename_dst[first].two = alloc_filespec(two->path);
|
|
fill_filespec(rename_dst[first].two, two->sha1, two->mode);
|
|
rename_dst[first].pair = NULL;
|
|
return &(rename_dst[first]);
|
|
}
|
|
|
|
/* Table of rename/copy src files */
|
|
static struct diff_rename_src {
|
|
struct diff_filespec *one;
|
|
unsigned short score; /* to remember the break score */
|
|
} *rename_src;
|
|
static int rename_src_nr, rename_src_alloc;
|
|
|
|
static struct diff_rename_src *register_rename_src(struct diff_filespec *one,
|
|
unsigned short score)
|
|
{
|
|
int first, last;
|
|
|
|
first = 0;
|
|
last = rename_src_nr;
|
|
while (last > first) {
|
|
int next = (last + first) >> 1;
|
|
struct diff_rename_src *src = &(rename_src[next]);
|
|
int cmp = strcmp(one->path, src->one->path);
|
|
if (!cmp)
|
|
return src;
|
|
if (cmp < 0) {
|
|
last = next;
|
|
continue;
|
|
}
|
|
first = next+1;
|
|
}
|
|
|
|
/* insert to make it at "first" */
|
|
if (rename_src_alloc <= rename_src_nr) {
|
|
rename_src_alloc = alloc_nr(rename_src_alloc);
|
|
rename_src = xrealloc(rename_src,
|
|
rename_src_alloc * sizeof(*rename_src));
|
|
}
|
|
rename_src_nr++;
|
|
if (first < rename_src_nr)
|
|
memmove(rename_src + first + 1, rename_src + first,
|
|
(rename_src_nr - first - 1) * sizeof(*rename_src));
|
|
rename_src[first].one = one;
|
|
rename_src[first].score = score;
|
|
return &(rename_src[first]);
|
|
}
|
|
|
|
static int basename_same(struct diff_filespec *src, struct diff_filespec *dst)
|
|
{
|
|
int src_len = strlen(src->path), dst_len = strlen(dst->path);
|
|
while (src_len && dst_len) {
|
|
char c1 = src->path[--src_len];
|
|
char c2 = dst->path[--dst_len];
|
|
if (c1 != c2)
|
|
return 0;
|
|
if (c1 == '/')
|
|
return 1;
|
|
}
|
|
return (!src_len || src->path[src_len - 1] == '/') &&
|
|
(!dst_len || dst->path[dst_len - 1] == '/');
|
|
}
|
|
|
|
struct diff_score {
|
|
int src; /* index in rename_src */
|
|
int dst; /* index in rename_dst */
|
|
int score;
|
|
int name_score;
|
|
};
|
|
|
|
static int estimate_similarity(struct diff_filespec *src,
|
|
struct diff_filespec *dst,
|
|
int minimum_score)
|
|
{
|
|
/* src points at a file that existed in the original tree (or
|
|
* optionally a file in the destination tree) and dst points
|
|
* at a newly created file. They may be quite similar, in which
|
|
* case we want to say src is renamed to dst or src is copied into
|
|
* dst, and then some edit has been applied to dst.
|
|
*
|
|
* Compare them and return how similar they are, representing
|
|
* the score as an integer between 0 and MAX_SCORE.
|
|
*
|
|
* When there is an exact match, it is considered a better
|
|
* match than anything else; the destination does not even
|
|
* call into this function in that case.
|
|
*/
|
|
unsigned long max_size, delta_size, base_size, src_copied, literal_added;
|
|
unsigned long delta_limit;
|
|
int score;
|
|
|
|
/* We deal only with regular files. Symlink renames are handled
|
|
* only when they are exact matches --- in other words, no edits
|
|
* after renaming.
|
|
*/
|
|
if (!S_ISREG(src->mode) || !S_ISREG(dst->mode))
|
|
return 0;
|
|
|
|
max_size = ((src->size > dst->size) ? src->size : dst->size);
|
|
base_size = ((src->size < dst->size) ? src->size : dst->size);
|
|
delta_size = max_size - base_size;
|
|
|
|
/* We would not consider edits that change the file size so
|
|
* drastically. delta_size must be smaller than
|
|
* (MAX_SCORE-minimum_score)/MAX_SCORE * min(src->size, dst->size).
|
|
*
|
|
* Note that base_size == 0 case is handled here already
|
|
* and the final score computation below would not have a
|
|
* divide-by-zero issue.
|
|
*/
|
|
if (base_size * (MAX_SCORE-minimum_score) < delta_size * MAX_SCORE)
|
|
return 0;
|
|
|
|
if ((!src->cnt_data && diff_populate_filespec(src, 0))
|
|
|| (!dst->cnt_data && diff_populate_filespec(dst, 0)))
|
|
return 0; /* error but caught downstream */
|
|
|
|
|
|
delta_limit = (unsigned long)
|
|
(base_size * (MAX_SCORE-minimum_score) / MAX_SCORE);
|
|
if (diffcore_count_changes(src, dst,
|
|
&src->cnt_data, &dst->cnt_data,
|
|
delta_limit,
|
|
&src_copied, &literal_added))
|
|
return 0;
|
|
|
|
/* How similar are they?
|
|
* what percentage of material in dst are from source?
|
|
*/
|
|
if (!dst->size)
|
|
score = 0; /* should not happen */
|
|
else
|
|
score = (int)(src_copied * MAX_SCORE / max_size);
|
|
return score;
|
|
}
|
|
|
|
static void record_rename_pair(int dst_index, int src_index, int score)
|
|
{
|
|
struct diff_filespec *src, *dst;
|
|
struct diff_filepair *dp;
|
|
|
|
if (rename_dst[dst_index].pair)
|
|
die("internal error: dst already matched.");
|
|
|
|
src = rename_src[src_index].one;
|
|
src->rename_used++;
|
|
src->count++;
|
|
|
|
dst = rename_dst[dst_index].two;
|
|
dst->count++;
|
|
|
|
dp = diff_queue(NULL, src, dst);
|
|
dp->renamed_pair = 1;
|
|
if (!strcmp(src->path, dst->path))
|
|
dp->score = rename_src[src_index].score;
|
|
else
|
|
dp->score = score;
|
|
rename_dst[dst_index].pair = dp;
|
|
}
|
|
|
|
/*
|
|
* We sort the rename similarity matrix with the score, in descending
|
|
* order (the most similar first).
|
|
*/
|
|
static int score_compare(const void *a_, const void *b_)
|
|
{
|
|
const struct diff_score *a = a_, *b = b_;
|
|
|
|
if (a->score == b->score)
|
|
return b->name_score - a->name_score;
|
|
|
|
return b->score - a->score;
|
|
}
|
|
|
|
struct file_similarity {
|
|
int src_dst, index;
|
|
struct diff_filespec *filespec;
|
|
struct file_similarity *next;
|
|
};
|
|
|
|
static int find_identical_files(struct file_similarity *src,
|
|
struct file_similarity *dst)
|
|
{
|
|
int renames = 0;
|
|
|
|
/*
|
|
* Walk over all the destinations ...
|
|
*/
|
|
do {
|
|
struct diff_filespec *one = dst->filespec;
|
|
struct file_similarity *p, *best;
|
|
int i = 100;
|
|
|
|
/*
|
|
* .. to find the best source match
|
|
*/
|
|
best = NULL;
|
|
for (p = src; p; p = p->next) {
|
|
struct diff_filespec *two = p->filespec;
|
|
|
|
/* False hash collission? */
|
|
if (hashcmp(one->sha1, two->sha1))
|
|
continue;
|
|
/* Non-regular files? If so, the modes must match! */
|
|
if (!S_ISREG(one->mode) || !S_ISREG(two->mode)) {
|
|
if (one->mode != two->mode)
|
|
continue;
|
|
}
|
|
best = p;
|
|
if (basename_same(one, two))
|
|
break;
|
|
|
|
/* Too many identical alternatives? Pick one */
|
|
if (!--i)
|
|
break;
|
|
}
|
|
if (best) {
|
|
record_rename_pair(dst->index, best->index, MAX_SCORE);
|
|
renames++;
|
|
}
|
|
} while ((dst = dst->next) != NULL);
|
|
return renames;
|
|
}
|
|
|
|
/*
|
|
* Note: the rest of the rename logic depends on this
|
|
* phase also populating all the filespecs for any
|
|
* entry that isn't matched up with an exact rename.
|
|
*/
|
|
static void free_similarity_list(struct file_similarity *p)
|
|
{
|
|
while (p) {
|
|
struct file_similarity *entry = p;
|
|
p = p->next;
|
|
|
|
/* Stupid special case, see note above! */
|
|
diff_populate_filespec(entry->filespec, 0);
|
|
free(entry);
|
|
}
|
|
}
|
|
|
|
static int find_same_files(void *ptr)
|
|
{
|
|
int ret;
|
|
struct file_similarity *p = ptr;
|
|
struct file_similarity *src = NULL, *dst = NULL;
|
|
|
|
/* Split the hash list up into sources and destinations */
|
|
do {
|
|
struct file_similarity *entry = p;
|
|
p = p->next;
|
|
if (entry->src_dst < 0) {
|
|
entry->next = src;
|
|
src = entry;
|
|
} else {
|
|
entry->next = dst;
|
|
dst = entry;
|
|
}
|
|
} while (p);
|
|
|
|
/*
|
|
* If we have both sources *and* destinations, see if
|
|
* we can match them up
|
|
*/
|
|
ret = (src && dst) ? find_identical_files(src, dst) : 0;
|
|
|
|
/* Free the hashes and return the number of renames found */
|
|
free_similarity_list(src);
|
|
free_similarity_list(dst);
|
|
return ret;
|
|
}
|
|
|
|
static unsigned int hash_filespec(struct diff_filespec *filespec)
|
|
{
|
|
unsigned int hash;
|
|
if (!filespec->sha1_valid) {
|
|
if (diff_populate_filespec(filespec, 0))
|
|
return 0;
|
|
hash_sha1_file(filespec->data, filespec->size, "blob", filespec->sha1);
|
|
}
|
|
memcpy(&hash, filespec->sha1, sizeof(hash));
|
|
return hash;
|
|
}
|
|
|
|
static void insert_file_table(struct hash_table *table, int src_dst, int index, struct diff_filespec *filespec)
|
|
{
|
|
void **pos;
|
|
unsigned int hash;
|
|
struct file_similarity *entry = xmalloc(sizeof(*entry));
|
|
|
|
entry->src_dst = src_dst;
|
|
entry->index = index;
|
|
entry->filespec = filespec;
|
|
entry->next = NULL;
|
|
|
|
hash = hash_filespec(filespec);
|
|
pos = insert_hash(hash, entry, table);
|
|
|
|
/* We already had an entry there? */
|
|
if (pos) {
|
|
entry->next = *pos;
|
|
*pos = entry;
|
|
}
|
|
}
|
|
|
|
/*
|
|
* Find exact renames first.
|
|
*
|
|
* The first round matches up the up-to-date entries,
|
|
* and then during the second round we try to match
|
|
* cache-dirty entries as well.
|
|
*/
|
|
static int find_exact_renames(void)
|
|
{
|
|
int i;
|
|
struct hash_table file_table;
|
|
|
|
init_hash(&file_table);
|
|
for (i = 0; i < rename_src_nr; i++)
|
|
insert_file_table(&file_table, -1, i, rename_src[i].one);
|
|
|
|
for (i = 0; i < rename_dst_nr; i++)
|
|
insert_file_table(&file_table, 1, i, rename_dst[i].two);
|
|
|
|
/* Find the renames */
|
|
i = for_each_hash(&file_table, find_same_files);
|
|
|
|
/* .. and free the hash data structure */
|
|
free_hash(&file_table);
|
|
|
|
return i;
|
|
}
|
|
|
|
void diffcore_rename(struct diff_options *options)
|
|
{
|
|
int detect_rename = options->detect_rename;
|
|
int minimum_score = options->rename_score;
|
|
int rename_limit = options->rename_limit;
|
|
struct diff_queue_struct *q = &diff_queued_diff;
|
|
struct diff_queue_struct outq;
|
|
struct diff_score *mx;
|
|
int i, j, rename_count;
|
|
int num_create, num_src, dst_cnt;
|
|
|
|
if (!minimum_score)
|
|
minimum_score = DEFAULT_RENAME_SCORE;
|
|
|
|
for (i = 0; i < q->nr; i++) {
|
|
struct diff_filepair *p = q->queue[i];
|
|
if (!DIFF_FILE_VALID(p->one)) {
|
|
if (!DIFF_FILE_VALID(p->two))
|
|
continue; /* unmerged */
|
|
else if (options->single_follow &&
|
|
strcmp(options->single_follow, p->two->path))
|
|
continue; /* not interested */
|
|
else
|
|
locate_rename_dst(p->two, 1);
|
|
}
|
|
else if (!DIFF_FILE_VALID(p->two)) {
|
|
/*
|
|
* If the source is a broken "delete", and
|
|
* they did not really want to get broken,
|
|
* that means the source actually stays.
|
|
* So we increment the "rename_used" score
|
|
* by one, to indicate ourselves as a user
|
|
*/
|
|
if (p->broken_pair && !p->score)
|
|
p->one->rename_used++;
|
|
register_rename_src(p->one, p->score);
|
|
}
|
|
else if (detect_rename == DIFF_DETECT_COPY) {
|
|
/*
|
|
* Increment the "rename_used" score by
|
|
* one, to indicate ourselves as a user.
|
|
*/
|
|
p->one->rename_used++;
|
|
register_rename_src(p->one, p->score);
|
|
}
|
|
}
|
|
if (rename_dst_nr == 0 || rename_src_nr == 0)
|
|
goto cleanup; /* nothing to do */
|
|
|
|
/*
|
|
* We really want to cull the candidates list early
|
|
* with cheap tests in order to avoid doing deltas.
|
|
*/
|
|
rename_count = find_exact_renames();
|
|
|
|
/*
|
|
* This basically does a test for the rename matrix not
|
|
* growing larger than a "rename_limit" square matrix, ie:
|
|
*
|
|
* rename_dst_nr * rename_src_nr > rename_limit * rename_limit
|
|
*
|
|
* but handles the potential overflow case specially (and we
|
|
* assume at least 32-bit integers)
|
|
*/
|
|
if (rename_limit <= 0 || rename_limit > 32767)
|
|
rename_limit = 32767;
|
|
if (rename_dst_nr > rename_limit && rename_src_nr > rename_limit)
|
|
goto cleanup;
|
|
if (rename_dst_nr * rename_src_nr > rename_limit * rename_limit)
|
|
goto cleanup;
|
|
|
|
/* Have we run out the created file pool? If so we can avoid
|
|
* doing the delta matrix altogether.
|
|
*/
|
|
if (rename_count == rename_dst_nr)
|
|
goto cleanup;
|
|
|
|
if (minimum_score == MAX_SCORE)
|
|
goto cleanup;
|
|
|
|
num_create = (rename_dst_nr - rename_count);
|
|
num_src = rename_src_nr;
|
|
mx = xmalloc(sizeof(*mx) * num_create * num_src);
|
|
for (dst_cnt = i = 0; i < rename_dst_nr; i++) {
|
|
int base = dst_cnt * num_src;
|
|
struct diff_filespec *two = rename_dst[i].two;
|
|
if (rename_dst[i].pair)
|
|
continue; /* dealt with exact match already. */
|
|
for (j = 0; j < rename_src_nr; j++) {
|
|
struct diff_filespec *one = rename_src[j].one;
|
|
struct diff_score *m = &mx[base+j];
|
|
m->src = j;
|
|
m->dst = i;
|
|
m->score = estimate_similarity(one, two,
|
|
minimum_score);
|
|
m->name_score = basename_same(one, two);
|
|
diff_free_filespec_blob(one);
|
|
}
|
|
/* We do not need the text anymore */
|
|
diff_free_filespec_blob(two);
|
|
dst_cnt++;
|
|
}
|
|
/* cost matrix sorted by most to least similar pair */
|
|
qsort(mx, num_create * num_src, sizeof(*mx), score_compare);
|
|
for (i = 0; i < num_create * num_src; i++) {
|
|
struct diff_rename_dst *dst = &rename_dst[mx[i].dst];
|
|
if (dst->pair)
|
|
continue; /* already done, either exact or fuzzy. */
|
|
if (mx[i].score < minimum_score)
|
|
break; /* there is no more usable pair. */
|
|
record_rename_pair(mx[i].dst, mx[i].src, mx[i].score);
|
|
rename_count++;
|
|
}
|
|
free(mx);
|
|
|
|
cleanup:
|
|
/* At this point, we have found some renames and copies and they
|
|
* are recorded in rename_dst. The original list is still in *q.
|
|
*/
|
|
outq.queue = NULL;
|
|
outq.nr = outq.alloc = 0;
|
|
for (i = 0; i < q->nr; i++) {
|
|
struct diff_filepair *p = q->queue[i];
|
|
struct diff_filepair *pair_to_free = NULL;
|
|
|
|
if (!DIFF_FILE_VALID(p->one) && DIFF_FILE_VALID(p->two)) {
|
|
/*
|
|
* Creation
|
|
*
|
|
* We would output this create record if it has
|
|
* not been turned into a rename/copy already.
|
|
*/
|
|
struct diff_rename_dst *dst =
|
|
locate_rename_dst(p->two, 0);
|
|
if (dst && dst->pair) {
|
|
diff_q(&outq, dst->pair);
|
|
pair_to_free = p;
|
|
}
|
|
else
|
|
/* no matching rename/copy source, so
|
|
* record this as a creation.
|
|
*/
|
|
diff_q(&outq, p);
|
|
}
|
|
else if (DIFF_FILE_VALID(p->one) && !DIFF_FILE_VALID(p->two)) {
|
|
/*
|
|
* Deletion
|
|
*
|
|
* We would output this delete record if:
|
|
*
|
|
* (1) this is a broken delete and the counterpart
|
|
* broken create remains in the output; or
|
|
* (2) this is not a broken delete, and rename_dst
|
|
* does not have a rename/copy to move p->one->path
|
|
* out of existence.
|
|
*
|
|
* Otherwise, the counterpart broken create
|
|
* has been turned into a rename-edit; or
|
|
* delete did not have a matching create to
|
|
* begin with.
|
|
*/
|
|
if (DIFF_PAIR_BROKEN(p)) {
|
|
/* broken delete */
|
|
struct diff_rename_dst *dst =
|
|
locate_rename_dst(p->one, 0);
|
|
if (dst && dst->pair)
|
|
/* counterpart is now rename/copy */
|
|
pair_to_free = p;
|
|
}
|
|
else {
|
|
if (p->one->rename_used)
|
|
/* this path remains */
|
|
pair_to_free = p;
|
|
}
|
|
|
|
if (pair_to_free)
|
|
;
|
|
else
|
|
diff_q(&outq, p);
|
|
}
|
|
else if (!diff_unmodified_pair(p))
|
|
/* all the usual ones need to be kept */
|
|
diff_q(&outq, p);
|
|
else
|
|
/* no need to keep unmodified pairs */
|
|
pair_to_free = p;
|
|
|
|
if (pair_to_free)
|
|
diff_free_filepair(pair_to_free);
|
|
}
|
|
diff_debug_queue("done copying original", &outq);
|
|
|
|
free(q->queue);
|
|
*q = outq;
|
|
diff_debug_queue("done collapsing", q);
|
|
|
|
for (i = 0; i < rename_dst_nr; i++)
|
|
free_filespec(rename_dst[i].two);
|
|
|
|
free(rename_dst);
|
|
rename_dst = NULL;
|
|
rename_dst_nr = rename_dst_alloc = 0;
|
|
free(rename_src);
|
|
rename_src = NULL;
|
|
rename_src_nr = rename_src_alloc = 0;
|
|
return;
|
|
}
|