git/diffcore-rename.c

734 lines
19 KiB
C
Raw Normal View History

/*
*
* Copyright (C) 2005 Junio C Hamano
*/
#include "cache.h"
#include "diff.h"
#include "diffcore.h"
#include "object-store.h"
#include "hashmap.h"
#include "progress.h"
#include "promisor-remote.h"
/* Table of rename/copy destinations */
static struct diff_rename_dst {
struct diff_filespec *two;
struct diff_filepair *pair;
} *rename_dst;
static int rename_dst_nr, rename_dst_alloc;
static int find_rename_dst(struct diff_filespec *two)
{
int first, last;
first = 0;
last = rename_dst_nr;
while (last > first) {
int next = first + ((last - first) >> 1);
struct diff_rename_dst *dst = &(rename_dst[next]);
int cmp = strcmp(two->path, dst->two->path);
if (!cmp)
return next;
if (cmp < 0) {
last = next;
continue;
}
first = next+1;
}
return -first - 1;
}
static struct diff_rename_dst *locate_rename_dst(struct diff_filespec *two)
{
int ofs = find_rename_dst(two);
return ofs < 0 ? NULL : &rename_dst[ofs];
}
/*
* Returns 0 on success, -1 if we found a duplicate.
*/
static int add_rename_dst(struct diff_filespec *two)
{
int first = find_rename_dst(two);
if (first >= 0)
return -1;
first = -first - 1;
/* insert to make it at "first" */
ALLOC_GROW(rename_dst, rename_dst_nr + 1, rename_dst_alloc);
rename_dst_nr++;
if (first < rename_dst_nr)
MOVE_ARRAY(rename_dst + first + 1, rename_dst + first,
rename_dst_nr - first - 1);
rename_dst[first].two = alloc_filespec(two->path);
fill_filespec(rename_dst[first].two, &two->oid, two->oid_valid,
two->mode);
rename_dst[first].pair = NULL;
return 0;
}
/* Table of rename/copy src files */
static struct diff_rename_src {
struct diff_filepair *p;
unsigned short score; /* to remember the break score */
} *rename_src;
static int rename_src_nr, rename_src_alloc;
static struct diff_rename_src *register_rename_src(struct diff_filepair *p)
{
int first, last;
struct diff_filespec *one = p->one;
unsigned short score = p->score;
first = 0;
last = rename_src_nr;
while (last > first) {
int next = first + ((last - first) >> 1);
struct diff_rename_src *src = &(rename_src[next]);
int cmp = strcmp(one->path, src->p->one->path);
if (!cmp)
return src;
if (cmp < 0) {
last = next;
continue;
}
first = next+1;
}
/* insert to make it at "first" */
ALLOC_GROW(rename_src, rename_src_nr + 1, rename_src_alloc);
rename_src_nr++;
if (first < rename_src_nr)
MOVE_ARRAY(rename_src + first + 1, rename_src + first,
rename_src_nr - first - 1);
rename_src[first].p = p;
rename_src[first].score = score;
return &(rename_src[first]);
}
static int basename_same(struct diff_filespec *src, struct diff_filespec *dst)
{
int src_len = strlen(src->path), dst_len = strlen(dst->path);
while (src_len && dst_len) {
char c1 = src->path[--src_len];
char c2 = dst->path[--dst_len];
if (c1 != c2)
return 0;
if (c1 == '/')
return 1;
}
return (!src_len || src->path[src_len - 1] == '/') &&
(!dst_len || dst->path[dst_len - 1] == '/');
}
struct diff_score {
int src; /* index in rename_src */
int dst; /* index in rename_dst */
unsigned short score;
short name_score;
};
struct prefetch_options {
struct repository *repo;
int skip_unmodified;
};
static void prefetch(void *prefetch_options)
{
struct prefetch_options *options = prefetch_options;
int i;
struct oid_array to_fetch = OID_ARRAY_INIT;
for (i = 0; i < rename_dst_nr; i++) {
if (rename_dst[i].pair)
/*
* The loop in diffcore_rename() will not need these
* blobs, so skip prefetching.
*/
continue; /* already found exact match */
diff_add_if_missing(options->repo, &to_fetch,
rename_dst[i].two);
}
for (i = 0; i < rename_src_nr; i++) {
if (options->skip_unmodified &&
diff_unmodified_pair(rename_src[i].p))
/*
* The loop in diffcore_rename() will not need these
* blobs, so skip prefetching.
*/
continue;
diff_add_if_missing(options->repo, &to_fetch,
rename_src[i].p->one);
}
promisor_remote_get_direct(options->repo, to_fetch.oid, to_fetch.nr);
oid_array_clear(&to_fetch);
}
static int estimate_similarity(struct repository *r,
struct diff_filespec *src,
struct diff_filespec *dst,
int minimum_score,
int skip_unmodified)
{
/* src points at a file that existed in the original tree (or
* optionally a file in the destination tree) and dst points
* at a newly created file. They may be quite similar, in which
* case we want to say src is renamed to dst or src is copied into
* dst, and then some edit has been applied to dst.
*
* Compare them and return how similar they are, representing
* the score as an integer between 0 and MAX_SCORE.
*
* When there is an exact match, it is considered a better
* match than anything else; the destination does not even
* call into this function in that case.
*/
unsigned long max_size, delta_size, base_size, src_copied, literal_added;
int score;
struct diff_populate_filespec_options dpf_options = {
.check_size_only = 1
};
struct prefetch_options prefetch_options = {r, skip_unmodified};
if (r == the_repository && has_promisor_remote()) {
dpf_options.missing_object_cb = prefetch;
dpf_options.missing_object_data = &prefetch_options;
}
/* We deal only with regular files. Symlink renames are handled
* only when they are exact matches --- in other words, no edits
* after renaming.
*/
if (!S_ISREG(src->mode) || !S_ISREG(dst->mode))
return 0;
/*
* Need to check that source and destination sizes are
* filled in before comparing them.
*
* If we already have "cnt_data" filled in, we know it's
* all good (avoid checking the size for zero, as that
* is a possible size - we really should have a flag to
* say whether the size is valid or not!)
*/
if (!src->cnt_data &&
diff_populate_filespec(r, src, &dpf_options))
return 0;
if (!dst->cnt_data &&
diff_populate_filespec(r, dst, &dpf_options))
return 0;
max_size = ((src->size > dst->size) ? src->size : dst->size);
base_size = ((src->size < dst->size) ? src->size : dst->size);
delta_size = max_size - base_size;
/* We would not consider edits that change the file size so
* drastically. delta_size must be smaller than
* (MAX_SCORE-minimum_score)/MAX_SCORE * min(src->size, dst->size).
*
* Note that base_size == 0 case is handled here already
* and the final score computation below would not have a
* divide-by-zero issue.
*/
if (max_size * (MAX_SCORE-minimum_score) < delta_size * MAX_SCORE)
return 0;
dpf_options.check_size_only = 0;
if (!src->cnt_data && diff_populate_filespec(r, src, &dpf_options))
return 0;
if (!dst->cnt_data && diff_populate_filespec(r, dst, &dpf_options))
return 0;
if (diffcore_count_changes(r, src, dst,
&src->cnt_data, &dst->cnt_data,
&src_copied, &literal_added))
return 0;
/* How similar are they?
* what percentage of material in dst are from source?
*/
if (!dst->size)
score = 0; /* should not happen */
else
score = (int)(src_copied * MAX_SCORE / max_size);
return score;
}
static void record_rename_pair(int dst_index, int src_index, int score)
{
struct diff_filespec *src, *dst;
struct diff_filepair *dp;
[PATCH] Rename/copy detection fix. The rename/copy detection logic in earlier round was only good enough to show patch output and discussion on the mailing list about the diff-raw format updates revealed many problems with it. This patch fixes all the ones known to me, without making things I want to do later impossible, mostly related to patch reordering. (1) Earlier rename/copy detector determined which one is rename and which one is copy too early, which made it impossible to later introduce diffcore transformers to reorder patches. This patch fixes it by moving that logic to the very end of the processing. (2) Earlier output routine diff_flush() was pruning all the "no-change" entries indiscriminatingly. This was done due to my false assumption that one of the requirements in the diff-raw output was not to show such an entry (which resulted in my incorrect comment about "diff-helper never being able to be equivalent to built-in diff driver"). My special thanks go to Linus for correcting me about this. When we produce diff-raw output, for the downstream to be able to tell renames from copies, sometimes it _is_ necessary to output "no-change" entries, and this patch adds diffcore_prune() function for doing it. (3) Earlier diff_filepair structure was trying to be not too specific about rename/copy operations, but the purpose of the structure was to record one or two paths, which _was_ indeed about rename/copy. This patch discards xfrm_msg field which was trying to be generic for this wrong reason, and introduces a couple of fields (rename_score and rename_rank) that are explicitly specific to rename/copy logic. One thing to note is that the information in a single diff_filepair structure _still_ does not distinguish renames from copies, and it is deliberately so. This is to allow patches to be reordered in later stages. (4) This patch also adds some tests about diff-raw format output and makes sure that necessary "no-change" entries appear on the output. Signed-off-by: Junio C Hamano <junkio@cox.net> Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2005-05-23 04:26:09 +00:00
if (rename_dst[dst_index].pair)
die("internal error: dst already matched.");
src = rename_src[src_index].p->one;
src->rename_used++;
src->count++;
dst = rename_dst[dst_index].two;
dst->count++;
dp = diff_queue(NULL, src, dst);
dp->renamed_pair = 1;
if (!strcmp(src->path, dst->path))
dp->score = rename_src[src_index].score;
else
dp->score = score;
rename_dst[dst_index].pair = dp;
}
/*
* We sort the rename similarity matrix with the score, in descending
* order (the most similar first).
*/
static int score_compare(const void *a_, const void *b_)
{
const struct diff_score *a = a_, *b = b_;
/* sink the unused ones to the bottom */
if (a->dst < 0)
return (0 <= b->dst);
else if (b->dst < 0)
return -1;
if (a->score == b->score)
return b->name_score - a->name_score;
return b->score - a->score;
}
struct file_similarity {
struct hashmap_entry entry;
int index;
struct diff_filespec *filespec;
};
static unsigned int hash_filespec(struct repository *r,
struct diff_filespec *filespec)
{
if (!filespec->oid_valid) {
if (diff_populate_filespec(r, filespec, NULL))
return 0;
hash_object_file(r->hash_algo, filespec->data, filespec->size,
"blob", &filespec->oid);
}
return oidhash(&filespec->oid);
}
static int find_identical_files(struct hashmap *srcs,
int dst_index,
struct diff_options *options)
{
int renames = 0;
struct diff_filespec *target = rename_dst[dst_index].two;
struct file_similarity *p, *best = NULL;
int i = 100, best_score = -1;
unsigned int hash = hash_filespec(options->repo, target);
/*
* Find the best source match for specified destination.
*/
p = hashmap_get_entry_from_hash(srcs, hash, NULL,
struct file_similarity, entry);
hashmap_for_each_entry_from(srcs, p, entry) {
int score;
struct diff_filespec *source = p->filespec;
/* False hash collision? */
if (!oideq(&source->oid, &target->oid))
continue;
/* Non-regular files? If so, the modes must match! */
if (!S_ISREG(source->mode) || !S_ISREG(target->mode)) {
if (source->mode != target->mode)
continue;
}
/* Give higher scores to sources that haven't been used already */
score = !source->rename_used;
if (source->rename_used && options->detect_rename != DIFF_DETECT_COPY)
continue;
score += basename_same(source, target);
if (score > best_score) {
best = p;
best_score = score;
if (score == 2)
break;
}
/* Too many identical alternatives? Pick one */
if (!--i)
break;
}
if (best) {
record_rename_pair(dst_index, best->index, MAX_SCORE);
renames++;
}
return renames;
}
static void insert_file_table(struct repository *r,
struct hashmap *table, int index,
struct diff_filespec *filespec)
{
struct file_similarity *entry = xmalloc(sizeof(*entry));
entry->index = index;
entry->filespec = filespec;
hashmap_entry_init(&entry->entry, hash_filespec(r, filespec));
hashmap_add(table, &entry->entry);
}
/*
* Find exact renames first.
*
* The first round matches up the up-to-date entries,
* and then during the second round we try to match
* cache-dirty entries as well.
*/
static int find_exact_renames(struct diff_options *options)
{
int i, renames = 0;
struct hashmap file_table;
2016-03-30 08:35:07 +00:00
/* Add all sources to the hash table in reverse order, because
* later on they will be retrieved in LIFO order.
*/
hashmap_init(&file_table, NULL, NULL, rename_src_nr);
2016-03-30 08:35:07 +00:00
for (i = rename_src_nr-1; i >= 0; i--)
insert_file_table(options->repo,
&file_table, i,
rename_src[i].p->one);
/* Walk the destinations and find best source match */
for (i = 0; i < rename_dst_nr; i++)
renames += find_identical_files(&file_table, i, options);
/* Free the hash data structure and entries */
hashmap_clear_and_free(&file_table, struct file_similarity, entry);
return renames;
}
#define NUM_CANDIDATE_PER_DST 4
static void record_if_better(struct diff_score m[], struct diff_score *o)
{
int i, worst;
/* find the worst one */
worst = 0;
for (i = 1; i < NUM_CANDIDATE_PER_DST; i++)
if (score_compare(&m[i], &m[worst]) > 0)
worst = i;
/* is it better than the worst one? */
if (score_compare(&m[worst], o) > 0)
m[worst] = *o;
}
/*
* Returns:
* 0 if we are under the limit;
* 1 if we need to disable inexact rename detection;
* 2 if we would be under the limit if we were given -C instead of -C -C.
*/
static int too_many_rename_candidates(int num_create,
struct diff_options *options)
{
int rename_limit = options->rename_limit;
int num_src = rename_src_nr;
int i;
options->needed_rename_limit = 0;
/*
* This basically does a test for the rename matrix not
* growing larger than a "rename_limit" square matrix, ie:
*
* num_create * num_src > rename_limit * rename_limit
*/
if (rename_limit <= 0)
rename_limit = 32767;
if ((num_create <= rename_limit || num_src <= rename_limit) &&
((uint64_t)num_create * (uint64_t)num_src
<= (uint64_t)rename_limit * (uint64_t)rename_limit))
return 0;
options->needed_rename_limit =
num_src > num_create ? num_src : num_create;
/* Are we running under -C -C? */
diff: make struct diff_flags members lowercase Now that the flags stored in struct diff_flags are being accessed directly and not through macros, change all struct members from being uppercase to lowercase. This conversion is done using the following semantic patch: @@ expression E; @@ - E.RECURSIVE + E.recursive @@ expression E; @@ - E.TREE_IN_RECURSIVE + E.tree_in_recursive @@ expression E; @@ - E.BINARY + E.binary @@ expression E; @@ - E.TEXT + E.text @@ expression E; @@ - E.FULL_INDEX + E.full_index @@ expression E; @@ - E.SILENT_ON_REMOVE + E.silent_on_remove @@ expression E; @@ - E.FIND_COPIES_HARDER + E.find_copies_harder @@ expression E; @@ - E.FOLLOW_RENAMES + E.follow_renames @@ expression E; @@ - E.RENAME_EMPTY + E.rename_empty @@ expression E; @@ - E.HAS_CHANGES + E.has_changes @@ expression E; @@ - E.QUICK + E.quick @@ expression E; @@ - E.NO_INDEX + E.no_index @@ expression E; @@ - E.ALLOW_EXTERNAL + E.allow_external @@ expression E; @@ - E.EXIT_WITH_STATUS + E.exit_with_status @@ expression E; @@ - E.REVERSE_DIFF + E.reverse_diff @@ expression E; @@ - E.CHECK_FAILED + E.check_failed @@ expression E; @@ - E.RELATIVE_NAME + E.relative_name @@ expression E; @@ - E.IGNORE_SUBMODULES + E.ignore_submodules @@ expression E; @@ - E.DIRSTAT_CUMULATIVE + E.dirstat_cumulative @@ expression E; @@ - E.DIRSTAT_BY_FILE + E.dirstat_by_file @@ expression E; @@ - E.ALLOW_TEXTCONV + E.allow_textconv @@ expression E; @@ - E.TEXTCONV_SET_VIA_CMDLINE + E.textconv_set_via_cmdline @@ expression E; @@ - E.DIFF_FROM_CONTENTS + E.diff_from_contents @@ expression E; @@ - E.DIRTY_SUBMODULES + E.dirty_submodules @@ expression E; @@ - E.IGNORE_UNTRACKED_IN_SUBMODULES + E.ignore_untracked_in_submodules @@ expression E; @@ - E.IGNORE_DIRTY_SUBMODULES + E.ignore_dirty_submodules @@ expression E; @@ - E.OVERRIDE_SUBMODULE_CONFIG + E.override_submodule_config @@ expression E; @@ - E.DIRSTAT_BY_LINE + E.dirstat_by_line @@ expression E; @@ - E.FUNCCONTEXT + E.funccontext @@ expression E; @@ - E.PICKAXE_IGNORE_CASE + E.pickaxe_ignore_case @@ expression E; @@ - E.DEFAULT_FOLLOW_RENAMES + E.default_follow_renames Signed-off-by: Brandon Williams <bmwill@google.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>
2017-10-31 18:19:11 +00:00
if (!options->flags.find_copies_harder)
return 1;
/* Would we bust the limit if we were running under -C? */
for (num_src = i = 0; i < rename_src_nr; i++) {
if (diff_unmodified_pair(rename_src[i].p))
continue;
num_src++;
}
if ((num_create <= rename_limit || num_src <= rename_limit) &&
((uint64_t)num_create * (uint64_t)num_src
<= (uint64_t)rename_limit * (uint64_t)rename_limit))
return 2;
return 1;
}
static int find_renames(struct diff_score *mx, int dst_cnt, int minimum_score, int copies)
{
int count = 0, i;
for (i = 0; i < dst_cnt * NUM_CANDIDATE_PER_DST; i++) {
struct diff_rename_dst *dst;
if ((mx[i].dst < 0) ||
(mx[i].score < minimum_score))
break; /* there is no more usable pair. */
dst = &rename_dst[mx[i].dst];
if (dst->pair)
continue; /* already done, either exact or fuzzy. */
if (!copies && rename_src[mx[i].src].p->one->rename_used)
continue;
record_rename_pair(mx[i].dst, mx[i].src, mx[i].score);
count++;
}
return count;
}
void diffcore_rename(struct diff_options *options)
{
int detect_rename = options->detect_rename;
int minimum_score = options->rename_score;
struct diff_queue_struct *q = &diff_queued_diff;
struct diff_queue_struct outq;
struct diff_score *mx;
int i, j, rename_count, skip_unmodified = 0;
int num_create, dst_cnt;
struct progress *progress = NULL;
if (!minimum_score)
minimum_score = DEFAULT_RENAME_SCORE;
for (i = 0; i < q->nr; i++) {
struct diff_filepair *p = q->queue[i];
if (!DIFF_FILE_VALID(p->one)) {
if (!DIFF_FILE_VALID(p->two))
continue; /* unmerged */
else if (options->single_follow &&
strcmp(options->single_follow, p->two->path))
continue; /* not interested */
diff: make struct diff_flags members lowercase Now that the flags stored in struct diff_flags are being accessed directly and not through macros, change all struct members from being uppercase to lowercase. This conversion is done using the following semantic patch: @@ expression E; @@ - E.RECURSIVE + E.recursive @@ expression E; @@ - E.TREE_IN_RECURSIVE + E.tree_in_recursive @@ expression E; @@ - E.BINARY + E.binary @@ expression E; @@ - E.TEXT + E.text @@ expression E; @@ - E.FULL_INDEX + E.full_index @@ expression E; @@ - E.SILENT_ON_REMOVE + E.silent_on_remove @@ expression E; @@ - E.FIND_COPIES_HARDER + E.find_copies_harder @@ expression E; @@ - E.FOLLOW_RENAMES + E.follow_renames @@ expression E; @@ - E.RENAME_EMPTY + E.rename_empty @@ expression E; @@ - E.HAS_CHANGES + E.has_changes @@ expression E; @@ - E.QUICK + E.quick @@ expression E; @@ - E.NO_INDEX + E.no_index @@ expression E; @@ - E.ALLOW_EXTERNAL + E.allow_external @@ expression E; @@ - E.EXIT_WITH_STATUS + E.exit_with_status @@ expression E; @@ - E.REVERSE_DIFF + E.reverse_diff @@ expression E; @@ - E.CHECK_FAILED + E.check_failed @@ expression E; @@ - E.RELATIVE_NAME + E.relative_name @@ expression E; @@ - E.IGNORE_SUBMODULES + E.ignore_submodules @@ expression E; @@ - E.DIRSTAT_CUMULATIVE + E.dirstat_cumulative @@ expression E; @@ - E.DIRSTAT_BY_FILE + E.dirstat_by_file @@ expression E; @@ - E.ALLOW_TEXTCONV + E.allow_textconv @@ expression E; @@ - E.TEXTCONV_SET_VIA_CMDLINE + E.textconv_set_via_cmdline @@ expression E; @@ - E.DIFF_FROM_CONTENTS + E.diff_from_contents @@ expression E; @@ - E.DIRTY_SUBMODULES + E.dirty_submodules @@ expression E; @@ - E.IGNORE_UNTRACKED_IN_SUBMODULES + E.ignore_untracked_in_submodules @@ expression E; @@ - E.IGNORE_DIRTY_SUBMODULES + E.ignore_dirty_submodules @@ expression E; @@ - E.OVERRIDE_SUBMODULE_CONFIG + E.override_submodule_config @@ expression E; @@ - E.DIRSTAT_BY_LINE + E.dirstat_by_line @@ expression E; @@ - E.FUNCCONTEXT + E.funccontext @@ expression E; @@ - E.PICKAXE_IGNORE_CASE + E.pickaxe_ignore_case @@ expression E; @@ - E.DEFAULT_FOLLOW_RENAMES + E.default_follow_renames Signed-off-by: Brandon Williams <bmwill@google.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>
2017-10-31 18:19:11 +00:00
else if (!options->flags.rename_empty &&
is_empty_blob_oid(&p->two->oid))
teach diffcore-rename to optionally ignore empty content Our rename detection is a heuristic, matching pairs of removed and added files with similar or identical content. It's unlikely to be wrong when there is actual content to compare, and we already take care not to do inexact rename detection when there is not enough content to produce good results. However, we always do exact rename detection, even when the blob is tiny or empty. It's easy to get false positives with an empty blob, simply because it is an obvious content to use as a boilerplate (e.g., when telling git that an empty directory is worth tracking via an empty .gitignore). This patch lets callers specify whether or not they are interested in using empty files as rename sources and destinations. The default is "yes", keeping the original behavior. It works by detecting the empty-blob sha1 for rename sources and destinations. One more flexible alternative would be to allow the caller to specify a minimum size for a blob to be "interesting" for rename detection. But that would catch small boilerplate files, not large ones (e.g., if you had the GPL COPYING file in many directories). A better alternative would be to allow a "-rename" gitattribute to allow boilerplate files to be marked as such. I'll leave the complexity of that solution until such time as somebody actually wants it. The complaints we've seen so far revolve around empty files, so let's start with the simple thing. Signed-off-by: Jeff King <peff@peff.net> Signed-off-by: Junio C Hamano <gitster@pobox.com>
2012-03-22 22:52:13 +00:00
continue;
else if (add_rename_dst(p->two) < 0) {
warning("skipping rename detection, detected"
" duplicate destination '%s'",
p->two->path);
goto cleanup;
}
}
diff: make struct diff_flags members lowercase Now that the flags stored in struct diff_flags are being accessed directly and not through macros, change all struct members from being uppercase to lowercase. This conversion is done using the following semantic patch: @@ expression E; @@ - E.RECURSIVE + E.recursive @@ expression E; @@ - E.TREE_IN_RECURSIVE + E.tree_in_recursive @@ expression E; @@ - E.BINARY + E.binary @@ expression E; @@ - E.TEXT + E.text @@ expression E; @@ - E.FULL_INDEX + E.full_index @@ expression E; @@ - E.SILENT_ON_REMOVE + E.silent_on_remove @@ expression E; @@ - E.FIND_COPIES_HARDER + E.find_copies_harder @@ expression E; @@ - E.FOLLOW_RENAMES + E.follow_renames @@ expression E; @@ - E.RENAME_EMPTY + E.rename_empty @@ expression E; @@ - E.HAS_CHANGES + E.has_changes @@ expression E; @@ - E.QUICK + E.quick @@ expression E; @@ - E.NO_INDEX + E.no_index @@ expression E; @@ - E.ALLOW_EXTERNAL + E.allow_external @@ expression E; @@ - E.EXIT_WITH_STATUS + E.exit_with_status @@ expression E; @@ - E.REVERSE_DIFF + E.reverse_diff @@ expression E; @@ - E.CHECK_FAILED + E.check_failed @@ expression E; @@ - E.RELATIVE_NAME + E.relative_name @@ expression E; @@ - E.IGNORE_SUBMODULES + E.ignore_submodules @@ expression E; @@ - E.DIRSTAT_CUMULATIVE + E.dirstat_cumulative @@ expression E; @@ - E.DIRSTAT_BY_FILE + E.dirstat_by_file @@ expression E; @@ - E.ALLOW_TEXTCONV + E.allow_textconv @@ expression E; @@ - E.TEXTCONV_SET_VIA_CMDLINE + E.textconv_set_via_cmdline @@ expression E; @@ - E.DIFF_FROM_CONTENTS + E.diff_from_contents @@ expression E; @@ - E.DIRTY_SUBMODULES + E.dirty_submodules @@ expression E; @@ - E.IGNORE_UNTRACKED_IN_SUBMODULES + E.ignore_untracked_in_submodules @@ expression E; @@ - E.IGNORE_DIRTY_SUBMODULES + E.ignore_dirty_submodules @@ expression E; @@ - E.OVERRIDE_SUBMODULE_CONFIG + E.override_submodule_config @@ expression E; @@ - E.DIRSTAT_BY_LINE + E.dirstat_by_line @@ expression E; @@ - E.FUNCCONTEXT + E.funccontext @@ expression E; @@ - E.PICKAXE_IGNORE_CASE + E.pickaxe_ignore_case @@ expression E; @@ - E.DEFAULT_FOLLOW_RENAMES + E.default_follow_renames Signed-off-by: Brandon Williams <bmwill@google.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>
2017-10-31 18:19:11 +00:00
else if (!options->flags.rename_empty &&
is_empty_blob_oid(&p->one->oid))
teach diffcore-rename to optionally ignore empty content Our rename detection is a heuristic, matching pairs of removed and added files with similar or identical content. It's unlikely to be wrong when there is actual content to compare, and we already take care not to do inexact rename detection when there is not enough content to produce good results. However, we always do exact rename detection, even when the blob is tiny or empty. It's easy to get false positives with an empty blob, simply because it is an obvious content to use as a boilerplate (e.g., when telling git that an empty directory is worth tracking via an empty .gitignore). This patch lets callers specify whether or not they are interested in using empty files as rename sources and destinations. The default is "yes", keeping the original behavior. It works by detecting the empty-blob sha1 for rename sources and destinations. One more flexible alternative would be to allow the caller to specify a minimum size for a blob to be "interesting" for rename detection. But that would catch small boilerplate files, not large ones (e.g., if you had the GPL COPYING file in many directories). A better alternative would be to allow a "-rename" gitattribute to allow boilerplate files to be marked as such. I'll leave the complexity of that solution until such time as somebody actually wants it. The complaints we've seen so far revolve around empty files, so let's start with the simple thing. Signed-off-by: Jeff King <peff@peff.net> Signed-off-by: Junio C Hamano <gitster@pobox.com>
2012-03-22 22:52:13 +00:00
continue;
diffcore-rename: don't consider unmerged path as source Since e9c8409 (diff-index --cached --raw: show tree entry on the LHS for unmerged entries., 2007-01-05), an unmerged entry should be detected by using DIFF_PAIR_UNMERGED(p), not by noticing both one and two sides of the filepair records mode=0 entries. However, it forgot to update some parts of the rename detection logic. This only makes difference in the "diff --cached" codepath where an unmerged filepair carries information on the entries that came from the tree. It probably hasn't been noticed for a long time because nobody would run "diff -M" during a conflict resolution, but "git status" uses rename detection when it internally runs "diff-index" and "diff-files" and gives nonsense results. In an unmerged pair, "one" side can have a valid filespec to record the tree entry (e.g. what's in HEAD) when running "diff --cached". This can be used as a rename source to other paths in the index that are not unmerged. The path that is unmerged by definition does not have the final content yet (i.e. "two" side cannot have a valid filespec), so it can never be a rename destination. Use the DIFF_PAIR_UNMERGED() to detect unmerged filepair correctly, and allow the valid "one" side of an unmerged filepair to be considered a potential rename source, but never to be considered a rename destination. Commit message and first two test cases by Junio, the rest by Martin. Signed-off-by: Martin von Zweigbergk <martin.von.zweigbergk@gmail.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>
2011-03-24 02:41:01 +00:00
else if (!DIFF_PAIR_UNMERGED(p) && !DIFF_FILE_VALID(p->two)) {
/*
* If the source is a broken "delete", and
* they did not really want to get broken,
* that means the source actually stays.
* So we increment the "rename_used" score
* by one, to indicate ourselves as a user
*/
if (p->broken_pair && !p->score)
p->one->rename_used++;
register_rename_src(p);
}
else if (detect_rename == DIFF_DETECT_COPY) {
/*
* Increment the "rename_used" score by
* one, to indicate ourselves as a user.
*/
p->one->rename_used++;
register_rename_src(p);
}
}
if (rename_dst_nr == 0 || rename_src_nr == 0)
goto cleanup; /* nothing to do */
Do exact rename detection regardless of rename limits Now that the exact rename detection is linear-time (with a very small constant factor to boot), there is no longer any reason to limit it by the number of files involved. In some trivial testing, I created a repository with a directory that had a hundred thousand files in it (all with different contents), and then moved that directory to show the effects of renaming 100,000 files. With the new code, that resulted in [torvalds@woody big-rename]$ time ~/git/git show -C | wc -l 400006 real 0m2.071s user 0m1.520s sys 0m0.576s ie the code can correctly detect the hundred thousand renames in about 2 seconds (the number "400006" comes from four lines for each rename: diff --git a/really-big-dir/file-1-1-1-1-1 b/moved-big-dir/file-1-1-1-1-1 similarity index 100% rename from really-big-dir/file-1-1-1-1-1 rename to moved-big-dir/file-1-1-1-1-1 and the extra six lines is from a one-liner commit message and all the commit information and spacing). Most of those two seconds weren't even really the rename detection, it's really all the other stuff needed to get there. With the old code, this wouldn't have been practically possible. Doing a pairwise check of the ten billion possible pairs would have been prohibitively expensive. In fact, even with the rename limiter in place, the old code would waste a lot of time just on the diff_filespec checks, and despite not even trying to find renames, it used to look like: [torvalds@woody big-rename]$ time git show -C | wc -l 1400006 real 0m12.337s user 0m12.285s sys 0m0.192s ie we used to take 12 seconds for this load and not even do any rename detection! (The number 1400006 comes from fourteen lines per file moved: seven lines each for the delete and the create of a one-liner file, and the same extra six lines of commit information). Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org> Signed-off-by: Junio C Hamano <gitster@pobox.com>
2007-10-25 18:24:47 +00:00
/*
* We really want to cull the candidates list early
* with cheap tests in order to avoid doing deltas.
*/
rename_count = find_exact_renames(options);
Do exact rename detection regardless of rename limits Now that the exact rename detection is linear-time (with a very small constant factor to boot), there is no longer any reason to limit it by the number of files involved. In some trivial testing, I created a repository with a directory that had a hundred thousand files in it (all with different contents), and then moved that directory to show the effects of renaming 100,000 files. With the new code, that resulted in [torvalds@woody big-rename]$ time ~/git/git show -C | wc -l 400006 real 0m2.071s user 0m1.520s sys 0m0.576s ie the code can correctly detect the hundred thousand renames in about 2 seconds (the number "400006" comes from four lines for each rename: diff --git a/really-big-dir/file-1-1-1-1-1 b/moved-big-dir/file-1-1-1-1-1 similarity index 100% rename from really-big-dir/file-1-1-1-1-1 rename to moved-big-dir/file-1-1-1-1-1 and the extra six lines is from a one-liner commit message and all the commit information and spacing). Most of those two seconds weren't even really the rename detection, it's really all the other stuff needed to get there. With the old code, this wouldn't have been practically possible. Doing a pairwise check of the ten billion possible pairs would have been prohibitively expensive. In fact, even with the rename limiter in place, the old code would waste a lot of time just on the diff_filespec checks, and despite not even trying to find renames, it used to look like: [torvalds@woody big-rename]$ time git show -C | wc -l 1400006 real 0m12.337s user 0m12.285s sys 0m0.192s ie we used to take 12 seconds for this load and not even do any rename detection! (The number 1400006 comes from fourteen lines per file moved: seven lines each for the delete and the create of a one-liner file, and the same extra six lines of commit information). Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org> Signed-off-by: Junio C Hamano <gitster@pobox.com>
2007-10-25 18:24:47 +00:00
/* Did we only want exact renames? */
if (minimum_score == MAX_SCORE)
goto cleanup;
/*
* Calculate how many renames are left (but all the source
* files still remain as options for rename/copies!)
*/
num_create = (rename_dst_nr - rename_count);
/* All done? */
if (!num_create)
goto cleanup;
switch (too_many_rename_candidates(num_create, options)) {
case 1:
goto cleanup;
case 2:
options->degraded_cc_to_c = 1;
skip_unmodified = 1;
break;
default:
break;
}
if (options->show_rename_progress) {
progress: simplify "delayed" progress API We used to expose the full power of the delayed progress API to the callers, so that they can specify, not just the message to show and expected total amount of work that is used to compute the percentage of work performed so far, the percent-threshold parameter P and the delay-seconds parameter N. The progress meter starts to show at N seconds into the operation only if we have not yet completed P per-cent of the total work. Most callers used either (0%, 2s) or (50%, 1s) as (P, N), but there are oddballs that chose more random-looking values like 95%. For a smoother workload, (50%, 1s) would allow us to start showing the progress meter earlier than (0%, 2s), while keeping the chance of not showing progress meter for long running operation the same as the latter. For a task that would take 2s or more to complete, it is likely that less than half of it would complete within the first second, if the workload is smooth. But for a spiky workload whose earlier part is easier, such a setting is likely to fail to show the progress meter entirely and (0%, 2s) is more appropriate. But that is merely a theory. Realistically, it is of dubious value to ask each codepath to carefully consider smoothness of their workload and specify their own setting by passing two extra parameters. Let's simplify the API by dropping both parameters and have everybody use (0%, 2s). Oh, by the way, the percent-threshold parameter and the structure member were consistently misspelled, which also is now fixed ;-) Helped-by: Jeff King <peff@peff.net> Signed-off-by: Junio C Hamano <gitster@pobox.com>
2017-08-19 17:39:41 +00:00
progress = start_delayed_progress(
_("Performing inexact rename detection"),
(uint64_t)rename_dst_nr * (uint64_t)rename_src_nr);
}
mx = xcalloc(st_mult(NUM_CANDIDATE_PER_DST, num_create), sizeof(*mx));
for (dst_cnt = i = 0; i < rename_dst_nr; i++) {
struct diff_filespec *two = rename_dst[i].two;
struct diff_score *m;
if (rename_dst[i].pair)
continue; /* dealt with exact match already. */
m = &mx[dst_cnt * NUM_CANDIDATE_PER_DST];
for (j = 0; j < NUM_CANDIDATE_PER_DST; j++)
m[j].dst = -1;
for (j = 0; j < rename_src_nr; j++) {
struct diff_filespec *one = rename_src[j].p->one;
struct diff_score this_src;
if (skip_unmodified &&
diff_unmodified_pair(rename_src[j].p))
continue;
this_src.score = estimate_similarity(options->repo,
one, two,
minimum_score,
skip_unmodified);
this_src.name_score = basename_same(one, two);
this_src.dst = i;
this_src.src = j;
record_if_better(m, &this_src);
/*
* Once we run estimate_similarity,
* We do not need the text anymore.
*/
diff_free_filespec_blob(one);
diff_free_filespec_blob(two);
}
dst_cnt++;
display_progress(progress, (uint64_t)(i+1)*(uint64_t)rename_src_nr);
}
stop_progress(&progress);
/* cost matrix sorted by most to least similar pair */
STABLE_QSORT(mx, dst_cnt * NUM_CANDIDATE_PER_DST, score_compare);
rename_count += find_renames(mx, dst_cnt, minimum_score, 0);
if (detect_rename == DIFF_DETECT_COPY)
rename_count += find_renames(mx, dst_cnt, minimum_score, 1);
free(mx);
cleanup:
/* At this point, we have found some renames and copies and they
* are recorded in rename_dst. The original list is still in *q.
*/
DIFF_QUEUE_CLEAR(&outq);
for (i = 0; i < q->nr; i++) {
struct diff_filepair *p = q->queue[i];
struct diff_filepair *pair_to_free = NULL;
diffcore-rename: don't consider unmerged path as source Since e9c8409 (diff-index --cached --raw: show tree entry on the LHS for unmerged entries., 2007-01-05), an unmerged entry should be detected by using DIFF_PAIR_UNMERGED(p), not by noticing both one and two sides of the filepair records mode=0 entries. However, it forgot to update some parts of the rename detection logic. This only makes difference in the "diff --cached" codepath where an unmerged filepair carries information on the entries that came from the tree. It probably hasn't been noticed for a long time because nobody would run "diff -M" during a conflict resolution, but "git status" uses rename detection when it internally runs "diff-index" and "diff-files" and gives nonsense results. In an unmerged pair, "one" side can have a valid filespec to record the tree entry (e.g. what's in HEAD) when running "diff --cached". This can be used as a rename source to other paths in the index that are not unmerged. The path that is unmerged by definition does not have the final content yet (i.e. "two" side cannot have a valid filespec), so it can never be a rename destination. Use the DIFF_PAIR_UNMERGED() to detect unmerged filepair correctly, and allow the valid "one" side of an unmerged filepair to be considered a potential rename source, but never to be considered a rename destination. Commit message and first two test cases by Junio, the rest by Martin. Signed-off-by: Martin von Zweigbergk <martin.von.zweigbergk@gmail.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>
2011-03-24 02:41:01 +00:00
if (DIFF_PAIR_UNMERGED(p)) {
diff_q(&outq, p);
}
else if (!DIFF_FILE_VALID(p->one) && DIFF_FILE_VALID(p->two)) {
/*
* Creation
*
* We would output this create record if it has
* not been turned into a rename/copy already.
*/
struct diff_rename_dst *dst = locate_rename_dst(p->two);
if (dst && dst->pair) {
diff_q(&outq, dst->pair);
pair_to_free = p;
}
else
/* no matching rename/copy source, so
* record this as a creation.
*/
diff_q(&outq, p);
}
else if (DIFF_FILE_VALID(p->one) && !DIFF_FILE_VALID(p->two)) {
/*
* Deletion
*
* We would output this delete record if:
*
* (1) this is a broken delete and the counterpart
* broken create remains in the output; or
* (2) this is not a broken delete, and rename_dst
* does not have a rename/copy to move p->one->path
* out of existence.
*
* Otherwise, the counterpart broken create
* has been turned into a rename-edit; or
* delete did not have a matching create to
* begin with.
*/
if (DIFF_PAIR_BROKEN(p)) {
/* broken delete */
struct diff_rename_dst *dst = locate_rename_dst(p->one);
if (dst && dst->pair)
/* counterpart is now rename/copy */
pair_to_free = p;
}
else {
if (p->one->rename_used)
/* this path remains */
pair_to_free = p;
}
if (pair_to_free)
;
else
diff_q(&outq, p);
}
else if (!diff_unmodified_pair(p))
/* all the usual ones need to be kept */
diff_q(&outq, p);
else
/* no need to keep unmodified pairs */
pair_to_free = p;
if (pair_to_free)
diff_free_filepair(pair_to_free);
}
diff_debug_queue("done copying original", &outq);
free(q->queue);
*q = outq;
diff_debug_queue("done collapsing", q);
for (i = 0; i < rename_dst_nr; i++)
free_filespec(rename_dst[i].two);
FREE_AND_NULL(rename_dst);
rename_dst_nr = rename_dst_alloc = 0;
FREE_AND_NULL(rename_src);
rename_src_nr = rename_src_alloc = 0;
return;
}