git/preload-index.c

179 lines
4.3 KiB
C
Raw Normal View History

/*
* Copyright (C) 2008 Linus Torvalds
*/
#include "cache.h"
#include "pathspec.h"
#include "dir.h"
#include "environment.h"
#include "fsmonitor.h"
#include "gettext.h"
#include "config.h"
#include "progress.h"
#include "thread-utils.h"
#include "repository.h"
#include "symlinks.h"
#include "trace2.h"
/*
* Mostly randomly chosen maximum thread counts: we
* cap the parallelism to 20 threads, and we want
* to have at least 500 lstat's per thread for it to
* be worth starting a thread.
*/
#define MAX_PARALLEL (20)
#define THREAD_COST (500)
struct progress_data {
unsigned long n;
struct progress *progress;
pthread_mutex_t mutex;
};
struct thread_data {
pthread_t pthread;
struct index_state *index;
struct pathspec pathspec;
struct progress_data *progress;
int offset, nr;
int t2_nr_lstat;
};
static void *preload_thread(void *_data)
{
int nr, last_nr;
struct thread_data *p = _data;
struct index_state *index = p->index;
struct cache_entry **cep = index->cache + p->offset;
struct cache_def cache = CACHE_DEF_INIT;
nr = p->nr;
if (nr + p->offset > index->cache_nr)
nr = index->cache_nr - p->offset;
last_nr = nr;
do {
struct cache_entry *ce = *cep++;
struct stat st;
if (ce_stage(ce))
continue;
Make ce_uptodate() trustworthy again The rule has always been that a cache entry that is ce_uptodate(ce) means that we already have checked the work tree entity and we know there is no change in the work tree compared to the index, and nobody should have to double check. Note that false ce_uptodate(ce) does not mean it is known to be dirty---it only means we don't know if it is clean. There are a few codepaths (refresh-index and preload-index are among them) that mark a cache entry as up-to-date based solely on the return value from ie_match_stat(); this function uses lstat() to see if the work tree entity has been touched, and for a submodule entry, if its HEAD points at the same commit as the commit recorded in the index of the superproject (a submodule that is not even cloned is considered clean). A submodule is no longer considered unmodified merely because its HEAD matches the index of the superproject these days, in order to prevent people from forgetting to commit in the submodule and updating the superproject index with the new submodule commit, before commiting the state in the superproject. However, the patch to do so didn't update the codepath that marks cache entries up-to-date based on the updated definition and instead worked it around by saying "we don't trust the return value of ce_uptodate() for submodules." This makes ce_uptodate() trustworthy again by not marking submodule entries up-to-date. The next step _could_ be to introduce a few "in-core" flag bits to cache_entry structure to record "this entry is _known_ to be dirty", call is_submodule_modified() from ie_match_stat(), and use these new bits to avoid running this rather expensive check more than once, but that can be a separate patch. Signed-off-by: Junio C Hamano <gitster@pobox.com>
2010-01-24 08:10:20 +00:00
if (S_ISGITLINK(ce->ce_mode))
continue;
if (ce_uptodate(ce))
continue;
if (ce_skip_worktree(ce))
continue;
if (ce->ce_flags & CE_FSMONITOR_VALID)
continue;
if (p->progress && !(nr & 31)) {
struct progress_data *pd = p->progress;
pthread_mutex_lock(&pd->mutex);
pd->n += last_nr - nr;
display_progress(pd->progress, pd->n);
pthread_mutex_unlock(&pd->mutex);
last_nr = nr;
}
if (!ce_path_match(index, ce, &p->pathspec, NULL))
continue;
if (threaded_has_symlink_leading_path(&cache, ce->name, ce_namelen(ce)))
continue;
p->t2_nr_lstat++;
if (lstat(ce->name, &st))
continue;
if (ie_match_stat(index, ce, &st, CE_MATCH_RACY_IS_DIRTY|CE_MATCH_IGNORE_FSMONITOR))
continue;
ce_mark_uptodate(ce);
mark_fsmonitor_valid(): mark the index as changed if needed Without this bug fix, t7519's four "status doesn't detect unreported modifications" test cases would fail occasionally (and, oddly enough, *a lot* more frequently on Windows). The reason is that these test cases intentionally use the side effect of `git status` to re-write the index if any updates were detected: they first clean the worktree, run `git status` to update the index as well as show the output to the casual reader, then make the worktree dirty again and expect no changes to reported if running with a mocked fsmonitor hook. The problem with this strategy was that the index was written during said `git status` on the clean worktree for the *wrong* reason: not because the index was marked as changed (it wasn't), but because the recorded mtimes were racy with the index' own mtime. As the mtime granularity on Windows is 100 nanoseconds (see e.g. https://docs.microsoft.com/en-us/windows/desktop/SysInfo/file-times), the mtimes of the files are often enough *not* racy with the index', so that that `git status` call currently does not always update the index (including the fsmonitor extension), causing the test case to fail. The obvious fix: if we change *any* index entry's `CE_FSMONITOR_VALID` flag, we should also mark the index as changed. That will cause the index to be written upon `git status`, *including* an updated fsmonitor extension. Side note: Even though the reader might think that the t7519 issue should be *much* more prevalent on Linux, given that the ext4 filesystem (that seems to be used by every Linux distribution) stores mtimes in nanosecond precision. However, ext4 uses `current_kernel_time()` (see https://unix.stackexchange.com/questions/11599#comment762968_11599; it is *amazingly* hard to find any proper source of information about such ext4 questions) whose accuracy seems to depend on many factors but is safely worse than the 100-nanosecond granularity of NTFS (again, it is *horribly* hard to find anything remotely authoritative about this question). So it seems that the racy index condition that hid the bug fixed by this patch simply is a lot more likely on Linux than on Windows. But not impossible ;-) Signed-off-by: Johannes Schindelin <johannes.schindelin@gmx.de> Signed-off-by: Junio C Hamano <gitster@pobox.com>
2019-05-24 12:23:48 +00:00
mark_fsmonitor_valid(index, ce);
} while (--nr > 0);
if (p->progress) {
struct progress_data *pd = p->progress;
pthread_mutex_lock(&pd->mutex);
display_progress(pd->progress, pd->n + last_nr);
pthread_mutex_unlock(&pd->mutex);
}
cache_def_clear(&cache);
return NULL;
}
void preload_index(struct index_state *index,
const struct pathspec *pathspec,
unsigned int refresh_flags)
{
int threads, i, work, offset;
struct thread_data data[MAX_PARALLEL];
struct progress_data pd;
int t2_sum_lstat = 0;
if (!HAVE_THREADS || !core_preload_index)
return;
threads = index->cache_nr / THREAD_COST;
if ((index->cache_nr > 1) && (threads < 2) && git_env_bool("GIT_TEST_PRELOAD_INDEX", 0))
threads = 2;
if (threads < 2)
return;
trace2_region_enter("index", "preload", NULL);
trace_performance_enter();
if (threads > MAX_PARALLEL)
threads = MAX_PARALLEL;
offset = 0;
work = DIV_ROUND_UP(index->cache_nr, threads);
memset(&data, 0, sizeof(data));
memset(&pd, 0, sizeof(pd));
if (refresh_flags & REFRESH_PROGRESS && isatty(2)) {
pd.progress = start_delayed_progress(_("Refreshing index"), index->cache_nr);
pthread_mutex_init(&pd.mutex, NULL);
}
for (i = 0; i < threads; i++) {
struct thread_data *p = data+i;
int err;
p->index = index;
if (pathspec)
copy_pathspec(&p->pathspec, pathspec);
p->offset = offset;
p->nr = work;
if (pd.progress)
p->progress = &pd;
offset += work;
err = pthread_create(&p->pthread, NULL, preload_thread, p);
if (err)
die(_("unable to create threaded lstat: %s"), strerror(err));
}
for (i = 0; i < threads; i++) {
struct thread_data *p = data+i;
if (pthread_join(p->pthread, NULL))
die("unable to join threaded lstat");
t2_sum_lstat += p->t2_nr_lstat;
}
stop_progress(&pd.progress);
if (pathspec) {
/* earlier we made deep copies for each thread to work with */
for (i = 0; i < threads; i++)
clear_pathspec(&data[i].pathspec);
}
trace_performance_leave("preload index");
trace2_data_intmax("index", NULL, "preload/sum_lstat", t2_sum_lstat);
trace2_region_leave("index", "preload", NULL);
}
int repo_read_index_preload(struct repository *repo,
const struct pathspec *pathspec,
unsigned int refresh_flags)
{
int retval = repo_read_index(repo);
preload_index(repo->index, pathspec, refresh_flags);
return retval;
}