diff --git a/Makefile b/Makefile index 775ffaac27..544385f3e3 100644 --- a/Makefile +++ b/Makefile @@ -250,8 +250,7 @@ LIB_OBJS = \ revision.o pager.o tree-walk.o xdiff-interface.o \ write_or_die.o trace.o list-objects.o grep.o \ alloc.o merge-file.o path-list.o help.o unpack-trees.o $(DIFF_OBJS) \ - color.o wt-status.o archive-zip.o archive-tar.o \ - utf8.o + color.o wt-status.o archive-zip.o archive-tar.o shallow.o utf8.o BUILTIN_OBJS = \ builtin-add.o \ diff --git a/commit.c b/commit.c index 3167ce62ac..59ea77c577 100644 --- a/commit.c +++ b/commit.c @@ -1,6 +1,7 @@ #include "cache.h" #include "tag.h" #include "commit.h" +#include "pkt-line.h" int save_commit_buffer = 1; @@ -221,6 +222,8 @@ static void prepare_commit_graft(void) return; graft_file = get_graft_file(); read_graft_file(graft_file); + /* make sure shallows are read */ + is_repository_shallow(); commit_graft_prepared = 1; } @@ -234,6 +237,37 @@ static struct commit_graft *lookup_commit_graft(const unsigned char *sha1) return commit_graft[pos]; } +int write_shallow_commits(int fd, int use_pack_protocol) +{ + int i, count = 0; + for (i = 0; i < commit_graft_nr; i++) + if (commit_graft[i]->nr_parent < 0) { + const char *hex = + sha1_to_hex(commit_graft[i]->sha1); + count++; + if (use_pack_protocol) + packet_write(fd, "shallow %s", hex); + else { + write(fd, hex, 40); + write(fd, "\n", 1); + } + } + return count; +} + +int unregister_shallow(const unsigned char *sha1) +{ + int pos = commit_graft_pos(sha1); + if (pos < 0) + return -1; + if (pos + 1 < commit_graft_nr) + memcpy(commit_graft + pos, commit_graft + pos + 1, + sizeof(struct commit_graft *) + * (commit_graft_nr - pos - 1)); + commit_graft_nr--; + return 0; +} + int parse_commit_buffer(struct commit *item, void *buffer, unsigned long size) { char *tail = buffer; diff --git a/commit.h b/commit.h index 10eea9f26f..936f8fce30 100644 --- a/commit.h +++ b/commit.h @@ -97,7 +97,7 @@ void sort_in_topological_order_fn(struct commit_list ** list, int lifo, struct commit_graft { unsigned char sha1[20]; - int nr_parent; + int nr_parent; /* < 0 if shallow commit */ unsigned char parent[FLEX_ARRAY][20]; /* more */ }; @@ -107,5 +107,12 @@ int read_graft_file(const char *graft_file); extern struct commit_list *get_merge_bases(struct commit *rev1, struct commit *rev2, int cleanup); +extern int register_shallow(const unsigned char *sha1); +extern int unregister_shallow(const unsigned char *sha1); +extern int write_shallow_commits(int fd, int use_pack_protocol); +extern int is_repository_shallow(); +extern struct commit_list *get_shallow_commits(struct object_array *heads, + int depth, int shallow_flag, int not_shallow_flag); + int in_merge_bases(struct commit *rev1, struct commit *rev2); #endif /* COMMIT_H */ diff --git a/fetch-pack.c b/fetch-pack.c index 92322cf4da..c527bf9e96 100644 --- a/fetch-pack.c +++ b/fetch-pack.c @@ -10,8 +10,9 @@ static int keep_pack; static int quiet; static int verbose; static int fetch_all; +static int depth; static const char fetch_pack_usage[] = -"git-fetch-pack [--all] [-q] [-v] [-k] [--thin] [--exec=upload-pack] [host:]directory ..."; +"git-fetch-pack [--all] [-q] [-v] [-k] [--thin] [--exec=upload-pack] [--depth=] [host:]directory ..."; static const char *exec = "git-upload-pack"; #define COMPLETE (1U << 0) @@ -179,10 +180,41 @@ static int find_common(int fd[2], unsigned char *result_sha1, packet_write(fd[1], "want %s\n", sha1_to_hex(remote)); fetching++; } + if (is_repository_shallow()) + write_shallow_commits(fd[1], 1); + if (depth > 0) + packet_write(fd[1], "deepen %d", depth); packet_flush(fd[1]); if (!fetching) return 1; + if (depth > 0) { + char line[1024]; + unsigned char sha1[20]; + int len; + + while ((len = packet_read_line(fd[0], line, sizeof(line)))) { + if (!strncmp("shallow ", line, 8)) { + if (get_sha1_hex(line + 8, sha1)) + die("invalid shallow line: %s", line); + register_shallow(sha1); + continue; + } + if (!strncmp("unshallow ", line, 10)) { + if (get_sha1_hex(line + 10, sha1)) + die("invalid unshallow line: %s", line); + if (!lookup_object(sha1)) + die("object not found: %s", line); + /* make sure that it is parsed as shallow */ + parse_object(sha1); + if (unregister_shallow(sha1)) + die("no shallow found: %s", line); + continue; + } + die("expected shallow/unshallow, got %s", line); + } + } + flushes = 0; retval = -1; while ((sha1 = get_rev())) { @@ -309,7 +341,8 @@ static void filter_refs(struct ref **refs, int nr_match, char **match) if (!memcmp(ref->name, "refs/", 5) && check_ref_format(ref->name + 5)) ; /* trash */ - else if (fetch_all) { + else if (fetch_all && + (!depth || strncmp(ref->name, "refs/tags/", 10) )) { *newtail = ref; ref->next = NULL; newtail = &ref->next; @@ -368,9 +401,11 @@ static int everything_local(struct ref **refs, int nr_match, char **match) } } - for_each_ref(mark_complete, NULL); - if (cutoff) - mark_recent_complete_commits(cutoff); + if (!depth) { + for_each_ref(mark_complete, NULL); + if (cutoff) + mark_recent_complete_commits(cutoff); + } /* * Mark all complete remote refs as common refs. @@ -522,6 +557,8 @@ static int fetch_pack(int fd[2], int nr_match, char **match) int status; get_remote_heads(fd[0], &ref, 0, NULL, 0); + if (is_repository_shallow() && !server_supports("shallow")) + die("Server does not support shallow clients"); if (server_supports("multi_ack")) { if (verbose) fprintf(stderr, "Server supports multi_ack\n"); @@ -594,6 +631,8 @@ int main(int argc, char **argv) char *dest = NULL, **heads; int fd[2]; pid_t pid; + struct stat st; + struct lock_file lock; setup_git_directory(); @@ -627,6 +666,12 @@ int main(int argc, char **argv) verbose = 1; continue; } + if (!strncmp("--depth=", arg, 8)) { + depth = strtol(arg + 8, NULL, 0); + if (stat(git_path("shallow"), &st)) + st.st_mtime = 0; + continue; + } usage(fetch_pack_usage); } dest = arg; @@ -659,5 +704,34 @@ int main(int argc, char **argv) } } + if (!ret && depth > 0) { + struct cache_time mtime; + char *shallow = git_path("shallow"); + int fd; + + mtime.sec = st.st_mtime; +#ifdef USE_NSEC + mtime.usec = st.st_mtim.usec; +#endif + if (stat(shallow, &st)) { + if (mtime.sec) + die("shallow file was removed during fetch"); + } else if (st.st_mtime != mtime.sec +#ifdef USE_NSEC + || st.st_mtim.usec != mtime.usec +#endif + ) + die("shallow file was changed during fetch"); + + fd = hold_lock_file_for_update(&lock, shallow, 1); + if (!write_shallow_commits(fd, 0)) { + unlink(shallow); + rollback_lock_file(&lock); + } else { + close(fd); + commit_lock_file(&lock); + } + } + return !!ret; } diff --git a/git-clone.sh b/git-clone.sh index 490f3e48db..f37eb9d105 100755 --- a/git-clone.sh +++ b/git-clone.sh @@ -14,7 +14,7 @@ die() { } usage() { - die "Usage: $0 [--template=] [--reference ] [--bare] [-l [-s]] [-q] [-u ] [--origin ] [-n] []" + die "Usage: $0 [--template=] [--reference ] [--bare] [-l [-s]] [-q] [-u ] [--origin ] [--depth ] [-n] []" } get_repo_base() { @@ -120,6 +120,7 @@ reference= origin= origin_override= use_separate_remote=t +depth= while case "$#,$1" in 0,*) break ;; @@ -163,6 +164,10 @@ while *,-u|*,--upload-pack) shift upload_pack="--exec=$1" ;; + 1,--depth) usage;; + *,--depth) + shift + depth="--depth=$1";; *,-*) usage ;; *) break ;; esac @@ -267,6 +272,10 @@ yes,yes) *) case "$repo" in rsync://*) + case "$depth" in + "") ;; + *) die "shallow over rsync not supported" ;; + esac rsync $quiet -av --ignore-existing \ --exclude info "$repo/objects/" "$GIT_DIR/objects/" || exit @@ -295,6 +304,10 @@ yes,yes) git-ls-remote "$repo" >"$GIT_DIR/CLONE_HEAD" || exit 1 ;; https://*|http://*|ftp://*) + case "$depth" in + "") ;; + *) die "shallow over http or ftp not supported" ;; + esac if test -z "@@NO_CURL@@" then clone_dumb_http "$repo" "$D" @@ -304,8 +317,8 @@ yes,yes) ;; *) case "$upload_pack" in - '') git-fetch-pack --all -k $quiet "$repo" ;; - *) git-fetch-pack --all -k $quiet "$upload_pack" "$repo" ;; + '') git-fetch-pack --all -k $quiet $depth "$repo" ;; + *) git-fetch-pack --all -k $quiet "$upload_pack" $depth "$repo" ;; esac >"$GIT_DIR/CLONE_HEAD" || die "fetch-pack from '$repo' failed." ;; diff --git a/git-fetch.sh b/git-fetch.sh index 7fb47f69fb..8bd11f8b60 100755 --- a/git-fetch.sh +++ b/git-fetch.sh @@ -28,6 +28,7 @@ update_head_ok= exec= upload_pack= keep= +shallow_depth= while case "$#" in 0) break ;; esac do case "$1" in @@ -60,6 +61,13 @@ do -k|--k|--ke|--kee|--keep) keep='-k -k' ;; + --depth=*) + shallow_depth="--depth=`expr "z$1" : 'z-[^=]*=\(.*\)'`" + ;; + --depth) + shift + shallow_depth="--depth=$1" + ;; -*) usage ;; @@ -300,6 +308,8 @@ fetch_main () { # There are transports that can fetch only one head at a time... case "$remote" in http://* | https://* | ftp://*) + test -n "$shallow_depth" && + die "shallow clone with http not supported" proto=`expr "$remote" : '\([^:]*\):'` if [ -n "$GIT_SSL_NO_VERIFY" ]; then curl_extra_args="-k" @@ -326,6 +336,8 @@ fetch_main () { git-http-fetch -v -a "$head" "$remote/" || exit ;; rsync://*) + test -n "$shallow_depth" && + die "shallow clone with rsync not supported" TMP_HEAD="$GIT_DIR/TMP_HEAD" rsync -L -q "$remote/$remote_name" "$TMP_HEAD" || exit 1 head=$(git-rev-parse --verify TMP_HEAD) @@ -373,7 +385,7 @@ fetch_main () { pack_lockfile= IFS=" $LF" ( - git-fetch-pack --thin $exec $keep "$remote" $rref || echo failed "$remote" + git-fetch-pack --thin $exec $keep $shallow_depth "$remote" $rref || echo failed "$remote" ) | while read sha1 remote_name do @@ -446,6 +458,8 @@ case "$no_tags$tags" in case "$taglist" in '') ;; ?*) + # do not deepen a shallow tree when following tags + shallow_depth= fetch_main "$taglist" || exit ;; esac esac diff --git a/shallow.c b/shallow.c new file mode 100644 index 0000000000..3d53d17423 --- /dev/null +++ b/shallow.c @@ -0,0 +1,104 @@ +#include "cache.h" +#include "commit.h" +#include "tag.h" + +static int is_shallow = -1; + +int register_shallow(const unsigned char *sha1) +{ + struct commit_graft *graft = + xmalloc(sizeof(struct commit_graft)); + struct commit *commit = lookup_commit(sha1); + + hashcpy(graft->sha1, sha1); + graft->nr_parent = -1; + if (commit && commit->object.parsed) + commit->parents = NULL; + return register_commit_graft(graft, 0); +} + +int is_repository_shallow() +{ + FILE *fp; + char buf[1024]; + + if (is_shallow >= 0) + return is_shallow; + + fp = fopen(git_path("shallow"), "r"); + if (!fp) { + is_shallow = 0; + return is_shallow; + } + is_shallow = 1; + + while (fgets(buf, sizeof(buf), fp)) { + unsigned char sha1[20]; + if (get_sha1_hex(buf, sha1)) + die("bad shallow line: %s", buf); + register_shallow(sha1); + } + fclose(fp); + return is_shallow; +} + +struct commit_list *get_shallow_commits(struct object_array *heads, int depth, + int shallow_flag, int not_shallow_flag) +{ + int i = 0, cur_depth = 0; + struct commit_list *result = NULL; + struct object_array stack = {0, 0, NULL}; + struct commit *commit = NULL; + + while (commit || i < heads->nr || stack.nr) { + struct commit_list *p; + if (!commit) { + if (i < heads->nr) { + commit = (struct commit *) + deref_tag(heads->objects[i++].item, NULL, 0); + if (commit->object.type != OBJ_COMMIT) { + commit = NULL; + continue; + } + if (!commit->util) + commit->util = xmalloc(sizeof(int)); + *(int *)commit->util = 0; + cur_depth = 0; + } else { + commit = (struct commit *) + stack.objects[--stack.nr].item; + cur_depth = *(int *)commit->util; + } + } + parse_commit(commit); + commit->object.flags |= not_shallow_flag; + cur_depth++; + for (p = commit->parents, commit = NULL; p; p = p->next) { + if (!p->item->util) { + int *pointer = xmalloc(sizeof(int)); + p->item->util = pointer; + *pointer = cur_depth; + } else { + int *pointer = p->item->util; + if (cur_depth >= *pointer) + continue; + *pointer = cur_depth; + } + if (cur_depth < depth) { + if (p->next) + add_object_array(&p->item->object, + NULL, &stack); + else { + commit = p->item; + cur_depth = *(int *)commit->util; + } + } else { + commit_list_insert(p->item, &result); + p->item->object.flags |= shallow_flag; + } + } + } + + return result; +} + diff --git a/t/t5500-fetch-pack.sh b/t/t5500-fetch-pack.sh index f7625a6f46..77c3c575d8 100755 --- a/t/t5500-fetch-pack.sh +++ b/t/t5500-fetch-pack.sh @@ -128,4 +128,54 @@ pull_to_client 2nd "B" $((64*3)) pull_to_client 3rd "A" $((1*3)) # old fails +test_expect_success "clone shallow" "git-clone --depth 2 . shallow" + +(cd shallow; git-count-objects -v) > count.shallow + +test_expect_success "clone shallow object count" \ + "test \"in-pack: 18\" = \"$(grep in-pack count.shallow)\"" + +count_output () { + sed -e '/^in-pack:/d' -e '/^packs:/d' -e '/: 0$/d' "$1" +} + +test_expect_success "clone shallow object count (part 2)" ' + test -z "$(count_output count.shallow)" +' + +test_expect_success "fsck in shallow repo" \ + "(cd shallow; git-fsck-objects --full)" + +#test_done; exit + +add B66 $B65 +add B67 $B66 + +test_expect_success "pull in shallow repo" \ + "(cd shallow; git pull .. B)" + +(cd shallow; git-count-objects -v) > count.shallow +test_expect_success "clone shallow object count" \ + "test \"count: 6\" = \"$(grep count count.shallow)\"" + +add B68 $B67 +add B69 $B68 + +test_expect_success "deepening pull in shallow repo" \ + "(cd shallow; git pull --depth 4 .. B)" + +(cd shallow; git-count-objects -v) > count.shallow +test_expect_success "clone shallow object count" \ + "test \"count: 12\" = \"$(grep count count.shallow)\"" + +test_expect_success "deepening fetch in shallow repo" \ + "(cd shallow; git fetch --depth 4 .. A:A)" + +(cd shallow; git-count-objects -v) > count.shallow +test_expect_success "clone shallow object count" \ + "test \"count: 18\" = \"$(grep count count.shallow)\"" + +test_expect_failure "pull in shallow repo with missing merge base" \ + "(cd shallow; git pull --depth 4 .. A)" + test_done diff --git a/upload-pack.c b/upload-pack.c index 32b06b2e66..c568ef066c 100644 --- a/upload-pack.c +++ b/upload-pack.c @@ -6,6 +6,9 @@ #include "object.h" #include "commit.h" #include "exec_cmd.h" +#include "diff.h" +#include "revision.h" +#include "list-objects.h" static const char upload_pack_usage[] = "git-upload-pack [--strict] [--timeout=nn] "; @@ -16,6 +19,10 @@ static const char upload_pack_usage[] = "git-upload-pack [--strict] [--timeout=n #define COMMON_KNOWN (1u << 14) #define REACHABLE (1u << 15) +#define SHALLOW (1u << 16) +#define NOT_SHALLOW (1u << 17) +#define CLIENT_SHALLOW (1u << 18) + static unsigned long oldest_have; static int multi_ack, nr_our_refs; @@ -54,6 +61,40 @@ static ssize_t send_client_data(int fd, const char *data, ssize_t sz) return safe_write(fd, data, sz); } +FILE *pack_pipe = NULL; +static void show_commit(struct commit *commit) +{ + if (commit->object.flags & BOUNDARY) + fputc('-', pack_pipe); + if (fputs(sha1_to_hex(commit->object.sha1), pack_pipe) < 0) + die("broken output pipe"); + fputc('\n', pack_pipe); + fflush(pack_pipe); + free(commit->buffer); + commit->buffer = NULL; +} + +static void show_object(struct object_array_entry *p) +{ + /* An object with name "foo\n0000000..." can be used to + * confuse downstream git-pack-objects very badly. + */ + const char *ep = strchr(p->name, '\n'); + if (ep) { + fprintf(pack_pipe, "%s %.*s\n", sha1_to_hex(p->item->sha1), + (int) (ep - p->name), + p->name); + } + else + fprintf(pack_pipe, "%s %s\n", + sha1_to_hex(p->item->sha1), p->name); +} + +static void show_edge(struct commit *commit) +{ + fprintf(pack_pipe, "-%s\n", sha1_to_hex(commit->object.sha1)); +} + static void create_pack_file(void) { /* Pipes between rev-list to pack-objects, pack-objects to us @@ -75,48 +116,40 @@ static void create_pack_file(void) if (!pid_rev_list) { int i; - int args; - const char **argv; - const char **p; - char *buf; + struct rev_info revs; + + pack_pipe = fdopen(lp_pipe[1], "w"); + + if (create_full_pack) + use_thin_pack = 0; /* no point doing it */ + init_revisions(&revs, NULL); + revs.tag_objects = 1; + revs.tree_objects = 1; + revs.blob_objects = 1; + if (use_thin_pack) + revs.edge_hint = 1; if (create_full_pack) { - args = 10; - use_thin_pack = 0; /* no point doing it */ - } - else - args = have_obj.nr + want_obj.nr + 5; - p = xmalloc(args * sizeof(char *)); - argv = (const char **) p; - buf = xmalloc(args * 45); - - dup2(lp_pipe[1], 1); - close(0); - close(lp_pipe[0]); - close(lp_pipe[1]); - *p++ = "rev-list"; - *p++ = use_thin_pack ? "--objects-edge" : "--objects"; - if (create_full_pack) - *p++ = "--all"; - else { + const char *args[] = {"rev-list", "--all", NULL}; + setup_revisions(2, args, &revs, NULL); + } else { for (i = 0; i < want_obj.nr; i++) { struct object *o = want_obj.objects[i].item; - *p++ = buf; - memcpy(buf, sha1_to_hex(o->sha1), 41); - buf += 41; + /* why??? */ + o->flags &= ~UNINTERESTING; + add_pending_object(&revs, o, NULL); } - } - if (!create_full_pack) for (i = 0; i < have_obj.nr; i++) { struct object *o = have_obj.objects[i].item; - *p++ = buf; - *buf++ = '^'; - memcpy(buf, sha1_to_hex(o->sha1), 41); - buf += 41; + o->flags |= UNINTERESTING; + add_pending_object(&revs, o, NULL); } - *p++ = NULL; - execv_git_cmd(argv); - die("git-upload-pack: unable to exec git-rev-list"); + setup_revisions(0, NULL, &revs, NULL); + } + prepare_revision_walk(&revs); + mark_edges_uninteresting(revs.commits, &revs, show_edge); + traverse_commit_list(&revs, show_commit, show_object); + exit(0); } if (pipe(pu_pipe) < 0) @@ -456,8 +489,9 @@ static int get_common_commits(void) static void receive_needs(void) { + struct object_array shallows = {0, 0, NULL}; static char line[1000]; - int len; + int len, depth = 0; for (;;) { struct object *o; @@ -465,8 +499,29 @@ static void receive_needs(void) len = packet_read_line(0, line, sizeof(line)); reset_timeout(); if (!len) - return; + break; + if (!strncmp("shallow ", line, 8)) { + unsigned char sha1[20]; + struct object *object; + use_thin_pack = 0; + if (get_sha1(line + 8, sha1)) + die("invalid shallow line: %s", line); + object = parse_object(sha1); + if (!object) + die("did not find object for %s", line); + object->flags |= CLIENT_SHALLOW; + add_object_array(object, NULL, &shallows); + continue; + } + if (!strncmp("deepen ", line, 7)) { + char *end; + use_thin_pack = 0; + depth = strtol(line + 7, &end, 0); + if (end == line + 7 || depth <= 0) + die("Invalid deepen: %s", line); + continue; + } if (strncmp("want ", line, 5) || get_sha1_hex(line+5, sha1_buf)) die("git-upload-pack: protocol error, " @@ -498,11 +553,58 @@ static void receive_needs(void) add_object_array(o, NULL, &want_obj); } } + if (depth == 0 && shallows.nr == 0) + return; + if (depth > 0) { + struct commit_list *result, *backup; + int i; + backup = result = get_shallow_commits(&want_obj, depth, + SHALLOW, NOT_SHALLOW); + while (result) { + struct object *object = &result->item->object; + if (!(object->flags & (CLIENT_SHALLOW|NOT_SHALLOW))) { + packet_write(1, "shallow %s", + sha1_to_hex(object->sha1)); + register_shallow(object->sha1); + } + result = result->next; + } + free_commit_list(backup); + for (i = 0; i < shallows.nr; i++) { + struct object *object = shallows.objects[i].item; + if (object->flags & NOT_SHALLOW) { + struct commit_list *parents; + packet_write(1, "unshallow %s", + sha1_to_hex(object->sha1)); + object->flags &= ~CLIENT_SHALLOW; + /* make sure the real parents are parsed */ + unregister_shallow(object->sha1); + object->parsed = 0; + parse_commit((struct commit *)object); + parents = ((struct commit *)object)->parents; + while (parents) { + add_object_array(&parents->item->object, + NULL, &want_obj); + parents = parents->next; + } + } + /* make sure commit traversal conforms to client */ + register_shallow(object->sha1); + } + packet_flush(1); + } else + if (shallows.nr > 0) { + int i; + for (i = 0; i < shallows.nr; i++) + register_shallow(shallows.objects[i].item->sha1); + } + free(shallows.objects); } static int send_ref(const char *refname, const unsigned char *sha1, int flag, void *cb_data) { - static const char *capabilities = "multi_ack thin-pack side-band side-band-64k ofs-delta"; + static const char *capabilities = "multi_ack thin-pack side-band" + " side-band-64k ofs-delta shallow"; struct object *o = parse_object(sha1); if (!o)