development/git
development/git
1
0
Fork 0
mirror of https://github.com/git/git synced 2024-08-27 19:49:21 +00:00
Code Issues Packages Projects Releases Wiki Activity

Modularize commit-walker

Browse source 
This turns the extern functions to be provided by the backend into a
struct of pointers, renames the functions to be more
namespace-friendly, and updates http-fetch to this interface. It
removes the unused include from http-push.c. It makes git-http-fetch a
builtin (with the implementation a separate file, accessible
directly).

Signed-off-by: Daniel Barkalow <barkalow@iabervon.org>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
This commit is contained in:
Daniel Barkalow 2007-09-10 23:02:45 -04:00 committed by Junio C Hamano
parent fbdeef948b
commit 30ae764b1e
9 changed files with 271 additions and 230 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

17
Makefile
View file

@ -514,7 +514,9 @@ else
CC_LD_DYNPATH = -R CC_LD_DYNPATH = -R
endif endif
ifndef NO_CURL ifdef NO_CURL
BASIC_CFLAGS += -DNO_CURL
else
ifdef CURLDIR ifdef CURLDIR
# Try "-Wl,-rpath=$(CURLDIR)/$(lib)" in such a case. # Try "-Wl,-rpath=$(CURLDIR)/$(lib)" in such a case.
BASIC_CFLAGS += -I$(CURLDIR)/include BASIC_CFLAGS += -I$(CURLDIR)/include
@ -522,7 +524,9 @@ ifndef NO_CURL
else else
CURL_LIBCURL = -lcurl CURL_LIBCURL = -lcurl
endif endif
PROGRAMS += git-http-fetch$X BUILTIN_OBJS += builtin-http-fetch.o
EXTLIBS += $(CURL_LIBCURL)
LIB_OBJS += http.o walker.o http-walker.o
curl_check := $(shell (echo 070908; curl-config --vernum) | sort -r | sed -ne 2p) curl_check := $(shell (echo 070908; curl-config --vernum) | sort -r | sed -ne 2p)
ifeq "$(curl_check)" "070908" ifeq "$(curl_check)" "070908"
ifndef NO_EXPAT ifndef NO_EXPAT
@ -884,7 +888,7 @@ http.o: http.c GIT-CFLAGS
$(QUIET_CC)$(CC) -o $*.o -c $(ALL_CFLAGS) -DGIT_USER_AGENT='"git/$(GIT_VERSION)"' $< $(QUIET_CC)$(CC) -o $*.o -c $(ALL_CFLAGS) -DGIT_USER_AGENT='"git/$(GIT_VERSION)"' $<
ifdef NO_EXPAT ifdef NO_EXPAT
http-fetch.o: http-fetch.c http.h GIT-CFLAGS http-walker.o: http-walker.c http.h GIT-CFLAGS
$(QUIET_CC)$(CC) -o $*.o -c $(ALL_CFLAGS) -DNO_EXPAT $< $(QUIET_CC)$(CC) -o $*.o -c $(ALL_CFLAGS) -DNO_EXPAT $<
endif endif
@ -893,16 +897,13 @@ git-%$X: %.o $(GITLIBS)
git-imap-send$X: imap-send.o $(LIB_FILE) git-imap-send$X: imap-send.o $(LIB_FILE)
http.o http-fetch.o http-push.o: http.h http.o http-walker.o http-push.o: http.h
git-http-fetch$X: fetch.o http.o http-fetch.o $(GITLIBS)
$(QUIET_LINK)$(CC) $(ALL_CFLAGS) -o $@ $(ALL_LDFLAGS) $(filter %.o,$^) \
$(LIBS) $(CURL_LIBCURL) $(EXPAT_LIBEXPAT)
git-http-push$X: revision.o http.o http-push.o $(GITLIBS) git-http-push$X: revision.o http.o http-push.o $(GITLIBS)
$(QUIET_LINK)$(CC) $(ALL_CFLAGS) -o $@ $(ALL_LDFLAGS) $(filter %.o,$^) \ $(QUIET_LINK)$(CC) $(ALL_CFLAGS) -o $@ $(ALL_LDFLAGS) $(filter %.o,$^) \
$(LIBS) $(CURL_LIBCURL) $(EXPAT_LIBEXPAT) $(LIBS) $(CURL_LIBCURL) $(EXPAT_LIBEXPAT)
$(LIB_OBJS) $(BUILTIN_OBJS) fetch.o: $(LIB_H) $(LIB_OBJS) $(BUILTIN_OBJS) walker.o: $(LIB_H)
$(patsubst git-%$X,%.o,$(PROGRAMS)): $(LIB_H) $(wildcard */*.h) $(patsubst git-%$X,%.o,$(PROGRAMS)): $(LIB_H) $(wildcard */*.h)
$(DIFF_OBJS): diffcore.h $(DIFF_OBJS): diffcore.h

77
builtin-http-fetch.c Normal file
View file

@ -0,0 +1,77 @@
#include "cache.h"
#include "walker.h"
int cmd_http_fetch(int argc, const char **argv, const char *prefix)
{
struct walker *walker;
int commits_on_stdin = 0;
int commits;
const char **write_ref = NULL;
char **commit_id;
const char *url;
int arg = 1;
int rc = 0;
int get_tree = 0;
int get_history = 0;
int get_all = 0;
int get_verbosely = 0;
int get_recover = 0;
git_config(git_default_config);
while (arg < argc && argv[arg][0] == '-') {
if (argv[arg][1] == 't') {
get_tree = 1;
} else if (argv[arg][1] == 'c') {
get_history = 1;
} else if (argv[arg][1] == 'a') {
get_all = 1;
get_tree = 1;
get_history = 1;
} else if (argv[arg][1] == 'v') {
get_verbosely = 1;
} else if (argv[arg][1] == 'w') {
write_ref = &argv[arg + 1];
arg++;
} else if (!strcmp(argv[arg], "--recover")) {
get_recover = 1;
} else if (!strcmp(argv[arg], "--stdin")) {
commits_on_stdin = 1;
}
arg++;
}
if (argc < arg + 2 - commits_on_stdin) {
usage("git-http-fetch [-c] [-t] [-a] [-v] [--recover] [-w ref] [--stdin] commit-id url");
return 1;
}
if (commits_on_stdin) {
commits = walker_targets_stdin(&commit_id, &write_ref);
} else {
commit_id = (char **) &argv[arg++];
commits = 1;
}
url = argv[arg];
walker = get_http_walker(url);
walker->get_tree = get_tree;
walker->get_history = get_history;
walker->get_all = get_all;
walker->get_verbosely = get_verbosely;
walker->get_recover = get_recover;
rc = walker_fetch(walker, commits, commit_id, write_ref, url);
if (commits_on_stdin)
walker_targets_free(commits, commit_id, write_ref);
if (walker->corrupt_object_found) {
fprintf(stderr,
"Some loose object were found to be corrupt, but they might be just\n"
"a false '404 Not Found' error message sent with incorrect HTTP\n"
"status code. Suggest running git-fsck.\n");
}
walker_free(walker);
return rc;
}

1
builtin.h
View file

@ -40,6 +40,7 @@ extern int cmd_gc(int argc, const char **argv, const char *prefix);
extern int cmd_get_tar_commit_id(int argc, const char **argv, const char *prefix); extern int cmd_get_tar_commit_id(int argc, const char **argv, const char *prefix);
extern int cmd_grep(int argc, const char **argv, const char *prefix); extern int cmd_grep(int argc, const char **argv, const char *prefix);
extern int cmd_help(int argc, const char **argv, const char *prefix); extern int cmd_help(int argc, const char **argv, const char *prefix);
extern int cmd_http_fetch(int argc, const char **argv, const char *prefix);
extern int cmd_init_db(int argc, const char **argv, const char *prefix); extern int cmd_init_db(int argc, const char **argv, const char *prefix);
extern int cmd_log(int argc, const char **argv, const char *prefix); extern int cmd_log(int argc, const char **argv, const char *prefix);
extern int cmd_log_reflog(int argc, const char **argv, const char *prefix); extern int cmd_log_reflog(int argc, const char **argv, const char *prefix);

54
fetch.h
View file

@ -1,54 +0,0 @@
#ifndef PULL_H
#define PULL_H
/*
* Fetch object given SHA1 from the remote, and store it locally under
* GIT_OBJECT_DIRECTORY. Return 0 on success, -1 on failure. To be
* provided by the particular implementation.
*/
extern int fetch(unsigned char *sha1);
/*
* Fetch the specified object and store it locally; fetch() will be
* called later to determine success. To be provided by the particular
* implementation.
*/
extern void prefetch(unsigned char *sha1);
/*
* Fetch ref (relative to $GIT_DIR/refs) from the remote, and store
* the 20-byte SHA1 in sha1. Return 0 on success, -1 on failure. To
* be provided by the particular implementation.
*/
extern int fetch_ref(char *ref, unsigned char *sha1);
/* Set to fetch the target tree. */
extern int get_tree;
/* Set to fetch the commit history. */
extern int get_history;
/* Set to fetch the trees in the commit history. */
extern int get_all;
/* Set to be verbose */
extern int get_verbosely;
/* Set to check on all reachable objects. */
extern int get_recover;
/* Report what we got under get_verbosely */
extern void pull_say(const char *, const char *);
/* Load pull targets from stdin */
extern int pull_targets_stdin(char ***target, const char ***write_ref);
/* Free up loaded targets */
extern void pull_targets_free(int targets, char **target, const char **write_ref);
/* If write_ref is set, the ref filename to write the target value to. */
/* If write_ref_log_details is set, additional text will appear in the ref log. */
extern int pull(int targets, char **target, const char **write_ref,
const char *write_ref_log_details);
#endif /* PULL_H */

3
git.c
View file

@ -344,6 +344,9 @@ static void handle_internal_command(int argc, const char **argv)
{ "get-tar-commit-id", cmd_get_tar_commit_id }, { "get-tar-commit-id", cmd_get_tar_commit_id },
{ "grep", cmd_grep, RUN_SETUP | USE_PAGER }, { "grep", cmd_grep, RUN_SETUP | USE_PAGER },
{ "help", cmd_help }, { "help", cmd_help },
#ifndef NO_CURL
{ "http-fetch", cmd_http_fetch, RUN_SETUP },
#endif
{ "init", cmd_init_db }, { "init", cmd_init_db },
{ "init-db", cmd_init_db }, { "init-db", cmd_init_db },
{ "log", cmd_log, RUN_SETUP | USE_PAGER }, { "log", cmd_log, RUN_SETUP | USE_PAGER },

1
http-push.c
View file

@ -1,7 +1,6 @@
#include "cache.h" #include "cache.h"
#include "commit.h" #include "commit.h"
#include "pack.h" #include "pack.h"
#include "fetch.h"
#include "tag.h" #include "tag.h"
#include "blob.h" #include "blob.h"
#include "http.h" #include "http.h"

228
http-fetch.c → http-walker.c
View file

@ -1,19 +1,12 @@
#include "cache.h" #include "cache.h"
#include "commit.h" #include "commit.h"
#include "pack.h" #include "pack.h"
#include "fetch.h" #include "walker.h"
#include "http.h" #include "http.h"
#define PREV_BUF_SIZE 4096 #define PREV_BUF_SIZE 4096
#define RANGE_HEADER_SIZE 30 #define RANGE_HEADER_SIZE 30
static int commits_on_stdin;
static int got_alternates = -1;
static int corrupt_object_found;
static struct curl_slist *no_pragma_header;
struct alt_base struct alt_base
{ {
char *base; char *base;
@ -22,8 +15,6 @@ struct alt_base
struct alt_base *next; struct alt_base *next;
}; };
static struct alt_base *alt;
enum object_request_state { enum object_request_state {
WAITING, WAITING,
ABORTED, ABORTED,
@ -33,6 +24,7 @@ enum object_request_state {
struct object_request struct object_request
{ {
struct walker *walker;
unsigned char sha1[20]; unsigned char sha1[20];
struct alt_base *repo; struct alt_base *repo;
char *url; char *url;
@ -53,6 +45,7 @@ struct object_request
}; };
struct alternates_request { struct alternates_request {
struct walker *walker;
const char *base; const char *base;
char *url; char *url;
struct buffer *buffer; struct buffer *buffer;
@ -60,6 +53,13 @@ struct alternates_request {
int http_specific; int http_specific;
}; };
struct walker_data {
const char *url;
int got_alternates;
struct alt_base *alt;
struct curl_slist *no_pragma_header;
};
static struct object_request *object_queue_head; static struct object_request *object_queue_head;
static size_t fwrite_sha1_file(void *ptr, size_t eltsize, size_t nmemb, static size_t fwrite_sha1_file(void *ptr, size_t eltsize, size_t nmemb,
@ -103,11 +103,12 @@ static int missing__target(int code, int result)
#define missing_target(a) missing__target((a)->http_code, (a)->curl_result) #define missing_target(a) missing__target((a)->http_code, (a)->curl_result)
static void fetch_alternates(const char *base); static void fetch_alternates(struct walker *walker, const char *base);
static void process_object_response(void *callback_data); static void process_object_response(void *callback_data);
static void start_object_request(struct object_request *obj_req) static void start_object_request(struct walker *walker,
struct object_request *obj_req)
{ {
char *hex = sha1_to_hex(obj_req->sha1); char *hex = sha1_to_hex(obj_req->sha1);
char prevfile[PATH_MAX]; char prevfile[PATH_MAX];
@ -120,6 +121,7 @@ static void start_object_request(struct object_request *obj_req)
char range[RANGE_HEADER_SIZE]; char range[RANGE_HEADER_SIZE];
struct curl_slist *range_header = NULL; struct curl_slist *range_header = NULL;
struct active_request_slot *slot; struct active_request_slot *slot;
struct walker_data *data = walker->data;
snprintf(prevfile, sizeof(prevfile), "%s.prev", obj_req->filename); snprintf(prevfile, sizeof(prevfile), "%s.prev", obj_req->filename);
unlink(prevfile); unlink(prevfile);
@ -212,12 +214,12 @@ static void start_object_request(struct object_request *obj_req)
curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_sha1_file); curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_sha1_file);
curl_easy_setopt(slot->curl, CURLOPT_ERRORBUFFER, obj_req->errorstr); curl_easy_setopt(slot->curl, CURLOPT_ERRORBUFFER, obj_req->errorstr);
curl_easy_setopt(slot->curl, CURLOPT_URL, url); curl_easy_setopt(slot->curl, CURLOPT_URL, url);
curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, no_pragma_header); curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, data->no_pragma_header);
/* If we have successfully processed data from a previous fetch /* If we have successfully processed data from a previous fetch
attempt, only fetch the data we don't already have. */ attempt, only fetch the data we don't already have. */
if (prev_posn>0) { if (prev_posn>0) {
if (get_verbosely) if (walker->get_verbosely)
fprintf(stderr, fprintf(stderr,
"Resuming fetch of object %s at byte %ld\n", "Resuming fetch of object %s at byte %ld\n",
hex, prev_posn); hex, prev_posn);
@ -268,13 +270,16 @@ static void finish_object_request(struct object_request *obj_req)
move_temp_to_file(obj_req->tmpfile, obj_req->filename); move_temp_to_file(obj_req->tmpfile, obj_req->filename);
if (obj_req->rename == 0) if (obj_req->rename == 0)
pull_say("got %s\n", sha1_to_hex(obj_req->sha1)); walker_say(obj_req->walker, "got %s\n", sha1_to_hex(obj_req->sha1));
} }
static void process_object_response(void *callback_data) static void process_object_response(void *callback_data)
{ {
struct object_request *obj_req = struct object_request *obj_req =
(struct object_request *)callback_data; (struct object_request *)callback_data;
struct walker *walker = obj_req->walker;
struct walker_data *data = walker->data;
struct alt_base *alt = data->alt;
obj_req->curl_result = obj_req->slot->curl_result; obj_req->curl_result = obj_req->slot->curl_result;
obj_req->http_code = obj_req->slot->http_code; obj_req->http_code = obj_req->slot->http_code;
@ -283,13 +288,13 @@ static void process_object_response(void *callback_data)
/* Use alternates if necessary */ /* Use alternates if necessary */
if (missing_target(obj_req)) { if (missing_target(obj_req)) {
fetch_alternates(alt->base); fetch_alternates(walker, alt->base);
if (obj_req->repo->next != NULL) { if (obj_req->repo->next != NULL) {
obj_req->repo = obj_req->repo =
obj_req->repo->next; obj_req->repo->next;
close(obj_req->local); close(obj_req->local);
obj_req->local = -1; obj_req->local = -1;
start_object_request(obj_req); start_object_request(walker, obj_req);
return; return;
} }
} }
@ -317,7 +322,7 @@ static void release_object_request(struct object_request *obj_req)
} }
#ifdef USE_CURL_MULTI #ifdef USE_CURL_MULTI
static int fill_active_slot(void *unused) static int fill_active_slot(struct walker *walker)
{ {
struct object_request *obj_req; struct object_request *obj_req;
@ -326,7 +331,7 @@ static int fill_active_slot(void *unused)
if (has_sha1_file(obj_req->sha1)) if (has_sha1_file(obj_req->sha1))
obj_req->state = COMPLETE; obj_req->state = COMPLETE;
else { else {
start_object_request(obj_req); start_object_request(walker, obj_req);
return 1; return 1;
} }
} }
@ -335,15 +340,17 @@ static int fill_active_slot(void *unused)
} }
#endif #endif
void prefetch(unsigned char *sha1) static void prefetch(struct walker *walker, unsigned char *sha1)
{ {
struct object_request *newreq; struct object_request *newreq;
struct object_request *tail; struct object_request *tail;
struct walker_data *data = walker->data;
char *filename = sha1_file_name(sha1); char *filename = sha1_file_name(sha1);
newreq = xmalloc(sizeof(*newreq)); newreq = xmalloc(sizeof(*newreq));
newreq->walker = walker;
hashcpy(newreq->sha1, sha1); hashcpy(newreq->sha1, sha1);
newreq->repo = alt; newreq->repo = data->alt;
newreq->url = NULL; newreq->url = NULL;
newreq->local = -1; newreq->local = -1;
newreq->state = WAITING; newreq->state = WAITING;
@ -369,7 +376,7 @@ void prefetch(unsigned char *sha1)
#endif #endif
} }
static int fetch_index(struct alt_base *repo, unsigned char *sha1) static int fetch_index(struct walker *walker, struct alt_base *repo, unsigned char *sha1)
{ {
char *hex = sha1_to_hex(sha1); char *hex = sha1_to_hex(sha1);
char *filename; char *filename;
@ -378,6 +385,7 @@ static int fetch_index(struct alt_base *repo, unsigned char *sha1)
long prev_posn = 0; long prev_posn = 0;
char range[RANGE_HEADER_SIZE]; char range[RANGE_HEADER_SIZE];
struct curl_slist *range_header = NULL; struct curl_slist *range_header = NULL;
struct walker_data *data = walker->data;
FILE *indexfile; FILE *indexfile;
struct active_request_slot *slot; struct active_request_slot *slot;
@ -386,7 +394,7 @@ static int fetch_index(struct alt_base *repo, unsigned char *sha1)
if (has_pack_index(sha1)) if (has_pack_index(sha1))
return 0; return 0;
if (get_verbosely) if (walker->get_verbosely)
fprintf(stderr, "Getting index for pack %s\n", hex); fprintf(stderr, "Getting index for pack %s\n", hex);
url = xmalloc(strlen(repo->base) + 64); url = xmalloc(strlen(repo->base) + 64);
@ -404,14 +412,14 @@ static int fetch_index(struct alt_base *repo, unsigned char *sha1)
curl_easy_setopt(slot->curl, CURLOPT_FILE, indexfile); curl_easy_setopt(slot->curl, CURLOPT_FILE, indexfile);
curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite); curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite);
curl_easy_setopt(slot->curl, CURLOPT_URL, url); curl_easy_setopt(slot->curl, CURLOPT_URL, url);
curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, no_pragma_header); curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, data->no_pragma_header);
slot->local = indexfile; slot->local = indexfile;
/* If there is data present from a previous transfer attempt, /* If there is data present from a previous transfer attempt,
resume where it left off */ resume where it left off */
prev_posn = ftell(indexfile); prev_posn = ftell(indexfile);
if (prev_posn>0) { if (prev_posn>0) {
if (get_verbosely) if (walker->get_verbosely)
fprintf(stderr, fprintf(stderr,
"Resuming fetch of index for pack %s at byte %ld\n", "Resuming fetch of index for pack %s at byte %ld\n",
hex, prev_posn); hex, prev_posn);
@ -437,13 +445,13 @@ static int fetch_index(struct alt_base *repo, unsigned char *sha1)
return move_temp_to_file(tmpfile, filename); return move_temp_to_file(tmpfile, filename);
} }
static int setup_index(struct alt_base *repo, unsigned char *sha1) static int setup_index(struct walker *walker, struct alt_base *repo, unsigned char *sha1)
{ {
struct packed_git *new_pack; struct packed_git *new_pack;
if (has_pack_file(sha1)) if (has_pack_file(sha1))
return 0; /* don't list this as something we can get */ return 0; /* don't list this as something we can get */
if (fetch_index(repo, sha1)) if (fetch_index(walker, repo, sha1))
return -1; return -1;
new_pack = parse_pack_index(sha1); new_pack = parse_pack_index(sha1);
@ -456,8 +464,10 @@ static void process_alternates_response(void *callback_data)
{ {
struct alternates_request *alt_req = struct alternates_request *alt_req =
(struct alternates_request *)callback_data; (struct alternates_request *)callback_data;
struct walker *walker = alt_req->walker;
struct walker_data *cdata = walker->data;
struct active_request_slot *slot = alt_req->slot; struct active_request_slot *slot = alt_req->slot;
struct alt_base *tail = alt; struct alt_base *tail = cdata->alt;
const char *base = alt_req->base; const char *base = alt_req->base;
static const char null_byte = '\0'; static const char null_byte = '\0';
char *data; char *data;
@ -478,7 +488,7 @@ static void process_alternates_response(void *callback_data)
if (slot->finished != NULL) if (slot->finished != NULL)
(*slot->finished) = 0; (*slot->finished) = 0;
if (!start_active_slot(slot)) { if (!start_active_slot(slot)) {
got_alternates = -1; cdata->got_alternates = -1;
slot->in_use = 0; slot->in_use = 0;
if (slot->finished != NULL) if (slot->finished != NULL)
(*slot->finished) = 1; (*slot->finished) = 1;
@ -487,7 +497,7 @@ static void process_alternates_response(void *callback_data)
} }
} else if (slot->curl_result != CURLE_OK) { } else if (slot->curl_result != CURLE_OK) {
if (!missing_target(slot)) { if (!missing_target(slot)) {
got_alternates = -1; cdata->got_alternates = -1;
return; return;
} }
} }
@ -564,7 +574,7 @@ static void process_alternates_response(void *callback_data)
memcpy(target + serverlen, data + i, memcpy(target + serverlen, data + i,
posn - i - 7); posn - i - 7);
target[serverlen + posn - i - 7] = 0; target[serverlen + posn - i - 7] = 0;
if (get_verbosely) if (walker->get_verbosely)
fprintf(stderr, fprintf(stderr,
"Also look at %s\n", target); "Also look at %s\n", target);
newalt = xmalloc(sizeof(*newalt)); newalt = xmalloc(sizeof(*newalt));
@ -581,39 +591,40 @@ static void process_alternates_response(void *callback_data)
i = posn + 1; i = posn + 1;
} }
got_alternates = 1; cdata->got_alternates = 1;
} }
static void fetch_alternates(const char *base) static void fetch_alternates(struct walker *walker, const char *base)
{ {
struct buffer buffer; struct buffer buffer;
char *url; char *url;
char *data; char *data;
struct active_request_slot *slot; struct active_request_slot *slot;
struct alternates_request alt_req; struct alternates_request alt_req;
struct walker_data *cdata = walker->data;
/* If another request has already started fetching alternates, /* If another request has already started fetching alternates,
wait for them to arrive and return to processing this request's wait for them to arrive and return to processing this request's
curl message */ curl message */
#ifdef USE_CURL_MULTI #ifdef USE_CURL_MULTI
while (got_alternates == 0) { while (cdata->got_alternates == 0) {
step_active_slots(); step_active_slots();
} }
#endif #endif
/* Nothing to do if they've already been fetched */ /* Nothing to do if they've already been fetched */
if (got_alternates == 1) if (cdata->got_alternates == 1)
return; return;
/* Start the fetch */ /* Start the fetch */
got_alternates = 0; cdata->got_alternates = 0;
data = xmalloc(4096); data = xmalloc(4096);
buffer.size = 4096; buffer.size = 4096;
buffer.posn = 0; buffer.posn = 0;
buffer.buffer = data; buffer.buffer = data;
if (get_verbosely) if (walker->get_verbosely)
fprintf(stderr, "Getting alternates list for %s\n", base); fprintf(stderr, "Getting alternates list for %s\n", base);
url = xmalloc(strlen(base) + 31); url = xmalloc(strlen(base) + 31);
@ -623,6 +634,7 @@ static void fetch_alternates(const char *base)
may fail and need to have alternates loaded before continuing */ may fail and need to have alternates loaded before continuing */
slot = get_active_slot(); slot = get_active_slot();
slot->callback_func = process_alternates_response; slot->callback_func = process_alternates_response;
alt_req.walker = walker;
slot->callback_data = &alt_req; slot->callback_data = &alt_req;
curl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer); curl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer);
@ -638,13 +650,13 @@ static void fetch_alternates(const char *base)
if (start_active_slot(slot)) if (start_active_slot(slot))
run_active_slot(slot); run_active_slot(slot);
else else
got_alternates = -1; cdata->got_alternates = -1;
free(data); free(data);
free(url); free(url);
} }
static int fetch_indices(struct alt_base *repo) static int fetch_indices(struct walker *walker, struct alt_base *repo)
{ {
unsigned char sha1[20]; unsigned char sha1[20];
char *url; char *url;
@ -663,7 +675,7 @@ static int fetch_indices(struct alt_base *repo)
buffer.posn = 0; buffer.posn = 0;
buffer.buffer = data; buffer.buffer = data;
if (get_verbosely) if (walker->get_verbosely)
fprintf(stderr, "Getting pack list for %s\n", repo->base); fprintf(stderr, "Getting pack list for %s\n", repo->base);
url = xmalloc(strlen(repo->base) + 21); url = xmalloc(strlen(repo->base) + 21);
@ -703,7 +715,7 @@ static int fetch_indices(struct alt_base *repo)
!prefixcmp(data + i, " pack-") && !prefixcmp(data + i, " pack-") &&
!prefixcmp(data + i + 46, ".pack\n")) { !prefixcmp(data + i + 46, ".pack\n")) {
get_sha1_hex(data + i + 6, sha1); get_sha1_hex(data + i + 6, sha1);
setup_index(repo, sha1); setup_index(walker, repo, sha1);
i += 51; i += 51;
break; break;
} }
@ -719,7 +731,7 @@ static int fetch_indices(struct alt_base *repo)
return 0; return 0;
} }
static int fetch_pack(struct alt_base *repo, unsigned char *sha1) static int fetch_pack(struct walker *walker, struct alt_base *repo, unsigned char *sha1)
{ {
char *url; char *url;
struct packed_git *target; struct packed_git *target;
@ -731,17 +743,18 @@ static int fetch_pack(struct alt_base *repo, unsigned char *sha1)
long prev_posn = 0; long prev_posn = 0;
char range[RANGE_HEADER_SIZE]; char range[RANGE_HEADER_SIZE];
struct curl_slist *range_header = NULL; struct curl_slist *range_header = NULL;
struct walker_data *data = walker->data;
struct active_request_slot *slot; struct active_request_slot *slot;
struct slot_results results; struct slot_results results;
if (fetch_indices(repo)) if (fetch_indices(walker, repo))
return -1; return -1;
target = find_sha1_pack(sha1, repo->packs); target = find_sha1_pack(sha1, repo->packs);
if (!target) if (!target)
return -1; return -1;
if (get_verbosely) { if (walker->get_verbosely) {
fprintf(stderr, "Getting pack %s\n", fprintf(stderr, "Getting pack %s\n",
sha1_to_hex(target->sha1)); sha1_to_hex(target->sha1));
fprintf(stderr, " which contains %s\n", fprintf(stderr, " which contains %s\n",
@ -764,14 +777,14 @@ static int fetch_pack(struct alt_base *repo, unsigned char *sha1)
curl_easy_setopt(slot->curl, CURLOPT_FILE, packfile); curl_easy_setopt(slot->curl, CURLOPT_FILE, packfile);
curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite); curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite);
curl_easy_setopt(slot->curl, CURLOPT_URL, url); curl_easy_setopt(slot->curl, CURLOPT_URL, url);
curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, no_pragma_header); curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, data->no_pragma_header);
slot->local = packfile; slot->local = packfile;
/* If there is data present from a previous transfer attempt, /* If there is data present from a previous transfer attempt,
resume where it left off */ resume where it left off */
prev_posn = ftell(packfile); prev_posn = ftell(packfile);
if (prev_posn>0) { if (prev_posn>0) {
if (get_verbosely) if (walker->get_verbosely)
fprintf(stderr, fprintf(stderr,
"Resuming fetch of pack %s at byte %ld\n", "Resuming fetch of pack %s at byte %ld\n",
sha1_to_hex(target->sha1), prev_posn); sha1_to_hex(target->sha1), prev_posn);
@ -825,7 +838,7 @@ static void abort_object_request(struct object_request *obj_req)
release_object_request(obj_req); release_object_request(obj_req);
} }
static int fetch_object(struct alt_base *repo, unsigned char *sha1) static int fetch_object(struct walker *walker, struct alt_base *repo, unsigned char *sha1)
{ {
char *hex = sha1_to_hex(sha1); char *hex = sha1_to_hex(sha1);
int ret = 0; int ret = 0;
@ -846,7 +859,7 @@ static int fetch_object(struct alt_base *repo, unsigned char *sha1)
step_active_slots(); step_active_slots();
} }
#else #else
start_object_request(obj_req); start_object_request(walker, obj_req);
#endif #endif
while (obj_req->state == ACTIVE) { while (obj_req->state == ACTIVE) {
@ -867,7 +880,7 @@ static int fetch_object(struct alt_base *repo, unsigned char *sha1)
obj_req->errorstr, obj_req->curl_result, obj_req->errorstr, obj_req->curl_result,
obj_req->http_code, hex); obj_req->http_code, hex);
} else if (obj_req->zret != Z_STREAM_END) { } else if (obj_req->zret != Z_STREAM_END) {
corrupt_object_found++; walker->corrupt_object_found++;
ret = error("File %s (%s) corrupt", hex, obj_req->url); ret = error("File %s (%s) corrupt", hex, obj_req->url);
} else if (hashcmp(obj_req->sha1, obj_req->real_sha1)) { } else if (hashcmp(obj_req->sha1, obj_req->real_sha1)) {
ret = error("File %s has bad hash", hex); ret = error("File %s has bad hash", hex);
@ -880,20 +893,21 @@ static int fetch_object(struct alt_base *repo, unsigned char *sha1)
return ret; return ret;
} }
int fetch(unsigned char *sha1) static int fetch(struct walker *walker, unsigned char *sha1)
{ {
struct alt_base *altbase = alt; struct walker_data *data = walker->data;
struct alt_base *altbase = data->alt;
if (!fetch_object(altbase, sha1)) if (!fetch_object(walker, altbase, sha1))
return 0; return 0;
while (altbase) { while (altbase) {
if (!fetch_pack(altbase, sha1)) if (!fetch_pack(walker, altbase, sha1))
return 0; return 0;
fetch_alternates(alt->base); fetch_alternates(walker, data->alt->base);
altbase = altbase->next; altbase = altbase->next;
} }
return error("Unable to find %s under %s", sha1_to_hex(sha1), return error("Unable to find %s under %s", sha1_to_hex(sha1),
alt->base); data->alt->base);
} }
static inline int needs_quote(int ch) static inline int needs_quote(int ch)
@ -942,12 +956,13 @@ static char *quote_ref_url(const char *base, const char *ref)
return qref; return qref;
} }
int fetch_ref(char *ref, unsigned char *sha1) static int fetch_ref(struct walker *walker, char *ref, unsigned char *sha1)
{ {
char *url; char *url;
char hex[42]; char hex[42];
struct buffer buffer; struct buffer buffer;
const char *base = alt->base; struct walker_data *data = walker->data;
const char *base = data->alt->base;
struct active_request_slot *slot; struct active_request_slot *slot;
struct slot_results results; struct slot_results results;
buffer.size = 41; buffer.size = 41;
@ -976,84 +991,45 @@ int fetch_ref(char *ref, unsigned char *sha1)
return 0; return 0;
} }
int main(int argc, const char **argv) static void cleanup(struct walker *walker)
{
struct walker_data *data = walker->data;
http_cleanup();
curl_slist_free_all(data->no_pragma_header);
}
struct walker *get_http_walker(const char *url)
{ {
int commits;
const char **write_ref = NULL;
char **commit_id;
const char *url;
char *s; char *s;
int arg = 1; struct walker_data *data = xmalloc(sizeof(struct walker_data));
int rc = 0; struct walker *walker = xmalloc(sizeof(struct walker));
setup_git_directory();
git_config(git_default_config);
while (arg < argc && argv[arg][0] == '-') {
if (argv[arg][1] == 't') {
get_tree = 1;
} else if (argv[arg][1] == 'c') {
get_history = 1;
} else if (argv[arg][1] == 'a') {
get_all = 1;
get_tree = 1;
get_history = 1;
} else if (argv[arg][1] == 'v') {
get_verbosely = 1;
} else if (argv[arg][1] == 'w') {
write_ref = &argv[arg + 1];
arg++;
} else if (!strcmp(argv[arg], "--recover")) {
get_recover = 1;
} else if (!strcmp(argv[arg], "--stdin")) {
commits_on_stdin = 1;
}
arg++;
}
if (argc < arg + 2 - commits_on_stdin) {
usage("git-http-fetch [-c] [-t] [-a] [-v] [--recover] [-w ref] [--stdin] commit-id url");
return 1;
}
if (commits_on_stdin) {
commits = pull_targets_stdin(&commit_id, &write_ref);
} else {
commit_id = (char **) &argv[arg++];
commits = 1;
}
url = argv[arg];
http_init(); http_init();
no_pragma_header = curl_slist_append(no_pragma_header, "Pragma:"); data->no_pragma_header = curl_slist_append(NULL, "Pragma:");
alt = xmalloc(sizeof(*alt)); data->alt = xmalloc(sizeof(*data->alt));
alt->base = xmalloc(strlen(url) + 1); data->alt->base = xmalloc(strlen(url) + 1);
strcpy(alt->base, url); strcpy(data->alt->base, url);
for (s = alt->base + strlen(alt->base) - 1; *s == '/'; --s) for (s = data->alt->base + strlen(data->alt->base) - 1; *s == '/'; --s)
*s = 0; *s = 0;
alt->got_indices = 0;
alt->packs = NULL; data->alt->got_indices = 0;
alt->next = NULL; data->alt->packs = NULL;
data->alt->next = NULL;
data->got_alternates = -1;
walker->corrupt_object_found = 0;
walker->fetch = fetch;
walker->fetch_ref = fetch_ref;
walker->prefetch = prefetch;
walker->cleanup = cleanup;
walker->data = data;
#ifdef USE_CURL_MULTI #ifdef USE_CURL_MULTI
add_fill_function(NULL, fill_active_slot); add_fill_function(walker, (int (*)(void *)) fill_active_slot);
#endif #endif
if (pull(commits, commit_id, write_ref, url)) return walker;
rc = 1;
http_cleanup();
curl_slist_free_all(no_pragma_header);
if (commits_on_stdin)
pull_targets_free(commits, commit_id, write_ref);
if (corrupt_object_found) {
fprintf(stderr,
"Some loose object were found to be corrupt, but they might be just\n"
"a false '404 Not Found' error message sent with incorrect HTTP\n"
"status code. Suggest running git-fsck.\n");
}
return rc;
} }

83
fetch.c → walker.c
View file

@ -1,5 +1,5 @@
#include "cache.h" #include "cache.h"
#include "fetch.h" #include "walker.h"
#include "commit.h" #include "commit.h"
#include "tree.h" #include "tree.h"
#include "tree-walk.h" #include "tree-walk.h"
@ -8,16 +8,11 @@
#include "refs.h" #include "refs.h"
#include "strbuf.h" #include "strbuf.h"
int get_tree = 0;
int get_history = 0;
int get_all = 0;
int get_verbosely = 0;
int get_recover = 0;
static unsigned char current_commit_sha1[20]; static unsigned char current_commit_sha1[20];
void pull_say(const char *fmt, const char *hex) void walker_say(struct walker *walker, const char *fmt, const char *hex)
{ {
if (get_verbosely) if (walker->get_verbosely)
fprintf(stderr, fmt, hex); fprintf(stderr, fmt, hex);
} }
@ -32,9 +27,9 @@ static void report_missing(const struct object *obj)
sha1_to_hex(current_commit_sha1)); sha1_to_hex(current_commit_sha1));
} }
static int process(struct object *obj); static int process(struct walker *walker, struct object *obj);
static int process_tree(struct tree *tree) static int process_tree(struct walker *walker, struct tree *tree)
{ {
struct tree_desc desc; struct tree_desc desc;
struct name_entry entry; struct name_entry entry;
@ -59,7 +54,7 @@ static int process_tree(struct tree *tree)
if (blob) if (blob)
obj = &blob->object; obj = &blob->object;
} }
if (!obj || process(obj)) if (!obj || process(walker, obj))
return -1; return -1;
} }
free(tree->buffer); free(tree->buffer);
@ -74,7 +69,7 @@ static int process_tree(struct tree *tree)
static struct commit_list *complete = NULL; static struct commit_list *complete = NULL;
static int process_commit(struct commit *commit) static int process_commit(struct walker *walker, struct commit *commit)
{ {
if (parse_commit(commit)) if (parse_commit(commit))
return -1; return -1;
@ -88,43 +83,43 @@ static int process_commit(struct commit *commit)
hashcpy(current_commit_sha1, commit->object.sha1); hashcpy(current_commit_sha1, commit->object.sha1);
pull_say("walk %s\n", sha1_to_hex(commit->object.sha1)); walker_say(walker, "walk %s\n", sha1_to_hex(commit->object.sha1));
if (get_tree) { if (walker->get_tree) {
if (process(&commit->tree->object)) if (process(walker, &commit->tree->object))
return -1; return -1;
if (!get_all) if (!walker->get_all)
get_tree = 0; walker->get_tree = 0;
} }
if (get_history) { if (walker->get_history) {
struct commit_list *parents = commit->parents; struct commit_list *parents = commit->parents;
for (; parents; parents = parents->next) { for (; parents; parents = parents->next) {
if (process(&parents->item->object)) if (process(walker, &parents->item->object))
return -1; return -1;
} }
} }
return 0; return 0;
} }
static int process_tag(struct tag *tag) static int process_tag(struct walker *walker, struct tag *tag)
{ {
if (parse_tag(tag)) if (parse_tag(tag))
return -1; return -1;
return process(tag->tagged); return process(walker, tag->tagged);
} }
static struct object_list *process_queue = NULL; static struct object_list *process_queue = NULL;
static struct object_list **process_queue_end = &process_queue; static struct object_list **process_queue_end = &process_queue;
static int process_object(struct object *obj) static int process_object(struct walker *walker, struct object *obj)
{ {
if (obj->type == OBJ_COMMIT) { if (obj->type == OBJ_COMMIT) {
if (process_commit((struct commit *)obj)) if (process_commit(walker, (struct commit *)obj))
return -1; return -1;
return 0; return 0;
} }
if (obj->type == OBJ_TREE) { if (obj->type == OBJ_TREE) {
if (process_tree((struct tree *)obj)) if (process_tree(walker, (struct tree *)obj))
return -1; return -1;
return 0; return 0;
} }
@ -132,7 +127,7 @@ static int process_object(struct object *obj)
return 0; return 0;
} }
if (obj->type == OBJ_TAG) { if (obj->type == OBJ_TAG) {
if (process_tag((struct tag *)obj)) if (process_tag(walker, (struct tag *)obj))
return -1; return -1;
return 0; return 0;
} }
@ -141,7 +136,7 @@ static int process_object(struct object *obj)
typename(obj->type), sha1_to_hex(obj->sha1)); typename(obj->type), sha1_to_hex(obj->sha1));
} }
static int process(struct object *obj) static int process(struct walker *walker, struct object *obj)
{ {
if (obj->flags & SEEN) if (obj->flags & SEEN)
return 0; return 0;
@ -154,7 +149,7 @@ static int process(struct object *obj)
else { else {
if (obj->flags & COMPLETE) if (obj->flags & COMPLETE)
return 0; return 0;
prefetch(obj->sha1); walker->prefetch(walker, obj->sha1);
} }
object_list_insert(obj, process_queue_end); object_list_insert(obj, process_queue_end);
@ -162,7 +157,7 @@ static int process(struct object *obj)
return 0; return 0;
} }
static int loop(void) static int loop(struct walker *walker)
{ {
struct object_list *elem; struct object_list *elem;
@ -178,25 +173,25 @@ static int loop(void)
* the queue because we needed to fetch it first. * the queue because we needed to fetch it first.
*/ */
if (! (obj->flags & TO_SCAN)) { if (! (obj->flags & TO_SCAN)) {
if (fetch(obj->sha1)) { if (walker->fetch(walker, obj->sha1)) {
report_missing(obj); report_missing(obj);
return -1; return -1;
} }
} }
if (!obj->type) if (!obj->type)
parse_object(obj->sha1); parse_object(obj->sha1);
if (process_object(obj)) if (process_object(walker, obj))
return -1; return -1;
} }
return 0; return 0;
} }
static int interpret_target(char *target, unsigned char *sha1) static int interpret_target(struct walker *walker, char *target, unsigned char *sha1)
{ {
if (!get_sha1_hex(target, sha1)) if (!get_sha1_hex(target, sha1))
return 0; return 0;
if (!check_ref_format(target)) { if (!check_ref_format(target)) {
if (!fetch_ref(target, sha1)) { if (!walker->fetch_ref(walker, target, sha1)) {
return 0; return 0;
} }
} }
@ -213,7 +208,7 @@ static int mark_complete(const char *path, const unsigned char *sha1, int flag,
return 0; return 0;
} }
int pull_targets_stdin(char ***target, const char ***write_ref) int walker_targets_stdin(char ***target, const char ***write_ref)
{ {
int targets = 0, targets_alloc = 0; int targets = 0, targets_alloc = 0;
struct strbuf buf; struct strbuf buf;
@ -243,7 +238,7 @@ int pull_targets_stdin(char ***target, const char ***write_ref)
return targets; return targets;
} }
void pull_targets_free(int targets, char **target, const char **write_ref) void walker_targets_free(int targets, char **target, const char **write_ref)
{ {
while (targets--) { while (targets--) {
free(target[targets]); free(target[targets]);
@ -252,8 +247,8 @@ void pull_targets_free(int targets, char **target, const char **write_ref)
} }
} }
int pull(int targets, char **target, const char **write_ref, int walker_fetch(struct walker *walker, int targets, char **target,
const char *write_ref_log_details) const char **write_ref, const char *write_ref_log_details)
{ {
struct ref_lock **lock = xcalloc(targets, sizeof(struct ref_lock *)); struct ref_lock **lock = xcalloc(targets, sizeof(struct ref_lock *));
unsigned char *sha1 = xmalloc(targets * 20); unsigned char *sha1 = xmalloc(targets * 20);
@ -275,19 +270,19 @@ int pull(int targets, char **target, const char **write_ref,
} }
} }
if (!get_recover) if (!walker->get_recover)
for_each_ref(mark_complete, NULL); for_each_ref(mark_complete, NULL);
for (i = 0; i < targets; i++) { for (i = 0; i < targets; i++) {
if (interpret_target(target[i], &sha1[20 * i])) { if (interpret_target(walker, target[i], &sha1[20 * i])) {
error("Could not interpret %s as something to pull", target[i]); error("Could not interpret %s as something to pull", target[i]);
goto unlock_and_fail; goto unlock_and_fail;
} }
if (process(lookup_unknown_object(&sha1[20 * i]))) if (process(walker, lookup_unknown_object(&sha1[20 * i])))
goto unlock_and_fail; goto unlock_and_fail;
} }
if (loop()) if (loop(walker))
goto unlock_and_fail; goto unlock_and_fail;
if (write_ref_log_details) { if (write_ref_log_details) {
@ -308,10 +303,16 @@ int pull(int targets, char **target, const char **write_ref,
return 0; return 0;
unlock_and_fail: unlock_and_fail:
for (i = 0; i < targets; i++) for (i = 0; i < targets; i++)
if (lock[i]) if (lock[i])
unlock_ref(lock[i]); unlock_ref(lock[i]);
return -1; return -1;
} }
void walker_free(struct walker *walker)
{
walker->cleanup(walker);
free(walker);
}

37
walker.h Normal file
View file

@ -0,0 +1,37 @@
#ifndef WALKER_H
#define WALKER_H
struct walker {
void *data;
int (*fetch_ref)(struct walker *, char *ref, unsigned char *sha1);
void (*prefetch)(struct walker *, unsigned char *sha1);
int (*fetch)(struct walker *, unsigned char *sha1);
void (*cleanup)(struct walker *);
int get_tree;
int get_history;
int get_all;
int get_verbosely;
int get_recover;
int corrupt_object_found;
};
/* Report what we got under get_verbosely */
void walker_say(struct walker *walker, const char *, const char *);
/* Load pull targets from stdin */
int walker_targets_stdin(char ***target, const char ***write_ref);
/* Free up loaded targets */
void walker_targets_free(int targets, char **target, const char **write_ref);
/* If write_ref is set, the ref filename to write the target value to. */
/* If write_ref_log_details is set, additional text will appear in the ref log. */
int walker_fetch(struct walker *impl, int targets, char **target,
const char **write_ref, const char *write_ref_log_details);
void walker_free(struct walker *walker);
struct walker *get_http_walker(const char *url);
#endif /* WALKER_H */