2007-09-11 03:03:00 +00:00
|
|
|
#ifndef FETCH_PACK_H
|
|
|
|
|
#define FETCH_PACK_H
|
|
|
|
|
|
2012-09-09 06:19:40 +00:00
|
|
|
#include "string-list.h"
|
2013-07-08 20:56:53 +00:00
|
|
|
#include "run-command.h"
|
2018-03-15 17:31:28 +00:00
|
|
|
#include "protocol.h"
|
2017-12-08 15:58:40 +00:00
|
|
|
#include "list-objects-filter-options.h"
|
2012-09-09 06:19:40 +00:00
|
|
|
|
2017-03-31 01:40:00 +00:00
|
|
|
struct oid_array;
|
2013-12-05 13:02:39 +00:00
|
|
|
|
2011-03-16 07:08:34 +00:00
|
|
|
struct fetch_pack_args {
|
2007-09-11 03:03:00 +00:00
|
|
|
const char *uploadpack;
|
|
|
|
|
int unpacklimit;
|
|
|
|
|
int depth;
|
2016-06-12 10:53:59 +00:00
|
|
|
const char *deepen_since;
|
2016-06-12 10:54:04 +00:00
|
|
|
const struct string_list *deepen_not;
|
2017-12-08 15:58:40 +00:00
|
|
|
struct list_objects_filter_options filter_options;
|
2018-04-23 22:46:24 +00:00
|
|
|
const struct string_list *server_options;
|
2018-07-02 22:39:44 +00:00
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* If not NULL, during packfile negotiation, fetch-pack will send "have"
|
|
|
|
|
* lines only with these tips and their ancestors.
|
|
|
|
|
*/
|
|
|
|
|
const struct oid_array *negotiation_tips;
|
|
|
|
|
|
fetch, upload-pack: --deepen=N extends shallow boundary by N commits
In git-fetch, --depth argument is always relative with the latest
remote refs. This makes it a bit difficult to cover this use case,
where the user wants to make the shallow history, say 3 levels
deeper. It would work if remote refs have not moved yet, but nobody
can guarantee that, especially when that use case is performed a
couple months after the last clone or "git fetch --depth". Also,
modifying shallow boundary using --depth does not work well with
clones created by --since or --not.
This patch fixes that. A new argument --deepen=<N> will add <N> more (*)
parent commits to the current history regardless of where remote refs
are.
Have/Want negotiation is still respected. So if remote refs move, the
server will send two chunks: one between "have" and "want" and another
to extend shallow history. In theory, the client could send no "want"s
in order to get the second chunk only. But the protocol does not allow
that. Either you send no want lines, which means ls-remote; or you
have to send at least one want line that carries deep-relative to the
server..
The main work was done by Dongcan Jiang. I fixed it up here and there.
And of course all the bugs belong to me.
(*) We could even support --deepen=<N> where <N> is negative. In that
case we can cut some history from the shallow clone. This operation
(and --depth=<shorter depth>) does not require interaction with remote
side (and more complicated to implement as a result).
Helped-by: Duy Nguyen <pclouds@gmail.com>
Helped-by: Eric Sunshine <sunshine@sunshineco.com>
Helped-by: Junio C Hamano <gitster@pobox.com>
Signed-off-by: Dongcan Jiang <dongcan.jiang@gmail.com>
Signed-off-by: Nguyễn Thái Ngọc Duy <pclouds@gmail.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2016-06-12 10:54:09 +00:00
|
|
|
unsigned deepen_relative:1;
|
2013-12-05 13:02:38 +00:00
|
|
|
unsigned quiet:1;
|
|
|
|
|
unsigned keep_pack:1;
|
|
|
|
|
unsigned lock_pack:1;
|
|
|
|
|
unsigned use_thin_pack:1;
|
|
|
|
|
unsigned fetch_all:1;
|
|
|
|
|
unsigned stdin_refs:1;
|
2014-01-17 20:21:14 +00:00
|
|
|
unsigned diag_url:1;
|
2013-12-05 13:02:38 +00:00
|
|
|
unsigned verbose:1;
|
|
|
|
|
unsigned no_progress:1;
|
|
|
|
|
unsigned include_tag:1;
|
|
|
|
|
unsigned stateless_rpc:1;
|
|
|
|
|
unsigned check_self_contained_and_connected:1;
|
|
|
|
|
unsigned self_contained_and_connected:1;
|
2013-12-05 13:02:39 +00:00
|
|
|
unsigned cloning:1;
|
2013-12-05 13:02:42 +00:00
|
|
|
unsigned update_shallow:1;
|
2016-06-12 10:53:56 +00:00
|
|
|
unsigned deepen:1;
|
introduce fetch-object: fetch one promisor object
Introduce fetch-object, providing the ability to fetch one object from a
promisor remote.
This uses fetch-pack. To do this, the transport mechanism has been
updated with 2 flags, "from-promisor" to indicate that the resulting
pack comes from a promisor remote (and thus should be annotated as such
by index-pack), and "no-dependents" to indicate that only the objects
themselves need to be fetched (but fetching additional objects is
nevertheless safe).
Whenever "no-dependents" is used, fetch-pack will refrain from using any
object flags, because it is most likely invoked as part of a dynamic
object fetch by another Git command (which may itself use object flags).
An alternative to this is to leave fetch-pack alone, and instead update
the allocation of flags so that fetch-pack's flags never overlap with
any others, but this will end up shrinking the number of flags available
to nearly every other Git command (that is, every Git command that
accesses objects), so the approach in this commit was used instead.
This will be tested in a subsequent commit.
Signed-off-by: Jonathan Tan <jonathantanmy@google.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2017-12-05 16:58:49 +00:00
|
|
|
unsigned from_promisor:1;
|
|
|
|
|
|
|
|
|
|
/*
|
fetch-pack: exclude blobs when lazy-fetching trees
A partial clone with missing trees can be obtained using "git clone
--filter=tree:none <repo>". In such a repository, when a tree needs to
be lazily fetched, any tree or blob it directly or indirectly references
is fetched as well, regardless of whether the original command required
those objects, or if the local repository already had some of them.
This is because the fetch protocol, which the lazy fetch uses, does not
allow clients to request that only the wanted objects be sent, which
would be the ideal solution. This patch implements a partial solution:
specify the "blob:none" filter, somewhat reducing the fetch payload.
This change has no effect when lazily fetching blobs (due to how filters
work). And if lazily fetching a commit (such repositories are difficult
to construct and is not a use case we support very well, but it is
possible), referenced commits and trees are still fetched - only the
blobs are not fetched.
The necessary code change is done in fetch_pack() instead of somewhere
closer to where the "filter" instruction is written to the wire so that
only one part of the code needs to be changed in order for users of all
protocol versions to benefit from this optimization.
Signed-off-by: Jonathan Tan <jonathantanmy@google.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2018-10-03 23:04:53 +00:00
|
|
|
* Attempt to fetch only the wanted objects, and not any objects
|
|
|
|
|
* referred to by them. Due to protocol limitations, extraneous
|
|
|
|
|
* objects may still be included. (When fetching non-blob
|
|
|
|
|
* objects, only blobs are excluded; when fetching a blob, the
|
|
|
|
|
* blob itself will still be sent. The client does not need to
|
|
|
|
|
* know whether a wanted object is a blob or not.)
|
|
|
|
|
*
|
introduce fetch-object: fetch one promisor object
Introduce fetch-object, providing the ability to fetch one object from a
promisor remote.
This uses fetch-pack. To do this, the transport mechanism has been
updated with 2 flags, "from-promisor" to indicate that the resulting
pack comes from a promisor remote (and thus should be annotated as such
by index-pack), and "no-dependents" to indicate that only the objects
themselves need to be fetched (but fetching additional objects is
nevertheless safe).
Whenever "no-dependents" is used, fetch-pack will refrain from using any
object flags, because it is most likely invoked as part of a dynamic
object fetch by another Git command (which may itself use object flags).
An alternative to this is to leave fetch-pack alone, and instead update
the allocation of flags so that fetch-pack's flags never overlap with
any others, but this will end up shrinking the number of flags available
to nearly every other Git command (that is, every Git command that
accesses objects), so the approach in this commit was used instead.
This will be tested in a subsequent commit.
Signed-off-by: Jonathan Tan <jonathantanmy@google.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2017-12-05 16:58:49 +00:00
|
|
|
* If 1, fetch_pack() will also not modify any object flags.
|
|
|
|
|
* This allows fetch_pack() to safely be called by any function,
|
|
|
|
|
* regardless of which object flags it uses (if any).
|
|
|
|
|
*/
|
|
|
|
|
unsigned no_dependents:1;
|
fetch-pack: write shallow, then check connectivity
When fetching, connectivity is checked after the shallow file is
updated. There are 2 issues with this: (1) the connectivity check is
only performed up to ancestors of existing refs (which is not thorough
enough if we were deepening an existing ref in the first place), and (2)
there is no rollback of the shallow file if the connectivity check
fails.
To solve (1), update the connectivity check to check the ancestry chain
completely in the case of a deepening fetch by refraining from passing
"--not --all" when invoking rev-list in connected.c.
To solve (2), have fetch_pack() perform its own connectivity check
before updating the shallow file. To support existing use cases in which
"git fetch-pack" is used to download objects without much regard as to
the connectivity of the resulting objects with respect to the existing
repository, the connectivity check is only done if necessary (that is,
the fetch is not a clone, and the fetch involves shallow/deepen
functionality). "git fetch" still performs its own connectivity check,
preserving correctness but sometimes performing redundant work. This
redundancy is mitigated by the fact that fetch_pack() reports if it has
performed a connectivity check itself, and if the transport supports
connect or stateless-connect, it will bubble up that report so that "git
fetch" knows not to perform the connectivity check in such a case.
This was noticed when a user tried to deepen an existing repository by
fetching with --no-shallow from a server that did not send all necessary
objects - the connectivity check as run by "git fetch" succeeded, but a
subsequent "git fsck" failed.
Signed-off-by: Jonathan Tan <jonathantanmy@google.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2018-07-02 22:08:43 +00:00
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* Because fetch_pack() overwrites the shallow file upon a
|
|
|
|
|
* successful deepening non-clone fetch, if this struct
|
|
|
|
|
* specifies such a fetch, fetch_pack() needs to perform a
|
|
|
|
|
* connectivity check before deciding if a fetch is successful
|
|
|
|
|
* (and overwriting the shallow file). fetch_pack() sets this
|
|
|
|
|
* field to 1 if such a connectivity check was performed.
|
|
|
|
|
*
|
|
|
|
|
* This is different from check_self_contained_and_connected
|
|
|
|
|
* in that the former allows existing objects in the
|
|
|
|
|
* repository to satisfy connectivity needs, whereas the
|
|
|
|
|
* latter doesn't.
|
|
|
|
|
*/
|
|
|
|
|
unsigned connectivity_checked:1;
|
2007-09-11 03:03:00 +00:00
|
|
|
};
|
|
|
|
|
|
2012-09-09 06:19:43 +00:00
|
|
|
/*
|
2013-01-29 22:02:15 +00:00
|
|
|
* sought represents remote references that should be updated from.
|
|
|
|
|
* On return, the names that were found on the remote will have been
|
|
|
|
|
* marked as such.
|
2012-09-09 06:19:43 +00:00
|
|
|
*/
|
2007-09-19 04:49:35 +00:00
|
|
|
struct ref *fetch_pack(struct fetch_pack_args *args,
|
2019-03-20 08:16:14 +00:00
|
|
|
int fd[],
|
2012-09-09 06:19:39 +00:00
|
|
|
const struct ref *ref,
|
2013-01-29 22:02:15 +00:00
|
|
|
struct ref **sought,
|
|
|
|
|
int nr_sought,
|
2017-03-31 01:40:00 +00:00
|
|
|
struct oid_array *shallow,
|
fetch-pack: support more than one pack lockfile
Whenever a fetch results in a packfile being downloaded, a .keep file is
generated, so that the packfile can be preserved (from, say, a running
"git repack") until refs are written referring to the contents of the
packfile.
In a subsequent patch, a successful fetch using protocol v2 may result
in more than one .keep file being generated. Therefore, teach
fetch_pack() and the transport mechanism to support multiple .keep
files.
Implementation notes:
- builtin/fetch-pack.c normally does not generate .keep files, and thus
is unaffected by this or future changes. However, it has an
undocumented "--lock-pack" feature, used by remote-curl.c when
implementing the "fetch" remote helper command. In keeping with the
remote helper protocol, only one "lock" line will ever be written;
the rest will result in warnings to stderr. However, in practice,
warnings will never be written because the remote-curl.c "fetch" is
only used for protocol v0/v1 (which will not generate multiple .keep
files). (Protocol v2 uses the "stateless-connect" command, not the
"fetch" command.)
- connected.c has an optimization in that connectivity checks on a ref
need not be done if the target object is in a pack known to be
self-contained and connected. If there are multiple packfiles, this
optimization can no longer be done.
Signed-off-by: Jonathan Tan <jonathantanmy@google.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2020-06-10 20:57:22 +00:00
|
|
|
struct string_list *pack_lockfiles,
|
2018-03-15 17:31:28 +00:00
|
|
|
enum protocol_version version);
|
2007-09-11 03:03:00 +00:00
|
|
|
|
2017-02-22 16:01:22 +00:00
|
|
|
/*
|
|
|
|
|
* Print an appropriate error message for each sought ref that wasn't
|
|
|
|
|
* matched. Return 0 if all sought refs were matched, otherwise 1.
|
|
|
|
|
*/
|
|
|
|
|
int report_unmatched_refs(struct ref **sought, int nr_sought);
|
|
|
|
|
|
2007-09-11 03:03:00 +00:00
|
|
|
#endif
|