2007-01-06 10:16:17 +00:00
|
|
|
#include "cache.h"
|
|
|
|
|
#include "refs.h"
|
|
|
|
|
#include "tag.h"
|
|
|
|
|
#include "commit.h"
|
|
|
|
|
#include "blob.h"
|
|
|
|
|
#include "diff.h"
|
|
|
|
|
#include "revision.h"
|
|
|
|
|
#include "reachable.h"
|
|
|
|
|
#include "cache-tree.h"
|
2011-11-05 12:00:08 +00:00
|
|
|
#include "progress.h"
|
2014-10-15 22:37:28 +00:00
|
|
|
#include "list-objects.h"
|
2017-08-18 22:20:38 +00:00
|
|
|
#include "packfile.h"
|
2017-08-23 12:36:59 +00:00
|
|
|
#include "worktree.h"
|
2007-01-06 10:16:17 +00:00
|
|
|
|
2011-11-08 05:37:00 +00:00
|
|
|
struct connectivity_progress {
|
|
|
|
|
struct progress *progress;
|
|
|
|
|
unsigned long count;
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
static void update_progress(struct connectivity_progress *cp)
|
|
|
|
|
{
|
|
|
|
|
cp->count++;
|
|
|
|
|
if ((cp->count & 1023) == 0)
|
|
|
|
|
display_progress(cp->progress, cp->count);
|
|
|
|
|
}
|
|
|
|
|
|
2015-05-25 18:39:00 +00:00
|
|
|
static int add_one_ref(const char *path, const struct object_id *oid,
|
|
|
|
|
int flag, void *cb_data)
|
2007-01-06 10:16:17 +00:00
|
|
|
{
|
|
|
|
|
struct rev_info *revs = (struct rev_info *)cb_data;
|
2015-09-28 14:01:25 +00:00
|
|
|
struct object *object;
|
2007-01-06 10:16:17 +00:00
|
|
|
|
2015-09-28 14:01:25 +00:00
|
|
|
if ((flag & REF_ISSYMREF) && (flag & REF_ISBROKEN)) {
|
|
|
|
|
warning("symbolic ref is dangling: %s", path);
|
|
|
|
|
return 0;
|
|
|
|
|
}
|
|
|
|
|
|
object: convert parse_object* to take struct object_id
Make parse_object, parse_object_or_die, and parse_object_buffer take a
pointer to struct object_id. Remove the temporary variables inserted
earlier, since they are no longer necessary. Transform all of the
callers using the following semantic patch:
@@
expression E1;
@@
- parse_object(E1.hash)
+ parse_object(&E1)
@@
expression E1;
@@
- parse_object(E1->hash)
+ parse_object(E1)
@@
expression E1, E2;
@@
- parse_object_or_die(E1.hash, E2)
+ parse_object_or_die(&E1, E2)
@@
expression E1, E2;
@@
- parse_object_or_die(E1->hash, E2)
+ parse_object_or_die(E1, E2)
@@
expression E1, E2, E3, E4, E5;
@@
- parse_object_buffer(E1.hash, E2, E3, E4, E5)
+ parse_object_buffer(&E1, E2, E3, E4, E5)
@@
expression E1, E2, E3, E4, E5;
@@
- parse_object_buffer(E1->hash, E2, E3, E4, E5)
+ parse_object_buffer(E1, E2, E3, E4, E5)
Signed-off-by: brian m. carlson <sandals@crustytoothpaste.net>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2017-05-06 22:10:38 +00:00
|
|
|
object = parse_object_or_die(oid, path);
|
2007-01-06 10:16:17 +00:00
|
|
|
add_pending_object(revs, object, "");
|
|
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
|
}
|
|
|
|
|
|
2014-10-15 22:37:28 +00:00
|
|
|
/*
|
|
|
|
|
* The traversal will have already marked us as SEEN, so we
|
|
|
|
|
* only need to handle any progress reporting here.
|
|
|
|
|
*/
|
list-objects: pass full pathname to callbacks
When we find a blob at "a/b/c", we currently pass this to
our show_object_fn callbacks as two components: "a/b/" and
"c". Callbacks which want the full value then call
path_name(), which concatenates the two. But this is an
inefficient interface; the path is a strbuf, and we could
simply append "c" to it temporarily, then roll back the
length, without creating a new copy.
So we could improve this by teaching the callsites of
path_name() this trick (and there are only 3). But we can
also notice that no callback actually cares about the
broken-down representation, and simply pass each callback
the full path "a/b/c" as a string. The callback code becomes
even simpler, then, as we do not have to worry about freeing
an allocated buffer, nor rolling back our modification to
the strbuf.
This is theoretically less efficient, as some callbacks
would not bother to format the final path component. But in
practice this is not measurable. Since we use the same
strbuf over and over, our work to grow it is amortized, and
we really only pay to memcpy a few bytes.
Signed-off-by: Jeff King <peff@peff.net>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2016-02-11 22:28:36 +00:00
|
|
|
static void mark_object(struct object *obj, const char *name, void *data)
|
2014-10-15 22:37:28 +00:00
|
|
|
{
|
|
|
|
|
update_progress(data);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static void mark_commit(struct commit *c, void *data)
|
|
|
|
|
{
|
list-objects: pass full pathname to callbacks
When we find a blob at "a/b/c", we currently pass this to
our show_object_fn callbacks as two components: "a/b/" and
"c". Callbacks which want the full value then call
path_name(), which concatenates the two. But this is an
inefficient interface; the path is a strbuf, and we could
simply append "c" to it temporarily, then roll back the
length, without creating a new copy.
So we could improve this by teaching the callsites of
path_name() this trick (and there are only 3). But we can
also notice that no callback actually cares about the
broken-down representation, and simply pass each callback
the full path "a/b/c" as a string. The callback code becomes
even simpler, then, as we do not have to worry about freeing
an allocated buffer, nor rolling back our modification to
the strbuf.
This is theoretically less efficient, as some callbacks
would not bother to format the final path component. But in
practice this is not measurable. Since we use the same
strbuf over and over, our work to grow it is amortized, and
we really only pay to memcpy a few bytes.
Signed-off-by: Jeff King <peff@peff.net>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2016-02-11 22:28:36 +00:00
|
|
|
mark_object(&c->object, NULL, data);
|
2014-10-15 22:37:28 +00:00
|
|
|
}
|
|
|
|
|
|
prune: keep objects reachable from recent objects
Our current strategy with prune is that an object falls into
one of three categories:
1. Reachable (from ref tips, reflogs, index, etc).
2. Not reachable, but recent (based on the --expire time).
3. Not reachable and not recent.
We keep objects from (1) and (2), but prune objects in (3).
The point of (2) is that these objects may be part of an
in-progress operation that has not yet updated any refs.
However, it is not always the case that objects for an
in-progress operation will have a recent mtime. For example,
the object database may have an old copy of a blob (from an
abandoned operation, a branch that was deleted, etc). If we
create a new tree that points to it, a simultaneous prune
will leave our tree, but delete the blob. Referencing that
tree with a commit will then work (we check that the tree is
in the object database, but not that all of its referred
objects are), as will mentioning the commit in a ref. But
the resulting repo is corrupt; we are missing the blob
reachable from a ref.
One way to solve this is to be more thorough when
referencing a sha1: make sure that not only do we have that
sha1, but that we have objects it refers to, and so forth
recursively. The problem is that this is very expensive.
Creating a parent link would require traversing the entire
object graph!
Instead, this patch pushes the extra work onto prune, which
runs less frequently (and has to look at the whole object
graph anyway). It creates a new category of objects: objects
which are not recent, but which are reachable from a recent
object. We do not prune these objects, just like the
reachable and recent ones.
This lets us avoid the recursive check above, because if we
have an object, even if it is unreachable, we should have
its referent. We can make a simple inductive argument that
with this patch, this property holds (that there are no
objects with missing referents in the repository):
0. When we have no objects, we have nothing to refer or be
referred to, so the property holds.
1. If we add objects to the repository, their direct
referents must generally exist (e.g., if you create a
tree, the blobs it references must exist; if you create
a commit to point at the tree, the tree must exist).
This is already the case before this patch. And it is
not 100% foolproof (you can make bogus objects using
`git hash-object`, for example), but it should be the
case for normal usage.
Therefore for any sequence of object additions, the
property will continue to hold.
2. If we remove objects from the repository, then we will
not remove a child object (like a blob) if an object
that refers to it is being kept. That is the part
implemented by this patch.
Note, however, that our reachability check and the
actual pruning are not atomic. So it _is_ still
possible to violate the property (e.g., an object
becomes referenced just as we are deleting it). This
patch is shooting for eliminating problems where the
mtimes of dependent objects differ by hours or days,
and one is dropped without the other. It does nothing
to help with short races.
Naively, the simplest way to implement this would be to add
all recent objects as tips to the reachability traversal.
However, this does not perform well. In a recently-packed
repository, all reachable objects will also be recent, and
therefore we have to look at each object twice. This patch
instead performs the reachability traversal, then follows up
with a second traversal for recent objects, skipping any
that have already been marked.
Signed-off-by: Jeff King <peff@peff.net>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2014-10-15 22:41:35 +00:00
|
|
|
struct recent_data {
|
|
|
|
|
struct rev_info *revs;
|
2017-04-26 19:29:31 +00:00
|
|
|
timestamp_t timestamp;
|
prune: keep objects reachable from recent objects
Our current strategy with prune is that an object falls into
one of three categories:
1. Reachable (from ref tips, reflogs, index, etc).
2. Not reachable, but recent (based on the --expire time).
3. Not reachable and not recent.
We keep objects from (1) and (2), but prune objects in (3).
The point of (2) is that these objects may be part of an
in-progress operation that has not yet updated any refs.
However, it is not always the case that objects for an
in-progress operation will have a recent mtime. For example,
the object database may have an old copy of a blob (from an
abandoned operation, a branch that was deleted, etc). If we
create a new tree that points to it, a simultaneous prune
will leave our tree, but delete the blob. Referencing that
tree with a commit will then work (we check that the tree is
in the object database, but not that all of its referred
objects are), as will mentioning the commit in a ref. But
the resulting repo is corrupt; we are missing the blob
reachable from a ref.
One way to solve this is to be more thorough when
referencing a sha1: make sure that not only do we have that
sha1, but that we have objects it refers to, and so forth
recursively. The problem is that this is very expensive.
Creating a parent link would require traversing the entire
object graph!
Instead, this patch pushes the extra work onto prune, which
runs less frequently (and has to look at the whole object
graph anyway). It creates a new category of objects: objects
which are not recent, but which are reachable from a recent
object. We do not prune these objects, just like the
reachable and recent ones.
This lets us avoid the recursive check above, because if we
have an object, even if it is unreachable, we should have
its referent. We can make a simple inductive argument that
with this patch, this property holds (that there are no
objects with missing referents in the repository):
0. When we have no objects, we have nothing to refer or be
referred to, so the property holds.
1. If we add objects to the repository, their direct
referents must generally exist (e.g., if you create a
tree, the blobs it references must exist; if you create
a commit to point at the tree, the tree must exist).
This is already the case before this patch. And it is
not 100% foolproof (you can make bogus objects using
`git hash-object`, for example), but it should be the
case for normal usage.
Therefore for any sequence of object additions, the
property will continue to hold.
2. If we remove objects from the repository, then we will
not remove a child object (like a blob) if an object
that refers to it is being kept. That is the part
implemented by this patch.
Note, however, that our reachability check and the
actual pruning are not atomic. So it _is_ still
possible to violate the property (e.g., an object
becomes referenced just as we are deleting it). This
patch is shooting for eliminating problems where the
mtimes of dependent objects differ by hours or days,
and one is dropped without the other. It does nothing
to help with short races.
Naively, the simplest way to implement this would be to add
all recent objects as tips to the reachability traversal.
However, this does not perform well. In a recently-packed
repository, all reachable objects will also be recent, and
therefore we have to look at each object twice. This patch
instead performs the reachability traversal, then follows up
with a second traversal for recent objects, skipping any
that have already been marked.
Signed-off-by: Jeff King <peff@peff.net>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2014-10-15 22:41:35 +00:00
|
|
|
};
|
|
|
|
|
|
2017-02-21 23:47:35 +00:00
|
|
|
static void add_recent_object(const struct object_id *oid,
|
2017-04-26 19:29:31 +00:00
|
|
|
timestamp_t mtime,
|
prune: keep objects reachable from recent objects
Our current strategy with prune is that an object falls into
one of three categories:
1. Reachable (from ref tips, reflogs, index, etc).
2. Not reachable, but recent (based on the --expire time).
3. Not reachable and not recent.
We keep objects from (1) and (2), but prune objects in (3).
The point of (2) is that these objects may be part of an
in-progress operation that has not yet updated any refs.
However, it is not always the case that objects for an
in-progress operation will have a recent mtime. For example,
the object database may have an old copy of a blob (from an
abandoned operation, a branch that was deleted, etc). If we
create a new tree that points to it, a simultaneous prune
will leave our tree, but delete the blob. Referencing that
tree with a commit will then work (we check that the tree is
in the object database, but not that all of its referred
objects are), as will mentioning the commit in a ref. But
the resulting repo is corrupt; we are missing the blob
reachable from a ref.
One way to solve this is to be more thorough when
referencing a sha1: make sure that not only do we have that
sha1, but that we have objects it refers to, and so forth
recursively. The problem is that this is very expensive.
Creating a parent link would require traversing the entire
object graph!
Instead, this patch pushes the extra work onto prune, which
runs less frequently (and has to look at the whole object
graph anyway). It creates a new category of objects: objects
which are not recent, but which are reachable from a recent
object. We do not prune these objects, just like the
reachable and recent ones.
This lets us avoid the recursive check above, because if we
have an object, even if it is unreachable, we should have
its referent. We can make a simple inductive argument that
with this patch, this property holds (that there are no
objects with missing referents in the repository):
0. When we have no objects, we have nothing to refer or be
referred to, so the property holds.
1. If we add objects to the repository, their direct
referents must generally exist (e.g., if you create a
tree, the blobs it references must exist; if you create
a commit to point at the tree, the tree must exist).
This is already the case before this patch. And it is
not 100% foolproof (you can make bogus objects using
`git hash-object`, for example), but it should be the
case for normal usage.
Therefore for any sequence of object additions, the
property will continue to hold.
2. If we remove objects from the repository, then we will
not remove a child object (like a blob) if an object
that refers to it is being kept. That is the part
implemented by this patch.
Note, however, that our reachability check and the
actual pruning are not atomic. So it _is_ still
possible to violate the property (e.g., an object
becomes referenced just as we are deleting it). This
patch is shooting for eliminating problems where the
mtimes of dependent objects differ by hours or days,
and one is dropped without the other. It does nothing
to help with short races.
Naively, the simplest way to implement this would be to add
all recent objects as tips to the reachability traversal.
However, this does not perform well. In a recently-packed
repository, all reachable objects will also be recent, and
therefore we have to look at each object twice. This patch
instead performs the reachability traversal, then follows up
with a second traversal for recent objects, skipping any
that have already been marked.
Signed-off-by: Jeff King <peff@peff.net>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2014-10-15 22:41:35 +00:00
|
|
|
struct recent_data *data)
|
|
|
|
|
{
|
|
|
|
|
struct object *obj;
|
|
|
|
|
enum object_type type;
|
|
|
|
|
|
|
|
|
|
if (mtime <= data->timestamp)
|
|
|
|
|
return;
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* We do not want to call parse_object here, because
|
|
|
|
|
* inflating blobs and trees could be very expensive.
|
|
|
|
|
* However, we do need to know the correct type for
|
|
|
|
|
* later processing, and the revision machinery expects
|
|
|
|
|
* commits and tags to have been parsed.
|
|
|
|
|
*/
|
2017-02-21 23:47:35 +00:00
|
|
|
type = sha1_object_info(oid->hash, NULL);
|
prune: keep objects reachable from recent objects
Our current strategy with prune is that an object falls into
one of three categories:
1. Reachable (from ref tips, reflogs, index, etc).
2. Not reachable, but recent (based on the --expire time).
3. Not reachable and not recent.
We keep objects from (1) and (2), but prune objects in (3).
The point of (2) is that these objects may be part of an
in-progress operation that has not yet updated any refs.
However, it is not always the case that objects for an
in-progress operation will have a recent mtime. For example,
the object database may have an old copy of a blob (from an
abandoned operation, a branch that was deleted, etc). If we
create a new tree that points to it, a simultaneous prune
will leave our tree, but delete the blob. Referencing that
tree with a commit will then work (we check that the tree is
in the object database, but not that all of its referred
objects are), as will mentioning the commit in a ref. But
the resulting repo is corrupt; we are missing the blob
reachable from a ref.
One way to solve this is to be more thorough when
referencing a sha1: make sure that not only do we have that
sha1, but that we have objects it refers to, and so forth
recursively. The problem is that this is very expensive.
Creating a parent link would require traversing the entire
object graph!
Instead, this patch pushes the extra work onto prune, which
runs less frequently (and has to look at the whole object
graph anyway). It creates a new category of objects: objects
which are not recent, but which are reachable from a recent
object. We do not prune these objects, just like the
reachable and recent ones.
This lets us avoid the recursive check above, because if we
have an object, even if it is unreachable, we should have
its referent. We can make a simple inductive argument that
with this patch, this property holds (that there are no
objects with missing referents in the repository):
0. When we have no objects, we have nothing to refer or be
referred to, so the property holds.
1. If we add objects to the repository, their direct
referents must generally exist (e.g., if you create a
tree, the blobs it references must exist; if you create
a commit to point at the tree, the tree must exist).
This is already the case before this patch. And it is
not 100% foolproof (you can make bogus objects using
`git hash-object`, for example), but it should be the
case for normal usage.
Therefore for any sequence of object additions, the
property will continue to hold.
2. If we remove objects from the repository, then we will
not remove a child object (like a blob) if an object
that refers to it is being kept. That is the part
implemented by this patch.
Note, however, that our reachability check and the
actual pruning are not atomic. So it _is_ still
possible to violate the property (e.g., an object
becomes referenced just as we are deleting it). This
patch is shooting for eliminating problems where the
mtimes of dependent objects differ by hours or days,
and one is dropped without the other. It does nothing
to help with short races.
Naively, the simplest way to implement this would be to add
all recent objects as tips to the reachability traversal.
However, this does not perform well. In a recently-packed
repository, all reachable objects will also be recent, and
therefore we have to look at each object twice. This patch
instead performs the reachability traversal, then follows up
with a second traversal for recent objects, skipping any
that have already been marked.
Signed-off-by: Jeff King <peff@peff.net>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2014-10-15 22:41:35 +00:00
|
|
|
if (type < 0)
|
2017-02-21 23:47:35 +00:00
|
|
|
die("unable to get object info for %s", oid_to_hex(oid));
|
prune: keep objects reachable from recent objects
Our current strategy with prune is that an object falls into
one of three categories:
1. Reachable (from ref tips, reflogs, index, etc).
2. Not reachable, but recent (based on the --expire time).
3. Not reachable and not recent.
We keep objects from (1) and (2), but prune objects in (3).
The point of (2) is that these objects may be part of an
in-progress operation that has not yet updated any refs.
However, it is not always the case that objects for an
in-progress operation will have a recent mtime. For example,
the object database may have an old copy of a blob (from an
abandoned operation, a branch that was deleted, etc). If we
create a new tree that points to it, a simultaneous prune
will leave our tree, but delete the blob. Referencing that
tree with a commit will then work (we check that the tree is
in the object database, but not that all of its referred
objects are), as will mentioning the commit in a ref. But
the resulting repo is corrupt; we are missing the blob
reachable from a ref.
One way to solve this is to be more thorough when
referencing a sha1: make sure that not only do we have that
sha1, but that we have objects it refers to, and so forth
recursively. The problem is that this is very expensive.
Creating a parent link would require traversing the entire
object graph!
Instead, this patch pushes the extra work onto prune, which
runs less frequently (and has to look at the whole object
graph anyway). It creates a new category of objects: objects
which are not recent, but which are reachable from a recent
object. We do not prune these objects, just like the
reachable and recent ones.
This lets us avoid the recursive check above, because if we
have an object, even if it is unreachable, we should have
its referent. We can make a simple inductive argument that
with this patch, this property holds (that there are no
objects with missing referents in the repository):
0. When we have no objects, we have nothing to refer or be
referred to, so the property holds.
1. If we add objects to the repository, their direct
referents must generally exist (e.g., if you create a
tree, the blobs it references must exist; if you create
a commit to point at the tree, the tree must exist).
This is already the case before this patch. And it is
not 100% foolproof (you can make bogus objects using
`git hash-object`, for example), but it should be the
case for normal usage.
Therefore for any sequence of object additions, the
property will continue to hold.
2. If we remove objects from the repository, then we will
not remove a child object (like a blob) if an object
that refers to it is being kept. That is the part
implemented by this patch.
Note, however, that our reachability check and the
actual pruning are not atomic. So it _is_ still
possible to violate the property (e.g., an object
becomes referenced just as we are deleting it). This
patch is shooting for eliminating problems where the
mtimes of dependent objects differ by hours or days,
and one is dropped without the other. It does nothing
to help with short races.
Naively, the simplest way to implement this would be to add
all recent objects as tips to the reachability traversal.
However, this does not perform well. In a recently-packed
repository, all reachable objects will also be recent, and
therefore we have to look at each object twice. This patch
instead performs the reachability traversal, then follows up
with a second traversal for recent objects, skipping any
that have already been marked.
Signed-off-by: Jeff King <peff@peff.net>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2014-10-15 22:41:35 +00:00
|
|
|
|
|
|
|
|
switch (type) {
|
|
|
|
|
case OBJ_TAG:
|
|
|
|
|
case OBJ_COMMIT:
|
object: convert parse_object* to take struct object_id
Make parse_object, parse_object_or_die, and parse_object_buffer take a
pointer to struct object_id. Remove the temporary variables inserted
earlier, since they are no longer necessary. Transform all of the
callers using the following semantic patch:
@@
expression E1;
@@
- parse_object(E1.hash)
+ parse_object(&E1)
@@
expression E1;
@@
- parse_object(E1->hash)
+ parse_object(E1)
@@
expression E1, E2;
@@
- parse_object_or_die(E1.hash, E2)
+ parse_object_or_die(&E1, E2)
@@
expression E1, E2;
@@
- parse_object_or_die(E1->hash, E2)
+ parse_object_or_die(E1, E2)
@@
expression E1, E2, E3, E4, E5;
@@
- parse_object_buffer(E1.hash, E2, E3, E4, E5)
+ parse_object_buffer(&E1, E2, E3, E4, E5)
@@
expression E1, E2, E3, E4, E5;
@@
- parse_object_buffer(E1->hash, E2, E3, E4, E5)
+ parse_object_buffer(E1, E2, E3, E4, E5)
Signed-off-by: brian m. carlson <sandals@crustytoothpaste.net>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2017-05-06 22:10:38 +00:00
|
|
|
obj = parse_object_or_die(oid, NULL);
|
prune: keep objects reachable from recent objects
Our current strategy with prune is that an object falls into
one of three categories:
1. Reachable (from ref tips, reflogs, index, etc).
2. Not reachable, but recent (based on the --expire time).
3. Not reachable and not recent.
We keep objects from (1) and (2), but prune objects in (3).
The point of (2) is that these objects may be part of an
in-progress operation that has not yet updated any refs.
However, it is not always the case that objects for an
in-progress operation will have a recent mtime. For example,
the object database may have an old copy of a blob (from an
abandoned operation, a branch that was deleted, etc). If we
create a new tree that points to it, a simultaneous prune
will leave our tree, but delete the blob. Referencing that
tree with a commit will then work (we check that the tree is
in the object database, but not that all of its referred
objects are), as will mentioning the commit in a ref. But
the resulting repo is corrupt; we are missing the blob
reachable from a ref.
One way to solve this is to be more thorough when
referencing a sha1: make sure that not only do we have that
sha1, but that we have objects it refers to, and so forth
recursively. The problem is that this is very expensive.
Creating a parent link would require traversing the entire
object graph!
Instead, this patch pushes the extra work onto prune, which
runs less frequently (and has to look at the whole object
graph anyway). It creates a new category of objects: objects
which are not recent, but which are reachable from a recent
object. We do not prune these objects, just like the
reachable and recent ones.
This lets us avoid the recursive check above, because if we
have an object, even if it is unreachable, we should have
its referent. We can make a simple inductive argument that
with this patch, this property holds (that there are no
objects with missing referents in the repository):
0. When we have no objects, we have nothing to refer or be
referred to, so the property holds.
1. If we add objects to the repository, their direct
referents must generally exist (e.g., if you create a
tree, the blobs it references must exist; if you create
a commit to point at the tree, the tree must exist).
This is already the case before this patch. And it is
not 100% foolproof (you can make bogus objects using
`git hash-object`, for example), but it should be the
case for normal usage.
Therefore for any sequence of object additions, the
property will continue to hold.
2. If we remove objects from the repository, then we will
not remove a child object (like a blob) if an object
that refers to it is being kept. That is the part
implemented by this patch.
Note, however, that our reachability check and the
actual pruning are not atomic. So it _is_ still
possible to violate the property (e.g., an object
becomes referenced just as we are deleting it). This
patch is shooting for eliminating problems where the
mtimes of dependent objects differ by hours or days,
and one is dropped without the other. It does nothing
to help with short races.
Naively, the simplest way to implement this would be to add
all recent objects as tips to the reachability traversal.
However, this does not perform well. In a recently-packed
repository, all reachable objects will also be recent, and
therefore we have to look at each object twice. This patch
instead performs the reachability traversal, then follows up
with a second traversal for recent objects, skipping any
that have already been marked.
Signed-off-by: Jeff King <peff@peff.net>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2014-10-15 22:41:35 +00:00
|
|
|
break;
|
|
|
|
|
case OBJ_TREE:
|
2017-05-06 22:10:17 +00:00
|
|
|
obj = (struct object *)lookup_tree(oid);
|
prune: keep objects reachable from recent objects
Our current strategy with prune is that an object falls into
one of three categories:
1. Reachable (from ref tips, reflogs, index, etc).
2. Not reachable, but recent (based on the --expire time).
3. Not reachable and not recent.
We keep objects from (1) and (2), but prune objects in (3).
The point of (2) is that these objects may be part of an
in-progress operation that has not yet updated any refs.
However, it is not always the case that objects for an
in-progress operation will have a recent mtime. For example,
the object database may have an old copy of a blob (from an
abandoned operation, a branch that was deleted, etc). If we
create a new tree that points to it, a simultaneous prune
will leave our tree, but delete the blob. Referencing that
tree with a commit will then work (we check that the tree is
in the object database, but not that all of its referred
objects are), as will mentioning the commit in a ref. But
the resulting repo is corrupt; we are missing the blob
reachable from a ref.
One way to solve this is to be more thorough when
referencing a sha1: make sure that not only do we have that
sha1, but that we have objects it refers to, and so forth
recursively. The problem is that this is very expensive.
Creating a parent link would require traversing the entire
object graph!
Instead, this patch pushes the extra work onto prune, which
runs less frequently (and has to look at the whole object
graph anyway). It creates a new category of objects: objects
which are not recent, but which are reachable from a recent
object. We do not prune these objects, just like the
reachable and recent ones.
This lets us avoid the recursive check above, because if we
have an object, even if it is unreachable, we should have
its referent. We can make a simple inductive argument that
with this patch, this property holds (that there are no
objects with missing referents in the repository):
0. When we have no objects, we have nothing to refer or be
referred to, so the property holds.
1. If we add objects to the repository, their direct
referents must generally exist (e.g., if you create a
tree, the blobs it references must exist; if you create
a commit to point at the tree, the tree must exist).
This is already the case before this patch. And it is
not 100% foolproof (you can make bogus objects using
`git hash-object`, for example), but it should be the
case for normal usage.
Therefore for any sequence of object additions, the
property will continue to hold.
2. If we remove objects from the repository, then we will
not remove a child object (like a blob) if an object
that refers to it is being kept. That is the part
implemented by this patch.
Note, however, that our reachability check and the
actual pruning are not atomic. So it _is_ still
possible to violate the property (e.g., an object
becomes referenced just as we are deleting it). This
patch is shooting for eliminating problems where the
mtimes of dependent objects differ by hours or days,
and one is dropped without the other. It does nothing
to help with short races.
Naively, the simplest way to implement this would be to add
all recent objects as tips to the reachability traversal.
However, this does not perform well. In a recently-packed
repository, all reachable objects will also be recent, and
therefore we have to look at each object twice. This patch
instead performs the reachability traversal, then follows up
with a second traversal for recent objects, skipping any
that have already been marked.
Signed-off-by: Jeff King <peff@peff.net>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2014-10-15 22:41:35 +00:00
|
|
|
break;
|
|
|
|
|
case OBJ_BLOB:
|
2017-05-06 22:10:14 +00:00
|
|
|
obj = (struct object *)lookup_blob(oid);
|
prune: keep objects reachable from recent objects
Our current strategy with prune is that an object falls into
one of three categories:
1. Reachable (from ref tips, reflogs, index, etc).
2. Not reachable, but recent (based on the --expire time).
3. Not reachable and not recent.
We keep objects from (1) and (2), but prune objects in (3).
The point of (2) is that these objects may be part of an
in-progress operation that has not yet updated any refs.
However, it is not always the case that objects for an
in-progress operation will have a recent mtime. For example,
the object database may have an old copy of a blob (from an
abandoned operation, a branch that was deleted, etc). If we
create a new tree that points to it, a simultaneous prune
will leave our tree, but delete the blob. Referencing that
tree with a commit will then work (we check that the tree is
in the object database, but not that all of its referred
objects are), as will mentioning the commit in a ref. But
the resulting repo is corrupt; we are missing the blob
reachable from a ref.
One way to solve this is to be more thorough when
referencing a sha1: make sure that not only do we have that
sha1, but that we have objects it refers to, and so forth
recursively. The problem is that this is very expensive.
Creating a parent link would require traversing the entire
object graph!
Instead, this patch pushes the extra work onto prune, which
runs less frequently (and has to look at the whole object
graph anyway). It creates a new category of objects: objects
which are not recent, but which are reachable from a recent
object. We do not prune these objects, just like the
reachable and recent ones.
This lets us avoid the recursive check above, because if we
have an object, even if it is unreachable, we should have
its referent. We can make a simple inductive argument that
with this patch, this property holds (that there are no
objects with missing referents in the repository):
0. When we have no objects, we have nothing to refer or be
referred to, so the property holds.
1. If we add objects to the repository, their direct
referents must generally exist (e.g., if you create a
tree, the blobs it references must exist; if you create
a commit to point at the tree, the tree must exist).
This is already the case before this patch. And it is
not 100% foolproof (you can make bogus objects using
`git hash-object`, for example), but it should be the
case for normal usage.
Therefore for any sequence of object additions, the
property will continue to hold.
2. If we remove objects from the repository, then we will
not remove a child object (like a blob) if an object
that refers to it is being kept. That is the part
implemented by this patch.
Note, however, that our reachability check and the
actual pruning are not atomic. So it _is_ still
possible to violate the property (e.g., an object
becomes referenced just as we are deleting it). This
patch is shooting for eliminating problems where the
mtimes of dependent objects differ by hours or days,
and one is dropped without the other. It does nothing
to help with short races.
Naively, the simplest way to implement this would be to add
all recent objects as tips to the reachability traversal.
However, this does not perform well. In a recently-packed
repository, all reachable objects will also be recent, and
therefore we have to look at each object twice. This patch
instead performs the reachability traversal, then follows up
with a second traversal for recent objects, skipping any
that have already been marked.
Signed-off-by: Jeff King <peff@peff.net>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2014-10-15 22:41:35 +00:00
|
|
|
break;
|
|
|
|
|
default:
|
|
|
|
|
die("unknown object type for %s: %s",
|
2018-02-14 18:59:24 +00:00
|
|
|
oid_to_hex(oid), type_name(type));
|
prune: keep objects reachable from recent objects
Our current strategy with prune is that an object falls into
one of three categories:
1. Reachable (from ref tips, reflogs, index, etc).
2. Not reachable, but recent (based on the --expire time).
3. Not reachable and not recent.
We keep objects from (1) and (2), but prune objects in (3).
The point of (2) is that these objects may be part of an
in-progress operation that has not yet updated any refs.
However, it is not always the case that objects for an
in-progress operation will have a recent mtime. For example,
the object database may have an old copy of a blob (from an
abandoned operation, a branch that was deleted, etc). If we
create a new tree that points to it, a simultaneous prune
will leave our tree, but delete the blob. Referencing that
tree with a commit will then work (we check that the tree is
in the object database, but not that all of its referred
objects are), as will mentioning the commit in a ref. But
the resulting repo is corrupt; we are missing the blob
reachable from a ref.
One way to solve this is to be more thorough when
referencing a sha1: make sure that not only do we have that
sha1, but that we have objects it refers to, and so forth
recursively. The problem is that this is very expensive.
Creating a parent link would require traversing the entire
object graph!
Instead, this patch pushes the extra work onto prune, which
runs less frequently (and has to look at the whole object
graph anyway). It creates a new category of objects: objects
which are not recent, but which are reachable from a recent
object. We do not prune these objects, just like the
reachable and recent ones.
This lets us avoid the recursive check above, because if we
have an object, even if it is unreachable, we should have
its referent. We can make a simple inductive argument that
with this patch, this property holds (that there are no
objects with missing referents in the repository):
0. When we have no objects, we have nothing to refer or be
referred to, so the property holds.
1. If we add objects to the repository, their direct
referents must generally exist (e.g., if you create a
tree, the blobs it references must exist; if you create
a commit to point at the tree, the tree must exist).
This is already the case before this patch. And it is
not 100% foolproof (you can make bogus objects using
`git hash-object`, for example), but it should be the
case for normal usage.
Therefore for any sequence of object additions, the
property will continue to hold.
2. If we remove objects from the repository, then we will
not remove a child object (like a blob) if an object
that refers to it is being kept. That is the part
implemented by this patch.
Note, however, that our reachability check and the
actual pruning are not atomic. So it _is_ still
possible to violate the property (e.g., an object
becomes referenced just as we are deleting it). This
patch is shooting for eliminating problems where the
mtimes of dependent objects differ by hours or days,
and one is dropped without the other. It does nothing
to help with short races.
Naively, the simplest way to implement this would be to add
all recent objects as tips to the reachability traversal.
However, this does not perform well. In a recently-packed
repository, all reachable objects will also be recent, and
therefore we have to look at each object twice. This patch
instead performs the reachability traversal, then follows up
with a second traversal for recent objects, skipping any
that have already been marked.
Signed-off-by: Jeff King <peff@peff.net>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2014-10-15 22:41:35 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (!obj)
|
2017-02-21 23:47:35 +00:00
|
|
|
die("unable to lookup %s", oid_to_hex(oid));
|
prune: keep objects reachable from recent objects
Our current strategy with prune is that an object falls into
one of three categories:
1. Reachable (from ref tips, reflogs, index, etc).
2. Not reachable, but recent (based on the --expire time).
3. Not reachable and not recent.
We keep objects from (1) and (2), but prune objects in (3).
The point of (2) is that these objects may be part of an
in-progress operation that has not yet updated any refs.
However, it is not always the case that objects for an
in-progress operation will have a recent mtime. For example,
the object database may have an old copy of a blob (from an
abandoned operation, a branch that was deleted, etc). If we
create a new tree that points to it, a simultaneous prune
will leave our tree, but delete the blob. Referencing that
tree with a commit will then work (we check that the tree is
in the object database, but not that all of its referred
objects are), as will mentioning the commit in a ref. But
the resulting repo is corrupt; we are missing the blob
reachable from a ref.
One way to solve this is to be more thorough when
referencing a sha1: make sure that not only do we have that
sha1, but that we have objects it refers to, and so forth
recursively. The problem is that this is very expensive.
Creating a parent link would require traversing the entire
object graph!
Instead, this patch pushes the extra work onto prune, which
runs less frequently (and has to look at the whole object
graph anyway). It creates a new category of objects: objects
which are not recent, but which are reachable from a recent
object. We do not prune these objects, just like the
reachable and recent ones.
This lets us avoid the recursive check above, because if we
have an object, even if it is unreachable, we should have
its referent. We can make a simple inductive argument that
with this patch, this property holds (that there are no
objects with missing referents in the repository):
0. When we have no objects, we have nothing to refer or be
referred to, so the property holds.
1. If we add objects to the repository, their direct
referents must generally exist (e.g., if you create a
tree, the blobs it references must exist; if you create
a commit to point at the tree, the tree must exist).
This is already the case before this patch. And it is
not 100% foolproof (you can make bogus objects using
`git hash-object`, for example), but it should be the
case for normal usage.
Therefore for any sequence of object additions, the
property will continue to hold.
2. If we remove objects from the repository, then we will
not remove a child object (like a blob) if an object
that refers to it is being kept. That is the part
implemented by this patch.
Note, however, that our reachability check and the
actual pruning are not atomic. So it _is_ still
possible to violate the property (e.g., an object
becomes referenced just as we are deleting it). This
patch is shooting for eliminating problems where the
mtimes of dependent objects differ by hours or days,
and one is dropped without the other. It does nothing
to help with short races.
Naively, the simplest way to implement this would be to add
all recent objects as tips to the reachability traversal.
However, this does not perform well. In a recently-packed
repository, all reachable objects will also be recent, and
therefore we have to look at each object twice. This patch
instead performs the reachability traversal, then follows up
with a second traversal for recent objects, skipping any
that have already been marked.
Signed-off-by: Jeff King <peff@peff.net>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2014-10-15 22:41:35 +00:00
|
|
|
|
|
|
|
|
add_pending_object(data->revs, obj, "");
|
|
|
|
|
}
|
|
|
|
|
|
2017-02-21 23:47:35 +00:00
|
|
|
static int add_recent_loose(const struct object_id *oid,
|
prune: keep objects reachable from recent objects
Our current strategy with prune is that an object falls into
one of three categories:
1. Reachable (from ref tips, reflogs, index, etc).
2. Not reachable, but recent (based on the --expire time).
3. Not reachable and not recent.
We keep objects from (1) and (2), but prune objects in (3).
The point of (2) is that these objects may be part of an
in-progress operation that has not yet updated any refs.
However, it is not always the case that objects for an
in-progress operation will have a recent mtime. For example,
the object database may have an old copy of a blob (from an
abandoned operation, a branch that was deleted, etc). If we
create a new tree that points to it, a simultaneous prune
will leave our tree, but delete the blob. Referencing that
tree with a commit will then work (we check that the tree is
in the object database, but not that all of its referred
objects are), as will mentioning the commit in a ref. But
the resulting repo is corrupt; we are missing the blob
reachable from a ref.
One way to solve this is to be more thorough when
referencing a sha1: make sure that not only do we have that
sha1, but that we have objects it refers to, and so forth
recursively. The problem is that this is very expensive.
Creating a parent link would require traversing the entire
object graph!
Instead, this patch pushes the extra work onto prune, which
runs less frequently (and has to look at the whole object
graph anyway). It creates a new category of objects: objects
which are not recent, but which are reachable from a recent
object. We do not prune these objects, just like the
reachable and recent ones.
This lets us avoid the recursive check above, because if we
have an object, even if it is unreachable, we should have
its referent. We can make a simple inductive argument that
with this patch, this property holds (that there are no
objects with missing referents in the repository):
0. When we have no objects, we have nothing to refer or be
referred to, so the property holds.
1. If we add objects to the repository, their direct
referents must generally exist (e.g., if you create a
tree, the blobs it references must exist; if you create
a commit to point at the tree, the tree must exist).
This is already the case before this patch. And it is
not 100% foolproof (you can make bogus objects using
`git hash-object`, for example), but it should be the
case for normal usage.
Therefore for any sequence of object additions, the
property will continue to hold.
2. If we remove objects from the repository, then we will
not remove a child object (like a blob) if an object
that refers to it is being kept. That is the part
implemented by this patch.
Note, however, that our reachability check and the
actual pruning are not atomic. So it _is_ still
possible to violate the property (e.g., an object
becomes referenced just as we are deleting it). This
patch is shooting for eliminating problems where the
mtimes of dependent objects differ by hours or days,
and one is dropped without the other. It does nothing
to help with short races.
Naively, the simplest way to implement this would be to add
all recent objects as tips to the reachability traversal.
However, this does not perform well. In a recently-packed
repository, all reachable objects will also be recent, and
therefore we have to look at each object twice. This patch
instead performs the reachability traversal, then follows up
with a second traversal for recent objects, skipping any
that have already been marked.
Signed-off-by: Jeff King <peff@peff.net>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2014-10-15 22:41:35 +00:00
|
|
|
const char *path, void *data)
|
|
|
|
|
{
|
|
|
|
|
struct stat st;
|
2017-02-21 23:47:35 +00:00
|
|
|
struct object *obj = lookup_object(oid->hash);
|
prune: keep objects reachable from recent objects
Our current strategy with prune is that an object falls into
one of three categories:
1. Reachable (from ref tips, reflogs, index, etc).
2. Not reachable, but recent (based on the --expire time).
3. Not reachable and not recent.
We keep objects from (1) and (2), but prune objects in (3).
The point of (2) is that these objects may be part of an
in-progress operation that has not yet updated any refs.
However, it is not always the case that objects for an
in-progress operation will have a recent mtime. For example,
the object database may have an old copy of a blob (from an
abandoned operation, a branch that was deleted, etc). If we
create a new tree that points to it, a simultaneous prune
will leave our tree, but delete the blob. Referencing that
tree with a commit will then work (we check that the tree is
in the object database, but not that all of its referred
objects are), as will mentioning the commit in a ref. But
the resulting repo is corrupt; we are missing the blob
reachable from a ref.
One way to solve this is to be more thorough when
referencing a sha1: make sure that not only do we have that
sha1, but that we have objects it refers to, and so forth
recursively. The problem is that this is very expensive.
Creating a parent link would require traversing the entire
object graph!
Instead, this patch pushes the extra work onto prune, which
runs less frequently (and has to look at the whole object
graph anyway). It creates a new category of objects: objects
which are not recent, but which are reachable from a recent
object. We do not prune these objects, just like the
reachable and recent ones.
This lets us avoid the recursive check above, because if we
have an object, even if it is unreachable, we should have
its referent. We can make a simple inductive argument that
with this patch, this property holds (that there are no
objects with missing referents in the repository):
0. When we have no objects, we have nothing to refer or be
referred to, so the property holds.
1. If we add objects to the repository, their direct
referents must generally exist (e.g., if you create a
tree, the blobs it references must exist; if you create
a commit to point at the tree, the tree must exist).
This is already the case before this patch. And it is
not 100% foolproof (you can make bogus objects using
`git hash-object`, for example), but it should be the
case for normal usage.
Therefore for any sequence of object additions, the
property will continue to hold.
2. If we remove objects from the repository, then we will
not remove a child object (like a blob) if an object
that refers to it is being kept. That is the part
implemented by this patch.
Note, however, that our reachability check and the
actual pruning are not atomic. So it _is_ still
possible to violate the property (e.g., an object
becomes referenced just as we are deleting it). This
patch is shooting for eliminating problems where the
mtimes of dependent objects differ by hours or days,
and one is dropped without the other. It does nothing
to help with short races.
Naively, the simplest way to implement this would be to add
all recent objects as tips to the reachability traversal.
However, this does not perform well. In a recently-packed
repository, all reachable objects will also be recent, and
therefore we have to look at each object twice. This patch
instead performs the reachability traversal, then follows up
with a second traversal for recent objects, skipping any
that have already been marked.
Signed-off-by: Jeff King <peff@peff.net>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2014-10-15 22:41:35 +00:00
|
|
|
|
|
|
|
|
if (obj && obj->flags & SEEN)
|
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
|
|
if (stat(path, &st) < 0) {
|
|
|
|
|
/*
|
|
|
|
|
* It's OK if an object went away during our iteration; this
|
|
|
|
|
* could be due to a simultaneous repack. But anything else
|
|
|
|
|
* we should abort, since we might then fail to mark objects
|
|
|
|
|
* which should not be pruned.
|
|
|
|
|
*/
|
|
|
|
|
if (errno == ENOENT)
|
|
|
|
|
return 0;
|
2017-02-21 23:47:35 +00:00
|
|
|
return error_errno("unable to stat %s", oid_to_hex(oid));
|
prune: keep objects reachable from recent objects
Our current strategy with prune is that an object falls into
one of three categories:
1. Reachable (from ref tips, reflogs, index, etc).
2. Not reachable, but recent (based on the --expire time).
3. Not reachable and not recent.
We keep objects from (1) and (2), but prune objects in (3).
The point of (2) is that these objects may be part of an
in-progress operation that has not yet updated any refs.
However, it is not always the case that objects for an
in-progress operation will have a recent mtime. For example,
the object database may have an old copy of a blob (from an
abandoned operation, a branch that was deleted, etc). If we
create a new tree that points to it, a simultaneous prune
will leave our tree, but delete the blob. Referencing that
tree with a commit will then work (we check that the tree is
in the object database, but not that all of its referred
objects are), as will mentioning the commit in a ref. But
the resulting repo is corrupt; we are missing the blob
reachable from a ref.
One way to solve this is to be more thorough when
referencing a sha1: make sure that not only do we have that
sha1, but that we have objects it refers to, and so forth
recursively. The problem is that this is very expensive.
Creating a parent link would require traversing the entire
object graph!
Instead, this patch pushes the extra work onto prune, which
runs less frequently (and has to look at the whole object
graph anyway). It creates a new category of objects: objects
which are not recent, but which are reachable from a recent
object. We do not prune these objects, just like the
reachable and recent ones.
This lets us avoid the recursive check above, because if we
have an object, even if it is unreachable, we should have
its referent. We can make a simple inductive argument that
with this patch, this property holds (that there are no
objects with missing referents in the repository):
0. When we have no objects, we have nothing to refer or be
referred to, so the property holds.
1. If we add objects to the repository, their direct
referents must generally exist (e.g., if you create a
tree, the blobs it references must exist; if you create
a commit to point at the tree, the tree must exist).
This is already the case before this patch. And it is
not 100% foolproof (you can make bogus objects using
`git hash-object`, for example), but it should be the
case for normal usage.
Therefore for any sequence of object additions, the
property will continue to hold.
2. If we remove objects from the repository, then we will
not remove a child object (like a blob) if an object
that refers to it is being kept. That is the part
implemented by this patch.
Note, however, that our reachability check and the
actual pruning are not atomic. So it _is_ still
possible to violate the property (e.g., an object
becomes referenced just as we are deleting it). This
patch is shooting for eliminating problems where the
mtimes of dependent objects differ by hours or days,
and one is dropped without the other. It does nothing
to help with short races.
Naively, the simplest way to implement this would be to add
all recent objects as tips to the reachability traversal.
However, this does not perform well. In a recently-packed
repository, all reachable objects will also be recent, and
therefore we have to look at each object twice. This patch
instead performs the reachability traversal, then follows up
with a second traversal for recent objects, skipping any
that have already been marked.
Signed-off-by: Jeff King <peff@peff.net>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2014-10-15 22:41:35 +00:00
|
|
|
}
|
|
|
|
|
|
2017-02-21 23:47:35 +00:00
|
|
|
add_recent_object(oid, st.st_mtime, data);
|
prune: keep objects reachable from recent objects
Our current strategy with prune is that an object falls into
one of three categories:
1. Reachable (from ref tips, reflogs, index, etc).
2. Not reachable, but recent (based on the --expire time).
3. Not reachable and not recent.
We keep objects from (1) and (2), but prune objects in (3).
The point of (2) is that these objects may be part of an
in-progress operation that has not yet updated any refs.
However, it is not always the case that objects for an
in-progress operation will have a recent mtime. For example,
the object database may have an old copy of a blob (from an
abandoned operation, a branch that was deleted, etc). If we
create a new tree that points to it, a simultaneous prune
will leave our tree, but delete the blob. Referencing that
tree with a commit will then work (we check that the tree is
in the object database, but not that all of its referred
objects are), as will mentioning the commit in a ref. But
the resulting repo is corrupt; we are missing the blob
reachable from a ref.
One way to solve this is to be more thorough when
referencing a sha1: make sure that not only do we have that
sha1, but that we have objects it refers to, and so forth
recursively. The problem is that this is very expensive.
Creating a parent link would require traversing the entire
object graph!
Instead, this patch pushes the extra work onto prune, which
runs less frequently (and has to look at the whole object
graph anyway). It creates a new category of objects: objects
which are not recent, but which are reachable from a recent
object. We do not prune these objects, just like the
reachable and recent ones.
This lets us avoid the recursive check above, because if we
have an object, even if it is unreachable, we should have
its referent. We can make a simple inductive argument that
with this patch, this property holds (that there are no
objects with missing referents in the repository):
0. When we have no objects, we have nothing to refer or be
referred to, so the property holds.
1. If we add objects to the repository, their direct
referents must generally exist (e.g., if you create a
tree, the blobs it references must exist; if you create
a commit to point at the tree, the tree must exist).
This is already the case before this patch. And it is
not 100% foolproof (you can make bogus objects using
`git hash-object`, for example), but it should be the
case for normal usage.
Therefore for any sequence of object additions, the
property will continue to hold.
2. If we remove objects from the repository, then we will
not remove a child object (like a blob) if an object
that refers to it is being kept. That is the part
implemented by this patch.
Note, however, that our reachability check and the
actual pruning are not atomic. So it _is_ still
possible to violate the property (e.g., an object
becomes referenced just as we are deleting it). This
patch is shooting for eliminating problems where the
mtimes of dependent objects differ by hours or days,
and one is dropped without the other. It does nothing
to help with short races.
Naively, the simplest way to implement this would be to add
all recent objects as tips to the reachability traversal.
However, this does not perform well. In a recently-packed
repository, all reachable objects will also be recent, and
therefore we have to look at each object twice. This patch
instead performs the reachability traversal, then follows up
with a second traversal for recent objects, skipping any
that have already been marked.
Signed-off-by: Jeff King <peff@peff.net>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2014-10-15 22:41:35 +00:00
|
|
|
return 0;
|
|
|
|
|
}
|
|
|
|
|
|
2017-02-21 23:47:35 +00:00
|
|
|
static int add_recent_packed(const struct object_id *oid,
|
prune: keep objects reachable from recent objects
Our current strategy with prune is that an object falls into
one of three categories:
1. Reachable (from ref tips, reflogs, index, etc).
2. Not reachable, but recent (based on the --expire time).
3. Not reachable and not recent.
We keep objects from (1) and (2), but prune objects in (3).
The point of (2) is that these objects may be part of an
in-progress operation that has not yet updated any refs.
However, it is not always the case that objects for an
in-progress operation will have a recent mtime. For example,
the object database may have an old copy of a blob (from an
abandoned operation, a branch that was deleted, etc). If we
create a new tree that points to it, a simultaneous prune
will leave our tree, but delete the blob. Referencing that
tree with a commit will then work (we check that the tree is
in the object database, but not that all of its referred
objects are), as will mentioning the commit in a ref. But
the resulting repo is corrupt; we are missing the blob
reachable from a ref.
One way to solve this is to be more thorough when
referencing a sha1: make sure that not only do we have that
sha1, but that we have objects it refers to, and so forth
recursively. The problem is that this is very expensive.
Creating a parent link would require traversing the entire
object graph!
Instead, this patch pushes the extra work onto prune, which
runs less frequently (and has to look at the whole object
graph anyway). It creates a new category of objects: objects
which are not recent, but which are reachable from a recent
object. We do not prune these objects, just like the
reachable and recent ones.
This lets us avoid the recursive check above, because if we
have an object, even if it is unreachable, we should have
its referent. We can make a simple inductive argument that
with this patch, this property holds (that there are no
objects with missing referents in the repository):
0. When we have no objects, we have nothing to refer or be
referred to, so the property holds.
1. If we add objects to the repository, their direct
referents must generally exist (e.g., if you create a
tree, the blobs it references must exist; if you create
a commit to point at the tree, the tree must exist).
This is already the case before this patch. And it is
not 100% foolproof (you can make bogus objects using
`git hash-object`, for example), but it should be the
case for normal usage.
Therefore for any sequence of object additions, the
property will continue to hold.
2. If we remove objects from the repository, then we will
not remove a child object (like a blob) if an object
that refers to it is being kept. That is the part
implemented by this patch.
Note, however, that our reachability check and the
actual pruning are not atomic. So it _is_ still
possible to violate the property (e.g., an object
becomes referenced just as we are deleting it). This
patch is shooting for eliminating problems where the
mtimes of dependent objects differ by hours or days,
and one is dropped without the other. It does nothing
to help with short races.
Naively, the simplest way to implement this would be to add
all recent objects as tips to the reachability traversal.
However, this does not perform well. In a recently-packed
repository, all reachable objects will also be recent, and
therefore we have to look at each object twice. This patch
instead performs the reachability traversal, then follows up
with a second traversal for recent objects, skipping any
that have already been marked.
Signed-off-by: Jeff King <peff@peff.net>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2014-10-15 22:41:35 +00:00
|
|
|
struct packed_git *p, uint32_t pos,
|
|
|
|
|
void *data)
|
|
|
|
|
{
|
2017-02-21 23:47:35 +00:00
|
|
|
struct object *obj = lookup_object(oid->hash);
|
prune: keep objects reachable from recent objects
Our current strategy with prune is that an object falls into
one of three categories:
1. Reachable (from ref tips, reflogs, index, etc).
2. Not reachable, but recent (based on the --expire time).
3. Not reachable and not recent.
We keep objects from (1) and (2), but prune objects in (3).
The point of (2) is that these objects may be part of an
in-progress operation that has not yet updated any refs.
However, it is not always the case that objects for an
in-progress operation will have a recent mtime. For example,
the object database may have an old copy of a blob (from an
abandoned operation, a branch that was deleted, etc). If we
create a new tree that points to it, a simultaneous prune
will leave our tree, but delete the blob. Referencing that
tree with a commit will then work (we check that the tree is
in the object database, but not that all of its referred
objects are), as will mentioning the commit in a ref. But
the resulting repo is corrupt; we are missing the blob
reachable from a ref.
One way to solve this is to be more thorough when
referencing a sha1: make sure that not only do we have that
sha1, but that we have objects it refers to, and so forth
recursively. The problem is that this is very expensive.
Creating a parent link would require traversing the entire
object graph!
Instead, this patch pushes the extra work onto prune, which
runs less frequently (and has to look at the whole object
graph anyway). It creates a new category of objects: objects
which are not recent, but which are reachable from a recent
object. We do not prune these objects, just like the
reachable and recent ones.
This lets us avoid the recursive check above, because if we
have an object, even if it is unreachable, we should have
its referent. We can make a simple inductive argument that
with this patch, this property holds (that there are no
objects with missing referents in the repository):
0. When we have no objects, we have nothing to refer or be
referred to, so the property holds.
1. If we add objects to the repository, their direct
referents must generally exist (e.g., if you create a
tree, the blobs it references must exist; if you create
a commit to point at the tree, the tree must exist).
This is already the case before this patch. And it is
not 100% foolproof (you can make bogus objects using
`git hash-object`, for example), but it should be the
case for normal usage.
Therefore for any sequence of object additions, the
property will continue to hold.
2. If we remove objects from the repository, then we will
not remove a child object (like a blob) if an object
that refers to it is being kept. That is the part
implemented by this patch.
Note, however, that our reachability check and the
actual pruning are not atomic. So it _is_ still
possible to violate the property (e.g., an object
becomes referenced just as we are deleting it). This
patch is shooting for eliminating problems where the
mtimes of dependent objects differ by hours or days,
and one is dropped without the other. It does nothing
to help with short races.
Naively, the simplest way to implement this would be to add
all recent objects as tips to the reachability traversal.
However, this does not perform well. In a recently-packed
repository, all reachable objects will also be recent, and
therefore we have to look at each object twice. This patch
instead performs the reachability traversal, then follows up
with a second traversal for recent objects, skipping any
that have already been marked.
Signed-off-by: Jeff King <peff@peff.net>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2014-10-15 22:41:35 +00:00
|
|
|
|
|
|
|
|
if (obj && obj->flags & SEEN)
|
|
|
|
|
return 0;
|
2017-02-21 23:47:35 +00:00
|
|
|
add_recent_object(oid, p->mtime, data);
|
prune: keep objects reachable from recent objects
Our current strategy with prune is that an object falls into
one of three categories:
1. Reachable (from ref tips, reflogs, index, etc).
2. Not reachable, but recent (based on the --expire time).
3. Not reachable and not recent.
We keep objects from (1) and (2), but prune objects in (3).
The point of (2) is that these objects may be part of an
in-progress operation that has not yet updated any refs.
However, it is not always the case that objects for an
in-progress operation will have a recent mtime. For example,
the object database may have an old copy of a blob (from an
abandoned operation, a branch that was deleted, etc). If we
create a new tree that points to it, a simultaneous prune
will leave our tree, but delete the blob. Referencing that
tree with a commit will then work (we check that the tree is
in the object database, but not that all of its referred
objects are), as will mentioning the commit in a ref. But
the resulting repo is corrupt; we are missing the blob
reachable from a ref.
One way to solve this is to be more thorough when
referencing a sha1: make sure that not only do we have that
sha1, but that we have objects it refers to, and so forth
recursively. The problem is that this is very expensive.
Creating a parent link would require traversing the entire
object graph!
Instead, this patch pushes the extra work onto prune, which
runs less frequently (and has to look at the whole object
graph anyway). It creates a new category of objects: objects
which are not recent, but which are reachable from a recent
object. We do not prune these objects, just like the
reachable and recent ones.
This lets us avoid the recursive check above, because if we
have an object, even if it is unreachable, we should have
its referent. We can make a simple inductive argument that
with this patch, this property holds (that there are no
objects with missing referents in the repository):
0. When we have no objects, we have nothing to refer or be
referred to, so the property holds.
1. If we add objects to the repository, their direct
referents must generally exist (e.g., if you create a
tree, the blobs it references must exist; if you create
a commit to point at the tree, the tree must exist).
This is already the case before this patch. And it is
not 100% foolproof (you can make bogus objects using
`git hash-object`, for example), but it should be the
case for normal usage.
Therefore for any sequence of object additions, the
property will continue to hold.
2. If we remove objects from the repository, then we will
not remove a child object (like a blob) if an object
that refers to it is being kept. That is the part
implemented by this patch.
Note, however, that our reachability check and the
actual pruning are not atomic. So it _is_ still
possible to violate the property (e.g., an object
becomes referenced just as we are deleting it). This
patch is shooting for eliminating problems where the
mtimes of dependent objects differ by hours or days,
and one is dropped without the other. It does nothing
to help with short races.
Naively, the simplest way to implement this would be to add
all recent objects as tips to the reachability traversal.
However, this does not perform well. In a recently-packed
repository, all reachable objects will also be recent, and
therefore we have to look at each object twice. This patch
instead performs the reachability traversal, then follows up
with a second traversal for recent objects, skipping any
that have already been marked.
Signed-off-by: Jeff King <peff@peff.net>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2014-10-15 22:41:35 +00:00
|
|
|
return 0;
|
|
|
|
|
}
|
|
|
|
|
|
2014-10-15 22:42:09 +00:00
|
|
|
int add_unseen_recent_objects_to_traversal(struct rev_info *revs,
|
2017-04-26 19:29:31 +00:00
|
|
|
timestamp_t timestamp)
|
prune: keep objects reachable from recent objects
Our current strategy with prune is that an object falls into
one of three categories:
1. Reachable (from ref tips, reflogs, index, etc).
2. Not reachable, but recent (based on the --expire time).
3. Not reachable and not recent.
We keep objects from (1) and (2), but prune objects in (3).
The point of (2) is that these objects may be part of an
in-progress operation that has not yet updated any refs.
However, it is not always the case that objects for an
in-progress operation will have a recent mtime. For example,
the object database may have an old copy of a blob (from an
abandoned operation, a branch that was deleted, etc). If we
create a new tree that points to it, a simultaneous prune
will leave our tree, but delete the blob. Referencing that
tree with a commit will then work (we check that the tree is
in the object database, but not that all of its referred
objects are), as will mentioning the commit in a ref. But
the resulting repo is corrupt; we are missing the blob
reachable from a ref.
One way to solve this is to be more thorough when
referencing a sha1: make sure that not only do we have that
sha1, but that we have objects it refers to, and so forth
recursively. The problem is that this is very expensive.
Creating a parent link would require traversing the entire
object graph!
Instead, this patch pushes the extra work onto prune, which
runs less frequently (and has to look at the whole object
graph anyway). It creates a new category of objects: objects
which are not recent, but which are reachable from a recent
object. We do not prune these objects, just like the
reachable and recent ones.
This lets us avoid the recursive check above, because if we
have an object, even if it is unreachable, we should have
its referent. We can make a simple inductive argument that
with this patch, this property holds (that there are no
objects with missing referents in the repository):
0. When we have no objects, we have nothing to refer or be
referred to, so the property holds.
1. If we add objects to the repository, their direct
referents must generally exist (e.g., if you create a
tree, the blobs it references must exist; if you create
a commit to point at the tree, the tree must exist).
This is already the case before this patch. And it is
not 100% foolproof (you can make bogus objects using
`git hash-object`, for example), but it should be the
case for normal usage.
Therefore for any sequence of object additions, the
property will continue to hold.
2. If we remove objects from the repository, then we will
not remove a child object (like a blob) if an object
that refers to it is being kept. That is the part
implemented by this patch.
Note, however, that our reachability check and the
actual pruning are not atomic. So it _is_ still
possible to violate the property (e.g., an object
becomes referenced just as we are deleting it). This
patch is shooting for eliminating problems where the
mtimes of dependent objects differ by hours or days,
and one is dropped without the other. It does nothing
to help with short races.
Naively, the simplest way to implement this would be to add
all recent objects as tips to the reachability traversal.
However, this does not perform well. In a recently-packed
repository, all reachable objects will also be recent, and
therefore we have to look at each object twice. This patch
instead performs the reachability traversal, then follows up
with a second traversal for recent objects, skipping any
that have already been marked.
Signed-off-by: Jeff King <peff@peff.net>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2014-10-15 22:41:35 +00:00
|
|
|
{
|
|
|
|
|
struct recent_data data;
|
|
|
|
|
int r;
|
|
|
|
|
|
|
|
|
|
data.revs = revs;
|
|
|
|
|
data.timestamp = timestamp;
|
|
|
|
|
|
reachable: only mark local objects as recent
When pruning and repacking a repository that has an
alternate object store configured, we may traverse a large
number of objects in the alternate. This serves no purpose,
and may be expensive to do. A longer explanation is below.
Commits d3038d2 and abcb865 taught prune and pack-objects
(respectively) to treat "recent" objects as tips for
reachability, so that we keep whole chunks of history. They
built on the object traversal in 660c889 (sha1_file: add
for_each iterators for loose and packed objects,
2014-10-15), which covers both local and alternate objects.
In both cases, covering alternate objects is unnecessary, as
both commands can only drop objects from the local
repository. In the case of prune, we traverse only the local
object directory. And in the case of repacking, while we may
or may not include local objects in our pack, we will never
reach into the alternate with "repack -d". The "-l" option
is only a question of whether we are migrating objects from
the alternate into our repository, or leaving them
untouched.
It is possible that we may drop an object that is depended
upon by another object in the alternate. For example,
imagine two repositories, A and B, with A pointing to B as
an alternate. Now imagine a commit that is in B which
references a tree that is only in A. Traversing from recent
objects in B might prevent A from dropping that tree. But
this case isn't worth covering. Repo B should take
responsibility for its own objects. It would never have had
the commit in the first place if it did not also have the
tree, and assuming it is using the same "keep recent chunks
of history" scheme, then it would itself keep the tree, as
well.
So checking the alternate objects is not worth doing, and
come with a significant performance impact. In both cases,
we skip any recent objects that have already been marked
SEEN (i.e., that we know are already reachable for prune, or
included in the pack for a repack). So there is a slight
waste of time in opening the alternate packs at all, only to
notice that we have already considered each object. But much
worse, the alternate repository may have a large number of
objects that are not reachable from the local repository at
all, and we end up adding them to the traversal.
We can fix this by considering only local unseen objects.
Signed-off-by: Jeff King <peff@peff.net>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2015-03-27 11:32:41 +00:00
|
|
|
r = for_each_loose_object(add_recent_loose, &data,
|
|
|
|
|
FOR_EACH_OBJECT_LOCAL_ONLY);
|
prune: keep objects reachable from recent objects
Our current strategy with prune is that an object falls into
one of three categories:
1. Reachable (from ref tips, reflogs, index, etc).
2. Not reachable, but recent (based on the --expire time).
3. Not reachable and not recent.
We keep objects from (1) and (2), but prune objects in (3).
The point of (2) is that these objects may be part of an
in-progress operation that has not yet updated any refs.
However, it is not always the case that objects for an
in-progress operation will have a recent mtime. For example,
the object database may have an old copy of a blob (from an
abandoned operation, a branch that was deleted, etc). If we
create a new tree that points to it, a simultaneous prune
will leave our tree, but delete the blob. Referencing that
tree with a commit will then work (we check that the tree is
in the object database, but not that all of its referred
objects are), as will mentioning the commit in a ref. But
the resulting repo is corrupt; we are missing the blob
reachable from a ref.
One way to solve this is to be more thorough when
referencing a sha1: make sure that not only do we have that
sha1, but that we have objects it refers to, and so forth
recursively. The problem is that this is very expensive.
Creating a parent link would require traversing the entire
object graph!
Instead, this patch pushes the extra work onto prune, which
runs less frequently (and has to look at the whole object
graph anyway). It creates a new category of objects: objects
which are not recent, but which are reachable from a recent
object. We do not prune these objects, just like the
reachable and recent ones.
This lets us avoid the recursive check above, because if we
have an object, even if it is unreachable, we should have
its referent. We can make a simple inductive argument that
with this patch, this property holds (that there are no
objects with missing referents in the repository):
0. When we have no objects, we have nothing to refer or be
referred to, so the property holds.
1. If we add objects to the repository, their direct
referents must generally exist (e.g., if you create a
tree, the blobs it references must exist; if you create
a commit to point at the tree, the tree must exist).
This is already the case before this patch. And it is
not 100% foolproof (you can make bogus objects using
`git hash-object`, for example), but it should be the
case for normal usage.
Therefore for any sequence of object additions, the
property will continue to hold.
2. If we remove objects from the repository, then we will
not remove a child object (like a blob) if an object
that refers to it is being kept. That is the part
implemented by this patch.
Note, however, that our reachability check and the
actual pruning are not atomic. So it _is_ still
possible to violate the property (e.g., an object
becomes referenced just as we are deleting it). This
patch is shooting for eliminating problems where the
mtimes of dependent objects differ by hours or days,
and one is dropped without the other. It does nothing
to help with short races.
Naively, the simplest way to implement this would be to add
all recent objects as tips to the reachability traversal.
However, this does not perform well. In a recently-packed
repository, all reachable objects will also be recent, and
therefore we have to look at each object twice. This patch
instead performs the reachability traversal, then follows up
with a second traversal for recent objects, skipping any
that have already been marked.
Signed-off-by: Jeff King <peff@peff.net>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2014-10-15 22:41:35 +00:00
|
|
|
if (r)
|
|
|
|
|
return r;
|
reachable: only mark local objects as recent
When pruning and repacking a repository that has an
alternate object store configured, we may traverse a large
number of objects in the alternate. This serves no purpose,
and may be expensive to do. A longer explanation is below.
Commits d3038d2 and abcb865 taught prune and pack-objects
(respectively) to treat "recent" objects as tips for
reachability, so that we keep whole chunks of history. They
built on the object traversal in 660c889 (sha1_file: add
for_each iterators for loose and packed objects,
2014-10-15), which covers both local and alternate objects.
In both cases, covering alternate objects is unnecessary, as
both commands can only drop objects from the local
repository. In the case of prune, we traverse only the local
object directory. And in the case of repacking, while we may
or may not include local objects in our pack, we will never
reach into the alternate with "repack -d". The "-l" option
is only a question of whether we are migrating objects from
the alternate into our repository, or leaving them
untouched.
It is possible that we may drop an object that is depended
upon by another object in the alternate. For example,
imagine two repositories, A and B, with A pointing to B as
an alternate. Now imagine a commit that is in B which
references a tree that is only in A. Traversing from recent
objects in B might prevent A from dropping that tree. But
this case isn't worth covering. Repo B should take
responsibility for its own objects. It would never have had
the commit in the first place if it did not also have the
tree, and assuming it is using the same "keep recent chunks
of history" scheme, then it would itself keep the tree, as
well.
So checking the alternate objects is not worth doing, and
come with a significant performance impact. In both cases,
we skip any recent objects that have already been marked
SEEN (i.e., that we know are already reachable for prune, or
included in the pack for a repack). So there is a slight
waste of time in opening the alternate packs at all, only to
notice that we have already considered each object. But much
worse, the alternate repository may have a large number of
objects that are not reachable from the local repository at
all, and we end up adding them to the traversal.
We can fix this by considering only local unseen objects.
Signed-off-by: Jeff King <peff@peff.net>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2015-03-27 11:32:41 +00:00
|
|
|
return for_each_packed_object(add_recent_packed, &data,
|
|
|
|
|
FOR_EACH_OBJECT_LOCAL_ONLY);
|
prune: keep objects reachable from recent objects
Our current strategy with prune is that an object falls into
one of three categories:
1. Reachable (from ref tips, reflogs, index, etc).
2. Not reachable, but recent (based on the --expire time).
3. Not reachable and not recent.
We keep objects from (1) and (2), but prune objects in (3).
The point of (2) is that these objects may be part of an
in-progress operation that has not yet updated any refs.
However, it is not always the case that objects for an
in-progress operation will have a recent mtime. For example,
the object database may have an old copy of a blob (from an
abandoned operation, a branch that was deleted, etc). If we
create a new tree that points to it, a simultaneous prune
will leave our tree, but delete the blob. Referencing that
tree with a commit will then work (we check that the tree is
in the object database, but not that all of its referred
objects are), as will mentioning the commit in a ref. But
the resulting repo is corrupt; we are missing the blob
reachable from a ref.
One way to solve this is to be more thorough when
referencing a sha1: make sure that not only do we have that
sha1, but that we have objects it refers to, and so forth
recursively. The problem is that this is very expensive.
Creating a parent link would require traversing the entire
object graph!
Instead, this patch pushes the extra work onto prune, which
runs less frequently (and has to look at the whole object
graph anyway). It creates a new category of objects: objects
which are not recent, but which are reachable from a recent
object. We do not prune these objects, just like the
reachable and recent ones.
This lets us avoid the recursive check above, because if we
have an object, even if it is unreachable, we should have
its referent. We can make a simple inductive argument that
with this patch, this property holds (that there are no
objects with missing referents in the repository):
0. When we have no objects, we have nothing to refer or be
referred to, so the property holds.
1. If we add objects to the repository, their direct
referents must generally exist (e.g., if you create a
tree, the blobs it references must exist; if you create
a commit to point at the tree, the tree must exist).
This is already the case before this patch. And it is
not 100% foolproof (you can make bogus objects using
`git hash-object`, for example), but it should be the
case for normal usage.
Therefore for any sequence of object additions, the
property will continue to hold.
2. If we remove objects from the repository, then we will
not remove a child object (like a blob) if an object
that refers to it is being kept. That is the part
implemented by this patch.
Note, however, that our reachability check and the
actual pruning are not atomic. So it _is_ still
possible to violate the property (e.g., an object
becomes referenced just as we are deleting it). This
patch is shooting for eliminating problems where the
mtimes of dependent objects differ by hours or days,
and one is dropped without the other. It does nothing
to help with short races.
Naively, the simplest way to implement this would be to add
all recent objects as tips to the reachability traversal.
However, this does not perform well. In a recently-packed
repository, all reachable objects will also be recent, and
therefore we have to look at each object twice. This patch
instead performs the reachability traversal, then follows up
with a second traversal for recent objects, skipping any
that have already been marked.
Signed-off-by: Jeff King <peff@peff.net>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2014-10-15 22:41:35 +00:00
|
|
|
}
|
|
|
|
|
|
2011-11-05 12:00:08 +00:00
|
|
|
void mark_reachable_objects(struct rev_info *revs, int mark_reflog,
|
2017-04-26 19:29:31 +00:00
|
|
|
timestamp_t mark_recent, struct progress *progress)
|
2007-01-06 10:16:17 +00:00
|
|
|
{
|
2011-11-08 05:37:00 +00:00
|
|
|
struct connectivity_progress cp;
|
|
|
|
|
|
2007-01-06 10:16:17 +00:00
|
|
|
/*
|
|
|
|
|
* Set up revision parsing, and mark us as being interested
|
|
|
|
|
* in all object types, not just commits.
|
|
|
|
|
*/
|
|
|
|
|
revs->tag_objects = 1;
|
|
|
|
|
revs->blob_objects = 1;
|
|
|
|
|
revs->tree_objects = 1;
|
|
|
|
|
|
|
|
|
|
/* Add all refs from the index file */
|
2014-10-17 00:44:30 +00:00
|
|
|
add_index_objects_to_pending(revs, 0);
|
2007-01-06 10:16:17 +00:00
|
|
|
|
|
|
|
|
/* Add all external refs */
|
2015-05-25 18:39:00 +00:00
|
|
|
for_each_ref(add_one_ref, revs);
|
2007-01-06 10:16:17 +00:00
|
|
|
|
2014-09-03 16:14:10 +00:00
|
|
|
/* detached HEAD is not included in the list above */
|
2015-05-25 18:39:00 +00:00
|
|
|
head_ref(add_one_ref, revs);
|
2017-08-23 12:36:59 +00:00
|
|
|
other_head_refs(add_one_ref, revs);
|
2014-09-03 16:14:10 +00:00
|
|
|
|
2007-02-03 18:25:43 +00:00
|
|
|
/* Add all reflog info */
|
2007-01-06 10:16:17 +00:00
|
|
|
if (mark_reflog)
|
2014-10-15 22:38:31 +00:00
|
|
|
add_reflogs_to_pending(revs, 0);
|
2007-01-06 10:16:17 +00:00
|
|
|
|
2011-11-08 05:37:00 +00:00
|
|
|
cp.progress = progress;
|
|
|
|
|
cp.count = 0;
|
|
|
|
|
|
2007-01-06 10:16:17 +00:00
|
|
|
/*
|
|
|
|
|
* Set up the revision walk - this will move all commits
|
|
|
|
|
* from the pending list to the commit walking list.
|
|
|
|
|
*/
|
2008-02-18 07:31:56 +00:00
|
|
|
if (prepare_revision_walk(revs))
|
|
|
|
|
die("revision walk setup failed");
|
2014-10-15 22:37:28 +00:00
|
|
|
traverse_commit_list(revs, mark_commit, mark_object, &cp);
|
prune: keep objects reachable from recent objects
Our current strategy with prune is that an object falls into
one of three categories:
1. Reachable (from ref tips, reflogs, index, etc).
2. Not reachable, but recent (based on the --expire time).
3. Not reachable and not recent.
We keep objects from (1) and (2), but prune objects in (3).
The point of (2) is that these objects may be part of an
in-progress operation that has not yet updated any refs.
However, it is not always the case that objects for an
in-progress operation will have a recent mtime. For example,
the object database may have an old copy of a blob (from an
abandoned operation, a branch that was deleted, etc). If we
create a new tree that points to it, a simultaneous prune
will leave our tree, but delete the blob. Referencing that
tree with a commit will then work (we check that the tree is
in the object database, but not that all of its referred
objects are), as will mentioning the commit in a ref. But
the resulting repo is corrupt; we are missing the blob
reachable from a ref.
One way to solve this is to be more thorough when
referencing a sha1: make sure that not only do we have that
sha1, but that we have objects it refers to, and so forth
recursively. The problem is that this is very expensive.
Creating a parent link would require traversing the entire
object graph!
Instead, this patch pushes the extra work onto prune, which
runs less frequently (and has to look at the whole object
graph anyway). It creates a new category of objects: objects
which are not recent, but which are reachable from a recent
object. We do not prune these objects, just like the
reachable and recent ones.
This lets us avoid the recursive check above, because if we
have an object, even if it is unreachable, we should have
its referent. We can make a simple inductive argument that
with this patch, this property holds (that there are no
objects with missing referents in the repository):
0. When we have no objects, we have nothing to refer or be
referred to, so the property holds.
1. If we add objects to the repository, their direct
referents must generally exist (e.g., if you create a
tree, the blobs it references must exist; if you create
a commit to point at the tree, the tree must exist).
This is already the case before this patch. And it is
not 100% foolproof (you can make bogus objects using
`git hash-object`, for example), but it should be the
case for normal usage.
Therefore for any sequence of object additions, the
property will continue to hold.
2. If we remove objects from the repository, then we will
not remove a child object (like a blob) if an object
that refers to it is being kept. That is the part
implemented by this patch.
Note, however, that our reachability check and the
actual pruning are not atomic. So it _is_ still
possible to violate the property (e.g., an object
becomes referenced just as we are deleting it). This
patch is shooting for eliminating problems where the
mtimes of dependent objects differ by hours or days,
and one is dropped without the other. It does nothing
to help with short races.
Naively, the simplest way to implement this would be to add
all recent objects as tips to the reachability traversal.
However, this does not perform well. In a recently-packed
repository, all reachable objects will also be recent, and
therefore we have to look at each object twice. This patch
instead performs the reachability traversal, then follows up
with a second traversal for recent objects, skipping any
that have already been marked.
Signed-off-by: Jeff King <peff@peff.net>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2014-10-15 22:41:35 +00:00
|
|
|
|
|
|
|
|
if (mark_recent) {
|
|
|
|
|
revs->ignore_missing_links = 1;
|
|
|
|
|
if (add_unseen_recent_objects_to_traversal(revs, mark_recent))
|
|
|
|
|
die("unable to mark recent objects");
|
|
|
|
|
if (prepare_revision_walk(revs))
|
|
|
|
|
die("revision walk setup failed");
|
|
|
|
|
traverse_commit_list(revs, mark_commit, mark_object, &cp);
|
|
|
|
|
}
|
|
|
|
|
|
2011-11-08 05:37:00 +00:00
|
|
|
display_progress(cp.progress, cp.count);
|
2007-01-06 10:16:17 +00:00
|
|
|
}
|