2013-09-15 15:33:20 +00:00
|
|
|
#include "builtin.h"
|
|
|
|
#include "cache.h"
|
|
|
|
#include "dir.h"
|
|
|
|
#include "parse-options.h"
|
|
|
|
#include "run-command.h"
|
|
|
|
#include "sigchain.h"
|
|
|
|
#include "strbuf.h"
|
|
|
|
#include "string-list.h"
|
|
|
|
#include "argv-array.h"
|
|
|
|
|
|
|
|
static int delta_base_offset = 1;
|
repack: add `repack.packKeptObjects` config var
The git-repack command always passes `--honor-pack-keep`
to pack-objects. This has traditionally been a good thing,
as we do not want to duplicate those objects in a new pack,
and we are not going to delete the old pack.
However, when bitmaps are in use, it is important for a full
repack to include all reachable objects, even if they may be
duplicated in a .keep pack. Otherwise, we cannot generate
the bitmaps, as the on-disk format requires the set of
objects in the pack to be fully closed.
Even if the repository does not generally have .keep files,
a simultaneous push could cause a race condition in which a
.keep file exists at the moment of a repack. The repack may
try to include those objects in one of two situations:
1. The pushed .keep pack contains objects that were
already in the repository (e.g., blobs due to a revert of
an old commit).
2. Receive-pack updates the refs, making the objects
reachable, but before it removes the .keep file, the
repack runs.
In either case, we may prefer to duplicate some objects in
the new, full pack, and let the next repack (after the .keep
file is cleaned up) take care of removing them.
This patch introduces both a command-line and config option
to disable the `--honor-pack-keep` option. By default, it
is triggered when pack.writeBitmaps (or `--write-bitmap-index`
is turned on), but specifying it explicitly can override the
behavior (e.g., in cases where you prefer .keep files to
bitmaps, but only when they are present).
Note that this option just disables the pack-objects
behavior. We still leave packs with a .keep in place, as we
do not necessarily know that we have duplicated all of their
objects.
Signed-off-by: Jeff King <peff@peff.net>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2014-03-03 20:04:20 +00:00
|
|
|
static int pack_kept_objects = -1;
|
2014-06-10 20:19:38 +00:00
|
|
|
static int write_bitmaps;
|
2013-09-15 15:33:20 +00:00
|
|
|
static char *packdir, *packtmp;
|
|
|
|
|
|
|
|
static const char *const git_repack_usage[] = {
|
2015-01-13 07:44:47 +00:00
|
|
|
N_("git repack [<options>]"),
|
2013-09-15 15:33:20 +00:00
|
|
|
NULL
|
|
|
|
};
|
|
|
|
|
|
|
|
static int repack_config(const char *var, const char *value, void *cb)
|
|
|
|
{
|
|
|
|
if (!strcmp(var, "repack.usedeltabaseoffset")) {
|
|
|
|
delta_base_offset = git_config_bool(var, value);
|
|
|
|
return 0;
|
|
|
|
}
|
repack: add `repack.packKeptObjects` config var
The git-repack command always passes `--honor-pack-keep`
to pack-objects. This has traditionally been a good thing,
as we do not want to duplicate those objects in a new pack,
and we are not going to delete the old pack.
However, when bitmaps are in use, it is important for a full
repack to include all reachable objects, even if they may be
duplicated in a .keep pack. Otherwise, we cannot generate
the bitmaps, as the on-disk format requires the set of
objects in the pack to be fully closed.
Even if the repository does not generally have .keep files,
a simultaneous push could cause a race condition in which a
.keep file exists at the moment of a repack. The repack may
try to include those objects in one of two situations:
1. The pushed .keep pack contains objects that were
already in the repository (e.g., blobs due to a revert of
an old commit).
2. Receive-pack updates the refs, making the objects
reachable, but before it removes the .keep file, the
repack runs.
In either case, we may prefer to duplicate some objects in
the new, full pack, and let the next repack (after the .keep
file is cleaned up) take care of removing them.
This patch introduces both a command-line and config option
to disable the `--honor-pack-keep` option. By default, it
is triggered when pack.writeBitmaps (or `--write-bitmap-index`
is turned on), but specifying it explicitly can override the
behavior (e.g., in cases where you prefer .keep files to
bitmaps, but only when they are present).
Note that this option just disables the pack-objects
behavior. We still leave packs with a .keep in place, as we
do not necessarily know that we have duplicated all of their
objects.
Signed-off-by: Jeff King <peff@peff.net>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2014-03-03 20:04:20 +00:00
|
|
|
if (!strcmp(var, "repack.packkeptobjects")) {
|
|
|
|
pack_kept_objects = git_config_bool(var, value);
|
|
|
|
return 0;
|
|
|
|
}
|
2014-06-10 20:20:30 +00:00
|
|
|
if (!strcmp(var, "repack.writebitmaps") ||
|
|
|
|
!strcmp(var, "pack.writebitmaps")) {
|
2014-06-10 20:10:07 +00:00
|
|
|
write_bitmaps = git_config_bool(var, value);
|
2014-06-10 20:09:23 +00:00
|
|
|
return 0;
|
|
|
|
}
|
2013-09-15 15:33:20 +00:00
|
|
|
return git_default_config(var, value, cb);
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Remove temporary $GIT_OBJECT_DIRECTORY/pack/.tmp-$$-pack-* files.
|
|
|
|
*/
|
|
|
|
static void remove_temporary_files(void)
|
|
|
|
{
|
|
|
|
struct strbuf buf = STRBUF_INIT;
|
|
|
|
size_t dirlen, prefixlen;
|
|
|
|
DIR *dir;
|
|
|
|
struct dirent *e;
|
|
|
|
|
|
|
|
dir = opendir(packdir);
|
|
|
|
if (!dir)
|
|
|
|
return;
|
|
|
|
|
|
|
|
/* Point at the slash at the end of ".../objects/pack/" */
|
|
|
|
dirlen = strlen(packdir) + 1;
|
|
|
|
strbuf_addstr(&buf, packtmp);
|
|
|
|
/* Hold the length of ".tmp-%d-pack-" */
|
|
|
|
prefixlen = buf.len - dirlen;
|
|
|
|
|
|
|
|
while ((e = readdir(dir))) {
|
|
|
|
if (strncmp(e->d_name, buf.buf + dirlen, prefixlen))
|
|
|
|
continue;
|
|
|
|
strbuf_setlen(&buf, dirlen);
|
|
|
|
strbuf_addstr(&buf, e->d_name);
|
|
|
|
unlink(buf.buf);
|
|
|
|
}
|
|
|
|
closedir(dir);
|
|
|
|
strbuf_release(&buf);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void remove_pack_on_signal(int signo)
|
|
|
|
{
|
|
|
|
remove_temporary_files();
|
|
|
|
sigchain_pop(signo);
|
|
|
|
raise(signo);
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Adds all packs hex strings to the fname list, which do not
|
|
|
|
* have a corresponding .keep file.
|
|
|
|
*/
|
|
|
|
static void get_non_kept_pack_filenames(struct string_list *fname_list)
|
|
|
|
{
|
|
|
|
DIR *dir;
|
|
|
|
struct dirent *e;
|
|
|
|
char *fname;
|
|
|
|
|
|
|
|
if (!(dir = opendir(packdir)))
|
|
|
|
return;
|
|
|
|
|
|
|
|
while ((e = readdir(dir)) != NULL) {
|
2014-06-30 16:58:51 +00:00
|
|
|
size_t len;
|
|
|
|
if (!strip_suffix(e->d_name, ".pack", &len))
|
2013-09-15 15:33:20 +00:00
|
|
|
continue;
|
|
|
|
|
|
|
|
fname = xmemdupz(e->d_name, len);
|
|
|
|
|
|
|
|
if (!file_exists(mkpath("%s/%s.keep", packdir, fname)))
|
|
|
|
string_list_append_nodup(fname_list, fname);
|
|
|
|
else
|
|
|
|
free(fname);
|
|
|
|
}
|
|
|
|
closedir(dir);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void remove_redundant_pack(const char *dir_name, const char *base_name)
|
|
|
|
{
|
2013-12-21 14:00:31 +00:00
|
|
|
const char *exts[] = {".pack", ".idx", ".keep", ".bitmap"};
|
2013-09-15 15:33:20 +00:00
|
|
|
int i;
|
|
|
|
struct strbuf buf = STRBUF_INIT;
|
|
|
|
size_t plen;
|
|
|
|
|
|
|
|
strbuf_addf(&buf, "%s/%s", dir_name, base_name);
|
|
|
|
plen = buf.len;
|
|
|
|
|
|
|
|
for (i = 0; i < ARRAY_SIZE(exts); i++) {
|
|
|
|
strbuf_setlen(&buf, plen);
|
|
|
|
strbuf_addstr(&buf, exts[i]);
|
|
|
|
unlink(buf.buf);
|
|
|
|
}
|
|
|
|
strbuf_release(&buf);
|
|
|
|
}
|
|
|
|
|
|
|
|
#define ALL_INTO_ONE 1
|
|
|
|
#define LOOSEN_UNREACHABLE 2
|
|
|
|
|
|
|
|
int cmd_repack(int argc, const char **argv, const char *prefix)
|
|
|
|
{
|
2013-12-21 14:00:23 +00:00
|
|
|
struct {
|
|
|
|
const char *name;
|
2013-12-21 14:00:27 +00:00
|
|
|
unsigned optional:1;
|
2013-12-21 14:00:23 +00:00
|
|
|
} exts[] = {
|
|
|
|
{".pack"},
|
|
|
|
{".idx"},
|
2013-12-21 14:00:31 +00:00
|
|
|
{".bitmap", 1},
|
2013-12-21 14:00:23 +00:00
|
|
|
};
|
2014-08-19 19:09:35 +00:00
|
|
|
struct child_process cmd = CHILD_PROCESS_INIT;
|
2013-09-15 15:33:20 +00:00
|
|
|
struct string_list_item *item;
|
|
|
|
struct string_list names = STRING_LIST_INIT_DUP;
|
|
|
|
struct string_list rollback = STRING_LIST_INIT_NODUP;
|
|
|
|
struct string_list existing_packs = STRING_LIST_INIT_DUP;
|
|
|
|
struct strbuf line = STRBUF_INIT;
|
2013-12-17 13:43:58 +00:00
|
|
|
int ext, ret, failed;
|
2013-09-15 15:33:20 +00:00
|
|
|
FILE *out;
|
|
|
|
|
|
|
|
/* variables to be filled by option parsing */
|
|
|
|
int pack_everything = 0;
|
|
|
|
int delete_redundant = 0;
|
2014-01-23 01:28:30 +00:00
|
|
|
const char *unpack_unreachable = NULL;
|
repack: propagate pack-objects options as strings
In the original shell version of git-repack, any options
destined for pack-objects were left as strings, and passed
as a whole. Since the C rewrite in commit a1bbc6c (repack:
rewrite the shell script in C, 2013-09-15), we now parse
these values to integers internally, then reformat the
integers when passing the option to pack-objects.
This has the advantage that we catch format errors earlier
(i.e., when repack is invoked, rather than when pack-objects
is invoked).
It has three disadvantages, though:
1. Our internal data types may not be the right size. In
the case of "--window-memory" and "--max-pack-size",
these are "unsigned long" in pack-objects, but we can
only represent a regular "int".
2. Our parsing routines might not be the same as those of
pack-objects. For the two options above, pack-objects
understands "100m" to mean "100 megabytes", but repack
does not.
3. We have to keep a sentinel value to know whether it is
worth passing the option along. In the case of
"--window-memory", we currently do not pass it if the
value is "0". But that is a meaningful value to
pack-objects, where it overrides any configured value.
We can fix all of these by simply passing the strings from
the user along to pack-objects verbatim. This does not
actually fix anything for "--depth" or "--window", but these
are converted, too, for consistency.
Signed-off-by: Jeff King <peff@peff.net>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2014-01-23 01:30:13 +00:00
|
|
|
const char *window = NULL, *window_memory = NULL;
|
|
|
|
const char *depth = NULL;
|
|
|
|
const char *max_pack_size = NULL;
|
2013-09-15 15:33:20 +00:00
|
|
|
int no_reuse_delta = 0, no_reuse_object = 0;
|
|
|
|
int no_update_server_info = 0;
|
|
|
|
int quiet = 0;
|
|
|
|
int local = 0;
|
|
|
|
|
|
|
|
struct option builtin_repack_options[] = {
|
|
|
|
OPT_BIT('a', NULL, &pack_everything,
|
|
|
|
N_("pack everything in a single pack"), ALL_INTO_ONE),
|
|
|
|
OPT_BIT('A', NULL, &pack_everything,
|
|
|
|
N_("same as -a, and turn unreachable objects loose"),
|
|
|
|
LOOSEN_UNREACHABLE | ALL_INTO_ONE),
|
|
|
|
OPT_BOOL('d', NULL, &delete_redundant,
|
|
|
|
N_("remove redundant packs, and run git-prune-packed")),
|
|
|
|
OPT_BOOL('f', NULL, &no_reuse_delta,
|
|
|
|
N_("pass --no-reuse-delta to git-pack-objects")),
|
|
|
|
OPT_BOOL('F', NULL, &no_reuse_object,
|
|
|
|
N_("pass --no-reuse-object to git-pack-objects")),
|
|
|
|
OPT_BOOL('n', NULL, &no_update_server_info,
|
|
|
|
N_("do not run git-update-server-info")),
|
|
|
|
OPT__QUIET(&quiet, N_("be quiet")),
|
|
|
|
OPT_BOOL('l', "local", &local,
|
|
|
|
N_("pass --local to git-pack-objects")),
|
2014-06-10 20:10:07 +00:00
|
|
|
OPT_BOOL('b', "write-bitmap-index", &write_bitmaps,
|
2013-12-21 14:00:31 +00:00
|
|
|
N_("write bitmap index")),
|
2013-09-15 15:33:20 +00:00
|
|
|
OPT_STRING(0, "unpack-unreachable", &unpack_unreachable, N_("approxidate"),
|
|
|
|
N_("with -A, do not loosen objects older than this")),
|
repack: propagate pack-objects options as strings
In the original shell version of git-repack, any options
destined for pack-objects were left as strings, and passed
as a whole. Since the C rewrite in commit a1bbc6c (repack:
rewrite the shell script in C, 2013-09-15), we now parse
these values to integers internally, then reformat the
integers when passing the option to pack-objects.
This has the advantage that we catch format errors earlier
(i.e., when repack is invoked, rather than when pack-objects
is invoked).
It has three disadvantages, though:
1. Our internal data types may not be the right size. In
the case of "--window-memory" and "--max-pack-size",
these are "unsigned long" in pack-objects, but we can
only represent a regular "int".
2. Our parsing routines might not be the same as those of
pack-objects. For the two options above, pack-objects
understands "100m" to mean "100 megabytes", but repack
does not.
3. We have to keep a sentinel value to know whether it is
worth passing the option along. In the case of
"--window-memory", we currently do not pass it if the
value is "0". But that is a meaningful value to
pack-objects, where it overrides any configured value.
We can fix all of these by simply passing the strings from
the user along to pack-objects verbatim. This does not
actually fix anything for "--depth" or "--window", but these
are converted, too, for consistency.
Signed-off-by: Jeff King <peff@peff.net>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2014-01-23 01:30:13 +00:00
|
|
|
OPT_STRING(0, "window", &window, N_("n"),
|
2013-09-15 15:33:20 +00:00
|
|
|
N_("size of the window used for delta compression")),
|
repack: propagate pack-objects options as strings
In the original shell version of git-repack, any options
destined for pack-objects were left as strings, and passed
as a whole. Since the C rewrite in commit a1bbc6c (repack:
rewrite the shell script in C, 2013-09-15), we now parse
these values to integers internally, then reformat the
integers when passing the option to pack-objects.
This has the advantage that we catch format errors earlier
(i.e., when repack is invoked, rather than when pack-objects
is invoked).
It has three disadvantages, though:
1. Our internal data types may not be the right size. In
the case of "--window-memory" and "--max-pack-size",
these are "unsigned long" in pack-objects, but we can
only represent a regular "int".
2. Our parsing routines might not be the same as those of
pack-objects. For the two options above, pack-objects
understands "100m" to mean "100 megabytes", but repack
does not.
3. We have to keep a sentinel value to know whether it is
worth passing the option along. In the case of
"--window-memory", we currently do not pass it if the
value is "0". But that is a meaningful value to
pack-objects, where it overrides any configured value.
We can fix all of these by simply passing the strings from
the user along to pack-objects verbatim. This does not
actually fix anything for "--depth" or "--window", but these
are converted, too, for consistency.
Signed-off-by: Jeff King <peff@peff.net>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2014-01-23 01:30:13 +00:00
|
|
|
OPT_STRING(0, "window-memory", &window_memory, N_("bytes"),
|
2013-09-15 15:33:20 +00:00
|
|
|
N_("same as the above, but limit memory size instead of entries count")),
|
repack: propagate pack-objects options as strings
In the original shell version of git-repack, any options
destined for pack-objects were left as strings, and passed
as a whole. Since the C rewrite in commit a1bbc6c (repack:
rewrite the shell script in C, 2013-09-15), we now parse
these values to integers internally, then reformat the
integers when passing the option to pack-objects.
This has the advantage that we catch format errors earlier
(i.e., when repack is invoked, rather than when pack-objects
is invoked).
It has three disadvantages, though:
1. Our internal data types may not be the right size. In
the case of "--window-memory" and "--max-pack-size",
these are "unsigned long" in pack-objects, but we can
only represent a regular "int".
2. Our parsing routines might not be the same as those of
pack-objects. For the two options above, pack-objects
understands "100m" to mean "100 megabytes", but repack
does not.
3. We have to keep a sentinel value to know whether it is
worth passing the option along. In the case of
"--window-memory", we currently do not pass it if the
value is "0". But that is a meaningful value to
pack-objects, where it overrides any configured value.
We can fix all of these by simply passing the strings from
the user along to pack-objects verbatim. This does not
actually fix anything for "--depth" or "--window", but these
are converted, too, for consistency.
Signed-off-by: Jeff King <peff@peff.net>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2014-01-23 01:30:13 +00:00
|
|
|
OPT_STRING(0, "depth", &depth, N_("n"),
|
2013-09-15 15:33:20 +00:00
|
|
|
N_("limits the maximum delta depth")),
|
repack: propagate pack-objects options as strings
In the original shell version of git-repack, any options
destined for pack-objects were left as strings, and passed
as a whole. Since the C rewrite in commit a1bbc6c (repack:
rewrite the shell script in C, 2013-09-15), we now parse
these values to integers internally, then reformat the
integers when passing the option to pack-objects.
This has the advantage that we catch format errors earlier
(i.e., when repack is invoked, rather than when pack-objects
is invoked).
It has three disadvantages, though:
1. Our internal data types may not be the right size. In
the case of "--window-memory" and "--max-pack-size",
these are "unsigned long" in pack-objects, but we can
only represent a regular "int".
2. Our parsing routines might not be the same as those of
pack-objects. For the two options above, pack-objects
understands "100m" to mean "100 megabytes", but repack
does not.
3. We have to keep a sentinel value to know whether it is
worth passing the option along. In the case of
"--window-memory", we currently do not pass it if the
value is "0". But that is a meaningful value to
pack-objects, where it overrides any configured value.
We can fix all of these by simply passing the strings from
the user along to pack-objects verbatim. This does not
actually fix anything for "--depth" or "--window", but these
are converted, too, for consistency.
Signed-off-by: Jeff King <peff@peff.net>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2014-01-23 01:30:13 +00:00
|
|
|
OPT_STRING(0, "max-pack-size", &max_pack_size, N_("bytes"),
|
2013-09-15 15:33:20 +00:00
|
|
|
N_("maximum size of each packfile")),
|
repack: add `repack.packKeptObjects` config var
The git-repack command always passes `--honor-pack-keep`
to pack-objects. This has traditionally been a good thing,
as we do not want to duplicate those objects in a new pack,
and we are not going to delete the old pack.
However, when bitmaps are in use, it is important for a full
repack to include all reachable objects, even if they may be
duplicated in a .keep pack. Otherwise, we cannot generate
the bitmaps, as the on-disk format requires the set of
objects in the pack to be fully closed.
Even if the repository does not generally have .keep files,
a simultaneous push could cause a race condition in which a
.keep file exists at the moment of a repack. The repack may
try to include those objects in one of two situations:
1. The pushed .keep pack contains objects that were
already in the repository (e.g., blobs due to a revert of
an old commit).
2. Receive-pack updates the refs, making the objects
reachable, but before it removes the .keep file, the
repack runs.
In either case, we may prefer to duplicate some objects in
the new, full pack, and let the next repack (after the .keep
file is cleaned up) take care of removing them.
This patch introduces both a command-line and config option
to disable the `--honor-pack-keep` option. By default, it
is triggered when pack.writeBitmaps (or `--write-bitmap-index`
is turned on), but specifying it explicitly can override the
behavior (e.g., in cases where you prefer .keep files to
bitmaps, but only when they are present).
Note that this option just disables the pack-objects
behavior. We still leave packs with a .keep in place, as we
do not necessarily know that we have duplicated all of their
objects.
Signed-off-by: Jeff King <peff@peff.net>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2014-03-03 20:04:20 +00:00
|
|
|
OPT_BOOL(0, "pack-kept-objects", &pack_kept_objects,
|
|
|
|
N_("repack objects in packs marked with .keep")),
|
2013-09-15 15:33:20 +00:00
|
|
|
OPT_END()
|
|
|
|
};
|
|
|
|
|
|
|
|
git_config(repack_config, NULL);
|
|
|
|
|
|
|
|
argc = parse_options(argc, argv, prefix, builtin_repack_options,
|
|
|
|
git_repack_usage, 0);
|
|
|
|
|
repack: add `repack.packKeptObjects` config var
The git-repack command always passes `--honor-pack-keep`
to pack-objects. This has traditionally been a good thing,
as we do not want to duplicate those objects in a new pack,
and we are not going to delete the old pack.
However, when bitmaps are in use, it is important for a full
repack to include all reachable objects, even if they may be
duplicated in a .keep pack. Otherwise, we cannot generate
the bitmaps, as the on-disk format requires the set of
objects in the pack to be fully closed.
Even if the repository does not generally have .keep files,
a simultaneous push could cause a race condition in which a
.keep file exists at the moment of a repack. The repack may
try to include those objects in one of two situations:
1. The pushed .keep pack contains objects that were
already in the repository (e.g., blobs due to a revert of
an old commit).
2. Receive-pack updates the refs, making the objects
reachable, but before it removes the .keep file, the
repack runs.
In either case, we may prefer to duplicate some objects in
the new, full pack, and let the next repack (after the .keep
file is cleaned up) take care of removing them.
This patch introduces both a command-line and config option
to disable the `--honor-pack-keep` option. By default, it
is triggered when pack.writeBitmaps (or `--write-bitmap-index`
is turned on), but specifying it explicitly can override the
behavior (e.g., in cases where you prefer .keep files to
bitmaps, but only when they are present).
Note that this option just disables the pack-objects
behavior. We still leave packs with a .keep in place, as we
do not necessarily know that we have duplicated all of their
objects.
Signed-off-by: Jeff King <peff@peff.net>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2014-03-03 20:04:20 +00:00
|
|
|
if (pack_kept_objects < 0)
|
2014-06-10 20:19:38 +00:00
|
|
|
pack_kept_objects = write_bitmaps;
|
repack: add `repack.packKeptObjects` config var
The git-repack command always passes `--honor-pack-keep`
to pack-objects. This has traditionally been a good thing,
as we do not want to duplicate those objects in a new pack,
and we are not going to delete the old pack.
However, when bitmaps are in use, it is important for a full
repack to include all reachable objects, even if they may be
duplicated in a .keep pack. Otherwise, we cannot generate
the bitmaps, as the on-disk format requires the set of
objects in the pack to be fully closed.
Even if the repository does not generally have .keep files,
a simultaneous push could cause a race condition in which a
.keep file exists at the moment of a repack. The repack may
try to include those objects in one of two situations:
1. The pushed .keep pack contains objects that were
already in the repository (e.g., blobs due to a revert of
an old commit).
2. Receive-pack updates the refs, making the objects
reachable, but before it removes the .keep file, the
repack runs.
In either case, we may prefer to duplicate some objects in
the new, full pack, and let the next repack (after the .keep
file is cleaned up) take care of removing them.
This patch introduces both a command-line and config option
to disable the `--honor-pack-keep` option. By default, it
is triggered when pack.writeBitmaps (or `--write-bitmap-index`
is turned on), but specifying it explicitly can override the
behavior (e.g., in cases where you prefer .keep files to
bitmaps, but only when they are present).
Note that this option just disables the pack-objects
behavior. We still leave packs with a .keep in place, as we
do not necessarily know that we have duplicated all of their
objects.
Signed-off-by: Jeff King <peff@peff.net>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2014-03-03 20:04:20 +00:00
|
|
|
|
2013-09-15 15:33:20 +00:00
|
|
|
packdir = mkpathdup("%s/pack", get_object_directory());
|
|
|
|
packtmp = mkpathdup("%s/.tmp-%d-pack", packdir, (int)getpid());
|
|
|
|
|
|
|
|
sigchain_push_common(remove_pack_on_signal);
|
|
|
|
|
2014-11-09 13:49:54 +00:00
|
|
|
argv_array_push(&cmd.args, "pack-objects");
|
|
|
|
argv_array_push(&cmd.args, "--keep-true-parents");
|
repack: add `repack.packKeptObjects` config var
The git-repack command always passes `--honor-pack-keep`
to pack-objects. This has traditionally been a good thing,
as we do not want to duplicate those objects in a new pack,
and we are not going to delete the old pack.
However, when bitmaps are in use, it is important for a full
repack to include all reachable objects, even if they may be
duplicated in a .keep pack. Otherwise, we cannot generate
the bitmaps, as the on-disk format requires the set of
objects in the pack to be fully closed.
Even if the repository does not generally have .keep files,
a simultaneous push could cause a race condition in which a
.keep file exists at the moment of a repack. The repack may
try to include those objects in one of two situations:
1. The pushed .keep pack contains objects that were
already in the repository (e.g., blobs due to a revert of
an old commit).
2. Receive-pack updates the refs, making the objects
reachable, but before it removes the .keep file, the
repack runs.
In either case, we may prefer to duplicate some objects in
the new, full pack, and let the next repack (after the .keep
file is cleaned up) take care of removing them.
This patch introduces both a command-line and config option
to disable the `--honor-pack-keep` option. By default, it
is triggered when pack.writeBitmaps (or `--write-bitmap-index`
is turned on), but specifying it explicitly can override the
behavior (e.g., in cases where you prefer .keep files to
bitmaps, but only when they are present).
Note that this option just disables the pack-objects
behavior. We still leave packs with a .keep in place, as we
do not necessarily know that we have duplicated all of their
objects.
Signed-off-by: Jeff King <peff@peff.net>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2014-03-03 20:04:20 +00:00
|
|
|
if (!pack_kept_objects)
|
2014-11-09 13:49:54 +00:00
|
|
|
argv_array_push(&cmd.args, "--honor-pack-keep");
|
|
|
|
argv_array_push(&cmd.args, "--non-empty");
|
|
|
|
argv_array_push(&cmd.args, "--all");
|
|
|
|
argv_array_push(&cmd.args, "--reflog");
|
|
|
|
argv_array_push(&cmd.args, "--indexed-objects");
|
2013-09-15 15:33:20 +00:00
|
|
|
if (window)
|
2014-11-09 13:49:54 +00:00
|
|
|
argv_array_pushf(&cmd.args, "--window=%s", window);
|
2013-09-15 15:33:20 +00:00
|
|
|
if (window_memory)
|
2014-11-09 13:49:54 +00:00
|
|
|
argv_array_pushf(&cmd.args, "--window-memory=%s", window_memory);
|
2013-09-15 15:33:20 +00:00
|
|
|
if (depth)
|
2014-11-09 13:49:54 +00:00
|
|
|
argv_array_pushf(&cmd.args, "--depth=%s", depth);
|
2013-09-15 15:33:20 +00:00
|
|
|
if (max_pack_size)
|
2014-11-09 13:49:54 +00:00
|
|
|
argv_array_pushf(&cmd.args, "--max-pack-size=%s", max_pack_size);
|
2013-09-15 15:33:20 +00:00
|
|
|
if (no_reuse_delta)
|
2014-11-09 13:49:54 +00:00
|
|
|
argv_array_pushf(&cmd.args, "--no-reuse-delta");
|
2013-09-15 15:33:20 +00:00
|
|
|
if (no_reuse_object)
|
2014-11-09 13:49:54 +00:00
|
|
|
argv_array_pushf(&cmd.args, "--no-reuse-object");
|
2014-06-10 20:19:38 +00:00
|
|
|
if (write_bitmaps)
|
2014-11-09 13:49:54 +00:00
|
|
|
argv_array_push(&cmd.args, "--write-bitmap-index");
|
2013-09-15 15:33:20 +00:00
|
|
|
|
|
|
|
if (pack_everything & ALL_INTO_ONE) {
|
|
|
|
get_non_kept_pack_filenames(&existing_packs);
|
|
|
|
|
|
|
|
if (existing_packs.nr && delete_redundant) {
|
2015-03-20 18:43:13 +00:00
|
|
|
if (unpack_unreachable) {
|
2014-11-09 13:49:54 +00:00
|
|
|
argv_array_pushf(&cmd.args,
|
2013-09-15 15:33:20 +00:00
|
|
|
"--unpack-unreachable=%s",
|
|
|
|
unpack_unreachable);
|
2015-03-20 18:43:13 +00:00
|
|
|
argv_array_push(&cmd.env_array, "GIT_REF_PARANOIA=1");
|
|
|
|
} else if (pack_everything & LOOSEN_UNREACHABLE) {
|
2014-11-09 13:49:54 +00:00
|
|
|
argv_array_push(&cmd.args,
|
2013-09-15 15:33:20 +00:00
|
|
|
"--unpack-unreachable");
|
2015-03-20 18:43:13 +00:00
|
|
|
} else {
|
|
|
|
argv_array_push(&cmd.env_array, "GIT_REF_PARANOIA=1");
|
|
|
|
}
|
2013-09-15 15:33:20 +00:00
|
|
|
}
|
|
|
|
} else {
|
2014-11-09 13:49:54 +00:00
|
|
|
argv_array_push(&cmd.args, "--unpacked");
|
|
|
|
argv_array_push(&cmd.args, "--incremental");
|
2013-09-15 15:33:20 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
if (local)
|
2014-11-09 13:49:54 +00:00
|
|
|
argv_array_push(&cmd.args, "--local");
|
2013-09-15 15:33:20 +00:00
|
|
|
if (quiet)
|
2014-11-09 13:49:54 +00:00
|
|
|
argv_array_push(&cmd.args, "--quiet");
|
2013-09-15 15:33:20 +00:00
|
|
|
if (delta_base_offset)
|
2014-11-09 13:49:54 +00:00
|
|
|
argv_array_push(&cmd.args, "--delta-base-offset");
|
2013-09-15 15:33:20 +00:00
|
|
|
|
2014-11-09 13:49:54 +00:00
|
|
|
argv_array_push(&cmd.args, packtmp);
|
2013-09-15 15:33:20 +00:00
|
|
|
|
|
|
|
cmd.git_cmd = 1;
|
|
|
|
cmd.out = -1;
|
|
|
|
cmd.no_stdin = 1;
|
|
|
|
|
|
|
|
ret = start_command(&cmd);
|
|
|
|
if (ret)
|
2013-09-15 15:33:21 +00:00
|
|
|
return ret;
|
2013-09-15 15:33:20 +00:00
|
|
|
|
|
|
|
out = xfdopen(cmd.out, "r");
|
|
|
|
while (strbuf_getline(&line, out, '\n') != EOF) {
|
|
|
|
if (line.len != 40)
|
|
|
|
die("repack: Expecting 40 character sha1 lines only from pack-objects.");
|
|
|
|
string_list_append(&names, line.buf);
|
|
|
|
}
|
|
|
|
fclose(out);
|
|
|
|
ret = finish_command(&cmd);
|
|
|
|
if (ret)
|
2013-09-15 15:33:21 +00:00
|
|
|
return ret;
|
2013-09-15 15:33:20 +00:00
|
|
|
|
2013-12-17 13:43:58 +00:00
|
|
|
if (!names.nr && !quiet)
|
2013-09-15 15:33:20 +00:00
|
|
|
printf("Nothing new to pack.\n");
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Ok we have prepared all new packfiles.
|
|
|
|
* First see if there are packs of the same name and if so
|
|
|
|
* if we can move them out of the way (this can happen if we
|
|
|
|
* repacked immediately after packing fully.
|
|
|
|
*/
|
|
|
|
failed = 0;
|
|
|
|
for_each_string_list_item(item, &names) {
|
2013-12-21 14:00:19 +00:00
|
|
|
for (ext = 0; ext < ARRAY_SIZE(exts); ext++) {
|
2015-08-10 09:35:38 +00:00
|
|
|
char *fname, *fname_old;
|
2014-02-02 15:09:56 +00:00
|
|
|
fname = mkpathdup("%s/pack-%s%s", packdir,
|
2013-12-21 14:00:23 +00:00
|
|
|
item->string, exts[ext].name);
|
2013-09-15 15:33:20 +00:00
|
|
|
if (!file_exists(fname)) {
|
|
|
|
free(fname);
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
|
2015-08-10 09:35:38 +00:00
|
|
|
fname_old = mkpathdup("%s/old-%s%s", packdir,
|
2013-12-21 14:00:23 +00:00
|
|
|
item->string, exts[ext].name);
|
2013-09-15 15:33:20 +00:00
|
|
|
if (file_exists(fname_old))
|
|
|
|
if (unlink(fname_old))
|
|
|
|
failed = 1;
|
|
|
|
|
|
|
|
if (!failed && rename(fname, fname_old)) {
|
|
|
|
free(fname);
|
2015-08-10 09:35:38 +00:00
|
|
|
free(fname_old);
|
2013-09-15 15:33:20 +00:00
|
|
|
failed = 1;
|
|
|
|
break;
|
|
|
|
} else {
|
|
|
|
string_list_append(&rollback, fname);
|
2015-08-10 09:35:38 +00:00
|
|
|
free(fname_old);
|
2013-09-15 15:33:20 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
if (failed)
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
if (failed) {
|
|
|
|
struct string_list rollback_failure = STRING_LIST_INIT_DUP;
|
|
|
|
for_each_string_list_item(item, &rollback) {
|
2015-08-10 09:35:38 +00:00
|
|
|
char *fname, *fname_old;
|
2013-09-15 15:33:20 +00:00
|
|
|
fname = mkpathdup("%s/%s", packdir, item->string);
|
2015-08-10 09:35:38 +00:00
|
|
|
fname_old = mkpathdup("%s/old-%s", packdir, item->string);
|
2013-09-15 15:33:20 +00:00
|
|
|
if (rename(fname_old, fname))
|
|
|
|
string_list_append(&rollback_failure, fname);
|
|
|
|
free(fname);
|
2015-08-10 09:35:38 +00:00
|
|
|
free(fname_old);
|
2013-09-15 15:33:20 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
if (rollback_failure.nr) {
|
|
|
|
int i;
|
|
|
|
fprintf(stderr,
|
|
|
|
"WARNING: Some packs in use have been renamed by\n"
|
|
|
|
"WARNING: prefixing old- to their name, in order to\n"
|
|
|
|
"WARNING: replace them with the new version of the\n"
|
|
|
|
"WARNING: file. But the operation failed, and the\n"
|
|
|
|
"WARNING: attempt to rename them back to their\n"
|
|
|
|
"WARNING: original names also failed.\n"
|
|
|
|
"WARNING: Please rename them in %s manually:\n", packdir);
|
|
|
|
for (i = 0; i < rollback_failure.nr; i++)
|
|
|
|
fprintf(stderr, "WARNING: old-%s -> %s\n",
|
|
|
|
rollback_failure.items[i].string,
|
|
|
|
rollback_failure.items[i].string);
|
|
|
|
}
|
|
|
|
exit(1);
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Now the ones with the same name are out of the way... */
|
|
|
|
for_each_string_list_item(item, &names) {
|
2013-12-21 14:00:19 +00:00
|
|
|
for (ext = 0; ext < ARRAY_SIZE(exts); ext++) {
|
2013-09-15 15:33:20 +00:00
|
|
|
char *fname, *fname_old;
|
|
|
|
struct stat statbuffer;
|
2013-12-21 14:00:27 +00:00
|
|
|
int exists = 0;
|
2013-09-15 15:33:20 +00:00
|
|
|
fname = mkpathdup("%s/pack-%s%s",
|
2013-12-21 14:00:23 +00:00
|
|
|
packdir, item->string, exts[ext].name);
|
2013-09-15 15:33:20 +00:00
|
|
|
fname_old = mkpathdup("%s-%s%s",
|
2013-12-21 14:00:23 +00:00
|
|
|
packtmp, item->string, exts[ext].name);
|
2013-09-15 15:33:20 +00:00
|
|
|
if (!stat(fname_old, &statbuffer)) {
|
|
|
|
statbuffer.st_mode &= ~(S_IWUSR | S_IWGRP | S_IWOTH);
|
|
|
|
chmod(fname_old, statbuffer.st_mode);
|
2013-12-21 14:00:27 +00:00
|
|
|
exists = 1;
|
|
|
|
}
|
|
|
|
if (exists || !exts[ext].optional) {
|
|
|
|
if (rename(fname_old, fname))
|
|
|
|
die_errno(_("renaming '%s' failed"), fname_old);
|
2013-09-15 15:33:20 +00:00
|
|
|
}
|
|
|
|
free(fname);
|
|
|
|
free(fname_old);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Remove the "old-" files */
|
|
|
|
for_each_string_list_item(item, &names) {
|
2013-12-21 14:00:19 +00:00
|
|
|
for (ext = 0; ext < ARRAY_SIZE(exts); ext++) {
|
2015-08-10 09:35:38 +00:00
|
|
|
char *fname;
|
|
|
|
fname = mkpathdup("%s/old-%s%s",
|
|
|
|
packdir,
|
|
|
|
item->string,
|
|
|
|
exts[ext].name);
|
2013-09-15 15:33:22 +00:00
|
|
|
if (remove_path(fname))
|
|
|
|
warning(_("removing '%s' failed"), fname);
|
2015-08-10 09:35:38 +00:00
|
|
|
free(fname);
|
2013-09-15 15:33:20 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
/* End of pack replacement. */
|
|
|
|
|
|
|
|
if (delete_redundant) {
|
2014-09-13 07:28:01 +00:00
|
|
|
int opts = 0;
|
2014-11-25 08:02:35 +00:00
|
|
|
string_list_sort(&names);
|
2013-09-15 15:33:20 +00:00
|
|
|
for_each_string_list_item(item, &existing_packs) {
|
|
|
|
char *sha1;
|
|
|
|
size_t len = strlen(item->string);
|
|
|
|
if (len < 40)
|
|
|
|
continue;
|
|
|
|
sha1 = item->string + len - 40;
|
|
|
|
if (!string_list_has_string(&names, sha1))
|
|
|
|
remove_redundant_pack(packdir, item->string);
|
|
|
|
}
|
2014-09-13 07:28:01 +00:00
|
|
|
if (!quiet && isatty(2))
|
|
|
|
opts |= PRUNE_PACKED_VERBOSE;
|
|
|
|
prune_packed_objects(opts);
|
2013-09-15 15:33:20 +00:00
|
|
|
}
|
|
|
|
|
2014-09-13 07:28:01 +00:00
|
|
|
if (!no_update_server_info)
|
|
|
|
update_server_info(0);
|
2013-09-15 15:33:20 +00:00
|
|
|
remove_temporary_files();
|
|
|
|
string_list_clear(&names, 0);
|
|
|
|
string_list_clear(&rollback, 0);
|
|
|
|
string_list_clear(&existing_packs, 0);
|
|
|
|
strbuf_release(&line);
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|