2005-04-18 20:04:43 +00:00
|
|
|
/*
|
|
|
|
* GIT - The information manager from hell
|
|
|
|
*
|
|
|
|
* Copyright (C) Linus Torvalds, 2005
|
|
|
|
*
|
|
|
|
* This handles basic git sha1 object files - packing, unpacking,
|
|
|
|
* creation etc.
|
|
|
|
*/
|
|
|
|
#include "cache.h"
|
2005-06-27 10:35:33 +00:00
|
|
|
#include "delta.h"
|
2005-06-28 21:21:02 +00:00
|
|
|
#include "pack.h"
|
2006-04-02 12:44:09 +00:00
|
|
|
#include "blob.h"
|
|
|
|
#include "commit.h"
|
|
|
|
#include "tag.h"
|
|
|
|
#include "tree.h"
|
2007-04-10 04:20:29 +00:00
|
|
|
#include "refs.h"
|
2005-04-18 20:04:43 +00:00
|
|
|
|
2005-04-23 18:09:32 +00:00
|
|
|
#ifndef O_NOATIME
|
|
|
|
#if defined(__linux__) && (defined(__i386__) || defined(__PPC__))
|
|
|
|
#define O_NOATIME 01000000
|
|
|
|
#else
|
|
|
|
#define O_NOATIME 0
|
|
|
|
#endif
|
|
|
|
#endif
|
|
|
|
|
2007-01-10 04:07:11 +00:00
|
|
|
#ifdef NO_C99_FORMAT
|
|
|
|
#define SZ_FMT "lu"
|
2007-11-21 20:27:19 +00:00
|
|
|
static unsigned long sz_fmt(size_t s) { return (unsigned long)s; }
|
2007-01-10 04:07:11 +00:00
|
|
|
#else
|
|
|
|
#define SZ_FMT "zu"
|
2007-11-21 20:27:19 +00:00
|
|
|
static size_t sz_fmt(size_t s) { return s; }
|
2007-01-10 04:07:11 +00:00
|
|
|
#endif
|
|
|
|
|
2006-08-15 17:23:48 +00:00
|
|
|
const unsigned char null_sha1[20];
|
2005-09-30 21:02:47 +00:00
|
|
|
|
2005-04-23 18:09:32 +00:00
|
|
|
static unsigned int sha1_file_open_flag = O_NOATIME;
|
|
|
|
|
2007-05-30 17:32:19 +00:00
|
|
|
const signed char hexval_table[256] = {
|
2006-09-20 23:04:46 +00:00
|
|
|
-1, -1, -1, -1, -1, -1, -1, -1, /* 00-07 */
|
|
|
|
-1, -1, -1, -1, -1, -1, -1, -1, /* 08-0f */
|
|
|
|
-1, -1, -1, -1, -1, -1, -1, -1, /* 10-17 */
|
|
|
|
-1, -1, -1, -1, -1, -1, -1, -1, /* 18-1f */
|
|
|
|
-1, -1, -1, -1, -1, -1, -1, -1, /* 20-27 */
|
|
|
|
-1, -1, -1, -1, -1, -1, -1, -1, /* 28-2f */
|
|
|
|
0, 1, 2, 3, 4, 5, 6, 7, /* 30-37 */
|
|
|
|
8, 9, -1, -1, -1, -1, -1, -1, /* 38-3f */
|
|
|
|
-1, 10, 11, 12, 13, 14, 15, -1, /* 40-47 */
|
|
|
|
-1, -1, -1, -1, -1, -1, -1, -1, /* 48-4f */
|
|
|
|
-1, -1, -1, -1, -1, -1, -1, -1, /* 50-57 */
|
|
|
|
-1, -1, -1, -1, -1, -1, -1, -1, /* 58-5f */
|
|
|
|
-1, 10, 11, 12, 13, 14, 15, -1, /* 60-67 */
|
|
|
|
-1, -1, -1, -1, -1, -1, -1, -1, /* 68-67 */
|
|
|
|
-1, -1, -1, -1, -1, -1, -1, -1, /* 70-77 */
|
|
|
|
-1, -1, -1, -1, -1, -1, -1, -1, /* 78-7f */
|
|
|
|
-1, -1, -1, -1, -1, -1, -1, -1, /* 80-87 */
|
|
|
|
-1, -1, -1, -1, -1, -1, -1, -1, /* 88-8f */
|
|
|
|
-1, -1, -1, -1, -1, -1, -1, -1, /* 90-97 */
|
|
|
|
-1, -1, -1, -1, -1, -1, -1, -1, /* 98-9f */
|
|
|
|
-1, -1, -1, -1, -1, -1, -1, -1, /* a0-a7 */
|
|
|
|
-1, -1, -1, -1, -1, -1, -1, -1, /* a8-af */
|
|
|
|
-1, -1, -1, -1, -1, -1, -1, -1, /* b0-b7 */
|
|
|
|
-1, -1, -1, -1, -1, -1, -1, -1, /* b8-bf */
|
|
|
|
-1, -1, -1, -1, -1, -1, -1, -1, /* c0-c7 */
|
|
|
|
-1, -1, -1, -1, -1, -1, -1, -1, /* c8-cf */
|
|
|
|
-1, -1, -1, -1, -1, -1, -1, -1, /* d0-d7 */
|
|
|
|
-1, -1, -1, -1, -1, -1, -1, -1, /* d8-df */
|
|
|
|
-1, -1, -1, -1, -1, -1, -1, -1, /* e0-e7 */
|
|
|
|
-1, -1, -1, -1, -1, -1, -1, -1, /* e8-ef */
|
|
|
|
-1, -1, -1, -1, -1, -1, -1, -1, /* f0-f7 */
|
|
|
|
-1, -1, -1, -1, -1, -1, -1, -1, /* f8-ff */
|
|
|
|
};
|
2005-04-18 20:04:43 +00:00
|
|
|
|
|
|
|
int get_sha1_hex(const char *hex, unsigned char *sha1)
|
|
|
|
{
|
|
|
|
int i;
|
|
|
|
for (i = 0; i < 20; i++) {
|
|
|
|
unsigned int val = (hexval(hex[0]) << 4) | hexval(hex[1]);
|
|
|
|
if (val & ~0xff)
|
|
|
|
return -1;
|
|
|
|
*sha1++ = val;
|
|
|
|
hex += 2;
|
|
|
|
}
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2005-07-06 08:11:52 +00:00
|
|
|
int safe_create_leading_directories(char *path)
|
|
|
|
{
|
|
|
|
char *pos = path;
|
2006-02-10 01:56:13 +00:00
|
|
|
struct stat st;
|
|
|
|
|
2007-11-13 20:05:00 +00:00
|
|
|
if (is_absolute_path(path))
|
2005-11-06 23:36:15 +00:00
|
|
|
pos++;
|
2005-07-06 08:11:52 +00:00
|
|
|
|
|
|
|
while (pos) {
|
|
|
|
pos = strchr(pos, '/');
|
|
|
|
if (!pos)
|
|
|
|
break;
|
|
|
|
*pos = 0;
|
2006-02-10 01:56:13 +00:00
|
|
|
if (!stat(path, &st)) {
|
|
|
|
/* path exists */
|
|
|
|
if (!S_ISDIR(st.st_mode)) {
|
2005-07-06 08:11:52 +00:00
|
|
|
*pos = '/';
|
2006-02-10 01:56:13 +00:00
|
|
|
return -3;
|
2005-07-06 08:11:52 +00:00
|
|
|
}
|
2005-12-22 22:13:56 +00:00
|
|
|
}
|
2006-02-10 01:56:13 +00:00
|
|
|
else if (mkdir(path, 0777)) {
|
|
|
|
*pos = '/';
|
|
|
|
return -1;
|
|
|
|
}
|
2005-12-22 22:13:56 +00:00
|
|
|
else if (adjust_shared_perm(path)) {
|
|
|
|
*pos = '/';
|
|
|
|
return -2;
|
|
|
|
}
|
2005-07-06 08:11:52 +00:00
|
|
|
*pos++ = '/';
|
|
|
|
}
|
|
|
|
return 0;
|
|
|
|
}
|
2005-07-05 18:31:32 +00:00
|
|
|
|
2005-04-18 20:04:43 +00:00
|
|
|
char * sha1_to_hex(const unsigned char *sha1)
|
|
|
|
{
|
2006-05-04 00:21:08 +00:00
|
|
|
static int bufno;
|
|
|
|
static char hexbuffer[4][50];
|
2005-04-18 20:04:43 +00:00
|
|
|
static const char hex[] = "0123456789abcdef";
|
2006-05-04 00:21:08 +00:00
|
|
|
char *buffer = hexbuffer[3 & ++bufno], *buf = buffer;
|
2005-04-18 20:04:43 +00:00
|
|
|
int i;
|
|
|
|
|
|
|
|
for (i = 0; i < 20; i++) {
|
|
|
|
unsigned int val = *sha1++;
|
|
|
|
*buf++ = hex[val >> 4];
|
|
|
|
*buf++ = hex[val & 0xf];
|
|
|
|
}
|
2005-12-22 17:55:59 +00:00
|
|
|
*buf = '\0';
|
|
|
|
|
2005-04-18 20:04:43 +00:00
|
|
|
return buffer;
|
|
|
|
}
|
|
|
|
|
2005-05-07 07:38:04 +00:00
|
|
|
static void fill_sha1_path(char *pathbuf, const unsigned char *sha1)
|
|
|
|
{
|
|
|
|
int i;
|
|
|
|
for (i = 0; i < 20; i++) {
|
|
|
|
static char hex[] = "0123456789abcdef";
|
|
|
|
unsigned int val = sha1[i];
|
|
|
|
char *pos = pathbuf + i*2 + (i > 0);
|
|
|
|
*pos++ = hex[val >> 4];
|
|
|
|
*pos = hex[val & 0xf];
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2005-04-18 20:04:43 +00:00
|
|
|
/*
|
|
|
|
* NOTE! This returns a statically allocated buffer, so you have to be
|
2008-01-03 14:18:07 +00:00
|
|
|
* careful about using it. Do an "xstrdup()" if you need to save the
|
2005-04-18 20:04:43 +00:00
|
|
|
* filename.
|
2005-05-07 07:38:04 +00:00
|
|
|
*
|
|
|
|
* Also note that this returns the location for creating. Reading
|
|
|
|
* SHA1 file can happen from any alternate directory listed in the
|
2005-05-10 00:57:56 +00:00
|
|
|
* DB_ENVIRONMENT environment variable if it is not found in
|
2005-05-07 07:38:04 +00:00
|
|
|
* the primary object database.
|
2005-04-18 20:04:43 +00:00
|
|
|
*/
|
|
|
|
char *sha1_file_name(const unsigned char *sha1)
|
|
|
|
{
|
|
|
|
static char *name, *base;
|
|
|
|
|
|
|
|
if (!base) {
|
2005-05-10 00:57:56 +00:00
|
|
|
const char *sha1_file_directory = get_object_directory();
|
2005-04-18 20:04:43 +00:00
|
|
|
int len = strlen(sha1_file_directory);
|
2005-04-26 19:00:58 +00:00
|
|
|
base = xmalloc(len + 60);
|
2005-04-18 20:04:43 +00:00
|
|
|
memcpy(base, sha1_file_directory, len);
|
|
|
|
memset(base+len, 0, 60);
|
|
|
|
base[len] = '/';
|
|
|
|
base[len+3] = '/';
|
|
|
|
name = base + len + 1;
|
|
|
|
}
|
2005-05-07 07:38:04 +00:00
|
|
|
fill_sha1_path(name, sha1);
|
2005-04-18 20:04:43 +00:00
|
|
|
return base;
|
|
|
|
}
|
|
|
|
|
2005-08-01 00:53:44 +00:00
|
|
|
char *sha1_pack_name(const unsigned char *sha1)
|
|
|
|
{
|
|
|
|
static const char hex[] = "0123456789abcdef";
|
|
|
|
static char *name, *base, *buf;
|
|
|
|
int i;
|
|
|
|
|
|
|
|
if (!base) {
|
|
|
|
const char *sha1_file_directory = get_object_directory();
|
|
|
|
int len = strlen(sha1_file_directory);
|
|
|
|
base = xmalloc(len + 60);
|
|
|
|
sprintf(base, "%s/pack/pack-1234567890123456789012345678901234567890.pack", sha1_file_directory);
|
|
|
|
name = base + len + 11;
|
|
|
|
}
|
|
|
|
|
|
|
|
buf = name;
|
|
|
|
|
|
|
|
for (i = 0; i < 20; i++) {
|
|
|
|
unsigned int val = *sha1++;
|
|
|
|
*buf++ = hex[val >> 4];
|
|
|
|
*buf++ = hex[val & 0xf];
|
|
|
|
}
|
2007-06-07 07:04:01 +00:00
|
|
|
|
2005-08-01 00:53:44 +00:00
|
|
|
return base;
|
|
|
|
}
|
|
|
|
|
|
|
|
char *sha1_pack_index_name(const unsigned char *sha1)
|
|
|
|
{
|
|
|
|
static const char hex[] = "0123456789abcdef";
|
|
|
|
static char *name, *base, *buf;
|
|
|
|
int i;
|
|
|
|
|
|
|
|
if (!base) {
|
|
|
|
const char *sha1_file_directory = get_object_directory();
|
|
|
|
int len = strlen(sha1_file_directory);
|
|
|
|
base = xmalloc(len + 60);
|
|
|
|
sprintf(base, "%s/pack/pack-1234567890123456789012345678901234567890.idx", sha1_file_directory);
|
|
|
|
name = base + len + 11;
|
|
|
|
}
|
|
|
|
|
|
|
|
buf = name;
|
|
|
|
|
|
|
|
for (i = 0; i < 20; i++) {
|
|
|
|
unsigned int val = *sha1++;
|
|
|
|
*buf++ = hex[val >> 4];
|
|
|
|
*buf++ = hex[val & 0xf];
|
|
|
|
}
|
2007-06-07 07:04:01 +00:00
|
|
|
|
2005-08-01 00:53:44 +00:00
|
|
|
return base;
|
|
|
|
}
|
|
|
|
|
2005-08-15 00:25:57 +00:00
|
|
|
struct alternate_object_database *alt_odb_list;
|
|
|
|
static struct alternate_object_database **alt_odb_tail;
|
2005-05-07 07:38:04 +00:00
|
|
|
|
2006-05-07 18:19:21 +00:00
|
|
|
static void read_info_alternates(const char * alternates, int depth);
|
|
|
|
|
2005-05-08 20:51:13 +00:00
|
|
|
/*
|
|
|
|
* Prepare alternate object database registry.
|
2005-08-15 00:25:57 +00:00
|
|
|
*
|
|
|
|
* The variable alt_odb_list points at the list of struct
|
|
|
|
* alternate_object_database. The elements on this list come from
|
|
|
|
* non-empty elements from colon separated ALTERNATE_DB_ENVIRONMENT
|
|
|
|
* environment variable, and $GIT_OBJECT_DIRECTORY/info/alternates,
|
2005-12-05 06:48:43 +00:00
|
|
|
* whose contents is similar to that environment variable but can be
|
|
|
|
* LF separated. Its base points at a statically allocated buffer that
|
2005-08-15 00:25:57 +00:00
|
|
|
* contains "/the/directory/corresponding/to/.git/objects/...", while
|
|
|
|
* its name points just after the slash at the end of ".git/objects/"
|
|
|
|
* in the example above, and has enough space to hold 40-byte hex
|
|
|
|
* SHA1, an extra slash for the first level indirection, and the
|
|
|
|
* terminating NUL.
|
2005-05-08 20:51:13 +00:00
|
|
|
*/
|
2006-05-07 18:19:21 +00:00
|
|
|
static int link_alt_odb_entry(const char * entry, int len, const char * relative_base, int depth)
|
2005-05-07 07:38:04 +00:00
|
|
|
{
|
2006-05-07 18:19:21 +00:00
|
|
|
struct stat st;
|
2005-12-05 06:48:43 +00:00
|
|
|
const char *objdir = get_object_directory();
|
2006-05-07 18:19:21 +00:00
|
|
|
struct alternate_object_database *ent;
|
|
|
|
struct alternate_object_database *alt;
|
|
|
|
/* 43 = 40-byte + 2 '/' + terminating NUL */
|
|
|
|
int pfxlen = len;
|
|
|
|
int entlen = pfxlen + 43;
|
2005-09-13 07:05:22 +00:00
|
|
|
int base_len = -1;
|
2005-08-15 00:25:57 +00:00
|
|
|
|
2007-11-13 20:05:00 +00:00
|
|
|
if (!is_absolute_path(entry) && relative_base) {
|
2006-05-07 18:19:21 +00:00
|
|
|
/* Relative alt-odb */
|
|
|
|
if (base_len < 0)
|
|
|
|
base_len = strlen(relative_base) + 1;
|
|
|
|
entlen += base_len;
|
|
|
|
pfxlen += base_len;
|
|
|
|
}
|
|
|
|
ent = xmalloc(sizeof(*ent) + entlen);
|
|
|
|
|
2007-11-13 20:05:00 +00:00
|
|
|
if (!is_absolute_path(entry) && relative_base) {
|
2006-05-07 18:19:21 +00:00
|
|
|
memcpy(ent->base, relative_base, base_len - 1);
|
|
|
|
ent->base[base_len - 1] = '/';
|
|
|
|
memcpy(ent->base + base_len, entry, len);
|
|
|
|
}
|
|
|
|
else
|
|
|
|
memcpy(ent->base, entry, pfxlen);
|
|
|
|
|
|
|
|
ent->name = ent->base + pfxlen + 1;
|
|
|
|
ent->base[pfxlen + 3] = '/';
|
|
|
|
ent->base[pfxlen] = ent->base[entlen-1] = 0;
|
|
|
|
|
|
|
|
/* Detect cases where alternate disappeared */
|
|
|
|
if (stat(ent->base, &st) || !S_ISDIR(st.st_mode)) {
|
|
|
|
error("object directory %s does not exist; "
|
|
|
|
"check .git/objects/info/alternates.",
|
|
|
|
ent->base);
|
|
|
|
free(ent);
|
|
|
|
return -1;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Prevent the common mistake of listing the same
|
|
|
|
* thing twice, or object directory itself.
|
|
|
|
*/
|
|
|
|
for (alt = alt_odb_list; alt; alt = alt->next) {
|
|
|
|
if (!memcmp(ent->base, alt->base, pfxlen)) {
|
|
|
|
free(ent);
|
|
|
|
return -1;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
if (!memcmp(ent->base, objdir, pfxlen)) {
|
|
|
|
free(ent);
|
|
|
|
return -1;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* add the alternate entry */
|
|
|
|
*alt_odb_tail = ent;
|
|
|
|
alt_odb_tail = &(ent->next);
|
|
|
|
ent->next = NULL;
|
|
|
|
|
|
|
|
/* recursively add alternates */
|
|
|
|
read_info_alternates(ent->base, depth + 1);
|
|
|
|
|
|
|
|
ent->base[pfxlen] = '/';
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void link_alt_odb_entries(const char *alt, const char *ep, int sep,
|
|
|
|
const char *relative_base, int depth)
|
|
|
|
{
|
|
|
|
const char *cp, *last;
|
|
|
|
|
|
|
|
if (depth > 5) {
|
|
|
|
error("%s: ignoring alternate object stores, nesting too deep.",
|
|
|
|
relative_base);
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
2005-08-15 00:25:57 +00:00
|
|
|
last = alt;
|
2005-08-17 01:22:05 +00:00
|
|
|
while (last < ep) {
|
|
|
|
cp = last;
|
|
|
|
if (cp < ep && *cp == '#') {
|
|
|
|
while (cp < ep && *cp != sep)
|
|
|
|
cp++;
|
|
|
|
last = cp + 1;
|
|
|
|
continue;
|
|
|
|
}
|
2006-05-07 18:19:21 +00:00
|
|
|
while (cp < ep && *cp != sep)
|
|
|
|
cp++;
|
2005-08-15 00:25:57 +00:00
|
|
|
if (last != cp) {
|
2007-11-13 20:05:00 +00:00
|
|
|
if (!is_absolute_path(last) && depth) {
|
2006-05-07 18:19:21 +00:00
|
|
|
error("%s: ignoring relative alternate object store %s",
|
|
|
|
relative_base, last);
|
|
|
|
} else {
|
|
|
|
link_alt_odb_entry(last, cp - last,
|
|
|
|
relative_base, depth);
|
2005-12-05 06:48:43 +00:00
|
|
|
}
|
2005-08-15 00:25:57 +00:00
|
|
|
}
|
2005-08-17 01:22:05 +00:00
|
|
|
while (cp < ep && *cp == sep)
|
2005-08-15 00:25:57 +00:00
|
|
|
cp++;
|
|
|
|
last = cp;
|
2005-08-17 01:22:05 +00:00
|
|
|
}
|
2005-08-15 00:25:57 +00:00
|
|
|
}
|
|
|
|
|
2006-05-07 18:19:21 +00:00
|
|
|
static void read_info_alternates(const char * relative_base, int depth)
|
2005-08-15 00:25:57 +00:00
|
|
|
{
|
2005-08-17 01:22:05 +00:00
|
|
|
char *map;
|
2007-03-07 01:44:37 +00:00
|
|
|
size_t mapsz;
|
2005-08-15 00:25:57 +00:00
|
|
|
struct stat st;
|
2007-07-03 10:40:20 +00:00
|
|
|
const char alt_file_name[] = "info/alternates";
|
|
|
|
/* Given that relative_base is no longer than PATH_MAX,
|
|
|
|
ensure that "path" has enough space to append "/", the
|
|
|
|
file name, "info/alternates", and a trailing NUL. */
|
|
|
|
char path[PATH_MAX + 1 + sizeof alt_file_name];
|
2006-05-07 18:19:21 +00:00
|
|
|
int fd;
|
2005-08-15 00:25:57 +00:00
|
|
|
|
2007-07-03 10:40:20 +00:00
|
|
|
sprintf(path, "%s/%s", relative_base, alt_file_name);
|
2005-08-15 00:25:57 +00:00
|
|
|
fd = open(path, O_RDONLY);
|
|
|
|
if (fd < 0)
|
|
|
|
return;
|
|
|
|
if (fstat(fd, &st) || (st.st_size == 0)) {
|
|
|
|
close(fd);
|
2005-06-28 21:56:57 +00:00
|
|
|
return;
|
2005-05-07 07:38:04 +00:00
|
|
|
}
|
2007-03-07 01:44:37 +00:00
|
|
|
mapsz = xsize_t(st.st_size);
|
|
|
|
map = xmmap(NULL, mapsz, PROT_READ, MAP_PRIVATE, fd, 0);
|
2005-08-15 00:25:57 +00:00
|
|
|
close(fd);
|
|
|
|
|
2007-03-07 01:44:37 +00:00
|
|
|
link_alt_odb_entries(map, map + mapsz, '\n', relative_base, depth);
|
2006-05-07 18:19:21 +00:00
|
|
|
|
2007-03-07 01:44:37 +00:00
|
|
|
munmap(map, mapsz);
|
2005-05-07 07:38:04 +00:00
|
|
|
}
|
|
|
|
|
2006-05-07 18:19:21 +00:00
|
|
|
void prepare_alt_odb(void)
|
|
|
|
{
|
2006-06-28 09:04:39 +00:00
|
|
|
const char *alt;
|
2006-05-07 18:19:21 +00:00
|
|
|
|
2007-05-26 05:24:40 +00:00
|
|
|
if (alt_odb_tail)
|
|
|
|
return;
|
|
|
|
|
2006-05-07 18:19:21 +00:00
|
|
|
alt = getenv(ALTERNATE_DB_ENVIRONMENT);
|
|
|
|
if (!alt) alt = "";
|
|
|
|
|
|
|
|
alt_odb_tail = &alt_odb_list;
|
|
|
|
link_alt_odb_entries(alt, alt + strlen(alt), ':', NULL, 0);
|
|
|
|
|
|
|
|
read_info_alternates(get_object_directory(), 0);
|
|
|
|
}
|
|
|
|
|
2005-05-07 07:38:04 +00:00
|
|
|
static char *find_sha1_file(const unsigned char *sha1, struct stat *st)
|
|
|
|
{
|
|
|
|
char *name = sha1_file_name(sha1);
|
2005-08-15 00:25:57 +00:00
|
|
|
struct alternate_object_database *alt;
|
2005-05-07 07:38:04 +00:00
|
|
|
|
|
|
|
if (!stat(name, st))
|
|
|
|
return name;
|
2005-06-28 21:56:57 +00:00
|
|
|
prepare_alt_odb();
|
2005-08-15 00:25:57 +00:00
|
|
|
for (alt = alt_odb_list; alt; alt = alt->next) {
|
|
|
|
name = alt->name;
|
2005-05-07 07:38:04 +00:00
|
|
|
fill_sha1_path(name, sha1);
|
2005-08-15 00:25:57 +00:00
|
|
|
if (!stat(alt->base, st))
|
|
|
|
return alt->base;
|
2005-05-07 07:38:04 +00:00
|
|
|
}
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
2006-12-23 07:34:28 +00:00
|
|
|
static unsigned int pack_used_ctr;
|
2006-12-23 07:34:47 +00:00
|
|
|
static unsigned int pack_mmap_calls;
|
|
|
|
static unsigned int peak_pack_open_windows;
|
|
|
|
static unsigned int pack_open_windows;
|
|
|
|
static size_t peak_pack_mapped;
|
2006-12-23 07:34:28 +00:00
|
|
|
static size_t pack_mapped;
|
2005-06-28 21:56:57 +00:00
|
|
|
struct packed_git *packed_git;
|
2005-06-27 10:35:33 +00:00
|
|
|
|
2007-06-13 08:22:51 +00:00
|
|
|
void pack_report(void)
|
2006-12-23 07:34:47 +00:00
|
|
|
{
|
|
|
|
fprintf(stderr,
|
2007-01-10 04:07:11 +00:00
|
|
|
"pack_report: getpagesize() = %10" SZ_FMT "\n"
|
|
|
|
"pack_report: core.packedGitWindowSize = %10" SZ_FMT "\n"
|
|
|
|
"pack_report: core.packedGitLimit = %10" SZ_FMT "\n",
|
2007-11-21 20:27:19 +00:00
|
|
|
sz_fmt(getpagesize()),
|
|
|
|
sz_fmt(packed_git_window_size),
|
|
|
|
sz_fmt(packed_git_limit));
|
2006-12-23 07:34:47 +00:00
|
|
|
fprintf(stderr,
|
|
|
|
"pack_report: pack_used_ctr = %10u\n"
|
|
|
|
"pack_report: pack_mmap_calls = %10u\n"
|
|
|
|
"pack_report: pack_open_windows = %10u / %10u\n"
|
2007-01-10 04:07:11 +00:00
|
|
|
"pack_report: pack_mapped = "
|
|
|
|
"%10" SZ_FMT " / %10" SZ_FMT "\n",
|
2006-12-23 07:34:47 +00:00
|
|
|
pack_used_ctr,
|
|
|
|
pack_mmap_calls,
|
|
|
|
pack_open_windows, peak_pack_open_windows,
|
2007-11-21 20:27:19 +00:00
|
|
|
sz_fmt(pack_mapped), sz_fmt(peak_pack_mapped));
|
2006-12-23 07:34:47 +00:00
|
|
|
}
|
|
|
|
|
2007-03-16 20:42:50 +00:00
|
|
|
static int check_packed_git_idx(const char *path, struct packed_git *p)
|
2005-06-27 10:35:33 +00:00
|
|
|
{
|
|
|
|
void *idx_map;
|
2007-03-16 20:42:50 +00:00
|
|
|
struct pack_idx_header *hdr;
|
2007-03-07 01:44:37 +00:00
|
|
|
size_t idx_size;
|
2007-04-09 05:06:35 +00:00
|
|
|
uint32_t version, nr, i, *index;
|
2006-01-19 04:26:14 +00:00
|
|
|
int fd = open(path, O_RDONLY);
|
2005-06-27 10:35:33 +00:00
|
|
|
struct stat st;
|
2007-03-16 20:42:50 +00:00
|
|
|
|
2005-06-27 10:35:33 +00:00
|
|
|
if (fd < 0)
|
|
|
|
return -1;
|
|
|
|
if (fstat(fd, &st)) {
|
|
|
|
close(fd);
|
|
|
|
return -1;
|
|
|
|
}
|
2007-03-07 01:44:37 +00:00
|
|
|
idx_size = xsize_t(st.st_size);
|
2007-03-07 01:44:11 +00:00
|
|
|
if (idx_size < 4 * 256 + 20 + 20) {
|
|
|
|
close(fd);
|
|
|
|
return error("index file %s is too small", path);
|
|
|
|
}
|
2006-12-24 05:47:23 +00:00
|
|
|
idx_map = xmmap(NULL, idx_size, PROT_READ, MAP_PRIVATE, fd, 0);
|
2005-06-27 10:35:33 +00:00
|
|
|
close(fd);
|
|
|
|
|
2007-03-16 20:42:50 +00:00
|
|
|
hdr = idx_map;
|
|
|
|
if (hdr->idx_signature == htonl(PACK_IDX_SIGNATURE)) {
|
2007-04-09 05:06:35 +00:00
|
|
|
version = ntohl(hdr->idx_version);
|
|
|
|
if (version < 2 || version > 2) {
|
|
|
|
munmap(idx_map, idx_size);
|
|
|
|
return error("index file %s is version %d"
|
|
|
|
" and is not supported by this binary"
|
|
|
|
" (try upgrading GIT to a newer version)",
|
|
|
|
path, version);
|
|
|
|
}
|
|
|
|
} else
|
|
|
|
version = 1;
|
2007-01-18 01:43:57 +00:00
|
|
|
|
2005-06-27 10:35:33 +00:00
|
|
|
nr = 0;
|
2007-03-16 20:42:50 +00:00
|
|
|
index = idx_map;
|
2007-04-09 05:06:35 +00:00
|
|
|
if (version > 1)
|
|
|
|
index += 2; /* skip index header */
|
2005-06-27 10:35:33 +00:00
|
|
|
for (i = 0; i < 256; i++) {
|
2007-03-07 01:44:19 +00:00
|
|
|
uint32_t n = ntohl(index[i]);
|
2007-03-07 01:44:11 +00:00
|
|
|
if (n < nr) {
|
|
|
|
munmap(idx_map, idx_size);
|
2007-01-18 01:43:57 +00:00
|
|
|
return error("non-monotonic index %s", path);
|
2007-03-07 01:44:11 +00:00
|
|
|
}
|
2005-06-27 10:35:33 +00:00
|
|
|
nr = n;
|
|
|
|
}
|
|
|
|
|
2007-04-09 05:06:35 +00:00
|
|
|
if (version == 1) {
|
|
|
|
/*
|
|
|
|
* Total size:
|
|
|
|
* - 256 index entries 4 bytes each
|
|
|
|
* - 24-byte entries * nr (20-byte sha1 + 4-byte offset)
|
|
|
|
* - 20-byte SHA1 of the packfile
|
|
|
|
* - 20-byte SHA1 file checksum
|
|
|
|
*/
|
|
|
|
if (idx_size != 4*256 + nr * 24 + 20 + 20) {
|
|
|
|
munmap(idx_map, idx_size);
|
2007-08-14 19:42:37 +00:00
|
|
|
return error("wrong index v1 file size in %s", path);
|
2007-04-09 05:06:35 +00:00
|
|
|
}
|
|
|
|
} else if (version == 2) {
|
|
|
|
/*
|
|
|
|
* Minimum size:
|
|
|
|
* - 8 bytes of header
|
|
|
|
* - 256 index entries 4 bytes each
|
|
|
|
* - 20-byte sha1 entry * nr
|
|
|
|
* - 4-byte crc entry * nr
|
|
|
|
* - 4-byte offset entry * nr
|
|
|
|
* - 20-byte SHA1 of the packfile
|
|
|
|
* - 20-byte SHA1 file checksum
|
|
|
|
* And after the 4-byte offset table might be a
|
|
|
|
* variable sized table containing 8-byte entries
|
|
|
|
* for offsets larger than 2^31.
|
|
|
|
*/
|
|
|
|
unsigned long min_size = 8 + 4*256 + nr*(20 + 4 + 4) + 20 + 20;
|
2007-06-26 21:34:02 +00:00
|
|
|
unsigned long max_size = min_size;
|
|
|
|
if (nr)
|
|
|
|
max_size += (nr - 1)*8;
|
|
|
|
if (idx_size < min_size || idx_size > max_size) {
|
2007-04-09 05:06:35 +00:00
|
|
|
munmap(idx_map, idx_size);
|
2007-08-14 19:42:37 +00:00
|
|
|
return error("wrong index v2 file size in %s", path);
|
2007-04-09 05:06:35 +00:00
|
|
|
}
|
2007-10-29 18:53:55 +00:00
|
|
|
if (idx_size != min_size &&
|
|
|
|
/*
|
|
|
|
* make sure we can deal with large pack offsets.
|
|
|
|
* 31-bit signed offset won't be enough, neither
|
|
|
|
* 32-bit unsigned one will be.
|
|
|
|
*/
|
|
|
|
(sizeof(off_t) <= 4)) {
|
|
|
|
munmap(idx_map, idx_size);
|
|
|
|
return error("pack too large for current definition of off_t in %s", path);
|
2007-04-09 05:06:35 +00:00
|
|
|
}
|
2007-03-07 01:44:11 +00:00
|
|
|
}
|
2005-06-27 10:35:33 +00:00
|
|
|
|
2007-04-09 05:06:35 +00:00
|
|
|
p->index_version = version;
|
2007-03-16 20:42:50 +00:00
|
|
|
p->index_data = idx_map;
|
|
|
|
p->index_size = idx_size;
|
2007-04-09 05:06:28 +00:00
|
|
|
p->num_objects = nr;
|
2005-06-27 10:35:33 +00:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2007-05-30 06:13:42 +00:00
|
|
|
int open_pack_index(struct packed_git *p)
|
2007-05-26 05:24:19 +00:00
|
|
|
{
|
|
|
|
char *idx_name;
|
|
|
|
int ret;
|
|
|
|
|
|
|
|
if (p->index_data)
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
idx_name = xstrdup(p->pack_name);
|
|
|
|
strcpy(idx_name + strlen(idx_name) - strlen(".pack"), ".idx");
|
|
|
|
ret = check_packed_git_idx(idx_name, p);
|
|
|
|
free(idx_name);
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
2006-12-23 07:34:44 +00:00
|
|
|
static void scan_windows(struct packed_git *p,
|
|
|
|
struct packed_git **lru_p,
|
|
|
|
struct pack_window **lru_w,
|
|
|
|
struct pack_window **lru_l)
|
2005-06-27 10:35:33 +00:00
|
|
|
{
|
2006-12-23 07:34:44 +00:00
|
|
|
struct pack_window *w, *w_l;
|
|
|
|
|
|
|
|
for (w_l = NULL, w = p->windows; w; w = w->next) {
|
|
|
|
if (!w->inuse_cnt) {
|
|
|
|
if (!*lru_w || w->last_used < (*lru_w)->last_used) {
|
|
|
|
*lru_p = p;
|
|
|
|
*lru_w = w;
|
|
|
|
*lru_l = w_l;
|
2006-12-23 07:34:23 +00:00
|
|
|
}
|
|
|
|
}
|
2006-12-23 07:34:44 +00:00
|
|
|
w_l = w;
|
2005-06-29 09:51:27 +00:00
|
|
|
}
|
2006-12-23 07:34:44 +00:00
|
|
|
}
|
|
|
|
|
Actually handle some-low memory conditions
Tim Ansell discovered his Debian server didn't permit git-daemon to
use as much memory as it needed to handle cloning a project with
a 128 MiB packfile. Filtering the strace provided by Tim of the
rev-list child showed this gem of a sequence:
open("./objects/pack/pack-*.pack", O_RDONLY|O_LARGEFILE <unfinished ...>
<... open resumed> ) = 5
OK, so the packfile is fd 5...
mmap2(NULL, 33554432, PROT_READ, MAP_PRIVATE, 5, 0 <unfinished ...>
<... mmap2 resumed> ) = 0xb5e2d000
and we mapped one 32 MiB window from it at position 0...
mmap2(NULL, 31020635, PROT_READ, MAP_PRIVATE, 5, 0x6000 <unfinished ...>
<... mmap2 resumed> ) = -1 ENOMEM (Cannot allocate memory)
And we asked for another window further into the file. But got
denied. In Tim's case this was due to a resource limit on the
git-daemon process, and its children.
Now where are we in the code? We're down inside use_pack(),
after we have called unuse_one_window() enough times to make sure
we stay within our allowed maximum window size. However since we
didn't unmap the prior window at 0xb5e2d000 we aren't exceeding
the current limit (which probably was just the defaults).
But we're actually down inside xmmap()...
So we release the window we do have (by calling release_pack_memory),
assuming there is some memory pressure...
munmap(0xb5e2d000, 33554432 <unfinished ...>
<... munmap resumed> ) = 0
close(5 <unfinished ...>
<... close resumed> ) = 0
And that was the last window in this packfile. So we closed it.
Way to go us. Our xmmap did not expect release_pack_memory to
close the fd its about to map...
mmap2(NULL, 31020635, PROT_READ, MAP_PRIVATE, 5, 0x6000 <unfinished ...>
<... mmap2 resumed> ) = -1 EBADF (Bad file descriptor)
And so the Linux kernel happily tells us f' off.
write(2, "fatal: ", 7 <unfinished ...>
<... write resumed> ) = 7
write(2, "Out of memory? mmap failed: Bad "..., 47 <unfinished ...>
<... write resumed> ) = 47
And we report the bad file descriptor error, and not the ENOMEM,
and die, claiming we are out of memory. But actually that mmap
should have succeeded, as we had enough memory for that window,
seeing as how we released the prior one.
Originally when I developed the sliding window mmap feature I had
this exact same bug in fast-import, and I dealt with it by handing
in the struct packed_git* we want to open the new window for, as the
caller wasn't prepared to reopen the packfile if unuse_one_window
closed it. The same is true here from xmmap, but the caller doesn't
have the struct packed_git* handy. So I'm using the file descriptor
instead to perform the same test.
Signed-off-by: Shawn O. Pearce <spearce@spearce.org>
Signed-off-by: Junio C Hamano <junkio@cox.net>
2007-04-25 08:02:27 +00:00
|
|
|
static int unuse_one_window(struct packed_git *current, int keep_fd)
|
2006-12-23 07:34:44 +00:00
|
|
|
{
|
|
|
|
struct packed_git *p, *lru_p = NULL;
|
|
|
|
struct pack_window *lru_w = NULL, *lru_l = NULL;
|
|
|
|
|
|
|
|
if (current)
|
|
|
|
scan_windows(current, &lru_p, &lru_w, &lru_l);
|
|
|
|
for (p = packed_git; p; p = p->next)
|
|
|
|
scan_windows(p, &lru_p, &lru_w, &lru_l);
|
2006-12-23 07:34:23 +00:00
|
|
|
if (lru_p) {
|
|
|
|
munmap(lru_w->base, lru_w->len);
|
|
|
|
pack_mapped -= lru_w->len;
|
|
|
|
if (lru_l)
|
|
|
|
lru_l->next = lru_w->next;
|
|
|
|
else {
|
|
|
|
lru_p->windows = lru_w->next;
|
Actually handle some-low memory conditions
Tim Ansell discovered his Debian server didn't permit git-daemon to
use as much memory as it needed to handle cloning a project with
a 128 MiB packfile. Filtering the strace provided by Tim of the
rev-list child showed this gem of a sequence:
open("./objects/pack/pack-*.pack", O_RDONLY|O_LARGEFILE <unfinished ...>
<... open resumed> ) = 5
OK, so the packfile is fd 5...
mmap2(NULL, 33554432, PROT_READ, MAP_PRIVATE, 5, 0 <unfinished ...>
<... mmap2 resumed> ) = 0xb5e2d000
and we mapped one 32 MiB window from it at position 0...
mmap2(NULL, 31020635, PROT_READ, MAP_PRIVATE, 5, 0x6000 <unfinished ...>
<... mmap2 resumed> ) = -1 ENOMEM (Cannot allocate memory)
And we asked for another window further into the file. But got
denied. In Tim's case this was due to a resource limit on the
git-daemon process, and its children.
Now where are we in the code? We're down inside use_pack(),
after we have called unuse_one_window() enough times to make sure
we stay within our allowed maximum window size. However since we
didn't unmap the prior window at 0xb5e2d000 we aren't exceeding
the current limit (which probably was just the defaults).
But we're actually down inside xmmap()...
So we release the window we do have (by calling release_pack_memory),
assuming there is some memory pressure...
munmap(0xb5e2d000, 33554432 <unfinished ...>
<... munmap resumed> ) = 0
close(5 <unfinished ...>
<... close resumed> ) = 0
And that was the last window in this packfile. So we closed it.
Way to go us. Our xmmap did not expect release_pack_memory to
close the fd its about to map...
mmap2(NULL, 31020635, PROT_READ, MAP_PRIVATE, 5, 0x6000 <unfinished ...>
<... mmap2 resumed> ) = -1 EBADF (Bad file descriptor)
And so the Linux kernel happily tells us f' off.
write(2, "fatal: ", 7 <unfinished ...>
<... write resumed> ) = 7
write(2, "Out of memory? mmap failed: Bad "..., 47 <unfinished ...>
<... write resumed> ) = 47
And we report the bad file descriptor error, and not the ENOMEM,
and die, claiming we are out of memory. But actually that mmap
should have succeeded, as we had enough memory for that window,
seeing as how we released the prior one.
Originally when I developed the sliding window mmap feature I had
this exact same bug in fast-import, and I dealt with it by handing
in the struct packed_git* we want to open the new window for, as the
caller wasn't prepared to reopen the packfile if unuse_one_window
closed it. The same is true here from xmmap, but the caller doesn't
have the struct packed_git* handy. So I'm using the file descriptor
instead to perform the same test.
Signed-off-by: Shawn O. Pearce <spearce@spearce.org>
Signed-off-by: Junio C Hamano <junkio@cox.net>
2007-04-25 08:02:27 +00:00
|
|
|
if (!lru_p->windows && lru_p->pack_fd != keep_fd) {
|
2006-12-23 07:34:23 +00:00
|
|
|
close(lru_p->pack_fd);
|
|
|
|
lru_p->pack_fd = -1;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
free(lru_w);
|
2006-12-23 07:34:47 +00:00
|
|
|
pack_open_windows--;
|
2006-12-23 07:34:23 +00:00
|
|
|
return 1;
|
|
|
|
}
|
|
|
|
return 0;
|
2005-06-29 09:51:27 +00:00
|
|
|
}
|
|
|
|
|
Actually handle some-low memory conditions
Tim Ansell discovered his Debian server didn't permit git-daemon to
use as much memory as it needed to handle cloning a project with
a 128 MiB packfile. Filtering the strace provided by Tim of the
rev-list child showed this gem of a sequence:
open("./objects/pack/pack-*.pack", O_RDONLY|O_LARGEFILE <unfinished ...>
<... open resumed> ) = 5
OK, so the packfile is fd 5...
mmap2(NULL, 33554432, PROT_READ, MAP_PRIVATE, 5, 0 <unfinished ...>
<... mmap2 resumed> ) = 0xb5e2d000
and we mapped one 32 MiB window from it at position 0...
mmap2(NULL, 31020635, PROT_READ, MAP_PRIVATE, 5, 0x6000 <unfinished ...>
<... mmap2 resumed> ) = -1 ENOMEM (Cannot allocate memory)
And we asked for another window further into the file. But got
denied. In Tim's case this was due to a resource limit on the
git-daemon process, and its children.
Now where are we in the code? We're down inside use_pack(),
after we have called unuse_one_window() enough times to make sure
we stay within our allowed maximum window size. However since we
didn't unmap the prior window at 0xb5e2d000 we aren't exceeding
the current limit (which probably was just the defaults).
But we're actually down inside xmmap()...
So we release the window we do have (by calling release_pack_memory),
assuming there is some memory pressure...
munmap(0xb5e2d000, 33554432 <unfinished ...>
<... munmap resumed> ) = 0
close(5 <unfinished ...>
<... close resumed> ) = 0
And that was the last window in this packfile. So we closed it.
Way to go us. Our xmmap did not expect release_pack_memory to
close the fd its about to map...
mmap2(NULL, 31020635, PROT_READ, MAP_PRIVATE, 5, 0x6000 <unfinished ...>
<... mmap2 resumed> ) = -1 EBADF (Bad file descriptor)
And so the Linux kernel happily tells us f' off.
write(2, "fatal: ", 7 <unfinished ...>
<... write resumed> ) = 7
write(2, "Out of memory? mmap failed: Bad "..., 47 <unfinished ...>
<... write resumed> ) = 47
And we report the bad file descriptor error, and not the ENOMEM,
and die, claiming we are out of memory. But actually that mmap
should have succeeded, as we had enough memory for that window,
seeing as how we released the prior one.
Originally when I developed the sliding window mmap feature I had
this exact same bug in fast-import, and I dealt with it by handing
in the struct packed_git* we want to open the new window for, as the
caller wasn't prepared to reopen the packfile if unuse_one_window
closed it. The same is true here from xmmap, but the caller doesn't
have the struct packed_git* handy. So I'm using the file descriptor
instead to perform the same test.
Signed-off-by: Shawn O. Pearce <spearce@spearce.org>
Signed-off-by: Junio C Hamano <junkio@cox.net>
2007-04-25 08:02:27 +00:00
|
|
|
void release_pack_memory(size_t need, int fd)
|
2006-12-24 05:47:19 +00:00
|
|
|
{
|
|
|
|
size_t cur = pack_mapped;
|
Actually handle some-low memory conditions
Tim Ansell discovered his Debian server didn't permit git-daemon to
use as much memory as it needed to handle cloning a project with
a 128 MiB packfile. Filtering the strace provided by Tim of the
rev-list child showed this gem of a sequence:
open("./objects/pack/pack-*.pack", O_RDONLY|O_LARGEFILE <unfinished ...>
<... open resumed> ) = 5
OK, so the packfile is fd 5...
mmap2(NULL, 33554432, PROT_READ, MAP_PRIVATE, 5, 0 <unfinished ...>
<... mmap2 resumed> ) = 0xb5e2d000
and we mapped one 32 MiB window from it at position 0...
mmap2(NULL, 31020635, PROT_READ, MAP_PRIVATE, 5, 0x6000 <unfinished ...>
<... mmap2 resumed> ) = -1 ENOMEM (Cannot allocate memory)
And we asked for another window further into the file. But got
denied. In Tim's case this was due to a resource limit on the
git-daemon process, and its children.
Now where are we in the code? We're down inside use_pack(),
after we have called unuse_one_window() enough times to make sure
we stay within our allowed maximum window size. However since we
didn't unmap the prior window at 0xb5e2d000 we aren't exceeding
the current limit (which probably was just the defaults).
But we're actually down inside xmmap()...
So we release the window we do have (by calling release_pack_memory),
assuming there is some memory pressure...
munmap(0xb5e2d000, 33554432 <unfinished ...>
<... munmap resumed> ) = 0
close(5 <unfinished ...>
<... close resumed> ) = 0
And that was the last window in this packfile. So we closed it.
Way to go us. Our xmmap did not expect release_pack_memory to
close the fd its about to map...
mmap2(NULL, 31020635, PROT_READ, MAP_PRIVATE, 5, 0x6000 <unfinished ...>
<... mmap2 resumed> ) = -1 EBADF (Bad file descriptor)
And so the Linux kernel happily tells us f' off.
write(2, "fatal: ", 7 <unfinished ...>
<... write resumed> ) = 7
write(2, "Out of memory? mmap failed: Bad "..., 47 <unfinished ...>
<... write resumed> ) = 47
And we report the bad file descriptor error, and not the ENOMEM,
and die, claiming we are out of memory. But actually that mmap
should have succeeded, as we had enough memory for that window,
seeing as how we released the prior one.
Originally when I developed the sliding window mmap feature I had
this exact same bug in fast-import, and I dealt with it by handing
in the struct packed_git* we want to open the new window for, as the
caller wasn't prepared to reopen the packfile if unuse_one_window
closed it. The same is true here from xmmap, but the caller doesn't
have the struct packed_git* handy. So I'm using the file descriptor
instead to perform the same test.
Signed-off-by: Shawn O. Pearce <spearce@spearce.org>
Signed-off-by: Junio C Hamano <junkio@cox.net>
2007-04-25 08:02:27 +00:00
|
|
|
while (need >= (cur - pack_mapped) && unuse_one_window(NULL, fd))
|
2006-12-24 05:47:19 +00:00
|
|
|
; /* nothing */
|
|
|
|
}
|
|
|
|
|
2008-01-18 03:57:00 +00:00
|
|
|
void close_pack_windows(struct packed_git *p)
|
|
|
|
{
|
|
|
|
while (p->windows) {
|
|
|
|
struct pack_window *w = p->windows;
|
|
|
|
|
|
|
|
if (w->inuse_cnt)
|
|
|
|
die("pack '%s' still has open windows to it",
|
|
|
|
p->pack_name);
|
|
|
|
munmap(w->base, w->len);
|
|
|
|
pack_mapped -= w->len;
|
|
|
|
pack_open_windows--;
|
|
|
|
p->windows = w->next;
|
|
|
|
free(w);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2006-12-23 07:34:08 +00:00
|
|
|
void unuse_pack(struct pack_window **w_cursor)
|
2005-06-29 09:51:27 +00:00
|
|
|
{
|
2006-12-23 07:34:08 +00:00
|
|
|
struct pack_window *w = *w_cursor;
|
|
|
|
if (w) {
|
|
|
|
w->inuse_cnt--;
|
|
|
|
*w_cursor = NULL;
|
|
|
|
}
|
2005-06-27 10:35:33 +00:00
|
|
|
}
|
|
|
|
|
2007-02-02 08:00:03 +00:00
|
|
|
/*
|
|
|
|
* Do not call this directly as this leaks p->pack_fd on error return;
|
|
|
|
* call open_packed_git() instead.
|
|
|
|
*/
|
|
|
|
static int open_packed_git_1(struct packed_git *p)
|
2005-06-27 10:35:33 +00:00
|
|
|
{
|
2006-12-23 07:34:01 +00:00
|
|
|
struct stat st;
|
|
|
|
struct pack_header hdr;
|
|
|
|
unsigned char sha1[20];
|
|
|
|
unsigned char *idx_sha1;
|
2006-12-29 08:30:01 +00:00
|
|
|
long fd_flag;
|
2006-12-23 07:34:01 +00:00
|
|
|
|
2007-05-26 05:24:19 +00:00
|
|
|
if (!p->index_data && open_pack_index(p))
|
|
|
|
return error("packfile %s index unavailable", p->pack_name);
|
|
|
|
|
2006-12-23 07:34:01 +00:00
|
|
|
p->pack_fd = open(p->pack_name, O_RDONLY);
|
|
|
|
if (p->pack_fd < 0 || fstat(p->pack_fd, &st))
|
2007-02-01 20:52:33 +00:00
|
|
|
return -1;
|
2006-12-23 07:34:01 +00:00
|
|
|
|
|
|
|
/* If we created the struct before we had the pack we lack size. */
|
2005-08-01 00:53:44 +00:00
|
|
|
if (!p->pack_size) {
|
|
|
|
if (!S_ISREG(st.st_mode))
|
2007-02-01 20:52:33 +00:00
|
|
|
return error("packfile %s not a regular file", p->pack_name);
|
2005-08-01 00:53:44 +00:00
|
|
|
p->pack_size = st.st_size;
|
2006-12-23 07:34:01 +00:00
|
|
|
} else if (p->pack_size != st.st_size)
|
2007-02-01 20:52:33 +00:00
|
|
|
return error("packfile %s size changed", p->pack_name);
|
2006-12-23 07:34:01 +00:00
|
|
|
|
2006-12-29 08:30:01 +00:00
|
|
|
/* We leave these file descriptors open with sliding mmap;
|
|
|
|
* there is no point keeping them open across exec(), though.
|
|
|
|
*/
|
|
|
|
fd_flag = fcntl(p->pack_fd, F_GETFD, 0);
|
|
|
|
if (fd_flag < 0)
|
2007-02-01 20:52:33 +00:00
|
|
|
return error("cannot determine file descriptor flags");
|
2006-12-29 08:30:01 +00:00
|
|
|
fd_flag |= FD_CLOEXEC;
|
|
|
|
if (fcntl(p->pack_fd, F_SETFD, fd_flag) == -1)
|
2007-02-01 20:52:33 +00:00
|
|
|
return error("cannot set FD_CLOEXEC");
|
2006-12-29 08:30:01 +00:00
|
|
|
|
2006-12-23 07:34:01 +00:00
|
|
|
/* Verify we recognize this pack file format. */
|
2007-01-14 06:01:49 +00:00
|
|
|
if (read_in_full(p->pack_fd, &hdr, sizeof(hdr)) != sizeof(hdr))
|
2007-02-01 20:52:33 +00:00
|
|
|
return error("file %s is far too short to be a packfile", p->pack_name);
|
2006-12-23 07:34:01 +00:00
|
|
|
if (hdr.hdr_signature != htonl(PACK_SIGNATURE))
|
2007-02-01 20:52:33 +00:00
|
|
|
return error("file %s is not a GIT packfile", p->pack_name);
|
2006-12-23 07:34:01 +00:00
|
|
|
if (!pack_version_ok(hdr.hdr_version))
|
2007-02-01 20:52:33 +00:00
|
|
|
return error("packfile %s is version %u and not supported"
|
2006-12-23 07:34:01 +00:00
|
|
|
" (try upgrading GIT to a newer version)",
|
|
|
|
p->pack_name, ntohl(hdr.hdr_version));
|
|
|
|
|
|
|
|
/* Verify the pack matches its index. */
|
2007-04-09 05:06:28 +00:00
|
|
|
if (p->num_objects != ntohl(hdr.hdr_entries))
|
2007-02-01 20:52:33 +00:00
|
|
|
return error("packfile %s claims to have %u objects"
|
2007-04-09 05:06:28 +00:00
|
|
|
" while index indicates %u objects",
|
|
|
|
p->pack_name, ntohl(hdr.hdr_entries),
|
|
|
|
p->num_objects);
|
2006-12-23 07:34:01 +00:00
|
|
|
if (lseek(p->pack_fd, p->pack_size - sizeof(sha1), SEEK_SET) == -1)
|
2007-02-01 20:52:33 +00:00
|
|
|
return error("end of packfile %s is unavailable", p->pack_name);
|
2007-01-14 06:01:49 +00:00
|
|
|
if (read_in_full(p->pack_fd, sha1, sizeof(sha1)) != sizeof(sha1))
|
2007-02-01 20:52:33 +00:00
|
|
|
return error("packfile %s signature is unavailable", p->pack_name);
|
2007-03-16 20:42:50 +00:00
|
|
|
idx_sha1 = ((unsigned char *)p->index_data) + p->index_size - 40;
|
2006-12-23 07:34:01 +00:00
|
|
|
if (hashcmp(sha1, idx_sha1))
|
2007-02-01 20:52:33 +00:00
|
|
|
return error("packfile %s does not match index", p->pack_name);
|
|
|
|
return 0;
|
2006-12-23 07:34:01 +00:00
|
|
|
}
|
|
|
|
|
2007-02-02 08:00:03 +00:00
|
|
|
static int open_packed_git(struct packed_git *p)
|
|
|
|
{
|
|
|
|
if (!open_packed_git_1(p))
|
|
|
|
return 0;
|
|
|
|
if (p->pack_fd != -1) {
|
|
|
|
close(p->pack_fd);
|
|
|
|
p->pack_fd = -1;
|
|
|
|
}
|
|
|
|
return -1;
|
|
|
|
}
|
|
|
|
|
2007-03-07 01:44:30 +00:00
|
|
|
static int in_window(struct pack_window *win, off_t offset)
|
2006-12-23 07:34:28 +00:00
|
|
|
{
|
|
|
|
/* We must promise at least 20 bytes (one hash) after the
|
|
|
|
* offset is available from this window, otherwise the offset
|
|
|
|
* is not actually in this window and a different window (which
|
|
|
|
* has that one hash excess) must be used. This is to support
|
|
|
|
* the object header and delta base parsing routines below.
|
|
|
|
*/
|
|
|
|
off_t win_off = win->offset;
|
|
|
|
return win_off <= offset
|
|
|
|
&& (offset + 20) <= (win_off + win->len);
|
|
|
|
}
|
|
|
|
|
2006-12-23 07:34:08 +00:00
|
|
|
unsigned char* use_pack(struct packed_git *p,
|
|
|
|
struct pack_window **w_cursor,
|
2007-03-07 01:44:30 +00:00
|
|
|
off_t offset,
|
2006-12-23 07:34:08 +00:00
|
|
|
unsigned int *left)
|
2006-12-23 07:34:01 +00:00
|
|
|
{
|
2006-12-23 07:34:28 +00:00
|
|
|
struct pack_window *win = *w_cursor;
|
2006-12-23 07:34:08 +00:00
|
|
|
|
2007-02-01 20:52:33 +00:00
|
|
|
if (p->pack_fd == -1 && open_packed_git(p))
|
|
|
|
die("packfile %s cannot be accessed", p->pack_name);
|
2006-12-23 07:34:28 +00:00
|
|
|
|
|
|
|
/* Since packfiles end in a hash of their content and its
|
|
|
|
* pointless to ask for an offset into the middle of that
|
|
|
|
* hash, and the in_window function above wouldn't match
|
|
|
|
* don't allow an offset too close to the end of the file.
|
|
|
|
*/
|
|
|
|
if (offset > (p->pack_size - 20))
|
|
|
|
die("offset beyond end of packfile (truncated pack?)");
|
|
|
|
|
|
|
|
if (!win || !in_window(win, offset)) {
|
|
|
|
if (win)
|
|
|
|
win->inuse_cnt--;
|
|
|
|
for (win = p->windows; win; win = win->next) {
|
|
|
|
if (in_window(win, offset))
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
if (!win) {
|
2007-02-14 17:11:40 +00:00
|
|
|
size_t window_align = packed_git_window_size / 2;
|
2007-03-07 01:44:37 +00:00
|
|
|
off_t len;
|
2006-12-23 07:34:28 +00:00
|
|
|
win = xcalloc(1, sizeof(*win));
|
2007-02-14 17:11:40 +00:00
|
|
|
win->offset = (offset / window_align) * window_align;
|
2007-03-07 01:44:37 +00:00
|
|
|
len = p->pack_size - win->offset;
|
|
|
|
if (len > packed_git_window_size)
|
|
|
|
len = packed_git_window_size;
|
|
|
|
win->len = (size_t)len;
|
2006-12-23 07:34:28 +00:00
|
|
|
pack_mapped += win->len;
|
2006-12-23 07:34:44 +00:00
|
|
|
while (packed_git_limit < pack_mapped
|
Actually handle some-low memory conditions
Tim Ansell discovered his Debian server didn't permit git-daemon to
use as much memory as it needed to handle cloning a project with
a 128 MiB packfile. Filtering the strace provided by Tim of the
rev-list child showed this gem of a sequence:
open("./objects/pack/pack-*.pack", O_RDONLY|O_LARGEFILE <unfinished ...>
<... open resumed> ) = 5
OK, so the packfile is fd 5...
mmap2(NULL, 33554432, PROT_READ, MAP_PRIVATE, 5, 0 <unfinished ...>
<... mmap2 resumed> ) = 0xb5e2d000
and we mapped one 32 MiB window from it at position 0...
mmap2(NULL, 31020635, PROT_READ, MAP_PRIVATE, 5, 0x6000 <unfinished ...>
<... mmap2 resumed> ) = -1 ENOMEM (Cannot allocate memory)
And we asked for another window further into the file. But got
denied. In Tim's case this was due to a resource limit on the
git-daemon process, and its children.
Now where are we in the code? We're down inside use_pack(),
after we have called unuse_one_window() enough times to make sure
we stay within our allowed maximum window size. However since we
didn't unmap the prior window at 0xb5e2d000 we aren't exceeding
the current limit (which probably was just the defaults).
But we're actually down inside xmmap()...
So we release the window we do have (by calling release_pack_memory),
assuming there is some memory pressure...
munmap(0xb5e2d000, 33554432 <unfinished ...>
<... munmap resumed> ) = 0
close(5 <unfinished ...>
<... close resumed> ) = 0
And that was the last window in this packfile. So we closed it.
Way to go us. Our xmmap did not expect release_pack_memory to
close the fd its about to map...
mmap2(NULL, 31020635, PROT_READ, MAP_PRIVATE, 5, 0x6000 <unfinished ...>
<... mmap2 resumed> ) = -1 EBADF (Bad file descriptor)
And so the Linux kernel happily tells us f' off.
write(2, "fatal: ", 7 <unfinished ...>
<... write resumed> ) = 7
write(2, "Out of memory? mmap failed: Bad "..., 47 <unfinished ...>
<... write resumed> ) = 47
And we report the bad file descriptor error, and not the ENOMEM,
and die, claiming we are out of memory. But actually that mmap
should have succeeded, as we had enough memory for that window,
seeing as how we released the prior one.
Originally when I developed the sliding window mmap feature I had
this exact same bug in fast-import, and I dealt with it by handing
in the struct packed_git* we want to open the new window for, as the
caller wasn't prepared to reopen the packfile if unuse_one_window
closed it. The same is true here from xmmap, but the caller doesn't
have the struct packed_git* handy. So I'm using the file descriptor
instead to perform the same test.
Signed-off-by: Shawn O. Pearce <spearce@spearce.org>
Signed-off-by: Junio C Hamano <junkio@cox.net>
2007-04-25 08:02:27 +00:00
|
|
|
&& unuse_one_window(p, p->pack_fd))
|
2006-12-23 07:34:28 +00:00
|
|
|
; /* nothing */
|
2006-12-24 05:47:23 +00:00
|
|
|
win->base = xmmap(NULL, win->len,
|
2006-12-23 07:34:28 +00:00
|
|
|
PROT_READ, MAP_PRIVATE,
|
|
|
|
p->pack_fd, win->offset);
|
|
|
|
if (win->base == MAP_FAILED)
|
2006-12-23 07:34:41 +00:00
|
|
|
die("packfile %s cannot be mapped: %s",
|
|
|
|
p->pack_name,
|
|
|
|
strerror(errno));
|
2006-12-23 07:34:47 +00:00
|
|
|
pack_mmap_calls++;
|
|
|
|
pack_open_windows++;
|
|
|
|
if (pack_mapped > peak_pack_mapped)
|
|
|
|
peak_pack_mapped = pack_mapped;
|
|
|
|
if (pack_open_windows > peak_pack_open_windows)
|
|
|
|
peak_pack_open_windows = pack_open_windows;
|
2006-12-23 07:34:28 +00:00
|
|
|
win->next = p->windows;
|
|
|
|
p->windows = win;
|
|
|
|
}
|
2005-06-27 10:35:33 +00:00
|
|
|
}
|
2006-12-23 07:34:08 +00:00
|
|
|
if (win != *w_cursor) {
|
|
|
|
win->last_used = pack_used_ctr++;
|
|
|
|
win->inuse_cnt++;
|
|
|
|
*w_cursor = win;
|
|
|
|
}
|
2006-12-23 07:34:28 +00:00
|
|
|
offset -= win->offset;
|
2006-12-23 07:34:08 +00:00
|
|
|
if (left)
|
2007-03-07 01:44:37 +00:00
|
|
|
*left = win->len - xsize_t(offset);
|
2006-12-23 07:34:08 +00:00
|
|
|
return win->base + offset;
|
2005-06-27 10:35:33 +00:00
|
|
|
}
|
|
|
|
|
2007-03-16 20:42:50 +00:00
|
|
|
struct packed_git *add_packed_git(const char *path, int path_len, int local)
|
2005-06-27 10:35:33 +00:00
|
|
|
{
|
|
|
|
struct stat st;
|
2007-03-16 20:42:50 +00:00
|
|
|
struct packed_git *p = xmalloc(sizeof(*p) + path_len + 2);
|
2005-06-27 10:35:33 +00:00
|
|
|
|
2007-03-16 20:42:50 +00:00
|
|
|
/*
|
|
|
|
* Make sure a corresponding .pack file exists and that
|
|
|
|
* the index looks sane.
|
|
|
|
*/
|
|
|
|
path_len -= strlen(".idx");
|
|
|
|
if (path_len < 1)
|
2005-06-27 10:35:33 +00:00
|
|
|
return NULL;
|
2007-03-16 20:42:50 +00:00
|
|
|
memcpy(p->pack_name, path, path_len);
|
|
|
|
strcpy(p->pack_name + path_len, ".pack");
|
2007-05-26 05:24:19 +00:00
|
|
|
if (stat(p->pack_name, &st) || !S_ISREG(st.st_mode)) {
|
2007-03-16 20:42:50 +00:00
|
|
|
free(p);
|
2005-06-27 10:35:33 +00:00
|
|
|
return NULL;
|
|
|
|
}
|
2007-03-16 20:42:50 +00:00
|
|
|
|
2005-06-27 10:35:33 +00:00
|
|
|
/* ok, it looks sane as far as we can check without
|
|
|
|
* actually mapping the pack file.
|
|
|
|
*/
|
2007-05-26 05:24:19 +00:00
|
|
|
p->index_version = 0;
|
|
|
|
p->index_data = NULL;
|
|
|
|
p->index_size = 0;
|
|
|
|
p->num_objects = 0;
|
2005-06-27 10:35:33 +00:00
|
|
|
p->pack_size = st.st_size;
|
|
|
|
p->next = NULL;
|
2006-12-23 07:33:44 +00:00
|
|
|
p->windows = NULL;
|
2006-12-23 07:34:01 +00:00
|
|
|
p->pack_fd = -1;
|
2005-10-13 22:38:28 +00:00
|
|
|
p->pack_local = local;
|
2007-03-09 11:52:12 +00:00
|
|
|
p->mtime = st.st_mtime;
|
2007-03-16 20:42:50 +00:00
|
|
|
if (path_len < 40 || get_sha1_hex(path + path_len - 40, p->sha1))
|
|
|
|
hashclr(p->sha1);
|
2005-06-27 10:35:33 +00:00
|
|
|
return p;
|
|
|
|
}
|
|
|
|
|
2005-08-01 00:53:44 +00:00
|
|
|
struct packed_git *parse_pack_index(unsigned char *sha1)
|
2005-08-16 04:10:03 +00:00
|
|
|
{
|
|
|
|
char *path = sha1_pack_index_name(sha1);
|
|
|
|
return parse_pack_index_file(sha1, path);
|
|
|
|
}
|
|
|
|
|
2007-03-16 20:42:50 +00:00
|
|
|
struct packed_git *parse_pack_index_file(const unsigned char *sha1,
|
|
|
|
const char *idx_path)
|
2005-08-01 00:53:44 +00:00
|
|
|
{
|
2007-03-16 20:42:50 +00:00
|
|
|
const char *path = sha1_pack_name(sha1);
|
|
|
|
struct packed_git *p = xmalloc(sizeof(*p) + strlen(path) + 2);
|
2005-08-01 00:53:44 +00:00
|
|
|
|
2007-03-16 20:42:50 +00:00
|
|
|
if (check_packed_git_idx(idx_path, p)) {
|
|
|
|
free(p);
|
2005-08-01 00:53:44 +00:00
|
|
|
return NULL;
|
2007-03-16 20:42:50 +00:00
|
|
|
}
|
2005-08-01 00:53:44 +00:00
|
|
|
|
|
|
|
strcpy(p->pack_name, path);
|
|
|
|
p->pack_size = 0;
|
|
|
|
p->next = NULL;
|
2006-12-23 07:33:44 +00:00
|
|
|
p->windows = NULL;
|
2006-12-23 07:34:01 +00:00
|
|
|
p->pack_fd = -1;
|
2006-08-23 06:49:00 +00:00
|
|
|
hashcpy(p->sha1, sha1);
|
2005-08-01 00:53:44 +00:00
|
|
|
return p;
|
|
|
|
}
|
|
|
|
|
|
|
|
void install_packed_git(struct packed_git *pack)
|
|
|
|
{
|
|
|
|
pack->next = packed_git;
|
|
|
|
packed_git = pack;
|
|
|
|
}
|
|
|
|
|
2005-10-13 22:38:28 +00:00
|
|
|
static void prepare_packed_git_one(char *objdir, int local)
|
2005-06-27 10:35:33 +00:00
|
|
|
{
|
2007-07-03 10:40:20 +00:00
|
|
|
/* Ensure that this buffer is large enough so that we can
|
|
|
|
append "/pack/" without clobbering the stack even if
|
|
|
|
strlen(objdir) were PATH_MAX. */
|
|
|
|
char path[PATH_MAX + 1 + 4 + 1 + 1];
|
2005-06-27 10:35:33 +00:00
|
|
|
int len;
|
|
|
|
DIR *dir;
|
|
|
|
struct dirent *de;
|
|
|
|
|
|
|
|
sprintf(path, "%s/pack", objdir);
|
|
|
|
len = strlen(path);
|
|
|
|
dir = opendir(path);
|
2006-02-18 00:14:52 +00:00
|
|
|
if (!dir) {
|
2006-02-22 19:16:38 +00:00
|
|
|
if (errno != ENOENT)
|
2006-02-23 01:47:10 +00:00
|
|
|
error("unable to open object pack directory: %s: %s",
|
2006-02-22 19:16:38 +00:00
|
|
|
path, strerror(errno));
|
2005-06-27 10:35:33 +00:00
|
|
|
return;
|
2006-02-18 00:14:52 +00:00
|
|
|
}
|
2005-06-27 10:35:33 +00:00
|
|
|
path[len++] = '/';
|
|
|
|
while ((de = readdir(dir)) != NULL) {
|
|
|
|
int namelen = strlen(de->d_name);
|
|
|
|
struct packed_git *p;
|
|
|
|
|
2006-08-11 12:01:45 +00:00
|
|
|
if (!has_extension(de->d_name, ".idx"))
|
2005-06-27 10:35:33 +00:00
|
|
|
continue;
|
|
|
|
|
2007-07-03 10:40:20 +00:00
|
|
|
if (len + namelen + 1 > sizeof(path))
|
|
|
|
continue;
|
|
|
|
|
2007-02-01 20:52:27 +00:00
|
|
|
/* Don't reopen a pack we already have. */
|
2005-06-27 10:35:33 +00:00
|
|
|
strcpy(path + len, de->d_name);
|
2006-06-02 16:49:32 +00:00
|
|
|
for (p = packed_git; p; p = p->next) {
|
|
|
|
if (!memcmp(path, p->pack_name, len + namelen - 4))
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
if (p)
|
|
|
|
continue;
|
2007-02-01 20:52:27 +00:00
|
|
|
/* See if it really is a valid .idx file with corresponding
|
|
|
|
* .pack file that we can map.
|
|
|
|
*/
|
2005-10-13 22:38:28 +00:00
|
|
|
p = add_packed_git(path, len + namelen, local);
|
2005-06-27 10:35:33 +00:00
|
|
|
if (!p)
|
|
|
|
continue;
|
2007-02-01 20:52:22 +00:00
|
|
|
install_packed_git(p);
|
2005-06-27 10:35:33 +00:00
|
|
|
}
|
2005-07-06 06:52:17 +00:00
|
|
|
closedir(dir);
|
2005-06-27 10:35:33 +00:00
|
|
|
}
|
|
|
|
|
2007-03-09 11:52:12 +00:00
|
|
|
static int sort_pack(const void *a_, const void *b_)
|
|
|
|
{
|
|
|
|
struct packed_git *a = *((struct packed_git **)a_);
|
|
|
|
struct packed_git *b = *((struct packed_git **)b_);
|
|
|
|
int st;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Local packs tend to contain objects specific to our
|
|
|
|
* variant of the project than remote ones. In addition,
|
|
|
|
* remote ones could be on a network mounted filesystem.
|
|
|
|
* Favor local ones for these reasons.
|
|
|
|
*/
|
|
|
|
st = a->pack_local - b->pack_local;
|
|
|
|
if (st)
|
|
|
|
return -st;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Younger packs tend to contain more recent objects,
|
|
|
|
* and more recent objects tend to get accessed more
|
|
|
|
* often.
|
|
|
|
*/
|
|
|
|
if (a->mtime < b->mtime)
|
|
|
|
return 1;
|
|
|
|
else if (a->mtime == b->mtime)
|
|
|
|
return 0;
|
|
|
|
return -1;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void rearrange_packed_git(void)
|
|
|
|
{
|
|
|
|
struct packed_git **ary, *p;
|
|
|
|
int i, n;
|
|
|
|
|
|
|
|
for (n = 0, p = packed_git; p; p = p->next)
|
|
|
|
n++;
|
|
|
|
if (n < 2)
|
|
|
|
return;
|
|
|
|
|
|
|
|
/* prepare an array of packed_git for easier sorting */
|
|
|
|
ary = xcalloc(n, sizeof(struct packed_git *));
|
|
|
|
for (n = 0, p = packed_git; p; p = p->next)
|
|
|
|
ary[n++] = p;
|
|
|
|
|
|
|
|
qsort(ary, n, sizeof(struct packed_git *), sort_pack);
|
|
|
|
|
|
|
|
/* link them back again */
|
|
|
|
for (i = 0; i < n - 1; i++)
|
|
|
|
ary[i]->next = ary[i + 1];
|
|
|
|
ary[n - 1]->next = NULL;
|
|
|
|
packed_git = ary[0];
|
|
|
|
|
|
|
|
free(ary);
|
|
|
|
}
|
|
|
|
|
2006-06-02 15:32:23 +00:00
|
|
|
static int prepare_packed_git_run_once = 0;
|
2005-06-28 21:56:57 +00:00
|
|
|
void prepare_packed_git(void)
|
2005-06-27 10:35:33 +00:00
|
|
|
{
|
2005-08-15 00:25:57 +00:00
|
|
|
struct alternate_object_database *alt;
|
2005-06-27 10:35:33 +00:00
|
|
|
|
2006-06-02 15:32:23 +00:00
|
|
|
if (prepare_packed_git_run_once)
|
2005-06-27 10:35:33 +00:00
|
|
|
return;
|
2005-10-13 22:38:28 +00:00
|
|
|
prepare_packed_git_one(get_object_directory(), 1);
|
2005-06-28 21:56:57 +00:00
|
|
|
prepare_alt_odb();
|
2005-08-15 00:25:57 +00:00
|
|
|
for (alt = alt_odb_list; alt; alt = alt->next) {
|
2005-12-05 06:48:43 +00:00
|
|
|
alt->name[-1] = 0;
|
2005-10-13 22:38:28 +00:00
|
|
|
prepare_packed_git_one(alt->base, 0);
|
2005-12-05 06:48:43 +00:00
|
|
|
alt->name[-1] = '/';
|
2005-06-27 10:35:33 +00:00
|
|
|
}
|
2007-03-09 11:52:12 +00:00
|
|
|
rearrange_packed_git();
|
2006-06-02 15:32:23 +00:00
|
|
|
prepare_packed_git_run_once = 1;
|
|
|
|
}
|
|
|
|
|
2006-11-01 22:06:21 +00:00
|
|
|
void reprepare_packed_git(void)
|
2006-06-02 15:32:23 +00:00
|
|
|
{
|
|
|
|
prepare_packed_git_run_once = 0;
|
|
|
|
prepare_packed_git();
|
2005-06-27 10:35:33 +00:00
|
|
|
}
|
|
|
|
|
2005-06-03 15:05:39 +00:00
|
|
|
int check_sha1_signature(const unsigned char *sha1, void *map, unsigned long size, const char *type)
|
2005-04-18 20:04:43 +00:00
|
|
|
{
|
|
|
|
unsigned char real_sha1[20];
|
2006-10-15 12:02:18 +00:00
|
|
|
hash_sha1_file(map, size, type, real_sha1);
|
2006-08-17 18:54:57 +00:00
|
|
|
return hashcmp(sha1, real_sha1) ? -1 : 0;
|
2005-04-18 20:04:43 +00:00
|
|
|
}
|
|
|
|
|
2007-06-13 08:42:05 +00:00
|
|
|
static void *map_sha1_file(const unsigned char *sha1, unsigned long *size)
|
2005-04-18 20:04:43 +00:00
|
|
|
{
|
|
|
|
struct stat st;
|
|
|
|
void *map;
|
2005-04-23 18:09:32 +00:00
|
|
|
int fd;
|
2005-05-07 07:38:04 +00:00
|
|
|
char *filename = find_sha1_file(sha1, &st);
|
|
|
|
|
|
|
|
if (!filename) {
|
|
|
|
return NULL;
|
|
|
|
}
|
2005-04-18 20:04:43 +00:00
|
|
|
|
2005-04-23 18:09:32 +00:00
|
|
|
fd = open(filename, O_RDONLY | sha1_file_open_flag);
|
2005-04-18 20:04:43 +00:00
|
|
|
if (fd < 0) {
|
2005-04-23 18:09:32 +00:00
|
|
|
/* See if it works without O_NOATIME */
|
|
|
|
switch (sha1_file_open_flag) {
|
|
|
|
default:
|
|
|
|
fd = open(filename, O_RDONLY);
|
|
|
|
if (fd >= 0)
|
|
|
|
break;
|
|
|
|
/* Fallthrough */
|
|
|
|
case 0:
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
2005-06-27 10:35:33 +00:00
|
|
|
/* If it failed once, it will probably fail again.
|
|
|
|
* Stop using O_NOATIME
|
|
|
|
*/
|
2005-04-23 18:09:32 +00:00
|
|
|
sha1_file_open_flag = 0;
|
2005-04-18 20:04:43 +00:00
|
|
|
}
|
2007-03-07 01:44:37 +00:00
|
|
|
*size = xsize_t(st.st_size);
|
|
|
|
map = xmmap(NULL, *size, PROT_READ, MAP_PRIVATE, fd, 0);
|
2005-04-18 20:04:43 +00:00
|
|
|
close(fd);
|
|
|
|
return map;
|
|
|
|
}
|
|
|
|
|
2007-05-09 18:42:42 +00:00
|
|
|
static int legacy_loose_object(unsigned char *map)
|
2006-07-17 22:04:47 +00:00
|
|
|
{
|
|
|
|
unsigned int word;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Is it a zlib-compressed buffer? If so, the first byte
|
|
|
|
* must be 0x78 (15-bit window size, deflated), and the
|
|
|
|
* first 16-bit word is evenly divisible by 31
|
|
|
|
*/
|
|
|
|
word = (map[0] << 8) + map[1];
|
|
|
|
if (map[0] == 0x78 && !(word % 31))
|
|
|
|
return 1;
|
|
|
|
else
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2006-09-04 04:09:18 +00:00
|
|
|
unsigned long unpack_object_header_gently(const unsigned char *buf, unsigned long len, enum object_type *type, unsigned long *sizep)
|
2005-06-02 00:54:59 +00:00
|
|
|
{
|
2006-09-01 22:17:01 +00:00
|
|
|
unsigned shift;
|
2006-07-11 19:48:08 +00:00
|
|
|
unsigned char c;
|
|
|
|
unsigned long size;
|
2006-09-01 22:17:01 +00:00
|
|
|
unsigned long used = 0;
|
|
|
|
|
|
|
|
c = buf[used++];
|
|
|
|
*type = (c >> 4) & 7;
|
|
|
|
size = c & 15;
|
|
|
|
shift = 4;
|
|
|
|
while (c & 0x80) {
|
|
|
|
if (len <= used)
|
|
|
|
return 0;
|
|
|
|
if (sizeof(long) * 8 <= shift)
|
|
|
|
return 0;
|
|
|
|
c = buf[used++];
|
|
|
|
size += (c & 0x7f) << shift;
|
|
|
|
shift += 7;
|
|
|
|
}
|
|
|
|
*sizep = size;
|
|
|
|
return used;
|
|
|
|
}
|
|
|
|
|
|
|
|
static int unpack_sha1_header(z_stream *stream, unsigned char *map, unsigned long mapsize, void *buffer, unsigned long bufsiz)
|
|
|
|
{
|
|
|
|
unsigned long size, used;
|
|
|
|
static const char valid_loose_object_type[8] = {
|
|
|
|
0, /* OBJ_EXT */
|
|
|
|
1, 1, 1, 1, /* "commit", "tree", "blob", "tag" */
|
|
|
|
0, /* "delta" and others are invalid in a loose object */
|
2006-07-11 19:48:08 +00:00
|
|
|
};
|
2006-09-01 22:17:01 +00:00
|
|
|
enum object_type type;
|
2006-07-11 19:48:08 +00:00
|
|
|
|
2005-06-02 00:54:59 +00:00
|
|
|
/* Get the data stream */
|
|
|
|
memset(stream, 0, sizeof(*stream));
|
|
|
|
stream->next_in = map;
|
|
|
|
stream->avail_in = mapsize;
|
|
|
|
stream->next_out = buffer;
|
2006-07-11 19:48:08 +00:00
|
|
|
stream->avail_out = bufsiz;
|
|
|
|
|
2006-07-17 22:04:47 +00:00
|
|
|
if (legacy_loose_object(map)) {
|
2006-07-11 19:48:08 +00:00
|
|
|
inflateInit(stream);
|
|
|
|
return inflate(stream, 0);
|
|
|
|
}
|
|
|
|
|
2007-05-09 18:42:42 +00:00
|
|
|
|
|
|
|
/*
|
|
|
|
* There used to be a second loose object header format which
|
|
|
|
* was meant to mimic the in-pack format, allowing for direct
|
|
|
|
* copy of the object data. This format turned up not to be
|
|
|
|
* really worth it and we don't write it any longer. But we
|
|
|
|
* can still read it.
|
|
|
|
*/
|
2006-09-01 22:17:01 +00:00
|
|
|
used = unpack_object_header_gently(map, mapsize, &type, &size);
|
|
|
|
if (!used || !valid_loose_object_type[type])
|
2006-07-11 19:48:08 +00:00
|
|
|
return -1;
|
2006-09-01 22:17:01 +00:00
|
|
|
map += used;
|
|
|
|
mapsize -= used;
|
2006-07-11 19:48:08 +00:00
|
|
|
|
|
|
|
/* Set up the stream for the rest.. */
|
|
|
|
stream->next_in = map;
|
|
|
|
stream->avail_in = mapsize;
|
2005-06-02 00:54:59 +00:00
|
|
|
inflateInit(stream);
|
2006-07-11 19:48:08 +00:00
|
|
|
|
|
|
|
/* And generate the fake traditional header */
|
2006-09-01 22:17:01 +00:00
|
|
|
stream->total_out = 1 + snprintf(buffer, bufsiz, "%s %lu",
|
2007-02-26 19:55:58 +00:00
|
|
|
typename(type), size);
|
2006-07-11 19:48:08 +00:00
|
|
|
return 0;
|
2005-06-02 00:54:59 +00:00
|
|
|
}
|
|
|
|
|
2007-03-05 08:21:37 +00:00
|
|
|
static void *unpack_sha1_rest(z_stream *stream, void *buffer, unsigned long size, const unsigned char *sha1)
|
2005-06-02 14:57:25 +00:00
|
|
|
{
|
|
|
|
int bytes = strlen(buffer) + 1;
|
2005-06-21 20:04:33 +00:00
|
|
|
unsigned char *buf = xmalloc(1+size);
|
2006-07-11 19:48:08 +00:00
|
|
|
unsigned long n;
|
2007-03-05 08:21:37 +00:00
|
|
|
int status = Z_OK;
|
2005-06-02 14:57:25 +00:00
|
|
|
|
2006-07-11 19:48:08 +00:00
|
|
|
n = stream->total_out - bytes;
|
|
|
|
if (n > size)
|
|
|
|
n = size;
|
|
|
|
memcpy(buf, (char *) buffer + bytes, n);
|
|
|
|
bytes = n;
|
2007-03-20 05:49:53 +00:00
|
|
|
if (bytes <= size) {
|
|
|
|
/*
|
|
|
|
* The above condition must be (bytes <= size), not
|
|
|
|
* (bytes < size). In other words, even though we
|
|
|
|
* expect no more output and set avail_out to zer0,
|
|
|
|
* the input zlib stream may have bytes that express
|
|
|
|
* "this concludes the stream", and we *do* want to
|
|
|
|
* eat that input.
|
|
|
|
*
|
|
|
|
* Otherwise we would not be able to test that we
|
|
|
|
* consumed all the input to reach the expected size;
|
|
|
|
* we also want to check that zlib tells us that all
|
|
|
|
* went well with status == Z_STREAM_END at the end.
|
|
|
|
*/
|
2005-06-02 14:57:25 +00:00
|
|
|
stream->next_out = buf + bytes;
|
|
|
|
stream->avail_out = size - bytes;
|
2007-03-05 08:21:37 +00:00
|
|
|
while (status == Z_OK)
|
|
|
|
status = inflate(stream, Z_FINISH);
|
2005-06-02 14:57:25 +00:00
|
|
|
}
|
|
|
|
buf[size] = 0;
|
2007-03-20 05:49:53 +00:00
|
|
|
if (status == Z_STREAM_END && !stream->avail_in) {
|
2007-03-05 08:21:37 +00:00
|
|
|
inflateEnd(stream);
|
|
|
|
return buf;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (status < 0)
|
|
|
|
error("corrupt loose object '%s'", sha1_to_hex(sha1));
|
|
|
|
else if (stream->avail_in)
|
|
|
|
error("garbage at end of loose object '%s'",
|
|
|
|
sha1_to_hex(sha1));
|
|
|
|
free(buf);
|
|
|
|
return NULL;
|
2005-06-02 14:57:25 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* We used to just use "sscanf()", but that's actually way
|
|
|
|
* too permissive for what we want to check. So do an anal
|
|
|
|
* object header parse by hand.
|
|
|
|
*/
|
2007-02-26 19:55:59 +00:00
|
|
|
static int parse_sha1_header(const char *hdr, unsigned long *sizep)
|
2005-06-02 14:57:25 +00:00
|
|
|
{
|
2007-02-26 19:55:59 +00:00
|
|
|
char type[10];
|
2005-06-02 14:57:25 +00:00
|
|
|
int i;
|
|
|
|
unsigned long size;
|
|
|
|
|
|
|
|
/*
|
2007-06-07 07:04:01 +00:00
|
|
|
* The type can be at most ten bytes (including the
|
2005-06-02 14:57:25 +00:00
|
|
|
* terminating '\0' that we add), and is followed by
|
2007-02-26 19:55:59 +00:00
|
|
|
* a space.
|
2005-06-02 14:57:25 +00:00
|
|
|
*/
|
2007-02-26 19:55:59 +00:00
|
|
|
i = 0;
|
2005-06-02 14:57:25 +00:00
|
|
|
for (;;) {
|
|
|
|
char c = *hdr++;
|
|
|
|
if (c == ' ')
|
|
|
|
break;
|
2007-02-26 19:55:59 +00:00
|
|
|
type[i++] = c;
|
|
|
|
if (i >= sizeof(type))
|
2005-06-02 14:57:25 +00:00
|
|
|
return -1;
|
|
|
|
}
|
2007-02-26 19:55:59 +00:00
|
|
|
type[i] = 0;
|
2005-06-02 14:57:25 +00:00
|
|
|
|
|
|
|
/*
|
|
|
|
* The length must follow immediately, and be in canonical
|
|
|
|
* decimal format (ie "010" is not valid).
|
|
|
|
*/
|
|
|
|
size = *hdr++ - '0';
|
|
|
|
if (size > 9)
|
|
|
|
return -1;
|
|
|
|
if (size) {
|
|
|
|
for (;;) {
|
|
|
|
unsigned long c = *hdr - '0';
|
|
|
|
if (c > 9)
|
|
|
|
break;
|
|
|
|
hdr++;
|
|
|
|
size = size * 10 + c;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
*sizep = size;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* The length must be followed by a zero byte
|
|
|
|
*/
|
2007-02-26 19:55:59 +00:00
|
|
|
return *hdr ? -1 : type_from_string(type);
|
2005-06-02 14:57:25 +00:00
|
|
|
}
|
|
|
|
|
2007-03-05 08:21:37 +00:00
|
|
|
static void *unpack_sha1_file(void *map, unsigned long mapsize, enum object_type *type, unsigned long *size, const unsigned char *sha1)
|
2005-04-18 20:04:43 +00:00
|
|
|
{
|
2005-06-02 14:57:25 +00:00
|
|
|
int ret;
|
2005-04-18 20:04:43 +00:00
|
|
|
z_stream stream;
|
2005-06-02 14:57:25 +00:00
|
|
|
char hdr[8192];
|
2005-04-18 20:04:43 +00:00
|
|
|
|
2005-06-02 14:57:25 +00:00
|
|
|
ret = unpack_sha1_header(&stream, map, mapsize, hdr, sizeof(hdr));
|
2007-02-26 19:55:59 +00:00
|
|
|
if (ret < Z_OK || (*type = parse_sha1_header(hdr, size)) < 0)
|
2005-04-18 20:04:43 +00:00
|
|
|
return NULL;
|
|
|
|
|
2007-03-05 08:21:37 +00:00
|
|
|
return unpack_sha1_rest(&stream, hdr, *size, sha1);
|
2005-04-18 20:04:43 +00:00
|
|
|
}
|
|
|
|
|
2007-04-16 16:31:56 +00:00
|
|
|
unsigned long get_size_from_delta(struct packed_git *p,
|
|
|
|
struct pack_window **w_curs,
|
|
|
|
off_t curpos)
|
|
|
|
{
|
|
|
|
const unsigned char *data;
|
|
|
|
unsigned char delta_head[20], *in;
|
|
|
|
z_stream stream;
|
|
|
|
int st;
|
|
|
|
|
|
|
|
memset(&stream, 0, sizeof(stream));
|
|
|
|
stream.next_out = delta_head;
|
|
|
|
stream.avail_out = sizeof(delta_head);
|
|
|
|
|
|
|
|
inflateInit(&stream);
|
|
|
|
do {
|
|
|
|
in = use_pack(p, w_curs, curpos, &stream.avail_in);
|
|
|
|
stream.next_in = in;
|
|
|
|
st = inflate(&stream, Z_FINISH);
|
|
|
|
curpos += stream.next_in - in;
|
|
|
|
} while ((st == Z_OK || st == Z_BUF_ERROR) &&
|
|
|
|
stream.total_out < sizeof(delta_head));
|
|
|
|
inflateEnd(&stream);
|
|
|
|
if ((st != Z_STREAM_END) && stream.total_out != sizeof(delta_head))
|
|
|
|
die("delta data unpack-initial failed");
|
|
|
|
|
|
|
|
/* Examine the initial part of the delta to figure out
|
|
|
|
* the result size.
|
|
|
|
*/
|
|
|
|
data = delta_head;
|
|
|
|
|
|
|
|
/* ignore base size */
|
|
|
|
get_delta_hdr_size(&data, delta_head+sizeof(delta_head));
|
|
|
|
|
|
|
|
/* Read the result size */
|
|
|
|
return get_delta_hdr_size(&data, delta_head+sizeof(delta_head));
|
|
|
|
}
|
|
|
|
|
2007-03-07 01:44:30 +00:00
|
|
|
static off_t get_delta_base(struct packed_git *p,
|
2006-12-23 07:34:08 +00:00
|
|
|
struct pack_window **w_curs,
|
2007-03-07 01:44:30 +00:00
|
|
|
off_t *curpos,
|
2007-02-26 19:55:59 +00:00
|
|
|
enum object_type type,
|
2007-03-07 01:44:30 +00:00
|
|
|
off_t delta_obj_offset)
|
2006-09-21 04:06:49 +00:00
|
|
|
{
|
2007-02-26 19:55:56 +00:00
|
|
|
unsigned char *base_info = use_pack(p, w_curs, *curpos, NULL);
|
2007-03-07 01:44:30 +00:00
|
|
|
off_t base_offset;
|
2006-09-21 04:06:49 +00:00
|
|
|
|
2006-12-23 07:34:18 +00:00
|
|
|
/* use_pack() assured us we have [base_info, base_info + 20)
|
|
|
|
* as a range that we can look at without walking off the
|
|
|
|
* end of the mapped window. Its actually the hash size
|
|
|
|
* that is assured. An OFS_DELTA longer than the hash size
|
|
|
|
* is stupid, as then a REF_DELTA would be smaller to store.
|
|
|
|
*/
|
2007-02-26 19:55:59 +00:00
|
|
|
if (type == OBJ_OFS_DELTA) {
|
2006-09-21 04:06:49 +00:00
|
|
|
unsigned used = 0;
|
|
|
|
unsigned char c = base_info[used++];
|
|
|
|
base_offset = c & 127;
|
|
|
|
while (c & 128) {
|
|
|
|
base_offset += 1;
|
2007-04-09 05:06:29 +00:00
|
|
|
if (!base_offset || MSB(base_offset, 7))
|
2006-09-21 04:06:49 +00:00
|
|
|
die("offset value overflow for delta base object");
|
|
|
|
c = base_info[used++];
|
|
|
|
base_offset = (base_offset << 7) + (c & 127);
|
|
|
|
}
|
|
|
|
base_offset = delta_obj_offset - base_offset;
|
|
|
|
if (base_offset >= delta_obj_offset)
|
|
|
|
die("delta base offset out of bound");
|
2007-02-26 19:55:56 +00:00
|
|
|
*curpos += used;
|
2007-02-26 19:55:59 +00:00
|
|
|
} else if (type == OBJ_REF_DELTA) {
|
2006-09-21 04:06:49 +00:00
|
|
|
/* The base entry _must_ be in the same pack */
|
|
|
|
base_offset = find_pack_entry_one(base_info, p);
|
|
|
|
if (!base_offset)
|
|
|
|
die("failed to find delta-pack base object %s",
|
|
|
|
sha1_to_hex(base_info));
|
2007-02-26 19:55:56 +00:00
|
|
|
*curpos += 20;
|
2006-09-21 04:06:49 +00:00
|
|
|
} else
|
|
|
|
die("I am totally screwed");
|
2007-02-26 19:55:56 +00:00
|
|
|
return base_offset;
|
2006-09-21 04:06:49 +00:00
|
|
|
}
|
|
|
|
|
2005-07-01 00:15:39 +00:00
|
|
|
/* forward declaration for a mutually recursive function */
|
2007-03-07 01:44:30 +00:00
|
|
|
static int packed_object_info(struct packed_git *p, off_t offset,
|
2007-02-26 19:55:59 +00:00
|
|
|
unsigned long *sizep);
|
2005-07-01 00:15:39 +00:00
|
|
|
|
2006-09-21 04:05:37 +00:00
|
|
|
static int packed_delta_info(struct packed_git *p,
|
2006-12-23 07:34:08 +00:00
|
|
|
struct pack_window **w_curs,
|
2007-03-07 01:44:30 +00:00
|
|
|
off_t curpos,
|
2007-02-26 19:55:59 +00:00
|
|
|
enum object_type type,
|
2007-03-07 01:44:30 +00:00
|
|
|
off_t obj_offset,
|
2006-09-21 04:05:37 +00:00
|
|
|
unsigned long *sizep)
|
2005-06-28 06:58:08 +00:00
|
|
|
{
|
2007-03-07 01:44:30 +00:00
|
|
|
off_t base_offset;
|
2005-07-01 00:15:39 +00:00
|
|
|
|
2007-02-26 19:55:59 +00:00
|
|
|
base_offset = get_delta_base(p, w_curs, &curpos, type, obj_offset);
|
|
|
|
type = packed_object_info(p, base_offset, NULL);
|
2005-07-01 00:15:39 +00:00
|
|
|
|
2005-07-01 00:13:07 +00:00
|
|
|
/* We choose to only get the type of the base object and
|
|
|
|
* ignore potentially corrupt pack file that expects the delta
|
|
|
|
* based on a base with a wrong size. This saves tons of
|
|
|
|
* inflate() calls.
|
|
|
|
*/
|
2007-04-16 16:31:56 +00:00
|
|
|
if (sizep)
|
|
|
|
*sizep = get_size_from_delta(p, w_curs, curpos);
|
2007-02-26 19:55:59 +00:00
|
|
|
|
|
|
|
return type;
|
2005-06-28 06:58:08 +00:00
|
|
|
}
|
|
|
|
|
2007-02-26 19:55:56 +00:00
|
|
|
static int unpack_object_header(struct packed_git *p,
|
|
|
|
struct pack_window **w_curs,
|
2007-03-07 01:44:30 +00:00
|
|
|
off_t *curpos,
|
2007-02-26 19:55:56 +00:00
|
|
|
unsigned long *sizep)
|
2005-06-28 21:21:02 +00:00
|
|
|
{
|
2006-12-23 07:34:08 +00:00
|
|
|
unsigned char *base;
|
|
|
|
unsigned int left;
|
2006-09-01 22:17:01 +00:00
|
|
|
unsigned long used;
|
2007-02-26 19:55:56 +00:00
|
|
|
enum object_type type;
|
2005-06-28 21:21:02 +00:00
|
|
|
|
2006-12-23 07:34:18 +00:00
|
|
|
/* use_pack() assures us we have [base, base + 20) available
|
|
|
|
* as a range that we can look at at. (Its actually the hash
|
2007-02-04 04:49:16 +00:00
|
|
|
* size that is assured.) With our object header encoding
|
2006-12-23 07:34:18 +00:00
|
|
|
* the maximum deflated object size is 2^137, which is just
|
|
|
|
* insane, so we know won't exceed what we have been given.
|
|
|
|
*/
|
2007-02-26 19:55:56 +00:00
|
|
|
base = use_pack(p, w_curs, *curpos, &left);
|
|
|
|
used = unpack_object_header_gently(base, left, &type, sizep);
|
2006-09-01 22:17:01 +00:00
|
|
|
if (!used)
|
|
|
|
die("object offset outside of pack file");
|
2007-02-26 19:55:56 +00:00
|
|
|
*curpos += used;
|
2006-09-01 22:17:01 +00:00
|
|
|
|
2007-02-26 19:55:56 +00:00
|
|
|
return type;
|
2005-06-28 21:21:02 +00:00
|
|
|
}
|
|
|
|
|
2007-02-26 19:55:59 +00:00
|
|
|
const char *packed_object_info_detail(struct packed_git *p,
|
2007-03-07 01:44:30 +00:00
|
|
|
off_t obj_offset,
|
2007-02-26 19:55:59 +00:00
|
|
|
unsigned long *size,
|
|
|
|
unsigned long *store_size,
|
|
|
|
unsigned int *delta_chain_length,
|
|
|
|
unsigned char *base_sha1)
|
2005-07-01 00:17:20 +00:00
|
|
|
{
|
2006-12-23 07:34:08 +00:00
|
|
|
struct pack_window *w_curs = NULL;
|
2007-03-07 01:44:30 +00:00
|
|
|
off_t curpos;
|
|
|
|
unsigned long dummy;
|
2006-09-21 04:05:37 +00:00
|
|
|
unsigned char *next_sha1;
|
2007-02-26 19:55:59 +00:00
|
|
|
enum object_type type;
|
2005-07-01 00:17:20 +00:00
|
|
|
|
2006-09-21 04:05:37 +00:00
|
|
|
*delta_chain_length = 0;
|
2007-02-26 19:55:56 +00:00
|
|
|
curpos = obj_offset;
|
2007-02-26 19:55:59 +00:00
|
|
|
type = unpack_object_header(p, &w_curs, &curpos, size);
|
2006-09-21 04:05:37 +00:00
|
|
|
|
|
|
|
for (;;) {
|
2007-02-26 19:55:59 +00:00
|
|
|
switch (type) {
|
2006-09-21 04:05:37 +00:00
|
|
|
default:
|
2006-12-20 18:34:05 +00:00
|
|
|
die("pack %s contains unknown object type %d",
|
2007-02-26 19:55:59 +00:00
|
|
|
p->pack_name, type);
|
2006-09-21 04:05:37 +00:00
|
|
|
case OBJ_COMMIT:
|
|
|
|
case OBJ_TREE:
|
|
|
|
case OBJ_BLOB:
|
|
|
|
case OBJ_TAG:
|
|
|
|
*store_size = 0; /* notyet */
|
2006-12-23 07:34:08 +00:00
|
|
|
unuse_pack(&w_curs);
|
2007-02-26 19:55:59 +00:00
|
|
|
return typename(type);
|
2006-09-21 04:06:49 +00:00
|
|
|
case OBJ_OFS_DELTA:
|
2007-02-26 19:55:59 +00:00
|
|
|
obj_offset = get_delta_base(p, &w_curs, &curpos, type, obj_offset);
|
2006-09-21 04:06:49 +00:00
|
|
|
if (*delta_chain_length == 0) {
|
2007-02-26 19:55:56 +00:00
|
|
|
/* TODO: find base_sha1 as pointed by curpos */
|
2007-03-07 01:44:08 +00:00
|
|
|
hashclr(base_sha1);
|
2006-09-21 04:06:49 +00:00
|
|
|
}
|
|
|
|
break;
|
|
|
|
case OBJ_REF_DELTA:
|
2007-02-26 19:55:56 +00:00
|
|
|
next_sha1 = use_pack(p, &w_curs, curpos, NULL);
|
2006-09-21 04:05:37 +00:00
|
|
|
if (*delta_chain_length == 0)
|
|
|
|
hashcpy(base_sha1, next_sha1);
|
2007-02-26 19:55:56 +00:00
|
|
|
obj_offset = find_pack_entry_one(next_sha1, p);
|
2006-09-21 04:05:37 +00:00
|
|
|
break;
|
|
|
|
}
|
|
|
|
(*delta_chain_length)++;
|
2007-02-26 19:55:56 +00:00
|
|
|
curpos = obj_offset;
|
2007-02-26 19:55:59 +00:00
|
|
|
type = unpack_object_header(p, &w_curs, &curpos, &dummy);
|
2005-07-01 00:17:20 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2007-03-07 01:44:30 +00:00
|
|
|
static int packed_object_info(struct packed_git *p, off_t obj_offset,
|
2007-02-26 19:55:59 +00:00
|
|
|
unsigned long *sizep)
|
2005-06-27 10:35:33 +00:00
|
|
|
{
|
2006-12-23 07:34:08 +00:00
|
|
|
struct pack_window *w_curs = NULL;
|
2007-03-07 01:44:30 +00:00
|
|
|
unsigned long size;
|
|
|
|
off_t curpos = obj_offset;
|
2007-02-26 19:55:59 +00:00
|
|
|
enum object_type type;
|
2005-06-28 06:58:08 +00:00
|
|
|
|
2007-02-26 19:55:59 +00:00
|
|
|
type = unpack_object_header(p, &w_curs, &curpos, &size);
|
2005-06-28 06:58:08 +00:00
|
|
|
|
2007-02-26 19:55:59 +00:00
|
|
|
switch (type) {
|
2006-09-21 04:06:49 +00:00
|
|
|
case OBJ_OFS_DELTA:
|
|
|
|
case OBJ_REF_DELTA:
|
2007-02-26 19:55:59 +00:00
|
|
|
type = packed_delta_info(p, &w_curs, curpos,
|
|
|
|
type, obj_offset, sizep);
|
2007-02-26 19:55:56 +00:00
|
|
|
break;
|
2005-06-28 21:21:02 +00:00
|
|
|
case OBJ_COMMIT:
|
|
|
|
case OBJ_TREE:
|
|
|
|
case OBJ_BLOB:
|
|
|
|
case OBJ_TAG:
|
2007-02-26 19:55:56 +00:00
|
|
|
if (sizep)
|
|
|
|
*sizep = size;
|
2005-06-28 16:58:23 +00:00
|
|
|
break;
|
2005-06-27 10:35:33 +00:00
|
|
|
default:
|
2006-12-20 18:34:05 +00:00
|
|
|
die("pack %s contains unknown object type %d",
|
2007-02-26 19:55:59 +00:00
|
|
|
p->pack_name, type);
|
2005-06-27 10:35:33 +00:00
|
|
|
}
|
2007-02-26 19:55:56 +00:00
|
|
|
unuse_pack(&w_curs);
|
2007-02-26 19:55:59 +00:00
|
|
|
return type;
|
2005-06-27 10:35:33 +00:00
|
|
|
}
|
|
|
|
|
2006-08-26 08:12:27 +00:00
|
|
|
static void *unpack_compressed_entry(struct packed_git *p,
|
2006-12-23 07:34:08 +00:00
|
|
|
struct pack_window **w_curs,
|
2007-03-07 01:44:30 +00:00
|
|
|
off_t curpos,
|
2006-08-26 08:12:27 +00:00
|
|
|
unsigned long size)
|
2006-08-26 08:10:43 +00:00
|
|
|
{
|
|
|
|
int st;
|
|
|
|
z_stream stream;
|
2006-12-23 07:34:13 +00:00
|
|
|
unsigned char *buffer, *in;
|
2006-08-26 08:10:43 +00:00
|
|
|
|
|
|
|
buffer = xmalloc(size + 1);
|
|
|
|
buffer[size] = 0;
|
|
|
|
memset(&stream, 0, sizeof(stream));
|
|
|
|
stream.next_out = buffer;
|
|
|
|
stream.avail_out = size;
|
|
|
|
|
|
|
|
inflateInit(&stream);
|
2006-12-23 07:34:13 +00:00
|
|
|
do {
|
2007-02-26 19:55:56 +00:00
|
|
|
in = use_pack(p, w_curs, curpos, &stream.avail_in);
|
2006-12-23 07:34:13 +00:00
|
|
|
stream.next_in = in;
|
|
|
|
st = inflate(&stream, Z_FINISH);
|
2007-02-26 19:55:56 +00:00
|
|
|
curpos += stream.next_in - in;
|
2006-12-23 07:34:13 +00:00
|
|
|
} while (st == Z_OK || st == Z_BUF_ERROR);
|
2006-08-26 08:10:43 +00:00
|
|
|
inflateEnd(&stream);
|
|
|
|
if ((st != Z_STREAM_END) || stream.total_out != size) {
|
|
|
|
free(buffer);
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
|
|
|
return buffer;
|
|
|
|
}
|
|
|
|
|
2007-03-17 19:44:06 +00:00
|
|
|
#define MAX_DELTA_CACHE (256)
|
|
|
|
|
2007-03-19 05:14:37 +00:00
|
|
|
static size_t delta_base_cached;
|
2007-03-19 20:31:04 +00:00
|
|
|
|
|
|
|
static struct delta_base_cache_lru_list {
|
|
|
|
struct delta_base_cache_lru_list *prev;
|
|
|
|
struct delta_base_cache_lru_list *next;
|
|
|
|
} delta_base_cache_lru = { &delta_base_cache_lru, &delta_base_cache_lru };
|
|
|
|
|
2007-03-17 19:44:06 +00:00
|
|
|
static struct delta_base_cache_entry {
|
2007-03-19 20:31:04 +00:00
|
|
|
struct delta_base_cache_lru_list lru;
|
|
|
|
void *data;
|
2007-03-17 19:44:06 +00:00
|
|
|
struct packed_git *p;
|
|
|
|
off_t base_offset;
|
|
|
|
unsigned long size;
|
|
|
|
enum object_type type;
|
|
|
|
} delta_base_cache[MAX_DELTA_CACHE];
|
|
|
|
|
|
|
|
static unsigned long pack_entry_hash(struct packed_git *p, off_t base_offset)
|
|
|
|
{
|
|
|
|
unsigned long hash;
|
|
|
|
|
|
|
|
hash = (unsigned long)p + (unsigned long)base_offset;
|
|
|
|
hash += (hash >> 8) + (hash >> 16);
|
2007-03-19 20:28:51 +00:00
|
|
|
return hash % MAX_DELTA_CACHE;
|
2007-03-17 19:44:06 +00:00
|
|
|
}
|
|
|
|
|
2007-03-17 19:42:15 +00:00
|
|
|
static void *cache_or_unpack_entry(struct packed_git *p, off_t base_offset,
|
2007-03-18 01:13:57 +00:00
|
|
|
unsigned long *base_size, enum object_type *type, int keep_cache)
|
2007-03-17 19:42:15 +00:00
|
|
|
{
|
2007-03-17 19:44:06 +00:00
|
|
|
void *ret;
|
|
|
|
unsigned long hash = pack_entry_hash(p, base_offset);
|
|
|
|
struct delta_base_cache_entry *ent = delta_base_cache + hash;
|
|
|
|
|
|
|
|
ret = ent->data;
|
|
|
|
if (ret && ent->p == p && ent->base_offset == base_offset)
|
|
|
|
goto found_cache_entry;
|
2007-03-17 19:42:15 +00:00
|
|
|
return unpack_entry(p, base_offset, type, base_size);
|
2007-03-17 19:44:06 +00:00
|
|
|
|
|
|
|
found_cache_entry:
|
2007-03-19 05:14:37 +00:00
|
|
|
if (!keep_cache) {
|
2007-03-18 01:13:57 +00:00
|
|
|
ent->data = NULL;
|
2007-03-19 20:31:04 +00:00
|
|
|
ent->lru.next->prev = ent->lru.prev;
|
|
|
|
ent->lru.prev->next = ent->lru.next;
|
2007-03-19 05:14:37 +00:00
|
|
|
delta_base_cached -= ent->size;
|
2007-09-15 22:32:36 +00:00
|
|
|
} else {
|
|
|
|
ret = xmemdupz(ent->data, ent->size);
|
2007-03-18 01:13:57 +00:00
|
|
|
}
|
2007-03-17 19:44:06 +00:00
|
|
|
*type = ent->type;
|
|
|
|
*base_size = ent->size;
|
|
|
|
return ret;
|
2007-03-17 19:42:15 +00:00
|
|
|
}
|
|
|
|
|
2007-03-19 05:14:37 +00:00
|
|
|
static inline void release_delta_base_cache(struct delta_base_cache_entry *ent)
|
|
|
|
{
|
|
|
|
if (ent->data) {
|
|
|
|
free(ent->data);
|
|
|
|
ent->data = NULL;
|
2007-03-19 20:31:04 +00:00
|
|
|
ent->lru.next->prev = ent->lru.prev;
|
|
|
|
ent->lru.prev->next = ent->lru.next;
|
2007-03-19 05:14:37 +00:00
|
|
|
delta_base_cached -= ent->size;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2007-03-17 19:42:15 +00:00
|
|
|
static void add_delta_base_cache(struct packed_git *p, off_t base_offset,
|
|
|
|
void *base, unsigned long base_size, enum object_type type)
|
|
|
|
{
|
2007-03-19 20:31:04 +00:00
|
|
|
unsigned long hash = pack_entry_hash(p, base_offset);
|
2007-03-17 19:44:06 +00:00
|
|
|
struct delta_base_cache_entry *ent = delta_base_cache + hash;
|
2007-03-19 20:31:04 +00:00
|
|
|
struct delta_base_cache_lru_list *lru;
|
2007-03-17 19:44:06 +00:00
|
|
|
|
2007-03-19 05:14:37 +00:00
|
|
|
release_delta_base_cache(ent);
|
|
|
|
delta_base_cached += base_size;
|
2007-03-19 20:31:04 +00:00
|
|
|
|
|
|
|
for (lru = delta_base_cache_lru.next;
|
|
|
|
delta_base_cached > delta_base_cache_limit
|
|
|
|
&& lru != &delta_base_cache_lru;
|
|
|
|
lru = lru->next) {
|
|
|
|
struct delta_base_cache_entry *f = (void *)lru;
|
2007-03-19 05:14:37 +00:00
|
|
|
if (f->type == OBJ_BLOB)
|
|
|
|
release_delta_base_cache(f);
|
|
|
|
}
|
2007-03-19 20:31:04 +00:00
|
|
|
for (lru = delta_base_cache_lru.next;
|
|
|
|
delta_base_cached > delta_base_cache_limit
|
|
|
|
&& lru != &delta_base_cache_lru;
|
|
|
|
lru = lru->next) {
|
|
|
|
struct delta_base_cache_entry *f = (void *)lru;
|
|
|
|
release_delta_base_cache(f);
|
|
|
|
}
|
2007-03-19 05:14:37 +00:00
|
|
|
|
2007-03-17 19:44:06 +00:00
|
|
|
ent->p = p;
|
|
|
|
ent->base_offset = base_offset;
|
|
|
|
ent->type = type;
|
|
|
|
ent->data = base;
|
|
|
|
ent->size = base_size;
|
2007-03-19 20:31:04 +00:00
|
|
|
ent->lru.next = &delta_base_cache_lru;
|
|
|
|
ent->lru.prev = delta_base_cache_lru.prev;
|
|
|
|
delta_base_cache_lru.prev->next = &ent->lru;
|
|
|
|
delta_base_cache_lru.prev = &ent->lru;
|
2007-03-17 19:42:15 +00:00
|
|
|
}
|
|
|
|
|
2006-08-26 08:12:27 +00:00
|
|
|
static void *unpack_delta_entry(struct packed_git *p,
|
2006-12-23 07:34:08 +00:00
|
|
|
struct pack_window **w_curs,
|
2007-03-07 01:44:30 +00:00
|
|
|
off_t curpos,
|
2005-06-27 10:35:33 +00:00
|
|
|
unsigned long delta_size,
|
2007-03-07 01:44:30 +00:00
|
|
|
off_t obj_offset,
|
2007-02-26 19:55:59 +00:00
|
|
|
enum object_type *type,
|
2006-08-26 08:12:27 +00:00
|
|
|
unsigned long *sizep)
|
2005-06-27 10:35:33 +00:00
|
|
|
{
|
2006-08-26 08:11:02 +00:00
|
|
|
void *delta_data, *result, *base;
|
2007-03-07 01:44:30 +00:00
|
|
|
unsigned long base_size;
|
|
|
|
off_t base_offset;
|
2006-09-21 04:05:37 +00:00
|
|
|
|
2007-02-26 19:55:59 +00:00
|
|
|
base_offset = get_delta_base(p, w_curs, &curpos, *type, obj_offset);
|
2007-03-18 01:13:57 +00:00
|
|
|
base = cache_or_unpack_entry(p, base_offset, &base_size, type, 0);
|
2006-03-19 21:43:42 +00:00
|
|
|
if (!base)
|
2007-03-07 01:44:30 +00:00
|
|
|
die("failed to read delta base object"
|
|
|
|
" at %"PRIuMAX" from %s",
|
|
|
|
(uintmax_t)base_offset, p->pack_name);
|
2006-03-19 21:43:42 +00:00
|
|
|
|
2007-02-26 19:55:56 +00:00
|
|
|
delta_data = unpack_compressed_entry(p, w_curs, curpos, delta_size);
|
2007-08-25 07:26:04 +00:00
|
|
|
if (!delta_data)
|
|
|
|
die("failed to unpack compressed delta"
|
|
|
|
" at %"PRIuMAX" from %s",
|
|
|
|
(uintmax_t)curpos, p->pack_name);
|
2005-06-27 10:35:33 +00:00
|
|
|
result = patch_delta(base, base_size,
|
|
|
|
delta_data, delta_size,
|
2007-02-26 19:55:59 +00:00
|
|
|
sizep);
|
2005-06-27 10:35:33 +00:00
|
|
|
if (!result)
|
|
|
|
die("failed to apply delta");
|
|
|
|
free(delta_data);
|
2007-03-17 19:42:15 +00:00
|
|
|
add_delta_base_cache(p, base_offset, base, base_size, *type);
|
2005-06-27 10:35:33 +00:00
|
|
|
return result;
|
|
|
|
}
|
|
|
|
|
2007-03-07 01:44:30 +00:00
|
|
|
void *unpack_entry(struct packed_git *p, off_t obj_offset,
|
2007-02-26 19:55:59 +00:00
|
|
|
enum object_type *type, unsigned long *sizep)
|
2005-06-27 10:35:33 +00:00
|
|
|
{
|
2006-12-23 07:34:08 +00:00
|
|
|
struct pack_window *w_curs = NULL;
|
2007-03-07 01:44:30 +00:00
|
|
|
off_t curpos = obj_offset;
|
2007-02-26 19:55:59 +00:00
|
|
|
void *data;
|
2005-06-27 10:35:33 +00:00
|
|
|
|
2007-02-26 19:55:59 +00:00
|
|
|
*type = unpack_object_header(p, &w_curs, &curpos, sizep);
|
|
|
|
switch (*type) {
|
2006-09-21 04:06:49 +00:00
|
|
|
case OBJ_OFS_DELTA:
|
|
|
|
case OBJ_REF_DELTA:
|
2007-02-26 19:55:59 +00:00
|
|
|
data = unpack_delta_entry(p, &w_curs, curpos, *sizep,
|
|
|
|
obj_offset, type, sizep);
|
2006-12-23 07:33:25 +00:00
|
|
|
break;
|
2005-06-28 21:21:02 +00:00
|
|
|
case OBJ_COMMIT:
|
|
|
|
case OBJ_TREE:
|
|
|
|
case OBJ_BLOB:
|
|
|
|
case OBJ_TAG:
|
2007-02-26 19:55:59 +00:00
|
|
|
data = unpack_compressed_entry(p, &w_curs, curpos, *sizep);
|
2006-12-23 07:33:25 +00:00
|
|
|
break;
|
2005-06-27 10:35:33 +00:00
|
|
|
default:
|
2007-02-26 19:55:59 +00:00
|
|
|
die("unknown object type %i in %s", *type, p->pack_name);
|
2005-06-27 10:35:33 +00:00
|
|
|
}
|
2006-12-23 07:34:08 +00:00
|
|
|
unuse_pack(&w_curs);
|
2007-02-26 19:55:59 +00:00
|
|
|
return data;
|
2005-06-27 10:35:33 +00:00
|
|
|
}
|
|
|
|
|
2007-05-26 05:24:19 +00:00
|
|
|
const unsigned char *nth_packed_object_sha1(struct packed_git *p,
|
2007-04-04 20:49:04 +00:00
|
|
|
uint32_t n)
|
2005-06-28 21:56:57 +00:00
|
|
|
{
|
2007-03-16 20:42:50 +00:00
|
|
|
const unsigned char *index = p->index_data;
|
2007-05-26 05:24:19 +00:00
|
|
|
if (!index) {
|
|
|
|
if (open_pack_index(p))
|
|
|
|
return NULL;
|
|
|
|
index = p->index_data;
|
|
|
|
}
|
2007-04-09 05:06:28 +00:00
|
|
|
if (n >= p->num_objects)
|
2007-04-04 20:49:04 +00:00
|
|
|
return NULL;
|
2007-04-09 05:06:35 +00:00
|
|
|
index += 4 * 256;
|
|
|
|
if (p->index_version == 1) {
|
|
|
|
return index + 24 * n + 4;
|
|
|
|
} else {
|
|
|
|
index += 8;
|
|
|
|
return index + 20 * n;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
static off_t nth_packed_object_offset(const struct packed_git *p, uint32_t n)
|
|
|
|
{
|
|
|
|
const unsigned char *index = p->index_data;
|
|
|
|
index += 4 * 256;
|
|
|
|
if (p->index_version == 1) {
|
|
|
|
return ntohl(*((uint32_t *)(index + 24 * n)));
|
|
|
|
} else {
|
|
|
|
uint32_t off;
|
|
|
|
index += 8 + p->num_objects * (20 + 4);
|
|
|
|
off = ntohl(*((uint32_t *)(index + 4 * n)));
|
|
|
|
if (!(off & 0x80000000))
|
|
|
|
return off;
|
|
|
|
index += p->num_objects * 4 + (off & 0x7fffffff) * 8;
|
|
|
|
return (((uint64_t)ntohl(*((uint32_t *)(index + 0)))) << 32) |
|
|
|
|
ntohl(*((uint32_t *)(index + 4)));
|
|
|
|
}
|
2005-06-28 21:56:57 +00:00
|
|
|
}
|
|
|
|
|
2007-03-07 01:44:30 +00:00
|
|
|
off_t find_pack_entry_one(const unsigned char *sha1,
|
2006-09-21 04:05:37 +00:00
|
|
|
struct packed_git *p)
|
2005-06-27 10:35:33 +00:00
|
|
|
{
|
2007-03-16 20:42:50 +00:00
|
|
|
const uint32_t *level1_ofs = p->index_data;
|
|
|
|
const unsigned char *index = p->index_data;
|
2007-04-09 05:06:35 +00:00
|
|
|
unsigned hi, lo;
|
2007-03-16 20:42:50 +00:00
|
|
|
|
2007-05-26 05:24:19 +00:00
|
|
|
if (!index) {
|
|
|
|
if (open_pack_index(p))
|
|
|
|
return 0;
|
|
|
|
level1_ofs = p->index_data;
|
|
|
|
index = p->index_data;
|
|
|
|
}
|
2007-04-09 05:06:35 +00:00
|
|
|
if (p->index_version > 1) {
|
|
|
|
level1_ofs += 2;
|
|
|
|
index += 8;
|
|
|
|
}
|
2007-03-16 20:42:50 +00:00
|
|
|
index += 4 * 256;
|
2007-04-09 05:06:35 +00:00
|
|
|
hi = ntohl(level1_ofs[*sha1]);
|
|
|
|
lo = ((*sha1 == 0x0) ? 0 : ntohl(level1_ofs[*sha1 - 1]));
|
2005-06-27 10:35:33 +00:00
|
|
|
|
|
|
|
do {
|
2007-04-09 05:06:35 +00:00
|
|
|
unsigned mi = (lo + hi) / 2;
|
|
|
|
unsigned x = (p->index_version > 1) ? (mi * 20) : (mi * 24 + 4);
|
|
|
|
int cmp = hashcmp(index + x, sha1);
|
2006-09-21 04:05:37 +00:00
|
|
|
if (!cmp)
|
2007-04-09 05:06:35 +00:00
|
|
|
return nth_packed_object_offset(p, mi);
|
2005-06-27 10:35:33 +00:00
|
|
|
if (cmp > 0)
|
|
|
|
hi = mi;
|
|
|
|
else
|
|
|
|
lo = mi+1;
|
|
|
|
} while (lo < hi);
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2007-09-17 06:15:19 +00:00
|
|
|
int matches_pack_name(struct packed_git *p, const char *name)
|
2006-10-29 09:37:11 +00:00
|
|
|
{
|
|
|
|
const char *last_c, *c;
|
|
|
|
|
2007-09-17 06:15:19 +00:00
|
|
|
if (!strcmp(p->pack_name, name))
|
|
|
|
return 1;
|
2006-10-29 09:37:11 +00:00
|
|
|
|
|
|
|
for (c = p->pack_name, last_c = c; *c;)
|
|
|
|
if (*c == '/')
|
|
|
|
last_c = ++c;
|
|
|
|
else
|
|
|
|
++c;
|
2007-09-17 06:15:19 +00:00
|
|
|
if (!strcmp(last_c, name))
|
|
|
|
return 1;
|
2006-10-29 09:37:11 +00:00
|
|
|
|
2007-09-17 06:15:19 +00:00
|
|
|
return 0;
|
2006-10-29 09:37:11 +00:00
|
|
|
}
|
|
|
|
|
2006-09-06 09:12:09 +00:00
|
|
|
static int find_pack_entry(const unsigned char *sha1, struct pack_entry *e, const char **ignore_packed)
|
2005-06-27 10:35:33 +00:00
|
|
|
{
|
2007-05-31 02:48:13 +00:00
|
|
|
static struct packed_git *last_found = (void *)1;
|
2005-06-27 10:35:33 +00:00
|
|
|
struct packed_git *p;
|
2007-03-07 01:44:30 +00:00
|
|
|
off_t offset;
|
2006-09-21 04:05:37 +00:00
|
|
|
|
2005-06-27 10:35:33 +00:00
|
|
|
prepare_packed_git();
|
2007-05-31 02:48:13 +00:00
|
|
|
if (!packed_git)
|
|
|
|
return 0;
|
|
|
|
p = (last_found == (void *)1) ? packed_git : last_found;
|
2005-06-27 10:35:33 +00:00
|
|
|
|
2007-05-31 02:48:13 +00:00
|
|
|
do {
|
2006-09-06 09:12:09 +00:00
|
|
|
if (ignore_packed) {
|
|
|
|
const char **ig;
|
|
|
|
for (ig = ignore_packed; *ig; ig++)
|
2007-09-17 06:15:19 +00:00
|
|
|
if (matches_pack_name(p, *ig))
|
2006-09-06 09:12:09 +00:00
|
|
|
break;
|
|
|
|
if (*ig)
|
2007-05-31 02:48:13 +00:00
|
|
|
goto next;
|
2006-09-06 09:12:09 +00:00
|
|
|
}
|
2007-05-31 02:48:13 +00:00
|
|
|
|
2006-09-21 04:05:37 +00:00
|
|
|
offset = find_pack_entry_one(sha1, p);
|
|
|
|
if (offset) {
|
Don't find objects in packs which aren't available anymore.
Matthias Lederhofer identified a race condition where a Git reader
process was able to locate an object in a packed_git index, but
was then preempted while a `git repack -a -d` ran and completed.
By the time the reader was able to seek in the packfile to get the
object data, the packfile no longer existed on disk.
In this particular case the reader process did not attempt to
open the packfile before it was deleted, so it did not already
have the pack_fd field popuplated. With the packfile itself gone,
there was no way for the reader to open it and fetch the data.
I'm fixing the race condition by teaching find_pack_entry to ignore
a packed_git whose packfile is not currently open and which cannot
be opened. If none of the currently known packs can supply the
object, we will return 0 and the caller will decide the object is
not available. If this is the first attempt at finding an object,
the caller will reprepare_packed_git and try again. If it was
the second attempt, the caller will typically return NULL back,
and an error message about a missing object will be reported.
This patch does not address the situation of a reader which is
being starved out by a tight sequence of `git repack -a -d` runs.
In this particular case the reader will try twice, probably fail
both times, and declare the object in question cannot be found.
As it is highly unlikely that a real world `git repack -a -d` can
complete faster than a reader can open a packfile, so I don't think
this is a huge concern.
Signed-off-by: Shawn O. Pearce <spearce@spearce.org>
Signed-off-by: Junio C Hamano <junkio@cox.net>
2007-02-01 20:52:38 +00:00
|
|
|
/*
|
|
|
|
* We are about to tell the caller where they can
|
|
|
|
* locate the requested object. We better make
|
|
|
|
* sure the packfile is still here and can be
|
|
|
|
* accessed before supplying that answer, as
|
|
|
|
* it may have been deleted since the index
|
|
|
|
* was loaded!
|
|
|
|
*/
|
|
|
|
if (p->pack_fd == -1 && open_packed_git(p)) {
|
|
|
|
error("packfile %s cannot be accessed", p->pack_name);
|
2007-05-31 02:48:13 +00:00
|
|
|
goto next;
|
Don't find objects in packs which aren't available anymore.
Matthias Lederhofer identified a race condition where a Git reader
process was able to locate an object in a packed_git index, but
was then preempted while a `git repack -a -d` ran and completed.
By the time the reader was able to seek in the packfile to get the
object data, the packfile no longer existed on disk.
In this particular case the reader process did not attempt to
open the packfile before it was deleted, so it did not already
have the pack_fd field popuplated. With the packfile itself gone,
there was no way for the reader to open it and fetch the data.
I'm fixing the race condition by teaching find_pack_entry to ignore
a packed_git whose packfile is not currently open and which cannot
be opened. If none of the currently known packs can supply the
object, we will return 0 and the caller will decide the object is
not available. If this is the first attempt at finding an object,
the caller will reprepare_packed_git and try again. If it was
the second attempt, the caller will typically return NULL back,
and an error message about a missing object will be reported.
This patch does not address the situation of a reader which is
being starved out by a tight sequence of `git repack -a -d` runs.
In this particular case the reader will try twice, probably fail
both times, and declare the object in question cannot be found.
As it is highly unlikely that a real world `git repack -a -d` can
complete faster than a reader can open a packfile, so I don't think
this is a huge concern.
Signed-off-by: Shawn O. Pearce <spearce@spearce.org>
Signed-off-by: Junio C Hamano <junkio@cox.net>
2007-02-01 20:52:38 +00:00
|
|
|
}
|
2006-09-21 04:05:37 +00:00
|
|
|
e->offset = offset;
|
|
|
|
e->p = p;
|
|
|
|
hashcpy(e->sha1, sha1);
|
2007-05-31 02:48:13 +00:00
|
|
|
last_found = p;
|
2005-06-27 10:35:33 +00:00
|
|
|
return 1;
|
2006-09-21 04:05:37 +00:00
|
|
|
}
|
2007-05-31 02:48:13 +00:00
|
|
|
|
|
|
|
next:
|
|
|
|
if (p == last_found)
|
|
|
|
p = packed_git;
|
|
|
|
else
|
|
|
|
p = p->next;
|
|
|
|
if (p == last_found)
|
|
|
|
p = p->next;
|
|
|
|
} while (p);
|
2005-06-27 10:35:33 +00:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2007-06-07 07:04:01 +00:00
|
|
|
struct packed_git *find_sha1_pack(const unsigned char *sha1,
|
2005-08-01 00:53:44 +00:00
|
|
|
struct packed_git *packs)
|
|
|
|
{
|
|
|
|
struct packed_git *p;
|
|
|
|
|
|
|
|
for (p = packs; p; p = p->next) {
|
2006-09-21 04:05:37 +00:00
|
|
|
if (find_pack_entry_one(sha1, p))
|
2005-08-01 00:53:44 +00:00
|
|
|
return p;
|
|
|
|
}
|
|
|
|
return NULL;
|
2007-02-26 19:55:59 +00:00
|
|
|
|
2005-08-01 00:53:44 +00:00
|
|
|
}
|
|
|
|
|
2007-02-26 19:55:59 +00:00
|
|
|
static int sha1_loose_object_info(const unsigned char *sha1, unsigned long *sizep)
|
2005-06-02 22:20:54 +00:00
|
|
|
{
|
2005-06-27 10:34:06 +00:00
|
|
|
int status;
|
2005-06-02 22:20:54 +00:00
|
|
|
unsigned long mapsize, size;
|
|
|
|
void *map;
|
|
|
|
z_stream stream;
|
2007-02-26 19:55:55 +00:00
|
|
|
char hdr[32];
|
2005-06-02 22:20:54 +00:00
|
|
|
|
2006-07-17 22:04:47 +00:00
|
|
|
map = map_sha1_file(sha1, &mapsize);
|
2006-11-27 23:18:55 +00:00
|
|
|
if (!map)
|
|
|
|
return error("unable to find %s", sha1_to_hex(sha1));
|
2005-06-27 10:34:06 +00:00
|
|
|
if (unpack_sha1_header(&stream, map, mapsize, hdr, sizeof(hdr)) < 0)
|
|
|
|
status = error("unable to unpack %s header",
|
|
|
|
sha1_to_hex(sha1));
|
2007-02-26 19:55:59 +00:00
|
|
|
else if ((status = parse_sha1_header(hdr, &size)) < 0)
|
2005-06-27 10:34:06 +00:00
|
|
|
status = error("unable to parse %s header", sha1_to_hex(sha1));
|
2007-02-26 19:55:59 +00:00
|
|
|
else if (sizep)
|
|
|
|
*sizep = size;
|
2005-06-02 22:20:54 +00:00
|
|
|
inflateEnd(&stream);
|
|
|
|
munmap(map, mapsize);
|
|
|
|
return status;
|
|
|
|
}
|
|
|
|
|
2007-02-26 19:55:59 +00:00
|
|
|
int sha1_object_info(const unsigned char *sha1, unsigned long *sizep)
|
2006-11-27 23:18:55 +00:00
|
|
|
{
|
|
|
|
struct pack_entry e;
|
|
|
|
|
|
|
|
if (!find_pack_entry(sha1, &e, NULL)) {
|
|
|
|
reprepare_packed_git();
|
|
|
|
if (!find_pack_entry(sha1, &e, NULL))
|
2007-02-26 19:55:59 +00:00
|
|
|
return sha1_loose_object_info(sha1, sizep);
|
2006-11-27 23:18:55 +00:00
|
|
|
}
|
2007-02-26 19:55:59 +00:00
|
|
|
return packed_object_info(e.p, e.offset, sizep);
|
2006-11-27 23:18:55 +00:00
|
|
|
}
|
|
|
|
|
2007-02-26 19:55:59 +00:00
|
|
|
static void *read_packed_sha1(const unsigned char *sha1,
|
|
|
|
enum object_type *type, unsigned long *size)
|
2005-06-27 10:35:33 +00:00
|
|
|
{
|
|
|
|
struct pack_entry e;
|
|
|
|
|
2007-01-22 20:29:45 +00:00
|
|
|
if (!find_pack_entry(sha1, &e, NULL))
|
2005-06-27 10:35:33 +00:00
|
|
|
return NULL;
|
2007-01-22 20:29:45 +00:00
|
|
|
else
|
2007-03-18 01:13:57 +00:00
|
|
|
return cache_or_unpack_entry(e.p, e.offset, size, type, 1);
|
2005-06-27 10:35:33 +00:00
|
|
|
}
|
|
|
|
|
2007-02-05 05:42:38 +00:00
|
|
|
/*
|
|
|
|
* This is meant to hold a *small* number of objects that you would
|
|
|
|
* want read_sha1_file() to be able to return, but yet you do not want
|
|
|
|
* to write them into the object store (e.g. a browse-only
|
|
|
|
* application).
|
|
|
|
*/
|
|
|
|
static struct cached_object {
|
|
|
|
unsigned char sha1[20];
|
2007-02-26 19:55:59 +00:00
|
|
|
enum object_type type;
|
2007-02-05 05:42:38 +00:00
|
|
|
void *buf;
|
|
|
|
unsigned long size;
|
|
|
|
} *cached_objects;
|
|
|
|
static int cached_object_nr, cached_object_alloc;
|
|
|
|
|
2008-02-13 11:25:04 +00:00
|
|
|
static struct cached_object empty_tree = {
|
|
|
|
/* empty tree sha1: 4b825dc642cb6eb9a060e54bf8d69288fbee4904 */
|
|
|
|
"\x4b\x82\x5d\xc6\x42\xcb\x6e\xb9\xa0\x60"
|
|
|
|
"\xe5\x4b\xf8\xd6\x92\x88\xfb\xee\x49\x04",
|
|
|
|
OBJ_TREE,
|
|
|
|
"",
|
|
|
|
0
|
|
|
|
};
|
|
|
|
|
2007-02-05 05:42:38 +00:00
|
|
|
static struct cached_object *find_cached_object(const unsigned char *sha1)
|
|
|
|
{
|
|
|
|
int i;
|
|
|
|
struct cached_object *co = cached_objects;
|
|
|
|
|
|
|
|
for (i = 0; i < cached_object_nr; i++, co++) {
|
|
|
|
if (!hashcmp(co->sha1, sha1))
|
|
|
|
return co;
|
|
|
|
}
|
2008-02-13 11:25:04 +00:00
|
|
|
if (!hashcmp(sha1, empty_tree.sha1))
|
|
|
|
return &empty_tree;
|
2007-02-05 05:42:38 +00:00
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
2007-02-26 19:55:59 +00:00
|
|
|
int pretend_sha1_file(void *buf, unsigned long len, enum object_type type,
|
|
|
|
unsigned char *sha1)
|
2007-02-05 05:42:38 +00:00
|
|
|
{
|
|
|
|
struct cached_object *co;
|
|
|
|
|
2007-02-26 19:55:59 +00:00
|
|
|
hash_sha1_file(buf, len, typename(type), sha1);
|
2007-02-05 05:42:38 +00:00
|
|
|
if (has_sha1_file(sha1) || find_cached_object(sha1))
|
|
|
|
return 0;
|
|
|
|
if (cached_object_alloc <= cached_object_nr) {
|
|
|
|
cached_object_alloc = alloc_nr(cached_object_alloc);
|
|
|
|
cached_objects = xrealloc(cached_objects,
|
|
|
|
sizeof(*cached_objects) *
|
|
|
|
cached_object_alloc);
|
|
|
|
}
|
|
|
|
co = &cached_objects[cached_object_nr++];
|
|
|
|
co->size = len;
|
2007-02-26 19:55:59 +00:00
|
|
|
co->type = type;
|
2007-02-16 01:02:06 +00:00
|
|
|
co->buf = xmalloc(len);
|
|
|
|
memcpy(co->buf, buf, len);
|
2007-02-05 05:42:38 +00:00
|
|
|
hashcpy(co->sha1, sha1);
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2007-02-26 19:55:59 +00:00
|
|
|
void *read_sha1_file(const unsigned char *sha1, enum object_type *type,
|
|
|
|
unsigned long *size)
|
2005-04-18 20:04:43 +00:00
|
|
|
{
|
|
|
|
unsigned long mapsize;
|
|
|
|
void *map, *buf;
|
2007-02-05 05:42:38 +00:00
|
|
|
struct cached_object *co;
|
|
|
|
|
|
|
|
co = find_cached_object(sha1);
|
|
|
|
if (co) {
|
2007-02-26 19:55:59 +00:00
|
|
|
*type = co->type;
|
2007-02-05 05:42:38 +00:00
|
|
|
*size = co->size;
|
2007-09-15 22:32:36 +00:00
|
|
|
return xmemdupz(co->buf, co->size);
|
2007-02-05 05:42:38 +00:00
|
|
|
}
|
2005-04-18 20:04:43 +00:00
|
|
|
|
2007-01-22 20:29:45 +00:00
|
|
|
buf = read_packed_sha1(sha1, type, size);
|
|
|
|
if (buf)
|
|
|
|
return buf;
|
2006-07-17 22:04:47 +00:00
|
|
|
map = map_sha1_file(sha1, &mapsize);
|
2005-04-18 20:04:43 +00:00
|
|
|
if (map) {
|
2007-03-05 08:21:37 +00:00
|
|
|
buf = unpack_sha1_file(map, mapsize, type, size, sha1);
|
2005-04-18 20:04:43 +00:00
|
|
|
munmap(map, mapsize);
|
|
|
|
return buf;
|
|
|
|
}
|
2006-06-02 15:32:23 +00:00
|
|
|
reprepare_packed_git();
|
2007-01-22 20:29:45 +00:00
|
|
|
return read_packed_sha1(sha1, type, size);
|
2005-04-18 20:04:43 +00:00
|
|
|
}
|
|
|
|
|
2005-04-28 23:42:27 +00:00
|
|
|
void *read_object_with_reference(const unsigned char *sha1,
|
2007-02-26 19:55:59 +00:00
|
|
|
const char *required_type_name,
|
2005-04-28 23:42:27 +00:00
|
|
|
unsigned long *size,
|
|
|
|
unsigned char *actual_sha1_return)
|
2005-04-21 01:06:49 +00:00
|
|
|
{
|
2007-02-26 19:55:59 +00:00
|
|
|
enum object_type type, required_type;
|
2005-04-21 01:06:49 +00:00
|
|
|
void *buffer;
|
|
|
|
unsigned long isize;
|
2005-04-28 23:42:27 +00:00
|
|
|
unsigned char actual_sha1[20];
|
2005-04-21 01:06:49 +00:00
|
|
|
|
2007-02-26 19:55:59 +00:00
|
|
|
required_type = type_from_string(required_type_name);
|
2006-08-23 06:49:00 +00:00
|
|
|
hashcpy(actual_sha1, sha1);
|
2005-04-28 23:42:27 +00:00
|
|
|
while (1) {
|
|
|
|
int ref_length = -1;
|
|
|
|
const char *ref_type = NULL;
|
2005-04-21 01:06:49 +00:00
|
|
|
|
2007-02-26 19:55:59 +00:00
|
|
|
buffer = read_sha1_file(actual_sha1, &type, &isize);
|
2005-04-28 23:42:27 +00:00
|
|
|
if (!buffer)
|
|
|
|
return NULL;
|
2007-02-26 19:55:59 +00:00
|
|
|
if (type == required_type) {
|
2005-04-28 23:42:27 +00:00
|
|
|
*size = isize;
|
|
|
|
if (actual_sha1_return)
|
2006-08-23 06:49:00 +00:00
|
|
|
hashcpy(actual_sha1_return, actual_sha1);
|
2005-04-28 23:42:27 +00:00
|
|
|
return buffer;
|
|
|
|
}
|
|
|
|
/* Handle references */
|
2007-02-26 19:55:59 +00:00
|
|
|
else if (type == OBJ_COMMIT)
|
2005-04-28 23:42:27 +00:00
|
|
|
ref_type = "tree ";
|
2007-02-26 19:55:59 +00:00
|
|
|
else if (type == OBJ_TAG)
|
2005-04-28 23:42:27 +00:00
|
|
|
ref_type = "object ";
|
|
|
|
else {
|
|
|
|
free(buffer);
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
ref_length = strlen(ref_type);
|
2005-04-21 01:06:49 +00:00
|
|
|
|
2005-04-28 23:42:27 +00:00
|
|
|
if (memcmp(buffer, ref_type, ref_length) ||
|
2006-06-18 15:18:09 +00:00
|
|
|
get_sha1_hex((char *) buffer + ref_length, actual_sha1)) {
|
2005-04-28 23:42:27 +00:00
|
|
|
free(buffer);
|
|
|
|
return NULL;
|
|
|
|
}
|
2005-08-08 18:44:43 +00:00
|
|
|
free(buffer);
|
2005-04-28 23:42:27 +00:00
|
|
|
/* Now we have the ID of the referred-to object in
|
|
|
|
* actual_sha1. Check again. */
|
2005-04-21 01:06:49 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2007-03-20 20:02:09 +00:00
|
|
|
static void write_sha1_file_prepare(const void *buf, unsigned long len,
|
2006-10-15 12:02:03 +00:00
|
|
|
const char *type, unsigned char *sha1,
|
2007-02-26 19:55:55 +00:00
|
|
|
char *hdr, int *hdrlen)
|
2005-06-28 02:03:13 +00:00
|
|
|
{
|
|
|
|
SHA_CTX c;
|
|
|
|
|
|
|
|
/* Generate the header */
|
2007-02-26 19:55:55 +00:00
|
|
|
*hdrlen = sprintf(hdr, "%s %lu", type, len)+1;
|
2005-06-28 02:03:13 +00:00
|
|
|
|
|
|
|
/* Sha1.. */
|
|
|
|
SHA1_Init(&c);
|
|
|
|
SHA1_Update(&c, hdr, *hdrlen);
|
|
|
|
SHA1_Update(&c, buf, len);
|
|
|
|
SHA1_Final(sha1, &c);
|
|
|
|
}
|
|
|
|
|
Create object subdirectories on demand
This makes it possible to have a "sparse" git object subdirectory
structure, something that has become much more attractive now that people
use pack-files all the time.
As a result of pack-files, a git object directory doesn't necessarily have
any individual objects lying around, and in that case it's just wasting
space to keep the empty first-level object directories around: on many
filesystems the 256 empty directories will be aboue 1MB of diskspace.
Even more importantly, after you re-pack a project that _used_ to be
unpacked, you could be left with huge directories that no longer contain
anything, but that waste space and take time to look through.
With this change, "git prune-packed" can just do an rmdir() on the
directories, and they'll get removed if empty, and re-created on demand.
This patch also tries to fix up "write_sha1_from_fd()" to use the new
common infrastructure for creating the object files, closing a hole where
we might otherwise leave half-written objects in the object database.
[jc: I unoptimized the part that really removes the fan-out directories
to ease transition. init-db still wastes 1MB of diskspace to hold 256
empty fan-outs, and prune-packed rmdir()'s the grown but empty directories,
but runs mkdir() immediately after that -- reducing the saving from 150KB
to 146KB. These parts will be re-introduced when everybody has the
on-demand capability.]
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
Signed-off-by: Junio C Hamano <junkio@cox.net>
2005-10-08 22:54:01 +00:00
|
|
|
/*
|
|
|
|
* Link the tempfile to the final place, possibly creating the
|
|
|
|
* last directory level as you do so.
|
|
|
|
*
|
|
|
|
* Returns the errno on failure, 0 on success.
|
|
|
|
*/
|
2006-09-01 07:17:47 +00:00
|
|
|
static int link_temp_to_file(const char *tmpfile, const char *filename)
|
Create object subdirectories on demand
This makes it possible to have a "sparse" git object subdirectory
structure, something that has become much more attractive now that people
use pack-files all the time.
As a result of pack-files, a git object directory doesn't necessarily have
any individual objects lying around, and in that case it's just wasting
space to keep the empty first-level object directories around: on many
filesystems the 256 empty directories will be aboue 1MB of diskspace.
Even more importantly, after you re-pack a project that _used_ to be
unpacked, you could be left with huge directories that no longer contain
anything, but that waste space and take time to look through.
With this change, "git prune-packed" can just do an rmdir() on the
directories, and they'll get removed if empty, and re-created on demand.
This patch also tries to fix up "write_sha1_from_fd()" to use the new
common infrastructure for creating the object files, closing a hole where
we might otherwise leave half-written objects in the object database.
[jc: I unoptimized the part that really removes the fan-out directories
to ease transition. init-db still wastes 1MB of diskspace to hold 256
empty fan-outs, and prune-packed rmdir()'s the grown but empty directories,
but runs mkdir() immediately after that -- reducing the saving from 150KB
to 146KB. These parts will be re-introduced when everybody has the
on-demand capability.]
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
Signed-off-by: Junio C Hamano <junkio@cox.net>
2005-10-08 22:54:01 +00:00
|
|
|
{
|
|
|
|
int ret;
|
2006-07-12 05:00:16 +00:00
|
|
|
char *dir;
|
Create object subdirectories on demand
This makes it possible to have a "sparse" git object subdirectory
structure, something that has become much more attractive now that people
use pack-files all the time.
As a result of pack-files, a git object directory doesn't necessarily have
any individual objects lying around, and in that case it's just wasting
space to keep the empty first-level object directories around: on many
filesystems the 256 empty directories will be aboue 1MB of diskspace.
Even more importantly, after you re-pack a project that _used_ to be
unpacked, you could be left with huge directories that no longer contain
anything, but that waste space and take time to look through.
With this change, "git prune-packed" can just do an rmdir() on the
directories, and they'll get removed if empty, and re-created on demand.
This patch also tries to fix up "write_sha1_from_fd()" to use the new
common infrastructure for creating the object files, closing a hole where
we might otherwise leave half-written objects in the object database.
[jc: I unoptimized the part that really removes the fan-out directories
to ease transition. init-db still wastes 1MB of diskspace to hold 256
empty fan-outs, and prune-packed rmdir()'s the grown but empty directories,
but runs mkdir() immediately after that -- reducing the saving from 150KB
to 146KB. These parts will be re-introduced when everybody has the
on-demand capability.]
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
Signed-off-by: Junio C Hamano <junkio@cox.net>
2005-10-08 22:54:01 +00:00
|
|
|
|
|
|
|
if (!link(tmpfile, filename))
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
/*
|
2006-07-12 05:00:16 +00:00
|
|
|
* Try to mkdir the last path component if that failed.
|
Create object subdirectories on demand
This makes it possible to have a "sparse" git object subdirectory
structure, something that has become much more attractive now that people
use pack-files all the time.
As a result of pack-files, a git object directory doesn't necessarily have
any individual objects lying around, and in that case it's just wasting
space to keep the empty first-level object directories around: on many
filesystems the 256 empty directories will be aboue 1MB of diskspace.
Even more importantly, after you re-pack a project that _used_ to be
unpacked, you could be left with huge directories that no longer contain
anything, but that waste space and take time to look through.
With this change, "git prune-packed" can just do an rmdir() on the
directories, and they'll get removed if empty, and re-created on demand.
This patch also tries to fix up "write_sha1_from_fd()" to use the new
common infrastructure for creating the object files, closing a hole where
we might otherwise leave half-written objects in the object database.
[jc: I unoptimized the part that really removes the fan-out directories
to ease transition. init-db still wastes 1MB of diskspace to hold 256
empty fan-outs, and prune-packed rmdir()'s the grown but empty directories,
but runs mkdir() immediately after that -- reducing the saving from 150KB
to 146KB. These parts will be re-introduced when everybody has the
on-demand capability.]
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
Signed-off-by: Junio C Hamano <junkio@cox.net>
2005-10-08 22:54:01 +00:00
|
|
|
*
|
|
|
|
* Re-try the "link()" regardless of whether the mkdir
|
|
|
|
* succeeds, since a race might mean that somebody
|
|
|
|
* else succeeded.
|
|
|
|
*/
|
|
|
|
ret = errno;
|
2006-07-12 05:00:16 +00:00
|
|
|
dir = strrchr(filename, '/');
|
|
|
|
if (dir) {
|
|
|
|
*dir = 0;
|
2006-11-03 02:02:17 +00:00
|
|
|
if (!mkdir(filename, 0777) && adjust_shared_perm(filename)) {
|
2006-10-31 23:56:58 +00:00
|
|
|
*dir = '/';
|
2006-07-12 05:00:16 +00:00
|
|
|
return -2;
|
2006-10-31 23:56:58 +00:00
|
|
|
}
|
2006-07-12 05:00:16 +00:00
|
|
|
*dir = '/';
|
|
|
|
if (!link(tmpfile, filename))
|
|
|
|
return 0;
|
|
|
|
ret = errno;
|
Create object subdirectories on demand
This makes it possible to have a "sparse" git object subdirectory
structure, something that has become much more attractive now that people
use pack-files all the time.
As a result of pack-files, a git object directory doesn't necessarily have
any individual objects lying around, and in that case it's just wasting
space to keep the empty first-level object directories around: on many
filesystems the 256 empty directories will be aboue 1MB of diskspace.
Even more importantly, after you re-pack a project that _used_ to be
unpacked, you could be left with huge directories that no longer contain
anything, but that waste space and take time to look through.
With this change, "git prune-packed" can just do an rmdir() on the
directories, and they'll get removed if empty, and re-created on demand.
This patch also tries to fix up "write_sha1_from_fd()" to use the new
common infrastructure for creating the object files, closing a hole where
we might otherwise leave half-written objects in the object database.
[jc: I unoptimized the part that really removes the fan-out directories
to ease transition. init-db still wastes 1MB of diskspace to hold 256
empty fan-outs, and prune-packed rmdir()'s the grown but empty directories,
but runs mkdir() immediately after that -- reducing the saving from 150KB
to 146KB. These parts will be re-introduced when everybody has the
on-demand capability.]
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
Signed-off-by: Junio C Hamano <junkio@cox.net>
2005-10-08 22:54:01 +00:00
|
|
|
}
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Move the just written object into its final resting place
|
|
|
|
*/
|
2006-09-01 07:17:47 +00:00
|
|
|
int move_temp_to_file(const char *tmpfile, const char *filename)
|
Create object subdirectories on demand
This makes it possible to have a "sparse" git object subdirectory
structure, something that has become much more attractive now that people
use pack-files all the time.
As a result of pack-files, a git object directory doesn't necessarily have
any individual objects lying around, and in that case it's just wasting
space to keep the empty first-level object directories around: on many
filesystems the 256 empty directories will be aboue 1MB of diskspace.
Even more importantly, after you re-pack a project that _used_ to be
unpacked, you could be left with huge directories that no longer contain
anything, but that waste space and take time to look through.
With this change, "git prune-packed" can just do an rmdir() on the
directories, and they'll get removed if empty, and re-created on demand.
This patch also tries to fix up "write_sha1_from_fd()" to use the new
common infrastructure for creating the object files, closing a hole where
we might otherwise leave half-written objects in the object database.
[jc: I unoptimized the part that really removes the fan-out directories
to ease transition. init-db still wastes 1MB of diskspace to hold 256
empty fan-outs, and prune-packed rmdir()'s the grown but empty directories,
but runs mkdir() immediately after that -- reducing the saving from 150KB
to 146KB. These parts will be re-introduced when everybody has the
on-demand capability.]
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
Signed-off-by: Junio C Hamano <junkio@cox.net>
2005-10-08 22:54:01 +00:00
|
|
|
{
|
|
|
|
int ret = link_temp_to_file(tmpfile, filename);
|
2005-10-26 17:27:36 +00:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Coda hack - coda doesn't like cross-directory links,
|
|
|
|
* so we fall back to a rename, which will mean that it
|
|
|
|
* won't be able to check collisions, but that's not a
|
|
|
|
* big deal.
|
|
|
|
*
|
|
|
|
* The same holds for FAT formatted media.
|
|
|
|
*
|
|
|
|
* When this succeeds, we just return 0. We have nothing
|
|
|
|
* left to unlink.
|
|
|
|
*/
|
|
|
|
if (ret && ret != EEXIST) {
|
|
|
|
if (!rename(tmpfile, filename))
|
Create object subdirectories on demand
This makes it possible to have a "sparse" git object subdirectory
structure, something that has become much more attractive now that people
use pack-files all the time.
As a result of pack-files, a git object directory doesn't necessarily have
any individual objects lying around, and in that case it's just wasting
space to keep the empty first-level object directories around: on many
filesystems the 256 empty directories will be aboue 1MB of diskspace.
Even more importantly, after you re-pack a project that _used_ to be
unpacked, you could be left with huge directories that no longer contain
anything, but that waste space and take time to look through.
With this change, "git prune-packed" can just do an rmdir() on the
directories, and they'll get removed if empty, and re-created on demand.
This patch also tries to fix up "write_sha1_from_fd()" to use the new
common infrastructure for creating the object files, closing a hole where
we might otherwise leave half-written objects in the object database.
[jc: I unoptimized the part that really removes the fan-out directories
to ease transition. init-db still wastes 1MB of diskspace to hold 256
empty fan-outs, and prune-packed rmdir()'s the grown but empty directories,
but runs mkdir() immediately after that -- reducing the saving from 150KB
to 146KB. These parts will be re-introduced when everybody has the
on-demand capability.]
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
Signed-off-by: Junio C Hamano <junkio@cox.net>
2005-10-08 22:54:01 +00:00
|
|
|
return 0;
|
2005-10-25 23:41:20 +00:00
|
|
|
ret = errno;
|
Create object subdirectories on demand
This makes it possible to have a "sparse" git object subdirectory
structure, something that has become much more attractive now that people
use pack-files all the time.
As a result of pack-files, a git object directory doesn't necessarily have
any individual objects lying around, and in that case it's just wasting
space to keep the empty first-level object directories around: on many
filesystems the 256 empty directories will be aboue 1MB of diskspace.
Even more importantly, after you re-pack a project that _used_ to be
unpacked, you could be left with huge directories that no longer contain
anything, but that waste space and take time to look through.
With this change, "git prune-packed" can just do an rmdir() on the
directories, and they'll get removed if empty, and re-created on demand.
This patch also tries to fix up "write_sha1_from_fd()" to use the new
common infrastructure for creating the object files, closing a hole where
we might otherwise leave half-written objects in the object database.
[jc: I unoptimized the part that really removes the fan-out directories
to ease transition. init-db still wastes 1MB of diskspace to hold 256
empty fan-outs, and prune-packed rmdir()'s the grown but empty directories,
but runs mkdir() immediately after that -- reducing the saving from 150KB
to 146KB. These parts will be re-introduced when everybody has the
on-demand capability.]
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
Signed-off-by: Junio C Hamano <junkio@cox.net>
2005-10-08 22:54:01 +00:00
|
|
|
}
|
|
|
|
unlink(tmpfile);
|
|
|
|
if (ret) {
|
|
|
|
if (ret != EEXIST) {
|
2006-11-09 12:52:05 +00:00
|
|
|
return error("unable to write sha1 filename %s: %s\n", filename, strerror(ret));
|
Create object subdirectories on demand
This makes it possible to have a "sparse" git object subdirectory
structure, something that has become much more attractive now that people
use pack-files all the time.
As a result of pack-files, a git object directory doesn't necessarily have
any individual objects lying around, and in that case it's just wasting
space to keep the empty first-level object directories around: on many
filesystems the 256 empty directories will be aboue 1MB of diskspace.
Even more importantly, after you re-pack a project that _used_ to be
unpacked, you could be left with huge directories that no longer contain
anything, but that waste space and take time to look through.
With this change, "git prune-packed" can just do an rmdir() on the
directories, and they'll get removed if empty, and re-created on demand.
This patch also tries to fix up "write_sha1_from_fd()" to use the new
common infrastructure for creating the object files, closing a hole where
we might otherwise leave half-written objects in the object database.
[jc: I unoptimized the part that really removes the fan-out directories
to ease transition. init-db still wastes 1MB of diskspace to hold 256
empty fan-outs, and prune-packed rmdir()'s the grown but empty directories,
but runs mkdir() immediately after that -- reducing the saving from 150KB
to 146KB. These parts will be re-introduced when everybody has the
on-demand capability.]
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
Signed-off-by: Junio C Hamano <junkio@cox.net>
2005-10-08 22:54:01 +00:00
|
|
|
}
|
|
|
|
/* FIXME!!! Collision check here ? */
|
|
|
|
}
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2006-05-24 15:30:54 +00:00
|
|
|
static int write_buffer(int fd, const void *buf, size_t len)
|
|
|
|
{
|
2007-01-12 04:23:00 +00:00
|
|
|
if (write_in_full(fd, buf, len) < 0)
|
2007-01-08 15:58:23 +00:00
|
|
|
return error("file write error (%s)", strerror(errno));
|
2006-05-24 15:30:54 +00:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2007-03-20 20:02:09 +00:00
|
|
|
int hash_sha1_file(const void *buf, unsigned long len, const char *type,
|
2006-10-14 10:45:36 +00:00
|
|
|
unsigned char *sha1)
|
|
|
|
{
|
2007-02-26 19:55:55 +00:00
|
|
|
char hdr[32];
|
2006-10-14 10:45:36 +00:00
|
|
|
int hdrlen;
|
|
|
|
write_sha1_file_prepare(buf, len, type, sha1, hdr, &hdrlen);
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2005-05-18 12:14:09 +00:00
|
|
|
int write_sha1_file(void *buf, unsigned long len, const char *type, unsigned char *returnsha1)
|
2005-04-18 20:04:43 +00:00
|
|
|
{
|
Be more careful about zlib return values
When creating a new object, we use "deflate(stream, Z_FINISH)" in a loop
until it no longer returns Z_OK, and then we do "deflateEnd()" to finish
up business.
That should all work, but the fact is, it's not how you're _supposed_ to
use the zlib return values properly:
- deflate() should never return Z_OK in the first place, except if we
need to increase the output buffer size (which we're not doing, and
should never need to do, since we pre-allocated a buffer that is
supposed to be able to hold the output in full). So the "while()" loop
was incorrect: Z_OK doesn't actually mean "ok, continue", it means "ok,
allocate more memory for me and continue"!
- if we got an error return, we would consider it to be end-of-stream,
but it could be some internal zlib error. In short, we should check
for Z_STREAM_END explicitly, since that's the only valid return value
anyway for the Z_FINISH case.
- we never checked deflateEnd() return codes at all.
Now, admittedly, none of these issues should ever happen, unless there is
some internal bug in zlib. So this patch should make zero difference, but
it seems to be the right thing to do.
We should probablybe anal and check the return value of "deflateInit()"
too!
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
Signed-off-by: Junio C Hamano <junkio@cox.net>
2007-03-20 18:38:34 +00:00
|
|
|
int size, ret;
|
2005-05-18 12:14:09 +00:00
|
|
|
unsigned char *compressed;
|
2005-04-18 20:04:43 +00:00
|
|
|
z_stream stream;
|
|
|
|
unsigned char sha1[20];
|
2005-04-20 16:28:05 +00:00
|
|
|
char *filename;
|
2005-05-03 18:46:16 +00:00
|
|
|
static char tmpfile[PATH_MAX];
|
2007-02-26 19:55:55 +00:00
|
|
|
char hdr[32];
|
Create object subdirectories on demand
This makes it possible to have a "sparse" git object subdirectory
structure, something that has become much more attractive now that people
use pack-files all the time.
As a result of pack-files, a git object directory doesn't necessarily have
any individual objects lying around, and in that case it's just wasting
space to keep the empty first-level object directories around: on many
filesystems the 256 empty directories will be aboue 1MB of diskspace.
Even more importantly, after you re-pack a project that _used_ to be
unpacked, you could be left with huge directories that no longer contain
anything, but that waste space and take time to look through.
With this change, "git prune-packed" can just do an rmdir() on the
directories, and they'll get removed if empty, and re-created on demand.
This patch also tries to fix up "write_sha1_from_fd()" to use the new
common infrastructure for creating the object files, closing a hole where
we might otherwise leave half-written objects in the object database.
[jc: I unoptimized the part that really removes the fan-out directories
to ease transition. init-db still wastes 1MB of diskspace to hold 256
empty fan-outs, and prune-packed rmdir()'s the grown but empty directories,
but runs mkdir() immediately after that -- reducing the saving from 150KB
to 146KB. These parts will be re-introduced when everybody has the
on-demand capability.]
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
Signed-off-by: Junio C Hamano <junkio@cox.net>
2005-10-08 22:54:01 +00:00
|
|
|
int fd, hdrlen;
|
2005-04-25 17:19:53 +00:00
|
|
|
|
2005-06-28 02:03:13 +00:00
|
|
|
/* Normally if we have it in the pack then we do not bother writing
|
|
|
|
* it out into .git/objects/??/?{38} file.
|
|
|
|
*/
|
2006-10-15 12:02:03 +00:00
|
|
|
write_sha1_file_prepare(buf, len, type, sha1, hdr, &hdrlen);
|
|
|
|
filename = sha1_file_name(sha1);
|
2005-04-20 16:28:05 +00:00
|
|
|
if (returnsha1)
|
2006-08-23 06:49:00 +00:00
|
|
|
hashcpy(returnsha1, sha1);
|
2005-06-28 02:03:13 +00:00
|
|
|
if (has_sha1_file(sha1))
|
|
|
|
return 0;
|
2005-05-03 18:46:16 +00:00
|
|
|
fd = open(filename, O_RDONLY);
|
|
|
|
if (fd >= 0) {
|
2005-04-20 16:28:05 +00:00
|
|
|
/*
|
2005-05-03 18:46:16 +00:00
|
|
|
* FIXME!!! We might do collision checking here, but we'd
|
|
|
|
* need to uncompress the old file and check it. Later.
|
2005-04-20 16:28:05 +00:00
|
|
|
*/
|
2005-05-03 18:46:16 +00:00
|
|
|
close(fd);
|
2005-04-20 16:28:05 +00:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2005-05-03 18:46:16 +00:00
|
|
|
if (errno != ENOENT) {
|
2006-11-09 12:52:05 +00:00
|
|
|
return error("sha1 file %s: %s\n", filename, strerror(errno));
|
2005-05-03 18:46:16 +00:00
|
|
|
}
|
|
|
|
|
2007-03-24 15:58:22 +00:00
|
|
|
snprintf(tmpfile, sizeof(tmpfile), "%s/tmp_obj_XXXXXX", get_object_directory());
|
2005-05-07 07:38:04 +00:00
|
|
|
|
2005-05-03 18:46:16 +00:00
|
|
|
fd = mkstemp(tmpfile);
|
|
|
|
if (fd < 0) {
|
2006-11-09 12:52:05 +00:00
|
|
|
if (errno == EPERM)
|
|
|
|
return error("insufficient permission for adding an object to repository database %s\n", get_object_directory());
|
|
|
|
else
|
|
|
|
return error("unable to create temporary sha1 filename %s: %s\n", tmpfile, strerror(errno));
|
2005-05-03 18:46:16 +00:00
|
|
|
}
|
|
|
|
|
2005-04-18 20:04:43 +00:00
|
|
|
/* Set it up */
|
|
|
|
memset(&stream, 0, sizeof(stream));
|
2006-07-03 20:11:47 +00:00
|
|
|
deflateInit(&stream, zlib_compression_level);
|
2006-07-11 19:48:08 +00:00
|
|
|
size = 8 + deflateBound(&stream, len+hdrlen);
|
2005-04-26 19:00:58 +00:00
|
|
|
compressed = xmalloc(size);
|
2005-04-18 20:04:43 +00:00
|
|
|
|
|
|
|
/* Compress it */
|
|
|
|
stream.next_out = compressed;
|
|
|
|
stream.avail_out = size;
|
2005-04-25 17:19:53 +00:00
|
|
|
|
|
|
|
/* First header.. */
|
2007-02-26 19:55:55 +00:00
|
|
|
stream.next_in = (unsigned char *)hdr;
|
2005-04-25 17:19:53 +00:00
|
|
|
stream.avail_in = hdrlen;
|
2007-05-09 18:42:42 +00:00
|
|
|
while (deflate(&stream, 0) == Z_OK)
|
|
|
|
/* nothing */;
|
2005-04-25 17:19:53 +00:00
|
|
|
|
|
|
|
/* Then the data itself.. */
|
|
|
|
stream.next_in = buf;
|
|
|
|
stream.avail_in = len;
|
Be more careful about zlib return values
When creating a new object, we use "deflate(stream, Z_FINISH)" in a loop
until it no longer returns Z_OK, and then we do "deflateEnd()" to finish
up business.
That should all work, but the fact is, it's not how you're _supposed_ to
use the zlib return values properly:
- deflate() should never return Z_OK in the first place, except if we
need to increase the output buffer size (which we're not doing, and
should never need to do, since we pre-allocated a buffer that is
supposed to be able to hold the output in full). So the "while()" loop
was incorrect: Z_OK doesn't actually mean "ok, continue", it means "ok,
allocate more memory for me and continue"!
- if we got an error return, we would consider it to be end-of-stream,
but it could be some internal zlib error. In short, we should check
for Z_STREAM_END explicitly, since that's the only valid return value
anyway for the Z_FINISH case.
- we never checked deflateEnd() return codes at all.
Now, admittedly, none of these issues should ever happen, unless there is
some internal bug in zlib. So this patch should make zero difference, but
it seems to be the right thing to do.
We should probablybe anal and check the return value of "deflateInit()"
too!
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
Signed-off-by: Junio C Hamano <junkio@cox.net>
2007-03-20 18:38:34 +00:00
|
|
|
ret = deflate(&stream, Z_FINISH);
|
|
|
|
if (ret != Z_STREAM_END)
|
|
|
|
die("unable to deflate new object %s (%d)", sha1_to_hex(sha1), ret);
|
|
|
|
|
|
|
|
ret = deflateEnd(&stream);
|
|
|
|
if (ret != Z_OK)
|
|
|
|
die("deflateEnd on object %s failed (%d)", sha1_to_hex(sha1), ret);
|
|
|
|
|
2005-04-18 20:04:43 +00:00
|
|
|
size = stream.total_out;
|
|
|
|
|
2006-05-24 15:30:54 +00:00
|
|
|
if (write_buffer(fd, compressed, size) < 0)
|
|
|
|
die("unable to write sha1 file");
|
2005-05-03 18:46:16 +00:00
|
|
|
fchmod(fd, 0444);
|
2007-03-27 19:55:44 +00:00
|
|
|
if (close(fd))
|
|
|
|
die("unable to write sha1 file");
|
2005-05-07 08:27:00 +00:00
|
|
|
free(compressed);
|
2005-04-18 20:04:43 +00:00
|
|
|
|
Create object subdirectories on demand
This makes it possible to have a "sparse" git object subdirectory
structure, something that has become much more attractive now that people
use pack-files all the time.
As a result of pack-files, a git object directory doesn't necessarily have
any individual objects lying around, and in that case it's just wasting
space to keep the empty first-level object directories around: on many
filesystems the 256 empty directories will be aboue 1MB of diskspace.
Even more importantly, after you re-pack a project that _used_ to be
unpacked, you could be left with huge directories that no longer contain
anything, but that waste space and take time to look through.
With this change, "git prune-packed" can just do an rmdir() on the
directories, and they'll get removed if empty, and re-created on demand.
This patch also tries to fix up "write_sha1_from_fd()" to use the new
common infrastructure for creating the object files, closing a hole where
we might otherwise leave half-written objects in the object database.
[jc: I unoptimized the part that really removes the fan-out directories
to ease transition. init-db still wastes 1MB of diskspace to hold 256
empty fan-outs, and prune-packed rmdir()'s the grown but empty directories,
but runs mkdir() immediately after that -- reducing the saving from 150KB
to 146KB. These parts will be re-introduced when everybody has the
on-demand capability.]
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
Signed-off-by: Junio C Hamano <junkio@cox.net>
2005-10-08 22:54:01 +00:00
|
|
|
return move_temp_to_file(tmpfile, filename);
|
2005-04-18 20:04:43 +00:00
|
|
|
}
|
2005-04-24 01:47:23 +00:00
|
|
|
|
2006-05-24 15:30:54 +00:00
|
|
|
/*
|
|
|
|
* We need to unpack and recompress the object for writing
|
|
|
|
* it out to a different file.
|
|
|
|
*/
|
|
|
|
static void *repack_object(const unsigned char *sha1, unsigned long *objsize)
|
2005-07-10 22:25:38 +00:00
|
|
|
{
|
2006-05-24 15:30:54 +00:00
|
|
|
size_t size;
|
2005-07-10 22:25:38 +00:00
|
|
|
z_stream stream;
|
2006-05-24 15:30:54 +00:00
|
|
|
unsigned char *unpacked;
|
|
|
|
unsigned long len;
|
2007-02-26 19:55:59 +00:00
|
|
|
enum object_type type;
|
2007-02-26 19:55:55 +00:00
|
|
|
char hdr[32];
|
2006-05-24 15:30:54 +00:00
|
|
|
int hdrlen;
|
|
|
|
void *buf;
|
2005-08-08 18:45:36 +00:00
|
|
|
|
2006-07-10 06:57:51 +00:00
|
|
|
/* need to unpack and recompress it by itself */
|
2007-02-26 19:55:59 +00:00
|
|
|
unpacked = read_packed_sha1(sha1, &type, &len);
|
2007-01-22 20:29:45 +00:00
|
|
|
if (!unpacked)
|
|
|
|
error("cannot read sha1_file for %s", sha1_to_hex(sha1));
|
2005-07-10 22:25:38 +00:00
|
|
|
|
2007-02-26 19:55:59 +00:00
|
|
|
hdrlen = sprintf(hdr, "%s %lu", typename(type), len) + 1;
|
2005-07-10 22:25:38 +00:00
|
|
|
|
2006-05-24 15:30:54 +00:00
|
|
|
/* Set it up */
|
|
|
|
memset(&stream, 0, sizeof(stream));
|
2006-07-03 20:11:47 +00:00
|
|
|
deflateInit(&stream, zlib_compression_level);
|
2006-05-24 15:30:54 +00:00
|
|
|
size = deflateBound(&stream, len + hdrlen);
|
|
|
|
buf = xmalloc(size);
|
2005-07-10 22:25:38 +00:00
|
|
|
|
2006-05-24 15:30:54 +00:00
|
|
|
/* Compress it */
|
|
|
|
stream.next_out = buf;
|
|
|
|
stream.avail_out = size;
|
2005-07-10 22:25:38 +00:00
|
|
|
|
2006-05-24 15:30:54 +00:00
|
|
|
/* First header.. */
|
|
|
|
stream.next_in = (void *)hdr;
|
|
|
|
stream.avail_in = hdrlen;
|
|
|
|
while (deflate(&stream, 0) == Z_OK)
|
|
|
|
/* nothing */;
|
2005-08-08 18:45:36 +00:00
|
|
|
|
2006-05-24 15:30:54 +00:00
|
|
|
/* Then the data itself.. */
|
|
|
|
stream.next_in = unpacked;
|
|
|
|
stream.avail_in = len;
|
|
|
|
while (deflate(&stream, Z_FINISH) == Z_OK)
|
|
|
|
/* nothing */;
|
|
|
|
deflateEnd(&stream);
|
|
|
|
free(unpacked);
|
2005-08-08 18:45:36 +00:00
|
|
|
|
2006-05-24 15:30:54 +00:00
|
|
|
*objsize = stream.total_out;
|
|
|
|
return buf;
|
|
|
|
}
|
|
|
|
|
|
|
|
int write_sha1_to_fd(int fd, const unsigned char *sha1)
|
|
|
|
{
|
|
|
|
int retval;
|
|
|
|
unsigned long objsize;
|
2006-07-17 22:04:47 +00:00
|
|
|
void *buf = map_sha1_file(sha1, &objsize);
|
2006-05-24 15:30:54 +00:00
|
|
|
|
|
|
|
if (buf) {
|
|
|
|
retval = write_buffer(fd, buf, objsize);
|
|
|
|
munmap(buf, objsize);
|
|
|
|
return retval;
|
|
|
|
}
|
|
|
|
|
|
|
|
buf = repack_object(sha1, &objsize);
|
|
|
|
retval = write_buffer(fd, buf, objsize);
|
|
|
|
free(buf);
|
|
|
|
return retval;
|
2005-07-10 22:25:38 +00:00
|
|
|
}
|
|
|
|
|
2005-08-02 23:46:29 +00:00
|
|
|
int write_sha1_from_fd(const unsigned char *sha1, int fd, char *buffer,
|
|
|
|
size_t bufsize, size_t *bufposn)
|
2005-04-24 01:47:23 +00:00
|
|
|
{
|
Create object subdirectories on demand
This makes it possible to have a "sparse" git object subdirectory
structure, something that has become much more attractive now that people
use pack-files all the time.
As a result of pack-files, a git object directory doesn't necessarily have
any individual objects lying around, and in that case it's just wasting
space to keep the empty first-level object directories around: on many
filesystems the 256 empty directories will be aboue 1MB of diskspace.
Even more importantly, after you re-pack a project that _used_ to be
unpacked, you could be left with huge directories that no longer contain
anything, but that waste space and take time to look through.
With this change, "git prune-packed" can just do an rmdir() on the
directories, and they'll get removed if empty, and re-created on demand.
This patch also tries to fix up "write_sha1_from_fd()" to use the new
common infrastructure for creating the object files, closing a hole where
we might otherwise leave half-written objects in the object database.
[jc: I unoptimized the part that really removes the fan-out directories
to ease transition. init-db still wastes 1MB of diskspace to hold 256
empty fan-outs, and prune-packed rmdir()'s the grown but empty directories,
but runs mkdir() immediately after that -- reducing the saving from 150KB
to 146KB. These parts will be re-introduced when everybody has the
on-demand capability.]
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
Signed-off-by: Junio C Hamano <junkio@cox.net>
2005-10-08 22:54:01 +00:00
|
|
|
char tmpfile[PATH_MAX];
|
2005-04-24 01:47:23 +00:00
|
|
|
int local;
|
|
|
|
z_stream stream;
|
|
|
|
unsigned char real_sha1[20];
|
2005-05-18 12:14:09 +00:00
|
|
|
unsigned char discard[4096];
|
2005-04-24 01:47:23 +00:00
|
|
|
int ret;
|
|
|
|
SHA_CTX c;
|
|
|
|
|
2007-03-24 15:58:22 +00:00
|
|
|
snprintf(tmpfile, sizeof(tmpfile), "%s/tmp_obj_XXXXXX", get_object_directory());
|
2005-04-24 01:47:23 +00:00
|
|
|
|
Create object subdirectories on demand
This makes it possible to have a "sparse" git object subdirectory
structure, something that has become much more attractive now that people
use pack-files all the time.
As a result of pack-files, a git object directory doesn't necessarily have
any individual objects lying around, and in that case it's just wasting
space to keep the empty first-level object directories around: on many
filesystems the 256 empty directories will be aboue 1MB of diskspace.
Even more importantly, after you re-pack a project that _used_ to be
unpacked, you could be left with huge directories that no longer contain
anything, but that waste space and take time to look through.
With this change, "git prune-packed" can just do an rmdir() on the
directories, and they'll get removed if empty, and re-created on demand.
This patch also tries to fix up "write_sha1_from_fd()" to use the new
common infrastructure for creating the object files, closing a hole where
we might otherwise leave half-written objects in the object database.
[jc: I unoptimized the part that really removes the fan-out directories
to ease transition. init-db still wastes 1MB of diskspace to hold 256
empty fan-outs, and prune-packed rmdir()'s the grown but empty directories,
but runs mkdir() immediately after that -- reducing the saving from 150KB
to 146KB. These parts will be re-introduced when everybody has the
on-demand capability.]
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
Signed-off-by: Junio C Hamano <junkio@cox.net>
2005-10-08 22:54:01 +00:00
|
|
|
local = mkstemp(tmpfile);
|
2006-11-09 12:52:05 +00:00
|
|
|
if (local < 0) {
|
|
|
|
if (errno == EPERM)
|
|
|
|
return error("insufficient permission for adding an object to repository database %s\n", get_object_directory());
|
|
|
|
else
|
|
|
|
return error("unable to create temporary sha1 filename %s: %s\n", tmpfile, strerror(errno));
|
|
|
|
}
|
2005-04-24 01:47:23 +00:00
|
|
|
|
|
|
|
memset(&stream, 0, sizeof(stream));
|
|
|
|
|
|
|
|
inflateInit(&stream);
|
|
|
|
|
|
|
|
SHA1_Init(&c);
|
|
|
|
|
|
|
|
do {
|
|
|
|
ssize_t size;
|
2005-08-02 23:46:29 +00:00
|
|
|
if (*bufposn) {
|
|
|
|
stream.avail_in = *bufposn;
|
2005-08-09 20:54:40 +00:00
|
|
|
stream.next_in = (unsigned char *) buffer;
|
2005-08-02 23:46:29 +00:00
|
|
|
do {
|
|
|
|
stream.next_out = discard;
|
|
|
|
stream.avail_out = sizeof(discard);
|
|
|
|
ret = inflate(&stream, Z_SYNC_FLUSH);
|
|
|
|
SHA1_Update(&c, discard, sizeof(discard) -
|
|
|
|
stream.avail_out);
|
|
|
|
} while (stream.avail_in && ret == Z_OK);
|
2006-05-24 15:30:54 +00:00
|
|
|
if (write_buffer(local, buffer, *bufposn - stream.avail_in) < 0)
|
|
|
|
die("unable to write sha1 file");
|
2005-08-02 23:46:29 +00:00
|
|
|
memmove(buffer, buffer + *bufposn - stream.avail_in,
|
|
|
|
stream.avail_in);
|
|
|
|
*bufposn = stream.avail_in;
|
|
|
|
if (ret != Z_OK)
|
|
|
|
break;
|
|
|
|
}
|
2007-01-08 15:58:08 +00:00
|
|
|
size = xread(fd, buffer + *bufposn, bufsize - *bufposn);
|
2005-04-24 01:47:23 +00:00
|
|
|
if (size <= 0) {
|
|
|
|
close(local);
|
Create object subdirectories on demand
This makes it possible to have a "sparse" git object subdirectory
structure, something that has become much more attractive now that people
use pack-files all the time.
As a result of pack-files, a git object directory doesn't necessarily have
any individual objects lying around, and in that case it's just wasting
space to keep the empty first-level object directories around: on many
filesystems the 256 empty directories will be aboue 1MB of diskspace.
Even more importantly, after you re-pack a project that _used_ to be
unpacked, you could be left with huge directories that no longer contain
anything, but that waste space and take time to look through.
With this change, "git prune-packed" can just do an rmdir() on the
directories, and they'll get removed if empty, and re-created on demand.
This patch also tries to fix up "write_sha1_from_fd()" to use the new
common infrastructure for creating the object files, closing a hole where
we might otherwise leave half-written objects in the object database.
[jc: I unoptimized the part that really removes the fan-out directories
to ease transition. init-db still wastes 1MB of diskspace to hold 256
empty fan-outs, and prune-packed rmdir()'s the grown but empty directories,
but runs mkdir() immediately after that -- reducing the saving from 150KB
to 146KB. These parts will be re-introduced when everybody has the
on-demand capability.]
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
Signed-off-by: Junio C Hamano <junkio@cox.net>
2005-10-08 22:54:01 +00:00
|
|
|
unlink(tmpfile);
|
2005-04-24 01:47:23 +00:00
|
|
|
if (!size)
|
|
|
|
return error("Connection closed?");
|
|
|
|
perror("Reading from connection");
|
|
|
|
return -1;
|
|
|
|
}
|
2005-08-02 23:46:29 +00:00
|
|
|
*bufposn += size;
|
|
|
|
} while (1);
|
2005-04-24 01:47:23 +00:00
|
|
|
inflateEnd(&stream);
|
|
|
|
|
2007-03-24 16:02:27 +00:00
|
|
|
fchmod(local, 0444);
|
2007-03-26 13:57:23 +00:00
|
|
|
if (close(local) != 0)
|
|
|
|
die("unable to write sha1 file");
|
2005-04-24 01:47:23 +00:00
|
|
|
SHA1_Final(real_sha1, &c);
|
|
|
|
if (ret != Z_STREAM_END) {
|
Create object subdirectories on demand
This makes it possible to have a "sparse" git object subdirectory
structure, something that has become much more attractive now that people
use pack-files all the time.
As a result of pack-files, a git object directory doesn't necessarily have
any individual objects lying around, and in that case it's just wasting
space to keep the empty first-level object directories around: on many
filesystems the 256 empty directories will be aboue 1MB of diskspace.
Even more importantly, after you re-pack a project that _used_ to be
unpacked, you could be left with huge directories that no longer contain
anything, but that waste space and take time to look through.
With this change, "git prune-packed" can just do an rmdir() on the
directories, and they'll get removed if empty, and re-created on demand.
This patch also tries to fix up "write_sha1_from_fd()" to use the new
common infrastructure for creating the object files, closing a hole where
we might otherwise leave half-written objects in the object database.
[jc: I unoptimized the part that really removes the fan-out directories
to ease transition. init-db still wastes 1MB of diskspace to hold 256
empty fan-outs, and prune-packed rmdir()'s the grown but empty directories,
but runs mkdir() immediately after that -- reducing the saving from 150KB
to 146KB. These parts will be re-introduced when everybody has the
on-demand capability.]
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
Signed-off-by: Junio C Hamano <junkio@cox.net>
2005-10-08 22:54:01 +00:00
|
|
|
unlink(tmpfile);
|
2005-04-24 01:47:23 +00:00
|
|
|
return error("File %s corrupted", sha1_to_hex(sha1));
|
|
|
|
}
|
2006-08-17 18:54:57 +00:00
|
|
|
if (hashcmp(sha1, real_sha1)) {
|
Create object subdirectories on demand
This makes it possible to have a "sparse" git object subdirectory
structure, something that has become much more attractive now that people
use pack-files all the time.
As a result of pack-files, a git object directory doesn't necessarily have
any individual objects lying around, and in that case it's just wasting
space to keep the empty first-level object directories around: on many
filesystems the 256 empty directories will be aboue 1MB of diskspace.
Even more importantly, after you re-pack a project that _used_ to be
unpacked, you could be left with huge directories that no longer contain
anything, but that waste space and take time to look through.
With this change, "git prune-packed" can just do an rmdir() on the
directories, and they'll get removed if empty, and re-created on demand.
This patch also tries to fix up "write_sha1_from_fd()" to use the new
common infrastructure for creating the object files, closing a hole where
we might otherwise leave half-written objects in the object database.
[jc: I unoptimized the part that really removes the fan-out directories
to ease transition. init-db still wastes 1MB of diskspace to hold 256
empty fan-outs, and prune-packed rmdir()'s the grown but empty directories,
but runs mkdir() immediately after that -- reducing the saving from 150KB
to 146KB. These parts will be re-introduced when everybody has the
on-demand capability.]
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
Signed-off-by: Junio C Hamano <junkio@cox.net>
2005-10-08 22:54:01 +00:00
|
|
|
unlink(tmpfile);
|
2006-02-23 01:47:10 +00:00
|
|
|
return error("File %s has bad hash", sha1_to_hex(sha1));
|
2005-04-24 01:47:23 +00:00
|
|
|
}
|
Create object subdirectories on demand
This makes it possible to have a "sparse" git object subdirectory
structure, something that has become much more attractive now that people
use pack-files all the time.
As a result of pack-files, a git object directory doesn't necessarily have
any individual objects lying around, and in that case it's just wasting
space to keep the empty first-level object directories around: on many
filesystems the 256 empty directories will be aboue 1MB of diskspace.
Even more importantly, after you re-pack a project that _used_ to be
unpacked, you could be left with huge directories that no longer contain
anything, but that waste space and take time to look through.
With this change, "git prune-packed" can just do an rmdir() on the
directories, and they'll get removed if empty, and re-created on demand.
This patch also tries to fix up "write_sha1_from_fd()" to use the new
common infrastructure for creating the object files, closing a hole where
we might otherwise leave half-written objects in the object database.
[jc: I unoptimized the part that really removes the fan-out directories
to ease transition. init-db still wastes 1MB of diskspace to hold 256
empty fan-outs, and prune-packed rmdir()'s the grown but empty directories,
but runs mkdir() immediately after that -- reducing the saving from 150KB
to 146KB. These parts will be re-introduced when everybody has the
on-demand capability.]
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
Signed-off-by: Junio C Hamano <junkio@cox.net>
2005-10-08 22:54:01 +00:00
|
|
|
|
|
|
|
return move_temp_to_file(tmpfile, sha1_file_name(sha1));
|
2005-04-24 01:47:23 +00:00
|
|
|
}
|
|
|
|
|
2005-08-01 00:53:44 +00:00
|
|
|
int has_pack_index(const unsigned char *sha1)
|
|
|
|
{
|
|
|
|
struct stat st;
|
|
|
|
if (stat(sha1_pack_index_name(sha1), &st))
|
|
|
|
return 0;
|
|
|
|
return 1;
|
|
|
|
}
|
|
|
|
|
|
|
|
int has_pack_file(const unsigned char *sha1)
|
|
|
|
{
|
|
|
|
struct stat st;
|
|
|
|
if (stat(sha1_pack_name(sha1), &st))
|
|
|
|
return 0;
|
|
|
|
return 1;
|
|
|
|
}
|
|
|
|
|
2006-09-06 09:12:09 +00:00
|
|
|
int has_sha1_pack(const unsigned char *sha1, const char **ignore_packed)
|
2005-07-03 20:06:36 +00:00
|
|
|
{
|
|
|
|
struct pack_entry e;
|
2006-09-06 09:12:09 +00:00
|
|
|
return find_pack_entry(sha1, &e, ignore_packed);
|
2005-07-03 20:06:36 +00:00
|
|
|
}
|
|
|
|
|
2005-04-24 01:47:23 +00:00
|
|
|
int has_sha1_file(const unsigned char *sha1)
|
|
|
|
{
|
|
|
|
struct stat st;
|
2005-06-27 10:35:33 +00:00
|
|
|
struct pack_entry e;
|
|
|
|
|
2006-09-06 09:12:09 +00:00
|
|
|
if (find_pack_entry(sha1, &e, NULL))
|
2005-06-27 10:35:33 +00:00
|
|
|
return 1;
|
2005-07-11 07:00:55 +00:00
|
|
|
return find_sha1_file(sha1, &st) ? 1 : 0;
|
2005-04-24 01:47:23 +00:00
|
|
|
}
|
2005-05-02 06:45:49 +00:00
|
|
|
|
2006-05-23 18:19:04 +00:00
|
|
|
int index_pipe(unsigned char *sha1, int fd, const char *type, int write_object)
|
|
|
|
{
|
2007-09-10 10:35:09 +00:00
|
|
|
struct strbuf buf;
|
2006-05-23 18:19:04 +00:00
|
|
|
int ret;
|
|
|
|
|
2007-09-10 10:35:09 +00:00
|
|
|
strbuf_init(&buf, 0);
|
|
|
|
if (strbuf_read(&buf, fd, 4096) < 0) {
|
|
|
|
strbuf_release(&buf);
|
2005-12-10 22:25:24 +00:00
|
|
|
return -1;
|
|
|
|
}
|
2006-05-23 18:19:04 +00:00
|
|
|
|
2005-12-10 22:25:24 +00:00
|
|
|
if (!type)
|
2006-04-02 12:44:09 +00:00
|
|
|
type = blob_type;
|
2005-12-10 22:25:24 +00:00
|
|
|
if (write_object)
|
2007-09-10 10:35:09 +00:00
|
|
|
ret = write_sha1_file(buf.buf, buf.len, type, sha1);
|
2006-10-14 10:45:36 +00:00
|
|
|
else
|
2007-09-10 10:35:09 +00:00
|
|
|
ret = hash_sha1_file(buf.buf, buf.len, type, sha1);
|
|
|
|
strbuf_release(&buf);
|
|
|
|
|
2005-12-10 22:25:24 +00:00
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
2007-02-28 19:45:56 +00:00
|
|
|
int index_fd(unsigned char *sha1, int fd, struct stat *st, int write_object,
|
2007-02-28 19:52:04 +00:00
|
|
|
enum object_type type, const char *path)
|
2005-05-02 06:45:49 +00:00
|
|
|
{
|
2007-03-07 01:44:37 +00:00
|
|
|
size_t size = xsize_t(st->st_size);
|
2007-03-07 01:44:17 +00:00
|
|
|
void *buf = NULL;
|
Lazy man's auto-CRLF
It currently does NOT know about file attributes, so it does its
conversion purely based on content. Maybe that is more in the "git
philosophy" anyway, since content is king, but I think we should try to do
the file attributes to turn it off on demand.
Anyway, BY DEFAULT it is off regardless, because it requires a
[core]
AutoCRLF = true
in your config file to be enabled. We could make that the default for
Windows, of course, the same way we do some other things (filemode etc).
But you can actually enable it on UNIX, and it will cause:
- "git update-index" will write blobs without CRLF
- "git diff" will diff working tree files without CRLF
- "git checkout" will write files to the working tree _with_ CRLF
and things work fine.
Funnily, it actually shows an odd file in git itself:
git clone -n git test-crlf
cd test-crlf
git config core.autocrlf true
git checkout
git diff
shows a diff for "Documentation/docbook-xsl.css". Why? Because we have
actually checked in that file *with* CRLF! So when "core.autocrlf" is
true, we'll always generate a *different* hash for it in the index,
because the index hash will be for the content _without_ CRLF.
Is this complete? I dunno. It seems to work for me. It doesn't use the
filename at all right now, and that's probably a deficiency (we could
certainly make the "is_binary()" heuristics also take standard filename
heuristics into account).
I don't pass in the filename at all for the "index_fd()" case
(git-update-index), so that would need to be passed around, but this
actually works fine.
NOTE NOTE NOTE! The "is_binary()" heuristics are totally made-up by yours
truly. I will not guarantee that they work at all reasonable. Caveat
emptor. But it _is_ simple, and it _is_ safe, since it's all off by
default.
The patch is pretty simple - the biggest part is the new "convert.c" file,
but even that is really just basic stuff that anybody can write in
"Teaching C 101" as a final project for their first class in programming.
Not to say that it's bug-free, of course - but at least we're not talking
about rocket surgery here.
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
Signed-off-by: Junio C Hamano <junkio@cox.net>
2007-02-13 19:07:23 +00:00
|
|
|
int ret, re_allocated = 0;
|
2005-05-02 06:45:49 +00:00
|
|
|
|
|
|
|
if (size)
|
2006-12-24 05:47:23 +00:00
|
|
|
buf = xmmap(NULL, size, PROT_READ, MAP_PRIVATE, fd, 0);
|
2005-05-02 06:45:49 +00:00
|
|
|
close(fd);
|
|
|
|
|
2005-07-08 23:51:55 +00:00
|
|
|
if (!type)
|
2007-02-28 19:45:56 +00:00
|
|
|
type = OBJ_BLOB;
|
Lazy man's auto-CRLF
It currently does NOT know about file attributes, so it does its
conversion purely based on content. Maybe that is more in the "git
philosophy" anyway, since content is king, but I think we should try to do
the file attributes to turn it off on demand.
Anyway, BY DEFAULT it is off regardless, because it requires a
[core]
AutoCRLF = true
in your config file to be enabled. We could make that the default for
Windows, of course, the same way we do some other things (filemode etc).
But you can actually enable it on UNIX, and it will cause:
- "git update-index" will write blobs without CRLF
- "git diff" will diff working tree files without CRLF
- "git checkout" will write files to the working tree _with_ CRLF
and things work fine.
Funnily, it actually shows an odd file in git itself:
git clone -n git test-crlf
cd test-crlf
git config core.autocrlf true
git checkout
git diff
shows a diff for "Documentation/docbook-xsl.css". Why? Because we have
actually checked in that file *with* CRLF! So when "core.autocrlf" is
true, we'll always generate a *different* hash for it in the index,
because the index hash will be for the content _without_ CRLF.
Is this complete? I dunno. It seems to work for me. It doesn't use the
filename at all right now, and that's probably a deficiency (we could
certainly make the "is_binary()" heuristics also take standard filename
heuristics into account).
I don't pass in the filename at all for the "index_fd()" case
(git-update-index), so that would need to be passed around, but this
actually works fine.
NOTE NOTE NOTE! The "is_binary()" heuristics are totally made-up by yours
truly. I will not guarantee that they work at all reasonable. Caveat
emptor. But it _is_ simple, and it _is_ safe, since it's all off by
default.
The patch is pretty simple - the biggest part is the new "convert.c" file,
but even that is really just basic stuff that anybody can write in
"Teaching C 101" as a final project for their first class in programming.
Not to say that it's bug-free, of course - but at least we're not talking
about rocket surgery here.
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
Signed-off-by: Junio C Hamano <junkio@cox.net>
2007-02-13 19:07:23 +00:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Convert blobs to git internal format
|
|
|
|
*/
|
2007-02-28 19:57:39 +00:00
|
|
|
if ((type == OBJ_BLOB) && S_ISREG(st->st_mode)) {
|
Rewrite convert_to_{git,working_tree} to use strbuf's.
* Now, those functions take an "out" strbuf argument, where they store their
result if any. In that case, it also returns 1, else it returns 0.
* those functions support "in place" editing, in the sense that it's OK to
call them this way:
convert_to_git(path, sb->buf, sb->len, sb);
When doable, conversions are done in place for real, else the strbuf
content is just replaced with the new one, transparentely for the caller.
If you want to create a new filter working this way, being the accumulation
of filter1, filter2, ... filtern, then your meta_filter would be:
int meta_filter(..., const char *src, size_t len, struct strbuf *sb)
{
int ret = 0;
ret |= filter1(...., src, len, sb);
if (ret) {
src = sb->buf;
len = sb->len;
}
ret |= filter2(...., src, len, sb);
if (ret) {
src = sb->buf;
len = sb->len;
}
....
return ret | filtern(..., src, len, sb);
}
That's why subfilters the convert_to_* functions called were also rewritten
to work this way.
Signed-off-by: Pierre Habouzit <madcoder@debian.org>
Acked-by: Linus Torvalds <torvalds@linux-foundation.org>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2007-09-16 13:51:04 +00:00
|
|
|
struct strbuf nbuf;
|
|
|
|
strbuf_init(&nbuf, 0);
|
|
|
|
if (convert_to_git(path, buf, size, &nbuf)) {
|
2007-04-19 00:05:03 +00:00
|
|
|
munmap(buf, size);
|
2007-09-27 10:58:23 +00:00
|
|
|
buf = strbuf_detach(&nbuf, &size);
|
Lazy man's auto-CRLF
It currently does NOT know about file attributes, so it does its
conversion purely based on content. Maybe that is more in the "git
philosophy" anyway, since content is king, but I think we should try to do
the file attributes to turn it off on demand.
Anyway, BY DEFAULT it is off regardless, because it requires a
[core]
AutoCRLF = true
in your config file to be enabled. We could make that the default for
Windows, of course, the same way we do some other things (filemode etc).
But you can actually enable it on UNIX, and it will cause:
- "git update-index" will write blobs without CRLF
- "git diff" will diff working tree files without CRLF
- "git checkout" will write files to the working tree _with_ CRLF
and things work fine.
Funnily, it actually shows an odd file in git itself:
git clone -n git test-crlf
cd test-crlf
git config core.autocrlf true
git checkout
git diff
shows a diff for "Documentation/docbook-xsl.css". Why? Because we have
actually checked in that file *with* CRLF! So when "core.autocrlf" is
true, we'll always generate a *different* hash for it in the index,
because the index hash will be for the content _without_ CRLF.
Is this complete? I dunno. It seems to work for me. It doesn't use the
filename at all right now, and that's probably a deficiency (we could
certainly make the "is_binary()" heuristics also take standard filename
heuristics into account).
I don't pass in the filename at all for the "index_fd()" case
(git-update-index), so that would need to be passed around, but this
actually works fine.
NOTE NOTE NOTE! The "is_binary()" heuristics are totally made-up by yours
truly. I will not guarantee that they work at all reasonable. Caveat
emptor. But it _is_ simple, and it _is_ safe, since it's all off by
default.
The patch is pretty simple - the biggest part is the new "convert.c" file,
but even that is really just basic stuff that anybody can write in
"Teaching C 101" as a final project for their first class in programming.
Not to say that it's bug-free, of course - but at least we're not talking
about rocket surgery here.
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
Signed-off-by: Junio C Hamano <junkio@cox.net>
2007-02-13 19:07:23 +00:00
|
|
|
re_allocated = 1;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2005-07-08 23:51:55 +00:00
|
|
|
if (write_object)
|
2007-02-28 19:45:56 +00:00
|
|
|
ret = write_sha1_file(buf, size, typename(type), sha1);
|
2006-10-14 10:45:36 +00:00
|
|
|
else
|
2007-02-28 19:45:56 +00:00
|
|
|
ret = hash_sha1_file(buf, size, typename(type), sha1);
|
Lazy man's auto-CRLF
It currently does NOT know about file attributes, so it does its
conversion purely based on content. Maybe that is more in the "git
philosophy" anyway, since content is king, but I think we should try to do
the file attributes to turn it off on demand.
Anyway, BY DEFAULT it is off regardless, because it requires a
[core]
AutoCRLF = true
in your config file to be enabled. We could make that the default for
Windows, of course, the same way we do some other things (filemode etc).
But you can actually enable it on UNIX, and it will cause:
- "git update-index" will write blobs without CRLF
- "git diff" will diff working tree files without CRLF
- "git checkout" will write files to the working tree _with_ CRLF
and things work fine.
Funnily, it actually shows an odd file in git itself:
git clone -n git test-crlf
cd test-crlf
git config core.autocrlf true
git checkout
git diff
shows a diff for "Documentation/docbook-xsl.css". Why? Because we have
actually checked in that file *with* CRLF! So when "core.autocrlf" is
true, we'll always generate a *different* hash for it in the index,
because the index hash will be for the content _without_ CRLF.
Is this complete? I dunno. It seems to work for me. It doesn't use the
filename at all right now, and that's probably a deficiency (we could
certainly make the "is_binary()" heuristics also take standard filename
heuristics into account).
I don't pass in the filename at all for the "index_fd()" case
(git-update-index), so that would need to be passed around, but this
actually works fine.
NOTE NOTE NOTE! The "is_binary()" heuristics are totally made-up by yours
truly. I will not guarantee that they work at all reasonable. Caveat
emptor. But it _is_ simple, and it _is_ safe, since it's all off by
default.
The patch is pretty simple - the biggest part is the new "convert.c" file,
but even that is really just basic stuff that anybody can write in
"Teaching C 101" as a final project for their first class in programming.
Not to say that it's bug-free, of course - but at least we're not talking
about rocket surgery here.
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
Signed-off-by: Junio C Hamano <junkio@cox.net>
2007-02-13 19:07:23 +00:00
|
|
|
if (re_allocated) {
|
|
|
|
free(buf);
|
|
|
|
return ret;
|
|
|
|
}
|
2005-05-03 18:46:16 +00:00
|
|
|
if (size)
|
|
|
|
munmap(buf, size);
|
|
|
|
return ret;
|
2005-05-02 06:45:49 +00:00
|
|
|
}
|
2005-10-07 10:42:00 +00:00
|
|
|
|
|
|
|
int index_path(unsigned char *sha1, const char *path, struct stat *st, int write_object)
|
|
|
|
{
|
|
|
|
int fd;
|
|
|
|
char *target;
|
2007-03-07 01:44:37 +00:00
|
|
|
size_t len;
|
2005-10-07 10:42:00 +00:00
|
|
|
|
|
|
|
switch (st->st_mode & S_IFMT) {
|
|
|
|
case S_IFREG:
|
|
|
|
fd = open(path, O_RDONLY);
|
|
|
|
if (fd < 0)
|
|
|
|
return error("open(\"%s\"): %s", path,
|
|
|
|
strerror(errno));
|
2007-02-28 19:52:04 +00:00
|
|
|
if (index_fd(sha1, fd, st, write_object, OBJ_BLOB, path) < 0)
|
2005-10-07 10:42:00 +00:00
|
|
|
return error("%s: failed to insert into database",
|
|
|
|
path);
|
|
|
|
break;
|
|
|
|
case S_IFLNK:
|
2007-03-07 01:44:37 +00:00
|
|
|
len = xsize_t(st->st_size);
|
|
|
|
target = xmalloc(len + 1);
|
|
|
|
if (readlink(path, target, len + 1) != st->st_size) {
|
2005-10-07 10:42:00 +00:00
|
|
|
char *errstr = strerror(errno);
|
|
|
|
free(target);
|
|
|
|
return error("readlink(\"%s\"): %s", path,
|
|
|
|
errstr);
|
|
|
|
}
|
2006-10-14 10:45:36 +00:00
|
|
|
if (!write_object)
|
2007-03-07 01:44:37 +00:00
|
|
|
hash_sha1_file(target, len, blob_type, sha1);
|
|
|
|
else if (write_sha1_file(target, len, blob_type, sha1))
|
2005-10-07 10:42:00 +00:00
|
|
|
return error("%s: failed to insert into database",
|
|
|
|
path);
|
|
|
|
free(target);
|
|
|
|
break;
|
2007-04-10 04:20:29 +00:00
|
|
|
case S_IFDIR:
|
|
|
|
return resolve_gitlink_ref(path, "HEAD", sha1);
|
2005-10-07 10:42:00 +00:00
|
|
|
default:
|
|
|
|
return error("%s: unsupported file type", path);
|
|
|
|
}
|
|
|
|
return 0;
|
|
|
|
}
|
2007-01-23 05:55:18 +00:00
|
|
|
|
|
|
|
int read_pack_header(int fd, struct pack_header *header)
|
|
|
|
{
|
|
|
|
char *c = (char*)header;
|
|
|
|
ssize_t remaining = sizeof(struct pack_header);
|
|
|
|
do {
|
|
|
|
ssize_t r = xread(fd, c, remaining);
|
|
|
|
if (r <= 0)
|
|
|
|
/* "eof before pack header was fully read" */
|
|
|
|
return PH_ERROR_EOF;
|
|
|
|
remaining -= r;
|
|
|
|
c += r;
|
|
|
|
} while (remaining > 0);
|
|
|
|
if (header->hdr_signature != htonl(PACK_SIGNATURE))
|
|
|
|
/* "protocol error (pack signature mismatch detected)" */
|
|
|
|
return PH_ERROR_PACK_SIGNATURE;
|
|
|
|
if (!pack_version_ok(header->hdr_version))
|
|
|
|
/* "protocol error (pack version unsupported)" */
|
|
|
|
return PH_ERROR_PROTOCOL;
|
|
|
|
return 0;
|
|
|
|
}
|