git/convert.h

159 lines
5 KiB
C
Raw Normal View History

/*
* Copyright (c) 2011, Google Inc.
*/
#ifndef CONVERT_H
#define CONVERT_H
convert: permit passing additional metadata to filter processes There are a variety of situations where a filter process can make use of some additional metadata. For example, some people find the ident filter too limiting and would like to include the commit or the branch in their smudged files. This information isn't available during checkout as HEAD hasn't been updated at that point, and it wouldn't be available in archives either. Let's add a way to pass this metadata down to the filter. We pass the blob we're operating on, the treeish (preferring the commit over the tree if one exists), and the ref we're operating on. Note that we won't pass this information in all cases, such as when renormalizing or when we're performing diffs, since it doesn't make sense in those cases. The data we currently get from the filter process looks like the following: command=smudge pathname=git.c 0000 With this change, we'll get data more like this: command=smudge pathname=git.c refname=refs/tags/v2.25.1 treeish=c522f061d551c9bb8684a7c3859b2ece4499b56b blob=7be7ad34bd053884ec48923706e70c81719a8660 0000 There are a couple things to note about this approach. For operations like checkout, treeish will always be a commit, since we cannot check out individual trees, but for other operations, like archive, we can end up operating on only a particular tree, so we'll provide only a tree as the treeish. Similar comments apply for refname, since there are a variety of cases in which we won't have a ref. This commit wires up the code to print this information, but doesn't pass any of it at this point. In a future commit, we'll have various code paths pass the actual useful data down. Signed-off-by: brian m. carlson <bk2204@github.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>
2020-03-16 18:05:02 +00:00
#include "hash.h"
#include "string-list.h"
struct index_state;
struct strbuf;
#define CONV_EOL_RNDTRP_DIE (1<<0) /* Die if CRLF to LF to CRLF is different */
#define CONV_EOL_RNDTRP_WARN (1<<1) /* Warn if CRLF to LF to CRLF is different */
#define CONV_EOL_RENORMALIZE (1<<2) /* Convert CRLF to LF */
#define CONV_EOL_KEEP_CRLF (1<<3) /* Keep CRLF line endings as is */
#define CONV_WRITE_OBJECT (1<<4) /* Content is written to the index */
extern int global_conv_flags_eol;
enum auto_crlf {
AUTO_CRLF_FALSE = 0,
AUTO_CRLF_TRUE = 1,
AUTO_CRLF_INPUT = -1
};
extern enum auto_crlf auto_crlf;
enum eol {
EOL_UNSET,
EOL_CRLF,
EOL_LF,
#ifdef NATIVE_CRLF
EOL_NATIVE = EOL_CRLF
#else
EOL_NATIVE = EOL_LF
#endif
};
enum ce_delay_state {
CE_NO_DELAY = 0,
CE_CAN_DELAY = 1,
CE_RETRY = 2
};
struct delayed_checkout {
/*
* State of the currently processed cache entry. If the state is
* CE_CAN_DELAY, then the filter can delay the current cache entry.
* If the state is CE_RETRY, then this signals the filter that the
* cache entry was requested before.
*/
enum ce_delay_state state;
/* List of filter drivers that signaled delayed blobs. */
struct string_list filters;
/* List of delayed blobs identified by their path. */
struct string_list paths;
};
convert: permit passing additional metadata to filter processes There are a variety of situations where a filter process can make use of some additional metadata. For example, some people find the ident filter too limiting and would like to include the commit or the branch in their smudged files. This information isn't available during checkout as HEAD hasn't been updated at that point, and it wouldn't be available in archives either. Let's add a way to pass this metadata down to the filter. We pass the blob we're operating on, the treeish (preferring the commit over the tree if one exists), and the ref we're operating on. Note that we won't pass this information in all cases, such as when renormalizing or when we're performing diffs, since it doesn't make sense in those cases. The data we currently get from the filter process looks like the following: command=smudge pathname=git.c 0000 With this change, we'll get data more like this: command=smudge pathname=git.c refname=refs/tags/v2.25.1 treeish=c522f061d551c9bb8684a7c3859b2ece4499b56b blob=7be7ad34bd053884ec48923706e70c81719a8660 0000 There are a couple things to note about this approach. For operations like checkout, treeish will always be a commit, since we cannot check out individual trees, but for other operations, like archive, we can end up operating on only a particular tree, so we'll provide only a tree as the treeish. Similar comments apply for refname, since there are a variety of cases in which we won't have a ref. This commit wires up the code to print this information, but doesn't pass any of it at this point. In a future commit, we'll have various code paths pass the actual useful data down. Signed-off-by: brian m. carlson <bk2204@github.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>
2020-03-16 18:05:02 +00:00
struct checkout_metadata {
const char *refname;
struct object_id treeish;
struct object_id blob;
};
extern enum eol core_eol;
extern char *check_roundtrip_encoding;
const char *get_cached_convert_stats_ascii(const struct index_state *istate,
const char *path);
const char *get_wt_convert_stats_ascii(const char *path);
const char *get_convert_attr_ascii(const struct index_state *istate,
const char *path);
/* returns 1 if *dst was used */
int convert_to_git(const struct index_state *istate,
const char *path, const char *src, size_t len,
struct strbuf *dst, int conv_flags);
int convert_to_working_tree(const struct index_state *istate,
const char *path, const char *src,
convert: permit passing additional metadata to filter processes There are a variety of situations where a filter process can make use of some additional metadata. For example, some people find the ident filter too limiting and would like to include the commit or the branch in their smudged files. This information isn't available during checkout as HEAD hasn't been updated at that point, and it wouldn't be available in archives either. Let's add a way to pass this metadata down to the filter. We pass the blob we're operating on, the treeish (preferring the commit over the tree if one exists), and the ref we're operating on. Note that we won't pass this information in all cases, such as when renormalizing or when we're performing diffs, since it doesn't make sense in those cases. The data we currently get from the filter process looks like the following: command=smudge pathname=git.c 0000 With this change, we'll get data more like this: command=smudge pathname=git.c refname=refs/tags/v2.25.1 treeish=c522f061d551c9bb8684a7c3859b2ece4499b56b blob=7be7ad34bd053884ec48923706e70c81719a8660 0000 There are a couple things to note about this approach. For operations like checkout, treeish will always be a commit, since we cannot check out individual trees, but for other operations, like archive, we can end up operating on only a particular tree, so we'll provide only a tree as the treeish. Similar comments apply for refname, since there are a variety of cases in which we won't have a ref. This commit wires up the code to print this information, but doesn't pass any of it at this point. In a future commit, we'll have various code paths pass the actual useful data down. Signed-off-by: brian m. carlson <bk2204@github.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>
2020-03-16 18:05:02 +00:00
size_t len, struct strbuf *dst,
const struct checkout_metadata *meta);
int async_convert_to_working_tree(const struct index_state *istate,
const char *path, const char *src,
size_t len, struct strbuf *dst,
convert: permit passing additional metadata to filter processes There are a variety of situations where a filter process can make use of some additional metadata. For example, some people find the ident filter too limiting and would like to include the commit or the branch in their smudged files. This information isn't available during checkout as HEAD hasn't been updated at that point, and it wouldn't be available in archives either. Let's add a way to pass this metadata down to the filter. We pass the blob we're operating on, the treeish (preferring the commit over the tree if one exists), and the ref we're operating on. Note that we won't pass this information in all cases, such as when renormalizing or when we're performing diffs, since it doesn't make sense in those cases. The data we currently get from the filter process looks like the following: command=smudge pathname=git.c 0000 With this change, we'll get data more like this: command=smudge pathname=git.c refname=refs/tags/v2.25.1 treeish=c522f061d551c9bb8684a7c3859b2ece4499b56b blob=7be7ad34bd053884ec48923706e70c81719a8660 0000 There are a couple things to note about this approach. For operations like checkout, treeish will always be a commit, since we cannot check out individual trees, but for other operations, like archive, we can end up operating on only a particular tree, so we'll provide only a tree as the treeish. Similar comments apply for refname, since there are a variety of cases in which we won't have a ref. This commit wires up the code to print this information, but doesn't pass any of it at this point. In a future commit, we'll have various code paths pass the actual useful data down. Signed-off-by: brian m. carlson <bk2204@github.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>
2020-03-16 18:05:02 +00:00
const struct checkout_metadata *meta,
void *dco);
int async_query_available_blobs(const char *cmd,
struct string_list *available_paths);
int renormalize_buffer(const struct index_state *istate,
const char *path, const char *src, size_t len,
struct strbuf *dst);
static inline int would_convert_to_git(const struct index_state *istate,
const char *path)
{
return convert_to_git(istate, path, NULL, 0, NULL, 0);
}
convert: stream from fd to required clean filter to reduce used address space The data is streamed to the filter process anyway. Better avoid mapping the file if possible. This is especially useful if a clean filter reduces the size, for example if it computes a sha1 for binary data, like git media. The file size that the previous implementation could handle was limited by the available address space; large files for example could not be handled with (32-bit) msysgit. The new implementation can filter files of any size as long as the filter output is small enough. The new code path is only taken if the filter is required. The filter consumes data directly from the fd. If it fails, the original data is not immediately available. The condition can easily be handled as a fatal error, which is expected for a required filter anyway. If the filter was not required, the condition would need to be handled in a different way, like seeking to 0 and reading the data. But this would require more restructuring of the code and is probably not worth it. The obvious approach of falling back to reading all data would not help achieving the main purpose of this patch, which is to handle large files with limited address space. If reading all data is an option, we can simply take the old code path right away and mmap the entire file. The environment variable GIT_MMAP_LIMIT, which has been introduced in a previous commit is used to test that the expected code path is taken. A related test that exercises required filters is modified to verify that the data actually has been modified on its way from the file system to the object store. Signed-off-by: Steffen Prohaska <prohaska@zib.de> Signed-off-by: Junio C Hamano <gitster@pobox.com>
2014-08-26 15:23:25 +00:00
/* Precondition: would_convert_to_git_filter_fd(path) == true */
void convert_to_git_filter_fd(const struct index_state *istate,
const char *path, int fd,
struct strbuf *dst,
int conv_flags);
int would_convert_to_git_filter_fd(const struct index_state *istate,
const char *path);
/*
* Initialize the checkout metadata with the given values. Any argument may be
* NULL if it is not applicable. The treeish should be a commit if that is
* available, and a tree otherwise.
*
* The refname is not copied and must be valid for the lifetime of the struct.
* THe object IDs are copied.
*/
void init_checkout_metadata(struct checkout_metadata *meta, const char *refname,
const struct object_id *treeish,
const struct object_id *blob);
/* Copy the metadata from src to dst, updating the blob. */
void clone_checkout_metadata(struct checkout_metadata *dst,
const struct checkout_metadata *src,
const struct object_id *blob);
/*
* Reset the internal list of attributes used by convert_to_git and
* convert_to_working_tree.
*/
void reset_parsed_attributes(void);
/*****************************************************************
*
* Streaming conversion support
*
*****************************************************************/
struct stream_filter; /* opaque */
struct stream_filter *get_stream_filter(const struct index_state *istate,
const char *path,
const struct object_id *);
void free_stream_filter(struct stream_filter *);
int is_null_stream_filter(struct stream_filter *);
/*
* Use as much input up to *isize_p and fill output up to *osize_p;
* update isize_p and osize_p to indicate how much buffer space was
* consumed and filled. Return 0 on success, non-zero on error.
*
* Some filters may need to buffer the input and look-ahead inside it
* to decide what to output, and they may consume more than zero bytes
* of input and still not produce any output. After feeding all the
* input, pass NULL as input and keep calling this function, to let
* such filters know there is no more input coming and it is time for
* them to produce the remaining output based on the buffered input.
*/
int stream_filter(struct stream_filter *,
const char *input, size_t *isize_p,
char *output, size_t *osize_p);
#endif /* CONVERT_H */