diff --git a/contrib/xz/src/liblzma/api/lzma/container.h b/contrib/xz/src/liblzma/api/lzma/container.h index 2849fbfd3c51..4cbb1b7bf7b4 100644 --- a/contrib/xz/src/liblzma/api/lzma/container.h +++ b/contrib/xz/src/liblzma/api/lzma/container.h @@ -435,6 +435,34 @@ extern LZMA_API(lzma_ret) lzma_stream_encoder_mt( lzma_nothrow lzma_attr_warn_unused_result; +/** + * \brief Calculate recommended Block size for multithreaded .xz encoder + * + * This calculates a recommended Block size for multithreaded encoding given + * a filter chain. This is used internally by lzma_stream_encoder_mt() to + * determine the Block size if the block_size member is not set to the + * special value of 0 in the lzma_mt options struct. + * + * If one wishes to change the filters between Blocks, this function is + * helpful to set the block_size member of the lzma_mt struct before calling + * lzma_stream_encoder_mt(). Since the block_size member represents the + * maximum possible Block size for the multithreaded .xz encoder, one can + * use this function to find the maximum recommended Block size based on + * all planned filter chains. Otherwise, the multithreaded encoder will + * base its maximum Block size on the first filter chain used (if the + * block_size member is not set), which may unnecessarily limit the Block + * size for a later filter chain. + * + * \param filters Array of filters terminated with + * .id == LZMA_VLI_UNKNOWN. + * + * \return Recommended Block size in bytes, or UINT64_MAX if + * an error occurred. + */ +extern LZMA_API(uint64_t) lzma_mt_block_size(const lzma_filter *filters) + lzma_nothrow; + + /** * \brief Initialize .lzma encoder (legacy file format) * diff --git a/contrib/xz/src/liblzma/common/filter_encoder.c b/contrib/xz/src/liblzma/common/filter_encoder.c index 46fe8af1c153..0699bcee62a4 100644 --- a/contrib/xz/src/liblzma/common/filter_encoder.c +++ b/contrib/xz/src/liblzma/common/filter_encoder.c @@ -33,7 +33,8 @@ typedef struct { /// Calculates the recommended Uncompressed Size for .xz Blocks to /// which the input data can be split to make multithreaded /// encoding possible. If this is NULL, it is assumed that - /// the encoder is fast enough with single thread. + /// the encoder is fast enough with single thread. If the options + /// are invalid, UINT64_MAX is returned. uint64_t (*block_size)(const void *options); /// Tells the size of the Filter Properties field. If options are @@ -248,26 +249,29 @@ lzma_raw_encoder_memusage(const lzma_filter *filters) } -extern uint64_t +extern LZMA_API(uint64_t) lzma_mt_block_size(const lzma_filter *filters) { + if (filters == NULL) + return UINT64_MAX; + uint64_t max = 0; for (size_t i = 0; filters[i].id != LZMA_VLI_UNKNOWN; ++i) { const lzma_filter_encoder *const fe = encoder_find(filters[i].id); + if (fe == NULL) + return UINT64_MAX; + if (fe->block_size != NULL) { const uint64_t size = fe->block_size(filters[i].options); - if (size == 0) - return 0; - if (size > max) max = size; } } - return max; + return max == 0 ? UINT64_MAX : max; } diff --git a/contrib/xz/src/liblzma/common/filter_encoder.h b/contrib/xz/src/liblzma/common/filter_encoder.h index f1d5683fe793..da92be8b34da 100644 --- a/contrib/xz/src/liblzma/common/filter_encoder.h +++ b/contrib/xz/src/liblzma/common/filter_encoder.h @@ -1,6 +1,6 @@ /////////////////////////////////////////////////////////////////////////////// // -/// \file filter_encoder.c +/// \file filter_encoder.h /// \brief Filter ID mapping to filter-specific functions // // Author: Lasse Collin @@ -16,10 +16,6 @@ #include "common.h" -// FIXME: Might become a part of the public API. -extern uint64_t lzma_mt_block_size(const lzma_filter *filters); - - extern lzma_ret lzma_raw_encoder_init( lzma_next_coder *next, const lzma_allocator *allocator, const lzma_filter *filters); diff --git a/contrib/xz/src/liblzma/common/stream_encoder_mt.c b/contrib/xz/src/liblzma/common/stream_encoder_mt.c index f64de9bdbc57..64de526b6a17 100644 --- a/contrib/xz/src/liblzma/common/stream_encoder_mt.c +++ b/contrib/xz/src/liblzma/common/stream_encoder_mt.c @@ -979,20 +979,18 @@ get_options(const lzma_mt *options, lzma_options_easy *opt_easy, *filters = opt_easy->filters; } - // Block size - if (options->block_size > 0) { - if (options->block_size > BLOCK_SIZE_MAX) - return LZMA_OPTIONS_ERROR; - + // If the Block size is not set, determine it from the filter chain. + if (options->block_size > 0) *block_size = options->block_size; - } else { - // Determine the Block size from the filter chain. + else *block_size = lzma_mt_block_size(*filters); - if (*block_size == 0) - return LZMA_OPTIONS_ERROR; - assert(*block_size <= BLOCK_SIZE_MAX); - } + // UINT64_MAX > BLOCK_SIZE_MAX, so the second condition + // should be optimized out by any reasonable compiler. + // The second condition should be there in the unlikely event that + // the macros change and UINT64_MAX < BLOCK_SIZE_MAX. + if (*block_size > BLOCK_SIZE_MAX || *block_size == UINT64_MAX) + return LZMA_OPTIONS_ERROR; // Calculate the maximum amount output that a single output buffer // may need to hold. This is the same as the maximum total size of diff --git a/contrib/xz/src/liblzma/liblzma_generic.map b/contrib/xz/src/liblzma/liblzma_generic.map index bb82167ed57a..b251d366e15c 100644 --- a/contrib/xz/src/liblzma/liblzma_generic.map +++ b/contrib/xz/src/liblzma/liblzma_generic.map @@ -119,3 +119,8 @@ global: lzma_str_list_filters; lzma_str_to_filters; } XZ_5.2; + +XZ_5.5.0alpha { +global: + lzma_mt_block_size; +} XZ_5.4; diff --git a/contrib/xz/src/liblzma/liblzma_linux.map b/contrib/xz/src/liblzma/liblzma_linux.map index 449f5fd682db..25b393883693 100644 --- a/contrib/xz/src/liblzma/liblzma_linux.map +++ b/contrib/xz/src/liblzma/liblzma_linux.map @@ -134,3 +134,8 @@ global: lzma_str_list_filters; lzma_str_to_filters; } XZ_5.2; + +XZ_5.5.0alpha { +global: + lzma_mt_block_size; +} XZ_5.4; diff --git a/contrib/xz/src/liblzma/lz/lz_encoder.c b/contrib/xz/src/liblzma/lz/lz_encoder.c index 5489085a0860..8e724a035a13 100644 --- a/contrib/xz/src/liblzma/lz/lz_encoder.c +++ b/contrib/xz/src/liblzma/lz/lz_encoder.c @@ -196,9 +196,7 @@ lz_encoder_prepare(lzma_mf *mf, const lzma_allocator *allocator, // For now, the dictionary size is limited to 1.5 GiB. This may grow // in the future if needed, but it needs a little more work than just // changing this check. - if (lz_options->dict_size < LZMA_DICT_SIZE_MIN - || lz_options->dict_size - > (UINT32_C(1) << 30) + (UINT32_C(1) << 29) + if (!IS_ENC_DICT_SIZE_VALID(lz_options->dict_size) || lz_options->nice_len > lz_options->match_len_max) return true; diff --git a/contrib/xz/src/liblzma/lz/lz_encoder.h b/contrib/xz/src/liblzma/lz/lz_encoder.h index ffcba02ce931..b71f11805e50 100644 --- a/contrib/xz/src/liblzma/lz/lz_encoder.h +++ b/contrib/xz/src/liblzma/lz/lz_encoder.h @@ -17,6 +17,14 @@ #include "common.h" +// For now, the dictionary size is limited to 1.5 GiB. This may grow +// in the future if needed, but it needs a little more work than just +// changing this check. +#define IS_ENC_DICT_SIZE_VALID(size) \ + ((size) >= LZMA_DICT_SIZE_MIN \ + && (size) <= (UINT32_C(1) << 30) + (UINT32_C(1) << 29)) + + /// A table of these is used by the LZ-based encoder to hold /// the length-distance pairs found by the match finder. typedef struct { diff --git a/contrib/xz/src/liblzma/lzma/lzma2_encoder.c b/contrib/xz/src/liblzma/lzma/lzma2_encoder.c index 4b6b23118d70..5043a07e0fdf 100644 --- a/contrib/xz/src/liblzma/lzma/lzma2_encoder.c +++ b/contrib/xz/src/liblzma/lzma/lzma2_encoder.c @@ -409,6 +409,9 @@ lzma_lzma2_block_size(const void *options) { const lzma_options_lzma *const opt = options; + if (!IS_ENC_DICT_SIZE_VALID(opt->dict_size)) + return UINT64_MAX; + // Use at least 1 MiB to keep compression ratio better. return my_max((uint64_t)(opt->dict_size) * 3, UINT64_C(1) << 20); } diff --git a/lib/liblzma/Symbol.map b/lib/liblzma/Symbol.map index 9532da0a5bde..938b6191b4f4 100644 --- a/lib/liblzma/Symbol.map +++ b/lib/liblzma/Symbol.map @@ -114,6 +114,10 @@ XZ_5.4 { lzma_str_to_filters; }; +XZ_5.6 { + lzma_mt_block_size; +}; + XZprivate_1.0 { lzma_alloc; lzma_alloc_zero; @@ -173,7 +177,6 @@ XZprivate_1.0 { lzma_mf_hc3_skip; lzma_mf_hc4_find; lzma_mf_hc4_skip; - lzma_mt_block_size; lzma_next_end; lzma_next_filter_init; lzma_next_filter_update; diff --git a/lib/liblzma/Versions.def b/lib/liblzma/Versions.def index dea89514beda..25064d9d50e7 100644 --- a/lib/liblzma/Versions.def +++ b/lib/liblzma/Versions.def @@ -8,6 +8,9 @@ XZ_5.2 { XZ_5.4 { } XZ_5.2; -XZprivate_1.0 { +XZ_5.6 { } XZ_5.4; +XZprivate_1.0 { +} XZ_5.6; +