system/systemd
system/systemd
1
0
Fork 0
mirror of https://github.com/systemd/systemd synced 2024-10-04 15:21:01 +00:00
Code Issues Packages Projects Releases Wiki Activity

dynamically load compression libraries

Browse source 
Dynamically load liblz4, libzstd and liblzma with dlopen().
This helps to reduce the size of the initrd image when these libraries
are not really needed.
This commit is contained in:
Matteo Croce 2024-02-27 21:28:14 +01:00
parent 1c20c9f4fc
commit 3fc72d5413
17 changed files with 413 additions and 212 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

13
meson.build
View file

@ -1371,16 +1371,19 @@ conf.set10('HAVE_BZIP2', libbzip2.found())
libxz = dependency('liblzma',
required : get_option('xz'))
conf.set10('HAVE_XZ', libxz.found())
libxz_cflags = libxz.partial_dependency(includes: true, compile_args: true)
liblz4 = dependency('liblz4',
version : '>= 1.3.0',
required : get_option('lz4'))
conf.set10('HAVE_LZ4', liblz4.found())
liblz4_cflags = liblz4.partial_dependency(includes: true, compile_args: true)
libzstd = dependency('libzstd',
version : '>= 1.4.0',
required : get_option('zstd'))
conf.set10('HAVE_ZSTD', libzstd.found())
libzstd_cflags = libzstd.partial_dependency(includes: true, compile_args: true)
conf.set10('HAVE_COMPRESSION', libxz.found() or liblz4.found() or libzstd.found())
@ -1951,8 +1954,7 @@ libsystemd = shared_library(
link_args : ['-shared',
'-Wl,--version-script=' + libsystemd_sym_path],
link_with : [libbasic,
libbasic_gcrypt,
libbasic_compress],
libbasic_gcrypt],
link_whole : [libsystemd_static],
dependencies : [librt,
threads,
@ -1969,7 +1971,6 @@ install_libsystemd_static = static_library(
libsystemd_sources,
basic_sources,
basic_gcrypt_sources,
basic_compress_sources,
fundamental_sources,
include_directories : libsystemd_includes,
build_by_default : static_libsystemd != 'false',
@ -1981,12 +1982,12 @@ install_libsystemd_static = static_library(
libcap,
libdl,
libgcrypt,
liblz4,
liblz4_cflags,
libmount,
libopenssl,
librt,
libxz,
libzstd,
libxz_cflags,
libzstd_cflags,
threads,
userspace],
c_args : libsystemd_c_args + (static_libsystemd_pic ? [] : ['-fno-PIC']))

414
src/basic/compress.c
View file

@ -12,11 +12,6 @@
#include <lzma.h>
#endif
#if HAVE_LZ4
#include <lz4.h>
#include <lz4frame.h>
#endif
#if HAVE_ZSTD
#include <zstd.h>
#include <zstd_errors.h>
@ -34,16 +29,52 @@
#include "unaligned.h"
#if HAVE_LZ4
DEFINE_TRIVIAL_CLEANUP_FUNC_FULL(LZ4F_compressionContext_t, LZ4F_freeCompressionContext, NULL);
DEFINE_TRIVIAL_CLEANUP_FUNC_FULL(LZ4F_decompressionContext_t, LZ4F_freeDecompressionContext, NULL);
static void *lz4_dl = NULL;
static DLSYM_FUNCTION(LZ4F_compressBegin);
static DLSYM_FUNCTION(LZ4F_compressBound);
static DLSYM_FUNCTION(LZ4F_compressEnd);
static DLSYM_FUNCTION(LZ4F_compressUpdate);
static DLSYM_FUNCTION(LZ4F_createCompressionContext);
static DLSYM_FUNCTION(LZ4F_createDecompressionContext);
static DLSYM_FUNCTION(LZ4F_decompress);
static DLSYM_FUNCTION(LZ4F_freeCompressionContext);
static DLSYM_FUNCTION(LZ4F_freeDecompressionContext);
static DLSYM_FUNCTION(LZ4F_isError);
DLSYM_FUNCTION(LZ4_compress_default);
DLSYM_FUNCTION(LZ4_decompress_safe);
DLSYM_FUNCTION(LZ4_decompress_safe_partial);
DLSYM_FUNCTION(LZ4_versionNumber);
DEFINE_TRIVIAL_CLEANUP_FUNC_FULL(LZ4F_compressionContext_t, sym_LZ4F_freeCompressionContext, NULL);
DEFINE_TRIVIAL_CLEANUP_FUNC_FULL(LZ4F_decompressionContext_t, sym_LZ4F_freeDecompressionContext, NULL);
#endif
#if HAVE_ZSTD
DEFINE_TRIVIAL_CLEANUP_FUNC_FULL(ZSTD_CCtx*, ZSTD_freeCCtx, NULL);
DEFINE_TRIVIAL_CLEANUP_FUNC_FULL(ZSTD_DCtx*, ZSTD_freeDCtx, NULL);
static void *zstd_dl = NULL;
static DLSYM_FUNCTION(ZSTD_CCtx_setParameter);
static DLSYM_FUNCTION(ZSTD_compress);
static DLSYM_FUNCTION(ZSTD_compressStream2);
static DLSYM_FUNCTION(ZSTD_createCCtx);
static DLSYM_FUNCTION(ZSTD_createDCtx);
static DLSYM_FUNCTION(ZSTD_CStreamInSize);
static DLSYM_FUNCTION(ZSTD_CStreamOutSize);
static DLSYM_FUNCTION(ZSTD_decompressStream);
static DLSYM_FUNCTION(ZSTD_DStreamInSize);
static DLSYM_FUNCTION(ZSTD_DStreamOutSize);
static DLSYM_FUNCTION(ZSTD_freeCCtx);
static DLSYM_FUNCTION(ZSTD_freeDCtx);
static DLSYM_FUNCTION(ZSTD_getErrorCode);
static DLSYM_FUNCTION(ZSTD_getErrorName);
static DLSYM_FUNCTION(ZSTD_getFrameContentSize);
static DLSYM_FUNCTION(ZSTD_isError);
DEFINE_TRIVIAL_CLEANUP_FUNC_FULL(ZSTD_CCtx*, sym_ZSTD_freeCCtx, NULL);
DEFINE_TRIVIAL_CLEANUP_FUNC_FULL(ZSTD_DCtx*, sym_ZSTD_freeDCtx, NULL);
static int zstd_ret_to_errno(size_t ret) {
switch (ZSTD_getErrorCode(ret)) {
switch (sym_ZSTD_getErrorCode(ret)) {
case ZSTD_error_dstSize_tooSmall:
return -ENOBUFS;
case ZSTD_error_memory_allocation:
@ -54,6 +85,27 @@ static int zstd_ret_to_errno(size_t ret) {
}
#endif
#if HAVE_XZ
static void *lzma_dl = NULL;
static DLSYM_FUNCTION(lzma_code);
static DLSYM_FUNCTION(lzma_easy_encoder);
static DLSYM_FUNCTION(lzma_end);
static DLSYM_FUNCTION(lzma_stream_buffer_encode);
static DLSYM_FUNCTION(lzma_stream_decoder);
/* We can't just do _cleanup_(sym_lzma_end) because a compiler bug makes
* this fail with:
* ../src/basic/compress.c: In function ‘decompress_blob_xz’:
* ../src/basic/compress.c:304:9: error: cleanup argument not a function
* 304 | _cleanup_(sym_lzma_end) lzma_stream s = LZMA_STREAM_INIT;
* | ^~~~~~~~~
*/
static inline void lzma_end_wrapper(lzma_stream *ls) {
sym_lzma_end(ls);
}
#endif
#define ALIGN_8(l) ALIGN_TO(l, sizeof(size_t))
static const char* const compression_table[_COMPRESSION_MAX] = {
@ -75,8 +127,28 @@ bool compression_supported(Compression c) {
return c >= 0 && c < _COMPRESSION_MAX && FLAGS_SET(supported, 1U << c);
}
#if HAVE_XZ
int dlopen_lzma(void) {
return dlopen_many_sym_or_warn(
&lzma_dl,
"liblzma.so.5", LOG_DEBUG,
DLSYM_ARG(lzma_code),
DLSYM_ARG(lzma_easy_encoder),
DLSYM_ARG(lzma_end),
DLSYM_ARG(lzma_stream_buffer_encode),
DLSYM_ARG(lzma_stream_decoder));
}
#endif
int compress_blob_xz(const void *src, uint64_t src_size,
void *dst, size_t dst_alloc_size, size_t *dst_size) {
assert(src);
assert(src_size > 0);
assert(dst);
assert(dst_alloc_size > 0);
assert(dst_size);
#if HAVE_XZ
static const lzma_options_lzma opt = {
1u << 20u, NULL, 0, LZMA_LC_DEFAULT, LZMA_LP_DEFAULT,
@ -88,12 +160,11 @@ int compress_blob_xz(const void *src, uint64_t src_size,
};
lzma_ret ret;
size_t out_pos = 0;
int r;
assert(src);
assert(src_size > 0);
assert(dst);
assert(dst_alloc_size > 0);
assert(dst_size);
r = dlopen_lzma();
if (r < 0)
return r;
/* Returns < 0 if we couldn't compress the data or the
* compressed result is longer than the original */
@ -101,7 +172,7 @@ int compress_blob_xz(const void *src, uint64_t src_size,
if (src_size < 80)
return -ENOBUFS;
ret = lzma_stream_buffer_encode((lzma_filter*) filters, LZMA_CHECK_NONE, NULL,
ret = sym_lzma_stream_buffer_encode((lzma_filter*) filters, LZMA_CHECK_NONE, NULL,
src, src_size, dst, &out_pos, dst_alloc_size);
if (ret != LZMA_OK)
return -ENOBUFS;
@ -113,10 +184,30 @@ int compress_blob_xz(const void *src, uint64_t src_size,
#endif
}
#if HAVE_LZ4
int dlopen_lz4(void) {
return dlopen_many_sym_or_warn(
&lz4_dl,
"liblz4.so.1", LOG_DEBUG,
DLSYM_ARG(LZ4F_compressBegin),
DLSYM_ARG(LZ4F_compressBound),
DLSYM_ARG(LZ4F_compressEnd),
DLSYM_ARG(LZ4F_compressUpdate),
DLSYM_ARG(LZ4F_createCompressionContext),
DLSYM_ARG(LZ4F_createDecompressionContext),
DLSYM_ARG(LZ4F_decompress),
DLSYM_ARG(LZ4F_freeCompressionContext),
DLSYM_ARG(LZ4F_freeDecompressionContext),
DLSYM_ARG(LZ4F_isError),
DLSYM_ARG(LZ4_compress_default),
DLSYM_ARG(LZ4_decompress_safe),
DLSYM_ARG(LZ4_decompress_safe_partial),
DLSYM_ARG(LZ4_versionNumber));
}
#endif
int compress_blob_lz4(const void *src, uint64_t src_size,
void *dst, size_t dst_alloc_size, size_t *dst_size) {
#if HAVE_LZ4
int r;
assert(src);
assert(src_size > 0);
@ -124,13 +215,19 @@ int compress_blob_lz4(const void *src, uint64_t src_size,
assert(dst_alloc_size > 0);
assert(dst_size);
#if HAVE_LZ4
int r;
r = dlopen_lz4();
if (r < 0)
return r;
/* Returns < 0 if we couldn't compress the data or the
* compressed result is longer than the original */
if (src_size < 9)
return -ENOBUFS;
r = LZ4_compress_default(src, (char*)dst + 8, src_size, (int) dst_alloc_size - 8);
r = sym_LZ4_compress_default(src, (char*)dst + 8, src_size, (int) dst_alloc_size - 8);
if (r <= 0)
return -ENOBUFS;
@ -143,11 +240,33 @@ int compress_blob_lz4(const void *src, uint64_t src_size,
#endif
}
#if HAVE_ZSTD
int dlopen_zstd(void) {
return dlopen_many_sym_or_warn(
&zstd_dl,
"libzstd.so.1", LOG_DEBUG,
DLSYM_ARG(ZSTD_getErrorCode),
DLSYM_ARG(ZSTD_compress),
DLSYM_ARG(ZSTD_getFrameContentSize),
DLSYM_ARG(ZSTD_decompressStream),
DLSYM_ARG(ZSTD_getErrorName),
DLSYM_ARG(ZSTD_DStreamOutSize),
DLSYM_ARG(ZSTD_CStreamInSize),
DLSYM_ARG(ZSTD_CStreamOutSize),
DLSYM_ARG(ZSTD_CCtx_setParameter),
DLSYM_ARG(ZSTD_compressStream2),
DLSYM_ARG(ZSTD_DStreamInSize),
DLSYM_ARG(ZSTD_freeCCtx),
DLSYM_ARG(ZSTD_freeDCtx),
DLSYM_ARG(ZSTD_isError),
DLSYM_ARG(ZSTD_createDCtx),
DLSYM_ARG(ZSTD_createCCtx));
}
#endif
int compress_blob_zstd(
const void *src, uint64_t src_size,
void *dst, size_t dst_alloc_size, size_t *dst_size) {
#if HAVE_ZSTD
size_t k;
assert(src);
assert(src_size > 0);
@ -155,8 +274,16 @@ int compress_blob_zstd(
assert(dst_alloc_size > 0);
assert(dst_size);
k = ZSTD_compress(dst, dst_alloc_size, src, src_size, 0);
if (ZSTD_isError(k))
#if HAVE_ZSTD
size_t k;
int r;
r = dlopen_zstd();
if (r < 0)
return r;
k = sym_ZSTD_compress(dst, dst_alloc_size, src, src_size, 0);
if (sym_ZSTD_isError(k))
return zstd_ret_to_errno(k);
*dst_size = k;
@ -173,17 +300,22 @@ int decompress_blob_xz(
size_t* dst_size,
size_t dst_max) {
#if HAVE_XZ
_cleanup_(lzma_end) lzma_stream s = LZMA_STREAM_INIT;
lzma_ret ret;
size_t space;
assert(src);
assert(src_size > 0);
assert(dst);
assert(dst_size);
ret = lzma_stream_decoder(&s, UINT64_MAX, 0);
#if HAVE_XZ
_cleanup_(lzma_end_wrapper) lzma_stream s = LZMA_STREAM_INIT;
lzma_ret ret;
size_t space;
int r;
r = dlopen_lzma();
if (r < 0)
return r;
ret = sym_lzma_stream_decoder(&s, UINT64_MAX, 0);
if (ret != LZMA_OK)
return -ENOMEM;
@ -200,7 +332,7 @@ int decompress_blob_xz(
for (;;) {
size_t used;
ret = lzma_code(&s, LZMA_FINISH);
ret = sym_lzma_code(&s, LZMA_FINISH);
if (ret == LZMA_STREAM_END)
break;
@ -235,15 +367,19 @@ int decompress_blob_lz4(
size_t* dst_size,
size_t dst_max) {
#if HAVE_LZ4
char* out;
int r, size; /* LZ4 uses int for size */
assert(src);
assert(src_size > 0);
assert(dst);
assert(dst_size);
#if HAVE_LZ4
char* out;
int r, size; /* LZ4 uses int for size */
r = dlopen_lz4();
if (r < 0)
return r;
if (src_size <= 8)
return -EBADMSG;
@ -254,7 +390,7 @@ int decompress_blob_lz4(
if (!out)
return -ENOMEM;
r = LZ4_decompress_safe((char*)src + 8, out, src_size - 8, size);
r = sym_LZ4_decompress_safe((char*)src + 8, out, src_size - 8, size);
if (r < 0 || r != size)
return -EBADMSG;
@ -272,15 +408,20 @@ int decompress_blob_zstd(
size_t *dst_size,
size_t dst_max) {
#if HAVE_ZSTD
uint64_t size;
assert(src);
assert(src_size > 0);
assert(dst);
assert(dst_size);
size = ZSTD_getFrameContentSize(src, src_size);
#if HAVE_ZSTD
uint64_t size;
int r;
r = dlopen_zstd();
if (r < 0)
return r;
size = sym_ZSTD_getFrameContentSize(src, src_size);
if (IN_SET(size, ZSTD_CONTENTSIZE_ERROR, ZSTD_CONTENTSIZE_UNKNOWN))
return -EBADMSG;
@ -289,10 +430,10 @@ int decompress_blob_zstd(
if (size > SIZE_MAX)
return -E2BIG;
if (!(greedy_realloc(dst, MAX(ZSTD_DStreamOutSize(), size), 1)))
if (!(greedy_realloc(dst, MAX(sym_ZSTD_DStreamOutSize(), size), 1)))
return -ENOMEM;
_cleanup_(ZSTD_freeDCtxp) ZSTD_DCtx *dctx = ZSTD_createDCtx();
_cleanup_(sym_ZSTD_freeDCtxp) ZSTD_DCtx *dctx = sym_ZSTD_createDCtx();
if (!dctx)
return -ENOMEM;
@ -305,9 +446,9 @@ int decompress_blob_zstd(
.size = MALLOC_SIZEOF_SAFE(*dst),
};
size_t k = ZSTD_decompressStream(dctx, &output, &input);
if (ZSTD_isError(k)) {
log_debug("ZSTD decoder failed: %s", ZSTD_getErrorName(k));
size_t k = sym_ZSTD_decompressStream(dctx, &output, &input);
if (sym_ZSTD_isError(k)) {
log_debug("ZSTD decoder failed: %s", sym_ZSTD_getErrorName(k));
return zstd_ret_to_errno(k);
}
assert(output.pos >= size);
@ -351,11 +492,6 @@ int decompress_startswith_xz(
size_t prefix_len,
uint8_t extra) {
#if HAVE_XZ
_cleanup_(lzma_end) lzma_stream s = LZMA_STREAM_INIT;
size_t allocated;
lzma_ret ret;
/* Checks whether the decompressed blob starts with the mentioned prefix. The byte extra needs to
* follow the prefix */
@ -364,7 +500,17 @@ int decompress_startswith_xz(
assert(buffer);
assert(prefix);
ret = lzma_stream_decoder(&s, UINT64_MAX, 0);
#if HAVE_XZ
_cleanup_(lzma_end_wrapper) lzma_stream s = LZMA_STREAM_INIT;
size_t allocated;
lzma_ret ret;
int r;
r = dlopen_lzma();
if (r < 0)
return r;
ret = sym_lzma_stream_decoder(&s, UINT64_MAX, 0);
if (ret != LZMA_OK)
return -EBADMSG;
@ -380,7 +526,7 @@ int decompress_startswith_xz(
s.avail_out = allocated;
for (;;) {
ret = lzma_code(&s, LZMA_FINISH);
ret = sym_lzma_code(&s, LZMA_FINISH);
if (!IN_SET(ret, LZMA_OK, LZMA_STREAM_END))
return -EBADMSG;
@ -414,18 +560,22 @@ int decompress_startswith_lz4(
size_t prefix_len,
uint8_t extra) {
#if HAVE_LZ4
/* Checks whether the decompressed blob starts with the mentioned prefix. The byte extra needs to
* follow the prefix */
size_t allocated;
int r;
assert(src);
assert(src_size > 0);
assert(buffer);
assert(prefix);
#if HAVE_LZ4
size_t allocated;
int r;
r = dlopen_lz4();
if (r < 0)
return r;
if (src_size <= 8)
return -EBADMSG;
@ -433,7 +583,7 @@ int decompress_startswith_lz4(
return -ENOMEM;
allocated = MALLOC_SIZEOF_SAFE(*buffer);
r = LZ4_decompress_safe_partial(
r = sym_LZ4_decompress_safe_partial(
(char*)src + 8,
*buffer,
src_size - 8,
@ -447,7 +597,7 @@ int decompress_startswith_lz4(
if (r < 0 || (size_t) r < prefix_len + 1) {
size_t size;
if (LZ4_versionNumber() >= 10803)
if (sym_LZ4_versionNumber() >= 10803)
/* We trust that the newer lz4 decompresses the number of bytes we
* requested if available in the compressed string. */
return 0;
@ -482,24 +632,31 @@ int decompress_startswith_zstd(
const void *prefix,
size_t prefix_len,
uint8_t extra) {
#if HAVE_ZSTD
assert(src);
assert(src_size > 0);
assert(buffer);
assert(prefix);
uint64_t size = ZSTD_getFrameContentSize(src, src_size);
#if HAVE_ZSTD
int r;
r = dlopen_zstd();
if (r < 0)
return r;
uint64_t size = sym_ZSTD_getFrameContentSize(src, src_size);
if (IN_SET(size, ZSTD_CONTENTSIZE_ERROR, ZSTD_CONTENTSIZE_UNKNOWN))
return -EBADMSG;
if (size < prefix_len + 1)
return 0; /* Decompressed text too short to match the prefix and extra */
_cleanup_(ZSTD_freeDCtxp) ZSTD_DCtx *dctx = ZSTD_createDCtx();
_cleanup_(sym_ZSTD_freeDCtxp) ZSTD_DCtx *dctx = sym_ZSTD_createDCtx();
if (!dctx)
return -ENOMEM;
if (!(greedy_realloc(buffer, MAX(ZSTD_DStreamOutSize(), prefix_len + 1), 1)))
if (!(greedy_realloc(buffer, MAX(sym_ZSTD_DStreamOutSize(), prefix_len + 1), 1)))
return -ENOMEM;
ZSTD_inBuffer input = {
@ -512,9 +669,9 @@ int decompress_startswith_zstd(
};
size_t k;
k = ZSTD_decompressStream(dctx, &output, &input);
if (ZSTD_isError(k)) {
log_debug("ZSTD decoder failed: %s", ZSTD_getErrorName(k));
k = sym_ZSTD_decompressStream(dctx, &output, &input);
if (sym_ZSTD_isError(k)) {
log_debug("ZSTD decoder failed: %s", sym_ZSTD_getErrorName(k));
return zstd_ret_to_errno(k);
}
assert(output.pos >= prefix_len + 1);
@ -559,16 +716,21 @@ int decompress_startswith(
}
int compress_stream_xz(int fdf, int fdt, uint64_t max_bytes, uint64_t *ret_uncompressed_size) {
#if HAVE_XZ
_cleanup_(lzma_end) lzma_stream s = LZMA_STREAM_INIT;
lzma_ret ret;
uint8_t buf[BUFSIZ], out[BUFSIZ];
lzma_action action = LZMA_RUN;
assert(fdf >= 0);
assert(fdt >= 0);
ret = lzma_easy_encoder(&s, LZMA_PRESET_DEFAULT, LZMA_CHECK_CRC64);
#if HAVE_XZ
_cleanup_(lzma_end_wrapper) lzma_stream s = LZMA_STREAM_INIT;
lzma_ret ret;
uint8_t buf[BUFSIZ], out[BUFSIZ];
lzma_action action = LZMA_RUN;
int r;
r = dlopen_lzma();
if (r < 0)
return r;
ret = sym_lzma_easy_encoder(&s, LZMA_PRESET_DEFAULT, LZMA_CHECK_CRC64);
if (ret != LZMA_OK)
return log_error_errno(SYNTHETIC_ERRNO(EINVAL),
"Failed to initialize XZ encoder: code %u",
@ -603,7 +765,7 @@ int compress_stream_xz(int fdf, int fdt, uint64_t max_bytes, uint64_t *ret_uncom
s.avail_out = sizeof(out);
}
ret = lzma_code(&s, action);
ret = sym_lzma_code(&s, action);
if (!IN_SET(ret, LZMA_OK, LZMA_STREAM_END))
return log_error_errno(SYNTHETIC_ERRNO(EBADMSG),
"Compression failed: code %u",
@ -641,7 +803,7 @@ int compress_stream_lz4(int fdf, int fdt, uint64_t max_bytes, uint64_t *ret_unco
#if HAVE_LZ4
LZ4F_errorCode_t c;
_cleanup_(LZ4F_freeCompressionContextp) LZ4F_compressionContext_t ctx = NULL;
_cleanup_(sym_LZ4F_freeCompressionContextp) LZ4F_compressionContext_t ctx = NULL;
_cleanup_free_ void *in_buff = NULL;
_cleanup_free_ char *out_buff = NULL;
size_t out_allocsize, n, offset = 0, frame_size;
@ -651,11 +813,15 @@ int compress_stream_lz4(int fdf, int fdt, uint64_t max_bytes, uint64_t *ret_unco
.frameInfo.blockSizeID = 5,
};
c = LZ4F_createCompressionContext(&ctx, LZ4F_VERSION);
if (LZ4F_isError(c))
r = dlopen_lz4();
if (r < 0)
return r;
c = sym_LZ4F_createCompressionContext(&ctx, LZ4F_VERSION);
if (sym_LZ4F_isError(c))
return -ENOMEM;
frame_size = LZ4F_compressBound(LZ4_BUFSIZE, &preferences);
frame_size = sym_LZ4F_compressBound(LZ4_BUFSIZE, &preferences);
out_allocsize = frame_size + 64*1024; /* add some space for header and trailer */
out_buff = malloc(out_allocsize);
if (!out_buff)
@ -665,8 +831,8 @@ int compress_stream_lz4(int fdf, int fdt, uint64_t max_bytes, uint64_t *ret_unco
if (!in_buff)
return -ENOMEM;
n = offset = total_out = LZ4F_compressBegin(ctx, out_buff, out_allocsize, &preferences);
if (LZ4F_isError(n))
n = offset = total_out = sym_LZ4F_compressBegin(ctx, out_buff, out_allocsize, &preferences);
if (sym_LZ4F_isError(n))
return -EINVAL;
log_debug("Buffer size is %zu bytes, header size %zu bytes.", out_allocsize, n);
@ -679,9 +845,9 @@ int compress_stream_lz4(int fdf, int fdt, uint64_t max_bytes, uint64_t *ret_unco
return k;
if (k == 0)
break;
n = LZ4F_compressUpdate(ctx, out_buff + offset, out_allocsize - offset,
n = sym_LZ4F_compressUpdate(ctx, out_buff + offset, out_allocsize - offset,
in_buff, k, NULL);
if (LZ4F_isError(n))
if (sym_LZ4F_isError(n))
return -ENOTRECOVERABLE;
total_in += k;
@ -700,8 +866,8 @@ int compress_stream_lz4(int fdf, int fdt, uint64_t max_bytes, uint64_t *ret_unco
}
}
n = LZ4F_compressEnd(ctx, out_buff + offset, out_allocsize - offset, NULL);
if (LZ4F_isError(n))
n = sym_LZ4F_compressEnd(ctx, out_buff + offset, out_allocsize - offset, NULL);
if (sym_LZ4F_isError(n))
return -ENOTRECOVERABLE;
offset += n;
@ -724,18 +890,22 @@ int compress_stream_lz4(int fdf, int fdt, uint64_t max_bytes, uint64_t *ret_unco
}
int decompress_stream_xz(int fdf, int fdt, uint64_t max_bytes) {
assert(fdf >= 0);
assert(fdt >= 0);
#if HAVE_XZ
_cleanup_(lzma_end) lzma_stream s = LZMA_STREAM_INIT;
_cleanup_(lzma_end_wrapper) lzma_stream s = LZMA_STREAM_INIT;
lzma_ret ret;
uint8_t buf[BUFSIZ], out[BUFSIZ];
lzma_action action = LZMA_RUN;
int r;
assert(fdf >= 0);
assert(fdt >= 0);
r = dlopen_lzma();
if (r < 0)
return r;
ret = lzma_stream_decoder(&s, UINT64_MAX, 0);
ret = sym_lzma_stream_decoder(&s, UINT64_MAX, 0);
if (ret != LZMA_OK)
return log_debug_errno(SYNTHETIC_ERRNO(ENOMEM),
"Failed to initialize XZ decoder: code %u",
@ -761,7 +931,7 @@ int decompress_stream_xz(int fdf, int fdt, uint64_t max_bytes) {
s.avail_out = sizeof(out);
}
ret = lzma_code(&s, action);
ret = sym_lzma_code(&s, action);
if (!IN_SET(ret, LZMA_OK, LZMA_STREAM_END))
return log_debug_errno(SYNTHETIC_ERRNO(EBADMSG),
"Decompression failed: code %u",
@ -801,15 +971,19 @@ int decompress_stream_xz(int fdf, int fdt, uint64_t max_bytes) {
int decompress_stream_lz4(int in, int out, uint64_t max_bytes) {
#if HAVE_LZ4
size_t c;
_cleanup_(LZ4F_freeDecompressionContextp) LZ4F_decompressionContext_t ctx = NULL;
_cleanup_(sym_LZ4F_freeDecompressionContextp) LZ4F_decompressionContext_t ctx = NULL;
_cleanup_free_ char *buf = NULL;
char *src;
struct stat st;
int r = 0;
int r;
size_t total_in = 0, total_out = 0;
c = LZ4F_createDecompressionContext(&ctx, LZ4F_VERSION);
if (LZ4F_isError(c))
r = dlopen_lz4();
if (r < 0)
return r;
c = sym_LZ4F_createDecompressionContext(&ctx, LZ4F_VERSION);
if (sym_LZ4F_isError(c))
return -ENOMEM;
if (fstat(in, &st) < 0)
@ -830,8 +1004,8 @@ int decompress_stream_lz4(int in, int out, uint64_t max_bytes) {
size_t produced = LZ4_BUFSIZE;
size_t used = st.st_size - total_in;
c = LZ4F_decompress(ctx, buf, &produced, src + total_in, &used, NULL);
if (LZ4F_isError(c)) {
c = sym_LZ4F_decompress(ctx, buf, &produced, src + total_in, &used, NULL);
if (sym_LZ4F_isError(c)) {
r = -EBADMSG;
goto cleanup;
}
@ -853,6 +1027,7 @@ int decompress_stream_lz4(int in, int out, uint64_t max_bytes) {
log_debug("LZ4 decompression finished (%zu -> %zu bytes, %.1f%%)",
total_in, total_out,
total_in > 0 ? (double) total_out / total_in * 100 : 0.0);
r = 0;
cleanup:
munmap(src, st.st_size);
return r;
@ -863,28 +1038,33 @@ int decompress_stream_lz4(int in, int out, uint64_t max_bytes) {
}
int compress_stream_zstd(int fdf, int fdt, uint64_t max_bytes, uint64_t *ret_uncompressed_size) {
assert(fdf >= 0);
assert(fdt >= 0);
#if HAVE_ZSTD
_cleanup_(ZSTD_freeCCtxp) ZSTD_CCtx *cctx = NULL;
_cleanup_(sym_ZSTD_freeCCtxp) ZSTD_CCtx *cctx = NULL;
_cleanup_free_ void *in_buff = NULL, *out_buff = NULL;
size_t in_allocsize, out_allocsize;
size_t z;
uint64_t left = max_bytes, in_bytes = 0;
int r;
assert(fdf >= 0);
assert(fdt >= 0);
r = dlopen_zstd();
if (r < 0)
return r;
/* Create the context and buffers */
in_allocsize = ZSTD_CStreamInSize();
out_allocsize = ZSTD_CStreamOutSize();
in_allocsize = sym_ZSTD_CStreamInSize();
out_allocsize = sym_ZSTD_CStreamOutSize();
in_buff = malloc(in_allocsize);
out_buff = malloc(out_allocsize);
cctx = ZSTD_createCCtx();
cctx = sym_ZSTD_createCCtx();
if (!cctx || !out_buff || !in_buff)
return -ENOMEM;
z = ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1);
if (ZSTD_isError(z))
log_debug("Failed to enable ZSTD checksum, ignoring: %s", ZSTD_getErrorName(z));
z = sym_ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1);
if (sym_ZSTD_isError(z))
log_debug("Failed to enable ZSTD checksum, ignoring: %s", sym_ZSTD_getErrorName(z));
/* This loop read from the input file, compresses that entire chunk,
* and writes all output produced to the output file.
@ -919,12 +1099,12 @@ int compress_stream_zstd(int fdf, int fdt, uint64_t max_bytes, uint64_t *ret_unc
* output to the file so we can reuse the buffer next
* iteration.
*/
remaining = ZSTD_compressStream2(
remaining = sym_ZSTD_compressStream2(
cctx, &output, &input,
is_last_chunk ? ZSTD_e_end : ZSTD_e_continue);
if (ZSTD_isError(remaining)) {
log_debug("ZSTD encoder failed: %s", ZSTD_getErrorName(remaining));
if (sym_ZSTD_isError(remaining)) {
log_debug("ZSTD encoder failed: %s", sym_ZSTD_getErrorName(remaining));
return zstd_ret_to_errno(remaining);
}
@ -968,22 +1148,26 @@ int compress_stream_zstd(int fdf, int fdt, uint64_t max_bytes, uint64_t *ret_unc
}
int decompress_stream_zstd(int fdf, int fdt, uint64_t max_bytes) {
assert(fdf >= 0);
assert(fdt >= 0);
#if HAVE_ZSTD
_cleanup_(ZSTD_freeDCtxp) ZSTD_DCtx *dctx = NULL;
_cleanup_(sym_ZSTD_freeDCtxp) ZSTD_DCtx *dctx = NULL;
_cleanup_free_ void *in_buff = NULL, *out_buff = NULL;
size_t in_allocsize, out_allocsize;
size_t last_result = 0;
uint64_t left = max_bytes, in_bytes = 0;
int r;
assert(fdf >= 0);
assert(fdt >= 0);
r = dlopen_zstd();
if (r < 0)
return r;
/* Create the context and buffers */
in_allocsize = ZSTD_DStreamInSize();
out_allocsize = ZSTD_DStreamOutSize();
in_allocsize = sym_ZSTD_DStreamInSize();
out_allocsize = sym_ZSTD_DStreamOutSize();
in_buff = malloc(in_allocsize);
out_buff = malloc(out_allocsize);
dctx = ZSTD_createDCtx();
dctx = sym_ZSTD_createDCtx();
if (!dctx || !out_buff || !in_buff)
return -ENOMEM;
@ -1032,8 +1216,8 @@ int decompress_stream_zstd(int fdf, int fdt, uint64_t max_bytes) {
* for instance if the last decompression call returned
* an error.
*/
last_result = ZSTD_decompressStream(dctx, &output, &input);
if (ZSTD_isError(last_result)) {
last_result = sym_ZSTD_decompressStream(dctx, &output, &input);
if (sym_ZSTD_isError(last_result)) {
has_error = true;
break;
}
@ -1059,7 +1243,7 @@ int decompress_stream_zstd(int fdf, int fdt, uint64_t max_bytes) {
* on a frame, but we reached the end of the file! We assume
* this is an error, and the input was truncated.
*/
log_debug("ZSTD decoder failed: %s", ZSTD_getErrorName(last_result));
log_debug("ZSTD decoder failed: %s", sym_ZSTD_getErrorName(last_result));
return zstd_ret_to_errno(last_result);
}

25
src/basic/compress.h
View file

@ -6,6 +6,13 @@
#include <stdint.h>
#include <unistd.h>
#if HAVE_LZ4
#include <lz4.h>
#include <lz4frame.h>
#endif
#include "dlfcn-util.h"
typedef enum Compression {
COMPRESSION_NONE,
COMPRESSION_XZ,
@ -63,6 +70,24 @@ int decompress_stream_xz(int fdf, int fdt, uint64_t max_size);
int decompress_stream_lz4(int fdf, int fdt, uint64_t max_size);
int decompress_stream_zstd(int fdf, int fdt, uint64_t max_size);
#if HAVE_LZ4
DLSYM_PROTOTYPE(LZ4_compress_default);
DLSYM_PROTOTYPE(LZ4_decompress_safe);
DLSYM_PROTOTYPE(LZ4_decompress_safe_partial);
DLSYM_PROTOTYPE(LZ4_versionNumber);
int dlopen_lz4(void);
#endif
#if HAVE_ZSTD
int dlopen_zstd(void);
#endif
#if HAVE_XZ
int dlopen_lzma(void);
#endif
static inline int compress_blob(
Compression compression,
const void *src, uint64_t src_size,

21
src/basic/meson.build
View file

@ -17,6 +17,7 @@ basic_sources = files(
'cgroup-util.c',
'chase.c',
'chattr-util.c',
'compress.c',
'conf-files.c',
'confidential-virt.c',
'devnum-util.c',
@ -275,8 +276,11 @@ libbasic = static_library(
fundamental_sources,
include_directories : basic_includes,
dependencies : [libcap,
liblz4_cflags,
libm,
librt,
libxz_cflags,
libzstd_cflags,
threads,
userspace],
c_args : ['-fvisibility=default'],
@ -300,20 +304,3 @@ libbasic_gcrypt = static_library(
build_by_default : false)
############################################################
basic_compress_sources = files(
'compress.c',
)
# A convenience library that is separate from libbasic to avoid unnecessary
# linking to the compression libraries.
libbasic_compress = static_library(
'basic-compress',
basic_compress_sources,
include_directories : basic_includes,
dependencies : [liblz4,
libxz,
libzstd,
userspace],
c_args : ['-fvisibility=default'],
build_by_default : false)

2
src/boot/efi/meson.build
View file

@ -28,7 +28,7 @@ efi_fuzz_template = fuzz_template + efitest_base
executables += [
efi_test_template + {
'sources' : files('test-bcd.c'),
'dependencies' : libzstd,
'dependencies' : libzstd_cflags,
'conditions' : ['ENABLE_BOOTLOADER', 'HAVE_ZSTD'],
},
efi_test_template + {

15
src/coredump/meson.build
View file

@ -5,15 +5,10 @@ systemd_coredump_sources = files(
'coredump-vacuum.c',
)
common_link_with = [
libshared,
libbasic_compress,
]
common_dependencies = [
liblz4,
libxz,
libzstd,
liblz4_cflags,
libxz_cflags,
libzstd_cflags,
threads,
]
@ -22,7 +17,7 @@ executables += [
'name' : 'systemd-coredump',
'conditions' : ['ENABLE_COREDUMP'],
'sources' : systemd_coredump_sources,
'link_with' : common_link_with,
'link_with' : [libshared],
'dependencies' : common_dependencies + [libacl],
},
executable_template + {
@ -30,7 +25,7 @@ executables += [
'public' : true,
'conditions' : ['ENABLE_COREDUMP'],
'sources' : files('coredumpctl.c'),
'link_with' : common_link_with,
'link_with' : [libshared],
'dependencies' : common_dependencies,
},
test_template + {

10
src/journal-remote/meson.build
View file

@ -22,9 +22,9 @@ libsystemd_journal_remote = static_library(
libsystemd_journal_remote_sources,
include_directories : includes,
dependencies : [libgnutls,
liblz4,
liblz4_cflags,
libmicrohttpd,
libxz,
libxz_cflags,
threads,
userspace],
build_by_default : false)
@ -38,9 +38,9 @@ systemd_journal_gatewayd_sources = files(
common_deps = [
libgnutls,
liblz4,
libxz,
libzstd,
liblz4_cflags,
libxz_cflags,
libzstd_cflags,
threads,
]

20
src/journal/meson.build
View file

@ -63,10 +63,10 @@ executables += [
libshared,
],
'dependencies' : [
liblz4,
liblz4_cflags,
libselinux,
libxz,
libzstd,
libxz_cflags,
libzstd_cflags,
threads,
],
},
@ -95,26 +95,26 @@ executables += [
'link_with' : journalctl_link_with,
'dependencies' : [
libdl,
liblz4,
libxz,
libzstd,
liblz4_cflags,
libxz_cflags,
libzstd_cflags,
threads,
],
},
journal_test_template + {
'sources' : files('test-journald-config.c'),
'dependencies' : [
liblz4,
liblz4_cflags,
libselinux,
libxz,
libxz_cflags,
],
},
journal_test_template + {
'sources' : files('test-journald-syslog.c'),
'dependencies' : [
liblz4,
liblz4_cflags,
libselinux,
libxz,
libxz_cflags,
threads,
],
},

3
src/libsystemd/meson.build
View file

@ -118,8 +118,7 @@ libsystemd_static = static_library(
libsystemd_sources,
include_directories : libsystemd_includes,
c_args : libsystemd_c_args,
link_with : [libbasic,
libbasic_compress],
link_with : [libbasic],
dependencies : [threads,
librt,
userspace],

6
src/login/meson.build
View file

@ -65,9 +65,9 @@ executables += [
'conditions' : ['ENABLE_LOGIND'],
'sources' : loginctl_sources,
'dependencies' : [
liblz4,
libxz,
libzstd,
liblz4_cflags,
libxz_cflags,
libzstd_cflags,
threads,
],
},

6
src/machine/meson.build
View file

@ -35,9 +35,9 @@ executables += [
'conditions' : ['ENABLE_MACHINED'],
'sources' : files('machinectl.c'),
'dependencies' : [
liblz4,
libxz,
libzstd,
liblz4_cflags,
libxz_cflags,
libzstd_cflags,
threads,
],
},

6
src/pstore/meson.build
View file

@ -7,9 +7,9 @@ executables += [
'sources' : files('pstore.c'),
'dependencies' : [
libacl,
liblz4,
libxz,
libzstd,
liblz4_cflags,
libxz_cflags,
libzstd_cflags,
threads,
],
},

6
src/shared/meson.build
View file

@ -324,7 +324,7 @@ libshared_deps = [threads,
libgcrypt,
libiptc_cflags,
libkmod,
liblz4,
liblz4_cflags,
libmount,
libopenssl,
libp11kit_cflags,
@ -333,8 +333,8 @@ libshared_deps = [threads,
libseccomp,
libselinux,
libxenctrl_cflags,
libxz,
libzstd]
libxz_cflags,
libzstd_cflags]
libshared_sym_path = meson.current_source_dir() / 'libshared.sym'
libshared_build_dir = meson.current_build_dir()

6
src/systemctl/meson.build
View file

@ -56,10 +56,10 @@ executables += [
'link_with' : systemctl_link_with,
'dependencies' : [
libcap,
liblz4,
liblz4_cflags,
libselinux,
libxz,
libzstd,
libxz_cflags,
libzstd_cflags,
threads,
],
},

10
src/test/meson.build
View file

@ -247,18 +247,12 @@ executables += [
},
test_template + {
'sources' : files('test-compress-benchmark.c'),
'link_with' : [
libbasic_compress,
libshared,
],
'link_with' : [libshared],
'timeout' : 90,
},
test_template + {
'sources' : files('test-compress.c'),
'link_with' : [
libbasic_compress,
libshared,
],
'link_with' : [libshared],
},
test_template + {
'sources' : files('test-cryptolib.c'),

49
src/test/test-compress.c
View file

@ -6,6 +6,7 @@
#include <lz4.h>
#endif
#include "dlfcn-util.h"
#include "alloc-util.h"
#include "compress.h"
#include "fd-util.h"
@ -241,16 +242,16 @@ static void test_lz4_decompress_partial(void) {
memset(&huge[STRLEN("HUGE=")], 'x', HUGE_SIZE - STRLEN("HUGE=") - 1);
huge[HUGE_SIZE - 1] = '\0';
r = LZ4_compress_default(huge, buf, HUGE_SIZE, buf_size);
r = sym_LZ4_compress_default(huge, buf, HUGE_SIZE, buf_size);
assert_se(r >= 0);
compressed = r;
log_info("Compressed %i → %zu", HUGE_SIZE, compressed);
r = LZ4_decompress_safe(buf, huge, r, HUGE_SIZE);
r = sym_LZ4_decompress_safe(buf, huge, r, HUGE_SIZE);
assert_se(r >= 0);
log_info("Decompressed → %i", r);
r = LZ4_decompress_safe_partial(buf, huge,
r = sym_LZ4_decompress_safe_partial(buf, huge,
compressed,
12, HUGE_SIZE);
assert_se(r >= 0);
@ -258,10 +259,10 @@ static void test_lz4_decompress_partial(void) {
for (size_t size = 1; size < sizeof(buf2); size++) {
/* This failed in older lz4s but works in newer ones. */
r = LZ4_decompress_safe_partial(buf, buf2, compressed, size, size);
r = sym_LZ4_decompress_safe_partial(buf, buf2, compressed, size, size);
log_info("Decompressed partial %zu/%zu → %i (%s)", size, size, r,
r < 0 ? "bad" : "good");
if (r >= 0 && LZ4_versionNumber() >= 10803)
if (r >= 0 && sym_LZ4_versionNumber() >= 10803)
/* lz4 <= 1.8.2 should fail that test, let's only check for newer ones */
assert_se(memcmp(buf2, huge, r) == 0);
}
@ -316,28 +317,30 @@ int main(int argc, char *argv[]) {
#endif
#if HAVE_LZ4
test_compress_decompress("LZ4", compress_blob_lz4, decompress_blob_lz4,
text, sizeof(text), false);
test_compress_decompress("LZ4", compress_blob_lz4, decompress_blob_lz4,
data, sizeof(data), true);
if (dlopen_lz4() >= 0) {
test_compress_decompress("LZ4", compress_blob_lz4, decompress_blob_lz4,
text, sizeof(text), false);
test_compress_decompress("LZ4", compress_blob_lz4, decompress_blob_lz4,
data, sizeof(data), true);
test_decompress_startswith("LZ4",
compress_blob_lz4, decompress_startswith_lz4,
text, sizeof(text), false);
test_decompress_startswith("LZ4",
compress_blob_lz4, decompress_startswith_lz4,
data, sizeof(data), true);
test_decompress_startswith("LZ4",
compress_blob_lz4, decompress_startswith_lz4,
huge, HUGE_SIZE, true);
test_decompress_startswith("LZ4",
compress_blob_lz4, decompress_startswith_lz4,
text, sizeof(text), false);
test_decompress_startswith("LZ4",
compress_blob_lz4, decompress_startswith_lz4,
data, sizeof(data), true);
test_decompress_startswith("LZ4",
compress_blob_lz4, decompress_startswith_lz4,
huge, HUGE_SIZE, true);
test_compress_stream("LZ4", "lz4cat",
compress_stream_lz4, decompress_stream_lz4, srcfile);
test_compress_stream("LZ4", "lz4cat",
compress_stream_lz4, decompress_stream_lz4, srcfile);
test_lz4_decompress_partial();
test_decompress_startswith_short("LZ4", compress_blob_lz4, decompress_startswith_lz4);
test_lz4_decompress_partial();
test_decompress_startswith_short("LZ4", compress_blob_lz4, decompress_startswith_lz4);
} else
log_error("/* Can't load liblz4 */");
#else
log_info("/* LZ4 test skipped */");
#endif

13
src/test/test-dlopen-so.c
View file

@ -4,6 +4,7 @@
#include <stdlib.h>
#include "bpf-dlopen.h"
#include "compress.h"
#include "cryptsetup-util.h"
#include "elf-util.h"
#include "idn-util.h"
@ -75,6 +76,18 @@ static int run(int argc, char **argv) {
assert_se(dlopen_libarchive() >= 0);
#endif
#if HAVE_LZ4
assert_se(dlopen_lz4() >= 0);
#endif
#if HAVE_ZSTD
assert_se(dlopen_zstd() >= 0);
#endif
#if HAVE_XZ
assert_se(dlopen_lzma() >= 0);
#endif
return 0;
}