ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity
Files
pfs-rust
zstd-rs/programs/fileio.c
T
ddidderr da0e2c8380 fix(build): restore complete mixed Rust build matrix 
`make all` exercises substantially more than the default zstd binary. It also
builds compression-only and decompression-only archives, older contrib tools
that compile program C shims directly, and the single-file amalgamations. The
Rust migration had changed symbol ownership without updating every one of
those feature and link boundaries.

The first failure came from `fileio_asyncio`: it is always compiled, but its
codec bindings unconditionally referenced both `zstd_compress` and
`zstd_decompress`. A compression-only archive therefore required disabled
decoder modules, and the decompression-only configuration had the symmetric
problem. Gate the concrete codec imports, callback types, helpers, and exports
with their Cargo features. Keep the format probe compilable without the
legacy decoder predicate when decompression is disabled.

Once that boundary compiled, the remaining `make all` paths exposed related
integration gaps. Move the C decompression projection declarations out of the
compression preprocessor block, while leaving destination callback types
shared. Link the Rust CLI helpers archive into zlibWrapper, pzstd, and
largeNbDicts, whose util/time/data-generator C files are now declaration shims
rather than implementations.

The generated single-file C sources also call Rust-owned symbols now. Build
and link an appropriately featured Rust archive in their native smoke tests,
and include the pool configuration shim in the decoder case. The full
amalgamation combines `zstd_lazy.c` and `zstd_opt.c` into one translation unit,
so guard their otherwise translation-unit-local dictionary mode enum against
duplicate definition.

A Rust-backed amalgamation is no longer a standalone Emscripten input. Remove
the obsolete emcc/Docker path and report that limitation explicitly; restoring
the WebAssembly smoke test requires a Rust WebAssembly archive and a defined
cross-language amalgamation contract.

Test Plan:
- `cargo check --manifest-path rust/Cargo.toml --no-default-features --features compression` -- passed
- `cargo check --manifest-path rust/Cargo.toml --no-default-features --features decompression` -- passed
- `sh -n build/single_file_libs/build_decoder_test.sh build/single_file_libs/build_library_test.sh` -- passed
- `make all` -- passed, including native single-file and seekable-format tests
- `git diff --cached --check` -- passed
2026-07-22 06:56:06 +02:00

5595 lines
226 KiB
C
Raw Permalink Blame History

/*
* Copyright (c) Meta Platforms, Inc. and affiliates.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
* in the COPYING file in the root directory of this source tree).
* You may select, at your option, one of the above-listed licenses.
*/
/* *************************************
* Compiler Options
***************************************/
#ifdef _MSC_VER /* Visual */
# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
# pragma warning(disable : 4204) /* non-constant aggregate initializer */
#endif
#if defined(__MINGW32__) && !defined(_POSIX_SOURCE)
# define _POSIX_SOURCE 1 /* disable %llu warnings with MinGW on Windows */
#endif
/*-*************************************
* Includes
***************************************/
#include "platform.h" /* Large Files support, SET_BINARY_MODE */
#include "util.h" /* UTIL_getFileSize, UTIL_isRegularFile, UTIL_isSameFile */
#include <stdio.h> /* fprintf, open, fdopen, fread, _fileno, stdin, stdout */
#include <stdlib.h> /* malloc, free */
#include <string.h> /* strcmp, strlen */
#include <stddef.h> /* offsetof */
#include <time.h> /* clock_t, to measure process time */
#include <fcntl.h> /* O_WRONLY */
#include <assert.h>
#include <errno.h> /* errno */
#include <limits.h> /* INT_MAX */
#include <signal.h>
#include "timefn.h" /* UTIL_getTime, UTIL_clockSpanMicro */
#if defined (_MSC_VER)
# include <sys/stat.h>
# include <io.h>
#endif
#include "fileio.h"
#include "fileio_asyncio.h"
#include "fileio_common.h"
FIO_display_prefs_t g_display_prefs = {2, FIO_ps_auto};
UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER;
#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_magicNumber, ZSTD_frameHeaderSize_max */
#include "../lib/zstd.h"
#include "../lib/zstd_errors.h" /* ZSTD_error_frameParameter_windowTooLarge */
#if defined(ZSTD_GZCOMPRESS) || defined(ZSTD_GZDECOMPRESS)
# include <zlib.h>
# if !defined(z_const)
# define z_const
# endif
#endif
#if defined(ZSTD_LZMACOMPRESS) || defined(ZSTD_LZMADECOMPRESS)
# include <lzma.h>
#endif
#define LZ4_MAGICNUMBER 0x184D2204
#if defined(ZSTD_LZ4COMPRESS) || defined(ZSTD_LZ4DECOMPRESS)
# define LZ4F_ENABLE_OBSOLETE_ENUMS
# include <lz4frame.h>
# include <lz4.h>
#endif
char const* FIO_zlibVersion(void)
{
#if defined(ZSTD_GZCOMPRESS) || defined(ZSTD_GZDECOMPRESS)
return zlibVersion();
#else
return "Unsupported";
#endif
}
char const* FIO_lz4Version(void)
{
#if defined(ZSTD_LZ4COMPRESS) || defined(ZSTD_LZ4DECOMPRESS)
/* LZ4_versionString() added in v1.7.3 */
# if LZ4_VERSION_NUMBER >= 10703
return LZ4_versionString();
# else
# define ZSTD_LZ4_VERSION LZ4_VERSION_MAJOR.LZ4_VERSION_MINOR.LZ4_VERSION_RELEASE
# define ZSTD_LZ4_VERSION_STRING ZSTD_EXPAND_AND_QUOTE(ZSTD_LZ4_VERSION)
return ZSTD_LZ4_VERSION_STRING;
# endif
#else
return "Unsupported";
#endif
}
char const* FIO_lzmaVersion(void)
{
#if defined(ZSTD_LZMACOMPRESS) || defined(ZSTD_LZMADECOMPRESS)
return lzma_version_string();
#else
return "Unsupported";
#endif
}
/*-*************************************
* Constants
***************************************/
#define ADAPT_WINDOWLOG_DEFAULT 23 /* 8 MB */
#define DICTSIZE_MAX (32 MB) /* protection against large input (attack scenario) */
#define FNSPACE 30
/* Default file permissions 0666 (modulated by umask) */
/* Temporary restricted file permissions are used when we're going to
* chmod/chown at the end of the operation. */
#if !defined(_WIN32)
/* These macros aren't defined on windows. */
#define DEFAULT_FILE_PERMISSIONS (S_IRUSR|S_IWUSR|S_IRGRP|S_IWGRP|S_IROTH|S_IWOTH)
#define TEMPORARY_FILE_PERMISSIONS (S_IRUSR|S_IWUSR)
#else
#define DEFAULT_FILE_PERMISSIONS (0666)
#define TEMPORARY_FILE_PERMISSIONS (0600)
#endif
/*-************************************
* Signal (Ctrl-C trapping)
**************************************/
static const char* g_artefact = NULL;
static void INThandler(int sig)
{
assert(sig==SIGINT); (void)sig;
#if !defined(_MSC_VER)
signal(sig, SIG_IGN); /* this invocation generates a buggy warning in Visual Studio */
#endif
if (g_artefact) {
assert(UTIL_isRegularFile(g_artefact));
remove(g_artefact);
}
DISPLAY("\n");
exit(2);
}
static void addHandler(char const* dstFileName)
{
if (UTIL_isRegularFile(dstFileName)) {
g_artefact = dstFileName;
signal(SIGINT, INThandler);
} else {
g_artefact = NULL;
}
}
/* Idempotent */
static void clearHandler(void)
{
if (g_artefact) signal(SIGINT, SIG_DFL);
g_artefact = NULL;
}
/*-*********************************************************
* Termination signal trapping (Print debug stack trace)
***********************************************************/
#if defined(__has_feature) && !defined(BACKTRACE_ENABLE) /* Clang compiler */
# if (__has_feature(address_sanitizer))
# define BACKTRACE_ENABLE 0
# endif /* __has_feature(address_sanitizer) */
#elif defined(__SANITIZE_ADDRESS__) && !defined(BACKTRACE_ENABLE) /* GCC compiler */
# define BACKTRACE_ENABLE 0
#endif
#if !defined(BACKTRACE_ENABLE)
/* automatic detector : backtrace enabled by default on linux+glibc and osx */
# if (defined(__linux__) && (defined(__GLIBC__) && !defined(__UCLIBC__))) \
|| (defined(__APPLE__) && defined(__MACH__))
# define BACKTRACE_ENABLE 1
# else
# define BACKTRACE_ENABLE 0
# endif
#endif
/* note : after this point, BACKTRACE_ENABLE is necessarily defined */
#if BACKTRACE_ENABLE
#include <execinfo.h> /* backtrace, backtrace_symbols */
#define MAX_STACK_FRAMES 50
static void ABRThandler(int sig) {
const char* name;
void* addrlist[MAX_STACK_FRAMES];
char** symbollist;
int addrlen, i;
switch (sig) {
case SIGABRT: name = "SIGABRT"; break;
case SIGFPE: name = "SIGFPE"; break;
case SIGILL: name = "SIGILL"; break;
case SIGINT: name = "SIGINT"; break;
case SIGSEGV: name = "SIGSEGV"; break;
default: name = "UNKNOWN";
}
DISPLAY("Caught %s signal, printing stack:\n", name);
/* Retrieve current stack addresses. */
addrlen = backtrace(addrlist, MAX_STACK_FRAMES);
if (addrlen == 0) {
DISPLAY("\n");
return;
}
/* Create readable strings to each frame. */
symbollist = backtrace_symbols(addrlist, addrlen);
/* Print the stack trace, excluding calls handling the signal. */
for (i = ZSTD_START_SYMBOLLIST_FRAME; i < addrlen; i++) {
DISPLAY("%s\n", symbollist[i]);
}
free(symbollist);
/* Reset and raise the signal so default handler runs. */
signal(sig, SIG_DFL);
raise(sig);
}
#endif
void FIO_addAbortHandler(void)
{
#if BACKTRACE_ENABLE
signal(SIGABRT, ABRThandler);
signal(SIGFPE, ABRThandler);
signal(SIGILL, ABRThandler);
signal(SIGSEGV, ABRThandler);
signal(SIGBUS, ABRThandler);
#endif
}
/*-*************************************
* Parameters: FIO_ctx_t
***************************************/
/* typedef'd to FIO_ctx_t within fileio.h */
struct FIO_ctx_s {
/* file i/o info */
int nbFilesTotal;
int hasStdinInput;
int hasStdoutOutput;
/* file i/o state */
int currFileIdx;
int nbFilesProcessed;
size_t totalBytesInput;
size_t totalBytesOutput;
};
/* Keep the Rust preference/context layouts in lock-step with these C
* definitions. The Rust side uses the same #[repr(C)] field order. */
typedef char FIO_rust_prefs_compression_type_offset[
(offsetof(FIO_prefs_t, compressionType) == 0) ? 1 : -1];
typedef char FIO_rust_prefs_stream_size_offset[
(offsetof(FIO_prefs_t, streamSrcSize) == 16 * sizeof(int)) ? 1 : -1];
typedef char FIO_rust_prefs_target_size_offset[
(offsetof(FIO_prefs_t, targetCBlockSize)
== 16 * sizeof(int) + sizeof(size_t)) ? 1 : -1];
typedef char FIO_rust_prefs_src_hint_offset[
(offsetof(FIO_prefs_t, srcSizeHint)
== 16 * sizeof(int) + 2 * sizeof(size_t)) ? 1 : -1];
typedef char FIO_rust_prefs_mmap_dict_offset[
(offsetof(FIO_prefs_t, mmapDict)
== 16 * sizeof(int) + 2 * sizeof(size_t) + 13 * sizeof(int)) ? 1 : -1];
typedef char FIO_rust_prefs_size[
(sizeof(FIO_prefs_t)
== 16 * sizeof(int) + 2 * sizeof(size_t) + 14 * sizeof(int)) ? 1 : -1];
typedef char FIO_rust_ctx_total_input_offset[
(offsetof(FIO_ctx_t, totalBytesInput)
== ((5 * sizeof(int) + sizeof(size_t) - 1) / sizeof(size_t)) * sizeof(size_t)) ? 1 : -1];
typedef char FIO_rust_ctx_total_output_offset[
(offsetof(FIO_ctx_t, totalBytesOutput)
== offsetof(FIO_ctx_t, totalBytesInput) + sizeof(size_t)) ? 1 : -1];
typedef char FIO_rust_ctx_size[
(sizeof(FIO_ctx_t)
== offsetof(FIO_ctx_t, totalBytesOutput) + sizeof(size_t)) ? 1 : -1];
typedef char FIO_rust_compression_params_strategy_offset[
(offsetof(ZSTD_compressionParameters, strategy)
== 6 * sizeof(unsigned)) ? 1 : -1];
typedef char FIO_rust_compression_params_size[
(sizeof(ZSTD_compressionParameters)
== 7 * sizeof(unsigned)) ? 1 : -1];
int FIO_shouldDisplayFileSummary(const FIO_ctx_t* fCtx);
int FIO_shouldDisplayMultipleFileSummary(const FIO_ctx_t* fCtx);
enum {
FIO_RUST_MULTIPLE_SUMMARY_DECOMPRESSION = 0,
FIO_RUST_MULTIPLE_SUMMARY_COMPRESSION = 1
};
typedef void (*FIO_rust_multiple_summary_progress_fn)(void* opaque);
typedef void (*FIO_rust_compress_multiple_summary_fn)(
void* opaque, int nbFilesProcessed, U64 totalBytesInput, U64 totalBytesOutput);
typedef void (*FIO_rust_decompress_multiple_summary_fn)(
void* opaque, int nbFilesProcessed, U64 totalBytesOutput);
typedef struct {
void* opaque;
FIO_rust_multiple_summary_progress_fn displayProgress;
FIO_rust_compress_multiple_summary_fn displayCompression;
FIO_rust_decompress_multiple_summary_fn displayDecompression;
} FIO_rust_multiple_summary_callbacks_t;
typedef char FIO_rust_multiple_summary_callback_sizes[
(sizeof(FIO_rust_multiple_summary_progress_fn) == sizeof(void*)
&& sizeof(FIO_rust_compress_multiple_summary_fn) == sizeof(void*)
&& sizeof(FIO_rust_decompress_multiple_summary_fn) == sizeof(void*)) ? 1 : -1];
typedef char FIO_rust_multiple_summary_opaque_offset[
(offsetof(FIO_rust_multiple_summary_callbacks_t, opaque) == 0) ? 1 : -1];
typedef char FIO_rust_multiple_summary_progress_offset[
(offsetof(FIO_rust_multiple_summary_callbacks_t, displayProgress)
== sizeof(void*)) ? 1 : -1];
typedef char FIO_rust_multiple_summary_compression_offset[
(offsetof(FIO_rust_multiple_summary_callbacks_t, displayCompression)
== 2 * sizeof(void*)) ? 1 : -1];
typedef char FIO_rust_multiple_summary_decompression_offset[
(offsetof(FIO_rust_multiple_summary_callbacks_t, displayDecompression)
== 3 * sizeof(void*)) ? 1 : -1];
typedef char FIO_rust_multiple_summary_callbacks_size[
(sizeof(FIO_rust_multiple_summary_callbacks_t) == 4 * sizeof(void*)) ? 1 : -1];
void FIO_rust_displayMultipleFileSummary(
int nbFilesTotal, int nbFilesProcessed,
U64 totalBytesInput, U64 totalBytesOutput, int summaryKind,
const FIO_rust_multiple_summary_callbacks_t* callbacks);
typedef void (*FIO_rust_multi_files_concat_fatal_fn)(void* opaque);
typedef void (*FIO_rust_multi_files_concat_warning_fn)(void* opaque,
int hasStdoutOutput,
const char* outFileName);
typedef void (*FIO_rust_multi_files_concat_disable_remove_fn)(void* opaque);
typedef void (*FIO_rust_multi_files_concat_quiet_abort_fn)(void* opaque);
typedef int (*FIO_rust_multi_files_concat_confirmation_fn)(void* opaque,
int hasStdinInput);
typedef struct {
void* opaque;
FIO_rust_multi_files_concat_fatal_fn fatalStdoutRemove;
FIO_rust_multi_files_concat_fatal_fn fatalTestRemove;
FIO_rust_multi_files_concat_warning_fn displayWarning;
FIO_rust_multi_files_concat_disable_remove_fn disableRemove;
FIO_rust_multi_files_concat_quiet_abort_fn displayQuietAbort;
FIO_rust_multi_files_concat_confirmation_fn requireConfirmation;
} FIO_rust_multi_files_concat_callbacks_t;
typedef char FIO_rust_multi_files_concat_callback_sizes[
(sizeof(FIO_rust_multi_files_concat_fatal_fn) == sizeof(void*)
&& sizeof(FIO_rust_multi_files_concat_warning_fn) == sizeof(void*)
&& sizeof(FIO_rust_multi_files_concat_disable_remove_fn) == sizeof(void*)
&& sizeof(FIO_rust_multi_files_concat_quiet_abort_fn) == sizeof(void*)
&& sizeof(FIO_rust_multi_files_concat_confirmation_fn) == sizeof(void*)) ? 1 : -1];
typedef char FIO_rust_multi_files_concat_opaque_offset[
(offsetof(FIO_rust_multi_files_concat_callbacks_t, opaque) == 0) ? 1 : -1];
typedef char FIO_rust_multi_files_concat_fatal_stdout_offset[
(offsetof(FIO_rust_multi_files_concat_callbacks_t, fatalStdoutRemove)
== sizeof(void*)) ? 1 : -1];
typedef char FIO_rust_multi_files_concat_fatal_test_offset[
(offsetof(FIO_rust_multi_files_concat_callbacks_t, fatalTestRemove)
== 2 * sizeof(void*)) ? 1 : -1];
typedef char FIO_rust_multi_files_concat_warning_offset[
(offsetof(FIO_rust_multi_files_concat_callbacks_t, displayWarning)
== 3 * sizeof(void*)) ? 1 : -1];
typedef char FIO_rust_multi_files_concat_disable_remove_offset[
(offsetof(FIO_rust_multi_files_concat_callbacks_t, disableRemove)
== 4 * sizeof(void*)) ? 1 : -1];
typedef char FIO_rust_multi_files_concat_quiet_abort_offset[
(offsetof(FIO_rust_multi_files_concat_callbacks_t, displayQuietAbort)
== 5 * sizeof(void*)) ? 1 : -1];
typedef char FIO_rust_multi_files_concat_confirmation_offset[
(offsetof(FIO_rust_multi_files_concat_callbacks_t, requireConfirmation)
== 6 * sizeof(void*)) ? 1 : -1];
typedef char FIO_rust_multi_files_concat_callbacks_size[
(sizeof(FIO_rust_multi_files_concat_callbacks_t) == 7 * sizeof(void*)) ? 1 : -1];
int FIO_rust_multiFilesConcatWarning(int nbFilesTotal,
int hasStdoutOutput,
int testMode,
int hasOutputFile,
int removeSrcFile,
int overwrite,
int displayLevel,
int displayLevelCutoff,
int hasStdinInput,
const char* outFileName,
const FIO_rust_multi_files_concat_callbacks_t* callbacks);
/*-*************************************
* Parameters: Initialization
***************************************/
#define FIO_OVERLAP_LOG_NOTSET 9999
#define FIO_LDM_PARAM_NOTSET 9999
/* These policy and filesystem leaves are implemented by the Rust CLI archive.
* The declarations in fileio.h remain the C ABI shims while C keeps private
* resource/layout state and exact diagnostics around the Rust policy calls. */
/* FIO_highbit64() is a Rust-owned scalar policy leaf. */
unsigned FIO_highbit64(unsigned long long v);
unsigned long long FIO_getLargestFileSize(const char** inFileNames, unsigned nbFiles);
int FIO_rust_adjustParamsForPatchFromMode(FIO_prefs_t* prefs,
ZSTD_compressionParameters* comprParams,
unsigned long long dictSize,
unsigned long long maxSrcFileSize,
ZSTD_compressionParameters cParams,
unsigned* fileWindowLog,
int* autoLdm,
int* optimalParser);
const char* FIO_determineCompressedName(const char* srcFileName, const char* outDirName, const char* suffix);
const char* FIO_rust_determineDstName(const char* srcFileName, const char* outDirName,
const char* const* suffixList, const char* suffixListStr);
enum {
FIO_RUST_COMPRESS_DST_TRANSFER_STAT_NO = 0,
FIO_RUST_COMPRESS_DST_TRANSFER_STAT_YES = 1
};
typedef char FIO_rust_compress_dst_transfer_stat_values[
(FIO_RUST_COMPRESS_DST_TRANSFER_STAT_NO == 0
&& FIO_RUST_COMPRESS_DST_TRANSFER_STAT_YES == 1) ? 1 : -1];
int FIO_rust_compressDestinationTransferStat(int sourceIsStdin,
int destinationIsStdout,
int sourceIsRegular);
int FIO_rust_adjustMemLimitForPatchFromMode(FIO_prefs_t* prefs,
unsigned long long dictSize,
unsigned long long maxSrcFileSize);
int FIO_rust_getDictFileStat(const char* fileName, stat_t* statBuf);
typedef char FIO_rust_dict_buffer_type_values[
(FIO_mallocDict == 0 && FIO_mmapDict == 1) ? 1 : -1];
typedef char FIO_rust_mmap_policy_values[
(ZSTD_ps_auto == 0 && ZSTD_ps_enable == 1 && ZSTD_ps_disable == 2) ? 1 : -1];
int FIO_rust_selectDictBufferType(int mmapDict,
int patchFromMode,
unsigned long long dictSize,
unsigned long long memLimit);
enum {
FIO_RUST_OPEN_SRC_SUCCESS = 0,
FIO_RUST_OPEN_SRC_STAT_FAILED = 1,
FIO_RUST_OPEN_SRC_NON_REGULAR = 2,
FIO_RUST_OPEN_SRC_FOPEN_FAILED = 3,
FIO_RUST_OPEN_SRC_STDIN = 4,
};
int FIO_rust_openSrcFile(int allowBlockDevices,
const char* srcFileName,
stat_t* statbuf,
FILE** outFile);
enum {
FIO_RUST_OPEN_DST_SUCCESS = 0,
FIO_RUST_OPEN_DST_SETBUF_FAILED = 1,
FIO_RUST_OPEN_DST_TEST_MODE = 2,
FIO_RUST_OPEN_DST_STDOUT = 3,
FIO_RUST_OPEN_DST_SAME_FILE = 4,
FIO_RUST_OPEN_DST_EXISTING = 5,
FIO_RUST_OPEN_DST_NULL_DEVICE_REGULAR = 6,
FIO_RUST_OPEN_DST_OPEN_FAILED = 7,
};
enum {
FIO_RUST_OPEN_DST_CONFIRMATION_UNASKED = -1,
FIO_RUST_OPEN_DST_CONFIRMATION_ACCEPT = 0,
FIO_RUST_OPEN_DST_CONFIRMATION_ABORT = 1,
};
enum {
FIO_RUST_OPEN_DST_ACTION_TEST_MODE = 0,
FIO_RUST_OPEN_DST_ACTION_STDOUT = 1,
FIO_RUST_OPEN_DST_ACTION_SAME_FILE = 2,
FIO_RUST_OPEN_DST_ACTION_ADJUST_SPARSE = 3,
FIO_RUST_OPEN_DST_ACTION_NULL_DEVICE_REGULAR = 4,
FIO_RUST_OPEN_DST_ACTION_EXISTING_QUIET_ABORT = 5,
FIO_RUST_OPEN_DST_ACTION_EXISTING_PROMPT = 6,
FIO_RUST_OPEN_DST_ACTION_EXISTING_ABORT = 7,
FIO_RUST_OPEN_DST_ACTION_EXISTING_REMOVE = 8,
FIO_RUST_OPEN_DST_ACTION_OPEN_FAILED = 9,
FIO_RUST_OPEN_DST_ACTION_SETBUF_FAILED = 10,
FIO_RUST_OPEN_DST_ACTION_SUCCESS = 11,
FIO_RUST_OPEN_DST_ACTION_INVALID = 12,
};
int FIO_rust_openDstFileAction(int status, int overwrite, int displayLevel,
int confirmation, int sparseAdjusted);
typedef void (*FIO_rust_open_dst_action_fn)(void* opaque, int action,
const char* dstFileName);
typedef void (*FIO_rust_open_dst_adjust_sparse_fn)(void* opaque,
int isDstRegFile);
typedef void* (*FIO_rust_open_dst_stdout_fn)(void* opaque);
typedef int (*FIO_rust_open_dst_confirm_fn)(void* opaque);
typedef void (*FIO_rust_open_dst_remove_fn)(void* opaque,
const char* dstFileName);
typedef struct {
void* opaque;
FIO_rust_open_dst_action_fn displayAction;
FIO_rust_open_dst_adjust_sparse_fn adjustSparse;
FIO_rust_open_dst_stdout_fn useStdout;
FIO_rust_open_dst_confirm_fn confirm;
FIO_rust_open_dst_remove_fn removeExisting;
} FIO_rust_openDstFileCallbacks_t;
typedef char FIO_rust_open_dst_callback_sizes[
(sizeof(FIO_rust_open_dst_action_fn) == sizeof(void*)
&& sizeof(FIO_rust_open_dst_adjust_sparse_fn) == sizeof(void*)
&& sizeof(FIO_rust_open_dst_stdout_fn) == sizeof(void*)
&& sizeof(FIO_rust_open_dst_confirm_fn) == sizeof(void*)
&& sizeof(FIO_rust_open_dst_remove_fn) == sizeof(void*)) ? 1 : -1];
typedef char FIO_rust_open_dst_callback_offsets[
(offsetof(FIO_rust_openDstFileCallbacks_t, opaque) == 0
&& offsetof(FIO_rust_openDstFileCallbacks_t, displayAction) == sizeof(void*)
&& offsetof(FIO_rust_openDstFileCallbacks_t, adjustSparse) == 2 * sizeof(void*)
&& offsetof(FIO_rust_openDstFileCallbacks_t, useStdout) == 3 * sizeof(void*)
&& offsetof(FIO_rust_openDstFileCallbacks_t, confirm) == 4 * sizeof(void*)
&& offsetof(FIO_rust_openDstFileCallbacks_t, removeExisting) == 5 * sizeof(void*)
&& sizeof(FIO_rust_openDstFileCallbacks_t) == 6 * sizeof(void*)) ? 1 : -1];
void* FIO_rust_openDstFileOrchestrate(
int testMode,
int overwrite,
int displayLevel,
const char* srcFileName,
const char* dstFileName,
int mode,
const FIO_rust_openDstFileCallbacks_t* callbacks);
int FIO_rust_openDstFile(int testMode,
int allowExisting,
const char* srcFileName,
const char* dstFileName,
int mode,
int* isDstRegFile,
FILE** outFile);
int FIO_rust_setDictBufferMalloc(const char* fileName,
unsigned long long expectedFileSize,
size_t maxSize,
void** buffer,
size_t* loadedSize);
/* Rust owns the dictionary-loader status classification. C keeps the
* metadata, ownership handles, and exact diagnostics around the existing
* narrow loader status ABI. */
enum {
FIO_RUST_DICT_DIAGNOSTIC_OK = 0,
FIO_RUST_DICT_DIAGNOSTIC_OPEN_FAILED = 1,
FIO_RUST_DICT_DIAGNOSTIC_TOO_LARGE = 2,
FIO_RUST_DICT_DIAGNOSTIC_ALLOCATION_FAILED = 3,
FIO_RUST_DICT_DIAGNOSTIC_READ_FAILED = 4,
FIO_RUST_DICT_DIAGNOSTIC_MAP_FAILED = 5,
FIO_RUST_DICT_DIAGNOSTIC_VIEW_FAILED = 6,
FIO_RUST_DICT_DIAGNOSTIC_INVALID = 7,
};
int FIO_rust_dictLoadDiagnostic(int dictBufferType, int status);
#if (PLATFORM_POSIX_VERSION > 0) || defined(_MSC_VER) || defined(_WIN32)
enum {
FIO_DICT_MMAP_SUCCESS = 0,
FIO_DICT_MMAP_OPEN_FAILED = 1,
FIO_DICT_MMAP_TOO_LARGE = 2,
FIO_DICT_MMAP_FAILED = 3,
FIO_DICT_MMAP_VIEW_FAILED = 4,
};
int FIO_rust_setDictBufferMMap(const char* fileName,
unsigned long long expectedFileSize,
size_t maxSize,
void** buffer,
size_t* mappedSize,
void** dictHandle);
#endif
void FIO_rust_freeDict(int dictBufferType,
void** buffer,
size_t* bufferSize,
void** dictHandle);
int FIO_rust_removeFile(const char* path);
enum {
FIO_REMOVE_SUCCESS = 0,
FIO_REMOVE_STAT_FAILED = 1,
FIO_REMOVE_NON_REGULAR = 2,
FIO_REMOVE_FAILED = 3
};
enum {
FIO_RUST_REMOVE_FILE_ACTION_SUCCESS = 0,
FIO_RUST_REMOVE_FILE_ACTION_STAT_FAILED = 1,
FIO_RUST_REMOVE_FILE_ACTION_NON_REGULAR = 2,
FIO_RUST_REMOVE_FILE_ACTION_FAILED = 3,
FIO_RUST_REMOVE_FILE_ACTION_INVALID = 4
};
typedef char FIO_rust_remove_file_status_values[
(FIO_REMOVE_SUCCESS == 0
&& FIO_REMOVE_STAT_FAILED == 1
&& FIO_REMOVE_NON_REGULAR == 2
&& FIO_REMOVE_FAILED == 3) ? 1 : -1];
typedef char FIO_rust_remove_file_action_values[
(FIO_RUST_REMOVE_FILE_ACTION_SUCCESS == 0
&& FIO_RUST_REMOVE_FILE_ACTION_STAT_FAILED == 1
&& FIO_RUST_REMOVE_FILE_ACTION_NON_REGULAR == 2
&& FIO_RUST_REMOVE_FILE_ACTION_FAILED == 3
&& FIO_RUST_REMOVE_FILE_ACTION_INVALID == 4) ? 1 : -1];
int FIO_rust_removeFileAction(int status);
int FIO_rust_passThrough(ReadPoolCtx_t* readCtx, WritePoolCtx_t* writeCtx);
enum {
FIO_RUST_ZSTD_FRAME_OK = 0,
FIO_RUST_ZSTD_FRAME_DECODING_ERROR = 1,
FIO_RUST_ZSTD_FRAME_PREMATURE_END = 2,
};
enum {
FIO_RUST_ZSTD_FRAME_ACTION_OK = 0,
FIO_RUST_ZSTD_FRAME_ACTION_DECODING_ERROR = 1,
FIO_RUST_ZSTD_FRAME_ACTION_PREMATURE_END = 2,
FIO_RUST_ZSTD_FRAME_ACTION_INVALID = 3,
};
int FIO_rust_decompressZstdFrameAction(int status);
typedef void (*FIO_rust_frame_progress_fn)(void* opaque,
const char* srcFileName,
U64 decodedSize);
int FIO_rust_decompressZstdFrames(void* fCtx,
void* dctx,
ReadPoolCtx_t* readCtx,
WritePoolCtx_t* writeCtx,
const char* srcFileName,
U64 alreadyDecoded,
U64* decodedSize,
size_t* zstdError,
FIO_rust_frame_progress_fn progress);
typedef void (*FIO_rust_zstd_frame_decoding_error_fn)(void* opaque,
const char* srcFileName,
size_t zstdError);
typedef void (*FIO_rust_zstd_frame_premature_end_fn)(void* opaque,
const char* srcFileName);
typedef struct {
void* callbackContext;
void* fCtx;
void* dctx;
ReadPoolCtx_t* readCtx;
WritePoolCtx_t* writeCtx;
const char* srcFileName;
U64 alreadyDecoded;
FIO_rust_frame_progress_fn progress;
FIO_rust_zstd_frame_decoding_error_fn displayDecodingError;
FIO_rust_zstd_frame_premature_end_fn displayPrematureEnd;
} FIO_rust_zstd_frame_policy_state;
typedef char FIO_rust_zstd_frame_policy_state_layout[
(offsetof(FIO_rust_zstd_frame_policy_state, callbackContext) == 0
&& offsetof(FIO_rust_zstd_frame_policy_state, fCtx) == sizeof(void*)
&& offsetof(FIO_rust_zstd_frame_policy_state, dctx) == 2 * sizeof(void*)
&& offsetof(FIO_rust_zstd_frame_policy_state, readCtx) == 3 * sizeof(void*)
&& offsetof(FIO_rust_zstd_frame_policy_state, writeCtx) == 4 * sizeof(void*)
&& offsetof(FIO_rust_zstd_frame_policy_state, srcFileName) == 5 * sizeof(void*)
&& offsetof(FIO_rust_zstd_frame_policy_state, alreadyDecoded) == 6 * sizeof(void*)
&& offsetof(FIO_rust_zstd_frame_policy_state, progress) == 7 * sizeof(void*)
&& offsetof(FIO_rust_zstd_frame_policy_state, displayDecodingError)
== 8 * sizeof(void*)
&& offsetof(FIO_rust_zstd_frame_policy_state, displayPrematureEnd)
== 9 * sizeof(void*)
&& sizeof(FIO_rust_zstd_frame_policy_state) == 10 * sizeof(void*)
&& sizeof(FIO_rust_zstd_frame_decoding_error_fn) == sizeof(void*)
&& sizeof(FIO_rust_zstd_frame_premature_end_fn) == sizeof(void*))
? 1 : -1];
U64 FIO_rust_decompressZstdFramePolicy(
const FIO_rust_zstd_frame_policy_state* state);
enum {
FIO_RUST_DECOMPRESS_OK = 0,
FIO_RUST_DECOMPRESS_PASS_THROUGH = 1,
FIO_RUST_DECOMPRESS_EMPTY_INPUT = 2,
FIO_RUST_DECOMPRESS_SHORT_INPUT = 3,
FIO_RUST_DECOMPRESS_GZIP_UNSUPPORTED = 4,
FIO_RUST_DECOMPRESS_LZMA_UNSUPPORTED = 5,
FIO_RUST_DECOMPRESS_LZ4_UNSUPPORTED = 6,
FIO_RUST_DECOMPRESS_FRAME_ERROR = 7,
FIO_RUST_DECOMPRESS_UNSUPPORTED_FORMAT = 8,
FIO_RUST_DECOMPRESS_PASS_THROUGH_ERROR = 9,
FIO_RUST_DECOMPRESS_ZSTD_UNSUPPORTED = 10
};
/* Rust owns the decompression status-to-action policy. C keeps the exact
* diagnostic strings and display callback around the resulting action. */
enum {
FIO_RUST_DECOMPRESS_ACTION_NOOP = 0,
FIO_RUST_DECOMPRESS_ACTION_EMPTY_INPUT = 1,
FIO_RUST_DECOMPRESS_ACTION_SHORT_INPUT = 2,
FIO_RUST_DECOMPRESS_ACTION_GZIP_UNSUPPORTED = 3,
FIO_RUST_DECOMPRESS_ACTION_LZMA_UNSUPPORTED = 4,
FIO_RUST_DECOMPRESS_ACTION_LZ4_UNSUPPORTED = 5,
FIO_RUST_DECOMPRESS_ACTION_UNSUPPORTED_FORMAT = 6,
FIO_RUST_DECOMPRESS_ACTION_INVALID = 7
};
typedef int (*FIO_rust_decompress_frame_fn)(void* opaque,
const char* srcFileName,
U64 alreadyDecoded,
U64* frameSize,
size_t* errorCode,
int mode);
typedef int (*FIO_rust_pass_through_fn)(void* opaque);
typedef void (*FIO_rust_decompress_status_fn)(void* opaque,
int status,
const char* srcFileName);
typedef void (*FIO_rust_decompress_finish_fn)(void* opaque,
const char* srcFileName,
U64 decodedSize);
typedef struct {
void* opaque;
FIO_rust_decompress_frame_fn decode_zstd;
FIO_rust_decompress_frame_fn decode_gzip;
FIO_rust_decompress_frame_fn decode_lzma;
FIO_rust_decompress_frame_fn decode_lz4;
FIO_rust_pass_through_fn pass_through;
FIO_rust_decompress_status_fn report_status;
FIO_rust_decompress_finish_fn finish;
} FIO_rust_decompress_callbacks_t;
int FIO_rust_decompressFrames(ReadPoolCtx_t* readCtx,
const char* srcFileName,
int passThrough,
U64* decodedSize,
const FIO_rust_decompress_callbacks_t* callbacks);
int FIO_rust_finishDecompressFrames(int status,
const char* srcFileName,
U64 decodedSize,
const FIO_rust_decompress_callbacks_t* callbacks);
int FIO_rust_decompressStatusAction(int status);
int FIO_rust_decompressPassThroughPolicy(int passThrough,
int overwrite,
int destinationIsStdout);
typedef void (*FIO_rust_decompress_read_fill_fn)(
void* opaque, size_t requested, const unsigned char** buffer, size_t* loaded);
typedef void (*FIO_rust_decompress_read_consume_fn)(void* opaque, size_t consumed);
typedef void (*FIO_rust_decompress_write_acquire_fn)(
void* opaque, void** job, unsigned char** buffer, size_t* bufferSize);
typedef void (*FIO_rust_decompress_write_enqueue_fn)(
void* opaque, void** job, size_t usedBufferSize,
unsigned char** buffer, size_t* bufferSize);
typedef void (*FIO_rust_decompress_write_release_fn)(void* opaque, void* job);
typedef void (*FIO_rust_decompress_sparse_write_end_fn)(void* opaque);
typedef struct {
void* readOpaque;
void* writeOpaque;
size_t readBufferSize;
FIO_rust_decompress_read_fill_fn readFill;
FIO_rust_decompress_read_consume_fn readConsume;
FIO_rust_decompress_write_acquire_fn writeAcquire;
FIO_rust_decompress_write_enqueue_fn writeEnqueue;
FIO_rust_decompress_write_release_fn writeRelease;
FIO_rust_decompress_sparse_write_end_fn sparseWriteEnd;
} FIO_rust_decompress_io_projection_t;
enum {
FIO_RUST_GZIP_DECOMPRESS_OK = 0,
FIO_RUST_GZIP_DECOMPRESS_INIT_ERROR = 1,
FIO_RUST_GZIP_DECOMPRESS_BUF_ERROR = 2,
FIO_RUST_GZIP_DECOMPRESS_INFLATE_ERROR = 3,
FIO_RUST_GZIP_DECOMPRESS_END_ERROR = 4,
FIO_RUST_GZIP_DECOMPRESS_INVALID_PROJECTION = 5
};
typedef int (*FIO_rust_gzip_decompress_init_fn)(void* opaque);
typedef int (*FIO_rust_gzip_decompress_inflate_fn)(
void* opaque, const unsigned char* input, size_t inputSize,
unsigned char* output, size_t outputSize, int flush,
size_t* consumed, size_t* produced);
typedef int (*FIO_rust_gzip_decompress_end_fn)(void* opaque);
typedef struct {
FIO_rust_decompress_io_projection_t io;
void* zlibOpaque;
FIO_rust_gzip_decompress_init_fn zlibInit;
FIO_rust_gzip_decompress_inflate_fn zlibInflate;
FIO_rust_gzip_decompress_end_fn zlibEnd;
} FIO_rust_gzip_decompress_projection_t;
int FIO_rust_decompressGzipFrame(
const FIO_rust_gzip_decompress_projection_t* projection,
U64* frameSize, int* zlibResult);
enum {
FIO_RUST_LZMA_DECOMPRESS_OK = 0,
FIO_RUST_LZMA_DECOMPRESS_INIT_ERROR = 1,
FIO_RUST_LZMA_DECOMPRESS_BUF_ERROR = 2,
FIO_RUST_LZMA_DECOMPRESS_CODE_ERROR = 3,
FIO_RUST_LZMA_DECOMPRESS_INVALID_PROJECTION = 4
};
typedef int (*FIO_rust_lzma_decompress_init_fn)(void* opaque, int plainLzma);
typedef int (*FIO_rust_lzma_decompress_code_fn)(
void* opaque, const unsigned char* input, size_t inputSize,
unsigned char* output, size_t outputSize, int action,
size_t* consumed, size_t* produced);
typedef void (*FIO_rust_lzma_decompress_end_fn)(void* opaque);
typedef struct {
FIO_rust_decompress_io_projection_t io;
void* lzmaOpaque;
FIO_rust_lzma_decompress_init_fn lzmaInit;
FIO_rust_lzma_decompress_code_fn lzmaCode;
FIO_rust_lzma_decompress_end_fn lzmaEnd;
} FIO_rust_lzma_decompress_projection_t;
int FIO_rust_decompressLzmaFrame(
const FIO_rust_lzma_decompress_projection_t* projection,
int plainLzma, U64* frameSize, int* lzmaResult);
enum {
FIO_RUST_LZ4_DECOMPRESS_OK = 0,
FIO_RUST_LZ4_DECOMPRESS_CREATE_ERROR = 1,
FIO_RUST_LZ4_DECOMPRESS_CODE_ERROR = 2,
FIO_RUST_LZ4_DECOMPRESS_UNFINISHED = 3,
FIO_RUST_LZ4_DECOMPRESS_INVALID_PROJECTION = 4
};
typedef int (*FIO_rust_lz4_decompress_create_fn)(
void* opaque, unsigned version, size_t* result);
typedef int (*FIO_rust_lz4_decompress_code_fn)(
void* opaque, unsigned char* output, size_t* outputSize,
const unsigned char* input, size_t* inputSize, size_t* nextToLoad);
typedef void (*FIO_rust_lz4_decompress_free_fn)(void* opaque);
typedef void (*FIO_rust_lz4_decompress_progress_fn)(void* opaque, U64 decodedSize);
typedef struct {
FIO_rust_decompress_io_projection_t io;
void* codecOpaque;
void* progressOpaque;
unsigned version;
FIO_rust_lz4_decompress_create_fn create;
FIO_rust_lz4_decompress_code_fn code;
FIO_rust_lz4_decompress_free_fn freeContext;
FIO_rust_lz4_decompress_progress_fn progress;
} FIO_rust_lz4_decompress_projection_t;
int FIO_rust_decompressLz4Frame(
const FIO_rust_lz4_decompress_projection_t* projection,
U64* frameSize, size_t* lz4Result);
void FIO_rust_displayCompressionParameters(const FIO_prefs_t* prefs);
#ifdef ZSTD_LZ4COMPRESS
int FIO_LZ4_GetBlockSize_FromBlockId(int id);
#endif
/*-*************************************
* Parameters: Display Options
***************************************/
/*-*************************************
* Parameters: Setters
***************************************/
/* FIO_prefs_t functions */
/*-*************************************
* Functions
***************************************/
/** FIO_removeFile() :
* @result : Unlink `fileName`, even if it's read-only */
static int FIO_removeFile(const char* path)
{
int const status = FIO_rust_removeFile(path);
switch (FIO_rust_removeFileAction(status)) {
case FIO_RUST_REMOVE_FILE_ACTION_STAT_FAILED:
DISPLAYLEVEL(2, "zstd: Failed to stat %s while trying to remove it\n", path);
return 0;
case FIO_RUST_REMOVE_FILE_ACTION_NON_REGULAR:
DISPLAYLEVEL(2, "zstd: Refusing to remove non-regular file %s\n", path);
return 0;
case FIO_RUST_REMOVE_FILE_ACTION_SUCCESS:
return 0;
case FIO_RUST_REMOVE_FILE_ACTION_FAILED:
case FIO_RUST_REMOVE_FILE_ACTION_INVALID:
default:
return 1;
}
}
/** FIO_openSrcFile() :
* condition : `srcFileName` must be non-NULL. `prefs` may be NULL.
* @result : FILE* to `srcFileName`, or NULL if it fails */
static FILE* FIO_openSrcFile(const FIO_prefs_t* const prefs, const char* srcFileName, stat_t* statbuf)
{
int allowBlockDevices = prefs != NULL ? prefs->allowBlockDevices : 0;
assert(srcFileName != NULL);
assert(statbuf != NULL);
{ FILE* f = NULL;
int const status = FIO_rust_openSrcFile(allowBlockDevices,
srcFileName,
statbuf,
&f);
if (status == FIO_RUST_OPEN_SRC_STDIN) {
DISPLAYLEVEL(4,"Using stdin for input \n");
SET_BINARY_MODE(stdin);
return stdin;
}
if (status == FIO_RUST_OPEN_SRC_STAT_FAILED) {
DISPLAYLEVEL(1, "zstd: can't stat %s : %s -- ignored \n",
srcFileName, strerror(errno));
return NULL;
}
if (status == FIO_RUST_OPEN_SRC_NON_REGULAR) {
DISPLAYLEVEL(1, "zstd: %s is not a regular file -- ignored \n",
srcFileName);
return NULL;
}
if (status == FIO_RUST_OPEN_SRC_FOPEN_FAILED) {
DISPLAYLEVEL(1, "zstd: %s: %s \n", srcFileName, strerror(errno));
return NULL;
}
assert(status == FIO_RUST_OPEN_SRC_SUCCESS);
return f;
}
}
/* C-owned state for the Rust destination-open policy callbacks. */
typedef struct {
FIO_ctx_t* fCtx;
FIO_prefs_t* prefs;
const char* dstFileName;
} FIO_openDstFileCallbackContext_t;
static void
FIO_openDstAdjustSparse(void* opaque, int isDstRegFile)
{
FIO_openDstFileCallbackContext_t* const context =
(FIO_openDstFileCallbackContext_t*)opaque;
if (context->prefs->sparseFileSupport == 1) {
context->prefs->sparseFileSupport = ZSTD_SPARSE_DEFAULT;
if (!isDstRegFile) {
context->prefs->sparseFileSupport = 0;
DISPLAYLEVEL(4, "Sparse File Support is disabled when output is not a file \n");
}
}
}
static void*
FIO_openDstUseStdout(void* opaque)
{
FIO_openDstFileCallbackContext_t* const context =
(FIO_openDstFileCallbackContext_t*)opaque;
DISPLAYLEVEL(4,"Using stdout for output \n");
SET_BINARY_MODE(stdout);
if (context->prefs->sparseFileSupport == 1) {
context->prefs->sparseFileSupport = 0;
DISPLAYLEVEL(4, "Sparse File Support is automatically disabled on stdout ; try --sparse \n");
}
return stdout;
}
static void
FIO_openDstDisplayAction(void* opaque, int action, const char* dstFileName)
{
(void)opaque;
switch (action) {
case FIO_RUST_OPEN_DST_ACTION_SAME_FILE:
DISPLAYLEVEL(1, "zstd: Refusing to open an output file which will overwrite the input file \n");
break;
case FIO_RUST_OPEN_DST_ACTION_NULL_DEVICE_REGULAR:
EXM_THROW(40, "%s is unexpectedly categorized as a regular file",
dstFileName);
return;
case FIO_RUST_OPEN_DST_ACTION_EXISTING_QUIET_ABORT:
DISPLAYLEVEL(1, "zstd: %s already exists; not overwritten \n",
dstFileName);
break;
case FIO_RUST_OPEN_DST_ACTION_OPEN_FAILED:
DISPLAYLEVEL(1, "zstd: %s: %s\n", dstFileName, strerror(errno));
break;
case FIO_RUST_OPEN_DST_ACTION_SETBUF_FAILED:
DISPLAYLEVEL(2, "Warning: setvbuf failed for %s\n", dstFileName);
break;
case FIO_RUST_OPEN_DST_ACTION_INVALID:
default:
assert(0);
break;
}
}
static int
FIO_openDstConfirm(void* opaque)
{
FIO_openDstFileCallbackContext_t* const context =
(FIO_openDstFileCallbackContext_t*)opaque;
DISPLAY("zstd: %s already exists; ", context->dstFileName);
return UTIL_requireUserConfirmation(
"overwrite (y/n) ? ", "Not overwritten \n", "yY",
context->fCtx->hasStdinInput)
? FIO_RUST_OPEN_DST_CONFIRMATION_ABORT
: FIO_RUST_OPEN_DST_CONFIRMATION_ACCEPT;
}
static void
FIO_openDstRemoveExisting(void* opaque, const char* dstFileName)
{
(void)opaque;
/* Keep the existing C wrapper so its stat/non-regular diagnostics remain
* unchanged. Rust calls this only after the overwrite policy allows it. */
(void)FIO_removeFile(dstFileName);
}
/** FIO_openDstFile() :
* condition : `dstFileName` must be non-NULL.
* @result : FILE* to `dstFileName`, or NULL if it fails */
static FILE*
FIO_openDstFile(FIO_ctx_t* fCtx, FIO_prefs_t* const prefs,
const char* srcFileName, const char* dstFileName,
const int mode)
{
FIO_openDstFileCallbackContext_t context = { fCtx, prefs, dstFileName };
FIO_rust_openDstFileCallbacks_t callbacks = {
&context,
FIO_openDstDisplayAction,
FIO_openDstAdjustSparse,
FIO_openDstUseStdout,
FIO_openDstConfirm,
FIO_openDstRemoveExisting
};
return (FILE*)FIO_rust_openDstFileOrchestrate(
prefs->testMode, prefs->overwrite, g_display_prefs.displayLevel,
srcFileName, dstFileName, mode, &callbacks);
}
/* FIO_getDictFileStat() :
*/
static void FIO_getDictFileStat(const char* fileName, stat_t* dictFileStat) {
enum {
FIO_DICT_STAT_SUCCESS = 0,
FIO_DICT_STAT_FAILED = 1,
FIO_DICT_STAT_NON_REGULAR = 2,
};
int status;
assert(dictFileStat != NULL);
if (fileName == NULL) return;
status = FIO_rust_getDictFileStat(fileName, dictFileStat);
if (status == FIO_DICT_STAT_FAILED) {
EXM_THROW(31, "Stat failed on dictionary file %s: %s", fileName, strerror(errno));
}
if (status == FIO_DICT_STAT_NON_REGULAR) {
EXM_THROW(32, "Dictionary %s must be a regular file.", fileName);
}
assert(status == FIO_DICT_STAT_SUCCESS);
}
/* FIO_setDictBufferMalloc() :
* allocates a buffer, pointed by `dict->dictBuffer`,
* loads `filename` content into it, up to DICTSIZE_MAX bytes.
* @return : loaded size
* if fileName==NULL, returns 0 and a NULL pointer
*/
static size_t FIO_setDictBufferMalloc(FIO_Dict_t* dict, const char* fileName, FIO_prefs_t* const prefs, stat_t* dictFileStat)
{
U64 fileSize;
size_t loadedSize = 0;
size_t const dictSizeMax = prefs->patchFromMode ? prefs->memLimit : DICTSIZE_MAX;
void** bufferPtr = &dict->dictBuffer;
enum {
FIO_DICT_LOAD_SUCCESS = 0,
FIO_DICT_LOAD_OPEN_FAILED = 1,
FIO_DICT_LOAD_TOO_LARGE = 2,
FIO_DICT_LOAD_ALLOCATION_FAILED = 3,
FIO_DICT_LOAD_READ_FAILED = 4,
};
int status;
int diagnostic;
assert(bufferPtr != NULL);
assert(dictFileStat != NULL);
*bufferPtr = NULL;
if (fileName == NULL) return 0;
DISPLAYLEVEL(4,"Loading %s as dictionary \n", fileName);
fileSize = UTIL_getFileSizeStat(dictFileStat);
if (fileSize > dictSizeMax) {
EXM_THROW(34, "Dictionary file %s is too large (> %u bytes)",
fileName, (unsigned)dictSizeMax); /* avoid extreme cases */
}
status = FIO_rust_setDictBufferMalloc(fileName,
(unsigned long long)fileSize,
dictSizeMax,
bufferPtr,
&loadedSize);
diagnostic = FIO_rust_dictLoadDiagnostic((int)FIO_mallocDict, status);
switch (diagnostic) {
case FIO_RUST_DICT_DIAGNOSTIC_OK:
return loadedSize;
case FIO_RUST_DICT_DIAGNOSTIC_OPEN_FAILED:
EXM_THROW(33, "Couldn't open dictionary %s: %s", fileName, strerror(errno));
case FIO_RUST_DICT_DIAGNOSTIC_TOO_LARGE:
EXM_THROW(34, "Dictionary file %s is too large (> %u bytes)",
fileName, (unsigned)dictSizeMax); /* avoid extreme cases */
case FIO_RUST_DICT_DIAGNOSTIC_ALLOCATION_FAILED:
EXM_THROW(34, "%s", strerror(errno));
case FIO_RUST_DICT_DIAGNOSTIC_READ_FAILED:
EXM_THROW(35, "Error reading dictionary file %s : %s",
fileName, strerror(errno));
default:
assert(0); /* unexpected Rust diagnostic */
return 0;
}
}
#if (PLATFORM_POSIX_VERSION > 0)
static size_t FIO_setDictBufferMMap(FIO_Dict_t* dict, const char* fileName, FIO_prefs_t* const prefs, stat_t* dictFileStat)
{
U64 fileSize;
size_t const dictSizeMax = prefs->patchFromMode ? prefs->memLimit : DICTSIZE_MAX;
void** bufferPtr = &dict->dictBuffer;
int status;
int diagnostic;
assert(bufferPtr != NULL);
assert(dictFileStat != NULL);
*bufferPtr = NULL;
if (fileName == NULL) return 0;
DISPLAYLEVEL(4,"Loading %s as dictionary \n", fileName);
fileSize = UTIL_getFileSizeStat(dictFileStat);
if (fileSize > dictSizeMax) {
EXM_THROW(34, "Dictionary file %s is too large (> %u bytes)",
fileName, (unsigned)dictSizeMax); /* avoid extreme cases */
}
status = FIO_rust_setDictBufferMMap(fileName,
(unsigned long long)fileSize,
dictSizeMax,
bufferPtr,
&dict->dictBufferSize,
NULL);
diagnostic = FIO_rust_dictLoadDiagnostic((int)FIO_mmapDict, status);
switch (diagnostic) {
case FIO_RUST_DICT_DIAGNOSTIC_OK:
return dict->dictBufferSize;
case FIO_RUST_DICT_DIAGNOSTIC_OPEN_FAILED:
EXM_THROW(33, "Couldn't open dictionary %s: %s", fileName, strerror(errno));
case FIO_RUST_DICT_DIAGNOSTIC_TOO_LARGE:
EXM_THROW(34, "Dictionary file %s is too large (> %u bytes)",
fileName, (unsigned)dictSizeMax); /* avoid extreme cases */
case FIO_RUST_DICT_DIAGNOSTIC_MAP_FAILED:
EXM_THROW(34, "%s", strerror(errno));
default:
assert(0); /* unexpected Rust diagnostic */
return 0;
}
}
#elif defined(_MSC_VER) || defined(_WIN32)
static size_t FIO_setDictBufferMMap(FIO_Dict_t* dict, const char* fileName, FIO_prefs_t* const prefs, stat_t* dictFileStat)
{
U64 fileSize;
size_t const dictSizeMax = prefs->patchFromMode ? prefs->memLimit : DICTSIZE_MAX;
void** bufferPtr = &dict->dictBuffer;
int status;
int diagnostic;
assert(bufferPtr != NULL);
assert(dictFileStat != NULL);
*bufferPtr = NULL;
if (fileName == NULL) return 0;
DISPLAYLEVEL(4,"Loading %s as dictionary \n", fileName);
fileSize = UTIL_getFileSizeStat(dictFileStat);
if (fileSize > dictSizeMax) {
EXM_THROW(34, "Dictionary file %s is too large (> %u bytes)",
fileName, (unsigned)dictSizeMax); /* avoid extreme cases */
}
status = FIO_rust_setDictBufferMMap(fileName,
(unsigned long long)fileSize,
dictSizeMax,
bufferPtr,
&dict->dictBufferSize,
(void**)&dict->dictHandle);
diagnostic = FIO_rust_dictLoadDiagnostic((int)FIO_mmapDict, status);
switch (diagnostic) {
case FIO_RUST_DICT_DIAGNOSTIC_OK:
return dict->dictBufferSize;
case FIO_RUST_DICT_DIAGNOSTIC_OPEN_FAILED:
EXM_THROW(33, "Couldn't open dictionary %s: %s", fileName, strerror(errno));
case FIO_RUST_DICT_DIAGNOSTIC_TOO_LARGE:
EXM_THROW(34, "Dictionary file %s is too large (> %u bytes)",
fileName, (unsigned)dictSizeMax); /* avoid extreme cases */
case FIO_RUST_DICT_DIAGNOSTIC_MAP_FAILED:
EXM_THROW(35, "Couldn't map dictionary %s: %s", fileName, strerror(errno));
case FIO_RUST_DICT_DIAGNOSTIC_VIEW_FAILED:
EXM_THROW(36, "%s", strerror(errno));
default:
assert(0); /* unexpected Rust diagnostic */
return 0;
}
}
#else
static size_t FIO_setDictBufferMMap(FIO_Dict_t* dict, const char* fileName, FIO_prefs_t* const prefs, stat_t* dictFileStat)
{
return FIO_setDictBufferMalloc(dict, fileName, prefs, dictFileStat);
}
#endif
static void FIO_freeDict(FIO_Dict_t* dict) {
if (dict->dictBufferType != FIO_mallocDict
&& dict->dictBufferType != FIO_mmapDict) {
assert(0); /* Should not reach this case */
return;
}
FIO_rust_freeDict((int)dict->dictBufferType,
&dict->dictBuffer,
&dict->dictBufferSize,
#if defined(_MSC_VER) || defined(_WIN32)
(void**)&dict->dictHandle
#else
NULL
#endif
);
}
static void FIO_initDict(FIO_Dict_t* dict, const char* fileName, FIO_prefs_t* const prefs, stat_t* dictFileStat, FIO_dictBufferType_t dictBufferType) {
#if defined(_MSC_VER) || defined(_WIN32)
dict->dictHandle = NULL;
#endif
dict->dictBufferType = dictBufferType;
if (dict->dictBufferType == FIO_mallocDict) {
dict->dictBufferSize = FIO_setDictBufferMalloc(dict, fileName, prefs, dictFileStat);
} else if (dict->dictBufferType == FIO_mmapDict) {
dict->dictBufferSize = FIO_setDictBufferMMap(dict, fileName, prefs, dictFileStat);
} else {
assert(0); /* Should not reach this case */
}
}
static void FIO_adjustMemLimitForPatchFromMode(FIO_prefs_t* const prefs,
unsigned long long const dictSize,
unsigned long long const maxSrcFileSize)
{
enum {
FIO_PATCH_MEM_LIMIT_SUCCESS = 0,
FIO_PATCH_MEM_LIMIT_UNKNOWN_SIZE = 1,
FIO_PATCH_MEM_LIMIT_TOO_LARGE = 2
};
unsigned const maxWindowSize = (1U << ZSTD_WINDOWLOG_MAX);
int const status = FIO_rust_adjustMemLimitForPatchFromMode(prefs, dictSize, maxSrcFileSize);
if (status == FIO_PATCH_MEM_LIMIT_UNKNOWN_SIZE)
EXM_THROW(42, "Using --patch-from with stdin requires --stream-size");
if (status == FIO_PATCH_MEM_LIMIT_TOO_LARGE)
EXM_THROW(42, "Can't handle files larger than %u GB\n", maxWindowSize/(1 GB));
assert(status == FIO_PATCH_MEM_LIMIT_SUCCESS);
}
/* FIO_multiFilesConcatWarning() :
* This function supplies the C-owned diagnostics and prompt callbacks for the
* Rust policy/order wrapper used when processing multiple files with -o or -c.
* Returns 1 if the console should abort, 0 if console should proceed.
*
* If output is stdout or test mode is active, check that `--rm` disabled.
*
* If there is just 1 file to process, zstd will proceed as usual.
* If each file get processed into its own separate destination file, proceed as usual.
*
* When multiple files are processed into a single output,
* display a warning message, then disable --rm if it's set.
*
* If -f is specified or if output is stdout, just proceed.
* If output is set with -o, prompt for confirmation.
*/
typedef struct {
const FIO_ctx_t* fCtx;
FIO_prefs_t* prefs;
} FIO_multiFilesConcatCallbackContext_t;
static void
FIO_multiFilesConcatFatalStdoutRemove(void* opaque)
{
(void)opaque;
/* this should not happen ; hard fail, to protect user's data
* note: this should rather be an assert(), but we want to be certain that user's data will not be wiped out in case it nonetheless happen */
EXM_THROW(43, "It's not allowed to remove input files when processed output is piped to stdout. "
"This scenario is not supposed to be possible. "
"This is a programming error. File an issue for it to be fixed.");
}
static void
FIO_multiFilesConcatFatalTestRemove(void* opaque)
{
(void)opaque;
/* this should not happen ; hard fail, to protect user's data
* note: this should rather be an assert(), but we want to be certain that user's data will not be wiped out in case it nonetheless happen */
EXM_THROW(43, "Test mode shall not remove input files! "
"This scenario is not supposed to be possible. "
"This is a programming error. File an issue for it to be fixed.");
}
static void
FIO_multiFilesConcatDisplayWarning(void* opaque, int hasStdoutOutput,
const char* outFileName)
{
(void)opaque;
if (hasStdoutOutput) {
DISPLAYLEVEL(2, "zstd: WARNING: all input files will be processed and concatenated into stdout. \n");
} else {
DISPLAYLEVEL(2, "zstd: WARNING: all input files will be processed and concatenated into a single output file: %s \n", outFileName);
}
DISPLAYLEVEL(2, "The concatenated output CANNOT regenerate original file names nor directory structure. \n")
}
static void
FIO_multiFilesConcatDisableRemove(void* opaque)
{
FIO_multiFilesConcatCallbackContext_t* const context =
(FIO_multiFilesConcatCallbackContext_t*)opaque;
DISPLAYLEVEL(2, "Since it's a destructive operation, input files will not be removed. \n");
context->prefs->removeSrcFile = 0;
}
static void
FIO_multiFilesConcatDisplayQuietAbort(void* opaque)
{
(void)opaque;
/* multiple files concatenated into single destination file using -o without -f */
/* quiet mode => no prompt => fail automatically */
DISPLAYLEVEL(1, "Concatenating multiple processed inputs into a single output loses file metadata. \n");
DISPLAYLEVEL(1, "Aborting. \n");
}
static int
FIO_multiFilesConcatRequireConfirmation(void* opaque, int hasStdinInput)
{
(void)opaque;
return UTIL_requireUserConfirmation("Proceed? (y/n): ", "Aborting...", "yY", hasStdinInput);
}
static int FIO_multiFilesConcatWarning(const FIO_ctx_t* fCtx, FIO_prefs_t* prefs, const char* outFileName, int displayLevelCutoff)
{
FIO_multiFilesConcatCallbackContext_t context = { fCtx, prefs };
FIO_rust_multi_files_concat_callbacks_t callbacks;
callbacks.opaque = &context;
callbacks.fatalStdoutRemove = FIO_multiFilesConcatFatalStdoutRemove;
callbacks.fatalTestRemove = FIO_multiFilesConcatFatalTestRemove;
callbacks.displayWarning = FIO_multiFilesConcatDisplayWarning;
callbacks.disableRemove = FIO_multiFilesConcatDisableRemove;
callbacks.displayQuietAbort = FIO_multiFilesConcatDisplayQuietAbort;
callbacks.requireConfirmation = FIO_multiFilesConcatRequireConfirmation;
return FIO_rust_multiFilesConcatWarning(
fCtx->nbFilesTotal, fCtx->hasStdoutOutput, prefs->testMode,
outFileName != NULL, prefs->removeSrcFile, prefs->overwrite,
g_display_prefs.displayLevel, displayLevelCutoff, fCtx->hasStdinInput,
outFileName, &callbacks);
}
/* Rust owns the final multi-file summary decision and ordering. These
* callbacks retain the exact C display sinks and format strings. */
static void FIO_rust_multipleSummaryDisplayProgress(void* opaque)
{
(void)opaque;
DISPLAY_PROGRESS("\r%79s\r", "");
}
static void FIO_rust_compressMultipleSummary(void* opaque, int nbFilesProcessed,
U64 totalBytesInput, U64 totalBytesOutput)
{
UTIL_HumanReadableSize_t const hr_isize =
UTIL_makeHumanReadableSize(totalBytesInput);
UTIL_HumanReadableSize_t const hr_osize =
UTIL_makeHumanReadableSize(totalBytesOutput);
(void)opaque;
if (totalBytesInput == 0) {
DISPLAY_SUMMARY("%3d files compressed : (%6.*f%4s => %6.*f%4s)\n",
nbFilesProcessed,
hr_isize.precision, hr_isize.value, hr_isize.suffix,
hr_osize.precision, hr_osize.value, hr_osize.suffix);
} else {
DISPLAY_SUMMARY("%3d files compressed : %.2f%% (%6.*f%4s => %6.*f%4s)\n",
nbFilesProcessed,
(double)totalBytesOutput/(double)totalBytesInput*100,
hr_isize.precision, hr_isize.value, hr_isize.suffix,
hr_osize.precision, hr_osize.value, hr_osize.suffix);
}
}
static void FIO_rust_decompressMultipleSummary(void* opaque, int nbFilesProcessed,
U64 totalBytesOutput)
{
(void)opaque;
DISPLAY_SUMMARY("%d files decompressed : %6llu bytes total \n",
nbFilesProcessed, (unsigned long long)totalBytesOutput);
}
static void FIO_displayMultipleFileSummary(const FIO_ctx_t* fCtx, int summaryKind)
{
FIO_rust_multiple_summary_callbacks_t callbacks;
callbacks.opaque = NULL;
callbacks.displayProgress = FIO_rust_multipleSummaryDisplayProgress;
callbacks.displayCompression = FIO_rust_compressMultipleSummary;
callbacks.displayDecompression = FIO_rust_decompressMultipleSummary;
FIO_rust_displayMultipleFileSummary(
fCtx->nbFilesTotal, fCtx->nbFilesProcessed,
(U64)fCtx->totalBytesInput, (U64)fCtx->totalBytesOutput,
summaryKind, &callbacks);
}
typedef int (*FIO_rust_multiple_destination_warning_fn)(void* opaque);
typedef void* (*FIO_rust_multiple_destination_open_fn)(void* opaque);
typedef void (*FIO_rust_multiple_destination_attach_fn)(
void* opaque, void* destination);
typedef int (*FIO_rust_multiple_destination_close_fn)(void* opaque);
#ifndef ZSTD_NOCOMPRESS
/* **********************************************************************
* Compression
************************************************************************/
typedef struct {
FIO_Dict_t dict;
const char* dictFileName;
stat_t dictFileStat;
ZSTD_CStream* cctx;
WritePoolCtx_t *writeCtx;
ReadPoolCtx_t *readCtx;
} cRess_t;
enum {
FIO_RUST_COMPRESS_OK = 0,
FIO_RUST_COMPRESS_GZIP_UNSUPPORTED = 1,
FIO_RUST_COMPRESS_LZMA_UNSUPPORTED = 2,
FIO_RUST_COMPRESS_LZ4_UNSUPPORTED = 3,
FIO_RUST_COMPRESS_ZSTD_UNSUPPORTED = 4
};
enum {
FIO_RUST_COMPRESS_DIAGNOSTIC_OK = 0,
FIO_RUST_COMPRESS_DIAGNOSTIC_GZIP_UNSUPPORTED = 1,
FIO_RUST_COMPRESS_DIAGNOSTIC_LZMA_UNSUPPORTED = 2,
FIO_RUST_COMPRESS_DIAGNOSTIC_LZ4_UNSUPPORTED = 3,
FIO_RUST_COMPRESS_DIAGNOSTIC_ZSTD_UNSUPPORTED = 4,
FIO_RUST_COMPRESS_DIAGNOSTIC_UNKNOWN = 5
};
int FIO_rust_compressFilenameDiagnostic(int status);
typedef unsigned long long (*FIO_rust_compress_zstd_fn)(
void* fCtx, void* prefs, void* ress,
const char* srcFileName, U64 srcFileSize,
int compressionLevel, U64* readsize);
typedef unsigned long long (*FIO_rust_compress_gzip_fn)(
void* ress, const char* srcFileName, U64 srcFileSize,
int compressionLevel, U64* readsize);
typedef unsigned long long (*FIO_rust_compress_lzma_fn)(
void* ress, const char* srcFileName, U64 srcFileSize,
int compressionLevel, U64* readsize, int plainLzma);
typedef unsigned long long (*FIO_rust_compress_lz4_fn)(
void* ress, const char* srcFileName, U64 srcFileSize,
int compressionLevel, int checksumFlag, U64* readsize);
typedef void (*FIO_rust_compress_input_display_fn)(
void* opaque, const char* srcFileName, U64 fileSize);
typedef void (*FIO_rust_compress_status_display_fn)(
void* opaque, void* fCtx, const char* dstFileName,
const char* srcFileName, U64 readsize, U64 compressedfilesize);
typedef struct {
void* opaque;
FIO_rust_compress_zstd_fn compress_zstd;
FIO_rust_compress_gzip_fn compress_gzip;
FIO_rust_compress_lzma_fn compress_lzma;
FIO_rust_compress_lz4_fn compress_lz4;
FIO_rust_compress_input_display_fn display_input;
FIO_rust_compress_status_display_fn display_status;
} FIO_rust_compress_callbacks_t;
typedef int (*FIO_rust_compress_multiple_file_fn)(
void* opaque, const char* dstFileName, const char* srcFileName);
typedef struct {
void* fCtx;
const char** inFileNamesTable;
const char* outFileName;
void* opaque;
FIO_rust_compress_multiple_file_fn compressFile;
} FIO_rust_compress_multiple_projection_t;
typedef char FIO_rust_compress_multiple_fctx_offset[
(offsetof(FIO_rust_compress_multiple_projection_t, fCtx) == 0) ? 1 : -1];
typedef char FIO_rust_compress_multiple_input_names_offset[
(offsetof(FIO_rust_compress_multiple_projection_t, inFileNamesTable)
== sizeof(void*)) ? 1 : -1];
typedef char FIO_rust_compress_multiple_output_name_offset[
(offsetof(FIO_rust_compress_multiple_projection_t, outFileName)
== 2 * sizeof(void*)) ? 1 : -1];
typedef char FIO_rust_compress_multiple_opaque_offset[
(offsetof(FIO_rust_compress_multiple_projection_t, opaque)
== 3 * sizeof(void*)) ? 1 : -1];
typedef char FIO_rust_compress_multiple_callback_offset[
(offsetof(FIO_rust_compress_multiple_projection_t, compressFile)
== 4 * sizeof(void*)) ? 1 : -1];
typedef char FIO_rust_compress_multiple_projection_size[
(sizeof(FIO_rust_compress_multiple_projection_t)
== 4 * sizeof(void*) + sizeof(FIO_rust_compress_multiple_file_fn)) ? 1 : -1];
int FIO_rust_compressMultipleFilenames(
const FIO_rust_compress_multiple_projection_t* projection);
typedef struct {
const FIO_rust_compress_multiple_projection_t* files;
FIO_rust_multiple_destination_warning_fn warning;
FIO_rust_multiple_destination_open_fn openDestination;
FIO_rust_multiple_destination_attach_fn attachDestination;
FIO_rust_multiple_destination_close_fn closeDestination;
} FIO_rust_compress_multiple_destination_projection_t;
typedef char FIO_rust_compress_multiple_destination_files_offset[
(offsetof(FIO_rust_compress_multiple_destination_projection_t, files)
== 0) ? 1 : -1];
typedef char FIO_rust_compress_multiple_destination_warning_offset[
(offsetof(FIO_rust_compress_multiple_destination_projection_t, warning)
== sizeof(void*)) ? 1 : -1];
typedef char FIO_rust_compress_multiple_destination_open_offset[
(offsetof(FIO_rust_compress_multiple_destination_projection_t, openDestination)
== 2 * sizeof(void*)) ? 1 : -1];
typedef char FIO_rust_compress_multiple_destination_attach_offset[
(offsetof(FIO_rust_compress_multiple_destination_projection_t, attachDestination)
== 3 * sizeof(void*)) ? 1 : -1];
typedef char FIO_rust_compress_multiple_destination_close_offset[
(offsetof(FIO_rust_compress_multiple_destination_projection_t, closeDestination)
== 4 * sizeof(void*)) ? 1 : -1];
typedef char FIO_rust_compress_multiple_destination_callback_size[
(sizeof(FIO_rust_multiple_destination_warning_fn) == sizeof(void*)
&& sizeof(FIO_rust_multiple_destination_open_fn) == sizeof(void*)
&& sizeof(FIO_rust_multiple_destination_attach_fn) == sizeof(void*)
&& sizeof(FIO_rust_multiple_destination_close_fn) == sizeof(void*)) ? 1 : -1];
typedef char FIO_rust_compress_multiple_destination_projection_size[
(sizeof(FIO_rust_compress_multiple_destination_projection_t)
== 5 * sizeof(void*)) ? 1 : -1];
int FIO_rust_compressMultipleFilenamesWithDestination(
const FIO_rust_compress_multiple_destination_projection_t* projection);
typedef int (*FIO_rust_compress_multiple_separate_file_fn)(
void* opaque, const char* srcFileName);
/* Rust selects the destination-mode callback; both callbacks retain their
* private resources, path construction, diagnostics, and compression leaf. */
typedef struct {
void* fCtx;
const char** inFileNamesTable;
void* opaque;
int mirrorOutput;
FIO_rust_compress_multiple_separate_file_fn compressMirroredFile;
FIO_rust_compress_multiple_separate_file_fn compressFlatFile;
} FIO_rust_compress_multiple_separate_projection_t;
typedef char FIO_rust_compress_multiple_separate_fctx_offset[
(offsetof(FIO_rust_compress_multiple_separate_projection_t, fCtx) == 0) ? 1 : -1];
typedef char FIO_rust_compress_multiple_separate_input_names_offset[
(offsetof(FIO_rust_compress_multiple_separate_projection_t, inFileNamesTable)
== sizeof(void*)) ? 1 : -1];
typedef char FIO_rust_compress_multiple_separate_opaque_offset[
(offsetof(FIO_rust_compress_multiple_separate_projection_t, opaque)
== 2 * sizeof(void*)) ? 1 : -1];
typedef char FIO_rust_compress_multiple_separate_mode_offset[
(offsetof(FIO_rust_compress_multiple_separate_projection_t, mirrorOutput)
== 3 * sizeof(void*)) ? 1 : -1];
typedef char FIO_rust_compress_multiple_separate_mirrored_callback_offset[
(offsetof(FIO_rust_compress_multiple_separate_projection_t, compressMirroredFile)
== 4 * sizeof(void*)) ? 1 : -1];
typedef char FIO_rust_compress_multiple_separate_flat_callback_offset[
(offsetof(FIO_rust_compress_multiple_separate_projection_t, compressFlatFile)
== 5 * sizeof(void*)) ? 1 : -1];
typedef char FIO_rust_compress_multiple_separate_int_size[
(sizeof(int) <= sizeof(void*)) ? 1 : -1];
typedef char FIO_rust_compress_multiple_separate_callback_size[
(sizeof(FIO_rust_compress_multiple_separate_file_fn) == sizeof(void*)) ? 1 : -1];
typedef char FIO_rust_compress_multiple_separate_projection_size[
(sizeof(FIO_rust_compress_multiple_separate_projection_t)
== 6 * sizeof(void*)) ? 1 : -1];
int FIO_rust_compressMultipleSeparateFilenames(
const FIO_rust_compress_multiple_separate_projection_t* projection);
#endif /* #ifndef ZSTD_NOCOMPRESS */
#ifndef ZSTD_NODECOMPRESS
typedef int (*FIO_rust_decompress_multiple_file_fn)(
void* opaque, const char* outFileName, const char* srcFileName);
typedef struct {
void* fCtx;
const char** srcNamesTable;
const char* outFileName;
void* opaque;
FIO_rust_decompress_multiple_file_fn decompressFile;
} FIO_rust_decompress_multiple_projection_t;
typedef char FIO_rust_decompress_multiple_fctx_offset[
(offsetof(FIO_rust_decompress_multiple_projection_t, fCtx) == 0) ? 1 : -1];
typedef char FIO_rust_decompress_multiple_input_names_offset[
(offsetof(FIO_rust_decompress_multiple_projection_t, srcNamesTable)
== sizeof(void*)) ? 1 : -1];
typedef char FIO_rust_decompress_multiple_output_name_offset[
(offsetof(FIO_rust_decompress_multiple_projection_t, outFileName)
== 2 * sizeof(void*)) ? 1 : -1];
typedef char FIO_rust_decompress_multiple_opaque_offset[
(offsetof(FIO_rust_decompress_multiple_projection_t, opaque)
== 3 * sizeof(void*)) ? 1 : -1];
typedef char FIO_rust_decompress_multiple_callback_offset[
(offsetof(FIO_rust_decompress_multiple_projection_t, decompressFile)
== 4 * sizeof(void*)) ? 1 : -1];
typedef char FIO_rust_decompress_multiple_projection_size[
(sizeof(FIO_rust_decompress_multiple_projection_t)
== 4 * sizeof(void*) + sizeof(FIO_rust_decompress_multiple_file_fn)) ? 1 : -1];
int FIO_rust_decompressMultipleFilenames(
const FIO_rust_decompress_multiple_projection_t* projection);
typedef struct {
const FIO_rust_decompress_multiple_projection_t* files;
int destinationEnabled;
FIO_rust_multiple_destination_warning_fn warning;
FIO_rust_multiple_destination_open_fn openDestination;
FIO_rust_multiple_destination_attach_fn attachDestination;
FIO_rust_multiple_destination_close_fn closeDestination;
} FIO_rust_decompress_multiple_destination_projection_t;
typedef char FIO_rust_decompress_multiple_destination_files_offset[
(offsetof(FIO_rust_decompress_multiple_destination_projection_t, files)
== 0) ? 1 : -1];
typedef char FIO_rust_decompress_multiple_destination_enabled_offset[
(offsetof(FIO_rust_decompress_multiple_destination_projection_t, destinationEnabled)
== sizeof(void*)) ? 1 : -1];
typedef char FIO_rust_decompress_multiple_destination_warning_offset[
(offsetof(FIO_rust_decompress_multiple_destination_projection_t, warning)
== 2 * sizeof(void*)) ? 1 : -1];
typedef char FIO_rust_decompress_multiple_destination_open_offset[
(offsetof(FIO_rust_decompress_multiple_destination_projection_t, openDestination)
== 3 * sizeof(void*)) ? 1 : -1];
typedef char FIO_rust_decompress_multiple_destination_attach_offset[
(offsetof(FIO_rust_decompress_multiple_destination_projection_t, attachDestination)
== 4 * sizeof(void*)) ? 1 : -1];
typedef char FIO_rust_decompress_multiple_destination_close_offset[
(offsetof(FIO_rust_decompress_multiple_destination_projection_t, closeDestination)
== 5 * sizeof(void*)) ? 1 : -1];
typedef char FIO_rust_decompress_multiple_destination_int_size[
(sizeof(int) <= sizeof(void*)) ? 1 : -1];
typedef char FIO_rust_decompress_multiple_destination_projection_size[
(sizeof(FIO_rust_decompress_multiple_destination_projection_t)
== 6 * sizeof(void*)) ? 1 : -1];
int FIO_rust_decompressMultipleFilenamesWithDestination(
const FIO_rust_decompress_multiple_destination_projection_t* projection);
typedef int (*FIO_rust_decompress_multiple_separate_file_fn)(
void* opaque, const char* srcFileName);
typedef struct {
void* fCtx;
const char** srcNamesTable;
void* opaque;
FIO_rust_decompress_multiple_separate_file_fn decompressFile;
} FIO_rust_decompress_multiple_separate_projection_t;
typedef char FIO_rust_decompress_multiple_separate_fctx_offset[
(offsetof(FIO_rust_decompress_multiple_separate_projection_t, fCtx) == 0) ? 1 : -1];
typedef char FIO_rust_decompress_multiple_separate_input_names_offset[
(offsetof(FIO_rust_decompress_multiple_separate_projection_t, srcNamesTable)
== sizeof(void*)) ? 1 : -1];
typedef char FIO_rust_decompress_multiple_separate_opaque_offset[
(offsetof(FIO_rust_decompress_multiple_separate_projection_t, opaque)
== 2 * sizeof(void*)) ? 1 : -1];
typedef char FIO_rust_decompress_multiple_separate_callback_offset[
(offsetof(FIO_rust_decompress_multiple_separate_projection_t, decompressFile)
== 3 * sizeof(void*)) ? 1 : -1];
typedef char FIO_rust_decompress_multiple_separate_projection_size[
(sizeof(FIO_rust_decompress_multiple_separate_projection_t)
== 3 * sizeof(void*)
+ sizeof(FIO_rust_decompress_multiple_separate_file_fn)) ? 1 : -1];
int FIO_rust_decompressMultipleSeparateFilenames(
const FIO_rust_decompress_multiple_separate_projection_t* projection);
enum {
FIO_RUST_DECOMPRESS_SRC_OPEN_OK = 0,
};
typedef int (*FIO_rust_decompress_src_open_fn)(void* opaque, const char* srcFileName,
U64* fileSize, int* sourceIsRegular);
typedef void (*FIO_rust_decompress_src_async_fn)(void* opaque, int asyncMode);
typedef void (*FIO_rust_decompress_src_attach_fn)(void* opaque);
typedef int (*FIO_rust_decompress_src_close_fn)(void* opaque);
typedef int (*FIO_rust_decompress_dst_open_fn)(void* opaque,
const char* srcFileName,
const char* dstFileName,
int transferStat, int* dstFd);
typedef void (*FIO_rust_decompress_dst_attach_fn)(void* opaque);
typedef void (*FIO_rust_decompress_handler_fn)(void* opaque);
typedef int (*FIO_rust_decompress_file_fn)(void* opaque,
const char* dstFileName,
const char* srcFileName);
typedef void (*FIO_rust_decompress_dst_stat_fn)(void* opaque, int dstFd,
const char* dstFileName);
typedef int (*FIO_rust_decompress_dst_close_fn)(void* opaque);
typedef void (*FIO_rust_decompress_dst_remove_fn)(void* opaque, const char* dstFileName);
typedef int (*FIO_rust_decompress_src_remove_fn)(void* opaque, const char* srcFileName);
typedef int (*FIO_rust_decompress_src_directory_fn)(void* opaque,
const char* srcFileName);
/* Rust owns per-file source/directory ordering and destination ordering; the
* callback context keeps dRess_t, FILE/pool handles, diagnostics, and the
* format dispatcher private. */
typedef struct {
void* opaque;
const char* dstFileName;
const char* srcFileName;
int destinationAlreadyOpen;
int testMode;
int sourceIsStdin;
int destinationIsStdout;
int removeSource;
FIO_rust_decompress_src_open_fn openSource;
FIO_rust_decompress_src_async_fn setAsync;
FIO_rust_decompress_src_attach_fn attachSource;
FIO_rust_decompress_src_attach_fn detachSource;
FIO_rust_decompress_src_close_fn closeSource;
FIO_rust_decompress_dst_open_fn openDestination;
FIO_rust_decompress_dst_attach_fn attachDestination;
FIO_rust_decompress_handler_fn addHandler;
FIO_rust_decompress_file_fn decompress;
FIO_rust_decompress_handler_fn clearHandler;
FIO_rust_decompress_dst_stat_fn setFDStat;
FIO_rust_decompress_dst_close_fn closeDestination;
FIO_rust_decompress_handler_fn utimeDestination;
FIO_rust_decompress_dst_remove_fn removeDestination;
FIO_rust_decompress_src_remove_fn removeSourceFile;
FIO_rust_decompress_src_directory_fn sourceIsDirectory;
} FIO_rust_decompress_file_projection_t;
typedef char FIO_rust_decompress_file_opaque_offset[
(offsetof(FIO_rust_decompress_file_projection_t, opaque) == 0) ? 1 : -1];
typedef char FIO_rust_decompress_file_dst_name_offset[
(offsetof(FIO_rust_decompress_file_projection_t, dstFileName)
== sizeof(void*)) ? 1 : -1];
typedef char FIO_rust_decompress_file_src_name_offset[
(offsetof(FIO_rust_decompress_file_projection_t, srcFileName)
== 2 * sizeof(void*)) ? 1 : -1];
typedef char FIO_rust_decompress_file_destination_open_offset[
(offsetof(FIO_rust_decompress_file_projection_t, destinationAlreadyOpen)
== 3 * sizeof(void*)) ? 1 : -1];
typedef char FIO_rust_decompress_file_test_mode_offset[
(offsetof(FIO_rust_decompress_file_projection_t, testMode)
== 3 * sizeof(void*) + sizeof(int)) ? 1 : -1];
typedef char FIO_rust_decompress_file_stdin_offset[
(offsetof(FIO_rust_decompress_file_projection_t, sourceIsStdin)
== 3 * sizeof(void*) + 2 * sizeof(int)) ? 1 : -1];
typedef char FIO_rust_decompress_file_stdout_offset[
(offsetof(FIO_rust_decompress_file_projection_t, destinationIsStdout)
== 3 * sizeof(void*) + 3 * sizeof(int)) ? 1 : -1];
typedef char FIO_rust_decompress_file_remove_source_offset[
(offsetof(FIO_rust_decompress_file_projection_t, removeSource)
== 3 * sizeof(void*) + 4 * sizeof(int)) ? 1 : -1];
typedef char FIO_rust_decompress_file_callback_offset[
(offsetof(FIO_rust_decompress_file_projection_t, openSource)
== ((3 * sizeof(void*) + 5 * sizeof(int) + sizeof(void*) - 1)
/ sizeof(void*)) * sizeof(void*)) ? 1 : -1];
typedef char FIO_rust_decompress_file_source_directory_offset[
(offsetof(FIO_rust_decompress_file_projection_t, sourceIsDirectory)
== ((3 * sizeof(void*) + 5 * sizeof(int) + sizeof(void*) - 1)
/ sizeof(void*)) * sizeof(void*)
+ 15 * sizeof(void*)) ? 1 : -1];
typedef char FIO_rust_decompress_file_projection_size[
(sizeof(FIO_rust_decompress_file_projection_t)
== ((3 * sizeof(void*) + 5 * sizeof(int) + sizeof(void*) - 1)
/ sizeof(void*)) * sizeof(void*)
+ 16 * sizeof(FIO_rust_decompress_src_open_fn)) ? 1 : -1];
int FIO_rust_decompressFilename(
const FIO_rust_decompress_file_projection_t* projection);
#endif /* #ifndef ZSTD_NODECOMPRESS */
#ifndef ZSTD_NOCOMPRESS
enum {
FIO_RUST_ZSTD_OK = 0,
FIO_RUST_ZSTD_COMPRESS_ERROR = 1,
FIO_RUST_ZSTD_INCOMPLETE_INPUT = 2,
FIO_RUST_ZSTD_INVALID_PROJECTION = 3
};
enum {
FIO_RUST_ZSTD_DIAGNOSTIC_OK = 0,
FIO_RUST_ZSTD_DIAGNOSTIC_COMPRESS_ERROR = 1,
FIO_RUST_ZSTD_DIAGNOSTIC_INCOMPLETE_INPUT = 2,
FIO_RUST_ZSTD_DIAGNOSTIC_INVALID_PROJECTION = 3,
FIO_RUST_ZSTD_DIAGNOSTIC_UNKNOWN = 4
};
int FIO_rust_zstdCompressionDiagnostic(int status);
typedef size_t (*FIO_rust_zstd_read_fill_fn)(
void* opaque, size_t requested,
const unsigned char** buffer, size_t* loaded);
typedef void (*FIO_rust_zstd_read_consume_fn)(void* opaque, size_t consumed);
typedef void (*FIO_rust_zstd_write_acquire_fn)(
void* opaque, void** job, unsigned char** buffer, size_t* bufferSize);
typedef void (*FIO_rust_zstd_write_enqueue_fn)(
void* opaque, void** job, size_t usedBufferSize,
unsigned char** buffer, size_t* bufferSize);
typedef void (*FIO_rust_zstd_write_release_fn)(void* opaque, void* job);
typedef void (*FIO_rust_zstd_sparse_write_end_fn)(void* opaque);
typedef int (*FIO_rust_zstd_compress_stream_fn)(
void* opaque, const char* srcFileName, int directive,
const unsigned char* input, size_t inputSize, size_t inputPos,
unsigned char* output, size_t outputSize,
size_t* inputPosAfter, size_t* outputProduced,
size_t* toFlushNow, size_t* zstdResult);
typedef void (*FIO_rust_zstd_compress_display_fn)(
int directive, size_t inputPos, size_t inputSize, size_t outputProduced);
int FIO_rust_zstd_compressStream(
void* opaque, const char* srcFileName, int directive,
const unsigned char* input, size_t inputSize, size_t inputPos,
unsigned char* output, size_t outputSize,
size_t* inputPosAfter, size_t* outputProduced,
size_t* toFlushNow, size_t* zstdResult);
void FIO_rust_zstd_compressStreamDisplay(
int directive, size_t inputPos, size_t inputSize, size_t outputProduced);
typedef void (*FIO_rust_zstd_iteration_fn)(
void* opaque, const char* srcFileName, int* compressionLevel,
size_t oldInputPos, size_t newInputPos, size_t toFlushNow);
typedef struct FIO_rust_zstd_adapt_projection_s
FIO_rust_zstd_adapt_projection_t;
typedef struct {
U64 ingested;
U64 consumed;
U64 produced;
U64 flushed;
unsigned currentJobID;
unsigned nbActiveWorkers;
} FIO_rust_zstd_progression_t;
typedef char FIO_rust_zstd_progression_layout[
(offsetof(FIO_rust_zstd_progression_t, ingested) == 0
&& offsetof(FIO_rust_zstd_progression_t, consumed) == sizeof(U64)
&& offsetof(FIO_rust_zstd_progression_t, produced) == 2 * sizeof(U64)
&& offsetof(FIO_rust_zstd_progression_t, flushed) == 3 * sizeof(U64)
&& offsetof(FIO_rust_zstd_progression_t, currentJobID) == 4 * sizeof(U64)
&& offsetof(FIO_rust_zstd_progression_t, nbActiveWorkers)
== 4 * sizeof(U64) + sizeof(unsigned)
&& sizeof(FIO_rust_zstd_progression_t)
== 4 * sizeof(U64) + 2 * sizeof(unsigned)) ? 1 : -1];
typedef void (*FIO_rust_zstd_adaptive_progression_fn)(
void* opaque, FIO_rust_zstd_progression_t* progression);
typedef void (*FIO_rust_zstd_adaptive_set_parameter_fn)(
void* opaque, int compressionLevel);
typedef void (*FIO_rust_zstd_adaptive_diagnostic_fn)(
void* opaque, int diagnostic,
const FIO_rust_zstd_adapt_projection_t* projection);
typedef struct {
void* readOpaque;
void* writeOpaque;
void* codecOpaque;
void* policyOpaque;
size_t readBufferSize;
FIO_rust_zstd_read_fill_fn readFill;
FIO_rust_zstd_read_consume_fn readConsume;
FIO_rust_zstd_write_acquire_fn writeAcquire;
FIO_rust_zstd_write_enqueue_fn writeEnqueue;
FIO_rust_zstd_write_release_fn writeRelease;
FIO_rust_zstd_sparse_write_end_fn sparseWriteEnd;
FIO_rust_zstd_compress_stream_fn compressStream;
FIO_rust_zstd_iteration_fn iteration;
FIO_rust_zstd_compress_display_fn compressStreamDisplay;
int adaptiveMode;
int nbWorkers;
int minAdaptLevel;
int maxAdaptLevel;
int maxCLevel;
FIO_rust_zstd_adaptive_progression_fn adaptiveProgression;
FIO_rust_zstd_adaptive_set_parameter_fn adaptiveSetParameter;
FIO_rust_zstd_adaptive_diagnostic_fn adaptiveDiagnostic;
} FIO_rust_zstd_compress_projection_t;
typedef char FIO_rust_zstd_compress_projection_layout[
(offsetof(FIO_rust_zstd_compress_projection_t, adaptiveMode)
== 14 * sizeof(void*)
&& offsetof(FIO_rust_zstd_compress_projection_t, nbWorkers)
== 14 * sizeof(void*) + sizeof(int)
&& offsetof(FIO_rust_zstd_compress_projection_t, maxCLevel)
== 14 * sizeof(void*) + 4 * sizeof(int)
&& offsetof(FIO_rust_zstd_compress_projection_t, adaptiveProgression)
== ((14 * sizeof(void*) + 5 * sizeof(int) + sizeof(void*) - 1)
/ sizeof(void*)) * sizeof(void*)
&& offsetof(FIO_rust_zstd_compress_projection_t, adaptiveSetParameter)
== ((14 * sizeof(void*) + 5 * sizeof(int) + sizeof(void*) - 1)
/ sizeof(void*)) * sizeof(void*) + sizeof(void*)
&& offsetof(FIO_rust_zstd_compress_projection_t, adaptiveDiagnostic)
== ((14 * sizeof(void*) + 5 * sizeof(int) + sizeof(void*) - 1)
/ sizeof(void*)) * sizeof(void*) + 2 * sizeof(void*)
&& sizeof(FIO_rust_zstd_compress_projection_t)
== ((14 * sizeof(void*) + 5 * sizeof(int) + sizeof(void*) - 1)
/ sizeof(void*)) * sizeof(void*) + 3 * sizeof(void*))
? 1 : -1];
int FIO_rust_compressZstdFrame(
const FIO_rust_zstd_compress_projection_t* projection,
const char* srcFileName, U64 srcFileSize, int compressionLevel,
U64* readsize, U64* compressedSize, size_t* zstdResult);
/* Select the content size to pledge for the next zstd frame. Rust owns the
* source-size precedence policy; C retains the private CCtx call and error
* handling in FIO_rust_compressZstdCallback(). */
U64 FIO_rust_selectPledgedSrcSize(
U64 srcFileSize, U64 streamSrcSize,
U64 unknownSrcFileSize, U64 unknownPledgedSrcSize);
enum {
FIO_RUST_GZIP_OK = 0,
FIO_RUST_GZIP_INIT_ERROR = 1,
FIO_RUST_GZIP_DEFLATE_ERROR = 2,
FIO_RUST_GZIP_FINISH_ERROR = 3,
FIO_RUST_GZIP_END_ERROR = 4,
FIO_RUST_GZIP_INVALID_PROJECTION = 5
};
enum {
FIO_RUST_GZIP_DIAGNOSTIC_OK = 0,
FIO_RUST_GZIP_DIAGNOSTIC_INIT_ERROR = 1,
FIO_RUST_GZIP_DIAGNOSTIC_DEFLATE_ERROR = 2,
FIO_RUST_GZIP_DIAGNOSTIC_FINISH_ERROR = 3,
FIO_RUST_GZIP_DIAGNOSTIC_END_ERROR = 4,
FIO_RUST_GZIP_DIAGNOSTIC_INVALID_PROJECTION = 5,
FIO_RUST_GZIP_DIAGNOSTIC_UNKNOWN = 6
};
int FIO_rust_gzipCompressionDiagnostic(int status);
typedef void (*FIO_rust_gzip_read_fill_fn)(
void* opaque, size_t requested, const unsigned char** buffer, size_t* loaded);
typedef void (*FIO_rust_gzip_read_consume_fn)(void* opaque, size_t consumed);
typedef void (*FIO_rust_gzip_write_acquire_fn)(
void* opaque, void** job, unsigned char** buffer, size_t* bufferSize);
typedef void (*FIO_rust_gzip_write_enqueue_fn)(
void* opaque, void** job, size_t usedBufferSize,
unsigned char** buffer, size_t* bufferSize);
typedef void (*FIO_rust_gzip_write_release_fn)(void* opaque, void* job);
typedef void (*FIO_rust_gzip_sparse_write_end_fn)(void* opaque);
typedef int (*FIO_rust_gzip_zlib_init_fn)(void* opaque, int compressionLevel);
typedef int (*FIO_rust_gzip_zlib_deflate_fn)(
void* opaque, const unsigned char* input, size_t inputSize,
unsigned char* output, size_t outputSize, int flush,
size_t* consumed, size_t* produced);
typedef int (*FIO_rust_gzip_zlib_end_fn)(void* opaque);
typedef void (*FIO_rust_gzip_progress_fn)(
void* opaque, U64 srcFileSize, U64 inFileSize, U64 outFileSize);
typedef struct {
void* readOpaque;
void* writeOpaque;
void* zlibOpaque;
void* progressOpaque;
size_t readBufferSize;
FIO_rust_gzip_read_fill_fn readFill;
FIO_rust_gzip_read_consume_fn readConsume;
FIO_rust_gzip_write_acquire_fn writeAcquire;
FIO_rust_gzip_write_enqueue_fn writeEnqueue;
FIO_rust_gzip_write_release_fn writeRelease;
FIO_rust_gzip_sparse_write_end_fn sparseWriteEnd;
FIO_rust_gzip_zlib_init_fn zlibInit;
FIO_rust_gzip_zlib_deflate_fn zlibDeflate;
FIO_rust_gzip_zlib_end_fn zlibEnd;
FIO_rust_gzip_progress_fn progress;
} FIO_rust_gzip_compress_projection_t;
int FIO_rust_compressGzipFrame(
const FIO_rust_gzip_compress_projection_t* projection,
const char* srcFileName, U64 srcFileSize, int compressionLevel,
U64* readsize, U64* compressedSize, int* zlibResult);
enum {
FIO_RUST_LZMA_OK = 0,
FIO_RUST_LZMA_INIT_PRESET_ERROR = 1,
FIO_RUST_LZMA_INIT_ALONE_ERROR = 2,
FIO_RUST_LZMA_INIT_XZ_ERROR = 3,
FIO_RUST_LZMA_CODE_ERROR = 4,
FIO_RUST_LZMA_INVALID_PROJECTION = 5
};
enum {
FIO_RUST_LZMA_DIAGNOSTIC_OK = 0,
FIO_RUST_LZMA_DIAGNOSTIC_INIT_PRESET_ERROR = 1,
FIO_RUST_LZMA_DIAGNOSTIC_INIT_ALONE_ERROR = 2,
FIO_RUST_LZMA_DIAGNOSTIC_INIT_XZ_ERROR = 3,
FIO_RUST_LZMA_DIAGNOSTIC_CODE_ERROR = 4,
FIO_RUST_LZMA_DIAGNOSTIC_INVALID_PROJECTION = 5,
FIO_RUST_LZMA_DIAGNOSTIC_UNKNOWN = 6
};
int FIO_rust_lzmaCompressionDiagnostic(int status);
typedef void (*FIO_rust_lzma_read_fill_fn)(
void* opaque, size_t requested, const unsigned char** buffer, size_t* loaded);
typedef void (*FIO_rust_lzma_read_consume_fn)(void* opaque, size_t consumed);
typedef void (*FIO_rust_lzma_write_acquire_fn)(
void* opaque, void** job, unsigned char** buffer, size_t* bufferSize);
typedef void (*FIO_rust_lzma_write_enqueue_fn)(
void* opaque, void** job, size_t usedBufferSize,
unsigned char** buffer, size_t* bufferSize);
typedef void (*FIO_rust_lzma_write_release_fn)(void* opaque, void* job);
typedef void (*FIO_rust_lzma_sparse_write_end_fn)(void* opaque);
typedef int (*FIO_rust_lzma_init_fn)(
void* opaque, int compressionLevel, int plainLzma, int* lzmaResult);
typedef int (*FIO_rust_lzma_code_fn)(
void* opaque, const unsigned char* input, size_t inputSize,
unsigned char* output, size_t outputSize, int action,
size_t* consumed, size_t* produced);
typedef void (*FIO_rust_lzma_end_fn)(void* opaque);
typedef void (*FIO_rust_lzma_progress_fn)(
void* opaque, U64 srcFileSize, U64 inFileSize, U64 outFileSize);
typedef struct {
void* readOpaque;
void* writeOpaque;
void* lzmaOpaque;
void* progressOpaque;
size_t readBufferSize;
FIO_rust_lzma_read_fill_fn readFill;
FIO_rust_lzma_read_consume_fn readConsume;
FIO_rust_lzma_write_acquire_fn writeAcquire;
FIO_rust_lzma_write_enqueue_fn writeEnqueue;
FIO_rust_lzma_write_release_fn writeRelease;
FIO_rust_lzma_sparse_write_end_fn sparseWriteEnd;
FIO_rust_lzma_init_fn lzmaInit;
FIO_rust_lzma_code_fn lzmaCode;
FIO_rust_lzma_end_fn lzmaEnd;
FIO_rust_lzma_progress_fn progress;
} FIO_rust_lzma_compress_projection_t;
int FIO_rust_compressLzmaFrame(
const FIO_rust_lzma_compress_projection_t* projection,
const char* srcFileName, U64 srcFileSize, int compressionLevel,
int plainLzma, U64* readsize, U64* compressedSize, int* lzmaResult);
#ifdef ZSTD_LZ4COMPRESS
enum {
FIO_RUST_LZ4_OK = 0,
FIO_RUST_LZ4_CREATE_ERROR = 1,
FIO_RUST_LZ4_HEADER_ERROR = 2,
FIO_RUST_LZ4_UPDATE_ERROR = 3,
FIO_RUST_LZ4_END_ERROR = 4,
FIO_RUST_LZ4_INVALID_PROJECTION = 5
};
enum {
FIO_RUST_LZ4_DIAGNOSTIC_OK = 0,
FIO_RUST_LZ4_DIAGNOSTIC_CREATE_ERROR = 1,
FIO_RUST_LZ4_DIAGNOSTIC_HEADER_ERROR = 2,
FIO_RUST_LZ4_DIAGNOSTIC_UPDATE_ERROR = 3,
FIO_RUST_LZ4_DIAGNOSTIC_END_ERROR = 4,
FIO_RUST_LZ4_DIAGNOSTIC_INVALID_PROJECTION = 5,
FIO_RUST_LZ4_DIAGNOSTIC_UNKNOWN = 6
};
int FIO_rust_lz4CompressionDiagnostic(int status);
typedef int (*FIO_rust_lz4_create_fn)(void* opaque, unsigned version,
size_t* result);
typedef void (*FIO_rust_lz4_prepare_fn)(void* opaque, U64 srcFileSize,
int compressionLevel, int checksumFlag,
size_t blockSize, size_t bufferSize);
typedef int (*FIO_rust_lz4_begin_fn)(void* opaque, unsigned char* output,
size_t outputSize, size_t* result);
typedef int (*FIO_rust_lz4_update_fn)(void* opaque, unsigned char* output,
size_t outputSize, const unsigned char* input,
size_t inputSize, size_t* result);
typedef int (*FIO_rust_lz4_end_fn)(void* opaque, unsigned char* output,
size_t outputSize, size_t* result);
typedef void (*FIO_rust_lz4_free_fn)(void* opaque);
typedef size_t (*FIO_rust_lz4_read_fill_fn)(void* opaque, size_t requested,
const unsigned char** buffer,
size_t* loaded);
typedef void (*FIO_rust_lz4_read_consume_fn)(void* opaque, size_t consumed);
typedef void (*FIO_rust_lz4_write_acquire_fn)(void* opaque, void** job,
unsigned char** buffer,
size_t* bufferSize);
typedef void (*FIO_rust_lz4_write_enqueue_fn)(void* opaque, void** job,
size_t usedBufferSize,
unsigned char** buffer,
size_t* bufferSize);
typedef void (*FIO_rust_lz4_write_release_fn)(void* opaque, void* job);
typedef void (*FIO_rust_lz4_sparse_write_end_fn)(void* opaque);
typedef void (*FIO_rust_lz4_progress_fn)(void* opaque, U64 srcFileSize,
U64 inFileSize, U64 outFileSize);
typedef struct {
void* readOpaque;
void* writeOpaque;
void* codecOpaque;
void* progressOpaque;
unsigned version;
size_t blockSize;
FIO_rust_lz4_create_fn create;
FIO_rust_lz4_prepare_fn prepare;
FIO_rust_lz4_begin_fn begin;
FIO_rust_lz4_update_fn update;
FIO_rust_lz4_end_fn end;
FIO_rust_lz4_free_fn freeContext;
FIO_rust_lz4_read_fill_fn readFill;
FIO_rust_lz4_read_consume_fn readConsume;
FIO_rust_lz4_write_acquire_fn writeAcquire;
FIO_rust_lz4_write_enqueue_fn writeEnqueue;
FIO_rust_lz4_write_release_fn writeRelease;
FIO_rust_lz4_sparse_write_end_fn sparseWriteEnd;
FIO_rust_lz4_progress_fn progress;
} FIO_rust_lz4_compress_projection_t;
int FIO_rust_compressLz4Frame(
const FIO_rust_lz4_compress_projection_t* projection,
const char* srcFileName, U64 srcFileSize, int compressionLevel,
int checksumFlag, U64* readsize, U64* compressedSize,
size_t* lz4Result);
#endif
int FIO_rust_compressFilenameInternal(
void* fCtx, FIO_prefs_t* prefs, void* ress,
const char* dstFileName, const char* srcFileName,
U64 fileSize, int compressionLevel,
const FIO_rust_compress_callbacks_t* callbacks);
enum {
FIO_RUST_COMPRESS_DST_OPEN_OK = 0,
};
typedef int (*FIO_rust_compress_dst_open_fn)(void* opaque,
const char* srcFileName,
const char* dstFileName,
int transferStat, int* dstFd);
typedef void (*FIO_rust_compress_dst_handler_fn)(void* opaque);
typedef int (*FIO_rust_compress_dst_file_fn)(void* opaque,
const char* dstFileName,
const char* srcFileName,
int compressionLevel);
typedef void (*FIO_rust_compress_dst_stat_fn)(void* opaque, int dstFd,
const char* dstFileName);
typedef int (*FIO_rust_compress_dst_close_fn)(void* opaque);
typedef void (*FIO_rust_compress_dst_remove_fn)(void* opaque,
const char* dstFileName);
/* Rust owns the per-file destination lifecycle; C retains private resource
* handles, diagnostics, metadata operations, and format dispatch in opaque
* callbacks. */
typedef struct {
void* opaque;
const char* dstFileName;
const char* srcFileName;
int compressionLevel;
int destinationAlreadyOpen;
int sourceIsStdin;
int destinationIsStdout;
int sourceIsRegular;
FIO_rust_compress_dst_open_fn openDestination;
FIO_rust_compress_dst_handler_fn attachDestination;
FIO_rust_compress_dst_handler_fn addHandler;
FIO_rust_compress_dst_file_fn compress;
FIO_rust_compress_dst_handler_fn clearHandler;
FIO_rust_compress_dst_stat_fn setFDStat;
FIO_rust_compress_dst_close_fn closeDestination;
FIO_rust_compress_dst_handler_fn utimeDestination;
FIO_rust_compress_dst_remove_fn removeDestination;
} FIO_rust_compress_dst_projection_t;
typedef char FIO_rust_compress_dst_opaque_offset[
(offsetof(FIO_rust_compress_dst_projection_t, opaque) == 0) ? 1 : -1];
typedef char FIO_rust_compress_dst_dst_name_offset[
(offsetof(FIO_rust_compress_dst_projection_t, dstFileName)
== sizeof(void*)) ? 1 : -1];
typedef char FIO_rust_compress_dst_src_name_offset[
(offsetof(FIO_rust_compress_dst_projection_t, srcFileName)
== 2 * sizeof(void*)) ? 1 : -1];
typedef char FIO_rust_compress_dst_level_offset[
(offsetof(FIO_rust_compress_dst_projection_t, compressionLevel)
== 3 * sizeof(void*)) ? 1 : -1];
typedef char FIO_rust_compress_dst_opened_offset[
(offsetof(FIO_rust_compress_dst_projection_t, destinationAlreadyOpen)
== 3 * sizeof(void*) + sizeof(int)) ? 1 : -1];
typedef char FIO_rust_compress_dst_stdin_offset[
(offsetof(FIO_rust_compress_dst_projection_t, sourceIsStdin)
== 3 * sizeof(void*) + 2 * sizeof(int)) ? 1 : -1];
typedef char FIO_rust_compress_dst_stdout_offset[
(offsetof(FIO_rust_compress_dst_projection_t, destinationIsStdout)
== 3 * sizeof(void*) + 3 * sizeof(int)) ? 1 : -1];
typedef char FIO_rust_compress_dst_regular_offset[
(offsetof(FIO_rust_compress_dst_projection_t, sourceIsRegular)
== 3 * sizeof(void*) + 4 * sizeof(int)) ? 1 : -1];
typedef char FIO_rust_compress_dst_callback_offset[
(offsetof(FIO_rust_compress_dst_projection_t, openDestination)
== ((3 * sizeof(void*) + 5 * sizeof(int) + sizeof(void*) - 1)
/ sizeof(void*)) * sizeof(void*)) ? 1 : -1];
typedef char FIO_rust_compress_dst_projection_size[
(sizeof(FIO_rust_compress_dst_projection_t)
== ((3 * sizeof(void*) + 5 * sizeof(int) + sizeof(void*) - 1)
/ sizeof(void*)) * sizeof(void*)
+ 9 * sizeof(FIO_rust_compress_dst_open_fn)) ? 1 : -1];
int FIO_rust_compressFilenameDstFile(
const FIO_rust_compress_dst_projection_t* projection);
enum {
FIO_RUST_COMPRESS_SRC_STAT_FAILED = 0,
FIO_RUST_COMPRESS_SRC_STAT_OK = 1,
FIO_RUST_COMPRESS_SRC_DIRECTORY = 2,
FIO_RUST_COMPRESS_SRC_DICT_COLLISION = 3,
FIO_RUST_COMPRESS_SRC_OPEN_OK = 0,
};
typedef int (*FIO_rust_compress_src_stat_fn)(void* opaque, const char* srcFileName);
typedef int (*FIO_rust_compress_src_excluded_fn)(void* opaque, const char* srcFileName);
typedef int (*FIO_rust_compress_src_open_fn)(void* opaque, const char* srcFileName,
U64* fileSize);
typedef void (*FIO_rust_compress_src_async_fn)(void* opaque, int asyncMode);
typedef void (*FIO_rust_compress_src_attach_fn)(void* opaque);
typedef int (*FIO_rust_compress_src_compress_fn)(void* opaque,
const char* dstFileName,
const char* srcFileName,
int compressionLevel);
typedef void (*FIO_rust_compress_src_close_fn)(void* opaque);
typedef void (*FIO_rust_compress_src_remove_fn)(void* opaque, const char* srcFileName);
/* Rust owns the source policy and exclusion diagnostic; this projection
* exposes only scalar names and opaque callbacks for C-owned operations.
* FIO_ctx_t, cRess_t, stat_t, and all resource state remain private to the
* callback context below. */
typedef struct {
void* opaque;
const char* dstFileName;
const char* srcFileName;
int compressionLevel;
int sourceIsStdin;
int excludeCompressedFiles;
int removeSrcFile;
FIO_rust_compress_src_stat_fn statSource;
FIO_rust_compress_src_excluded_fn sourceIsExcluded;
FIO_rust_compress_src_open_fn openSource;
FIO_rust_compress_src_async_fn setAsync;
FIO_rust_compress_src_attach_fn attachSource;
FIO_rust_compress_src_compress_fn compress;
FIO_rust_compress_src_close_fn closeSource;
FIO_rust_compress_src_remove_fn removeSource;
} FIO_rust_compress_src_projection_t;
typedef char FIO_rust_compress_src_opaque_offset[
(offsetof(FIO_rust_compress_src_projection_t, opaque) == 0) ? 1 : -1];
typedef char FIO_rust_compress_src_dst_name_offset[
(offsetof(FIO_rust_compress_src_projection_t, dstFileName)
== sizeof(void*)) ? 1 : -1];
typedef char FIO_rust_compress_src_src_name_offset[
(offsetof(FIO_rust_compress_src_projection_t, srcFileName)
== 2 * sizeof(void*)) ? 1 : -1];
typedef char FIO_rust_compress_src_level_offset[
(offsetof(FIO_rust_compress_src_projection_t, compressionLevel)
== 3 * sizeof(void*)) ? 1 : -1];
typedef char FIO_rust_compress_src_stdin_offset[
(offsetof(FIO_rust_compress_src_projection_t, sourceIsStdin)
== 3 * sizeof(void*) + sizeof(int)) ? 1 : -1];
typedef char FIO_rust_compress_src_exclude_offset[
(offsetof(FIO_rust_compress_src_projection_t, excludeCompressedFiles)
== 3 * sizeof(void*) + 2 * sizeof(int)) ? 1 : -1];
typedef char FIO_rust_compress_src_remove_offset[
(offsetof(FIO_rust_compress_src_projection_t, removeSrcFile)
== 3 * sizeof(void*) + 3 * sizeof(int)) ? 1 : -1];
typedef char FIO_rust_compress_src_stat_callback_offset[
(offsetof(FIO_rust_compress_src_projection_t, statSource)
== 3 * sizeof(void*) + 4 * sizeof(int)) ? 1 : -1];
typedef char FIO_rust_compress_src_projection_size[
(sizeof(FIO_rust_compress_src_projection_t)
== 3 * sizeof(void*) + 4 * sizeof(int)
+ 8 * sizeof(FIO_rust_compress_src_stat_fn)) ? 1 : -1];
int FIO_rust_compressFilenameSrcFile(
const FIO_rust_compress_src_projection_t* projection);
int FIO_rust_compressSourceIsExcluded(void* opaque, const char* srcFileName);
static void FIO_adjustParamsForPatchFromMode(FIO_prefs_t* const prefs,
ZSTD_compressionParameters* comprParams,
unsigned long long const dictSize,
unsigned long long const maxSrcFileSize,
int cLevel)
{
enum {
FIO_PATCH_MEM_LIMIT_SUCCESS = 0,
FIO_PATCH_MEM_LIMIT_UNKNOWN_SIZE = 1,
FIO_PATCH_MEM_LIMIT_TOO_LARGE = 2
};
ZSTD_compressionParameters const cParams = ZSTD_getCParams(cLevel, (size_t)maxSrcFileSize, (size_t)dictSize);
unsigned fileWindowLog;
int autoLdm;
int optimalParser;
int const status = FIO_rust_adjustParamsForPatchFromMode(
prefs, comprParams, dictSize, maxSrcFileSize, cParams,
&fileWindowLog, &autoLdm, &optimalParser);
if (status == FIO_PATCH_MEM_LIMIT_UNKNOWN_SIZE)
EXM_THROW(42, "Using --patch-from with stdin requires --stream-size");
if (status == FIO_PATCH_MEM_LIMIT_TOO_LARGE) {
unsigned const maxWindowSize = (1U << ZSTD_WINDOWLOG_MAX);
EXM_THROW(42, "Can't handle files larger than %u GB\n", maxWindowSize/(1 GB));
}
assert(status == FIO_PATCH_MEM_LIMIT_SUCCESS);
if (fileWindowLog > ZSTD_WINDOWLOG_MAX)
DISPLAYLEVEL(1, "Max window log exceeded by file (compression ratio will suffer)\n");
if (autoLdm)
DISPLAYLEVEL(2, "long mode automatically triggered\n");
if (optimalParser) {
DISPLAYLEVEL(4, "[Optimal parser notes] Consider the following to improve patch size at the cost of speed:\n");
DISPLAYLEVEL(4, "- Set a larger targetLength (e.g. --zstd=targetLength=4096)\n");
DISPLAYLEVEL(4, "- Set a larger chainLog (e.g. --zstd=chainLog=%u)\n", ZSTD_CHAINLOG_MAX);
DISPLAYLEVEL(4, "- Set a larger LDM hashLog (e.g. --zstd=ldmHashLog=%u)\n", ZSTD_LDM_HASHLOG_MAX);
DISPLAYLEVEL(4, "- Set a smaller LDM rateLog (e.g. --zstd=ldmHashRateLog=%u)\n", ZSTD_LDM_HASHRATELOG_MIN);
DISPLAYLEVEL(4, "Also consider playing around with searchLog and hashLog\n");
}
}
typedef size_t (*FIO_rust_createCResources_set_parameter_f)(void* context,
int parameter,
int value);
typedef int (*FIO_rust_createCResources_is_error_f)(size_t result);
typedef void (*FIO_rust_createCResources_display_overlap_f)(int overlapLog);
typedef void (*FIO_rust_createCResources_create_cctx_f)(void* context);
typedef void (*FIO_rust_createCResources_prepare_dictionary_f)(
void* context, const FIO_prefs_t* prefs, const char* dictFileName,
U64 maxSrcFileSize, int cLevel, void* comprParams);
typedef void (*FIO_rust_createCResources_create_pool_f)(
void* context, const FIO_prefs_t* prefs);
typedef void (*FIO_rust_createCResources_validate_dictionary_f)(void* context);
typedef size_t (*FIO_rust_createCResources_load_dictionary_f)(
void* context, int patchFromMode);
typedef struct {
void* callbackContext;
const FIO_prefs_t* prefs;
ZSTD_compressionParameters comprParams;
int cLevel;
const char* dictFileName;
U64 maxSrcFileSize;
int multithreaded;
FIO_rust_createCResources_create_cctx_f createCctx;
FIO_rust_createCResources_prepare_dictionary_f prepareDictionary;
FIO_rust_createCResources_create_pool_f createWritePool;
FIO_rust_createCResources_create_pool_f createReadPool;
FIO_rust_createCResources_validate_dictionary_f validateDictionary;
FIO_rust_createCResources_load_dictionary_f loadDictionary;
FIO_rust_createCResources_set_parameter_f setParameter;
FIO_rust_createCResources_is_error_f isError;
FIO_rust_createCResources_display_overlap_f displayOverlap;
} FIO_rust_createCResourcesState;
typedef char FIO_rust_create_c_resources_state_context_offset[
(offsetof(FIO_rust_createCResourcesState, callbackContext) == 0) ? 1 : -1];
typedef char FIO_rust_create_c_resources_state_prefs_offset[
(offsetof(FIO_rust_createCResourcesState, prefs) == sizeof(void*)) ? 1 : -1];
typedef char FIO_rust_create_c_resources_state_params_offset[
(offsetof(FIO_rust_createCResourcesState, comprParams) == 2 * sizeof(void*)) ? 1 : -1];
typedef char FIO_rust_create_c_resources_state_level_offset[
(offsetof(FIO_rust_createCResourcesState, cLevel)
== 2 * sizeof(void*) + sizeof(ZSTD_compressionParameters)) ? 1 : -1];
typedef char FIO_rust_create_c_resources_state_dict_name_offset[
(offsetof(FIO_rust_createCResourcesState, dictFileName)
== ((2 * sizeof(void*) + sizeof(ZSTD_compressionParameters) + sizeof(int)
+ sizeof(void*) - 1) / sizeof(void*)) * sizeof(void*)) ? 1 : -1];
typedef char FIO_rust_create_c_resources_state_max_src_offset[
(offsetof(FIO_rust_createCResourcesState, maxSrcFileSize)
== offsetof(FIO_rust_createCResourcesState, dictFileName) + sizeof(void*)) ? 1 : -1];
typedef char FIO_rust_create_c_resources_state_mt_offset[
(offsetof(FIO_rust_createCResourcesState, multithreaded)
== offsetof(FIO_rust_createCResourcesState, maxSrcFileSize) + sizeof(U64)) ? 1 : -1];
typedef char FIO_rust_create_c_resources_state_set_parameter_offset[
(offsetof(FIO_rust_createCResourcesState, setParameter)
== ((offsetof(FIO_rust_createCResourcesState, multithreaded) + sizeof(int)
+ sizeof(void*) - 1) / sizeof(void*)) * sizeof(void*)
+ 6 * sizeof(void*)) ? 1 : -1];
typedef char FIO_rust_create_c_resources_state_is_error_offset[
(offsetof(FIO_rust_createCResourcesState, isError)
== offsetof(FIO_rust_createCResourcesState, setParameter) + sizeof(void*)) ? 1 : -1];
typedef char FIO_rust_create_c_resources_state_display_overlap_offset[
(offsetof(FIO_rust_createCResourcesState, displayOverlap)
== offsetof(FIO_rust_createCResourcesState, isError) + sizeof(void*)) ? 1 : -1];
typedef char FIO_rust_create_c_resources_state_size[
(sizeof(FIO_rust_createCResourcesState)
== offsetof(FIO_rust_createCResourcesState, displayOverlap) + sizeof(void*)) ? 1 : -1];
size_t FIO_rust_createCResources(const FIO_rust_createCResourcesState* state);
static size_t FIO_rust_createCResources_setParameter(void* context,
int parameter,
int value)
{
cRess_t* const ress = (cRess_t*)context;
if (parameter == ZSTD_c_nbWorkers)
DISPLAYLEVEL(5, "set nb workers = %u \n", (unsigned)value);
return ZSTD_CCtx_setParameter(ress->cctx, (ZSTD_cParameter)parameter, value);
}
static int FIO_rust_createCResources_isError(size_t result)
{
return ZSTD_isError(result);
}
static void FIO_rust_createCResources_displayOverlap(int overlapLog)
{
DISPLAYLEVEL(3,"set overlapLog = %u \n", overlapLog);
}
static void FIO_rust_createCResources_createCctx(void* context)
{
cRess_t* const ress = (cRess_t*)context;
ress->cctx = ZSTD_createCCtx();
if (ress->cctx == NULL)
EXM_THROW(30, "allocation error (%s): can't create ZSTD_CCtx",
strerror(errno));
}
static void FIO_rust_createCResources_prepareDictionary(
void* context, const FIO_prefs_t* prefs, const char* dictFileName,
U64 maxSrcFileSize, int cLevel, void* comprParams)
{
cRess_t* const ress = (cRess_t*)context;
ZSTD_compressionParameters* const params =
(ZSTD_compressionParameters*)comprParams;
unsigned long long dictSize = 0;
unsigned long long ssSize = 0;
FIO_dictBufferType_t dictBufferType;
FIO_getDictFileStat(dictFileName, &ress->dictFileStat);
/* need to update memLimit before calling createDictBuffer
* because of memLimit check inside it */
if (prefs->patchFromMode) {
dictSize = (unsigned long long)UTIL_getFileSizeStat(&ress->dictFileStat);
ssSize = (unsigned long long)prefs->streamSrcSize;
FIO_adjustParamsForPatchFromMode(
(FIO_prefs_t*)prefs, params, dictSize,
ssSize > 0 ? ssSize : maxSrcFileSize, cLevel);
}
dictBufferType = (FIO_dictBufferType_t)FIO_rust_selectDictBufferType(
prefs->mmapDict, prefs->patchFromMode, dictSize, prefs->memLimit);
FIO_initDict(&ress->dict, dictFileName, (FIO_prefs_t*)prefs,
&ress->dictFileStat, dictBufferType);
ress->dictFileName = dictFileName;
}
static void FIO_rust_createCResources_createWritePool(
void* context, const FIO_prefs_t* prefs)
{
cRess_t* const ress = (cRess_t*)context;
ress->writeCtx = AIO_WritePool_create(prefs, ZSTD_CStreamOutSize());
}
static void FIO_rust_createCResources_createReadPool(
void* context, const FIO_prefs_t* prefs)
{
cRess_t* const ress = (cRess_t*)context;
ress->readCtx = AIO_ReadPool_create(prefs, ZSTD_CStreamInSize());
}
static void FIO_rust_createCResources_validateDictionary(void* context)
{
cRess_t* const ress = (cRess_t*)context;
if (ress->dictFileName && (ress->dict.dictBuffer == NULL))
EXM_THROW(32, "allocation error : can't create dictBuffer");
}
static size_t FIO_rust_createCResources_loadDictionary(
void* context, int patchFromMode)
{
cRess_t* const ress = (cRess_t*)context;
if (patchFromMode)
return ZSTD_CCtx_refPrefix(
ress->cctx, ress->dict.dictBuffer, ress->dict.dictBufferSize);
return ZSTD_CCtx_loadDictionary_byReference(
ress->cctx, ress->dict.dictBuffer, ress->dict.dictBufferSize);
}
static cRess_t FIO_createCResources(FIO_prefs_t* const prefs,
const char* dictFileName, unsigned long long const maxSrcFileSize,
int cLevel, ZSTD_compressionParameters comprParams) {
cRess_t ress;
FIO_rust_createCResourcesState policy;
memset(&ress, 0, sizeof(ress));
DISPLAYLEVEL(6, "FIO_createCResources \n");
policy.callbackContext = &ress;
policy.prefs = prefs;
policy.comprParams = comprParams;
policy.cLevel = cLevel;
policy.dictFileName = dictFileName;
policy.maxSrcFileSize = maxSrcFileSize;
#ifdef ZSTD_MULTITHREAD
policy.multithreaded = 1;
#else
policy.multithreaded = 0;
#endif
policy.createCctx = FIO_rust_createCResources_createCctx;
policy.prepareDictionary = FIO_rust_createCResources_prepareDictionary;
policy.createWritePool = FIO_rust_createCResources_createWritePool;
policy.createReadPool = FIO_rust_createCResources_createReadPool;
policy.validateDictionary = FIO_rust_createCResources_validateDictionary;
policy.loadDictionary = FIO_rust_createCResources_loadDictionary;
policy.setParameter = FIO_rust_createCResources_setParameter;
policy.isError = FIO_rust_createCResources_isError;
policy.displayOverlap = FIO_rust_createCResources_displayOverlap;
CHECK( FIO_rust_createCResources(&policy) );
return ress;
}
typedef void (*FIO_rust_freeCResourcesCallback_f)(void* context);
typedef struct {
void* callbackContext;
FIO_rust_freeCResourcesCallback_f freeDict;
FIO_rust_freeCResourcesCallback_f freeWritePool;
FIO_rust_freeCResourcesCallback_f freeReadPool;
FIO_rust_freeCResourcesCallback_f freeCctx;
} FIO_rust_freeCResourcesState;
typedef char FIO_rust_free_c_resources_state_layout[
(offsetof(FIO_rust_freeCResourcesState, callbackContext) == 0
&& offsetof(FIO_rust_freeCResourcesState, freeDict) == sizeof(void*)
&& offsetof(FIO_rust_freeCResourcesState, freeWritePool)
== 2 * sizeof(void*)
&& offsetof(FIO_rust_freeCResourcesState, freeReadPool)
== 3 * sizeof(void*)
&& offsetof(FIO_rust_freeCResourcesState, freeCctx)
== 4 * sizeof(void*)
&& sizeof(FIO_rust_freeCResourcesState) == 5 * sizeof(void*))
? 1 : -1];
void FIO_rust_freeCResources(const FIO_rust_freeCResourcesState* state);
static void FIO_rust_freeCResources_dict(void* context)
{
cRess_t* const ress = (cRess_t*)context;
FIO_freeDict(&(ress->dict));
}
static void FIO_rust_freeCResources_writePool(void* context)
{
cRess_t* const ress = (cRess_t*)context;
AIO_WritePool_free(ress->writeCtx);
}
static void FIO_rust_freeCResources_readPool(void* context)
{
cRess_t* const ress = (cRess_t*)context;
AIO_ReadPool_free(ress->readCtx);
}
static void FIO_rust_freeCResources_cctx(void* context)
{
cRess_t* const ress = (cRess_t*)context;
ZSTD_freeCStream(ress->cctx); /* never fails */
}
static void FIO_freeCResources(cRess_t* const ress)
{
FIO_rust_freeCResourcesState const state = {
ress,
FIO_rust_freeCResources_dict,
FIO_rust_freeCResources_writePool,
FIO_rust_freeCResources_readPool,
FIO_rust_freeCResources_cctx
};
FIO_rust_freeCResources(&state);
}
#ifdef ZSTD_GZCOMPRESS
static void FIO_rust_gzip_readFill(void* opaque, size_t requested,
const unsigned char** buffer, size_t* loaded)
{
ReadPoolCtx_t* const readCtx = (ReadPoolCtx_t*)opaque;
AIO_ReadPool_fillBuffer(readCtx, requested);
*buffer = readCtx->srcBuffer;
*loaded = readCtx->srcBufferLoaded;
}
static void FIO_rust_gzip_readConsume(void* opaque, size_t consumed)
{
AIO_ReadPool_consumeBytes((ReadPoolCtx_t*)opaque, consumed);
}
static void FIO_rust_gzip_writeAcquire(void* opaque, void** job,
unsigned char** buffer, size_t* bufferSize)
{
IOJob_t* const writeJob = AIO_WritePool_acquireJob((WritePoolCtx_t*)opaque);
*job = writeJob;
*buffer = (unsigned char*)writeJob->buffer;
*bufferSize = writeJob->bufferSize;
}
static void FIO_rust_gzip_writeEnqueue(void* opaque, void** job,
size_t usedBufferSize,
unsigned char** buffer, size_t* bufferSize)
{
IOJob_t* writeJob = (IOJob_t*)*job;
writeJob->usedBufferSize = usedBufferSize;
AIO_WritePool_enqueueAndReacquireWriteJob(&writeJob);
*job = writeJob;
*buffer = (unsigned char*)writeJob->buffer;
*bufferSize = writeJob->bufferSize;
(void)opaque;
}
static void FIO_rust_gzip_writeRelease(void* opaque, void* job)
{
AIO_WritePool_releaseIoJob((IOJob_t*)job);
(void)opaque;
}
static void FIO_rust_gzip_sparseWriteEnd(void* opaque)
{
AIO_WritePool_sparseWriteEnd((WritePoolCtx_t*)opaque);
}
static int FIO_rust_gzip_zlibInit(void* opaque, int compressionLevel)
{
z_stream* const strm = (z_stream*)opaque;
strm->zalloc = Z_NULL;
strm->zfree = Z_NULL;
strm->opaque = Z_NULL;
return deflateInit2(strm, compressionLevel, Z_DEFLATED,
15 /* maxWindowLogSize */ + 16 /* gzip only */,
8, Z_DEFAULT_STRATEGY); /* see https://www.zlib.net/manual.html */
}
static int FIO_rust_gzip_zlibDeflate(void* opaque, const unsigned char* input,
size_t inputSize, unsigned char* output,
size_t outputSize, int flush,
size_t* consumed, size_t* produced)
{
z_stream* const strm = (z_stream*)opaque;
size_t const availBefore = inputSize;
size_t const outputBefore = outputSize;
int ret;
assert(inputSize <= UINT_MAX);
assert(outputSize <= UINT_MAX);
strm->next_in = (z_const unsigned char*)input;
strm->avail_in = (uInt)inputSize;
strm->next_out = output;
strm->avail_out = (uInt)outputSize;
ret = deflate(strm, flush);
*consumed = availBefore - strm->avail_in;
*produced = outputBefore - strm->avail_out;
return ret;
}
static int FIO_rust_gzip_zlibEnd(void* opaque)
{
return deflateEnd((z_stream*)opaque);
}
static void FIO_rust_gzip_progress(void* opaque, U64 srcFileSize,
U64 inFileSize, U64 outFileSize)
{
(void)opaque;
if (srcFileSize == UTIL_FILESIZE_UNKNOWN) {
DISPLAYUPDATE_PROGRESS(
"\rRead : %u MB ==> %.2f%% ",
(unsigned)(inFileSize>>20),
(double)outFileSize/(double)inFileSize*100)
} else {
DISPLAYUPDATE_PROGRESS(
"\rRead : %u / %u MB ==> %.2f%% ",
(unsigned)(inFileSize>>20), (unsigned)(srcFileSize>>20),
(double)outFileSize/(double)inFileSize*100);
}
}
#endif
#ifdef ZSTD_LZMACOMPRESS
static void FIO_rust_lzma_readFill(void* opaque, size_t requested,
const unsigned char** buffer, size_t* loaded)
{
ReadPoolCtx_t* const readCtx = (ReadPoolCtx_t*)opaque;
AIO_ReadPool_fillBuffer(readCtx, requested);
*buffer = readCtx->srcBuffer;
*loaded = readCtx->srcBufferLoaded;
}
static void FIO_rust_lzma_readConsume(void* opaque, size_t consumed)
{
AIO_ReadPool_consumeBytes((ReadPoolCtx_t*)opaque, consumed);
}
static void FIO_rust_lzma_writeAcquire(void* opaque, void** job,
unsigned char** buffer, size_t* bufferSize)
{
IOJob_t* const writeJob = AIO_WritePool_acquireJob((WritePoolCtx_t*)opaque);
*job = writeJob;
*buffer = (unsigned char*)writeJob->buffer;
*bufferSize = writeJob->bufferSize;
}
static void FIO_rust_lzma_writeEnqueue(void* opaque, void** job,
size_t usedBufferSize,
unsigned char** buffer, size_t* bufferSize)
{
IOJob_t* writeJob = (IOJob_t*)*job;
writeJob->usedBufferSize = usedBufferSize;
AIO_WritePool_enqueueAndReacquireWriteJob(&writeJob);
*job = writeJob;
*buffer = (unsigned char*)writeJob->buffer;
*bufferSize = writeJob->bufferSize;
(void)opaque;
}
static void FIO_rust_lzma_writeRelease(void* opaque, void* job)
{
AIO_WritePool_releaseIoJob((IOJob_t*)job);
(void)opaque;
}
static void FIO_rust_lzma_sparseWriteEnd(void* opaque)
{
AIO_WritePool_sparseWriteEnd((WritePoolCtx_t*)opaque);
}
static int FIO_rust_lzma_init(void* opaque, int compressionLevel,
int plainLzma, int* lzmaResult)
{
lzma_stream* const strm = (lzma_stream*)opaque;
lzma_ret ret;
*lzmaResult = (int)LZMA_OK;
if (plainLzma) {
lzma_options_lzma opt_lzma;
if (lzma_lzma_preset(&opt_lzma, (uint32_t)compressionLevel))
return FIO_RUST_LZMA_INIT_PRESET_ERROR;
ret = lzma_alone_encoder(strm, &opt_lzma); /* LZMA */
if (ret != LZMA_OK) {
*lzmaResult = (int)ret;
return FIO_RUST_LZMA_INIT_ALONE_ERROR;
}
} else {
ret = lzma_easy_encoder(strm, (uint32_t)compressionLevel,
LZMA_CHECK_CRC64); /* XZ */
if (ret != LZMA_OK) {
*lzmaResult = (int)ret;
return FIO_RUST_LZMA_INIT_XZ_ERROR;
}
}
return FIO_RUST_LZMA_OK;
}
static int FIO_rust_lzma_code(void* opaque, const unsigned char* input,
size_t inputSize, unsigned char* output,
size_t outputSize, int action,
size_t* consumed, size_t* produced)
{
lzma_stream* const strm = (lzma_stream*)opaque;
size_t const availInBefore = inputSize;
size_t const availOutBefore = outputSize;
lzma_ret ret;
strm->next_in = (const uint8_t*)input;
strm->avail_in = inputSize;
strm->next_out = (uint8_t*)output;
strm->avail_out = outputSize;
ret = lzma_code(strm, (lzma_action)action);
*consumed = availInBefore - strm->avail_in;
*produced = availOutBefore - strm->avail_out;
return (int)ret;
}
static void FIO_rust_lzma_end(void* opaque)
{
lzma_end((lzma_stream*)opaque);
}
static void FIO_rust_lzma_progress(void* opaque, U64 srcFileSize,
U64 inFileSize, U64 outFileSize)
{
(void)opaque;
if (srcFileSize == UTIL_FILESIZE_UNKNOWN)
DISPLAYUPDATE_PROGRESS("\rRead : %u MB ==> %.2f%%",
(unsigned)(inFileSize>>20),
(double)outFileSize/(double)inFileSize*100)
else
DISPLAYUPDATE_PROGRESS("\rRead : %u / %u MB ==> %.2f%%",
(unsigned)(inFileSize>>20), (unsigned)(srcFileSize>>20),
(double)outFileSize/(double)inFileSize*100);
}
static unsigned long long
FIO_compressLzmaFrame(cRess_t* ress,
const char* srcFileName, U64 const srcFileSize,
int compressionLevel, U64* readsize, int plain_lzma)
{
lzma_stream strm = LZMA_STREAM_INIT;
FIO_rust_lzma_compress_projection_t projection;
U64 compressedSize = 0;
int lzmaResult = (int)LZMA_OK;
int status;
int diagnostic;
memset(&projection, 0, sizeof(projection));
projection.readOpaque = (void*)ress->readCtx;
projection.writeOpaque = (void*)ress->writeCtx;
projection.lzmaOpaque = &strm;
projection.progressOpaque = NULL;
projection.readBufferSize = ZSTD_CStreamInSize();
projection.readFill = FIO_rust_lzma_readFill;
projection.readConsume = FIO_rust_lzma_readConsume;
projection.writeAcquire = FIO_rust_lzma_writeAcquire;
projection.writeEnqueue = FIO_rust_lzma_writeEnqueue;
projection.writeRelease = FIO_rust_lzma_writeRelease;
projection.sparseWriteEnd = FIO_rust_lzma_sparseWriteEnd;
projection.lzmaInit = FIO_rust_lzma_init;
projection.lzmaCode = FIO_rust_lzma_code;
projection.lzmaEnd = FIO_rust_lzma_end;
projection.progress = FIO_rust_lzma_progress;
status = FIO_rust_compressLzmaFrame(
&projection, srcFileName, srcFileSize, compressionLevel, plain_lzma,
readsize, &compressedSize, &lzmaResult);
diagnostic = FIO_rust_lzmaCompressionDiagnostic(status);
switch (diagnostic) {
case FIO_RUST_LZMA_DIAGNOSTIC_OK:
return compressedSize;
case FIO_RUST_LZMA_DIAGNOSTIC_INIT_PRESET_ERROR:
EXM_THROW(81, "zstd: %s: lzma_lzma_preset error", srcFileName);
case FIO_RUST_LZMA_DIAGNOSTIC_INIT_ALONE_ERROR:
EXM_THROW(82, "zstd: %s: lzma_alone_encoder error %d", srcFileName, lzmaResult);
case FIO_RUST_LZMA_DIAGNOSTIC_INIT_XZ_ERROR:
EXM_THROW(83, "zstd: %s: lzma_easy_encoder error %d", srcFileName, lzmaResult);
case FIO_RUST_LZMA_DIAGNOSTIC_CODE_ERROR:
EXM_THROW(84, "zstd: %s: lzma_code encoding error %d", srcFileName, lzmaResult);
case FIO_RUST_LZMA_DIAGNOSTIC_INVALID_PROJECTION:
EXM_THROW(84, "zstd: %s: lzma_code encoding error %d", srcFileName, lzmaResult);
default:
assert(diagnostic == FIO_RUST_LZMA_DIAGNOSTIC_UNKNOWN);
EXM_THROW(84, "zstd: %s: lzma_code encoding error %d", srcFileName, lzmaResult);
}
}
#endif
#ifdef ZSTD_LZ4COMPRESS
#if LZ4_VERSION_NUMBER <= 10600
#define LZ4F_blockLinked blockLinked
#define LZ4F_max64KB max64KB
#endif
typedef struct {
LZ4F_compressionContext_t ctx;
LZ4F_preferences_t prefs;
} FIO_rust_lz4_context_t;
static int FIO_rust_lz4_create(void* opaque, unsigned version, size_t* result)
{
FIO_rust_lz4_context_t* const context = (FIO_rust_lz4_context_t*)opaque;
LZ4F_errorCode_t const errorCode = LZ4F_createCompressionContext(&context->ctx, version);
*result = (size_t)errorCode;
return LZ4F_isError(errorCode);
}
static void FIO_rust_lz4_prepare(void* opaque, U64 srcFileSize,
int compressionLevel, int checksumFlag,
size_t blockSize, size_t bufferSize)
{
FIO_rust_lz4_context_t* const context = (FIO_rust_lz4_context_t*)opaque;
memset(&context->prefs, 0, sizeof(context->prefs));
/* autoflush off to mitigate a bug in lz4<=1.9.3 for compression level 12 */
context->prefs.autoFlush = 0;
context->prefs.compressionLevel = compressionLevel;
context->prefs.frameInfo.blockMode = LZ4F_blockLinked;
context->prefs.frameInfo.blockSizeID = LZ4F_max64KB;
context->prefs.frameInfo.contentChecksumFlag = (contentChecksum_t)checksumFlag;
#if LZ4_VERSION_NUMBER >= 10600
context->prefs.frameInfo.contentSize = (srcFileSize==UTIL_FILESIZE_UNKNOWN) ? 0 : srcFileSize;
#else
(void)srcFileSize;
#endif
assert(LZ4F_compressBound(blockSize, &context->prefs) <= bufferSize);
}
static int FIO_rust_lz4_begin(void* opaque, unsigned char* output,
size_t outputSize, size_t* result)
{
FIO_rust_lz4_context_t* const context = (FIO_rust_lz4_context_t*)opaque;
size_t const headerSize = LZ4F_compressBegin(context->ctx, output, outputSize,
&context->prefs);
*result = headerSize;
return LZ4F_isError(headerSize);
}
static int FIO_rust_lz4_update(void* opaque, unsigned char* output,
size_t outputSize, const unsigned char* input,
size_t inputSize, size_t* result)
{
FIO_rust_lz4_context_t* const context = (FIO_rust_lz4_context_t*)opaque;
size_t const outSize = LZ4F_compressUpdate(context->ctx, output, outputSize,
input, inputSize, NULL);
*result = outSize;
return LZ4F_isError(outSize);
}
static int FIO_rust_lz4_end(void* opaque, unsigned char* output,
size_t outputSize, size_t* result)
{
FIO_rust_lz4_context_t* const context = (FIO_rust_lz4_context_t*)opaque;
size_t const endSize = LZ4F_compressEnd(context->ctx, output, outputSize, NULL);
*result = endSize;
return LZ4F_isError(endSize);
}
static void FIO_rust_lz4_free(void* opaque)
{
FIO_rust_lz4_context_t* const context = (FIO_rust_lz4_context_t*)opaque;
LZ4F_freeCompressionContext(context->ctx);
context->ctx = NULL;
}
static size_t FIO_rust_lz4_readFill(void* opaque, size_t requested,
const unsigned char** buffer, size_t* loaded)
{
ReadPoolCtx_t* const readCtx = (ReadPoolCtx_t*)opaque;
size_t const added = AIO_ReadPool_fillBuffer(readCtx, requested);
*buffer = readCtx->srcBuffer;
*loaded = readCtx->srcBufferLoaded;
return added;
}
static void FIO_rust_lz4_readConsume(void* opaque, size_t consumed)
{
AIO_ReadPool_consumeBytes((ReadPoolCtx_t*)opaque, consumed);
}
static void FIO_rust_lz4_writeAcquire(void* opaque, void** job,
unsigned char** buffer, size_t* bufferSize)
{
IOJob_t* const writeJob = AIO_WritePool_acquireJob((WritePoolCtx_t*)opaque);
*job = writeJob;
*buffer = (unsigned char*)writeJob->buffer;
*bufferSize = writeJob->bufferSize;
}
static void FIO_rust_lz4_writeEnqueue(void* opaque, void** job,
size_t usedBufferSize,
unsigned char** buffer, size_t* bufferSize)
{
IOJob_t* writeJob = (IOJob_t*)*job;
writeJob->usedBufferSize = usedBufferSize;
AIO_WritePool_enqueueAndReacquireWriteJob(&writeJob);
*job = writeJob;
*buffer = (unsigned char*)writeJob->buffer;
*bufferSize = writeJob->bufferSize;
(void)opaque;
}
static void FIO_rust_lz4_writeRelease(void* opaque, void* job)
{
AIO_WritePool_releaseIoJob((IOJob_t*)job);
(void)opaque;
}
static void FIO_rust_lz4_sparseWriteEnd(void* opaque)
{
AIO_WritePool_sparseWriteEnd((WritePoolCtx_t*)opaque);
}
static void FIO_rust_lz4_progress(void* opaque, U64 srcFileSize,
U64 inFileSize, U64 outFileSize)
{
(void)opaque;
if (srcFileSize == UTIL_FILESIZE_UNKNOWN) {
DISPLAYUPDATE_PROGRESS("\rRead : %u MB ==> %.2f%%",
(unsigned)(inFileSize>>20),
(double)outFileSize/(double)inFileSize*100)
} else {
DISPLAYUPDATE_PROGRESS("\rRead : %u / %u MB ==> %.2f%%",
(unsigned)(inFileSize>>20), (unsigned)(srcFileSize>>20),
(double)outFileSize/(double)inFileSize*100);
}
}
static unsigned long long
FIO_compressLz4Frame(cRess_t* ress,
const char* srcFileName, U64 const srcFileSize,
int compressionLevel, int checksumFlag,
U64* readsize)
{
FIO_rust_lz4_context_t codec;
FIO_rust_lz4_compress_projection_t projection;
U64 compressedSize = 0;
size_t lz4Result = 0;
int status;
int diagnostic;
memset(&codec, 0, sizeof(codec));
memset(&projection, 0, sizeof(projection));
projection.readOpaque = ress->readCtx;
projection.writeOpaque = ress->writeCtx;
projection.codecOpaque = &codec;
projection.progressOpaque = NULL;
projection.version = LZ4F_VERSION;
projection.blockSize = (size_t)FIO_LZ4_GetBlockSize_FromBlockId(LZ4F_max64KB);
projection.create = FIO_rust_lz4_create;
projection.prepare = FIO_rust_lz4_prepare;
projection.begin = FIO_rust_lz4_begin;
projection.update = FIO_rust_lz4_update;
projection.end = FIO_rust_lz4_end;
projection.freeContext = FIO_rust_lz4_free;
projection.readFill = FIO_rust_lz4_readFill;
projection.readConsume = FIO_rust_lz4_readConsume;
projection.writeAcquire = FIO_rust_lz4_writeAcquire;
projection.writeEnqueue = FIO_rust_lz4_writeEnqueue;
projection.writeRelease = FIO_rust_lz4_writeRelease;
projection.sparseWriteEnd = FIO_rust_lz4_sparseWriteEnd;
projection.progress = FIO_rust_lz4_progress;
status = FIO_rust_compressLz4Frame(
&projection, srcFileName, srcFileSize, compressionLevel, checksumFlag,
readsize, &compressedSize, &lz4Result);
diagnostic = FIO_rust_lz4CompressionDiagnostic(status);
switch (diagnostic) {
case FIO_RUST_LZ4_DIAGNOSTIC_OK:
return compressedSize;
case FIO_RUST_LZ4_DIAGNOSTIC_CREATE_ERROR:
EXM_THROW(31, "zstd: failed to create lz4 compression context");
case FIO_RUST_LZ4_DIAGNOSTIC_HEADER_ERROR:
EXM_THROW(33, "File header generation failed : %s",
LZ4F_getErrorName(lz4Result));
case FIO_RUST_LZ4_DIAGNOSTIC_UPDATE_ERROR:
EXM_THROW(35, "zstd: %s: lz4 compression failed : %s",
srcFileName, LZ4F_getErrorName(lz4Result));
case FIO_RUST_LZ4_DIAGNOSTIC_END_ERROR:
EXM_THROW(38, "zstd: %s: lz4 end of file generation failed : %s",
srcFileName, LZ4F_getErrorName(lz4Result));
case FIO_RUST_LZ4_DIAGNOSTIC_INVALID_PROJECTION:
EXM_THROW(31, "zstd: failed to create lz4 compression context");
default:
assert(diagnostic == FIO_RUST_LZ4_DIAGNOSTIC_UNKNOWN);
EXM_THROW(31, "zstd: failed to create lz4 compression context");
}
}
#endif
/* Rust owns scalar adaptive state and policy. Keep the FIO context,
* preference, and ZSTD progression layouts private to this translation unit. */
struct FIO_rust_zstd_adapt_projection_s {
U64 consumed;
U64 previousConsumed;
unsigned nbActiveWorkers;
U64 newlyProduced;
U64 newlyFlushed;
unsigned flushWaiting;
unsigned inputBlocked;
unsigned inputPresented;
U64 newlyIngested;
U64 newlyConsumed;
int compressionLevel;
int minAdaptLevel;
int maxAdaptLevel;
int maxCLevel;
};
typedef char FIO_rust_zstd_adapt_projection_layout[
(offsetof(FIO_rust_zstd_adapt_projection_t, consumed) == 0
&& offsetof(FIO_rust_zstd_adapt_projection_t, previousConsumed)
== sizeof(U64)
&& offsetof(FIO_rust_zstd_adapt_projection_t, nbActiveWorkers)
== 2 * sizeof(U64)
&& offsetof(FIO_rust_zstd_adapt_projection_t, newlyProduced)
== (sizeof(void*) == 8 ? 24 : 20)
&& offsetof(FIO_rust_zstd_adapt_projection_t, newlyFlushed)
== (sizeof(void*) == 8 ? 32 : 28)
&& offsetof(FIO_rust_zstd_adapt_projection_t, flushWaiting)
== (sizeof(void*) == 8 ? 40 : 36)
&& offsetof(FIO_rust_zstd_adapt_projection_t, inputBlocked)
== (sizeof(void*) == 8 ? 44 : 40)
&& offsetof(FIO_rust_zstd_adapt_projection_t, inputPresented)
== (sizeof(void*) == 8 ? 48 : 44)
&& offsetof(FIO_rust_zstd_adapt_projection_t, newlyIngested)
== (sizeof(void*) == 8 ? 56 : 48)
&& offsetof(FIO_rust_zstd_adapt_projection_t, newlyConsumed)
== (sizeof(void*) == 8 ? 64 : 56)
&& offsetof(FIO_rust_zstd_adapt_projection_t, compressionLevel)
== (sizeof(void*) == 8 ? 72 : 64)
&& offsetof(FIO_rust_zstd_adapt_projection_t, minAdaptLevel)
== (sizeof(void*) == 8 ? 76 : 68)
&& offsetof(FIO_rust_zstd_adapt_projection_t, maxAdaptLevel)
== (sizeof(void*) == 8 ? 80 : 72)
&& offsetof(FIO_rust_zstd_adapt_projection_t, maxCLevel)
== (sizeof(void*) == 8 ? 84 : 76)
&& sizeof(FIO_rust_zstd_adapt_projection_t)
== (sizeof(void*) == 8 ? 88 : 80))
? 1 : -1];
enum {
FIO_RUST_ZSTD_ADAPT_DIAG_OUTPUT_BLOCKED,
FIO_RUST_ZSTD_ADAPT_DIAG_OUTPUT_BACKLOG,
FIO_RUST_ZSTD_ADAPT_DIAG_CHECK,
FIO_RUST_ZSTD_ADAPT_DIAG_INPUT_STARVATION,
FIO_RUST_ZSTD_ADAPT_DIAG_INPUT_STATS,
FIO_RUST_ZSTD_ADAPT_DIAG_RECOMMEND_FASTER,
FIO_RUST_ZSTD_ADAPT_DIAG_SLOWER_LEVEL,
FIO_RUST_ZSTD_ADAPT_DIAG_FASTER_LEVEL
};
typedef struct {
FIO_ctx_t* fCtx;
ZSTD_CCtx* cctx;
U64 srcFileSize;
UTIL_HumanReadableSize_t fileHrs;
} FIO_rust_zstd_projection_context_t;
static size_t FIO_rust_zstd_readFill(void* opaque, size_t requested,
const unsigned char** buffer, size_t* loaded)
{
ReadPoolCtx_t* const readCtx = (ReadPoolCtx_t*)opaque;
size_t const added = AIO_ReadPool_fillBuffer(readCtx, requested);
*buffer = readCtx->srcBuffer;
*loaded = readCtx->srcBufferLoaded;
DISPLAYLEVEL(6, "fread %u bytes from source \n", (unsigned)added);
return added;
}
static void FIO_rust_zstd_readConsume(void* opaque, size_t consumed)
{
AIO_ReadPool_consumeBytes((ReadPoolCtx_t*)opaque, consumed);
}
static void FIO_rust_zstd_writeAcquire(void* opaque, void** job,
unsigned char** buffer, size_t* bufferSize)
{
IOJob_t* const writeJob = AIO_WritePool_acquireJob((WritePoolCtx_t*)opaque);
*job = writeJob;
*buffer = (unsigned char*)writeJob->buffer;
*bufferSize = writeJob->bufferSize;
}
static void FIO_rust_zstd_writeEnqueue(void* opaque, void** job,
size_t usedBufferSize,
unsigned char** buffer, size_t* bufferSize)
{
IOJob_t* writeJob = (IOJob_t*)*job;
writeJob->usedBufferSize = usedBufferSize;
AIO_WritePool_enqueueAndReacquireWriteJob(&writeJob);
*job = writeJob;
*buffer = (unsigned char*)writeJob->buffer;
*bufferSize = writeJob->bufferSize;
(void)opaque;
}
static void FIO_rust_zstd_writeRelease(void* opaque, void* job)
{
AIO_WritePool_releaseIoJob((IOJob_t*)job);
(void)opaque;
}
static void FIO_rust_zstd_sparseWriteEnd(void* opaque)
{
AIO_WritePool_sparseWriteEnd((WritePoolCtx_t*)opaque);
}
static void FIO_rust_zstd_adaptiveProgression(
void* opaque, FIO_rust_zstd_progression_t* projection)
{
FIO_rust_zstd_projection_context_t* const context =
(FIO_rust_zstd_projection_context_t*)opaque;
ZSTD_frameProgression const zfp = ZSTD_getFrameProgression(context->cctx);
assert(projection != NULL);
projection->ingested = zfp.ingested;
projection->consumed = zfp.consumed;
projection->produced = zfp.produced;
projection->flushed = zfp.flushed;
projection->currentJobID = zfp.currentJobID;
projection->nbActiveWorkers = zfp.nbActiveWorkers;
}
static void FIO_rust_zstd_adaptiveSetParameter(void* opaque,
int compressionLevel)
{
FIO_rust_zstd_projection_context_t* const context =
(FIO_rust_zstd_projection_context_t*)opaque;
(void)ZSTD_CCtx_setParameter(context->cctx,
ZSTD_c_compressionLevel,
compressionLevel);
}
static void FIO_rust_zstd_adaptiveDiagnostic(
void* opaque, int diagnostic,
const FIO_rust_zstd_adapt_projection_t* projection)
{
(void)opaque;
switch (diagnostic) {
case FIO_RUST_ZSTD_ADAPT_DIAG_OUTPUT_BLOCKED:
DISPLAYLEVEL(6, "all buffers full : compression stopped => slow down \n")
break;
case FIO_RUST_ZSTD_ADAPT_DIAG_OUTPUT_BACKLOG:
assert(projection != NULL);
DISPLAYLEVEL(6, "compression faster than flush (%llu > %llu), and flushed was never slowed down by lack of production => slow down \n",
(unsigned long long)projection->newlyProduced,
(unsigned long long)projection->newlyFlushed);
break;
case FIO_RUST_ZSTD_ADAPT_DIAG_CHECK:
DISPLAYLEVEL(6, "compression level adaptation check \n")
break;
case FIO_RUST_ZSTD_ADAPT_DIAG_INPUT_STARVATION:
DISPLAYLEVEL(6, "input is never blocked => input is slower than ingestion \n");
break;
case FIO_RUST_ZSTD_ADAPT_DIAG_INPUT_STATS:
assert(projection != NULL);
assert(projection->inputPresented > 0);
DISPLAYLEVEL(6, "input blocked %u/%u(%.2f) - ingested:%u vs %u:consumed - flushed:%u vs %u:produced \n",
projection->inputBlocked, projection->inputPresented,
(double)projection->inputBlocked/
projection->inputPresented*100,
(unsigned)projection->newlyIngested,
(unsigned)projection->newlyConsumed,
(unsigned)projection->newlyFlushed,
(unsigned)projection->newlyProduced);
break;
case FIO_RUST_ZSTD_ADAPT_DIAG_RECOMMEND_FASTER:
assert(projection != NULL);
DISPLAYLEVEL(6, "recommend faster as in(%llu) >= (%llu)comp(%llu) <= out(%llu) \n",
(unsigned long long)projection->newlyIngested,
(unsigned long long)projection->newlyConsumed,
(unsigned long long)projection->newlyProduced,
(unsigned long long)projection->newlyFlushed);
break;
case FIO_RUST_ZSTD_ADAPT_DIAG_SLOWER_LEVEL:
DISPLAYLEVEL(6, "slower speed , higher compression \n")
break;
case FIO_RUST_ZSTD_ADAPT_DIAG_FASTER_LEVEL:
DISPLAYLEVEL(6, "faster speed , lighter compression \n")
break;
default:
assert(0);
break;
}
}
void FIO_rust_zstd_compressStreamDisplay(
int directive, size_t inputPos, size_t inputSize, size_t outputProduced)
{
DISPLAYLEVEL(6, "ZSTD_compress_generic(end:%u) => input pos(%u)<=(%u)size ; output generated %u bytes \n",
(unsigned)directive, (unsigned)inputPos,
(unsigned)inputSize, (unsigned)outputProduced);
}
static void FIO_rust_zstd_iteration(void* opaque, const char* srcFileName,
int* compressionLevel,
size_t oldInputPos, size_t newInputPos,
size_t toFlushNow)
{
FIO_rust_zstd_projection_context_t* const context =
(FIO_rust_zstd_projection_context_t*)opaque;
FIO_ctx_t* const fCtx = context->fCtx;
(void)oldInputPos;
(void)newInputPos;
(void)toFlushNow;
/* Keep progress formatting and the frame-progression query in C. */
if (SHOULD_DISPLAY_PROGRESS() && READY_FOR_UPDATE()) {
ZSTD_frameProgression const zfp = ZSTD_getFrameProgression(context->cctx);
double const cShare = (double)zfp.produced /
(double)(zfp.consumed + !zfp.consumed) * 100;
UTIL_HumanReadableSize_t const buffered_hrs =
UTIL_makeHumanReadableSize(zfp.ingested - zfp.consumed);
UTIL_HumanReadableSize_t const consumed_hrs =
UTIL_makeHumanReadableSize(zfp.consumed);
UTIL_HumanReadableSize_t const produced_hrs =
UTIL_makeHumanReadableSize(zfp.produced);
DELAY_NEXT_UPDATE();
DISPLAY_PROGRESS("\r%79s\r", "");
if (g_display_prefs.displayLevel >= 3) {
DISPLAY_PROGRESS(
"(L%i) Buffered:%5.*f%s - Consumed:%5.*f%s - Compressed:%5.*f%s => %.2f%% ",
*compressionLevel,
buffered_hrs.precision, buffered_hrs.value, buffered_hrs.suffix,
consumed_hrs.precision, consumed_hrs.value, consumed_hrs.suffix,
produced_hrs.precision, produced_hrs.value, produced_hrs.suffix,
cShare);
} else {
if (fCtx->nbFilesTotal > 1) {
size_t const srcFileNameSize = strlen(srcFileName);
if (srcFileNameSize > 18) {
const char* const truncatedSrcFileName = srcFileName + srcFileNameSize - 15;
DISPLAY_PROGRESS("Compress: %u/%u files. Current: ...%s ",
fCtx->currFileIdx+1, fCtx->nbFilesTotal,
truncatedSrcFileName);
} else {
DISPLAY_PROGRESS("Compress: %u/%u files. Current: %*s ",
fCtx->currFileIdx+1, fCtx->nbFilesTotal,
(int)(18-srcFileNameSize), srcFileName);
}
}
DISPLAY_PROGRESS("Read:%6.*f%4s ", consumed_hrs.precision,
consumed_hrs.value, consumed_hrs.suffix);
if (context->srcFileSize != UTIL_FILESIZE_UNKNOWN)
DISPLAY_PROGRESS("/%6.*f%4s", context->fileHrs.precision,
context->fileHrs.value, context->fileHrs.suffix);
DISPLAY_PROGRESS(" ==> %2.f%%", cShare);
}
}
}
static unsigned long long
FIO_rust_compressZstdCallback(void* fCtx, void* prefs, void* ress,
const char* srcFileName, U64 srcFileSize,
int compressionLevel, U64* readsize)
{
FIO_ctx_t* const fCtxPtr = (FIO_ctx_t*)fCtx;
FIO_prefs_t* const prefsPtr = (FIO_prefs_t*)prefs;
cRess_t const* const ressPtr = (const cRess_t*)ress;
FIO_rust_zstd_projection_context_t context;
FIO_rust_zstd_compress_projection_t projection;
U64 compressedSize = 0;
U64 pledgedSrcSize = ZSTD_CONTENTSIZE_UNKNOWN;
size_t zstdResult = 0;
int status;
int diagnostic;
memset(&context, 0, sizeof(context));
context.fCtx = fCtxPtr;
context.cctx = ressPtr->cctx;
context.srcFileSize = srcFileSize;
context.fileHrs = UTIL_makeHumanReadableSize(srcFileSize);
memset(&projection, 0, sizeof(projection));
projection.readOpaque = (void*)ressPtr->readCtx;
projection.writeOpaque = (void*)ressPtr->writeCtx;
projection.codecOpaque = (void*)ressPtr->cctx;
projection.policyOpaque = &context;
projection.readBufferSize = ZSTD_CStreamInSize();
projection.readFill = FIO_rust_zstd_readFill;
projection.readConsume = FIO_rust_zstd_readConsume;
projection.writeAcquire = FIO_rust_zstd_writeAcquire;
projection.writeEnqueue = FIO_rust_zstd_writeEnqueue;
projection.writeRelease = FIO_rust_zstd_writeRelease;
projection.sparseWriteEnd = FIO_rust_zstd_sparseWriteEnd;
projection.compressStream = FIO_rust_zstd_compressStream;
projection.iteration = FIO_rust_zstd_iteration;
projection.compressStreamDisplay = FIO_rust_zstd_compressStreamDisplay;
projection.adaptiveMode = prefsPtr->adaptiveMode;
projection.nbWorkers = prefsPtr->nbWorkers;
projection.minAdaptLevel = prefsPtr->minAdaptLevel;
projection.maxAdaptLevel = prefsPtr->maxAdaptLevel;
projection.maxCLevel = ZSTD_maxCLevel();
projection.adaptiveProgression = FIO_rust_zstd_adaptiveProgression;
projection.adaptiveSetParameter = FIO_rust_zstd_adaptiveSetParameter;
projection.adaptiveDiagnostic = FIO_rust_zstd_adaptiveDiagnostic;
DISPLAYLEVEL(6, "compression using zstd format \n");
/* Rust selects the source-size precedence. C retains the private CCtx
* operation and its exact diagnostic path. */
pledgedSrcSize = FIO_rust_selectPledgedSrcSize(
srcFileSize, (U64)prefsPtr->streamSrcSize,
UTIL_FILESIZE_UNKNOWN, ZSTD_CONTENTSIZE_UNKNOWN);
if (pledgedSrcSize != ZSTD_CONTENTSIZE_UNKNOWN)
CHECK(ZSTD_CCtx_setPledgedSrcSize(ressPtr->cctx, pledgedSrcSize));
{ int windowLog;
UTIL_HumanReadableSize_t windowSize;
CHECK(ZSTD_CCtx_getParameter(ressPtr->cctx, ZSTD_c_windowLog, &windowLog));
if (windowLog == 0) {
if (prefsPtr->ldmFlag) {
/* If long mode is set without a window size libzstd will set this size internally */
windowLog = ZSTD_WINDOWLOG_LIMIT_DEFAULT;
} else {
ZSTD_compressionParameters const cParams =
ZSTD_getCParams(compressionLevel, srcFileSize, 0);
windowLog = (int)cParams.windowLog;
}
}
windowSize = UTIL_makeHumanReadableSize(
MAX(1ULL, MIN(1ULL << windowLog, pledgedSrcSize)));
DISPLAYLEVEL(4, "Decompression will require %.*f%s of memory\n",
windowSize.precision, windowSize.value, windowSize.suffix);
}
status = FIO_rust_compressZstdFrame(
&projection, srcFileName, srcFileSize, compressionLevel,
readsize, &compressedSize, &zstdResult);
diagnostic = FIO_rust_zstdCompressionDiagnostic(status);
switch (diagnostic) {
case FIO_RUST_ZSTD_DIAGNOSTIC_OK:
return compressedSize;
case FIO_RUST_ZSTD_DIAGNOSTIC_COMPRESS_ERROR:
DISPLAYLEVEL(5, "%s \n",
"ZSTD_compressStream2(ress.cctx, &outBuff, &inBuff, directive)");
EXM_THROW(11, "%s", ZSTD_getErrorName(zstdResult));
case FIO_RUST_ZSTD_DIAGNOSTIC_INCOMPLETE_INPUT:
EXM_THROW(27, "Read error : Incomplete read : %llu / %llu B",
(unsigned long long)*readsize,
(unsigned long long)srcFileSize);
case FIO_RUST_ZSTD_DIAGNOSTIC_INVALID_PROJECTION:
EXM_THROW(11, "zstd compression projection is invalid");
default:
assert(diagnostic == FIO_RUST_ZSTD_DIAGNOSTIC_UNKNOWN);
EXM_THROW(11, "zstd compression projection is invalid");
}
}
#ifdef ZSTD_GZCOMPRESS
static unsigned long long
FIO_rust_compressGzipCallback(void* ress, const char* srcFileName,
U64 srcFileSize, int compressionLevel,
U64* readsize)
{
const cRess_t* const ressPtr = (const cRess_t*)ress;
FIO_rust_gzip_compress_projection_t projection;
z_stream strm;
U64 compressedSize = 0;
int zlibResult = Z_OK;
int status;
int diagnostic;
memset(&projection, 0, sizeof(projection));
projection.readOpaque = (void*)ressPtr->readCtx;
projection.writeOpaque = (void*)ressPtr->writeCtx;
projection.zlibOpaque = &strm;
projection.readBufferSize = ZSTD_CStreamInSize();
projection.readFill = FIO_rust_gzip_readFill;
projection.readConsume = FIO_rust_gzip_readConsume;
projection.writeAcquire = FIO_rust_gzip_writeAcquire;
projection.writeEnqueue = FIO_rust_gzip_writeEnqueue;
projection.writeRelease = FIO_rust_gzip_writeRelease;
projection.sparseWriteEnd = FIO_rust_gzip_sparseWriteEnd;
projection.zlibInit = FIO_rust_gzip_zlibInit;
projection.zlibDeflate = FIO_rust_gzip_zlibDeflate;
projection.zlibEnd = FIO_rust_gzip_zlibEnd;
projection.progress = FIO_rust_gzip_progress;
status = FIO_rust_compressGzipFrame(
&projection, srcFileName, srcFileSize, compressionLevel,
readsize, &compressedSize, &zlibResult);
diagnostic = FIO_rust_gzipCompressionDiagnostic(status);
switch (diagnostic) {
case FIO_RUST_GZIP_DIAGNOSTIC_OK:
return compressedSize;
case FIO_RUST_GZIP_DIAGNOSTIC_INIT_ERROR:
EXM_THROW(71, "zstd: %s: deflateInit2 error %d \n", srcFileName, zlibResult);
case FIO_RUST_GZIP_DIAGNOSTIC_DEFLATE_ERROR:
EXM_THROW(72, "zstd: %s: deflate error %d \n", srcFileName, zlibResult);
case FIO_RUST_GZIP_DIAGNOSTIC_FINISH_ERROR:
EXM_THROW(77, "zstd: %s: deflate error %d \n", srcFileName, zlibResult);
case FIO_RUST_GZIP_DIAGNOSTIC_END_ERROR:
EXM_THROW(79, "zstd: %s: deflateEnd error %d \n", srcFileName, zlibResult);
case FIO_RUST_GZIP_DIAGNOSTIC_INVALID_PROJECTION:
EXM_THROW(72, "zstd: %s: deflate error %d \n", srcFileName, zlibResult);
default:
assert(diagnostic == FIO_RUST_GZIP_DIAGNOSTIC_UNKNOWN);
EXM_THROW(72, "zstd: %s: deflate error %d \n", srcFileName, zlibResult);
}
}
#endif
#ifdef ZSTD_LZMACOMPRESS
static unsigned long long
FIO_rust_compressLzmaCallback(void* ress, const char* srcFileName,
U64 srcFileSize, int compressionLevel,
U64* readsize, int plainLzma)
{
return FIO_compressLzmaFrame((cRess_t*)ress, srcFileName, srcFileSize,
compressionLevel, readsize, plainLzma);
}
#endif
#ifdef ZSTD_LZ4COMPRESS
static unsigned long long
FIO_rust_compressLz4Callback(void* ress, const char* srcFileName,
U64 srcFileSize, int compressionLevel,
int checksumFlag, U64* readsize)
{
return FIO_compressLz4Frame((cRess_t*)ress, srcFileName, srcFileSize,
compressionLevel, checksumFlag, readsize);
}
#endif
typedef struct {
UTIL_time_t timeStart;
clock_t cpuStart;
} FIO_compressDisplayContext_t;
static void
FIO_rust_compressInputDisplayCallback(void* opaque, const char* srcFileName,
U64 fileSize)
{
(void)opaque;
DISPLAYLEVEL(5, "%s: %llu bytes \n", srcFileName,
(unsigned long long)fileSize);
}
static void
FIO_rust_compressStatusDisplayCallback(void* opaque, void* fCtxOpaque,
const char* dstFileName,
const char* srcFileName,
U64 readsize, U64 compressedfilesize)
{
FIO_compressDisplayContext_t const* const displayCtx =
(const FIO_compressDisplayContext_t*)opaque;
FIO_ctx_t* const fCtx = (FIO_ctx_t*)fCtxOpaque;
DISPLAY_PROGRESS("\r%79s\r", "");
if (FIO_shouldDisplayFileSummary(fCtx)) {
UTIL_HumanReadableSize_t hr_isize =
UTIL_makeHumanReadableSize((U64)readsize);
UTIL_HumanReadableSize_t hr_osize =
UTIL_makeHumanReadableSize((U64)compressedfilesize);
if (readsize == 0) {
DISPLAY_SUMMARY("%-20s : (%6.*f%s => %6.*f%s, %s) \n",
srcFileName,
hr_isize.precision, hr_isize.value, hr_isize.suffix,
hr_osize.precision, hr_osize.value, hr_osize.suffix,
dstFileName);
} else {
DISPLAY_SUMMARY("%-20s :%6.2f%% (%6.*f%s => %6.*f%s, %s) \n",
srcFileName,
(double)compressedfilesize / (double)readsize * 100,
hr_isize.precision, hr_isize.value, hr_isize.suffix,
hr_osize.precision, hr_osize.value, hr_osize.suffix,
dstFileName);
}
}
/* Keep the original elapsed-time and CPU-load diagnostics in C so their
* formatting and clock source remain identical to the old loop. */
{ clock_t const cpuEnd = clock();
double const cpuLoad_s = (double)(cpuEnd - displayCtx->cpuStart) / CLOCKS_PER_SEC;
U64 const timeLength_ns = UTIL_clockSpanNano(displayCtx->timeStart);
double const timeLength_s = (double)timeLength_ns / 1000000000;
double const cpuLoad_pct = (cpuLoad_s / timeLength_s) * 100;
DISPLAYLEVEL(4, "%-20s : Completed in %.2f sec (cpu load : %.0f%%)\n",
srcFileName, timeLength_s, cpuLoad_pct);
}
}
/*! FIO_compressFilename_internal() :
* same as FIO_compressFilename_extRess(), with `ress.desFile` already opened.
* @return : 0 : compression completed correctly,
* 1 : missing or pb opening srcFileName
*/
static int
FIO_compressFilename_internal(FIO_ctx_t* const fCtx,
FIO_prefs_t* const prefs,
cRess_t ress,
const char* dstFileName, const char* srcFileName,
int compressionLevel)
{
FIO_compressDisplayContext_t displayCtx = {
UTIL_getTime(),
clock()
};
U64 const fileSize = UTIL_getFileSize(srcFileName);
FIO_rust_compress_callbacks_t callbacks;
int status;
int diagnostic;
memset(&callbacks, 0, sizeof(callbacks));
callbacks.opaque = &displayCtx;
callbacks.compress_zstd = FIO_rust_compressZstdCallback;
#ifdef ZSTD_GZCOMPRESS
callbacks.compress_gzip = FIO_rust_compressGzipCallback;
#endif
#ifdef ZSTD_LZMACOMPRESS
callbacks.compress_lzma = FIO_rust_compressLzmaCallback;
#endif
#ifdef ZSTD_LZ4COMPRESS
callbacks.compress_lz4 = FIO_rust_compressLz4Callback;
#endif
callbacks.display_input = FIO_rust_compressInputDisplayCallback;
callbacks.display_status = FIO_rust_compressStatusDisplayCallback;
status = FIO_rust_compressFilenameInternal(
fCtx, prefs, &ress, dstFileName, srcFileName, fileSize,
compressionLevel, &callbacks);
diagnostic = FIO_rust_compressFilenameDiagnostic(status);
switch (diagnostic) {
case FIO_RUST_COMPRESS_DIAGNOSTIC_OK:
return 0;
case FIO_RUST_COMPRESS_DIAGNOSTIC_GZIP_UNSUPPORTED:
EXM_THROW(20, "zstd: %s: file cannot be compressed as gzip (zstd compiled without ZSTD_GZCOMPRESS) -- ignored \n",
srcFileName);
case FIO_RUST_COMPRESS_DIAGNOSTIC_LZMA_UNSUPPORTED:
EXM_THROW(20, "zstd: %s: file cannot be compressed as xz/lzma (zstd compiled without ZSTD_LZMACOMPRESS) -- ignored \n",
srcFileName);
case FIO_RUST_COMPRESS_DIAGNOSTIC_LZ4_UNSUPPORTED:
EXM_THROW(20, "zstd: %s: file cannot be compressed as lz4 (zstd compiled without ZSTD_LZ4COMPRESS) -- ignored \n",
srcFileName);
case FIO_RUST_COMPRESS_DIAGNOSTIC_ZSTD_UNSUPPORTED:
default:
assert(status == FIO_RUST_COMPRESS_ZSTD_UNSUPPORTED);
EXM_THROW(20, "zstd: %s: file cannot be compressed as zstd -- ignored \n",
srcFileName);
}
}
typedef struct {
FIO_ctx_t* fCtx;
FIO_prefs_t* prefs;
cRess_t* ress;
const char* dstFileName;
const stat_t* srcFileStat;
FILE* dstFile;
} FIO_rust_compress_dst_context_t;
static int FIO_rust_compressDestinationOpen(void* opaque,
const char* srcFileName,
const char* dstFileName,
int transferStat, int* dstFd)
{
FIO_rust_compress_dst_context_t* const context =
(FIO_rust_compress_dst_context_t*)opaque;
int const permissions = transferStat
? TEMPORARY_FILE_PERMISSIONS
: DEFAULT_FILE_PERMISSIONS;
DISPLAYLEVEL(6, "FIO_compressFilename_dstFile: opening dst: %s \n", dstFileName);
context->dstFile = FIO_openDstFile(
context->fCtx, context->prefs, srcFileName, dstFileName, permissions);
if (context->dstFile == NULL)
return 1;
*dstFd = fileno(context->dstFile);
return FIO_RUST_COMPRESS_DST_OPEN_OK;
}
static void FIO_rust_compressDestinationAttach(void* opaque)
{
FIO_rust_compress_dst_context_t* const context =
(FIO_rust_compress_dst_context_t*)opaque;
AIO_WritePool_setFile(context->ress->writeCtx, context->dstFile);
}
static void FIO_rust_compressAddHandler(void* opaque)
{
FIO_rust_compress_dst_context_t* const context =
(FIO_rust_compress_dst_context_t*)opaque;
/* Add the handler only after FIO_openDstFile() succeeds. */
addHandler(context->dstFileName);
}
static int FIO_rust_compressDestinationFile(void* opaque,
const char* dstFileName,
const char* srcFileName,
int compressionLevel)
{
FIO_rust_compress_dst_context_t* const context =
(FIO_rust_compress_dst_context_t*)opaque;
return FIO_compressFilename_internal(
context->fCtx, context->prefs, *context->ress,
dstFileName, srcFileName, compressionLevel);
}
static void FIO_rust_compressClearHandler(void* opaque)
{
(void)opaque;
clearHandler();
}
static void FIO_rust_compressSetFDStat(void* opaque, int dstFd,
const char* dstFileName)
{
FIO_rust_compress_dst_context_t* const context =
(FIO_rust_compress_dst_context_t*)opaque;
UTIL_setFDStat(dstFd, dstFileName, context->srcFileStat);
}
static int FIO_rust_compressDestinationClose(void* opaque)
{
FIO_rust_compress_dst_context_t* const context =
(FIO_rust_compress_dst_context_t*)opaque;
int const result = AIO_WritePool_closeFile(context->ress->writeCtx);
context->dstFile = NULL;
if (result) {
DISPLAYLEVEL(1, "zstd: %s: %s \n", context->dstFileName, strerror(errno));
return 1;
}
return 0;
}
static void FIO_rust_compressDestinationUtime(void* opaque)
{
FIO_rust_compress_dst_context_t* const context =
(FIO_rust_compress_dst_context_t*)opaque;
UTIL_utime(context->dstFileName, context->srcFileStat);
}
static void FIO_rust_compressDestinationRemove(void* opaque,
const char* dstFileName)
{
(void)opaque;
(void)FIO_removeFile(dstFileName);
}
/*! FIO_compressFilename_dstFile() :
* open the destination, or pass through if the write pool already owns it,
* then start compression with FIO_compressFilename_internal().
* Rust owns the lifecycle ordering and scalar metadata-transfer policy; C
* retains opaque resources, the private stat probe, metadata operations, and
* format-specific compression callbacks.
* note : ress.readCtx must already have a source file attached,
* so reach this function through FIO_compressFilename_srcFile().
* @return : 0 : compression completed correctly,
* 1 : pb
*/
static int FIO_compressFilename_dstFile(FIO_ctx_t* const fCtx,
FIO_prefs_t* const prefs,
cRess_t ress,
const char* dstFileName,
const char* srcFileName,
const stat_t* srcFileStat,
int compressionLevel)
{
FIO_rust_compress_dst_context_t context;
FIO_rust_compress_dst_projection_t projection;
assert(AIO_ReadPool_getFile(ress.readCtx) != NULL);
memset(&context, 0, sizeof(context));
context.fCtx = fCtx;
context.prefs = prefs;
context.ress = &ress;
context.dstFileName = dstFileName;
context.srcFileStat = srcFileStat;
memset(&projection, 0, sizeof(projection));
projection.opaque = &context;
projection.dstFileName = dstFileName;
projection.srcFileName = srcFileName;
projection.compressionLevel = compressionLevel;
projection.destinationAlreadyOpen = AIO_WritePool_getFile(ress.writeCtx) != NULL;
projection.sourceIsStdin = !strcmp(srcFileName, stdinmark);
projection.destinationIsStdout = !strcmp(dstFileName, stdoutmark);
/* C probes the private stat_t; Rust owns the stdin/stdout exceptions and
* normalizes the scalar consumed by the Rust destination lifecycle. */
projection.sourceIsRegular = FIO_rust_compressDestinationTransferStat(
projection.sourceIsStdin,
projection.destinationIsStdout,
UTIL_isRegularFileStat(srcFileStat));
projection.openDestination = FIO_rust_compressDestinationOpen;
projection.attachDestination = FIO_rust_compressDestinationAttach;
projection.addHandler = FIO_rust_compressAddHandler;
projection.compress = FIO_rust_compressDestinationFile;
projection.clearHandler = FIO_rust_compressClearHandler;
projection.setFDStat = FIO_rust_compressSetFDStat;
projection.closeDestination = FIO_rust_compressDestinationClose;
projection.utimeDestination = FIO_rust_compressDestinationUtime;
projection.removeDestination = FIO_rust_compressDestinationRemove;
return FIO_rust_compressFilenameDstFile(&projection);
}
typedef struct {
FIO_ctx_t* fCtx;
FIO_prefs_t* prefs;
cRess_t* ress;
stat_t srcFileStat;
FILE* srcFile;
} FIO_rust_compress_src_context_t;
static int FIO_rust_compressSourceStat(void* opaque, const char* srcFileName)
{
FIO_rust_compress_src_context_t* const context =
(FIO_rust_compress_src_context_t*)opaque;
if (!UTIL_stat(srcFileName, &context->srcFileStat)) {
/* Failure to stat at all is handled during opening. */
return FIO_RUST_COMPRESS_SRC_STAT_FAILED;
}
if (UTIL_isDirectoryStat(&context->srcFileStat)) {
DISPLAYLEVEL(1, "zstd: %s is a directory -- ignored \n", srcFileName);
return FIO_RUST_COMPRESS_SRC_DIRECTORY;
}
if (context->ress->dictFileName != NULL
&& UTIL_isSameFileStat(srcFileName, context->ress->dictFileName,
&context->srcFileStat, &context->ress->dictFileStat)) {
DISPLAYLEVEL(1, "zstd: cannot use %s as an input file and dictionary \n", srcFileName);
return FIO_RUST_COMPRESS_SRC_DICT_COLLISION;
}
return FIO_RUST_COMPRESS_SRC_STAT_OK;
}
static int FIO_rust_compressSourceOpen(void* opaque, const char* srcFileName,
U64* fileSize)
{
FIO_rust_compress_src_context_t* const context =
(FIO_rust_compress_src_context_t*)opaque;
context->srcFile = FIO_openSrcFile(context->prefs, srcFileName, &context->srcFileStat);
if (context->srcFile == NULL)
return 1;
*fileSize = strcmp(srcFileName, stdinmark)
? UTIL_getFileSizeStat(&context->srcFileStat)
: UTIL_FILESIZE_UNKNOWN;
return FIO_RUST_COMPRESS_SRC_OPEN_OK;
}
static void FIO_rust_compressSourceSetAsync(void* opaque, int asyncMode)
{
FIO_rust_compress_src_context_t* const context =
(FIO_rust_compress_src_context_t*)opaque;
AIO_ReadPool_setAsync(context->ress->readCtx, asyncMode);
AIO_WritePool_setAsync(context->ress->writeCtx, asyncMode);
}
static void FIO_rust_compressSourceAttach(void* opaque)
{
FIO_rust_compress_src_context_t* const context =
(FIO_rust_compress_src_context_t*)opaque;
AIO_ReadPool_setFile(context->ress->readCtx, context->srcFile);
}
static int FIO_rust_compressSourceCompress(void* opaque, const char* dstFileName,
const char* srcFileName, int compressionLevel)
{
FIO_rust_compress_src_context_t* const context =
(FIO_rust_compress_src_context_t*)opaque;
return FIO_compressFilename_dstFile(
context->fCtx, context->prefs, *context->ress,
dstFileName, srcFileName, &context->srcFileStat, compressionLevel);
}
static void FIO_rust_compressSourceClose(void* opaque)
{
FIO_rust_compress_src_context_t* const context =
(FIO_rust_compress_src_context_t*)opaque;
(void)AIO_ReadPool_closeFile(context->ress->readCtx);
context->srcFile = NULL;
}
static void FIO_rust_compressSourceRemove(void* opaque, const char* srcFileName)
{
(void)opaque;
/* After this point Ctrl-C must not remove both source and destination. */
clearHandler();
if (FIO_removeFile(srcFileName))
EXM_THROW(1, "zstd: %s: %s", srcFileName, strerror(errno));
}
/*! FIO_compressFilename_srcFile() :
* @return : 0 : compression completed correctly,
* 1 : missing or pb opening srcFileName
*/
static int
FIO_compressFilename_srcFile(FIO_ctx_t* const fCtx,
FIO_prefs_t* const prefs,
cRess_t ress,
const char* dstFileName,
const char* srcFileName,
int compressionLevel)
{
FIO_rust_compress_src_context_t context;
FIO_rust_compress_src_projection_t projection;
DISPLAYLEVEL(6, "FIO_compressFilename_srcFile: %s \n", srcFileName);
memset(&context, 0, sizeof(context));
context.fCtx = fCtx;
context.prefs = prefs;
context.ress = &ress;
memset(&projection, 0, sizeof(projection));
projection.opaque = &context;
projection.dstFileName = dstFileName;
projection.srcFileName = srcFileName;
projection.compressionLevel = compressionLevel;
projection.sourceIsStdin = !strcmp(srcFileName, stdinmark);
projection.excludeCompressedFiles = prefs->excludeCompressedFiles == 1;
projection.removeSrcFile = prefs->removeSrcFile;
projection.statSource = FIO_rust_compressSourceStat;
projection.sourceIsExcluded = FIO_rust_compressSourceIsExcluded;
projection.openSource = FIO_rust_compressSourceOpen;
projection.setAsync = FIO_rust_compressSourceSetAsync;
projection.attachSource = FIO_rust_compressSourceAttach;
projection.compress = FIO_rust_compressSourceCompress;
projection.closeSource = FIO_rust_compressSourceClose;
projection.removeSource = FIO_rust_compressSourceRemove;
return FIO_rust_compressFilenameSrcFile(&projection);
}
void FIO_displayCompressionParameters(const FIO_prefs_t* prefs)
{
assert(g_display_prefs.displayLevel >= 4);
FIO_rust_displayCompressionParameters(prefs);
}
int FIO_compressFilename(FIO_ctx_t* const fCtx, FIO_prefs_t* const prefs, const char* dstFileName,
const char* srcFileName, const char* dictFileName,
int compressionLevel, ZSTD_compressionParameters comprParams)
{
cRess_t ress = FIO_createCResources(prefs, dictFileName, UTIL_getFileSize(srcFileName), compressionLevel, comprParams);
int const result = FIO_compressFilename_srcFile(fCtx, prefs, ress, dstFileName, srcFileName, compressionLevel);
#define DISPLAY_LEVEL_DEFAULT 2
FIO_freeCResources(&ress);
return result;
}
typedef struct {
FIO_ctx_t* fCtx;
FIO_prefs_t* prefs;
cRess_t* ress;
int compressionLevel;
const char* outFileName;
} FIO_rust_compress_multiple_context_t;
static int FIO_rust_compressMultipleDestinationWarning(void* opaque)
{
FIO_rust_compress_multiple_context_t* const context =
(FIO_rust_compress_multiple_context_t*)opaque;
return FIO_multiFilesConcatWarning(
context->fCtx, context->prefs, context->outFileName, 1 /* displayLevelCutoff */);
}
static void* FIO_rust_compressMultipleDestinationOpen(void* opaque)
{
FIO_rust_compress_multiple_context_t* const context =
(FIO_rust_compress_multiple_context_t*)opaque;
return (void*)FIO_openDstFile(
context->fCtx, context->prefs, NULL,
context->outFileName, DEFAULT_FILE_PERMISSIONS);
}
static void FIO_rust_compressMultipleDestinationAttach(void* opaque, void* destination)
{
FIO_rust_compress_multiple_context_t* const context =
(FIO_rust_compress_multiple_context_t*)opaque;
AIO_WritePool_setFile(context->ress->writeCtx, (FILE*)destination);
}
static int FIO_rust_compressMultipleDestinationClose(void* opaque)
{
FIO_rust_compress_multiple_context_t* const context =
(FIO_rust_compress_multiple_context_t*)opaque;
if (AIO_WritePool_closeFile(context->ress->writeCtx))
EXM_THROW(29, "Write error (%s) : cannot properly close %s",
strerror(errno), context->outFileName);
return 0;
}
static int FIO_rust_compressMultipleFileCallback(void* opaque,
const char* dstFileName,
const char* srcFileName)
{
FIO_rust_compress_multiple_context_t* const context =
(FIO_rust_compress_multiple_context_t*)opaque;
return FIO_compressFilename_srcFile(
context->fCtx, context->prefs, *context->ress,
dstFileName, srcFileName, context->compressionLevel);
}
typedef struct {
FIO_ctx_t* fCtx;
FIO_prefs_t* prefs;
cRess_t* ress;
const char* outMirroredRootDirName;
const char* outDirName;
const char* suffix;
int compressionLevel;
} FIO_rust_compress_multiple_separate_context_t;
static int FIO_rust_compressMultipleSeparateMirroredFileCallback(void* opaque,
const char* srcFileName)
{
FIO_rust_compress_multiple_separate_context_t* const context =
(FIO_rust_compress_multiple_separate_context_t*)opaque;
char* const validMirroredDirName = UTIL_createMirroredDestDirName(
srcFileName, context->outMirroredRootDirName);
if (validMirroredDirName) {
const char* const dstFileName = FIO_determineCompressedName(
srcFileName, validMirroredDirName, context->suffix);
free(validMirroredDirName);
return FIO_compressFilename_srcFile(
context->fCtx, context->prefs, *context->ress,
dstFileName, srcFileName, context->compressionLevel);
}
DISPLAYLEVEL(2, "zstd: --output-dir-mirror cannot compress '%s' into '%s' \n",
srcFileName, context->outMirroredRootDirName);
return 1;
}
static int FIO_rust_compressMultipleSeparateFlatFileCallback(void* opaque,
const char* srcFileName)
{
FIO_rust_compress_multiple_separate_context_t* const context =
(FIO_rust_compress_multiple_separate_context_t*)opaque;
const char* const dstFileName = FIO_determineCompressedName(
srcFileName, context->outDirName, context->suffix);
return FIO_compressFilename_srcFile(
context->fCtx, context->prefs, *context->ress,
dstFileName, srcFileName, context->compressionLevel);
}
/* FIO_compressMultipleFilenames() :
* compress nbFiles files
* into either one destination (outFileName),
* or into one file each (outFileName == NULL, but suffix != NULL),
* or into a destination folder (specified with -O)
*/
int FIO_compressMultipleFilenames(FIO_ctx_t* const fCtx,
FIO_prefs_t* const prefs,
const char** inFileNamesTable,
const char* outMirroredRootDirName,
const char* outDirName,
const char* outFileName, const char* suffix,
const char* dictFileName, int compressionLevel,
ZSTD_compressionParameters comprParams)
{
int error = 0;
cRess_t ress = FIO_createCResources(prefs, dictFileName,
FIO_getLargestFileSize(inFileNamesTable, (unsigned)fCtx->nbFilesTotal),
compressionLevel, comprParams);
/* init */
assert(outFileName != NULL || suffix != NULL);
if (outFileName != NULL) { /* output into a single destination (stdout typically) */
FIO_rust_compress_multiple_context_t callbackContext = {
fCtx, prefs, &ress, compressionLevel, outFileName
};
FIO_rust_compress_multiple_projection_t files = {
fCtx, inFileNamesTable, outFileName,
&callbackContext, FIO_rust_compressMultipleFileCallback
};
FIO_rust_compress_multiple_destination_projection_t projection = {
&files,
FIO_rust_compressMultipleDestinationWarning,
FIO_rust_compressMultipleDestinationOpen,
FIO_rust_compressMultipleDestinationAttach,
FIO_rust_compressMultipleDestinationClose
};
error = FIO_rust_compressMultipleFilenamesWithDestination(&projection);
} else {
FIO_rust_compress_multiple_separate_context_t callbackContext = {
fCtx, prefs, &ress, outMirroredRootDirName,
outDirName, suffix, compressionLevel
};
FIO_rust_compress_multiple_separate_projection_t projection = {
fCtx, inFileNamesTable, &callbackContext,
outMirroredRootDirName != NULL,
FIO_rust_compressMultipleSeparateMirroredFileCallback,
FIO_rust_compressMultipleSeparateFlatFileCallback
};
if (outMirroredRootDirName)
UTIL_mirrorSourceFilesDirectories(inFileNamesTable, (unsigned)fCtx->nbFilesTotal, outMirroredRootDirName);
error = FIO_rust_compressMultipleSeparateFilenames(&projection);
if (outDirName)
FIO_checkFilenameCollisions(inFileNamesTable , (unsigned)fCtx->nbFilesTotal);
}
FIO_displayMultipleFileSummary(
fCtx, FIO_RUST_MULTIPLE_SUMMARY_COMPRESSION);
FIO_freeCResources(&ress);
return error;
}
#endif /* #ifndef ZSTD_NOCOMPRESS */
#ifndef ZSTD_NODECOMPRESS
/* **************************************************************************
* Decompression
***************************************************************************/
typedef struct {
FIO_Dict_t dict;
stat_t dictFileStat;
ZSTD_DStream* dctx;
WritePoolCtx_t *writeCtx;
ReadPoolCtx_t *readCtx;
} dRess_t;
typedef void (*FIO_rust_createDResourcesPrepareDictionaryCallback_f)(
void* context, FIO_prefs_t* prefs, const char* dictFileName);
typedef void (*FIO_rust_createDResourcesCreateDctxCallback_f)(void* context);
typedef size_t (*FIO_rust_createDResourcesSetMaxWindowSizeCallback_f)(
void* context, unsigned memLimit);
typedef size_t (*FIO_rust_createDResourcesSetParameterCallback_f)(
void* context, int value);
typedef size_t (*FIO_rust_createDResourcesLoadDictionaryCallback_f)(
void* context, FIO_prefs_t* prefs, const char* dictFileName);
typedef void (*FIO_rust_createDResourcesCreatePoolCallback_f)(
void* context, const FIO_prefs_t* prefs);
typedef int (*FIO_rust_createDResourcesIsErrorCallback_f)(size_t result);
typedef struct {
void* callbackContext;
FIO_prefs_t* prefs;
const char* dictFileName;
FIO_rust_createDResourcesPrepareDictionaryCallback_f prepareDictionary;
FIO_rust_createDResourcesCreateDctxCallback_f createDctx;
FIO_rust_createDResourcesSetMaxWindowSizeCallback_f setMaxWindowSize;
FIO_rust_createDResourcesSetParameterCallback_f setParameter;
FIO_rust_createDResourcesLoadDictionaryCallback_f loadDictionary;
FIO_rust_createDResourcesCreatePoolCallback_f createWritePool;
FIO_rust_createDResourcesCreatePoolCallback_f createReadPool;
FIO_rust_createDResourcesIsErrorCallback_f isError;
} FIO_rust_createDResourcesState;
typedef char FIO_rust_create_d_resources_state_layout[
(offsetof(FIO_rust_createDResourcesState, callbackContext) == 0
&& offsetof(FIO_rust_createDResourcesState, prefs) == sizeof(void*)
&& offsetof(FIO_rust_createDResourcesState, dictFileName)
== 2 * sizeof(void*)
&& offsetof(FIO_rust_createDResourcesState, prepareDictionary)
== 3 * sizeof(void*)
&& offsetof(FIO_rust_createDResourcesState, createDctx)
== 4 * sizeof(void*)
&& offsetof(FIO_rust_createDResourcesState, setMaxWindowSize)
== 5 * sizeof(void*)
&& offsetof(FIO_rust_createDResourcesState, setParameter)
== 6 * sizeof(void*)
&& offsetof(FIO_rust_createDResourcesState, loadDictionary)
== 7 * sizeof(void*)
&& offsetof(FIO_rust_createDResourcesState, createWritePool)
== 8 * sizeof(void*)
&& offsetof(FIO_rust_createDResourcesState, createReadPool)
== 9 * sizeof(void*)
&& offsetof(FIO_rust_createDResourcesState, isError)
== 10 * sizeof(void*)
&& sizeof(FIO_rust_createDResourcesState) == 11 * sizeof(void*))
? 1 : -1];
size_t FIO_rust_createDResources(const FIO_rust_createDResourcesState* state);
typedef void (*FIO_rust_freeDResourcesCallback_f)(void* context);
typedef struct {
void* callbackContext;
FIO_rust_freeDResourcesCallback_f freeDict;
FIO_rust_freeDResourcesCallback_f freeDctx;
FIO_rust_freeDResourcesCallback_f freeWritePool;
FIO_rust_freeDResourcesCallback_f freeReadPool;
} FIO_rust_freeDResourcesState;
typedef char FIO_rust_free_d_resources_state_layout[
(offsetof(FIO_rust_freeDResourcesState, callbackContext) == 0
&& offsetof(FIO_rust_freeDResourcesState, freeDict) == sizeof(void*)
&& offsetof(FIO_rust_freeDResourcesState, freeDctx)
== 2 * sizeof(void*)
&& offsetof(FIO_rust_freeDResourcesState, freeWritePool)
== 3 * sizeof(void*)
&& offsetof(FIO_rust_freeDResourcesState, freeReadPool)
== 4 * sizeof(void*)
&& sizeof(FIO_rust_freeDResourcesState) == 5 * sizeof(void*))
? 1 : -1];
void FIO_rust_freeDResources(const FIO_rust_freeDResourcesState* state);
static void FIO_rust_freeDResources_dict(void* context)
{
dRess_t* const ress = (dRess_t*)context;
FIO_freeDict(&(ress->dict));
}
static void FIO_rust_freeDResources_dctx(void* context)
{
dRess_t* const ress = (dRess_t*)context;
CHECK( ZSTD_freeDStream(ress->dctx) );
}
static void FIO_rust_freeDResources_writePool(void* context)
{
dRess_t* const ress = (dRess_t*)context;
AIO_WritePool_free(ress->writeCtx);
}
static void FIO_rust_freeDResources_readPool(void* context)
{
dRess_t* const ress = (dRess_t*)context;
AIO_ReadPool_free(ress->readCtx);
}
static void FIO_rust_createDResources_prepareDictionary(
void* context, FIO_prefs_t* prefs, const char* dictFileName)
{
dRess_t* const ress = (dRess_t*)context;
FIO_getDictFileStat(dictFileName, &ress->dictFileStat);
if (prefs->patchFromMode) {
unsigned long long const dictSize =
(unsigned long long)UTIL_getFileSizeStat(&ress->dictFileStat);
FIO_adjustMemLimitForPatchFromMode(
prefs, dictSize, 0 /* just use the dict size */);
}
}
static void FIO_rust_createDResources_createDctx(void* context)
{
dRess_t* const ress = (dRess_t*)context;
ress->dctx = ZSTD_createDStream();
if (ress->dctx == NULL)
EXM_THROW(60, "Error: %s : can't create ZSTD_DStream", strerror(errno));
}
static size_t FIO_rust_createDResources_setMaxWindowSize(
void* context, unsigned memLimit)
{
dRess_t* const ress = (dRess_t*)context;
return ZSTD_DCtx_setMaxWindowSize(ress->dctx, memLimit);
}
static size_t FIO_rust_createDResources_setParameter(void* context, int value)
{
dRess_t* const ress = (dRess_t*)context;
return ZSTD_DCtx_setParameter(
ress->dctx, ZSTD_d_forceIgnoreChecksum, value);
}
static size_t FIO_rust_createDResources_loadDictionary(
void* context, FIO_prefs_t* prefs, const char* dictFileName)
{
dRess_t* const ress = (dRess_t*)context;
unsigned long long const dictSize = prefs->patchFromMode
? (unsigned long long)UTIL_getFileSizeStat(&ress->dictFileStat)
: 0;
FIO_dictBufferType_t const dictBufferType =
(FIO_dictBufferType_t)FIO_rust_selectDictBufferType(
prefs->mmapDict, prefs->patchFromMode, dictSize,
prefs->memLimit);
FIO_initDict(&ress->dict, dictFileName, prefs, &ress->dictFileStat,
dictBufferType);
{ size_t const result = ZSTD_DCtx_reset(
ress->dctx, ZSTD_reset_session_only);
if (ZSTD_isError(result)) return result;
}
if (prefs->patchFromMode) {
return ZSTD_DCtx_refPrefix(
ress->dctx, ress->dict.dictBuffer, ress->dict.dictBufferSize);
}
return ZSTD_DCtx_loadDictionary_byReference(
ress->dctx, ress->dict.dictBuffer, ress->dict.dictBufferSize);
}
static void FIO_rust_createDResources_createWritePool(
void* context, const FIO_prefs_t* prefs)
{
dRess_t* const ress = (dRess_t*)context;
ress->writeCtx = AIO_WritePool_create(prefs, ZSTD_DStreamOutSize());
}
static void FIO_rust_createDResources_createReadPool(
void* context, const FIO_prefs_t* prefs)
{
dRess_t* const ress = (dRess_t*)context;
ress->readCtx = AIO_ReadPool_create(prefs, ZSTD_DStreamInSize());
}
static int FIO_rust_createDResources_isError(size_t result)
{
return ZSTD_isError(result);
}
static dRess_t FIO_createDResources(FIO_prefs_t* const prefs, const char* dictFileName)
{
dRess_t ress;
FIO_rust_createDResourcesState const state = {
&ress,
prefs,
dictFileName,
FIO_rust_createDResources_prepareDictionary,
FIO_rust_createDResources_createDctx,
FIO_rust_createDResources_setMaxWindowSize,
FIO_rust_createDResources_setParameter,
FIO_rust_createDResources_loadDictionary,
FIO_rust_createDResources_createWritePool,
FIO_rust_createDResources_createReadPool,
FIO_rust_createDResources_isError
};
memset(&ress, 0, sizeof(ress));
CHECK(FIO_rust_createDResources(&state));
return ress;
}
static void FIO_freeDResources(dRess_t ress)
{
FIO_rust_freeDResourcesState const state = {
&ress,
FIO_rust_freeDResources_dict,
FIO_rust_freeDResources_dctx,
FIO_rust_freeDResources_writePool,
FIO_rust_freeDResources_readPool
};
FIO_rust_freeDResources(&state);
}
enum {
FIO_RUST_ZSTD_ERROR_HELP_WINDOW_SIZE = 0,
FIO_RUST_ZSTD_ERROR_HELP_WINDOW_GUIDANCE = 1,
FIO_RUST_ZSTD_ERROR_HELP_UNSUPPORTED_WINDOW_LOG = 2
};
typedef void (*FIO_rust_zstdErrorHelpDisplayFn)(
void* callbackContext,
int diagnostic,
const char* srcFileName,
unsigned long long windowSize,
unsigned windowLog,
unsigned windowMB,
unsigned memLimit);
typedef struct {
void* callbackContext;
const char* srcFileName;
unsigned long long windowSize;
size_t headerError;
unsigned windowLog; /* Rust derives this from windowSize; retain ABI layout. */
unsigned memLimit;
int errorCode;
FIO_rust_zstdErrorHelpDisplayFn display;
} FIO_rust_zstdErrorHelpState;
typedef char FIO_rust_zstd_error_help_state_layout[
(offsetof(FIO_rust_zstdErrorHelpState, callbackContext) == 0
&& offsetof(FIO_rust_zstdErrorHelpState, srcFileName)
== sizeof(void*)
&& offsetof(FIO_rust_zstdErrorHelpState, windowSize)
== 2 * sizeof(void*)
&& offsetof(FIO_rust_zstdErrorHelpState, headerError)
== offsetof(FIO_rust_zstdErrorHelpState, windowSize)
+ sizeof(unsigned long long)
&& offsetof(FIO_rust_zstdErrorHelpState, windowLog)
== offsetof(FIO_rust_zstdErrorHelpState, headerError)
+ sizeof(size_t)
&& offsetof(FIO_rust_zstdErrorHelpState, memLimit)
== offsetof(FIO_rust_zstdErrorHelpState, windowLog)
+ sizeof(unsigned)
&& offsetof(FIO_rust_zstdErrorHelpState, errorCode)
== offsetof(FIO_rust_zstdErrorHelpState, memLimit)
+ sizeof(unsigned)
&& offsetof(FIO_rust_zstdErrorHelpState, display)
== ((offsetof(FIO_rust_zstdErrorHelpState, errorCode)
+ sizeof(int) + sizeof(void*) - 1)
/ sizeof(void*)) * sizeof(void*)
&& sizeof(FIO_rust_zstdErrorHelpState)
== offsetof(FIO_rust_zstdErrorHelpState, display)
+ sizeof(void*)
&& sizeof(FIO_rust_zstdErrorHelpDisplayFn) == sizeof(void*))
? 1 : -1];
void FIO_rust_zstdErrorHelp(const FIO_rust_zstdErrorHelpState* state);
static void FIO_rust_zstdErrorHelp_display(
void* callbackContext,
int diagnostic,
const char* srcFileName,
unsigned long long windowSize,
unsigned windowLog,
unsigned windowMB,
unsigned memLimit)
{
(void)callbackContext;
if (diagnostic == FIO_RUST_ZSTD_ERROR_HELP_WINDOW_SIZE) {
DISPLAYLEVEL(1, "%s : Window size larger than maximum : %llu > %u \n",
srcFileName, windowSize, memLimit);
return;
}
if (diagnostic == FIO_RUST_ZSTD_ERROR_HELP_WINDOW_GUIDANCE) {
DISPLAYLEVEL(1, "%s : Use --long=%u or --memory=%uMB \n",
srcFileName, windowLog, windowMB);
return;
}
assert(diagnostic == FIO_RUST_ZSTD_ERROR_HELP_UNSUPPORTED_WINDOW_LOG);
DISPLAYLEVEL(1, "%s : Window log larger than ZSTD_WINDOWLOG_MAX=%u; not supported \n",
srcFileName, ZSTD_WINDOWLOG_MAX);
}
/* FIO_zstdErrorHelp() :
* detailed error message when requested window size is too large */
static void
FIO_zstdErrorHelp(const FIO_prefs_t* const prefs,
const dRess_t* ress,
size_t err,
const char* srcFileName)
{
ZSTD_FrameHeader header;
int const errorCode = (int)ZSTD_getErrorCode(err);
size_t headerError = 1;
unsigned long long windowSize = 0;
unsigned memLimit = 0;
FIO_rust_zstdErrorHelpState state = { 0 };
if (errorCode == ZSTD_error_frameParameter_windowTooLarge) {
/* Keep codec/header parsing and the private read-pool layout in C. */
headerError = ZSTD_getFrameHeader(
&header, ress->readCtx->srcBuffer, ress->readCtx->srcBufferLoaded);
if (headerError == 0) {
windowSize = header.windowSize;
memLimit = prefs->memLimit;
}
}
state.callbackContext = NULL;
state.srcFileName = srcFileName;
state.windowSize = windowSize;
state.headerError = headerError;
state.memLimit = memLimit;
state.errorCode = errorCode;
state.display = FIO_rust_zstdErrorHelp_display;
FIO_rust_zstdErrorHelp(&state);
}
/** FIO_decompressFrame() :
* @return : size of decoded zstd frame, or an error code
*/
#define FIO_ERROR_FRAME_DECODING ((unsigned long long)(-2))
static void
FIO_decompressZstdFrameProgress(void* const opaque,
const char* const srcFileName,
U64 const decodedSize)
{
FIO_ctx_t* const fCtx = (FIO_ctx_t*)opaque;
const char* srcFName20 = srcFileName;
size_t const srcFileLength = strlen(srcFileName);
UTIL_HumanReadableSize_t const hrs = UTIL_makeHumanReadableSize(decodedSize);
/* display last 20 characters only when not --verbose */
if ((srcFileLength>20) && (g_display_prefs.displayLevel<3))
srcFName20 += srcFileLength-20;
if (fCtx->nbFilesTotal > 1) {
DISPLAYUPDATE_PROGRESS(
"\rDecompress: %2u/%2u files. Current: %s : %.*f%s... ",
fCtx->currFileIdx+1, fCtx->nbFilesTotal, srcFName20,
hrs.precision, hrs.value, hrs.suffix);
} else {
DISPLAYUPDATE_PROGRESS("\r%-20.20s : %.*f%s... ",
srcFName20, hrs.precision, hrs.value, hrs.suffix);
}
}
/* Rust owns the mixed-format loop, while these adapters keep each codec and
* the private dRess_t layout in this C translation unit. */
typedef struct {
FIO_ctx_t* fCtx;
dRess_t* ress;
const FIO_prefs_t* prefs;
} FIO_rust_decompression_projection_t;
#if defined(ZSTD_GZDECOMPRESS) || defined(ZSTD_LZMADECOMPRESS) || defined(ZSTD_LZ4DECOMPRESS)
static void FIO_rust_decompressReadFill(void* opaque, size_t requested,
const unsigned char** buffer, size_t* loaded)
{
ReadPoolCtx_t* const readCtx = (ReadPoolCtx_t*)opaque;
AIO_ReadPool_fillBuffer(readCtx, requested);
*buffer = (const unsigned char*)readCtx->srcBuffer;
*loaded = readCtx->srcBufferLoaded;
}
static void FIO_rust_decompressReadConsume(void* opaque, size_t consumed)
{
AIO_ReadPool_consumeBytes((ReadPoolCtx_t*)opaque, consumed);
}
static void FIO_rust_decompressWriteAcquire(void* opaque, void** job,
unsigned char** buffer, size_t* bufferSize)
{
IOJob_t* const writeJob = AIO_WritePool_acquireJob((WritePoolCtx_t*)opaque);
*job = writeJob;
*buffer = (unsigned char*)writeJob->buffer;
*bufferSize = writeJob->bufferSize;
}
static void FIO_rust_decompressWriteEnqueue(void* opaque, void** job,
size_t usedBufferSize,
unsigned char** buffer, size_t* bufferSize)
{
IOJob_t* writeJob = (IOJob_t*)*job;
writeJob->usedBufferSize = usedBufferSize;
AIO_WritePool_enqueueAndReacquireWriteJob(&writeJob);
*job = writeJob;
*buffer = (unsigned char*)writeJob->buffer;
*bufferSize = writeJob->bufferSize;
(void)opaque;
}
static void FIO_rust_decompressWriteRelease(void* opaque, void* job)
{
AIO_WritePool_releaseIoJob((IOJob_t*)job);
(void)opaque;
}
static void FIO_rust_decompressSparseWriteEnd(void* opaque)
{
AIO_WritePool_sparseWriteEnd((WritePoolCtx_t*)opaque);
}
static void FIO_rust_decompressInitIo(FIO_rust_decompress_io_projection_t* io,
const dRess_t* ress)
{
memset(io, 0, sizeof(*io));
io->readOpaque = (void*)ress->readCtx;
io->writeOpaque = (void*)ress->writeCtx;
io->readBufferSize = ZSTD_DStreamInSize();
io->readFill = FIO_rust_decompressReadFill;
io->readConsume = FIO_rust_decompressReadConsume;
io->writeAcquire = FIO_rust_decompressWriteAcquire;
io->writeEnqueue = FIO_rust_decompressWriteEnqueue;
io->writeRelease = FIO_rust_decompressWriteRelease;
io->sparseWriteEnd = FIO_rust_decompressSparseWriteEnd;
}
#endif
#ifdef ZSTD_GZDECOMPRESS
static int FIO_rust_gzip_decompressInit(void* opaque)
{
z_stream* const strm = (z_stream*)opaque;
strm->zalloc = Z_NULL;
strm->zfree = Z_NULL;
strm->opaque = Z_NULL;
return inflateInit2(strm, 15 /* maxWindowLogSize */ + 16 /* gzip only */);
}
static int FIO_rust_gzip_decompressInflate(void* opaque,
const unsigned char* input, size_t inputSize,
unsigned char* output, size_t outputSize,
int flush, size_t* consumed, size_t* produced)
{
z_stream* const strm = (z_stream*)opaque;
size_t const inputBefore = inputSize;
size_t const outputBefore = outputSize;
int ret;
assert(inputSize <= UINT_MAX);
assert(outputSize <= UINT_MAX);
strm->next_in = (z_const unsigned char*)input;
strm->avail_in = (uInt)inputSize;
strm->next_out = (Bytef*)output;
strm->avail_out = (uInt)outputSize;
ret = inflate(strm, flush);
*consumed = inputBefore - strm->avail_in;
*produced = outputBefore - strm->avail_out;
return ret;
}
static int FIO_rust_gzip_decompressEnd(void* opaque)
{
return inflateEnd((z_stream*)opaque);
}
static int FIO_rust_decompressGzFrameCallback(void* opaque,
const char* srcFileName,
U64 alreadyDecoded,
U64* frameSize,
size_t* errorCode,
int mode)
{
FIO_rust_decompression_projection_t* const projection =
(FIO_rust_decompression_projection_t*)opaque;
FIO_rust_gzip_decompress_projection_t codecProjection;
z_stream strm;
U64 decodedSize = 0;
int zlibResult = Z_OK;
int status;
(void)alreadyDecoded;
(void)mode;
memset(&strm, 0, sizeof(strm));
memset(&codecProjection, 0, sizeof(codecProjection));
FIO_rust_decompressInitIo(&codecProjection.io, projection->ress);
codecProjection.zlibOpaque = &strm;
codecProjection.zlibInit = FIO_rust_gzip_decompressInit;
codecProjection.zlibInflate = FIO_rust_gzip_decompressInflate;
codecProjection.zlibEnd = FIO_rust_gzip_decompressEnd;
status = FIO_rust_decompressGzipFrame(&codecProjection, &decodedSize, &zlibResult);
*errorCode = 0;
*frameSize = status == FIO_RUST_GZIP_DECOMPRESS_OK
? decodedSize : FIO_ERROR_FRAME_DECODING;
switch (status) {
case FIO_RUST_GZIP_DECOMPRESS_OK:
break;
case FIO_RUST_GZIP_DECOMPRESS_BUF_ERROR:
DISPLAYLEVEL(1, "zstd: %s: premature gz end \n", srcFileName);
break;
case FIO_RUST_GZIP_DECOMPRESS_INFLATE_ERROR:
DISPLAYLEVEL(1, "zstd: %s: inflate error %d \n", srcFileName, zlibResult);
break;
case FIO_RUST_GZIP_DECOMPRESS_END_ERROR:
DISPLAYLEVEL(1, "zstd: %s: inflateEnd error \n", srcFileName);
break;
default:
assert(status == FIO_RUST_GZIP_DECOMPRESS_INIT_ERROR
|| status == FIO_RUST_GZIP_DECOMPRESS_INVALID_PROJECTION);
break;
}
return status != FIO_RUST_GZIP_DECOMPRESS_OK;
}
#endif
#ifdef ZSTD_LZMADECOMPRESS
static int FIO_rust_lzma_decompressInit(void* opaque, int plainLzma)
{
lzma_stream* const strm = (lzma_stream*)opaque;
if (plainLzma)
return (int)lzma_alone_decoder(strm, UINT64_MAX); /* LZMA */
return (int)lzma_stream_decoder(strm, UINT64_MAX, 0); /* XZ */
}
static int FIO_rust_lzma_decompressCode(void* opaque,
const unsigned char* input, size_t inputSize,
unsigned char* output, size_t outputSize,
int action, size_t* consumed, size_t* produced)
{
lzma_stream* const strm = (lzma_stream*)opaque;
size_t const inputBefore = inputSize;
size_t const outputBefore = outputSize;
lzma_ret ret;
strm->next_in = (const uint8_t*)input;
strm->avail_in = inputSize;
strm->next_out = (uint8_t*)output;
strm->avail_out = outputSize;
ret = lzma_code(strm, (lzma_action)action);
*consumed = inputBefore - strm->avail_in;
*produced = outputBefore - strm->avail_out;
return (int)ret;
}
static void FIO_rust_lzma_decompressEnd(void* opaque)
{
lzma_end((lzma_stream*)opaque);
}
static int FIO_rust_decompressLzmaFrameCallback(void* opaque,
const char* srcFileName,
U64 alreadyDecoded,
U64* frameSize,
size_t* errorCode,
int mode)
{
FIO_rust_decompression_projection_t* const projection =
(FIO_rust_decompression_projection_t*)opaque;
FIO_rust_lzma_decompress_projection_t codecProjection;
lzma_stream strm = LZMA_STREAM_INIT;
U64 decodedSize = 0;
int lzmaResult = (int)LZMA_OK;
int status;
(void)alreadyDecoded;
memset(&codecProjection, 0, sizeof(codecProjection));
FIO_rust_decompressInitIo(&codecProjection.io, projection->ress);
codecProjection.lzmaOpaque = &strm;
codecProjection.lzmaInit = FIO_rust_lzma_decompressInit;
codecProjection.lzmaCode = FIO_rust_lzma_decompressCode;
codecProjection.lzmaEnd = FIO_rust_lzma_decompressEnd;
status = FIO_rust_decompressLzmaFrame(
&codecProjection, mode, &decodedSize, &lzmaResult);
*errorCode = 0;
*frameSize = status == FIO_RUST_LZMA_DECOMPRESS_OK
? decodedSize : FIO_ERROR_FRAME_DECODING;
switch (status) {
case FIO_RUST_LZMA_DECOMPRESS_OK:
break;
case FIO_RUST_LZMA_DECOMPRESS_INIT_ERROR:
DISPLAYLEVEL(1, "zstd: %s: %s error %d \n",
mode ? "lzma_alone_decoder" : "lzma_stream_decoder",
srcFileName, lzmaResult);
break;
case FIO_RUST_LZMA_DECOMPRESS_BUF_ERROR:
DISPLAYLEVEL(1, "zstd: %s: premature lzma end \n", srcFileName);
break;
case FIO_RUST_LZMA_DECOMPRESS_CODE_ERROR:
DISPLAYLEVEL(1, "zstd: %s: lzma_code decoding error %d \n",
srcFileName, lzmaResult);
break;
default:
assert(status == FIO_RUST_LZMA_DECOMPRESS_INVALID_PROJECTION);
break;
}
return status != FIO_RUST_LZMA_DECOMPRESS_OK;
}
#endif
#ifdef ZSTD_LZ4DECOMPRESS
static int FIO_rust_lz4_decompressCreate(void* opaque, unsigned version, size_t* result)
{
LZ4F_errorCode_t const errorCode = LZ4F_createDecompressionContext(
(LZ4F_decompressionContext_t*)opaque, version);
*result = (size_t)errorCode;
return LZ4F_isError(errorCode);
}
static int FIO_rust_lz4_decompressCode(void* opaque, unsigned char* output,
size_t* outputSize, const unsigned char* input,
size_t* inputSize, size_t* nextToLoad)
{
LZ4F_errorCode_t const result = LZ4F_decompress(
*(LZ4F_decompressionContext_t*)opaque, output, outputSize,
input, inputSize, NULL);
*nextToLoad = (size_t)result;
return LZ4F_isError(result);
}
static void FIO_rust_lz4_decompressFree(void* opaque)
{
LZ4F_freeDecompressionContext(*(LZ4F_decompressionContext_t*)opaque);
}
static void FIO_rust_lz4_decompressProgress(void* opaque, U64 decodedSize)
{
UTIL_HumanReadableSize_t const hrs = UTIL_makeHumanReadableSize(decodedSize);
(void)opaque;
DISPLAYUPDATE_PROGRESS("\rDecompressed : %.*f%s ",
hrs.precision, hrs.value, hrs.suffix);
}
static int FIO_rust_decompressLz4FrameCallback(void* opaque,
const char* srcFileName,
U64 alreadyDecoded,
U64* frameSize,
size_t* errorCode,
int mode)
{
FIO_rust_decompression_projection_t* const projection =
(FIO_rust_decompression_projection_t*)opaque;
FIO_rust_lz4_decompress_projection_t codecProjection;
LZ4F_decompressionContext_t dCtx;
U64 decodedSize = 0;
size_t lz4Result = 0;
int status;
(void)alreadyDecoded;
(void)mode;
memset(&codecProjection, 0, sizeof(codecProjection));
FIO_rust_decompressInitIo(&codecProjection.io, projection->ress);
codecProjection.codecOpaque = &dCtx;
codecProjection.progressOpaque = NULL;
codecProjection.version = LZ4F_VERSION;
codecProjection.create = FIO_rust_lz4_decompressCreate;
codecProjection.code = FIO_rust_lz4_decompressCode;
codecProjection.freeContext = FIO_rust_lz4_decompressFree;
codecProjection.progress = FIO_rust_lz4_decompressProgress;
status = FIO_rust_decompressLz4Frame(&codecProjection, &decodedSize, &lz4Result);
*errorCode = 0;
*frameSize = status == FIO_RUST_LZ4_DECOMPRESS_OK
? decodedSize : FIO_ERROR_FRAME_DECODING;
switch (status) {
case FIO_RUST_LZ4_DECOMPRESS_OK:
break;
case FIO_RUST_LZ4_DECOMPRESS_CREATE_ERROR:
DISPLAYLEVEL(1, "zstd: failed to create lz4 decompression context \n");
break;
case FIO_RUST_LZ4_DECOMPRESS_CODE_ERROR:
DISPLAYLEVEL(1, "zstd: %s: lz4 decompression error : %s \n",
srcFileName, LZ4F_getErrorName(lz4Result));
break;
case FIO_RUST_LZ4_DECOMPRESS_UNFINISHED:
DISPLAYLEVEL(1, "zstd: %s: unfinished lz4 stream \n", srcFileName);
break;
default:
assert(status == FIO_RUST_LZ4_DECOMPRESS_INVALID_PROJECTION);
break;
}
return status != FIO_RUST_LZ4_DECOMPRESS_OK;
}
#endif
static void FIO_rust_decompressZstdFrame_displayDecodingError(
void* opaque, const char* srcFileName, size_t zstdError)
{
FIO_rust_decompression_projection_t* const projection =
(FIO_rust_decompression_projection_t*)opaque;
DISPLAYLEVEL(1, "%s : Decoding error (36) : %s \n",
srcFileName, ZSTD_getErrorName(zstdError));
FIO_zstdErrorHelp(projection->prefs, projection->ress,
zstdError, srcFileName);
}
static void FIO_rust_decompressZstdFrame_displayPrematureEnd(
void* opaque, const char* srcFileName)
{
(void)opaque;
DISPLAYLEVEL(1, "%s : Read error (39) : premature end \n",
srcFileName);
}
static int FIO_rust_decompressZstdFrameCallback(void* opaque,
const char* srcFileName,
U64 alreadyDecoded,
U64* frameSize,
size_t* errorCode,
int mode)
{
FIO_rust_decompression_projection_t* const projection =
(FIO_rust_decompression_projection_t*)opaque;
FIO_rust_zstd_frame_policy_state state;
(void)mode;
state.callbackContext = projection;
state.fCtx = projection->fCtx;
state.dctx = projection->ress->dctx;
state.readCtx = projection->ress->readCtx;
state.writeCtx = projection->ress->writeCtx;
state.srcFileName = srcFileName;
state.alreadyDecoded = alreadyDecoded;
state.progress = FIO_decompressZstdFrameProgress;
state.displayDecodingError =
FIO_rust_decompressZstdFrame_displayDecodingError;
state.displayPrematureEnd =
FIO_rust_decompressZstdFrame_displayPrematureEnd;
*errorCode = 0;
*frameSize = FIO_rust_decompressZstdFramePolicy(&state);
return *frameSize == FIO_ERROR_FRAME_DECODING;
}
/* FIO_passThrough() : just copy input into output, for compatibility with gzip -df mode
* @return : 0 (no error) */
static int FIO_rust_decompressPassThroughCallback(void* opaque)
{
FIO_rust_decompression_projection_t* const projection =
(FIO_rust_decompression_projection_t*)opaque;
return FIO_rust_passThrough(projection->ress->readCtx,
projection->ress->writeCtx);
}
static void FIO_rust_decompressStatusCallback(void* opaque,
int status,
const char* srcFileName)
{
(void)opaque;
switch (FIO_rust_decompressStatusAction(status)) {
case FIO_RUST_DECOMPRESS_ACTION_EMPTY_INPUT:
DISPLAYLEVEL(1, "zstd: %s: unexpected end of file \n", srcFileName);
break;
case FIO_RUST_DECOMPRESS_ACTION_SHORT_INPUT:
DISPLAYLEVEL(1, "zstd: %s: unknown header \n", srcFileName);
break;
case FIO_RUST_DECOMPRESS_ACTION_GZIP_UNSUPPORTED:
DISPLAYLEVEL(1, "zstd: %s: gzip file cannot be uncompressed (zstd compiled without HAVE_ZLIB) -- ignored \n", srcFileName);
break;
case FIO_RUST_DECOMPRESS_ACTION_LZMA_UNSUPPORTED:
DISPLAYLEVEL(1, "zstd: %s: xz/lzma file cannot be uncompressed (zstd compiled without HAVE_LZMA) -- ignored \n", srcFileName);
break;
case FIO_RUST_DECOMPRESS_ACTION_LZ4_UNSUPPORTED:
DISPLAYLEVEL(1, "zstd: %s: lz4 file cannot be uncompressed (zstd compiled without HAVE_LZ4) -- ignored \n", srcFileName);
break;
case FIO_RUST_DECOMPRESS_ACTION_UNSUPPORTED_FORMAT:
DISPLAYLEVEL(1, "zstd: %s: unsupported format \n", srcFileName);
break;
case FIO_RUST_DECOMPRESS_ACTION_NOOP:
case FIO_RUST_DECOMPRESS_ACTION_INVALID:
default:
break;
}
}
static void FIO_rust_decompressFinalCallback(void* opaque,
const char* srcFileName,
U64 decodedSize)
{
FIO_rust_decompression_projection_t* const projection =
(FIO_rust_decompression_projection_t*)opaque;
projection->fCtx->totalBytesOutput += (size_t)decodedSize;
DISPLAY_PROGRESS("\r%79s\r", "");
if (FIO_shouldDisplayFileSummary(projection->fCtx))
DISPLAY_SUMMARY("%-20s: %llu bytes \n", srcFileName,
(unsigned long long)decodedSize);
}
/** FIO_decompressFrames() :
* Find and decode frames inside srcFile
* srcFile presumed opened and valid
* @return : 0 : OK
* 1 : error
*/
static int FIO_decompressFrames(FIO_ctx_t* const fCtx,
dRess_t ress, const FIO_prefs_t* const prefs,
const char* dstFileName, const char* srcFileName)
{
U64 filesize = 0;
int const passThrough = FIO_rust_decompressPassThroughPolicy(
prefs->passThrough,
prefs->overwrite,
prefs->passThrough == -1 ? !strcmp(dstFileName, stdoutmark) : 0);
FIO_rust_decompression_projection_t projection;
FIO_rust_decompress_callbacks_t callbacks;
int status;
assert(passThrough == 0 || passThrough == 1);
projection.fCtx = fCtx;
projection.ress = &ress;
projection.prefs = prefs;
memset(&callbacks, 0, sizeof(callbacks));
callbacks.opaque = &projection;
callbacks.decode_zstd = FIO_rust_decompressZstdFrameCallback;
#ifdef ZSTD_GZDECOMPRESS
callbacks.decode_gzip = FIO_rust_decompressGzFrameCallback;
#endif
#ifdef ZSTD_LZMADECOMPRESS
callbacks.decode_lzma = FIO_rust_decompressLzmaFrameCallback;
#endif
#ifdef ZSTD_LZ4DECOMPRESS
callbacks.decode_lz4 = FIO_rust_decompressLz4FrameCallback;
#endif
callbacks.pass_through = FIO_rust_decompressPassThroughCallback;
callbacks.report_status = FIO_rust_decompressStatusCallback;
callbacks.finish = FIO_rust_decompressFinalCallback;
status = FIO_rust_decompressFrames(ress.readCtx, srcFileName, passThrough,
&filesize, &callbacks);
return FIO_rust_finishDecompressFrames(status, srcFileName, filesize, &callbacks);
}
typedef struct {
FIO_ctx_t* fCtx;
FIO_prefs_t* prefs;
dRess_t* ress;
const char* dstFileName;
const char* srcFileName;
stat_t srcFileStat;
FILE* srcFile;
FILE* dstFile;
} FIO_rust_decompress_file_context_t;
static int FIO_rust_decompressSourceOpen(void* opaque, const char* srcFileName,
U64* fileSize, int* sourceIsRegular)
{
FIO_rust_decompress_file_context_t* const context =
(FIO_rust_decompress_file_context_t*)opaque;
context->srcFile = FIO_openSrcFile(context->prefs, srcFileName, &context->srcFileStat);
if (context->srcFile == NULL)
return 1;
*fileSize = strcmp(srcFileName, stdinmark)
? UTIL_getFileSizeStat(&context->srcFileStat)
: UTIL_FILESIZE_UNKNOWN;
*sourceIsRegular = strcmp(srcFileName, stdinmark)
? UTIL_isRegularFileStat(&context->srcFileStat)
: 0;
return FIO_RUST_DECOMPRESS_SRC_OPEN_OK;
}
static int FIO_rust_decompressSourceIsDirectory(void* opaque, const char* srcFileName)
{
/* Rust owns the rejection decision; keep the probe and exact diagnostic
* here because they depend on the CLI's private filesystem helpers. */
(void)opaque;
if (UTIL_isDirectory(srcFileName)) {
DISPLAYLEVEL(1, "zstd: %s is a directory -- ignored \n", srcFileName);
return 1;
}
return 0;
}
static void FIO_rust_decompressSourceSetAsync(void* opaque, int asyncMode)
{
FIO_rust_decompress_file_context_t* const context =
(FIO_rust_decompress_file_context_t*)opaque;
AIO_ReadPool_setAsync(context->ress->readCtx, asyncMode);
AIO_WritePool_setAsync(context->ress->writeCtx, asyncMode);
}
static void FIO_rust_decompressSourceAttach(void* opaque)
{
FIO_rust_decompress_file_context_t* const context =
(FIO_rust_decompress_file_context_t*)opaque;
AIO_ReadPool_setFile(context->ress->readCtx, context->srcFile);
}
static void FIO_rust_decompressSourceDetach(void* opaque)
{
FIO_rust_decompress_file_context_t* const context =
(FIO_rust_decompress_file_context_t*)opaque;
AIO_ReadPool_setFile(context->ress->readCtx, NULL);
}
static int FIO_rust_decompressSourceClose(void* opaque)
{
FIO_rust_decompress_file_context_t* const context =
(FIO_rust_decompress_file_context_t*)opaque;
int const result = fclose(context->srcFile);
context->srcFile = NULL;
if (result) {
DISPLAYLEVEL(1, "zstd: %s: %s \n", context->srcFileName, strerror(errno));
return 1;
}
return 0;
}
static int FIO_rust_decompressDestinationOpen(void* opaque,
const char* srcFileName,
const char* dstFileName,
int transferStat, int* dstFd)
{
FIO_rust_decompress_file_context_t* const context =
(FIO_rust_decompress_file_context_t*)opaque;
int const permissions = transferStat
? TEMPORARY_FILE_PERMISSIONS
: DEFAULT_FILE_PERMISSIONS;
context->dstFile = FIO_openDstFile(
context->fCtx, context->prefs, srcFileName, dstFileName, permissions);
if (context->dstFile == NULL)
return 1;
*dstFd = fileno(context->dstFile);
return 0;
}
static void FIO_rust_decompressDestinationAttach(void* opaque)
{
FIO_rust_decompress_file_context_t* const context =
(FIO_rust_decompress_file_context_t*)opaque;
AIO_WritePool_setFile(context->ress->writeCtx, context->dstFile);
}
static void FIO_rust_decompressAddHandler(void* opaque)
{
FIO_rust_decompress_file_context_t* const context =
(FIO_rust_decompress_file_context_t*)opaque;
addHandler(context->dstFileName);
}
static int FIO_rust_decompressFile(void* opaque,
const char* dstFileName,
const char* srcFileName)
{
FIO_rust_decompress_file_context_t* const context =
(FIO_rust_decompress_file_context_t*)opaque;
return FIO_decompressFrames(
context->fCtx, *context->ress, context->prefs, dstFileName, srcFileName);
}
static void FIO_rust_decompressClearHandler(void* opaque)
{
(void)opaque;
clearHandler();
}
static void FIO_rust_decompressSetFDStat(void* opaque, int dstFd,
const char* dstFileName)
{
FIO_rust_decompress_file_context_t* const context =
(FIO_rust_decompress_file_context_t*)opaque;
UTIL_setFDStat(dstFd, dstFileName, &context->srcFileStat);
}
static int FIO_rust_decompressDestinationClose(void* opaque)
{
FIO_rust_decompress_file_context_t* const context =
(FIO_rust_decompress_file_context_t*)opaque;
int const result = AIO_WritePool_closeFile(context->ress->writeCtx);
context->dstFile = NULL;
if (result) {
DISPLAYLEVEL(1, "zstd: %s: %s \n", context->dstFileName, strerror(errno));
return 1;
}
return 0;
}
static void FIO_rust_decompressDestinationUtime(void* opaque)
{
FIO_rust_decompress_file_context_t* const context =
(FIO_rust_decompress_file_context_t*)opaque;
UTIL_utime(context->dstFileName, &context->srcFileStat);
}
static void FIO_rust_decompressDestinationRemove(void* opaque, const char* dstFileName)
{
(void)opaque;
(void)FIO_removeFile(dstFileName);
}
static int FIO_rust_decompressSourceRemove(void* opaque, const char* srcFileName)
{
(void)opaque;
if (FIO_removeFile(srcFileName)) {
DISPLAYLEVEL(1, "zstd: %s: %s \n", srcFileName, strerror(errno));
return 1;
}
return 0;
}
static int FIO_decompressFile(FIO_ctx_t* const fCtx, FIO_prefs_t* const prefs,
dRess_t* const ress, const char* dstFileName,
const char* srcFileName)
{
FIO_rust_decompress_file_context_t context;
FIO_rust_decompress_file_projection_t projection;
memset(&context, 0, sizeof(context));
context.fCtx = fCtx;
context.prefs = prefs;
context.ress = ress;
context.dstFileName = dstFileName;
context.srcFileName = srcFileName;
memset(&projection, 0, sizeof(projection));
projection.opaque = &context;
projection.dstFileName = dstFileName;
projection.srcFileName = srcFileName;
projection.destinationAlreadyOpen = AIO_WritePool_getFile(ress->writeCtx) != NULL;
projection.testMode = prefs->testMode;
projection.sourceIsStdin = !strcmp(srcFileName, stdinmark);
projection.destinationIsStdout = !strcmp(dstFileName, stdoutmark);
projection.removeSource = prefs->removeSrcFile;
projection.openSource = FIO_rust_decompressSourceOpen;
projection.setAsync = FIO_rust_decompressSourceSetAsync;
projection.attachSource = FIO_rust_decompressSourceAttach;
projection.detachSource = FIO_rust_decompressSourceDetach;
projection.closeSource = FIO_rust_decompressSourceClose;
projection.openDestination = FIO_rust_decompressDestinationOpen;
projection.attachDestination = FIO_rust_decompressDestinationAttach;
projection.addHandler = FIO_rust_decompressAddHandler;
projection.decompress = FIO_rust_decompressFile;
projection.clearHandler = FIO_rust_decompressClearHandler;
projection.setFDStat = FIO_rust_decompressSetFDStat;
projection.closeDestination = FIO_rust_decompressDestinationClose;
projection.utimeDestination = FIO_rust_decompressDestinationUtime;
projection.removeDestination = FIO_rust_decompressDestinationRemove;
projection.removeSourceFile = FIO_rust_decompressSourceRemove;
projection.sourceIsDirectory = FIO_rust_decompressSourceIsDirectory;
return FIO_rust_decompressFilename(&projection);
}
typedef struct {
FIO_ctx_t* fCtx;
FIO_prefs_t* prefs;
dRess_t* ress;
const char* outFileName;
} FIO_rust_decompress_multiple_context_t;
static int FIO_rust_decompressMultipleDestinationWarning(void* opaque)
{
FIO_rust_decompress_multiple_context_t* const context =
(FIO_rust_decompress_multiple_context_t*)opaque;
return FIO_multiFilesConcatWarning(
context->fCtx, context->prefs, context->outFileName, 1 /* displayLevelCutoff */);
}
static void* FIO_rust_decompressMultipleDestinationOpen(void* opaque)
{
FIO_rust_decompress_multiple_context_t* const context =
(FIO_rust_decompress_multiple_context_t*)opaque;
FILE* const dstFile = FIO_openDstFile(
context->fCtx, context->prefs, NULL,
context->outFileName, DEFAULT_FILE_PERMISSIONS);
if (dstFile == 0) EXM_THROW(19, "cannot open %s", context->outFileName);
return (void*)dstFile;
}
static void FIO_rust_decompressMultipleDestinationAttach(void* opaque, void* destination)
{
FIO_rust_decompress_multiple_context_t* const context =
(FIO_rust_decompress_multiple_context_t*)opaque;
AIO_WritePool_setFile(context->ress->writeCtx, (FILE*)destination);
}
static int FIO_rust_decompressMultipleDestinationClose(void* opaque)
{
FIO_rust_decompress_multiple_context_t* const context =
(FIO_rust_decompress_multiple_context_t*)opaque;
if (AIO_WritePool_closeFile(context->ress->writeCtx))
EXM_THROW(72, "Write error : %s : cannot properly close output file",
strerror(errno));
return 0;
}
/* Keep private source/destination operations, format dispatch, and
* diagnostics in C; Rust owns per-file ordering and removal policy. */
static int FIO_rust_decompressMultipleFileCallback(void* opaque,
const char* outFileName,
const char* srcFileName)
{
FIO_rust_decompress_multiple_context_t* const context =
(FIO_rust_decompress_multiple_context_t*)opaque;
return FIO_decompressFile(
context->fCtx, context->prefs, context->ress, outFileName, srcFileName);
}
typedef struct {
FIO_ctx_t* fCtx;
FIO_prefs_t* prefs;
dRess_t* ress;
const char* outMirroredRootDirName;
const char* outDirName;
} FIO_rust_decompress_multiple_separate_context_t;
static const char *suffixList[];
static const char *suffixListStr;
/* Keep destination-name construction, diagnostics, and private operations in
* C; Rust owns per-file ordering and removal policy. */
static int FIO_rust_decompressMultipleSeparateFileCallback(void* opaque,
const char* srcFileName)
{
FIO_rust_decompress_multiple_separate_context_t* const context =
(FIO_rust_decompress_multiple_separate_context_t*)opaque;
const char* dstFileName = NULL;
if (context->outMirroredRootDirName) {
char* validMirroredDirName = UTIL_createMirroredDestDirName(
srcFileName, context->outMirroredRootDirName);
if (validMirroredDirName) {
dstFileName = FIO_rust_determineDstName(
srcFileName, validMirroredDirName, suffixList, suffixListStr);
free(validMirroredDirName);
} else {
DISPLAYLEVEL(2, "zstd: --output-dir-mirror cannot decompress '%s' into '%s'\n",
srcFileName, context->outMirroredRootDirName);
}
} else {
dstFileName = FIO_rust_determineDstName(
srcFileName, context->outDirName, suffixList, suffixListStr);
}
if (dstFileName == NULL) return 1;
return FIO_decompressFile(
context->fCtx, context->prefs, context->ress, dstFileName, srcFileName);
}
int FIO_decompressFilename(FIO_ctx_t* const fCtx, FIO_prefs_t* const prefs,
const char* dstFileName, const char* srcFileName,
const char* dictFileName)
{
dRess_t ress = FIO_createDResources(prefs, dictFileName);
int const decodingError = FIO_decompressFile(
fCtx, prefs, &ress, dstFileName, srcFileName);
FIO_freeDResources(ress);
return decodingError;
}
static const char *suffixList[] = {
ZSTD_EXTENSION,
TZSTD_EXTENSION,
#ifndef ZSTD_NODECOMPRESS
ZSTD_ALT_EXTENSION,
#endif
#ifdef ZSTD_GZDECOMPRESS
GZ_EXTENSION,
TGZ_EXTENSION,
#endif
#ifdef ZSTD_LZMADECOMPRESS
LZMA_EXTENSION,
XZ_EXTENSION,
TXZ_EXTENSION,
#endif
#ifdef ZSTD_LZ4DECOMPRESS
LZ4_EXTENSION,
TLZ4_EXTENSION,
#endif
NULL
};
static const char *suffixListStr =
ZSTD_EXTENSION "/" TZSTD_EXTENSION
#ifdef ZSTD_GZDECOMPRESS
"/" GZ_EXTENSION "/" TGZ_EXTENSION
#endif
#ifdef ZSTD_LZMADECOMPRESS
"/" LZMA_EXTENSION "/" XZ_EXTENSION "/" TXZ_EXTENSION
#endif
#ifdef ZSTD_LZ4DECOMPRESS
"/" LZ4_EXTENSION "/" TLZ4_EXTENSION
#endif
;
int
FIO_decompressMultipleFilenames(FIO_ctx_t* const fCtx,
FIO_prefs_t* const prefs,
const char** srcNamesTable,
const char* outMirroredRootDirName,
const char* outDirName, const char* outFileName,
const char* dictFileName)
{
int error = 0;
dRess_t ress = FIO_createDResources(prefs, dictFileName);
if (outFileName) {
FIO_rust_decompress_multiple_context_t callbackContext = {
fCtx, prefs, &ress, outFileName
};
FIO_rust_decompress_multiple_projection_t files = {
fCtx, srcNamesTable, outFileName,
&callbackContext, FIO_rust_decompressMultipleFileCallback
};
FIO_rust_decompress_multiple_destination_projection_t projection = {
&files,
!prefs->testMode,
FIO_rust_decompressMultipleDestinationWarning,
FIO_rust_decompressMultipleDestinationOpen,
FIO_rust_decompressMultipleDestinationAttach,
FIO_rust_decompressMultipleDestinationClose
};
error = FIO_rust_decompressMultipleFilenamesWithDestination(&projection);
} else {
FIO_rust_decompress_multiple_separate_context_t callbackContext = {
fCtx, prefs, &ress, outMirroredRootDirName, outDirName
};
FIO_rust_decompress_multiple_separate_projection_t projection = {
fCtx, srcNamesTable, &callbackContext,
FIO_rust_decompressMultipleSeparateFileCallback
};
if (outMirroredRootDirName)
UTIL_mirrorSourceFilesDirectories(srcNamesTable, (unsigned)fCtx->nbFilesTotal, outMirroredRootDirName);
error = FIO_rust_decompressMultipleSeparateFilenames(&projection);
if (outDirName)
FIO_checkFilenameCollisions(srcNamesTable , (unsigned)fCtx->nbFilesTotal);
}
FIO_displayMultipleFileSummary(
fCtx, FIO_RUST_MULTIPLE_SUMMARY_DECOMPRESSION);
FIO_freeDResources(ress);
return error;
}
/* **************************************************************************
* .zst file info (--list command)
***************************************************************************/
typedef struct {
U64 decompressedSize;
U64 compressedSize;
U64 windowSize;
int numActualFrames;
int numSkippableFrames;
int decompUnavailable;
int usesCheck;
BYTE checksum[4];
U32 nbFiles;
unsigned dictID;
} fileInfo_t;
typedef char FIO_rust_file_info_compressed_size_offset[
(offsetof(fileInfo_t, compressedSize) == sizeof(U64)) ? 1 : -1];
typedef char FIO_rust_file_info_window_size_offset[
(offsetof(fileInfo_t, windowSize) == 2 * sizeof(U64)) ? 1 : -1];
typedef char FIO_rust_file_info_frame_counts_offset[
(offsetof(fileInfo_t, numActualFrames) == 3 * sizeof(U64)) ? 1 : -1];
typedef char FIO_rust_file_info_checksum_offset[
(offsetof(fileInfo_t, checksum) == 3 * sizeof(U64) + 4 * sizeof(int)) ? 1 : -1];
typedef char FIO_rust_file_info_size[
(sizeof(fileInfo_t) == (sizeof(void*) == 8
? 7 * sizeof(U64)
: 13 * sizeof(U32))) ? 1 : -1];
typedef enum {
info_success=0,
info_frame_error=1,
info_not_zstd=2,
info_file_error=3,
info_truncated_input=4
} InfoError;
/* Rust owns the --list file policy/order. C keeps the exact diagnostics,
* metadata formatting, file I/O, and private fileInfo_t layout. */
enum {
FIO_RUST_LIST_FILE_ACTION_SUCCESS = 0,
FIO_RUST_LIST_FILE_ACTION_FRAME_ERROR = 1,
FIO_RUST_LIST_FILE_ACTION_NOT_ZSTD = 2,
FIO_RUST_LIST_FILE_ACTION_FILE_ERROR = 3,
FIO_RUST_LIST_FILE_ACTION_TRUNCATED_INPUT = 4,
FIO_RUST_LIST_FILE_ACTION_INVALID = 5
};
int FIO_rust_listFileStatusAction(int status);
/* Keep the private fileInfo_t layout in C. This projection contains only the
* fields needed by the --list total row and crosses the Rust policy boundary
* after the C-owned file-info and display callbacks have run. */
typedef struct {
U64 decompressedSize;
U64 compressedSize;
int numActualFrames;
int numSkippableFrames;
int decompUnavailable;
int usesCheck;
U32 nbFiles;
} FIO_listFileInfoProjection_t;
typedef int (*FIO_listFileGetInfoFn)(void* opaque, const char* fileName,
fileInfo_t* info);
typedef void (*FIO_listFileDisplayStatusFn)(void* opaque, int action,
const char* fileName,
int displayLevel);
typedef void (*FIO_listFileWriteOutputFn)(void* opaque, const char* data,
size_t size);
/* This callback state is a deliberate repr(C) bridge: filesystem access,
* frame analysis, and exact diagnostics stay in C while Rust owns row
* formatting and the per-file policy/order around it. */
typedef struct {
void* opaque;
FIO_listFileGetInfoFn getInfo;
FIO_listFileDisplayStatusFn displayStatus;
FIO_listFileWriteOutputFn writeOutput;
} FIO_listFileCallbacks_t;
typedef char FIO_rust_list_file_callbacks_get_info_offset[
(offsetof(FIO_listFileCallbacks_t, getInfo) == sizeof(void*)) ? 1 : -1];
typedef char FIO_rust_list_file_callbacks_status_offset[
(offsetof(FIO_listFileCallbacks_t, displayStatus)
== sizeof(void*) + sizeof(FIO_listFileGetInfoFn)) ? 1 : -1];
typedef char FIO_rust_list_file_callbacks_output_offset[
(offsetof(FIO_listFileCallbacks_t, writeOutput)
== sizeof(void*) + sizeof(FIO_listFileGetInfoFn)
+ sizeof(FIO_listFileDisplayStatusFn)) ? 1 : -1];
typedef struct {
void* opaque;
int (*isStdin)(void* opaque, const char* fileName);
void (*displayStdinError)(void* opaque);
void (*displayNoFilesError)(void* opaque);
void (*displayHeader)(void* opaque);
int (*listFile)(void* opaque, const char* fileName, int displayLevel,
FIO_listFileInfoProjection_t* info);
FIO_listFileWriteOutputFn writeOutput;
} FIO_listMultipleFilesCallbacks_t;
int FIO_rust_listFile(const char* inputName, int displayLevel,
FIO_listFileInfoProjection_t* output,
const FIO_listFileCallbacks_t* callbacks);
int FIO_rust_listMultipleFiles(unsigned numFiles, const char** filenameTable,
int displayLevel,
const FIO_listMultipleFilesCallbacks_t* callbacks);
enum {
FIO_RUST_ANALYZE_DIAG_SEEKED_PAST_FILE = 0,
FIO_RUST_ANALYZE_DIAG_INCOMPLETE_FRAME = 1,
FIO_RUST_ANALYZE_DIAG_RAN_OUT_OF_FRAMES = 2,
FIO_RUST_ANALYZE_DIAG_DECODE_FRAME_HEADER = 3,
FIO_RUST_ANALYZE_DIAG_FRAME_HEADER_SIZE = 4,
FIO_RUST_ANALYZE_DIAG_MOVE_TO_FRAME_HEADER_END = 5,
FIO_RUST_ANALYZE_DIAG_BLOCK_HEADER = 6,
FIO_RUST_ANALYZE_DIAG_UNSUPPORTED_BLOCK_TYPE = 7,
FIO_RUST_ANALYZE_DIAG_SKIP_BLOCK = 8,
FIO_RUST_ANALYZE_DIAG_CHECKSUM = 9,
FIO_RUST_ANALYZE_DIAG_SKIP_FRAME = 10,
FIO_RUST_ANALYZE_DIAG_MIXED_DICTIONARY_IDS = 11
};
int FIO_rust_analyzeFrames(fileInfo_t* info, FILE* srcFile);
void FIO_rust_analyzeFrames_display(int diagnostic,
unsigned long long filePosition,
unsigned long long fileSize);
#define ERROR_IF(c,n,...) { \
if (c) { \
DISPLAYLEVEL(1, __VA_ARGS__); \
DISPLAYLEVEL(1, " \n"); \
return n; \
} \
}
void
FIO_rust_analyzeFrames_display(int diagnostic,
unsigned long long filePosition,
unsigned long long fileSize)
{
switch (diagnostic) {
case FIO_RUST_ANALYZE_DIAG_SEEKED_PAST_FILE:
DISPLAYLEVEL(1,
"Error: seeked to position %llu, which is beyond file size of %llu\n",
filePosition, fileSize);
break;
case FIO_RUST_ANALYZE_DIAG_INCOMPLETE_FRAME:
DISPLAYLEVEL(1, "Error: reached end of file with incomplete frame");
break;
case FIO_RUST_ANALYZE_DIAG_RAN_OUT_OF_FRAMES:
DISPLAYLEVEL(1, "Error: did not reach end of file but ran out of frames");
break;
case FIO_RUST_ANALYZE_DIAG_DECODE_FRAME_HEADER:
DISPLAYLEVEL(1, "Error: could not decode frame header");
break;
case FIO_RUST_ANALYZE_DIAG_FRAME_HEADER_SIZE:
DISPLAYLEVEL(1, "Error: could not determine frame header size");
break;
case FIO_RUST_ANALYZE_DIAG_MOVE_TO_FRAME_HEADER_END:
DISPLAYLEVEL(1, "Error: could not move to end of frame header");
break;
case FIO_RUST_ANALYZE_DIAG_BLOCK_HEADER:
DISPLAYLEVEL(1, "Error while reading block header");
break;
case FIO_RUST_ANALYZE_DIAG_UNSUPPORTED_BLOCK_TYPE:
DISPLAYLEVEL(1, "Error: unsupported block type");
break;
case FIO_RUST_ANALYZE_DIAG_SKIP_BLOCK:
DISPLAYLEVEL(1, "Error: could not skip to end of block");
break;
case FIO_RUST_ANALYZE_DIAG_CHECKSUM:
DISPLAYLEVEL(1, "Error: could not read checksum");
break;
case FIO_RUST_ANALYZE_DIAG_SKIP_FRAME:
DISPLAYLEVEL(1, "Error: could not find end of skippable frame");
break;
case FIO_RUST_ANALYZE_DIAG_MIXED_DICTIONARY_IDS:
DISPLAY("WARNING: File contains multiple frames with different dictionary IDs. Showing dictID 0 instead");
return;
default:
assert(0);
return;
}
DISPLAYLEVEL(1, " \n");
}
static InfoError
getFileInfo_fileConfirmed(fileInfo_t* info, const char* inFileName)
{
InfoError status;
stat_t srcFileStat;
FILE* const srcFile = FIO_openSrcFile(NULL, inFileName, &srcFileStat);
ERROR_IF(srcFile == NULL, info_file_error, "Error: could not open source file %s", inFileName);
info->compressedSize = UTIL_getFileSizeStat(&srcFileStat);
status = (InfoError)FIO_rust_analyzeFrames(info, srcFile);
fclose(srcFile);
info->nbFiles = 1;
return status;
}
/** getFileInfo() :
* Reads information from file, stores in *info
* @return : InfoError status
*/
static InfoError
getFileInfo(fileInfo_t* info, const char* srcFileName)
{
ERROR_IF(!UTIL_isRegularFile(srcFileName),
info_file_error, "Error : %s is not a file", srcFileName);
return getFileInfo_fileConfirmed(info, srcFileName);
}
static int
FIO_listFileGetInfo(void* opaque, const char* inFileName, fileInfo_t* info)
{
(void)opaque;
return (int)getFileInfo(info, inFileName);
}
static void
FIO_listFileDisplayStatus(void* opaque, int action,
const char* inFileName, int displayLevel)
{
(void)opaque;
switch (action) {
case FIO_RUST_LIST_FILE_ACTION_FRAME_ERROR:
/* display error, but provide output */
DISPLAYLEVEL(1, "Error while parsing \"%s\" \n", inFileName);
break;
case FIO_RUST_LIST_FILE_ACTION_NOT_ZSTD:
DISPLAYOUT("File \"%s\" not compressed by zstd \n", inFileName);
if (displayLevel > 2) DISPLAYOUT("\n");
break;
case FIO_RUST_LIST_FILE_ACTION_FILE_ERROR:
/* error occurred while opening the file */
if (displayLevel > 2) DISPLAYOUT("\n");
break;
case FIO_RUST_LIST_FILE_ACTION_TRUNCATED_INPUT:
DISPLAYOUT("File \"%s\" is truncated \n", inFileName);
if (displayLevel > 2) DISPLAYOUT("\n");
break;
default:
assert(0);
break;
}
}
static void
FIO_listFileWriteOutput(void* opaque, const char* data, size_t size)
{
(void)opaque;
(void)fwrite(data, 1, size, stdout);
}
/* Rust owns this stateless --list input-name predicate. */
int FIO_listFileIsStdin(void* opaque, const char* fileName);
static void
FIO_listFileDisplayStdinError(void* opaque)
{
(void)opaque;
DISPLAYLEVEL(1, "zstd: --list does not support reading from standard input");
DISPLAYLEVEL(1, " \n");
}
static void
FIO_listFileDisplayNoFilesError(void* opaque)
{
(void)opaque;
if (!UTIL_isConsole(stdin)) {
DISPLAYLEVEL(1, "zstd: --list does not support reading from standard input \n");
}
DISPLAYLEVEL(1, "No files given \n");
}
static void
FIO_listFileDisplayHeader(void* opaque)
{
(void)opaque;
DISPLAYOUT("Frames Skips Compressed Uncompressed Ratio Check Filename\n");
}
static int
FIO_listFileCallback(void* opaque, const char* fileName, int displayLevel,
FIO_listFileInfoProjection_t* info)
{
FIO_listFileCallbacks_t callbacks;
(void)opaque;
memset(info, 0, sizeof(*info));
callbacks.opaque = NULL;
callbacks.getInfo = FIO_listFileGetInfo;
callbacks.displayStatus = FIO_listFileDisplayStatus;
callbacks.writeOutput = FIO_listFileWriteOutput;
return FIO_rust_listFile(fileName, displayLevel, info, &callbacks);
}
int FIO_listMultipleFiles(unsigned numFiles, const char** filenameTable, int displayLevel)
{
FIO_listMultipleFilesCallbacks_t callbacks;
callbacks.opaque = NULL;
callbacks.isStdin = FIO_listFileIsStdin;
callbacks.displayStdinError = FIO_listFileDisplayStdinError;
callbacks.displayNoFilesError = FIO_listFileDisplayNoFilesError;
callbacks.displayHeader = FIO_listFileDisplayHeader;
callbacks.listFile = FIO_listFileCallback;
callbacks.writeOutput = FIO_listFileWriteOutput;
return FIO_rust_listMultipleFiles(numFiles, filenameTable, displayLevel, &callbacks);
}
#endif /* #ifndef ZSTD_NODECOMPRESS */
Reference in New Issue View Git Blame Copy Permalink