feat(cli): move optional decompression loops into Rust
Move gzip, xz/LZMA, and LZ4 frame I/O loops into Rust projections while keeping zlib, liblzma, LZ4F, asynchronous pool operations, diagnostics, and private C handles in C callbacks. Preserve trailing-input accounting and format-dispatch behavior, and stop faulty LZ4 callbacks from producing an unbounded no-progress output loop while retaining the codec error value. Test Plan: - ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo test --manifest-path rust/Cargo.toml --lib lz4_decompress_projection_reports_errors_and_unfinished_input -- --test-threads=1 (1 passed) - ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo clippy --manifest-path rust/Cargo.toml --lib -- -D warnings (passed) - ulimit -v 41943040; make -B -C tests -j1 test-cli-tests (41 passed) - ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo test --manifest-path rust/Cargo.toml --all-targets -- --test-threads=1 (507 passed before final error-value assertion)
This commit is contained in:
+422
-261
@@ -466,6 +466,107 @@ int FIO_rust_finishDecompressFrames(int status,
|
||||
const char* srcFileName,
|
||||
U64 decodedSize,
|
||||
const FIO_rust_decompress_callbacks_t* callbacks);
|
||||
|
||||
typedef void (*FIO_rust_decompress_read_fill_fn)(
|
||||
void* opaque, size_t requested, const unsigned char** buffer, size_t* loaded);
|
||||
typedef void (*FIO_rust_decompress_read_consume_fn)(void* opaque, size_t consumed);
|
||||
typedef void (*FIO_rust_decompress_write_acquire_fn)(
|
||||
void* opaque, void** job, unsigned char** buffer, size_t* bufferSize);
|
||||
typedef void (*FIO_rust_decompress_write_enqueue_fn)(
|
||||
void* opaque, void** job, size_t usedBufferSize,
|
||||
unsigned char** buffer, size_t* bufferSize);
|
||||
typedef void (*FIO_rust_decompress_write_release_fn)(void* opaque, void* job);
|
||||
typedef void (*FIO_rust_decompress_sparse_write_end_fn)(void* opaque);
|
||||
|
||||
typedef struct {
|
||||
void* readOpaque;
|
||||
void* writeOpaque;
|
||||
size_t readBufferSize;
|
||||
FIO_rust_decompress_read_fill_fn readFill;
|
||||
FIO_rust_decompress_read_consume_fn readConsume;
|
||||
FIO_rust_decompress_write_acquire_fn writeAcquire;
|
||||
FIO_rust_decompress_write_enqueue_fn writeEnqueue;
|
||||
FIO_rust_decompress_write_release_fn writeRelease;
|
||||
FIO_rust_decompress_sparse_write_end_fn sparseWriteEnd;
|
||||
} FIO_rust_decompress_io_projection_t;
|
||||
|
||||
enum {
|
||||
FIO_RUST_GZIP_DECOMPRESS_OK = 0,
|
||||
FIO_RUST_GZIP_DECOMPRESS_INIT_ERROR = 1,
|
||||
FIO_RUST_GZIP_DECOMPRESS_BUF_ERROR = 2,
|
||||
FIO_RUST_GZIP_DECOMPRESS_INFLATE_ERROR = 3,
|
||||
FIO_RUST_GZIP_DECOMPRESS_END_ERROR = 4,
|
||||
FIO_RUST_GZIP_DECOMPRESS_INVALID_PROJECTION = 5
|
||||
};
|
||||
typedef int (*FIO_rust_gzip_decompress_init_fn)(void* opaque);
|
||||
typedef int (*FIO_rust_gzip_decompress_inflate_fn)(
|
||||
void* opaque, const unsigned char* input, size_t inputSize,
|
||||
unsigned char* output, size_t outputSize, int flush,
|
||||
size_t* consumed, size_t* produced);
|
||||
typedef int (*FIO_rust_gzip_decompress_end_fn)(void* opaque);
|
||||
typedef struct {
|
||||
FIO_rust_decompress_io_projection_t io;
|
||||
void* zlibOpaque;
|
||||
FIO_rust_gzip_decompress_init_fn zlibInit;
|
||||
FIO_rust_gzip_decompress_inflate_fn zlibInflate;
|
||||
FIO_rust_gzip_decompress_end_fn zlibEnd;
|
||||
} FIO_rust_gzip_decompress_projection_t;
|
||||
int FIO_rust_decompressGzipFrame(
|
||||
const FIO_rust_gzip_decompress_projection_t* projection,
|
||||
U64* frameSize, int* zlibResult);
|
||||
|
||||
enum {
|
||||
FIO_RUST_LZMA_DECOMPRESS_OK = 0,
|
||||
FIO_RUST_LZMA_DECOMPRESS_INIT_ERROR = 1,
|
||||
FIO_RUST_LZMA_DECOMPRESS_BUF_ERROR = 2,
|
||||
FIO_RUST_LZMA_DECOMPRESS_CODE_ERROR = 3,
|
||||
FIO_RUST_LZMA_DECOMPRESS_INVALID_PROJECTION = 4
|
||||
};
|
||||
typedef int (*FIO_rust_lzma_decompress_init_fn)(void* opaque, int plainLzma);
|
||||
typedef int (*FIO_rust_lzma_decompress_code_fn)(
|
||||
void* opaque, const unsigned char* input, size_t inputSize,
|
||||
unsigned char* output, size_t outputSize, int action,
|
||||
size_t* consumed, size_t* produced);
|
||||
typedef void (*FIO_rust_lzma_decompress_end_fn)(void* opaque);
|
||||
typedef struct {
|
||||
FIO_rust_decompress_io_projection_t io;
|
||||
void* lzmaOpaque;
|
||||
FIO_rust_lzma_decompress_init_fn lzmaInit;
|
||||
FIO_rust_lzma_decompress_code_fn lzmaCode;
|
||||
FIO_rust_lzma_decompress_end_fn lzmaEnd;
|
||||
} FIO_rust_lzma_decompress_projection_t;
|
||||
int FIO_rust_decompressLzmaFrame(
|
||||
const FIO_rust_lzma_decompress_projection_t* projection,
|
||||
int plainLzma, U64* frameSize, int* lzmaResult);
|
||||
|
||||
enum {
|
||||
FIO_RUST_LZ4_DECOMPRESS_OK = 0,
|
||||
FIO_RUST_LZ4_DECOMPRESS_CREATE_ERROR = 1,
|
||||
FIO_RUST_LZ4_DECOMPRESS_CODE_ERROR = 2,
|
||||
FIO_RUST_LZ4_DECOMPRESS_UNFINISHED = 3,
|
||||
FIO_RUST_LZ4_DECOMPRESS_INVALID_PROJECTION = 4
|
||||
};
|
||||
typedef int (*FIO_rust_lz4_decompress_create_fn)(
|
||||
void* opaque, unsigned version, size_t* result);
|
||||
typedef int (*FIO_rust_lz4_decompress_code_fn)(
|
||||
void* opaque, unsigned char* output, size_t* outputSize,
|
||||
const unsigned char* input, size_t* inputSize, size_t* nextToLoad);
|
||||
typedef void (*FIO_rust_lz4_decompress_free_fn)(void* opaque);
|
||||
typedef void (*FIO_rust_lz4_decompress_progress_fn)(void* opaque, U64 decodedSize);
|
||||
typedef struct {
|
||||
FIO_rust_decompress_io_projection_t io;
|
||||
void* codecOpaque;
|
||||
void* progressOpaque;
|
||||
unsigned version;
|
||||
FIO_rust_lz4_decompress_create_fn create;
|
||||
FIO_rust_lz4_decompress_code_fn code;
|
||||
FIO_rust_lz4_decompress_free_fn freeContext;
|
||||
FIO_rust_lz4_decompress_progress_fn progress;
|
||||
} FIO_rust_lz4_decompress_projection_t;
|
||||
int FIO_rust_decompressLz4Frame(
|
||||
const FIO_rust_lz4_decompress_projection_t* projection,
|
||||
U64* frameSize, size_t* lz4Result);
|
||||
|
||||
void FIO_rust_displayCompressionParameters(const FIO_prefs_t* prefs);
|
||||
#ifdef ZSTD_LZ4COMPRESS
|
||||
int FIO_rust_LZ4_GetBlockSize_FromBlockId(int id);
|
||||
@@ -2822,214 +2923,6 @@ FIO_decompressZstdFrames(FIO_ctx_t* const fCtx, dRess_t* ress,
|
||||
}
|
||||
|
||||
|
||||
#ifdef ZSTD_GZDECOMPRESS
|
||||
static unsigned long long
|
||||
FIO_decompressGzFrame(dRess_t* ress, const char* srcFileName)
|
||||
{
|
||||
unsigned long long outFileSize = 0;
|
||||
z_stream strm;
|
||||
int flush = Z_NO_FLUSH;
|
||||
int decodingError = 0;
|
||||
IOJob_t *writeJob = NULL;
|
||||
|
||||
strm.zalloc = Z_NULL;
|
||||
strm.zfree = Z_NULL;
|
||||
strm.opaque = Z_NULL;
|
||||
strm.next_in = 0;
|
||||
strm.avail_in = 0;
|
||||
/* see https://www.zlib.net/manual.html */
|
||||
if (inflateInit2(&strm, 15 /* maxWindowLogSize */ + 16 /* gzip only */) != Z_OK)
|
||||
return FIO_ERROR_FRAME_DECODING;
|
||||
|
||||
writeJob = AIO_WritePool_acquireJob(ress->writeCtx);
|
||||
strm.next_out = (Bytef*)writeJob->buffer;
|
||||
strm.avail_out = (uInt)writeJob->bufferSize;
|
||||
strm.avail_in = (uInt)ress->readCtx->srcBufferLoaded;
|
||||
strm.next_in = (z_const unsigned char*)ress->readCtx->srcBuffer;
|
||||
|
||||
for ( ; ; ) {
|
||||
int ret;
|
||||
if (strm.avail_in == 0) {
|
||||
AIO_ReadPool_consumeAndRefill(ress->readCtx);
|
||||
if (ress->readCtx->srcBufferLoaded == 0) flush = Z_FINISH;
|
||||
strm.next_in = (z_const unsigned char*)ress->readCtx->srcBuffer;
|
||||
strm.avail_in = (uInt)ress->readCtx->srcBufferLoaded;
|
||||
}
|
||||
ret = inflate(&strm, flush);
|
||||
if (ret == Z_BUF_ERROR) {
|
||||
DISPLAYLEVEL(1, "zstd: %s: premature gz end \n", srcFileName);
|
||||
decodingError = 1; break;
|
||||
}
|
||||
if (ret != Z_OK && ret != Z_STREAM_END) {
|
||||
DISPLAYLEVEL(1, "zstd: %s: inflate error %d \n", srcFileName, ret);
|
||||
decodingError = 1; break;
|
||||
}
|
||||
{ size_t const decompBytes = writeJob->bufferSize - strm.avail_out;
|
||||
if (decompBytes) {
|
||||
writeJob->usedBufferSize = decompBytes;
|
||||
AIO_WritePool_enqueueAndReacquireWriteJob(&writeJob);
|
||||
outFileSize += decompBytes;
|
||||
strm.next_out = (Bytef*)writeJob->buffer;
|
||||
strm.avail_out = (uInt)writeJob->bufferSize;
|
||||
}
|
||||
}
|
||||
if (ret == Z_STREAM_END) break;
|
||||
}
|
||||
|
||||
AIO_ReadPool_consumeBytes(ress->readCtx, ress->readCtx->srcBufferLoaded - strm.avail_in);
|
||||
|
||||
if ( (inflateEnd(&strm) != Z_OK) /* release resources ; error detected */
|
||||
&& (decodingError==0) ) {
|
||||
DISPLAYLEVEL(1, "zstd: %s: inflateEnd error \n", srcFileName);
|
||||
decodingError = 1;
|
||||
}
|
||||
AIO_WritePool_releaseIoJob(writeJob);
|
||||
AIO_WritePool_sparseWriteEnd(ress->writeCtx);
|
||||
return decodingError ? FIO_ERROR_FRAME_DECODING : outFileSize;
|
||||
}
|
||||
#endif
|
||||
|
||||
#ifdef ZSTD_LZMADECOMPRESS
|
||||
static unsigned long long
|
||||
FIO_decompressLzmaFrame(dRess_t* ress,
|
||||
const char* srcFileName, int plain_lzma)
|
||||
{
|
||||
unsigned long long outFileSize = 0;
|
||||
lzma_stream strm = LZMA_STREAM_INIT;
|
||||
lzma_action action = LZMA_RUN;
|
||||
lzma_ret initRet;
|
||||
int decodingError = 0;
|
||||
IOJob_t *writeJob = NULL;
|
||||
|
||||
strm.next_in = 0;
|
||||
strm.avail_in = 0;
|
||||
if (plain_lzma) {
|
||||
initRet = lzma_alone_decoder(&strm, UINT64_MAX); /* LZMA */
|
||||
} else {
|
||||
initRet = lzma_stream_decoder(&strm, UINT64_MAX, 0); /* XZ */
|
||||
}
|
||||
|
||||
if (initRet != LZMA_OK) {
|
||||
DISPLAYLEVEL(1, "zstd: %s: %s error %d \n",
|
||||
plain_lzma ? "lzma_alone_decoder" : "lzma_stream_decoder",
|
||||
srcFileName, initRet);
|
||||
return FIO_ERROR_FRAME_DECODING;
|
||||
}
|
||||
|
||||
writeJob = AIO_WritePool_acquireJob(ress->writeCtx);
|
||||
strm.next_out = (BYTE*)writeJob->buffer;
|
||||
strm.avail_out = writeJob->bufferSize;
|
||||
strm.next_in = (BYTE const*)ress->readCtx->srcBuffer;
|
||||
strm.avail_in = ress->readCtx->srcBufferLoaded;
|
||||
|
||||
for ( ; ; ) {
|
||||
lzma_ret ret;
|
||||
if (strm.avail_in == 0) {
|
||||
AIO_ReadPool_consumeAndRefill(ress->readCtx);
|
||||
if (ress->readCtx->srcBufferLoaded == 0) action = LZMA_FINISH;
|
||||
strm.next_in = (BYTE const*)ress->readCtx->srcBuffer;
|
||||
strm.avail_in = ress->readCtx->srcBufferLoaded;
|
||||
}
|
||||
ret = lzma_code(&strm, action);
|
||||
|
||||
if (ret == LZMA_BUF_ERROR) {
|
||||
DISPLAYLEVEL(1, "zstd: %s: premature lzma end \n", srcFileName);
|
||||
decodingError = 1; break;
|
||||
}
|
||||
if (ret != LZMA_OK && ret != LZMA_STREAM_END) {
|
||||
DISPLAYLEVEL(1, "zstd: %s: lzma_code decoding error %d \n",
|
||||
srcFileName, ret);
|
||||
decodingError = 1; break;
|
||||
}
|
||||
{ size_t const decompBytes = writeJob->bufferSize - strm.avail_out;
|
||||
if (decompBytes) {
|
||||
writeJob->usedBufferSize = decompBytes;
|
||||
AIO_WritePool_enqueueAndReacquireWriteJob(&writeJob);
|
||||
outFileSize += decompBytes;
|
||||
strm.next_out = (BYTE*)writeJob->buffer;
|
||||
strm.avail_out = writeJob->bufferSize;
|
||||
} }
|
||||
if (ret == LZMA_STREAM_END) break;
|
||||
}
|
||||
|
||||
AIO_ReadPool_consumeBytes(ress->readCtx, ress->readCtx->srcBufferLoaded - strm.avail_in);
|
||||
lzma_end(&strm);
|
||||
AIO_WritePool_releaseIoJob(writeJob);
|
||||
AIO_WritePool_sparseWriteEnd(ress->writeCtx);
|
||||
return decodingError ? FIO_ERROR_FRAME_DECODING : outFileSize;
|
||||
}
|
||||
#endif
|
||||
|
||||
#ifdef ZSTD_LZ4DECOMPRESS
|
||||
static unsigned long long
|
||||
FIO_decompressLz4Frame(dRess_t* ress, const char* srcFileName)
|
||||
{
|
||||
unsigned long long filesize = 0;
|
||||
LZ4F_errorCode_t nextToLoad = 4;
|
||||
LZ4F_decompressionContext_t dCtx;
|
||||
LZ4F_errorCode_t const errorCode = LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION);
|
||||
int decodingError = 0;
|
||||
IOJob_t *writeJob = NULL;
|
||||
|
||||
if (LZ4F_isError(errorCode)) {
|
||||
DISPLAYLEVEL(1, "zstd: failed to create lz4 decompression context \n");
|
||||
return FIO_ERROR_FRAME_DECODING;
|
||||
}
|
||||
|
||||
writeJob = AIO_WritePool_acquireJob(ress->writeCtx);
|
||||
|
||||
/* Main Loop */
|
||||
for (;nextToLoad;) {
|
||||
size_t pos = 0;
|
||||
size_t decodedBytes = writeJob->bufferSize;
|
||||
int fullBufferDecoded = 0;
|
||||
|
||||
/* Read input */
|
||||
AIO_ReadPool_fillBuffer(ress->readCtx, nextToLoad);
|
||||
if(!ress->readCtx->srcBufferLoaded) break; /* reached end of file */
|
||||
|
||||
while ((pos < ress->readCtx->srcBufferLoaded) || fullBufferDecoded) { /* still to read, or still to flush */
|
||||
/* Decode Input (at least partially) */
|
||||
size_t remaining = ress->readCtx->srcBufferLoaded - pos;
|
||||
decodedBytes = writeJob->bufferSize;
|
||||
nextToLoad = LZ4F_decompress(dCtx, writeJob->buffer, &decodedBytes, (char*)(ress->readCtx->srcBuffer)+pos,
|
||||
&remaining, NULL);
|
||||
if (LZ4F_isError(nextToLoad)) {
|
||||
DISPLAYLEVEL(1, "zstd: %s: lz4 decompression error : %s \n",
|
||||
srcFileName, LZ4F_getErrorName(nextToLoad));
|
||||
decodingError = 1; nextToLoad = 0; break;
|
||||
}
|
||||
pos += remaining;
|
||||
assert(pos <= ress->readCtx->srcBufferLoaded);
|
||||
fullBufferDecoded = decodedBytes == writeJob->bufferSize;
|
||||
|
||||
/* Write Block */
|
||||
if (decodedBytes) {
|
||||
UTIL_HumanReadableSize_t hrs;
|
||||
writeJob->usedBufferSize = decodedBytes;
|
||||
AIO_WritePool_enqueueAndReacquireWriteJob(&writeJob);
|
||||
filesize += decodedBytes;
|
||||
hrs = UTIL_makeHumanReadableSize(filesize);
|
||||
DISPLAYUPDATE_PROGRESS("\rDecompressed : %.*f%s ", hrs.precision, hrs.value, hrs.suffix);
|
||||
}
|
||||
|
||||
if (!nextToLoad) break;
|
||||
}
|
||||
AIO_ReadPool_consumeBytes(ress->readCtx, pos);
|
||||
}
|
||||
if (nextToLoad!=0) {
|
||||
DISPLAYLEVEL(1, "zstd: %s: unfinished lz4 stream \n", srcFileName);
|
||||
decodingError=1;
|
||||
}
|
||||
|
||||
LZ4F_freeDecompressionContext(dCtx);
|
||||
AIO_WritePool_releaseIoJob(writeJob);
|
||||
AIO_WritePool_sparseWriteEnd(ress->writeCtx);
|
||||
|
||||
return decodingError ? FIO_ERROR_FRAME_DECODING : filesize;
|
||||
}
|
||||
#endif
|
||||
|
||||
/* Rust owns the mixed-format loop, while these adapters keep each codec and
|
||||
* the private dRess_t layout in this C translation unit. */
|
||||
typedef struct {
|
||||
@@ -3038,6 +2931,327 @@ typedef struct {
|
||||
const FIO_prefs_t* prefs;
|
||||
} FIO_rust_decompression_projection_t;
|
||||
|
||||
#if defined(ZSTD_GZDECOMPRESS) || defined(ZSTD_LZMADECOMPRESS) || defined(ZSTD_LZ4DECOMPRESS)
|
||||
static void FIO_rust_decompressReadFill(void* opaque, size_t requested,
|
||||
const unsigned char** buffer, size_t* loaded)
|
||||
{
|
||||
ReadPoolCtx_t* const readCtx = (ReadPoolCtx_t*)opaque;
|
||||
AIO_ReadPool_fillBuffer(readCtx, requested);
|
||||
*buffer = (const unsigned char*)readCtx->srcBuffer;
|
||||
*loaded = readCtx->srcBufferLoaded;
|
||||
}
|
||||
|
||||
static void FIO_rust_decompressReadConsume(void* opaque, size_t consumed)
|
||||
{
|
||||
AIO_ReadPool_consumeBytes((ReadPoolCtx_t*)opaque, consumed);
|
||||
}
|
||||
|
||||
static void FIO_rust_decompressWriteAcquire(void* opaque, void** job,
|
||||
unsigned char** buffer, size_t* bufferSize)
|
||||
{
|
||||
IOJob_t* const writeJob = AIO_WritePool_acquireJob((WritePoolCtx_t*)opaque);
|
||||
*job = writeJob;
|
||||
*buffer = (unsigned char*)writeJob->buffer;
|
||||
*bufferSize = writeJob->bufferSize;
|
||||
}
|
||||
|
||||
static void FIO_rust_decompressWriteEnqueue(void* opaque, void** job,
|
||||
size_t usedBufferSize,
|
||||
unsigned char** buffer, size_t* bufferSize)
|
||||
{
|
||||
IOJob_t* writeJob = (IOJob_t*)*job;
|
||||
writeJob->usedBufferSize = usedBufferSize;
|
||||
AIO_WritePool_enqueueAndReacquireWriteJob(&writeJob);
|
||||
*job = writeJob;
|
||||
*buffer = (unsigned char*)writeJob->buffer;
|
||||
*bufferSize = writeJob->bufferSize;
|
||||
(void)opaque;
|
||||
}
|
||||
|
||||
static void FIO_rust_decompressWriteRelease(void* opaque, void* job)
|
||||
{
|
||||
AIO_WritePool_releaseIoJob((IOJob_t*)job);
|
||||
(void)opaque;
|
||||
}
|
||||
|
||||
static void FIO_rust_decompressSparseWriteEnd(void* opaque)
|
||||
{
|
||||
AIO_WritePool_sparseWriteEnd((WritePoolCtx_t*)opaque);
|
||||
}
|
||||
|
||||
static void FIO_rust_decompressInitIo(FIO_rust_decompress_io_projection_t* io,
|
||||
const dRess_t* ress)
|
||||
{
|
||||
memset(io, 0, sizeof(*io));
|
||||
io->readOpaque = (void*)ress->readCtx;
|
||||
io->writeOpaque = (void*)ress->writeCtx;
|
||||
io->readBufferSize = ZSTD_DStreamInSize();
|
||||
io->readFill = FIO_rust_decompressReadFill;
|
||||
io->readConsume = FIO_rust_decompressReadConsume;
|
||||
io->writeAcquire = FIO_rust_decompressWriteAcquire;
|
||||
io->writeEnqueue = FIO_rust_decompressWriteEnqueue;
|
||||
io->writeRelease = FIO_rust_decompressWriteRelease;
|
||||
io->sparseWriteEnd = FIO_rust_decompressSparseWriteEnd;
|
||||
}
|
||||
#endif
|
||||
|
||||
#ifdef ZSTD_GZDECOMPRESS
|
||||
static int FIO_rust_gzip_decompressInit(void* opaque)
|
||||
{
|
||||
z_stream* const strm = (z_stream*)opaque;
|
||||
strm->zalloc = Z_NULL;
|
||||
strm->zfree = Z_NULL;
|
||||
strm->opaque = Z_NULL;
|
||||
return inflateInit2(strm, 15 /* maxWindowLogSize */ + 16 /* gzip only */);
|
||||
}
|
||||
|
||||
static int FIO_rust_gzip_decompressInflate(void* opaque,
|
||||
const unsigned char* input, size_t inputSize,
|
||||
unsigned char* output, size_t outputSize,
|
||||
int flush, size_t* consumed, size_t* produced)
|
||||
{
|
||||
z_stream* const strm = (z_stream*)opaque;
|
||||
size_t const inputBefore = inputSize;
|
||||
size_t const outputBefore = outputSize;
|
||||
int ret;
|
||||
|
||||
assert(inputSize <= UINT_MAX);
|
||||
assert(outputSize <= UINT_MAX);
|
||||
strm->next_in = (z_const unsigned char*)input;
|
||||
strm->avail_in = (uInt)inputSize;
|
||||
strm->next_out = (Bytef*)output;
|
||||
strm->avail_out = (uInt)outputSize;
|
||||
ret = inflate(strm, flush);
|
||||
*consumed = inputBefore - strm->avail_in;
|
||||
*produced = outputBefore - strm->avail_out;
|
||||
return ret;
|
||||
}
|
||||
|
||||
static int FIO_rust_gzip_decompressEnd(void* opaque)
|
||||
{
|
||||
return inflateEnd((z_stream*)opaque);
|
||||
}
|
||||
|
||||
static int FIO_rust_decompressGzFrameCallback(void* opaque,
|
||||
const char* srcFileName,
|
||||
U64 alreadyDecoded,
|
||||
U64* frameSize,
|
||||
size_t* errorCode,
|
||||
int mode)
|
||||
{
|
||||
FIO_rust_decompression_projection_t* const projection =
|
||||
(FIO_rust_decompression_projection_t*)opaque;
|
||||
FIO_rust_gzip_decompress_projection_t codecProjection;
|
||||
z_stream strm;
|
||||
U64 decodedSize = 0;
|
||||
int zlibResult = Z_OK;
|
||||
int status;
|
||||
|
||||
(void)alreadyDecoded;
|
||||
(void)mode;
|
||||
memset(&strm, 0, sizeof(strm));
|
||||
memset(&codecProjection, 0, sizeof(codecProjection));
|
||||
FIO_rust_decompressInitIo(&codecProjection.io, projection->ress);
|
||||
codecProjection.zlibOpaque = &strm;
|
||||
codecProjection.zlibInit = FIO_rust_gzip_decompressInit;
|
||||
codecProjection.zlibInflate = FIO_rust_gzip_decompressInflate;
|
||||
codecProjection.zlibEnd = FIO_rust_gzip_decompressEnd;
|
||||
|
||||
status = FIO_rust_decompressGzipFrame(&codecProjection, &decodedSize, &zlibResult);
|
||||
*errorCode = 0;
|
||||
*frameSize = status == FIO_RUST_GZIP_DECOMPRESS_OK
|
||||
? decodedSize : FIO_ERROR_FRAME_DECODING;
|
||||
switch (status) {
|
||||
case FIO_RUST_GZIP_DECOMPRESS_OK:
|
||||
break;
|
||||
case FIO_RUST_GZIP_DECOMPRESS_BUF_ERROR:
|
||||
DISPLAYLEVEL(1, "zstd: %s: premature gz end \n", srcFileName);
|
||||
break;
|
||||
case FIO_RUST_GZIP_DECOMPRESS_INFLATE_ERROR:
|
||||
DISPLAYLEVEL(1, "zstd: %s: inflate error %d \n", srcFileName, zlibResult);
|
||||
break;
|
||||
case FIO_RUST_GZIP_DECOMPRESS_END_ERROR:
|
||||
DISPLAYLEVEL(1, "zstd: %s: inflateEnd error \n", srcFileName);
|
||||
break;
|
||||
default:
|
||||
assert(status == FIO_RUST_GZIP_DECOMPRESS_INIT_ERROR
|
||||
|| status == FIO_RUST_GZIP_DECOMPRESS_INVALID_PROJECTION);
|
||||
break;
|
||||
}
|
||||
return status != FIO_RUST_GZIP_DECOMPRESS_OK;
|
||||
}
|
||||
#endif
|
||||
|
||||
#ifdef ZSTD_LZMADECOMPRESS
|
||||
static int FIO_rust_lzma_decompressInit(void* opaque, int plainLzma)
|
||||
{
|
||||
lzma_stream* const strm = (lzma_stream*)opaque;
|
||||
if (plainLzma)
|
||||
return (int)lzma_alone_decoder(strm, UINT64_MAX); /* LZMA */
|
||||
return (int)lzma_stream_decoder(strm, UINT64_MAX, 0); /* XZ */
|
||||
}
|
||||
|
||||
static int FIO_rust_lzma_decompressCode(void* opaque,
|
||||
const unsigned char* input, size_t inputSize,
|
||||
unsigned char* output, size_t outputSize,
|
||||
int action, size_t* consumed, size_t* produced)
|
||||
{
|
||||
lzma_stream* const strm = (lzma_stream*)opaque;
|
||||
size_t const inputBefore = inputSize;
|
||||
size_t const outputBefore = outputSize;
|
||||
lzma_ret ret;
|
||||
|
||||
strm->next_in = (const uint8_t*)input;
|
||||
strm->avail_in = inputSize;
|
||||
strm->next_out = (uint8_t*)output;
|
||||
strm->avail_out = outputSize;
|
||||
ret = lzma_code(strm, (lzma_action)action);
|
||||
*consumed = inputBefore - strm->avail_in;
|
||||
*produced = outputBefore - strm->avail_out;
|
||||
return (int)ret;
|
||||
}
|
||||
|
||||
static void FIO_rust_lzma_decompressEnd(void* opaque)
|
||||
{
|
||||
lzma_end((lzma_stream*)opaque);
|
||||
}
|
||||
|
||||
static int FIO_rust_decompressLzmaFrameCallback(void* opaque,
|
||||
const char* srcFileName,
|
||||
U64 alreadyDecoded,
|
||||
U64* frameSize,
|
||||
size_t* errorCode,
|
||||
int mode)
|
||||
{
|
||||
FIO_rust_decompression_projection_t* const projection =
|
||||
(FIO_rust_decompression_projection_t*)opaque;
|
||||
FIO_rust_lzma_decompress_projection_t codecProjection;
|
||||
lzma_stream strm = LZMA_STREAM_INIT;
|
||||
U64 decodedSize = 0;
|
||||
int lzmaResult = (int)LZMA_OK;
|
||||
int status;
|
||||
|
||||
(void)alreadyDecoded;
|
||||
memset(&codecProjection, 0, sizeof(codecProjection));
|
||||
FIO_rust_decompressInitIo(&codecProjection.io, projection->ress);
|
||||
codecProjection.lzmaOpaque = &strm;
|
||||
codecProjection.lzmaInit = FIO_rust_lzma_decompressInit;
|
||||
codecProjection.lzmaCode = FIO_rust_lzma_decompressCode;
|
||||
codecProjection.lzmaEnd = FIO_rust_lzma_decompressEnd;
|
||||
|
||||
status = FIO_rust_decompressLzmaFrame(
|
||||
&codecProjection, mode, &decodedSize, &lzmaResult);
|
||||
*errorCode = 0;
|
||||
*frameSize = status == FIO_RUST_LZMA_DECOMPRESS_OK
|
||||
? decodedSize : FIO_ERROR_FRAME_DECODING;
|
||||
switch (status) {
|
||||
case FIO_RUST_LZMA_DECOMPRESS_OK:
|
||||
break;
|
||||
case FIO_RUST_LZMA_DECOMPRESS_INIT_ERROR:
|
||||
DISPLAYLEVEL(1, "zstd: %s: %s error %d \n",
|
||||
mode ? "lzma_alone_decoder" : "lzma_stream_decoder",
|
||||
srcFileName, lzmaResult);
|
||||
break;
|
||||
case FIO_RUST_LZMA_DECOMPRESS_BUF_ERROR:
|
||||
DISPLAYLEVEL(1, "zstd: %s: premature lzma end \n", srcFileName);
|
||||
break;
|
||||
case FIO_RUST_LZMA_DECOMPRESS_CODE_ERROR:
|
||||
DISPLAYLEVEL(1, "zstd: %s: lzma_code decoding error %d \n",
|
||||
srcFileName, lzmaResult);
|
||||
break;
|
||||
default:
|
||||
assert(status == FIO_RUST_LZMA_DECOMPRESS_INVALID_PROJECTION);
|
||||
break;
|
||||
}
|
||||
return status != FIO_RUST_LZMA_DECOMPRESS_OK;
|
||||
}
|
||||
#endif
|
||||
|
||||
#ifdef ZSTD_LZ4DECOMPRESS
|
||||
static int FIO_rust_lz4_decompressCreate(void* opaque, unsigned version, size_t* result)
|
||||
{
|
||||
LZ4F_errorCode_t const errorCode = LZ4F_createDecompressionContext(
|
||||
(LZ4F_decompressionContext_t*)opaque, version);
|
||||
*result = (size_t)errorCode;
|
||||
return LZ4F_isError(errorCode);
|
||||
}
|
||||
|
||||
static int FIO_rust_lz4_decompressCode(void* opaque, unsigned char* output,
|
||||
size_t* outputSize, const unsigned char* input,
|
||||
size_t* inputSize, size_t* nextToLoad)
|
||||
{
|
||||
LZ4F_errorCode_t const result = LZ4F_decompress(
|
||||
*(LZ4F_decompressionContext_t*)opaque, output, outputSize,
|
||||
input, inputSize, NULL);
|
||||
*nextToLoad = (size_t)result;
|
||||
return LZ4F_isError(result);
|
||||
}
|
||||
|
||||
static void FIO_rust_lz4_decompressFree(void* opaque)
|
||||
{
|
||||
LZ4F_freeDecompressionContext(*(LZ4F_decompressionContext_t*)opaque);
|
||||
}
|
||||
|
||||
static void FIO_rust_lz4_decompressProgress(void* opaque, U64 decodedSize)
|
||||
{
|
||||
UTIL_HumanReadableSize_t const hrs = UTIL_makeHumanReadableSize(decodedSize);
|
||||
(void)opaque;
|
||||
DISPLAYUPDATE_PROGRESS("\rDecompressed : %.*f%s ",
|
||||
hrs.precision, hrs.value, hrs.suffix);
|
||||
}
|
||||
|
||||
static int FIO_rust_decompressLz4FrameCallback(void* opaque,
|
||||
const char* srcFileName,
|
||||
U64 alreadyDecoded,
|
||||
U64* frameSize,
|
||||
size_t* errorCode,
|
||||
int mode)
|
||||
{
|
||||
FIO_rust_decompression_projection_t* const projection =
|
||||
(FIO_rust_decompression_projection_t*)opaque;
|
||||
FIO_rust_lz4_decompress_projection_t codecProjection;
|
||||
LZ4F_decompressionContext_t dCtx;
|
||||
U64 decodedSize = 0;
|
||||
size_t lz4Result = 0;
|
||||
int status;
|
||||
|
||||
(void)alreadyDecoded;
|
||||
(void)mode;
|
||||
memset(&codecProjection, 0, sizeof(codecProjection));
|
||||
FIO_rust_decompressInitIo(&codecProjection.io, projection->ress);
|
||||
codecProjection.codecOpaque = &dCtx;
|
||||
codecProjection.progressOpaque = NULL;
|
||||
codecProjection.version = LZ4F_VERSION;
|
||||
codecProjection.create = FIO_rust_lz4_decompressCreate;
|
||||
codecProjection.code = FIO_rust_lz4_decompressCode;
|
||||
codecProjection.freeContext = FIO_rust_lz4_decompressFree;
|
||||
codecProjection.progress = FIO_rust_lz4_decompressProgress;
|
||||
|
||||
status = FIO_rust_decompressLz4Frame(&codecProjection, &decodedSize, &lz4Result);
|
||||
*errorCode = 0;
|
||||
*frameSize = status == FIO_RUST_LZ4_DECOMPRESS_OK
|
||||
? decodedSize : FIO_ERROR_FRAME_DECODING;
|
||||
switch (status) {
|
||||
case FIO_RUST_LZ4_DECOMPRESS_OK:
|
||||
break;
|
||||
case FIO_RUST_LZ4_DECOMPRESS_CREATE_ERROR:
|
||||
DISPLAYLEVEL(1, "zstd: failed to create lz4 decompression context \n");
|
||||
break;
|
||||
case FIO_RUST_LZ4_DECOMPRESS_CODE_ERROR:
|
||||
DISPLAYLEVEL(1, "zstd: %s: lz4 decompression error : %s \n",
|
||||
srcFileName, LZ4F_getErrorName(lz4Result));
|
||||
break;
|
||||
case FIO_RUST_LZ4_DECOMPRESS_UNFINISHED:
|
||||
DISPLAYLEVEL(1, "zstd: %s: unfinished lz4 stream \n", srcFileName);
|
||||
break;
|
||||
default:
|
||||
assert(status == FIO_RUST_LZ4_DECOMPRESS_INVALID_PROJECTION);
|
||||
break;
|
||||
}
|
||||
return status != FIO_RUST_LZ4_DECOMPRESS_OK;
|
||||
}
|
||||
#endif
|
||||
|
||||
static int FIO_rust_decompressZstdFrameCallback(void* opaque,
|
||||
const char* srcFileName,
|
||||
U64 alreadyDecoded,
|
||||
@@ -3057,59 +3271,6 @@ static int FIO_rust_decompressZstdFrameCallback(void* opaque,
|
||||
return *frameSize == FIO_ERROR_FRAME_DECODING;
|
||||
}
|
||||
|
||||
#ifdef ZSTD_GZDECOMPRESS
|
||||
static int FIO_rust_decompressGzFrameCallback(void* opaque,
|
||||
const char* srcFileName,
|
||||
U64 alreadyDecoded,
|
||||
U64* frameSize,
|
||||
size_t* errorCode,
|
||||
int mode)
|
||||
{
|
||||
FIO_rust_decompression_projection_t* const projection =
|
||||
(FIO_rust_decompression_projection_t*)opaque;
|
||||
(void)alreadyDecoded;
|
||||
(void)mode;
|
||||
*errorCode = 0;
|
||||
*frameSize = FIO_decompressGzFrame(projection->ress, srcFileName);
|
||||
return *frameSize == FIO_ERROR_FRAME_DECODING;
|
||||
}
|
||||
#endif
|
||||
|
||||
#ifdef ZSTD_LZMADECOMPRESS
|
||||
static int FIO_rust_decompressLzmaFrameCallback(void* opaque,
|
||||
const char* srcFileName,
|
||||
U64 alreadyDecoded,
|
||||
U64* frameSize,
|
||||
size_t* errorCode,
|
||||
int mode)
|
||||
{
|
||||
FIO_rust_decompression_projection_t* const projection =
|
||||
(FIO_rust_decompression_projection_t*)opaque;
|
||||
(void)alreadyDecoded;
|
||||
*errorCode = 0;
|
||||
*frameSize = FIO_decompressLzmaFrame(projection->ress, srcFileName, mode);
|
||||
return *frameSize == FIO_ERROR_FRAME_DECODING;
|
||||
}
|
||||
#endif
|
||||
|
||||
#ifdef ZSTD_LZ4DECOMPRESS
|
||||
static int FIO_rust_decompressLz4FrameCallback(void* opaque,
|
||||
const char* srcFileName,
|
||||
U64 alreadyDecoded,
|
||||
U64* frameSize,
|
||||
size_t* errorCode,
|
||||
int mode)
|
||||
{
|
||||
FIO_rust_decompression_projection_t* const projection =
|
||||
(FIO_rust_decompression_projection_t*)opaque;
|
||||
(void)alreadyDecoded;
|
||||
(void)mode;
|
||||
*errorCode = 0;
|
||||
*frameSize = FIO_decompressLz4Frame(projection->ress, srcFileName);
|
||||
return *frameSize == FIO_ERROR_FRAME_DECODING;
|
||||
}
|
||||
#endif
|
||||
|
||||
static int FIO_rust_decompressPassThroughCallback(void* opaque)
|
||||
{
|
||||
FIO_rust_decompression_projection_t* const projection =
|
||||
|
||||
Reference in New Issue
Block a user