ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(cli): move --list frame analysis to Rust

Browse Source 
Keep the C --list orchestration, file ownership, fileInfo_t storage, and
DISPLAYLEVEL/reporting behavior in programs/fileio.c while replacing only the
frame scanner with a guarded Rust FFI leaf. The Rust implementation preserves
the C status values, fread lookahead, large-file seek/tell behavior, frame and
skippable-frame accounting, content-size and window updates, checksum capture,
RLE and invalid-block handling, dictionary-ID aggregation, and truncation
classification without taking ownership of FILE or the output structure.

A C diagnostic bridge keeps the existing CLI messages and warning formatting
outside the scanner. The Rust tests use temporary C streams and focused frame
buffers, including a test-only header API shim so the unit tests remain safe
and link independently of the full CLI binary. The decompression feature guard
also keeps the scanner absent from no-decompression CLI archives.

Test Plan:
- cargo test --no-default-features --features cli,compression,decompression,benchmark --lib fileio_prefs (45 passed)
- cargo test --no-default-features --features cli,compression,decompression,benchmark (152 passed)
- library and CLI clippy for main, benches, and tests, before and after nightly formatting (clean)
- cargo +nightly fmt -- --check for both library and CLI crates (clean)
- make -B -C programs -j2 zstd, zstd-small, zstd-frugal, and zstd-dictBuilder (passed)
- exact --list/-l playTests.sh blocks (passed)
- make -C tests -j2 test-cli-tests (41 passed)
- nm archive guard check confirmed FIO_rust_analyzeFrames only in the decompression-enabled CLI archive
- zstd-decompress remains blocked by the pre-existing DEFAULT_MAX_CLEVEL feature-gating error in rust/src/zstd_cli.rs
- zstd-compress remains blocked by pre-existing unresolved ZDICT_* references; neither blocker is in the owned files
This commit is contained in:
ddidderr
2026-07-18 16:31:55 +02:00
parent 6f905fdcc4
commit 17b610b73c
2 changed files with 795 additions and 91 deletions
Download Patch File Download Diff File Expand all files Collapse all files
programs/fileio.c
+72 -91
View File
@@ -2870,6 +2870,26 @@ typedef enum {
info_truncated_input=4
} InfoError;
enum {
FIO_RUST_ANALYZE_DIAG_SEEKED_PAST_FILE = 0,
FIO_RUST_ANALYZE_DIAG_INCOMPLETE_FRAME = 1,
FIO_RUST_ANALYZE_DIAG_RAN_OUT_OF_FRAMES = 2,
FIO_RUST_ANALYZE_DIAG_DECODE_FRAME_HEADER = 3,
FIO_RUST_ANALYZE_DIAG_FRAME_HEADER_SIZE = 4,
FIO_RUST_ANALYZE_DIAG_MOVE_TO_FRAME_HEADER_END = 5,
FIO_RUST_ANALYZE_DIAG_BLOCK_HEADER = 6,
FIO_RUST_ANALYZE_DIAG_UNSUPPORTED_BLOCK_TYPE = 7,
FIO_RUST_ANALYZE_DIAG_SKIP_BLOCK = 8,
FIO_RUST_ANALYZE_DIAG_CHECKSUM = 9,
FIO_RUST_ANALYZE_DIAG_SKIP_FRAME = 10,
FIO_RUST_ANALYZE_DIAG_MIXED_DICTIONARY_IDS = 11
};
int FIO_rust_analyzeFrames(fileInfo_t* info, FILE* srcFile);
void FIO_rust_analyzeFrames_display(int diagnostic,
unsigned long long filePosition,
unsigned long long fileSize);
#define ERROR_IF(c,n,...) { \
if (c) { \
DISPLAYLEVEL(1, __VA_ARGS__); \
@@ -2878,100 +2898,61 @@ typedef enum {
} \
}
void
FIO_rust_analyzeFrames_display(int diagnostic,
unsigned long long filePosition,
unsigned long long fileSize)
{
switch (diagnostic) {
case FIO_RUST_ANALYZE_DIAG_SEEKED_PAST_FILE:
DISPLAYLEVEL(1,
"Error: seeked to position %llu, which is beyond file size of %llu\n",
filePosition, fileSize);
break;
case FIO_RUST_ANALYZE_DIAG_INCOMPLETE_FRAME:
DISPLAYLEVEL(1, "Error: reached end of file with incomplete frame");
break;
case FIO_RUST_ANALYZE_DIAG_RAN_OUT_OF_FRAMES:
DISPLAYLEVEL(1, "Error: did not reach end of file but ran out of frames");
break;
case FIO_RUST_ANALYZE_DIAG_DECODE_FRAME_HEADER:
DISPLAYLEVEL(1, "Error: could not decode frame header");
break;
case FIO_RUST_ANALYZE_DIAG_FRAME_HEADER_SIZE:
DISPLAYLEVEL(1, "Error: could not determine frame header size");
break;
case FIO_RUST_ANALYZE_DIAG_MOVE_TO_FRAME_HEADER_END:
DISPLAYLEVEL(1, "Error: could not move to end of frame header");
break;
case FIO_RUST_ANALYZE_DIAG_BLOCK_HEADER:
DISPLAYLEVEL(1, "Error while reading block header");
break;
case FIO_RUST_ANALYZE_DIAG_UNSUPPORTED_BLOCK_TYPE:
DISPLAYLEVEL(1, "Error: unsupported block type");
break;
case FIO_RUST_ANALYZE_DIAG_SKIP_BLOCK:
DISPLAYLEVEL(1, "Error: could not skip to end of block");
break;
case FIO_RUST_ANALYZE_DIAG_CHECKSUM:
DISPLAYLEVEL(1, "Error: could not read checksum");
break;
case FIO_RUST_ANALYZE_DIAG_SKIP_FRAME:
DISPLAYLEVEL(1, "Error: could not find end of skippable frame");
break;
case FIO_RUST_ANALYZE_DIAG_MIXED_DICTIONARY_IDS:
DISPLAY("WARNING: File contains multiple frames with different dictionary IDs. Showing dictID 0 instead");
return;
default:
assert(0);
return;
}
DISPLAYLEVEL(1, " \n");
}
static InfoError
FIO_analyzeFrames(fileInfo_t* info, FILE* const srcFile)
{
/* begin analyzing frame */
for ( ; ; ) {
BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX];
size_t const numBytesRead = fread(headerBuffer, 1, sizeof(headerBuffer), srcFile);
if (numBytesRead < ZSTD_FRAMEHEADERSIZE_MIN(ZSTD_f_zstd1)) {
if ( feof(srcFile)
&& (numBytesRead == 0)
&& (info->compressedSize > 0)
&& (info->compressedSize != UTIL_FILESIZE_UNKNOWN) ) {
unsigned long long file_position = (unsigned long long) LONG_TELL(srcFile);
unsigned long long file_size = (unsigned long long) info->compressedSize;
ERROR_IF(file_position != file_size, info_truncated_input,
"Error: seeked to position %llu, which is beyond file size of %llu\n",
file_position,
file_size);
break; /* correct end of file => success */
}
ERROR_IF(feof(srcFile), info_not_zstd, "Error: reached end of file with incomplete frame");
ERROR_IF(1, info_frame_error, "Error: did not reach end of file but ran out of frames");
}
{ U32 const magicNumber = MEM_readLE32(headerBuffer);
/* Zstandard frame */
if (magicNumber == ZSTD_MAGICNUMBER) {
ZSTD_FrameHeader header;
U64 const frameContentSize = ZSTD_getFrameContentSize(headerBuffer, numBytesRead);
if ( frameContentSize == ZSTD_CONTENTSIZE_ERROR
|| frameContentSize == ZSTD_CONTENTSIZE_UNKNOWN ) {
info->decompUnavailable = 1;
} else {
info->decompressedSize += frameContentSize;
}
ERROR_IF(ZSTD_getFrameHeader(&header, headerBuffer, numBytesRead) != 0,
info_frame_error, "Error: could not decode frame header");
if (info->dictID != 0 && info->dictID != header.dictID) {
DISPLAY("WARNING: File contains multiple frames with different dictionary IDs. Showing dictID 0 instead");
info->dictID = 0;
} else {
info->dictID = header.dictID;
}
info->windowSize = header.windowSize;
/* move to the end of the frame header */
{ size_t const headerSize = ZSTD_frameHeaderSize(headerBuffer, numBytesRead);
ERROR_IF(ZSTD_isError(headerSize), info_frame_error, "Error: could not determine frame header size");
ERROR_IF(fseek(srcFile, ((long)headerSize)-((long)numBytesRead), SEEK_CUR) != 0,
info_frame_error, "Error: could not move to end of frame header");
}
/* skip all blocks in the frame */
{ int lastBlock = 0;
do {
BYTE blockHeaderBuffer[3];
ERROR_IF(fread(blockHeaderBuffer, 1, 3, srcFile) != 3,
info_frame_error, "Error while reading block header");
{ U32 const blockHeader = MEM_readLE24(blockHeaderBuffer);
U32 const blockTypeID = (blockHeader >> 1) & 3;
U32 const isRLE = (blockTypeID == 1);
U32 const isWrongBlock = (blockTypeID == 3);
long const blockSize = isRLE ? 1 : (long)(blockHeader >> 3);
ERROR_IF(isWrongBlock, info_frame_error, "Error: unsupported block type");
lastBlock = blockHeader & 1;
ERROR_IF(fseek(srcFile, blockSize, SEEK_CUR) != 0,
info_frame_error, "Error: could not skip to end of block");
}
} while (lastBlock != 1);
}
/* check if checksum is used */
{ BYTE const frameHeaderDescriptor = headerBuffer[4];
int const contentChecksumFlag = (frameHeaderDescriptor & (1 << 2)) >> 2;
if (contentChecksumFlag) {
info->usesCheck = 1;
ERROR_IF(fread(info->checksum, 1, 4, srcFile) != 4,
info_frame_error, "Error: could not read checksum");
} }
info->numActualFrames++;
}
/* Skippable frame */
else if ((magicNumber & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) {
U32 const frameSize = MEM_readLE32(headerBuffer + 4);
long const seek = (long)(8 + frameSize - numBytesRead);
ERROR_IF(LONG_SEEK(srcFile, seek, SEEK_CUR) != 0,
info_frame_error, "Error: could not find end of skippable frame");
info->numSkippableFrames++;
}
/* unknown content */
else {
return info_not_zstd;
}
} /* magic number analysis */
} /* end analyzing frames */
return info_success;
return (InfoError)FIO_rust_analyzeFrames(info, srcFile);
}