feat(cli): move --list frame analysis to Rust
Keep the C --list orchestration, file ownership, fileInfo_t storage, and DISPLAYLEVEL/reporting behavior in programs/fileio.c while replacing only the frame scanner with a guarded Rust FFI leaf. The Rust implementation preserves the C status values, fread lookahead, large-file seek/tell behavior, frame and skippable-frame accounting, content-size and window updates, checksum capture, RLE and invalid-block handling, dictionary-ID aggregation, and truncation classification without taking ownership of FILE or the output structure. A C diagnostic bridge keeps the existing CLI messages and warning formatting outside the scanner. The Rust tests use temporary C streams and focused frame buffers, including a test-only header API shim so the unit tests remain safe and link independently of the full CLI binary. The decompression feature guard also keeps the scanner absent from no-decompression CLI archives. Test Plan: - cargo test --no-default-features --features cli,compression,decompression,benchmark --lib fileio_prefs (45 passed) - cargo test --no-default-features --features cli,compression,decompression,benchmark (152 passed) - library and CLI clippy for main, benches, and tests, before and after nightly formatting (clean) - cargo +nightly fmt -- --check for both library and CLI crates (clean) - make -B -C programs -j2 zstd, zstd-small, zstd-frugal, and zstd-dictBuilder (passed) - exact --list/-l playTests.sh blocks (passed) - make -C tests -j2 test-cli-tests (41 passed) - nm archive guard check confirmed FIO_rust_analyzeFrames only in the decompression-enabled CLI archive - zstd-decompress remains blocked by the pre-existing DEFAULT_MAX_CLEVEL feature-gating error in rust/src/zstd_cli.rs - zstd-compress remains blocked by pre-existing unresolved ZDICT_* references; neither blocker is in the owned files
This commit is contained in:
+72
-91
@@ -2870,6 +2870,26 @@ typedef enum {
|
||||
info_truncated_input=4
|
||||
} InfoError;
|
||||
|
||||
enum {
|
||||
FIO_RUST_ANALYZE_DIAG_SEEKED_PAST_FILE = 0,
|
||||
FIO_RUST_ANALYZE_DIAG_INCOMPLETE_FRAME = 1,
|
||||
FIO_RUST_ANALYZE_DIAG_RAN_OUT_OF_FRAMES = 2,
|
||||
FIO_RUST_ANALYZE_DIAG_DECODE_FRAME_HEADER = 3,
|
||||
FIO_RUST_ANALYZE_DIAG_FRAME_HEADER_SIZE = 4,
|
||||
FIO_RUST_ANALYZE_DIAG_MOVE_TO_FRAME_HEADER_END = 5,
|
||||
FIO_RUST_ANALYZE_DIAG_BLOCK_HEADER = 6,
|
||||
FIO_RUST_ANALYZE_DIAG_UNSUPPORTED_BLOCK_TYPE = 7,
|
||||
FIO_RUST_ANALYZE_DIAG_SKIP_BLOCK = 8,
|
||||
FIO_RUST_ANALYZE_DIAG_CHECKSUM = 9,
|
||||
FIO_RUST_ANALYZE_DIAG_SKIP_FRAME = 10,
|
||||
FIO_RUST_ANALYZE_DIAG_MIXED_DICTIONARY_IDS = 11
|
||||
};
|
||||
|
||||
int FIO_rust_analyzeFrames(fileInfo_t* info, FILE* srcFile);
|
||||
void FIO_rust_analyzeFrames_display(int diagnostic,
|
||||
unsigned long long filePosition,
|
||||
unsigned long long fileSize);
|
||||
|
||||
#define ERROR_IF(c,n,...) { \
|
||||
if (c) { \
|
||||
DISPLAYLEVEL(1, __VA_ARGS__); \
|
||||
@@ -2878,100 +2898,61 @@ typedef enum {
|
||||
} \
|
||||
}
|
||||
|
||||
void
|
||||
FIO_rust_analyzeFrames_display(int diagnostic,
|
||||
unsigned long long filePosition,
|
||||
unsigned long long fileSize)
|
||||
{
|
||||
switch (diagnostic) {
|
||||
case FIO_RUST_ANALYZE_DIAG_SEEKED_PAST_FILE:
|
||||
DISPLAYLEVEL(1,
|
||||
"Error: seeked to position %llu, which is beyond file size of %llu\n",
|
||||
filePosition, fileSize);
|
||||
break;
|
||||
case FIO_RUST_ANALYZE_DIAG_INCOMPLETE_FRAME:
|
||||
DISPLAYLEVEL(1, "Error: reached end of file with incomplete frame");
|
||||
break;
|
||||
case FIO_RUST_ANALYZE_DIAG_RAN_OUT_OF_FRAMES:
|
||||
DISPLAYLEVEL(1, "Error: did not reach end of file but ran out of frames");
|
||||
break;
|
||||
case FIO_RUST_ANALYZE_DIAG_DECODE_FRAME_HEADER:
|
||||
DISPLAYLEVEL(1, "Error: could not decode frame header");
|
||||
break;
|
||||
case FIO_RUST_ANALYZE_DIAG_FRAME_HEADER_SIZE:
|
||||
DISPLAYLEVEL(1, "Error: could not determine frame header size");
|
||||
break;
|
||||
case FIO_RUST_ANALYZE_DIAG_MOVE_TO_FRAME_HEADER_END:
|
||||
DISPLAYLEVEL(1, "Error: could not move to end of frame header");
|
||||
break;
|
||||
case FIO_RUST_ANALYZE_DIAG_BLOCK_HEADER:
|
||||
DISPLAYLEVEL(1, "Error while reading block header");
|
||||
break;
|
||||
case FIO_RUST_ANALYZE_DIAG_UNSUPPORTED_BLOCK_TYPE:
|
||||
DISPLAYLEVEL(1, "Error: unsupported block type");
|
||||
break;
|
||||
case FIO_RUST_ANALYZE_DIAG_SKIP_BLOCK:
|
||||
DISPLAYLEVEL(1, "Error: could not skip to end of block");
|
||||
break;
|
||||
case FIO_RUST_ANALYZE_DIAG_CHECKSUM:
|
||||
DISPLAYLEVEL(1, "Error: could not read checksum");
|
||||
break;
|
||||
case FIO_RUST_ANALYZE_DIAG_SKIP_FRAME:
|
||||
DISPLAYLEVEL(1, "Error: could not find end of skippable frame");
|
||||
break;
|
||||
case FIO_RUST_ANALYZE_DIAG_MIXED_DICTIONARY_IDS:
|
||||
DISPLAY("WARNING: File contains multiple frames with different dictionary IDs. Showing dictID 0 instead");
|
||||
return;
|
||||
default:
|
||||
assert(0);
|
||||
return;
|
||||
}
|
||||
DISPLAYLEVEL(1, " \n");
|
||||
}
|
||||
|
||||
static InfoError
|
||||
FIO_analyzeFrames(fileInfo_t* info, FILE* const srcFile)
|
||||
{
|
||||
/* begin analyzing frame */
|
||||
for ( ; ; ) {
|
||||
BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX];
|
||||
size_t const numBytesRead = fread(headerBuffer, 1, sizeof(headerBuffer), srcFile);
|
||||
if (numBytesRead < ZSTD_FRAMEHEADERSIZE_MIN(ZSTD_f_zstd1)) {
|
||||
if ( feof(srcFile)
|
||||
&& (numBytesRead == 0)
|
||||
&& (info->compressedSize > 0)
|
||||
&& (info->compressedSize != UTIL_FILESIZE_UNKNOWN) ) {
|
||||
unsigned long long file_position = (unsigned long long) LONG_TELL(srcFile);
|
||||
unsigned long long file_size = (unsigned long long) info->compressedSize;
|
||||
ERROR_IF(file_position != file_size, info_truncated_input,
|
||||
"Error: seeked to position %llu, which is beyond file size of %llu\n",
|
||||
file_position,
|
||||
file_size);
|
||||
break; /* correct end of file => success */
|
||||
}
|
||||
ERROR_IF(feof(srcFile), info_not_zstd, "Error: reached end of file with incomplete frame");
|
||||
ERROR_IF(1, info_frame_error, "Error: did not reach end of file but ran out of frames");
|
||||
}
|
||||
{ U32 const magicNumber = MEM_readLE32(headerBuffer);
|
||||
/* Zstandard frame */
|
||||
if (magicNumber == ZSTD_MAGICNUMBER) {
|
||||
ZSTD_FrameHeader header;
|
||||
U64 const frameContentSize = ZSTD_getFrameContentSize(headerBuffer, numBytesRead);
|
||||
if ( frameContentSize == ZSTD_CONTENTSIZE_ERROR
|
||||
|| frameContentSize == ZSTD_CONTENTSIZE_UNKNOWN ) {
|
||||
info->decompUnavailable = 1;
|
||||
} else {
|
||||
info->decompressedSize += frameContentSize;
|
||||
}
|
||||
ERROR_IF(ZSTD_getFrameHeader(&header, headerBuffer, numBytesRead) != 0,
|
||||
info_frame_error, "Error: could not decode frame header");
|
||||
if (info->dictID != 0 && info->dictID != header.dictID) {
|
||||
DISPLAY("WARNING: File contains multiple frames with different dictionary IDs. Showing dictID 0 instead");
|
||||
info->dictID = 0;
|
||||
} else {
|
||||
info->dictID = header.dictID;
|
||||
}
|
||||
info->windowSize = header.windowSize;
|
||||
/* move to the end of the frame header */
|
||||
{ size_t const headerSize = ZSTD_frameHeaderSize(headerBuffer, numBytesRead);
|
||||
ERROR_IF(ZSTD_isError(headerSize), info_frame_error, "Error: could not determine frame header size");
|
||||
ERROR_IF(fseek(srcFile, ((long)headerSize)-((long)numBytesRead), SEEK_CUR) != 0,
|
||||
info_frame_error, "Error: could not move to end of frame header");
|
||||
}
|
||||
|
||||
/* skip all blocks in the frame */
|
||||
{ int lastBlock = 0;
|
||||
do {
|
||||
BYTE blockHeaderBuffer[3];
|
||||
ERROR_IF(fread(blockHeaderBuffer, 1, 3, srcFile) != 3,
|
||||
info_frame_error, "Error while reading block header");
|
||||
{ U32 const blockHeader = MEM_readLE24(blockHeaderBuffer);
|
||||
U32 const blockTypeID = (blockHeader >> 1) & 3;
|
||||
U32 const isRLE = (blockTypeID == 1);
|
||||
U32 const isWrongBlock = (blockTypeID == 3);
|
||||
long const blockSize = isRLE ? 1 : (long)(blockHeader >> 3);
|
||||
ERROR_IF(isWrongBlock, info_frame_error, "Error: unsupported block type");
|
||||
lastBlock = blockHeader & 1;
|
||||
ERROR_IF(fseek(srcFile, blockSize, SEEK_CUR) != 0,
|
||||
info_frame_error, "Error: could not skip to end of block");
|
||||
}
|
||||
} while (lastBlock != 1);
|
||||
}
|
||||
|
||||
/* check if checksum is used */
|
||||
{ BYTE const frameHeaderDescriptor = headerBuffer[4];
|
||||
int const contentChecksumFlag = (frameHeaderDescriptor & (1 << 2)) >> 2;
|
||||
if (contentChecksumFlag) {
|
||||
info->usesCheck = 1;
|
||||
ERROR_IF(fread(info->checksum, 1, 4, srcFile) != 4,
|
||||
info_frame_error, "Error: could not read checksum");
|
||||
} }
|
||||
info->numActualFrames++;
|
||||
}
|
||||
/* Skippable frame */
|
||||
else if ((magicNumber & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) {
|
||||
U32 const frameSize = MEM_readLE32(headerBuffer + 4);
|
||||
long const seek = (long)(8 + frameSize - numBytesRead);
|
||||
ERROR_IF(LONG_SEEK(srcFile, seek, SEEK_CUR) != 0,
|
||||
info_frame_error, "Error: could not find end of skippable frame");
|
||||
info->numSkippableFrames++;
|
||||
}
|
||||
/* unknown content */
|
||||
else {
|
||||
return info_not_zstd;
|
||||
}
|
||||
} /* magic number analysis */
|
||||
} /* end analyzing frames */
|
||||
return info_success;
|
||||
return (InfoError)FIO_rust_analyzeFrames(info, srcFile);
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -86,6 +86,518 @@ pub struct FIO_fileInfo_t {
|
||||
dictID: c_uint,
|
||||
}
|
||||
|
||||
#[cfg(feature = "decompression")]
|
||||
const FIO_INFO_SUCCESS: c_int = 0;
|
||||
#[cfg(feature = "decompression")]
|
||||
const FIO_INFO_FRAME_ERROR: c_int = 1;
|
||||
#[cfg(feature = "decompression")]
|
||||
const FIO_INFO_NOT_ZSTD: c_int = 2;
|
||||
#[cfg(feature = "decompression")]
|
||||
const FIO_INFO_TRUNCATED_INPUT: c_int = 4;
|
||||
#[cfg(feature = "decompression")]
|
||||
const ZSTD_MAGICNUMBER: u32 = 0xFD2F_B528;
|
||||
#[cfg(feature = "decompression")]
|
||||
const ZSTD_MAGIC_SKIPPABLE_START: u32 = 0x184D_2A50;
|
||||
#[cfg(feature = "decompression")]
|
||||
const ZSTD_MAGIC_SKIPPABLE_MASK: u32 = 0xFFFF_FFF0;
|
||||
#[cfg(feature = "decompression")]
|
||||
const ZSTD_CONTENTSIZE_ERROR: u64 = u64::MAX - 1;
|
||||
#[cfg(feature = "decompression")]
|
||||
const ZSTD_CONTENTSIZE_UNKNOWN: u64 = u64::MAX;
|
||||
#[cfg(feature = "decompression")]
|
||||
const ZSTD_FRAMEHEADERSIZE_MAX: usize = 18;
|
||||
#[cfg(feature = "decompression")]
|
||||
const ZSTD_FRAMEHEADERSIZE_MIN: usize = 6;
|
||||
|
||||
#[cfg(feature = "decompression")]
|
||||
const FIO_RUST_ANALYZE_DIAG_SEEKED_PAST_FILE: c_int = 0;
|
||||
#[cfg(feature = "decompression")]
|
||||
const FIO_RUST_ANALYZE_DIAG_INCOMPLETE_FRAME: c_int = 1;
|
||||
#[cfg(feature = "decompression")]
|
||||
const FIO_RUST_ANALYZE_DIAG_RAN_OUT_OF_FRAMES: c_int = 2;
|
||||
#[cfg(feature = "decompression")]
|
||||
const FIO_RUST_ANALYZE_DIAG_DECODE_FRAME_HEADER: c_int = 3;
|
||||
#[cfg(feature = "decompression")]
|
||||
const FIO_RUST_ANALYZE_DIAG_FRAME_HEADER_SIZE: c_int = 4;
|
||||
#[cfg(feature = "decompression")]
|
||||
const FIO_RUST_ANALYZE_DIAG_MOVE_TO_FRAME_HEADER_END: c_int = 5;
|
||||
#[cfg(feature = "decompression")]
|
||||
const FIO_RUST_ANALYZE_DIAG_BLOCK_HEADER: c_int = 6;
|
||||
#[cfg(feature = "decompression")]
|
||||
const FIO_RUST_ANALYZE_DIAG_UNSUPPORTED_BLOCK_TYPE: c_int = 7;
|
||||
#[cfg(feature = "decompression")]
|
||||
const FIO_RUST_ANALYZE_DIAG_SKIP_BLOCK: c_int = 8;
|
||||
#[cfg(feature = "decompression")]
|
||||
const FIO_RUST_ANALYZE_DIAG_CHECKSUM: c_int = 9;
|
||||
#[cfg(feature = "decompression")]
|
||||
const FIO_RUST_ANALYZE_DIAG_SKIP_FRAME: c_int = 10;
|
||||
#[cfg(feature = "decompression")]
|
||||
const FIO_RUST_ANALYZE_DIAG_MIXED_DICTIONARY_IDS: c_int = 11;
|
||||
|
||||
#[cfg(feature = "decompression")]
|
||||
#[repr(C)]
|
||||
#[derive(Clone, Copy, Default)]
|
||||
struct FIO_zstdFrameHeader {
|
||||
frameContentSize: u64,
|
||||
windowSize: u64,
|
||||
blockSizeMax: c_uint,
|
||||
frameType: c_int,
|
||||
headerSize: c_uint,
|
||||
dictID: c_uint,
|
||||
checksumFlag: c_uint,
|
||||
reserved1: c_uint,
|
||||
reserved2: c_uint,
|
||||
}
|
||||
|
||||
#[cfg(all(feature = "decompression", not(test)))]
|
||||
unsafe extern "C" {
|
||||
fn ZSTD_getFrameContentSize(src: *const c_void, srcSize: usize) -> u64;
|
||||
fn ZSTD_getFrameHeader(
|
||||
zfhPtr: *mut FIO_zstdFrameHeader,
|
||||
src: *const c_void,
|
||||
srcSize: usize,
|
||||
) -> usize;
|
||||
fn ZSTD_frameHeaderSize(src: *const c_void, srcSize: usize) -> usize;
|
||||
fn ZSTD_isError(code: usize) -> c_uint;
|
||||
fn FIO_rust_analyzeFrames_display(diagnostic: c_int, filePosition: u64, fileSize: u64);
|
||||
}
|
||||
|
||||
#[cfg(all(feature = "decompression", windows))]
|
||||
unsafe extern "C" {
|
||||
fn _fseeki64(file: *mut libc::FILE, offset: i64, whence: c_int) -> c_int;
|
||||
fn _ftelli64(file: *mut libc::FILE) -> i64;
|
||||
}
|
||||
|
||||
#[cfg(all(test, feature = "decompression"))]
|
||||
mod analyze_test_api {
|
||||
use super::*;
|
||||
use std::slice;
|
||||
|
||||
fn read_le16(bytes: &[u8], offset: usize) -> u16 {
|
||||
u16::from_le_bytes([bytes[offset], bytes[offset + 1]])
|
||||
}
|
||||
|
||||
fn read_le32(bytes: &[u8], offset: usize) -> u32 {
|
||||
u32::from_le_bytes([
|
||||
bytes[offset],
|
||||
bytes[offset + 1],
|
||||
bytes[offset + 2],
|
||||
bytes[offset + 3],
|
||||
])
|
||||
}
|
||||
|
||||
fn read_le64(bytes: &[u8], offset: usize) -> u64 {
|
||||
u64::from_le_bytes([
|
||||
bytes[offset],
|
||||
bytes[offset + 1],
|
||||
bytes[offset + 2],
|
||||
bytes[offset + 3],
|
||||
bytes[offset + 4],
|
||||
bytes[offset + 5],
|
||||
bytes[offset + 6],
|
||||
bytes[offset + 7],
|
||||
])
|
||||
}
|
||||
|
||||
fn parse_frame_header(bytes: &[u8]) -> Option<FIO_zstdFrameHeader> {
|
||||
if bytes.len() < ZSTD_FRAMEHEADERSIZE_MIN
|
||||
|| u32::from_le_bytes(bytes[0..4].try_into().ok()?) != ZSTD_MAGICNUMBER
|
||||
{
|
||||
return None;
|
||||
}
|
||||
|
||||
let descriptor = bytes[4];
|
||||
if descriptor & 0x08 != 0 {
|
||||
return None;
|
||||
}
|
||||
let singleSegment = descriptor & 0x20 != 0;
|
||||
let dictIdCode = descriptor & 3;
|
||||
let fcsCode = descriptor >> 6;
|
||||
let mut position = 5;
|
||||
let windowSize = if singleSegment {
|
||||
0
|
||||
} else {
|
||||
let windowDescriptor = *bytes.get(position)?;
|
||||
position += 1;
|
||||
let windowLog = usize::from(windowDescriptor >> 3) + 10;
|
||||
let base = 1u64.checked_shl(windowLog as u32)?;
|
||||
base + (base >> 3) * u64::from(windowDescriptor & 7)
|
||||
};
|
||||
let dictId = match dictIdCode {
|
||||
0 => 0,
|
||||
1 => {
|
||||
let value = u32::from(*bytes.get(position)?);
|
||||
position += 1;
|
||||
value
|
||||
}
|
||||
2 => {
|
||||
let value = u32::from(read_le16(bytes, position));
|
||||
position += 2;
|
||||
value
|
||||
}
|
||||
_ => {
|
||||
let value = read_le32(bytes, position);
|
||||
position += 4;
|
||||
value
|
||||
}
|
||||
};
|
||||
let frameContentSize = match fcsCode {
|
||||
0 if singleSegment => {
|
||||
let value = u64::from(*bytes.get(position)?);
|
||||
position += 1;
|
||||
value
|
||||
}
|
||||
0 => ZSTD_CONTENTSIZE_UNKNOWN,
|
||||
1 => {
|
||||
let value = u64::from(read_le16(bytes, position)) + 256;
|
||||
position += 2;
|
||||
value
|
||||
}
|
||||
2 => {
|
||||
let value = u64::from(read_le32(bytes, position));
|
||||
position += 4;
|
||||
value
|
||||
}
|
||||
_ => {
|
||||
let value = read_le64(bytes, position);
|
||||
position += 8;
|
||||
value
|
||||
}
|
||||
};
|
||||
let windowSize = if singleSegment {
|
||||
frameContentSize
|
||||
} else {
|
||||
windowSize
|
||||
};
|
||||
|
||||
Some(FIO_zstdFrameHeader {
|
||||
frameContentSize,
|
||||
windowSize,
|
||||
blockSizeMax: windowSize.min(128 << 10) as c_uint,
|
||||
frameType: 0,
|
||||
headerSize: position as c_uint,
|
||||
dictID: dictId,
|
||||
checksumFlag: c_uint::from(descriptor & 4 != 0),
|
||||
reserved1: 0,
|
||||
reserved2: 0,
|
||||
})
|
||||
}
|
||||
|
||||
pub unsafe fn get_frame_content_size(src: *const c_void, srcSize: usize) -> u64 {
|
||||
if src.is_null() || srcSize == 0 {
|
||||
return ZSTD_CONTENTSIZE_ERROR;
|
||||
}
|
||||
let bytes = unsafe { slice::from_raw_parts(src.cast::<u8>(), srcSize) };
|
||||
parse_frame_header(bytes).map_or(ZSTD_CONTENTSIZE_ERROR, |header| header.frameContentSize)
|
||||
}
|
||||
|
||||
pub unsafe fn get_frame_header(
|
||||
output: *mut FIO_zstdFrameHeader,
|
||||
src: *const c_void,
|
||||
srcSize: usize,
|
||||
) -> usize {
|
||||
if output.is_null() || src.is_null() || srcSize == 0 {
|
||||
return 1;
|
||||
}
|
||||
let bytes = unsafe { slice::from_raw_parts(src.cast::<u8>(), srcSize) };
|
||||
let Some(header) = parse_frame_header(bytes) else {
|
||||
return 1;
|
||||
};
|
||||
unsafe { output.write(header) };
|
||||
0
|
||||
}
|
||||
|
||||
pub unsafe fn frame_header_size(src: *const c_void, srcSize: usize) -> usize {
|
||||
if src.is_null() || srcSize == 0 {
|
||||
return usize::MAX;
|
||||
}
|
||||
let bytes = unsafe { slice::from_raw_parts(src.cast::<u8>(), srcSize) };
|
||||
parse_frame_header(bytes).map_or(usize::MAX, |header| header.headerSize as usize)
|
||||
}
|
||||
|
||||
pub fn is_error(code: usize) -> bool {
|
||||
code == usize::MAX
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "decompression")]
|
||||
#[inline]
|
||||
unsafe fn zstd_get_frame_content_size(src: *const c_void, src_size: usize) -> u64 {
|
||||
#[cfg(test)]
|
||||
{
|
||||
unsafe { analyze_test_api::get_frame_content_size(src, src_size) }
|
||||
}
|
||||
#[cfg(not(test))]
|
||||
{
|
||||
unsafe { ZSTD_getFrameContentSize(src, src_size) }
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "decompression")]
|
||||
#[inline]
|
||||
unsafe fn zstd_get_frame_header(
|
||||
header: *mut FIO_zstdFrameHeader,
|
||||
src: *const c_void,
|
||||
src_size: usize,
|
||||
) -> usize {
|
||||
#[cfg(test)]
|
||||
{
|
||||
unsafe { analyze_test_api::get_frame_header(header, src, src_size) }
|
||||
}
|
||||
#[cfg(not(test))]
|
||||
{
|
||||
unsafe { ZSTD_getFrameHeader(header, src, src_size) }
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "decompression")]
|
||||
#[inline]
|
||||
unsafe fn zstd_frame_header_size(src: *const c_void, src_size: usize) -> usize {
|
||||
#[cfg(test)]
|
||||
{
|
||||
analyze_test_api::frame_header_size(src, src_size)
|
||||
}
|
||||
#[cfg(not(test))]
|
||||
{
|
||||
unsafe { ZSTD_frameHeaderSize(src, src_size) }
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "decompression")]
|
||||
#[inline]
|
||||
unsafe fn zstd_is_error(code: usize) -> bool {
|
||||
#[cfg(test)]
|
||||
{
|
||||
analyze_test_api::is_error(code)
|
||||
}
|
||||
#[cfg(not(test))]
|
||||
{
|
||||
unsafe { ZSTD_isError(code) != 0 }
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "decompression")]
|
||||
#[inline]
|
||||
unsafe fn analyze_diagnostic(diagnostic: c_int, file_position: u64, file_size: u64) {
|
||||
#[cfg(test)]
|
||||
{
|
||||
let _ = (diagnostic, file_position, file_size);
|
||||
}
|
||||
#[cfg(not(test))]
|
||||
{
|
||||
unsafe { FIO_rust_analyzeFrames_display(diagnostic, file_position, file_size) };
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "decompression")]
|
||||
#[inline]
|
||||
unsafe fn seek_relative(file: *mut libc::FILE, mut offset: i64) -> bool {
|
||||
const SEEK_CHUNK: i64 = 1 << 30;
|
||||
|
||||
if offset < 0 {
|
||||
return unsafe { long_seek(file, offset) == 0 };
|
||||
}
|
||||
while offset != 0 {
|
||||
let step = offset.min(SEEK_CHUNK);
|
||||
if unsafe { long_seek(file, step) } != 0 {
|
||||
return false;
|
||||
}
|
||||
offset -= step;
|
||||
}
|
||||
true
|
||||
}
|
||||
|
||||
#[cfg(feature = "decompression")]
|
||||
#[inline]
|
||||
unsafe fn long_seek(file: *mut libc::FILE, offset: i64) -> c_int {
|
||||
#[cfg(unix)]
|
||||
{
|
||||
unsafe { libc::fseeko(file, offset as libc::off_t, libc::SEEK_CUR) }
|
||||
}
|
||||
#[cfg(windows)]
|
||||
{
|
||||
unsafe { _fseeki64(file, offset, libc::SEEK_CUR) }
|
||||
}
|
||||
#[cfg(not(any(unix, windows)))]
|
||||
{
|
||||
unsafe { libc::fseek(file, offset as std::os::raw::c_long, libc::SEEK_CUR) }
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "decompression")]
|
||||
#[inline]
|
||||
unsafe fn long_tell(file: *mut libc::FILE) -> i64 {
|
||||
#[cfg(unix)]
|
||||
{
|
||||
unsafe { libc::ftello(file) as i64 }
|
||||
}
|
||||
#[cfg(windows)]
|
||||
{
|
||||
unsafe { _ftelli64(file) }
|
||||
}
|
||||
#[cfg(not(any(unix, windows)))]
|
||||
{
|
||||
unsafe { libc::ftell(file) as i64 }
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "decompression")]
|
||||
#[no_mangle]
|
||||
pub unsafe extern "C" fn FIO_rust_analyzeFrames(
|
||||
info: *mut FIO_fileInfo_t,
|
||||
srcFile: *mut libc::FILE,
|
||||
) -> c_int {
|
||||
let info = unsafe { &mut *info };
|
||||
|
||||
loop {
|
||||
let mut header_buffer = [0u8; ZSTD_FRAMEHEADERSIZE_MAX];
|
||||
let num_bytes_read = unsafe {
|
||||
libc::fread(
|
||||
header_buffer.as_mut_ptr().cast::<c_void>(),
|
||||
1,
|
||||
header_buffer.len(),
|
||||
srcFile,
|
||||
)
|
||||
};
|
||||
if num_bytes_read < ZSTD_FRAMEHEADERSIZE_MIN {
|
||||
let at_eof = unsafe { libc::feof(srcFile) != 0 };
|
||||
if at_eof
|
||||
&& num_bytes_read == 0
|
||||
&& info.compressedSize > 0
|
||||
&& info.compressedSize != UTIL_FILESIZE_UNKNOWN
|
||||
{
|
||||
let file_position = unsafe { long_tell(srcFile) } as u64;
|
||||
let file_size = info.compressedSize;
|
||||
if file_position != file_size {
|
||||
unsafe {
|
||||
analyze_diagnostic(
|
||||
FIO_RUST_ANALYZE_DIAG_SEEKED_PAST_FILE,
|
||||
file_position,
|
||||
file_size,
|
||||
)
|
||||
};
|
||||
return FIO_INFO_TRUNCATED_INPUT;
|
||||
}
|
||||
break;
|
||||
}
|
||||
if at_eof {
|
||||
unsafe { analyze_diagnostic(FIO_RUST_ANALYZE_DIAG_INCOMPLETE_FRAME, 0, 0) };
|
||||
return FIO_INFO_NOT_ZSTD;
|
||||
}
|
||||
unsafe { analyze_diagnostic(FIO_RUST_ANALYZE_DIAG_RAN_OUT_OF_FRAMES, 0, 0) };
|
||||
return FIO_INFO_FRAME_ERROR;
|
||||
}
|
||||
|
||||
let magic_number = u32::from_le_bytes(header_buffer[0..4].try_into().unwrap());
|
||||
if magic_number == ZSTD_MAGICNUMBER {
|
||||
let frame_content_size = unsafe {
|
||||
zstd_get_frame_content_size(header_buffer.as_ptr().cast::<c_void>(), num_bytes_read)
|
||||
};
|
||||
if frame_content_size == ZSTD_CONTENTSIZE_ERROR
|
||||
|| frame_content_size == ZSTD_CONTENTSIZE_UNKNOWN
|
||||
{
|
||||
info.decompUnavailable = 1;
|
||||
} else {
|
||||
info.decompressedSize = info.decompressedSize.wrapping_add(frame_content_size);
|
||||
}
|
||||
|
||||
let mut header = FIO_zstdFrameHeader::default();
|
||||
let header_result = unsafe {
|
||||
zstd_get_frame_header(
|
||||
&mut header,
|
||||
header_buffer.as_ptr().cast::<c_void>(),
|
||||
num_bytes_read,
|
||||
)
|
||||
};
|
||||
if header_result != 0 {
|
||||
unsafe { analyze_diagnostic(FIO_RUST_ANALYZE_DIAG_DECODE_FRAME_HEADER, 0, 0) };
|
||||
return FIO_INFO_FRAME_ERROR;
|
||||
}
|
||||
if info.dictID != 0 && info.dictID != header.dictID {
|
||||
unsafe { analyze_diagnostic(FIO_RUST_ANALYZE_DIAG_MIXED_DICTIONARY_IDS, 0, 0) };
|
||||
info.dictID = 0;
|
||||
} else {
|
||||
info.dictID = header.dictID;
|
||||
}
|
||||
info.windowSize = header.windowSize;
|
||||
|
||||
let header_size = unsafe {
|
||||
zstd_frame_header_size(header_buffer.as_ptr().cast::<c_void>(), num_bytes_read)
|
||||
};
|
||||
if unsafe { zstd_is_error(header_size) } {
|
||||
unsafe { analyze_diagnostic(FIO_RUST_ANALYZE_DIAG_FRAME_HEADER_SIZE, 0, 0) };
|
||||
return FIO_INFO_FRAME_ERROR;
|
||||
}
|
||||
let header_offset = header_size as i64 - num_bytes_read as i64;
|
||||
if unsafe { !seek_relative(srcFile, header_offset) } {
|
||||
unsafe { analyze_diagnostic(FIO_RUST_ANALYZE_DIAG_MOVE_TO_FRAME_HEADER_END, 0, 0) };
|
||||
return FIO_INFO_FRAME_ERROR;
|
||||
}
|
||||
|
||||
let mut last_block = false;
|
||||
while !last_block {
|
||||
let mut block_header_buffer = [0u8; 3];
|
||||
let block_header_read = unsafe {
|
||||
libc::fread(
|
||||
block_header_buffer.as_mut_ptr().cast::<c_void>(),
|
||||
1,
|
||||
block_header_buffer.len(),
|
||||
srcFile,
|
||||
)
|
||||
};
|
||||
if block_header_read != block_header_buffer.len() {
|
||||
unsafe { analyze_diagnostic(FIO_RUST_ANALYZE_DIAG_BLOCK_HEADER, 0, 0) };
|
||||
return FIO_INFO_FRAME_ERROR;
|
||||
}
|
||||
let block_header = u32::from(block_header_buffer[0])
|
||||
| (u32::from(block_header_buffer[1]) << 8)
|
||||
| (u32::from(block_header_buffer[2]) << 16);
|
||||
let block_type_id = (block_header >> 1) & 3;
|
||||
let is_rle = block_type_id == 1;
|
||||
if block_type_id == 3 {
|
||||
unsafe {
|
||||
analyze_diagnostic(FIO_RUST_ANALYZE_DIAG_UNSUPPORTED_BLOCK_TYPE, 0, 0)
|
||||
};
|
||||
return FIO_INFO_FRAME_ERROR;
|
||||
}
|
||||
last_block = block_header & 1 != 0;
|
||||
let block_size = if is_rle {
|
||||
1
|
||||
} else {
|
||||
i64::from(block_header >> 3)
|
||||
};
|
||||
if unsafe { !seek_relative(srcFile, block_size) } {
|
||||
unsafe { analyze_diagnostic(FIO_RUST_ANALYZE_DIAG_SKIP_BLOCK, 0, 0) };
|
||||
return FIO_INFO_FRAME_ERROR;
|
||||
}
|
||||
}
|
||||
|
||||
if header_buffer[4] & (1 << 2) != 0 {
|
||||
info.usesCheck = 1;
|
||||
let checksum_read = unsafe {
|
||||
libc::fread(info.checksum.as_mut_ptr().cast::<c_void>(), 1, 4, srcFile)
|
||||
};
|
||||
if checksum_read != 4 {
|
||||
unsafe { analyze_diagnostic(FIO_RUST_ANALYZE_DIAG_CHECKSUM, 0, 0) };
|
||||
return FIO_INFO_FRAME_ERROR;
|
||||
}
|
||||
}
|
||||
info.numActualFrames = info.numActualFrames.wrapping_add(1);
|
||||
} else if magic_number & ZSTD_MAGIC_SKIPPABLE_MASK == ZSTD_MAGIC_SKIPPABLE_START {
|
||||
let frame_size = u32::from_le_bytes(header_buffer[4..8].try_into().unwrap());
|
||||
let seek = 8i64 + i64::from(frame_size) - num_bytes_read as i64;
|
||||
if unsafe { !seek_relative(srcFile, seek) } {
|
||||
unsafe { analyze_diagnostic(FIO_RUST_ANALYZE_DIAG_SKIP_FRAME, 0, 0) };
|
||||
return FIO_INFO_FRAME_ERROR;
|
||||
}
|
||||
info.numSkippableFrames = info.numSkippableFrames.wrapping_add(1);
|
||||
} else {
|
||||
return FIO_INFO_NOT_ZSTD;
|
||||
}
|
||||
}
|
||||
FIO_INFO_SUCCESS
|
||||
}
|
||||
|
||||
#[cfg(target_vendor = "apple")]
|
||||
const ZSTD_SPARSE_DEFAULT: c_int = 0;
|
||||
#[cfg(not(target_vendor = "apple"))]
|
||||
@@ -1142,6 +1654,89 @@ mod tests {
|
||||
std::env::temp_dir().join(format!("zstd-fileio-prefs-{}-{name}", std::process::id()))
|
||||
}
|
||||
|
||||
#[cfg(feature = "decompression")]
|
||||
struct TemporaryCFile(*mut libc::FILE);
|
||||
|
||||
#[cfg(feature = "decompression")]
|
||||
impl Drop for TemporaryCFile {
|
||||
fn drop(&mut self) {
|
||||
if !self.0.is_null() {
|
||||
unsafe { libc::fclose(self.0) };
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "decompression")]
|
||||
fn temporary_c_file(bytes: &[u8]) -> TemporaryCFile {
|
||||
let file = unsafe { libc::tmpfile() };
|
||||
assert!(!file.is_null(), "tmpfile failed");
|
||||
let written =
|
||||
unsafe { libc::fwrite(bytes.as_ptr().cast::<c_void>(), 1, bytes.len(), file) };
|
||||
assert_eq!(written, bytes.len());
|
||||
assert_eq!(unsafe { libc::fseek(file, 0, libc::SEEK_SET) }, 0);
|
||||
TemporaryCFile(file)
|
||||
}
|
||||
|
||||
#[cfg(feature = "decompression")]
|
||||
fn analyze_temporary_input(bytes: &[u8], compressed_size: u64) -> (c_int, FIO_fileInfo_t) {
|
||||
let file = temporary_c_file(bytes);
|
||||
let mut info = unsafe { std::mem::zeroed::<FIO_fileInfo_t>() };
|
||||
info.compressedSize = compressed_size;
|
||||
let status = unsafe { FIO_rust_analyzeFrames(&mut info, file.0) };
|
||||
(status, info)
|
||||
}
|
||||
|
||||
#[cfg(feature = "decompression")]
|
||||
fn zstd_test_frame(
|
||||
content_size: Option<u8>,
|
||||
block_payload: &[u8],
|
||||
block_type: u8,
|
||||
checksum: Option<[u8; 4]>,
|
||||
) -> Vec<u8> {
|
||||
let mut frame = vec![0x28, 0xB5, 0x2F, 0xFD];
|
||||
let mut descriptor = if content_size.is_some() { 0x20 } else { 0 };
|
||||
if checksum.is_some() {
|
||||
descriptor |= 1 << 2;
|
||||
}
|
||||
frame.push(descriptor);
|
||||
if let Some(content_size) = content_size {
|
||||
frame.push(content_size);
|
||||
} else {
|
||||
frame.push(0);
|
||||
}
|
||||
|
||||
let block_size = if block_type == 1 {
|
||||
1
|
||||
} else {
|
||||
block_payload.len()
|
||||
};
|
||||
let block_header =
|
||||
(u32::try_from(block_size).unwrap() << 3) | (u32::from(block_type) << 1) | 1;
|
||||
frame.extend_from_slice(&block_header.to_le_bytes()[..3]);
|
||||
frame.extend_from_slice(block_payload);
|
||||
if let Some(checksum) = checksum {
|
||||
frame.extend_from_slice(&checksum);
|
||||
}
|
||||
frame
|
||||
}
|
||||
|
||||
#[cfg(feature = "decompression")]
|
||||
fn zstd_test_frame_with_dict_id(dict_id: u8, content_size: u8) -> Vec<u8> {
|
||||
let mut frame = vec![0x28, 0xB5, 0x2F, 0xFD, 0x21, dict_id, content_size];
|
||||
let block_header = (u32::from(content_size) << 3) | 1;
|
||||
frame.extend_from_slice(&block_header.to_le_bytes()[..3]);
|
||||
frame.resize(frame.len() + usize::from(content_size), b'x');
|
||||
frame
|
||||
}
|
||||
|
||||
#[cfg(feature = "decompression")]
|
||||
fn skippable_test_frame(payload: &[u8]) -> Vec<u8> {
|
||||
let mut frame = vec![0x50, 0x2A, 0x4D, 0x18];
|
||||
frame.extend_from_slice(&u32::try_from(payload.len()).unwrap().to_le_bytes());
|
||||
frame.extend_from_slice(payload);
|
||||
frame
|
||||
}
|
||||
|
||||
fn prefs_with_mem_limit(mem_limit: c_uint) -> FIO_prefs_t {
|
||||
let mut prefs = unsafe { std::mem::zeroed::<FIO_prefs_t>() };
|
||||
prefs.memLimit = mem_limit;
|
||||
@@ -1621,6 +2216,134 @@ mod tests {
|
||||
assert_eq!(size_of::<FIO_fileInfo_t>(), expected_size);
|
||||
}
|
||||
|
||||
#[cfg(feature = "decompression")]
|
||||
#[test]
|
||||
fn analyze_frames_reads_a_normal_frame() {
|
||||
let frame = zstd_test_frame(Some(3), b"abc", 0, None);
|
||||
let (status, info) = analyze_temporary_input(&frame, frame.len() as u64);
|
||||
|
||||
assert_eq!(status, FIO_INFO_SUCCESS);
|
||||
assert_eq!(info.decompressedSize, 3);
|
||||
assert_eq!(info.windowSize, 3);
|
||||
assert_eq!(info.numActualFrames, 1);
|
||||
assert_eq!(info.numSkippableFrames, 0);
|
||||
assert_eq!(info.decompUnavailable, 0);
|
||||
assert_eq!(info.usesCheck, 0);
|
||||
}
|
||||
|
||||
#[cfg(feature = "decompression")]
|
||||
#[test]
|
||||
fn analyze_frames_marks_unknown_content_size_unavailable() {
|
||||
let frame = zstd_test_frame(None, b"abc", 0, None);
|
||||
let (status, info) = analyze_temporary_input(&frame, frame.len() as u64);
|
||||
|
||||
assert_eq!(status, FIO_INFO_SUCCESS);
|
||||
assert_eq!(info.decompressedSize, 0);
|
||||
assert_eq!(info.windowSize, 1024);
|
||||
assert_eq!(info.numActualFrames, 1);
|
||||
assert_eq!(info.decompUnavailable, 1);
|
||||
}
|
||||
|
||||
#[cfg(feature = "decompression")]
|
||||
#[test]
|
||||
fn analyze_frames_accumulates_concatenated_frames() {
|
||||
let first = zstd_test_frame(Some(3), b"abc", 0, None);
|
||||
let second = zstd_test_frame(Some(2), b"de", 0, None);
|
||||
let mut concatenated = first;
|
||||
concatenated.extend_from_slice(&second);
|
||||
let (status, info) = analyze_temporary_input(&concatenated, concatenated.len() as u64);
|
||||
|
||||
assert_eq!(status, FIO_INFO_SUCCESS);
|
||||
assert_eq!(info.decompressedSize, 5);
|
||||
assert_eq!(info.windowSize, 2);
|
||||
assert_eq!(info.numActualFrames, 2);
|
||||
assert_eq!(info.numSkippableFrames, 0);
|
||||
}
|
||||
|
||||
#[cfg(feature = "decompression")]
|
||||
#[test]
|
||||
fn analyze_frames_reports_mixed_dictionary_ids() {
|
||||
let first = zstd_test_frame_with_dict_id(1, 1);
|
||||
let second = zstd_test_frame_with_dict_id(2, 1);
|
||||
let mut concatenated = first;
|
||||
concatenated.extend_from_slice(&second);
|
||||
let (status, info) = analyze_temporary_input(&concatenated, concatenated.len() as u64);
|
||||
|
||||
assert_eq!(status, FIO_INFO_SUCCESS);
|
||||
assert_eq!(info.dictID, 0);
|
||||
assert_eq!(info.numActualFrames, 2);
|
||||
assert_eq!(info.decompressedSize, 2);
|
||||
}
|
||||
|
||||
#[cfg(feature = "decompression")]
|
||||
#[test]
|
||||
fn analyze_frames_skips_skippable_frames() {
|
||||
let mut input = skippable_test_frame(b"metadata");
|
||||
input.extend_from_slice(&zstd_test_frame(Some(3), b"abc", 0, None));
|
||||
let (status, info) = analyze_temporary_input(&input, input.len() as u64);
|
||||
|
||||
assert_eq!(status, FIO_INFO_SUCCESS);
|
||||
assert_eq!(info.numSkippableFrames, 1);
|
||||
assert_eq!(info.numActualFrames, 1);
|
||||
assert_eq!(info.decompressedSize, 3);
|
||||
}
|
||||
|
||||
#[cfg(feature = "decompression")]
|
||||
#[test]
|
||||
fn analyze_frames_records_checksums() {
|
||||
let checksum = [1, 2, 3, 4];
|
||||
let frame = zstd_test_frame(Some(3), b"abc", 0, Some(checksum));
|
||||
let (status, info) = analyze_temporary_input(&frame, frame.len() as u64);
|
||||
|
||||
assert_eq!(status, FIO_INFO_SUCCESS);
|
||||
assert_eq!(info.usesCheck, 1);
|
||||
assert_eq!(info.checksum, checksum);
|
||||
assert_eq!(info.numActualFrames, 1);
|
||||
}
|
||||
|
||||
#[cfg(feature = "decompression")]
|
||||
#[test]
|
||||
fn analyze_frames_skips_rle_block_payload() {
|
||||
let frame = zstd_test_frame(Some(5), b"x", 1, None);
|
||||
let (status, info) = analyze_temporary_input(&frame, frame.len() as u64);
|
||||
|
||||
assert_eq!(status, FIO_INFO_SUCCESS);
|
||||
assert_eq!(info.decompressedSize, 5);
|
||||
assert_eq!(info.numActualFrames, 1);
|
||||
}
|
||||
|
||||
#[cfg(feature = "decompression")]
|
||||
#[test]
|
||||
fn analyze_frames_rejects_invalid_magic() {
|
||||
let input = [0, 1, 2, 3, 4, 5];
|
||||
let (status, info) = analyze_temporary_input(&input, input.len() as u64);
|
||||
|
||||
assert_eq!(status, FIO_INFO_NOT_ZSTD);
|
||||
assert_eq!(info.numActualFrames, 0);
|
||||
assert_eq!(info.numSkippableFrames, 0);
|
||||
}
|
||||
|
||||
#[cfg(feature = "decompression")]
|
||||
#[test]
|
||||
fn analyze_frames_reports_a_truncated_block_as_truncated_input() {
|
||||
let mut input = zstd_test_frame(Some(3), b"abc", 0, None);
|
||||
input.pop();
|
||||
let (status, info) = analyze_temporary_input(&input, input.len() as u64);
|
||||
|
||||
assert_eq!(status, FIO_INFO_TRUNCATED_INPUT);
|
||||
assert_eq!(info.numActualFrames, 1);
|
||||
}
|
||||
|
||||
#[cfg(feature = "decompression")]
|
||||
#[test]
|
||||
fn analyze_frames_rejects_reserved_block_type() {
|
||||
let frame = zstd_test_frame(Some(0), &[], 3, None);
|
||||
let (status, info) = analyze_temporary_input(&frame, frame.len() as u64);
|
||||
|
||||
assert_eq!(status, FIO_INFO_FRAME_ERROR);
|
||||
assert_eq!(info.numActualFrames, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lz4_block_size_shim_preserves_the_block_id_formula() {
|
||||
assert_eq!(FIO_rust_LZ4_GetBlockSize_FromBlockId(0), 1 << 8);
|
||||
|
||||
Reference in New Issue
Block a user