diff --git a/programs/fileio.c b/programs/fileio.c index df7babb01..33caa474c 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -2870,6 +2870,26 @@ typedef enum { info_truncated_input=4 } InfoError; +enum { + FIO_RUST_ANALYZE_DIAG_SEEKED_PAST_FILE = 0, + FIO_RUST_ANALYZE_DIAG_INCOMPLETE_FRAME = 1, + FIO_RUST_ANALYZE_DIAG_RAN_OUT_OF_FRAMES = 2, + FIO_RUST_ANALYZE_DIAG_DECODE_FRAME_HEADER = 3, + FIO_RUST_ANALYZE_DIAG_FRAME_HEADER_SIZE = 4, + FIO_RUST_ANALYZE_DIAG_MOVE_TO_FRAME_HEADER_END = 5, + FIO_RUST_ANALYZE_DIAG_BLOCK_HEADER = 6, + FIO_RUST_ANALYZE_DIAG_UNSUPPORTED_BLOCK_TYPE = 7, + FIO_RUST_ANALYZE_DIAG_SKIP_BLOCK = 8, + FIO_RUST_ANALYZE_DIAG_CHECKSUM = 9, + FIO_RUST_ANALYZE_DIAG_SKIP_FRAME = 10, + FIO_RUST_ANALYZE_DIAG_MIXED_DICTIONARY_IDS = 11 +}; + +int FIO_rust_analyzeFrames(fileInfo_t* info, FILE* srcFile); +void FIO_rust_analyzeFrames_display(int diagnostic, + unsigned long long filePosition, + unsigned long long fileSize); + #define ERROR_IF(c,n,...) { \ if (c) { \ DISPLAYLEVEL(1, __VA_ARGS__); \ @@ -2878,100 +2898,61 @@ typedef enum { } \ } +void +FIO_rust_analyzeFrames_display(int diagnostic, + unsigned long long filePosition, + unsigned long long fileSize) +{ + switch (diagnostic) { + case FIO_RUST_ANALYZE_DIAG_SEEKED_PAST_FILE: + DISPLAYLEVEL(1, + "Error: seeked to position %llu, which is beyond file size of %llu\n", + filePosition, fileSize); + break; + case FIO_RUST_ANALYZE_DIAG_INCOMPLETE_FRAME: + DISPLAYLEVEL(1, "Error: reached end of file with incomplete frame"); + break; + case FIO_RUST_ANALYZE_DIAG_RAN_OUT_OF_FRAMES: + DISPLAYLEVEL(1, "Error: did not reach end of file but ran out of frames"); + break; + case FIO_RUST_ANALYZE_DIAG_DECODE_FRAME_HEADER: + DISPLAYLEVEL(1, "Error: could not decode frame header"); + break; + case FIO_RUST_ANALYZE_DIAG_FRAME_HEADER_SIZE: + DISPLAYLEVEL(1, "Error: could not determine frame header size"); + break; + case FIO_RUST_ANALYZE_DIAG_MOVE_TO_FRAME_HEADER_END: + DISPLAYLEVEL(1, "Error: could not move to end of frame header"); + break; + case FIO_RUST_ANALYZE_DIAG_BLOCK_HEADER: + DISPLAYLEVEL(1, "Error while reading block header"); + break; + case FIO_RUST_ANALYZE_DIAG_UNSUPPORTED_BLOCK_TYPE: + DISPLAYLEVEL(1, "Error: unsupported block type"); + break; + case FIO_RUST_ANALYZE_DIAG_SKIP_BLOCK: + DISPLAYLEVEL(1, "Error: could not skip to end of block"); + break; + case FIO_RUST_ANALYZE_DIAG_CHECKSUM: + DISPLAYLEVEL(1, "Error: could not read checksum"); + break; + case FIO_RUST_ANALYZE_DIAG_SKIP_FRAME: + DISPLAYLEVEL(1, "Error: could not find end of skippable frame"); + break; + case FIO_RUST_ANALYZE_DIAG_MIXED_DICTIONARY_IDS: + DISPLAY("WARNING: File contains multiple frames with different dictionary IDs. Showing dictID 0 instead"); + return; + default: + assert(0); + return; + } + DISPLAYLEVEL(1, " \n"); +} + static InfoError FIO_analyzeFrames(fileInfo_t* info, FILE* const srcFile) { - /* begin analyzing frame */ - for ( ; ; ) { - BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX]; - size_t const numBytesRead = fread(headerBuffer, 1, sizeof(headerBuffer), srcFile); - if (numBytesRead < ZSTD_FRAMEHEADERSIZE_MIN(ZSTD_f_zstd1)) { - if ( feof(srcFile) - && (numBytesRead == 0) - && (info->compressedSize > 0) - && (info->compressedSize != UTIL_FILESIZE_UNKNOWN) ) { - unsigned long long file_position = (unsigned long long) LONG_TELL(srcFile); - unsigned long long file_size = (unsigned long long) info->compressedSize; - ERROR_IF(file_position != file_size, info_truncated_input, - "Error: seeked to position %llu, which is beyond file size of %llu\n", - file_position, - file_size); - break; /* correct end of file => success */ - } - ERROR_IF(feof(srcFile), info_not_zstd, "Error: reached end of file with incomplete frame"); - ERROR_IF(1, info_frame_error, "Error: did not reach end of file but ran out of frames"); - } - { U32 const magicNumber = MEM_readLE32(headerBuffer); - /* Zstandard frame */ - if (magicNumber == ZSTD_MAGICNUMBER) { - ZSTD_FrameHeader header; - U64 const frameContentSize = ZSTD_getFrameContentSize(headerBuffer, numBytesRead); - if ( frameContentSize == ZSTD_CONTENTSIZE_ERROR - || frameContentSize == ZSTD_CONTENTSIZE_UNKNOWN ) { - info->decompUnavailable = 1; - } else { - info->decompressedSize += frameContentSize; - } - ERROR_IF(ZSTD_getFrameHeader(&header, headerBuffer, numBytesRead) != 0, - info_frame_error, "Error: could not decode frame header"); - if (info->dictID != 0 && info->dictID != header.dictID) { - DISPLAY("WARNING: File contains multiple frames with different dictionary IDs. Showing dictID 0 instead"); - info->dictID = 0; - } else { - info->dictID = header.dictID; - } - info->windowSize = header.windowSize; - /* move to the end of the frame header */ - { size_t const headerSize = ZSTD_frameHeaderSize(headerBuffer, numBytesRead); - ERROR_IF(ZSTD_isError(headerSize), info_frame_error, "Error: could not determine frame header size"); - ERROR_IF(fseek(srcFile, ((long)headerSize)-((long)numBytesRead), SEEK_CUR) != 0, - info_frame_error, "Error: could not move to end of frame header"); - } - - /* skip all blocks in the frame */ - { int lastBlock = 0; - do { - BYTE blockHeaderBuffer[3]; - ERROR_IF(fread(blockHeaderBuffer, 1, 3, srcFile) != 3, - info_frame_error, "Error while reading block header"); - { U32 const blockHeader = MEM_readLE24(blockHeaderBuffer); - U32 const blockTypeID = (blockHeader >> 1) & 3; - U32 const isRLE = (blockTypeID == 1); - U32 const isWrongBlock = (blockTypeID == 3); - long const blockSize = isRLE ? 1 : (long)(blockHeader >> 3); - ERROR_IF(isWrongBlock, info_frame_error, "Error: unsupported block type"); - lastBlock = blockHeader & 1; - ERROR_IF(fseek(srcFile, blockSize, SEEK_CUR) != 0, - info_frame_error, "Error: could not skip to end of block"); - } - } while (lastBlock != 1); - } - - /* check if checksum is used */ - { BYTE const frameHeaderDescriptor = headerBuffer[4]; - int const contentChecksumFlag = (frameHeaderDescriptor & (1 << 2)) >> 2; - if (contentChecksumFlag) { - info->usesCheck = 1; - ERROR_IF(fread(info->checksum, 1, 4, srcFile) != 4, - info_frame_error, "Error: could not read checksum"); - } } - info->numActualFrames++; - } - /* Skippable frame */ - else if ((magicNumber & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) { - U32 const frameSize = MEM_readLE32(headerBuffer + 4); - long const seek = (long)(8 + frameSize - numBytesRead); - ERROR_IF(LONG_SEEK(srcFile, seek, SEEK_CUR) != 0, - info_frame_error, "Error: could not find end of skippable frame"); - info->numSkippableFrames++; - } - /* unknown content */ - else { - return info_not_zstd; - } - } /* magic number analysis */ - } /* end analyzing frames */ - return info_success; + return (InfoError)FIO_rust_analyzeFrames(info, srcFile); } diff --git a/rust/src/fileio_prefs.rs b/rust/src/fileio_prefs.rs index 3862b83d8..f4fa7c732 100644 --- a/rust/src/fileio_prefs.rs +++ b/rust/src/fileio_prefs.rs @@ -86,6 +86,518 @@ pub struct FIO_fileInfo_t { dictID: c_uint, } +#[cfg(feature = "decompression")] +const FIO_INFO_SUCCESS: c_int = 0; +#[cfg(feature = "decompression")] +const FIO_INFO_FRAME_ERROR: c_int = 1; +#[cfg(feature = "decompression")] +const FIO_INFO_NOT_ZSTD: c_int = 2; +#[cfg(feature = "decompression")] +const FIO_INFO_TRUNCATED_INPUT: c_int = 4; +#[cfg(feature = "decompression")] +const ZSTD_MAGICNUMBER: u32 = 0xFD2F_B528; +#[cfg(feature = "decompression")] +const ZSTD_MAGIC_SKIPPABLE_START: u32 = 0x184D_2A50; +#[cfg(feature = "decompression")] +const ZSTD_MAGIC_SKIPPABLE_MASK: u32 = 0xFFFF_FFF0; +#[cfg(feature = "decompression")] +const ZSTD_CONTENTSIZE_ERROR: u64 = u64::MAX - 1; +#[cfg(feature = "decompression")] +const ZSTD_CONTENTSIZE_UNKNOWN: u64 = u64::MAX; +#[cfg(feature = "decompression")] +const ZSTD_FRAMEHEADERSIZE_MAX: usize = 18; +#[cfg(feature = "decompression")] +const ZSTD_FRAMEHEADERSIZE_MIN: usize = 6; + +#[cfg(feature = "decompression")] +const FIO_RUST_ANALYZE_DIAG_SEEKED_PAST_FILE: c_int = 0; +#[cfg(feature = "decompression")] +const FIO_RUST_ANALYZE_DIAG_INCOMPLETE_FRAME: c_int = 1; +#[cfg(feature = "decompression")] +const FIO_RUST_ANALYZE_DIAG_RAN_OUT_OF_FRAMES: c_int = 2; +#[cfg(feature = "decompression")] +const FIO_RUST_ANALYZE_DIAG_DECODE_FRAME_HEADER: c_int = 3; +#[cfg(feature = "decompression")] +const FIO_RUST_ANALYZE_DIAG_FRAME_HEADER_SIZE: c_int = 4; +#[cfg(feature = "decompression")] +const FIO_RUST_ANALYZE_DIAG_MOVE_TO_FRAME_HEADER_END: c_int = 5; +#[cfg(feature = "decompression")] +const FIO_RUST_ANALYZE_DIAG_BLOCK_HEADER: c_int = 6; +#[cfg(feature = "decompression")] +const FIO_RUST_ANALYZE_DIAG_UNSUPPORTED_BLOCK_TYPE: c_int = 7; +#[cfg(feature = "decompression")] +const FIO_RUST_ANALYZE_DIAG_SKIP_BLOCK: c_int = 8; +#[cfg(feature = "decompression")] +const FIO_RUST_ANALYZE_DIAG_CHECKSUM: c_int = 9; +#[cfg(feature = "decompression")] +const FIO_RUST_ANALYZE_DIAG_SKIP_FRAME: c_int = 10; +#[cfg(feature = "decompression")] +const FIO_RUST_ANALYZE_DIAG_MIXED_DICTIONARY_IDS: c_int = 11; + +#[cfg(feature = "decompression")] +#[repr(C)] +#[derive(Clone, Copy, Default)] +struct FIO_zstdFrameHeader { + frameContentSize: u64, + windowSize: u64, + blockSizeMax: c_uint, + frameType: c_int, + headerSize: c_uint, + dictID: c_uint, + checksumFlag: c_uint, + reserved1: c_uint, + reserved2: c_uint, +} + +#[cfg(all(feature = "decompression", not(test)))] +unsafe extern "C" { + fn ZSTD_getFrameContentSize(src: *const c_void, srcSize: usize) -> u64; + fn ZSTD_getFrameHeader( + zfhPtr: *mut FIO_zstdFrameHeader, + src: *const c_void, + srcSize: usize, + ) -> usize; + fn ZSTD_frameHeaderSize(src: *const c_void, srcSize: usize) -> usize; + fn ZSTD_isError(code: usize) -> c_uint; + fn FIO_rust_analyzeFrames_display(diagnostic: c_int, filePosition: u64, fileSize: u64); +} + +#[cfg(all(feature = "decompression", windows))] +unsafe extern "C" { + fn _fseeki64(file: *mut libc::FILE, offset: i64, whence: c_int) -> c_int; + fn _ftelli64(file: *mut libc::FILE) -> i64; +} + +#[cfg(all(test, feature = "decompression"))] +mod analyze_test_api { + use super::*; + use std::slice; + + fn read_le16(bytes: &[u8], offset: usize) -> u16 { + u16::from_le_bytes([bytes[offset], bytes[offset + 1]]) + } + + fn read_le32(bytes: &[u8], offset: usize) -> u32 { + u32::from_le_bytes([ + bytes[offset], + bytes[offset + 1], + bytes[offset + 2], + bytes[offset + 3], + ]) + } + + fn read_le64(bytes: &[u8], offset: usize) -> u64 { + u64::from_le_bytes([ + bytes[offset], + bytes[offset + 1], + bytes[offset + 2], + bytes[offset + 3], + bytes[offset + 4], + bytes[offset + 5], + bytes[offset + 6], + bytes[offset + 7], + ]) + } + + fn parse_frame_header(bytes: &[u8]) -> Option { + if bytes.len() < ZSTD_FRAMEHEADERSIZE_MIN + || u32::from_le_bytes(bytes[0..4].try_into().ok()?) != ZSTD_MAGICNUMBER + { + return None; + } + + let descriptor = bytes[4]; + if descriptor & 0x08 != 0 { + return None; + } + let singleSegment = descriptor & 0x20 != 0; + let dictIdCode = descriptor & 3; + let fcsCode = descriptor >> 6; + let mut position = 5; + let windowSize = if singleSegment { + 0 + } else { + let windowDescriptor = *bytes.get(position)?; + position += 1; + let windowLog = usize::from(windowDescriptor >> 3) + 10; + let base = 1u64.checked_shl(windowLog as u32)?; + base + (base >> 3) * u64::from(windowDescriptor & 7) + }; + let dictId = match dictIdCode { + 0 => 0, + 1 => { + let value = u32::from(*bytes.get(position)?); + position += 1; + value + } + 2 => { + let value = u32::from(read_le16(bytes, position)); + position += 2; + value + } + _ => { + let value = read_le32(bytes, position); + position += 4; + value + } + }; + let frameContentSize = match fcsCode { + 0 if singleSegment => { + let value = u64::from(*bytes.get(position)?); + position += 1; + value + } + 0 => ZSTD_CONTENTSIZE_UNKNOWN, + 1 => { + let value = u64::from(read_le16(bytes, position)) + 256; + position += 2; + value + } + 2 => { + let value = u64::from(read_le32(bytes, position)); + position += 4; + value + } + _ => { + let value = read_le64(bytes, position); + position += 8; + value + } + }; + let windowSize = if singleSegment { + frameContentSize + } else { + windowSize + }; + + Some(FIO_zstdFrameHeader { + frameContentSize, + windowSize, + blockSizeMax: windowSize.min(128 << 10) as c_uint, + frameType: 0, + headerSize: position as c_uint, + dictID: dictId, + checksumFlag: c_uint::from(descriptor & 4 != 0), + reserved1: 0, + reserved2: 0, + }) + } + + pub unsafe fn get_frame_content_size(src: *const c_void, srcSize: usize) -> u64 { + if src.is_null() || srcSize == 0 { + return ZSTD_CONTENTSIZE_ERROR; + } + let bytes = unsafe { slice::from_raw_parts(src.cast::(), srcSize) }; + parse_frame_header(bytes).map_or(ZSTD_CONTENTSIZE_ERROR, |header| header.frameContentSize) + } + + pub unsafe fn get_frame_header( + output: *mut FIO_zstdFrameHeader, + src: *const c_void, + srcSize: usize, + ) -> usize { + if output.is_null() || src.is_null() || srcSize == 0 { + return 1; + } + let bytes = unsafe { slice::from_raw_parts(src.cast::(), srcSize) }; + let Some(header) = parse_frame_header(bytes) else { + return 1; + }; + unsafe { output.write(header) }; + 0 + } + + pub unsafe fn frame_header_size(src: *const c_void, srcSize: usize) -> usize { + if src.is_null() || srcSize == 0 { + return usize::MAX; + } + let bytes = unsafe { slice::from_raw_parts(src.cast::(), srcSize) }; + parse_frame_header(bytes).map_or(usize::MAX, |header| header.headerSize as usize) + } + + pub fn is_error(code: usize) -> bool { + code == usize::MAX + } +} + +#[cfg(feature = "decompression")] +#[inline] +unsafe fn zstd_get_frame_content_size(src: *const c_void, src_size: usize) -> u64 { + #[cfg(test)] + { + unsafe { analyze_test_api::get_frame_content_size(src, src_size) } + } + #[cfg(not(test))] + { + unsafe { ZSTD_getFrameContentSize(src, src_size) } + } +} + +#[cfg(feature = "decompression")] +#[inline] +unsafe fn zstd_get_frame_header( + header: *mut FIO_zstdFrameHeader, + src: *const c_void, + src_size: usize, +) -> usize { + #[cfg(test)] + { + unsafe { analyze_test_api::get_frame_header(header, src, src_size) } + } + #[cfg(not(test))] + { + unsafe { ZSTD_getFrameHeader(header, src, src_size) } + } +} + +#[cfg(feature = "decompression")] +#[inline] +unsafe fn zstd_frame_header_size(src: *const c_void, src_size: usize) -> usize { + #[cfg(test)] + { + analyze_test_api::frame_header_size(src, src_size) + } + #[cfg(not(test))] + { + unsafe { ZSTD_frameHeaderSize(src, src_size) } + } +} + +#[cfg(feature = "decompression")] +#[inline] +unsafe fn zstd_is_error(code: usize) -> bool { + #[cfg(test)] + { + analyze_test_api::is_error(code) + } + #[cfg(not(test))] + { + unsafe { ZSTD_isError(code) != 0 } + } +} + +#[cfg(feature = "decompression")] +#[inline] +unsafe fn analyze_diagnostic(diagnostic: c_int, file_position: u64, file_size: u64) { + #[cfg(test)] + { + let _ = (diagnostic, file_position, file_size); + } + #[cfg(not(test))] + { + unsafe { FIO_rust_analyzeFrames_display(diagnostic, file_position, file_size) }; + } +} + +#[cfg(feature = "decompression")] +#[inline] +unsafe fn seek_relative(file: *mut libc::FILE, mut offset: i64) -> bool { + const SEEK_CHUNK: i64 = 1 << 30; + + if offset < 0 { + return unsafe { long_seek(file, offset) == 0 }; + } + while offset != 0 { + let step = offset.min(SEEK_CHUNK); + if unsafe { long_seek(file, step) } != 0 { + return false; + } + offset -= step; + } + true +} + +#[cfg(feature = "decompression")] +#[inline] +unsafe fn long_seek(file: *mut libc::FILE, offset: i64) -> c_int { + #[cfg(unix)] + { + unsafe { libc::fseeko(file, offset as libc::off_t, libc::SEEK_CUR) } + } + #[cfg(windows)] + { + unsafe { _fseeki64(file, offset, libc::SEEK_CUR) } + } + #[cfg(not(any(unix, windows)))] + { + unsafe { libc::fseek(file, offset as std::os::raw::c_long, libc::SEEK_CUR) } + } +} + +#[cfg(feature = "decompression")] +#[inline] +unsafe fn long_tell(file: *mut libc::FILE) -> i64 { + #[cfg(unix)] + { + unsafe { libc::ftello(file) as i64 } + } + #[cfg(windows)] + { + unsafe { _ftelli64(file) } + } + #[cfg(not(any(unix, windows)))] + { + unsafe { libc::ftell(file) as i64 } + } +} + +#[cfg(feature = "decompression")] +#[no_mangle] +pub unsafe extern "C" fn FIO_rust_analyzeFrames( + info: *mut FIO_fileInfo_t, + srcFile: *mut libc::FILE, +) -> c_int { + let info = unsafe { &mut *info }; + + loop { + let mut header_buffer = [0u8; ZSTD_FRAMEHEADERSIZE_MAX]; + let num_bytes_read = unsafe { + libc::fread( + header_buffer.as_mut_ptr().cast::(), + 1, + header_buffer.len(), + srcFile, + ) + }; + if num_bytes_read < ZSTD_FRAMEHEADERSIZE_MIN { + let at_eof = unsafe { libc::feof(srcFile) != 0 }; + if at_eof + && num_bytes_read == 0 + && info.compressedSize > 0 + && info.compressedSize != UTIL_FILESIZE_UNKNOWN + { + let file_position = unsafe { long_tell(srcFile) } as u64; + let file_size = info.compressedSize; + if file_position != file_size { + unsafe { + analyze_diagnostic( + FIO_RUST_ANALYZE_DIAG_SEEKED_PAST_FILE, + file_position, + file_size, + ) + }; + return FIO_INFO_TRUNCATED_INPUT; + } + break; + } + if at_eof { + unsafe { analyze_diagnostic(FIO_RUST_ANALYZE_DIAG_INCOMPLETE_FRAME, 0, 0) }; + return FIO_INFO_NOT_ZSTD; + } + unsafe { analyze_diagnostic(FIO_RUST_ANALYZE_DIAG_RAN_OUT_OF_FRAMES, 0, 0) }; + return FIO_INFO_FRAME_ERROR; + } + + let magic_number = u32::from_le_bytes(header_buffer[0..4].try_into().unwrap()); + if magic_number == ZSTD_MAGICNUMBER { + let frame_content_size = unsafe { + zstd_get_frame_content_size(header_buffer.as_ptr().cast::(), num_bytes_read) + }; + if frame_content_size == ZSTD_CONTENTSIZE_ERROR + || frame_content_size == ZSTD_CONTENTSIZE_UNKNOWN + { + info.decompUnavailable = 1; + } else { + info.decompressedSize = info.decompressedSize.wrapping_add(frame_content_size); + } + + let mut header = FIO_zstdFrameHeader::default(); + let header_result = unsafe { + zstd_get_frame_header( + &mut header, + header_buffer.as_ptr().cast::(), + num_bytes_read, + ) + }; + if header_result != 0 { + unsafe { analyze_diagnostic(FIO_RUST_ANALYZE_DIAG_DECODE_FRAME_HEADER, 0, 0) }; + return FIO_INFO_FRAME_ERROR; + } + if info.dictID != 0 && info.dictID != header.dictID { + unsafe { analyze_diagnostic(FIO_RUST_ANALYZE_DIAG_MIXED_DICTIONARY_IDS, 0, 0) }; + info.dictID = 0; + } else { + info.dictID = header.dictID; + } + info.windowSize = header.windowSize; + + let header_size = unsafe { + zstd_frame_header_size(header_buffer.as_ptr().cast::(), num_bytes_read) + }; + if unsafe { zstd_is_error(header_size) } { + unsafe { analyze_diagnostic(FIO_RUST_ANALYZE_DIAG_FRAME_HEADER_SIZE, 0, 0) }; + return FIO_INFO_FRAME_ERROR; + } + let header_offset = header_size as i64 - num_bytes_read as i64; + if unsafe { !seek_relative(srcFile, header_offset) } { + unsafe { analyze_diagnostic(FIO_RUST_ANALYZE_DIAG_MOVE_TO_FRAME_HEADER_END, 0, 0) }; + return FIO_INFO_FRAME_ERROR; + } + + let mut last_block = false; + while !last_block { + let mut block_header_buffer = [0u8; 3]; + let block_header_read = unsafe { + libc::fread( + block_header_buffer.as_mut_ptr().cast::(), + 1, + block_header_buffer.len(), + srcFile, + ) + }; + if block_header_read != block_header_buffer.len() { + unsafe { analyze_diagnostic(FIO_RUST_ANALYZE_DIAG_BLOCK_HEADER, 0, 0) }; + return FIO_INFO_FRAME_ERROR; + } + let block_header = u32::from(block_header_buffer[0]) + | (u32::from(block_header_buffer[1]) << 8) + | (u32::from(block_header_buffer[2]) << 16); + let block_type_id = (block_header >> 1) & 3; + let is_rle = block_type_id == 1; + if block_type_id == 3 { + unsafe { + analyze_diagnostic(FIO_RUST_ANALYZE_DIAG_UNSUPPORTED_BLOCK_TYPE, 0, 0) + }; + return FIO_INFO_FRAME_ERROR; + } + last_block = block_header & 1 != 0; + let block_size = if is_rle { + 1 + } else { + i64::from(block_header >> 3) + }; + if unsafe { !seek_relative(srcFile, block_size) } { + unsafe { analyze_diagnostic(FIO_RUST_ANALYZE_DIAG_SKIP_BLOCK, 0, 0) }; + return FIO_INFO_FRAME_ERROR; + } + } + + if header_buffer[4] & (1 << 2) != 0 { + info.usesCheck = 1; + let checksum_read = unsafe { + libc::fread(info.checksum.as_mut_ptr().cast::(), 1, 4, srcFile) + }; + if checksum_read != 4 { + unsafe { analyze_diagnostic(FIO_RUST_ANALYZE_DIAG_CHECKSUM, 0, 0) }; + return FIO_INFO_FRAME_ERROR; + } + } + info.numActualFrames = info.numActualFrames.wrapping_add(1); + } else if magic_number & ZSTD_MAGIC_SKIPPABLE_MASK == ZSTD_MAGIC_SKIPPABLE_START { + let frame_size = u32::from_le_bytes(header_buffer[4..8].try_into().unwrap()); + let seek = 8i64 + i64::from(frame_size) - num_bytes_read as i64; + if unsafe { !seek_relative(srcFile, seek) } { + unsafe { analyze_diagnostic(FIO_RUST_ANALYZE_DIAG_SKIP_FRAME, 0, 0) }; + return FIO_INFO_FRAME_ERROR; + } + info.numSkippableFrames = info.numSkippableFrames.wrapping_add(1); + } else { + return FIO_INFO_NOT_ZSTD; + } + } + FIO_INFO_SUCCESS +} + #[cfg(target_vendor = "apple")] const ZSTD_SPARSE_DEFAULT: c_int = 0; #[cfg(not(target_vendor = "apple"))] @@ -1142,6 +1654,89 @@ mod tests { std::env::temp_dir().join(format!("zstd-fileio-prefs-{}-{name}", std::process::id())) } + #[cfg(feature = "decompression")] + struct TemporaryCFile(*mut libc::FILE); + + #[cfg(feature = "decompression")] + impl Drop for TemporaryCFile { + fn drop(&mut self) { + if !self.0.is_null() { + unsafe { libc::fclose(self.0) }; + } + } + } + + #[cfg(feature = "decompression")] + fn temporary_c_file(bytes: &[u8]) -> TemporaryCFile { + let file = unsafe { libc::tmpfile() }; + assert!(!file.is_null(), "tmpfile failed"); + let written = + unsafe { libc::fwrite(bytes.as_ptr().cast::(), 1, bytes.len(), file) }; + assert_eq!(written, bytes.len()); + assert_eq!(unsafe { libc::fseek(file, 0, libc::SEEK_SET) }, 0); + TemporaryCFile(file) + } + + #[cfg(feature = "decompression")] + fn analyze_temporary_input(bytes: &[u8], compressed_size: u64) -> (c_int, FIO_fileInfo_t) { + let file = temporary_c_file(bytes); + let mut info = unsafe { std::mem::zeroed::() }; + info.compressedSize = compressed_size; + let status = unsafe { FIO_rust_analyzeFrames(&mut info, file.0) }; + (status, info) + } + + #[cfg(feature = "decompression")] + fn zstd_test_frame( + content_size: Option, + block_payload: &[u8], + block_type: u8, + checksum: Option<[u8; 4]>, + ) -> Vec { + let mut frame = vec![0x28, 0xB5, 0x2F, 0xFD]; + let mut descriptor = if content_size.is_some() { 0x20 } else { 0 }; + if checksum.is_some() { + descriptor |= 1 << 2; + } + frame.push(descriptor); + if let Some(content_size) = content_size { + frame.push(content_size); + } else { + frame.push(0); + } + + let block_size = if block_type == 1 { + 1 + } else { + block_payload.len() + }; + let block_header = + (u32::try_from(block_size).unwrap() << 3) | (u32::from(block_type) << 1) | 1; + frame.extend_from_slice(&block_header.to_le_bytes()[..3]); + frame.extend_from_slice(block_payload); + if let Some(checksum) = checksum { + frame.extend_from_slice(&checksum); + } + frame + } + + #[cfg(feature = "decompression")] + fn zstd_test_frame_with_dict_id(dict_id: u8, content_size: u8) -> Vec { + let mut frame = vec![0x28, 0xB5, 0x2F, 0xFD, 0x21, dict_id, content_size]; + let block_header = (u32::from(content_size) << 3) | 1; + frame.extend_from_slice(&block_header.to_le_bytes()[..3]); + frame.resize(frame.len() + usize::from(content_size), b'x'); + frame + } + + #[cfg(feature = "decompression")] + fn skippable_test_frame(payload: &[u8]) -> Vec { + let mut frame = vec![0x50, 0x2A, 0x4D, 0x18]; + frame.extend_from_slice(&u32::try_from(payload.len()).unwrap().to_le_bytes()); + frame.extend_from_slice(payload); + frame + } + fn prefs_with_mem_limit(mem_limit: c_uint) -> FIO_prefs_t { let mut prefs = unsafe { std::mem::zeroed::() }; prefs.memLimit = mem_limit; @@ -1621,6 +2216,134 @@ mod tests { assert_eq!(size_of::(), expected_size); } + #[cfg(feature = "decompression")] + #[test] + fn analyze_frames_reads_a_normal_frame() { + let frame = zstd_test_frame(Some(3), b"abc", 0, None); + let (status, info) = analyze_temporary_input(&frame, frame.len() as u64); + + assert_eq!(status, FIO_INFO_SUCCESS); + assert_eq!(info.decompressedSize, 3); + assert_eq!(info.windowSize, 3); + assert_eq!(info.numActualFrames, 1); + assert_eq!(info.numSkippableFrames, 0); + assert_eq!(info.decompUnavailable, 0); + assert_eq!(info.usesCheck, 0); + } + + #[cfg(feature = "decompression")] + #[test] + fn analyze_frames_marks_unknown_content_size_unavailable() { + let frame = zstd_test_frame(None, b"abc", 0, None); + let (status, info) = analyze_temporary_input(&frame, frame.len() as u64); + + assert_eq!(status, FIO_INFO_SUCCESS); + assert_eq!(info.decompressedSize, 0); + assert_eq!(info.windowSize, 1024); + assert_eq!(info.numActualFrames, 1); + assert_eq!(info.decompUnavailable, 1); + } + + #[cfg(feature = "decompression")] + #[test] + fn analyze_frames_accumulates_concatenated_frames() { + let first = zstd_test_frame(Some(3), b"abc", 0, None); + let second = zstd_test_frame(Some(2), b"de", 0, None); + let mut concatenated = first; + concatenated.extend_from_slice(&second); + let (status, info) = analyze_temporary_input(&concatenated, concatenated.len() as u64); + + assert_eq!(status, FIO_INFO_SUCCESS); + assert_eq!(info.decompressedSize, 5); + assert_eq!(info.windowSize, 2); + assert_eq!(info.numActualFrames, 2); + assert_eq!(info.numSkippableFrames, 0); + } + + #[cfg(feature = "decompression")] + #[test] + fn analyze_frames_reports_mixed_dictionary_ids() { + let first = zstd_test_frame_with_dict_id(1, 1); + let second = zstd_test_frame_with_dict_id(2, 1); + let mut concatenated = first; + concatenated.extend_from_slice(&second); + let (status, info) = analyze_temporary_input(&concatenated, concatenated.len() as u64); + + assert_eq!(status, FIO_INFO_SUCCESS); + assert_eq!(info.dictID, 0); + assert_eq!(info.numActualFrames, 2); + assert_eq!(info.decompressedSize, 2); + } + + #[cfg(feature = "decompression")] + #[test] + fn analyze_frames_skips_skippable_frames() { + let mut input = skippable_test_frame(b"metadata"); + input.extend_from_slice(&zstd_test_frame(Some(3), b"abc", 0, None)); + let (status, info) = analyze_temporary_input(&input, input.len() as u64); + + assert_eq!(status, FIO_INFO_SUCCESS); + assert_eq!(info.numSkippableFrames, 1); + assert_eq!(info.numActualFrames, 1); + assert_eq!(info.decompressedSize, 3); + } + + #[cfg(feature = "decompression")] + #[test] + fn analyze_frames_records_checksums() { + let checksum = [1, 2, 3, 4]; + let frame = zstd_test_frame(Some(3), b"abc", 0, Some(checksum)); + let (status, info) = analyze_temporary_input(&frame, frame.len() as u64); + + assert_eq!(status, FIO_INFO_SUCCESS); + assert_eq!(info.usesCheck, 1); + assert_eq!(info.checksum, checksum); + assert_eq!(info.numActualFrames, 1); + } + + #[cfg(feature = "decompression")] + #[test] + fn analyze_frames_skips_rle_block_payload() { + let frame = zstd_test_frame(Some(5), b"x", 1, None); + let (status, info) = analyze_temporary_input(&frame, frame.len() as u64); + + assert_eq!(status, FIO_INFO_SUCCESS); + assert_eq!(info.decompressedSize, 5); + assert_eq!(info.numActualFrames, 1); + } + + #[cfg(feature = "decompression")] + #[test] + fn analyze_frames_rejects_invalid_magic() { + let input = [0, 1, 2, 3, 4, 5]; + let (status, info) = analyze_temporary_input(&input, input.len() as u64); + + assert_eq!(status, FIO_INFO_NOT_ZSTD); + assert_eq!(info.numActualFrames, 0); + assert_eq!(info.numSkippableFrames, 0); + } + + #[cfg(feature = "decompression")] + #[test] + fn analyze_frames_reports_a_truncated_block_as_truncated_input() { + let mut input = zstd_test_frame(Some(3), b"abc", 0, None); + input.pop(); + let (status, info) = analyze_temporary_input(&input, input.len() as u64); + + assert_eq!(status, FIO_INFO_TRUNCATED_INPUT); + assert_eq!(info.numActualFrames, 1); + } + + #[cfg(feature = "decompression")] + #[test] + fn analyze_frames_rejects_reserved_block_type() { + let frame = zstd_test_frame(Some(0), &[], 3, None); + let (status, info) = analyze_temporary_input(&frame, frame.len() as u64); + + assert_eq!(status, FIO_INFO_FRAME_ERROR); + assert_eq!(info.numActualFrames, 0); + } + #[test] fn lz4_block_size_shim_preserves_the_block_id_formula() { assert_eq!(FIO_rust_LZ4_GetBlockSize_FromBlockId(0), 1 << 8);