ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(rust): port frame serialization

Browse Source 
Continue the incremental zstd_compress.c migration with its frame
serialization leaves: ZSTD_writeFrameHeader(), the public
ZSTD_writeSkippableFrame() ABI, and ZSTD_writeLastEmptyBlock() now live
in rust/src/zstd_compress_frame.rs.

ZSTD_writeFrameHeader() reads five fields out of ZSTD_CCtx_params, whose
layout is private to zstd_compress.c and sensitive to build
configuration.  Rather than mirror that structure in Rust, the C
function keeps its original static signature and forwards the scalar
fields to ZSTD_rust_writeFrameHeader(), so no parameter-structure layout
crosses the language boundary.  The other two functions take only
pointer/size arguments and are exported directly, replacing their C
bodies outright.

Behavior differences are limited to hardening in release builds: the
Rust leaf clamps a window-size shift that C would leave undefined for a
malformed windowLog, and ZSTD_writeSkippableFrame() rejects a srcSize +
header overflow instead of comparing against a wrapped sum.  Both paths
are unreachable through validated callers, so compressed output is
byte-identical.

Remaining zstd_compress.c work: parameter selection and validation,
context lifecycle, block dispatch, dictionary loading, and the streaming
state machine.

Test plan:
- cd rust && cargo fmt --check && cargo clippy --all-targets -- -D
  warnings && cargo test --all-targets (118 tests, includes new frame
  header/skippable/last-block reference vectors)
- make -C tests fuzzer && ./tests/fuzzer -i1 --no-big-tests
- make -C tests test-rust-lib-smoke
This commit is contained in:
ddidderr
2026-07-11 09:16:06 +02:00
parent fe7e24c770
commit 959e485201
4 changed files with 441 additions and 79 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/compress/zstd_compress.c
+15 -79
View File
@@ -30,6 +30,14 @@
#include "zstd_compress_superblock.h"
#include "../common/bits.h" /* ZSTD_highbit32, ZSTD_rotateRight_U64 */
/* Frame serialization lives in Rust. Keep its interface scalar so the
* large, configuration-sensitive CCtx parameter structure stays in C. */
size_t ZSTD_rust_writeFrameHeader(void* dst, size_t dstCapacity,
int noDictIDFlag, int checksumFlag,
int contentSizeFlag, int format,
U32 windowLog, U64 pledgedSrcSize,
U32 dictID);
/* ***************************************************************
* Tuning parameters
*****************************************************************/
@@ -4668,85 +4676,13 @@ static size_t ZSTD_writeFrameHeader(void* dst, size_t dstCapacity,
const ZSTD_CCtx_params* params,
U64 pledgedSrcSize, U32 dictID)
{
BYTE* const op = (BYTE*)dst;
U32 const dictIDSizeCodeLength = (dictID>0) + (dictID>=256) + (dictID>=65536); /* 0-3 */
U32 const dictIDSizeCode = params->fParams.noDictIDFlag ? 0 : dictIDSizeCodeLength; /* 0-3 */
U32 const checksumFlag = params->fParams.checksumFlag>0;
U32 const windowSize = (U32)1 << params->cParams.windowLog;
U32 const singleSegment = params->fParams.contentSizeFlag && (windowSize >= pledgedSrcSize);
BYTE const windowLogByte = (BYTE)((params->cParams.windowLog - ZSTD_WINDOWLOG_ABSOLUTEMIN) << 3);
U32 const fcsCode = params->fParams.contentSizeFlag ?
(pledgedSrcSize>=256) + (pledgedSrcSize>=65536+256) + (pledgedSrcSize>=0xFFFFFFFFU) : 0; /* 0-3 */
BYTE const frameHeaderDescriptionByte = (BYTE)(dictIDSizeCode + (checksumFlag<<2) + (singleSegment<<5) + (fcsCode<<6) );
size_t pos=0;
assert(!(params->fParams.contentSizeFlag && pledgedSrcSize == ZSTD_CONTENTSIZE_UNKNOWN));
RETURN_ERROR_IF(dstCapacity < ZSTD_FRAMEHEADERSIZE_MAX, dstSize_tooSmall,
"dst buf is too small to fit worst-case frame header size.");
DEBUGLOG(4, "ZSTD_writeFrameHeader : dictIDFlag : %u ; dictID : %u ; dictIDSizeCode : %u",
!params->fParams.noDictIDFlag, (unsigned)dictID, (unsigned)dictIDSizeCode);
if (params->format == ZSTD_f_zstd1) {
MEM_writeLE32(dst, ZSTD_MAGICNUMBER);
pos = 4;
}
op[pos++] = frameHeaderDescriptionByte;
if (!singleSegment) op[pos++] = windowLogByte;
switch(dictIDSizeCode)
{
default:
assert(0); /* impossible */
ZSTD_FALLTHROUGH;
case 0 : break;
case 1 : op[pos] = (BYTE)(dictID); pos++; break;
case 2 : MEM_writeLE16(op+pos, (U16)dictID); pos+=2; break;
case 3 : MEM_writeLE32(op+pos, dictID); pos+=4; break;
}
switch(fcsCode)
{
default:
assert(0); /* impossible */
ZSTD_FALLTHROUGH;
case 0 : if (singleSegment) op[pos++] = (BYTE)(pledgedSrcSize); break;
case 1 : MEM_writeLE16(op+pos, (U16)(pledgedSrcSize-256)); pos+=2; break;
case 2 : MEM_writeLE32(op+pos, (U32)(pledgedSrcSize)); pos+=4; break;
case 3 : MEM_writeLE64(op+pos, (U64)(pledgedSrcSize)); pos+=8; break;
}
return pos;
}
/* ZSTD_writeSkippableFrame_advanced() :
* Writes out a skippable frame with the specified magic number variant (16 are supported),
* from ZSTD_MAGIC_SKIPPABLE_START to ZSTD_MAGIC_SKIPPABLE_START+15, and the desired source data.
*
* Returns the total number of bytes written, or a ZSTD error code.
*/
size_t ZSTD_writeSkippableFrame(void* dst, size_t dstCapacity,
const void* src, size_t srcSize, unsigned magicVariant) {
BYTE* op = (BYTE*)dst;
RETURN_ERROR_IF(dstCapacity < srcSize + ZSTD_SKIPPABLEHEADERSIZE /* Skippable frame overhead */,
dstSize_tooSmall, "Not enough room for skippable frame");
RETURN_ERROR_IF(srcSize > (unsigned)0xFFFFFFFF, srcSize_wrong, "Src size too large for skippable frame");
RETURN_ERROR_IF(magicVariant > 15, parameter_outOfBound, "Skippable frame magic number variant not supported");
MEM_writeLE32(op, (U32)(ZSTD_MAGIC_SKIPPABLE_START + magicVariant));
MEM_writeLE32(op+4, (U32)srcSize);
ZSTD_memcpy(op+8, src, srcSize);
return srcSize + ZSTD_SKIPPABLEHEADERSIZE;
}
/* ZSTD_writeLastEmptyBlock() :
* output an empty Block with end-of-frame mark to complete a frame
* @return : size of data written into `dst` (== ZSTD_blockHeaderSize (defined in zstd_internal.h))
* or an error code if `dstCapacity` is too small (<ZSTD_blockHeaderSize)
*/
size_t ZSTD_writeLastEmptyBlock(void* dst, size_t dstCapacity)
{
RETURN_ERROR_IF(dstCapacity < ZSTD_blockHeaderSize, dstSize_tooSmall,
"dst buf is too small to write frame trailer empty block.");
{ U32 const cBlockHeader24 = 1 /*lastBlock*/ + (((U32)bt_raw)<<1); /* 0 size */
MEM_writeLE24(dst, cBlockHeader24);
return ZSTD_blockHeaderSize;
}
return ZSTD_rust_writeFrameHeader(dst, dstCapacity,
params->fParams.noDictIDFlag,
params->fParams.checksumFlag,
params->fParams.contentSizeFlag,
(int)params->format,
params->cParams.windowLog,
pledgedSrcSize, dictID);
}
void ZSTD_referenceExternalSequences(ZSTD_CCtx* cctx, rawSeq* seq, size_t nbSeq)