ddidderr
daeac5238c
Continue the incremental zstd_compress.c migration with its sequence statistics and seqStore entropy-compression layer. The following now live in rust/src/zstd_compress_stats.rs: - ZSTD_seqToCodes(), exported under its original name because the C dictionary builder (zdict.c) and decodecorpus link against it, - ZSTD_buildSequencesStatistics() and its dummy variant, whose result struct no longer crosses the language boundary, - ZSTD_entropyCompressSeqStore_internal(), _wExtLitBuffer(), and ZSTD_entropyCompressSeqStore(), - ZSTD_buildBlockEntropyStats() with its literals/sequences helpers, - ZSTD_copyBlockSequences() and the ZSTD_updateRep() rules it shares with the superblock writer. Boundary: ZSTD_CCtx and ZSTD_CCtx_params stay private to C. These paths read exactly two parameter fields, so the C shims keep the original static/exported function names and forward the strategy and ZSTD_literalsCompressionIsDisabled() as int scalars alongside the seqStore and entropy-table leaves. The leaf layouts (SeqDef, SeqStore_t, ZSTD_hufCTables_t, ZSTD_fseCTables_t, entropy metadata, SeqCollector, ZSTD_Sequence) are pinned by compile-time asserts in zstd_compress.c and by both-pointer-width layout tests in Rust. ZSTD_buildSeqStore, block dispatch/splitting, and the block-size estimation helpers remain C for a later slice. The superblock module previously round-tripped through the C export of ZSTD_buildBlockEntropyStats and mirrored the entropy leaf structs privately. It now calls the crate-internal builder directly, and the shared struct definitions moved to zstd_compress_stats; consequently ZSTD_rust_compressSuperBlock() takes the two parameter scalars instead of an opaque ZSTD_CCtx_params pointer, extracted by its C shim. Its layout and repcode tests moved with the definitions. One C helper family gets no shim: the static ZSTD_entropyCompressSeqStore_wExtLitBuffer() had a single caller and was folded into the Rust implementation. Byte-identity was verified against the pre-change compressor: COPYING, datagen -g5000000 -s7, and datagen -g300000 -s21 -P90, each at levels 1/3/9/19 and --fast=5, plus a superblock-heavy pass at level 19 with --target-compressed-block-size=1024; all 18 frames are byte-identical, covering the repeat-mode state machine, longOffsets, RLE/raw fallback, and dstSize_tooSmall paths through the block splitter and superblock. Test plan: - cd rust && cargo fmt --check && cargo clippy --all-targets -- -D warnings && cargo test --all-targets (131 tests: new reference vectors for seqToCodes, RLE table headers, empty-seqStore repeat copies, literal-stats type selection, and repcode resolution in copyBlockSequences) - cargo build --release --no-default-features --features compression and --features decompression - make -C tests fuzzer && ./tests/fuzzer -i1 --no-big-tests (covers ZSTD_generateSequences and ZDICT training over the Rust seqToCodes) - make -C tests test-rust-lib-smoke - ./tests/zstreamtest -i1 and ./tests/zstreamtest --newapi -t1 -i1 - ./tests/decodecorpus -t -T1s (decodecorpus links Rust seqToCodes) - /tmp/ref vs /tmp/got frame diff as described above: byte-identical
57 lines
2.5 KiB
C
57 lines
2.5 KiB
C
/*
|
|
* Copyright (c) Meta Platforms, Inc. and affiliates.
|
|
* All rights reserved.
|
|
*
|
|
* This source code is licensed under both the BSD-style license (found in the
|
|
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
|
* in the COPYING file in the root directory of this source tree).
|
|
* You may select, at your option, one of the above-listed licenses.
|
|
*/
|
|
|
|
/* The implementation lives in rust/src/zstd_compress_superblock.rs. Keep the
|
|
* context extraction here: ZSTD_CCtx is intentionally opaque to Rust, while
|
|
* the small sequence and entropy leaf layouts are asserted in both sources. */
|
|
#include "zstd_compress_superblock.h"
|
|
#include "zstd_compress_internal.h"
|
|
|
|
typedef char ZSTD_rust_superblock_seqdef_layout[(sizeof(SeqDef) == 8) ? 1 : -1];
|
|
typedef char ZSTD_rust_superblock_fse_layout[(sizeof(ZSTD_fseCTables_t) == 3552) ? 1 : -1];
|
|
typedef char ZSTD_rust_superblock_entropy_offset[
|
|
(offsetof(ZSTD_compressedBlockState_t, entropy) == 0) ? 1 : -1];
|
|
typedef char ZSTD_rust_superblock_rep_offset[
|
|
(offsetof(ZSTD_compressedBlockState_t, rep) == sizeof(ZSTD_entropyCTables_t)) ? 1 : -1];
|
|
typedef char ZSTD_rust_superblock_seqstore_long_length_pos[
|
|
(offsetof(SeqStore_t, longLengthPos) == 9 * sizeof(size_t) + 4) ? 1 : -1];
|
|
typedef char ZSTD_rust_superblock_seqstore_layout[
|
|
(sizeof(SeqStore_t) == 9 * sizeof(size_t) + 8) ? 1 : -1];
|
|
|
|
size_t ZSTD_rust_compressSuperBlock(
|
|
const SeqStore_t* seqStore,
|
|
const ZSTD_compressedBlockState_t* prevCBlock,
|
|
ZSTD_compressedBlockState_t* nextCBlock,
|
|
int strategy, int disableLiteralCompression,
|
|
void* workspace, size_t wkspSize,
|
|
int bmi2, U32 windowLog, size_t targetCBlockSize,
|
|
void* dst, size_t dstCapacity,
|
|
const void* src, size_t srcSize,
|
|
U32 lastBlock);
|
|
|
|
size_t ZSTD_compressSuperBlock(ZSTD_CCtx* zc,
|
|
void* dst, size_t dstCapacity,
|
|
const void* src, size_t srcSize,
|
|
unsigned lastBlock)
|
|
{
|
|
return ZSTD_rust_compressSuperBlock(
|
|
&zc->seqStore,
|
|
zc->blockState.prevCBlock,
|
|
zc->blockState.nextCBlock,
|
|
(int)zc->appliedParams.cParams.strategy,
|
|
ZSTD_literalsCompressionIsDisabled(&zc->appliedParams),
|
|
zc->tmpWorkspace, zc->tmpWkspSize,
|
|
zc->bmi2, zc->appliedParams.cParams.windowLog,
|
|
zc->appliedParams.targetCBlockSize,
|
|
dst, dstCapacity,
|
|
src, srcSize,
|
|
lastBlock);
|
|
}
|