From 62389c002f672fdd84bcb92a563b23d4cdd393fa Mon Sep 17 00:00:00 2001 From: ddidderr Date: Sun, 19 Jul 2026 08:21:07 +0200 Subject: [PATCH] feat(compress): move sequence-store policy into Rust Move sequence-store construction, small-block handling, repcode setup, last-literal storage, block-body no-compress transitions, target/split post-build policy, and public sequence conversion into Rust projections. Keep matchfinders, LDM, external sequence producers, workspaces, and private compression context state behind narrow C callbacks and preserve the existing C-facing sequence API. Test Plan: - ulimit -v 41943040; make -B -C lib -j1 lib (passed) - ulimit -v 41943040; make -B -C tests -j1 test-zstd (passed) - ulimit -v 41943040; FUZZERTEST=-T5s make -B -C tests -j1 test-fuzzer (284 passed) - ulimit -v 41943040; ZSTREAM_TESTTIME=-T2s make -B -C tests -j1 test-zstream (210 passed) - ulimit -v 41943040; DECODECORPUS_TESTTIME=-T10 make -B -C tests -j1 test-decodecorpus (1608 passed) - ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo test --manifest-path rust/Cargo.toml --no-default-features --features compression,decompression,dict-builder,legacy-v01,legacy-v02,legacy-v03,legacy-v04,legacy-v05,legacy-v06,legacy-v07 --all-targets -- --test-threads=1 (562 passed) --- lib/compress/zstd_compress.c | 523 +++++++++++++++----------------- rust/src/zstd_compress.rs | 558 ++++++++++++++++++++++++++++++++++- 2 files changed, 794 insertions(+), 287 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 05bcf8a6b..8e9bd1e49 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -293,6 +293,11 @@ size_t ZSTD_rust_compressBlockTargetCBlockSize( void* dst, size_t dstCapacity, const void* src, size_t srcSize, int bss, U32 lastBlock); +size_t ZSTD_rust_compressBlockTargetCBlockSizeAfterBuild( + const ZSTD_rust_targetCBlockSizeState* state, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + int bss, U32 lastBlock); typedef char ZSTD_rust_target_cblock_state_layout[ (offsetof(ZSTD_rust_targetCBlockSizeState, seqStore) == 0 && offsetof(ZSTD_rust_targetCBlockSizeState, prevCBlock) == sizeof(void*) @@ -337,6 +342,11 @@ size_t ZSTD_rust_compressBlockSplit( void* dst, size_t dstCapacity, const void* src, size_t blockSize, U32 lastBlock, size_t numSplits); +size_t ZSTD_rust_compressBlockSplitAfterBuild( + const ZSTD_rust_splitBlockState* state, + void* dst, size_t dstCapacity, + const void* src, size_t blockSize, + U32 lastBlock, size_t numSplits, int bss); typedef char ZSTD_rust_split_block_state_layout[ (offsetof(ZSTD_rust_splitBlockState, seqStore) == 0 && offsetof(ZSTD_rust_splitBlockState, partitions) == sizeof(void*) @@ -377,6 +387,11 @@ size_t ZSTD_rust_compressBlockInternal( void* dst, size_t dstCapacity, const void* src, size_t srcSize, U32 frame); +size_t ZSTD_rust_compressBlockInternalAfterBuild( + const ZSTD_rust_blockInternalState* state, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + U32 frame, int bss); typedef char ZSTD_rust_block_internal_state_layout[ (offsetof(ZSTD_rust_blockInternalState, seqStore) == 0 && offsetof(ZSTD_rust_blockInternalState, prevCBlock) == sizeof(void*) @@ -653,8 +668,6 @@ void ZSTD_rust_resetSeqStore(SeqStore_t* ssPtr); size_t ZSTD_rust_fastSequenceLengthSum(const ZSTD_Sequence* seqBuf, size_t seqBufSize); void ZSTD_rust_validateSeqStore(const SeqStore_t* seqStore, U32 minMatch); -size_t ZSTD_rust_convertSequencesNoRepcodes( - SeqDef* dstSeqs, const ZSTD_Sequence* inSeqs, size_t nbSequences); BlockSummary ZSTD_rust_get1BlockSummary(const ZSTD_Sequence* seqs, size_t nbSeqs); size_t ZSTD_rust_postProcessSequenceProducerResult( @@ -673,7 +686,6 @@ size_t ZSTD_rust_deriveBlockSplits( void* workspace, size_t workspaceSize); U32 ZSTD_rust_resolveRepcodeToRawOffset(const U32 rep[ZSTD_REP_NUM], U32 offBase, U32 ll0); -U32 ZSTD_rust_finalizeOffBase(U32 rawOffset, const U32 rep[ZSTD_REP_NUM], U32 ll0); size_t ZSTD_rust_loadCEntropy(ZSTD_compressedBlockState_t* bs, void* workspace, const void* dict, size_t dictSize); size_t ZSTD_rust_transferSequencesWBlockDelim( @@ -689,6 +701,47 @@ size_t ZSTD_rust_optimalBlockSize(const void* src, size_t srcSize, int strategy, S64 savings, void* workspace, size_t workspaceSize); +/* Sequence-store construction is Rust-owned at the orchestration boundary. + * Matchfinder, LDM, and external sequence-producer operations retain access + * to the private CCtx through these narrow C callbacks. */ +typedef void (*ZSTD_rust_buildSeqStoreSkip_f)(void* context, size_t srcSize); +typedef void (*ZSTD_rust_buildSeqStorePrepare_f)(void* context, + const void* src, + size_t srcSize); +typedef size_t (*ZSTD_rust_buildSeqStoreSelect_f)( + void* context, SeqStore_t* seqStore, U32 nextRep[ZSTD_REP_NUM], + const void* src, size_t srcSize, int* seqStoreComplete); +typedef struct { + SeqStore_t* seqStore; + ZSTD_compressedBlockState_t** prevCBlock; + ZSTD_compressedBlockState_t** nextCBlock; + void* callbackContext; + U32 minMatch; + int validateSeqStore; + ZSTD_rust_buildSeqStoreSkip_f skipSmallBlock; + ZSTD_rust_buildSeqStorePrepare_f prepareMatchState; + ZSTD_rust_buildSeqStoreSelect_f selectSequences; +} ZSTD_rust_buildSeqStoreState; +size_t ZSTD_rust_buildSeqStore(const ZSTD_rust_buildSeqStoreState* state, + const void* src, size_t srcSize); +typedef char ZSTD_rust_build_seq_store_state_layout[ + (offsetof(ZSTD_rust_buildSeqStoreState, seqStore) == 0 + && offsetof(ZSTD_rust_buildSeqStoreState, prevCBlock) == sizeof(void*) + && offsetof(ZSTD_rust_buildSeqStoreState, nextCBlock) == 2 * sizeof(void*) + && offsetof(ZSTD_rust_buildSeqStoreState, callbackContext) == 3 * sizeof(void*) + && offsetof(ZSTD_rust_buildSeqStoreState, minMatch) == 4 * sizeof(void*) + && offsetof(ZSTD_rust_buildSeqStoreState, validateSeqStore) + == 4 * sizeof(void*) + sizeof(U32) + && offsetof(ZSTD_rust_buildSeqStoreState, skipSmallBlock) + == 4 * sizeof(void*) + sizeof(U32) + sizeof(int) + && offsetof(ZSTD_rust_buildSeqStoreState, prepareMatchState) + == 5 * sizeof(void*) + sizeof(U32) + sizeof(int) + && offsetof(ZSTD_rust_buildSeqStoreState, selectSequences) + == 6 * sizeof(void*) + sizeof(U32) + sizeof(int) + && sizeof(ZSTD_rust_buildSeqStoreState) + == 7 * sizeof(void*) + sizeof(U32) + sizeof(int)) + ? 1 : -1]; + /* The sequence-compression loop receives only the state it actually reads or * updates. In particular, neither ZSTD_CCtx nor a C function pointer crosses * the Rust ABI. */ @@ -763,6 +816,25 @@ size_t ZSTD_rust_compressSequencesAndLiteralsInternal( void* dst, size_t dstCapacity, const ZSTD_Sequence* inSeqs, size_t nbSequences, const void* literals, size_t litSize, size_t srcSize); + +/* Public sequence conversion uses only the sequence store and the two + * compressed-block state slots. Keep the C entry point for existing callers, + * but move its conversion logic behind this explicit Rust projection. */ +typedef struct { + SeqStore_t* seqStore; + ZSTD_compressedBlockState_t** prevCBlock; + ZSTD_compressedBlockState_t** nextCBlock; +} ZSTD_rust_convertBlockSequencesState; +size_t ZSTD_rust_convertBlockSequences( + const ZSTD_rust_convertBlockSequencesState* state, + const ZSTD_Sequence* inSeqs, size_t nbSequences, + int repcodeResolution); +typedef char ZSTD_rust_convert_block_sequences_state_layout[ + (offsetof(ZSTD_rust_convertBlockSequencesState, seqStore) == 0 + && offsetof(ZSTD_rust_convertBlockSequencesState, prevCBlock) == sizeof(void*) + && offsetof(ZSTD_rust_convertBlockSequencesState, nextCBlock) == 2 * sizeof(void*) + && sizeof(ZSTD_rust_convertBlockSequencesState) == 3 * sizeof(void*)) + ? 1 : -1]; typedef char ZSTD_rust_sequence_literals_state_layout[ (offsetof(ZSTD_rust_sequenceLiteralsState, seqStore) == 0 && offsetof(ZSTD_rust_sequenceLiteralsState, prevCBlock) == sizeof(void*) @@ -2664,12 +2736,6 @@ ZSTD_BlockCompressor_f ZSTD_selectBlockCompressor(ZSTD_strategy strat, ZSTD_Para return selectedCompressor; } -static void ZSTD_storeLastLiterals(SeqStore_t* seqStorePtr, - const BYTE* anchor, size_t lastLLSize) -{ - ZSTD_rust_storeLastLiterals(seqStorePtr, anchor, lastLLSize); -} - void ZSTD_resetSeqStore(SeqStore_t* ssPtr) { ZSTD_rust_resetSeqStore(ssPtr); @@ -2698,19 +2764,6 @@ static size_t ZSTD_fastSequenceLengthSum(ZSTD_Sequence const* seqBuf, size_t seq return ZSTD_rust_fastSequenceLengthSum(seqBuf, seqBufSize); } -/** - * Function to validate sequences produced by a block compressor. - */ -static void ZSTD_validateSeqStore(const SeqStore_t* seqStore, const ZSTD_compressionParameters* cParams) -{ -#if DEBUGLEVEL >= 1 - ZSTD_rust_validateSeqStore(seqStore, (U32)cParams->minMatch); -#else - (void)seqStore; - (void)cParams; -#endif -} - static size_t ZSTD_transferSequences_wBlockDelim(ZSTD_CCtx* cctx, ZSTD_SequencePosition* seqPos, @@ -2720,169 +2773,172 @@ ZSTD_transferSequences_wBlockDelim(ZSTD_CCtx* cctx, typedef enum { ZSTDbss_compress, ZSTDbss_noCompress } ZSTD_BuildSeqStore_e; -static size_t ZSTD_buildSeqStore(ZSTD_CCtx* zc, const void* src, size_t srcSize) +static void ZSTD_rust_buildSeqStore_skipSmallBlock(void* context, size_t srcSize) { + ZSTD_CCtx* const zc = (ZSTD_CCtx*)context; + if (zc->appliedParams.cParams.strategy >= ZSTD_btopt) { + ZSTD_ldm_skipRawSeqStoreBytes(&zc->externSeqStore, srcSize); + } else { + ZSTD_ldm_skipSequences(&zc->externSeqStore, srcSize, + zc->appliedParams.cParams.minMatch); + } +} + +static void ZSTD_rust_buildSeqStore_prepareMatchState(void* context, + const void* src, + size_t srcSize) +{ + ZSTD_CCtx* const zc = (ZSTD_CCtx*)context; ZSTD_MatchState_t* const ms = &zc->blockState.matchState; DEBUGLOG(5, "ZSTD_buildSeqStore (srcSize=%zu)", srcSize); - assert(srcSize <= ZSTD_BLOCKSIZE_MAX); - /* Assert that we have correctly flushed the ctx params into the ms's copy */ + (void)srcSize; + /* Assert that we have correctly flushed the ctx params into the ms's copy. */ ZSTD_assertEqualCParams(zc->appliedParams.cParams, ms->cParams); - /* TODO: See 3090. We reduced MIN_CBLOCK_SIZE from 3 to 2 so to compensate we are adding - * additional 1. We need to revisit and change this logic to be more consistent */ - if (srcSize < MIN_CBLOCK_SIZE+ZSTD_blockHeaderSize+1+1) { - if (zc->appliedParams.cParams.strategy >= ZSTD_btopt) { - ZSTD_ldm_skipRawSeqStoreBytes(&zc->externSeqStore, srcSize); - } else { - ZSTD_ldm_skipSequences(&zc->externSeqStore, srcSize, zc->appliedParams.cParams.minMatch); - } - return ZSTDbss_noCompress; /* don't even attempt compression below a certain srcSize */ - } - ZSTD_resetSeqStore(&(zc->seqStore)); /* required for optimal parser to read stats from dictionary */ ms->opt.symbolCosts = &zc->blockState.prevCBlock->entropy; /* tell the optimal parser how we expect to compress literals */ ms->opt.literalCompressionMode = zc->appliedParams.literalCompressionMode; /* a gap between an attached dict and the current window is not safe, - * they must remain adjacent, - * and when that stops being the case, the dict must be unset */ + * they must remain adjacent, and when that stops being the case, the + * dict must be unset */ assert(ms->dictMatchState == NULL || ms->loadedDictEnd == ms->window.dictLimit); /* limited update after a very long match */ { const BYTE* const base = ms->window.base; const BYTE* const istart = (const BYTE*)src; const U32 curr = (U32)(istart-base); - if (sizeof(ptrdiff_t)==8) assert(istart - base < (ptrdiff_t)(U32)(-1)); /* ensure no overflow */ + if (sizeof(ptrdiff_t)==8) assert(istart - base < (ptrdiff_t)(U32)(-1)); if (curr > ms->nextToUpdate + 384) ms->nextToUpdate = curr - MIN(192, (U32)(curr - ms->nextToUpdate - 384)); } +} - /* select and store sequences */ - { ZSTD_dictMode_e const dictMode = ZSTD_matchState_dictMode(ms); - size_t lastLLSize; - { int i; - for (i = 0; i < ZSTD_REP_NUM; ++i) - zc->blockState.nextCBlock->rep[i] = zc->blockState.prevCBlock->rep[i]; - } - if (zc->externSeqStore.pos < zc->externSeqStore.size) { - assert(zc->appliedParams.ldmParams.enableLdm == ZSTD_ps_disable); +static size_t ZSTD_rust_buildSeqStore_selectSequences( + void* context, SeqStore_t* seqStore, U32 nextRep[ZSTD_REP_NUM], + const void* src, size_t srcSize, int* seqStoreComplete) +{ + ZSTD_CCtx* const zc = (ZSTD_CCtx*)context; + ZSTD_MatchState_t* const ms = &zc->blockState.matchState; + ZSTD_dictMode_e const dictMode = ZSTD_matchState_dictMode(ms); + size_t lastLLSize; + *seqStoreComplete = 0; - /* External matchfinder + LDM is technically possible, just not implemented yet. - * We need to revisit soon and implement it. */ - RETURN_ERROR_IF( - ZSTD_hasExtSeqProd(&zc->appliedParams), - parameter_combination_unsupported, - "Long-distance matching with external sequence producer enabled is not currently supported." - ); + if (zc->externSeqStore.pos < zc->externSeqStore.size) { + assert(zc->appliedParams.ldmParams.enableLdm == ZSTD_ps_disable); - /* Updates ldmSeqStore.pos */ - lastLLSize = - ZSTD_ldm_blockCompress(&zc->externSeqStore, - ms, &zc->seqStore, - zc->blockState.nextCBlock->rep, - zc->appliedParams.useRowMatchFinder, - src, srcSize); - assert(zc->externSeqStore.pos <= zc->externSeqStore.size); - } else if (zc->appliedParams.ldmParams.enableLdm == ZSTD_ps_enable) { - RawSeqStore_t ldmSeqStore = kNullRawSeqStore; + /* External matchfinder + LDM is technically possible, just not + * implemented yet. */ + RETURN_ERROR_IF( + ZSTD_hasExtSeqProd(&zc->appliedParams), + parameter_combination_unsupported, + "Long-distance matching with external sequence producer enabled is not currently supported." + ); - /* External matchfinder + LDM is technically possible, just not implemented yet. - * We need to revisit soon and implement it. */ - RETURN_ERROR_IF( - ZSTD_hasExtSeqProd(&zc->appliedParams), - parameter_combination_unsupported, - "Long-distance matching with external sequence producer enabled is not currently supported." - ); + lastLLSize = ZSTD_ldm_blockCompress( + &zc->externSeqStore, ms, seqStore, nextRep, + zc->appliedParams.useRowMatchFinder, src, srcSize); + assert(zc->externSeqStore.pos <= zc->externSeqStore.size); + } else if (zc->appliedParams.ldmParams.enableLdm == ZSTD_ps_enable) { + RawSeqStore_t ldmSeqStore = kNullRawSeqStore; - ldmSeqStore.seq = zc->ldmSequences; - ldmSeqStore.capacity = zc->maxNbLdmSequences; - /* Updates ldmSeqStore.size */ - FORWARD_IF_ERROR(ZSTD_ldm_generateSequences(&zc->ldmState, &ldmSeqStore, - &zc->appliedParams.ldmParams, - src, srcSize), ""); - /* Updates ldmSeqStore.pos */ - lastLLSize = - ZSTD_ldm_blockCompress(&ldmSeqStore, - ms, &zc->seqStore, - zc->blockState.nextCBlock->rep, - zc->appliedParams.useRowMatchFinder, - src, srcSize); - assert(ldmSeqStore.pos == ldmSeqStore.size); - } else if (ZSTD_hasExtSeqProd(&zc->appliedParams)) { - assert( - zc->extSeqBufCapacity >= ZSTD_sequenceBound(srcSize) - ); - assert(zc->appliedParams.extSeqProdFunc != NULL); + /* External matchfinder + LDM is technically possible, just not + * implemented yet. */ + RETURN_ERROR_IF( + ZSTD_hasExtSeqProd(&zc->appliedParams), + parameter_combination_unsupported, + "Long-distance matching with external sequence producer enabled is not currently supported." + ); - { U32 const windowSize = (U32)1 << zc->appliedParams.cParams.windowLog; + ldmSeqStore.seq = zc->ldmSequences; + ldmSeqStore.capacity = zc->maxNbLdmSequences; + FORWARD_IF_ERROR(ZSTD_ldm_generateSequences( + &zc->ldmState, &ldmSeqStore, &zc->appliedParams.ldmParams, + src, srcSize), ""); + lastLLSize = ZSTD_ldm_blockCompress( + &ldmSeqStore, ms, seqStore, nextRep, + zc->appliedParams.useRowMatchFinder, src, srcSize); + assert(ldmSeqStore.pos == ldmSeqStore.size); + } else if (ZSTD_hasExtSeqProd(&zc->appliedParams)) { + assert(zc->extSeqBufCapacity >= ZSTD_sequenceBound(srcSize)); + assert(zc->appliedParams.extSeqProdFunc != NULL); - size_t const nbExternalSeqs = (zc->appliedParams.extSeqProdFunc)( + { U32 const windowSize = (U32)1 << zc->appliedParams.cParams.windowLog; + size_t const nbExternalSeqs = (zc->appliedParams.extSeqProdFunc)( zc->appliedParams.extSeqProdState, - zc->extSeqBuf, - zc->extSeqBufCapacity, - src, srcSize, - NULL, 0, /* dict and dictSize, currently not supported */ - zc->appliedParams.compressionLevel, - windowSize - ); + zc->extSeqBuf, zc->extSeqBufCapacity, + src, srcSize, NULL, 0, + zc->appliedParams.compressionLevel, windowSize); + size_t const nbPostProcessedSeqs = ZSTD_postProcessSequenceProducerResult( + zc->extSeqBuf, nbExternalSeqs, + zc->extSeqBufCapacity, srcSize); - size_t const nbPostProcessedSeqs = ZSTD_postProcessSequenceProducerResult( - zc->extSeqBuf, - nbExternalSeqs, - zc->extSeqBufCapacity, - srcSize - ); + /* Return early if there is no error, since the delimiter already + * carries the block's final literals. */ + if (!ZSTD_isError(nbPostProcessedSeqs)) { + ZSTD_SequencePosition seqPos = {0,0,0}; + size_t const seqLenSum = ZSTD_fastSequenceLengthSum( + zc->extSeqBuf, nbPostProcessedSeqs); + RETURN_ERROR_IF(seqLenSum > srcSize, externalSequences_invalid, + "External sequences imply too large a block!"); + FORWARD_IF_ERROR(ZSTD_transferSequences_wBlockDelim( + zc, &seqPos, zc->extSeqBuf, nbPostProcessedSeqs, + src, srcSize, + zc->appliedParams.searchForExternalRepcodes), + "Failed to copy external sequences to seqStore!"); + ms->ldmSeqStore = NULL; + *seqStoreComplete = 1; + DEBUGLOG(5, "Copied %lu sequences from external sequence producer to internal seqStore.", + (unsigned long)nbExternalSeqs); + return 0; + } - /* Return early if there is no error, since we don't need to worry about last literals */ - if (!ZSTD_isError(nbPostProcessedSeqs)) { - ZSTD_SequencePosition seqPos = {0,0,0}; - size_t const seqLenSum = ZSTD_fastSequenceLengthSum(zc->extSeqBuf, nbPostProcessedSeqs); - RETURN_ERROR_IF(seqLenSum > srcSize, externalSequences_invalid, "External sequences imply too large a block!"); - FORWARD_IF_ERROR( - ZSTD_transferSequences_wBlockDelim( - zc, &seqPos, - zc->extSeqBuf, nbPostProcessedSeqs, - src, srcSize, - zc->appliedParams.searchForExternalRepcodes - ), - "Failed to copy external sequences to seqStore!" - ); - ms->ldmSeqStore = NULL; - DEBUGLOG(5, "Copied %lu sequences from external sequence producer to internal seqStore.", (unsigned long)nbExternalSeqs); - return ZSTDbss_compress; - } + if (!zc->appliedParams.enableMatchFinderFallback) + return nbPostProcessedSeqs; - /* Propagate the error if fallback is disabled */ - if (!zc->appliedParams.enableMatchFinderFallback) { - return nbPostProcessedSeqs; - } - - /* Fallback to software matchfinder */ - { ZSTD_BlockCompressor_f const blockCompressor = + { ZSTD_BlockCompressor_f const blockCompressor = ZSTD_selectBlockCompressor( - zc->appliedParams.cParams.strategy, - zc->appliedParams.useRowMatchFinder, - dictMode); - ms->ldmSeqStore = NULL; - DEBUGLOG( - 5, - "External sequence producer returned error code %lu. Falling back to internal parser.", - (unsigned long)nbExternalSeqs - ); - lastLLSize = blockCompressor(ms, &zc->seqStore, zc->blockState.nextCBlock->rep, src, srcSize); - } } - } else { /* not long range mode and no external matchfinder */ - ZSTD_BlockCompressor_f const blockCompressor = ZSTD_selectBlockCompressor( - zc->appliedParams.cParams.strategy, - zc->appliedParams.useRowMatchFinder, - dictMode); - ms->ldmSeqStore = NULL; - lastLLSize = blockCompressor(ms, &zc->seqStore, zc->blockState.nextCBlock->rep, src, srcSize); + zc->appliedParams.cParams.strategy, + zc->appliedParams.useRowMatchFinder, + dictMode); + ms->ldmSeqStore = NULL; + DEBUGLOG(5, "External sequence producer returned error code %lu. Falling back to internal parser.", + (unsigned long)nbExternalSeqs); + lastLLSize = blockCompressor(ms, seqStore, nextRep, src, srcSize); + } } - { const BYTE* const lastLiterals = (const BYTE*)src + srcSize - lastLLSize; - ZSTD_storeLastLiterals(&zc->seqStore, lastLiterals, lastLLSize); - } } - ZSTD_validateSeqStore(&zc->seqStore, &zc->appliedParams.cParams); - return ZSTDbss_compress; + } else { + ZSTD_BlockCompressor_f const blockCompressor = ZSTD_selectBlockCompressor( + zc->appliedParams.cParams.strategy, + zc->appliedParams.useRowMatchFinder, dictMode); + ms->ldmSeqStore = NULL; + lastLLSize = blockCompressor(ms, seqStore, nextRep, src, srcSize); + } + return lastLLSize; +} + +static void ZSTD_initBuildSeqStoreState( + ZSTD_CCtx* zc, ZSTD_rust_buildSeqStoreState* state) +{ + state->seqStore = &zc->seqStore; + state->prevCBlock = &zc->blockState.prevCBlock; + state->nextCBlock = &zc->blockState.nextCBlock; + state->callbackContext = zc; + state->minMatch = zc->appliedParams.cParams.minMatch; +#if DEBUGLEVEL >= 1 + state->validateSeqStore = 1; +#else + state->validateSeqStore = 0; +#endif + state->skipSmallBlock = ZSTD_rust_buildSeqStore_skipSmallBlock; + state->prepareMatchState = ZSTD_rust_buildSeqStore_prepareMatchState; + state->selectSequences = ZSTD_rust_buildSeqStore_selectSequences; +} + +static size_t ZSTD_buildSeqStore(ZSTD_CCtx* zc, const void* src, size_t srcSize) +{ + ZSTD_rust_buildSeqStoreState state; + ZSTD_initBuildSeqStoreState(zc, &state); + return ZSTD_rust_buildSeqStore(&state, src, srcSize); } /* ZSTD_sequenceBound() lives in rust/src/zstd_compress_api.rs. */ @@ -2980,11 +3036,16 @@ static size_t ZSTD_compressBlock_splitBlock_internal(ZSTD_CCtx* zc, void* dst, size_t dstCapacity, const void* src, size_t blockSize, - U32 lastBlock, U32 nbSeq) + U32 lastBlock, U32 nbSeq, int bss) { ZSTD_rust_splitBlockState state; U32* const partitions = zc->blockSplitCtx.partitions; /* splits plus the terminal boundary */ - size_t const numSplits = ZSTD_deriveBlockSplits(zc, partitions, nbSeq); + size_t numSplits = 0; + + if (bss == ZSTDbss_compress) { + numSplits = ZSTD_deriveBlockSplits(zc, partitions, nbSeq); + FORWARD_IF_ERROR(numSplits, "Deriving block splits failed!"); + } DEBUGLOG(5, "ZSTD_compressBlock_splitBlock_internal (dstCapacity=%u, dictLimit=%u, nextToUpdate=%u)", (unsigned)dstCapacity, (unsigned)zc->blockState.matchState.window.dictLimit, @@ -3003,8 +3064,9 @@ ZSTD_compressBlock_splitBlock_internal(ZSTD_CCtx* zc, state.disableLiteralCompression = ZSTD_literalsCompressionIsDisabled(&zc->appliedParams); state.bmi2 = zc->bmi2; state.isFirstBlock = zc->isFirstBlock; - return ZSTD_rust_compressBlockSplit( - &state, dst, dstCapacity, src, blockSize, lastBlock, numSplits); + return ZSTD_rust_compressBlockSplitAfterBuild( + &state, dst, dstCapacity, src, blockSize, lastBlock, + numSplits, bss); } static size_t @@ -3014,24 +3076,14 @@ ZSTD_compressBlock_splitBlock(ZSTD_CCtx* zc, { U32 nbSeq; size_t cSize; + size_t const bss = ZSTD_buildSeqStore(zc, src, srcSize); DEBUGLOG(5, "ZSTD_compressBlock_splitBlock"); assert(zc->appliedParams.postBlockSplitter == ZSTD_ps_enable); + FORWARD_IF_ERROR(bss, "ZSTD_buildSeqStore failed"); + nbSeq = (U32)(zc->seqStore.sequences - zc->seqStore.sequencesStart); - { const size_t bss = ZSTD_buildSeqStore(zc, src, srcSize); - FORWARD_IF_ERROR(bss, "ZSTD_buildSeqStore failed"); - if (bss == ZSTDbss_noCompress) { - if (zc->blockState.prevCBlock->entropy.fse.offcode_repeatMode == FSE_repeat_valid) - zc->blockState.prevCBlock->entropy.fse.offcode_repeatMode = FSE_repeat_check; - RETURN_ERROR_IF(zc->seqCollector.collectSequences, sequenceProducer_failed, "Uncompressible block"); - cSize = ZSTD_rust_noCompressBlock(dst, dstCapacity, src, srcSize, lastBlock); - FORWARD_IF_ERROR(cSize, "ZSTD_noCompressBlock failed"); - DEBUGLOG(5, "ZSTD_compressBlock_splitBlock: Nocompress block"); - return cSize; - } - nbSeq = (U32)(zc->seqStore.sequences - zc->seqStore.sequencesStart); - } - - cSize = ZSTD_compressBlock_splitBlock_internal(zc, dst, dstCapacity, src, srcSize, lastBlock, nbSeq); + cSize = ZSTD_compressBlock_splitBlock_internal( + zc, dst, dstCapacity, src, srcSize, lastBlock, nbSeq, (int)bss); FORWARD_IF_ERROR(cSize, "Splitting blocks failed!"); return cSize; } @@ -3042,19 +3094,11 @@ ZSTD_compressBlock_internal(ZSTD_CCtx* zc, const void* src, size_t srcSize, U32 frame) { ZSTD_rust_blockInternalState state; + size_t const bss = ZSTD_buildSeqStore(zc, src, srcSize); DEBUGLOG(5, "ZSTD_compressBlock_internal (dstCapacity=%u, dictLimit=%u, nextToUpdate=%u)", (unsigned)dstCapacity, (unsigned)zc->blockState.matchState.window.dictLimit, (unsigned)zc->blockState.matchState.nextToUpdate); - - { const size_t bss = ZSTD_buildSeqStore(zc, src, srcSize); - FORWARD_IF_ERROR(bss, "ZSTD_buildSeqStore failed"); - if (bss == ZSTDbss_noCompress) { - RETURN_ERROR_IF(zc->seqCollector.collectSequences, sequenceProducer_failed, "Uncompressible block"); - if (zc->blockState.prevCBlock->entropy.fse.offcode_repeatMode == FSE_repeat_valid) - zc->blockState.prevCBlock->entropy.fse.offcode_repeatMode = FSE_repeat_check; - return 0; - } - } + FORWARD_IF_ERROR(bss, "ZSTD_buildSeqStore failed"); state.seqStore = &zc->seqStore; state.prevCBlock = &zc->blockState.prevCBlock; @@ -3066,8 +3110,8 @@ ZSTD_compressBlock_internal(ZSTD_CCtx* zc, state.disableLiteralCompression = ZSTD_literalsCompressionIsDisabled(&zc->appliedParams); state.bmi2 = zc->bmi2; state.isFirstBlock = zc->isFirstBlock; - return ZSTD_rust_compressBlockInternal( - &state, dst, dstCapacity, src, srcSize, frame); + return ZSTD_rust_compressBlockInternalAfterBuild( + &state, dst, dstCapacity, src, srcSize, frame, (int)bss); } static size_t ZSTD_compressBlock_targetCBlockSize(ZSTD_CCtx* zc, @@ -3075,8 +3119,8 @@ static size_t ZSTD_compressBlock_targetCBlockSize(ZSTD_CCtx* zc, const void* src, size_t srcSize, U32 lastBlock) { - size_t cSize = 0; - const size_t bss = ZSTD_buildSeqStore(zc, src, srcSize); + size_t cSize; + size_t const bss = ZSTD_buildSeqStore(zc, src, srcSize); DEBUGLOG(5, "ZSTD_compressBlock_targetCBlockSize (dstCapacity=%u, dictLimit=%u, nextToUpdate=%u, srcSize=%zu)", (unsigned)dstCapacity, (unsigned)zc->blockState.matchState.window.dictLimit, (unsigned)zc->blockState.matchState.nextToUpdate, srcSize); FORWARD_IF_ERROR(bss, "ZSTD_buildSeqStore failed"); @@ -3093,14 +3137,10 @@ static size_t ZSTD_compressBlock_targetCBlockSize(ZSTD_CCtx* zc, state.windowLog = zc->appliedParams.cParams.windowLog; state.targetCBlockSize = zc->appliedParams.targetCBlockSize; state.isFirstBlock = zc->isFirstBlock; - cSize = ZSTD_rust_compressBlockTargetCBlockSize( + cSize = ZSTD_rust_compressBlockTargetCBlockSizeAfterBuild( &state, dst, dstCapacity, src, srcSize, (int)bss, lastBlock); } FORWARD_IF_ERROR(cSize, "ZSTD_compressBlock_targetCBlockSize_body failed"); - - if (zc->blockState.prevCBlock->entropy.fse.offcode_repeatMode == FSE_repeat_valid) - zc->blockState.prevCBlock->entropy.fse.offcode_repeatMode = FSE_repeat_check; - return cSize; } @@ -4952,12 +4992,6 @@ size_t ZSTD_compress2_c(ZSTD_CCtx* cctx, } } -/* Returns an offset code, given a sequence's raw offset, the ongoing repcode array, and whether litLength == 0 */ -static U32 ZSTD_finalizeOffBase(U32 rawOffset, const U32 rep[ZSTD_REP_NUM], U32 ll0) -{ - return ZSTD_rust_finalizeOffBase(rawOffset, rep, ll0); -} - /* The explicit-delimiter adapter is also used by the external sequence * producer path. Keep that C-context-facing call site separate from the * Rust-owned compressSequences block loop. */ @@ -5082,97 +5116,20 @@ size_t ZSTD_compressSequences(ZSTD_CCtx* cctx, return cSize; } -/* Sequence conversion is implemented in Rust; this wrapper keeps the - * existing C-side block orchestration and long-length bookkeeping intact. */ -static size_t convertSequences_noRepcodes( - SeqDef* dstSeqs, - const ZSTD_Sequence* inSeqs, - size_t nbSequences) -{ - return ZSTD_rust_convertSequencesNoRepcodes(dstSeqs, inSeqs, nbSequences); -} - -/* - * Precondition: Sequences must end on an explicit Block Delimiter - * @return: 0 on success, or an error code. - * Note: Sequence validation functionality has been disabled (removed). - * This is helpful to generate a lean main pipeline, improving performance. - * It may be re-inserted later. - */ +/* The public symbol remains C-facing for fullbench and the sequence API, but + * the sequence-store conversion itself is Rust-owned. */ size_t ZSTD_convertBlockSequences(ZSTD_CCtx* cctx, const ZSTD_Sequence* const inSeqs, size_t nbSequences, int repcodeResolution) { - Repcodes_t updatedRepcodes; - size_t seqNb = 0; + ZSTD_rust_convertBlockSequencesState state; DEBUGLOG(5, "ZSTD_convertBlockSequences (nbSequences = %zu)", nbSequences); - - RETURN_ERROR_IF(nbSequences >= cctx->seqStore.maxNbSeq, externalSequences_invalid, - "Not enough memory allocated. Try adjusting ZSTD_c_minMatch."); - - ZSTD_memcpy(updatedRepcodes.rep, cctx->blockState.prevCBlock->rep, sizeof(Repcodes_t)); - - /* check end condition */ - assert(nbSequences >= 1); - assert(inSeqs[nbSequences-1].matchLength == 0); - assert(inSeqs[nbSequences-1].offset == 0); - - /* Convert Sequences from public format to internal format */ - if (!repcodeResolution) { - size_t const longl = convertSequences_noRepcodes(cctx->seqStore.sequencesStart, inSeqs, nbSequences-1); - cctx->seqStore.sequences = cctx->seqStore.sequencesStart + nbSequences-1; - if (longl) { - DEBUGLOG(5, "long length"); - assert(cctx->seqStore.longLengthType == ZSTD_llt_none); - if (longl <= nbSequences-1) { - DEBUGLOG(5, "long match length detected at pos %zu", longl-1); - cctx->seqStore.longLengthType = ZSTD_llt_matchLength; - cctx->seqStore.longLengthPos = (U32)(longl-1); - } else { - DEBUGLOG(5, "long literals length detected at pos %zu", longl-nbSequences); - assert(longl <= 2* (nbSequences-1)); - cctx->seqStore.longLengthType = ZSTD_llt_literalLength; - cctx->seqStore.longLengthPos = (U32)(longl-(nbSequences-1)-1); - } - } - } else { - for (seqNb = 0; seqNb < nbSequences - 1 ; seqNb++) { - U32 const litLength = inSeqs[seqNb].litLength; - U32 const matchLength = inSeqs[seqNb].matchLength; - U32 const ll0 = (litLength == 0); - U32 const offBase = ZSTD_finalizeOffBase(inSeqs[seqNb].offset, updatedRepcodes.rep, ll0); - - DEBUGLOG(6, "Storing sequence: (of: %u, ml: %u, ll: %u)", offBase, matchLength, litLength); - ZSTD_storeSeqOnly(&cctx->seqStore, litLength, offBase, matchLength); - ZSTD_updateRep(updatedRepcodes.rep, offBase, ll0); - } - } - - /* If we skipped repcode search while parsing, we need to update repcodes now */ - if (!repcodeResolution && nbSequences > 1) { - U32* const rep = updatedRepcodes.rep; - - if (nbSequences >= 4) { - U32 lastSeqIdx = (U32)nbSequences - 2; /* index of last full sequence */ - rep[2] = inSeqs[lastSeqIdx - 2].offset; - rep[1] = inSeqs[lastSeqIdx - 1].offset; - rep[0] = inSeqs[lastSeqIdx].offset; - } else if (nbSequences == 3) { - rep[2] = rep[0]; - rep[1] = inSeqs[0].offset; - rep[0] = inSeqs[1].offset; - } else { - assert(nbSequences == 2); - rep[2] = rep[1]; - rep[1] = rep[0]; - rep[0] = inSeqs[0].offset; - } - } - - ZSTD_memcpy(cctx->blockState.nextCBlock->rep, updatedRepcodes.rep, sizeof(Repcodes_t)); - - return 0; + state.seqStore = &cctx->seqStore; + state.prevCBlock = &cctx->blockState.prevCBlock; + state.nextCBlock = &cctx->blockState.nextCBlock; + return ZSTD_rust_convertBlockSequences( + &state, inSeqs, nbSequences, repcodeResolution); } static size_t ZSTD_convertBlockSequencesForRust( diff --git a/rust/src/zstd_compress.rs b/rust/src/zstd_compress.rs index d44a5dff8..b3a09dbd7 100644 --- a/rust/src/zstd_compress.rs +++ b/rust/src/zstd_compress.rs @@ -13,6 +13,7 @@ //! `ZSTD_compressStream2(..., ZSTD_e_end)` path dispatch through Rust while //! retaining the C implementation for advanced and partial-stream cases. +use crate::common::MINMATCH; use crate::errors::{ERR_isError, ZstdErrorCode, ERROR}; #[cfg(not(test))] use crate::zstd_compress_api::ZSTD_compressBound; @@ -27,13 +28,16 @@ use crate::zstd_compress_params::{ }; use crate::zstd_compress_sequences::SeqDef; use crate::zstd_compress_stats::{ - SeqCollector, SeqStore_t, ZSTD_Sequence, ZSTD_SequencePosition, ZSTD_compressedBlockState_t, - ZSTD_entropyCTables_t, ZSTD_rust_confirmRepcodesAndEntropyTables, ZSTD_rust_copyBlockSequences, + update_rep, SeqCollector, SeqStore_t, ZSTD_Sequence, ZSTD_SequencePosition, + ZSTD_compressedBlockState_t, ZSTD_entropyCTables_t, ZSTD_rust_confirmRepcodesAndEntropyTables, + ZSTD_rust_convertSequencesNoRepcodes, ZSTD_rust_copyBlockSequences, ZSTD_rust_countSeqStoreLiteralsBytes, ZSTD_rust_countSeqStoreMatchBytes, ZSTD_rust_deriveSeqStoreChunk, ZSTD_rust_determineBlockSize, ZSTD_rust_entropyCompressSeqStore, - ZSTD_rust_entropyCompressSeqStore_internal, ZSTD_rust_get1BlockSummary, ZSTD_rust_isRLE, - ZSTD_rust_maybeRLE, ZSTD_rust_resetSeqStore, ZSTD_rust_seqStore_resolveOffCodes, + ZSTD_rust_entropyCompressSeqStore_internal, ZSTD_rust_finalizeOffBase, + ZSTD_rust_get1BlockSummary, ZSTD_rust_isRLE, ZSTD_rust_maybeRLE, ZSTD_rust_resetSeqStore, + ZSTD_rust_seqStore_resolveOffCodes, ZSTD_rust_storeLastLiterals, ZSTD_rust_transferSequencesNoDelim, ZSTD_rust_transferSequencesWBlockDelim, + ZSTD_rust_validateSeqStore, ZSTD_LLT_LITERAL_LENGTH, ZSTD_LLT_MATCH_LENGTH, }; use crate::zstd_compress_superblock::ZSTD_rust_compressSuperBlock; use std::ffi::c_void; @@ -110,12 +114,73 @@ const ZSTD_E_CONTINUE: c_int = 0; const ZSTD_E_FLUSH: c_int = 1; const ZSTD_CSTREAM_STAGE_LOAD: c_int = 1; const ZSTD_CSTREAM_STAGE_FLUSH: c_int = 2; +const ZSTD_BSS_COMPRESS: c_int = 0; +const ZSTD_BSS_NO_COMPRESS: c_int = 1; +const FSE_REPEAT_CHECK: c_int = 1; +const FSE_REPEAT_VALID: c_int = 2; type FrameChunkPrepareFn = unsafe extern "C" fn(*mut c_void, *const c_void, usize); type FrameChunkCompressFn = unsafe extern "C" fn(*mut c_void, *mut c_void, usize, *const c_void, usize, c_uint) -> usize; type FrameChunkChecksumFn = unsafe extern "C" fn(*mut c_void, *const c_void, usize); +type BuildSeqStoreSkipFn = unsafe extern "C" fn(*mut c_void, usize); +type BuildSeqStorePrepareFn = unsafe extern "C" fn(*mut c_void, *const c_void, usize); +type BuildSeqStoreSelectFn = unsafe extern "C" fn( + *mut c_void, + *mut SeqStore_t, + *mut u32, + *const c_void, + usize, + *mut c_int, +) -> usize; + +/// Explicit projection for the sequence-store builder. +/// +/// Rust owns the threshold/reset/repcode/literal-store orchestration. The +/// callbacks retain the private matchfinder, LDM, and external sequence +/// producer operations in C without passing `ZSTD_CCtx` across the ABI. +#[repr(C)] +pub struct ZSTD_rust_buildSeqStoreState { + seq_store: *mut SeqStore_t, + prev_c_block: *mut *mut ZSTD_compressedBlockState_t, + next_c_block: *mut *mut ZSTD_compressedBlockState_t, + callback_context: *mut c_void, + min_match: c_uint, + validate_seq_store: c_int, + skip_small_block: BuildSeqStoreSkipFn, + prepare_match_state: BuildSeqStorePrepareFn, + select_sequences: BuildSeqStoreSelectFn, +} + +const _: () = { + assert!(offset_of!(ZSTD_rust_buildSeqStoreState, seq_store) == 0); + assert!(offset_of!(ZSTD_rust_buildSeqStoreState, prev_c_block) == size_of::()); + assert!(offset_of!(ZSTD_rust_buildSeqStoreState, next_c_block) == 2 * size_of::()); + assert!(offset_of!(ZSTD_rust_buildSeqStoreState, callback_context) == 3 * size_of::()); + assert!(offset_of!(ZSTD_rust_buildSeqStoreState, min_match) == 4 * size_of::()); + assert!( + offset_of!(ZSTD_rust_buildSeqStoreState, validate_seq_store) + == 4 * size_of::() + size_of::() + ); + assert!( + offset_of!(ZSTD_rust_buildSeqStoreState, skip_small_block) + == 4 * size_of::() + size_of::() + size_of::() + ); + assert!( + offset_of!(ZSTD_rust_buildSeqStoreState, prepare_match_state) + == 5 * size_of::() + size_of::() + size_of::() + ); + assert!( + offset_of!(ZSTD_rust_buildSeqStoreState, select_sequences) + == 6 * size_of::() + size_of::() + size_of::() + ); + assert!( + size_of::() + == 7 * size_of::() + size_of::() + size_of::() + ); +}; + /// Explicit projection of the state used by `ZSTD_compress_frameChunk`. /// /// The Rust side owns the per-frame block loop and its savings/dispatch @@ -1036,6 +1101,25 @@ pub struct ZSTD_rust_sequenceLiteralsState { convert_block_sequences: SequenceLiteralsConvertFn, } +/// Explicit projection for the public block-sequence conversion entry point. +/// The surrounding `ZSTD_CCtx` remains opaque; only the sequence store and +/// compressed-block repcode slots are needed by the converter. +#[repr(C)] +pub struct ZSTD_rust_convertBlockSequencesState { + seq_store: *mut SeqStore_t, + prev_c_block: *mut *mut ZSTD_compressedBlockState_t, + next_c_block: *mut *mut ZSTD_compressedBlockState_t, +} + +const _: () = { + assert!(offset_of!(ZSTD_rust_convertBlockSequencesState, seq_store) == 0); + assert!(offset_of!(ZSTD_rust_convertBlockSequencesState, prev_c_block) == size_of::()); + assert!( + offset_of!(ZSTD_rust_convertBlockSequencesState, next_c_block) == 2 * size_of::() + ); + assert!(size_of::() == 3 * size_of::()); +}; + const _: () = { assert!(offset_of!(ZSTD_rust_sequenceLiteralsState, seq_store) == 0); assert!(offset_of!(ZSTD_rust_sequenceLiteralsState, prev_c_block) == size_of::()); @@ -1352,6 +1436,100 @@ impl SingleBlockSeams { } } +/// Rust-owned sequence-store boundary. C callbacks perform the operations +/// which need the private matchfinder or CCtx parameter/function-pointer state. +#[allow(clippy::too_many_arguments)] +unsafe fn build_seq_store_body_with( + state: &ZSTD_rust_buildSeqStoreState, + src: *const c_void, + src_size: usize, +) -> usize { + if state.seq_store.is_null() + || state.skip_small_block as usize == 0 + || state.prepare_match_state as usize == 0 + || state.select_sequences as usize == 0 + { + return ERROR(ZstdErrorCode::Generic); + } + + if src_size > ZSTD_BLOCKSIZE_MAX { + return ERROR(ZstdErrorCode::SrcSizeWrong); + } + + if src_size < MIN_COMPRESSIBLE_BLOCK_SIZE { + unsafe { (state.skip_small_block)(state.callback_context, src_size) }; + return ZSTD_BSS_NO_COMPRESS as usize; + } + + if src.is_null() || state.prev_c_block.is_null() || state.next_c_block.is_null() { + return ERROR(ZstdErrorCode::Generic); + } + + let prev_c_block = unsafe { *state.prev_c_block }; + let next_c_block = unsafe { *state.next_c_block }; + if prev_c_block.is_null() || next_c_block.is_null() { + return ERROR(ZstdErrorCode::Generic); + } + + unsafe { ZSTD_rust_resetSeqStore(state.seq_store) }; + unsafe { + (state.prepare_match_state)(state.callback_context, src, src_size); + } + + unsafe { + ptr::copy_nonoverlapping( + (*prev_c_block).rep.as_ptr(), + (*next_c_block).rep.as_mut_ptr(), + ZSTD_REP_NUM, + ); + } + + let mut seq_store_complete = 0; + let last_literals_size = unsafe { + (state.select_sequences)( + state.callback_context, + state.seq_store, + (*next_c_block).rep.as_mut_ptr(), + src, + src_size, + &mut seq_store_complete, + ) + }; + if ERR_isError(last_literals_size) { + return last_literals_size; + } + if seq_store_complete != 0 { + return ZSTD_BSS_COMPRESS as usize; + } + if last_literals_size > src_size { + return ERROR(ZstdErrorCode::Generic); + } + + unsafe { + ZSTD_rust_storeLastLiterals( + state.seq_store, + src.cast::().add(src_size - last_literals_size), + last_literals_size, + ); + } + if state.validate_seq_store != 0 { + unsafe { ZSTD_rust_validateSeqStore(state.seq_store, state.min_match) }; + } + ZSTD_BSS_COMPRESS as usize +} + +#[no_mangle] +pub unsafe extern "C" fn ZSTD_rust_buildSeqStore( + state: *const ZSTD_rust_buildSeqStoreState, + src: *const c_void, + src_size: usize, +) -> usize { + if state.is_null() { + return ERROR(ZstdErrorCode::Generic); + } + unsafe { build_seq_store_body_with(&*state, src, src_size) } +} + /// Rust implementation of `ZSTD_compressBlock_internal` after the C caller /// has built the sequence store. #[allow(clippy::too_many_arguments)] @@ -1441,6 +1619,40 @@ unsafe fn compress_block_internal_body_with( c_size } +unsafe fn compress_block_internal_after_build_body_with( + state: &ZSTD_rust_blockInternalState, + dst: *mut c_void, + dst_capacity: usize, + src: *const c_void, + src_size: usize, + frame: c_uint, + bss: c_int, + seams: SingleBlockSeams, +) -> usize { + if bss == ZSTD_BSS_NO_COMPRESS { + if state.seq_collector.is_null() || state.prev_c_block.is_null() { + return ERROR(ZstdErrorCode::Generic); + } + if unsafe { (*state.seq_collector).collectSequences } != 0 { + return ERROR(ZstdErrorCode::SequenceProducerFailed); + } + let prev_c_block = unsafe { *state.prev_c_block }; + if prev_c_block.is_null() { + return ERROR(ZstdErrorCode::Generic); + } + if unsafe { (*prev_c_block).entropy.fse.offcode_repeatMode } == FSE_REPEAT_VALID { + unsafe { (*prev_c_block).entropy.fse.offcode_repeatMode = FSE_REPEAT_CHECK }; + } + return 0; + } + if bss != ZSTD_BSS_COMPRESS { + return ERROR(ZstdErrorCode::Generic); + } + unsafe { + compress_block_internal_body_with(state, dst, dst_capacity, src, src_size, frame, seams) + } +} + #[no_mangle] pub unsafe extern "C" fn ZSTD_rust_compressBlockInternal( state: *const ZSTD_rust_blockInternalState, @@ -1466,6 +1678,33 @@ pub unsafe extern "C" fn ZSTD_rust_compressBlockInternal( } } +#[no_mangle] +pub unsafe extern "C" fn ZSTD_rust_compressBlockInternalAfterBuild( + state: *const ZSTD_rust_blockInternalState, + dst: *mut c_void, + dst_capacity: usize, + src: *const c_void, + src_size: usize, + frame: c_uint, + bss: c_int, +) -> usize { + if state.is_null() { + return ERROR(ZstdErrorCode::Generic); + } + unsafe { + compress_block_internal_after_build_body_with( + &*state, + dst, + dst_capacity, + src, + src_size, + frame, + bss, + SingleBlockSeams::production(), + ) + } +} + #[inline] fn sequence_block_action( is_first_block: c_int, @@ -1628,6 +1867,47 @@ unsafe fn compress_block_target_c_block_size_body_with( unsafe { ZSTD_rust_noCompressBlock(dst, dst_capacity, src, src_size, last_block) } } +unsafe fn compress_block_target_c_block_size_after_build_body_with( + state: &ZSTD_rust_targetCBlockSizeState, + dst: *mut c_void, + dst_capacity: usize, + src: *const c_void, + src_size: usize, + bss: c_int, + last_block: c_uint, + compress_super_block: TargetCBlockSuperBlockFn, +) -> usize { + if bss != ZSTD_BSS_COMPRESS && bss != ZSTD_BSS_NO_COMPRESS { + return ERROR(ZstdErrorCode::Generic); + } + let c_size = unsafe { + compress_block_target_c_block_size_body_with( + state, + dst, + dst_capacity, + src, + src_size, + bss, + last_block, + compress_super_block, + ) + }; + if ERR_isError(c_size) { + return c_size; + } + if state.prev_c_block.is_null() { + return ERROR(ZstdErrorCode::Generic); + } + let prev_c_block = unsafe { *state.prev_c_block }; + if prev_c_block.is_null() { + return ERROR(ZstdErrorCode::Generic); + } + if unsafe { (*prev_c_block).entropy.fse.offcode_repeatMode } == FSE_REPEAT_VALID { + unsafe { (*prev_c_block).entropy.fse.offcode_repeatMode = FSE_REPEAT_CHECK }; + } + c_size +} + /// C ABI entry point for the target-sized block body. The public and outer /// block APIs remain C-owned; this is only the body after `ZSTD_buildSeqStore`. #[no_mangle] @@ -1657,6 +1937,33 @@ pub unsafe extern "C" fn ZSTD_rust_compressBlockTargetCBlockSize( } } +#[no_mangle] +pub unsafe extern "C" fn ZSTD_rust_compressBlockTargetCBlockSizeAfterBuild( + state: *const ZSTD_rust_targetCBlockSizeState, + dst: *mut c_void, + dst_capacity: usize, + src: *const c_void, + src_size: usize, + bss: c_int, + last_block: c_uint, +) -> usize { + if state.is_null() { + return ERROR(ZstdErrorCode::Generic); + } + unsafe { + compress_block_target_c_block_size_after_build_body_with( + &*state, + dst, + dst_capacity, + src, + src_size, + bss, + last_block, + ZSTD_rust_compressSuperBlock, + ) + } +} + /// Rust implementation of `ZSTD_compressSeqStore_singleBlock`. /// /// The C caller still owns sequence-store construction, split-block control, @@ -1999,6 +2306,52 @@ unsafe fn compress_block_split_body_with( c_size } +unsafe fn compress_block_split_after_build_body_with( + state: &ZSTD_rust_splitBlockState, + dst: *mut c_void, + dst_capacity: usize, + src: *const c_void, + block_size: usize, + last_block: c_uint, + num_splits: usize, + bss: c_int, + seams: SingleBlockSeams, +) -> usize { + if bss == ZSTD_BSS_NO_COMPRESS { + if state.seq_collector.is_null() || state.prev_c_block.is_null() { + return ERROR(ZstdErrorCode::Generic); + } + if unsafe { (*state.seq_collector).collectSequences } != 0 { + return ERROR(ZstdErrorCode::SequenceProducerFailed); + } + let prev_c_block = unsafe { *state.prev_c_block }; + if prev_c_block.is_null() { + return ERROR(ZstdErrorCode::Generic); + } + if unsafe { (*prev_c_block).entropy.fse.offcode_repeatMode } == FSE_REPEAT_VALID { + unsafe { (*prev_c_block).entropy.fse.offcode_repeatMode = FSE_REPEAT_CHECK }; + } + return unsafe { + ZSTD_rust_noCompressBlock(dst, dst_capacity, src, block_size, last_block) + }; + } + if bss != ZSTD_BSS_COMPRESS { + return ERROR(ZstdErrorCode::Generic); + } + unsafe { + compress_block_split_body_with( + state, + dst, + dst_capacity, + src, + block_size, + last_block, + num_splits, + seams, + ) + } +} + #[no_mangle] pub unsafe extern "C" fn ZSTD_rust_compressBlockSplit( state: *const ZSTD_rust_splitBlockState, @@ -2026,6 +2379,203 @@ pub unsafe extern "C" fn ZSTD_rust_compressBlockSplit( } } +#[no_mangle] +pub unsafe extern "C" fn ZSTD_rust_compressBlockSplitAfterBuild( + state: *const ZSTD_rust_splitBlockState, + dst: *mut c_void, + dst_capacity: usize, + src: *const c_void, + block_size: usize, + last_block: c_uint, + num_splits: usize, + bss: c_int, +) -> usize { + if state.is_null() { + return ERROR(ZstdErrorCode::Generic); + } + unsafe { + compress_block_split_after_build_body_with( + &*state, + dst, + dst_capacity, + src, + block_size, + last_block, + num_splits, + bss, + SingleBlockSeams::production(), + ) + } +} + +unsafe fn store_converted_sequence( + seq_store: &mut SeqStore_t, + lit_length: u32, + off_base: u32, + match_length: u32, +) -> bool { + let sequence_index = unsafe { seq_store.sequences.offset_from(seq_store.sequencesStart) }; + if sequence_index < 0 || sequence_index as usize >= seq_store.maxNbSeq { + return false; + } + let ml_base = match (match_length as usize).checked_sub(MINMATCH) { + Some(value) => value, + None => return false, + }; + + if lit_length as usize > u16::MAX as usize { + if seq_store.longLengthType != 0 { + return false; + } + seq_store.longLengthType = ZSTD_LLT_LITERAL_LENGTH; + seq_store.longLengthPos = sequence_index as u32; + } + if ml_base > u16::MAX as usize { + if seq_store.longLengthType != 0 { + return false; + } + seq_store.longLengthType = ZSTD_LLT_MATCH_LENGTH; + seq_store.longLengthPos = sequence_index as u32; + } + + unsafe { + (*seq_store.sequences).litLength = lit_length as u16; + (*seq_store.sequences).offBase = off_base; + (*seq_store.sequences).mlBase = ml_base as u16; + seq_store.sequences = seq_store.sequences.add(1); + } + true +} + +unsafe fn convert_block_sequences_body_with( + state: &ZSTD_rust_convertBlockSequencesState, + in_seqs: *const ZSTD_Sequence, + nb_sequences: usize, + repcode_resolution: c_int, +) -> usize { + if state.seq_store.is_null() + || state.prev_c_block.is_null() + || state.next_c_block.is_null() + || in_seqs.is_null() + || nb_sequences == 0 + { + return ERROR(ZstdErrorCode::ExternalSequencesInvalid); + } + + let seq_store = unsafe { &mut *state.seq_store }; + if nb_sequences >= seq_store.maxNbSeq { + return ERROR(ZstdErrorCode::ExternalSequencesInvalid); + } + let delimiter = unsafe { *in_seqs.add(nb_sequences - 1) }; + if delimiter.matchLength != 0 || delimiter.offset != 0 { + return ERROR(ZstdErrorCode::ExternalSequencesInvalid); + } + + let prev_c_block = unsafe { *state.prev_c_block }; + let next_c_block = unsafe { *state.next_c_block }; + if prev_c_block.is_null() || next_c_block.is_null() { + return ERROR(ZstdErrorCode::ExternalSequencesInvalid); + } + + let mut updated_repcodes = [0u32; ZSTD_REP_NUM]; + unsafe { + ptr::copy_nonoverlapping( + (*prev_c_block).rep.as_ptr(), + updated_repcodes.as_mut_ptr(), + ZSTD_REP_NUM, + ); + } + + if repcode_resolution == 0 { + let full_sequence_count = nb_sequences - 1; + let long_length = unsafe { + ZSTD_rust_convertSequencesNoRepcodes( + seq_store.sequencesStart, + in_seqs, + full_sequence_count, + ) + }; + if long_length != 0 { + if seq_store.longLengthType != 0 { + return ERROR(ZstdErrorCode::ExternalSequencesInvalid); + } + if long_length <= full_sequence_count { + seq_store.longLengthType = ZSTD_LLT_MATCH_LENGTH; + seq_store.longLengthPos = (long_length - 1) as u32; + } else if long_length <= full_sequence_count.saturating_mul(2) { + seq_store.longLengthType = ZSTD_LLT_LITERAL_LENGTH; + seq_store.longLengthPos = (long_length - nb_sequences) as u32; + } else { + return ERROR(ZstdErrorCode::ExternalSequencesInvalid); + } + } + seq_store.sequences = unsafe { seq_store.sequencesStart.add(full_sequence_count) }; + + if nb_sequences > 1 { + let rep = &mut updated_repcodes; + if nb_sequences >= 4 { + let last_seq_idx = nb_sequences - 2; + rep[2] = unsafe { (*in_seqs.add(last_seq_idx - 2)).offset }; + rep[1] = unsafe { (*in_seqs.add(last_seq_idx - 1)).offset }; + rep[0] = unsafe { (*in_seqs.add(last_seq_idx)).offset }; + } else if nb_sequences == 3 { + rep[2] = rep[0]; + rep[1] = unsafe { (*in_seqs).offset }; + rep[0] = unsafe { (*in_seqs.add(1)).offset }; + } else { + rep[2] = rep[1]; + rep[1] = rep[0]; + rep[0] = unsafe { (*in_seqs).offset }; + } + } + } else { + for index in 0..(nb_sequences - 1) { + let sequence = unsafe { *in_seqs.add(index) }; + let ll0 = sequence.litLength == 0; + let off_base = unsafe { + ZSTD_rust_finalizeOffBase( + sequence.offset, + updated_repcodes.as_ptr(), + u32::from(ll0), + ) + }; + if !unsafe { + store_converted_sequence( + seq_store, + sequence.litLength, + off_base, + sequence.matchLength, + ) + } { + return ERROR(ZstdErrorCode::ExternalSequencesInvalid); + } + update_rep(&mut updated_repcodes, off_base, ll0); + } + } + + unsafe { + ptr::copy_nonoverlapping( + updated_repcodes.as_ptr(), + (*next_c_block).rep.as_mut_ptr(), + ZSTD_REP_NUM, + ); + } + 0 +} + +#[no_mangle] +pub unsafe extern "C" fn ZSTD_rust_convertBlockSequences( + state: *const ZSTD_rust_convertBlockSequencesState, + in_seqs: *const ZSTD_Sequence, + nb_sequences: usize, + repcode_resolution: c_int, +) -> usize { + if state.is_null() { + return ERROR(ZstdErrorCode::ExternalSequencesInvalid); + } + unsafe { convert_block_sequences_body_with(&*state, in_seqs, nb_sequences, repcode_resolution) } +} + /// Select the strategy used by the simple compression entry points. /// /// This is the Rust equivalent of the strategy portion of