diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index fd37d9b71..05448a420 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -38,6 +38,62 @@ size_t ZSTD_rust_writeFrameHeader(void* dst, size_t dstCapacity, U32 windowLog, U64 pledgedSrcSize, U32 dictID); +/* Context-free compression-parameter selection and sizing leaves live in + * Rust (rust/src/zstd_compress_params.rs). This file retains + * configuration-sensitive policy: the excluded-block-compressor strategy + * cascade, private ZSTD_CCtx_params handling, and sanitizer workspace + * policy, which it feeds to the leaves as explicit scalar inputs. The + * ZSTD_CParamMode_e and ZSTD_ParamSwitch_e enums are passed as int; the + * Rust side mirrors their values. */ +int ZSTD_rust_params_maxCLevel(void); +int ZSTD_rust_params_minCLevel(void); +int ZSTD_rust_params_defaultCLevel(void); +ZSTD_bounds ZSTD_rust_params_getBounds(int param); +size_t ZSTD_rust_params_checkCParams(ZSTD_compressionParameters cParams); +ZSTD_compressionParameters +ZSTD_rust_params_clampCParams(ZSTD_compressionParameters cParams); +U32 ZSTD_rust_params_cycleLog(U32 hashLog, int strategy); +ZSTD_compressionParameters +ZSTD_rust_params_selectCParams(int compressionLevel, U64 srcSizeHint, + size_t dictSize, int mode); +ZSTD_compressionParameters +ZSTD_rust_params_adjustCParams(ZSTD_compressionParameters cParams, U64 srcSize, + size_t dictSize, int mode, + int useRowMatchFinder); +ZSTD_parameters ZSTD_rust_params_makeParams(ZSTD_compressionParameters cParams); +size_t ZSTD_rust_params_maxNbSeq(size_t blockSize, U32 minMatch, + int useSequenceProducer); +size_t ZSTD_rust_params_resolveMaxBlockSize(size_t maxBlockSize); + +typedef struct { + U32 hashLog3Max; + size_t matchTSize; + size_t optimalTSize; + size_t asanRedzoneSize; +} ZSTD_rustMatchStateSizing; +size_t ZSTD_rust_params_estimateMatchStateSize( + ZSTD_compressionParameters cParams, int useRowMatchFinder, + int enableDedicatedDictSearch, U32 forCCtx, + const ZSTD_rustMatchStateSizing* sizing); + +typedef struct { + size_t cdictSize; + size_t hufWorkspaceSize; + U32 hashLog3Max; + size_t matchTSize; + size_t optimalTSize; + size_t asanRedzoneSize; +} ZSTD_rustCDictSizing; +size_t ZSTD_rust_params_estimateCDictSizeFromCParams( + size_t dictSize, ZSTD_compressionParameters cParams, + int dictLoadMethod, const ZSTD_rustCDictSizing* sizing); + +#if ZSTD_ADDRESS_SANITIZER && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE) +# define ZSTD_RUST_ASAN_REDZONE_SIZE ((size_t)ZSTD_CWKSP_ASAN_REDZONE_SIZE) +#else +# define ZSTD_RUST_ASAN_REDZONE_SIZE ((size_t)0) +#endif + /* *************************************************************** * Tuning parameters *****************************************************************/ @@ -276,11 +332,7 @@ static int ZSTD_resolveExternalSequenceValidation(int mode) { /* Resolves maxBlockSize to the default if no value is present. */ static size_t ZSTD_resolveMaxBlockSize(size_t maxBlockSize) { - if (maxBlockSize == 0) { - return ZSTD_BLOCKSIZE_MAX; - } else { - return maxBlockSize; - } + return ZSTD_rust_params_resolveMaxBlockSize(maxBlockSize); } static ZSTD_ParamSwitch_e ZSTD_resolveExternalRepcodeSearch(ZSTD_ParamSwitch_e value, int cLevel) { @@ -420,45 +472,17 @@ ZSTD_bounds ZSTD_cParam_getBounds(ZSTD_cParameter param) switch(param) { + /* The compression level and the seven core compression parameters are + * bounded by the Rust leaf. */ case ZSTD_c_compressionLevel: - bounds.lowerBound = ZSTD_minCLevel(); - bounds.upperBound = ZSTD_maxCLevel(); - return bounds; - case ZSTD_c_windowLog: - bounds.lowerBound = ZSTD_WINDOWLOG_MIN; - bounds.upperBound = ZSTD_WINDOWLOG_MAX; - return bounds; - case ZSTD_c_hashLog: - bounds.lowerBound = ZSTD_HASHLOG_MIN; - bounds.upperBound = ZSTD_HASHLOG_MAX; - return bounds; - case ZSTD_c_chainLog: - bounds.lowerBound = ZSTD_CHAINLOG_MIN; - bounds.upperBound = ZSTD_CHAINLOG_MAX; - return bounds; - case ZSTD_c_searchLog: - bounds.lowerBound = ZSTD_SEARCHLOG_MIN; - bounds.upperBound = ZSTD_SEARCHLOG_MAX; - return bounds; - case ZSTD_c_minMatch: - bounds.lowerBound = ZSTD_MINMATCH_MIN; - bounds.upperBound = ZSTD_MINMATCH_MAX; - return bounds; - case ZSTD_c_targetLength: - bounds.lowerBound = ZSTD_TARGETLENGTH_MIN; - bounds.upperBound = ZSTD_TARGETLENGTH_MAX; - return bounds; - case ZSTD_c_strategy: - bounds.lowerBound = ZSTD_STRATEGY_MIN; - bounds.upperBound = ZSTD_STRATEGY_MAX; - return bounds; + return ZSTD_rust_params_getBounds((int)param); case ZSTD_c_contentSizeFlag: bounds.lowerBound = 0; @@ -1385,14 +1409,7 @@ size_t ZSTD_CCtx_reset(ZSTD_CCtx* cctx, ZSTD_ResetDirective reset) @return : 0, or an error code if one value is beyond authorized range */ size_t ZSTD_checkCParams(ZSTD_compressionParameters cParams) { - BOUNDCHECK(ZSTD_c_windowLog, (int)cParams.windowLog); - BOUNDCHECK(ZSTD_c_chainLog, (int)cParams.chainLog); - BOUNDCHECK(ZSTD_c_hashLog, (int)cParams.hashLog); - BOUNDCHECK(ZSTD_c_searchLog, (int)cParams.searchLog); - BOUNDCHECK(ZSTD_c_minMatch, (int)cParams.minMatch); - BOUNDCHECK(ZSTD_c_targetLength,(int)cParams.targetLength); - BOUNDCHECK(ZSTD_c_strategy, (int)cParams.strategy); - return 0; + return ZSTD_rust_params_checkCParams(cParams); } /** ZSTD_clampCParams() : @@ -1401,63 +1418,14 @@ size_t ZSTD_checkCParams(ZSTD_compressionParameters cParams) static ZSTD_compressionParameters ZSTD_clampCParams(ZSTD_compressionParameters cParams) { -# define CLAMP_TYPE(cParam, val, type) \ - do { \ - ZSTD_bounds const bounds = ZSTD_cParam_getBounds(cParam); \ - if ((int)valbounds.upperBound) val=(type)bounds.upperBound; \ - } while (0) -# define CLAMP(cParam, val) CLAMP_TYPE(cParam, val, unsigned) - CLAMP(ZSTD_c_windowLog, cParams.windowLog); - CLAMP(ZSTD_c_chainLog, cParams.chainLog); - CLAMP(ZSTD_c_hashLog, cParams.hashLog); - CLAMP(ZSTD_c_searchLog, cParams.searchLog); - CLAMP(ZSTD_c_minMatch, cParams.minMatch); - CLAMP(ZSTD_c_targetLength,cParams.targetLength); - CLAMP_TYPE(ZSTD_c_strategy,cParams.strategy, ZSTD_strategy); - return cParams; + return ZSTD_rust_params_clampCParams(cParams); } /** ZSTD_cycleLog() : * condition for correct operation : hashLog > 1 */ U32 ZSTD_cycleLog(U32 hashLog, ZSTD_strategy strat) { - U32 const btScale = ((U32)strat >= (U32)ZSTD_btlazy2); - return hashLog - btScale; -} - -/** ZSTD_dictAndWindowLog() : - * Returns an adjusted window log that is large enough to fit the source and the dictionary. - * The zstd format says that the entire dictionary is valid if one byte of the dictionary - * is within the window. So the hashLog and chainLog should be large enough to reference both - * the dictionary and the window. So we must use this adjusted dictAndWindowLog when downsizing - * the hashLog and windowLog. - * NOTE: srcSize must not be ZSTD_CONTENTSIZE_UNKNOWN. - */ -static U32 ZSTD_dictAndWindowLog(U32 windowLog, U64 srcSize, U64 dictSize) -{ - const U64 maxWindowSize = 1ULL << ZSTD_WINDOWLOG_MAX; - /* No dictionary ==> No change */ - if (dictSize == 0) { - return windowLog; - } - assert(windowLog <= ZSTD_WINDOWLOG_MAX); - assert(srcSize != ZSTD_CONTENTSIZE_UNKNOWN); /* Handled in ZSTD_adjustCParams_internal() */ - { - U64 const windowSize = 1ULL << windowLog; - U64 const dictAndWindowSize = dictSize + windowSize; - /* If the window size is already large enough to fit both the source and the dictionary - * then just use the window size. Otherwise adjust so that it fits the dictionary and - * the window. - */ - if (windowSize >= dictSize + srcSize) { - return windowLog; /* Window size large enough already */ - } else if (dictAndWindowSize >= maxWindowSize) { - return ZSTD_WINDOWLOG_MAX; /* Larger than max window log */ - } else { - return ZSTD_highbit32((U32)dictAndWindowSize - 1) + 1; - } - } + return ZSTD_rust_params_cycleLog(hashLog, (int)strat); } /** ZSTD_adjustCParams_internal() : @@ -1474,8 +1442,6 @@ ZSTD_adjustCParams_internal(ZSTD_compressionParameters cPar, ZSTD_CParamMode_e mode, ZSTD_ParamSwitch_e useRowMatchFinder) { - const U64 minSrcSize = 513; /* (1<<9) + 1 */ - const U64 maxWindowResize = 1ULL << (ZSTD_WINDOWLOG_MAX-1); assert(ZSTD_checkCParams(cPar)==0); /* Cascade the selected strategy down to the next-highest one built into @@ -1520,90 +1486,13 @@ ZSTD_adjustCParams_internal(ZSTD_compressionParameters cPar, } #endif - switch (mode) { - case ZSTD_cpm_unknown: - case ZSTD_cpm_noAttachDict: - /* If we don't know the source size, don't make any - * assumptions about it. We will already have selected - * smaller parameters if a dictionary is in use. - */ - break; - case ZSTD_cpm_createCDict: - /* Assume a small source size when creating a dictionary - * with an unknown source size. - */ - if (dictSize && srcSize == ZSTD_CONTENTSIZE_UNKNOWN) - srcSize = minSrcSize; - break; - case ZSTD_cpm_attachDict: - /* Dictionary has its own dedicated parameters which have - * already been selected. We are selecting parameters - * for only the source. - */ - dictSize = 0; - break; - default: - assert(0); - break; - } - - /* resize windowLog if input is small enough, to use less memory */ - if ( (srcSize <= maxWindowResize) - && (dictSize <= maxWindowResize) ) { - U32 const tSize = (U32)(srcSize + dictSize); - static U32 const hashSizeMin = 1 << ZSTD_HASHLOG_MIN; - U32 const srcLog = (tSize < hashSizeMin) ? ZSTD_HASHLOG_MIN : - ZSTD_highbit32(tSize-1) + 1; - if (cPar.windowLog > srcLog) cPar.windowLog = srcLog; - } - if (srcSize != ZSTD_CONTENTSIZE_UNKNOWN) { - U32 const dictAndWindowLog = ZSTD_dictAndWindowLog(cPar.windowLog, (U64)srcSize, (U64)dictSize); - U32 const cycleLog = ZSTD_cycleLog(cPar.chainLog, cPar.strategy); - if (cPar.hashLog > dictAndWindowLog+1) cPar.hashLog = dictAndWindowLog+1; - if (cycleLog > dictAndWindowLog) - cPar.chainLog -= (cycleLog - dictAndWindowLog); - } - - if (cPar.windowLog < ZSTD_WINDOWLOG_ABSOLUTEMIN) - cPar.windowLog = ZSTD_WINDOWLOG_ABSOLUTEMIN; /* minimum wlog required for valid frame header */ - - /* We can't use more than 32 bits of hash in total, so that means that we require: - * (hashLog + 8) <= 32 && (chainLog + 8) <= 32 - */ - if (mode == ZSTD_cpm_createCDict && ZSTD_CDictIndicesAreTagged(&cPar)) { - U32 const maxShortCacheHashLog = 32 - ZSTD_SHORT_CACHE_TAG_BITS; - if (cPar.hashLog > maxShortCacheHashLog) { - cPar.hashLog = maxShortCacheHashLog; - } - if (cPar.chainLog > maxShortCacheHashLog) { - cPar.chainLog = maxShortCacheHashLog; - } - } - - - /* At this point, we aren't 100% sure if we are using the row match finder. - * Unless it is explicitly disabled, conservatively assume that it is enabled. - * In this case it will only be disabled for small sources, so shrinking the - * hash log a little bit shouldn't result in any ratio loss. - */ - if (useRowMatchFinder == ZSTD_ps_auto) - useRowMatchFinder = ZSTD_ps_enable; - - /* We can't hash more than 32-bits in total. So that means that we require: - * (hashLog - rowLog + 8) <= 32 - */ - if (ZSTD_rowMatchFinderUsed(cPar.strategy, useRowMatchFinder)) { - /* Switch to 32-entry rows if searchLog is 5 (or more) */ - U32 const rowLog = BOUNDED(4, cPar.searchLog, 6); - U32 const maxRowHashLog = 32 - ZSTD_ROW_HASH_TAG_BITS; - U32 const maxHashLog = maxRowHashLog + rowLog; - assert(cPar.hashLog >= rowLog); - if (cPar.hashLog > maxHashLog) { - cPar.hashLog = maxHashLog; - } - } - - return cPar; + /* The remaining adjustment logic is context-free and lives in Rust. The + * short-cache and row-hash tag widths the leaf assumes are fixed + * private-header constants; keep them checked here. */ + ZSTD_STATIC_ASSERT(ZSTD_SHORT_CACHE_TAG_BITS == 8); + ZSTD_STATIC_ASSERT(ZSTD_ROW_HASH_TAG_BITS == 8); + return ZSTD_rust_params_adjustCParams(cPar, (U64)srcSize, dictSize, + (int)mode, (int)useRowMatchFinder); } ZSTD_compressionParameters @@ -1654,47 +1543,31 @@ ZSTD_sizeof_matchState(const ZSTD_compressionParameters* const cParams, const int enableDedicatedDictSearch, const U32 forCCtx) { - /* chain table size should be 0 for fast or row-hash strategies */ - size_t const chainSize = ZSTD_allocateChainTable(cParams->strategy, useRowMatchFinder, enableDedicatedDictSearch && !forCCtx) - ? ((size_t)1 << cParams->chainLog) - : 0; - size_t const hSize = ((size_t)1) << cParams->hashLog; - U32 const hashLog3 = (forCCtx && cParams->minMatch==3) ? MIN(ZSTD_HASHLOG3_MAX, cParams->windowLog) : 0; - size_t const h3Size = hashLog3 ? ((size_t)1) << hashLog3 : 0; - /* We don't use ZSTD_cwksp_alloc_size() here because the tables aren't - * surrounded by redzones in ASAN. */ - size_t const tableSpace = chainSize * sizeof(U32) - + hSize * sizeof(U32) - + h3Size * sizeof(U32); - size_t const optPotentialSpace = - ZSTD_cwksp_aligned64_alloc_size((MaxML+1) * sizeof(U32)) - + ZSTD_cwksp_aligned64_alloc_size((MaxLL+1) * sizeof(U32)) - + ZSTD_cwksp_aligned64_alloc_size((MaxOff+1) * sizeof(U32)) - + ZSTD_cwksp_aligned64_alloc_size((1<strategy, useRowMatchFinder) - ? ZSTD_cwksp_aligned64_alloc_size(hSize) - : 0; - size_t const optSpace = (forCCtx && (cParams->strategy >= ZSTD_btopt)) - ? optPotentialSpace - : 0; - size_t const slackSpace = ZSTD_cwksp_slack_space_required(); + ZSTD_rustMatchStateSizing sizing; + sizing.hashLog3Max = ZSTD_HASHLOG3_MAX; + sizing.matchTSize = sizeof(ZSTD_match_t); + sizing.optimalTSize = sizeof(ZSTD_optimal_t); + sizing.asanRedzoneSize = ZSTD_RUST_ASAN_REDZONE_SIZE; + /* The Rust leaf hardcodes the frozen format bounds and workspace rules; + * keep them checked against the private headers here. */ + ZSTD_STATIC_ASSERT(MaxML == 52 && MaxLL == 35 && MaxOff == 31); + ZSTD_STATIC_ASSERT(Litbits == 8 && ZSTD_OPT_SIZE == 4099); + ZSTD_STATIC_ASSERT(ZSTD_CWKSP_ALIGNMENT_BYTES == 64); /* tables are guaranteed to be sized in multiples of 64 bytes (or 16 uint32_t) */ ZSTD_STATIC_ASSERT(ZSTD_HASHLOG_MIN >= 4 && ZSTD_WINDOWLOG_MIN >= 4 && ZSTD_CHAINLOG_MIN >= 4); assert(useRowMatchFinder != ZSTD_ps_auto); - DEBUGLOG(4, "chainSize: %u - hSize: %u - h3Size: %u", - (U32)chainSize, (U32)hSize, (U32)h3Size); - return tableSpace + optSpace + slackSpace + lazyAdditionalSpace; + return ZSTD_rust_params_estimateMatchStateSize(*cParams, + (int)useRowMatchFinder, + enableDedicatedDictSearch, + forCCtx, &sizing); } /* Helper function for calculating memory requirements. * Gives a tighter bound than ZSTD_sequenceBound() by taking minMatch into account. */ static size_t ZSTD_maxNbSeq(size_t blockSize, unsigned minMatch, int useSequenceProducer) { - U32 const divider = (minMatch==3 || useSequenceProducer) ? 3 : 4; - return blockSize / divider; + return ZSTD_rust_params_maxNbSeq(blockSize, minMatch, useSequenceProducer); } static size_t ZSTD_estimateCCtxSize_usingCCtxParams_internal( @@ -5430,15 +5303,15 @@ size_t ZSTD_estimateCDictSize_advanced( size_t dictSize, ZSTD_compressionParameters cParams, ZSTD_dictLoadMethod_e dictLoadMethod) { - DEBUGLOG(5, "sizeof(ZSTD_CDict) : %u", (unsigned)sizeof(ZSTD_CDict)); - return ZSTD_cwksp_alloc_size(sizeof(ZSTD_CDict)) - + ZSTD_cwksp_alloc_size(HUF_WORKSPACE_SIZE) - /* enableDedicatedDictSearch == 1 ensures that CDict estimation will not be too small - * in case we are using DDS with row-hash. */ - + ZSTD_sizeof_matchState(&cParams, ZSTD_resolveRowMatchFinderMode(ZSTD_ps_auto, &cParams), - /* enableDedicatedDictSearch */ 1, /* forCCtx */ 0) - + (dictLoadMethod == ZSTD_dlm_byRef ? 0 - : ZSTD_cwksp_alloc_size(ZSTD_cwksp_align(dictSize, sizeof(void *)))); + ZSTD_rustCDictSizing sizing; + sizing.cdictSize = sizeof(ZSTD_CDict); + sizing.hufWorkspaceSize = HUF_WORKSPACE_SIZE; + sizing.hashLog3Max = ZSTD_HASHLOG3_MAX; + sizing.matchTSize = sizeof(ZSTD_match_t); + sizing.optimalTSize = sizeof(ZSTD_optimal_t); + sizing.asanRedzoneSize = ZSTD_RUST_ASAN_REDZONE_SIZE; + return ZSTD_rust_params_estimateCDictSizeFromCParams( + dictSize, cParams, (int)dictLoadMethod, &sizing); } size_t ZSTD_estimateCDictSize(size_t dictSize, int compressionLevel) @@ -7576,11 +7449,12 @@ size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output) /*-===== Pre-defined compression levels =====-*/ -#include "clevels.h" +/* The compression-level tables (formerly included from clevels.h) live in + * rust/src/zstd_compress_params.rs. */ -int ZSTD_maxCLevel(void) { return ZSTD_MAX_CLEVEL; } -int ZSTD_minCLevel(void) { return (int)-ZSTD_TARGETLENGTH_MAX; } -int ZSTD_defaultCLevel(void) { return ZSTD_CLEVEL_DEFAULT; } +int ZSTD_maxCLevel(void) { return ZSTD_rust_params_maxCLevel(); } +int ZSTD_minCLevel(void) { return ZSTD_rust_params_minCLevel(); } +int ZSTD_defaultCLevel(void) { return ZSTD_rust_params_defaultCLevel(); } static ZSTD_compressionParameters ZSTD_dedicatedDictSearch_getCParams(int const compressionLevel, size_t const dictSize) { @@ -7639,26 +7513,6 @@ static void ZSTD_dedicatedDictSearch_revertCParams( } } -static U64 ZSTD_getCParamRowSize(U64 srcSizeHint, size_t dictSize, ZSTD_CParamMode_e mode) -{ - switch (mode) { - case ZSTD_cpm_unknown: - case ZSTD_cpm_noAttachDict: - case ZSTD_cpm_createCDict: - break; - case ZSTD_cpm_attachDict: - dictSize = 0; - break; - default: - assert(0); - break; - } - { int const unknown = srcSizeHint == ZSTD_CONTENTSIZE_UNKNOWN; - size_t const addedSize = unknown && dictSize > 0 ? 500 : 0; - return unknown && dictSize == 0 ? ZSTD_CONTENTSIZE_UNKNOWN : srcSizeHint+dictSize+addedSize; - } -} - /*! ZSTD_getCParams_internal() : * @return ZSTD_compressionParameters structure for a selected compression level, srcSize and dictSize. * Note: srcSizeHint 0 means 0, use ZSTD_CONTENTSIZE_UNKNOWN for unknown. @@ -7666,27 +7520,12 @@ static U64 ZSTD_getCParamRowSize(U64 srcSizeHint, size_t dictSize, ZSTD_CParamMo * Note: `mode` controls how we treat the `dictSize`. See docs for `ZSTD_CParamMode_e`. */ static ZSTD_compressionParameters ZSTD_getCParams_internal(int compressionLevel, unsigned long long srcSizeHint, size_t dictSize, ZSTD_CParamMode_e mode) { - U64 const rSize = ZSTD_getCParamRowSize(srcSizeHint, dictSize, mode); - U32 const tableID = (rSize <= 256 KB) + (rSize <= 128 KB) + (rSize <= 16 KB); - int row; - DEBUGLOG(5, "ZSTD_getCParams_internal (cLevel=%i)", compressionLevel); - - /* row */ - if (compressionLevel == 0) row = ZSTD_CLEVEL_DEFAULT; /* 0 == default */ - else if (compressionLevel < 0) row = 0; /* entry 0 is baseline for fast mode */ - else if (compressionLevel > ZSTD_MAX_CLEVEL) row = ZSTD_MAX_CLEVEL; - else row = compressionLevel; - - { ZSTD_compressionParameters cp = ZSTD_defaultCParameters[tableID][row]; - DEBUGLOG(5, "ZSTD_getCParams_internal selected tableID: %u row: %u strat: %u", tableID, row, (U32)cp.strategy); - /* acceleration factor */ - if (compressionLevel < 0) { - int const clampedCompressionLevel = MAX(ZSTD_minCLevel(), compressionLevel); - cp.targetLength = (unsigned)(-clampedCompressionLevel); - } - /* refine parameters based on srcSize & dictSize */ - return ZSTD_adjustCParams_internal(cp, srcSizeHint, dictSize, mode, ZSTD_ps_auto); - } + /* Table selection is context-free and lives in Rust; the adjustment step + * stays behind ZSTD_adjustCParams_internal() so this build's strategy + * cascade applies. */ + ZSTD_compressionParameters const cp = ZSTD_rust_params_selectCParams( + compressionLevel, srcSizeHint, dictSize, (int)mode); + return ZSTD_adjustCParams_internal(cp, srcSizeHint, dictSize, mode, ZSTD_ps_auto); } /*! ZSTD_getCParams() : @@ -7705,13 +7544,9 @@ ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, unsigned long l static ZSTD_parameters ZSTD_getParams_internal(int compressionLevel, unsigned long long srcSizeHint, size_t dictSize, ZSTD_CParamMode_e mode) { - ZSTD_parameters params; ZSTD_compressionParameters const cParams = ZSTD_getCParams_internal(compressionLevel, srcSizeHint, dictSize, mode); DEBUGLOG(5, "ZSTD_getParams (cLevel=%i)", compressionLevel); - ZSTD_memset(¶ms, 0, sizeof(params)); - params.cParams = cParams; - params.fParams.contentSizeFlag = 1; - return params; + return ZSTD_rust_params_makeParams(cParams); } /*! ZSTD_getParams() : diff --git a/rust/README.md b/rust/README.md index 362a9fce6..5eea53a91 100644 --- a/rust/README.md +++ b/rust/README.md @@ -34,6 +34,12 @@ zstd ABI: - `zstd_compress_frame` serializes frame headers, skippable frames, and the last empty block; it takes scalar frame parameters so the C-owned `ZSTD_CCtx_params` layout never crosses the language boundary. + - `zstd_compress_params` owns the compression-level tables (formerly + `clevels.h`), parameter bounds, clamping, validation, table selection, + source/dictionary adjustment, and match-state/CDict size estimation. + The C integration layer keeps the public `ZSTD_*` symbols and feeds the + leaves configuration-owned scalars: the excluded-block-compressor + strategy cascade, struct sizes, and sanitizer redzone policy. - `zstd_fast` and `zstd_double_fast` implement the single- and two-table fast block match finders, including attached and external dictionary paths. - `zstd_lazy` implements greedy, lazy, lazy2, and binary-tree matching, diff --git a/rust/src/lib.rs b/rust/src/lib.rs index c415f1531..a7f47a257 100644 --- a/rust/src/lib.rs +++ b/rust/src/lib.rs @@ -28,6 +28,8 @@ pub mod zstd_compress_frame; #[cfg(feature = "compression")] pub mod zstd_compress_literals; #[cfg(feature = "compression")] +pub mod zstd_compress_params; +#[cfg(feature = "compression")] pub mod zstd_compress_sequences; #[cfg(feature = "compression")] pub mod zstd_compress_superblock; diff --git a/rust/src/zstd_compress_params.rs b/rust/src/zstd_compress_params.rs new file mode 100644 index 000000000..85a75c87f --- /dev/null +++ b/rust/src/zstd_compress_params.rs @@ -0,0 +1,1048 @@ +#![allow(non_camel_case_types)] +#![allow(non_snake_case)] +#![allow(clippy::missing_safety_doc)] +#![allow(clippy::too_many_arguments)] + +//! Context-free compression-parameter selection and sizing leaves. +//! +//! This module deliberately does **not** own the public `ZSTD_*` symbols yet. +//! `zstd_compress.c` still owns configuration-sensitive policy: excluded block +//! compressors, private `ZSTD_CCtx_params` layouts, LDM workspace sizing, and +//! ASAN workspace policy. The C integration layer can select a raw table +//! entry here, apply its configured strategy cascade, then use the adjustment +//! and sizing leaves below. That keeps the Rust implementation independent of +//! C preprocessor state while retaining byte-for-byte C policy for reduced +//! builds. + +use crate::errors::{ZstdErrorCode, ERROR}; +use std::mem::size_of; +use std::os::raw::c_int; + +pub const ZSTD_CONTENTSIZE_UNKNOWN: u64 = u64::MAX; + +const ZSTD_CLEVEL_DEFAULT: c_int = 3; +const ZSTD_MAX_CLEVEL: c_int = 22; +const ZSTD_TARGETLENGTH_MAX: c_int = 1 << 17; +const ZSTD_BLOCKSIZE_MAX: usize = 1 << 17; + +const ZSTD_WINDOWLOG_MIN: c_int = 10; +#[cfg(target_pointer_width = "32")] +const ZSTD_WINDOWLOG_MAX: c_int = 30; +#[cfg(not(target_pointer_width = "32"))] +const ZSTD_WINDOWLOG_MAX: c_int = 31; +const ZSTD_HASHLOG_MIN: c_int = 6; +const ZSTD_HASHLOG_MAX: c_int = 30; +const ZSTD_CHAINLOG_MIN: c_int = ZSTD_HASHLOG_MIN; +#[cfg(target_pointer_width = "32")] +const ZSTD_CHAINLOG_MAX: c_int = 29; +#[cfg(not(target_pointer_width = "32"))] +const ZSTD_CHAINLOG_MAX: c_int = 30; +const ZSTD_SEARCHLOG_MIN: c_int = 1; +const ZSTD_SEARCHLOG_MAX: c_int = ZSTD_WINDOWLOG_MAX - 1; +const ZSTD_MINMATCH_MIN: c_int = 3; +const ZSTD_MINMATCH_MAX: c_int = 7; +const ZSTD_TARGETLENGTH_MIN: c_int = 0; + +const ZSTD_FAST: c_int = 1; +const ZSTD_DFAST: c_int = 2; +const ZSTD_GREEDY: c_int = 3; +const ZSTD_LAZY: c_int = 4; +const ZSTD_LAZY2: c_int = 5; +const ZSTD_BTLAZY2: c_int = 6; +const ZSTD_BTOPT: c_int = 7; +const ZSTD_BTULTRA: c_int = 8; +const ZSTD_BTULTRA2: c_int = 9; + +const ZSTD_C_COMPRESSION_LEVEL: c_int = 100; +const ZSTD_C_WINDOW_LOG: c_int = 101; +const ZSTD_C_HASH_LOG: c_int = 102; +const ZSTD_C_CHAIN_LOG: c_int = 103; +const ZSTD_C_SEARCH_LOG: c_int = 104; +const ZSTD_C_MIN_MATCH: c_int = 105; +const ZSTD_C_TARGET_LENGTH: c_int = 106; +const ZSTD_C_STRATEGY: c_int = 107; + +/// Private compression-parameter modes from `zstd_compress_internal.h`. +/// +/// They are ABI-compatible with `ZSTD_CParamMode_e` and intentionally kept as +/// integer constants because that enum remains private C API for now. +pub const ZSTD_RUST_CPM_NO_ATTACH_DICT: c_int = 0; +pub const ZSTD_RUST_CPM_ATTACH_DICT: c_int = 1; +pub const ZSTD_RUST_CPM_CREATE_CDICT: c_int = 2; +pub const ZSTD_RUST_CPM_UNKNOWN: c_int = 3; + +/// `ZSTD_ParamSwitch_e` values used by the adjustment and sizing leaves. +pub const ZSTD_RUST_PS_AUTO: c_int = 0; +pub const ZSTD_RUST_PS_ENABLE: c_int = 1; +pub const ZSTD_RUST_PS_DISABLE: c_int = 2; + +/// ABI-compatible `ZSTD_compressionParameters` from `zstd.h`. +#[repr(C)] +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct ZSTD_compressionParameters { + pub windowLog: u32, + pub chainLog: u32, + pub hashLog: u32, + pub searchLog: u32, + pub minMatch: u32, + pub targetLength: u32, + pub strategy: c_int, +} + +/// ABI-compatible `ZSTD_frameParameters` from `zstd.h`. +#[repr(C)] +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct ZSTD_frameParameters { + pub contentSizeFlag: c_int, + pub checksumFlag: c_int, + pub noDictIDFlag: c_int, +} + +/// ABI-compatible `ZSTD_parameters` from `zstd.h`. +#[repr(C)] +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct ZSTD_parameters { + pub cParams: ZSTD_compressionParameters, + pub fParams: ZSTD_frameParameters, +} + +/// ABI-compatible `ZSTD_bounds` from `zstd.h`. +#[repr(C)] +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct ZSTD_bounds { + pub error: usize, + pub lowerBound: c_int, + pub upperBound: c_int, +} + +/* `clevels.h`, represented as raw fields so the table stays compact and easy + * to compare mechanically against its C source. Field order is W, C, H, S, + * L, TL, strategy. */ +const DEFAULT_CPARAMS: [[[u32; 7]; 23]; 4] = [ + [ + [19, 12, 13, 1, 6, 1, ZSTD_FAST as u32], + [19, 13, 14, 1, 7, 0, ZSTD_FAST as u32], + [20, 15, 16, 1, 6, 0, ZSTD_FAST as u32], + [21, 16, 17, 1, 5, 0, ZSTD_DFAST as u32], + [21, 18, 18, 1, 5, 0, ZSTD_DFAST as u32], + [21, 18, 19, 3, 5, 2, ZSTD_GREEDY as u32], + [21, 18, 19, 3, 5, 4, ZSTD_LAZY as u32], + [21, 19, 20, 4, 5, 8, ZSTD_LAZY as u32], + [21, 19, 20, 4, 5, 16, ZSTD_LAZY2 as u32], + [22, 20, 21, 4, 5, 16, ZSTD_LAZY2 as u32], + [22, 21, 22, 5, 5, 16, ZSTD_LAZY2 as u32], + [22, 21, 22, 6, 5, 16, ZSTD_LAZY2 as u32], + [22, 22, 23, 6, 5, 32, ZSTD_LAZY2 as u32], + [22, 22, 22, 4, 5, 32, ZSTD_BTLAZY2 as u32], + [22, 22, 23, 5, 5, 32, ZSTD_BTLAZY2 as u32], + [22, 23, 23, 6, 5, 32, ZSTD_BTLAZY2 as u32], + [22, 22, 22, 5, 5, 48, ZSTD_BTOPT as u32], + [23, 23, 22, 5, 4, 64, ZSTD_BTOPT as u32], + [23, 23, 22, 6, 3, 64, ZSTD_BTULTRA as u32], + [23, 24, 22, 7, 3, 256, ZSTD_BTULTRA2 as u32], + [25, 25, 23, 7, 3, 256, ZSTD_BTULTRA2 as u32], + [26, 26, 24, 7, 3, 512, ZSTD_BTULTRA2 as u32], + [27, 27, 25, 9, 3, 999, ZSTD_BTULTRA2 as u32], + ], + [ + [18, 12, 13, 1, 5, 1, ZSTD_FAST as u32], + [18, 13, 14, 1, 6, 0, ZSTD_FAST as u32], + [18, 14, 14, 1, 5, 0, ZSTD_DFAST as u32], + [18, 16, 16, 1, 4, 0, ZSTD_DFAST as u32], + [18, 16, 17, 3, 5, 2, ZSTD_GREEDY as u32], + [18, 17, 18, 5, 5, 2, ZSTD_GREEDY as u32], + [18, 18, 19, 3, 5, 4, ZSTD_LAZY as u32], + [18, 18, 19, 4, 4, 4, ZSTD_LAZY as u32], + [18, 18, 19, 4, 4, 8, ZSTD_LAZY2 as u32], + [18, 18, 19, 5, 4, 8, ZSTD_LAZY2 as u32], + [18, 18, 19, 6, 4, 8, ZSTD_LAZY2 as u32], + [18, 18, 19, 5, 4, 12, ZSTD_BTLAZY2 as u32], + [18, 19, 19, 7, 4, 12, ZSTD_BTLAZY2 as u32], + [18, 18, 19, 4, 4, 16, ZSTD_BTOPT as u32], + [18, 18, 19, 4, 3, 32, ZSTD_BTOPT as u32], + [18, 18, 19, 6, 3, 128, ZSTD_BTOPT as u32], + [18, 19, 19, 6, 3, 128, ZSTD_BTULTRA as u32], + [18, 19, 19, 8, 3, 256, ZSTD_BTULTRA as u32], + [18, 19, 19, 6, 3, 128, ZSTD_BTULTRA2 as u32], + [18, 19, 19, 8, 3, 256, ZSTD_BTULTRA2 as u32], + [18, 19, 19, 10, 3, 512, ZSTD_BTULTRA2 as u32], + [18, 19, 19, 12, 3, 512, ZSTD_BTULTRA2 as u32], + [18, 19, 19, 13, 3, 999, ZSTD_BTULTRA2 as u32], + ], + [ + [17, 12, 12, 1, 5, 1, ZSTD_FAST as u32], + [17, 12, 13, 1, 6, 0, ZSTD_FAST as u32], + [17, 13, 15, 1, 5, 0, ZSTD_FAST as u32], + [17, 15, 16, 2, 5, 0, ZSTD_DFAST as u32], + [17, 17, 17, 2, 4, 0, ZSTD_DFAST as u32], + [17, 16, 17, 3, 4, 2, ZSTD_GREEDY as u32], + [17, 16, 17, 3, 4, 4, ZSTD_LAZY as u32], + [17, 16, 17, 3, 4, 8, ZSTD_LAZY2 as u32], + [17, 16, 17, 4, 4, 8, ZSTD_LAZY2 as u32], + [17, 16, 17, 5, 4, 8, ZSTD_LAZY2 as u32], + [17, 16, 17, 6, 4, 8, ZSTD_LAZY2 as u32], + [17, 17, 17, 5, 4, 8, ZSTD_BTLAZY2 as u32], + [17, 18, 17, 7, 4, 12, ZSTD_BTLAZY2 as u32], + [17, 18, 17, 3, 4, 12, ZSTD_BTOPT as u32], + [17, 18, 17, 4, 3, 32, ZSTD_BTOPT as u32], + [17, 18, 17, 6, 3, 256, ZSTD_BTOPT as u32], + [17, 18, 17, 6, 3, 128, ZSTD_BTULTRA as u32], + [17, 18, 17, 8, 3, 256, ZSTD_BTULTRA as u32], + [17, 18, 17, 10, 3, 512, ZSTD_BTULTRA as u32], + [17, 18, 17, 5, 3, 256, ZSTD_BTULTRA2 as u32], + [17, 18, 17, 7, 3, 512, ZSTD_BTULTRA2 as u32], + [17, 18, 17, 9, 3, 512, ZSTD_BTULTRA2 as u32], + [17, 18, 17, 11, 3, 999, ZSTD_BTULTRA2 as u32], + ], + [ + [14, 12, 13, 1, 5, 1, ZSTD_FAST as u32], + [14, 14, 15, 1, 5, 0, ZSTD_FAST as u32], + [14, 14, 15, 1, 4, 0, ZSTD_FAST as u32], + [14, 14, 15, 2, 4, 0, ZSTD_DFAST as u32], + [14, 14, 14, 4, 4, 2, ZSTD_GREEDY as u32], + [14, 14, 14, 3, 4, 4, ZSTD_LAZY as u32], + [14, 14, 14, 4, 4, 8, ZSTD_LAZY2 as u32], + [14, 14, 14, 6, 4, 8, ZSTD_LAZY2 as u32], + [14, 14, 14, 8, 4, 8, ZSTD_LAZY2 as u32], + [14, 15, 14, 5, 4, 8, ZSTD_BTLAZY2 as u32], + [14, 15, 14, 9, 4, 8, ZSTD_BTLAZY2 as u32], + [14, 15, 14, 3, 4, 12, ZSTD_BTOPT as u32], + [14, 15, 14, 4, 3, 24, ZSTD_BTOPT as u32], + [14, 15, 14, 5, 3, 32, ZSTD_BTULTRA as u32], + [14, 15, 15, 6, 3, 64, ZSTD_BTULTRA as u32], + [14, 15, 15, 7, 3, 256, ZSTD_BTULTRA as u32], + [14, 15, 15, 5, 3, 48, ZSTD_BTULTRA2 as u32], + [14, 15, 15, 6, 3, 128, ZSTD_BTULTRA2 as u32], + [14, 15, 15, 7, 3, 256, ZSTD_BTULTRA2 as u32], + [14, 15, 15, 8, 3, 256, ZSTD_BTULTRA2 as u32], + [14, 15, 15, 8, 3, 512, ZSTD_BTULTRA2 as u32], + [14, 15, 15, 9, 3, 512, ZSTD_BTULTRA2 as u32], + [14, 15, 15, 10, 3, 999, ZSTD_BTULTRA2 as u32], + ], +]; + +#[inline] +fn cparams_from_row(row: [u32; 7]) -> ZSTD_compressionParameters { + ZSTD_compressionParameters { + windowLog: row[0], + chainLog: row[1], + hashLog: row[2], + searchLog: row[3], + minMatch: row[4], + targetLength: row[5], + strategy: row[6] as c_int, + } +} + +#[inline] +fn bounds(param: c_int) -> ZSTD_bounds { + let (lowerBound, upperBound) = match param { + ZSTD_C_COMPRESSION_LEVEL => (ZSTD_rust_params_minCLevel(), ZSTD_rust_params_maxCLevel()), + ZSTD_C_WINDOW_LOG => (ZSTD_WINDOWLOG_MIN, ZSTD_WINDOWLOG_MAX), + ZSTD_C_HASH_LOG => (ZSTD_HASHLOG_MIN, ZSTD_HASHLOG_MAX), + ZSTD_C_CHAIN_LOG => (ZSTD_CHAINLOG_MIN, ZSTD_CHAINLOG_MAX), + ZSTD_C_SEARCH_LOG => (ZSTD_SEARCHLOG_MIN, ZSTD_SEARCHLOG_MAX), + ZSTD_C_MIN_MATCH => (ZSTD_MINMATCH_MIN, ZSTD_MINMATCH_MAX), + ZSTD_C_TARGET_LENGTH => (ZSTD_TARGETLENGTH_MIN, ZSTD_TARGETLENGTH_MAX), + ZSTD_C_STRATEGY => (ZSTD_FAST, ZSTD_BTULTRA2), + _ => { + return ZSTD_bounds { + error: ERROR(ZstdErrorCode::ParameterUnsupported), + lowerBound: 0, + upperBound: 0, + } + } + }; + ZSTD_bounds { + error: 0, + lowerBound, + upperBound, + } +} + +#[inline] +fn within_bounds(param: c_int, value: c_int) -> bool { + let bounds = bounds(param); + bounds.error == 0 && value >= bounds.lowerBound && value <= bounds.upperBound +} + +#[inline] +fn clamp_unsigned(value: u32, param: c_int) -> u32 { + let bounds = bounds(param); + debug_assert_eq!(bounds.error, 0); + let signed = value as c_int; + if signed < bounds.lowerBound { + bounds.lowerBound as u32 + } else if signed > bounds.upperBound { + bounds.upperBound as u32 + } else { + value + } +} + +#[inline] +fn clamp_cparams(mut cparams: ZSTD_compressionParameters) -> ZSTD_compressionParameters { + cparams.windowLog = clamp_unsigned(cparams.windowLog, ZSTD_C_WINDOW_LOG); + cparams.chainLog = clamp_unsigned(cparams.chainLog, ZSTD_C_CHAIN_LOG); + cparams.hashLog = clamp_unsigned(cparams.hashLog, ZSTD_C_HASH_LOG); + cparams.searchLog = clamp_unsigned(cparams.searchLog, ZSTD_C_SEARCH_LOG); + cparams.minMatch = clamp_unsigned(cparams.minMatch, ZSTD_C_MIN_MATCH); + cparams.targetLength = clamp_unsigned(cparams.targetLength, ZSTD_C_TARGET_LENGTH); + let strategy_bounds = bounds(ZSTD_C_STRATEGY); + if cparams.strategy < strategy_bounds.lowerBound { + cparams.strategy = strategy_bounds.lowerBound; + } else if cparams.strategy > strategy_bounds.upperBound { + cparams.strategy = strategy_bounds.upperBound; + } + cparams +} + +#[inline] +fn highbit32(value: u32) -> u32 { + debug_assert_ne!(value, 0); + u32::BITS - 1 - value.leading_zeros() +} + +#[inline] +fn cycle_log(hash_log: u32, strategy: c_int) -> u32 { + hash_log.wrapping_sub((strategy >= ZSTD_BTLAZY2) as u32) +} + +#[inline] +fn strategy_supports_row_match_finder(strategy: c_int) -> bool { + (ZSTD_GREEDY..=ZSTD_LAZY2).contains(&strategy) +} + +#[inline] +fn row_match_finder_used(strategy: c_int, mode: c_int) -> bool { + debug_assert_ne!(mode, ZSTD_RUST_PS_AUTO); + strategy_supports_row_match_finder(strategy) && mode == ZSTD_RUST_PS_ENABLE +} + +#[inline] +fn resolve_row_match_finder(mode: c_int, cparams: ZSTD_compressionParameters) -> c_int { + if mode != ZSTD_RUST_PS_AUTO { + return mode; + } + if strategy_supports_row_match_finder(cparams.strategy) && cparams.windowLog > 14 { + ZSTD_RUST_PS_ENABLE + } else { + ZSTD_RUST_PS_DISABLE + } +} + +#[inline] +fn dict_and_window_log(window_log: u32, src_size: u64, dict_size: u64) -> u32 { + /* 1ULL << ZSTD_WINDOWLOG_MAX, which is smaller for 32-bit builds. */ + const MAX_WINDOW_SIZE: u64 = 1u64 << ZSTD_WINDOWLOG_MAX; + if dict_size == 0 { + return window_log; + } + debug_assert!(window_log <= ZSTD_WINDOWLOG_MAX as u32); + debug_assert_ne!(src_size, ZSTD_CONTENTSIZE_UNKNOWN); + let window_size = 1u64 << window_log; + if window_size >= dict_size.wrapping_add(src_size) { + window_log + } else { + let dict_and_window_size = dict_size.wrapping_add(window_size); + if dict_and_window_size >= MAX_WINDOW_SIZE { + ZSTD_WINDOWLOG_MAX as u32 + } else { + highbit32((dict_and_window_size as u32).wrapping_sub(1)) + 1 + } + } +} + +/// Adjusts a *validated, configuration-resolved* parameter set. +/// +/// This is the direct Rust leaf for `ZSTD_adjustCParams_internal()` after the +/// C wrapper has applied its `ZSTD_EXCLUDE_*_BLOCK_COMPRESSOR` cascade. In +/// particular, it does not select a fallback strategy itself. `srcSize == 0` +/// means a known empty input here, just as it does in the C internal helper; +/// the public C wrapper must translate zero to `ZSTD_CONTENTSIZE_UNKNOWN`. +fn adjust_cparams( + mut cparams: ZSTD_compressionParameters, + mut src_size: u64, + mut dict_size: usize, + mode: c_int, + mut use_row_match_finder: c_int, +) -> ZSTD_compressionParameters { + debug_assert_eq!(check_cparams(cparams), 0); + + const MIN_SRC_SIZE: u64 = 513; + /* 1ULL << (ZSTD_WINDOWLOG_MAX - 1), which is smaller for 32-bit builds. */ + const MAX_WINDOW_RESIZE: u64 = 1u64 << (ZSTD_WINDOWLOG_MAX - 1); + + match mode { + ZSTD_RUST_CPM_UNKNOWN | ZSTD_RUST_CPM_NO_ATTACH_DICT => {} + ZSTD_RUST_CPM_CREATE_CDICT => { + if dict_size != 0 && src_size == ZSTD_CONTENTSIZE_UNKNOWN { + src_size = MIN_SRC_SIZE; + } + } + ZSTD_RUST_CPM_ATTACH_DICT => dict_size = 0, + _ => debug_assert!(false, "invalid ZSTD_CParamMode_e"), + } + + if src_size <= MAX_WINDOW_RESIZE && (dict_size as u64) <= MAX_WINDOW_RESIZE { + let total_size = src_size.wrapping_add(dict_size as u64) as u32; + let hash_size_min = 1u32 << ZSTD_HASHLOG_MIN; + let src_log = if total_size < hash_size_min { + ZSTD_HASHLOG_MIN as u32 + } else { + highbit32(total_size.wrapping_sub(1)) + 1 + }; + cparams.windowLog = cparams.windowLog.min(src_log); + } + + if src_size != ZSTD_CONTENTSIZE_UNKNOWN { + let dict_and_window_log = + dict_and_window_log(cparams.windowLog, src_size, dict_size as u64); + let cycle_log = cycle_log(cparams.chainLog, cparams.strategy); + cparams.hashLog = cparams.hashLog.min(dict_and_window_log + 1); + if cycle_log > dict_and_window_log { + cparams.chainLog = cparams + .chainLog + .wrapping_sub(cycle_log - dict_and_window_log); + } + } + + cparams.windowLog = cparams.windowLog.max(ZSTD_WINDOWLOG_MIN as u32); + + /* The short-cache tags used for fast and dfast CDicts consume eight bits + * of each 32-bit index. This is a fixed private-header constant today; + * if it becomes configurable, the C shim must pass it as an explicit + * adjustment input rather than silently changing this leaf. */ + if mode == ZSTD_RUST_CPM_CREATE_CDICT + && (cparams.strategy == ZSTD_FAST || cparams.strategy == ZSTD_DFAST) + { + const SHORT_CACHE_TAG_BITS: u32 = 8; + let max_short_cache_hash_log = 32 - SHORT_CACHE_TAG_BITS; + cparams.hashLog = cparams.hashLog.min(max_short_cache_hash_log); + cparams.chainLog = cparams.chainLog.min(max_short_cache_hash_log); + } + + if use_row_match_finder == ZSTD_RUST_PS_AUTO { + use_row_match_finder = ZSTD_RUST_PS_ENABLE; + } + if row_match_finder_used(cparams.strategy, use_row_match_finder) { + const ROW_HASH_TAG_BITS: u32 = 8; + let row_log = cparams.searchLog.clamp(4, 6); + let max_hash_log = (32 - ROW_HASH_TAG_BITS) + row_log; + debug_assert!(cparams.hashLog >= row_log); + cparams.hashLog = cparams.hashLog.min(max_hash_log); + } + + cparams +} + +#[inline] +fn get_cparam_row_size(src_size_hint: u64, dict_size: usize, mode: c_int) -> u64 { + let mut dict_size = dict_size as u64; + match mode { + ZSTD_RUST_CPM_UNKNOWN | ZSTD_RUST_CPM_NO_ATTACH_DICT | ZSTD_RUST_CPM_CREATE_CDICT => {} + ZSTD_RUST_CPM_ATTACH_DICT => dict_size = 0, + _ => debug_assert!(false, "invalid ZSTD_CParamMode_e"), + } + let unknown = src_size_hint == ZSTD_CONTENTSIZE_UNKNOWN; + let added_size = if unknown && dict_size > 0 { 500 } else { 0 }; + if unknown && dict_size == 0 { + ZSTD_CONTENTSIZE_UNKNOWN + } else { + src_size_hint + .wrapping_add(dict_size) + .wrapping_add(added_size) + } +} + +/// Selects a raw compression-level-table entry, without strategy cascading or +/// source/dictionary adjustment. +/// +/// A C wrapper must apply its active excluded-compressor cascade to the +/// returned strategy before calling [`ZSTD_rust_params_adjustCParams`]. +fn select_cparams( + compression_level: c_int, + src_size_hint: u64, + dict_size: usize, + mode: c_int, +) -> ZSTD_compressionParameters { + let row_size = get_cparam_row_size(src_size_hint, dict_size, mode); + let table_id = usize::from(row_size <= 256 * 1024) + + usize::from(row_size <= 128 * 1024) + + usize::from(row_size <= 16 * 1024); + let row = if compression_level == 0 { + ZSTD_CLEVEL_DEFAULT + } else if compression_level < 0 { + 0 + } else { + compression_level.min(ZSTD_MAX_CLEVEL) + } as usize; + + let mut cparams = cparams_from_row(DEFAULT_CPARAMS[table_id][row]); + if compression_level < 0 { + let clamped = compression_level.max(ZSTD_rust_params_minCLevel()); + cparams.targetLength = (-clamped) as u32; + } + cparams +} + +#[inline] +fn make_params(cparams: ZSTD_compressionParameters) -> ZSTD_parameters { + ZSTD_parameters { + cParams: cparams, + fParams: ZSTD_frameParameters { + contentSizeFlag: 1, + checksumFlag: 0, + noDictIDFlag: 0, + }, + } +} + +#[inline] +fn check_cparams(cparams: ZSTD_compressionParameters) -> usize { + if !within_bounds(ZSTD_C_WINDOW_LOG, cparams.windowLog as c_int) + || !within_bounds(ZSTD_C_CHAIN_LOG, cparams.chainLog as c_int) + || !within_bounds(ZSTD_C_HASH_LOG, cparams.hashLog as c_int) + || !within_bounds(ZSTD_C_SEARCH_LOG, cparams.searchLog as c_int) + || !within_bounds(ZSTD_C_MIN_MATCH, cparams.minMatch as c_int) + || !within_bounds(ZSTD_C_TARGET_LENGTH, cparams.targetLength as c_int) + || !within_bounds(ZSTD_C_STRATEGY, cparams.strategy) + { + ERROR(ZstdErrorCode::ParameterOutOfBound) + } else { + 0 + } +} + +/// Returns the highest table-backed compression level (`ZSTD_MAX_CLEVEL`). +#[no_mangle] +pub extern "C" fn ZSTD_rust_params_maxCLevel() -> c_int { + ZSTD_MAX_CLEVEL +} + +/// Returns the lowest public fast level (`-ZSTD_TARGETLENGTH_MAX`). +#[no_mangle] +pub extern "C" fn ZSTD_rust_params_minCLevel() -> c_int { + -ZSTD_TARGETLENGTH_MAX +} + +/// Returns the default compression level from `zstd.h`. +#[no_mangle] +pub extern "C" fn ZSTD_rust_params_defaultCLevel() -> c_int { + ZSTD_CLEVEL_DEFAULT +} + +/// Returns bounds for the seven core compression parameters and compression +/// level. Other `ZSTD_cParameter` cases remain C-owned. +#[no_mangle] +pub extern "C" fn ZSTD_rust_params_getBounds(param: c_int) -> ZSTD_bounds { + bounds(param) +} + +/// Checks the seven fields of `ZSTD_compressionParameters`. +#[no_mangle] +pub extern "C" fn ZSTD_rust_params_checkCParams(cparams: ZSTD_compressionParameters) -> usize { + check_cparams(cparams) +} + +/// Clamps the seven fields of `ZSTD_compressionParameters` to public bounds. +#[no_mangle] +pub extern "C" fn ZSTD_rust_params_clampCParams( + cparams: ZSTD_compressionParameters, +) -> ZSTD_compressionParameters { + clamp_cparams(cparams) +} + +/// C ABI for `ZSTD_cycleLog()`. +#[no_mangle] +pub extern "C" fn ZSTD_rust_params_cycleLog(hashLog: u32, strategy: c_int) -> u32 { + cycle_log(hashLog, strategy) +} + +/// C ABI for private `ZSTD_getCParamRowSize()`. +#[no_mangle] +pub extern "C" fn ZSTD_rust_params_getCParamRowSize( + srcSizeHint: u64, + dictSize: usize, + mode: c_int, +) -> u64 { + get_cparam_row_size(srcSizeHint, dictSize, mode) +} + +/// Returns the unadjusted compression-level-table entry. +#[no_mangle] +pub extern "C" fn ZSTD_rust_params_selectCParams( + compressionLevel: c_int, + srcSizeHint: u64, + dictSize: usize, + mode: c_int, +) -> ZSTD_compressionParameters { + select_cparams(compressionLevel, srcSizeHint, dictSize, mode) +} + +/// Adjusts a validated, strategy-resolved C parameter set. +/// +/// See [`adjust_cparams`] for the required C-side policy step. +#[no_mangle] +pub extern "C" fn ZSTD_rust_params_adjustCParams( + cparams: ZSTD_compressionParameters, + srcSize: u64, + dictSize: usize, + mode: c_int, + useRowMatchFinder: c_int, +) -> ZSTD_compressionParameters { + adjust_cparams(cparams, srcSize, dictSize, mode, useRowMatchFinder) +} + +/// Builds `ZSTD_parameters` with the public default frame parameters. +#[no_mangle] +pub extern "C" fn ZSTD_rust_params_makeParams( + cparams: ZSTD_compressionParameters, +) -> ZSTD_parameters { + make_params(cparams) +} + +/// Runtime sizes needed to reproduce `ZSTD_sizeof_matchState()` without +/// exposing private C structures to Rust. +/// +/// `asanRedzoneSize` is `ZSTD_CWKSP_ASAN_REDZONE_SIZE` when workspace +/// poisoning is enabled and zero otherwise. `matchTSize` and `optimalTSize` +/// must be `sizeof(ZSTD_match_t)` and `sizeof(ZSTD_optimal_t)` respectively. +#[repr(C)] +#[derive(Clone, Copy, Debug, Default)] +pub struct ZSTD_rustMatchStateSizing { + pub hashLog3Max: u32, + pub matchTSize: usize, + pub optimalTSize: usize, + pub asanRedzoneSize: usize, +} + +/// C-only layout inputs for `ZSTD_estimateCDictSize_advanced()`. +#[repr(C)] +#[derive(Clone, Copy, Debug, Default)] +pub struct ZSTD_rustCDictSizing { + pub cdictSize: usize, + pub hufWorkspaceSize: usize, + pub hashLog3Max: u32, + pub matchTSize: usize, + pub optimalTSize: usize, + pub asanRedzoneSize: usize, +} + +#[inline] +fn cwksp_alloc_size(size: usize, asan_redzone_size: usize) -> usize { + if size == 0 { + 0 + } else { + size.wrapping_add(asan_redzone_size.wrapping_mul(2)) + } +} + +#[inline] +fn cwksp_align(size: usize, alignment: usize) -> usize { + debug_assert!(alignment.is_power_of_two()); + size.wrapping_add(alignment - 1) & !(alignment - 1) +} + +#[inline] +fn cwksp_aligned64_alloc_size(size: usize, asan_redzone_size: usize) -> usize { + cwksp_alloc_size(cwksp_align(size, 64), asan_redzone_size) +} + +#[inline] +fn shift_size(log: u32) -> usize { + debug_assert!(log < usize::BITS); + 1usize << log +} + +#[inline] +fn allocate_chain_table(strategy: c_int, use_row_match_finder: c_int, for_dds_dict: bool) -> bool { + for_dds_dict + || (strategy != ZSTD_FAST && !row_match_finder_used(strategy, use_row_match_finder)) +} + +fn estimate_match_state_size( + cparams: ZSTD_compressionParameters, + use_row_match_finder: c_int, + enable_dedicated_dict_search: bool, + for_cctx: bool, + sizing: ZSTD_rustMatchStateSizing, +) -> usize { + if check_cparams(cparams) != 0 || use_row_match_finder == ZSTD_RUST_PS_AUTO { + return 0; + } + + let chain_size = if allocate_chain_table( + cparams.strategy, + use_row_match_finder, + enable_dedicated_dict_search && !for_cctx, + ) { + shift_size(cparams.chainLog) + } else { + 0 + }; + let hash_size = shift_size(cparams.hashLog); + let hash_log_3 = if for_cctx && cparams.minMatch == 3 { + cparams.windowLog.min(sizing.hashLog3Max) + } else { + 0 + }; + let hash3_size = if hash_log_3 == 0 { + 0 + } else { + shift_size(hash_log_3) + }; + let table_space = chain_size + .wrapping_mul(size_of::()) + .wrapping_add(hash_size.wrapping_mul(size_of::())) + .wrapping_add(hash3_size.wrapping_mul(size_of::())); + + let redzone = sizing.asanRedzoneSize; + let opt_potential_space = cwksp_aligned64_alloc_size((52 + 1) * size_of::(), redzone) + .wrapping_add(cwksp_aligned64_alloc_size( + (35 + 1) * size_of::(), + redzone, + )) + .wrapping_add(cwksp_aligned64_alloc_size( + (31 + 1) * size_of::(), + redzone, + )) + .wrapping_add(cwksp_aligned64_alloc_size( + (1 << 8) * size_of::(), + redzone, + )) + .wrapping_add(cwksp_aligned64_alloc_size( + 4099usize.wrapping_mul(sizing.matchTSize), + redzone, + )) + .wrapping_add(cwksp_aligned64_alloc_size( + 4099usize.wrapping_mul(sizing.optimalTSize), + redzone, + )); + let lazy_additional_space = if row_match_finder_used(cparams.strategy, use_row_match_finder) { + cwksp_aligned64_alloc_size(hash_size, redzone) + } else { + 0 + }; + let opt_space = if for_cctx && cparams.strategy >= ZSTD_BTOPT { + opt_potential_space + } else { + 0 + }; + let slack_space = 2 * 64; + + table_space + .wrapping_add(opt_space) + .wrapping_add(slack_space) + .wrapping_add(lazy_additional_space) +} + +/// Pure leaf for private `ZSTD_sizeof_matchState()`. +/// +/// `useRowMatchFinder` must already be resolved to enable or disable. A NULL +/// `sizing` pointer or invalid C parameters returns zero; C never supplies +/// either in a valid estimator call. +#[no_mangle] +pub unsafe extern "C" fn ZSTD_rust_params_estimateMatchStateSize( + cparams: ZSTD_compressionParameters, + useRowMatchFinder: c_int, + enableDedicatedDictSearch: c_int, + forCCtx: u32, + sizing: *const ZSTD_rustMatchStateSizing, +) -> usize { + if sizing.is_null() { + return 0; + } + let sizing = unsafe { *sizing }; + estimate_match_state_size( + cparams, + useRowMatchFinder, + enableDedicatedDictSearch != 0, + forCCtx != 0, + sizing, + ) +} + +#[inline] +fn max_nb_seq(block_size: usize, min_match: u32, use_sequence_producer: bool) -> usize { + let divider = if min_match == 3 || use_sequence_producer { + 3 + } else { + 4 + }; + block_size / divider +} + +/// Pure leaf for private `ZSTD_maxNbSeq()`. +#[no_mangle] +pub extern "C" fn ZSTD_rust_params_maxNbSeq( + blockSize: usize, + minMatch: u32, + useSequenceProducer: c_int, +) -> usize { + max_nb_seq(blockSize, minMatch, useSequenceProducer != 0) +} + +/// Pure leaf for private `ZSTD_resolveMaxBlockSize()`. +#[no_mangle] +pub extern "C" fn ZSTD_rust_params_resolveMaxBlockSize(maxBlockSize: usize) -> usize { + if maxBlockSize == 0 { + ZSTD_BLOCKSIZE_MAX + } else { + maxBlockSize + } +} + +fn estimate_cdict_size_from_cparams( + dict_size: usize, + cparams: ZSTD_compressionParameters, + dict_load_method: c_int, + sizing: ZSTD_rustCDictSizing, +) -> usize { + if check_cparams(cparams) != 0 { + return 0; + } + let match_state_sizing = ZSTD_rustMatchStateSizing { + hashLog3Max: sizing.hashLog3Max, + matchTSize: sizing.matchTSize, + optimalTSize: sizing.optimalTSize, + asanRedzoneSize: sizing.asanRedzoneSize, + }; + let row_match_finder = resolve_row_match_finder(ZSTD_RUST_PS_AUTO, cparams); + let copied_dict_space = if dict_load_method == 1 { + 0 + } else { + cwksp_alloc_size( + cwksp_align(dict_size, size_of::()), + sizing.asanRedzoneSize, + ) + }; + cwksp_alloc_size(sizing.cdictSize, sizing.asanRedzoneSize) + .wrapping_add(cwksp_alloc_size( + sizing.hufWorkspaceSize, + sizing.asanRedzoneSize, + )) + .wrapping_add(estimate_match_state_size( + cparams, + row_match_finder, + true, + false, + match_state_sizing, + )) + .wrapping_add(copied_dict_space) +} + +/// Parameter-only leaf for `ZSTD_estimateCDictSize_advanced()`. +/// +/// C retains ownership of `sizeof(ZSTD_CDict)`, HUF workspace configuration, +/// and sanitizer workspace policy, then passes them in `sizing`. A NULL sizing +/// pointer returns zero. +#[no_mangle] +pub unsafe extern "C" fn ZSTD_rust_params_estimateCDictSizeFromCParams( + dictSize: usize, + cparams: ZSTD_compressionParameters, + dictLoadMethod: c_int, + sizing: *const ZSTD_rustCDictSizing, +) -> usize { + if sizing.is_null() { + return 0; + } + estimate_cdict_size_from_cparams(dictSize, cparams, dictLoadMethod, unsafe { *sizing }) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::errors::ERR_isError; + use std::mem::{align_of, size_of}; + + fn adjust_public( + cparams: ZSTD_compressionParameters, + src_size: u64, + dict_size: usize, + ) -> ZSTD_compressionParameters { + let src_size = if src_size == 0 { + ZSTD_CONTENTSIZE_UNKNOWN + } else { + src_size + }; + adjust_cparams( + clamp_cparams(cparams), + src_size, + dict_size, + ZSTD_RUST_CPM_UNKNOWN, + ZSTD_RUST_PS_AUTO, + ) + } + + #[test] + fn abi_parameter_layouts_match_zstd_h() { + assert_eq!( + size_of::(), + 7 * size_of::() + ); + assert_eq!(align_of::(), align_of::()); + assert_eq!(size_of::(), 3 * size_of::()); + assert_eq!(size_of::(), 10 * size_of::()); + assert_eq!( + size_of::(), + size_of::() + 2 * size_of::() + ); + } + + #[test] + fn level_tables_match_representative_clevels_entries() { + let large = select_cparams(3, ZSTD_CONTENTSIZE_UNKNOWN, 0, ZSTD_RUST_CPM_UNKNOWN); + assert_eq!( + large, + ZSTD_compressionParameters { + windowLog: 21, + chainLog: 16, + hashLog: 17, + searchLog: 1, + minMatch: 5, + targetLength: 0, + strategy: ZSTD_DFAST, + } + ); + + let small = select_cparams(3, 16 * 1024, 0, ZSTD_RUST_CPM_UNKNOWN); + assert_eq!( + small, + ZSTD_compressionParameters { + windowLog: 14, + chainLog: 14, + hashLog: 15, + searchLog: 2, + minMatch: 4, + targetLength: 0, + strategy: ZSTD_DFAST, + } + ); + + let fast = select_cparams(-5, ZSTD_CONTENTSIZE_UNKNOWN, 0, ZSTD_RUST_CPM_UNKNOWN); + assert_eq!(fast.strategy, ZSTD_FAST); + assert_eq!(fast.targetLength, 5); + assert_eq!( + select_cparams(999, ZSTD_CONTENTSIZE_UNKNOWN, 0, ZSTD_RUST_CPM_UNKNOWN), + select_cparams(22, ZSTD_CONTENTSIZE_UNKNOWN, 0, ZSTD_RUST_CPM_UNKNOWN) + ); + } + + #[test] + fn adjustment_clamps_and_downsizes_like_the_c_leaf() { + let input = ZSTD_compressionParameters { + windowLog: 31, + chainLog: 30, + hashLog: 30, + searchLog: 1, + minMatch: 3, + targetLength: 0, + strategy: ZSTD_BTOPT, + }; + let adjusted = adjust_public(input, 1, 0); + assert_eq!(adjusted.windowLog, 10); + /* C applies WINDOWLOG_ABSOLUTEMIN only after it has used the smaller + * temporary window to downsize the hash and chain logs. */ + assert_eq!(adjusted.hashLog, 7); + assert_eq!(adjusted.chainLog, 7); + assert_eq!(ZSTD_rust_params_checkCParams(adjusted), 0); + + let clamped = ZSTD_rust_params_clampCParams(ZSTD_compressionParameters { + windowLog: u32::MAX, + chainLog: 0, + hashLog: 0, + searchLog: 0, + minMatch: 0, + targetLength: u32::MAX, + strategy: 99, + }); + assert_eq!(clamped.windowLog, ZSTD_WINDOWLOG_MIN as u32); + assert_eq!(clamped.chainLog, ZSTD_CHAINLOG_MIN as u32); + assert_eq!(clamped.hashLog, ZSTD_HASHLOG_MIN as u32); + assert_eq!(clamped.searchLog, ZSTD_SEARCHLOG_MIN as u32); + assert_eq!(clamped.minMatch, ZSTD_MINMATCH_MIN as u32); + assert_eq!(clamped.targetLength, ZSTD_TARGETLENGTH_MIN as u32); + assert_eq!(clamped.strategy, ZSTD_BTULTRA2); + } + + #[test] + fn checking_and_bounds_preserve_the_public_error_contract() { + let valid = select_cparams(1, ZSTD_CONTENTSIZE_UNKNOWN, 0, ZSTD_RUST_CPM_UNKNOWN); + assert_eq!(check_cparams(valid), 0); + let mut invalid = valid; + invalid.minMatch = 2; + assert!(ERR_isError(check_cparams(invalid))); + let bounds = ZSTD_rust_params_getBounds(ZSTD_C_HASH_LOG); + assert_eq!(bounds.error, 0); + assert_eq!(bounds.lowerBound, 6); + assert_eq!(bounds.upperBound, 30); + assert!(ERR_isError(ZSTD_rust_params_getBounds(-1).error)); + } + + #[test] + fn params_default_frame_flags_match_c() { + let cparams = select_cparams(5, ZSTD_CONTENTSIZE_UNKNOWN, 0, ZSTD_RUST_CPM_UNKNOWN); + let params = ZSTD_rust_params_makeParams(cparams); + assert_eq!(params.cParams, cparams); + assert_eq!(params.fParams.contentSizeFlag, 1); + assert_eq!(params.fParams.checksumFlag, 0); + assert_eq!(params.fParams.noDictIDFlag, 0); + } + + #[test] + fn row_size_preserves_unknown_dictionary_overflow_semantics() { + assert_eq!( + get_cparam_row_size(ZSTD_CONTENTSIZE_UNKNOWN, 0, ZSTD_RUST_CPM_UNKNOWN), + ZSTD_CONTENTSIZE_UNKNOWN + ); + assert_eq!( + get_cparam_row_size(ZSTD_CONTENTSIZE_UNKNOWN, 1, ZSTD_RUST_CPM_UNKNOWN), + 500 + ); + assert_eq!( + get_cparam_row_size(128 * 1024, 1, ZSTD_RUST_CPM_ATTACH_DICT), + 128 * 1024 + ); + } + + #[test] + fn match_state_sizing_reproduces_table_and_row_rules() { + let sizing = ZSTD_rustMatchStateSizing { + hashLog3Max: 17, + matchTSize: 16, + optimalTSize: 32, + asanRedzoneSize: 0, + }; + let fast = ZSTD_compressionParameters { + windowLog: 10, + chainLog: 10, + hashLog: 10, + searchLog: 1, + minMatch: 4, + targetLength: 0, + strategy: ZSTD_FAST, + }; + assert_eq!( + estimate_match_state_size(fast, ZSTD_RUST_PS_DISABLE, false, true, sizing), + 4096 + 128 + ); + + let row = ZSTD_compressionParameters { + windowLog: 15, + chainLog: 10, + hashLog: 10, + searchLog: 4, + minMatch: 4, + targetLength: 0, + strategy: ZSTD_GREEDY, + }; + assert_eq!( + estimate_match_state_size(row, ZSTD_RUST_PS_ENABLE, false, true, sizing), + 4096 + 1024 + 128 + ); + assert_eq!(ZSTD_rust_params_maxNbSeq(100, 3, 0), 33); + assert_eq!(ZSTD_rust_params_maxNbSeq(100, 4, 0), 25); + assert_eq!(ZSTD_rust_params_resolveMaxBlockSize(0), 128 * 1024); + } +}