diff --git a/.github/workflows/dev-long-tests.yml b/.github/workflows/dev-long-tests.yml index deb5242b0..fa603e401 100644 --- a/.github/workflows/dev-long-tests.yml +++ b/.github/workflows/dev-long-tests.yml @@ -156,15 +156,30 @@ jobs: make libc6install CFLAGS="-O3 -m32" FUZZER_FLAGS="--long-tests" make uasan-fuzztest - clang-asan-ubsan-fuzz32: + clang-asan-fuzz32: runs-on: ubuntu-20.04 steps: - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # tag=v4.1.1 - - name: clang + ASan + UBSan + Fuzz Test 32bit + - name: clang + ASan + Fuzz Test 32bit run: | sudo apt-get -qqq update make libc6install - CC=clang CFLAGS="-O3 -m32" FUZZER_FLAGS="--long-tests" make uasan-fuzztest + CC=clang CFLAGS="-O3 -m32" FUZZER_FLAGS="--long-tests" make asan-fuzztest + +# The following test seems to have issues on github CI specifically, +# it does not provide the `__mulodi4` instruction emulation +# required for signed 64-bit multiplication. +# Replaced by asan-only test (above) +# +# clang-asan-ubsan-fuzz32: +# runs-on: ubuntu-20.04 +# steps: +# - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # tag=v4.1.1 +# - name: clang + ASan + UBSan + Fuzz Test 32bit +# run: | +# sudo apt-get -qqq update +# make libc6install +# CC=clang CFLAGS="-O3 -m32" FUZZER_FLAGS="--long-tests" make uasan-fuzztest asan-ubsan-regression: runs-on: ubuntu-20.04 diff --git a/.github/workflows/dev-short-tests.yml b/.github/workflows/dev-short-tests.yml index 7713a897c..99aa166c9 100644 --- a/.github/workflows/dev-short-tests.yml +++ b/.github/workflows/dev-short-tests.yml @@ -232,10 +232,10 @@ jobs: -Dbin_tests=true \ -Dbin_contrib=true \ -Ddefault_library=both \ - build/meson builddir - ninja -C builddir/ - meson test -C builddir/ --print-errorlogs - meson install -C builddir --destdir staging/ + build/meson mesonBuild + ninja -C mesonBuild/ + meson test -C mesonBuild/ --print-errorlogs + meson install -C mesonBuild --destdir staging/ meson-mingw-cross-compilation: runs-on: ubuntu-latest diff --git a/Makefile b/Makefile index 5598cdd03..95d45afb4 100644 --- a/Makefile +++ b/Makefile @@ -145,7 +145,7 @@ clean: $(Q)$(MAKE) -C contrib/largeNbDicts $@ > $(VOID) $(Q)$(MAKE) -C contrib/externalSequenceProducer $@ > $(VOID) $(Q)$(RM) zstd$(EXT) zstdmt$(EXT) tmp* - $(Q)$(RM) -r lz4 cmakebuild install + $(Q)$(RM) -r lz4 cmakebuild mesonbuild install @echo Cleaning completed #------------------------------------------------------------------------------ @@ -415,6 +415,24 @@ cmakebuild: $(CMAKE) --build cmakebuild --target install -- -j V=1 cd cmakebuild; ctest -V -L Medium +MESON ?= meson +NINJA ?= ninja + +.PHONY: mesonbuild +mesonbuild: + $(MESON) setup \ + --buildtype=debugoptimized \ + -Db_lundef=false \ + -Dauto_features=enabled \ + -Dbin_programs=true \ + -Dbin_tests=true \ + -Dbin_contrib=true \ + -Ddefault_library=both \ + build/meson mesonbuild + $(NINJA) -C mesonbuild/ + $(MESON) test -C mesonbuild/ --print-errorlogs + $(MESON) install -C mesonbuild --destdir staging/ + .PHONY: c89build gnu90build c99build gnu99build c11build bmix64build bmix32build bmi32build staticAnalyze c89build: clean $(CC) -v diff --git a/build/VS2010/fullbench/fullbench.vcxproj b/build/VS2010/fullbench/fullbench.vcxproj index 2e0a042b0..f60cfe726 100644 --- a/build/VS2010/fullbench/fullbench.vcxproj +++ b/build/VS2010/fullbench/fullbench.vcxproj @@ -170,6 +170,7 @@ + diff --git a/build/VS2010/fuzzer/fuzzer.vcxproj b/build/VS2010/fuzzer/fuzzer.vcxproj index 91974ec7f..ccbd2517d 100644 --- a/build/VS2010/fuzzer/fuzzer.vcxproj +++ b/build/VS2010/fuzzer/fuzzer.vcxproj @@ -170,6 +170,7 @@ + diff --git a/build/VS2010/libzstd-dll/libzstd-dll.vcxproj b/build/VS2010/libzstd-dll/libzstd-dll.vcxproj index 97ad2fcb2..6925e0f9e 100644 --- a/build/VS2010/libzstd-dll/libzstd-dll.vcxproj +++ b/build/VS2010/libzstd-dll/libzstd-dll.vcxproj @@ -34,6 +34,7 @@ + diff --git a/build/VS2010/libzstd/libzstd.vcxproj b/build/VS2010/libzstd/libzstd.vcxproj index 614e6292a..82a2d8268 100644 --- a/build/VS2010/libzstd/libzstd.vcxproj +++ b/build/VS2010/libzstd/libzstd.vcxproj @@ -34,6 +34,7 @@ + diff --git a/build/VS2010/zstd/zstd.vcxproj b/build/VS2010/zstd/zstd.vcxproj index 5a5237f0e..0558687f5 100644 --- a/build/VS2010/zstd/zstd.vcxproj +++ b/build/VS2010/zstd/zstd.vcxproj @@ -35,6 +35,7 @@ + diff --git a/build/meson/lib/meson.build b/build/meson/lib/meson.build index 4ff926f9c..d086fc2d7 100644 --- a/build/meson/lib/meson.build +++ b/build/meson/lib/meson.build @@ -30,6 +30,7 @@ libzstd_sources = [join_paths(zstd_rootdir, 'lib/common/entropy_common.c'), join_paths(zstd_rootdir, 'lib/compress/zstd_compress_literals.c'), join_paths(zstd_rootdir, 'lib/compress/zstd_compress_sequences.c'), join_paths(zstd_rootdir, 'lib/compress/zstd_compress_superblock.c'), + join_paths(zstd_rootdir, 'lib/compress/zstd_preSplit.c'), join_paths(zstd_rootdir, 'lib/compress/zstdmt_compress.c'), join_paths(zstd_rootdir, 'lib/compress/zstd_fast.c'), join_paths(zstd_rootdir, 'lib/compress/zstd_double_fast.c'), diff --git a/build/single_file_libs/zstd-in.c b/build/single_file_libs/zstd-in.c index e6fca9e4a..f381ecc4f 100644 --- a/build/single_file_libs/zstd-in.c +++ b/build/single_file_libs/zstd-in.c @@ -69,6 +69,7 @@ #include "compress/zstd_compress_literals.c" #include "compress/zstd_compress_sequences.c" #include "compress/zstd_compress_superblock.c" +#include "compress/zstd_preSplit.c" #include "compress/zstd_compress.c" #include "compress/zstd_double_fast.c" #include "compress/zstd_fast.c" diff --git a/lib/common/compiler.h b/lib/common/compiler.h index fdf0dd18a..b6cbcee03 100644 --- a/lib/common/compiler.h +++ b/lib/common/compiler.h @@ -278,6 +278,12 @@ * Alignment check *****************************************************************/ +/* @return 1 if @u is a 2^n value, 0 otherwise + * useful to check a value is valid for alignment restrictions */ +MEM_STATIC int ZSTD_isPower2(size_t u) { + return (u & (u-1)) == 0; +} + /* this test was initially positioned in mem.h, * but this file is removed (or replaced) for linux kernel * so it's now hosted in compiler.h, diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index aad25049d..437bd1fbe 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -137,11 +137,12 @@ ZSTD_CCtx* ZSTD_initStaticCCtx(void* workspace, size_t workspaceSize) ZSTD_cwksp_move(&cctx->workspace, &ws); cctx->staticSize = workspaceSize; - /* statically sized space. entropyWorkspace never moves (but prev/next block swap places) */ - if (!ZSTD_cwksp_check_available(&cctx->workspace, ENTROPY_WORKSPACE_SIZE + 2 * sizeof(ZSTD_compressedBlockState_t))) return NULL; + /* statically sized space. tmpWorkspace never moves (but prev/next block swap places) */ + if (!ZSTD_cwksp_check_available(&cctx->workspace, TMP_WORKSPACE_SIZE + 2 * sizeof(ZSTD_compressedBlockState_t))) return NULL; cctx->blockState.prevCBlock = (ZSTD_compressedBlockState_t*)ZSTD_cwksp_reserve_object(&cctx->workspace, sizeof(ZSTD_compressedBlockState_t)); cctx->blockState.nextCBlock = (ZSTD_compressedBlockState_t*)ZSTD_cwksp_reserve_object(&cctx->workspace, sizeof(ZSTD_compressedBlockState_t)); - cctx->entropyWorkspace = (U32*)ZSTD_cwksp_reserve_object(&cctx->workspace, ENTROPY_WORKSPACE_SIZE); + cctx->tmpWorkspace = ZSTD_cwksp_reserve_object(&cctx->workspace, TMP_WORKSPACE_SIZE); + cctx->tmpWkspSize = TMP_WORKSPACE_SIZE; cctx->bmi2 = ZSTD_cpuid_bmi2(ZSTD_cpuid()); return cctx; } @@ -883,7 +884,7 @@ size_t ZSTD_CCtxParams_setParameter(ZSTD_CCtx_params* CCtxParams, value = ZSTDMT_JOBSIZE_MIN; FORWARD_IF_ERROR(ZSTD_cParam_clampBounds(param, &value), ""); assert(value >= 0); - CCtxParams->jobSize = value; + CCtxParams->jobSize = (size_t)value; return CCtxParams->jobSize; #endif @@ -1002,7 +1003,8 @@ size_t ZSTD_CCtxParams_setParameter(ZSTD_CCtx_params* CCtxParams, case ZSTD_c_maxBlockSize: if (value!=0) /* 0 ==> default */ BOUNDCHECK(ZSTD_c_maxBlockSize, value); - CCtxParams->maxBlockSize = value; + assert(value>=0); + CCtxParams->maxBlockSize = (size_t)value; return CCtxParams->maxBlockSize; case ZSTD_c_searchForExternalRepcodes: @@ -1630,7 +1632,8 @@ ZSTD_compressionParameters ZSTD_getCParamsFromCCtxParams( { ZSTD_compressionParameters cParams; if (srcSizeHint == ZSTD_CONTENTSIZE_UNKNOWN && CCtxParams->srcSizeHint > 0) { - srcSizeHint = CCtxParams->srcSizeHint; + assert(CCtxParams->srcSizeHint>=0); + srcSizeHint = (U64)CCtxParams->srcSizeHint; } cParams = ZSTD_getCParams_internal(CCtxParams->compressionLevel, srcSizeHint, dictSize, mode); if (CCtxParams->ldmParams.enableLdm == ZSTD_ps_enable) cParams.windowLog = ZSTD_LDM_DEFAULT_WINDOW_LOG; @@ -1659,14 +1662,14 @@ ZSTD_sizeof_matchState(const ZSTD_compressionParameters* const cParams, + hSize * sizeof(U32) + h3Size * sizeof(U32); size_t const optPotentialSpace = - ZSTD_cwksp_aligned_alloc_size((MaxML+1) * sizeof(U32)) - + ZSTD_cwksp_aligned_alloc_size((MaxLL+1) * sizeof(U32)) - + ZSTD_cwksp_aligned_alloc_size((MaxOff+1) * sizeof(U32)) - + ZSTD_cwksp_aligned_alloc_size((1<strategy, useRowMatchFinder) - ? ZSTD_cwksp_aligned_alloc_size(hSize) + ? ZSTD_cwksp_aligned64_alloc_size(hSize) : 0; size_t const optSpace = (forCCtx && (cParams->strategy >= ZSTD_btopt)) ? optPotentialSpace @@ -1704,16 +1707,16 @@ static size_t ZSTD_estimateCCtxSize_usingCCtxParams_internal( size_t const blockSize = MIN(ZSTD_resolveMaxBlockSize(maxBlockSize), windowSize); size_t const maxNbSeq = ZSTD_maxNbSeq(blockSize, cParams->minMatch, useSequenceProducer); size_t const tokenSpace = ZSTD_cwksp_alloc_size(WILDCOPY_OVERLENGTH + blockSize) - + ZSTD_cwksp_aligned_alloc_size(maxNbSeq * sizeof(seqDef)) + + ZSTD_cwksp_aligned64_alloc_size(maxNbSeq * sizeof(seqDef)) + 3 * ZSTD_cwksp_alloc_size(maxNbSeq * sizeof(BYTE)); - size_t const entropySpace = ZSTD_cwksp_alloc_size(ENTROPY_WORKSPACE_SIZE); + size_t const tmpWorkSpace = ZSTD_cwksp_alloc_size(TMP_WORKSPACE_SIZE); size_t const blockStateSpace = 2 * ZSTD_cwksp_alloc_size(sizeof(ZSTD_compressedBlockState_t)); size_t const matchStateSize = ZSTD_sizeof_matchState(cParams, useRowMatchFinder, /* enableDedicatedDictSearch */ 0, /* forCCtx */ 1); size_t const ldmSpace = ZSTD_ldm_getTableSize(*ldmParams); size_t const maxNbLdmSeq = ZSTD_ldm_getMaxNbSeq(*ldmParams, blockSize); size_t const ldmSeqSpace = ldmParams->enableLdm == ZSTD_ps_enable ? - ZSTD_cwksp_aligned_alloc_size(maxNbLdmSeq * sizeof(rawSeq)) : 0; + ZSTD_cwksp_aligned64_alloc_size(maxNbLdmSeq * sizeof(rawSeq)) : 0; size_t const bufferSpace = ZSTD_cwksp_alloc_size(buffInSize) @@ -1723,12 +1726,12 @@ static size_t ZSTD_estimateCCtxSize_usingCCtxParams_internal( size_t const maxNbExternalSeq = ZSTD_sequenceBound(blockSize); size_t const externalSeqSpace = useSequenceProducer - ? ZSTD_cwksp_aligned_alloc_size(maxNbExternalSeq * sizeof(ZSTD_Sequence)) + ? ZSTD_cwksp_aligned64_alloc_size(maxNbExternalSeq * sizeof(ZSTD_Sequence)) : 0; size_t const neededSpace = cctxSpace + - entropySpace + + tmpWorkSpace + blockStateSpace + ldmSpace + ldmSeqSpace + @@ -2029,7 +2032,7 @@ ZSTD_reset_matchState(ZSTD_matchState_t* ms, ZSTD_advanceHashSalt(ms); } else { /* When we are not salting we want to always memset the memory */ - ms->tagTable = (BYTE*) ZSTD_cwksp_reserve_aligned(ws, tagTableSize); + ms->tagTable = (BYTE*) ZSTD_cwksp_reserve_aligned64(ws, tagTableSize); ZSTD_memset(ms->tagTable, 0, tagTableSize); ms->hashSalt = 0; } @@ -2043,12 +2046,12 @@ ZSTD_reset_matchState(ZSTD_matchState_t* ms, /* opt parser space */ if ((forWho == ZSTD_resetTarget_CCtx) && (cParams->strategy >= ZSTD_btopt)) { DEBUGLOG(4, "reserving optimal parser space"); - ms->opt.litFreq = (unsigned*)ZSTD_cwksp_reserve_aligned(ws, (1<opt.litLengthFreq = (unsigned*)ZSTD_cwksp_reserve_aligned(ws, (MaxLL+1) * sizeof(unsigned)); - ms->opt.matchLengthFreq = (unsigned*)ZSTD_cwksp_reserve_aligned(ws, (MaxML+1) * sizeof(unsigned)); - ms->opt.offCodeFreq = (unsigned*)ZSTD_cwksp_reserve_aligned(ws, (MaxOff+1) * sizeof(unsigned)); - ms->opt.matchTable = (ZSTD_match_t*)ZSTD_cwksp_reserve_aligned(ws, ZSTD_OPT_SIZE * sizeof(ZSTD_match_t)); - ms->opt.priceTable = (ZSTD_optimal_t*)ZSTD_cwksp_reserve_aligned(ws, ZSTD_OPT_SIZE * sizeof(ZSTD_optimal_t)); + ms->opt.litFreq = (unsigned*)ZSTD_cwksp_reserve_aligned64(ws, (1<opt.litLengthFreq = (unsigned*)ZSTD_cwksp_reserve_aligned64(ws, (MaxLL+1) * sizeof(unsigned)); + ms->opt.matchLengthFreq = (unsigned*)ZSTD_cwksp_reserve_aligned64(ws, (MaxML+1) * sizeof(unsigned)); + ms->opt.offCodeFreq = (unsigned*)ZSTD_cwksp_reserve_aligned64(ws, (MaxOff+1) * sizeof(unsigned)); + ms->opt.matchTable = (ZSTD_match_t*)ZSTD_cwksp_reserve_aligned64(ws, ZSTD_OPT_SIZE * sizeof(ZSTD_match_t)); + ms->opt.priceTable = (ZSTD_optimal_t*)ZSTD_cwksp_reserve_aligned64(ws, ZSTD_OPT_SIZE * sizeof(ZSTD_optimal_t)); } ms->cParams = *cParams; @@ -2164,15 +2167,16 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, DEBUGLOG(5, "reserving object space"); /* Statically sized space. - * entropyWorkspace never moves, + * tmpWorkspace never moves, * though prev/next block swap places */ assert(ZSTD_cwksp_check_available(ws, 2 * sizeof(ZSTD_compressedBlockState_t))); zc->blockState.prevCBlock = (ZSTD_compressedBlockState_t*) ZSTD_cwksp_reserve_object(ws, sizeof(ZSTD_compressedBlockState_t)); RETURN_ERROR_IF(zc->blockState.prevCBlock == NULL, memory_allocation, "couldn't allocate prevCBlock"); zc->blockState.nextCBlock = (ZSTD_compressedBlockState_t*) ZSTD_cwksp_reserve_object(ws, sizeof(ZSTD_compressedBlockState_t)); RETURN_ERROR_IF(zc->blockState.nextCBlock == NULL, memory_allocation, "couldn't allocate nextCBlock"); - zc->entropyWorkspace = (U32*) ZSTD_cwksp_reserve_object(ws, ENTROPY_WORKSPACE_SIZE); - RETURN_ERROR_IF(zc->entropyWorkspace == NULL, memory_allocation, "couldn't allocate entropyWorkspace"); + zc->tmpWorkspace = ZSTD_cwksp_reserve_object(ws, TMP_WORKSPACE_SIZE); + RETURN_ERROR_IF(zc->tmpWorkspace == NULL, memory_allocation, "couldn't allocate tmpWorkspace"); + zc->tmpWkspSize = TMP_WORKSPACE_SIZE; } } ZSTD_cwksp_clear(ws); @@ -2205,15 +2209,15 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, needsIndexReset, ZSTD_resetTarget_CCtx), ""); - zc->seqStore.sequencesStart = (seqDef*)ZSTD_cwksp_reserve_aligned(ws, maxNbSeq * sizeof(seqDef)); + zc->seqStore.sequencesStart = (seqDef*)ZSTD_cwksp_reserve_aligned64(ws, maxNbSeq * sizeof(seqDef)); /* ldm hash table */ if (params->ldmParams.enableLdm == ZSTD_ps_enable) { /* TODO: avoid memset? */ size_t const ldmHSize = ((size_t)1) << params->ldmParams.hashLog; - zc->ldmState.hashTable = (ldmEntry_t*)ZSTD_cwksp_reserve_aligned(ws, ldmHSize * sizeof(ldmEntry_t)); + zc->ldmState.hashTable = (ldmEntry_t*)ZSTD_cwksp_reserve_aligned64(ws, ldmHSize * sizeof(ldmEntry_t)); ZSTD_memset(zc->ldmState.hashTable, 0, ldmHSize * sizeof(ldmEntry_t)); - zc->ldmSequences = (rawSeq*)ZSTD_cwksp_reserve_aligned(ws, maxNbLdmSeq * sizeof(rawSeq)); + zc->ldmSequences = (rawSeq*)ZSTD_cwksp_reserve_aligned64(ws, maxNbLdmSeq * sizeof(rawSeq)); zc->maxNbLdmSequences = maxNbLdmSeq; ZSTD_window_init(&zc->ldmState.window); @@ -2225,7 +2229,7 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, size_t const maxNbExternalSeq = ZSTD_sequenceBound(blockSize); zc->extSeqBufCapacity = maxNbExternalSeq; zc->extSeqBuf = - (ZSTD_Sequence*)ZSTD_cwksp_reserve_aligned(ws, maxNbExternalSeq * sizeof(ZSTD_Sequence)); + (ZSTD_Sequence*)ZSTD_cwksp_reserve_aligned64(ws, maxNbExternalSeq * sizeof(ZSTD_Sequence)); } /* buffers */ @@ -2444,7 +2448,8 @@ static size_t ZSTD_resetCCtx_byCopyingCDict(ZSTD_CCtx* cctx, } /* Zero the hashTable3, since the cdict never fills it */ - { int const h3log = cctx->blockState.matchState.hashLog3; + assert(cctx->blockState.matchState.hashLog3 <= 31); + { U32 const h3log = cctx->blockState.matchState.hashLog3; size_t const h3Size = h3log ? ((size_t)1 << h3log) : 0; assert(cdict->matchState.hashLog3 == 0); ZSTD_memset(cctx->blockState.matchState.hashTable3, 0, h3Size * sizeof(U32)); @@ -3891,14 +3896,14 @@ ZSTD_buildEntropyStatisticsAndEstimateSubBlockSize(seqStore_t* seqStore, ZSTD_CC &zc->blockState.nextCBlock->entropy, &zc->appliedParams, entropyMetadata, - zc->entropyWorkspace, ENTROPY_WORKSPACE_SIZE), ""); + zc->tmpWorkspace, zc->tmpWkspSize), ""); return ZSTD_estimateBlockSize( seqStore->litStart, (size_t)(seqStore->lit - seqStore->litStart), seqStore->ofCode, seqStore->llCode, seqStore->mlCode, (size_t)(seqStore->sequences - seqStore->sequencesStart), &zc->blockState.nextCBlock->entropy, entropyMetadata, - zc->entropyWorkspace, ENTROPY_WORKSPACE_SIZE, + zc->tmpWorkspace, zc->tmpWkspSize, (int)(entropyMetadata->hufMetadata.hType == set_compressed), 1); } @@ -4064,7 +4069,7 @@ ZSTD_compressSeqStore_singleBlock(ZSTD_CCtx* zc, &zc->appliedParams, op + ZSTD_blockHeaderSize, dstCapacity - ZSTD_blockHeaderSize, srcSize, - zc->entropyWorkspace, ENTROPY_WORKSPACE_SIZE /* statically allocated in resetCCtx */, + zc->tmpWorkspace, zc->tmpWkspSize /* statically allocated in resetCCtx */, zc->bmi2); FORWARD_IF_ERROR(cSeqsSize, "ZSTD_entropyCompressSeqStore failed!"); @@ -4354,7 +4359,7 @@ ZSTD_compressBlock_internal(ZSTD_CCtx* zc, &zc->appliedParams, dst, dstCapacity, srcSize, - zc->entropyWorkspace, ENTROPY_WORKSPACE_SIZE /* statically allocated in resetCCtx */, + zc->tmpWorkspace, zc->tmpWkspSize /* statically allocated in resetCCtx */, zc->bmi2); if (frame && @@ -4484,6 +4489,34 @@ static void ZSTD_overflowCorrectIfNeeded(ZSTD_matchState_t* ms, } } +#include "zstd_preSplit.h" + +static size_t ZSTD_optimalBlockSize(ZSTD_CCtx* cctx, const void* src, size_t srcSize, size_t blockSizeMax, ZSTD_strategy strat, S64 savings) +{ + /* note: conservatively only split full blocks (128 KB) currently. + * While it's possible to go lower, let's keep it simple for a first implementation. + * Besides, benefits of splitting are reduced when blocks are already small. + */ + if (srcSize < 128 KB || blockSizeMax < 128 KB) + return MIN(srcSize, blockSizeMax); + /* do not split incompressible data though: + * ensure a 3 bytes per full block overhead limit. + * Note: as a consequence, the first full block skips the splitting detector. + */ + if (savings < 3) return 128 KB; + /* dynamic splitting has a cpu cost for analysis, + * due to that cost it's only used for higher levels */ + if (strat >= ZSTD_btopt) + return ZSTD_splitBlock(src, srcSize, blockSizeMax, split_lvl2, cctx->tmpWorkspace, cctx->tmpWkspSize); + if (strat >= ZSTD_lazy2) + return ZSTD_splitBlock(src, srcSize, blockSizeMax, split_lvl1, cctx->tmpWorkspace, cctx->tmpWkspSize); + /* blind split strategy + * heuristic value, tested as being "generally better". + * no cpu cost, but can over-split homegeneous data. + */ + return 92 KB; +} + /*! ZSTD_compress_frameChunk() : * Compress a chunk of data into one or multiple blocks. * All blocks will be terminated, all input will be consumed. @@ -4496,29 +4529,31 @@ static size_t ZSTD_compress_frameChunk(ZSTD_CCtx* cctx, const void* src, size_t srcSize, U32 lastFrameChunk) { - size_t blockSize = cctx->blockSize; + size_t blockSizeMax = cctx->blockSize; size_t remaining = srcSize; const BYTE* ip = (const BYTE*)src; BYTE* const ostart = (BYTE*)dst; BYTE* op = ostart; U32 const maxDist = (U32)1 << cctx->appliedParams.cParams.windowLog; + S64 savings = (S64)cctx->consumedSrcSize - (S64)cctx->producedCSize; assert(cctx->appliedParams.cParams.windowLog <= ZSTD_WINDOWLOG_MAX); - DEBUGLOG(4, "ZSTD_compress_frameChunk (blockSize=%u)", (unsigned)blockSize); + DEBUGLOG(4, "ZSTD_compress_frameChunk (blockSizeMax=%u)", (unsigned)blockSizeMax); if (cctx->appliedParams.fParams.checksumFlag && srcSize) XXH64_update(&cctx->xxhState, src, srcSize); while (remaining) { ZSTD_matchState_t* const ms = &cctx->blockState.matchState; - U32 const lastBlock = lastFrameChunk & (blockSize >= remaining); + size_t const blockSize = ZSTD_optimalBlockSize(cctx, ip, remaining, blockSizeMax, cctx->appliedParams.cParams.strategy, savings); + U32 const lastBlock = lastFrameChunk & (blockSize == remaining); + assert(blockSize <= remaining); /* TODO: See 3090. We reduced MIN_CBLOCK_SIZE from 3 to 2 so to compensate we are adding * additional 1. We need to revisit and change this logic to be more consistent */ RETURN_ERROR_IF(dstCapacity < ZSTD_blockHeaderSize + MIN_CBLOCK_SIZE + 1, dstSize_tooSmall, "not enough space to store compressed block"); - if (remaining < blockSize) blockSize = remaining; ZSTD_overflowCorrectIfNeeded( ms, &cctx->workspace, &cctx->appliedParams, ip, ip + blockSize); @@ -4556,6 +4591,21 @@ static size_t ZSTD_compress_frameChunk(ZSTD_CCtx* cctx, } } /* if (ZSTD_useTargetCBlockSize(&cctx->appliedParams))*/ + /* @savings is employed to ensure that splitting doesn't worsen expansion of incompressible data. + * Without splitting, the maximum expansion is 3 bytes per full block. + * An adversarial input could attempt to fudge the split detector, + * and make it split incompressible data, resulting in more block headers. + * Note that, since ZSTD_COMPRESSBOUND() assumes a worst case scenario of 1KB per block, + * and the splitter never creates blocks that small (current lower limit is 8 KB), + * there is already no risk to expand beyond ZSTD_COMPRESSBOUND() limit. + * But if the goal is to not expand by more than 3-bytes per 128 KB full block, + * then yes, it becomes possible to make the block splitter oversplit incompressible data. + * Using @savings, we enforce an even more conservative condition, + * requiring the presence of enough savings (at least 3 bytes) to authorize splitting, + * otherwise only full blocks are used. + * But being conservative is fine, + * since splitting barely compressible blocks is not fruitful anyway */ + savings += (S64)blockSize - (S64)cSize; ip += blockSize; assert(remaining >= blockSize); @@ -5022,7 +5072,7 @@ size_t ZSTD_loadCEntropy(ZSTD_compressedBlockState_t* bs, void* workspace, RETURN_ERROR_IF(bs->rep[u] > dictContentSize, dictionary_corrupted, ""); } } } - return dictPtr - (const BYTE*)dict; + return (size_t)(dictPtr - (const BYTE*)dict); } /* Dictionary format : @@ -5145,11 +5195,11 @@ static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* cctx, cctx->blockState.prevCBlock, &cctx->blockState.matchState, &cctx->ldmState, &cctx->workspace, &cctx->appliedParams, cdict->dictContent, cdict->dictContentSize, cdict->dictContentType, dtlm, - ZSTD_tfp_forCCtx, cctx->entropyWorkspace) + ZSTD_tfp_forCCtx, cctx->tmpWorkspace) : ZSTD_compress_insertDictionary( cctx->blockState.prevCBlock, &cctx->blockState.matchState, &cctx->ldmState, &cctx->workspace, &cctx->appliedParams, dict, dictSize, - dictContentType, dtlm, ZSTD_tfp_forCCtx, cctx->entropyWorkspace); + dictContentType, dtlm, ZSTD_tfp_forCCtx, cctx->tmpWorkspace); FORWARD_IF_ERROR(dictID, "ZSTD_compress_insertDictionary failed"); assert(dictID <= UINT_MAX); cctx->dictID = (U32)dictID; @@ -6847,7 +6897,7 @@ ZSTD_compressSequences_internal(ZSTD_CCtx* cctx, &cctx->appliedParams, op + ZSTD_blockHeaderSize /* Leave space for block header */, dstCapacity - ZSTD_blockHeaderSize, blockSize, - cctx->entropyWorkspace, ENTROPY_WORKSPACE_SIZE /* statically allocated in resetCCtx */, + cctx->tmpWorkspace, cctx->tmpWkspSize /* statically allocated in resetCCtx */, cctx->bmi2); FORWARD_IF_ERROR(compressedSeqsSize, "Compressing sequences of block failed"); DEBUGLOG(5, "Compressed sequences size: %zu", compressedSeqsSize); diff --git a/lib/compress/zstd_compress_internal.h b/lib/compress/zstd_compress_internal.h index 622903398..e76bbda71 100644 --- a/lib/compress/zstd_compress_internal.h +++ b/lib/compress/zstd_compress_internal.h @@ -24,6 +24,7 @@ # include "zstdmt_compress.h" #endif #include "../common/bits.h" /* ZSTD_highbit32, ZSTD_NbCommonBytes */ +#include "zstd_preSplit.h" /* ZSTD_SLIPBLOCK_WORKSPACESIZE */ #if defined (__cplusplus) extern "C" { @@ -376,6 +377,7 @@ struct ZSTD_CCtx_params_s { #define COMPRESS_SEQUENCES_WORKSPACE_SIZE (sizeof(unsigned) * (MaxSeq + 2)) #define ENTROPY_WORKSPACE_SIZE (HUF_WORKSPACE_SIZE + COMPRESS_SEQUENCES_WORKSPACE_SIZE) +#define TMP_WORKSPACE_SIZE (MAX(ENTROPY_WORKSPACE_SIZE, ZSTD_SLIPBLOCK_WORKSPACESIZE)) /** * Indicates whether this compression proceeds directly from user-provided @@ -432,7 +434,8 @@ struct ZSTD_CCtx_s { size_t maxNbLdmSequences; rawSeqStore_t externSeqStore; /* Mutable reference to external sequences */ ZSTD_blockState_t blockState; - U32* entropyWorkspace; /* entropy workspace of ENTROPY_WORKSPACE_SIZE bytes */ + void* tmpWorkspace; /* used as substitute of stack space - must be aligned for S64 type */ + size_t tmpWkspSize; /* Whether we are streaming or not */ ZSTD_buffered_policy_e bufferedPolicy; @@ -798,8 +801,8 @@ ZSTD_count_2segments(const BYTE* ip, const BYTE* match, size_t const matchLength = ZSTD_count(ip, match, vEnd); if (match + matchLength != mEnd) return matchLength; DEBUGLOG(7, "ZSTD_count_2segments: found a 2-parts match (current length==%zu)", matchLength); - DEBUGLOG(7, "distance from match beginning to end dictionary = %zi", mEnd - match); - DEBUGLOG(7, "distance from current pos to end buffer = %zi", iEnd - ip); + DEBUGLOG(7, "distance from match beginning to end dictionary = %i", (int)(mEnd - match)); + DEBUGLOG(7, "distance from current pos to end buffer = %i", (int)(iEnd - ip)); DEBUGLOG(7, "next byte : ip==%02X, istart==%02X", ip[matchLength], *iStart); DEBUGLOG(7, "final match length = %zu", matchLength + ZSTD_count(ip+matchLength, iStart, iEnd)); return matchLength + ZSTD_count(ip+matchLength, iStart, iEnd); diff --git a/lib/compress/zstd_compress_superblock.c b/lib/compress/zstd_compress_superblock.c index 628a2dccd..97321bc76 100644 --- a/lib/compress/zstd_compress_superblock.c +++ b/lib/compress/zstd_compress_superblock.c @@ -674,7 +674,7 @@ size_t ZSTD_compressSuperBlock(ZSTD_CCtx* zc, &zc->blockState.nextCBlock->entropy, &zc->appliedParams, &entropyMetadata, - zc->entropyWorkspace, ENTROPY_WORKSPACE_SIZE /* statically allocated in resetCCtx */), ""); + zc->tmpWorkspace, zc->tmpWkspSize /* statically allocated in resetCCtx */), ""); return ZSTD_compressSubBlock_multi(&zc->seqStore, zc->blockState.prevCBlock, @@ -684,5 +684,5 @@ size_t ZSTD_compressSuperBlock(ZSTD_CCtx* zc, dst, dstCapacity, src, srcSize, zc->bmi2, lastBlock, - zc->entropyWorkspace, ENTROPY_WORKSPACE_SIZE /* statically allocated in resetCCtx */); + zc->tmpWorkspace, zc->tmpWkspSize /* statically allocated in resetCCtx */); } diff --git a/lib/compress/zstd_cwksp.h b/lib/compress/zstd_cwksp.h index dcd485cb5..aa4c3e49e 100644 --- a/lib/compress/zstd_cwksp.h +++ b/lib/compress/zstd_cwksp.h @@ -17,6 +17,7 @@ #include "../common/allocations.h" /* ZSTD_customMalloc, ZSTD_customFree */ #include "../common/zstd_internal.h" #include "../common/portability_macros.h" +#include "../common/compiler.h" /* ZS2_isPower2 */ #if defined (__cplusplus) extern "C" { @@ -206,9 +207,9 @@ MEM_STATIC void ZSTD_cwksp_assert_internal_consistency(ZSTD_cwksp* ws) { /** * Align must be a power of 2. */ -MEM_STATIC size_t ZSTD_cwksp_align(size_t size, size_t const align) { +MEM_STATIC size_t ZSTD_cwksp_align(size_t size, size_t align) { size_t const mask = align - 1; - assert((align & mask) == 0); + assert(ZSTD_isPower2(align)); return (size + mask) & ~mask; } @@ -222,7 +223,7 @@ MEM_STATIC size_t ZSTD_cwksp_align(size_t size, size_t const align) { * to figure out how much space you need for the matchState tables. Everything * else is though. * - * Do not use for sizing aligned buffers. Instead, use ZSTD_cwksp_aligned_alloc_size(). + * Do not use for sizing aligned buffers. Instead, use ZSTD_cwksp_aligned64_alloc_size(). */ MEM_STATIC size_t ZSTD_cwksp_alloc_size(size_t size) { if (size == 0) @@ -234,12 +235,16 @@ MEM_STATIC size_t ZSTD_cwksp_alloc_size(size_t size) { #endif } +MEM_STATIC size_t ZSTD_cwksp_aligned_alloc_size(size_t size, size_t alignment) { + return ZSTD_cwksp_alloc_size(ZSTD_cwksp_align(size, alignment)); +} + /** * Returns an adjusted alloc size that is the nearest larger multiple of 64 bytes. * Used to determine the number of bytes required for a given "aligned". */ -MEM_STATIC size_t ZSTD_cwksp_aligned_alloc_size(size_t size) { - return ZSTD_cwksp_alloc_size(ZSTD_cwksp_align(size, ZSTD_CWKSP_ALIGNMENT_BYTES)); +MEM_STATIC size_t ZSTD_cwksp_aligned64_alloc_size(size_t size) { + return ZSTD_cwksp_aligned_alloc_size(size, ZSTD_CWKSP_ALIGNMENT_BYTES); } /** @@ -262,7 +267,7 @@ MEM_STATIC size_t ZSTD_cwksp_slack_space_required(void) { MEM_STATIC size_t ZSTD_cwksp_bytes_to_align_ptr(void* ptr, const size_t alignBytes) { size_t const alignBytesMask = alignBytes - 1; size_t const bytes = (alignBytes - ((size_t)ptr & (alignBytesMask))) & alignBytesMask; - assert((alignBytes & alignBytesMask) == 0); + assert(ZSTD_isPower2(alignBytes)); assert(bytes < alignBytes); return bytes; } @@ -271,8 +276,12 @@ MEM_STATIC size_t ZSTD_cwksp_bytes_to_align_ptr(void* ptr, const size_t alignByt * Returns the initial value for allocStart which is used to determine the position from * which we can allocate from the end of the workspace. */ -MEM_STATIC void* ZSTD_cwksp_initialAllocStart(ZSTD_cwksp* ws) { - return (void*)((size_t)ws->workspaceEnd & ~(ZSTD_CWKSP_ALIGNMENT_BYTES-1)); +MEM_STATIC void* ZSTD_cwksp_initialAllocStart(ZSTD_cwksp* ws) +{ + char* endPtr = (char*)ws->workspaceEnd; + assert(ZSTD_isPower2(ZSTD_CWKSP_ALIGNMENT_BYTES)); + endPtr = endPtr - ((size_t)endPtr % ZSTD_CWKSP_ALIGNMENT_BYTES); + return (void*)endPtr; } /** @@ -404,7 +413,7 @@ MEM_STATIC void* ZSTD_cwksp_reserve_aligned_init_once(ZSTD_cwksp* ws, size_t byt { size_t const alignedBytes = ZSTD_cwksp_align(bytes, ZSTD_CWKSP_ALIGNMENT_BYTES); void* ptr = ZSTD_cwksp_reserve_internal(ws, alignedBytes, ZSTD_cwksp_alloc_aligned_init_once); - assert(((size_t)ptr & (ZSTD_CWKSP_ALIGNMENT_BYTES-1))== 0); + assert(((size_t)ptr & (ZSTD_CWKSP_ALIGNMENT_BYTES-1)) == 0); if(ptr && ptr < ws->initOnceStart) { /* We assume the memory following the current allocation is either: * 1. Not usable as initOnce memory (end of workspace) @@ -424,11 +433,12 @@ MEM_STATIC void* ZSTD_cwksp_reserve_aligned_init_once(ZSTD_cwksp* ws, size_t byt /** * Reserves and returns memory sized on and aligned on ZSTD_CWKSP_ALIGNMENT_BYTES (64 bytes). */ -MEM_STATIC void* ZSTD_cwksp_reserve_aligned(ZSTD_cwksp* ws, size_t bytes) +MEM_STATIC void* ZSTD_cwksp_reserve_aligned64(ZSTD_cwksp* ws, size_t bytes) { - void* ptr = ZSTD_cwksp_reserve_internal(ws, ZSTD_cwksp_align(bytes, ZSTD_CWKSP_ALIGNMENT_BYTES), - ZSTD_cwksp_alloc_aligned); - assert(((size_t)ptr & (ZSTD_CWKSP_ALIGNMENT_BYTES-1))== 0); + void* const ptr = ZSTD_cwksp_reserve_internal(ws, + ZSTD_cwksp_align(bytes, ZSTD_CWKSP_ALIGNMENT_BYTES), + ZSTD_cwksp_alloc_aligned); + assert(((size_t)ptr & (ZSTD_CWKSP_ALIGNMENT_BYTES-1)) == 0); return ptr; } @@ -474,7 +484,7 @@ MEM_STATIC void* ZSTD_cwksp_reserve_table(ZSTD_cwksp* ws, size_t bytes) #endif assert((bytes & (ZSTD_CWKSP_ALIGNMENT_BYTES-1)) == 0); - assert(((size_t)alloc & (ZSTD_CWKSP_ALIGNMENT_BYTES-1))== 0); + assert(((size_t)alloc & (ZSTD_CWKSP_ALIGNMENT_BYTES-1)) == 0); return alloc; } @@ -520,6 +530,20 @@ MEM_STATIC void* ZSTD_cwksp_reserve_object(ZSTD_cwksp* ws, size_t bytes) return alloc; } +/** + * with alignment control + * Note : should happen only once, at workspace first initialization + */ +MEM_STATIC void* ZSTD_cwksp_reserve_object_aligned(ZSTD_cwksp* ws, size_t byteSize, size_t alignment) +{ + size_t const mask = alignment - 1; + size_t const surplus = (alignment > sizeof(void*)) ? alignment - sizeof(void*) : 0; + void* const start = ZSTD_cwksp_reserve_object(ws, byteSize + surplus); + if (start == NULL) return NULL; + if (surplus == 0) return start; + assert(ZSTD_isPower2(alignment)); + return (void*)(((size_t)start + surplus) & ~mask); +} MEM_STATIC void ZSTD_cwksp_mark_tables_dirty(ZSTD_cwksp* ws) { diff --git a/lib/compress/zstd_opt.c b/lib/compress/zstd_opt.c index 8a4345b31..e4a8ee81e 100644 --- a/lib/compress/zstd_opt.c +++ b/lib/compress/zstd_opt.c @@ -1197,7 +1197,7 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms, for (cur = 1; cur <= last_pos; cur++) { const BYTE* const inr = ip + cur; assert(cur <= ZSTD_OPT_NUM); - DEBUGLOG(7, "cPos:%zi==rPos:%u", inr-istart, cur); + DEBUGLOG(7, "cPos:%i==rPos:%u", (int)(inr-istart), cur); /* Fix current position with one literal if cheaper */ { U32 const litlen = opt[cur-1].litlen + 1; @@ -1207,8 +1207,8 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms, assert(price < 1000000000); /* overflow check */ if (price <= opt[cur].price) { ZSTD_optimal_t const prevMatch = opt[cur]; - DEBUGLOG(7, "cPos:%zi==rPos:%u : better price (%.2f<=%.2f) using literal (ll==%u) (hist:%u,%u,%u)", - inr-istart, cur, ZSTD_fCost(price), ZSTD_fCost(opt[cur].price), litlen, + DEBUGLOG(7, "cPos:%i==rPos:%u : better price (%.2f<=%.2f) using literal (ll==%u) (hist:%u,%u,%u)", + (int)(inr-istart), cur, ZSTD_fCost(price), ZSTD_fCost(opt[cur].price), litlen, opt[cur-1].rep[0], opt[cur-1].rep[1], opt[cur-1].rep[2]); opt[cur] = opt[cur-1]; opt[cur].litlen = litlen; @@ -1240,8 +1240,8 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms, } } } else { - DEBUGLOG(7, "cPos:%zi==rPos:%u : literal would cost more (%.2f>%.2f)", - inr-istart, cur, ZSTD_fCost(price), ZSTD_fCost(opt[cur].price)); + DEBUGLOG(7, "cPos:%i==rPos:%u : literal would cost more (%.2f>%.2f)", + (int)(inr-istart), cur, ZSTD_fCost(price), ZSTD_fCost(opt[cur].price)); } } @@ -1284,8 +1284,8 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms, } { U32 const longestML = matches[nbMatches-1].len; - DEBUGLOG(7, "cPos:%zi==rPos:%u, found %u matches, of longest ML=%u", - inr-istart, cur, nbMatches, longestML); + DEBUGLOG(7, "cPos:%i==rPos:%u, found %u matches, of longest ML=%u", + (int)(inr-istart), cur, nbMatches, longestML); if ( (longestML > sufficient_len) || (cur + longestML >= ZSTD_OPT_NUM) @@ -1411,8 +1411,8 @@ _shortestPath: /* cur, last_pos, best_mlen, best_off have to be set */ U32 const mlen = opt[storePos].mlen; U32 const offBase = opt[storePos].off; U32 const advance = llen + mlen; - DEBUGLOG(6, "considering seq starting at %zi, llen=%u, mlen=%u", - anchor - istart, (unsigned)llen, (unsigned)mlen); + DEBUGLOG(6, "considering seq starting at %i, llen=%u, mlen=%u", + (int)(anchor - istart), (unsigned)llen, (unsigned)mlen); if (mlen==0) { /* only literals => must be last "sequence", actually starting a new stream of sequences */ assert(storePos == storeEnd); /* must be last sequence */ diff --git a/lib/compress/zstd_preSplit.c b/lib/compress/zstd_preSplit.c new file mode 100644 index 000000000..c658f6fe5 --- /dev/null +++ b/lib/compress/zstd_preSplit.c @@ -0,0 +1,181 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#include "../common/compiler.h" /* ZSTD_ALIGNOF */ +#include "../common/mem.h" /* S64 */ +#include "../common/zstd_deps.h" /* ZSTD_memset */ +#include "../common/zstd_internal.h" /* ZSTD_STATIC_ASSERT */ +#include "zstd_preSplit.h" + + +#define BLOCKSIZE_MIN 3500 +#define THRESHOLD_PENALTY_RATE 16 +#define THRESHOLD_BASE (THRESHOLD_PENALTY_RATE - 2) +#define THRESHOLD_PENALTY 3 + +#define HASHLENGTH 2 +#define HASHLOG 10 +#define HASHTABLESIZE (1 << HASHLOG) +#define HASHMASK (HASHTABLESIZE - 1) +#define KNUTH 0x9e3779b9 + +static unsigned hash2(const void *p) +{ + return (U32)(MEM_read16(p)) * KNUTH >> (32 - HASHLOG); +} + + +typedef struct { + unsigned events[HASHTABLESIZE]; + size_t nbEvents; +} Fingerprint; +typedef struct { + Fingerprint pastEvents; + Fingerprint newEvents; +} FPStats; + +static void initStats(FPStats* fpstats) +{ + ZSTD_memset(fpstats, 0, sizeof(FPStats)); +} + +FORCE_INLINE_TEMPLATE void addEvents_generic(Fingerprint* fp, const void* src, size_t srcSize, size_t samplingRate) +{ + const char* p = (const char*)src; + size_t limit = srcSize - HASHLENGTH + 1; + size_t n; + assert(srcSize >= HASHLENGTH); + for (n = 0; n < limit; n+=samplingRate) { + fp->events[hash2(p+n)]++; + } + fp->nbEvents += limit/samplingRate; +} + +#define ADDEVENTS_RATE(_rate) ZSTD_addEvents_##_rate + +#define ZSTD_GEN_ADDEVENTS_SAMPLE(_rate) \ + static void ADDEVENTS_RATE(_rate)(Fingerprint* fp, const void* src, size_t srcSize) \ + { \ + addEvents_generic(fp, src, srcSize, _rate); \ + } + +ZSTD_GEN_ADDEVENTS_SAMPLE(1) +ZSTD_GEN_ADDEVENTS_SAMPLE(5) + + +typedef void (*addEvents_f)(Fingerprint* fp, const void* src, size_t srcSize); + +static void recordFingerprint(Fingerprint* fp, const void* src, size_t s, addEvents_f addEvents) +{ + ZSTD_memset(fp, 0, sizeof(*fp)); + addEvents(fp, src, s); +} + +static U64 abs64(S64 s64) { return (U64)((s64 < 0) ? -s64 : s64); } + +static U64 fpDistance(const Fingerprint* fp1, const Fingerprint* fp2) +{ + U64 distance = 0; + size_t n; + for (n = 0; n < HASHTABLESIZE; n++) { + distance += + abs64((S64)fp1->events[n] * (S64)fp2->nbEvents - (S64)fp2->events[n] * (S64)fp1->nbEvents); + } + return distance; +} + +/* Compare newEvents with pastEvents + * return 1 when considered "too different" + */ +static int compareFingerprints(const Fingerprint* ref, + const Fingerprint* newfp, + int penalty) +{ + assert(ref->nbEvents > 0); + assert(newfp->nbEvents > 0); + { U64 p50 = (U64)ref->nbEvents * (U64)newfp->nbEvents; + U64 deviation = fpDistance(ref, newfp); + U64 threshold = p50 * (U64)(THRESHOLD_BASE + penalty) / THRESHOLD_PENALTY_RATE; + return deviation >= threshold; + } +} + +static void mergeEvents(Fingerprint* acc, const Fingerprint* newfp) +{ + size_t n; + for (n = 0; n < HASHTABLESIZE; n++) { + acc->events[n] += newfp->events[n]; + } + acc->nbEvents += newfp->nbEvents; +} + +static void flushEvents(FPStats* fpstats) +{ + size_t n; + for (n = 0; n < HASHTABLESIZE; n++) { + fpstats->pastEvents.events[n] = fpstats->newEvents.events[n]; + } + fpstats->pastEvents.nbEvents = fpstats->newEvents.nbEvents; + ZSTD_memset(&fpstats->newEvents, 0, sizeof(fpstats->newEvents)); +} + +static void removeEvents(Fingerprint* acc, const Fingerprint* slice) +{ + size_t n; + for (n = 0; n < HASHTABLESIZE; n++) { + assert(acc->events[n] >= slice->events[n]); + acc->events[n] -= slice->events[n]; + } + acc->nbEvents -= slice->nbEvents; +} + +#define CHUNKSIZE (8 << 10) +/* Note: technically, we use CHUNKSIZE, so that's 8 KB */ +static size_t ZSTD_splitBlock_byChunks(const void* src, size_t srcSize, + size_t blockSizeMax, addEvents_f f, + void* workspace, size_t wkspSize) +{ + FPStats* const fpstats = (FPStats*)workspace; + const char* p = (const char*)src; + int penalty = THRESHOLD_PENALTY; + size_t pos = 0; + if (srcSize <= blockSizeMax) return srcSize; + assert(blockSizeMax == (128 << 10)); + assert(workspace != NULL); + assert((size_t)workspace % ZSTD_ALIGNOF(FPStats) == 0); + ZSTD_STATIC_ASSERT(ZSTD_SLIPBLOCK_WORKSPACESIZE >= sizeof(FPStats)); + assert(wkspSize >= sizeof(FPStats)); (void)wkspSize; + + initStats(fpstats); + recordFingerprint(&fpstats->pastEvents, p, CHUNKSIZE, f); + for (pos = CHUNKSIZE; pos <= blockSizeMax - CHUNKSIZE; pos += CHUNKSIZE) { + recordFingerprint(&fpstats->newEvents, p + pos, CHUNKSIZE, f); + if (compareFingerprints(&fpstats->pastEvents, &fpstats->newEvents, penalty)) { + return pos; + } else { + mergeEvents(&fpstats->pastEvents, &fpstats->newEvents); + if (penalty > 0) penalty--; + } + } + assert(pos == blockSizeMax); + return blockSizeMax; + (void)flushEvents; (void)removeEvents; +} + +size_t ZSTD_splitBlock(const void* src, size_t srcSize, + size_t blockSizeMax, ZSTD_SplitBlock_strategy_e splitStrat, + void* workspace, size_t wkspSize) +{ + if (splitStrat == split_lvl2) + return ZSTD_splitBlock_byChunks(src, srcSize, blockSizeMax, ADDEVENTS_RATE(1), workspace, wkspSize); + + assert(splitStrat == split_lvl1); + return ZSTD_splitBlock_byChunks(src, srcSize, blockSizeMax, ADDEVENTS_RATE(5), workspace, wkspSize); +} diff --git a/lib/compress/zstd_preSplit.h b/lib/compress/zstd_preSplit.h new file mode 100644 index 000000000..2c87d34a6 --- /dev/null +++ b/lib/compress/zstd_preSplit.h @@ -0,0 +1,40 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_PRESPLIT_H +#define ZSTD_PRESPLIT_H + +#include /* size_t */ + +#if defined (__cplusplus) +extern "C" { +#endif + +typedef enum { split_lvl1, split_lvl2 } ZSTD_SplitBlock_strategy_e; + +#define ZSTD_SLIPBLOCK_WORKSPACESIZE 8208 + +/* note: + * @workspace must be aligned on 8-bytes boundaries + * @wkspSize must be at least >= ZSTD_SLIPBLOCK_WORKSPACESIZE + * note2: + * for the time being, this function only accepts full 128 KB blocks, + * therefore @blockSizeMax must be == 128 KB. + * This could be extended to smaller sizes in the future. + */ +size_t ZSTD_splitBlock(const void* src, size_t srcSize, + size_t blockSizeMax, ZSTD_SplitBlock_strategy_e splitStrat, + void* workspace, size_t wkspSize); + +#if defined (__cplusplus) +} +#endif + +#endif /* ZSTD_PRESPLIT_H */ diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index 26c9457fb..6aac5dd3f 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -1057,7 +1057,7 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx, } ZSTD_DCtx_trace_end(dctx, (U64)(op-ostart), (U64)(ip-istart), /* streaming */ 0); /* Allow caller to get size read */ - DEBUGLOG(4, "ZSTD_decompressFrame: decompressed frame of size %zi, consuming %zi bytes of input", op-ostart, ip - (const BYTE*)*srcPtr); + DEBUGLOG(4, "ZSTD_decompressFrame: decompressed frame of size %i, consuming %i bytes of input", (int)(op-ostart), (int)(ip - (const BYTE*)*srcPtr)); *srcPtr = ip; *srcSizePtr = remainingSrcSize; return (size_t)(op-ostart); diff --git a/tests/fuzzer.c b/tests/fuzzer.c index ead3cc8dd..f5a894354 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -3653,16 +3653,19 @@ static int basicUnitTests(U32 const seed, double compressibility) ZSTD_freeDCtx(dctx); } - /* long rle test */ + /* rle detection test: must compress better blocks with a single identical byte repeated */ { size_t sampleSize = 0; - size_t expectedCompressedSize = 39; /* block 1, 2: compressed, block 3: RLE, zstd 1.4.4 */ - DISPLAYLEVEL(3, "test%3i : Long RLE test : ", testNb++); - memset((char*)CNBuffer+sampleSize, 'B', 256 KB - 1); - sampleSize += 256 KB - 1; - memset((char*)CNBuffer+sampleSize, 'A', 96 KB); - sampleSize += 96 KB; + size_t maxCompressedSize = 46; /* block 1, 2: compressed, block 3: RLE, zstd 1.4.4 */ + DISPLAYLEVEL(3, "test%3i : RLE detection test : ", testNb++); + memset((char*)CNBuffer+sampleSize, 'B', 256 KB - 2); + sampleSize += 256 KB - 2; + memset((char*)CNBuffer+sampleSize, 'A', 100 KB); + sampleSize += 100 KB; cSize = ZSTD_compress(compressedBuffer, ZSTD_compressBound(sampleSize), CNBuffer, sampleSize, 1); - if (ZSTD_isError(cSize) || cSize > expectedCompressedSize) goto _output_error; + if (ZSTD_isError(cSize) || cSize > maxCompressedSize) { + DISPLAYLEVEL(4, "error: cSize %u > %u expected ! \n", (unsigned)cSize, (unsigned)maxCompressedSize); + goto _output_error; + } { CHECK_NEWV(regenSize, ZSTD_decompress(decodedBuffer, sampleSize, compressedBuffer, cSize)); if (regenSize!=sampleSize) goto _output_error; } DISPLAYLEVEL(3, "OK \n"); diff --git a/tests/regression/results.csv b/tests/regression/results.csv index e39abd66d..ea5ed58d5 100644 --- a/tests/regression/results.csv +++ b/tests/regression/results.csv @@ -1,63 +1,63 @@ Data, Config, Method, Total compressed size -silesia.tar, level -5, compress simple, 6861055 -silesia.tar, level -3, compress simple, 6505483 -silesia.tar, level -1, compress simple, 6179047 -silesia.tar, level 0, compress simple, 4851407 -silesia.tar, level 1, compress simple, 5327717 -silesia.tar, level 3, compress simple, 4851407 -silesia.tar, level 4, compress simple, 4788821 -silesia.tar, level 5, compress simple, 4679004 -silesia.tar, level 6, compress simple, 4614561 -silesia.tar, level 7, compress simple, 4579828 -silesia.tar, level 9, compress simple, 4555448 -silesia.tar, level 13, compress simple, 4502956 -silesia.tar, level 16, compress simple, 4360385 -silesia.tar, level 19, compress simple, 4260939 -silesia.tar, uncompressed literals, compress simple, 4851407 -silesia.tar, uncompressed literals optimal, compress simple, 4260939 -silesia.tar, huffman literals, compress simple, 6179047 -github.tar, level -5, compress simple, 52115 -github.tar, level -3, compress simple, 45678 -github.tar, level -1, compress simple, 42560 -github.tar, level 0, compress simple, 38884 -github.tar, level 1, compress simple, 39200 -github.tar, level 3, compress simple, 38884 -github.tar, level 4, compress simple, 38880 -github.tar, level 5, compress simple, 39651 -github.tar, level 6, compress simple, 39282 -github.tar, level 7, compress simple, 38005 +silesia.tar, level -5, compress simple, 6860782 +silesia.tar, level -3, compress simple, 6507448 +silesia.tar, level -1, compress simple, 6175582 +silesia.tar, level 0, compress simple, 4843429 +silesia.tar, level 1, compress simple, 5316794 +silesia.tar, level 3, compress simple, 4843429 +silesia.tar, level 4, compress simple, 4780657 +silesia.tar, level 5, compress simple, 4669650 +silesia.tar, level 6, compress simple, 4604925 +silesia.tar, level 7, compress simple, 4570873 +silesia.tar, level 9, compress simple, 4537558 +silesia.tar, level 13, compress simple, 4484732 +silesia.tar, level 16, compress simple, 4355572 +silesia.tar, level 19, compress simple, 4257629 +silesia.tar, uncompressed literals, compress simple, 4843429 +silesia.tar, uncompressed literals optimal, compress simple, 4257629 +silesia.tar, huffman literals, compress simple, 6175582 +github.tar, level -5, compress simple, 52320 +github.tar, level -3, compress simple, 45823 +github.tar, level -1, compress simple, 42740 +github.tar, level 0, compress simple, 39073 +github.tar, level 1, compress simple, 39444 +github.tar, level 3, compress simple, 39073 +github.tar, level 4, compress simple, 39068 +github.tar, level 5, compress simple, 39808 +github.tar, level 6, compress simple, 39455 +github.tar, level 7, compress simple, 38168 github.tar, level 9, compress simple, 36723 github.tar, level 13, compress simple, 35501 github.tar, level 16, compress simple, 40466 github.tar, level 19, compress simple, 32262 -github.tar, uncompressed literals, compress simple, 38884 +github.tar, uncompressed literals, compress simple, 39073 github.tar, uncompressed literals optimal, compress simple, 32262 -github.tar, huffman literals, compress simple, 42560 -silesia, level -5, compress cctx, 6857372 -silesia, level -3, compress cctx, 6503412 -silesia, level -1, compress cctx, 6172202 -silesia, level 0, compress cctx, 4839431 -silesia, level 1, compress cctx, 5306632 -silesia, level 3, compress cctx, 4839431 -silesia, level 4, compress cctx, 4776557 -silesia, level 5, compress cctx, 4667668 -silesia, level 6, compress cctx, 4604351 -silesia, level 7, compress cctx, 4570271 -silesia, level 9, compress cctx, 4545850 -silesia, level 13, compress cctx, 4493990 -silesia, level 16, compress cctx, 4359652 -silesia, level 19, compress cctx, 4266582 -silesia, long distance mode, compress cctx, 4839431 -silesia, multithreaded, compress cctx, 4839431 -silesia, multithreaded long distance mode, compress cctx, 4839431 +github.tar, huffman literals, compress simple, 42740 +silesia, level -5, compress cctx, 6857776 +silesia, level -3, compress cctx, 6504756 +silesia, level -1, compress cctx, 6174127 +silesia, level 0, compress cctx, 4838417 +silesia, level 1, compress cctx, 5306931 +silesia, level 3, compress cctx, 4838417 +silesia, level 4, compress cctx, 4775359 +silesia, level 5, compress cctx, 4666524 +silesia, level 6, compress cctx, 4602031 +silesia, level 7, compress cctx, 4567954 +silesia, level 9, compress cctx, 4540520 +silesia, level 13, compress cctx, 4488969 +silesia, level 16, compress cctx, 4356799 +silesia, level 19, compress cctx, 4265851 +silesia, long distance mode, compress cctx, 4838417 +silesia, multithreaded, compress cctx, 4838417 +silesia, multithreaded long distance mode, compress cctx, 4838417 silesia, small window log, compress cctx, 7082907 -silesia, small hash log, compress cctx, 6526141 -silesia, small chain log, compress cctx, 4912197 -silesia, explicit params, compress cctx, 4794318 -silesia, uncompressed literals, compress cctx, 4839431 -silesia, uncompressed literals optimal, compress cctx, 4266582 -silesia, huffman literals, compress cctx, 6172202 -silesia, multithreaded with advanced params, compress cctx, 4839431 +silesia, small hash log, compress cctx, 6525510 +silesia, small chain log, compress cctx, 4912248 +silesia, explicit params, compress cctx, 4792640 +silesia, uncompressed literals, compress cctx, 4838417 +silesia, uncompressed literals optimal, compress cctx, 4265851 +silesia, huffman literals, compress cctx, 6174127 +silesia, multithreaded with advanced params, compress cctx, 4838417 github, level -5, compress cctx, 204407 github, level -5 with dict, compress cctx, 47581 github, level -3, compress cctx, 193253 @@ -97,57 +97,57 @@ github, uncompressed literals, compress github, uncompressed literals optimal, compress cctx, 132879 github, huffman literals, compress cctx, 175468 github, multithreaded with advanced params, compress cctx, 141069 -silesia, level -5, zstdcli, 6857420 -silesia, level -3, zstdcli, 6503460 -silesia, level -1, zstdcli, 6172250 -silesia, level 0, zstdcli, 4839479 -silesia, level 1, zstdcli, 5306680 -silesia, level 3, zstdcli, 4839479 -silesia, level 4, zstdcli, 4776605 -silesia, level 5, zstdcli, 4667716 -silesia, level 6, zstdcli, 4604399 -silesia, level 7, zstdcli, 4570319 -silesia, level 9, zstdcli, 4545898 -silesia, level 13, zstdcli, 4494038 -silesia, level 16, zstdcli, 4359700 -silesia, level 19, zstdcli, 4266630 -silesia, long distance mode, zstdcli, 4831224 -silesia, multithreaded, zstdcli, 4839479 -silesia, multithreaded long distance mode, zstdcli, 4831224 +silesia, level -5, zstdcli, 6856073 +silesia, level -3, zstdcli, 6505580 +silesia, level -1, zstdcli, 6172649 +silesia, level 0, zstdcli, 4838997 +silesia, level 1, zstdcli, 5306179 +silesia, level 3, zstdcli, 4838997 +silesia, level 4, zstdcli, 4775769 +silesia, level 5, zstdcli, 4666931 +silesia, level 6, zstdcli, 4602509 +silesia, level 7, zstdcli, 4568265 +silesia, level 9, zstdcli, 4540123 +silesia, level 13, zstdcli, 4488478 +silesia, level 16, zstdcli, 4358150 +silesia, level 19, zstdcli, 4265929 +silesia, long distance mode, zstdcli, 4830467 +silesia, multithreaded, zstdcli, 4838997 +silesia, multithreaded long distance mode, zstdcli, 4830467 silesia, small window log, zstdcli, 7094528 -silesia, small hash log, zstdcli, 6526189 -silesia, small chain log, zstdcli, 4912245 -silesia, explicit params, zstdcli, 4795840 -silesia, uncompressed literals, zstdcli, 5117005 -silesia, uncompressed literals optimal, zstdcli, 4316928 -silesia, huffman literals, zstdcli, 5321417 -silesia, multithreaded with advanced params, zstdcli, 5117005 -silesia.tar, level -5, zstdcli, 6862049 -silesia.tar, level -3, zstdcli, 6506509 -silesia.tar, level -1, zstdcli, 6179789 -silesia.tar, level 0, zstdcli, 4851482 -silesia.tar, level 1, zstdcli, 5329010 -silesia.tar, level 3, zstdcli, 4851482 -silesia.tar, level 4, zstdcli, 4789729 -silesia.tar, level 5, zstdcli, 4679860 -silesia.tar, level 6, zstdcli, 4615355 -silesia.tar, level 7, zstdcli, 4581791 -silesia.tar, level 9, zstdcli, 4555452 -silesia.tar, level 13, zstdcli, 4502960 -silesia.tar, level 16, zstdcli, 4360389 -silesia.tar, level 19, zstdcli, 4260943 -silesia.tar, no source size, zstdcli, 4851478 -silesia.tar, long distance mode, zstdcli, 4843159 -silesia.tar, multithreaded, zstdcli, 4851482 -silesia.tar, multithreaded long distance mode, zstdcli, 4843159 +silesia, small hash log, zstdcli, 6527214 +silesia, small chain log, zstdcli, 4911647 +silesia, explicit params, zstdcli, 4794378 +silesia, uncompressed literals, zstdcli, 5117843 +silesia, uncompressed literals optimal, zstdcli, 4316761 +silesia, huffman literals, zstdcli, 5320793 +silesia, multithreaded with advanced params, zstdcli, 5117843 +silesia.tar, level -5, zstdcli, 6860173 +silesia.tar, level -3, zstdcli, 6507196 +silesia.tar, level -1, zstdcli, 6177719 +silesia.tar, level 0, zstdcli, 4841906 +silesia.tar, level 1, zstdcli, 5313917 +silesia.tar, level 3, zstdcli, 4841906 +silesia.tar, level 4, zstdcli, 4780121 +silesia.tar, level 5, zstdcli, 4671144 +silesia.tar, level 6, zstdcli, 4606407 +silesia.tar, level 7, zstdcli, 4572666 +silesia.tar, level 9, zstdcli, 4542599 +silesia.tar, level 13, zstdcli, 4489976 +silesia.tar, level 16, zstdcli, 4356959 +silesia.tar, level 19, zstdcli, 4259787 +silesia.tar, no source size, zstdcli, 4841902 +silesia.tar, long distance mode, zstdcli, 4834266 +silesia.tar, multithreaded, zstdcli, 4841906 +silesia.tar, multithreaded long distance mode, zstdcli, 4834266 silesia.tar, small window log, zstdcli, 7100110 -silesia.tar, small hash log, zstdcli, 6529264 -silesia.tar, small chain log, zstdcli, 4917022 -silesia.tar, explicit params, zstdcli, 4821112 -silesia.tar, uncompressed literals, zstdcli, 5118944 -silesia.tar, uncompressed literals optimal, zstdcli, 4308455 -silesia.tar, huffman literals, zstdcli, 5342074 -silesia.tar, multithreaded with advanced params, zstdcli, 5118944 +silesia.tar, small hash log, zstdcli, 6530007 +silesia.tar, small chain log, zstdcli, 4916091 +silesia.tar, explicit params, zstdcli, 4809323 +silesia.tar, uncompressed literals, zstdcli, 5116446 +silesia.tar, uncompressed literals optimal, zstdcli, 4306844 +silesia.tar, huffman literals, zstdcli, 5326463 +silesia.tar, multithreaded with advanced params, zstdcli, 5116446 github, level -5, zstdcli, 206407 github, level -5 with dict, zstdcli, 47832 github, level -3, zstdcli, 195253 @@ -187,25 +187,25 @@ github, uncompressed literals, zstdcli, github, uncompressed literals optimal, zstdcli, 154667 github, huffman literals, zstdcli, 144365 github, multithreaded with advanced params, zstdcli, 167909 -github.tar, level -5, zstdcli, 52119 -github.tar, level -5 with dict, zstdcli, 51101 -github.tar, level -3, zstdcli, 45682 -github.tar, level -3 with dict, zstdcli, 44738 -github.tar, level -1, zstdcli, 42564 -github.tar, level -1 with dict, zstdcli, 41357 -github.tar, level 0, zstdcli, 38888 -github.tar, level 0 with dict, zstdcli, 37999 -github.tar, level 1, zstdcli, 39204 -github.tar, level 1 with dict, zstdcli, 38123 -github.tar, level 3, zstdcli, 38888 -github.tar, level 3 with dict, zstdcli, 37999 -github.tar, level 4, zstdcli, 38884 -github.tar, level 4 with dict, zstdcli, 37952 -github.tar, level 5, zstdcli, 39655 -github.tar, level 5 with dict, zstdcli, 39073 -github.tar, level 6, zstdcli, 39286 -github.tar, level 6 with dict, zstdcli, 38647 -github.tar, level 7, zstdcli, 38009 +github.tar, level -5, zstdcli, 52308 +github.tar, level -5 with dict, zstdcli, 51318 +github.tar, level -3, zstdcli, 45815 +github.tar, level -3 with dict, zstdcli, 44883 +github.tar, level -1, zstdcli, 42688 +github.tar, level -1 with dict, zstdcli, 41510 +github.tar, level 0, zstdcli, 39071 +github.tar, level 0 with dict, zstdcli, 38140 +github.tar, level 1, zstdcli, 39404 +github.tar, level 1 with dict, zstdcli, 38320 +github.tar, level 3, zstdcli, 39071 +github.tar, level 3 with dict, zstdcli, 38140 +github.tar, level 4, zstdcli, 39067 +github.tar, level 4 with dict, zstdcli, 38082 +github.tar, level 5, zstdcli, 39815 +github.tar, level 5 with dict, zstdcli, 39221 +github.tar, level 6, zstdcli, 39455 +github.tar, level 6 with dict, zstdcli, 38787 +github.tar, level 7, zstdcli, 38177 github.tar, level 7 with dict, zstdcli, 37861 github.tar, level 9, zstdcli, 36727 github.tar, level 9 with dict, zstdcli, 36686 @@ -215,87 +215,87 @@ github.tar, level 16, zstdcli, github.tar, level 16 with dict, zstdcli, 33379 github.tar, level 19, zstdcli, 32266 github.tar, level 19 with dict, zstdcli, 32705 -github.tar, no source size, zstdcli, 38885 -github.tar, no source size with dict, zstdcli, 38115 -github.tar, long distance mode, zstdcli, 40227 -github.tar, multithreaded, zstdcli, 38888 -github.tar, multithreaded long distance mode, zstdcli, 40227 +github.tar, no source size, zstdcli, 39068 +github.tar, no source size with dict, zstdcli, 38274 +github.tar, long distance mode, zstdcli, 40381 +github.tar, multithreaded, zstdcli, 39071 +github.tar, multithreaded long distance mode, zstdcli, 40381 github.tar, small window log, zstdcli, 198539 github.tar, small hash log, zstdcli, 129874 github.tar, small chain log, zstdcli, 41673 -github.tar, explicit params, zstdcli, 41385 -github.tar, uncompressed literals, zstdcli, 41566 +github.tar, explicit params, zstdcli, 41588 +github.tar, uncompressed literals, zstdcli, 41704 github.tar, uncompressed literals optimal, zstdcli, 35360 -github.tar, huffman literals, zstdcli, 38857 -github.tar, multithreaded with advanced params, zstdcli, 41566 -silesia, level -5, advanced one pass, 6857372 -silesia, level -3, advanced one pass, 6503412 -silesia, level -1, advanced one pass, 6172202 -silesia, level 0, advanced one pass, 4839431 -silesia, level 1, advanced one pass, 5306632 -silesia, level 3, advanced one pass, 4839431 -silesia, level 4, advanced one pass, 4776557 -silesia, level 5 row 1, advanced one pass, 4667668 -silesia, level 5 row 2, advanced one pass, 4670326 -silesia, level 5, advanced one pass, 4667668 -silesia, level 6, advanced one pass, 4604351 -silesia, level 7 row 1, advanced one pass, 4570271 -silesia, level 7 row 2, advanced one pass, 4565169 -silesia, level 7, advanced one pass, 4570271 -silesia, level 9, advanced one pass, 4545850 -silesia, level 11 row 1, advanced one pass, 4505658 -silesia, level 11 row 2, advanced one pass, 4503429 -silesia, level 12 row 1, advanced one pass, 4505658 -silesia, level 12 row 2, advanced one pass, 4503429 -silesia, level 13, advanced one pass, 4493990 -silesia, level 16, advanced one pass, 4359652 -silesia, level 19, advanced one pass, 4266582 -silesia, no source size, advanced one pass, 4839431 -silesia, long distance mode, advanced one pass, 4831158 -silesia, multithreaded, advanced one pass, 4839431 -silesia, multithreaded long distance mode, advanced one pass, 4831176 +github.tar, huffman literals, zstdcli, 39046 +github.tar, multithreaded with advanced params, zstdcli, 41704 +silesia, level -5, advanced one pass, 6857776 +silesia, level -3, advanced one pass, 6504756 +silesia, level -1, advanced one pass, 6174127 +silesia, level 0, advanced one pass, 4838417 +silesia, level 1, advanced one pass, 5306931 +silesia, level 3, advanced one pass, 4838417 +silesia, level 4, advanced one pass, 4775359 +silesia, level 5 row 1, advanced one pass, 4666524 +silesia, level 5 row 2, advanced one pass, 4668976 +silesia, level 5, advanced one pass, 4666524 +silesia, level 6, advanced one pass, 4602031 +silesia, level 7 row 1, advanced one pass, 4567954 +silesia, level 7 row 2, advanced one pass, 4562774 +silesia, level 7, advanced one pass, 4567954 +silesia, level 9, advanced one pass, 4540520 +silesia, level 11 row 1, advanced one pass, 4500472 +silesia, level 11 row 2, advanced one pass, 4498174 +silesia, level 12 row 1, advanced one pass, 4500472 +silesia, level 12 row 2, advanced one pass, 4498174 +silesia, level 13, advanced one pass, 4488969 +silesia, level 16, advanced one pass, 4356799 +silesia, level 19, advanced one pass, 4265851 +silesia, no source size, advanced one pass, 4838417 +silesia, long distance mode, advanced one pass, 4830097 +silesia, multithreaded, advanced one pass, 4838949 +silesia, multithreaded long distance mode, advanced one pass, 4830419 silesia, small window log, advanced one pass, 7094480 -silesia, small hash log, advanced one pass, 6526141 -silesia, small chain log, advanced one pass, 4912197 -silesia, explicit params, advanced one pass, 4795840 -silesia, uncompressed literals, advanced one pass, 5116957 -silesia, uncompressed literals optimal, advanced one pass, 4316880 -silesia, huffman literals, advanced one pass, 5321369 -silesia, multithreaded with advanced params, advanced one pass, 5116957 -silesia.tar, level -5, advanced one pass, 6861055 -silesia.tar, level -3, advanced one pass, 6505483 -silesia.tar, level -1, advanced one pass, 6179047 -silesia.tar, level 0, advanced one pass, 4851407 -silesia.tar, level 1, advanced one pass, 5327717 -silesia.tar, level 3, advanced one pass, 4851407 -silesia.tar, level 4, advanced one pass, 4788821 -silesia.tar, level 5 row 1, advanced one pass, 4679004 -silesia.tar, level 5 row 2, advanced one pass, 4682334 -silesia.tar, level 5, advanced one pass, 4679004 -silesia.tar, level 6, advanced one pass, 4614561 -silesia.tar, level 7 row 1, advanced one pass, 4579828 -silesia.tar, level 7 row 2, advanced one pass, 4575602 -silesia.tar, level 7, advanced one pass, 4579828 -silesia.tar, level 9, advanced one pass, 4555448 -silesia.tar, level 11 row 1, advanced one pass, 4514962 -silesia.tar, level 11 row 2, advanced one pass, 4513816 -silesia.tar, level 12 row 1, advanced one pass, 4514517 -silesia.tar, level 12 row 2, advanced one pass, 4514007 -silesia.tar, level 13, advanced one pass, 4502956 -silesia.tar, level 16, advanced one pass, 4360385 -silesia.tar, level 19, advanced one pass, 4260939 -silesia.tar, no source size, advanced one pass, 4851407 -silesia.tar, long distance mode, advanced one pass, 4837775 -silesia.tar, multithreaded, advanced one pass, 4851478 -silesia.tar, multithreaded long distance mode, advanced one pass, 4843155 +silesia, small hash log, advanced one pass, 6525510 +silesia, small chain log, advanced one pass, 4912248 +silesia, explicit params, advanced one pass, 4794219 +silesia, uncompressed literals, advanced one pass, 5117482 +silesia, uncompressed literals optimal, advanced one pass, 4316644 +silesia, huffman literals, advanced one pass, 5321686 +silesia, multithreaded with advanced params, advanced one pass, 5117795 +silesia.tar, level -5, advanced one pass, 6860782 +silesia.tar, level -3, advanced one pass, 6507448 +silesia.tar, level -1, advanced one pass, 6175582 +silesia.tar, level 0, advanced one pass, 4843429 +silesia.tar, level 1, advanced one pass, 5316794 +silesia.tar, level 3, advanced one pass, 4843429 +silesia.tar, level 4, advanced one pass, 4780657 +silesia.tar, level 5 row 1, advanced one pass, 4669650 +silesia.tar, level 5 row 2, advanced one pass, 4673119 +silesia.tar, level 5, advanced one pass, 4669650 +silesia.tar, level 6, advanced one pass, 4604925 +silesia.tar, level 7 row 1, advanced one pass, 4570873 +silesia.tar, level 7 row 2, advanced one pass, 4566314 +silesia.tar, level 7, advanced one pass, 4570873 +silesia.tar, level 9, advanced one pass, 4537558 +silesia.tar, level 11 row 1, advanced one pass, 4496590 +silesia.tar, level 11 row 2, advanced one pass, 4495225 +silesia.tar, level 12 row 1, advanced one pass, 4496084 +silesia.tar, level 12 row 2, advanced one pass, 4495434 +silesia.tar, level 13, advanced one pass, 4484732 +silesia.tar, level 16, advanced one pass, 4355572 +silesia.tar, level 19, advanced one pass, 4257629 +silesia.tar, no source size, advanced one pass, 4843429 +silesia.tar, long distance mode, advanced one pass, 4830453 +silesia.tar, multithreaded, advanced one pass, 4841902 +silesia.tar, multithreaded long distance mode, advanced one pass, 4834262 silesia.tar, small window log, advanced one pass, 7100064 -silesia.tar, small hash log, advanced one pass, 6529206 -silesia.tar, small chain log, advanced one pass, 4917041 -silesia.tar, explicit params, advanced one pass, 4807274 -silesia.tar, uncompressed literals, advanced one pass, 5118848 -silesia.tar, uncompressed literals optimal, advanced one pass, 4308451 -silesia.tar, huffman literals, advanced one pass, 5341705 -silesia.tar, multithreaded with advanced params, advanced one pass, 5118940 +silesia.tar, small hash log, advanced one pass, 6530222 +silesia.tar, small chain log, advanced one pass, 4915689 +silesia.tar, explicit params, advanced one pass, 4797958 +silesia.tar, uncompressed literals, advanced one pass, 5116329 +silesia.tar, uncompressed literals optimal, advanced one pass, 4306289 +silesia.tar, huffman literals, advanced one pass, 5331382 +silesia.tar, multithreaded with advanced params, advanced one pass, 5116442 github, level -5, advanced one pass, 204407 github, level -5 with dict, advanced one pass, 45832 github, level -3, advanced one pass, 193253 @@ -421,74 +421,74 @@ github, uncompressed literals, advanced github, uncompressed literals optimal, advanced one pass, 152667 github, huffman literals, advanced one pass, 142365 github, multithreaded with advanced params, advanced one pass, 165909 -github.tar, level -5, advanced one pass, 52115 -github.tar, level -5 with dict, advanced one pass, 51097 -github.tar, level -3, advanced one pass, 45678 -github.tar, level -3 with dict, advanced one pass, 44734 -github.tar, level -1, advanced one pass, 42560 -github.tar, level -1 with dict, advanced one pass, 41353 -github.tar, level 0, advanced one pass, 38884 -github.tar, level 0 with dict, advanced one pass, 37995 -github.tar, level 0 with dict dms, advanced one pass, 38114 -github.tar, level 0 with dict dds, advanced one pass, 38114 -github.tar, level 0 with dict copy, advanced one pass, 37995 -github.tar, level 0 with dict load, advanced one pass, 37956 -github.tar, level 1, advanced one pass, 39200 -github.tar, level 1 with dict, advanced one pass, 38119 -github.tar, level 1 with dict dms, advanced one pass, 38406 -github.tar, level 1 with dict dds, advanced one pass, 38406 -github.tar, level 1 with dict copy, advanced one pass, 38119 -github.tar, level 1 with dict load, advanced one pass, 38364 -github.tar, level 3, advanced one pass, 38884 -github.tar, level 3 with dict, advanced one pass, 37995 -github.tar, level 3 with dict dms, advanced one pass, 38114 -github.tar, level 3 with dict dds, advanced one pass, 38114 -github.tar, level 3 with dict copy, advanced one pass, 37995 -github.tar, level 3 with dict load, advanced one pass, 37956 -github.tar, level 4, advanced one pass, 38880 -github.tar, level 4 with dict, advanced one pass, 37948 -github.tar, level 4 with dict dms, advanced one pass, 37995 -github.tar, level 4 with dict dds, advanced one pass, 37995 -github.tar, level 4 with dict copy, advanced one pass, 37948 -github.tar, level 4 with dict load, advanced one pass, 37927 -github.tar, level 5 row 1, advanced one pass, 39651 -github.tar, level 5 row 1 with dict dms, advanced one pass, 39043 -github.tar, level 5 row 1 with dict dds, advanced one pass, 39069 -github.tar, level 5 row 1 with dict copy, advanced one pass, 39145 -github.tar, level 5 row 1 with dict load, advanced one pass, 39000 -github.tar, level 5 row 2, advanced one pass, 39701 -github.tar, level 5 row 2 with dict dms, advanced one pass, 39365 -github.tar, level 5 row 2 with dict dds, advanced one pass, 39233 -github.tar, level 5 row 2 with dict copy, advanced one pass, 39715 -github.tar, level 5 row 2 with dict load, advanced one pass, 39158 -github.tar, level 5, advanced one pass, 39651 -github.tar, level 5 with dict, advanced one pass, 39145 -github.tar, level 5 with dict dms, advanced one pass, 39043 -github.tar, level 5 with dict dds, advanced one pass, 39069 -github.tar, level 5 with dict copy, advanced one pass, 39145 -github.tar, level 5 with dict load, advanced one pass, 39000 -github.tar, level 6, advanced one pass, 39282 -github.tar, level 6 with dict, advanced one pass, 38656 -github.tar, level 6 with dict dms, advanced one pass, 38640 -github.tar, level 6 with dict dds, advanced one pass, 38643 -github.tar, level 6 with dict copy, advanced one pass, 38656 -github.tar, level 6 with dict load, advanced one pass, 38647 -github.tar, level 7 row 1, advanced one pass, 38005 +github.tar, level -5, advanced one pass, 52320 +github.tar, level -5 with dict, advanced one pass, 51321 +github.tar, level -3, advanced one pass, 45823 +github.tar, level -3 with dict, advanced one pass, 44882 +github.tar, level -1, advanced one pass, 42740 +github.tar, level -1 with dict, advanced one pass, 41506 +github.tar, level 0, advanced one pass, 39073 +github.tar, level 0 with dict, advanced one pass, 38146 +github.tar, level 0 with dict dms, advanced one pass, 38262 +github.tar, level 0 with dict dds, advanced one pass, 38262 +github.tar, level 0 with dict copy, advanced one pass, 38146 +github.tar, level 0 with dict load, advanced one pass, 38107 +github.tar, level 1, advanced one pass, 39444 +github.tar, level 1 with dict, advanced one pass, 38338 +github.tar, level 1 with dict dms, advanced one pass, 38641 +github.tar, level 1 with dict dds, advanced one pass, 38641 +github.tar, level 1 with dict copy, advanced one pass, 38338 +github.tar, level 1 with dict load, advanced one pass, 38592 +github.tar, level 3, advanced one pass, 39073 +github.tar, level 3 with dict, advanced one pass, 38146 +github.tar, level 3 with dict dms, advanced one pass, 38262 +github.tar, level 3 with dict dds, advanced one pass, 38262 +github.tar, level 3 with dict copy, advanced one pass, 38146 +github.tar, level 3 with dict load, advanced one pass, 38107 +github.tar, level 4, advanced one pass, 39068 +github.tar, level 4 with dict, advanced one pass, 38081 +github.tar, level 4 with dict dms, advanced one pass, 38135 +github.tar, level 4 with dict dds, advanced one pass, 38135 +github.tar, level 4 with dict copy, advanced one pass, 38081 +github.tar, level 4 with dict load, advanced one pass, 38088 +github.tar, level 5 row 1, advanced one pass, 39808 +github.tar, level 5 row 1 with dict dms, advanced one pass, 39185 +github.tar, level 5 row 1 with dict dds, advanced one pass, 39213 +github.tar, level 5 row 1 with dict copy, advanced one pass, 39284 +github.tar, level 5 row 1 with dict load, advanced one pass, 39155 +github.tar, level 5 row 2, advanced one pass, 39851 +github.tar, level 5 row 2 with dict dms, advanced one pass, 39498 +github.tar, level 5 row 2 with dict dds, advanced one pass, 39364 +github.tar, level 5 row 2 with dict copy, advanced one pass, 39843 +github.tar, level 5 row 2 with dict load, advanced one pass, 39312 +github.tar, level 5, advanced one pass, 39808 +github.tar, level 5 with dict, advanced one pass, 39284 +github.tar, level 5 with dict dms, advanced one pass, 39185 +github.tar, level 5 with dict dds, advanced one pass, 39213 +github.tar, level 5 with dict copy, advanced one pass, 39284 +github.tar, level 5 with dict load, advanced one pass, 39155 +github.tar, level 6, advanced one pass, 39455 +github.tar, level 6 with dict, advanced one pass, 38808 +github.tar, level 6 with dict dms, advanced one pass, 38792 +github.tar, level 6 with dict dds, advanced one pass, 38790 +github.tar, level 6 with dict copy, advanced one pass, 38808 +github.tar, level 6 with dict load, advanced one pass, 38796 +github.tar, level 7 row 1, advanced one pass, 38168 github.tar, level 7 row 1 with dict dms, advanced one pass, 37832 github.tar, level 7 row 1 with dict dds, advanced one pass, 37857 github.tar, level 7 row 1 with dict copy, advanced one pass, 37839 -github.tar, level 7 row 1 with dict load, advanced one pass, 37286 -github.tar, level 7 row 2, advanced one pass, 38077 +github.tar, level 7 row 1 with dict load, advanced one pass, 37445 +github.tar, level 7 row 2, advanced one pass, 38235 github.tar, level 7 row 2 with dict dms, advanced one pass, 38012 github.tar, level 7 row 2 with dict dds, advanced one pass, 38014 github.tar, level 7 row 2 with dict copy, advanced one pass, 38101 -github.tar, level 7 row 2 with dict load, advanced one pass, 37402 -github.tar, level 7, advanced one pass, 38005 +github.tar, level 7 row 2 with dict load, advanced one pass, 37552 +github.tar, level 7, advanced one pass, 38168 github.tar, level 7 with dict, advanced one pass, 37839 github.tar, level 7 with dict dms, advanced one pass, 37832 github.tar, level 7 with dict dds, advanced one pass, 37857 github.tar, level 7 with dict copy, advanced one pass, 37839 -github.tar, level 7 with dict load, advanced one pass, 37286 +github.tar, level 7 with dict load, advanced one pass, 37445 github.tar, level 9, advanced one pass, 36723 github.tar, level 9 with dict, advanced one pass, 36531 github.tar, level 9 with dict dms, advanced one pass, 36615 @@ -533,87 +533,87 @@ github.tar, level 19 with dict dms, advanced github.tar, level 19 with dict dds, advanced one pass, 32565 github.tar, level 19 with dict copy, advanced one pass, 32701 github.tar, level 19 with dict load, advanced one pass, 32428 -github.tar, no source size, advanced one pass, 38884 -github.tar, no source size with dict, advanced one pass, 37995 -github.tar, long distance mode, advanced one pass, 40242 -github.tar, multithreaded, advanced one pass, 38884 -github.tar, multithreaded long distance mode, advanced one pass, 40223 +github.tar, no source size, advanced one pass, 39073 +github.tar, no source size with dict, advanced one pass, 38146 +github.tar, long distance mode, advanced one pass, 40402 +github.tar, multithreaded, advanced one pass, 39067 +github.tar, multithreaded long distance mode, advanced one pass, 40377 github.tar, small window log, advanced one pass, 198535 github.tar, small hash log, advanced one pass, 129870 github.tar, small chain log, advanced one pass, 41669 -github.tar, explicit params, advanced one pass, 41385 -github.tar, uncompressed literals, advanced one pass, 41562 +github.tar, explicit params, advanced one pass, 41605 +github.tar, uncompressed literals, advanced one pass, 41709 github.tar, uncompressed literals optimal, advanced one pass, 35356 -github.tar, huffman literals, advanced one pass, 38853 -github.tar, multithreaded with advanced params, advanced one pass, 41562 -silesia, level -5, advanced one pass small out, 6857372 -silesia, level -3, advanced one pass small out, 6503412 -silesia, level -1, advanced one pass small out, 6172202 -silesia, level 0, advanced one pass small out, 4839431 -silesia, level 1, advanced one pass small out, 5306632 -silesia, level 3, advanced one pass small out, 4839431 -silesia, level 4, advanced one pass small out, 4776557 -silesia, level 5 row 1, advanced one pass small out, 4667668 -silesia, level 5 row 2, advanced one pass small out, 4670326 -silesia, level 5, advanced one pass small out, 4667668 -silesia, level 6, advanced one pass small out, 4604351 -silesia, level 7 row 1, advanced one pass small out, 4570271 -silesia, level 7 row 2, advanced one pass small out, 4565169 -silesia, level 7, advanced one pass small out, 4570271 -silesia, level 9, advanced one pass small out, 4545850 -silesia, level 11 row 1, advanced one pass small out, 4505658 -silesia, level 11 row 2, advanced one pass small out, 4503429 -silesia, level 12 row 1, advanced one pass small out, 4505658 -silesia, level 12 row 2, advanced one pass small out, 4503429 -silesia, level 13, advanced one pass small out, 4493990 -silesia, level 16, advanced one pass small out, 4359652 -silesia, level 19, advanced one pass small out, 4266582 -silesia, no source size, advanced one pass small out, 4839431 -silesia, long distance mode, advanced one pass small out, 4831158 -silesia, multithreaded, advanced one pass small out, 4839431 -silesia, multithreaded long distance mode, advanced one pass small out, 4831176 +github.tar, huffman literals, advanced one pass, 39099 +github.tar, multithreaded with advanced params, advanced one pass, 41700 +silesia, level -5, advanced one pass small out, 6857776 +silesia, level -3, advanced one pass small out, 6504756 +silesia, level -1, advanced one pass small out, 6174127 +silesia, level 0, advanced one pass small out, 4838417 +silesia, level 1, advanced one pass small out, 5306931 +silesia, level 3, advanced one pass small out, 4838417 +silesia, level 4, advanced one pass small out, 4775359 +silesia, level 5 row 1, advanced one pass small out, 4666524 +silesia, level 5 row 2, advanced one pass small out, 4668976 +silesia, level 5, advanced one pass small out, 4666524 +silesia, level 6, advanced one pass small out, 4602031 +silesia, level 7 row 1, advanced one pass small out, 4567954 +silesia, level 7 row 2, advanced one pass small out, 4562774 +silesia, level 7, advanced one pass small out, 4567954 +silesia, level 9, advanced one pass small out, 4540520 +silesia, level 11 row 1, advanced one pass small out, 4500472 +silesia, level 11 row 2, advanced one pass small out, 4498174 +silesia, level 12 row 1, advanced one pass small out, 4500472 +silesia, level 12 row 2, advanced one pass small out, 4498174 +silesia, level 13, advanced one pass small out, 4488969 +silesia, level 16, advanced one pass small out, 4356799 +silesia, level 19, advanced one pass small out, 4265851 +silesia, no source size, advanced one pass small out, 4838417 +silesia, long distance mode, advanced one pass small out, 4830097 +silesia, multithreaded, advanced one pass small out, 4838949 +silesia, multithreaded long distance mode, advanced one pass small out, 4830419 silesia, small window log, advanced one pass small out, 7094480 -silesia, small hash log, advanced one pass small out, 6526141 -silesia, small chain log, advanced one pass small out, 4912197 -silesia, explicit params, advanced one pass small out, 4795840 -silesia, uncompressed literals, advanced one pass small out, 5116957 -silesia, uncompressed literals optimal, advanced one pass small out, 4316880 -silesia, huffman literals, advanced one pass small out, 5321369 -silesia, multithreaded with advanced params, advanced one pass small out, 5116957 -silesia.tar, level -5, advanced one pass small out, 6861055 -silesia.tar, level -3, advanced one pass small out, 6505483 -silesia.tar, level -1, advanced one pass small out, 6179047 -silesia.tar, level 0, advanced one pass small out, 4851407 -silesia.tar, level 1, advanced one pass small out, 5327717 -silesia.tar, level 3, advanced one pass small out, 4851407 -silesia.tar, level 4, advanced one pass small out, 4788821 -silesia.tar, level 5 row 1, advanced one pass small out, 4679004 -silesia.tar, level 5 row 2, advanced one pass small out, 4682334 -silesia.tar, level 5, advanced one pass small out, 4679004 -silesia.tar, level 6, advanced one pass small out, 4614561 -silesia.tar, level 7 row 1, advanced one pass small out, 4579828 -silesia.tar, level 7 row 2, advanced one pass small out, 4575602 -silesia.tar, level 7, advanced one pass small out, 4579828 -silesia.tar, level 9, advanced one pass small out, 4555448 -silesia.tar, level 11 row 1, advanced one pass small out, 4514962 -silesia.tar, level 11 row 2, advanced one pass small out, 4513816 -silesia.tar, level 12 row 1, advanced one pass small out, 4514517 -silesia.tar, level 12 row 2, advanced one pass small out, 4514007 -silesia.tar, level 13, advanced one pass small out, 4502956 -silesia.tar, level 16, advanced one pass small out, 4360385 -silesia.tar, level 19, advanced one pass small out, 4260939 -silesia.tar, no source size, advanced one pass small out, 4851407 -silesia.tar, long distance mode, advanced one pass small out, 4837775 -silesia.tar, multithreaded, advanced one pass small out, 4851478 -silesia.tar, multithreaded long distance mode, advanced one pass small out, 4843155 +silesia, small hash log, advanced one pass small out, 6525510 +silesia, small chain log, advanced one pass small out, 4912248 +silesia, explicit params, advanced one pass small out, 4794219 +silesia, uncompressed literals, advanced one pass small out, 5117482 +silesia, uncompressed literals optimal, advanced one pass small out, 4316644 +silesia, huffman literals, advanced one pass small out, 5321686 +silesia, multithreaded with advanced params, advanced one pass small out, 5117795 +silesia.tar, level -5, advanced one pass small out, 6860782 +silesia.tar, level -3, advanced one pass small out, 6507448 +silesia.tar, level -1, advanced one pass small out, 6175582 +silesia.tar, level 0, advanced one pass small out, 4843429 +silesia.tar, level 1, advanced one pass small out, 5316794 +silesia.tar, level 3, advanced one pass small out, 4843429 +silesia.tar, level 4, advanced one pass small out, 4780657 +silesia.tar, level 5 row 1, advanced one pass small out, 4669650 +silesia.tar, level 5 row 2, advanced one pass small out, 4673119 +silesia.tar, level 5, advanced one pass small out, 4669650 +silesia.tar, level 6, advanced one pass small out, 4604925 +silesia.tar, level 7 row 1, advanced one pass small out, 4570873 +silesia.tar, level 7 row 2, advanced one pass small out, 4566314 +silesia.tar, level 7, advanced one pass small out, 4570873 +silesia.tar, level 9, advanced one pass small out, 4537558 +silesia.tar, level 11 row 1, advanced one pass small out, 4496590 +silesia.tar, level 11 row 2, advanced one pass small out, 4495225 +silesia.tar, level 12 row 1, advanced one pass small out, 4496084 +silesia.tar, level 12 row 2, advanced one pass small out, 4495434 +silesia.tar, level 13, advanced one pass small out, 4484732 +silesia.tar, level 16, advanced one pass small out, 4355572 +silesia.tar, level 19, advanced one pass small out, 4257629 +silesia.tar, no source size, advanced one pass small out, 4843429 +silesia.tar, long distance mode, advanced one pass small out, 4830453 +silesia.tar, multithreaded, advanced one pass small out, 4841902 +silesia.tar, multithreaded long distance mode, advanced one pass small out, 4834262 silesia.tar, small window log, advanced one pass small out, 7100064 -silesia.tar, small hash log, advanced one pass small out, 6529206 -silesia.tar, small chain log, advanced one pass small out, 4917041 -silesia.tar, explicit params, advanced one pass small out, 4807274 -silesia.tar, uncompressed literals, advanced one pass small out, 5118848 -silesia.tar, uncompressed literals optimal, advanced one pass small out, 4308451 -silesia.tar, huffman literals, advanced one pass small out, 5341705 -silesia.tar, multithreaded with advanced params, advanced one pass small out, 5118940 +silesia.tar, small hash log, advanced one pass small out, 6530222 +silesia.tar, small chain log, advanced one pass small out, 4915689 +silesia.tar, explicit params, advanced one pass small out, 4797958 +silesia.tar, uncompressed literals, advanced one pass small out, 5116329 +silesia.tar, uncompressed literals optimal, advanced one pass small out, 4306289 +silesia.tar, huffman literals, advanced one pass small out, 5331382 +silesia.tar, multithreaded with advanced params, advanced one pass small out, 5116442 github, level -5, advanced one pass small out, 204407 github, level -5 with dict, advanced one pass small out, 45832 github, level -3, advanced one pass small out, 193253 @@ -739,74 +739,74 @@ github, uncompressed literals, advanced github, uncompressed literals optimal, advanced one pass small out, 152667 github, huffman literals, advanced one pass small out, 142365 github, multithreaded with advanced params, advanced one pass small out, 165909 -github.tar, level -5, advanced one pass small out, 52115 -github.tar, level -5 with dict, advanced one pass small out, 51097 -github.tar, level -3, advanced one pass small out, 45678 -github.tar, level -3 with dict, advanced one pass small out, 44734 -github.tar, level -1, advanced one pass small out, 42560 -github.tar, level -1 with dict, advanced one pass small out, 41353 -github.tar, level 0, advanced one pass small out, 38884 -github.tar, level 0 with dict, advanced one pass small out, 37995 -github.tar, level 0 with dict dms, advanced one pass small out, 38114 -github.tar, level 0 with dict dds, advanced one pass small out, 38114 -github.tar, level 0 with dict copy, advanced one pass small out, 37995 -github.tar, level 0 with dict load, advanced one pass small out, 37956 -github.tar, level 1, advanced one pass small out, 39200 -github.tar, level 1 with dict, advanced one pass small out, 38119 -github.tar, level 1 with dict dms, advanced one pass small out, 38406 -github.tar, level 1 with dict dds, advanced one pass small out, 38406 -github.tar, level 1 with dict copy, advanced one pass small out, 38119 -github.tar, level 1 with dict load, advanced one pass small out, 38364 -github.tar, level 3, advanced one pass small out, 38884 -github.tar, level 3 with dict, advanced one pass small out, 37995 -github.tar, level 3 with dict dms, advanced one pass small out, 38114 -github.tar, level 3 with dict dds, advanced one pass small out, 38114 -github.tar, level 3 with dict copy, advanced one pass small out, 37995 -github.tar, level 3 with dict load, advanced one pass small out, 37956 -github.tar, level 4, advanced one pass small out, 38880 -github.tar, level 4 with dict, advanced one pass small out, 37948 -github.tar, level 4 with dict dms, advanced one pass small out, 37995 -github.tar, level 4 with dict dds, advanced one pass small out, 37995 -github.tar, level 4 with dict copy, advanced one pass small out, 37948 -github.tar, level 4 with dict load, advanced one pass small out, 37927 -github.tar, level 5 row 1, advanced one pass small out, 39651 -github.tar, level 5 row 1 with dict dms, advanced one pass small out, 39043 -github.tar, level 5 row 1 with dict dds, advanced one pass small out, 39069 -github.tar, level 5 row 1 with dict copy, advanced one pass small out, 39145 -github.tar, level 5 row 1 with dict load, advanced one pass small out, 39000 -github.tar, level 5 row 2, advanced one pass small out, 39701 -github.tar, level 5 row 2 with dict dms, advanced one pass small out, 39365 -github.tar, level 5 row 2 with dict dds, advanced one pass small out, 39233 -github.tar, level 5 row 2 with dict copy, advanced one pass small out, 39715 -github.tar, level 5 row 2 with dict load, advanced one pass small out, 39158 -github.tar, level 5, advanced one pass small out, 39651 -github.tar, level 5 with dict, advanced one pass small out, 39145 -github.tar, level 5 with dict dms, advanced one pass small out, 39043 -github.tar, level 5 with dict dds, advanced one pass small out, 39069 -github.tar, level 5 with dict copy, advanced one pass small out, 39145 -github.tar, level 5 with dict load, advanced one pass small out, 39000 -github.tar, level 6, advanced one pass small out, 39282 -github.tar, level 6 with dict, advanced one pass small out, 38656 -github.tar, level 6 with dict dms, advanced one pass small out, 38640 -github.tar, level 6 with dict dds, advanced one pass small out, 38643 -github.tar, level 6 with dict copy, advanced one pass small out, 38656 -github.tar, level 6 with dict load, advanced one pass small out, 38647 -github.tar, level 7 row 1, advanced one pass small out, 38005 +github.tar, level -5, advanced one pass small out, 52320 +github.tar, level -5 with dict, advanced one pass small out, 51321 +github.tar, level -3, advanced one pass small out, 45823 +github.tar, level -3 with dict, advanced one pass small out, 44882 +github.tar, level -1, advanced one pass small out, 42740 +github.tar, level -1 with dict, advanced one pass small out, 41506 +github.tar, level 0, advanced one pass small out, 39073 +github.tar, level 0 with dict, advanced one pass small out, 38146 +github.tar, level 0 with dict dms, advanced one pass small out, 38262 +github.tar, level 0 with dict dds, advanced one pass small out, 38262 +github.tar, level 0 with dict copy, advanced one pass small out, 38146 +github.tar, level 0 with dict load, advanced one pass small out, 38107 +github.tar, level 1, advanced one pass small out, 39444 +github.tar, level 1 with dict, advanced one pass small out, 38338 +github.tar, level 1 with dict dms, advanced one pass small out, 38641 +github.tar, level 1 with dict dds, advanced one pass small out, 38641 +github.tar, level 1 with dict copy, advanced one pass small out, 38338 +github.tar, level 1 with dict load, advanced one pass small out, 38592 +github.tar, level 3, advanced one pass small out, 39073 +github.tar, level 3 with dict, advanced one pass small out, 38146 +github.tar, level 3 with dict dms, advanced one pass small out, 38262 +github.tar, level 3 with dict dds, advanced one pass small out, 38262 +github.tar, level 3 with dict copy, advanced one pass small out, 38146 +github.tar, level 3 with dict load, advanced one pass small out, 38107 +github.tar, level 4, advanced one pass small out, 39068 +github.tar, level 4 with dict, advanced one pass small out, 38081 +github.tar, level 4 with dict dms, advanced one pass small out, 38135 +github.tar, level 4 with dict dds, advanced one pass small out, 38135 +github.tar, level 4 with dict copy, advanced one pass small out, 38081 +github.tar, level 4 with dict load, advanced one pass small out, 38088 +github.tar, level 5 row 1, advanced one pass small out, 39808 +github.tar, level 5 row 1 with dict dms, advanced one pass small out, 39185 +github.tar, level 5 row 1 with dict dds, advanced one pass small out, 39213 +github.tar, level 5 row 1 with dict copy, advanced one pass small out, 39284 +github.tar, level 5 row 1 with dict load, advanced one pass small out, 39155 +github.tar, level 5 row 2, advanced one pass small out, 39851 +github.tar, level 5 row 2 with dict dms, advanced one pass small out, 39498 +github.tar, level 5 row 2 with dict dds, advanced one pass small out, 39364 +github.tar, level 5 row 2 with dict copy, advanced one pass small out, 39843 +github.tar, level 5 row 2 with dict load, advanced one pass small out, 39312 +github.tar, level 5, advanced one pass small out, 39808 +github.tar, level 5 with dict, advanced one pass small out, 39284 +github.tar, level 5 with dict dms, advanced one pass small out, 39185 +github.tar, level 5 with dict dds, advanced one pass small out, 39213 +github.tar, level 5 with dict copy, advanced one pass small out, 39284 +github.tar, level 5 with dict load, advanced one pass small out, 39155 +github.tar, level 6, advanced one pass small out, 39455 +github.tar, level 6 with dict, advanced one pass small out, 38808 +github.tar, level 6 with dict dms, advanced one pass small out, 38792 +github.tar, level 6 with dict dds, advanced one pass small out, 38790 +github.tar, level 6 with dict copy, advanced one pass small out, 38808 +github.tar, level 6 with dict load, advanced one pass small out, 38796 +github.tar, level 7 row 1, advanced one pass small out, 38168 github.tar, level 7 row 1 with dict dms, advanced one pass small out, 37832 github.tar, level 7 row 1 with dict dds, advanced one pass small out, 37857 github.tar, level 7 row 1 with dict copy, advanced one pass small out, 37839 -github.tar, level 7 row 1 with dict load, advanced one pass small out, 37286 -github.tar, level 7 row 2, advanced one pass small out, 38077 +github.tar, level 7 row 1 with dict load, advanced one pass small out, 37445 +github.tar, level 7 row 2, advanced one pass small out, 38235 github.tar, level 7 row 2 with dict dms, advanced one pass small out, 38012 github.tar, level 7 row 2 with dict dds, advanced one pass small out, 38014 github.tar, level 7 row 2 with dict copy, advanced one pass small out, 38101 -github.tar, level 7 row 2 with dict load, advanced one pass small out, 37402 -github.tar, level 7, advanced one pass small out, 38005 +github.tar, level 7 row 2 with dict load, advanced one pass small out, 37552 +github.tar, level 7, advanced one pass small out, 38168 github.tar, level 7 with dict, advanced one pass small out, 37839 github.tar, level 7 with dict dms, advanced one pass small out, 37832 github.tar, level 7 with dict dds, advanced one pass small out, 37857 github.tar, level 7 with dict copy, advanced one pass small out, 37839 -github.tar, level 7 with dict load, advanced one pass small out, 37286 +github.tar, level 7 with dict load, advanced one pass small out, 37445 github.tar, level 9, advanced one pass small out, 36723 github.tar, level 9 with dict, advanced one pass small out, 36531 github.tar, level 9 with dict dms, advanced one pass small out, 36615 @@ -851,33 +851,33 @@ github.tar, level 19 with dict dms, advanced github.tar, level 19 with dict dds, advanced one pass small out, 32565 github.tar, level 19 with dict copy, advanced one pass small out, 32701 github.tar, level 19 with dict load, advanced one pass small out, 32428 -github.tar, no source size, advanced one pass small out, 38884 -github.tar, no source size with dict, advanced one pass small out, 37995 -github.tar, long distance mode, advanced one pass small out, 40242 -github.tar, multithreaded, advanced one pass small out, 38884 -github.tar, multithreaded long distance mode, advanced one pass small out, 40223 +github.tar, no source size, advanced one pass small out, 39073 +github.tar, no source size with dict, advanced one pass small out, 38146 +github.tar, long distance mode, advanced one pass small out, 40402 +github.tar, multithreaded, advanced one pass small out, 39067 +github.tar, multithreaded long distance mode, advanced one pass small out, 40377 github.tar, small window log, advanced one pass small out, 198535 github.tar, small hash log, advanced one pass small out, 129870 github.tar, small chain log, advanced one pass small out, 41669 -github.tar, explicit params, advanced one pass small out, 41385 -github.tar, uncompressed literals, advanced one pass small out, 41562 +github.tar, explicit params, advanced one pass small out, 41605 +github.tar, uncompressed literals, advanced one pass small out, 41709 github.tar, uncompressed literals optimal, advanced one pass small out, 35356 -github.tar, huffman literals, advanced one pass small out, 38853 -github.tar, multithreaded with advanced params, advanced one pass small out, 41562 -silesia, level -5, advanced streaming, 6854744 -silesia, level -3, advanced streaming, 6503319 -silesia, level -1, advanced streaming, 6172207 -silesia, level 0, advanced streaming, 4839431 -silesia, level 1, advanced streaming, 5306388 -silesia, level 3, advanced streaming, 4839431 -silesia, level 4, advanced streaming, 4776557 -silesia, level 5 row 1, advanced streaming, 4667668 -silesia, level 5 row 2, advanced streaming, 4670326 -silesia, level 5, advanced streaming, 4667668 -silesia, level 6, advanced streaming, 4604351 -silesia, level 7 row 1, advanced streaming, 4570271 -silesia, level 7 row 2, advanced streaming, 4565169 -silesia, level 7, advanced streaming, 4570271 +github.tar, huffman literals, advanced one pass small out, 39099 +github.tar, multithreaded with advanced params, advanced one pass small out, 41700 +silesia, level -5, advanced streaming, 6856699 +silesia, level -3, advanced streaming, 6505694 +silesia, level -1, advanced streaming, 6174139 +silesia, level 0, advanced streaming, 4839173 +silesia, level 1, advanced streaming, 5305682 +silesia, level 3, advanced streaming, 4839173 +silesia, level 4, advanced streaming, 4776074 +silesia, level 5 row 1, advanced streaming, 4667049 +silesia, level 5 row 2, advanced streaming, 4669474 +silesia, level 5, advanced streaming, 4667049 +silesia, level 6, advanced streaming, 4602956 +silesia, level 7 row 1, advanced streaming, 4568785 +silesia, level 7 row 2, advanced streaming, 4563722 +silesia, level 7, advanced streaming, 4568785 silesia, level 9, advanced streaming, 4545850 silesia, level 11 row 1, advanced streaming, 4505658 silesia, level 11 row 2, advanced streaming, 4503429 @@ -886,32 +886,32 @@ silesia, level 12 row 2, advanced silesia, level 13, advanced streaming, 4493990 silesia, level 16, advanced streaming, 4359652 silesia, level 19, advanced streaming, 4266582 -silesia, no source size, advanced streaming, 4839395 -silesia, long distance mode, advanced streaming, 4831158 -silesia, multithreaded, advanced streaming, 4839431 -silesia, multithreaded long distance mode, advanced streaming, 4831176 +silesia, no source size, advanced streaming, 4839137 +silesia, long distance mode, advanced streaming, 4831125 +silesia, multithreaded, advanced streaming, 4838949 +silesia, multithreaded long distance mode, advanced streaming, 4830419 silesia, small window log, advanced streaming, 7110591 silesia, small hash log, advanced streaming, 6526141 silesia, small chain log, advanced streaming, 4912197 -silesia, explicit params, advanced streaming, 4795857 -silesia, uncompressed literals, advanced streaming, 5116957 +silesia, explicit params, advanced streaming, 4795053 +silesia, uncompressed literals, advanced streaming, 5117625 silesia, uncompressed literals optimal, advanced streaming, 4316880 -silesia, huffman literals, advanced streaming, 5321370 -silesia, multithreaded with advanced params, advanced streaming, 5116957 -silesia.tar, level -5, advanced streaming, 6856523 -silesia.tar, level -3, advanced streaming, 6505954 -silesia.tar, level -1, advanced streaming, 6179056 -silesia.tar, level 0, advanced streaming, 4858632 -silesia.tar, level 1, advanced streaming, 5327708 -silesia.tar, level 3, advanced streaming, 4858632 -silesia.tar, level 4, advanced streaming, 4796895 -silesia.tar, level 5 row 1, advanced streaming, 4679020 -silesia.tar, level 5 row 2, advanced streaming, 4682355 -silesia.tar, level 5, advanced streaming, 4679020 -silesia.tar, level 6, advanced streaming, 4614558 -silesia.tar, level 7 row 1, advanced streaming, 4579823 -silesia.tar, level 7 row 2, advanced streaming, 4575601 -silesia.tar, level 7, advanced streaming, 4579823 +silesia, huffman literals, advanced streaming, 5321812 +silesia, multithreaded with advanced params, advanced streaming, 5117795 +silesia.tar, level -5, advanced streaming, 6857458 +silesia.tar, level -3, advanced streaming, 6508562 +silesia.tar, level -1, advanced streaming, 6178057 +silesia.tar, level 0, advanced streaming, 4849720 +silesia.tar, level 1, advanced streaming, 5311452 +silesia.tar, level 3, advanced streaming, 4849720 +silesia.tar, level 4, advanced streaming, 4787534 +silesia.tar, level 5 row 1, advanced streaming, 4669420 +silesia.tar, level 5 row 2, advanced streaming, 4673610 +silesia.tar, level 5, advanced streaming, 4669420 +silesia.tar, level 6, advanced streaming, 4604125 +silesia.tar, level 7 row 1, advanced streaming, 4570206 +silesia.tar, level 7 row 2, advanced streaming, 4565792 +silesia.tar, level 7, advanced streaming, 4570206 silesia.tar, level 9, advanced streaming, 4555445 silesia.tar, level 11 row 1, advanced streaming, 4514959 silesia.tar, level 11 row 2, advanced streaming, 4513810 @@ -920,18 +920,18 @@ silesia.tar, level 12 row 2, advanced silesia.tar, level 13, advanced streaming, 4502956 silesia.tar, level 16, advanced streaming, 4360385 silesia.tar, level 19, advanced streaming, 4260939 -silesia.tar, no source size, advanced streaming, 4858628 -silesia.tar, long distance mode, advanced streaming, 4837775 -silesia.tar, multithreaded, advanced streaming, 4851478 -silesia.tar, multithreaded long distance mode, advanced streaming, 4843155 +silesia.tar, no source size, advanced streaming, 4849716 +silesia.tar, long distance mode, advanced streaming, 4829339 +silesia.tar, multithreaded, advanced streaming, 4841902 +silesia.tar, multithreaded long distance mode, advanced streaming, 4834262 silesia.tar, small window log, advanced streaming, 7117024 silesia.tar, small hash log, advanced streaming, 6529209 silesia.tar, small chain log, advanced streaming, 4917021 -silesia.tar, explicit params, advanced streaming, 4807288 -silesia.tar, uncompressed literals, advanced streaming, 5126542 +silesia.tar, explicit params, advanced streaming, 4797673 +silesia.tar, uncompressed literals, advanced streaming, 5124811 silesia.tar, uncompressed literals optimal, advanced streaming, 4308451 -silesia.tar, huffman literals, advanced streaming, 5341712 -silesia.tar, multithreaded with advanced params, advanced streaming, 5118940 +silesia.tar, huffman literals, advanced streaming, 5326278 +silesia.tar, multithreaded with advanced params, advanced streaming, 5116442 github, level -5, advanced streaming, 204407 github, level -5 with dict, advanced streaming, 45832 github, level -3, advanced streaming, 193253 @@ -1057,74 +1057,74 @@ github, uncompressed literals, advanced github, uncompressed literals optimal, advanced streaming, 152667 github, huffman literals, advanced streaming, 142365 github, multithreaded with advanced params, advanced streaming, 165909 -github.tar, level -5, advanced streaming, 52152 -github.tar, level -5 with dict, advanced streaming, 51181 -github.tar, level -3, advanced streaming, 45678 -github.tar, level -3 with dict, advanced streaming, 44734 -github.tar, level -1, advanced streaming, 42560 -github.tar, level -1 with dict, advanced streaming, 41353 -github.tar, level 0, advanced streaming, 38884 -github.tar, level 0 with dict, advanced streaming, 37995 -github.tar, level 0 with dict dms, advanced streaming, 38114 -github.tar, level 0 with dict dds, advanced streaming, 38114 -github.tar, level 0 with dict copy, advanced streaming, 37995 -github.tar, level 0 with dict load, advanced streaming, 37956 -github.tar, level 1, advanced streaming, 39200 -github.tar, level 1 with dict, advanced streaming, 38119 -github.tar, level 1 with dict dms, advanced streaming, 38406 -github.tar, level 1 with dict dds, advanced streaming, 38406 -github.tar, level 1 with dict copy, advanced streaming, 38119 -github.tar, level 1 with dict load, advanced streaming, 38364 -github.tar, level 3, advanced streaming, 38884 -github.tar, level 3 with dict, advanced streaming, 37995 -github.tar, level 3 with dict dms, advanced streaming, 38114 -github.tar, level 3 with dict dds, advanced streaming, 38114 -github.tar, level 3 with dict copy, advanced streaming, 37995 -github.tar, level 3 with dict load, advanced streaming, 37956 -github.tar, level 4, advanced streaming, 38880 -github.tar, level 4 with dict, advanced streaming, 37948 -github.tar, level 4 with dict dms, advanced streaming, 37995 -github.tar, level 4 with dict dds, advanced streaming, 37995 -github.tar, level 4 with dict copy, advanced streaming, 37948 -github.tar, level 4 with dict load, advanced streaming, 37927 -github.tar, level 5 row 1, advanced streaming, 39651 -github.tar, level 5 row 1 with dict dms, advanced streaming, 39043 -github.tar, level 5 row 1 with dict dds, advanced streaming, 39069 -github.tar, level 5 row 1 with dict copy, advanced streaming, 39145 -github.tar, level 5 row 1 with dict load, advanced streaming, 39000 -github.tar, level 5 row 2, advanced streaming, 39701 -github.tar, level 5 row 2 with dict dms, advanced streaming, 39365 -github.tar, level 5 row 2 with dict dds, advanced streaming, 39233 -github.tar, level 5 row 2 with dict copy, advanced streaming, 39715 -github.tar, level 5 row 2 with dict load, advanced streaming, 39158 -github.tar, level 5, advanced streaming, 39651 -github.tar, level 5 with dict, advanced streaming, 39145 -github.tar, level 5 with dict dms, advanced streaming, 39043 -github.tar, level 5 with dict dds, advanced streaming, 39069 -github.tar, level 5 with dict copy, advanced streaming, 39145 -github.tar, level 5 with dict load, advanced streaming, 39000 -github.tar, level 6, advanced streaming, 39282 -github.tar, level 6 with dict, advanced streaming, 38656 -github.tar, level 6 with dict dms, advanced streaming, 38640 -github.tar, level 6 with dict dds, advanced streaming, 38643 -github.tar, level 6 with dict copy, advanced streaming, 38656 -github.tar, level 6 with dict load, advanced streaming, 38647 -github.tar, level 7 row 1, advanced streaming, 38005 +github.tar, level -5, advanced streaming, 52384 +github.tar, level -5 with dict, advanced streaming, 51420 +github.tar, level -3, advanced streaming, 45907 +github.tar, level -3 with dict, advanced streaming, 44973 +github.tar, level -1, advanced streaming, 42777 +github.tar, level -1 with dict, advanced streaming, 41603 +github.tar, level 0, advanced streaming, 39249 +github.tar, level 0 with dict, advanced streaming, 38309 +github.tar, level 0 with dict dms, advanced streaming, 38445 +github.tar, level 0 with dict dds, advanced streaming, 38445 +github.tar, level 0 with dict copy, advanced streaming, 38309 +github.tar, level 0 with dict load, advanced streaming, 38281 +github.tar, level 1, advanced streaming, 39550 +github.tar, level 1 with dict, advanced streaming, 38502 +github.tar, level 1 with dict dms, advanced streaming, 38770 +github.tar, level 1 with dict dds, advanced streaming, 38770 +github.tar, level 1 with dict copy, advanced streaming, 38502 +github.tar, level 1 with dict load, advanced streaming, 38716 +github.tar, level 3, advanced streaming, 39249 +github.tar, level 3 with dict, advanced streaming, 38309 +github.tar, level 3 with dict dms, advanced streaming, 38445 +github.tar, level 3 with dict dds, advanced streaming, 38445 +github.tar, level 3 with dict copy, advanced streaming, 38309 +github.tar, level 3 with dict load, advanced streaming, 38281 +github.tar, level 4, advanced streaming, 39247 +github.tar, level 4 with dict, advanced streaming, 38270 +github.tar, level 4 with dict dms, advanced streaming, 38317 +github.tar, level 4 with dict dds, advanced streaming, 38317 +github.tar, level 4 with dict copy, advanced streaming, 38270 +github.tar, level 4 with dict load, advanced streaming, 38259 +github.tar, level 5 row 1, advanced streaming, 39986 +github.tar, level 5 row 1 with dict dms, advanced streaming, 39370 +github.tar, level 5 row 1 with dict dds, advanced streaming, 39396 +github.tar, level 5 row 1 with dict copy, advanced streaming, 39465 +github.tar, level 5 row 1 with dict load, advanced streaming, 39332 +github.tar, level 5 row 2, advanced streaming, 40038 +github.tar, level 5 row 2 with dict dms, advanced streaming, 39690 +github.tar, level 5 row 2 with dict dds, advanced streaming, 39555 +github.tar, level 5 row 2 with dict copy, advanced streaming, 40036 +github.tar, level 5 row 2 with dict load, advanced streaming, 39498 +github.tar, level 5, advanced streaming, 39986 +github.tar, level 5 with dict, advanced streaming, 39465 +github.tar, level 5 with dict dms, advanced streaming, 39370 +github.tar, level 5 with dict dds, advanced streaming, 39396 +github.tar, level 5 with dict copy, advanced streaming, 39465 +github.tar, level 5 with dict load, advanced streaming, 39332 +github.tar, level 6, advanced streaming, 39565 +github.tar, level 6 with dict, advanced streaming, 38894 +github.tar, level 6 with dict dms, advanced streaming, 38901 +github.tar, level 6 with dict dds, advanced streaming, 38903 +github.tar, level 6 with dict copy, advanced streaming, 38894 +github.tar, level 6 with dict load, advanced streaming, 38906 +github.tar, level 7 row 1, advanced streaming, 38295 github.tar, level 7 row 1 with dict dms, advanced streaming, 37832 github.tar, level 7 row 1 with dict dds, advanced streaming, 37857 github.tar, level 7 row 1 with dict copy, advanced streaming, 37839 -github.tar, level 7 row 1 with dict load, advanced streaming, 37286 -github.tar, level 7 row 2, advanced streaming, 38077 +github.tar, level 7 row 1 with dict load, advanced streaming, 37565 +github.tar, level 7 row 2, advanced streaming, 38369 github.tar, level 7 row 2 with dict dms, advanced streaming, 38012 github.tar, level 7 row 2 with dict dds, advanced streaming, 38014 github.tar, level 7 row 2 with dict copy, advanced streaming, 38101 -github.tar, level 7 row 2 with dict load, advanced streaming, 37402 -github.tar, level 7, advanced streaming, 38005 +github.tar, level 7 row 2 with dict load, advanced streaming, 37688 +github.tar, level 7, advanced streaming, 38295 github.tar, level 7 with dict, advanced streaming, 37839 github.tar, level 7 with dict dms, advanced streaming, 37832 github.tar, level 7 with dict dds, advanced streaming, 37857 github.tar, level 7 with dict copy, advanced streaming, 37839 -github.tar, level 7 with dict load, advanced streaming, 37286 +github.tar, level 7 with dict load, advanced streaming, 37565 github.tar, level 9, advanced streaming, 36723 github.tar, level 9 with dict, advanced streaming, 36531 github.tar, level 9 with dict dms, advanced streaming, 36615 @@ -1169,55 +1169,55 @@ github.tar, level 19 with dict dms, advanced github.tar, level 19 with dict dds, advanced streaming, 32565 github.tar, level 19 with dict copy, advanced streaming, 32701 github.tar, level 19 with dict load, advanced streaming, 32428 -github.tar, no source size, advanced streaming, 38881 -github.tar, no source size with dict, advanced streaming, 38111 -github.tar, long distance mode, advanced streaming, 40242 -github.tar, multithreaded, advanced streaming, 38884 -github.tar, multithreaded long distance mode, advanced streaming, 40223 +github.tar, no source size, advanced streaming, 39246 +github.tar, no source size with dict, advanced streaming, 38442 +github.tar, long distance mode, advanced streaming, 40569 +github.tar, multithreaded, advanced streaming, 39067 +github.tar, multithreaded long distance mode, advanced streaming, 40377 github.tar, small window log, advanced streaming, 199553 github.tar, small hash log, advanced streaming, 129870 github.tar, small chain log, advanced streaming, 41669 -github.tar, explicit params, advanced streaming, 41385 -github.tar, uncompressed literals, advanced streaming, 41562 +github.tar, explicit params, advanced streaming, 41768 +github.tar, uncompressed literals, advanced streaming, 41784 github.tar, uncompressed literals optimal, advanced streaming, 35356 -github.tar, huffman literals, advanced streaming, 38853 -github.tar, multithreaded with advanced params, advanced streaming, 41562 -silesia, level -5, old streaming, 6854744 -silesia, level -3, old streaming, 6503319 -silesia, level -1, old streaming, 6172207 -silesia, level 0, old streaming, 4839431 -silesia, level 1, old streaming, 5306388 -silesia, level 3, old streaming, 4839431 -silesia, level 4, old streaming, 4776557 -silesia, level 5, old streaming, 4667668 -silesia, level 6, old streaming, 4604351 -silesia, level 7, old streaming, 4570271 +github.tar, huffman literals, advanced streaming, 39226 +github.tar, multithreaded with advanced params, advanced streaming, 41700 +silesia, level -5, old streaming, 6856699 +silesia, level -3, old streaming, 6505694 +silesia, level -1, old streaming, 6174139 +silesia, level 0, old streaming, 4839173 +silesia, level 1, old streaming, 5305682 +silesia, level 3, old streaming, 4839173 +silesia, level 4, old streaming, 4776074 +silesia, level 5, old streaming, 4667049 +silesia, level 6, old streaming, 4602956 +silesia, level 7, old streaming, 4568785 silesia, level 9, old streaming, 4545850 silesia, level 13, old streaming, 4493990 silesia, level 16, old streaming, 4359652 silesia, level 19, old streaming, 4266582 -silesia, no source size, old streaming, 4839395 -silesia, uncompressed literals, old streaming, 4839431 +silesia, no source size, old streaming, 4839137 +silesia, uncompressed literals, old streaming, 4839173 silesia, uncompressed literals optimal, old streaming, 4266582 -silesia, huffman literals, old streaming, 6172207 -silesia.tar, level -5, old streaming, 6856523 -silesia.tar, level -3, old streaming, 6505954 -silesia.tar, level -1, old streaming, 6179056 -silesia.tar, level 0, old streaming, 4858632 -silesia.tar, level 1, old streaming, 5327708 -silesia.tar, level 3, old streaming, 4858632 -silesia.tar, level 4, old streaming, 4796895 -silesia.tar, level 5, old streaming, 4679020 -silesia.tar, level 6, old streaming, 4614558 -silesia.tar, level 7, old streaming, 4579823 +silesia, huffman literals, old streaming, 6174139 +silesia.tar, level -5, old streaming, 6857458 +silesia.tar, level -3, old streaming, 6508562 +silesia.tar, level -1, old streaming, 6178057 +silesia.tar, level 0, old streaming, 4849720 +silesia.tar, level 1, old streaming, 5311452 +silesia.tar, level 3, old streaming, 4849720 +silesia.tar, level 4, old streaming, 4787534 +silesia.tar, level 5, old streaming, 4669420 +silesia.tar, level 6, old streaming, 4604125 +silesia.tar, level 7, old streaming, 4570206 silesia.tar, level 9, old streaming, 4555445 silesia.tar, level 13, old streaming, 4502956 silesia.tar, level 16, old streaming, 4360385 silesia.tar, level 19, old streaming, 4260939 -silesia.tar, no source size, old streaming, 4858628 -silesia.tar, uncompressed literals, old streaming, 4858632 +silesia.tar, no source size, old streaming, 4849716 +silesia.tar, uncompressed literals, old streaming, 4849720 silesia.tar, uncompressed literals optimal, old streaming, 4260939 -silesia.tar, huffman literals, old streaming, 6179056 +silesia.tar, huffman literals, old streaming, 6178057 github, level -5, old streaming, 204407 github, level -5 with dict, old streaming, 45832 github, level -3, old streaming, 193253 @@ -1251,25 +1251,25 @@ github, no source size with dict, old stre github, uncompressed literals, old streaming, 136331 github, uncompressed literals optimal, old streaming, 132879 github, huffman literals, old streaming, 175468 -github.tar, level -5, old streaming, 52152 -github.tar, level -5 with dict, old streaming, 51181 -github.tar, level -3, old streaming, 45678 -github.tar, level -3 with dict, old streaming, 44734 -github.tar, level -1, old streaming, 42560 -github.tar, level -1 with dict, old streaming, 41353 -github.tar, level 0, old streaming, 38884 -github.tar, level 0 with dict, old streaming, 37995 -github.tar, level 1, old streaming, 39200 -github.tar, level 1 with dict, old streaming, 38119 -github.tar, level 3, old streaming, 38884 -github.tar, level 3 with dict, old streaming, 37995 -github.tar, level 4, old streaming, 38880 -github.tar, level 4 with dict, old streaming, 37948 -github.tar, level 5, old streaming, 39651 -github.tar, level 5 with dict, old streaming, 39145 -github.tar, level 6, old streaming, 39282 -github.tar, level 6 with dict, old streaming, 38656 -github.tar, level 7, old streaming, 38005 +github.tar, level -5, old streaming, 52384 +github.tar, level -5 with dict, old streaming, 51420 +github.tar, level -3, old streaming, 45907 +github.tar, level -3 with dict, old streaming, 44973 +github.tar, level -1, old streaming, 42777 +github.tar, level -1 with dict, old streaming, 41603 +github.tar, level 0, old streaming, 39249 +github.tar, level 0 with dict, old streaming, 38309 +github.tar, level 1, old streaming, 39550 +github.tar, level 1 with dict, old streaming, 38502 +github.tar, level 3, old streaming, 39249 +github.tar, level 3 with dict, old streaming, 38309 +github.tar, level 4, old streaming, 39247 +github.tar, level 4 with dict, old streaming, 38270 +github.tar, level 5, old streaming, 39986 +github.tar, level 5 with dict, old streaming, 39465 +github.tar, level 6, old streaming, 39565 +github.tar, level 6 with dict, old streaming, 38894 +github.tar, level 7, old streaming, 38295 github.tar, level 7 with dict, old streaming, 37839 github.tar, level 9, old streaming, 36723 github.tar, level 9 with dict, old streaming, 36531 @@ -1279,63 +1279,63 @@ github.tar, level 16, old stre github.tar, level 16 with dict, old streaming, 33375 github.tar, level 19, old streaming, 32262 github.tar, level 19 with dict, old streaming, 32701 -github.tar, no source size, old streaming, 38881 -github.tar, no source size with dict, old streaming, 38111 -github.tar, uncompressed literals, old streaming, 38884 +github.tar, no source size, old streaming, 39246 +github.tar, no source size with dict, old streaming, 38442 +github.tar, uncompressed literals, old streaming, 39249 github.tar, uncompressed literals optimal, old streaming, 32262 -github.tar, huffman literals, old streaming, 42560 -silesia, level -5, old streaming advanced, 6854744 -silesia, level -3, old streaming advanced, 6503319 -silesia, level -1, old streaming advanced, 6172207 -silesia, level 0, old streaming advanced, 4839431 -silesia, level 1, old streaming advanced, 5306388 -silesia, level 3, old streaming advanced, 4839431 -silesia, level 4, old streaming advanced, 4776557 -silesia, level 5, old streaming advanced, 4667668 -silesia, level 6, old streaming advanced, 4604351 -silesia, level 7, old streaming advanced, 4570271 +github.tar, huffman literals, old streaming, 42777 +silesia, level -5, old streaming advanced, 6856699 +silesia, level -3, old streaming advanced, 6505694 +silesia, level -1, old streaming advanced, 6174139 +silesia, level 0, old streaming advanced, 4839173 +silesia, level 1, old streaming advanced, 5305682 +silesia, level 3, old streaming advanced, 4839173 +silesia, level 4, old streaming advanced, 4776074 +silesia, level 5, old streaming advanced, 4667049 +silesia, level 6, old streaming advanced, 4602956 +silesia, level 7, old streaming advanced, 4568785 silesia, level 9, old streaming advanced, 4545850 silesia, level 13, old streaming advanced, 4493990 silesia, level 16, old streaming advanced, 4359652 silesia, level 19, old streaming advanced, 4266582 -silesia, no source size, old streaming advanced, 4839395 -silesia, long distance mode, old streaming advanced, 4839431 -silesia, multithreaded, old streaming advanced, 4839431 -silesia, multithreaded long distance mode, old streaming advanced, 4839431 +silesia, no source size, old streaming advanced, 4839137 +silesia, long distance mode, old streaming advanced, 4839173 +silesia, multithreaded, old streaming advanced, 4839173 +silesia, multithreaded long distance mode, old streaming advanced, 4839173 silesia, small window log, old streaming advanced, 7110591 silesia, small hash log, old streaming advanced, 6526141 silesia, small chain log, old streaming advanced, 4912197 -silesia, explicit params, old streaming advanced, 4795857 -silesia, uncompressed literals, old streaming advanced, 4839431 +silesia, explicit params, old streaming advanced, 4795053 +silesia, uncompressed literals, old streaming advanced, 4839173 silesia, uncompressed literals optimal, old streaming advanced, 4266582 -silesia, huffman literals, old streaming advanced, 6172207 -silesia, multithreaded with advanced params, old streaming advanced, 4839431 -silesia.tar, level -5, old streaming advanced, 6856523 -silesia.tar, level -3, old streaming advanced, 6505954 -silesia.tar, level -1, old streaming advanced, 6179056 -silesia.tar, level 0, old streaming advanced, 4858632 -silesia.tar, level 1, old streaming advanced, 5327708 -silesia.tar, level 3, old streaming advanced, 4858632 -silesia.tar, level 4, old streaming advanced, 4796895 -silesia.tar, level 5, old streaming advanced, 4679020 -silesia.tar, level 6, old streaming advanced, 4614558 -silesia.tar, level 7, old streaming advanced, 4579823 +silesia, huffman literals, old streaming advanced, 6174139 +silesia, multithreaded with advanced params, old streaming advanced, 4839173 +silesia.tar, level -5, old streaming advanced, 6857458 +silesia.tar, level -3, old streaming advanced, 6508562 +silesia.tar, level -1, old streaming advanced, 6178057 +silesia.tar, level 0, old streaming advanced, 4849720 +silesia.tar, level 1, old streaming advanced, 5311452 +silesia.tar, level 3, old streaming advanced, 4849720 +silesia.tar, level 4, old streaming advanced, 4787534 +silesia.tar, level 5, old streaming advanced, 4669420 +silesia.tar, level 6, old streaming advanced, 4604125 +silesia.tar, level 7, old streaming advanced, 4570206 silesia.tar, level 9, old streaming advanced, 4555445 silesia.tar, level 13, old streaming advanced, 4502956 silesia.tar, level 16, old streaming advanced, 4360385 silesia.tar, level 19, old streaming advanced, 4260939 -silesia.tar, no source size, old streaming advanced, 4858628 -silesia.tar, long distance mode, old streaming advanced, 4858632 -silesia.tar, multithreaded, old streaming advanced, 4858632 -silesia.tar, multithreaded long distance mode, old streaming advanced, 4858632 +silesia.tar, no source size, old streaming advanced, 4849716 +silesia.tar, long distance mode, old streaming advanced, 4849720 +silesia.tar, multithreaded, old streaming advanced, 4849720 +silesia.tar, multithreaded long distance mode, old streaming advanced, 4849720 silesia.tar, small window log, old streaming advanced, 7117027 silesia.tar, small hash log, old streaming advanced, 6529209 silesia.tar, small chain log, old streaming advanced, 4917021 -silesia.tar, explicit params, old streaming advanced, 4807288 -silesia.tar, uncompressed literals, old streaming advanced, 4858632 +silesia.tar, explicit params, old streaming advanced, 4797673 +silesia.tar, uncompressed literals, old streaming advanced, 4849720 silesia.tar, uncompressed literals optimal, old streaming advanced, 4260939 -silesia.tar, huffman literals, old streaming advanced, 6179056 -silesia.tar, multithreaded with advanced params, old streaming advanced, 4858632 +silesia.tar, huffman literals, old streaming advanced, 6178057 +silesia.tar, multithreaded with advanced params, old streaming advanced, 4849720 github, level -5, old streaming advanced, 213265 github, level -5 with dict, old streaming advanced, 46708 github, level -3, old streaming advanced, 196126 @@ -1377,26 +1377,26 @@ github, uncompressed literals, old stre github, uncompressed literals optimal, old streaming advanced, 132879 github, huffman literals, old streaming advanced, 181107 github, multithreaded with advanced params, old streaming advanced, 141101 -github.tar, level -5, old streaming advanced, 52152 -github.tar, level -5 with dict, old streaming advanced, 51129 -github.tar, level -3, old streaming advanced, 45678 -github.tar, level -3 with dict, old streaming advanced, 44986 -github.tar, level -1, old streaming advanced, 42560 -github.tar, level -1 with dict, old streaming advanced, 41650 -github.tar, level 0, old streaming advanced, 38884 -github.tar, level 0 with dict, old streaming advanced, 38013 -github.tar, level 1, old streaming advanced, 39200 -github.tar, level 1 with dict, old streaming advanced, 38359 -github.tar, level 3, old streaming advanced, 38884 -github.tar, level 3 with dict, old streaming advanced, 38013 -github.tar, level 4, old streaming advanced, 38880 -github.tar, level 4 with dict, old streaming advanced, 38063 -github.tar, level 5, old streaming advanced, 39651 -github.tar, level 5 with dict, old streaming advanced, 39018 -github.tar, level 6, old streaming advanced, 39282 -github.tar, level 6 with dict, old streaming advanced, 38635 -github.tar, level 7, old streaming advanced, 38005 -github.tar, level 7 with dict, old streaming advanced, 37264 +github.tar, level -5, old streaming advanced, 52384 +github.tar, level -5 with dict, old streaming advanced, 51359 +github.tar, level -3, old streaming advanced, 45907 +github.tar, level -3 with dict, old streaming advanced, 45211 +github.tar, level -1, old streaming advanced, 42777 +github.tar, level -1 with dict, old streaming advanced, 41865 +github.tar, level 0, old streaming advanced, 39249 +github.tar, level 0 with dict, old streaming advanced, 38327 +github.tar, level 1, old streaming advanced, 39550 +github.tar, level 1 with dict, old streaming advanced, 38714 +github.tar, level 3, old streaming advanced, 39249 +github.tar, level 3 with dict, old streaming advanced, 38327 +github.tar, level 4, old streaming advanced, 39247 +github.tar, level 4 with dict, old streaming advanced, 38382 +github.tar, level 5, old streaming advanced, 39986 +github.tar, level 5 with dict, old streaming advanced, 39347 +github.tar, level 6, old streaming advanced, 39565 +github.tar, level 6 with dict, old streaming advanced, 38889 +github.tar, level 7, old streaming advanced, 38295 +github.tar, level 7 with dict, old streaming advanced, 37534 github.tar, level 9, old streaming advanced, 36723 github.tar, level 9 with dict, old streaming advanced, 36241 github.tar, level 13, old streaming advanced, 35501 @@ -1405,19 +1405,19 @@ github.tar, level 16, old stre github.tar, level 16 with dict, old streaming advanced, 38578 github.tar, level 19, old streaming advanced, 32262 github.tar, level 19 with dict, old streaming advanced, 32678 -github.tar, no source size, old streaming advanced, 38881 -github.tar, no source size with dict, old streaming advanced, 38076 -github.tar, long distance mode, old streaming advanced, 38884 -github.tar, multithreaded, old streaming advanced, 38884 -github.tar, multithreaded long distance mode, old streaming advanced, 38884 +github.tar, no source size, old streaming advanced, 39246 +github.tar, no source size with dict, old streaming advanced, 38399 +github.tar, long distance mode, old streaming advanced, 39249 +github.tar, multithreaded, old streaming advanced, 39249 +github.tar, multithreaded long distance mode, old streaming advanced, 39249 github.tar, small window log, old streaming advanced, 199556 github.tar, small hash log, old streaming advanced, 129870 github.tar, small chain log, old streaming advanced, 41669 -github.tar, explicit params, old streaming advanced, 41385 -github.tar, uncompressed literals, old streaming advanced, 38884 +github.tar, explicit params, old streaming advanced, 41768 +github.tar, uncompressed literals, old streaming advanced, 39249 github.tar, uncompressed literals optimal, old streaming advanced, 32262 -github.tar, huffman literals, old streaming advanced, 42560 -github.tar, multithreaded with advanced params, old streaming advanced, 38884 +github.tar, huffman literals, old streaming advanced, 42777 +github.tar, multithreaded with advanced params, old streaming advanced, 39249 github, level -5 with dict, old streaming cdict, 45832 github, level -3 with dict, old streaming cdict, 44671 github, level -1 with dict, old streaming cdict, 41825 @@ -1433,21 +1433,21 @@ github, level 13 with dict, old stre github, level 16 with dict, old streaming cdict, 37902 github, level 19 with dict, old streaming cdict, 37916 github, no source size with dict, old streaming cdict, 40652 -github.tar, level -5 with dict, old streaming cdict, 51286 -github.tar, level -3 with dict, old streaming cdict, 45147 -github.tar, level -1 with dict, old streaming cdict, 41865 -github.tar, level 0 with dict, old streaming cdict, 37956 -github.tar, level 1 with dict, old streaming cdict, 38364 -github.tar, level 3 with dict, old streaming cdict, 37956 -github.tar, level 4 with dict, old streaming cdict, 37927 -github.tar, level 5 with dict, old streaming cdict, 39000 -github.tar, level 6 with dict, old streaming cdict, 38647 -github.tar, level 7 with dict, old streaming cdict, 37286 +github.tar, level -5 with dict, old streaming cdict, 51512 +github.tar, level -3 with dict, old streaming cdict, 45379 +github.tar, level -1 with dict, old streaming cdict, 42084 +github.tar, level 0 with dict, old streaming cdict, 38281 +github.tar, level 1 with dict, old streaming cdict, 38716 +github.tar, level 3 with dict, old streaming cdict, 38281 +github.tar, level 4 with dict, old streaming cdict, 38259 +github.tar, level 5 with dict, old streaming cdict, 39332 +github.tar, level 6 with dict, old streaming cdict, 38906 +github.tar, level 7 with dict, old streaming cdict, 37565 github.tar, level 9 with dict, old streaming cdict, 36322 github.tar, level 13 with dict, old streaming cdict, 36010 github.tar, level 16 with dict, old streaming cdict, 39081 github.tar, level 19 with dict, old streaming cdict, 32428 -github.tar, no source size with dict, old streaming cdict, 38111 +github.tar, no source size with dict, old streaming cdict, 38442 github, level -5 with dict, old streaming advanced cdict, 46708 github, level -3 with dict, old streaming advanced cdict, 45476 github, level -1 with dict, old streaming advanced cdict, 42060 @@ -1463,18 +1463,18 @@ github, level 13 with dict, old stre github, level 16 with dict, old streaming advanced cdict, 40804 github, level 19 with dict, old streaming advanced cdict, 37916 github, no source size with dict, old streaming advanced cdict, 40608 -github.tar, level -5 with dict, old streaming advanced cdict, 50791 -github.tar, level -3 with dict, old streaming advanced cdict, 44926 -github.tar, level -1 with dict, old streaming advanced cdict, 41482 -github.tar, level 0 with dict, old streaming advanced cdict, 38013 -github.tar, level 1 with dict, old streaming advanced cdict, 38168 -github.tar, level 3 with dict, old streaming advanced cdict, 38013 -github.tar, level 4 with dict, old streaming advanced cdict, 38063 -github.tar, level 5 with dict, old streaming advanced cdict, 39018 -github.tar, level 6 with dict, old streaming advanced cdict, 38635 -github.tar, level 7 with dict, old streaming advanced cdict, 37264 +github.tar, level -5 with dict, old streaming advanced cdict, 51019 +github.tar, level -3 with dict, old streaming advanced cdict, 45149 +github.tar, level -1 with dict, old streaming advanced cdict, 41694 +github.tar, level 0 with dict, old streaming advanced cdict, 38327 +github.tar, level 1 with dict, old streaming advanced cdict, 38513 +github.tar, level 3 with dict, old streaming advanced cdict, 38327 +github.tar, level 4 with dict, old streaming advanced cdict, 38382 +github.tar, level 5 with dict, old streaming advanced cdict, 39347 +github.tar, level 6 with dict, old streaming advanced cdict, 38889 +github.tar, level 7 with dict, old streaming advanced cdict, 37534 github.tar, level 9 with dict, old streaming advanced cdict, 36241 github.tar, level 13 with dict, old streaming advanced cdict, 35807 github.tar, level 16 with dict, old streaming advanced cdict, 38578 github.tar, level 19 with dict, old streaming advanced cdict, 32678 -github.tar, no source size with dict, old streaming advanced cdict, 38076 +github.tar, no source size with dict, old streaming advanced cdict, 38399 diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c index 2c25adf3c..8df86d985 100644 --- a/tests/zstreamtest.c +++ b/tests/zstreamtest.c @@ -441,7 +441,7 @@ static int basicUnitTests(U32 seed, double compressibility, int bigTests) size_t const gfhError = ZSTD_getFrameHeader(&fhi, cStart, cSize); if (gfhError!=0) goto _output_error; DISPLAYLEVEL(5, " (windowSize : %u) ", (unsigned)fhi.windowSize); - { size_t const s = ZSTD_estimateDStreamSize(fhi.windowSize) + { size_t const s = ZSTD_estimateDStreamSize((size_t)fhi.windowSize) /* uses ZSTD_initDStream_usingDict() */ + ZSTD_estimateDDictSize(dictSize, ZSTD_dlm_byCopy); if (ZSTD_isError(s)) goto _output_error; @@ -958,7 +958,7 @@ static int basicUnitTests(U32 seed, double compressibility, int bigTests) break; out.size = MIN(out.size + cSize / 4, compressedBufferSize); } - CHECK_Z(ZSTD_decompress(decodedBuffer, CNBufferSize, compressedBuffer, cSize)); + CHECK_Z(ZSTD_decompress(decodedBuffer, CNBufferSize, compressedBuffer, out.pos)); DISPLAYLEVEL(3, "OK \n"); DISPLAYLEVEL(3, "test%3i : ZSTD_compressStream2() ZSTD_c_stableInBuffer modify buffer : ", testNb++); @@ -1551,7 +1551,7 @@ static int basicUnitTests(U32 seed, double compressibility, int bigTests) DISPLAYLEVEL(3, "test%3i : check dictionary FSE tables can represent every code : ", testNb++); { unsigned const kMaxWindowLog = 24; unsigned value; - ZSTD_compressionParameters cParams = ZSTD_getCParams(3, 1U << kMaxWindowLog, 1024); + ZSTD_compressionParameters cParams = ZSTD_getCParams(3, 1ULL << kMaxWindowLog, 1024); ZSTD_CDict* cdict; ZSTD_DDict* ddict; SEQ_stream seq = SEQ_initStream(0x87654321);