diff --git a/.github/workflows/dev-long-tests.yml b/.github/workflows/dev-long-tests.yml index 3fa2ceea2..1c8c9ec55 100644 --- a/.github/workflows/dev-long-tests.yml +++ b/.github/workflows/dev-long-tests.yml @@ -65,6 +65,17 @@ jobs: - name: thread sanitizer fuzztest run: CC=clang make tsan-fuzztest + + big-tests-zstreamtest32: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3 + - name: zstream tests in 32bit mode, with big tests + run: | + sudo apt-get -qqq update + make libc6install + CC=clang make -C tests test-zstream32 FUZZER_FLAGS="--big-tests" + # lasts ~23mn gcc-8-asan-ubsan-testzstd: runs-on: ubuntu-latest diff --git a/lib/common/zstd_internal.h b/lib/common/zstd_internal.h index 94670a03d..8878aa100 100644 --- a/lib/common/zstd_internal.h +++ b/lib/common/zstd_internal.h @@ -346,7 +346,7 @@ typedef struct { } ZSTD_frameSizeInfo; /* decompress & legacy */ const seqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx); /* compress & dictBuilder */ -void ZSTD_seqToCodes(const seqStore_t* seqStorePtr); /* compress, dictBuilder, decodeCorpus (shouldn't get its definition from here) */ +int ZSTD_seqToCodes(const seqStore_t* seqStorePtr); /* compress, dictBuilder, decodeCorpus (shouldn't get its definition from here) */ /* custom memory allocation functions */ void* ZSTD_customMalloc(size_t size, ZSTD_customMem customMem); diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 07e62bf5c..579044dc9 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -2559,7 +2559,7 @@ static void ZSTD_reduceIndex (ZSTD_matchState_t* ms, ZSTD_CCtx_params const* par /* See doc/zstd_compression_format.md for detailed format description */ -void ZSTD_seqToCodes(const seqStore_t* seqStorePtr) +int ZSTD_seqToCodes(const seqStore_t* seqStorePtr) { const seqDef* const sequences = seqStorePtr->sequencesStart; BYTE* const llCodeTable = seqStorePtr->llCode; @@ -2567,18 +2567,24 @@ void ZSTD_seqToCodes(const seqStore_t* seqStorePtr) BYTE* const mlCodeTable = seqStorePtr->mlCode; U32 const nbSeq = (U32)(seqStorePtr->sequences - seqStorePtr->sequencesStart); U32 u; + int longOffsets = 0; assert(nbSeq <= seqStorePtr->maxNbSeq); for (u=0; u= STREAM_ACCUMULATOR_MIN)); + if (MEM_32bits() && ofCode >= STREAM_ACCUMULATOR_MIN) + longOffsets = 1; } if (seqStorePtr->longLengthType==ZSTD_llt_literalLength) llCodeTable[seqStorePtr->longLengthPos] = MaxLL; if (seqStorePtr->longLengthType==ZSTD_llt_matchLength) mlCodeTable[seqStorePtr->longLengthPos] = MaxML; + return longOffsets; } /* ZSTD_useTargetCBlockSize(): @@ -2612,6 +2618,7 @@ typedef struct { U32 MLtype; size_t size; size_t lastCountSize; /* Accounts for bug in 1.3.4. More detail in ZSTD_entropyCompressSeqStore_internal() */ + int longOffsets; } ZSTD_symbolEncodingTypeStats_t; /* ZSTD_buildSequencesStatistics(): @@ -2642,7 +2649,7 @@ ZSTD_buildSequencesStatistics( stats.lastCountSize = 0; /* convert length/distances into codes */ - ZSTD_seqToCodes(seqStorePtr); + stats.longOffsets = ZSTD_seqToCodes(seqStorePtr); assert(op <= oend); assert(nbSeq != 0); /* ZSTD_selectEncodingType() divides by nbSeq */ /* build CTable for Literal Lengths */ @@ -2756,7 +2763,6 @@ ZSTD_entropyCompressSeqStore_internal( void* entropyWorkspace, size_t entropyWkspSize, const int bmi2) { - const int longOffsets = cctxParams->cParams.windowLog >= STREAM_ACCUMULATOR_MIN; ZSTD_strategy const strategy = cctxParams->cParams.strategy; unsigned* count = (unsigned*)entropyWorkspace; FSE_CTable* CTable_LitLength = nextEntropy->fse.litlengthCTable; @@ -2771,6 +2777,7 @@ ZSTD_entropyCompressSeqStore_internal( BYTE* const oend = ostart + dstCapacity; BYTE* op = ostart; size_t lastCountSize; + int longOffsets = 0; entropyWorkspace = count + (MaxSeq + 1); entropyWkspSize -= (MaxSeq + 1) * sizeof(*count); @@ -2832,6 +2839,7 @@ ZSTD_entropyCompressSeqStore_internal( *seqHead = (BYTE)((stats.LLtype<<6) + (stats.Offtype<<4) + (stats.MLtype<<2)); lastCountSize = stats.lastCountSize; op += stats.size; + longOffsets = stats.longOffsets; } { size_t const bitstreamSize = ZSTD_encodeSequences( @@ -3476,7 +3484,7 @@ ZSTD_buildBlockEntropyStats_literals(void* const src, size_t srcSize, static ZSTD_symbolEncodingTypeStats_t ZSTD_buildDummySequencesStatistics(ZSTD_fseCTables_t* nextEntropy) { - ZSTD_symbolEncodingTypeStats_t stats = {set_basic, set_basic, set_basic, 0, 0}; + ZSTD_symbolEncodingTypeStats_t stats = {set_basic, set_basic, set_basic, 0, 0, 0}; nextEntropy->litlength_repeatMode = FSE_repeat_none; nextEntropy->offcode_repeatMode = FSE_repeat_none; nextEntropy->matchlength_repeatMode = FSE_repeat_none; diff --git a/tests/Makefile b/tests/Makefile index 8226176cc..3eed19ea8 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -373,7 +373,7 @@ test-zstream: zstreamtest $(QEMU_SYS) ./zstreamtest --newapi -t1 $(ZSTREAM_TESTTIME) $(FUZZER_FLAGS) test-zstream32: zstreamtest32 - $(QEMU_SYS) ./zstreamtest32 $(ZSTREAM_TESTTIME) $(FUZZER_FLAGS) + $(QEMU_SYS) ./zstreamtest32 -v $(ZSTREAM_TESTTIME) $(FUZZER_FLAGS) test-longmatch: longmatch $(QEMU_SYS) ./longmatch diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c index fa8a1ae37..db1e25188 100644 --- a/tests/zstreamtest.c +++ b/tests/zstreamtest.c @@ -2222,6 +2222,66 @@ static int basicUnitTests(U32 seed, double compressibility, int bigTests) } DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3i : Testing large offset with small window size: ", testNb++); + { + ZSTD_CCtx* cctx = ZSTD_createCCtx(); + ZSTD_DCtx* dctx = ZSTD_createDCtx(); + + /* Test large offset, small window size*/ + { + size_t srcSize = 21; + void* const src = CNBuffer; + size_t dstSize = ZSTD_compressBound(srcSize); + void* const dst = compressedBuffer; + size_t const kNbSequences = 4; + ZSTD_Sequence* sequences = malloc(sizeof(ZSTD_Sequence) * kNbSequences); + void* const checkBuf = malloc(srcSize); + const size_t largeDictSize = 1 << 25; + ZSTD_CDict* cdict = NULL; + ZSTD_DDict* ddict = NULL; + + /* Generate large dictionary */ + void* dictBuffer = calloc(largeDictSize, 1); + ZSTD_compressionParameters cParams = ZSTD_getCParams(1, srcSize, largeDictSize); + cParams.minMatch = ZSTD_MINMATCH_MIN; + cParams.hashLog = ZSTD_HASHLOG_MIN; + cParams.chainLog = ZSTD_CHAINLOG_MIN; + + cdict = ZSTD_createCDict_advanced(dictBuffer, largeDictSize, ZSTD_dlm_byRef, ZSTD_dct_rawContent, cParams, ZSTD_defaultCMem); + ddict = ZSTD_createDDict_advanced(dictBuffer, largeDictSize, ZSTD_dlm_byRef, ZSTD_dct_rawContent, ZSTD_defaultCMem); + + ZSTD_CCtx_refCDict(cctx, cdict); + ZSTD_DCtx_refDDict(dctx, ddict); + + sequences[0] = (ZSTD_Sequence) {3, 3, 3, 0}; + sequences[1] = (ZSTD_Sequence) {1 << 25, 0, 3, 0}; + sequences[2] = (ZSTD_Sequence) {1 << 25, 0, 9, 0}; + sequences[3] = (ZSTD_Sequence) {3, 0, 3, 0}; + + cSize = ZSTD_compressSequences(cctx, dst, dstSize, + sequences, kNbSequences, + src, srcSize); + + CHECK(ZSTD_isError(cSize), "Should not throw an error"); + + { + size_t dSize = ZSTD_decompressDCtx(dctx, checkBuf, srcSize, dst, cSize); + CHECK(ZSTD_isError(dSize), "Should not throw an error"); + CHECK(memcmp(src, checkBuf, srcSize) != 0, "Corruption!"); + } + + free(sequences); + free(checkBuf); + free(dictBuffer); + ZSTD_freeCDict(cdict); + ZSTD_freeDDict(ddict); + } + ZSTD_freeCCtx(cctx); + ZSTD_freeDCtx(dctx); + } + DISPLAYLEVEL(3, "OK \n"); + _end: FUZ_freeDictionary(dictionary); ZSTD_freeCStream(zc); @@ -3034,6 +3094,7 @@ int main(int argc, const char** argv) if (!strcmp(argument, "--newapi")) { selected_api=advanced_api; testNb += !testNb; continue; } if (!strcmp(argument, "--no-big-tests")) { bigTests=0; continue; } + if (!strcmp(argument, "--big-tests")) { bigTests=1; continue; } argument++; while (*argument!=0) {