Merge pull request #3023 from facebook/fix_seqCompress_withDelimiter

fix sequence compression API in Explicit Delimiter mode

lib/common/error_private.c

@@ -27,7 +27,7 @@ const char* ERR_getErrorString(ERR_enum code)
     case PREFIX(version_unsupported): return "Version not supported";
     case PREFIX(frameParameter_unsupported): return "Unsupported frame parameter";
     case PREFIX(frameParameter_windowTooLarge): return "Frame requires too much memory for decoding";
-    case PREFIX(corruption_detected): return "Corrupted block detected";
+    case PREFIX(corruption_detected): return "Data corruption detected";
     case PREFIX(checksum_wrong): return "Restored data doesn't match checksum";
     case PREFIX(parameter_unsupported): return "Unsupported parameter";
     case PREFIX(parameter_outOfBound): return "Parameter is out of bound";

lib/compress/zstd_compress.c

@@ -2598,7 +2598,7 @@ ZSTD_entropyCompressSeqStore_internal(seqStore_t* seqStorePtr,
     entropyWorkspace = count + (MaxSeq + 1);
     entropyWkspSize -= (MaxSeq + 1) * sizeof(*count);
 
-    DEBUGLOG(4, "ZSTD_entropyCompressSeqStore_internal (nbSeq=%zu)", nbSeq);
+    DEBUGLOG(5, "ZSTD_entropyCompressSeqStore_internal (nbSeq=%zu)", nbSeq);
     ZSTD_STATIC_ASSERT(HUF_WORKSPACE_SIZE >= (1<<MAX(MLFSELog,LLFSELog)));
     assert(entropyWkspSize >= HUF_WORKSPACE_SIZE);
 
@@ -2642,11 +2642,10 @@ ZSTD_entropyCompressSeqStore_internal(seqStore_t* seqStorePtr,
         ZSTD_memcpy(&nextEntropy->fse, &prevEntropy->fse, sizeof(prevEntropy->fse));
         return (size_t)(op - ostart);
     }
-    {
+    {   BYTE* seqHead = op++;
-        ZSTD_symbolEncodingTypeStats_t stats;
-        BYTE* seqHead = op++;
         /* build stats for sequences */
-        stats = ZSTD_buildSequencesStatistics(seqStorePtr, nbSeq,
+        const ZSTD_symbolEncodingTypeStats_t stats =
+                ZSTD_buildSequencesStatistics(seqStorePtr, nbSeq,
                                              &prevEntropy->fse, &nextEntropy->fse,
                                               op, oend,
                                               strategy, count,
@@ -5752,9 +5751,9 @@ size_t ZSTD_compress2(ZSTD_CCtx* cctx,
 }
 
 typedef struct {
-    U32 idx;             /* Index in array of ZSTD_Sequence */
+    U32 idx;            /* Index in array of ZSTD_Sequence */
-    U32 posInSequence;   /* Position within sequence at idx */
+    U32 posInSequence;  /* Position within sequence at idx */
-    size_t posInSrc;        /* Number of bytes given by sequences provided so far */
+    size_t posInSrc;    /* Number of bytes given by sequences provided so far */
 } ZSTD_sequencePosition;
 
 /* ZSTD_validateSequence() :
@@ -5809,6 +5808,8 @@ ZSTD_copySequencesToSeqStoreExplicitBlockDelim(ZSTD_CCtx* cctx,
     repcodes_t updatedRepcodes;
     U32 dictSize;
 
+    DEBUGLOG(5, "ZSTD_copySequencesToSeqStoreExplicitBlockDelim (blockSize = %zu)", blockSize);
+
     if (cctx->cdict) {
         dictSize = (U32)cctx->cdict->dictContentSize;
     } else if (cctx->prefixDict.dict) {
@@ -5995,6 +5996,57 @@ static ZSTD_sequenceCopier ZSTD_selectSequenceCopier(ZSTD_sequenceFormat_e mode)
     return sequenceCopier;
 }
 
+/* Discover the size of next block by searching for the delimiter.
+ * Note that a block delimiter **must** exist in this mode,
+ * otherwise it's an input error.
+ * The block size retrieved will be later compared to ensure it remains within bounds */
+static size_t
+blockSize_explicitDelimiter(const ZSTD_Sequence* inSeqs, size_t inSeqsSize, ZSTD_sequencePosition seqPos)
+{
+    int end = 0;
+    size_t blockSize = 0;
+    size_t spos = seqPos.idx;
+    DEBUGLOG(6, "blockSize_explicitDelimiter : seq %zu / %zu", spos, inSeqsSize);
+    assert(spos <= inSeqsSize);
+    while (spos < inSeqsSize) {
+        end = (inSeqs[spos].offset == 0);
+        blockSize += inSeqs[spos].litLength + inSeqs[spos].matchLength;
+        if (end) {
+            if (inSeqs[spos].matchLength != 0)
+                RETURN_ERROR(corruption_detected, "delimiter format error : both matchlength and offset must be == 0");
+            break;
+        }
+        spos++;
+    }
+    if (!end)
+        RETURN_ERROR(corruption_detected, "Reached end of sequences without finding a block delimiter");
+    return blockSize;
+}
+
+/* More a "target" block size */
+static size_t blockSize_noDelimiter(size_t blockSize, size_t remaining)
+{
+    int const lastBlock = (remaining <= blockSize);
+    return lastBlock ? remaining : blockSize;
+}
+
+static size_t determine_blockSize(ZSTD_sequenceFormat_e mode,
+                           size_t blockSize, size_t remaining,
+                     const ZSTD_Sequence* inSeqs, size_t inSeqsSize, ZSTD_sequencePosition seqPos)
+{
+    DEBUGLOG(6, "determine_blockSize : remainingSize = %zu", remaining);
+    if (mode == ZSTD_sf_noBlockDelimiters)
+        return blockSize_noDelimiter(blockSize, remaining);
+    {   size_t const explicitBlockSize = blockSize_explicitDelimiter(inSeqs, inSeqsSize, seqPos);
+        FORWARD_IF_ERROR(explicitBlockSize, "Error while determining block size with explicit delimiters");
+        if (explicitBlockSize > blockSize)
+            RETURN_ERROR(corruption_detected, "sequences incorrectly define a too large block");
+        if (explicitBlockSize > remaining)
+            RETURN_ERROR(srcSize_wrong, "sequences define a frame longer than source");
+        return explicitBlockSize;
+    }
+}
+
 /* Compress, block-by-block, all of the sequences given.
  *
  * Returns the cumulative size of all compressed blocks (including their headers),
@@ -6007,9 +6059,6 @@ ZSTD_compressSequences_internal(ZSTD_CCtx* cctx,
                           const void* src, size_t srcSize)
 {
     size_t cSize = 0;
-    U32 lastBlock;
-    size_t blockSize;
-    size_t compressedSeqsSize;
     size_t remaining = srcSize;
     ZSTD_sequencePosition seqPos = {0, 0, 0};
 
@@ -6029,12 +6078,17 @@ ZSTD_compressSequences_internal(ZSTD_CCtx* cctx,
     }
 
     while (remaining) {
+        size_t compressedSeqsSize;
         size_t cBlockSize;
         size_t additionalByteAdjustment;
-        lastBlock = remaining <= cctx->blockSize;
+        size_t blockSize = determine_blockSize(cctx->appliedParams.blockDelimiters,
-        blockSize = lastBlock ? (U32)remaining : (U32)cctx->blockSize;
+                                        cctx->blockSize, remaining,
+                                        inSeqs, inSeqsSize, seqPos);
+        U32 const lastBlock = (blockSize == remaining);
+        FORWARD_IF_ERROR(blockSize, "Error while trying to determine block size");
+        assert(blockSize <= remaining);
         ZSTD_resetSeqStore(&cctx->seqStore);
-        DEBUGLOG(4, "Working on new block. Blocksize: %zu", blockSize);
+        DEBUGLOG(5, "Working on new block. Blocksize: %zu", blockSize);
 
         additionalByteAdjustment = sequenceCopier(cctx, &seqPos, inSeqs, inSeqsSize, ip, blockSize);
         FORWARD_IF_ERROR(additionalByteAdjustment, "Bad sequence copy");
@@ -6044,7 +6098,7 @@ ZSTD_compressSequences_internal(ZSTD_CCtx* cctx,
         if (blockSize < MIN_CBLOCK_SIZE+ZSTD_blockHeaderSize+1) {
             cBlockSize = ZSTD_noCompressBlock(op, dstCapacity, ip, blockSize, lastBlock);
             FORWARD_IF_ERROR(cBlockSize, "Nocompress block failed");
-            DEBUGLOG(4, "Block too small, writing out nocompress block: cSize: %zu", cBlockSize);
+            DEBUGLOG(5, "Block too small, writing out nocompress block: cSize: %zu", cBlockSize);
             cSize += cBlockSize;
             ip += blockSize;
             op += cBlockSize;
@@ -6061,7 +6115,7 @@ ZSTD_compressSequences_internal(ZSTD_CCtx* cctx,
                                 cctx->entropyWorkspace, ENTROPY_WORKSPACE_SIZE /* statically allocated in resetCCtx */,
                                 cctx->bmi2);
         FORWARD_IF_ERROR(compressedSeqsSize, "Compressing sequences of block failed");
-        DEBUGLOG(4, "Compressed sequences size: %zu", compressedSeqsSize);
+        DEBUGLOG(5, "Compressed sequences size: %zu", compressedSeqsSize);
 
         if (!cctx->isFirstBlock &&
             ZSTD_maybeRLE(&cctx->seqStore) &&
@@ -6077,11 +6131,11 @@ ZSTD_compressSequences_internal(ZSTD_CCtx* cctx,
             /* ZSTD_noCompressBlock writes the block header as well */
             cBlockSize = ZSTD_noCompressBlock(op, dstCapacity, ip, blockSize, lastBlock);
             FORWARD_IF_ERROR(cBlockSize, "Nocompress block failed");
-            DEBUGLOG(4, "Writing out nocompress block, size: %zu", cBlockSize);
+            DEBUGLOG(5, "Writing out nocompress block, size: %zu", cBlockSize);
         } else if (compressedSeqsSize == 1) {
             cBlockSize = ZSTD_rleCompressBlock(op, dstCapacity, *ip, blockSize, lastBlock);
             FORWARD_IF_ERROR(cBlockSize, "RLE compress block failed");
-            DEBUGLOG(4, "Writing out RLE block, size: %zu", cBlockSize);
+            DEBUGLOG(5, "Writing out RLE block, size: %zu", cBlockSize);
         } else {
             U32 cBlockHeader;
             /* Error checking and repcodes update */
@@ -6093,11 +6147,11 @@ ZSTD_compressSequences_internal(ZSTD_CCtx* cctx,
             cBlockHeader = lastBlock + (((U32)bt_compressed)<<1) + (U32)(compressedSeqsSize << 3);
             MEM_writeLE24(op, cBlockHeader);
             cBlockSize = ZSTD_blockHeaderSize + compressedSeqsSize;
-            DEBUGLOG(4, "Writing out compressed block, size: %zu", cBlockSize);
+            DEBUGLOG(5, "Writing out compressed block, size: %zu", cBlockSize);
         }
 
         cSize += cBlockSize;
-        DEBUGLOG(4, "cSize running total: %zu", cSize);
+        DEBUGLOG(5, "cSize running total: %zu", cSize);
 
         if (lastBlock) {
             break;
@@ -6110,10 +6164,12 @@ ZSTD_compressSequences_internal(ZSTD_CCtx* cctx,
         }
     }
 
+    DEBUGLOG(4, "cSize final total: %zu", cSize);
     return cSize;
 }
 
-size_t ZSTD_compressSequences(ZSTD_CCtx* const cctx, void* dst, size_t dstCapacity,
+size_t ZSTD_compressSequences(ZSTD_CCtx* cctx,
+                              void* dst, size_t dstCapacity,
                               const ZSTD_Sequence* inSeqs, size_t inSeqsSize,
                               const void* src, size_t srcSize)
 {

lib/zstd.h

@@ -1375,23 +1375,25 @@ typedef enum {
 } ZSTD_sequenceFormat_e;
 
 /*! ZSTD_generateSequences() :
- * Generate sequences using ZSTD_compress2, given a source buffer.
+ * Generate sequences using ZSTD_compress2(), given a source buffer.
  *
  * Each block will end with a dummy sequence
  * with offset == 0, matchLength == 0, and litLength == length of last literals.
  * litLength may be == 0, and if so, then the sequence of (of: 0 ml: 0 ll: 0)
  * simply acts as a block delimiter.
  *
- * zc can be used to insert custom compression params.
+ * @zc can be used to insert custom compression params.
- * This function invokes ZSTD_compress2
+ * This function invokes ZSTD_compress2().
  *
  * The output of this function can be fed into ZSTD_compressSequences() with CCtx
  * setting of ZSTD_c_blockDelimiters as ZSTD_sf_explicitBlockDelimiters
  * @return : number of sequences generated
  */
 
-ZSTDLIB_STATIC_API size_t ZSTD_generateSequences(ZSTD_CCtx* zc, ZSTD_Sequence* outSeqs,
+ZSTDLIB_STATIC_API size_t
-                                          size_t outSeqsSize, const void* src, size_t srcSize);
+ZSTD_generateSequences( ZSTD_CCtx* zc,
+                        ZSTD_Sequence* outSeqs, size_t outSeqsSize,
+                        const void* src, size_t srcSize);
 
 /*! ZSTD_mergeBlockDelimiters() :
  * Given an array of ZSTD_Sequence, remove all sequences that represent block delimiters/last literals
@@ -1432,11 +1434,12 @@ ZSTDLIB_STATIC_API size_t ZSTD_mergeBlockDelimiters(ZSTD_Sequence* sequences, si
  * Note: Repcodes are, as of now, always re-calculated within this function, so ZSTD_Sequence::rep is unused.
  * Note 2: Once we integrate ability to ingest repcodes, the explicit block delims mode must respect those repcodes exactly,
  *         and cannot emit an RLE block that disagrees with the repcode history
- * @return : final compressed size or a ZSTD error.
+ * @return : final compressed size, or a ZSTD error code.
  */
-ZSTDLIB_STATIC_API size_t ZSTD_compressSequences(ZSTD_CCtx* const cctx, void* dst, size_t dstSize,
+ZSTDLIB_STATIC_API size_t
-                                  const ZSTD_Sequence* inSeqs, size_t inSeqsSize,
+ZSTD_compressSequences( ZSTD_CCtx* cctx, void* dst, size_t dstSize,
-                                  const void* src, size_t srcSize);
+                        const ZSTD_Sequence* inSeqs, size_t inSeqsSize,
+                        const void* src, size_t srcSize);
 
 
 /*! ZSTD_writeSkippableFrame() :

tests/fuzz/sequence_compression_api.c

@@ -26,8 +26,8 @@
 #include "zstd_helpers.h"
 #include "fuzz_data_producer.h"
 
-static ZSTD_CCtx *cctx = NULL;
+static ZSTD_CCtx* cctx = NULL;
-static ZSTD_DCtx *dctx = NULL;
+static ZSTD_DCtx* dctx = NULL;
 static void* literalsBuffer = NULL;
 static void* generatedSrc = NULL;
 static ZSTD_Sequence* generatedSequences = NULL;
@@ -55,7 +55,7 @@ static uint32_t FUZZ_RDG_rand(uint32_t* src)
 /* Make a pseudorandom string - this simple function exists to avoid
  * taking a dependency on datagen.h to have RDG_genBuffer().
  */
-static char *generatePseudoRandomString(char *str, size_t size) {
+static char* generatePseudoRandomString(char* str, size_t size) {
     const char charset[] = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJK1234567890!@#$^&*()_";
     uint32_t seed = 0;
     if (size) {
@@ -69,7 +69,10 @@ static char *generatePseudoRandomString(char *str, size_t size) {
 
 /* Returns size of source buffer */
 static size_t decodeSequences(void* dst, size_t nbSequences,
-                              size_t literalsSize, const void* dict, size_t dictSize) {
+                              size_t literalsSize,
+                              const void* dict, size_t dictSize,
+                              ZSTD_sequenceFormat_e mode)
+{
     const uint8_t* litPtr = literalsBuffer;
     const uint8_t* const litBegin = literalsBuffer;
     const uint8_t* const litEnd = litBegin + literalsSize;
@@ -78,21 +81,20 @@ static size_t decodeSequences(void* dst, size_t nbSequences,
     const uint8_t* const oend = (uint8_t*)dst + ZSTD_FUZZ_GENERATED_SRC_MAXSIZE;
     size_t generatedSrcBufferSize = 0;
     size_t bytesWritten = 0;
-    uint32_t lastLLSize;
 
     for (size_t i = 0; i < nbSequences; ++i) {
-        FUZZ_ASSERT(generatedSequences[i].matchLength != 0);
+        /* block boundary */
-        FUZZ_ASSERT(generatedSequences[i].offset != 0);
+        if (generatedSequences[i].offset == 0)
+            FUZZ_ASSERT(generatedSequences[i].matchLength == 0);
 
         if (litPtr + generatedSequences[i].litLength > litEnd) {
             litPtr = litBegin;
         }
-        ZSTD_memcpy(op, litPtr, generatedSequences[i].litLength);
+        memcpy(op, litPtr, generatedSequences[i].litLength);
         bytesWritten += generatedSequences[i].litLength;
         op += generatedSequences[i].litLength;
         litPtr += generatedSequences[i].litLength;
 
-        FUZZ_ASSERT(generatedSequences[i].offset != 0);
         /* Copy over the match */
         {   size_t matchLength = generatedSequences[i].matchLength;
             size_t j = 0;
@@ -109,7 +111,7 @@ static size_t decodeSequences(void* dst, size_t nbSequences,
                 }
             }
             for (; j < matchLength; ++j) {
-                op[j] = op[j-(int)generatedSequences[i].offset];
+                op[j] = op[j - generatedSequences[i].offset];
             }
             op += j;
             FUZZ_ASSERT(generatedSequences[i].matchLength == j + k);
@@ -118,55 +120,65 @@ static size_t decodeSequences(void* dst, size_t nbSequences,
     }
     generatedSrcBufferSize = bytesWritten;
     FUZZ_ASSERT(litPtr <= litEnd);
-    lastLLSize = (uint32_t)(litEnd - litPtr);
+    if (mode == ZSTD_sf_noBlockDelimiters) {
-    if (lastLLSize <= oend - op) {
+        const uint32_t lastLLSize = (uint32_t)(litEnd - litPtr);
-        ZSTD_memcpy(op, litPtr, lastLLSize);
+        if (lastLLSize <= oend - op) {
-        generatedSrcBufferSize += lastLLSize;
+            memcpy(op, litPtr, lastLLSize);
-    }
+            generatedSrcBufferSize += lastLLSize;
+    }   }
     return generatedSrcBufferSize;
 }
 
 /* Returns nb sequences generated
- * TODO: Add repcode fuzzing once we support repcode match splits
+ * Note : random sequences are always valid in ZSTD_sf_noBlockDelimiters mode.
+ * However, it can fail with ZSTD_sf_explicitBlockDelimiters,
+ * due to potential lack of space in
  */
 static size_t generateRandomSequences(FUZZ_dataProducer_t* producer,
                                       size_t literalsSizeLimit, size_t dictSize,
-                                      size_t windowLog) {
+                                      size_t windowLog, ZSTD_sequenceFormat_e mode)
+{
+    const uint32_t repCode = 0;  /* not used by sequence ingestion api */
+    const uint32_t windowSize = 1 << windowLog;
+    const uint32_t blockSizeMax = MIN(128 << 10, 1 << windowLog);
+    uint32_t matchLengthMax = ZSTD_FUZZ_MATCHLENGTH_MAXSIZE;
     uint32_t bytesGenerated = 0;
     uint32_t nbSeqGenerated = 0;
-    uint32_t litLength;
-    uint32_t matchLength;
-    uint32_t matchBound;
-    uint32_t offset;
-    uint32_t offsetBound;
-    uint32_t repCode = 0;
     uint32_t isFirstSequence = 1;
-    uint32_t windowSize = 1 << windowLog;
+    uint32_t blockSize = 0;
 
-    while (nbSeqGenerated < ZSTD_FUZZ_MAX_NBSEQ
+    if (mode == ZSTD_sf_explicitBlockDelimiters) {
+        /* ensure that no sequence can be larger than one block */
+        literalsSizeLimit = MIN(literalsSizeLimit, blockSizeMax/2);
+        matchLengthMax = MIN(matchLengthMax, blockSizeMax/2);
+    }
+
+    while ( nbSeqGenerated < ZSTD_FUZZ_MAX_NBSEQ-1
          && bytesGenerated < ZSTD_FUZZ_GENERATED_SRC_MAXSIZE
          && !FUZZ_dataProducer_empty(producer)) {
-        matchBound = ZSTD_FUZZ_MATCHLENGTH_MAXSIZE;
+        uint32_t matchLength;
-        litLength = isFirstSequence && dictSize == 0 ? FUZZ_dataProducer_uint32Range(producer, 1, literalsSizeLimit)
+        uint32_t matchBound = matchLengthMax;
-                                                     : FUZZ_dataProducer_uint32Range(producer, 0, literalsSizeLimit);
+        uint32_t offset;
+        uint32_t offsetBound;
+        const uint32_t minLitLength = (isFirstSequence && (dictSize == 0));
+        const uint32_t litLength = FUZZ_dataProducer_uint32Range(producer, minLitLength, (uint32_t)literalsSizeLimit);
         bytesGenerated += litLength;
         if (bytesGenerated > ZSTD_FUZZ_GENERATED_SRC_MAXSIZE) {
             break;
         }
-        offsetBound = bytesGenerated > windowSize ? windowSize : bytesGenerated + dictSize;
+        offsetBound = (bytesGenerated > windowSize) ? windowSize : bytesGenerated + (uint32_t)dictSize;
         offset = FUZZ_dataProducer_uint32Range(producer, 1, offsetBound);
         if (dictSize > 0 && bytesGenerated <= windowSize) {
             /* Prevent match length from being such that it would be associated with an offset too large
              * from the decoder's perspective. If not possible (match would be too small),
              * then reduce the offset if necessary.
              */
-            size_t bytesToReachWindowSize = windowSize - bytesGenerated;
+            const size_t bytesToReachWindowSize = windowSize - bytesGenerated;
             if (bytesToReachWindowSize < ZSTD_MINMATCH_MIN) {
-                uint32_t newOffsetBound = offsetBound > windowSize ? windowSize : offsetBound;
+                const uint32_t newOffsetBound = offsetBound > windowSize ? windowSize : offsetBound;
                 offset = FUZZ_dataProducer_uint32Range(producer, 1, newOffsetBound);
             } else {
-                matchBound = bytesToReachWindowSize > ZSTD_FUZZ_MATCHLENGTH_MAXSIZE ?
+                matchBound = MIN(matchLengthMax, (uint32_t)bytesToReachWindowSize);
-                             ZSTD_FUZZ_MATCHLENGTH_MAXSIZE : bytesToReachWindowSize;
             }
         }
         matchLength = FUZZ_dataProducer_uint32Range(producer, ZSTD_MINMATCH_MIN, matchBound);
@@ -174,9 +186,35 @@ static size_t generateRandomSequences(FUZZ_dataProducer_t* producer,
         if (bytesGenerated > ZSTD_FUZZ_GENERATED_SRC_MAXSIZE) {
             break;
         }
-        ZSTD_Sequence seq = {offset, litLength, matchLength, repCode};
+        {   ZSTD_Sequence seq = {offset, litLength, matchLength, repCode};
-        generatedSequences[nbSeqGenerated++] = seq;
+            const uint32_t lastLits = FUZZ_dataProducer_uint32Range(producer, 0, litLength);
-        isFirstSequence = 0;
+            #define SPLITPROB 6000
+            #define SPLITMARK 5234
+            const int split = (FUZZ_dataProducer_uint32Range(producer, 0, SPLITPROB) == SPLITMARK);
+            if (mode == ZSTD_sf_explicitBlockDelimiters) {
+                const size_t seqSize = seq.litLength + seq.matchLength;
+                if (blockSize + seqSize > blockSizeMax) {  /* reaching limit : must end block now */
+                    const ZSTD_Sequence endBlock = {0, 0, 0, 0};
+                    generatedSequences[nbSeqGenerated++] = endBlock;
+                    blockSize = seqSize;
+                }
+                if (split) {
+                    const ZSTD_Sequence endBlock = {0, lastLits, 0, 0};
+                    generatedSequences[nbSeqGenerated++] = endBlock;
+                    assert(lastLits <= seq.litLength);
+                    seq.litLength -= lastLits;
+                    blockSize = seqSize - lastLits;
+                } else {
+                    blockSize += seqSize;
+                }
+            }
+            generatedSequences[nbSeqGenerated++] = seq;
+            isFirstSequence = 0;
+    }   }
+    if (mode == ZSTD_sf_explicitBlockDelimiters) {
+        /* always end sequences with a block delimiter */
+        const ZSTD_Sequence endBlock = {0, 0, 0, 0};
+        generatedSequences[nbSeqGenerated++] = endBlock;
     }
 
     return nbSeqGenerated;
@@ -187,12 +225,11 @@ static size_t roundTripTest(void *result, size_t resultCapacity,
                             size_t srcSize,
                             const void *dict, size_t dictSize,
                             size_t generatedSequencesSize,
-                            size_t wLog, unsigned cLevel, unsigned hasDict)
+                            int wLog, int cLevel, unsigned hasDict,
+                            ZSTD_sequenceFormat_e mode)
 {
     size_t cSize;
     size_t dSize;
-    ZSTD_CDict* cdict = NULL;
-    ZSTD_DDict* ddict = NULL;
 
     ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
     ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 0);
@@ -200,8 +237,7 @@ static size_t roundTripTest(void *result, size_t resultCapacity,
     ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, wLog);
     ZSTD_CCtx_setParameter(cctx, ZSTD_c_minMatch, ZSTD_MINMATCH_MIN);
     ZSTD_CCtx_setParameter(cctx, ZSTD_c_validateSequences, 1);
-    /* TODO: Add block delim mode fuzzing */
+    ZSTD_CCtx_setParameter(cctx, ZSTD_c_blockDelimiters, mode);
-    ZSTD_CCtx_setParameter(cctx, ZSTD_c_blockDelimiters, ZSTD_sf_noBlockDelimiters);
     if (hasDict) {
         FUZZ_ZASSERT(ZSTD_CCtx_loadDictionary(cctx, dict, dictSize));
         FUZZ_ZASSERT(ZSTD_DCtx_loadDictionary(dctx, dict, dictSize));
@@ -214,16 +250,10 @@ static size_t roundTripTest(void *result, size_t resultCapacity,
     dSize = ZSTD_decompressDCtx(dctx, result, resultCapacity, compressed, cSize);
     FUZZ_ZASSERT(dSize);
 
-    if (cdict) {
-        ZSTD_freeCDict(cdict);
-    }
-    if (ddict) {
-        ZSTD_freeDDict(ddict);
-    }
     return dSize;
 }
 
-int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size)
+int LLVMFuzzerTestOneInput(const uint8_t* src, size_t size)
 {
     void* rBuf;
     size_t rBufSize;
@@ -231,15 +261,18 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size)
     size_t cBufSize;
     size_t generatedSrcSize;
     size_t nbSequences;
-    void* dictBuffer;
+    void* dictBuffer = NULL;
     size_t dictSize = 0;
     unsigned hasDict;
     unsigned wLog;
     int cLevel;
+    ZSTD_sequenceFormat_e mode;
 
-    FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(src, size);
+    FUZZ_dataProducer_t* const producer = FUZZ_dataProducer_create(src, size);
+    FUZZ_ASSERT(producer);
     if (literalsBuffer == NULL) {
         literalsBuffer = FUZZ_malloc(ZSTD_FUZZ_GENERATED_LITERALS_SIZE);
+        FUZZ_ASSERT(literalsBuffer);
         literalsBuffer = generatePseudoRandomString(literalsBuffer, ZSTD_FUZZ_GENERATED_LITERALS_SIZE);
     }
 
@@ -247,11 +280,13 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size)
     if (hasDict) {
         dictSize = FUZZ_dataProducer_uint32Range(producer, 1, ZSTD_FUZZ_GENERATED_DICT_MAXSIZE);
         dictBuffer = FUZZ_malloc(dictSize);
+        FUZZ_ASSERT(dictBuffer);
         dictBuffer = generatePseudoRandomString(dictBuffer, dictSize);
     }
     /* Generate window log first so we dont generate offsets too large */
     wLog = FUZZ_dataProducer_uint32Range(producer, ZSTD_WINDOWLOG_MIN, ZSTD_WINDOWLOG_MAX_32);
     cLevel = FUZZ_dataProducer_int32Range(producer, -3, 22);
+    mode = (ZSTD_sequenceFormat_e)FUZZ_dataProducer_int32Range(producer, 0, 1);
 
     if (!generatedSequences) {
         generatedSequences = FUZZ_malloc(sizeof(ZSTD_Sequence)*ZSTD_FUZZ_MAX_NBSEQ);
@@ -259,8 +294,8 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size)
     if (!generatedSrc) {
         generatedSrc = FUZZ_malloc(ZSTD_FUZZ_GENERATED_SRC_MAXSIZE);
     }
-    nbSequences = generateRandomSequences(producer, ZSTD_FUZZ_GENERATED_LITERALS_SIZE, dictSize, wLog);
+    nbSequences = generateRandomSequences(producer, ZSTD_FUZZ_GENERATED_LITERALS_SIZE, dictSize, wLog, mode);
-    generatedSrcSize = decodeSequences(generatedSrc, nbSequences, ZSTD_FUZZ_GENERATED_LITERALS_SIZE, dictBuffer, dictSize);
+    generatedSrcSize = decodeSequences(generatedSrc, nbSequences, ZSTD_FUZZ_GENERATED_LITERALS_SIZE, dictBuffer, dictSize, mode);
     cBufSize = ZSTD_compressBound(generatedSrcSize);
     cBuf = FUZZ_malloc(cBufSize);
 
@@ -276,14 +311,15 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size)
         FUZZ_ASSERT(dctx);
     }
 
-    size_t const result = roundTripTest(rBuf, rBufSize,
+    {   const size_t result = roundTripTest(rBuf, rBufSize,
                                         cBuf, cBufSize,
                                         generatedSrcSize,
                                         dictBuffer, dictSize,
                                         nbSequences,
-                                        wLog, cLevel, hasDict);
+                                        (int)wLog, cLevel, hasDict, mode);
-    FUZZ_ZASSERT(result);
+        FUZZ_ZASSERT(result);
-    FUZZ_ASSERT_MSG(result == generatedSrcSize, "Incorrect regenerated size");
+        FUZZ_ASSERT_MSG(result == generatedSrcSize, "Incorrect regenerated size");
+    }
     FUZZ_ASSERT_MSG(!FUZZ_memcmp(generatedSrc, rBuf, generatedSrcSize), "Corruption!");
 
     free(rBuf);

tests/fuzzer.c

@@ -3114,18 +3114,17 @@ static int basicUnitTests(U32 const seed, double compressibility)
 
     DISPLAYLEVEL(3, "test%3i : ZSTD_getSequences followed by ZSTD_compressSequences : ", testNb++);
     {
-        size_t srcSize = 500 KB;
+        const size_t srcSize = 500 KB;
-        BYTE* src = (BYTE*)CNBuffer;
+        const BYTE* const src = (BYTE*)CNBuffer;
-        BYTE* dst = (BYTE*)compressedBuffer;
+        BYTE* const dst = (BYTE*)compressedBuffer;
-        size_t dstSize = ZSTD_compressBound(srcSize);
+        const size_t dstCapacity = ZSTD_compressBound(srcSize);
-        size_t decompressSize = srcSize;
+        const size_t decompressSize = srcSize;
-        char* decompressBuffer = (char*)malloc(decompressSize);
+        char* const decompressBuffer = (char*)malloc(decompressSize);
         size_t compressedSize;
-        size_t dSize;
 
-        ZSTD_CCtx* cctx = ZSTD_createCCtx();
+        ZSTD_CCtx* const cctx = ZSTD_createCCtx();
-        ZSTD_Sequence* seqs = (ZSTD_Sequence*)malloc(srcSize * sizeof(ZSTD_Sequence));
+        ZSTD_Sequence* const seqs = (ZSTD_Sequence*)malloc(srcSize * sizeof(ZSTD_Sequence));
-        size_t seqsSize;
+        size_t nbSeqs;
 
         if (seqs == NULL) goto _output_error;
         assert(cctx != NULL);
@@ -3133,36 +3132,37 @@ static int basicUnitTests(U32 const seed, double compressibility)
         /* Populate src with random data */
         RDG_genBuffer(CNBuffer, srcSize, compressibility, 0., seed);
 
-        /* Test with block delimiters roundtrip */
+        /* Roundtrip Test with block delimiters generated by ZSTD_generateSequences() */
-        seqsSize = ZSTD_generateSequences(cctx, seqs, srcSize, src, srcSize);
+        nbSeqs = ZSTD_generateSequences(cctx, seqs, srcSize, src, srcSize);
         ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
         ZSTD_CCtx_setParameter(cctx, ZSTD_c_blockDelimiters, ZSTD_sf_explicitBlockDelimiters);
-        compressedSize = ZSTD_compressSequences(cctx, dst, dstSize, seqs, seqsSize, src, srcSize);
+        compressedSize = ZSTD_compressSequences(cctx, dst, dstCapacity, seqs, nbSeqs, src, srcSize);
         if (ZSTD_isError(compressedSize)) {
             DISPLAY("Error in sequence compression with block delims\n");
             goto _output_error;
         }
-        dSize = ZSTD_decompress(decompressBuffer, decompressSize, dst, compressedSize);
+        {   size_t const dSize = ZSTD_decompress(decompressBuffer, decompressSize, dst, compressedSize);
-        if (ZSTD_isError(dSize)) {
+            if (ZSTD_isError(dSize)) {
-            DISPLAY("Error in sequence compression roundtrip with block delims\n");
+                DISPLAY("Error in sequence compression roundtrip with block delims\n");
-            goto _output_error;
+                goto _output_error;
-        }
+        }   }
         assert(!memcmp(decompressBuffer, src, srcSize));
 
-        /* Test with no block delimiters roundtrip */
+        /* Roundtrip Test with no block delimiters  */
-        seqsSize = ZSTD_mergeBlockDelimiters(seqs, seqsSize);
+        {   size_t const nbSeqsAfterMerge = ZSTD_mergeBlockDelimiters(seqs, nbSeqs);
-        ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
+            ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
-        ZSTD_CCtx_setParameter(cctx, ZSTD_c_blockDelimiters, ZSTD_sf_noBlockDelimiters);
+            ZSTD_CCtx_setParameter(cctx, ZSTD_c_blockDelimiters, ZSTD_sf_noBlockDelimiters);
-        compressedSize = ZSTD_compressSequences(cctx, dst, dstSize, seqs, seqsSize, src, srcSize);
+            compressedSize = ZSTD_compressSequences(cctx, dst, dstCapacity, seqs, nbSeqsAfterMerge, src, srcSize);
+        }
         if (ZSTD_isError(compressedSize)) {
             DISPLAY("Error in sequence compression with no block delims\n");
             goto _output_error;
         }
-        dSize = ZSTD_decompress(decompressBuffer, decompressSize, dst, compressedSize);
+        {   size_t const dSize = ZSTD_decompress(decompressBuffer, decompressSize, dst, compressedSize);
-        if (ZSTD_isError(dSize)) {
+            if (ZSTD_isError(dSize)) {
-            DISPLAY("Error in sequence compression roundtrip with no block delims\n");
+                DISPLAY("Error in sequence compression roundtrip with no block delims\n");
-            goto _output_error;
+                goto _output_error;
-        }
+        }   }
         assert(!memcmp(decompressBuffer, src, srcSize));
 
         ZSTD_freeCCtx(cctx);
@@ -3968,9 +3968,9 @@ static int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, U32 const
         DISPLAYLEVEL(5, "fuzzer t%u: Bufferless streaming compression test \n", testNb);
         {   U32 const testLog = FUZ_rand(&lseed) % maxSrcLog;
             U32 const dictLog = FUZ_rand(&lseed) % maxSrcLog;
-            int const cLevel = (FUZ_rand(&lseed) %
+            int const cLevel = (int)(FUZ_rand(&lseed) %
-                                (ZSTD_maxCLevel() -
+                                ((U32)ZSTD_maxCLevel() -
-                                 (MAX(testLog, dictLog) / cLevelLimiter))) +
+                                 (MAX(testLog, dictLog) / (U32)cLevelLimiter))) +
                                1;
             maxTestSize = FUZ_rLogLength(&lseed, testLog);
             if (maxTestSize >= dstBufferSize) maxTestSize = dstBufferSize-1;
@@ -4066,7 +4066,7 @@ _cleanup:
     free(cBuffer);
     free(dstBuffer);
     free(mirrorBuffer);
-    return result;
+    return (int)result;
 
 _output_error:
     result = 1;
@@ -4103,7 +4103,7 @@ static unsigned readU32FromChar(const char** stringPtr)
 {
     unsigned result = 0;
     while ((**stringPtr >='0') && (**stringPtr <='9'))
-        result *= 10, result += **stringPtr - '0', (*stringPtr)++ ;
+        result *= 10, result += (unsigned)(**stringPtr - '0'), (*stringPtr)++ ;
     if ((**stringPtr=='K') || (**stringPtr=='M')) {
         result <<= 10;
         if (**stringPtr=='M') result <<= 10;
@@ -4245,7 +4245,7 @@ int main(int argc, const char** argv)
         }
     }
     if (!result)
-        result = fuzzerTests(seed, nbTests, testNb, maxDuration, ((double)proba) / 100, bigTests);
+        result = fuzzerTests(seed, (unsigned)nbTests, (unsigned)testNb, maxDuration, ((double)proba) / 100, bigTests);
     if (mainPause) {
         int unused;
         DISPLAY("Press Enter \n");