Merge branch 'dev' of github.com:Cyan4973/zstd into dev

Makefile

@@ -44,7 +44,7 @@ clean:
 	@$(MAKE) -C $(PRGDIR) $@ > $(VOID)
 	@$(MAKE) -C $(TESTDIR) $@ > $(VOID)
 	@$(MAKE) -C $(ZWRAPDIR) $@ > $(VOID)
-	@rm -f zstd
+	@$(RM) zstd
 	@echo Cleaning completed
 
 
@@ -121,7 +121,7 @@ endif
 ifneq (,$(filter $(HOST_OS),MSYS POSIX))
 cmaketest:
 	cmake --version
-	rm -rf projects/cmake/build
+	$(RM) -r projects/cmake/build
 	mkdir projects/cmake/build
 	cd projects/cmake/build ; cmake -DPREFIX:STRING=~/install_test_dir $(CMAKE_PARAMS) .. ; $(MAKE) install ; $(MAKE) uninstall
 

README.md

@@ -48,7 +48,7 @@ For a larger picture including very slow modes, [click on this link](images/DCsp
 
 Previous charts provide results applicable to typical file and stream scenarios (several MB). Small data comes with different perspectives. The smaller the amount of data to compress, the more difficult it is to achieve any significant compression.
 
-This problem is common to any compression algorithm. The reason is, compression algorithms learn from past data how to compress future data. But at the beginning of a new file, there is no "past" to build upon.
+This problem is common to many compression algorithms. The reason is, compression algorithms learn from past data how to compress future data. But at the beginning of a new file, there is no "past" to build upon.
 
 To solve this situation, Zstd offers a __training mode__, which can be used to tune the algorithm for a selected type of data, by providing it with a few samples. The result of the training is stored in a file called "dictionary", which can be loaded before compression and decompression. Using this dictionary, the compression ratio achievable on small data improves dramatically:
 

examples/Makefile

@@ -61,14 +61,15 @@ clean:
 
 test: all
 	cp README.md tmp
+	cp Makefile tmp2
 	@echo starting simple compression
 	./simple_compression tmp
 	./simple_decompression tmp.zst
-	./streaming_decompression tmp.zst
+	./streaming_decompression tmp.zst > /dev/null
 	@echo starting streaming compression
 	./streaming_compression tmp
-	./streaming_decompression tmp.zst
+	./streaming_decompression tmp.zst > /dev/null
 	@echo starting dictionary compression
-	./dictionary_compression tmp README.md
+	./dictionary_compression tmp2 tmp README.md
-	./dictionary_decompression tmp.zst README.md
+	./dictionary_decompression tmp2.zst tmp.zst README.md
 	@echo tests completed

examples/README.md

@@ -1,18 +1,31 @@
 Zstandard library : usage examples
 ==================================
 
-- [Simple compression](simple_compression.c)
+- [Simple compression](simple_compression.c) :
   Compress a single file.
   Introduces usage of : `ZSTD_compress()`
 
-- [Simple decompression](simple_decompression.c)
+- [Simple decompression](simple_decompression.c) :
-  Decompress a single file compressed by zstd.
+  Decompress a single file.
+  Only compatible with simple compression.
+  Result remains in memory.
   Introduces usage of : `ZSTD_decompress()`
 
-- [Dictionary compression](dictionary_compression.c)
+- [Streaming compression](streaming_compression.c) :
+  Compress a single file.
+  Introduces usage of : `ZSTD_compressStream()`
+
+- [Streaming decompression](streaming_decompression.c) :
+  Decompress a single file compressed by zstd.
+  Compatible with both simple and streaming compression.
+  Result is sent to stdout.
+  Introduces usage of : `ZSTD_decompressStream()`
+
+- [Dictionary compression](dictionary_compression.c) :
   Compress multiple files using the same dictionary.
   Introduces usage of : `ZSTD_createCDict()` and `ZSTD_compress_usingCDict()`
 
-- [Dictionary decompression](dictionary_decompression.c)
+- [Dictionary decompression](dictionary_decompression.c) :
   Decompress multiple files using the same dictionary.
+  Result remains in memory.
   Introduces usage of : `ZSTD_createDDict()` and `ZSTD_decompress_usingDDict()`

examples/dictionary_compression.c

@@ -73,12 +73,12 @@ static void saveFile_orDie(const char* fileName, const void* buff, size_t buffSi
 
 /* createDict() :
    `dictFileName` is supposed to have been created using `zstd --train` */
-static ZSTD_CDict* createCDict_orDie(const char* dictFileName)
+static ZSTD_CDict* createCDict_orDie(const char* dictFileName, int cLevel)
 {
     size_t dictSize;
     printf("loading dictionary %s \n", dictFileName);
     void* const dictBuffer = loadFile_orDie(dictFileName, &dictSize);
-    ZSTD_CDict* const cdict = ZSTD_createCDict(dictBuffer, dictSize, 3);
+    ZSTD_CDict* const cdict = ZSTD_createCDict(dictBuffer, dictSize, cLevel);
     if (!cdict) {
         fprintf(stderr, "ZSTD_createCDict error \n");
         exit(7);
@@ -96,6 +96,7 @@ static void compress(const char* fname, const char* oname, const ZSTD_CDict* cdi
     void* const cBuff = malloc_orDie(cBuffSize);
 
     ZSTD_CCtx* const cctx = ZSTD_createCCtx();
+    if (cctx==NULL) { fprintf(stderr, "ZSTD_createCCtx() error \n"); exit(10); }
     size_t const cSize = ZSTD_compress_usingCDict(cctx, cBuff, cBuffSize, fBuff, fSize, cdict);
     if (ZSTD_isError(cSize)) {
         fprintf(stderr, "error compressing %s : %s \n", fname, ZSTD_getErrorName(cSize));
@@ -107,7 +108,7 @@ static void compress(const char* fname, const char* oname, const ZSTD_CDict* cdi
     /* success */
     printf("%25s : %6u -> %7u - %s \n", fname, (unsigned)fSize, (unsigned)cSize, oname);
 
-    ZSTD_freeCCtx(cctx);
+    ZSTD_freeCCtx(cctx);   /* never fails */
     free(fBuff);
     free(cBuff);
 }
@@ -127,6 +128,7 @@ static char* createOutFilename_orDie(const char* filename)
 int main(int argc, const char** argv)
 {
     const char* const exeName = argv[0];
+    int const cLevel = 3;
 
     if (argc<3) {
         fprintf(stderr, "wrong arguments\n");
@@ -137,7 +139,7 @@ int main(int argc, const char** argv)
 
     /* load dictionary only once */
     const char* const dictName = argv[argc-1];
-    ZSTD_CDict* const dictPtr = createCDict_orDie(dictName);
+    ZSTD_CDict* const dictPtr = createCDict_orDie(dictName, cLevel);
 
     int u;
     for (u=1; u<argc-1; u++) {

examples/dictionary_decompression.c

@@ -81,11 +81,11 @@ static void decompress(const char* fname, const ZSTD_DDict* ddict)
         fprintf(stderr, "%s : original size unknown \n", fname);
         exit(6);
     }
-    void* const rBuff = malloc_orDie(rSize);
+    void* const rBuff = malloc_orDie((size_t)rSize);
 
     ZSTD_DCtx* const dctx = ZSTD_createDCtx();
+    if (dctx==NULL) { fprintf(stderr, "ZSTD_createDCtx() error \n"); exit(10); }
     size_t const dSize = ZSTD_decompress_usingDDict(dctx, rBuff, rSize, cBuff, cSize, ddict);
-
     if (dSize != rSize) {
         fprintf(stderr, "error decoding %s : %s \n", fname, ZSTD_getErrorName(dSize));
         exit(7);

examples/simple_compression.c

@@ -53,7 +53,7 @@ static void* loadFile_orDie(const char* fileName, size_t* size)
         fprintf(stderr, "fread: %s : %s \n", fileName, strerror(errno));
         exit(4);
     }
-    fclose(inFile);
+    fclose(inFile);  /* can't fail, read only */
     *size = buffSize;
     return buffer;
 }
@@ -101,7 +101,7 @@ static const char* createOutFilename_orDie(const char* filename)
 {
     size_t const inL = strlen(filename);
     size_t const outL = inL + 5;
-    void* outSpace = malloc_orDie(outL);
+    void* const outSpace = malloc_orDie(outL);
     memset(outSpace, 0, outL);
     strcat(outSpace, filename);
     strcat(outSpace, ".zst");

examples/simple_decompression.c

@@ -53,7 +53,7 @@ static void* loadFile_X(const char* fileName, size_t* size)
         printf("fread: %s : %s \n", fileName, strerror(errno));
         exit(4);
     }
-    fclose(inFile);
+    fclose(inFile);   /* can't fail (read only) */
     *size = buffSize;
     return buffer;
 }
@@ -68,7 +68,7 @@ static void decompress(const char* fname)
         printf("%s : original size unknown \n", fname);
         exit(5);
     }
-    void* const rBuff = malloc_X(rSize);
+    void* const rBuff = malloc_X((size_t)rSize);
 
     size_t const dSize = ZSTD_decompress(rBuff, rSize, cBuff, cSize);
 

examples/streaming_compression.c

@@ -69,13 +69,13 @@ static void compressFile_orDie(const char* fname, const char* outName, int cLeve
     void*  const buffIn  = malloc_orDie(buffInSize);
     size_t const buffOutSize = ZSTD_CStreamOutSize();;
     void*  const buffOut = malloc_orDie(buffOutSize);
-    size_t read, toRead = buffInSize;
 
     ZSTD_CStream* const cstream = ZSTD_createCStream();
     if (cstream==NULL) { fprintf(stderr, "ZSTD_createCStream() error \n"); exit(10); }
     size_t const initResult = ZSTD_initCStream(cstream, cLevel);
     if (ZSTD_isError(initResult)) { fprintf(stderr, "ZSTD_initCStream() error \n"); exit(11); }
 
+    size_t read, toRead = buffInSize;
     while( (read = fread_orDie(buffIn, toRead, fin)) ) {
         ZSTD_inBuffer input = { buffIn, read, 0 };
         while (input.pos < input.size) {
@@ -86,10 +86,11 @@ static void compressFile_orDie(const char* fname, const char* outName, int cLeve
     }
 
     ZSTD_outBuffer output = { buffOut, buffOutSize, 0 };
-    size_t const remainingToFlush = ZSTD_endStream(cstream, &output);
+    size_t const remainingToFlush = ZSTD_endStream(cstream, &output);   /* close frame */
     if (remainingToFlush) { fprintf(stderr, "not fully flushed"); exit(12); }
     fwrite_orDie(buffOut, output.pos, fout);
 
+    ZSTD_freeCStream(cstream);
     fclose_orDie(fout);
     fclose_orDie(fin);
     free(buffIn);

examples/streaming_decompression.c

@@ -42,6 +42,15 @@ static size_t fread_orDie(void* buffer, size_t sizeToRead, FILE* file)
     exit(4);
 }
 
+static size_t fwrite_orDie(const void* buffer, size_t sizeToWrite, FILE* file)
+{
+    size_t const writtenSize = fwrite(buffer, 1, sizeToWrite, file);
+    if (writtenSize == sizeToWrite) return sizeToWrite;   /* good */
+    /* error */
+    perror("fwrite");
+    exit(5);
+}
+
 static size_t fclose_orDie(FILE* file)
 {
     if (!fclose(file)) return 0;
@@ -54,28 +63,30 @@ static size_t fclose_orDie(FILE* file)
 static void decompressFile_orDie(const char* fname)
 {
     FILE* const fin  = fopen_orDie(fname, "rb");
-    size_t const buffInSize = ZSTD_DStreamInSize();;
+    size_t const buffInSize = ZSTD_DStreamInSize();
     void*  const buffIn  = malloc_orDie(buffInSize);
-    size_t const buffOutSize = ZSTD_DStreamOutSize();;
+    FILE* const fout = stdout;
+    size_t const buffOutSize = ZSTD_DStreamOutSize();  /* Guarantee to successfully flush at least one complete compressed block in all circumstances. */
     void*  const buffOut = malloc_orDie(buffOutSize);
-    size_t read, toRead = buffInSize;
 
     ZSTD_DStream* const dstream = ZSTD_createDStream();
     if (dstream==NULL) { fprintf(stderr, "ZSTD_createDStream() error \n"); exit(10); }
     size_t const initResult = ZSTD_initDStream(dstream);
-    if (ZSTD_isError(initResult)) { fprintf(stderr, "ZSTD_initDStream() error \n"); exit(11); }
+    if (ZSTD_isError(initResult)) { fprintf(stderr, "ZSTD_initDStream() error : %s \n", ZSTD_getErrorName(initResult)); exit(11); }
 
+    size_t read, toRead = initResult;
     while ( (read = fread_orDie(buffIn, toRead, fin)) ) {
         ZSTD_inBuffer input = { buffIn, read, 0 };
         while (input.pos < input.size) {
             ZSTD_outBuffer output = { buffOut, buffOutSize, 0 };
-            toRead = ZSTD_decompressStream(dstream, &output , &input);
+            toRead = ZSTD_decompressStream(dstream, &output , &input);  /* toRead : size of next compressed block */
-            /* note : data is just "sinked" into buffOut
+            fwrite_orDie(buffOut, output.pos, fout);
-               a more complete example would write it to disk or stdout */
         }
     }
 
+    ZSTD_freeDStream(dstream);
     fclose_orDie(fin);
+    fclose_orDie(fout);
     free(buffIn);
     free(buffOut);
 }
@@ -87,14 +98,12 @@ int main(int argc, const char** argv)
     const char* const inFilename = argv[1];
 
     if (argc!=2) {
-        printf("wrong arguments\n");
+        fprintf(stderr, "wrong arguments\n");
-        printf("usage:\n");
+        fprintf(stderr, "usage:\n");
-        printf("%s FILE\n", exeName);
+        fprintf(stderr, "%s FILE\n", exeName);
         return 1;
     }
 
     decompressFile_orDie(inFilename);
-    printf("%s correctly decoded (in memory). \n", inFilename);
-
     return 0;
 }

lib/compress/zstd_compress.c

@@ -122,13 +122,12 @@ const seqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx)   /* hidden interface *
 }
 
 
-#define CLAMPCHECK(val,min,max) { if ((val<min) | (val>max)) return ERROR(compressionParameter_unsupported); }
-
 /** ZSTD_checkParams() :
     ensure param values remain within authorized range.
     @return : 0, or an error code if one value is beyond authorized range */
 size_t ZSTD_checkCParams(ZSTD_compressionParameters cParams)
 {
+#   define CLAMPCHECK(val,min,max) { if ((val<min) | (val>max)) return ERROR(compressionParameter_unsupported); }
     CLAMPCHECK(cParams.windowLog, ZSTD_WINDOWLOG_MIN, ZSTD_WINDOWLOG_MAX);
     CLAMPCHECK(cParams.chainLog, ZSTD_CHAINLOG_MIN, ZSTD_CHAINLOG_MAX);
     CLAMPCHECK(cParams.hashLog, ZSTD_HASHLOG_MIN, ZSTD_HASHLOG_MAX);

lib/decompress/zstd_decompress.c

@@ -1286,6 +1286,7 @@ struct ZSTD_DStream_s {
     void* legacyContext;
     U32 previousLegacyVersion;
     U32 legacyVersion;
+    U32 hostageByte;
 };   /* typedef'd to ZSTD_DStream within "zstd.h" */
 
 
@@ -1349,6 +1350,7 @@ size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t di
         zds->dictSize = dictSize;
     }
     zds->legacyVersion = 0;
+    zds->hostageByte = 0;
     return ZSTD_frameHeaderSize_prefix;
 }
 
@@ -1371,11 +1373,11 @@ size_t ZSTD_setDStreamParameter(ZSTD_DStream* zds,
 
 size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds)
 {
-    return sizeof(*zds) + ZSTD_sizeof_DCtx(zds->zd) + zds->inBuffSize + zds->outBuffSize;
+    return sizeof(*zds) + ZSTD_sizeof_DCtx(zds->zd) + zds->inBuffSize + zds->outBuffSize + zds->dictSize;
 }
 
 
-/* *** Decompression *** */
+/* *****   Decompression   ***** */
 
 MEM_STATIC size_t ZSTD_limitCopy(void* dst, size_t dstCapacity, const void* src, size_t srcSize)
 {
@@ -1445,7 +1447,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
             zds->fParams.windowSize = MAX(zds->fParams.windowSize, 1U << ZSTD_WINDOWLOG_ABSOLUTEMIN);
             if (zds->fParams.windowSize > zds->maxWindowSize) return ERROR(frameParameter_unsupported);
 
-            /* Frame header instruct buffer sizes */
+            /* Adapt buffer sizes to frame header instructions */
             {   size_t const blockSize = MIN(zds->fParams.windowSize, ZSTD_BLOCKSIZE_ABSOLUTEMAX);
                 size_t const neededOutSize = zds->fParams.windowSize + blockSize;
                 zds->blockSize = blockSize;
@@ -1522,7 +1524,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
                         zds->outStart = zds->outEnd = 0;
                     break;
                 }
-                /* cannot flush everything */
+                /* cannot complete flush */
                 someMoreWork = 0;
                 break;
             }
@@ -1533,8 +1535,21 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
     input->pos += (size_t)(ip-istart);
     output->pos += (size_t)(op-ostart);
     {   size_t nextSrcSizeHint = ZSTD_nextSrcSizeToDecompress(zds->zd);
-        if (!nextSrcSizeHint) return (zds->outEnd != zds->outStart);   /* return 0 only if fully flushed too */
+        if (!nextSrcSizeHint) {   /* frame fully decoded */
-        nextSrcSizeHint += ZSTD_blockHeaderSize * (ZSTD_nextInputType(zds->zd) == ZSTDnit_block);
+            if (zds->outEnd == zds->outStart) {  /* output fully flushed */
+                if (zds->hostageByte) {
+                    if (input->pos >= input->size) { zds->stage = zdss_read; return 1; }  /* can't release hostage (not present) */
+                    input->pos++;  /* release hostage */
+                }
+                return 0;
+            }
+            if (!zds->hostageByte) { /* output not fully flushed; keep last byte as hostage; will be released when all output is flushed */
+                input->pos--;   /* note : pos > 0, otherwise, impossible to finish reading last block */
+                zds->hostageByte=1;
+            }
+            return 1;
+        }
+        nextSrcSizeHint += ZSTD_blockHeaderSize * (ZSTD_nextInputType(zds->zd) == ZSTDnit_block);   /* preload header of next block */
         if (zds->inPos > nextSrcSizeHint) return ERROR(GENERIC);   /* should never happen */
         nextSrcSizeHint -= zds->inPos;   /* already loaded*/
         return nextSrcSizeHint;

lib/zstd.h

@@ -229,7 +229,7 @@ ZSTDLIB_API ZSTD_CStream* ZSTD_createCStream(void);
 ZSTDLIB_API size_t ZSTD_freeCStream(ZSTD_CStream* zcs);
 
 ZSTDLIB_API size_t ZSTD_CStreamInSize(void);    /**< recommended size for input buffer */
-ZSTDLIB_API size_t ZSTD_CStreamOutSize(void);   /**< recommended size for output buffer */
+ZSTDLIB_API size_t ZSTD_CStreamOutSize(void);   /**< recommended size for output buffer. Guarantee to successfully flush at least one complete compressed block in all circumstances. */
 
 ZSTDLIB_API size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel);
 ZSTDLIB_API size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuffer* input);
@@ -252,15 +252,13 @@ ZSTDLIB_API size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output);
 *
 *  Use ZSTD_decompressStream() repetitively to consume your input.
 *  The function will update both `pos` fields.
-*  If `input.pos < input.size`, some input is not consumed.
+*  If `input.pos < input.size`, some input has not been consumed.
 *  It's up to the caller to present again remaining data.
-*  If `output.pos == output.size`, there is probably some more data to flush, still stored inside internal buffers.
+*  If `output.pos < output.size`, decoder has flushed everything it could.
 *  @return : 0 when a frame is completely decoded and fully flushed,
 *            an error code, which can be tested using ZSTD_isError(),
-*            any value > 0, which means there is still some work to do to complete the frame.
+*            any other value > 0, which means there is still some work to do to complete the frame.
-*            In general, the return value is a suggested next input size (merely a hint, to help latency).
+*            The return value is a suggested next input size (just an hint, to help latency).
-*            1 is a special value, which means either "there is still some data to flush", or "need 1 more byte as input".
-*            In which case, start by flushing. When flush is completed, if return value is still `1`, it means "need 1 more byte".
 * *******************************************************************************/
 
 typedef struct ZSTD_DStream_s ZSTD_DStream;
@@ -268,7 +266,7 @@ ZSTDLIB_API ZSTD_DStream* ZSTD_createDStream(void);
 ZSTDLIB_API size_t ZSTD_freeDStream(ZSTD_DStream* zds);
 
 ZSTDLIB_API size_t ZSTD_DStreamInSize(void);    /*!< recommended size for input buffer */
-ZSTDLIB_API size_t ZSTD_DStreamOutSize(void);   /*!< recommended size for output buffer */
+ZSTDLIB_API size_t ZSTD_DStreamOutSize(void);   /*!< recommended size for output buffer. Guarantee to successfully flush at least one complete block in all circumstances. */
 
 ZSTDLIB_API size_t ZSTD_initDStream(ZSTD_DStream* zds);
 ZSTDLIB_API size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inBuffer* input);

programs/fileio.c

@@ -20,7 +20,7 @@
  ***************************************/
 #ifndef ZSTD_LEGACY_SUPPORT
 /* LEGACY_SUPPORT :
- *  decompressor can decode older formats (starting from Zstd 0.1+) */
+ *  decompressor can decode older formats (starting from zstd 0.1+) */
 #  define ZSTD_LEGACY_SUPPORT 1
 #endif
 
@@ -613,22 +613,22 @@ unsigned long long FIO_decompressFrame(dRess_t ress,
     while (1) {
         ZSTD_inBuffer  inBuff = { ress.srcBuffer, readSize, 0 };
         ZSTD_outBuffer outBuff= { ress.dstBuffer, ress.dstBufferSize, 0 };
-        size_t const toRead = ZSTD_decompressStream(ress.dctx, &outBuff, &inBuff );
+        size_t const readSizeHint = ZSTD_decompressStream(ress.dctx, &outBuff, &inBuff );
-        if (ZSTD_isError(toRead)) EXM_THROW(36, "Decoding error : %s", ZSTD_getErrorName(toRead));
+        if (ZSTD_isError(readSizeHint)) EXM_THROW(36, "Decoding error : %s", ZSTD_getErrorName(readSizeHint));
 
         /* Write block */
         storedSkips = FIO_fwriteSparse(foutput, ress.dstBuffer, outBuff.pos, storedSkips);
         frameSize += outBuff.pos;
         DISPLAYUPDATE(2, "\rDecoded : %u MB...     ", (U32)(frameSize>>20) );
 
-        if (toRead == 0) break;   /* end of frame */
+        if (readSizeHint == 0) break;   /* end of frame */
         if (inBuff.size != inBuff.pos) EXM_THROW(37, "Decoding error : should consume entire input");
 
         /* Fill input buffer */
-        if (toRead > ress.srcBufferSize) EXM_THROW(38, "too large block");
+        {   size_t const toRead = MIN(readSizeHint, ress.srcBufferSize);  /* support large skippable frames */
             readSize = fread(ress.srcBuffer, 1, toRead, finput);
-        if (readSize == 0) EXM_THROW(39, "Read error : premature end");
+            if (readSize < toRead) EXM_THROW(39, "Read error : premature end");
-    }
+    }   }
 
     FIO_fwriteSparseEnd(foutput, storedSkips);
 
@@ -686,7 +686,7 @@ static int FIO_decompressSrcFile(dRess_t ress, const char* srcFileName)
             if (readSomething==0) { DISPLAY("zstd: %s: unexpected end of file \n", srcFileName); fclose(srcFile); return 1; }  /* srcFileName is empty */
             break;   /* no more input */
         }
-        readSomething = 1;
+        readSomething = 1;   /* there is at least >= 4 bytes in srcFile */
         if (sizeCheck != toRead) { DISPLAY("zstd: %s: unknown header \n", srcFileName); fclose(srcFile); return 1; }  /* srcFileName is empty */
         {   U32 const magic = MEM_readLE32(ress.srcBuffer);
             if (((magic & 0xFFFFFFF0U) != ZSTD_MAGIC_SKIPPABLE_START) & (magic != ZSTD_MAGICNUMBER)

tests/playTests.sh

@@ -45,7 +45,9 @@ file $ZSTD
 $ECHO "\n**** simple tests **** "
 
 ./datagen > tmp
+$ECHO "test : basic compression "
 $ZSTD -f tmp                      # trivial compression case, creates tmp.zst
+$ECHO "test : basic decompression"
 $ZSTD -df tmp.zst                 # trivial decompression case (overwrites tmp)
 $ECHO "test : too large compression level (must fail)"
 $ZSTD -99 -f tmp  # too large compression level, automatic sized down

tests/zstreamtest.c

@@ -58,7 +58,7 @@ static U32 g_displayLevel = 2;
             if ((FUZ_GetClockSpan(g_displayClock) > g_refreshRate) || (g_displayLevel>=4)) \
             { g_displayClock = clock(); DISPLAY(__VA_ARGS__); \
             if (g_displayLevel>=4) fflush(stdout); } }
-static const clock_t g_refreshRate = CLOCKS_PER_SEC * 15 / 100;
+static const clock_t g_refreshRate = CLOCKS_PER_SEC / 6;
 static clock_t g_displayClock = 0;
 
 static clock_t g_clockTime = 0;
@@ -118,8 +118,7 @@ static void freeFunction(void* opaque, void* address)
 
 static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem customMem)
 {
-    int testResult = 0;
+    size_t const CNBufferSize = COMPRESSIBLE_NOISE_LENGTH;
-    size_t CNBufferSize = COMPRESSIBLE_NOISE_LENGTH;
     void* CNBuffer = malloc(CNBufferSize);
     size_t const skippableFrameSize = 11;
     size_t const compressedBufferSize = (8 + skippableFrameSize) + ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH);
@@ -127,6 +126,7 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
     size_t const decodedBufferSize = CNBufferSize;
     void* decodedBuffer = malloc(decodedBufferSize);
     size_t cSize;
+    int testResult = 0;
     U32 testNb=0;
     ZSTD_CStream* zc = ZSTD_createCStream_advanced(customMem);
     ZSTD_DStream* zd = ZSTD_createDStream_advanced(customMem);
@@ -437,7 +437,7 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compres
         {   U32 const testLog = FUZ_rand(&lseed) % maxSrcLog;
             U32 const cLevel = (FUZ_rand(&lseed) % (ZSTD_maxCLevel() - (testLog/3))) + 1;
             maxTestSize = FUZ_rLogLength(&lseed, testLog);
-            dictSize  = (FUZ_rand(&lseed)==1) ? FUZ_randomLength(&lseed, maxSampleLog) : 0;
+            dictSize  = ((FUZ_rand(&lseed)&63)==1) ? FUZ_randomLength(&lseed, maxSampleLog) : 0;
             /* random dictionary selection */
             {   size_t const dictStart = FUZ_rand(&lseed) % (srcBufferSize - dictSize);
                 dict = srcBuffer + dictStart;
@@ -446,7 +446,7 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compres
                 params.fParams.checksumFlag = FUZ_rand(&lseed) & 1;
                 params.fParams.noDictIDFlag = FUZ_rand(&lseed) & 1;
                 {   size_t const initError = ZSTD_initCStream_advanced(zc, dict, dictSize, params, 0);
-                    CHECK (ZSTD_isError(initError),"init error : %s", ZSTD_getErrorName(initError));
+                    CHECK (ZSTD_isError(initError),"ZSTD_initCStream_advanced error : %s", ZSTD_getErrorName(initError));
         }   }   }
 
         /* multi-segments compression test */

zstd_compression_format.md

@@ -301,6 +301,7 @@ This is a variable size field, which contains
 the ID of the dictionary required to properly decode the frame.
 Note that this field is optional. When it's not present,
 it's up to the caller to make sure it uses the correct dictionary.
+Format is little-endian.
 
 Field size depends on `Dictionary_ID_flag`.
 1 byte can represent an ID 0-255.
@@ -731,7 +732,7 @@ This size is deducted from `blockSize - literalSectionSize`.
 
 #### `Sequences_Section_Header`
 
-Consists in 2 items :
+Consists of 2 items:
 - `Number_of_Sequences`
 - Symbol compression modes
 
@@ -872,7 +873,7 @@ and can be translated into an `Offset_Value` using the following formulas :
 Offset_Value = (1 << offsetCode) + readNBits(offsetCode);
 if (Offset_Value > 3) offset = Offset_Value - 3;
 ```
-It means that maximum `Offset_Value` is `2^(N+1))-1` and it supports back-reference distance up to `2^(N+1))-4`
+It means that maximum `Offset_Value` is `(2^(N+1))-1` and it supports back-reference distance up to `(2^(N+1))-4`
 but is limited by [maximum back-reference distance](#window_descriptor).
 
 `Offset_Value` from 1 to 3 are special : they define "repeat codes",
@@ -893,7 +894,7 @@ If any sequence in the compressed block requires an offset larger than this,
 it's not possible to use the default distribution to represent it.
 
 ```
-short offsetCodes_defaultDistribution[53] =
+short offsetCodes_defaultDistribution[29] =
         { 1, 1, 1, 1, 1, 1, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1,
           1, 1, 1, 1, 1, 1, 1, 1,-1,-1,-1,-1,-1 };
 ```