Merge pull request #47 from Cyan4973/dev

Dev

.travis.yml

@@ -1,6 +1,6 @@
 language: c
 compiler: gcc
-script: make test-travis
+script: make $ZSTD_TRAVIS_CI_ENV
 before_install:
   - sudo apt-get update  -qq
   - sudo apt-get install -qq gcc-arm-linux-gnueabi
@@ -11,11 +11,13 @@ before_install:
 
 env:
   - ZSTD_TRAVIS_CI_ENV=travis-install
-  - ZSTD_TRAVIS_CI_ENV=test-all  
   - ZSTD_TRAVIS_CI_ENV=clangtest  
   - ZSTD_TRAVIS_CI_ENV=gpptest  
   - ZSTD_TRAVIS_CI_ENV=armtest  
+  - ZSTD_TRAVIS_CI_ENV=test  
+  - ZSTD_TRAVIS_CI_ENV="-C programs test32"  
   - ZSTD_TRAVIS_CI_ENV=sanitize
+  - ZSTD_TRAVIS_CI_ENV="-C programs memtest"  
 
 matrix:
   fast_finish: true

Makefile

@@ -32,40 +32,25 @@
 # ################################################################
 
 # Version number
-export VERSION=0.1.0
-export RELEASE=r$(VERSION)
+export VERSION := 0.1.2
 
-DESTDIR?=
-PREFIX ?= /usr
-
-LIBDIR ?= $(PREFIX)/lib
-INCLUDEDIR=$(PREFIX)/include
 PRGDIR  = programs
 ZSTDDIR = lib
 
-# Select test target for Travis CI's Build Matrix
-ifneq (,$(filter test-%,$(ZSTD_TRAVIS_CI_ENV)))
-TRAVIS_TARGET=prg-travis
-else
-TRAVIS_TARGET=$(ZSTD_TRAVIS_CI_ENV)
-endif
-
-
 .PHONY: clean
 
-default: zstdprograms
+default: zstdprogram
 
 all: 
-	@cd $(ZSTDDIR); $(MAKE) -e all
-	@cd $(PRGDIR); $(MAKE) -e all
+	$(MAKE) -C $(ZSTDDIR) $@
+	$(MAKE) -C $(PRGDIR) $@
 
-zstdprograms:
-	@cd $(PRGDIR); $(MAKE) -e
+zstdprogram:
+	$(MAKE) -C $(PRGDIR)
 
 clean:
-	@cd $(PRGDIR); $(MAKE) clean
-	@cd $(ZSTDDIR); $(MAKE) clean
-#	@cd examples; $(MAKE) clean
+	$(MAKE) -C $(ZSTDDIR) $@
+	$(MAKE) -C $(PRGDIR) $@
 	@echo Cleaning completed
 
 
@@ -74,23 +59,18 @@ clean:
 ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU))
 
 install:
-	@cd $(ZSTDDIR); $(MAKE) -e install
-	@cd $(PRGDIR); $(MAKE) -e install
+	$(MAKE) -C $(ZSTDDIR) $@
+	$(MAKE) -C $(PRGDIR) $@
 
 uninstall:
-	@cd $(ZSTDDIR); $(MAKE) uninstall
-	@cd $(PRGDIR); $(MAKE) uninstall
+	$(MAKE) -C $(ZSTDDIR) $@
+	$(MAKE) -C $(PRGDIR) $@
 
 travis-install:
 	sudo $(MAKE) install
 
 test:
-	@cd $(PRGDIR); $(MAKE) -e test
-
-test-travis: $(TRAVIS_TARGET)
-
-prg-travis:
-	@cd $(PRGDIR); $(MAKE) -e $(ZSTD_TRAVIS_CI_ENV)
+	$(MAKE) -C $(PRGDIR) $@
 
 clangtest: clean
 	clang -v
@@ -100,8 +80,8 @@ gpptest: clean
 	$(MAKE) all CC=g++ CFLAGS="-O3 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Werror"
 
 armtest: clean
-	cd $(ZSTDDIR); $(MAKE) -e all CC=arm-linux-gnueabi-gcc MOREFLAGS="-Werror"
-	cd $(PRGDIR); $(MAKE) -e CC=arm-linux-gnueabi-gcc MOREFLAGS="-Werror"
+	$(MAKE) -C $(ZSTDDIR) -e all CC=arm-linux-gnueabi-gcc MOREFLAGS="-Werror"
+	$(MAKE) -C $(PRGDIR) -e CC=arm-linux-gnueabi-gcc MOREFLAGS="-Werror"
 
 sanitize: clean
 	$(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=undefined"

NEWS

@@ -1,3 +1,9 @@
-r0
-initial release
+v0.1.2
+
+v0.1.1
+fix compression bug
+detects write-flush errors
+
+v0.1.0
+first release
 

lib/Makefile

@@ -32,17 +32,16 @@
 # ################################################################
 
 # Version numbers
-VERSION?= 0
+VERSION?= 0.1.2
 LIBVER_MAJOR=`sed -n '/define ZSTD_VERSION_MAJOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < zstd.h`
 LIBVER_MINOR=`sed -n '/define ZSTD_VERSION_MINOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < zstd.h`
 LIBVER_PATCH=`sed -n '/define ZSTD_VERSION_RELEASE/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < zstd.h`
 LIBVER  = $(LIBVER_MAJOR).$(LIBVER_MINOR).$(LIBVER_PATCH)
 
 DESTDIR?=
-PREFIX ?= /usr
+PREFIX ?= /usr/local
 CFLAGS ?= -O3
-CFLAGS += -std=c99 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Wstrict-prototypes
-LDFLAGS = -I.
+CFLAGS += -I. -std=c99 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Wstrict-prototypes
 FLAGS   = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) $(MOREFLAGS)
 
 LIBDIR ?= $(PREFIX)/lib

lib/fse.c

@@ -135,7 +135,7 @@ typedef   signed long long  S64;
  * Method 1 : `__packed` statement. It depends on compiler extension (ie, not portable).
  *            This method is safe if your compiler supports it, and *generally* as fast or faster than `memcpy`.
  * Method 2 : direct access. This method is portable but violate C standard.
- *            It can generate buggy code on targets which generate assembly depending on alignment.
+ *            It can generate buggy code on targets generating assembly depending on alignment.
  *            But in some circumstances, it's the only known way to get the most performance (ie GCC + ARMv6)
  * See http://fastcompression.blogspot.fr/2015/08/accessing-unaligned-memory.html for details.
  * Prefer these methods in priority order (0 > 1 > 2)
@@ -914,8 +914,8 @@ size_t FSE_readNCount (short* normalizedCounter, unsigned* maxSVPtr, unsigned* t
                 else
                 {
                     bitCount -= (int)(8 * (iend - 4 - ip));
-					ip = iend - 4;
-				}
+                    ip = iend - 4;
+                }
                 bitStream = FSE_readLE32(ip) >> (bitCount & 31);
             }
         }
@@ -967,20 +967,20 @@ void  FSE_freeCTable (FSE_CTable* ct)
 /* provides the minimum logSize to safely represent a distribution */
 static unsigned FSE_minTableLog(size_t srcSize, unsigned maxSymbolValue)
 {
-	U32 minBitsSrc = FSE_highbit32((U32)(srcSize - 1)) + 1;
-	U32 minBitsSymbols = FSE_highbit32(maxSymbolValue) + 2;
-	U32 minBits = minBitsSrc < minBitsSymbols ? minBitsSrc : minBitsSymbols;
-	return minBits;
+    U32 minBitsSrc = FSE_highbit32((U32)(srcSize - 1)) + 1;
+    U32 minBitsSymbols = FSE_highbit32(maxSymbolValue) + 2;
+    U32 minBits = minBitsSrc < minBitsSymbols ? minBitsSrc : minBitsSymbols;
+    return minBits;
 }
 
 unsigned FSE_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue)
 {
-	U32 maxBitsSrc = FSE_highbit32((U32)(srcSize - 1)) - 2;
+    U32 maxBitsSrc = FSE_highbit32((U32)(srcSize - 1)) - 2;
     U32 tableLog = maxTableLog;
-	U32 minBits = FSE_minTableLog(srcSize, maxSymbolValue);
+    U32 minBits = FSE_minTableLog(srcSize, maxSymbolValue);
     if (tableLog==0) tableLog = FSE_DEFAULT_TABLELOG;
-	if (maxBitsSrc < tableLog) tableLog = maxBitsSrc;   /* Accuracy can be reduced */
-	if (minBits > tableLog) tableLog = minBits;   /* Need a minimum to safely represent all symbol values */
+    if (maxBitsSrc < tableLog) tableLog = maxBitsSrc;   /* Accuracy can be reduced */
+    if (minBits > tableLog) tableLog = minBits;   /* Need a minimum to safely represent all symbol values */
     if (tableLog < FSE_MIN_TABLELOG) tableLog = FSE_MIN_TABLELOG;
     if (tableLog > FSE_MAX_TABLELOG) tableLog = FSE_MAX_TABLELOG;
     return tableLog;
@@ -1398,6 +1398,7 @@ size_t FSE_compress2 (void* dst, size_t dstSize, const void* src, size_t srcSize
     errorCode = FSE_count (count, &maxSymbolValue, ip, srcSize);
     if (FSE_isError(errorCode)) return errorCode;
     if (errorCode == srcSize) return 1;
+    if (errorCode == 1) return 0;   /* each symbol only present once */
     if (errorCode < (srcSize >> 7)) return 0;   /* Heuristic : not compressible enough */
 
     tableLog = FSE_optimalTableLog(tableLog, srcSize, maxSymbolValue);
@@ -1529,12 +1530,14 @@ size_t FSE_initDStream(FSE_DStream_t* bitD, const void* srcBuffer, size_t srcSiz
  */
 static size_t FSE_lookBits(FSE_DStream_t* bitD, U32 nbBits)
 {
-    return ((bitD->bitContainer << (bitD->bitsConsumed & ((sizeof(bitD->bitContainer)*8)-1))) >> 1) >> (((sizeof(bitD->bitContainer)*8)-1)-nbBits);
+    const U32 bitMask = sizeof(bitD->bitContainer)*8 - 1;
+    return ((bitD->bitContainer << (bitD->bitsConsumed & bitMask)) >> 1) >> ((bitMask-nbBits) & bitMask);
 }
 
 static size_t FSE_lookBitsFast(FSE_DStream_t* bitD, U32 nbBits)   /* only if nbBits >= 1 !! */
 {
-    return (bitD->bitContainer << bitD->bitsConsumed) >> ((sizeof(bitD->bitContainer)*8)-nbBits);
+    const U32 bitMask = sizeof(bitD->bitContainer)*8 - 1;
+    return (bitD->bitContainer << (bitD->bitsConsumed & bitMask)) >> (((bitMask+1)-nbBits) & bitMask);
 }
 
 static void FSE_skipBits(FSE_DStream_t* bitD, U32 nbBits)
@@ -1566,8 +1569,8 @@ size_t FSE_readBitsFast(FSE_DStream_t* bitD, U32 nbBits)   /* only if nbBits >=
 
 unsigned FSE_reloadDStream(FSE_DStream_t* bitD)
 {
-	if (bitD->bitsConsumed > (sizeof(bitD->bitContainer)*8))  /* should never happen */
-		return FSE_DStream_tooFar;
+    if (bitD->bitsConsumed > (sizeof(bitD->bitContainer)*8))  /* should never happen */
+        return FSE_DStream_tooFar;
 
     if (bitD->ptr >= bitD->start + sizeof(bitD->bitContainer))
     {
@@ -1831,7 +1834,7 @@ size_t HUF_writeCTable (void* dst, size_t maxDstSize, const HUF_CElt* tree, U32
         if (maxSymbolValue > (241-128)) return (size_t)-FSE_ERROR_GENERIC;   /* not implemented (not possible with current format) */
         if (((maxSymbolValue+1)/2) + 1 > maxDstSize) return (size_t)-FSE_ERROR_dstSize_tooSmall;   /* not enough space within dst buffer */
         op[0] = (BYTE)(128 /*special case*/ + 0 /* Not Compressible */ + (maxSymbolValue-1));
-		huffWeight[maxSymbolValue] = 0;   /* to be sure it doesn't cause issue in final combination */
+        huffWeight[maxSymbolValue] = 0;   /* to be sure it doesn't cause issue in final combination */
         for (n=0; n<maxSymbolValue; n+=2)
             op[(n/2)+1] = (BYTE)((huffWeight[n] << 4) + huffWeight[n+1]);
         return ((maxSymbolValue+1)/2) + 1;
@@ -1875,7 +1878,7 @@ static U32 HUF_setMaxHeight(nodeElt* huffNode, U32 lastNonNull, U32 maxNbBits)
             U32 rankLast[HUF_MAX_TABLELOG];
             U32 currentNbBits = maxNbBits;
             int pos;
-			memset(rankLast, 0xF0, sizeof(rankLast));
+            memset(rankLast, 0xF0, sizeof(rankLast));
             for (pos=n ; pos >= 0; pos--)
             {
                 if (huffNode[pos].nbBits >= currentNbBits) continue;
@@ -1904,24 +1907,30 @@ static U32 HUF_setMaxHeight(nodeElt* huffNode, U32 lastNonNull, U32 maxNbBits)
                 if (rankLast[nBitsToDecrease-1] == noOne)
                     rankLast[nBitsToDecrease-1] = rankLast[nBitsToDecrease];   // now there is one elt
                 huffNode[rankLast[nBitsToDecrease]].nbBits ++;
-                rankLast[nBitsToDecrease]--;
-				if (huffNode[rankLast[nBitsToDecrease]].nbBits != maxNbBits-nBitsToDecrease)
-                    rankLast[nBitsToDecrease] = noOne;   // rank list emptied
+                if (rankLast[nBitsToDecrease] == 0)
+                    rankLast[nBitsToDecrease] = noOne;
+                else
+                {
+                    rankLast[nBitsToDecrease]--;
+                    if (huffNode[rankLast[nBitsToDecrease]].nbBits != maxNbBits-nBitsToDecrease)
+                        rankLast[nBitsToDecrease] = noOne;   // rank list emptied
+                }
+            }
+
+            while (totalCost < 0)   /* Sometimes, cost correction overshoot */
+            {
+                if (rankLast[1] == noOne)   /* special case, no weight 1, let's find it back at n */
+                {
+                    while (huffNode[n].nbBits == maxNbBits) n--;
+                    huffNode[n+1].nbBits--;
+                    rankLast[1] = n+1;
+                    totalCost++;
+                    continue;
+                }
+                huffNode[ rankLast[1] + 1 ].nbBits--;
+                rankLast[1]++;
+                totalCost ++;
             }
-			while (totalCost < 0)   // Sometimes, cost correction overshoot
-			{
-				if (rankLast[1] == noOne)   /* special case, no weight 1, let's find it back at n */
-				{
-					while (huffNode[n].nbBits == maxNbBits) n--;
-					huffNode[n+1].nbBits--;
-					rankLast[1] = n+1;
-					totalCost++;
-					continue;
-				}
-				huffNode[ rankLast[1] + 1 ].nbBits--;
-				rankLast[1]++;
-				totalCost ++;
-			}
         }
     }
 
@@ -1972,7 +1981,7 @@ size_t HUF_buildCTable (HUF_CElt* tree, const U32* count, U32 maxSymbolValue, U3
     /* safety checks */
     if (maxNbBits == 0) maxNbBits = HUF_DEFAULT_TABLELOG;
     if (maxSymbolValue > HUF_MAX_SYMBOL_VALUE) return (size_t)-FSE_ERROR_GENERIC;
-	memset(huffNode0, 0, sizeof(huffNode0));
+    memset(huffNode0, 0, sizeof(huffNode0));
 
     // sort, decreasing order
     HUF_sort(huffNode, count, maxSymbolValue);
@@ -2057,7 +2066,7 @@ size_t HUF_compress_usingCTable(void* dst, size_t dstSize, const void* src, size
     FSE_CStream_t bitC;
 
     /* init */
-	if (dstSize < 8) return 0;
+    if (dstSize < 8) return 0;
     op += 6;   /* jump Table -- could be optimized by delta / deviation */
     errorCode = FSE_initCStream(&bitC, op, oend-op);
     if (FSE_isError(errorCode)) return 0;

lib/zstd.c

@@ -314,6 +314,7 @@ typedef struct ZSTD_Cctx_s
 #else
     U32 hashTable[HASH_TABLESIZE];
 #endif
+	BYTE buffer[WORKPLACESIZE];
 } cctxi_t;
 
 
@@ -321,12 +322,7 @@ ZSTD_Cctx* ZSTD_createCCtx(void)
 {
     ZSTD_Cctx* ctx = (ZSTD_Cctx*) malloc( sizeof(ZSTD_Cctx) );
     if (ctx==NULL) return NULL;
-    ctx->seqStore.buffer = malloc(WORKPLACESIZE);
-    if (ctx->seqStore.buffer==NULL)
-    {
-        free(ctx);
-        return NULL;
-    }
+	ctx->seqStore.buffer = ctx->buffer;
     ctx->seqStore.offsetStart = (U32*) (ctx->seqStore.buffer);
     ctx->seqStore.offCodeStart = (BYTE*) (ctx->seqStore.offsetStart + (BLOCKSIZE>>2));
     ctx->seqStore.litStart = ctx->seqStore.offCodeStart + (BLOCKSIZE>>2);
@@ -344,7 +340,6 @@ void ZSTD_resetCCtx(ZSTD_Cctx* ctx)
 
 size_t ZSTD_freeCCtx(ZSTD_Cctx* ctx)
 {
-    free(ctx->seqStore.buffer);
     free(ctx);
     return 0;
 }
@@ -542,11 +537,11 @@ static size_t ZSTD_compressLiterals (void* dst, size_t dstSize,
     const size_t minGain = ZSTD_minGain(srcSize);
     BYTE* const ostart = (BYTE*)dst;
     size_t hsize;
-	static const size_t LHSIZE = 5;
+    static const size_t LHSIZE = 5;
 
-	if (dstSize < LHSIZE+1) return (size_t)-ZSTD_ERROR_maxDstSize_tooSmall;   /* not enough space for compression */
+    if (dstSize < LHSIZE+1) return (size_t)-ZSTD_ERROR_maxDstSize_tooSmall;   /* not enough space for compression */
 
-	hsize = HUF_compress(ostart+LHSIZE, dstSize-LHSIZE, src, srcSize);
+    hsize = HUF_compress(ostart+LHSIZE, dstSize-LHSIZE, src, srcSize);
     if (hsize<2) return hsize;   /* special cases */
     if (hsize >= srcSize - minGain) return 0;
 
@@ -619,8 +614,8 @@ static size_t ZSTD_compressSequences(BYTE* dst, size_t maxDstSize,
     }
 
     /* Sequences Header */
-	if ((oend-op) < 2+3+6)  /* nbSeq + dumpsLength + 3*rleCTable*/
-		return (size_t)-ZSTD_ERROR_maxDstSize_tooSmall;
+    if ((oend-op) < 2+3+6)  /* nbSeq + dumpsLength + 3*rleCTable*/
+        return (size_t)-ZSTD_ERROR_maxDstSize_tooSmall;
     ZSTD_writeLE16(op, (U16)nbSeq); op+=2;
     seqHead = op;
 
@@ -640,7 +635,7 @@ static size_t ZSTD_compressSequences(BYTE* dst, size_t maxDstSize,
             op[2] = (BYTE)(dumpsLength);
             op += 3;
         }
-		if ((size_t)(oend-op) < dumpsLength+6) return (size_t)-ZSTD_ERROR_maxDstSize_tooSmall;
+        if ((size_t)(oend-op) < dumpsLength+6) return (size_t)-ZSTD_ERROR_maxDstSize_tooSmall;
         memcpy(op, seqStorePtr->dumpsStart, dumpsLength);
         op += dumpsLength;
     }
@@ -661,11 +656,11 @@ static size_t ZSTD_compressSequences(BYTE* dst, size_t maxDstSize,
     }
     else
     {
-		size_t NCountSize;
+        size_t NCountSize;
         tableLog = FSE_optimalTableLog(LLFSELog, nbSeq, max);
         FSE_normalizeCount(norm, tableLog, count, nbSeq, max);
-		NCountSize = FSE_writeNCount(op, oend-op, norm, max, tableLog);   /* overflow protected */
-		if (FSE_isError(NCountSize)) return (size_t)-ZSTD_ERROR_GENERIC;
+        NCountSize = FSE_writeNCount(op, oend-op, norm, max, tableLog);   /* overflow protected */
+        if (FSE_isError(NCountSize)) return (size_t)-ZSTD_ERROR_GENERIC;
         op += NCountSize;
         FSE_buildCTable(CTable_LitLength, norm, max, tableLog);
         LLtype = bt_compressed;
@@ -696,11 +691,11 @@ static size_t ZSTD_compressSequences(BYTE* dst, size_t maxDstSize,
     }
     else
     {
-		size_t NCountSize;
+        size_t NCountSize;
         tableLog = FSE_optimalTableLog(OffFSELog, nbSeq, max);
         FSE_normalizeCount(norm, tableLog, count, nbSeq, max);
-		NCountSize = FSE_writeNCount(op, oend-op, norm, max, tableLog);   /* overflow protected */
-		if (FSE_isError(NCountSize)) return (size_t)-ZSTD_ERROR_GENERIC;
+        NCountSize = FSE_writeNCount(op, oend-op, norm, max, tableLog);   /* overflow protected */
+        if (FSE_isError(NCountSize)) return (size_t)-ZSTD_ERROR_GENERIC;
         op += NCountSize;
         FSE_buildCTable(CTable_OffsetBits, norm, max, tableLog);
         Offtype = bt_compressed;
@@ -722,11 +717,11 @@ static size_t ZSTD_compressSequences(BYTE* dst, size_t maxDstSize,
     }
     else
     {
-		size_t NCountSize;
+        size_t NCountSize;
         tableLog = FSE_optimalTableLog(MLFSELog, nbSeq, max);
         FSE_normalizeCount(norm, tableLog, count, nbSeq, max);
-		NCountSize = FSE_writeNCount(op, oend-op, norm, max, tableLog);   /* overflow protected */
-		if (FSE_isError(NCountSize)) return (size_t)-ZSTD_ERROR_GENERIC;
+        NCountSize = FSE_writeNCount(op, oend-op, norm, max, tableLog);   /* overflow protected */
+        if (FSE_isError(NCountSize)) return (size_t)-ZSTD_ERROR_GENERIC;
         op += NCountSize;
         FSE_buildCTable(CTable_MatchLength, norm, max, tableLog);
         MLtype = bt_compressed;
@@ -1028,7 +1023,7 @@ size_t ZSTD_compressContinue(ZSTD_Cctx*  cctx, void* dst, size_t maxDstSize, con
         size_t blockSize = BLOCKSIZE;
         if (blockSize > srcSize) blockSize = srcSize;
 
-		if (maxDstSize < 2*ZSTD_blockHeaderSize+1)  /* one RLE block + endMark */
+        if (maxDstSize < 2*ZSTD_blockHeaderSize+1)  /* one RLE block + endMark */
             return (size_t)-ZSTD_ERROR_maxDstSize_tooSmall;
 
         /* update hash table */
@@ -1239,7 +1234,7 @@ size_t ZSTD_decodeLiteralsBlock(void* ctx,
 }
 
 
-size_t ZSTD_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr,
+size_t ZSTD_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* dumpsLengthPtr,
                          FSE_DTable* DTableLL, FSE_DTable* DTableML, FSE_DTable* DTableOffb,
                          const void* src, size_t srcSize)
 {
@@ -1250,8 +1245,8 @@ size_t ZSTD_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr,
     U32 LLlog, Offlog, MLlog;
     size_t dumpsLength;
 
-	/* check */
-	if (srcSize < 5) return (size_t)-ZSTD_ERROR_SrcSize;
+    /* check */
+    if (srcSize < 5) return (size_t)-ZSTD_ERROR_SrcSize;
 
     /* SeqHead */
     *nbSeq = ZSTD_readLE16(ip); ip+=2;
@@ -1272,9 +1267,10 @@ size_t ZSTD_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr,
     }
     *dumpsPtr = ip;
     ip += dumpsLength;
+    *dumpsLengthPtr = dumpsLength;
 
-	/* check */
-	if (ip > iend-3) return (size_t)-ZSTD_ERROR_SrcSize; /* min : all 3 are "raw", hence no header, but at least xxLog bits per type */
+    /* check */
+    if (ip > iend-3) return (size_t)-ZSTD_ERROR_SrcSize; /* min : all 3 are "raw", hence no header, but at least xxLog bits per type */
 
     /* sequences */
     {
@@ -1295,7 +1291,7 @@ size_t ZSTD_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr,
             max = MaxLL;
             headerSize = FSE_readNCount(norm, &max, &LLlog, ip, iend-ip);
             if (FSE_isError(headerSize)) return (size_t)-ZSTD_ERROR_GENERIC;
-			if (LLlog > LLFSELog) return (size_t)-ZSTD_ERROR_corruption;
+            if (LLlog > LLFSELog) return (size_t)-ZSTD_ERROR_corruption;
             ip += headerSize;
             FSE_buildDTable(DTableLL, norm, max, LLlog);
         }
@@ -1314,7 +1310,7 @@ size_t ZSTD_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr,
             max = MaxOff;
             headerSize = FSE_readNCount(norm, &max, &Offlog, ip, iend-ip);
             if (FSE_isError(headerSize)) return (size_t)-ZSTD_ERROR_GENERIC;
-			if (Offlog > OffFSELog) return (size_t)-ZSTD_ERROR_corruption;
+            if (Offlog > OffFSELog) return (size_t)-ZSTD_ERROR_corruption;
             ip += headerSize;
             FSE_buildDTable(DTableOffb, norm, max, Offlog);
         }
@@ -1333,7 +1329,7 @@ size_t ZSTD_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr,
             max = MaxML;
             headerSize = FSE_readNCount(norm, &max, &MLlog, ip, iend-ip);
             if (FSE_isError(headerSize)) return (size_t)-ZSTD_ERROR_GENERIC;
-			if (MLlog > MLFSELog) return (size_t)-ZSTD_ERROR_corruption;
+            if (MLlog > MLFSELog) return (size_t)-ZSTD_ERROR_corruption;
             ip += headerSize;
             FSE_buildDTable(DTableML, norm, max, MLlog);
         }
@@ -1356,6 +1352,7 @@ typedef struct {
     FSE_DState_t stateML;
     size_t prevOffset;
     const BYTE* dumps;
+    const BYTE* dumpsEnd;
 } seqState_t;
 
 
@@ -1366,6 +1363,7 @@ static void ZSTD_decodeSequence(seq_t* seq, seqState_t* seqState)
     size_t offset;
     size_t matchLength;
     const BYTE* dumps = seqState->dumps;
+    const BYTE* const de = seqState->dumpsEnd;
 
     /* Literal length */
     litLength = FSE_decodeSymbol(&(seqState->stateLL), &(seqState->DStream));
@@ -1373,12 +1371,15 @@ static void ZSTD_decodeSequence(seq_t* seq, seqState_t* seqState)
     seqState->prevOffset = seq->offset;
     if (litLength == MaxLL)
     {
-        U32 add = *dumps++;
+        U32 add = dumps<de ? *dumps++ : 0;
         if (add < 255) litLength += add;
         else
         {
-            litLength = ZSTD_readLE32(dumps) & 0xFFFFFF;
-            dumps += 3;
+            if (dumps<=(de-3))
+            {
+                litLength = ZSTD_readLE32(dumps) & 0xFFFFFF;  /* no pb : dumps is always followed by seq tables > 1 byte */
+                dumps += 3;
+            }
         }
     }
 
@@ -1389,7 +1390,7 @@ static void ZSTD_decodeSequence(seq_t* seq, seqState_t* seqState)
         if (ZSTD_32bits()) FSE_reloadDStream(&(seqState->DStream));
         nbBits = offsetCode - 1;
         if (offsetCode==0) nbBits = 0;   /* cmove */
-        offset = ((size_t)1 << nbBits) + FSE_readBits(&(seqState->DStream), nbBits);
+        offset = ((size_t)1 << (nbBits & ((sizeof(offset)*8)-1))) + FSE_readBits(&(seqState->DStream), nbBits);
         if (ZSTD_32bits()) FSE_reloadDStream(&(seqState->DStream));
         if (offsetCode==0) offset = prevOffset;
     }
@@ -1398,12 +1399,15 @@ static void ZSTD_decodeSequence(seq_t* seq, seqState_t* seqState)
     matchLength = FSE_decodeSymbol(&(seqState->stateML), &(seqState->DStream));
     if (matchLength == MaxML)
     {
-        U32 add = *dumps++;
+        U32 add = dumps<de ? *dumps++ : 0;
         if (add < 255) matchLength += add;
         else
         {
-            matchLength = ZSTD_readLE32(dumps) & 0xFFFFFF;   /* no pb : dumps is always followed by seq tables > 1 byte */
-            dumps += 3;
+            if (dumps<=(de-3))
+            {
+                matchLength = ZSTD_readLE32(dumps) & 0xFFFFFF;  /* no pb : dumps is always followed by seq tables > 1 byte */
+                dumps += 3;
+            }
         }
     }
     matchLength += MINMATCH;
@@ -1417,9 +1421,9 @@ static void ZSTD_decodeSequence(seq_t* seq, seqState_t* seqState)
 
 
 static size_t ZSTD_execSequence(BYTE* op,
-								seq_t sequence,
-								const BYTE** litPtr, const BYTE* const litLimit,
-								BYTE* const base, BYTE* const oend)
+                                seq_t sequence,
+                                const BYTE** litPtr, const BYTE* const litLimit,
+                                BYTE* const base, BYTE* const oend)
 {
     static const int dec32table[] = {0, 1, 2, 1, 4, 4, 4, 4};   /* added */
     static const int dec64table[] = {8, 8, 8, 7, 8, 9,10,11};   /* substracted */
@@ -1430,7 +1434,7 @@ static size_t ZSTD_execSequence(BYTE* op,
 
     /* check */
     if (endMatch > oend) return (size_t)-ZSTD_ERROR_maxDstSize_tooSmall;   /* overwrite beyond dst buffer */
-	if (litEnd > litLimit) return (size_t)-ZSTD_ERROR_corruption;
+    if (litEnd > litLimit) return (size_t)-ZSTD_ERROR_corruption;
     if (sequence.matchLength > (size_t)(*litPtr-op))  return (size_t)-ZSTD_ERROR_maxDstSize_tooSmall;    /* overwrite literal segment */
 
     /* copy Literals */
@@ -1442,18 +1446,18 @@ static size_t ZSTD_execSequence(BYTE* op,
     *litPtr = litEnd;   /* update for next sequence */
 
     /* check : last match must be at a minimum distance of 8 from end of dest buffer */
-	if (oend-op < 8) return (size_t)-ZSTD_ERROR_maxDstSize_tooSmall;
+    if (oend-op < 8) return (size_t)-ZSTD_ERROR_maxDstSize_tooSmall;
 
-	/* copy Match */
+    /* copy Match */
     {
         const U32 overlapRisk = (((size_t)(litEnd - endMatch)) < 12);
         const BYTE* match = op - sequence.offset;            /* possible underflow at op - offset ? */
         size_t qutt = 12;
         U64 saved[2];
 
-		/* check */
-		if (match < base) return (size_t)-ZSTD_ERROR_corruption;
-		if (sequence.offset > (size_t)base) return (size_t)-ZSTD_ERROR_corruption;
+        /* check */
+        if (match < base) return (size_t)-ZSTD_ERROR_corruption;
+        if (sequence.offset > (size_t)base) return (size_t)-ZSTD_ERROR_corruption;
 
         /* save beginning of literal sequence, in case of write overlap */
         if (overlapRisk)
@@ -1473,7 +1477,7 @@ static size_t ZSTD_execSequence(BYTE* op,
             ZSTD_copy4(op+4, match);
             match -= dec64;
         } else { ZSTD_copy8(op, match); }
-		op += 8; match += 8;
+        op += 8; match += 8;
 
         if (endMatch > oend-12)
         {
@@ -1498,11 +1502,11 @@ static size_t ZSTD_execSequence(BYTE* op,
 typedef struct ZSTD_Dctx_s
 {
     U32 LLTable[FSE_DTABLE_SIZE_U32(LLFSELog)];
-	U32 OffTable[FSE_DTABLE_SIZE_U32(OffFSELog)];
-	U32 MLTable[FSE_DTABLE_SIZE_U32(MLFSELog)];
+    U32 OffTable[FSE_DTABLE_SIZE_U32(OffFSELog)];
+    U32 MLTable[FSE_DTABLE_SIZE_U32(MLFSELog)];
     void* previousDstEnd;
     void* base;
-	size_t expected;
+    size_t expected;
     blockType_t bType;
     U32 phase;
 } dctx_t;
@@ -1514,24 +1518,24 @@ static size_t ZSTD_decompressSequences(
                          const void* seqStart, size_t seqSize,
                          const BYTE* litStart, size_t litSize)
 {
-	dctx_t* dctx = (dctx_t*)ctx;
+    dctx_t* dctx = (dctx_t*)ctx;
     const BYTE* ip = (const BYTE*)seqStart;
     const BYTE* const iend = ip + seqSize;
     BYTE* const ostart = (BYTE* const)dst;
     BYTE* op = ostart;
     BYTE* const oend = ostart + maxDstSize;
-    size_t errorCode;
+    size_t errorCode, dumpsLength;
     const BYTE* litPtr = litStart;
     const BYTE* const litEnd = litStart + litSize;
     int nbSeq;
     const BYTE* dumps;
     U32* DTableLL = dctx->LLTable;
-	U32* DTableML = dctx->MLTable;
+    U32* DTableML = dctx->MLTable;
     U32* DTableOffb = dctx->OffTable;
-	BYTE* const base = (BYTE*) (dctx->base);
+    BYTE* const base = (BYTE*) (dctx->base);
 
     /* Build Decoding Tables */
-    errorCode = ZSTD_decodeSeqHeaders(&nbSeq, &dumps,
+    errorCode = ZSTD_decodeSeqHeaders(&nbSeq, &dumps, &dumpsLength,
                                       DTableLL, DTableML, DTableOffb,
                                       ip, iend-ip);
     if (ZSTD_isError(errorCode)) return errorCode;
@@ -1544,6 +1548,7 @@ static size_t ZSTD_decompressSequences(
 
         memset(&sequence, 0, sizeof(sequence));
         seqState.dumps = dumps;
+        seqState.dumpsEnd = dumps + dumpsLength;
         seqState.prevOffset = 1;
         errorCode = FSE_initDStream(&(seqState.DStream), ip, iend-ip);
         if (FSE_isError(errorCode)) return (size_t)-ZSTD_ERROR_corruption;
@@ -1551,7 +1556,7 @@ static size_t ZSTD_decompressSequences(
         FSE_initDState(&(seqState.stateOffb), &(seqState.DStream), DTableOffb);
         FSE_initDState(&(seqState.stateML), &(seqState.DStream), DTableML);
 
-        for ( ; (FSE_reloadDStream(&(seqState.DStream)) < FSE_DStream_completed) || (nbSeq>0) ; )
+        for ( ; (FSE_reloadDStream(&(seqState.DStream)) <= FSE_DStream_completed) && (nbSeq>0) ; )
         {
             size_t oneSeqSize;
             nbSeq--;
@@ -1562,7 +1567,7 @@ static size_t ZSTD_decompressSequences(
         }
 
         /* check if reached exact end */
-        if (FSE_reloadDStream(&(seqState.DStream)) > FSE_DStream_completed) return (size_t)-ZSTD_ERROR_corruption;   /* requested too much : data is corrupted */
+        if ( !FSE_endOfDStream(&(seqState.DStream)) ) return (size_t)-ZSTD_ERROR_corruption;   /* requested too much : data is corrupted */
         if (nbSeq<0) return (size_t)-ZSTD_ERROR_corruption;   /* requested too many sequences : data is corrupted */
 
         /* last literal segment */
@@ -1658,8 +1663,8 @@ static size_t ZSTD_decompressDCtx(void* ctx, void* dst, size_t maxDstSize, const
 
 size_t ZSTD_decompress(void* dst, size_t maxDstSize, const void* src, size_t srcSize)
 {
-	dctx_t ctx;
-	ctx.base = dst;
+    dctx_t ctx;
+    ctx.base = dst;
     return ZSTD_decompressDCtx(&ctx, dst, maxDstSize, src, srcSize);
 }
 
@@ -1668,14 +1673,20 @@ size_t ZSTD_decompress(void* dst, size_t maxDstSize, const void* src, size_t src
 *  Streaming Decompression API
 *******************************/
 
+size_t ZSTD_resetDCtx(ZSTD_Dctx* dctx)
+{
+    dctx->expected = ZSTD_frameHeaderSize;
+    dctx->phase = 0;
+    dctx->previousDstEnd = NULL;
+    dctx->base = NULL;
+    return 0;
+}
+
 ZSTD_Dctx* ZSTD_createDCtx(void)
 {
     ZSTD_Dctx* dctx = (ZSTD_Dctx*)malloc(sizeof(ZSTD_Dctx));
     if (dctx==NULL) return NULL;
-    dctx->expected = ZSTD_frameHeaderSize;
-    dctx->phase = 0;
-	dctx->previousDstEnd = NULL;
-	dctx->base = NULL;
+    ZSTD_resetDCtx(dctx);
     return dctx;
 }
 
@@ -1685,7 +1696,6 @@ size_t ZSTD_freeDCtx(ZSTD_Dctx* dctx)
     return 0;
 }
 
-
 size_t ZSTD_nextSrcSizeToDecompress(ZSTD_Dctx* dctx)
 {
     return ((dctx_t*)dctx)->expected;
@@ -1697,8 +1707,8 @@ size_t ZSTD_decompressContinue(ZSTD_Dctx* dctx, void* dst, size_t maxDstSize, co
 
     /* Sanity check */
     if (srcSize != ctx->expected) return (size_t)-ZSTD_ERROR_SrcSize;
-	if (dst != ctx->previousDstEnd)  /* not contiguous */
-		ctx->base = dst;
+    if (dst != ctx->previousDstEnd)  /* not contiguous */
+        ctx->base = dst;
 
     /* Decompress : frame header */
     if (ctx->phase == 0)
@@ -1754,7 +1764,7 @@ size_t ZSTD_decompressContinue(ZSTD_Dctx* dctx, void* dst, size_t maxDstSize, co
         }
         ctx->phase = 1;
         ctx->expected = ZSTD_blockHeaderSize;
-		ctx->previousDstEnd = (void*)( ((char*)dst) + rSize);
+        ctx->previousDstEnd = (void*)( ((char*)dst) + rSize);
         return rSize;
     }
 

lib/zstd.h

@@ -47,7 +47,7 @@ extern "C" {
 **************************************/
 #define ZSTD_VERSION_MAJOR    0    /* for breaking interface changes  */
 #define ZSTD_VERSION_MINOR    1    /* for new (non-breaking) interface capabilities */
-#define ZSTD_VERSION_RELEASE  0    /* for tweaks, bug-fixes, or development */
+#define ZSTD_VERSION_RELEASE  2    /* for tweaks, bug-fixes, or development */
 #define ZSTD_VERSION_NUMBER  (ZSTD_VERSION_MAJOR *100*100 + ZSTD_VERSION_MINOR *100 + ZSTD_VERSION_RELEASE)
 unsigned ZSTD_versionNumber (void);
 

lib/zstd_static.h

@@ -32,6 +32,11 @@
 */
 #pragma once
 
+/* The objects defined into this file should be considered experimental.
+ * They are not labelled stable, as their prototype may change in the future.
+ * You can use them for tests, provide feedback, or if you can endure risk of future changes.
+ */
+
 #if defined (__cplusplus)
 extern "C" {
 #endif
@@ -56,16 +61,16 @@ size_t ZSTD_compressEnd(ZSTD_Cctx* cctx, void* dst, size_t maxDstSize);
 
 typedef struct ZSTD_Dctx_s ZSTD_Dctx;
 ZSTD_Dctx* ZSTD_createDCtx(void);
+size_t     ZSTD_resetDCtx(ZSTD_Dctx* dctx);
 size_t     ZSTD_freeDCtx(ZSTD_Dctx* dctx);
 
 size_t ZSTD_nextSrcSizeToDecompress(ZSTD_Dctx* dctx);
 size_t ZSTD_decompressContinue(ZSTD_Dctx* dctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize);
 /*
   Use above functions alternatively.
-  ZSTD_nextSrcSizeToDecompress() tells how much bytes to provide as input to ZSTD_decompressContinue().
-  This value is expected to be provided, precisely, as 'srcSize'.
-  Otherwise, compression will fail (result is an error code, which can be tested using ZSTD_isError() )
-  ZSTD_decompressContinue() result is the number of bytes regenerated within 'dst'.
+  ZSTD_nextSrcSizeToDecompress() tells how much bytes to provide as 'srcSize' to ZSTD_decompressContinue().
+  ZSTD_decompressContinue() will use previous data blocks to improve compresson if they are located prior to current block.
+  Result is the number of bytes regenerated within 'dst'.
   It can be zero, which is not an error; it just means ZSTD_decompressContinue() has decoded some header.
 */
 

programs/Makefile

@@ -30,21 +30,18 @@
 # fullbench32: Same as fullbench, but forced to compile in 32-bits mode
 # ##########################################################################
 
-RELEASE?= v0.1.0
+VERSION?= v0.1.2
 
 DESTDIR?=
-PREFIX ?= /usr
+PREFIX ?= /usr/local
 CFLAGS ?= -O3
-CFLAGS += -std=c99 -Wall -Wextra -Wundef -Wshadow -Wcast-qual -Wcast-align -Wstrict-prototypes -DZSTD_VERSION=\"$(RELEASE)\"
+CFLAGS += -std=c99 -Wall -Wextra -Wundef -Wshadow -Wcast-qual -Wcast-align -Wstrict-prototypes -DZSTD_VERSION=\"$(VERSION)\"
 LDFLAGS = -I../lib
 FLAGS   = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) $(MOREFLAGS)
 
-BINDIR=$(PREFIX)/bin
-MANDIR=$(PREFIX)/share/man/man1
-ZSTDDIR=../lib
-
-TEST_FILES = COPYING
-TEST_TARGETS=test-native
+BINDIR  = $(PREFIX)/bin
+MANDIR  = $(PREFIX)/share/man/man1
+ZSTDDIR = ../lib
 
 
 # Define *.exe as extension for Windows systems
@@ -103,26 +100,42 @@ install: zstd
 	@ln -sf zstd$(EXT) $(DESTDIR)$(BINDIR)/unzstd
 	@echo Installing man pages
 	@install -m 644 zstd.1 $(DESTDIR)$(MANDIR)/zstd.1
-	@install -m 644 zstdcat.1 $(DESTDIR)$(MANDIR)/zstdcat.1
-	@install -m 644 unzstd.1 $(DESTDIR)$(MANDIR)/unzstd.1
+	@ln -sf zstd.1 $(DESTDIR)$(MANDIR)/zstdcat.1
+	@ln -sf zstd.1 $(DESTDIR)$(MANDIR)/unzstd.1
 	@echo zstd installation completed
 
 uninstall:
 	rm -f $(DESTDIR)$(BINDIR)/zstdcat
 	rm -f $(DESTDIR)$(BINDIR)/unzstd
 	[ -x $(DESTDIR)$(BINDIR)/zstd$(EXT) ] && rm -f $(DESTDIR)$(BINDIR)/zstd$(EXT)
+	rm -f $(DESTDIR)$(MANDIR)/zstdcat.1
+	rm -f $(DESTDIR)$(MANDIR)/unzstd.1
 	[ -f $(DESTDIR)$(MANDIR)/zstd.1 ] && rm -f $(DESTDIR)$(MANDIR)/zstd.1
-	[ -f $(DESTDIR)$(MANDIR)/zstdcat.1 ] && rm -f $(DESTDIR)$(MANDIR)/zstdcat.1
-	[ -f $(DESTDIR)$(MANDIR)/unzstd.1 ] && rm -f $(DESTDIR)$(MANDIR)/unzstd.1
 	@echo zstd programs successfully uninstalled
 
 test: test-zstd test-fullbench test-fuzzer
 
 test32: test-zstd32 test-fullbench32 test-fuzzer32
 
-test-all: test test32 test-mem
+test-all: test test32 memtest
 
 test-zstd: zstd datagen
+	@echo "\n**** frame concatenation **** "
+	@echo "hello " > hello.tmp
+	@echo "world!" > world.tmp
+	@cat hello.tmp world.tmp > helloworld.tmp
+	./zstd hello.tmp > hello.zstd
+	./zstd world.tmp > world.zstd
+	@cat hello.zstd world.zstd > helloworld.zstd
+	./zstd -d helloworld.zstd > result.tmp
+	cat result.tmp
+	sdiff helloworld.tmp result.tmp
+	@rm *.tmp *.zstd
+	@echo frame concatenation test completed
+	@echo "**** flush write error test **** "
+	echo foo | ./zstd > /dev/full; if [ $$? -eq 0 ] ; then echo "write error not detected!"; false; fi
+	echo foo | ./zstd | ./zstd -d > /dev/full; if [ $$? -eq 0 ] ; then echo "write error not detected!"; false; fi
+	@echo "**** zstd round-trip tests **** "
 	./datagen          | ./zstd -v    | ./zstd -d > $(VOID)
 	./datagen -g256MB  | ./zstd -v    | ./zstd -d > $(VOID)
 	./datagen -g6GB -P99 | ./zstd -vq | ./zstd -d > $(VOID)
@@ -146,15 +159,15 @@ test-fuzzer: fuzzer
 test-fuzzer32: fuzzer32
 	./fuzzer32
 
-test-mem: zstd datagen fuzzer fullbench
+memtest: zstd datagen fuzzer fullbench
 	@echo "\n ---- valgrind tests : memory analyzer ----"
-	valgrind --leak-check=yes --error-exitcode=1 ./datagen -g50M > /dev/null
+	valgrind --leak-check=yes --error-exitcode=1 ./datagen -g50M > $(VOID)
 	./datagen -g16KB > tmp
-	valgrind --leak-check=yes --error-exitcode=1 ./zstd -vf tmp /dev/null
+	valgrind --leak-check=yes --error-exitcode=1 ./zstd -vf tmp $(VOID)
 	./datagen -g64MB > tmp
-	valgrind --leak-check=yes --error-exitcode=1 ./zstd -vf tmp /dev/null
+	valgrind --leak-check=yes --error-exitcode=1 ./zstd -vf tmp $(VOID)
 	@rm tmp
-	valgrind --leak-check=yes --error-exitcode=1 ./fuzzer -i128 -t1
+	valgrind --leak-check=yes --error-exitcode=1 ./fuzzer -i1000 -t1
 	valgrind --leak-check=yes --error-exitcode=1 ./fullbench -i1
 
 endif

programs/fileio.c

@@ -52,6 +52,7 @@
 #include <stdlib.h>   /* malloc, free */
 #include <string.h>   /* strcmp, strlen */
 #include <time.h>     /* clock */
+#include <errno.h>    /* errno */
 #include "fileio.h"
 #include "zstd_static.h"
 
@@ -254,7 +255,7 @@ unsigned long long FIO_compressFilename(const char* output_filename, const char*
     if (ZSTD_isError(cSize)) EXM_THROW(22, "Compression error : cannot create frame header");
 
     sizeCheck = fwrite(outBuff, 1, cSize, foutput);
-    if (sizeCheck!=cSize) EXM_THROW(23, "Write error : cannot write header");
+    if (sizeCheck!=cSize) EXM_THROW(23, "Write error : cannot write header into %s", output_filename);
     compressedfilesize += cSize;
 
     /* Main compression loop */
@@ -276,7 +277,7 @@ unsigned long long FIO_compressFilename(const char* output_filename, const char*
 
         /* Write cBlock */
         sizeCheck = fwrite(outBuff, 1, cSize, foutput);
-        if (sizeCheck!=cSize) EXM_THROW(25, "Write error : cannot write compressed block");
+        if (sizeCheck!=cSize) EXM_THROW(25, "Write error : cannot write compressed block into %s", output_filename);
         compressedfilesize += cSize;
         inSlot += inSize;
 
@@ -288,7 +289,7 @@ unsigned long long FIO_compressFilename(const char* output_filename, const char*
     if (ZSTD_isError(cSize)) EXM_THROW(26, "Compression error : cannot create frame end");
 
     sizeCheck = fwrite(outBuff, 1, cSize, foutput);
-    if (sizeCheck!=cSize) EXM_THROW(27, "Write error : cannot write frame end");
+    if (sizeCheck!=cSize) EXM_THROW(27, "Write error : cannot write frame end into %s", output_filename);
     compressedfilesize += cSize;
 
     /* Status */
@@ -299,54 +300,27 @@ unsigned long long FIO_compressFilename(const char* output_filename, const char*
     /* clean */
     free(inBuff);
     free(outBuff);
-    fclose(finput);
-    fclose(foutput);
     ZSTD_freeCCtx(ctx);
+    fclose(finput);
+    if (fclose(foutput)) EXM_THROW(28, "Write error : cannot properly close %s", output_filename);
 
     return compressedfilesize;
 }
 
 
 #define MAXHEADERSIZE FIO_FRAMEHEADERSIZE+3
-unsigned long long FIO_decompressFilename(const char* output_filename, const char* input_filename)
+unsigned long long FIO_decompressFrame(FILE* foutput, FILE* finput,
+                                       BYTE* inBuff, size_t inBuffSize,
+                                       BYTE* outBuff, size_t outBuffSize,
+                                       ZSTD_Dctx* dctx)
 {
-    FILE* finput, *foutput;
-    BYTE* inBuff;
-    size_t inBuffSize;
-    BYTE* outBuff, *op, *oend;
-    size_t outBuffSize;
-    U32   blockSize = 128 KB;
-    U32   wNbBlocks = 4;
+    BYTE* op = outBuff;
+    BYTE* const oend = outBuff + outBuffSize;
     U64   filesize = 0;
-    BYTE* header[MAXHEADERSIZE];
-    ZSTD_Dctx* dctx;
     size_t toRead;
     size_t sizeCheck;
 
 
-    /* Init */
-    FIO_getFileHandles(&finput, &foutput, input_filename, output_filename);
-    dctx = ZSTD_createDCtx();
-
-    /* check header */
-    toRead = ZSTD_nextSrcSizeToDecompress(dctx);
-    if (toRead > MAXHEADERSIZE) EXM_THROW(30, "Not enough memory to read header");
-    sizeCheck = fread(header, (size_t)1, toRead, finput);
-    if (sizeCheck != toRead) EXM_THROW(31, "Read error : cannot read header");
-    sizeCheck = ZSTD_decompressContinue(dctx, NULL, 0, header, toRead);   // Decode frame header
-    if (ZSTD_isError(sizeCheck)) EXM_THROW(32, "Error decoding header");
-
-    /* Here later : blockSize determination */
-
-    /* Allocate Memory */
-    inBuffSize = blockSize + FIO_blockHeaderSize;
-    inBuff  = (BYTE*)malloc(inBuffSize);
-    outBuffSize = wNbBlocks * blockSize;
-    outBuff = (BYTE*)malloc(outBuffSize);
-    op = outBuff;
-    oend = outBuff + outBuffSize;
-    if (!inBuff || !outBuff) EXM_THROW(33, "Allocation error : not enough memory");
-
     /* Main decompression Loop */
     toRead = ZSTD_nextSrcSizeToDecompress(dctx);
     while (toRead)
@@ -379,15 +353,76 @@ unsigned long long FIO_decompressFilename(const char* output_filename, const cha
         toRead = ZSTD_nextSrcSizeToDecompress(dctx);
     }
 
+    return filesize;
+}
+
+
+unsigned long long FIO_decompressFilename(const char* output_filename, const char* input_filename)
+{
+    FILE* finput, *foutput;
+    BYTE* inBuff=NULL;
+    size_t inBuffSize = 0;
+    BYTE* outBuff=NULL;
+    size_t outBuffSize = 0;
+    U32   blockSize = 128 KB;
+    U32   wNbBlocks = 4;
+    U64   filesize = 0;
+    BYTE* header[MAXHEADERSIZE];
+    ZSTD_Dctx* dctx;
+    size_t toRead;
+    size_t sizeCheck;
+
+
+    /* Init */
+    FIO_getFileHandles(&finput, &foutput, input_filename, output_filename);
+    dctx = ZSTD_createDCtx();
+
+    /* for each frame */
+    for ( ; ; )
+    {
+        /* check header */
+        ZSTD_resetDCtx(dctx);
+        toRead = ZSTD_nextSrcSizeToDecompress(dctx);
+        if (toRead > MAXHEADERSIZE) EXM_THROW(30, "Not enough memory to read header");
+        sizeCheck = fread(header, (size_t)1, toRead, finput);
+        if (sizeCheck==0) break;   /* no more input */
+        if (sizeCheck != toRead) EXM_THROW(31, "Read error : cannot read header");
+        sizeCheck = ZSTD_decompressContinue(dctx, NULL, 0, header, toRead);   // Decode frame header
+        if (ZSTD_isError(sizeCheck)) EXM_THROW(32, "Error decoding header");
+
+        /* Here later : blockSize determination */
+
+        /* Allocate Memory (if needed) */
+        {
+            size_t newInBuffSize = blockSize + FIO_blockHeaderSize;
+            size_t newOutBuffSize = wNbBlocks * blockSize;
+            if (newInBuffSize > inBuffSize)
+            {
+                free(inBuff);
+                inBuffSize = newInBuffSize;
+                inBuff  = (BYTE*)malloc(inBuffSize);
+            }
+            if (newOutBuffSize > outBuffSize)
+            {
+                free(outBuff);
+                outBuffSize = newOutBuffSize;
+                outBuff  = (BYTE*)malloc(outBuffSize);
+            }
+        }
+        if (!inBuff || !outBuff) EXM_THROW(33, "Allocation error : not enough memory");
+
+        filesize += FIO_decompressFrame(foutput, finput, inBuff, inBuffSize, outBuff, outBuffSize, dctx);
+    }
+
     DISPLAYLEVEL(2, "\r%79s\r", "");
     DISPLAYLEVEL(2,"Decoded %llu bytes   \n", (long long unsigned)filesize);
 
     /* clean */
     free(inBuff);
     free(outBuff);
-    fclose(finput);
-    fclose(foutput);
     ZSTD_freeDCtx(dctx);
+    fclose(finput);
+    if (fclose(foutput)) EXM_THROW(38, "Write error : cannot properly close %s", output_filename);
 
     return filesize;
 }

programs/fullbench.c

@@ -229,7 +229,7 @@ typedef struct
 static size_t g_cSize = 0;
 
 extern size_t ZSTD_getcBlockSize(const void* src, size_t srcSize, blockProperties_t* bpPtr);
-extern size_t ZSTD_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, FSE_DTable* DTableLL, FSE_DTable* DTableML, FSE_DTable* DTableOffb, const void* src, size_t srcSize);
+extern size_t ZSTD_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* dumpsLengthPtr, FSE_DTable* DTableLL, FSE_DTable* DTableML, FSE_DTable* DTableOffb, const void* src, size_t srcSize);
 
 size_t local_ZSTD_compress(void* dst, size_t dstSize, void* buff2, const void* src, size_t srcSize)
 {
@@ -258,9 +258,10 @@ size_t local_ZSTD_decodeSeqHeaders(void* dst, size_t dstSize, void* buff2, const
 {
     U32 DTableML[1<<11], DTableLL[1<<10], DTableOffb[1<<9];
     const BYTE* dumps;
+    size_t length;
     int nbSeq;
     (void)src; (void)srcSize; (void)dst; (void)dstSize;
-    return ZSTD_decodeSeqHeaders(&nbSeq, &dumps, DTableLL, DTableML, DTableOffb, buff2, g_cSize);
+    return ZSTD_decodeSeqHeaders(&nbSeq, &dumps, &length, DTableLL, DTableML, DTableOffb, buff2, g_cSize);
 }
 
 size_t local_conditionalNull(void* dst, size_t dstSize, void* buff2, const void* src, size_t srcSize)
@@ -616,7 +617,7 @@ int main(int argc, char** argv)
 
                     /* Modify Nb Iterations */
                 case 'i':
-                    if ((argument[1] >='1') && (argument[1] <='9'))
+                    if ((argument[1] >='0') && (argument[1] <='9'))
                     {
                         int iters = argument[1] - '0';
                         BMK_SetNbIterations(iters);

programs/unzstd.1

@@ -1,31 +0,0 @@
-\"
-\" unzstd.1: This is a manual page for 'unzstd' program. This file is part of
-\" the zstd <https://github.com/Cyan4973/zstd/> project.
-\"
-
-\" No hyphenation
-.hy 0
-.nr HY 0
-
-.TH unzstd "1" "2014-06-20" "unzstd" "User Commands"
-.SH NAME
-\fBunzstd\fR - Utility based on zstd
-
-.SH SYNOPSIS
-.TP 5
-\fBunzstd\fR [\fBOPTIONS\fR] [-|INPUT-FILE]
-
-.SH DESCRIPTION
-.PP
-\fBunzstd\fR is an utility based on \fBzstd\fR, a fast lossless compression algorithm.
-
-\fBunzstd\fR decompress input file, it is equivalent to \fBzstd -d\fR,
-
-Available options are the same as \fBzstd\fR ones (man zstd).
-
-
-.SH BUGS
-Report bugs at:- https://github.com/Cyan4973/zstd/
-
-.SH AUTHOR
-Yann Collet

programs/zstd.1

@@ -1,26 +1,35 @@
 \"
 \" zstd.1: This is a manual page for 'zstd' program. This file is part of the
-\" zstd <https://github.com/Cyan4973/zstd> project.
+\" zstd <http://www.zstd.net/> project.
+\" Author: Yann Collet
 \"
 
 \" No hyphenation
 .hy 0
 .nr HY 0
 
-.TH zstd "1" "2015-01-22" "zstd" "User Commands"
+.TH zstd "1" "2015-08-22" "zstd" "User Commands"
 .SH NAME
-\fBzstd\fR - standard compression algorithm
+\fBzstd, unzstd, zstdcat\fR - Compress or decompress .zst files
 
 .SH SYNOPSIS
 .TP 5
 \fBzstd\fR [\fBOPTIONS\fR] [-|INPUT-FILE] <OUTPUT-FILE>
+.PP
+.B unzstd
+is equivalent to
+.BR "zstd \-d" 
+.br
+.B zstdcat
+is equivalent to
+.BR "zstd \-dc" 
+.br
 
 .SH DESCRIPTION
 .PP
-\fBzstd\fR is a fast lossless compression algorithm
-with highly parametrable compression strength and memory usage.
-It is based on the \fBLZ77\fR family, coupled with FSE entropy stage.
-zstd offers compression speed of 200 MB/s per core.
+\fBzstd\fR is a fast lossless compression algorithm.
+It is based on the \fBLZ77\fR family, with FSE & huff0 entropy stage.
+zstd offers compression speed > 200 MB/s per core.
 It also features a fast decoder, with speed > 500 MB/s per core.
 \fBzstd\fR supports the following options :
 
@@ -35,13 +44,13 @@ It also features a fast decoder, with speed > 500 MB/s per core.
 .B \-f
  overwrite output without prompting
 .TP
-.B \-h/\-H
+.BR \-h/\-H ", " --help
  display help/long help and exit
 .TP
-.B \-V
+.BR \-V ", " --version
  display Version number and exit
 .TP
-.B \-v
+.BR \-v ", " --verbose
  verbose mode
 .TP
 .B \-q
@@ -63,7 +72,7 @@ It also features a fast decoder, with speed > 500 MB/s per core.
  iteration loops [1-9](default : 3), benchmark mode only
 
 .SH BUGS
-Report bugs at:- https://github.com/Cyan4973/zstd
+Report bugs at:- https://github.com/Cyan4973/zstd/issues
 
 .SH AUTHOR
 Yann Collet

programs/zstdcat.1

@@ -1,32 +0,0 @@
-\"
-\" zstdcat.1: This is a manual page for 'zstdcat' program. This file is part of
-\" the zstd <https://github.com/Cyan4973/zstd/> project.
-\"
-
-\" No hyphenation
-.hy 0
-.nr HY 0
-
-.TH zstdcat "1" "2014-06-20" "zstdcat" "User Commands"
-.SH NAME
-\fBzstdcat\fR - Utility based on zstd
-
-.SH SYNOPSIS
-.TP 5
-\fBzstdcat\fR [\fBOPTIONS\fR] [-|INPUT-FILE]
-
-.SH DESCRIPTION
-.PP
-\fBzstdcat\fR is an utility based on \fBzstd\fR, a fast lossless compression algorithm.
-
-\fBzstdcat\fR decompress input file or stream, redirecting its output to the console.
-It is equivalent to \fBzstd -cd\fR,
-
-Available options are the same as \fBzstd\fR ones (man zstd).
-
-
-.SH BUGS
-Report bugs at:- https://github.com/Cyan4973/zstd/
-
-.SH AUTHOR
-Yann Collet

programs/zstdcli.c

@@ -86,8 +86,9 @@
 /**************************************
 *  Display Macros
 **************************************/
-#define DISPLAY(...)           fprintf(stderr, __VA_ARGS__)
+#define DISPLAY(...)           fprintf(displayOut, __VA_ARGS__)
 #define DISPLAYLEVEL(l, ...)   if (displayLevel>=l) { DISPLAY(__VA_ARGS__); }
+static FILE* displayOut;
 static unsigned displayLevel = 2;   // 0 : no display  // 1: errors  // 2 : + result + interaction + warnings ;  // 3 : + progression;  // 4 : + information
 
 
@@ -137,7 +138,7 @@ static int usage_advanced(const char* programName)
     //DISPLAY( " -t     : test compressed file integrity\n");
     DISPLAY( "Benchmark arguments :\n");
     DISPLAY( " -b     : benchmark file(s)\n");
-    DISPLAY( " -i#    : iteration loops [1-9](default : 3), benchmark mode only\n");
+    DISPLAY( " -i#    : iteration loops [1-9](default : 3)\n");
     return 0;
 }
 
@@ -173,30 +174,32 @@ int main(int argc, char** argv)
     char* dynNameSpace = NULL;
     char extension[] = ZSTD_EXTENSION;
 
+    displayOut = stderr;
     /* Pick out basename component. Don't rely on stdlib because of conflicting behaviour. */
     for (i = (int)strlen(programName); i > 0; i--)
     {
-        if (programName[i] == '/')
-        {
-            i++;
-            break;
-        }
+        if (programName[i] == '/') { i++; break; }
     }
     programName += i;
 
-    /* zstdcat behavior */
+    /* zstdcat preset behavior */
     if (!strcmp(programName, ZSTD_CAT)) { decode=1; forceStdout=1; displayLevel=1; outFileName=stdoutmark; }
 
-    /* unzstd behavior */
+    /* unzstd preset behavior */
     if (!strcmp(programName, ZSTD_UNZSTD))
         decode=1;
 
-    // command switches
+    /* command switches */
     for(i=1; i<argc; i++)
     {
         char* argument = argv[i];
 
-        if(!argument) continue;   // Protection if argument empty
+        if(!argument) continue;   /* Protection if argument empty */
+
+        /* long commands (--long-word) */
+        if (!strcmp(argument, "--version")) { displayOut=stdout; DISPLAY(WELCOME_MESSAGE); return 0; }
+        if (!strcmp(argument, "--help")) { displayOut=stdout; return usage_advanced(programName); }
+        if (!strcmp(argument, "--verbose")) { displayLevel=4; continue; }
 
         /* Decode commands (note : aggregated commands are allowed) */
         if (argument[0]=='-')
@@ -206,6 +209,7 @@ int main(int argc, char** argv)
             {
                 if (!inFileName) inFileName=stdinmark;
                 else outFileName=stdoutmark;
+                continue;
             }
 
             argument++;
@@ -215,9 +219,9 @@ int main(int argc, char** argv)
                 switch(argument[0])
                 {
                     /* Display help */
-                case 'V': DISPLAY(WELCOME_MESSAGE); return 0;   /* Version Only */
+                case 'V': displayOut=stdout; DISPLAY(WELCOME_MESSAGE); return 0;   /* Version Only */
                 case 'H':
-                case 'h': return usage_advanced(programName);
+                case 'h': displayOut=stdout; return usage_advanced(programName);
 
                     // Compression (default)
                 //case 'z': forceCompress = 1; break;