Merge pull request #47 from Cyan4973/dev

Dev

Makefile

@@ -32,7 +32,7 @@
 # ################################################################
 
 # Version number
-export VERSION=0.1.1
+export VERSION := 0.1.2
 
 PRGDIR  = programs
 ZSTDDIR = lib

NEWS

@@ -1,3 +1,9 @@
-r0
+v0.1.2
-initial release
+
+v0.1.1
+fix compression bug
+detects write-flush errors
+
+v0.1.0
+first release
 

lib/Makefile

@@ -32,7 +32,7 @@
 # ################################################################
 
 # Version numbers
-VERSION?= 0.1.1
+VERSION?= 0.1.2
 LIBVER_MAJOR=`sed -n '/define ZSTD_VERSION_MAJOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < zstd.h`
 LIBVER_MINOR=`sed -n '/define ZSTD_VERSION_MINOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < zstd.h`
 LIBVER_PATCH=`sed -n '/define ZSTD_VERSION_RELEASE/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < zstd.h`
@@ -41,8 +41,7 @@ LIBVER  = $(LIBVER_MAJOR).$(LIBVER_MINOR).$(LIBVER_PATCH)
 DESTDIR?=
 PREFIX ?= /usr/local
 CFLAGS ?= -O3
-CFLAGS += -std=c99 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Wstrict-prototypes
+CFLAGS += -I. -std=c99 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Wstrict-prototypes
-LDFLAGS = -I.
 FLAGS   = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) $(MOREFLAGS)
 
 LIBDIR ?= $(PREFIX)/lib

lib/zstd.c

@@ -314,6 +314,7 @@ typedef struct ZSTD_Cctx_s
 #else
     U32 hashTable[HASH_TABLESIZE];
 #endif
+	BYTE buffer[WORKPLACESIZE];
 } cctxi_t;
 
 
@@ -321,12 +322,7 @@ ZSTD_Cctx* ZSTD_createCCtx(void)
 {
     ZSTD_Cctx* ctx = (ZSTD_Cctx*) malloc( sizeof(ZSTD_Cctx) );
     if (ctx==NULL) return NULL;
-    ctx->seqStore.buffer = malloc(WORKPLACESIZE);
+	ctx->seqStore.buffer = ctx->buffer;
-    if (ctx->seqStore.buffer==NULL)
-    {
-        free(ctx);
-        return NULL;
-    }
     ctx->seqStore.offsetStart = (U32*) (ctx->seqStore.buffer);
     ctx->seqStore.offCodeStart = (BYTE*) (ctx->seqStore.offsetStart + (BLOCKSIZE>>2));
     ctx->seqStore.litStart = ctx->seqStore.offCodeStart + (BLOCKSIZE>>2);
@@ -344,7 +340,6 @@ void ZSTD_resetCCtx(ZSTD_Cctx* ctx)
 
 size_t ZSTD_freeCCtx(ZSTD_Cctx* ctx)
 {
-    free(ctx->seqStore.buffer);
     free(ctx);
     return 0;
 }
@@ -1239,7 +1234,7 @@ size_t ZSTD_decodeLiteralsBlock(void* ctx,
 }
 
 
-size_t ZSTD_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr,
+size_t ZSTD_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* dumpsLengthPtr,
                          FSE_DTable* DTableLL, FSE_DTable* DTableML, FSE_DTable* DTableOffb,
                          const void* src, size_t srcSize)
 {
@@ -1272,6 +1267,7 @@ size_t ZSTD_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr,
     }
     *dumpsPtr = ip;
     ip += dumpsLength;
+    *dumpsLengthPtr = dumpsLength;
 
     /* check */
     if (ip > iend-3) return (size_t)-ZSTD_ERROR_SrcSize; /* min : all 3 are "raw", hence no header, but at least xxLog bits per type */
@@ -1356,6 +1352,7 @@ typedef struct {
     FSE_DState_t stateML;
     size_t prevOffset;
     const BYTE* dumps;
+    const BYTE* dumpsEnd;
 } seqState_t;
 
 
@@ -1366,6 +1363,7 @@ static void ZSTD_decodeSequence(seq_t* seq, seqState_t* seqState)
     size_t offset;
     size_t matchLength;
     const BYTE* dumps = seqState->dumps;
+    const BYTE* const de = seqState->dumpsEnd;
 
     /* Literal length */
     litLength = FSE_decodeSymbol(&(seqState->stateLL), &(seqState->DStream));
@@ -1373,14 +1371,17 @@ static void ZSTD_decodeSequence(seq_t* seq, seqState_t* seqState)
     seqState->prevOffset = seq->offset;
     if (litLength == MaxLL)
     {
-        U32 add = *dumps++;
+        U32 add = dumps<de ? *dumps++ : 0;
         if (add < 255) litLength += add;
         else
         {
-            litLength = ZSTD_readLE32(dumps) & 0xFFFFFF;
+            if (dumps<=(de-3))
+            {
+                litLength = ZSTD_readLE32(dumps) & 0xFFFFFF;  /* no pb : dumps is always followed by seq tables > 1 byte */
                 dumps += 3;
             }
         }
+    }
 
     /* Offset */
     {
@@ -1398,14 +1399,17 @@ static void ZSTD_decodeSequence(seq_t* seq, seqState_t* seqState)
     matchLength = FSE_decodeSymbol(&(seqState->stateML), &(seqState->DStream));
     if (matchLength == MaxML)
     {
-        U32 add = *dumps++;
+        U32 add = dumps<de ? *dumps++ : 0;
         if (add < 255) matchLength += add;
         else
+        {
+            if (dumps<=(de-3))
             {
                 matchLength = ZSTD_readLE32(dumps) & 0xFFFFFF;  /* no pb : dumps is always followed by seq tables > 1 byte */
                 dumps += 3;
             }
         }
+    }
     matchLength += MINMATCH;
 
     /* save result */
@@ -1520,7 +1524,7 @@ static size_t ZSTD_decompressSequences(
     BYTE* const ostart = (BYTE* const)dst;
     BYTE* op = ostart;
     BYTE* const oend = ostart + maxDstSize;
-    size_t errorCode;
+    size_t errorCode, dumpsLength;
     const BYTE* litPtr = litStart;
     const BYTE* const litEnd = litStart + litSize;
     int nbSeq;
@@ -1531,7 +1535,7 @@ static size_t ZSTD_decompressSequences(
     BYTE* const base = (BYTE*) (dctx->base);
 
     /* Build Decoding Tables */
-    errorCode = ZSTD_decodeSeqHeaders(&nbSeq, &dumps,
+    errorCode = ZSTD_decodeSeqHeaders(&nbSeq, &dumps, &dumpsLength,
                                       DTableLL, DTableML, DTableOffb,
                                       ip, iend-ip);
     if (ZSTD_isError(errorCode)) return errorCode;
@@ -1544,6 +1548,7 @@ static size_t ZSTD_decompressSequences(
 
         memset(&sequence, 0, sizeof(sequence));
         seqState.dumps = dumps;
+        seqState.dumpsEnd = dumps + dumpsLength;
         seqState.prevOffset = 1;
         errorCode = FSE_initDStream(&(seqState.DStream), ip, iend-ip);
         if (FSE_isError(errorCode)) return (size_t)-ZSTD_ERROR_corruption;
@@ -1551,7 +1556,7 @@ static size_t ZSTD_decompressSequences(
         FSE_initDState(&(seqState.stateOffb), &(seqState.DStream), DTableOffb);
         FSE_initDState(&(seqState.stateML), &(seqState.DStream), DTableML);
 
-        for ( ; (FSE_reloadDStream(&(seqState.DStream)) < FSE_DStream_completed) || (nbSeq>0) ; )
+        for ( ; (FSE_reloadDStream(&(seqState.DStream)) <= FSE_DStream_completed) && (nbSeq>0) ; )
         {
             size_t oneSeqSize;
             nbSeq--;
@@ -1562,7 +1567,7 @@ static size_t ZSTD_decompressSequences(
         }
 
         /* check if reached exact end */
-        if (FSE_reloadDStream(&(seqState.DStream)) > FSE_DStream_completed) return (size_t)-ZSTD_ERROR_corruption;   /* requested too much : data is corrupted */
+        if ( !FSE_endOfDStream(&(seqState.DStream)) ) return (size_t)-ZSTD_ERROR_corruption;   /* requested too much : data is corrupted */
         if (nbSeq<0) return (size_t)-ZSTD_ERROR_corruption;   /* requested too many sequences : data is corrupted */
 
         /* last literal segment */
@@ -1668,14 +1673,20 @@ size_t ZSTD_decompress(void* dst, size_t maxDstSize, const void* src, size_t src
 *  Streaming Decompression API
 *******************************/
 
-ZSTD_Dctx* ZSTD_createDCtx(void)
+size_t ZSTD_resetDCtx(ZSTD_Dctx* dctx)
 {
-    ZSTD_Dctx* dctx = (ZSTD_Dctx*)malloc(sizeof(ZSTD_Dctx));
-    if (dctx==NULL) return NULL;
     dctx->expected = ZSTD_frameHeaderSize;
     dctx->phase = 0;
     dctx->previousDstEnd = NULL;
     dctx->base = NULL;
+    return 0;
+}
+
+ZSTD_Dctx* ZSTD_createDCtx(void)
+{
+    ZSTD_Dctx* dctx = (ZSTD_Dctx*)malloc(sizeof(ZSTD_Dctx));
+    if (dctx==NULL) return NULL;
+    ZSTD_resetDCtx(dctx);
     return dctx;
 }
 
@@ -1685,7 +1696,6 @@ size_t ZSTD_freeDCtx(ZSTD_Dctx* dctx)
     return 0;
 }
 
-
 size_t ZSTD_nextSrcSizeToDecompress(ZSTD_Dctx* dctx)
 {
     return ((dctx_t*)dctx)->expected;

lib/zstd.h

@@ -47,7 +47,7 @@ extern "C" {
 **************************************/
 #define ZSTD_VERSION_MAJOR    0    /* for breaking interface changes  */
 #define ZSTD_VERSION_MINOR    1    /* for new (non-breaking) interface capabilities */
-#define ZSTD_VERSION_RELEASE  0    /* for tweaks, bug-fixes, or development */
+#define ZSTD_VERSION_RELEASE  2    /* for tweaks, bug-fixes, or development */
 #define ZSTD_VERSION_NUMBER  (ZSTD_VERSION_MAJOR *100*100 + ZSTD_VERSION_MINOR *100 + ZSTD_VERSION_RELEASE)
 unsigned ZSTD_versionNumber (void);
 

lib/zstd_static.h

@@ -32,6 +32,11 @@
 */
 #pragma once
 
+/* The objects defined into this file should be considered experimental.
+ * They are not labelled stable, as their prototype may change in the future.
+ * You can use them for tests, provide feedback, or if you can endure risk of future changes.
+ */
+
 #if defined (__cplusplus)
 extern "C" {
 #endif
@@ -56,16 +61,16 @@ size_t ZSTD_compressEnd(ZSTD_Cctx* cctx, void* dst, size_t maxDstSize);
 
 typedef struct ZSTD_Dctx_s ZSTD_Dctx;
 ZSTD_Dctx* ZSTD_createDCtx(void);
+size_t     ZSTD_resetDCtx(ZSTD_Dctx* dctx);
 size_t     ZSTD_freeDCtx(ZSTD_Dctx* dctx);
 
 size_t ZSTD_nextSrcSizeToDecompress(ZSTD_Dctx* dctx);
 size_t ZSTD_decompressContinue(ZSTD_Dctx* dctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize);
 /*
   Use above functions alternatively.
-  ZSTD_nextSrcSizeToDecompress() tells how much bytes to provide as input to ZSTD_decompressContinue().
+  ZSTD_nextSrcSizeToDecompress() tells how much bytes to provide as 'srcSize' to ZSTD_decompressContinue().
-  This value is expected to be provided, precisely, as 'srcSize'.
+  ZSTD_decompressContinue() will use previous data blocks to improve compresson if they are located prior to current block.
-  Otherwise, compression will fail (result is an error code, which can be tested using ZSTD_isError() )
+  Result is the number of bytes regenerated within 'dst'.
-  ZSTD_decompressContinue() result is the number of bytes regenerated within 'dst'.
   It can be zero, which is not an error; it just means ZSTD_decompressContinue() has decoded some header.
 */
 

programs/Makefile

@@ -30,7 +30,7 @@
 # fullbench32: Same as fullbench, but forced to compile in 32-bits mode
 # ##########################################################################
 
-VERSION?= v0.1.1
+VERSION?= v0.1.2
 
 DESTDIR?=
 PREFIX ?= /usr/local
@@ -100,17 +100,17 @@ install: zstd
 	@ln -sf zstd$(EXT) $(DESTDIR)$(BINDIR)/unzstd
 	@echo Installing man pages
 	@install -m 644 zstd.1 $(DESTDIR)$(MANDIR)/zstd.1
-	@install -m 644 zstdcat.1 $(DESTDIR)$(MANDIR)/zstdcat.1
+	@ln -sf zstd.1 $(DESTDIR)$(MANDIR)/zstdcat.1
-	@install -m 644 unzstd.1 $(DESTDIR)$(MANDIR)/unzstd.1
+	@ln -sf zstd.1 $(DESTDIR)$(MANDIR)/unzstd.1
 	@echo zstd installation completed
 
 uninstall:
 	rm -f $(DESTDIR)$(BINDIR)/zstdcat
 	rm -f $(DESTDIR)$(BINDIR)/unzstd
 	[ -x $(DESTDIR)$(BINDIR)/zstd$(EXT) ] && rm -f $(DESTDIR)$(BINDIR)/zstd$(EXT)
+	rm -f $(DESTDIR)$(MANDIR)/zstdcat.1
+	rm -f $(DESTDIR)$(MANDIR)/unzstd.1
 	[ -f $(DESTDIR)$(MANDIR)/zstd.1 ] && rm -f $(DESTDIR)$(MANDIR)/zstd.1
-	[ -f $(DESTDIR)$(MANDIR)/zstdcat.1 ] && rm -f $(DESTDIR)$(MANDIR)/zstdcat.1
-	[ -f $(DESTDIR)$(MANDIR)/unzstd.1 ] && rm -f $(DESTDIR)$(MANDIR)/unzstd.1
 	@echo zstd programs successfully uninstalled
 
 test: test-zstd test-fullbench test-fuzzer
@@ -120,9 +120,22 @@ test32: test-zstd32 test-fullbench32 test-fuzzer32
 test-all: test test32 memtest
 
 test-zstd: zstd datagen
-	@echo "*** zstd cli write error test ***"
+	@echo "\n**** frame concatenation **** "
+	@echo "hello " > hello.tmp
+	@echo "world!" > world.tmp
+	@cat hello.tmp world.tmp > helloworld.tmp
+	./zstd hello.tmp > hello.zstd
+	./zstd world.tmp > world.zstd
+	@cat hello.zstd world.zstd > helloworld.zstd
+	./zstd -d helloworld.zstd > result.tmp
+	cat result.tmp
+	sdiff helloworld.tmp result.tmp
+	@rm *.tmp *.zstd
+	@echo frame concatenation test completed
+	@echo "**** flush write error test **** "
 	echo foo | ./zstd > /dev/full; if [ $$? -eq 0 ] ; then echo "write error not detected!"; false; fi
-	@echo "*** zstd round-trip tests *** "
+	echo foo | ./zstd | ./zstd -d > /dev/full; if [ $$? -eq 0 ] ; then echo "write error not detected!"; false; fi
+	@echo "**** zstd round-trip tests **** "
 	./datagen          | ./zstd -v    | ./zstd -d > $(VOID)
 	./datagen -g256MB  | ./zstd -v    | ./zstd -d > $(VOID)
 	./datagen -g6GB -P99 | ./zstd -vq | ./zstd -d > $(VOID)

programs/fileio.c

@@ -309,45 +309,18 @@ unsigned long long FIO_compressFilename(const char* output_filename, const char*
 
 
 #define MAXHEADERSIZE FIO_FRAMEHEADERSIZE+3
-unsigned long long FIO_decompressFilename(const char* output_filename, const char* input_filename)
+unsigned long long FIO_decompressFrame(FILE* foutput, FILE* finput,
+                                       BYTE* inBuff, size_t inBuffSize,
+                                       BYTE* outBuff, size_t outBuffSize,
+                                       ZSTD_Dctx* dctx)
 {
-    FILE* finput, *foutput;
+    BYTE* op = outBuff;
-    BYTE* inBuff;
+    BYTE* const oend = outBuff + outBuffSize;
-    size_t inBuffSize;
-    BYTE* outBuff, *op, *oend;
-    size_t outBuffSize;
-    U32   blockSize = 128 KB;
-    U32   wNbBlocks = 4;
     U64   filesize = 0;
-    BYTE* header[MAXHEADERSIZE];
-    ZSTD_Dctx* dctx;
     size_t toRead;
     size_t sizeCheck;
 
 
-    /* Init */
-    FIO_getFileHandles(&finput, &foutput, input_filename, output_filename);
-    dctx = ZSTD_createDCtx();
-
-    /* check header */
-    toRead = ZSTD_nextSrcSizeToDecompress(dctx);
-    if (toRead > MAXHEADERSIZE) EXM_THROW(30, "Not enough memory to read header");
-    sizeCheck = fread(header, (size_t)1, toRead, finput);
-    if (sizeCheck != toRead) EXM_THROW(31, "Read error : cannot read header");
-    sizeCheck = ZSTD_decompressContinue(dctx, NULL, 0, header, toRead);   // Decode frame header
-    if (ZSTD_isError(sizeCheck)) EXM_THROW(32, "Error decoding header");
-
-    /* Here later : blockSize determination */
-
-    /* Allocate Memory */
-    inBuffSize = blockSize + FIO_blockHeaderSize;
-    inBuff  = (BYTE*)malloc(inBuffSize);
-    outBuffSize = wNbBlocks * blockSize;
-    outBuff = (BYTE*)malloc(outBuffSize);
-    op = outBuff;
-    oend = outBuff + outBuffSize;
-    if (!inBuff || !outBuff) EXM_THROW(33, "Allocation error : not enough memory");
-
     /* Main decompression Loop */
     toRead = ZSTD_nextSrcSizeToDecompress(dctx);
     while (toRead)
@@ -380,15 +353,76 @@ unsigned long long FIO_decompressFilename(const char* output_filename, const cha
         toRead = ZSTD_nextSrcSizeToDecompress(dctx);
     }
 
+    return filesize;
+}
+
+
+unsigned long long FIO_decompressFilename(const char* output_filename, const char* input_filename)
+{
+    FILE* finput, *foutput;
+    BYTE* inBuff=NULL;
+    size_t inBuffSize = 0;
+    BYTE* outBuff=NULL;
+    size_t outBuffSize = 0;
+    U32   blockSize = 128 KB;
+    U32   wNbBlocks = 4;
+    U64   filesize = 0;
+    BYTE* header[MAXHEADERSIZE];
+    ZSTD_Dctx* dctx;
+    size_t toRead;
+    size_t sizeCheck;
+
+
+    /* Init */
+    FIO_getFileHandles(&finput, &foutput, input_filename, output_filename);
+    dctx = ZSTD_createDCtx();
+
+    /* for each frame */
+    for ( ; ; )
+    {
+        /* check header */
+        ZSTD_resetDCtx(dctx);
+        toRead = ZSTD_nextSrcSizeToDecompress(dctx);
+        if (toRead > MAXHEADERSIZE) EXM_THROW(30, "Not enough memory to read header");
+        sizeCheck = fread(header, (size_t)1, toRead, finput);
+        if (sizeCheck==0) break;   /* no more input */
+        if (sizeCheck != toRead) EXM_THROW(31, "Read error : cannot read header");
+        sizeCheck = ZSTD_decompressContinue(dctx, NULL, 0, header, toRead);   // Decode frame header
+        if (ZSTD_isError(sizeCheck)) EXM_THROW(32, "Error decoding header");
+
+        /* Here later : blockSize determination */
+
+        /* Allocate Memory (if needed) */
+        {
+            size_t newInBuffSize = blockSize + FIO_blockHeaderSize;
+            size_t newOutBuffSize = wNbBlocks * blockSize;
+            if (newInBuffSize > inBuffSize)
+            {
+                free(inBuff);
+                inBuffSize = newInBuffSize;
+                inBuff  = (BYTE*)malloc(inBuffSize);
+            }
+            if (newOutBuffSize > outBuffSize)
+            {
+                free(outBuff);
+                outBuffSize = newOutBuffSize;
+                outBuff  = (BYTE*)malloc(outBuffSize);
+            }
+        }
+        if (!inBuff || !outBuff) EXM_THROW(33, "Allocation error : not enough memory");
+
+        filesize += FIO_decompressFrame(foutput, finput, inBuff, inBuffSize, outBuff, outBuffSize, dctx);
+    }
+
     DISPLAYLEVEL(2, "\r%79s\r", "");
     DISPLAYLEVEL(2,"Decoded %llu bytes   \n", (long long unsigned)filesize);
 
     /* clean */
     free(inBuff);
     free(outBuff);
-    fclose(finput);
-    fclose(foutput);
     ZSTD_freeDCtx(dctx);
+    fclose(finput);
+    if (fclose(foutput)) EXM_THROW(38, "Write error : cannot properly close %s", output_filename);
 
     return filesize;
 }

programs/fullbench.c

@@ -229,7 +229,7 @@ typedef struct
 static size_t g_cSize = 0;
 
 extern size_t ZSTD_getcBlockSize(const void* src, size_t srcSize, blockProperties_t* bpPtr);
-extern size_t ZSTD_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, FSE_DTable* DTableLL, FSE_DTable* DTableML, FSE_DTable* DTableOffb, const void* src, size_t srcSize);
+extern size_t ZSTD_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* dumpsLengthPtr, FSE_DTable* DTableLL, FSE_DTable* DTableML, FSE_DTable* DTableOffb, const void* src, size_t srcSize);
 
 size_t local_ZSTD_compress(void* dst, size_t dstSize, void* buff2, const void* src, size_t srcSize)
 {
@@ -258,9 +258,10 @@ size_t local_ZSTD_decodeSeqHeaders(void* dst, size_t dstSize, void* buff2, const
 {
     U32 DTableML[1<<11], DTableLL[1<<10], DTableOffb[1<<9];
     const BYTE* dumps;
+    size_t length;
     int nbSeq;
     (void)src; (void)srcSize; (void)dst; (void)dstSize;
-    return ZSTD_decodeSeqHeaders(&nbSeq, &dumps, DTableLL, DTableML, DTableOffb, buff2, g_cSize);
+    return ZSTD_decodeSeqHeaders(&nbSeq, &dumps, &length, DTableLL, DTableML, DTableOffb, buff2, g_cSize);
 }
 
 size_t local_conditionalNull(void* dst, size_t dstSize, void* buff2, const void* src, size_t srcSize)

programs/unzstd.1

@@ -1 +0,0 @@
-zstd.1

programs/zstdcat.1

@@ -1 +0,0 @@
-zstd.1