Merge remote-tracking branch 'refs/remotes/upstream/dev' into dev

Makefile

@@ -32,7 +32,7 @@
 # ################################################################
 
 # Version number
-export VERSION := 0.4.4
+export VERSION := 0.4.5
 
 PRGDIR  = programs
 ZSTDDIR = lib

NEWS

@@ -1,3 +1,6 @@
+v0.4.5
+new : -m/--multiple : compress/decompress multiple files
+
 v0.4.4
 Fixed : high compression modes for Windows 32 bits
 new : external dictionary API extended to buffered mode and accessible through command line

lib/zstd_compress.c

@@ -2117,7 +2117,7 @@ size_t ZSTD_compressBegin(ZSTD_CCtx* ctx, void* dst, size_t maxDstSize, int comp
 }
 
 
-/** ZSTD_compressEnd
+/*! ZSTD_compressEnd
 *   Write frame epilogue
 *   @return : nb of bytes written into dst (or an error code) */
 size_t ZSTD_compressEnd(ZSTD_CCtx* ctx, void* dst, size_t maxDstSize)
@@ -2139,6 +2139,7 @@ size_t ZSTD_compressEnd(ZSTD_CCtx* ctx, void* dst, size_t maxDstSize)
 size_t ZSTD_compress_advanced (ZSTD_CCtx* ctx,
                                void* dst, size_t maxDstSize,
                          const void* src, size_t srcSize,
+                         const void* dict,size_t dictSize,
                                ZSTD_parameters params)
 {
     BYTE* const ostart = (BYTE*)dst;
@@ -2151,9 +2152,15 @@ size_t ZSTD_compress_advanced (ZSTD_CCtx* ctx,
     op += oSize;
     maxDstSize -= oSize;
 
+    /* dictionary */
+    if (dict)
+    {
+        oSize = ZSTD_compress_insertDictionary(ctx, dict, dictSize);
+        if (ZSTD_isError(oSize)) return oSize;
+    }
+
     /* body (compression) */
-    ctx->base = (const BYTE*)src;
+    oSize = ZSTD_compressContinue (ctx, op,  maxDstSize, src, srcSize);
-    oSize = ZSTD_compress_generic (ctx, op,  maxDstSize, src, srcSize);
     if(ZSTD_isError(oSize)) return oSize;
     op += oSize;
     maxDstSize -= oSize;
@@ -2166,9 +2173,14 @@ size_t ZSTD_compress_advanced (ZSTD_CCtx* ctx,
     return (op - ostart);
 }
 
+size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize, const void* dict, size_t dictSize, int compressionLevel)
+{
+    return ZSTD_compress_advanced(ctx, dst, maxDstSize, src, srcSize, dict, dictSize, ZSTD_getParams(compressionLevel, srcSize+dictSize));
+}
+
 size_t ZSTD_compressCCtx (ZSTD_CCtx* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize, int compressionLevel)
 {
-    return ZSTD_compress_advanced(ctx, dst, maxDstSize, src, srcSize, ZSTD_getParams(compressionLevel, srcSize));
+    return ZSTD_compress_advanced(ctx, dst, maxDstSize, src, srcSize, NULL, 0, ZSTD_getParams(compressionLevel, srcSize));
 }
 
 size_t ZSTD_compress(void* dst, size_t maxDstSize, const void* src, size_t srcSize, int compressionLevel)
@@ -2180,3 +2192,4 @@ size_t ZSTD_compress(void* dst, size_t maxDstSize, const void* src, size_t srcSi
     free(ctxBody.workSpace);   /* can't free ctxBody, since it's on stack; free heap content */
     return result;
 }
+

lib/zstd_decompress.c

@@ -676,7 +676,10 @@ static size_t ZSTD_decompressBlock(
 }
 
 
-size_t ZSTD_decompressDCtx(ZSTD_DCtx* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize)
+size_t ZSTD_decompress_usingDict(ZSTD_DCtx* ctx,
+                                 void* dst, size_t maxDstSize,
+                                 const void* src, size_t srcSize,
+                                 const void* dict, size_t dictSize)
 {
     const BYTE* ip = (const BYTE*)src;
     const BYTE* iend = ip + srcSize;
@@ -686,9 +689,19 @@ size_t ZSTD_decompressDCtx(ZSTD_DCtx* ctx, void* dst, size_t maxDstSize, const v
     size_t remainingSize = srcSize;
     blockProperties_t blockProperties;
 
-
     /* init */
+    ZSTD_resetDCtx(ctx);
+    if (dict)
+    {
+        ZSTD_decompress_insertDictionary(ctx, dict, dictSize);
+        ctx->dictEnd = ctx->previousDstEnd;
+        ctx->vBase = (const char*)dst - ((const char*)(ctx->previousDstEnd) - (const char*)(ctx->base));
+        ctx->base = dst;
+    }
+    else
+    {
         ctx->vBase = ctx->base = ctx->dictEnd = dst;
+    }
 
     /* Frame Header */
     {
@@ -749,10 +762,16 @@ size_t ZSTD_decompressDCtx(ZSTD_DCtx* ctx, void* dst, size_t maxDstSize, const v
     return op-ostart;
 }
 
+
+size_t ZSTD_decompressDCtx(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize)
+{
+    return ZSTD_decompress_usingDict(dctx, dst, maxDstSize, src, srcSize, NULL, 0);
+}
+
 size_t ZSTD_decompress(void* dst, size_t maxDstSize, const void* src, size_t srcSize)
 {
-    ZSTD_DCtx ctx;
+    ZSTD_DCtx dctx;
-    return ZSTD_decompressDCtx(&ctx, dst, maxDstSize, src, srcSize);
+    return ZSTD_decompressDCtx(&dctx, dst, maxDstSize, src, srcSize);
 }
 
 

lib/zstd_static.h

@@ -80,7 +80,7 @@ typedef struct
 
 
 /* *************************************
-*  Advanced function
+*  Advanced functions
 ***************************************/
 /** ZSTD_getParams
 *   return ZSTD_parameters structure for a selected compression level and srcSize.
@@ -91,16 +91,43 @@ ZSTDLIB_API ZSTD_parameters ZSTD_getParams(int compressionLevel, U64 srcSizeHint
 *   correct params value to remain within authorized range */
 ZSTDLIB_API void ZSTD_validateParams(ZSTD_parameters* params);
 
+/** ZSTD_compress_usingDict
+*   Same as ZSTD_compressCCtx(), using a Dictionary content as prefix
+*   Note : dict can be NULL, in which case, it's equivalent to ZSTD_compressCCtx() */
+ZSTDLIB_API size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx,
+                                           void* dst, size_t maxDstSize,
+                                     const void* src, size_t srcSize,
+                                     const void* dict,size_t dictSize,
+                                           int compressionLevel);
+
 /** ZSTD_compress_advanced
-*   Same as ZSTD_compressCCtx(), with fine-tune control of each compression parameter */
+*   Same as ZSTD_compress_usingDict(), with fine-tune control of each compression parameter */
 ZSTDLIB_API size_t ZSTD_compress_advanced (ZSTD_CCtx* ctx,
                                            void* dst, size_t maxDstSize,
                                      const void* src, size_t srcSize,
+                                     const void* dict,size_t dictSize,
                                            ZSTD_parameters params);
 
+/** Decompression context management */
+typedef struct ZSTD_DCtx_s ZSTD_DCtx;
+ZSTDLIB_API ZSTD_DCtx* ZSTD_createDCtx(void);
+ZSTDLIB_API size_t     ZSTD_freeDCtx(ZSTD_DCtx* dctx);
+
+/** ZSTD_decompressDCtx
+*   Same as ZSTD_decompress, with pre-allocated DCtx structure */
+size_t ZSTD_decompressDCtx(ZSTD_DCtx* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize);
+
+/** ZSTD_decompress_usingDict
+*   Same as ZSTD_decompressDCtx, using a Dictionary content as prefix
+*   Note : dict can be NULL, in which case, it's equivalent to ZSTD_decompressDCtx() */
+size_t ZSTD_decompress_usingDict(ZSTD_DCtx* ctx,
+                                 void* dst, size_t maxDstSize,
+                                 const void* src, size_t srcSize,
+                                 const void* dict, size_t dictSize);
+
 
 /* **************************************
-*  Streaming functions (bufferless mode)
+*  Streaming functions (direct mode)
 ****************************************/
 ZSTDLIB_API size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, void* dst, size_t maxDstSize, int compressionLevel);
 ZSTDLIB_API size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* ctx, void* dst, size_t maxDstSize, ZSTD_parameters params);
@@ -110,7 +137,7 @@ ZSTDLIB_API size_t ZSTD_compressContinue(ZSTD_CCtx* cctx, void* dst, size_t maxD
 ZSTDLIB_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t maxDstSize);
 
 /**
-  Streaming compression, bufferless mode
+  Streaming compression, direct mode (bufferless)
 
   A ZSTD_CCtx object is required to track streaming operations.
   Use ZSTD_createCCtx() / ZSTD_freeCCtx() to manage it.
@@ -131,14 +158,11 @@ ZSTDLIB_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t maxDstSiz
 
   Finish a frame with ZSTD_compressEnd(), which will write the epilogue.
   Without it, the frame will be considered incomplete by decoders.
-  You can then re-use ZSTD_CCtx to compress new frames.
+
+  You can then reuse ZSTD_CCtx to compress new frames.
 */
 
 
-typedef struct ZSTD_DCtx_s ZSTD_DCtx;
-ZSTDLIB_API ZSTD_DCtx* ZSTD_createDCtx(void);
-ZSTDLIB_API size_t     ZSTD_freeDCtx(ZSTD_DCtx* dctx);
-
 ZSTDLIB_API size_t ZSTD_resetDCtx(ZSTD_DCtx* dctx);
 ZSTDLIB_API size_t ZSTD_getFrameParams(ZSTD_parameters* params, const void* src, size_t srcSize);
 ZSTDLIB_API void   ZSTD_decompress_insertDictionary(ZSTD_DCtx* ctx, const void* src, size_t srcSize);
@@ -160,7 +184,8 @@ ZSTDLIB_API size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t ma
            >0 : means there is not enough data into src. Provides the expected size to successfully decode header.
            errorCode, which can be tested using ZSTD_isError() (For example, if it's not a ZSTD header)
 
-  Then, you can optionally insert a dictionary. This operation must mimic the compressor behavior, otherwise decompression will fail or be corrupted.
+  Then, you can optionally insert a dictionary.
+  This operation must mimic the compressor behavior, otherwise decompression will fail or be corrupted.
 
   Then it's possible to start decompression.
   Use ZSTD_nextSrcSizeToDecompress() and ZSTD_decompressContinue() alternatively.

programs/Makefile

@@ -30,7 +30,7 @@
 # fullbench32: Same as fullbench, but forced to compile in 32-bits mode
 # ##########################################################################
 
-VERSION?= 0.4.4
+VERSION?= 0.4.5
 
 DESTDIR?=
 PREFIX ?= /usr/local

programs/bench.c

@@ -63,7 +63,7 @@
 #endif
 
 #include "mem.h"
-#include "zstd.h"
+#include "zstd_static.h"
 #include "xxhash.h"
 #include "datagen.h"      /* RDG_genBuffer */
 
@@ -211,13 +211,12 @@ typedef struct
     size_t resSize;
 } blockParam_t;
 
-typedef size_t (*compressor_t) (void* dst, size_t maxDstSize, const void* src, size_t srcSize, int compressionLevel);
-
 #define MIN(a,b) ((a)<(b) ? (a) : (b))
 
 static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
                         const char* displayName, int cLevel,
-                        const size_t* fileSizes, U32 nbFiles)
+                        const size_t* fileSizes, U32 nbFiles,
+                        const void* dictBuffer, size_t dictBufferSize)
 {
     const size_t blockSize = (g_blockSize ? g_blockSize : srcSize) + (!srcSize);   /* avoid div by 0 */
     const U32 maxNbBlocks = (U32) ((srcSize + (blockSize-1)) / blockSize) + nbFiles;
@@ -225,7 +224,8 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
     const size_t maxCompressedSize = ZSTD_compressBound(srcSize) + (maxNbBlocks * 1024);   /* add some room for safety */
     void* const compressedBuffer = malloc(maxCompressedSize);
     void* const resultBuffer = malloc(srcSize);
-    const compressor_t compressor = ZSTD_compress;
+    ZSTD_CCtx* ctx = ZSTD_createCCtx();
+    ZSTD_DCtx* dctx = ZSTD_createDCtx();
     U64 crcOrig = XXH64(srcBuffer, srcSize, 0);
     U32 nbBlocks = 0;
 
@@ -233,7 +233,7 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
     if (strlen(displayName)>17) displayName += strlen(displayName)-17;   /* can only display 17 characters */
 
     /* Memory allocation & restrictions */
-    if (!compressedBuffer || !resultBuffer || !blockTable)
+    if (!compressedBuffer || !resultBuffer || !blockTable || !ctx || !dctx)
         EXM_THROW(31, "not enough memory");
 
     /* Init blockTable data */
@@ -292,7 +292,11 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
             while (BMK_GetMilliSpan(milliTime) < TIMELOOP)
             {
                 for (blockNb=0; blockNb<nbBlocks; blockNb++)
-                    blockTable[blockNb].cSize = compressor(blockTable[blockNb].cPtr,  blockTable[blockNb].cRoom, blockTable[blockNb].srcPtr,blockTable[blockNb].srcSize, cLevel);
+                    blockTable[blockNb].cSize = ZSTD_compress_usingDict(ctx,
+                                                              blockTable[blockNb].cPtr,  blockTable[blockNb].cRoom,
+                                                              blockTable[blockNb].srcPtr,blockTable[blockNb].srcSize,
+                                                              dictBuffer, dictBufferSize,
+                                                              cLevel);
                 nbLoops++;
             }
             milliTime = BMK_GetMilliSpan(milliTime);
@@ -315,8 +319,10 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
             for ( ; BMK_GetMilliSpan(milliTime) < TIMELOOP; nbLoops++)
             {
                 for (blockNb=0; blockNb<nbBlocks; blockNb++)
-                    blockTable[blockNb].resSize = ZSTD_decompress(blockTable[blockNb].resPtr, blockTable[blockNb].srcSize,
+                    blockTable[blockNb].resSize = ZSTD_decompress_usingDict(dctx,
-                                                                  blockTable[blockNb].cPtr, blockTable[blockNb].cSize);
+                                                                blockTable[blockNb].resPtr, blockTable[blockNb].srcSize,
+                                                                blockTable[blockNb].cPtr, blockTable[blockNb].cSize,
+                                                                dictBuffer, dictBufferSize);
             }
             milliTime = BMK_GetMilliSpan(milliTime);
 
@@ -345,11 +351,15 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
 
         if (crcOrig == crcCheck)
             DISPLAY("%2i-%-17.17s :%10i ->%10i (%5.3f),%6.1f MB/s ,%6.1f MB/s \n", cLevel, displayName, (int)srcSize, (int)cSize, ratio, (double)srcSize / fastestC / 1000., (double)srcSize / fastestD / 1000.);
+        else
+            DISPLAY("X \n");
     }
 
     /* clean up */
     free(compressedBuffer);
     free(resultBuffer);
+    ZSTD_freeCCtx(ctx);
+    ZSTD_freeDCtx(dctx);
     return 0;
 }
 
@@ -375,16 +385,23 @@ static size_t BMK_findMaxMem(U64 requiredMem)
 
 static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize,
                             const char* displayName, int cLevel,
-                            const size_t* fileSizes, unsigned nbFiles)
+                            const size_t* fileSizes, unsigned nbFiles,
+                            const void* dictBuffer, size_t dictBufferSize)
 {
     if (cLevel < 0)
     {
         int l;
         for (l=1; l <= -cLevel; l++)
-            BMK_benchMem(srcBuffer, benchedSize, displayName, l, fileSizes, nbFiles);
+            BMK_benchMem(srcBuffer, benchedSize,
+                         displayName, l,
+                         fileSizes, nbFiles,
+                         dictBuffer, dictBufferSize);
         return;
     }
-    BMK_benchMem(srcBuffer, benchedSize, displayName, cLevel, fileSizes, nbFiles);
+    BMK_benchMem(srcBuffer, benchedSize,
+                 displayName, cLevel,
+                 fileSizes, nbFiles,
+                 dictBuffer, dictBufferSize);
 }
 
 static U64 BMK_getTotalFileSize(const char** fileNamesTable, unsigned nbFiles)
@@ -420,22 +437,37 @@ static void BMK_loadFiles(void* buffer, size_t bufferSize,
     }
 }
 
-static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles, int cLevel)
+static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles,
+                               const char* dictFileName, int cLevel)
 {
     void* srcBuffer;
     size_t benchedSize;
-    size_t* fileSizes;
+    void* dictBuffer = NULL;
+    size_t dictBufferSize = 0;
+    size_t* fileSizes = (size_t*)malloc(nbFiles * sizeof(size_t));
     U64 totalSizeToLoad = BMK_getTotalFileSize(fileNamesTable, nbFiles);
     char mfName[20] = {0};
     const char* displayName = NULL;
 
+    if (!fileSizes) EXM_THROW(12, "not enough memory for fileSizes");
+
+    /* Load dictionary */
+    if (dictFileName != NULL)
+    {
+        U64 dictFileSize = BMK_getFileSize(dictFileName);
+        if (dictFileSize > 64 MB) EXM_THROW(10, "dictionary file %s too large", dictFileName);
+        dictBufferSize = (size_t)dictFileSize;
+        dictBuffer = malloc(dictBufferSize);
+        if (dictBuffer==NULL) EXM_THROW(11, "not enough memory for dictionary (%u bytes)", (U32)dictBufferSize);
+        BMK_loadFiles(dictBuffer, dictBufferSize, fileSizes, &dictFileName, 1);
+    }
+
     /* Memory allocation & restrictions */
     benchedSize = BMK_findMaxMem(totalSizeToLoad * 3) / 3;
     if ((U64)benchedSize > totalSizeToLoad) benchedSize = (size_t)totalSizeToLoad;
     if (benchedSize < totalSizeToLoad)
         DISPLAY("Not enough memory; testing %u MB only...\n", (U32)(benchedSize >> 20));
     srcBuffer = malloc(benchedSize);
-    fileSizes = (size_t*)malloc(nbFiles * sizeof(size_t));
     if (!srcBuffer) EXM_THROW(12, "not enough memory");
 
     /* Load input buffer */
@@ -446,10 +478,14 @@ static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles, in
     if (nbFiles > 1) displayName = mfName;
     else displayName = fileNamesTable[0];
 
-    BMK_benchCLevel(srcBuffer, benchedSize, displayName, cLevel, fileSizes, nbFiles);
+    BMK_benchCLevel(srcBuffer, benchedSize,
+                    displayName, cLevel,
+                    fileSizes, nbFiles,
+                    dictBuffer, dictBufferSize);
 
     /* clean up */
     free(srcBuffer);
+    free(dictBuffer);
     free(fileSizes);
 }
 
@@ -468,21 +504,22 @@ static void BMK_syntheticTest(int cLevel, double compressibility)
 
     /* Bench */
     snprintf (name, sizeof(name), "Synthetic %2u%%", (unsigned)(compressibility*100));
-    BMK_benchCLevel(srcBuffer, benchedSize, name, cLevel, &benchedSize, 1);
+    BMK_benchCLevel(srcBuffer, benchedSize, name, cLevel, &benchedSize, 1, NULL, 0);
 
     /* clean up */
     free(srcBuffer);
 }
 
 
-int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles, unsigned cLevel)
+int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles,
+                   const char* dictFileName, int cLevel)
 {
     double compressibility = (double)g_compressibilityDefault / 100;
 
     if (nbFiles == 0)
         BMK_syntheticTest(cLevel, compressibility);
     else
-        BMK_benchFileTable(fileNamesTable, nbFiles, cLevel);
+        BMK_benchFileTable(fileNamesTable, nbFiles, dictFileName, cLevel);
     return 0;
 }
 

programs/bench.h

@@ -26,7 +26,8 @@
 
 
 /* Main function */
-int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles, unsigned cLevel);
+int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles,
+                   const char* dictFileName, int cLevel);
 
 /* Set Parameters */
 void BMK_SetNbIterations(int nbLoops);

programs/fileio.c

@@ -175,52 +175,6 @@ static unsigned FIO_GetMilliSpan(clock_t nPrevious)
 }
 
 
-static void FIO_getFileHandles(FILE** pfinput, FILE** pfoutput, const char* input_filename, const char* output_filename)
-{
-    if (!strcmp (input_filename, stdinmark))
-    {
-        DISPLAYLEVEL(4,"Using stdin for input\n");
-        *pfinput = stdin;
-        SET_BINARY_MODE(stdin);
-    }
-    else
-    {
-        *pfinput = fopen(input_filename, "rb");
-    }
-
-    if (!strcmp (output_filename, stdoutmark))
-    {
-        DISPLAYLEVEL(4,"Using stdout for output\n");
-        *pfoutput = stdout;
-        SET_BINARY_MODE(stdout);
-    }
-    else
-    {
-        /* Check if destination file already exists */
-        *pfoutput=0;
-        if (strcmp(output_filename,nulmark)) *pfoutput = fopen( output_filename, "rb" );
-        if (*pfoutput!=0)
-        {
-            fclose(*pfoutput);
-            if (!g_overwrite)
-            {
-                char ch;
-                if (g_displayLevel <= 1)   /* No interaction possible */
-                    EXM_THROW(11, "Operation aborted : %s already exists", output_filename);
-                DISPLAYLEVEL(2, "Warning : %s already exists\n", output_filename);
-                DISPLAYLEVEL(2, "Overwrite ? (Y/N) : ");
-                ch = (char)getchar();
-                if ((ch!='Y') && (ch!='y')) EXM_THROW(11, "Operation aborted : %s already exists", output_filename);
-            }
-        }
-        *pfoutput = fopen( output_filename, "wb" );
-    }
-
-    if ( *pfinput==0 ) EXM_THROW(12, "Pb opening src : %s", input_filename);
-    if ( *pfoutput==0) EXM_THROW(13, "Pb opening dst : %s", output_filename);
-}
-
-
 static U64 FIO_getFileSize(const char* infilename)
 {
     int r;
@@ -236,60 +190,103 @@ static U64 FIO_getFileSize(const char* infilename)
 }
 
 
-static int FIO_getFiles(const char* input_filename, const char* output_filename,
+static int FIO_getFiles(FILE** fileOutPtr, FILE** fileInPtr,
-                        FILE** pfinput, FILE** pfoutput)
+                        const char* dstFileName, const char* srcFileName)
 {
-
+    if (!strcmp (srcFileName, stdinmark))
-    if (!strcmp (input_filename, stdinmark))
     {
         DISPLAYLEVEL(4,"Using stdin for input\n");
-        *pfinput = stdin;
+        *fileInPtr = stdin;
         SET_BINARY_MODE(stdin);
     }
     else
     {
-        *pfinput = fopen(input_filename, "rb");
+        *fileInPtr = fopen(srcFileName, "rb");
     }
 
-    if ( *pfinput==0 )
+    if ( *fileInPtr==0 )
     {
-        DISPLAYLEVEL(1, "Unable to access file for processing: %s\n", input_filename);
+        DISPLAYLEVEL(1, "Unable to access file for processing: %s\n", srcFileName);
         return 1;
     }
 
-    if (!strcmp (output_filename, stdoutmark))
+    if (!strcmp (dstFileName, stdoutmark))
     {
         DISPLAYLEVEL(4,"Using stdout for output\n");
-        *pfoutput = stdout;
+        *fileOutPtr = stdout;
         SET_BINARY_MODE(stdout);
     }
     else
     {
         /* Check if destination file already exists */
-        *pfoutput=0;
-        *pfoutput = fopen( output_filename, "rb" );
-        if (*pfoutput!=0)
-        {
-            fclose(*pfoutput);
         if (!g_overwrite)
         {
-                int ch = 'Y';
+            *fileOutPtr = fopen( dstFileName, "rb" );
-                DISPLAYLEVEL(2, "Warning : %s already exists\n", output_filename);
+            if (*fileOutPtr != 0)
-                if ((g_displayLevel <= 1) || (*pfinput == stdin))
+            {
-                    EXM_THROW(11, "Operation aborted : %s already exists", output_filename);   /* No interaction possible */
+                /* prompt for overwrite authorization */
-                DISPLAYLEVEL(2, "Overwrite ? (Y/n) : ");
+                int ch = 'N';
-                while((ch = getchar()) != '\n' && ch != EOF)   /* flush integrated */
+                fclose(*fileOutPtr);
-                if ((ch!='Y') && (ch!='y')) EXM_THROW(12, "No. Operation aborted : %s already exists", output_filename);
+                DISPLAY("Warning : %s already exists \n", dstFileName);
+                if ((g_displayLevel <= 1) || (*fileInPtr == stdin))
+                {
+                    /* No interaction possible */
+                    DISPLAY("Operation aborted : %s already exists \n", dstFileName);
+                    return 1;
+                }
+                DISPLAY("Overwrite ? (y/N) : ");
+                while((ch = getchar()) != '\n' && ch != EOF);   /* flush integrated */
+                if ((ch!='Y') && (ch!='y'))
+                {
+                    DISPLAY("No. Operation aborted : %s already exists \n", dstFileName);
+                    return 1;
                 }
             }
-        *pfoutput = fopen( output_filename, "wb" );
+        }
+        *fileOutPtr = fopen( dstFileName, "wb" );
     }
 
-    if (*pfoutput==0) EXM_THROW(13, "Pb opening %s", output_filename);
+    if (*fileOutPtr==0) EXM_THROW(13, "Pb opening %s", dstFileName);
 
     return 0;
 }
 
+/*!FIO_loadFile
+*  creates a buffer, pointed by *bufferPtr,
+*  loads "filename" content into it
+*  up to MAX_DICT_SIZE bytes
+*/
+static size_t FIO_loadFile(void** bufferPtr, const char* fileName)
+{
+    FILE* fileHandle;
+    size_t readSize;
+    U64 fileSize;
+
+    *bufferPtr = NULL;
+    if (fileName == NULL)
+        return 0;
+
+    DISPLAYLEVEL(4,"Loading %s as dictionary \n", fileName);
+    fileHandle = fopen(fileName, "rb");
+    if (fileHandle==0) EXM_THROW(31, "Error opening file %s", fileName);
+    fileSize = FIO_getFileSize(fileName);
+    if (fileSize > MAX_DICT_SIZE)
+    {
+        int seekResult;
+        if (fileSize > 1 GB) EXM_THROW(32, "Dictionary file %s is too large", fileName);   /* avoid extreme cases */
+        DISPLAYLEVEL(2,"Dictionary %s is too large : using last %u bytes only \n", fileName, MAX_DICT_SIZE);
+        seekResult = fseek(fileHandle, (long int)(fileSize-MAX_DICT_SIZE), SEEK_SET);   /* use end of file */
+        if (seekResult != 0) EXM_THROW(33, "Error seeking into file %s", fileName);
+        fileSize = MAX_DICT_SIZE;
+    }
+    *bufferPtr = (BYTE*)malloc((size_t)fileSize);
+    if (*bufferPtr==NULL) EXM_THROW(34, "Allocation error : not enough memory for dictBuffer");
+    readSize = fread(*bufferPtr, 1, (size_t)fileSize, fileHandle);
+    if (readSize!=fileSize) EXM_THROW(35, "Error reading dictionary file %s", fileName);
+    fclose(fileHandle);
+    return (size_t)fileSize;
+}
+
 
 /* **********************************************************************
 *  Compression
@@ -319,33 +316,7 @@ static cRess_t FIO_createCResources(const char* dictFileName)
     if (!ress.srcBuffer || !ress.dstBuffer) EXM_THROW(31, "Allocation error : not enough memory");
 
     /* dictionary */
-    ress.dictBuffer = NULL;
+    ress.dictBufferSize = FIO_loadFile(&(ress.dictBuffer), dictFileName);
-    ress.dictBufferSize = 0;
-    if (dictFileName)
-    {
-        FILE* dictHandle;
-        size_t readSize;
-        U64 dictSize;
-        DISPLAYLEVEL(4,"Using %s as dictionary \n", dictFileName);
-        dictHandle = fopen(dictFileName, "rb");
-        if (dictHandle==0) EXM_THROW(31, "Error opening dictionary file %s", dictFileName);
-        dictSize = FIO_getFileSize(dictFileName);
-        if (dictSize > MAX_DICT_SIZE)
-        {
-            int seekResult;
-            if (dictSize > 1 GB) EXM_THROW(32, "Dictionary file %s is too large", dictFileName);   /* avoid extreme cases */
-            DISPLAYLEVEL(2,"Dictionary %s is too large : using last %u bytes only \n", dictFileName, MAX_DICT_SIZE);
-            seekResult = fseek(dictHandle, (long int)(dictSize-MAX_DICT_SIZE), SEEK_SET);   /* use end of file */
-            if (seekResult != 0) EXM_THROW(33, "Error seeking into dictionary file %s", dictFileName);
-            dictSize = MAX_DICT_SIZE;
-        }
-        ress.dictBuffer = (BYTE*)malloc((size_t)dictSize);
-        if (ress.dictBuffer==NULL) EXM_THROW(34, "Allocation error : not enough memory for dictBuffer");
-        readSize = fread(ress.dictBuffer, 1, (size_t)dictSize, dictHandle);
-        if (readSize!=dictSize) EXM_THROW(35, "Error reading dictionary file %s", dictFileName);
-        fclose(dictHandle);
-        ress.dictBufferSize = (size_t)dictSize;
-    }
 
     return ress;
 }
@@ -367,7 +338,7 @@ static void FIO_freeCResources(cRess_t ress)
  *          1 : missing or pb opening srcFileName
  */
 static int FIO_compressFilename_extRess(cRess_t ress,
-                                        const char* srcFileName, const char* dstFileName,
+                                        const char* dstFileName, const char* srcFileName,
                                         int cLevel)
 {
     FILE* srcFile;
@@ -378,7 +349,7 @@ static int FIO_compressFilename_extRess(cRess_t ress,
     size_t sizeCheck, errorCode;
 
     /* File check */
-    if (FIO_getFiles(srcFileName, dstFileName, &srcFile, &dstFile)) return 1;
+    if (FIO_getFiles(&dstFile, &srcFile, dstFileName, srcFileName)) return 1;
 
     /* init */
     filesize = FIO_getFileSize(srcFileName) + dictSize;
@@ -443,125 +414,40 @@ static int FIO_compressFilename_extRess(cRess_t ress,
 }
 
 
-unsigned long long FIO_compressFilename(const char* output_filename, const char* input_filename,
+int FIO_compressFilename(const char* dstFileName, const char* srcFileName,
-                                        const char* dictFileName, int cLevel)
+                         const char* dictFileName, int compressionLevel)
 {
-    U64 filesize = 0;
+    clock_t start, end;
-    U64 compressedfilesize = 0;
+    cRess_t ress;
-    U64 dictSize = 0;
+    int issueWithSrcFile = 0;
-    BYTE* inBuff, *outBuff, *dictBuff=NULL;
-    size_t inBuffSize = ZBUFF_recommendedCInSize();
-    size_t outBuffSize = ZBUFF_recommendedCOutSize();
-    FILE* finput;
-    FILE* foutput;
-    size_t sizeCheck, errorCode;
-    ZBUFF_CCtx* ctx;
 
-    /* Allocate Memory */
+    /* Init */
-    ctx = ZBUFF_createCCtx();
+    start = clock();
-    inBuff  = (BYTE*)malloc(inBuffSize);
+    ress = FIO_createCResources(dictFileName);
-    outBuff = (BYTE*)malloc(outBuffSize);
-    if (!inBuff || !outBuff || !ctx) EXM_THROW(20, "Allocation error : not enough memory");
 
-    /* dictionary */
+    /* Compress File */
-    if (dictFileName)
+    issueWithSrcFile += FIO_compressFilename_extRess(ress, dstFileName, srcFileName, compressionLevel);
+
+    /* Free resources */
+    FIO_freeCResources(ress);
+
+    /* Final Status */
+    end = clock();
     {
-        FILE* dictHandle;
+        double seconds = (double)(end - start) / CLOCKS_PER_SEC;
-        size_t readSize;
+        DISPLAYLEVEL(4, "Completed in %.2f sec \n", seconds);
-        DISPLAYLEVEL(4,"Using %s as dictionary \n", dictFileName);
-        dictHandle = fopen(dictFileName, "rb");
-        if (dictHandle==0) EXM_THROW(21, "Error opening dictionary file %s", dictFileName);
-        dictSize = FIO_getFileSize(dictFileName);
-        if (dictSize > MAX_DICT_SIZE)
-        {
-            int seekResult;
-            if (dictSize > 1 GB) EXM_THROW(21, "Dictionary file %s is too large", dictFileName);   /* avoid extreme cases */
-            DISPLAYLEVEL(2,"Dictionary %s is too large : using last %u bytes only \n", dictFileName, MAX_DICT_SIZE);
-            seekResult = fseek(dictHandle, (long int)(dictSize-MAX_DICT_SIZE), SEEK_SET);   /* use end of file */
-            if (seekResult != 0) EXM_THROW(21, "Error seeking into dictionary file %s", dictFileName);
-            dictSize = MAX_DICT_SIZE;
-        }
-        dictBuff = (BYTE*)malloc((size_t)dictSize);
-        if (dictBuff==NULL) EXM_THROW(20, "Allocation error : not enough memory for dictBuff");
-        readSize = fread(dictBuff, 1, (size_t)dictSize, dictHandle);
-        if (readSize!=dictSize) EXM_THROW(21, "Error reading dictionary file %s", dictFileName);
-        fclose(dictHandle);
     }
 
-    /* init */
+    return issueWithSrcFile;
-    FIO_getFileHandles(&finput, &foutput, input_filename, output_filename);
-    filesize = FIO_getFileSize(input_filename) + dictSize;
-    errorCode = ZBUFF_compressInit_advanced(ctx, ZSTD_getParams(cLevel, filesize));
-    if (ZBUFF_isError(errorCode)) EXM_THROW(22, "Error initializing compression");
-    errorCode = ZBUFF_compressWithDictionary(ctx, dictBuff, (size_t)dictSize);
-    if (ZBUFF_isError(errorCode)) EXM_THROW(22, "Error initializing dictionary");
-
-    /* Main compression loop */
-    filesize = 0;
-    while (1)
-    {
-        size_t inSize;
-
-        /* Fill input Buffer */
-        inSize = fread(inBuff, (size_t)1, inBuffSize, finput);
-        if (inSize==0) break;
-        filesize += inSize;
-        DISPLAYUPDATE(2, "\rRead : %u MB  ", (U32)(filesize>>20));
-
-        {
-            /* Compress (buffered streaming ensures appropriate formatting) */
-            size_t usedInSize = inSize;
-            size_t cSize = outBuffSize;
-            size_t result = ZBUFF_compressContinue(ctx, outBuff, &cSize, inBuff, &usedInSize);
-            if (ZBUFF_isError(result))
-                EXM_THROW(23, "Compression error : %s ", ZBUFF_getErrorName(result));
-            if (inSize != usedInSize)
-                /* inBuff should be entirely consumed since buffer sizes are recommended ones */
-                EXM_THROW(24, "Compression error : input block not fully consumed");
-
-            /* Write cBlock */
-            sizeCheck = fwrite(outBuff, 1, cSize, foutput);
-            if (sizeCheck!=cSize) EXM_THROW(25, "Write error : cannot write compressed block into %s", output_filename);
-            compressedfilesize += cSize;
-        }
-
-        DISPLAYUPDATE(2, "\rRead : %u MB  ==> %.2f%%   ", (U32)(filesize>>20), (double)compressedfilesize/filesize*100);
-    }
-
-    /* End of Frame */
-    {
-        size_t cSize = outBuffSize;
-        size_t result = ZBUFF_compressEnd(ctx, outBuff, &cSize);
-        if (result!=0) EXM_THROW(26, "Compression error : cannot create frame end");
-
-        sizeCheck = fwrite(outBuff, 1, cSize, foutput);
-        if (sizeCheck!=cSize) EXM_THROW(27, "Write error : cannot write frame end into %s", output_filename);
-        compressedfilesize += cSize;
-    }
-
-    /* Status */
-    DISPLAYLEVEL(2, "\r%79s\r", "");
-    DISPLAYLEVEL(2,"Compressed %llu bytes into %llu bytes ==> %.2f%%\n",
-        (unsigned long long) filesize, (unsigned long long) compressedfilesize, (double)compressedfilesize/filesize*100);
-
-    /* clean */
-    free(inBuff);
-    free(outBuff);
-    free(dictBuff);
-    ZBUFF_freeCCtx(ctx);
-    fclose(finput);
-    if (fclose(foutput)) EXM_THROW(28, "Write error : cannot properly close %s", output_filename);
-
-    return compressedfilesize;
 }
 
 
 #define FNSPACE 30
-int FIO_compressMultipleFilenames(const char** inFileNamesTable, int ifntSize,
+int FIO_compressMultipleFilenames(const char** inFileNamesTable, unsigned nbFiles,
                                   const char* suffix,
                                   const char* dictFileName, int compressionLevel)
 {
-    int i;
+    unsigned u;
     int missed_files = 0;
     char* dstFileName = (char*)malloc(FNSPACE);
     size_t dfnSize = FNSPACE;
@@ -572,14 +458,14 @@ int FIO_compressMultipleFilenames(const char** inFileNamesTable, int ifntSize,
     ress = FIO_createCResources(dictFileName);
 
     /* loop on each file */
-    for (i=0; i<ifntSize; i++)
+    for (u=0; u<nbFiles; u++)
     {
-        size_t ifnSize = strlen(inFileNamesTable[i]);
+        size_t ifnSize = strlen(inFileNamesTable[u]);
         if (dfnSize <= ifnSize+suffixSize+1) { free(dstFileName); dfnSize = ifnSize + 20; dstFileName = (char*)malloc(dfnSize); }
-        strcpy(dstFileName, inFileNamesTable[i]);
+        strcpy(dstFileName, inFileNamesTable[u]);
         strcat(dstFileName, suffix);
 
-        missed_files += FIO_compressFilename_extRess(ress, inFileNamesTable[i], dstFileName, compressionLevel);
+        missed_files += FIO_compressFilename_extRess(ress, dstFileName, inFileNamesTable[u], compressionLevel);
     }
 
     /* Close & Free */
@@ -590,48 +476,80 @@ int FIO_compressMultipleFilenames(const char** inFileNamesTable, int ifntSize,
 }
 
 
-
-
 /* **************************************************************************
 *  Decompression
 ****************************************************************************/
+typedef struct {
+    void*  srcBuffer;
+    size_t srcBufferSize;
+    void*  dstBuffer;
+    size_t dstBufferSize;
+    void*  dictBuffer;
+    size_t dictBufferSize;
+    ZBUFF_DCtx* dctx;
+} dRess_t;
+
+static dRess_t FIO_createDResources(const char* dictFileName)
+{
+    dRess_t ress;
+
+    /* init */
+    ress.dctx = ZBUFF_createDCtx();
+    if (ress.dctx==NULL) EXM_THROW(60, "Can't create ZBUFF decompression context");
+
+    /* Allocate Memory */
+    ress.srcBufferSize = ZBUFF_recommendedDInSize();
+    ress.srcBuffer = malloc(ress.srcBufferSize);
+    ress.dstBufferSize = ZBUFF_recommendedDOutSize();
+    ress.dstBuffer = malloc(ress.dstBufferSize);
+    if (!ress.srcBuffer || !ress.dstBuffer) EXM_THROW(61, "Allocation error : not enough memory");
+
+    /* dictionary */
+    ress.dictBufferSize = FIO_loadFile(&(ress.dictBuffer), dictFileName);
+
+    return ress;
+}
+
+static void FIO_freeDResources(dRess_t ress)
+{
+    size_t errorCode = ZBUFF_freeDCtx(ress.dctx);
+    if (ZBUFF_isError(errorCode)) EXM_THROW(69, "Error : can't free ZBUFF context resource : %s", ZBUFF_getErrorName(errorCode));
+    free(ress.srcBuffer);
+    free(ress.dstBuffer);
+    free(ress.dictBuffer);
+}
 
 
-unsigned long long FIO_decompressFrame(FILE* foutput, FILE* finput,
+unsigned long long FIO_decompressFrame(dRess_t ress,
-                                       BYTE* inBuff, size_t inBuffSize, size_t alreadyLoaded,
+                                       FILE* foutput, FILE* finput, size_t alreadyLoaded)
-                                       BYTE* outBuff, size_t outBuffSize,
-                                       BYTE* dictBuff, size_t dictSize,
-                                       ZBUFF_DCtx* dctx)
 {
     U64    frameSize = 0;
     size_t readSize=alreadyLoaded;
 
     /* Main decompression Loop */
-    ZBUFF_decompressInit(dctx);
+    ZBUFF_decompressInit(ress.dctx);
-    ZBUFF_decompressWithDictionary(dctx, dictBuff, dictSize);
+    ZBUFF_decompressWithDictionary(ress.dctx, ress.dictBuffer, ress.dictBufferSize);
     while (1)
     {
         /* Decode */
         size_t sizeCheck;
-        size_t inSize=readSize, decodedSize=outBuffSize;
+        size_t inSize=readSize, decodedSize=ress.dstBufferSize;
-        size_t inStart=0;
+        size_t toRead = ZBUFF_decompressContinue(ress.dctx, ress.dstBuffer, &decodedSize, ress.srcBuffer, &inSize);
-        size_t toRead = ZBUFF_decompressContinue(dctx, outBuff, &decodedSize, inBuff+inStart, &inSize);
         if (ZBUFF_isError(toRead)) EXM_THROW(36, "Decoding error : %s", ZBUFF_getErrorName(toRead));
         readSize -= inSize;
-        inStart += inSize;
 
         /* Write block */
-        sizeCheck = fwrite(outBuff, 1, decodedSize, foutput);
+        sizeCheck = fwrite(ress.dstBuffer, 1, decodedSize, foutput);
         if (sizeCheck != decodedSize) EXM_THROW(37, "Write error : unable to write data block to destination file");
         frameSize += decodedSize;
         DISPLAYUPDATE(2, "\rDecoded : %u MB...     ", (U32)(frameSize>>20) );
 
         if (toRead == 0) break;
-        if (readSize) continue;   /* still some data left within inBuff */
+        if (readSize) EXM_THROW(38, "Decoding error : should consume entire input");
 
         /* Fill input buffer */
-        if (toRead > inBuffSize) EXM_THROW(34, "too large block");
+        if (toRead > ress.srcBufferSize) EXM_THROW(34, "too large block");
-        readSize = fread(inBuff, 1, toRead, finput);
+        readSize = fread(ress.srcBuffer, 1, toRead, finput);
         if (readSize != toRead) EXM_THROW(35, "Read error");
     }
 
@@ -639,88 +557,96 @@ unsigned long long FIO_decompressFrame(FILE* foutput, FILE* finput,
 }
 
 
-unsigned long long FIO_decompressFilename(const char* output_filename, const char* input_filename, const char* dictFileName)
+static int FIO_decompressFile_extRess(dRess_t ress,
+                                      const char* dstFileName, const char* srcFileName)
 {
-    FILE* finput, *foutput;
+    unsigned long long filesize = 0;
-    BYTE* inBuff=NULL;
+    FILE* srcFile;
-    size_t inBuffSize = ZBUFF_recommendedDInSize();
+    FILE* dstFile;
-    BYTE* outBuff=NULL;
-    size_t outBuffSize = ZBUFF_recommendedDOutSize();
-    BYTE* dictBuff=NULL;
-    size_t dictSize = 0;
-    U64   filesize = 0;
-    size_t toRead;
-
-    /* dictionary */
-    if (dictFileName)
-    {
-        FILE* dictHandle;
-        size_t readSize;
-        DISPLAYLEVEL(4,"Using %s as dictionary \n", dictFileName);
-        dictHandle = fopen(dictFileName, "rb");
-        if (dictHandle==0) EXM_THROW(21, "Error opening dictionary file %s", dictFileName);
-        dictSize = (size_t)FIO_getFileSize(dictFileName);
-        if (dictSize > MAX_DICT_SIZE)
-        {
-            int seekResult;
-            if (dictSize > 1 GB) EXM_THROW(21, "Dictionary file %s is too large", dictFileName);   /* avoid extreme cases */
-            DISPLAYLEVEL(2,"Dictionary %s is too large : using last %u bytes only \n", dictFileName, MAX_DICT_SIZE);
-            seekResult = fseek(dictHandle, (long int)(dictSize-MAX_DICT_SIZE), SEEK_SET);   /* use end of file */
-            if (seekResult != 0) EXM_THROW(21, "Error seeking into dictionary file %s", dictFileName);
-            dictSize = MAX_DICT_SIZE;
-        }
-        dictBuff = (BYTE*)malloc(dictSize);
-        if (dictBuff==NULL) EXM_THROW(20, "Allocation error : not enough memory for dictBuff");
-        readSize = fread(dictBuff, 1, (size_t)dictSize, dictHandle);
-        if (readSize!=dictSize) EXM_THROW(21, "Error reading dictionary file %s", dictFileName);
-        fclose(dictHandle);
-    }
 
     /* Init */
-    ZBUFF_DCtx* dctx = ZBUFF_createDCtx();
+    if (FIO_getFiles(&dstFile, &srcFile, dstFileName, srcFileName)) return 1;
-    FIO_getFileHandles(&finput, &foutput, input_filename, output_filename);
-
-    /* Allocate Memory (if needed) */
-    inBuff  = (BYTE*)malloc(inBuffSize);
-    outBuff  = (BYTE*)malloc(outBuffSize);
-    if (!inBuff || !outBuff) EXM_THROW(33, "Allocation error : not enough memory");
 
     /* for each frame */
     for ( ; ; )
     {
         size_t sizeCheck;
         /* check magic number -> version */
-        toRead = 4;
+        size_t toRead = 4;
-        sizeCheck = fread(inBuff, (size_t)1, toRead, finput);
+        sizeCheck = fread(ress.srcBuffer, (size_t)1, toRead, srcFile);
         if (sizeCheck==0) break;   /* no more input */
         if (sizeCheck != toRead) EXM_THROW(31, "Read error : cannot read header");
 #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1)
-        if (ZSTD_isLegacy(MEM_readLE32(inBuff)))
+        if (ZSTD_isLegacy(MEM_readLE32(ress.srcBuffer)))
         {
-            filesize += FIO_decompressLegacyFrame(foutput, finput, MEM_readLE32(inBuff));
+            filesize += FIO_decompressLegacyFrame(dstFile, srcFile, MEM_readLE32(ress.srcBuffer));
             continue;
         }
 #endif   /* ZSTD_LEGACY_SUPPORT */
 
-        filesize += FIO_decompressFrame(foutput, finput,
+        filesize += FIO_decompressFrame(ress, dstFile, srcFile, toRead);
-                                        inBuff, inBuffSize, toRead,
-                                        outBuff, outBuffSize,
-                                        dictBuff, dictSize,
-                                        dctx);
     }
 
+    /* Final Status */
     DISPLAYLEVEL(2, "\r%79s\r", "");
-    DISPLAYLEVEL(2, "Decoded %llu bytes   \n", (long long unsigned)filesize);
+    DISPLAYLEVEL(2, "Successfully decoded %llu bytes \n", filesize);
 
-    /* clean */
+    /* Close */
-    free(inBuff);
+    fclose(srcFile);
-    free(outBuff);
+    if (fclose(dstFile)) EXM_THROW(38, "Write error : cannot properly close %s", dstFileName);
-    free(dictBuff);
-    ZBUFF_freeDCtx(dctx);
-    fclose(finput);
-    if (fclose(foutput)) EXM_THROW(38, "Write error : cannot properly close %s", output_filename);
 
-    return filesize;
+    return 0;
 }
 
 
+int FIO_decompressFilename(const char* dstFileName, const char* srcFileName,
+                           const char* dictFileName)
+{
+    int missingFiles = 0;
+    dRess_t ress = FIO_createDResources(dictFileName);
+
+    missingFiles += FIO_decompressFile_extRess(ress, dstFileName, srcFileName);
+
+    FIO_freeDResources(ress);
+    return missingFiles;
+}
+
+
+#define MAXSUFFIXSIZE 8
+int FIO_decompressMultipleFilenames(const char** srcNamesTable, unsigned nbFiles,
+                                    const char* suffix,
+                                    const char* dictFileName)
+{
+    unsigned u;
+    int skippedFiles = 0;
+    int missingFiles = 0;
+    char* dstFileName = (char*)malloc(FNSPACE);
+    size_t dfnSize = FNSPACE;
+    const size_t suffixSize = strlen(suffix);
+    dRess_t ress;
+
+	if (dstFileName==NULL) EXM_THROW(70, "not enough memory for dstFileName");
+    ress = FIO_createDResources(dictFileName);
+
+    for (u=0; u<nbFiles; u++)
+    {
+        const char* srcFileName = srcNamesTable[u];
+        size_t sfnSize = strlen(srcFileName);
+        const char* suffixPtr = srcFileName + sfnSize - suffixSize;
+        if (dfnSize <= sfnSize-suffixSize+1) { free(dstFileName); dfnSize = sfnSize + 20; dstFileName = (char*)malloc(dfnSize); if (dstFileName==NULL) EXM_THROW(71, "not enough memory for dstFileName"); }
+        if (sfnSize <= suffixSize  ||  strcmp(suffixPtr, suffix) != 0)
+        {
+            DISPLAYLEVEL(1, "File extension doesn't match expected extension (%4s); will not process file: %s\n", suffix, srcFileName);
+            skippedFiles++;
+            continue;
+        }
+        memcpy(dstFileName, srcFileName, sfnSize - suffixSize);
+        dstFileName[sfnSize-suffixSize] = '\0';
+
+        missingFiles += FIO_decompressFile_extRess(ress, dstFileName, srcFileName);
+    }
+
+    FIO_freeDResources(ress);
+    free(dstFileName);
+    return missingFiles + skippedFiles;
+}

programs/fileio.h

@@ -52,26 +52,31 @@ void FIO_setNotificationLevel(unsigned level);
 /* *************************************
 *  Single File functions
 ***************************************/
-unsigned long long FIO_compressFilename (const char* outfilename, const char* infilename, const char* dictFileName, int compressionLevel);
+int FIO_compressFilename (const char* outfilename, const char* infilename, const char* dictFileName, int compressionLevel);
-unsigned long long FIO_decompressFilename (const char* outfilename, const char* infilename, const char* dictFileName);
+int FIO_decompressFilename (const char* outfilename, const char* infilename, const char* dictFileName);
 /**
 FIO_compressFilename :
-    @result : size of compressed file
+    @result : 0 == ok;  1 == pb with src file.
 
 FIO_decompressFilename :
-    @result : size of regenerated file
+    @result : 0 == ok;  1 == pb with src file.
 */
 
 
 /* *************************************
 *  Multiple File functions
 ***************************************/
-int FIO_compressMultipleFilenames(const char** inFileNamesTable, int ifntSize,
+int FIO_compressMultipleFilenames(const char** srcNamesTable, unsigned nbFiles,
                                   const char* suffix,
                                   const char* dictFileName, int compressionLevel);
+int FIO_decompressMultipleFilenames(const char** srcNamesTable, unsigned nbFiles,
+                                    const char* suffix,
+                                    const char* dictFileName);
 /**
 FIO_compressMultipleFilenames :
     @result : nb of missing files
+FIO_decompressMultipleFilenames :
+    @result : nb of missing or skipped files
 */
 
 

programs/paramgrill.c

@@ -342,6 +342,7 @@ static size_t BMK_benchParam(BMK_result_t* resultPtr,
                     blockTable[blockNb].cSize = ZSTD_compress_advanced(ctx,
                                                     blockTable[blockNb].cPtr,  blockTable[blockNb].cRoom,
                                                     blockTable[blockNb].srcPtr, blockTable[blockNb].srcSize,
+                                                    NULL, 0,
                                                     params);
                 nbLoops++;
             }

programs/playTests.sh

@@ -39,9 +39,12 @@ echo frame concatenation test completed
 
 echo "**** flush write error test **** "
 
+echo "echo foo | $ZSTD > /dev/full"
 echo foo | $ZSTD > /dev/full && die "write error not detected!"
+echo "echo foo | $ZSTD | $ZSTD -d > /dev/full"
 echo foo | $ZSTD | $ZSTD -d > /dev/full && die "write error not detected!"
 
+
 echo "*** dictionary tests *** "
 
 ./datagen > tmpDict
@@ -49,6 +52,19 @@ echo "*** dictionary tests *** "
 ./datagen -g1M | $ZSTD -D tmpDict | $ZSTD -D tmpDict -dv | md5sum > tmp2
 diff -q tmp1 tmp2
 
+echo "*** multiple files tests *** "
+
+./datagen -s1        > tmp1 2> /dev/null
+./datagen -s2 -g100K > tmp2 2> /dev/null
+./datagen -s3 -g1M   > tmp3 2> /dev/null
+$ZSTD -f -m tmp*
+ls -ls tmp*
+rm tmp1 tmp2 tmp3
+$ZSTD -df -m *.zst
+ls -ls tmp*
+$ZSTD -f -m tmp1 notHere tmp2 && die "missing file not detected!"
+rm tmp*
+
 echo "**** zstd round-trip tests **** "
 
 roundTripTest

programs/zstd.1

@@ -31,12 +31,21 @@ is equivalent to
 It is based on the \fBLZ77\fR family, with FSE & huff0 entropy stage.
 zstd offers compression speed > 200 MB/s per core.
 It also features a fast decoder, with speed > 500 MB/s per core.
+
+\fBzstd\fR command line is generally similar to gzip, but features the following differences :
+ - Original files are preserved
+ - By default, \fBzstd file1 file2\fR means : compress file1 \fBinto\fR file2.
+     Use \fB-m\fR command if you want : compress file1 into file1.zstd and file2 into file2.zst
+ - By default, when compressing files, \fBzstd\fR displays advancement notification and result summary.
+     Use \fB-q\fR to turn them off
+
+
 \fBzstd\fR supports the following options :
 
 .SH OPTIONS
 .TP
-.B \-1
+.B \-#
- fast compression (default)
+ # compression level [1-19](default:1)
 .TP
 .B \-d
  decompression
@@ -44,6 +53,14 @@ It also features a fast decoder, with speed > 500 MB/s per core.
 .B \-f
  overwrite output without prompting
 .TP
+.BR \-m ", " --multiple
+ multiple files mode
+ In this mode, multiple files on the command line means compression or decompression of each named file
+ Notifications are also turned off by default
+.TP
+.B \-D
+ Use next file as dictionary content for compress / decompression
+.TP
 .BR \-h/\-H ", " --help
  display help/long help and exit
 .TP
@@ -53,20 +70,17 @@ It also features a fast decoder, with speed > 500 MB/s per core.
 .BR \-v ", " --verbose
  verbose mode
 .TP
-.B \-q
+.BR \-q ", " --quiet
- suppress warnings; specify twice to suppress errors too
+ suppress warnings and notifications; specify twice to suppress errors too
 .TP
 .B \-c 
  force write to standard output, even if it is the console
 .TP
-.B \-t
- test compressed file integrity
-.TP
 .B \-z
  force compression
 .TP
-.B \-b
+.B \-b#
- benchmark file(s)
+ benchmark file(s) using compression level #
 .TP
 .B \-i#
  iteration loops [1-9](default : 3), benchmark mode only

programs/zstdcli.c

@@ -47,6 +47,7 @@
 #ifndef ZSTD_NOBENCH
 #  include "bench.h"  /* BMK_benchFiles, BMK_SetNbIterations */
 #endif
+#include "zstd.h"     /* ZSTD version numbers */
 
 
 /**************************************
@@ -72,7 +73,9 @@
 **************************************/
 #define COMPRESSOR_NAME "zstd command line interface"
 #ifndef ZSTD_VERSION
-#  define ZSTD_VERSION "v0.4.4"
+#  define QUOTE(str) #str
+#  define EXPAND_AND_QUOTE(str) QUOTE(str)
+#  define ZSTD_VERSION "v" EXPAND_AND_QUOTE(ZSTD_VERSION_MAJOR) "." EXPAND_AND_QUOTE(ZSTD_VERSION_MINOR) "." EXPAND_AND_QUOTE(ZSTD_VERSION_RELEASE)
 #endif
 #define AUTHOR "Yann Collet"
 #define WELCOME_MESSAGE "*** %s %i-bits %s, by %s (%s) ***\n", COMPRESSOR_NAME, (int)(sizeof(void*)*8), ZSTD_VERSION, AUTHOR, __DATE__
@@ -137,10 +140,9 @@ static int usage_advanced(const char* programName)
     DISPLAY( " -V     : display Version number and exit\n");
     DISPLAY( " -v     : verbose mode\n");
     DISPLAY( " -q     : suppress warnings; specify twice to suppress errors too\n");
-    DISPLAY( " -m     : multiple input filenames mode");
+    DISPLAY( " -m     : multiple input filenames mode \n");
     DISPLAY( " -c     : force write to standard output, even if it is the console\n");
     DISPLAY( " -D file: use file content as Dictionary \n");
-    //DISPLAY( " -t     : test compressed file integrity\n");
 #ifndef ZSTD_NOBENCH
     DISPLAY( "Benchmark arguments :\n");
     DISPLAY( " -b#    : benchmark file(s), using # compression level (default : 1) \n");
@@ -179,20 +181,20 @@ int main(int argCount, const char** argv)
         multiple=0,
         operationResult=0;
     unsigned cLevel = 1;
+    const char** filenameTable = (const char**)malloc(argCount * sizeof(const char*));   /* argCount >= 1 */
+    unsigned filenameIdx = 0;
     const char* programName = argv[0];
-    const char* inFileName = NULL;
     const char* outFileName = NULL;
     const char* dictFileName = NULL;
     char* dynNameSpace = NULL;
     const char extension[] = ZSTD_EXTENSION;
-    unsigned fileNameStart = 0;
-    unsigned nbFiles = 0;
     int rangeBench = 1;
 
     /* init */
-    (void)rangeBench; (void)nbFiles; (void)fileNameStart;   /* not used when ZSTD_NOBENCH set */
+    (void)rangeBench;   /* not used when ZSTD_NOBENCH set */
+    if (filenameTable==NULL) { DISPLAY("not enough memory\n"); exit(1); }
     displayOut = stderr;
-    /* Pick out basename component. Don't rely on stdlib because of conflicting behavior. */
+    /* Pick out program name from path. Don't rely on stdlib because of conflicting behavior */
     for (i = (int)strlen(programName); i > 0; i--) { if (programName[i] == '/') { i++; break; } }
     programName += i;
 
@@ -210,7 +212,9 @@ int main(int argCount, const char** argv)
         /* long commands (--long-word) */
         if (!strcmp(argument, "--version")) { displayOut=stdout; DISPLAY(WELCOME_MESSAGE); return 0; }
         if (!strcmp(argument, "--help")) { displayOut=stdout; return usage_advanced(programName); }
+        if (!strcmp(argument, "--multiple")) { multiple=1; continue; }
         if (!strcmp(argument, "--verbose")) { displayLevel=4; continue; }
+        if (!strcmp(argument, "--quiet")) { displayLevel--; continue; }
 
         /* Decode commands (note : aggregated commands are allowed) */
         if (argument[0]=='-')
@@ -218,9 +222,8 @@ int main(int argCount, const char** argv)
             /* '-' means stdin/stdout */
             if (argument[1]==0)
             {
-                if (!inFileName) inFileName=stdinmark;
+                if (!filenameIdx) { filenameIdx=1, filenameTable[0]=stdinmark; continue; }
-                else outFileName=stdoutmark;
+                outFileName=stdoutmark; continue;
-                continue;
             }
 
             argument++;
@@ -331,16 +334,8 @@ int main(int argCount, const char** argv)
             continue;
         }
 
-        /* first provided filename is input */
+        /* add filename to list */
-        if (!inFileName) { inFileName = argument; fileNameStart = i; nbFiles = argCount-i; continue; }
+        filenameTable[filenameIdx++] = argument;
-
-        /* second provided filename is output */
-        if (!outFileName)
-        {
-            outFileName = argument;
-            if (!strcmp (outFileName, nullString)) outFileName = nulmark;
-            continue;
-        }
     }
 
     /* Welcome message (if verbose) */
@@ -350,27 +345,28 @@ int main(int argCount, const char** argv)
     if (bench)
     {
 #ifndef ZSTD_NOBENCH
-        BMK_benchFiles(argv+fileNameStart, nbFiles, cLevel*rangeBench);
+        BMK_benchFiles(filenameTable, filenameIdx, dictFileName, cLevel*rangeBench);
 #endif
         goto _end;
     }
 
     /* No input filename ==> use stdin */
-    if(!inFileName) { inFileName=stdinmark; }
+    if(!filenameIdx) filenameIdx=1, filenameTable[0]=stdinmark;
 
     /* Check if input defined as console; trigger an error in this case */
-    if (!strcmp(inFileName, stdinmark) && IS_CONSOLE(stdin) ) return badusage(programName);
+    if (!strcmp(filenameTable[0], stdinmark) && IS_CONSOLE(stdin) ) return badusage(programName);
 
     /* No output filename ==> try to select one automatically (when possible) */
-    while (!outFileName)
+    if (filenameIdx>=2) outFileName = filenameTable[1];
+    while (!outFileName)   /* while : just to allow break statement */
     {
         if (!IS_CONSOLE(stdout)) { outFileName=stdoutmark; break; }   /* Default to stdout whenever possible (i.e. not a console) */
         if (!decode)   /* compression to file */
         {
-            size_t l = strlen(inFileName);
+            size_t l = strlen(filenameTable[0]);
             dynNameSpace = (char*)calloc(1,l+5);
             if (dynNameSpace==NULL) { DISPLAY("not enough memory\n"); exit(1); }
-            strcpy(dynNameSpace, inFileName);
+            strcpy(dynNameSpace, filenameTable[0]);
             strcpy(dynNameSpace+l, ZSTD_EXTENSION);
             outFileName = dynNameSpace;
             DISPLAYLEVEL(2, "Compressed filename will be : %s \n", outFileName);
@@ -378,8 +374,8 @@ int main(int argCount, const char** argv)
         }
         /* decompression to file (automatic name will work only if input filename has correct format extension) */
         {
-            size_t filenameSize = strlen(inFileName);
+            size_t filenameSize = strlen(filenameTable[0]);
-            if (strcmp(inFileName + (filenameSize-4), extension))
+            if (strcmp(filenameTable[0] + (filenameSize-4), extension))
             {
                  DISPLAYLEVEL(1, "unknown suffix - cannot determine destination filename\n");
                  return badusage(programName);
@@ -387,7 +383,7 @@ int main(int argCount, const char** argv)
             dynNameSpace = (char*)calloc(1,filenameSize+1);
             if (dynNameSpace==NULL) { DISPLAY("not enough memory\n"); exit(1); }
             outFileName = dynNameSpace;
-            strcpy(dynNameSpace, inFileName);
+            strcpy(dynNameSpace, filenameTable[0]);
             dynNameSpace[filenameSize-4]=0;
             DISPLAYLEVEL(2, "Decoding file %s \n", outFileName);
         }
@@ -397,23 +393,35 @@ int main(int argCount, const char** argv)
     if (!strcmp(outFileName,stdoutmark) && IS_CONSOLE(stdout) && !forceStdout) return badusage(programName);
 
     /* No warning message in pure pipe mode (stdin + stdout) or multiple mode */
-    if (!strcmp(inFileName, stdinmark) && !strcmp(outFileName,stdoutmark) && (displayLevel==2)) displayLevel=1;
+    if (!strcmp(filenameTable[0], stdinmark) && !strcmp(outFileName,stdoutmark) && (displayLevel==2)) displayLevel=1;
     if (multiple && (displayLevel==2)) displayLevel=1;
 
+    if ((!multiple) && (filenameIdx>2))
+    {
+        DISPLAY("Too many files on the command line (%u > 2). Do you mean -m ? \n", filenameIdx);
+        return filenameIdx;
+    }
+
     /* IO Stream/File */
     FIO_setNotificationLevel(displayLevel);
     if (decode)
-        FIO_decompressFilename(outFileName, inFileName, dictFileName);
+    {
+      if (multiple)
+        operationResult = FIO_decompressMultipleFilenames(filenameTable, filenameIdx, ZSTD_EXTENSION, dictFileName);
+      else
+        operationResult = FIO_decompressFilename(outFileName, filenameTable[0], dictFileName);
+    }
     else
     {
         if (multiple)
-          operationResult = FIO_compressMultipleFilenames(argv+fileNameStart, nbFiles, ZSTD_EXTENSION, dictFileName, cLevel);
+          operationResult = FIO_compressMultipleFilenames(filenameTable, filenameIdx, ZSTD_EXTENSION, dictFileName, cLevel);
         else
-          FIO_compressFilename(outFileName, inFileName, dictFileName, cLevel);
+          operationResult = FIO_compressFilename(outFileName, filenameTable[0], dictFileName, cLevel);
     }
 
 _end:
     if (main_pause) waitEnter();
     free(dynNameSpace);
+    free((void*)filenameTable);
     return operationResult;
 }