Merge pull request #26 from Cyan4973/streamAPI2

Stream api2

lib/fse.c

@@ -87,6 +87,8 @@
 #include "fse_static.h"
 
 
+#ifndef MEM_ACCESS_MODULE
+#define MEM_ACCESS_MODULE
 /****************************************************************
 *  Basic Types
 *****************************************************************/
@@ -109,6 +111,7 @@ typedef unsigned long long  U64;
 typedef   signed long long  S64;
 #endif
 
+#endif   /* MEM_ACCESS_MODULE */
 
 /****************************************************************
 *  Memory I/O
@@ -540,7 +543,6 @@ unsigned FSE_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxS
     if (tableLog==0) tableLog = FSE_DEFAULT_TABLELOG;
     if ((FSE_highbit32((U32)(srcSize - 1)) - 2) < tableLog) tableLog = FSE_highbit32((U32)(srcSize - 1)) - 2;   /* Accuracy can be reduced */
     if ((FSE_highbit32(maxSymbolValue+1)+1) > tableLog) tableLog = FSE_highbit32(maxSymbolValue+1)+1;   /* Need a minimum to safely represent all symbol values */
-    //if ((FSE_highbit32(maxSymbolValue)+2) > tableLog) tableLog = FSE_highbit32(maxSymbolValue)+2;   /* Need a minimum to safely represent all symbol values */
     if (tableLog < FSE_MIN_TABLELOG) tableLog = FSE_MIN_TABLELOG;
     if (tableLog > FSE_MAX_TABLELOG) tableLog = FSE_MAX_TABLELOG;
     return tableLog;
@@ -561,6 +563,8 @@ int FSE_compareRankT(const void* r1, const void* r2)
     return 2 * (R1->count < R2->count) - 1;
 }
 
+
+#if 0
 static size_t FSE_adjustNormSlow(short* norm, int pointsToRemove, const unsigned* count, U32 maxSymbolValue)
 {
     rank_t rank[FSE_MAX_SYMBOL_VALUE+2];
@@ -602,6 +606,100 @@ static size_t FSE_adjustNormSlow(short* norm, int pointsToRemove, const unsigned
     return 0;
 }
 
+#else
+
+/* Secondary normalization method.
+   To be used when primary method fails. */
+
+static size_t FSE_normalizeM2(short* norm, U32 tableLog, const unsigned* count, size_t total, U32 maxSymbolValue)
+{
+    U32 s;
+    U32 distributed = 0;
+    U32 ToDistribute;
+
+    /* Init */
+    U32 lowThreshold = (U32)(total >> tableLog);
+    U32 lowOne = (U32)((total * 3) >> (tableLog + 1));
+
+    for (s=0; s<=maxSymbolValue; s++)
+    {
+        if (count[s] == 0)
+        {
+            norm[s]=0;
+            continue;
+        }
+        if (count[s] <= lowThreshold)
+        {
+            norm[s] = -1;
+            distributed++;
+            total -= count[s];
+            continue;
+        }
+        if (count[s] <= lowOne)
+        {
+            norm[s] = 1;
+            distributed++;
+            total -= count[s];
+            continue;
+        }
+        norm[s]=-2;
+    }
+    ToDistribute = (1 << tableLog) - distributed;
+
+    if ((total / ToDistribute) > lowOne)
+    {
+        /* risk of rounding to zero */
+        lowOne = (U32)((total * 3) / (ToDistribute * 2));
+        for (s=0; s<=maxSymbolValue; s++)
+        {
+            if ((norm[s] == -2) && (count[s] <= lowOne))
+            {
+                norm[s] = 1;
+                distributed++;
+                total -= count[s];
+                continue;
+            }
+        }
+        ToDistribute = (1 << tableLog) - distributed;
+    }
+
+    if (distributed == maxSymbolValue+1)
+    {
+        /* all values are pretty poor;
+           probably incompressible data (should have already been detected);
+           find max, then give all remaining points to max */
+        U32 maxV = 0, maxC =0;
+        for (s=0; s<=maxSymbolValue; s++)
+            if (count[s] > maxC) maxV=s, maxC=count[s];
+        norm[maxV] += ToDistribute;
+        return 0;
+    }
+
+    {
+        U64 const vStepLog = 62 - tableLog;
+        U64 const mid = (1ULL << (vStepLog-1)) - 1;
+        U64 const rStep = ((((U64)1<<vStepLog) * ToDistribute) + mid) / total;   /* scale on remaining */
+        U64 tmpTotal = mid;
+        for (s=0; s<=maxSymbolValue; s++)
+        {
+            if (norm[s]==-2)
+            {
+                U64 end = tmpTotal + (count[s] * rStep);
+                U32 sStart = (U32)(tmpTotal >> vStepLog);
+                U32 sEnd = (U32)(end >> vStepLog);
+                U32 weight = sEnd - sStart;
+                if (weight < 1)
+                    return (size_t)-FSE_ERROR_GENERIC;
+                norm[s] = weight;
+                tmpTotal = end;
+            }
+        }
+    }
+
+    return 0;
+}
+#endif
+
 
 size_t FSE_normalizeCount (short* normalizedCounter, unsigned tableLog,
                            const unsigned* count, size_t total,
@@ -656,10 +754,9 @@ size_t FSE_normalizeCount (short* normalizedCounter, unsigned tableLog,
         }
         if (-stillToDistribute >= (normalizedCounter[largest] >> 1))
         {
-            /* corner case, need to converge towards normalization with caution */
+            /* corner case, need another normalization method */
-            size_t errorCode = FSE_adjustNormSlow(normalizedCounter, -stillToDistribute, count, maxSymbolValue);
+            size_t errorCode = FSE_normalizeM2(normalizedCounter, tableLog, count, total, maxSymbolValue);
             if (FSE_isError(errorCode)) return errorCode;
-            //FSE_adjustNormSlow(normalizedCounter, -stillToDistribute, count, maxSymbolValue);
         }
         else normalizedCounter[largest] += (short)stillToDistribute;
     }
@@ -869,12 +966,6 @@ size_t FSE_compress_usingCTable (void* dst, size_t dstSize,
 }
 
 
-static size_t FSE_compressRLE (BYTE *out, BYTE symbol)
-{
-    *out=symbol;
-    return 1;
-}
-
 size_t FSE_compressBound(size_t size) { return FSE_COMPRESSBOUND(size); }
 
 
@@ -901,8 +992,8 @@ size_t FSE_compress2 (void* dst, size_t dstSize, const void* src, size_t srcSize
     /* Scan input and build symbol stats */
     errorCode = FSE_count (count, ip, srcSize, &maxSymbolValue);
     if (FSE_isError(errorCode)) return errorCode;
-    if (errorCode == srcSize) return FSE_compressRLE (ostart, *istart);
+    if (errorCode == srcSize) return 1;
-    if (errorCode < ((srcSize * 7) >> 10)) return 0;   /* Heuristic : not compressible enough */
+    if (errorCode < (srcSize >> 7)) return 0;   /* Heuristic : not compressible enough */
 
     tableLog = FSE_optimalTableLog(tableLog, srcSize, maxSymbolValue);
     errorCode = FSE_normalizeCount (norm, tableLog, count, srcSize, maxSymbolValue);

lib/zstd.c

@@ -68,9 +68,6 @@
 #include <stdio.h>       /* debug : printf */
 #include "zstd_static.h"
 #if defined(__clang__) || defined(__GNUC__)
-#  ifdef __clang__
-#    pragma clang diagnostic ignored "-Wtypedef-redefinition"
-#  endif
 #  include "fse.c"       /* due to GCC/Clang inlining limitations, including *.c runs noticeably faster */
 #else
 #  include "fse_static.h"
@@ -99,6 +96,8 @@
 #endif
 
 
+#ifndef MEM_ACCESS_MODULE
+#define MEM_ACCESS_MODULE
 /********************************************************
 *  Basic Types
 *********************************************************/
@@ -119,6 +118,8 @@ typedef   signed int        S32;
 typedef unsigned long long  U64;
 #endif
 
+#endif   /* MEM_ACCESS_MODULE */
+
 
 /********************************************************
 *  Constants
@@ -129,9 +130,6 @@ static const U32 ZSTD_magicNumber = 0xFD2FB51C;   /* Initial (limited) frame for
 #define HASH_TABLESIZE (1 << HASH_LOG)
 #define HASH_MASK (HASH_TABLESIZE - 1)
 
-#define MAXD_LOG 16
-#define MAX_DISTANCE ((1 << MAXD_LOG) - 1)
-
 #define KNUTH 2654435761
 
 #define BIT7 128
@@ -141,10 +139,10 @@ static const U32 ZSTD_magicNumber = 0xFD2FB51C;   /* Initial (limited) frame for
 
 #define KB *(1 <<10)
 #define MB *(1 <<20)
-#define GB *(1U<<20)
+#define GB *(1U<<30)
 
 #define BLOCKSIZE (128 KB)                 /* define, for static allocation */
-static const U32 g_maxDistance = 512 KB;
+static const U32 g_maxDistance = 4 * BLOCKSIZE;
 static const U32 g_maxLimit = 1 GB;
 static const U32 g_searchStrength = 8;
 
@@ -616,25 +614,27 @@ static size_t ZSTD_compressLiterals (void* dst, size_t dstSize,
     size_t errorCode;
     const size_t minGain = ZSTD_minGain(srcSize);
 
-    // early out
+    /* early out */
     if (dstSize < FSE_compressBound(srcSize)) return (size_t)-ZSTD_ERROR_maxDstSize_tooSmall;
 
-    // Scan input and build symbol stats
+    /* Scan input and build symbol stats */
     errorCode = FSE_count (count, ip, srcSize, &maxSymbolValue);
     if (FSE_isError(errorCode)) return (size_t)-ZSTD_ERROR_GENERIC;
     if (errorCode == srcSize) return 1;
-    if (errorCode < ((srcSize * 7) >> 10)) return 0;
+    //if (errorCode < ((srcSize * 7) >> 10)) return 0;
+    //if (errorCode < (srcSize >> 7)) return 0;
+    if (errorCode < (srcSize >> 6)) return 0;   /* heuristic : probably not compressible enough */
 
     tableLog = FSE_optimalTableLog(tableLog, srcSize, maxSymbolValue);
     errorCode = (int)FSE_normalizeCount (norm, tableLog, count, srcSize, maxSymbolValue);
     if (FSE_isError(errorCode)) return (size_t)-ZSTD_ERROR_GENERIC;
 
-    // Write table description header
+    /* Write table description header */
     errorCode = FSE_writeHeader (op, FSE_MAX_HEADERSIZE, norm, maxSymbolValue, tableLog);
     if (FSE_isError(errorCode)) return (size_t)-ZSTD_ERROR_GENERIC;
     op += errorCode;
 
-    // Compress
+    /* Compress */
     errorCode = FSE_buildCTable (&CTable, norm, maxSymbolValue, tableLog);
     if (FSE_isError(errorCode)) return (size_t)-ZSTD_ERROR_GENERIC;
     errorCode = ZSTD_compressLiterals_usingCTable(op, oend - op, ip, srcSize, &CTable);
@@ -1708,24 +1708,24 @@ size_t ZSTD_decompress(void* dst, size_t maxDstSize, const void* src, size_t src
 }
 
 
-/******************************
+/*******************************
 *  Streaming Decompression API
-******************************/
+*******************************/
 
 typedef struct
 {
     U32 ctx[FSE_DTABLE_SIZE_U32(LLFSELog) + FSE_DTABLE_SIZE_U32(OffFSELog) + FSE_DTABLE_SIZE_U32(MLFSELog)];
     size_t expected;
     blockType_t bType;
-    U32 started;
+    U32 phase;
 } dctx_t;
 
 
 ZSTD_dctx_t ZSTD_createDCtx(void)
 {
     dctx_t* dctx = (dctx_t*)malloc(sizeof(dctx_t));
-    dctx->expected = 4 + ZSTD_blockHeaderSize;   // Frame Header + Block Header
+    dctx->expected = ZSTD_frameHeaderSize;
-    dctx->started = 0;
+    dctx->phase = 0;
     return (ZSTD_dctx_t)dctx;
 }
 
@@ -1736,7 +1736,7 @@ size_t ZSTD_freeDCtx(ZSTD_dctx_t dctx)
 }
 
 
-size_t ZSTD_getNextcBlockSize(ZSTD_dctx_t dctx)
+size_t ZSTD_nextSrcSizeToDecompress(ZSTD_dctx_t dctx)
 {
     return ((dctx_t*)dctx)->expected;
 }
@@ -1744,63 +1744,67 @@ size_t ZSTD_getNextcBlockSize(ZSTD_dctx_t dctx)
 size_t ZSTD_decompressContinue(ZSTD_dctx_t dctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize)
 {
     dctx_t* ctx = (dctx_t*)dctx;
-    size_t cSize = srcSize - ZSTD_blockHeaderSize;
-    size_t rSize;
 
-    // Sanity check
+    /* Sanity check */
     if (srcSize != ctx->expected) return (size_t)-ZSTD_ERROR_wrongSrcSize;
 
-    // Decompress
+    /* Decompress : frame header */
-    if (!ctx->started)
+    if (ctx->phase == 0)
     {
-        // Just check correct magic header
+        /* Check frame magic header */
         U32 magicNumber = ZSTD_readBE32(src);
         if (magicNumber != ZSTD_magicNumber) return (size_t)-ZSTD_ERROR_wrongMagicNumber;
-        rSize = 0;
+        ctx->phase = 1;
+        ctx->expected = ZSTD_blockHeaderSize;
+        return 0;
     }
-    else
+
+    /* Decompress : block header */
+    if (ctx->phase == 1)
     {
+        blockProperties_t bp;
+        size_t blockSize = ZSTD_getcBlockSize(src, ZSTD_blockHeaderSize, &bp);
+        if (ZSTD_isError(blockSize)) return blockSize;
+        if (bp.blockType == bt_end)
+        {
+            ctx->expected = 0;
+            ctx->phase = 0;
+        }
+        else
+        {
+            ctx->expected = blockSize;
+            ctx->bType = bp.blockType;
+            ctx->phase = 2;
+        }
+
+        return 0;
+    }
+
+    /* Decompress : block content */
+    {
+        size_t rSize;
         switch(ctx->bType)
         {
         case bt_compressed:
-            rSize = ZSTD_decompressBlock(ctx, dst, maxDstSize, src, cSize);
+            rSize = ZSTD_decompressBlock(ctx, dst, maxDstSize, src, srcSize);
             break;
         case bt_raw :
-            rSize = ZSTD_copyUncompressedBlock(dst, maxDstSize, src, cSize);
+            rSize = ZSTD_copyUncompressedBlock(dst, maxDstSize, src, srcSize);
             break;
         case bt_rle :
             return (size_t)-ZSTD_ERROR_GENERIC;   /* not yet handled */
             break;
-        case bt_end :
+        case bt_end :   /* should never happen (filtered at phase 1) */
             rSize = 0;
             break;
         default:
             return (size_t)-ZSTD_ERROR_GENERIC;
         }
+        ctx->phase = 1;
+        ctx->expected = ZSTD_blockHeaderSize;
+        return rSize;
     }
 
-    // Prepare next block
-    {
-        const BYTE* header = (const BYTE*)src;
-        blockProperties_t bp;
-        size_t blockSize;
-        header += cSize;
-        blockSize = ZSTD_getcBlockSize(header, ZSTD_blockHeaderSize, &bp);
-        if (ZSTD_isError(blockSize)) return blockSize;
-        if (bp.blockType == bt_end)
-        {
-            ctx->expected = 0;
-            ctx->started = 0;
-        }
-        else
-        {
-            ctx->expected = blockSize + ZSTD_blockHeaderSize;
-            ctx->bType = bp.blockType;
-            ctx->started = 1;
-        }
-    }
-
-    return rSize;
 }
 
 

lib/zstd.h

@@ -47,7 +47,7 @@ extern "C" {
 **************************************/
 #define ZSTD_VERSION_MAJOR    0    /* for breaking interface changes  */
 #define ZSTD_VERSION_MINOR    0    /* for new (non-breaking) interface capabilities */
-#define ZSTD_VERSION_RELEASE  1    /* for tweaks, bug-fixes, or development */
+#define ZSTD_VERSION_RELEASE  2    /* for tweaks, bug-fixes, or development */
 #define ZSTD_VERSION_NUMBER (ZSTD_VERSION_MAJOR *100*100 + ZSTD_VERSION_MINOR *100 + ZSTD_VERSION_RELEASE)
 unsigned ZSTD_versionNumber (void);
 

lib/zstd_static.h

@@ -57,9 +57,16 @@ typedef void* ZSTD_dctx_t;
 ZSTD_dctx_t ZSTD_createDCtx(void);
 size_t      ZSTD_freeDCtx(ZSTD_dctx_t dctx);
 
-size_t ZSTD_getNextcBlockSize(ZSTD_dctx_t dctx);
+size_t ZSTD_nextSrcSizeToDecompress(ZSTD_dctx_t dctx);
 size_t ZSTD_decompressContinue(ZSTD_dctx_t dctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize);
-
+/*
+  Use above functions alternatively.
+  ZSTD_nextSrcSizeToDecompress() tells how much bytes to provide as input to ZSTD_decompressContinue().
+  This value is expected to be provided, precisely, as 'srcSize'.
+  Otherwise, compression will fail (result is an error code, which can be tested using ZSTD_isError() )
+  ZSTD_decompressContinue() result is the number of bytes regenerated within 'dst'.
+  It can be zero, which is not an error; it just means ZSTD_decompressContinue() has decoded some header.
+*/
 
 /**************************************
 *  Error management

programs/fileio.c

@@ -327,10 +327,10 @@ unsigned long long FIO_decompressFilename(const char* output_filename, const cha
     /* Init */
     FIO_getFileHandles(&finput, &foutput, input_filename, output_filename);
     dctx = ZSTD_createDCtx();
-    toRead = ZSTD_getNextcBlockSize(dctx);
-    if (toRead > MAXHEADERSIZE) EXM_THROW(30, "Not enough memory to read header");
 
     /* check header */
+    toRead = ZSTD_nextSrcSizeToDecompress(dctx);
+    if (toRead > MAXHEADERSIZE) EXM_THROW(30, "Not enough memory to read header");
     sizeCheck = fread(header, (size_t)1, toRead, finput);
     if (sizeCheck != toRead) EXM_THROW(31, "Read error : cannot read header");
     sizeCheck = ZSTD_decompressContinue(dctx, NULL, 0, header, toRead);   // Decode frame header
@@ -348,7 +348,7 @@ unsigned long long FIO_decompressFilename(const char* output_filename, const cha
     if (!inBuff || !outBuff) EXM_THROW(33, "Allocation error : not enough memory");
 
     /* Main decompression Loop */
-    toRead = ZSTD_getNextcBlockSize(dctx);
+    toRead = ZSTD_nextSrcSizeToDecompress(dctx);
     while (toRead)
     {
         size_t readSize, decodedSize;
@@ -361,16 +361,19 @@ unsigned long long FIO_decompressFilename(const char* output_filename, const cha
         /* Decode block */
         decodedSize = ZSTD_decompressContinue(dctx, op, oend-op, inBuff, readSize);
 
-        /* Write block */
+        if (decodedSize)   /* not a header */
-        sizeCheck = fwrite(op, 1, decodedSize, foutput);
+        {
-        if (sizeCheck != decodedSize) EXM_THROW(35, "Write error : unable to write data block to destination file");
+            /* Write block */
-        filesize += decodedSize;
+            sizeCheck = fwrite(op, 1, decodedSize, foutput);
+            if (sizeCheck != decodedSize) EXM_THROW(35, "Write error : unable to write data block to destination file");
+            filesize += decodedSize;
+            op += decodedSize;
+            if (op==oend) op = outBuff;
+            DISPLAYUPDATE(2, "\rDecoded : %u MB...     ", (U32)(filesize>>20) );
+        }
 
         /* prepare for next Block */
-        op += decodedSize;
+        toRead = ZSTD_nextSrcSizeToDecompress(dctx);
-        if (op==oend) op = outBuff;
-        toRead = ZSTD_getNextcBlockSize(dctx);
-        DISPLAYUPDATE(2, "\rDecoded : %u MB...     ", (U32)(filesize>>20) );
     }
 
     DISPLAYLEVEL(2, "\r%79s\r", "");