first version supporting legacy streams (transparent decoding)

lib/common/error_private.h

@@ -93,6 +93,7 @@ ERR_STATIC const char* ERR_getErrorString(ERR_enum code)
     case PREFIX(no_error): return "No error detected";
     case PREFIX(GENERIC):  return "Error (generic)";
     case PREFIX(prefix_unknown): return "Unknown frame descriptor";
+    case PREFIX(version_unsupported): return "Version not supported";
     case PREFIX(parameter_unknown): return "Unknown parameter type";
     case PREFIX(frameParameter_unsupported): return "Unsupported frame parameter";
     case PREFIX(frameParameter_unsupportedBy32bits): return "Frame parameter unsupported in 32-bits mode";

lib/common/error_public.h

@@ -41,13 +41,14 @@ extern "C" {
 #include <stddef.h>   /* size_t */
 
 
-/* ****************************************
+/*-****************************************
 *  error codes list
 ******************************************/
 typedef enum {
   ZSTD_error_no_error,
   ZSTD_error_GENERIC,
   ZSTD_error_prefix_unknown,
+  ZSTD_error_version_unsupported,
   ZSTD_error_parameter_unknown,
   ZSTD_error_frameParameter_unsupported,
   ZSTD_error_frameParameter_unsupportedBy32bits,

lib/decompress/zstd_decompress.c

@@ -1308,8 +1308,8 @@ ZSTDLIB_API size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx,
     if (ZSTD_isLegacy(src, srcSize)) return ZSTD_decompressLegacy(dst, dstCapacity, src, srcSize, ddict->dict, ddict->dictSize);
 #endif
     return ZSTD_decompress_usingPreparedDCtx(dctx, ddict->refContext,
-                                           dst, dstCapacity,
+                                             dst, dstCapacity,
-                                           src, srcSize);
+                                             src, srcSize);
 }
 
 
@@ -1337,6 +1337,11 @@ struct ZSTD_DStream_s {
     BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX];
     size_t lhSize;
     ZSTD_customMem customMem;
+    void* dictContent;
+    size_t dictSize;
+    const void* dictSource;
+    void* legacyContext;
+    U32 legacyVersion;
 };   /* typedef'd to ZSTD_DStream within "zstd.h" */
 
 
@@ -1372,6 +1377,13 @@ size_t ZSTD_freeDStream(ZSTD_DStream* zds)
     ZSTD_freeDCtx(zds->zd);
     if (zds->inBuff) zds->customMem.customFree(zds->customMem.opaque, zds->inBuff);
     if (zds->outBuff) zds->customMem.customFree(zds->customMem.opaque, zds->outBuff);
+    if (zds->dictContent) zds->customMem.customFree(zds->customMem.opaque, zds->dictContent);
+#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1)
+    if (zds->legacyContext) {
+        ZSTD_freeLegacyStreamContext(zds->legacyContext, zds->legacyVersion);
+        zds->legacyContext = NULL;
+    }
+#endif
     zds->customMem.customFree(zds->customMem.opaque, zds);
     return 0;
 }
@@ -1386,7 +1398,18 @@ size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t di
 {
     zds->stage = zdss_loadHeader;
     zds->lhSize = zds->inPos = zds->outStart = zds->outEnd = 0;
-    return ZSTD_decompressBegin_usingDict(zds->zd, dict, dictSize);
+    if ((dict != zds->dictSource) || (dictSize != zds->dictSize)) {   /* new dictionary */
+        if (dictSize > zds->dictSize) {
+            if (zds->dictContent) zds->customMem.customFree(zds->customMem.opaque, zds->dictContent);
+            zds->dictContent = zds->customMem.customAlloc(zds->customMem.opaque, dictSize);
+            if (zds->dictContent == NULL) return ERROR(memory_allocation);
+        }
+        memcpy(zds->dictContent, dict, dictSize);
+        zds->dictSize = dictSize;
+    }
+    if (zds->legacyContext) zds->customMem.customFree(zds->customMem.opaque, zds->dictContent);   /* legacy restarts from scratch on each call, to detect restart */
+    zds->legacyVersion = 0;
+    return 0;
 }
 
 size_t ZSTD_initDStream(ZSTD_DStream* zds)
@@ -1432,6 +1455,11 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
     char* op = ostart;
     U32 someMoreWork = 1;
 
+#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1)
+    if (zds->legacyVersion)
+        return ZSTD_decompressLegacyStream(&zds->legacyContext, zds->legacyVersion, output, input);
+#endif
+
     while (someMoreWork) {
         switch(zds->stage)
         {
@@ -1439,8 +1467,19 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
             return ERROR(init_missing);
 
         case zdss_loadHeader :
-            {   size_t const hSize = ZSTD_getFrameParams(&(zds->fParams), zds->headerBuffer, zds->lhSize);
+            {   size_t const hSize = ZSTD_getFrameParams(&zds->fParams, zds->headerBuffer, zds->lhSize);
-                if (ZSTD_isError(hSize)) return hSize;
+                if (ZSTD_isError(hSize))
+#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1)
+                {   U32 const legacyVersion = ZSTD_isLegacy(istart, iend-istart)
+                    if (legacyVersion) {
+                        zds->legacyVersion = legacyVersion;
+                        return ZSTD_decompressLegacyStream(&zds->legacyContext, zds->legacyVersion, output, input);
+                    } else {
+                        return hSize; /* error */
+                }   }
+#else
+                    return hSize;
+#endif
                 if (hSize != 0) {   /* need more input */
                     size_t const toLoad = hSize - zds->lhSize;   /* if hSize!=0, hSize > zds->lhSize */
                     if (toLoad > (size_t)(iend-ip)) {   /* not enough input to load full header */
@@ -1454,6 +1493,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
             }   }
 
             /* Consume header */
+            ZSTD_decompressBegin_usingDict(zds->zd, zds->dictContent, zds->dictSize);
             {   size_t const h1Size = ZSTD_nextSrcSizeToDecompress(zds->zd);  /* == ZSTD_frameHeaderSize_min */
                 size_t const h1Result = ZSTD_decompressContinue(zds->zd, NULL, 0, zds->headerBuffer, h1Size);
                 if (ZSTD_isError(h1Result)) return h1Result;   /* should not happen : already checked */

lib/legacy/zstd_legacy.h

@@ -42,6 +42,7 @@ extern "C" {
 ***************************************/
 #include "mem.h"            /* MEM_STATIC */
 #include "error_private.h"  /* ERROR */
+#include "zstd.h"           /* ZSTD_inBuffer, ZSTD_outBuffer */
 #include "zstd_v01.h"
 #include "zstd_v02.h"
 #include "zstd_v03.h"
@@ -76,32 +77,30 @@ MEM_STATIC unsigned ZSTD_isLegacy(const void* src, size_t srcSize)
 
 MEM_STATIC unsigned long long ZSTD_getDecompressedSize_legacy(const void* src, size_t srcSize)
 {
-    if (srcSize < 4) return 0;
+    U32 const version = ZSTD_isLegacy(src, srcSize);
-
+    if (version < 5) return 0;  /* no decompressed size in frame header, or not a legacy format */
-    {   U32 const version = ZSTD_isLegacy(src, srcSize);
+    if (version==5) {
-        if (version < 5) return 0;  /* no decompressed size in frame header, or not a legacy format */
+        ZSTDv05_parameters fParams;
-        if (version==5) {
+        size_t const frResult = ZSTDv05_getFrameParams(&fParams, src, srcSize);
-            ZSTDv05_parameters fParams;
+        if (frResult != 0) return 0;
-            size_t const frResult = ZSTDv05_getFrameParams(&fParams, src, srcSize);
+        return fParams.srcSize;
-            if (frResult != 0) return 0;
-            return fParams.srcSize;
-        }
-        if (version==6) {
-            ZSTDv06_frameParams fParams;
-            size_t const frResult = ZSTDv06_getFrameParams(&fParams, src, srcSize);
-            if (frResult != 0) return 0;
-            return fParams.frameContentSize;
-        }
-        if (version==7) {
-            ZSTDv07_frameParams fParams;
-            size_t const frResult = ZSTDv07_getFrameParams(&fParams, src, srcSize);
-            if (frResult != 0) return 0;
-            return fParams.frameContentSize;
-        }
-        return 0;   /* should not be possible */
     }
+    if (version==6) {
+        ZSTDv06_frameParams fParams;
+        size_t const frResult = ZSTDv06_getFrameParams(&fParams, src, srcSize);
+        if (frResult != 0) return 0;
+        return fParams.frameContentSize;
+    }
+    if (version==7) {
+        ZSTDv07_frameParams fParams;
+        size_t const frResult = ZSTDv07_getFrameParams(&fParams, src, srcSize);
+        if (frResult != 0) return 0;
+        return fParams.frameContentSize;
+    }
+    return 0;   /* should not be possible */
 }
 
+
 MEM_STATIC size_t ZSTD_decompressLegacy(
                      void* dst, size_t dstCapacity,
                const void* src, size_t compressedSize,
@@ -148,6 +147,147 @@ MEM_STATIC size_t ZSTD_decompressLegacy(
 }
 
 
+MEM_STATIC void* ZSTD_createLegacyStreamContext(U32 version)
+{
+    switch(version)
+    {
+        default :
+        case 1 :
+        case 2 :
+        case 3 :
+            return NULL;
+        case 4 : return ZBUFFv04_createDCtx();
+        case 5 : return ZBUFFv05_createDCtx();
+        case 6 : return ZBUFFv06_createDCtx();
+        case 7 : return ZBUFFv07_createDCtx();
+    }
+}
+
+MEM_STATIC size_t ZSTD_freeLegacyStreamContext(void* legacyContext, U32 version)
+{
+    switch(version)
+    {
+        default :
+        case 1 :
+        case 2 :
+        case 3 :
+            return ERROR(version_unsupported);
+        case 4 : return ZBUFFv04_freeDCtx((ZBUFFv04_DCtx*)legacyContext);
+        case 5 : return ZBUFFv05_freeDCtx((ZBUFFv05_DCtx*)legacyContext);
+        case 6 : return ZBUFFv06_freeDCtx((ZBUFFv06_DCtx*)legacyContext);
+        case 7 : return ZBUFFv07_freeDCtx((ZBUFFv07_DCtx*)legacyContext);
+    }
+}
+
+
+MEM_STATIC void ZSTD_initLegacyStream(void* legacyContext, U32 version,
+                                      const void* dict, size_t dictSize)
+{
+    switch(version)
+    {
+        default :
+        case 1 :
+        case 2 :
+        case 3 :
+            return;
+        case 4 :
+        {
+            ZBUFFv04_DCtx* dctx = (ZBUFFv04_DCtx*) legacyContext;
+            ZBUFFv04_decompressInit(dctx);
+            ZBUFFv04_decompressWithDictionary(dctx, dict, dictSize);
+            return;
+        }
+        case 5 :
+        {
+            ZBUFFv05_DCtx* dctx = (ZBUFFv05_DCtx*) legacyContext;
+            ZBUFFv05_decompressInitDictionary(dctx, dict, dictSize);
+            return;
+        }
+        case 6 :
+        {
+            ZBUFFv06_DCtx* dctx = (ZBUFFv06_DCtx*) legacyContext;
+            ZBUFFv06_decompressInitDictionary(dctx, dict, dictSize);
+            return;
+        }
+        case 7 :
+        {
+            ZBUFFv07_DCtx* dctx = (ZBUFFv07_DCtx*) legacyContext;
+            ZBUFFv07_decompressInitDictionary(dctx, dict, dictSize);
+            return;
+        }
+    }
+}
+
+
+
+MEM_STATIC size_t ZSTD_decompressLegacyStream(void** legacyContext, U32 version,
+                                              ZSTD_outBuffer* output, ZSTD_inBuffer* input,
+                                              const void* dict, size_t dictSize)
+{
+    if (*legacyContext == NULL) {
+        *legacyContext = ZSTD_createLegacyStreamContext(version);
+        if (*legacyContext==NULL) return ERROR(memory_allocation);
+        ZSTD_initLegacyStream(*legacyContext, version, dict, dictSize);
+    }
+
+    switch(version)
+    {
+        default :
+        case 1 :
+        case 2 :
+        case 3 :
+            return ERROR(version_unsupported);
+        case 4 :
+            {
+                ZBUFFv04_DCtx* dctx = (ZBUFFv04_DCtx*) (*legacyContext);
+                const void* src = (const char*)input->src + input->pos;
+                size_t readSize = input->size - input->pos;
+                void* dst = (char*)output->dst + output->pos;
+                size_t decodedSize = output->size - output->pos;
+                size_t const hintSize = ZBUFFv04_decompressContinue(dctx, dst, &decodedSize, src, &readSize);
+                output->pos += decodedSize;
+                input->pos += readSize;
+                return hintSize;
+            }
+        case 5 :
+            {
+                ZBUFFv05_DCtx* dctx = (ZBUFFv05_DCtx*) (*legacyContext);
+                const void* src = (const char*)input->src + input->pos;
+                size_t readSize = input->size - input->pos;
+                void* dst = (char*)output->dst + output->pos;
+                size_t decodedSize = output->size - output->pos;
+                size_t const hintSize = ZBUFFv05_decompressContinue(dctx, dst, &decodedSize, src, &readSize);
+                output->pos += decodedSize;
+                input->pos += readSize;
+                return hintSize;
+            }
+        case 6 :
+            {
+                ZBUFFv06_DCtx* dctx = (ZBUFFv06_DCtx*) (*legacyContext);
+                const void* src = (const char*)input->src + input->pos;
+                size_t readSize = input->size - input->pos;
+                void* dst = (char*)output->dst + output->pos;
+                size_t decodedSize = output->size - output->pos;
+                size_t const hintSize = ZBUFFv06_decompressContinue(dctx, dst, &decodedSize, src, &readSize);
+                output->pos += decodedSize;
+                input->pos += readSize;
+                return hintSize;
+            }
+        case 7 :
+            {
+                ZBUFFv07_DCtx* dctx = (ZBUFFv07_DCtx*) (*legacyContext);
+                const void* src = (const char*)input->src + input->pos;
+                size_t readSize = input->size - input->pos;
+                void* dst = (char*)output->dst + output->pos;
+                size_t decodedSize = output->size - output->pos;
+                size_t const hintSize = ZBUFFv07_decompressContinue(dctx, dst, &decodedSize, src, &readSize);
+                output->pos += decodedSize;
+                input->pos += readSize;
+                return hintSize;
+            }
+    }
+}
+
 
 #if defined (__cplusplus)
 }

lib/zstd.h

@@ -208,7 +208,7 @@ typedef struct ZSTD_outBuffer_s {
 } ZSTD_outBuffer;
 
 
-/*======   compression   ======*/
+/*======   streaming compression   ======*/
 
 /*-***********************************************************************
 *  Streaming compression - howto
@@ -225,7 +225,9 @@ typedef struct ZSTD_outBuffer_s {
 *  The function will automatically update both `pos`.
 *  Note that it may not consume the entire input, in which case `pos < size`,
 *  and it's up to the caller to present again remaining data.
-*  @return : a hint to preferred nb of bytes to use as input for next function call (it's just a hint, to improve latency)
+*  @return : a size hint, preferred nb of bytes to use as input for next function call
+*           (it's just a hint, to help latency a little, any other value will work fine)
+*           (note : the size hint is guaranteed to be <= ZSTD_CStreamInSize() )
 *            or an error code, which can be tested using ZSTD_isError().
 *
 *  At any moment, it's possible to flush whatever data remains within buffer, using ZSTD_flushStream().