Merge pull request #645 from terrelln/cdict

Fix ZSTD_initCStream_usingCDict() and ZSTD_compress_usingCDict()

lib/compress/zstd_compress.c

@@ -2921,6 +2921,8 @@ size_t ZSTD_compress_usingCDict(ZSTD_CCtx* cctx,
     if (cdict->refContext->params.fParams.contentSizeFlag==1) {
         cctx->params.fParams.contentSizeFlag = 1;
         cctx->frameContentSize = srcSize;
+    } else {
+        cctx->params.fParams.contentSizeFlag = 0;
     }
 
     return ZSTD_compressEnd(cctx, dst, dstCapacity, src, srcSize);
@@ -3067,8 +3069,8 @@ size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict)
     ZSTD_parameters const params = ZSTD_getParamsFromCDict(cdict);
     size_t const initError =  ZSTD_initCStream_advanced(zcs, NULL, 0, params, 0);
     zcs->cdict = cdict;
-    zcs->cctx->dictID = params.fParams.noDictIDFlag ? 0 : cdict->refContext->dictID;
+    if (ZSTD_isError(initError)) return initError;
-    return initError;
+    return ZSTD_resetCStream_internal(zcs, 0);
 }
 
 size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel)

tests/fuzzer.c

@@ -404,6 +404,39 @@ static int basicUnitTests(U32 seed, double compressibility)
                   if (r != CNBuffSize) goto _output_error);
         DISPLAYLEVEL(4, "OK \n");
 
+        DISPLAYLEVEL(4, "test%3i : compress with preprocessed dictionary : ", testNb++);
+        {   ZSTD_parameters params = ZSTD_getParams(1, CNBuffSize, dictSize);
+            params.fParams.contentSizeFlag = 0;
+            {   ZSTD_customMem customMem = { NULL, NULL, NULL };
+                ZSTD_CDict* cdict = ZSTD_createCDict_advanced(dictBuffer, dictSize, 1, params, customMem);
+                cSize = ZSTD_compress_usingCDict(cctx, compressedBuffer, ZSTD_compressBound(CNBuffSize),
+                                                 CNBuffer, CNBuffSize, cdict);
+                ZSTD_freeCDict(cdict);
+                if (ZSTD_isError(cSize)) goto _output_error;
+            }
+        }
+        DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/CNBuffSize*100);
+
+        DISPLAYLEVEL(4, "test%3i : retrieve dictID from frame : ", testNb++);
+        {   U32 const did = ZSTD_getDictID_fromFrame(compressedBuffer, cSize);
+            if (did != dictID) goto _output_error;   /* non-conformant (content-only) dictionary */
+        }
+        DISPLAYLEVEL(4, "OK \n");
+
+        DISPLAYLEVEL(4, "test%3i : frame should not have content size : ", testNb++);
+        {   unsigned long long const contentSize = ZSTD_findDecompressedSize(compressedBuffer, cSize);
+            if (contentSize != ZSTD_CONTENTSIZE_UNKNOWN)  goto _output_error;  /* cdict contentSizeFlag not used */
+        }
+        DISPLAYLEVEL(4, "OK \n");
+
+        DISPLAYLEVEL(4, "test%3i : frame built with dictionary should be decompressible : ", testNb++);
+        CHECKPLUS(r, ZSTD_decompress_usingDict(dctx,
+                                       decodedBuffer, CNBuffSize,
+                                       compressedBuffer, cSize,
+                                       dictBuffer, dictSize),
+                  if (r != CNBuffSize) goto _output_error);
+        DISPLAYLEVEL(4, "OK \n");
+
         DISPLAYLEVEL(4, "test%3i : compress without dictID : ", testNb++);
         {   ZSTD_parameters p = ZSTD_getParams(3, CNBuffSize, dictSize);
             p.fParams.noDictIDFlag = 1;

tests/zstreamtest.c

@@ -438,6 +438,34 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
       if (!ZSTD_isError(r)) goto _output_error;  /* must fail : frame requires > 100 bytes */
       DISPLAYLEVEL(3, "OK (%s)\n", ZSTD_getErrorName(r)); }
 
+    DISPLAYLEVEL(3, "test%3i : check dictionary used : ", testNb++);
+    {   ZSTD_parameters params = ZSTD_getParams(1, CNBufferSize, dictionary.filled);
+        ZSTD_CDict* cdict;
+        params.fParams.noDictIDFlag = 1;
+        cdict = ZSTD_createCDict_advanced(dictionary.start, dictionary.filled, 1, params, customMem);
+        { size_t const initError = ZSTD_initCStream_usingCDict(zc, cdict);
+          if (ZSTD_isError(initError)) goto _output_error; }
+        cSize = 0;
+        outBuff.dst = compressedBuffer;
+        outBuff.size = compressedBufferSize;
+        outBuff.pos = 0;
+        inBuff.src = CNBuffer;
+        inBuff.size = CNBufferSize;
+        inBuff.pos = 0;
+        { size_t const r = ZSTD_compressStream(zc, &outBuff, &inBuff);
+          if (ZSTD_isError(r)) goto _output_error; }
+        if (inBuff.pos != inBuff.size) goto _output_error;   /* entire input should be consumed */
+        { size_t const r = ZSTD_endStream(zc, &outBuff);
+          if (r != 0) goto _output_error; }  /* error, or some data not flushed */
+        cSize = outBuff.pos;
+        ZSTD_freeCDict(cdict);
+        DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/CNBufferSize*100);
+
+        { size_t const r = ZSTD_decompress(decodedBuffer, CNBufferSize, compressedBuffer, cSize);
+          if (!ZSTD_isError(r)) goto _output_error; /* must fail : dictionary not used */
+          DISPLAYLEVEL(3, "OK (%s)\n", ZSTD_getErrorName(r)); }
+    }
+
     /* Unknown srcSize */
     DISPLAYLEVEL(3, "test%3i : pledgedSrcSize == 0 behaves properly : ", testNb++);
     {   ZSTD_parameters params = ZSTD_getParams(5, 0, 0);