Adding --long support for --patch-from (#1959)

* adding long support for patch-from * adding refPrefix to dictionary_decompress * adding refPrefix to dictionary_loader * conversion nit * triggering log mode on chainLog < fileLog and removing old threshold * adding refPrefix to dictionary_round_trip * adding docs * adding enableldm + forceWindow test for dict * separate patch-from logic into FIO_adjustParamsForPatchFromMode * moving memLimit adjustment to outside ifdefs (need for decomp) * removing refPrefix gate on dictionary_round_trip * rebase on top of dev refPrefix change * making sure refPrefx + ldm is < 1% of srcSize * combining notes for patch-from * moving memlimit logic inside fileio.c * adding display for optimal parser and long mode trigger * conversion nit * fuzzer found heap-overflow fix * another conversion nit * moving FIO_adjustMemLimitForPatchFromMode outside ifndef * making params immutable * moving memLimit update before createDictBuffer call * making maxSrcSize unsigned long long * making dictSize and maxSrcSize params unsigned long long * error on files larger than 4gb * extend refPrefix test to include round trip * conversion to size_t * making sure ldm is at least 10x better * removing break * including zstd_compress_internal and removing redundant macros * exposing ZSTD_cycleLog() * using cycleLog instead of chainLog * add some more docs about user optimizations * formatting
2020-04-17 15:58:53 -05:00
parent 88ecdc939b
commit 5b0a452cac
12 changed files with 318 additions and 96 deletions
@@ -42,10 +42,15 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size)
        ddict = ZSTD_createDDict(dict.buff, dict.size);
        FUZZ_ASSERT(ddict);
    } else {
-        FUZZ_ZASSERT(ZSTD_DCtx_loadDictionary_advanced(
+        if (FUZZ_dataProducer_uint32Range(producer, 0, 1) == 0)
+            FUZZ_ZASSERT(ZSTD_DCtx_loadDictionary_advanced(
                dctx, dict.buff, dict.size,
                (ZSTD_dictLoadMethod_e)FUZZ_dataProducer_uint32Range(producer, 0, 1),
                (ZSTD_dictContentType_e)FUZZ_dataProducer_uint32Range(producer, 0, 2)));
+        else
+            FUZZ_ZASSERT(ZSTD_DCtx_refPrefix_advanced(
+                dctx, dict.buff, dict.size,
+                (ZSTD_dictContentType_e)FUZZ_dataProducer_uint32Range(producer, 0, 2)));
    }

    {
@@ -28,10 +28,15 @@ static size_t compress(void* compressed, size_t compressedCapacity,
                       void const* source, size_t sourceSize,
                       void const* dict, size_t dictSize,
                       ZSTD_dictLoadMethod_e dictLoadMethod,
-                       ZSTD_dictContentType_e dictContentType)
+                       ZSTD_dictContentType_e dictContentType,
+                       int const refPrefix)
 {
    ZSTD_CCtx* cctx = ZSTD_createCCtx();
-    FUZZ_ZASSERT(ZSTD_CCtx_loadDictionary_advanced(
+    if (refPrefix)
+        FUZZ_ZASSERT(ZSTD_CCtx_refPrefix_advanced(
+            cctx, dict, dictSize, dictContentType));
+    else 
+        FUZZ_ZASSERT(ZSTD_CCtx_loadDictionary_advanced(
            cctx, dict, dictSize, dictLoadMethod, dictContentType));
    size_t const compressedSize = ZSTD_compress2(
            cctx, compressed, compressedCapacity, source, sourceSize);
@@ -43,10 +48,15 @@ static size_t decompress(void* result, size_t resultCapacity,
                         void const* compressed, size_t compressedSize,
                         void const* dict, size_t dictSize,
                       ZSTD_dictLoadMethod_e dictLoadMethod,
-                         ZSTD_dictContentType_e dictContentType)
+                         ZSTD_dictContentType_e dictContentType,
+                         int const refPrefix)
 {
    ZSTD_DCtx* dctx = ZSTD_createDCtx();
-    FUZZ_ZASSERT(ZSTD_DCtx_loadDictionary_advanced(
+    if (refPrefix)
+        FUZZ_ZASSERT(ZSTD_DCtx_refPrefix_advanced(
+            dctx, dict, dictSize, dictContentType));
+    else
+        FUZZ_ZASSERT(ZSTD_DCtx_loadDictionary_advanced(
            dctx, dict, dictSize, dictLoadMethod, dictContentType));
    size_t const resultSize = ZSTD_decompressDCtx(
            dctx, result, resultCapacity, compressed, compressedSize);
@@ -58,6 +68,7 @@ static size_t decompress(void* result, size_t resultCapacity,
 int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size)
 {
    FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(src, size);
+    int const refPrefix = FUZZ_dataProducer_uint32Range(producer, 0, 1) != 0;
    ZSTD_dictLoadMethod_e const dlm =
    size = FUZZ_dataProducer_uint32Range(producer, 0, 1);
    ZSTD_dictContentType_e const dct =
@@ -75,14 +86,14 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size)
    FUZZ_ASSERT(cBuf);

    size_t const cSize =
-            compress(cBuf, cBufSize, src, size, src, size, dlm, dct);
+            compress(cBuf, cBufSize, src, size, src, size, dlm, dct, refPrefix);
    /* compression failing is okay */
    if (ZSTD_isError(cSize)) {
      FUZZ_ASSERT_MSG(dct != ZSTD_dct_rawContent, "Raw must always succeed!");
      goto out;
    }
    size_t const rSize =
-            decompress(rBuf, size, cBuf, cSize, src, size, dlm, dct);
+            decompress(rBuf, size, cBuf, cSize, src, size, dlm, dct, refPrefix);
    FUZZ_ASSERT_MSG(rSize == size, "Incorrect regenerated size");
    FUZZ_ASSERT_MSG(!memcmp(src, rBuf, size), "Corruption!");

@@ -536,15 +536,15 @@ static int basicUnitTests(U32 const seed, double compressibility)

        BYTE* const dst = (BYTE*)compressedBuffer;
        ZSTD_DCtx* dctx = ZSTD_createDCtx();
-        
+
        /* create a large frame and then a bunch of small frames */
-        size_t srcSize = ZSTD_compress((void*)dst, 
+        size_t srcSize = ZSTD_compress((void*)dst,
            compressedBufferSize, CNBuffer, largeFrameSrcSize, 3);
-        for (i = 0; i < nbFrames; i++) 
-            srcSize += ZSTD_compress((void*)(dst + srcSize), 
-                compressedBufferSize - srcSize, CNBuffer, 
+        for (i = 0; i < nbFrames; i++)
+            srcSize += ZSTD_compress((void*)(dst + srcSize),
+                compressedBufferSize - srcSize, CNBuffer,
                smallFrameSrcSize, 3);
-            
+
        /* decompressStream and make sure that dctx size was reduced at least once */
        while (consumed < srcSize) {
            ZSTD_inBuffer in = {(void*)(dst + consumed), MIN(1, srcSize - consumed), 0};
@@ -566,6 +566,115 @@ static int basicUnitTests(U32 const seed, double compressibility)
    }
    DISPLAYLEVEL(3, "OK \n");

+    DISPLAYLEVEL(3, "test%3i : ldm fill dict out-of-bounds check", testNb++);
+    {
+        ZSTD_CCtx* const cctx = ZSTD_createCCtx();
+
+        size_t const size = (1U << 10);
+        size_t const dstCapacity = ZSTD_compressBound(size);
+        void* dict = (void*)malloc(size);
+        void* src = (void*)malloc(size);
+        void* dst = (void*)malloc(dstCapacity);
+
+        RDG_genBuffer(dict, size, 0.5, 0.5, seed);
+        RDG_genBuffer(src, size, 0.5, 0.5, seed);
+
+        CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, 1));
+        assert(!ZSTD_isError(ZSTD_compress_usingDict(cctx, dst, dstCapacity, src, size, dict, size, 3)));
+
+        ZSTD_freeCCtx(cctx);
+        free(dict);
+        free(src);
+        free(dst);
+    }
+    DISPLAYLEVEL(3, "OK \n");
+
+    DISPLAYLEVEL(3, "test%3i : testing dict compression with enableLdm and forceMaxWindow : ", testNb++);
+    {
+        ZSTD_CCtx* const cctx = ZSTD_createCCtx();
+        void* dict = (void*)malloc(CNBuffSize);
+
+        RDG_genBuffer(dict, CNBuffSize, 0.5, 0.5, seed);
+        RDG_genBuffer(CNBuffer, CNBuffSize, 0.6, 0.6, seed);
+
+        CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_forceMaxWindow, 1));
+        CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, 1));
+        assert(!ZSTD_isError(ZSTD_compress_usingDict(cctx, compressedBuffer, compressedBufferSize,
+            CNBuffer, CNBuffSize, dict, CNBuffSize, 3)));
+
+        ZSTD_freeCCtx(cctx);
+        free(dict);
+    }
+    DISPLAYLEVEL(3, "OK \n");
+
+    /* Note: this test takes 0.5 seconds to run */
+    DISPLAYLEVEL(3, "test%3i : testing refPrefx vs refPrefx + ldm (size comparison) : ", testNb++);
+    {
+        /* test a big buffer so that ldm can take effect */
+        size_t const size = 100 MB;
+        int const windowLog = 27;
+        size_t const dstSize = ZSTD_compressBound(size);
+
+        void* dict = (void*)malloc(size);
+        void* src = (void*)malloc(size);
+        void* dst = (void*)malloc(dstSize);
+        void* recon = (void*)malloc(size);
+
+        size_t refPrefixCompressedSize = 0;
+        size_t refPrefixLdmComrpessedSize = 0;
+        size_t reconSize = 0;
+
+        ZSTD_CCtx* const cctx = ZSTD_createCCtx();
+        ZSTD_DCtx* const dctx = ZSTD_createDCtx();
+
+        /* make dict and src the same uncompressible data */
+        RDG_genBuffer(src, size, 0, 0, seed);
+        memcpy(dict, src, size);
+        assert(!memcmp(dict, src, size));
+
+        /* set level 1 and windowLog to cover src */
+        CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 1));
+        CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, windowLog));
+
+        /* compress on level 1 using just refPrefix and no ldm */
+        ZSTD_CCtx_refPrefix(cctx, dict, size);
+        refPrefixCompressedSize = ZSTD_compress2(cctx, dst, dstSize, src, size);
+        assert(!ZSTD_isError(refPrefixCompressedSize));
+
+        /* test round trip just refPrefix */
+        ZSTD_DCtx_refPrefix(dctx, dict, size);
+        reconSize = ZSTD_decompressDCtx(dctx, recon, size, dst, refPrefixCompressedSize);
+        assert(!ZSTD_isError(reconSize));
+        assert(reconSize == size);
+        assert(!memcmp(recon, src, size));
+
+        /* compress on level 1 using refPrefix and ldm */
+        ZSTD_CCtx_refPrefix(cctx, dict, size);;
+        CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, 1))
+        refPrefixLdmComrpessedSize = ZSTD_compress2(cctx, dst, dstSize, src, size);
+        assert(!ZSTD_isError(refPrefixLdmComrpessedSize));
+
+        /* test round trip refPrefix + ldm*/
+        ZSTD_DCtx_refPrefix(dctx, dict, size);
+        reconSize = ZSTD_decompressDCtx(dctx, recon, size, dst, refPrefixLdmComrpessedSize);
+        assert(!ZSTD_isError(reconSize));
+        assert(reconSize == size);
+        assert(!memcmp(recon, src, size));
+
+        /* make sure that refPrefixCompressedSize is significantly greater */
+        assert(refPrefixCompressedSize > 10 * refPrefixLdmComrpessedSize);
+        /* make sure the ldm comrpessed size is less than 1% of original */
+        assert((double)refPrefixLdmComrpessedSize / (double)size < 0.01);
+
+        ZSTD_freeDCtx(dctx);
+        ZSTD_freeCCtx(cctx);
+        free(recon);
+        free(dict);
+        free(src);
+        free(dst);
+    }
+    DISPLAYLEVEL(3, "OK \n");
+
    DISPLAYLEVEL(3, "test%3d: superblock uncompressible data, too many nocompress superblocks : ", testNb++);
    {
        ZSTD_CCtx* const cctx = ZSTD_createCCtx();
@@ -1262,11 +1262,20 @@ println "\n===> patch-from tests"

 datagen -g1000 -P50 > tmp_dict
 datagen -g1000 -P10 > tmp_patch
-zstd --memory=10000 --patch-from=tmp_dict tmp_patch -o tmp_patch_diff
-zstd -d --memory=10000 --patch-from=tmp_dict tmp_patch_diff -o tmp_patch_recon
+zstd --patch-from=tmp_dict tmp_patch -o tmp_patch_diff
+zstd -d --patch-from=tmp_dict tmp_patch_diff -o tmp_patch_recon
 $DIFF -s tmp_patch_recon tmp_patch
 rm -rf tmp_*

+println "\n===> patch-from recursive tests"
+
+mkdir tmp_dir
+datagen > tmp_dir/tmp1 
+datagen > tmp_dir/tmp2
+datagen > tmp_dict
+zstd --patch-from=tmp_dict -r tmp_dir && die
+rm -rf tmp* 
+
 println "\n===>   large files tests "

 roundTripTest -g270000000 1