From add7ed2d4a09c94e5a5b3b80682a858cbb3ecba1 Mon Sep 17 00:00:00 2001
From: Nick Terrell <terrelln@fb.com>
Date: Thu, 14 May 2020 11:50:48 -0700
Subject: [PATCH 1/2] [lib] Fix bug in loading LDM dictionary in MT mode

Exposed when loading a dictionary < LDM minMatch bytes in MT mode.

Test Plan:
```
CC=clang make -j zstreamtest MOREFLAGS="-O0 -fsanitize=address"
./zstreamtest -vv -i100000000 -t1 --newapi -s7065 -t3925297
```

TODO: Add an explicit test that loads a small dictionary in MT mode
---
 lib/compress/zstd_compress.c |  2 +-
 lib/compress/zstd_ldm.c      | 12 +++++++-----
 2 files changed, 8 insertions(+), 6 deletions(-)

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 33c3b4ecb..2cfa93051 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -2846,7 +2846,7 @@ static size_t ZSTD_loadDictionaryContent(ZSTD_matchState_t* ms,
 
         ZSTD_overflowCorrectIfNeeded(ms, ws, params, ip, ichunk);
 
-        if (params->ldmParams.enableLdm && ls != NULL && srcSize >= params->ldmParams.minMatchLength)
+        if (params->ldmParams.enableLdm && ls != NULL)
             ZSTD_ldm_fillHashTable(ls, (const BYTE*)src, (const BYTE*)src + srcSize, &params->ldmParams);
 
         switch(params->cParams.strategy)
diff --git a/lib/compress/zstd_ldm.c b/lib/compress/zstd_ldm.c
index 2a1c2cd62..22f362807 100644
--- a/lib/compress/zstd_ldm.c
+++ b/lib/compress/zstd_ldm.c
@@ -228,11 +228,13 @@ void ZSTD_ldm_fillHashTable(
             ldmState_t* state, const BYTE* ip,
             const BYTE* iend, ldmParams_t const* params)
 {
-    U64 startingHash = ZSTD_rollingHash_compute(ip, params->minMatchLength);
-    ZSTD_ldm_fillLdmHashTable(
-        state, startingHash, ip, iend - params->minMatchLength, state->window.base,
-        params->hashLog - params->bucketSizeLog,
-        *params);
+    if ((size_t)(iend - ip) >= params->minMatchLength) {
+        U64 startingHash = ZSTD_rollingHash_compute(ip, params->minMatchLength);
+        ZSTD_ldm_fillLdmHashTable(
+            state, startingHash, ip, iend - params->minMatchLength, state->window.base,
+            params->hashLog - params->bucketSizeLog,
+            *params);
+    }
 }
 
 

From bf0591e1e25a8ee7f524fceac286cb35eac67e3f Mon Sep 17 00:00:00 2001
From: Nick Terrell <terrelln@fb.com>
Date: Thu, 14 May 2020 12:06:55 -0700
Subject: [PATCH 2/2] [test] Expose the LDM+MT+dict bug in a unit test

---
 tests/fuzzer.c | 29 +++++++++++++++++++++++++++++
 1 file changed, 29 insertions(+)

diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index 9b01bc944..bbbcb31ad 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -2697,6 +2697,35 @@ static int basicUnitTests(U32 const seed, double compressibility)
         free(dict);
     }
     DISPLAYLEVEL(3, "OK \n");
+
+    DISPLAYLEVEL(3, "test%3i : small dictionary with multithreading and LDM ", testNb++);
+    {
+        ZSTD_CCtx* cctx = ZSTD_createCCtx();
+        /* A little more than ZSTDMT_JOBSIZE_MIN */
+        size_t const srcSize = 1 MB + 5;
+        size_t const dictSize = 10;
+        size_t const dstSize = ZSTD_compressBound(srcSize);
+        void* const src = CNBuffer;
+        void* const dst = compressedBuffer;
+        void* dict = (void*)malloc(dictSize);
+
+        RDG_genBuffer(src, srcSize, compressibility, 0.5, seed);
+        RDG_genBuffer(dict, dictSize, compressibility, 0., seed);
+
+        /* Make sure there is no ZSTD_MAGIC_NUMBER */
+        memset(dict, 0, sizeof(U32));
+
+        /* Enable MT, LDM, and use refPrefix() for a small dict */
+        CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 2));
+        CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, 1));
+        CHECK_Z(ZSTD_CCtx_refPrefix(cctx, dict, dictSize));
+
+        CHECK_Z(ZSTD_compress2(cctx, dst, dstSize, src, srcSize));
+
+        ZSTD_freeCCtx(cctx);
+        free(dict);
+    }
+    DISPLAYLEVEL(3, "OK \n");
 #endif
 
     /* note : this test is rather long, it would be great to find a way to speed up its execution */