From 0932de54bc0dbc6d29008b9e6087b88721ca8bb0 Mon Sep 17 00:00:00 2001 From: Nick Terrell Date: Mon, 26 Aug 2019 18:19:29 -0700 Subject: [PATCH 1/2] [dictBuilder] Fix deadlock in *COVER error case The COVER and FASTCOVER dictionary builders can deadlock when dictionary construction errors, likely because there are too few samples, or too few distinct dmers. The deadlock only occurs when there are errors. Fixes #1746. --- lib/dictBuilder/cover.c | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/lib/dictBuilder/cover.c b/lib/dictBuilder/cover.c index 621996759..4721205da 100644 --- a/lib/dictBuilder/cover.c +++ b/lib/dictBuilder/cover.c @@ -919,13 +919,12 @@ void COVER_best_finish(COVER_best_t *best, ZDICT_cover_params_t parameters, } } /* Save the dictionary, parameters, and size */ - if (!dict) { - return; + if (dict) { + memcpy(best->dict, dict, dictSize); + best->dictSize = dictSize; + best->parameters = parameters; + best->compressedSize = compressedSize; } - memcpy(best->dict, dict, dictSize); - best->dictSize = dictSize; - best->parameters = parameters; - best->compressedSize = compressedSize; } if (liveJobs == 0) { ZSTD_pthread_cond_broadcast(&best->cond); From 793c71d6aeb775dddc8f8825d36d88fc495cc8c1 Mon Sep 17 00:00:00 2001 From: Nick Terrell Date: Mon, 26 Aug 2019 18:32:08 -0700 Subject: [PATCH 2/2] [test][dictBuilder] Add multithreaded tests --- tests/playTests.sh | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/playTests.sh b/tests/playTests.sh index ad096fddd..19fc514f6 100755 --- a/tests/playTests.sh +++ b/tests/playTests.sh @@ -526,6 +526,15 @@ $ZSTD -o tmpDict --train "$TESTDIR"/*.c "$PRGDIR"/*.c test -f tmpDict $ZSTD --train "$TESTDIR"/*.c "$PRGDIR"/*.c test -f dictionary +println "- Test dictionary training fails" +echo "000000000000000000000000000000000" > tmpz +$ZSTD --train tmpz tmpz tmpz tmpz tmpz tmpz tmpz tmpz tmpz && die "Dictionary training should fail : source is all zeros" +if [ -n "$hasMT" ] +then + $ZSTD --train -T0 tmpz tmpz tmpz tmpz tmpz tmpz tmpz tmpz tmpz && die "Dictionary training should fail : source is all zeros" + println "- Create dictionary with multithreading enabled" + $ZSTD --train -T0 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict +fi rm tmp* dictionary