ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #2341 from senhuang42/ldm_optimized_for_opt_parser

Browse Source 
Integrate long distance matches into optimal parser
This commit is contained in:
Yann Collet
2020-10-13 13:09:07 -07:00
committed by GitHub
parent 1e47a75abf d6911b86be
commit f5d5cd3b40
7 changed files with 261 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/fuzzer.c
+38
View File
@@ -793,6 +793,44 @@ static int basicUnitTests(U32 const seed, double compressibility)
}
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(3, "test%3i : testing ldm no regressions in size for opt parser : ", testNb++);
{
size_t cSizeLdm;
size_t cSizeNoLdm;
ZSTD_CCtx* const cctx = ZSTD_createCCtx();
RDG_genBuffer(CNBuffer, CNBuffSize, 0.5, 0.5, seed);
/* Enable checksum to verify round trip. */
CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1));
CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, 1));
CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 19));
/* Round trip once with ldm. */
cSizeLdm = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize);
CHECK_Z(cSizeLdm);
CHECK_Z(ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSizeLdm));
ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1));
CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, 0));
CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 19));
/* Round trip once without ldm. */
cSizeNoLdm = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize);
CHECK_Z(cSizeNoLdm);
CHECK_Z(ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSizeNoLdm));
if (cSizeLdm > cSizeNoLdm) {
DISPLAY("Using long mode should not cause regressions for btopt+\n");
testResult = 1;
goto _end;
}
ZSTD_freeCCtx(cctx);
}
DISPLAYLEVEL(3, "OK \n");
/* Note: this test takes 0.5 seconds to run */
DISPLAYLEVEL(3, "test%3i : testing refPrefx vs refPrefx + ldm (size comparison) : ", testNb++);
{
tests/playTests.sh
+24
View File
@@ -1206,6 +1206,7 @@ roundTripTest -g1000K "1 --single-thread --long"
roundTripTest -g517K "6 --single-thread --long"
roundTripTest -g516K "16 --single-thread --long"
roundTripTest -g518K "19 --single-thread --long"
roundTripTest -g2M "22 --single-thread --ultra --long"
fileRoundTripTest -g5M "3 --single-thread --long"
@@ -1215,6 +1216,7 @@ then
println "\n===> zstdmt round-trip tests "
roundTripTest -g4M "1 -T0"
roundTripTest -g8M "3 -T2"
roundTripTest -g8M "19 -T0 --long"
roundTripTest -g8000K "2 --threads=2"
fileRoundTripTest -g4M "19 -T2 -B1M"
@@ -1333,6 +1335,28 @@ roundTripTest -g1M -P50 "1 --single-thread --long=29" " --long=28 --memory=512MB
roundTripTest -g1M -P50 "1 --single-thread --long=29" " --zstd=wlog=28 --memory=512MB"
println "\n===> zstd long distance matching with optimal parser compressed size tests "
optCSize16=$(datagen -g511K | zstd -16 -c | wc -c)
longCSize16=$(datagen -g511K | zstd -16 --long -c | wc -c)
optCSize19=$(datagen -g2M | zstd -19 -c | wc -c)
longCSize19=$(datagen -g2M | zstd -19 --long -c | wc -c)
optCSize19wlog23=$(datagen -g2M | zstd -19 -c --zstd=wlog=23 | wc -c)
longCSize19wlog23=$(datagen -g2M | zstd -19 -c --long=23 | wc -c)
optCSize22=$(datagen -g900K | zstd -22 --ultra -c | wc -c)
longCSize22=$(datagen -g900K | zstd -22 --ultra --long -c | wc -c)
if [ "$longCSize16" -gt "$optCSize16" ]; then
echo using --long on compression level 16 should not cause compressed size regression
exit 1
elif [ "$longCSize19" -gt "$optCSize19" ]; then
echo using --long on compression level 19 should not cause compressed size regression
exit 1
elif [ "$longCSize19wlog23" -gt "$optCSize19wlog23" ]; then
echo using --long on compression level 19 with wLog=23 should not cause compressed size regression
exit 1
elif [ "$longCSize22" -gt "$optCSize22" ]; then
echo using --long on compression level 22 should not cause compressed size regression
exit 1
fi
if [ "$1" != "--test-large-data" ]; then