From ca0cfa3dbd5616a4497ea4aabb7fc45f490527e4 Mon Sep 17 00:00:00 2001 From: Nick Terrell Date: Thu, 27 Sep 2018 12:48:29 -0700 Subject: [PATCH 1/3] [zstreamtest] Reduce memory of newapi tests We could allocate up to 2^28 bytes of memory when using 2 threads with window log = 24. Now, we limit it to 2^26 bytes of memory when not running big tests. I chose max window log = 22 since that is the maximum source size when big tests are disabled. Hopefully this will be enough to reduce or eliminate the test failures. --- tests/zstreamtest.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c index 96136a625..3959f5d7d 100644 --- a/tests/zstreamtest.c +++ b/tests/zstreamtest.c @@ -1808,7 +1808,7 @@ static int fuzzerTests_newAPI(U32 seed, U32 nbTests, unsigned startTest, } { U64 const pledgedSrcSize = (FUZ_rand(&lseed) & 3) ? ZSTD_CONTENTSIZE_UNKNOWN : maxTestSize; ZSTD_compressionParameters cParams = ZSTD_getCParams(cLevel, pledgedSrcSize, dictSize); - static const U32 windowLogMax = 24; + const U32 windowLogMax = bigTests ? 24 : 22; if (dictSize) DISPLAYLEVEL(5, "t%u: with dictionary of size : %zu \n", testNb, dictSize); From 7ee910e86b5a4c6ee7479959df547b775ae169a1 Mon Sep 17 00:00:00 2001 From: Nick Terrell Date: Thu, 27 Sep 2018 13:55:24 -0700 Subject: [PATCH 2/3] More aggressive limitations --- tests/zstreamtest.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c index 3959f5d7d..b56009323 100644 --- a/tests/zstreamtest.c +++ b/tests/zstreamtest.c @@ -1808,7 +1808,8 @@ static int fuzzerTests_newAPI(U32 seed, U32 nbTests, unsigned startTest, } { U64 const pledgedSrcSize = (FUZ_rand(&lseed) & 3) ? ZSTD_CONTENTSIZE_UNKNOWN : maxTestSize; ZSTD_compressionParameters cParams = ZSTD_getCParams(cLevel, pledgedSrcSize, dictSize); - const U32 windowLogMax = bigTests ? 24 : 22; + const U32 windowLogMax = bigTests ? 24 : 20; + const U32 searchLogMax = bigTests ? 15 : 13; if (dictSize) DISPLAYLEVEL(5, "t%u: with dictionary of size : %zu \n", testNb, dictSize); @@ -1818,6 +1819,7 @@ static int fuzzerTests_newAPI(U32 seed, U32 nbTests, unsigned startTest, cParams.hashLog += (FUZ_rand(&lseed) & 3) - 1; cParams.chainLog += (FUZ_rand(&lseed) & 3) - 1; cParams.searchLog += (FUZ_rand(&lseed) & 3) - 1; + cParams.searchLog = MIN(searchLogMax, cParams.searchLog); cParams.searchLength += (FUZ_rand(&lseed) & 3) - 1; cParams.targetLength = (U32)((cParams.targetLength + 1 ) * (0.5 + ((double)(FUZ_rand(&lseed) & 127) / 128))); cParams = ZSTD_adjustCParams(cParams, pledgedSrcSize, dictSize); From d8c73cd607bb56cc0e4ed77bd2ca59d58916fd60 Mon Sep 17 00:00:00 2001 From: Nick Terrell Date: Thu, 27 Sep 2018 15:49:31 -0700 Subject: [PATCH 3/3] Reset number of threads less often --- tests/zstreamtest.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c index b56009323..105e63e7d 100644 --- a/tests/zstreamtest.c +++ b/tests/zstreamtest.c @@ -1864,8 +1864,9 @@ static int fuzzerTests_newAPI(U32 seed, U32 nbTests, unsigned startTest, CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, pledgedSrcSize) ); } - /* multi-threading parameters */ - { U32 const nbThreadsCandidate = (FUZ_rand(&lseed) & 4) + 1; + /* multi-threading parameters. Only adjust ocassionally for small tests. */ + if (bigTests || (FUZ_rand(&lseed) & 0xF) == 0xF) { + U32 const nbThreadsCandidate = (FUZ_rand(&lseed) & 4) + 1; U32 const nbThreadsAdjusted = (windowLogMalus < nbThreadsCandidate) ? nbThreadsCandidate - windowLogMalus : 1; U32 const nbThreads = MIN(nbThreadsAdjusted, nbThreadsMax); DISPLAYLEVEL(5, "t%u: nbThreads : %u \n", testNb, nbThreads);