ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #1165 from facebook/ctxSizeDown

Browse Source 
Dynamic context downsize
This commit is contained in:
Yann Collet
2018-06-07 14:44:32 -07:00
committed by GitHub
parent b27c7389e3 3cdb342bd5
commit 8ef75547ef
5 changed files with 106 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/fuzzer.c
+41 -5
View File
@@ -450,14 +450,17 @@ static int basicUnitTests(U32 seed, double compressibility)
}
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(3, "test%3d : large window log smaller data : ", testNb++);
/* this test is really too long, and should be made faster */
DISPLAYLEVEL(3, "test%3d : overflow protection with large windowLog : ", testNb++);
{ ZSTD_CCtx* const cctx = ZSTD_createCCtx();
ZSTD_parameters params = ZSTD_getParams(1, ZSTD_CONTENTSIZE_UNKNOWN, 0);
size_t const nbCompressions = (1U << 31) / CNBuffSize + 1;
size_t i;
ZSTD_parameters params = ZSTD_getParams(-9, ZSTD_CONTENTSIZE_UNKNOWN, 0);
size_t const nbCompressions = ((1U << 31) / CNBuffSize) + 1; /* ensure U32 overflow protection is triggered */
size_t cnb;
assert(cctx != NULL);
params.fParams.contentSizeFlag = 0;
params.cParams.windowLog = ZSTD_WINDOWLOG_MAX;
for (i = 0; i < nbCompressions; ++i) {
for (cnb = 0; cnb < nbCompressions; ++cnb) {
DISPLAYLEVEL(6, "run %zu / %zu \n", cnb, nbCompressions);
CHECK_Z( ZSTD_compressBegin_advanced(cctx, NULL, 0, params, ZSTD_CONTENTSIZE_UNKNOWN) ); /* re-use same parameters */
CHECK_Z( ZSTD_compressEnd(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize) );
}
@@ -465,6 +468,39 @@ static int basicUnitTests(U32 seed, double compressibility)
}
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(3, "test%3d : size down context : ", testNb++);
{ ZSTD_CCtx* const largeCCtx = ZSTD_createCCtx();
assert(largeCCtx != NULL);
CHECK_Z( ZSTD_compressBegin(largeCCtx, 19) ); /* streaming implies ZSTD_CONTENTSIZE_UNKNOWN, which maximizes memory usage */
CHECK_Z( ZSTD_compressEnd(largeCCtx, compressedBuffer, compressedBufferSize, CNBuffer, 1) );
{ size_t const largeCCtxSize = ZSTD_sizeof_CCtx(largeCCtx); /* size of context must be measured after compression */
{ ZSTD_CCtx* const smallCCtx = ZSTD_createCCtx();
assert(smallCCtx != NULL);
CHECK_Z(ZSTD_compressCCtx(smallCCtx, compressedBuffer, compressedBufferSize, CNBuffer, 1, 1));
{ size_t const smallCCtxSize = ZSTD_sizeof_CCtx(smallCCtx);
DISPLAYLEVEL(5, "(large) %zuKB > 32*%zuKB (small) : ",
largeCCtxSize>>10, smallCCtxSize>>10);
assert(largeCCtxSize > 32* smallCCtxSize); /* note : "too large" definition is handled within zstd_compress.c .
* make this test case extreme, so that it doesn't depend on a possibly fluctuating definition */
}
ZSTD_freeCCtx(smallCCtx);
}
{ U32 const maxNbAttempts = 1100; /* nb of usages before triggering size down is handled within zstd_compress.c.
* currently defined as 128x, but could be adjusted in the future.
* make this test long enough so that it's not too much tied to the current definition within zstd_compress.c */
U32 u;
for (u=0; u<maxNbAttempts; u++) {
CHECK_Z(ZSTD_compressCCtx(largeCCtx, compressedBuffer, compressedBufferSize, CNBuffer, 1, 1));
if (ZSTD_sizeof_CCtx(largeCCtx) < largeCCtxSize) break; /* sized down */
}
DISPLAYLEVEL(5, "size down after %u attempts : ", u);
if (u==maxNbAttempts) goto _output_error; /* no sizedown happened */
}
}
ZSTD_freeCCtx(largeCCtx);
}
DISPLAYLEVEL(3, "OK \n");
/* Static CCtx tests */
#define STATIC_CCTX_LEVEL 3
DISPLAYLEVEL(3, "test%3i : create static CCtx for level %u :", testNb++, STATIC_CCTX_LEVEL);