ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

initial commit

Browse Source
This commit is contained in:
Danielle Rozenblit
2023-02-09 07:37:37 -08:00
parent dc39409a03
commit 610c8b9e33
385 changed files with 7518 additions and 4039 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/fuzzer.c
+447 -90
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) Yann Collet, Facebook, Inc.
* Copyright (c) Meta Platforms, Inc. and affiliates.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
@@ -25,7 +25,8 @@
#include <stdlib.h> /* free */
#include <stdio.h> /* fgets, sscanf */
#include <string.h> /* strcmp */
#undef NDEBUG
#include <time.h> /* time(), time_t */
#undef NDEBUG /* always enable assert() */
#include <assert.h>
#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_compressContinue, ZSTD_compressBlock */
#include "debug.h" /* DEBUG_STATIC_ASSERT */
@@ -82,8 +83,8 @@ static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER;
void FUZ_bug976(void);
void FUZ_bug976(void)
{ /* these constants shall not depend on MIN() macro */
assert(ZSTD_HASHLOG_MAX < 31);
assert(ZSTD_CHAINLOG_MAX < 31);
DEBUG_STATIC_ASSERT(ZSTD_HASHLOG_MAX < 31);
DEBUG_STATIC_ASSERT(ZSTD_CHAINLOG_MAX < 31);
}
@@ -118,23 +119,24 @@ static U32 FUZ_highbit32(U32 v32)
/*=============================================
* Test macros
=============================================*/
#define CHECK_Z(f) { \
size_t const err = f; \
if (ZSTD_isError(err)) { \
DISPLAY("Error => %s : %s ", \
#f, ZSTD_getErrorName(err)); \
exit(1); \
#define CHECK(fn) { if(!(fn)) { DISPLAYLEVEL(1, "Error : test (%s) failed \n", #fn); exit(1); } }
#define CHECK_Z(f) { \
size_t const err = f; \
if (ZSTD_isError(err)) { \
DISPLAY("Error => %s : %s ", \
#f, ZSTD_getErrorName(err)); \
exit(1); \
} }
#define CHECK_VAR(var, fn) var = fn; if (ZSTD_isError(var)) { DISPLAYLEVEL(1, "%s : fails : %s \n", #fn, ZSTD_getErrorName(var)); goto _output_error; }
#define CHECK_VAR(var, fn) var = fn; if (ZSTD_isError(var)) { DISPLAYLEVEL(1, "%s : fails : %s \n", #fn, ZSTD_getErrorName(var)); exit(1); }
#define CHECK_NEWV(var, fn) size_t const CHECK_VAR(var, fn)
#define CHECK(fn) { CHECK_NEWV(__err, fn); }
#define CHECKPLUS(var, fn, more) { CHECK_NEWV(var, fn); more; }
#define CHECK_OP(op, lhs, rhs) { \
if (!((lhs) op (rhs))) { \
DISPLAY("Error L%u => FAILED %s %s %s ", __LINE__, #lhs, #op, #rhs); \
goto _output_error; \
exit(1); \
} \
}
#define CHECK_EQ(lhs, rhs) CHECK_OP(==, lhs, rhs)
@@ -338,6 +340,7 @@ static void FUZ_decodeSequences(BYTE* dst, ZSTD_Sequence* seqs, size_t seqsSize,
}
#ifdef ZSTD_MULTITHREAD
typedef struct {
ZSTD_CCtx* cctx;
ZSTD_threadPool* pool;
@@ -429,8 +432,8 @@ static int threadPoolTests(void) {
ZSTD_pthread_create(&t1, NULL, threadPoolTests_compressionJob, &p1);
ZSTD_pthread_create(&t2, NULL, threadPoolTests_compressionJob, &p2);
ZSTD_pthread_join(t1, NULL);
ZSTD_pthread_join(t2, NULL);
ZSTD_pthread_join(t1);
ZSTD_pthread_join(t2);
assert(!memcmp(decodedBuffer, decodedBuffer2, CNBuffSize));
free(decodedBuffer2);
@@ -461,6 +464,101 @@ _output_error:
* Unit tests
=============================================*/
static void test_compressBound(unsigned tnb)
{
DISPLAYLEVEL(3, "test%3u : compressBound : ", tnb);
/* check ZSTD_compressBound == ZSTD_COMPRESSBOUND
* for a large range of known valid values */
DEBUG_STATIC_ASSERT(sizeof(size_t) >= 4);
{ int s;
for (s=0; s<30; s++) {
size_t const w = (size_t)1 << s;
CHECK_EQ(ZSTD_compressBound(w), ZSTD_COMPRESSBOUND(w));
} }
/* Ensure error if srcSize too big */
{ size_t const w = ZSTD_MAX_INPUT_SIZE + 1;
CHECK(ZSTD_isError(ZSTD_compressBound(w))); /* must fail */
CHECK_EQ(ZSTD_COMPRESSBOUND(w), 0);
}
DISPLAYLEVEL(3, "OK \n");
}
static void test_decompressBound(unsigned tnb)
{
DISPLAYLEVEL(3, "test%3u : decompressBound : ", tnb);
/* Simple compression, with size : should provide size; */
{ const char example[] = "abcd";
char cBuffer[ZSTD_COMPRESSBOUND(sizeof(example))];
size_t const cSize = ZSTD_compress(cBuffer, sizeof(cBuffer), example, sizeof(example), 0);
CHECK_Z(cSize);
CHECK_EQ(ZSTD_decompressBound(cBuffer, cSize), (unsigned long long)sizeof(example));
}
/* Simple small compression without size : should provide 1 block size */
{ char cBuffer[ZSTD_COMPRESSBOUND(0)];
ZSTD_outBuffer out = { cBuffer, sizeof(cBuffer), 0 };
ZSTD_inBuffer in = { NULL, 0, 0 };
ZSTD_CCtx* const cctx = ZSTD_createCCtx();
assert(cctx);
CHECK_Z( ZSTD_initCStream(cctx, 0) );
CHECK_Z( ZSTD_compressStream(cctx, &out, &in) );
CHECK_EQ( ZSTD_endStream(cctx, &out), 0 );
CHECK_EQ( ZSTD_decompressBound(cBuffer, out.pos), ZSTD_BLOCKSIZE_MAX );
ZSTD_freeCCtx(cctx);
}
/* Attempt to overflow 32-bit intermediate multiplication result
* This requires dBound >= 4 GB, aka 2^32.
* This requires 2^32 / 2^17 = 2^15 blocks
* => create 2^15 blocks (can be empty, or just 1 byte). */
{ const char input[] = "a";
size_t const nbBlocks = (1 << 15) + 1;
size_t blockNb;
size_t const outCapacity = 1 << 18; /* large margin */
char* const outBuffer = malloc (outCapacity);
ZSTD_outBuffer out = { outBuffer, outCapacity, 0 };
ZSTD_CCtx* const cctx = ZSTD_createCCtx();
assert(cctx);
assert(outBuffer);
CHECK_Z( ZSTD_initCStream(cctx, 0) );
for (blockNb=0; blockNb<nbBlocks; blockNb++) {
ZSTD_inBuffer in = { input, sizeof(input), 0 };
CHECK_Z( ZSTD_compressStream(cctx, &out, &in) );
CHECK_EQ( ZSTD_flushStream(cctx, &out), 0 );
}
CHECK_EQ( ZSTD_endStream(cctx, &out), 0 );
CHECK( ZSTD_decompressBound(outBuffer, out.pos) > 0x100000000LLU /* 4 GB */ );
ZSTD_freeCCtx(cctx);
free(outBuffer);
}
DISPLAYLEVEL(3, "OK \n");
}
static void test_setCParams(unsigned tnb)
{
ZSTD_CCtx* const cctx = ZSTD_createCCtx();
ZSTD_compressionParameters cparams;
assert(cctx);
DISPLAYLEVEL(3, "test%3u : ZSTD_CCtx_setCParams : ", tnb);
/* valid cparams */
cparams = ZSTD_getCParams(1, 0, 0);
CHECK_Z(ZSTD_CCtx_setCParams(cctx, cparams));
/* invalid cparams (must fail) */
cparams.windowLog = 99;
CHECK(ZSTD_isError(ZSTD_CCtx_setCParams(cctx, cparams)));
free(cctx);
DISPLAYLEVEL(3, "OK \n");
}
static int basicUnitTests(U32 const seed, double compressibility)
{
size_t const CNBuffSize = 5 MB;
@@ -507,6 +605,12 @@ static int basicUnitTests(U32 const seed, double compressibility)
DISPLAYLEVEL(3, "%u (OK) \n", vn);
}
test_compressBound(testNb++);
test_decompressBound(testNb++);
test_setCParams(testNb++);
DISPLAYLEVEL(3, "test%3u : ZSTD_adjustCParams : ", testNb++);
{
ZSTD_compressionParameters params;
@@ -580,6 +684,17 @@ static int basicUnitTests(U32 const seed, double compressibility)
if (r != CNBuffSize) goto _output_error; }
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(3, "test%3i : decompress %u bytes with Huffman assembly disabled : ", testNb++, (unsigned)CNBuffSize);
{
ZSTD_DCtx* dctx = ZSTD_createDCtx();
size_t r;
CHECK_Z(ZSTD_DCtx_setParameter(dctx, ZSTD_d_disableHuffmanAssembly, 1));
r = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize);
if (r != CNBuffSize || memcmp(decodedBuffer, CNBuffer, CNBuffSize)) goto _output_error;
ZSTD_freeDCtx(dctx);
}
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(3, "test%3i : check decompressed result : ", testNb++);
{ size_t u;
for (u=0; u<CNBuffSize; u++) {
@@ -1117,6 +1232,60 @@ static int basicUnitTests(U32 const seed, double compressibility)
}
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(3, "test%3i : in-place decompression : ", testNb++);
cSize = ZSTD_compress(compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize, -ZSTD_BLOCKSIZE_MAX);
CHECK_Z(cSize);
CHECK_LT(CNBuffSize, cSize);
{
size_t const margin = ZSTD_decompressionMargin(compressedBuffer, cSize);
size_t const outputSize = (CNBuffSize + margin);
char* output = malloc(outputSize);
char* input = output + outputSize - cSize;
CHECK_LT(cSize, CNBuffSize + margin);
CHECK(output != NULL);
CHECK_Z(margin);
CHECK(margin <= ZSTD_DECOMPRESSION_MARGIN(CNBuffSize, ZSTD_BLOCKSIZE_MAX));
memcpy(input, compressedBuffer, cSize);
{
size_t const dSize = ZSTD_decompress(output, outputSize, input, cSize);
CHECK_Z(dSize);
CHECK_EQ(dSize, CNBuffSize);
}
CHECK(!memcmp(output, CNBuffer, CNBuffSize));
free(output);
}
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(3, "test%3i : in-place decompression with 2 frames : ", testNb++);
cSize = ZSTD_compress(compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize / 3, -ZSTD_BLOCKSIZE_MAX);
CHECK_Z(cSize);
{
size_t const cSize2 = ZSTD_compress((char*)compressedBuffer + cSize, compressedBufferSize - cSize, (char const*)CNBuffer + (CNBuffSize / 3), CNBuffSize / 3, -ZSTD_BLOCKSIZE_MAX);
CHECK_Z(cSize2);
cSize += cSize2;
}
{
size_t const srcSize = (CNBuffSize / 3) * 2;
size_t const margin = ZSTD_decompressionMargin(compressedBuffer, cSize);
size_t const outputSize = (CNBuffSize + margin);
char* output = malloc(outputSize);
char* input = output + outputSize - cSize;
CHECK_LT(cSize, CNBuffSize + margin);
CHECK(output != NULL);
CHECK_Z(margin);
memcpy(input, compressedBuffer, cSize);
{
size_t const dSize = ZSTD_decompress(output, outputSize, input, cSize);
CHECK_Z(dSize);
CHECK_EQ(dSize, srcSize);
}
CHECK(!memcmp(output, CNBuffer, srcSize));
free(output);
}
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(3, "test%3d: superblock uncompressible data, too many nocompress superblocks : ", testNb++);
{
ZSTD_CCtx* const cctx = ZSTD_createCCtx();
@@ -1579,7 +1748,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(3, "test%3i : init CCtx for small level %u (should work again) : ", testNb++, 1);
CHECK( ZSTD_compressBegin(staticCCtx, 1) );
CHECK_Z( ZSTD_compressBegin(staticCCtx, 1) );
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(3, "test%3i : use CStream on CCtx-sized static context (should fail) : ", testNb++);
@@ -1622,6 +1791,94 @@ static int basicUnitTests(U32 const seed, double compressibility)
if (!ZSTD_isError(r)) goto _output_error;
}
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(3, "test%3i : test estimation functions with default cctx params : ", testNb++);
{
// Test ZSTD_estimateCCtxSize_usingCCtxParams
{
ZSTD_CCtx_params* params = ZSTD_createCCtxParams();
size_t const cctxSizeDefault = ZSTD_estimateCCtxSize_usingCCtxParams(params);
staticCCtx = ZSTD_initStaticCCtx(staticCCtxBuffer, cctxSizeDefault);
CHECK_VAR(cSize, ZSTD_compressCCtx(staticCCtx,
compressedBuffer, compressedBufferSize,
CNBuffer, CNBuffSize, 3));
{
size_t const r = ZSTD_decompressDCtx(staticDCtx,
decodedBuffer, CNBuffSize,
compressedBuffer, cSize);
if (r != CNBuffSize) goto _output_error;
if (memcmp(decodedBuffer, CNBuffer, CNBuffSize)) goto _output_error;
}
ZSTD_freeCCtxParams(params);
}
// Test ZSTD_estimateCStreamSize_usingCCtxParams
{
ZSTD_CCtx_params* params = ZSTD_createCCtxParams();
size_t const cctxSizeDefault = ZSTD_estimateCStreamSize_usingCCtxParams(params);
staticCCtx = ZSTD_initStaticCCtx(staticCCtxBuffer, cctxSizeDefault);
CHECK_VAR(cSize, ZSTD_compressCCtx(staticCCtx,
compressedBuffer, compressedBufferSize,
CNBuffer, CNBuffSize, 3) );
{
size_t const r = ZSTD_decompressDCtx(staticDCtx,
decodedBuffer, CNBuffSize,
compressedBuffer, cSize);
if (r != CNBuffSize) goto _output_error;
if (memcmp(decodedBuffer, CNBuffer, CNBuffSize)) goto _output_error;
}
ZSTD_freeCCtxParams(params);
}
}
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(3, "test%3i : test estimation functions with maxBlockSize = 0 : ", testNb++);
{
// Test ZSTD_estimateCCtxSize_usingCCtxParams
{
ZSTD_CCtx_params* params = ZSTD_createCCtxParams();
size_t cctxSizeDefault;
CHECK_Z(ZSTD_CCtxParams_setParameter(params, ZSTD_c_maxBlockSize, 0));
cctxSizeDefault = ZSTD_estimateCCtxSize_usingCCtxParams(params);
staticCCtx = ZSTD_initStaticCCtx(staticCCtxBuffer, cctxSizeDefault);
CHECK_VAR(cSize, ZSTD_compressCCtx(staticCCtx,
compressedBuffer, compressedBufferSize,
CNBuffer, CNBuffSize, 3) );
{
size_t const r = ZSTD_decompressDCtx(staticDCtx,
decodedBuffer, CNBuffSize,
compressedBuffer, cSize);
if (r != CNBuffSize) goto _output_error;
if (memcmp(decodedBuffer, CNBuffer, CNBuffSize)) goto _output_error;
}
ZSTD_freeCCtxParams(params);
}
// Test ZSTD_estimateCStreamSize_usingCCtxParams
{
ZSTD_CCtx_params* params = ZSTD_createCCtxParams();
size_t cctxSizeDefault;
CHECK_Z(ZSTD_CCtxParams_setParameter(params, ZSTD_c_maxBlockSize, 0));
cctxSizeDefault = ZSTD_estimateCStreamSize_usingCCtxParams(params);
staticCCtx = ZSTD_initStaticCCtx(staticCCtxBuffer, cctxSizeDefault);
CHECK_VAR(cSize, ZSTD_compressCCtx(staticCCtx,
compressedBuffer, compressedBufferSize,
CNBuffer, CNBuffSize, 3) );
{
size_t const r = ZSTD_decompressDCtx(staticDCtx,
decodedBuffer, CNBuffSize,
compressedBuffer, cSize);
if (r != CNBuffSize) goto _output_error;
if (memcmp(decodedBuffer, CNBuffer, CNBuffSize)) goto _output_error;
}
ZSTD_freeCCtxParams(params);
}
}
DISPLAYLEVEL(3, "OK \n");
}
free(staticCCtxBuffer);
free(staticDCtxBuffer);
@@ -1647,8 +1904,8 @@ static int basicUnitTests(U32 const seed, double compressibility)
testResult = 1;
goto _end;
}
CHECK( ZSTD_CCtx_setParameter(mtctx, ZSTD_c_nbWorkers, 2) );
CHECK( ZSTD_CCtx_setParameter(mtctx, ZSTD_c_compressionLevel, 1) );
CHECK_Z( ZSTD_CCtx_setParameter(mtctx, ZSTD_c_nbWorkers, 2) );
CHECK_Z( ZSTD_CCtx_setParameter(mtctx, ZSTD_c_compressionLevel, 1) );
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(3, "test%3u : compress %u bytes with 2 threads : ", testNb++, (unsigned)CNBuffSize);
@@ -1678,9 +1935,9 @@ static int basicUnitTests(U32 const seed, double compressibility)
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(3, "test%3i : compress -T2 with checksum : ", testNb++);
CHECK( ZSTD_CCtx_setParameter(mtctx, ZSTD_c_checksumFlag, 1) );
CHECK( ZSTD_CCtx_setParameter(mtctx, ZSTD_c_contentSizeFlag, 1) );
CHECK( ZSTD_CCtx_setParameter(mtctx, ZSTD_c_overlapLog, 3) );
CHECK_Z( ZSTD_CCtx_setParameter(mtctx, ZSTD_c_checksumFlag, 1) );
CHECK_Z( ZSTD_CCtx_setParameter(mtctx, ZSTD_c_contentSizeFlag, 1) );
CHECK_Z( ZSTD_CCtx_setParameter(mtctx, ZSTD_c_overlapLog, 3) );
CHECK_VAR(cSize, ZSTD_compress2(mtctx,
compressedBuffer, compressedBufferSize,
CNBuffer, CNBuffSize) );
@@ -1699,11 +1956,11 @@ static int basicUnitTests(U32 const seed, double compressibility)
ZSTD_DCtx* const dctx = ZSTD_createDCtx();
char out[32];
if (cctx == NULL || dctx == NULL) goto _output_error;
CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_contentSizeFlag, 0) );
CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_contentSizeFlag, 0) );
CHECK_VAR(cSize, ZSTD_compress2(cctx, out, sizeof(out), NULL, 0) );
DISPLAYLEVEL(3, "OK (%u bytes)\n", (unsigned)cSize);
CHECK( ZSTD_DCtx_setParameter(dctx, ZSTD_d_windowLogMax, 10) );
CHECK_Z( ZSTD_DCtx_setParameter(dctx, ZSTD_d_windowLogMax, 10) );
{ char const* outPtr = out;
ZSTD_inBuffer inBuffer = { outPtr, cSize, 0 };
ZSTD_outBuffer outBuffer = { NULL, 0, 0 };
@@ -1718,9 +1975,9 @@ static int basicUnitTests(U32 const seed, double compressibility)
DISPLAYLEVEL(3, "test%3i : compress with block splitting : ", testNb++)
{ ZSTD_CCtx* cctx = ZSTD_createCCtx();
CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_useBlockSplitter, ZSTD_ps_enable) );
CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_useBlockSplitter, ZSTD_ps_enable) );
cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize);
CHECK(cSize);
CHECK_Z(cSize);
ZSTD_freeCCtx(cctx);
}
DISPLAYLEVEL(3, "OK \n");
@@ -1728,13 +1985,13 @@ static int basicUnitTests(U32 const seed, double compressibility)
DISPLAYLEVEL(3, "test%3i : compress -T2 with/without literals compression : ", testNb++)
{ ZSTD_CCtx* cctx = ZSTD_createCCtx();
size_t cSize1, cSize2;
CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 1) );
CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 2) );
CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 1) );
CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 2) );
cSize1 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize);
CHECK(cSize1);
CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_literalCompressionMode, ZSTD_ps_disable) );
CHECK_Z(cSize1);
CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_literalCompressionMode, ZSTD_ps_disable) );
cSize2 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize);
CHECK(cSize2);
CHECK_Z(cSize2);
CHECK_LT(cSize1, cSize2);
ZSTD_freeCCtx(cctx);
}
@@ -1745,10 +2002,10 @@ static int basicUnitTests(U32 const seed, double compressibility)
/* Set rsyncable and don't give the ZSTD_compressBound(CNBuffSize) so
* ZSTDMT is forced to not take the shortcut.
*/
CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 1) );
CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 1) );
CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_rsyncable, 1) );
CHECK( ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize - 1, CNBuffer, CNBuffSize) );
CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 1) );
CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 1) );
CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_rsyncable, 1) );
CHECK_Z( ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize - 1, CNBuffer, CNBuffSize) );
ZSTD_freeCCtx(cctx);
}
DISPLAYLEVEL(3, "OK \n");
@@ -1758,22 +2015,22 @@ static int basicUnitTests(U32 const seed, double compressibility)
int const jobSize = 512 KB;
int value;
/* Check that the overlap log and job size are unset. */
CHECK( ZSTD_CCtxParams_getParameter(params, ZSTD_c_overlapLog, &value) );
CHECK_Z( ZSTD_CCtxParams_getParameter(params, ZSTD_c_overlapLog, &value) );
CHECK_EQ(value, 0);
CHECK( ZSTD_CCtxParams_getParameter(params, ZSTD_c_jobSize, &value) );
CHECK_Z( ZSTD_CCtxParams_getParameter(params, ZSTD_c_jobSize, &value) );
CHECK_EQ(value, 0);
/* Set and check the overlap log and job size. */
CHECK( ZSTD_CCtxParams_setParameter(params, ZSTD_c_overlapLog, 5) );
CHECK( ZSTD_CCtxParams_setParameter(params, ZSTD_c_jobSize, jobSize) );
CHECK( ZSTD_CCtxParams_getParameter(params, ZSTD_c_overlapLog, &value) );
CHECK_Z( ZSTD_CCtxParams_setParameter(params, ZSTD_c_overlapLog, 5) );
CHECK_Z( ZSTD_CCtxParams_setParameter(params, ZSTD_c_jobSize, jobSize) );
CHECK_Z( ZSTD_CCtxParams_getParameter(params, ZSTD_c_overlapLog, &value) );
CHECK_EQ(value, 5);
CHECK( ZSTD_CCtxParams_getParameter(params, ZSTD_c_jobSize, &value) );
CHECK_Z( ZSTD_CCtxParams_getParameter(params, ZSTD_c_jobSize, &value) );
CHECK_EQ(value, jobSize);
/* Set the number of workers and check the overlap log and job size. */
CHECK( ZSTD_CCtxParams_setParameter(params, ZSTD_c_nbWorkers, 2) );
CHECK( ZSTD_CCtxParams_getParameter(params, ZSTD_c_overlapLog, &value) );
CHECK_Z( ZSTD_CCtxParams_setParameter(params, ZSTD_c_nbWorkers, 2) );
CHECK_Z( ZSTD_CCtxParams_getParameter(params, ZSTD_c_overlapLog, &value) );
CHECK_EQ(value, 5);
CHECK( ZSTD_CCtxParams_getParameter(params, ZSTD_c_jobSize, &value) );
CHECK_Z( ZSTD_CCtxParams_getParameter(params, ZSTD_c_jobSize, &value) );
CHECK_EQ(value, jobSize);
ZSTD_freeCCtxParams(params);
}
@@ -1884,8 +2141,8 @@ static int basicUnitTests(U32 const seed, double compressibility)
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(3, "test%3i : load dictionary into context : ", testNb++);
CHECK( ZSTD_compressBegin_usingDict(ctxOrig, CNBuffer, dictSize, 2) );
CHECK( ZSTD_copyCCtx(ctxDuplicated, ctxOrig, 0) ); /* Begin_usingDict implies unknown srcSize, so match that */
CHECK_Z( ZSTD_compressBegin_usingDict(ctxOrig, CNBuffer, dictSize, 2) );
CHECK_Z( ZSTD_copyCCtx(ctxDuplicated, ctxOrig, 0) ); /* Begin_usingDict implies unknown srcSize, so match that */
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(3, "test%3i : compress with flat dictionary : ", testNb++);
@@ -1945,8 +2202,8 @@ static int basicUnitTests(U32 const seed, double compressibility)
DISPLAYLEVEL(3, "test%3i : check content size on duplicated context : ", testNb++);
{ size_t const testSize = CNBuffSize / 3;
CHECK( ZSTD_compressBegin(ctxOrig, ZSTD_defaultCLevel()) );
CHECK( ZSTD_copyCCtx(ctxDuplicated, ctxOrig, testSize) );
CHECK_Z( ZSTD_compressBegin(ctxOrig, ZSTD_defaultCLevel()) );
CHECK_Z( ZSTD_copyCCtx(ctxDuplicated, ctxOrig, testSize) );
CHECK_VAR(cSize, ZSTD_compressEnd(ctxDuplicated, compressedBuffer, ZSTD_compressBound(testSize),
(const char*)CNBuffer + dictSize, testSize) );
@@ -2346,6 +2603,27 @@ static int basicUnitTests(U32 const seed, double compressibility)
}
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(3, "test%3d : bufferless api with cdict : ", testNb++);
{ ZSTD_CDict* const cdict = ZSTD_createCDict(dictBuffer, dictSize, 1);
ZSTD_DCtx* const dctx = ZSTD_createDCtx();
ZSTD_frameParameters const fParams = { 0, 1, 0 };
size_t cBlockSize;
cSize = 0;
CHECK_Z(ZSTD_compressBegin_usingCDict_advanced(cctx, cdict, fParams, ZSTD_CONTENTSIZE_UNKNOWN));
cBlockSize = ZSTD_compressContinue(cctx, (char*)compressedBuffer + cSize, compressedBufferSize - cSize, CNBuffer, 1000);
CHECK_Z(cBlockSize);
cSize += cBlockSize;
cBlockSize = ZSTD_compressEnd(cctx, (char*)compressedBuffer + cSize, compressedBufferSize - cSize, (char const*)CNBuffer + 2000, 1000);
CHECK_Z(cBlockSize);
cSize += cBlockSize;
CHECK_Z(ZSTD_decompress_usingDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, dictBuffer, dictSize));
ZSTD_freeCDict(cdict);
ZSTD_freeDCtx(dctx);
}
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(3, "test%3i : Building cdict w/ ZSTD_dct_fullDict on a good dictionary : ", testNb++);
{ ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize);
ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_fullDict, cParams, ZSTD_defaultCMem);
@@ -2586,6 +2864,90 @@ static int basicUnitTests(U32 const seed, double compressibility)
}
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(3, "test%3i : ZSTD_fast attach dictionary with hashLog = 25 and chainLog = 25 : ", testNb++);
{
ZSTD_CCtx_params* cctxParams = ZSTD_createCCtxParams();
ZSTD_customMem customMem = {NULL, NULL, NULL};
ZSTD_DCtx* dctx = ZSTD_createDCtx();
ZSTD_CDict* cdict;
CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_strategy, ZSTD_fast));
/* Set windowLog to 25 so hash/chain logs don't get sized down */
CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_windowLog, 25));
CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_hashLog, 25));
CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_chainLog, 25));
/* Set srcSizeHint to 2^25 so hash/chain logs don't get sized down */
CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_srcSizeHint, 1u << 25));
cdict = ZSTD_createCDict_advanced2(dictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_auto, cctxParams, customMem);
CHECK_Z(ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters));
CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_forceAttachDict, ZSTD_dictForceAttach));
CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1));
CHECK_Z(ZSTD_CCtx_refCDict(cctx, cdict));
cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize);
CHECK_Z(cSize);
CHECK_Z(ZSTD_decompress_usingDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, dictBuffer, dictSize));
ZSTD_freeCDict(cdict);
ZSTD_freeDCtx(dctx);
ZSTD_freeCCtxParams(cctxParams);
}
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(3, "test%3i : ZSTD_dfast attach dictionary with hashLog = 25 and chainLog = 25 : ", testNb++);
{
ZSTD_CCtx_params* cctxParams = ZSTD_createCCtxParams();
ZSTD_customMem customMem = {NULL, NULL, NULL};
ZSTD_DCtx* dctx = ZSTD_createDCtx();
ZSTD_CDict* cdict;
CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_strategy, ZSTD_dfast));
/* Set windowLog to 25 so hash/chain logs don't get sized down */
CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_windowLog, 25));
CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_hashLog, 25));
CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_chainLog, 25));
/* Set srcSizeHint to 2^25 so hash/chain logs don't get sized down */
CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_srcSizeHint, 1u << 25));
cdict = ZSTD_createCDict_advanced2(dictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_auto, cctxParams, customMem);
CHECK_Z(ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters));
CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_forceAttachDict, ZSTD_dictForceAttach));
CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1));
CHECK_Z(ZSTD_CCtx_refCDict(cctx, cdict));
cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize);
CHECK_Z(cSize);
CHECK_Z(ZSTD_decompress_usingDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, dictBuffer, dictSize));
ZSTD_freeCDict(cdict);
ZSTD_freeDCtx(dctx);
ZSTD_freeCCtxParams(cctxParams);
}
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(3, "test%3i : ZSTD_lazy attach dictionary with hashLog = 29 and searchLog = 4 : ", testNb++);
if (MEM_64bits()) {
ZSTD_CCtx_params* cctxParams = ZSTD_createCCtxParams();
ZSTD_customMem customMem = {NULL, NULL, NULL};
ZSTD_DCtx* dctx = ZSTD_createDCtx();
ZSTD_CDict* cdict;
CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_strategy, ZSTD_lazy));
/* Force enable row based match finder, and disable dedicated dict search. */
CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_useRowMatchFinder, ZSTD_ps_enable));
CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_enableDedicatedDictSearch, 0));
CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_searchLog, 4));
/* Set windowLog to 29 so hash/chain logs don't get sized down */
CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_windowLog, 29));
CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_hashLog, 29));
/* Set srcSizeHint to 2^29 so hash/chain logs don't get sized down */
CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_srcSizeHint, 1u << 29));
cdict = ZSTD_createCDict_advanced2(dictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_auto, cctxParams, customMem);
CHECK_Z(ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters));
CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_forceAttachDict, ZSTD_dictForceAttach));
CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1));
CHECK_Z(ZSTD_CCtx_refCDict(cctx, cdict));
cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize);
CHECK_Z(cSize);
CHECK_Z(ZSTD_decompress_usingDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, dictBuffer, dictSize));
ZSTD_freeCDict(cdict);
ZSTD_freeDCtx(dctx);
ZSTD_freeCCtxParams(cctxParams);
}
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(3, "test%3i : Dictionary with non-default repcodes : ", testNb++);
{ U32 u; for (u=0; u<nbSamples; u++) samplesSizes[u] = sampleUnitSize; }
dictSize = ZDICT_trainFromBuffer(dictBuffer, dictSize,
@@ -2780,7 +3142,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
size_t const wrongSrcSize = (srcSize + 1000);
ZSTD_parameters params = ZSTD_getParams(1, wrongSrcSize, 0);
params.fParams.contentSizeFlag = 1;
CHECK( ZSTD_compressBegin_advanced(cctx, NULL, 0, params, wrongSrcSize) );
CHECK_Z( ZSTD_compressBegin_advanced(cctx, NULL, 0, params, wrongSrcSize) );
{ size_t const result = ZSTD_compressEnd(cctx, decodedBuffer, CNBuffSize, CNBuffer, srcSize);
if (!ZSTD_isError(result)) goto _output_error;
if (ZSTD_getErrorCode(result) != ZSTD_error_srcSize_wrong) goto _output_error;
@@ -2800,7 +3162,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
CNBuffer, srcSize, compressionLevel);
if (ZSTD_isError(cSize_1pass)) goto _output_error;
CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, compressionLevel) );
CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, compressionLevel) );
{ size_t const compressionResult = ZSTD_compress2(cctx,
compressedBuffer, compressedBufferSize,
CNBuffer, srcSize);
@@ -2819,13 +3181,13 @@ static int basicUnitTests(U32 const seed, double compressibility)
{ ZSTD_CCtx* const cctx = ZSTD_createCCtx();
DISPLAYLEVEL(3, "test%3i : parameters in order : ", testNb++);
assert(cctx != NULL);
CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 2) );
CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable) );
CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, 18) );
CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 2) );
CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable) );
CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, 18) );
{ size_t const compressedSize = ZSTD_compress2(cctx,
compressedBuffer, ZSTD_compressBound(inputSize),
CNBuffer, inputSize);
CHECK(compressedSize);
CHECK_Z(compressedSize);
cSize = compressedSize;
xxh64 = XXH64(compressedBuffer, compressedSize, 0);
}
@@ -2835,13 +3197,13 @@ static int basicUnitTests(U32 const seed, double compressibility)
{ ZSTD_CCtx* cctx = ZSTD_createCCtx();
DISPLAYLEVEL(3, "test%3i : parameters disordered : ", testNb++);
CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, 18) );
CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable) );
CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 2) );
CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, 18) );
CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable) );
CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 2) );
{ size_t const result = ZSTD_compress2(cctx,
compressedBuffer, ZSTD_compressBound(inputSize),
CNBuffer, inputSize);
CHECK(result);
CHECK_Z(result);
if (result != cSize) goto _output_error; /* must result in same compressed result, hence same size */
if (XXH64(compressedBuffer, result, 0) != xxh64) goto _output_error; /* must result in exactly same content, hence same hash */
DISPLAYLEVEL(3, "OK (compress : %u -> %u bytes)\n", (unsigned)inputSize, (unsigned)result);
@@ -2856,7 +3218,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
DISPLAYLEVEL(3, "test%3i : get dParameter bounds ", testNb++);
{ ZSTD_bounds const bounds = ZSTD_dParam_getBounds(ZSTD_d_windowLogMax);
CHECK(bounds.error);
CHECK_Z(bounds.error);
}
DISPLAYLEVEL(3, "OK \n");
@@ -2890,7 +3252,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
/* basic block compression */
DISPLAYLEVEL(3, "test%3i : magic-less format test : ", testNb++);
CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_format, ZSTD_f_zstd1_magicless) );
CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_format, ZSTD_f_zstd1_magicless) );
{ ZSTD_inBuffer in = { CNBuffer, inputSize, 0 };
ZSTD_outBuffer out = { compressedBuffer, ZSTD_compressBound(inputSize), 0 };
size_t const result = ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end);
@@ -2908,7 +3270,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
DISPLAYLEVEL(3, "test%3i : decompress of magic-less frame : ", testNb++);
ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters);
CHECK( ZSTD_DCtx_setParameter(dctx, ZSTD_d_format, ZSTD_f_zstd1_magicless) );
CHECK_Z( ZSTD_DCtx_setParameter(dctx, ZSTD_d_format, ZSTD_f_zstd1_magicless) );
{ ZSTD_frameHeader zfh;
size_t const zfhrt = ZSTD_getFrameHeader_advanced(&zfh, compressedBuffer, cSize, ZSTD_f_zstd1_magicless);
if (zfhrt != 0) goto _output_error;
@@ -2930,7 +3292,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
/* basic block compression */
DISPLAYLEVEL(3, "test%3i : empty magic-less format test : ", testNb++);
CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_format, ZSTD_f_zstd1_magicless) );
CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_format, ZSTD_f_zstd1_magicless) );
{ ZSTD_inBuffer in = { CNBuffer, 0, 0 };
ZSTD_outBuffer out = { compressedBuffer, ZSTD_compressBound(0), 0 };
size_t const result = ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end);
@@ -2942,7 +3304,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
DISPLAYLEVEL(3, "test%3i : decompress of empty magic-less frame : ", testNb++);
ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters);
CHECK( ZSTD_DCtx_setParameter(dctx, ZSTD_d_format, ZSTD_f_zstd1_magicless) );
CHECK_Z( ZSTD_DCtx_setParameter(dctx, ZSTD_d_format, ZSTD_f_zstd1_magicless) );
/* one shot */
{ size_t const result = ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize);
if (result != 0) goto _output_error;
@@ -2977,13 +3339,13 @@ static int basicUnitTests(U32 const seed, double compressibility)
int check;
if (ZSTD_isError(bounds.error))
continue;
CHECK(ZSTD_DCtx_getParameter(dctx, dParam, &value1));
CHECK_Z(ZSTD_DCtx_getParameter(dctx, dParam, &value1));
value2 = (value1 != bounds.lowerBound) ? bounds.lowerBound : bounds.upperBound;
CHECK(ZSTD_DCtx_setParameter(dctx, dParam, value2));
CHECK(ZSTD_DCtx_getParameter(dctx, dParam, &check));
CHECK_Z(ZSTD_DCtx_setParameter(dctx, dParam, value2));
CHECK_Z(ZSTD_DCtx_getParameter(dctx, dParam, &check));
if (check != value2) goto _output_error;
CHECK(ZSTD_DCtx_reset(dctx, ZSTD_reset_parameters));
CHECK(ZSTD_DCtx_getParameter(dctx, dParam, &check));
CHECK_Z(ZSTD_DCtx_reset(dctx, ZSTD_reset_parameters));
CHECK_Z(ZSTD_DCtx_getParameter(dctx, dParam, &check));
if (check != value1) goto _output_error;
}
ZSTD_freeDCtx(dctx);
@@ -3000,21 +3362,21 @@ static int basicUnitTests(U32 const seed, double compressibility)
/* basic block compression */
DISPLAYLEVEL(3, "test%3i : Block compression test : ", testNb++);
CHECK( ZSTD_compressBegin(cctx, 5) );
CHECK( ZSTD_getBlockSize(cctx) >= blockSize);
CHECK_Z( ZSTD_compressBegin(cctx, 5) );
CHECK_Z( ZSTD_getBlockSize(cctx) >= blockSize);
CHECK_VAR(cSize, ZSTD_compressBlock(cctx, compressedBuffer, ZSTD_compressBound(blockSize), CNBuffer, blockSize) );
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(3, "test%3i : Block decompression test : ", testNb++);
CHECK( ZSTD_decompressBegin(dctx) );
CHECK_Z( ZSTD_decompressBegin(dctx) );
{ CHECK_NEWV(r, ZSTD_decompressBlock(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize) );
if (r != blockSize) goto _output_error; }
DISPLAYLEVEL(3, "OK \n");
/* very long stream of block compression */
DISPLAYLEVEL(3, "test%3i : Huge block streaming compression test : ", testNb++);
CHECK( ZSTD_compressBegin(cctx, -199) ); /* we just want to quickly overflow internal U32 index */
CHECK( ZSTD_getBlockSize(cctx) >= blockSize);
CHECK_Z( ZSTD_compressBegin(cctx, -199) ); /* we just want to quickly overflow internal U32 index */
CHECK_Z( ZSTD_getBlockSize(cctx) >= blockSize);
{ U64 const toCompress = 5000000000ULL; /* > 4 GB */
U64 compressed = 0;
while (compressed < toCompress) {
@@ -3027,7 +3389,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
/* dictionary block compression */
DISPLAYLEVEL(3, "test%3i : Dictionary Block compression test : ", testNb++);
CHECK( ZSTD_compressBegin_usingDict(cctx, CNBuffer, dictSize, 5) );
CHECK_Z( ZSTD_compressBegin_usingDict(cctx, CNBuffer, dictSize, 5) );
CHECK_VAR(cSize, ZSTD_compressBlock(cctx, compressedBuffer, ZSTD_compressBound(blockSize), (char*)CNBuffer+dictSize, blockSize));
RDG_genBuffer((char*)CNBuffer+dictSize+blockSize, blockSize, 0.0, 0.0, seed); /* create a non-compressible second block */
{ CHECK_NEWV(r, ZSTD_compressBlock(cctx, (char*)compressedBuffer+cSize, ZSTD_compressBound(blockSize), (char*)CNBuffer+dictSize+blockSize, blockSize) ); /* for cctx history consistency */
@@ -3038,7 +3400,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(3, "test%3i : Dictionary Block decompression test : ", testNb++);
CHECK( ZSTD_decompressBegin_usingDict(dctx, CNBuffer, dictSize) );
CHECK_Z( ZSTD_decompressBegin_usingDict(dctx, CNBuffer, dictSize) );
{ CHECK_NEWV( r, ZSTD_decompressBlock(dctx, decodedBuffer, blockSize, compressedBuffer, cSize) );
if (r != blockSize) {
DISPLAYLEVEL(1, "ZSTD_decompressBlock() with _usingDict() fails : %u, instead of %u expected \n", (unsigned)r, (unsigned)blockSize);
@@ -3057,8 +3419,8 @@ static int basicUnitTests(U32 const seed, double compressibility)
DISPLAYLEVEL(3, "test%3i : Block compression with CDict : ", testNb++);
{ ZSTD_CDict* const cdict = ZSTD_createCDict(CNBuffer, dictSize, 3);
if (cdict==NULL) goto _output_error;
CHECK( ZSTD_compressBegin_usingCDict(cctx, cdict) );
CHECK( ZSTD_compressBlock(cctx, compressedBuffer, ZSTD_compressBound(blockSize), (char*)CNBuffer+dictSize, blockSize) );
CHECK_Z( ZSTD_compressBegin_usingCDict(cctx, cdict) );
CHECK_Z( ZSTD_compressBlock(cctx, compressedBuffer, ZSTD_compressBound(blockSize), (char*)CNBuffer+dictSize, blockSize) );
ZSTD_freeCDict(cdict);
}
DISPLAYLEVEL(3, "OK \n");
@@ -3227,7 +3589,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
size_t const bound = ZSTD_compressBound(_3BYTESTESTLENGTH);
size_t nbSeq = 1;
while (nbSeq <= maxNbSeq) {
CHECK(ZSTD_compressCCtx(cctx, compressedBuffer, bound, CNBuffer, nbSeq * 3, 19));
CHECK_Z(ZSTD_compressCCtx(cctx, compressedBuffer, bound, CNBuffer, nbSeq * 3, 19));
/* Check every sequence for the first 100, then skip more rapidly. */
if (nbSeq < 100) {
++nbSeq;
@@ -3256,7 +3618,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
size_t const bound = ZSTD_compressBound(CNBuffSize);
size_t size = 1;
while (size <= CNBuffSize) {
CHECK(ZSTD_compressCCtx(cctx, compressedBuffer, bound, CNBuffer, size, 3));
CHECK_Z(ZSTD_compressCCtx(cctx, compressedBuffer, bound, CNBuffer, size, 3));
/* Check every size for the first 100, then skip more rapidly. */
if (size < 100) {
++size;
@@ -3291,7 +3653,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
void* const outputBuffer = malloc(outputSize);
ZSTD_CCtx* const cctx = ZSTD_createCCtx();
if (!outputBuffer || !cctx) goto _output_error;
CHECK(ZSTD_compress_usingDict(cctx, outputBuffer, outputSize, CNBuffer, inputSize, dictBuffer, dictSize, 1));
CHECK_Z(ZSTD_compress_usingDict(cctx, outputBuffer, outputSize, CNBuffer, inputSize, dictBuffer, dictSize, 1));
free(outputBuffer);
ZSTD_freeCCtx(cctx);
}
@@ -3378,7 +3740,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
DISPLAYLEVEL(3, "test%3i : testing bitwise intrinsics PR#3045: ", testNb++);
{
U32 seed_copy = seed; // need non-const seed to avoid compiler warning for FUZ_rand(&seed)
U32 seed_copy = seed; /* need non-const seed to avoid compiler warning for FUZ_rand(&seed) */
U32 rand32 = FUZ_rand(&seed_copy);
U64 rand64 = ((U64)FUZ_rand(&seed_copy) << 32) | FUZ_rand(&seed_copy);
U32 lowbit_only_32 = 1;
@@ -3386,8 +3748,8 @@ static int basicUnitTests(U32 const seed, double compressibility)
U32 highbit_only_32 = (U32)1 << 31;
U64 highbit_only_64 = (U64)1 << 63;
U32 i;
if (rand32 == 0) rand32 = 1; // CLZ and CTZ are undefined on 0
if (rand64 == 0) rand64 = 1; // CLZ and CTZ are undefined on 0
if (rand32 == 0) rand32 = 1; /* CLZ and CTZ are undefined on 0 */
if (rand64 == 0) rand64 = 1; /* CLZ and CTZ are undefined on 0 */
/* Test ZSTD_countTrailingZeros32 */
CHECK_EQ(ZSTD_countTrailingZeros32(lowbit_only_32), 0u);
@@ -3628,7 +3990,7 @@ static int longUnitTests(U32 const seed, double compressibility)
while (approxIndex <= (maxIndex / 4) * 3) {
CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush));
approxIndex += in.pos;
CHECK(in.pos == in.size);
CHECK_Z(in.pos == in.size);
in.pos = 0;
out.pos = 0;
}
@@ -3654,7 +4016,7 @@ static int longUnitTests(U32 const seed, double compressibility)
while (approxIndex <= maxIndex) {
CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush));
approxIndex += in.pos;
CHECK(in.pos == in.size);
CHECK_Z(in.pos == in.size);
in.pos = 0;
out.pos = 0;
}
@@ -3737,7 +4099,7 @@ static int longUnitTests(U32 const seed, double compressibility)
RDG_genBuffer(dict, dictSize, 0.5, 0.5, seed);
RDG_genBuffer(CNBuffer, CNBuffSize, 0.6, 0.6, seed);
CHECK(cctx_params != NULL);
CHECK_Z(cctx_params != NULL);
for (dictSize = CNBuffSize; dictSize; dictSize = dictSize >> 3) {
DISPLAYLEVEL(3, "\n Testing with dictSize %u ", (U32)dictSize);
@@ -3780,11 +4142,6 @@ _end:
free(compressedBuffer);
free(decodedBuffer);
return testResult;
_output_error:
testResult = 1;
DISPLAY("Error detected in Unit tests ! \n");
goto _end;
}