ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

modify sequence compression api fuzzer

Browse Source
This commit is contained in:
Danielle Rozenblit
2023-01-23 07:55:11 -08:00
parent 23a356cdde
commit 638d502002
16 changed files with 587 additions and 199 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/decodecorpus.c
+16 -11
View File
@@ -25,21 +25,13 @@
#include "zdict.h"
/* Direct access to internal compression functions is required */
#include "zstd_compress.c" /* ZSTD_resetSeqStore, ZSTD_storeSeq, *_TO_OFFBASE, HIST_countFast_wksp, HIST_isError */
#include "compress/zstd_compress.c" /* ZSTD_resetSeqStore, ZSTD_storeSeq, *_TO_OFFBASE, HIST_countFast_wksp, HIST_isError */
#define XXH_STATIC_LINKING_ONLY
#include "xxhash.h" /* XXH64 */
#ifndef MIN
#define MIN(a, b) ((a) < (b) ? (a) : (b))
#endif
#ifndef MAX_PATH
#ifdef PATH_MAX
#define MAX_PATH PATH_MAX
#else
#define MAX_PATH 256
#endif
#if !(defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */))
# define inline /* disable */
#endif
/*-************************************
@@ -71,6 +63,7 @@ static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER;
} \
} while (0)
/*-*******************************************************
* Random function
*********************************************************/
@@ -176,6 +169,14 @@ const char* BLOCK_TYPES[] = {"raw", "rle", "compressed"};
#define MIN_SEQ_LEN (3)
#define MAX_NB_SEQ ((ZSTD_BLOCKSIZE_MAX + MIN_SEQ_LEN - 1) / MIN_SEQ_LEN)
#ifndef MAX_PATH
#ifdef PATH_MAX
#define MAX_PATH PATH_MAX
#else
#define MAX_PATH 256
#endif
#endif
BYTE CONTENT_BUFFER[MAX_DECOMPRESSED_SIZE];
BYTE FRAME_BUFFER[MAX_DECOMPRESSED_SIZE * 2];
BYTE LITERAL_BUFFER[ZSTD_BLOCKSIZE_MAX];
@@ -241,6 +242,10 @@ typedef enum {
gt_block, /* generate compressed blocks without block/frame headers */
} genType_e;
#ifndef MIN
#define MIN(a, b) ((a) < (b) ? (a) : (b))
#endif
/*-*******************************************************
* Global variables (set from command line)
*********************************************************/
tests/external_matchfinder.h
+1 -1
View File
@@ -32,4 +32,4 @@ size_t zstreamExternalMatchFinder(
size_t windowSize
);
#endif // EXTERNAL_MATCHFINDER
#endif /* EXTERNAL_MATCHFINDER */
tests/fuzz/sequence_compression_api.c
+105 -54
View File
@@ -22,6 +22,7 @@
#include <stdio.h>
#include <string.h>
#include <time.h>
#include <sys/mman.h>
#include "fuzz_helpers.h"
#include "zstd_helpers.h"
#include "fuzz_data_producer.h"
@@ -32,11 +33,17 @@ static void* literalsBuffer = NULL;
static void* generatedSrc = NULL;
static ZSTD_Sequence* generatedSequences = NULL;
static void* dictBuffer = NULL;
static ZSTD_CDict* cdict = NULL;
static ZSTD_DDict* ddict = NULL;
#define ZSTD_FUZZ_GENERATED_SRC_MAXSIZE (1 << 20) /* Allow up to 1MB generated data */
#define ZSTD_FUZZ_GENERATED_LITERALS_SIZE (1 << 20) /* Fixed size 1MB literals buffer */
#define ZSTD_FUZZ_MATCHLENGTH_MAXSIZE (1 << 18) /* Allow up to 256KB matches */
#define ZSTD_FUZZ_GENERATED_DICT_MAXSIZE (1 << 18) /* Allow up to a 256KB dict */
#define ZSTD_FUZZ_GENERATED_LITERALS_SIZE (1 << 18) /* Fixed size 256KB literals buffer */
#define ZSTD_FUZZ_GENERATED_DICT_MAXSIZE (1 << ZSTD_WINDOWLOG_MAX_32) /* Allow up to 1 << ZSTD_WINDOWLOG_MAX_32 dictionary */
#define ZSTD_FUZZ_MAX_NBSEQ (1 << 17) /* Maximum of 128K sequences */
#define ZSTD_FUZZ_DICT_FILE "sequence_fuzz_dictionary"
/* Deterministic random number generator */
#define FUZZ_RDG_rotl32(x,r) ((x << r) | (x >> (32 - r)))
@@ -55,9 +62,9 @@ static uint32_t FUZZ_RDG_rand(uint32_t* src)
/* Make a pseudorandom string - this simple function exists to avoid
* taking a dependency on datagen.h to have RDG_genBuffer().
*/
static char* generatePseudoRandomString(char* str, size_t size) {
static char* generatePseudoRandomString(char* str, size_t size, FUZZ_dataProducer_t* producer) {
const char charset[] = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJK1234567890!@#$^&*()_";
uint32_t seed = 0;
uint32_t seed = FUZZ_dataProducer_uint32(producer);
if (size) {
for (size_t n = 0; n < size; n++) {
int key = FUZZ_RDG_rand(&seed) % (int) (sizeof charset - 1);
@@ -67,6 +74,26 @@ static char* generatePseudoRandomString(char* str, size_t size) {
return str;
}
/*
* Create large dictionary file
*/
static void generateDictFile(size_t size, FUZZ_dataProducer_t* producer) {
char c;
FILE *dictFile;
const char charset[] = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJK1234567890!@#$^&*()_";
uint32_t seed = FUZZ_dataProducer_uint32(producer);
dictFile = fopen(ZSTD_FUZZ_DICT_FILE, "w");
FUZZ_ASSERT(dictFile);
while (size) {
c = FUZZ_RDG_rand(&seed) % (int) (sizeof charset - 1);
fputc(c, dictFile);
size--;
}
fclose(dictFile);
}
/* Returns size of source buffer */
static size_t decodeSequences(void* dst, size_t nbSequences,
size_t literalsSize,
@@ -100,14 +127,14 @@ static size_t decodeSequences(void* dst, size_t nbSequences,
size_t j = 0;
size_t k = 0;
if (dictSize != 0) {
if (generatedSequences[i].offset > bytesWritten) {
/* Offset goes into the dictionary */
size_t offsetFromEndOfDict = generatedSequences[i].offset - bytesWritten;
for (; k < offsetFromEndOfDict && k < matchLength; ++k) {
op[k] = dictPtr[dictSize - offsetFromEndOfDict + k];
if (generatedSequences[i].offset > bytesWritten) { /* Offset goes into the dictionary */
size_t dictOffset = generatedSequences[i].offset - bytesWritten;
size_t matchInDict = MIN(matchLength, dictOffset);
for (; k < matchInDict; ++k) {
op[k] = dictPtr[dictSize - dictOffset + k];
}
matchLength -= k;
op += k;
matchLength -= matchInDict;
op += matchInDict;
}
}
for (; j < matchLength; ++j) {
@@ -138,9 +165,9 @@ static size_t generateRandomSequences(FUZZ_dataProducer_t* producer,
size_t literalsSizeLimit, size_t dictSize,
size_t windowLog, ZSTD_sequenceFormat_e mode)
{
const uint32_t repCode = 0; /* not used by sequence ingestion api */
const uint32_t windowSize = 1 << windowLog;
const uint32_t blockSizeMax = MIN(128 << 10, 1 << windowLog);
const uint32_t repCode = 0; /* Not used by sequence ingestion api */
size_t windowSize = 1ULL << windowLog;
size_t blockSizeMax = MIN(ZSTD_BLOCKSIZE_MAX, windowSize);
uint32_t matchLengthMax = ZSTD_FUZZ_MATCHLENGTH_MAXSIZE;
uint32_t bytesGenerated = 0;
uint32_t nbSeqGenerated = 0;
@@ -148,12 +175,12 @@ static size_t generateRandomSequences(FUZZ_dataProducer_t* producer,
uint32_t blockSize = 0;
if (mode == ZSTD_sf_explicitBlockDelimiters) {
/* ensure that no sequence can be larger than one block */
/* Ensure that no sequence can be larger than one block */
literalsSizeLimit = MIN(literalsSizeLimit, blockSizeMax/2);
matchLengthMax = MIN(matchLengthMax, blockSizeMax/2);
}
while ( nbSeqGenerated < ZSTD_FUZZ_MAX_NBSEQ - 3 /* extra room for explicit delimiters */
while ( nbSeqGenerated < ZSTD_FUZZ_MAX_NBSEQ - 3 /* Extra room for explicit delimiters */
&& bytesGenerated < ZSTD_FUZZ_GENERATED_SRC_MAXSIZE
&& !FUZZ_dataProducer_empty(producer)) {
uint32_t matchLength;
@@ -210,38 +237,31 @@ static size_t generateRandomSequences(FUZZ_dataProducer_t* producer,
}
generatedSequences[nbSeqGenerated++] = seq;
isFirstSequence = 0;
} }
}
}
if (mode == ZSTD_sf_explicitBlockDelimiters) {
/* always end sequences with a block delimiter */
const ZSTD_Sequence endBlock = {0, 0, 0, 0};
assert(nbSeqGenerated < ZSTD_FUZZ_MAX_NBSEQ);
generatedSequences[nbSeqGenerated++] = endBlock;
}
return nbSeqGenerated;
}
static size_t roundTripTest(void* result, size_t resultCapacity,
void* compressed, size_t compressedCapacity,
const void* src, size_t srcSize,
const void* dict, size_t dictSize,
const ZSTD_Sequence* seqs, size_t seqSize,
int wLog, int cLevel, unsigned hasDict,
unsigned hasDict,
ZSTD_sequenceFormat_e mode)
{
size_t cSize;
size_t dSize;
ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 0);
ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, cLevel);
ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, wLog);
ZSTD_CCtx_setParameter(cctx, ZSTD_c_minMatch, ZSTD_MINMATCH_MIN);
ZSTD_CCtx_setParameter(cctx, ZSTD_c_validateSequences, 1);
ZSTD_CCtx_setParameter(cctx, ZSTD_c_blockDelimiters, mode);
if (hasDict) {
FUZZ_ZASSERT(ZSTD_CCtx_loadDictionary(cctx, dict, dictSize));
FUZZ_ZASSERT(ZSTD_DCtx_loadDictionary(dctx, dict, dictSize));
FUZZ_ZASSERT(ZSTD_CCtx_refCDict(cctx, cdict));
FUZZ_ZASSERT(ZSTD_DCtx_refDDict(dctx, ddict));
}
cSize = ZSTD_compressSequences(cctx, compressed, compressedCapacity,
@@ -272,7 +292,6 @@ int LLVMFuzzerTestOneInput(const uint8_t* src, size_t size)
size_t cBufSize;
size_t generatedSrcSize;
size_t nbSequences;
void* dictBuffer = NULL;
size_t dictSize = 0;
unsigned hasDict;
unsigned wLog;
@@ -281,23 +300,66 @@ int LLVMFuzzerTestOneInput(const uint8_t* src, size_t size)
FUZZ_dataProducer_t* const producer = FUZZ_dataProducer_create(src, size);
FUZZ_ASSERT(producer);
if (literalsBuffer == NULL) {
if (!cctx) {
cctx = ZSTD_createCCtx();
FUZZ_ASSERT(cctx);
}
if (!dctx) {
dctx = ZSTD_createDCtx();
FUZZ_ASSERT(dctx);
}
/* Generate window log first so we don't generate offsets too large */
wLog = FUZZ_dataProducer_uint32Range(producer, ZSTD_WINDOWLOG_MIN, ZSTD_WINDOWLOG_MAX);
cLevel = FUZZ_dataProducer_int32Range(producer, -3, 22);
mode = (ZSTD_sequenceFormat_e)FUZZ_dataProducer_int32Range(producer, 0, 1);
ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 0);
ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, cLevel);
ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, wLog);
ZSTD_CCtx_setParameter(cctx, ZSTD_c_minMatch, ZSTD_MINMATCH_MIN);
ZSTD_CCtx_setParameter(cctx, ZSTD_c_validateSequences, 1);
ZSTD_CCtx_setParameter(cctx, ZSTD_c_blockDelimiters, mode);
ZSTD_CCtx_setParameter(cctx, ZSTD_c_forceAttachDict, ZSTD_dictForceAttach);
if (!literalsBuffer) {
literalsBuffer = FUZZ_malloc(ZSTD_FUZZ_GENERATED_LITERALS_SIZE);
FUZZ_ASSERT(literalsBuffer);
literalsBuffer = generatePseudoRandomString(literalsBuffer, ZSTD_FUZZ_GENERATED_LITERALS_SIZE);
literalsBuffer = generatePseudoRandomString(literalsBuffer, ZSTD_FUZZ_GENERATED_LITERALS_SIZE, producer);
}
if (!dictBuffer) { /* Generate global dictionary buffer */
FILE* dictFile;
ZSTD_compressionParameters cParams;
/* Generate a large dictionary file and mmap to buffer */
generateDictFile(ZSTD_FUZZ_GENERATED_DICT_MAXSIZE, producer);
dictFile = fopen(ZSTD_FUZZ_DICT_FILE, "r");
dictBuffer = mmap(NULL, ZSTD_FUZZ_GENERATED_DICT_MAXSIZE, PROT_READ, MAP_PRIVATE, fileno(dictFile), 0);
FUZZ_ASSERT(dictBuffer);
fclose(dictFile);
/* Create global cdict and ddict*/
cParams = ZSTD_getCParams(1, ZSTD_FUZZ_GENERATED_SRC_MAXSIZE, ZSTD_FUZZ_GENERATED_DICT_MAXSIZE);
cParams.minMatch = ZSTD_MINMATCH_MIN;
cParams.hashLog = ZSTD_HASHLOG_MIN;
cParams.chainLog = ZSTD_CHAINLOG_MIN;
cdict = ZSTD_createCDict_advanced(dictBuffer, ZSTD_FUZZ_GENERATED_DICT_MAXSIZE, ZSTD_dlm_byRef, ZSTD_dct_rawContent, cParams, ZSTD_defaultCMem);
ddict = ZSTD_createDDict_advanced(dictBuffer, ZSTD_FUZZ_GENERATED_DICT_MAXSIZE, ZSTD_dlm_byRef, ZSTD_dct_rawContent, ZSTD_defaultCMem);
FUZZ_ASSERT(cdict);
FUZZ_ASSERT(ddict);
}
FUZZ_ASSERT(cdict);
FUZZ_ASSERT(ddict);
hasDict = FUZZ_dataProducer_uint32Range(producer, 0, 1);
if (hasDict) {
dictSize = FUZZ_dataProducer_uint32Range(producer, 1, ZSTD_FUZZ_GENERATED_DICT_MAXSIZE);
dictBuffer = FUZZ_malloc(dictSize);
FUZZ_ASSERT(dictBuffer);
dictBuffer = generatePseudoRandomString(dictBuffer, dictSize);
dictSize = ZSTD_FUZZ_GENERATED_DICT_MAXSIZE;
}
/* Generate window log first so we don't generate offsets too large */
wLog = FUZZ_dataProducer_uint32Range(producer, ZSTD_WINDOWLOG_MIN, ZSTD_WINDOWLOG_MAX_32);
cLevel = FUZZ_dataProducer_int32Range(producer, -3, 22);
mode = (ZSTD_sequenceFormat_e)FUZZ_dataProducer_int32Range(producer, 0, 1);
if (!generatedSequences) {
generatedSequences = FUZZ_malloc(sizeof(ZSTD_Sequence)*ZSTD_FUZZ_MAX_NBSEQ);
@@ -305,8 +367,10 @@ int LLVMFuzzerTestOneInput(const uint8_t* src, size_t size)
if (!generatedSrc) {
generatedSrc = FUZZ_malloc(ZSTD_FUZZ_GENERATED_SRC_MAXSIZE);
}
nbSequences = generateRandomSequences(producer, ZSTD_FUZZ_GENERATED_LITERALS_SIZE, dictSize, wLog, mode);
generatedSrcSize = decodeSequences(generatedSrc, nbSequences, ZSTD_FUZZ_GENERATED_LITERALS_SIZE, dictBuffer, dictSize, mode);
/* Note : in explicit block delimiters mode,
* the fuzzer might generate a lot of small blocks.
* In which case, the final compressed size might be > ZSTD_compressBound().
@@ -318,30 +382,17 @@ int LLVMFuzzerTestOneInput(const uint8_t* src, size_t size)
rBufSize = generatedSrcSize;
rBuf = FUZZ_malloc(rBufSize);
if (!cctx) {
cctx = ZSTD_createCCtx();
FUZZ_ASSERT(cctx);
}
if (!dctx) {
dctx = ZSTD_createDCtx();
FUZZ_ASSERT(dctx);
}
{ const size_t result = roundTripTest(rBuf, rBufSize,
cBuf, cBufSize,
generatedSrc, generatedSrcSize,
dictBuffer, dictSize,
generatedSequences, nbSequences,
(int)wLog, cLevel, hasDict, mode);
hasDict, mode);
FUZZ_ASSERT(result <= generatedSrcSize); /* can be 0 when no round-trip */
}
free(rBuf);
free(cBuf);
FUZZ_dataProducer_free(producer);
if (hasDict) {
free(dictBuffer);
}
#ifndef STATEFUL_FUZZING
ZSTD_freeCCtx(cctx); cctx = NULL;
ZSTD_freeDCtx(dctx); dctx = NULL;
tests/fuzz/zstd_helpers.c
+1
View File
@@ -115,6 +115,7 @@ void FUZZ_setRandomParameters(ZSTD_CCtx *cctx, size_t srcSize, FUZZ_dataProducer
setRand(cctx, ZSTD_c_useBlockSplitter, 0, 2, producer);
setRand(cctx, ZSTD_c_deterministicRefPrefix, 0, 1, producer);
setRand(cctx, ZSTD_c_prefetchCDictTables, 0, 2, producer);
setRand(cctx, ZSTD_c_maxBlockSize, ZSTD_BLOCKSIZE_MAX_MIN, ZSTD_BLOCKSIZE_MAX, producer);
if (FUZZ_dataProducer_uint32Range(producer, 0, 1) == 0) {
setRand(cctx, ZSTD_c_srcSizeHint, ZSTD_SRCSIZEHINT_MIN, 2 * srcSize, producer);
}
tests/fuzzer.c
+101 -12
View File
@@ -25,7 +25,8 @@
#include <stdlib.h> /* free */
#include <stdio.h> /* fgets, sscanf */
#include <string.h> /* strcmp */
#undef NDEBUG
#include <time.h> /* time(), time_t */
#undef NDEBUG /* always enable assert() */
#include <assert.h>
#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_compressContinue, ZSTD_compressBlock */
#include "debug.h" /* DEBUG_STATIC_ASSERT */
@@ -476,7 +477,7 @@ static void test_compressBound(unsigned tnb)
CHECK_EQ(ZSTD_compressBound(w), ZSTD_COMPRESSBOUND(w));
} }
// Ensure error if srcSize too big
/* Ensure error if srcSize too big */
{ size_t const w = ZSTD_MAX_INPUT_SIZE + 1;
CHECK(ZSTD_isError(ZSTD_compressBound(w))); /* must fail */
CHECK_EQ(ZSTD_COMPRESSBOUND(w), 0);
@@ -489,7 +490,7 @@ static void test_decompressBound(unsigned tnb)
{
DISPLAYLEVEL(3, "test%3u : decompressBound : ", tnb);
// Simple compression, with size : should provide size;
/* Simple compression, with size : should provide size; */
{ const char example[] = "abcd";
char cBuffer[ZSTD_COMPRESSBOUND(sizeof(example))];
size_t const cSize = ZSTD_compress(cBuffer, sizeof(cBuffer), example, sizeof(example), 0);
@@ -497,7 +498,7 @@ static void test_decompressBound(unsigned tnb)
CHECK_EQ(ZSTD_decompressBound(cBuffer, cSize), (unsigned long long)sizeof(example));
}
// Simple small compression without size : should provide 1 block size
/* Simple small compression without size : should provide 1 block size */
{ char cBuffer[ZSTD_COMPRESSBOUND(0)];
ZSTD_outBuffer out = { cBuffer, sizeof(cBuffer), 0 };
ZSTD_inBuffer in = { NULL, 0, 0 };
@@ -510,14 +511,14 @@ static void test_decompressBound(unsigned tnb)
ZSTD_freeCCtx(cctx);
}
// Attempt to overflow 32-bit intermediate multiplication result
// This requires dBound >= 4 GB, aka 2^32.
// This requires 2^32 / 2^17 = 2^15 blocks
// => create 2^15 blocks (can be empty, or just 1 byte).
/* Attempt to overflow 32-bit intermediate multiplication result
* This requires dBound >= 4 GB, aka 2^32.
* This requires 2^32 / 2^17 = 2^15 blocks
* => create 2^15 blocks (can be empty, or just 1 byte). */
{ const char input[] = "a";
size_t const nbBlocks = (1 << 15) + 1;
size_t blockNb;
size_t const outCapacity = 1 << 18; // large margin
size_t const outCapacity = 1 << 18; /* large margin */
char* const outBuffer = malloc (outCapacity);
ZSTD_outBuffer out = { outBuffer, outCapacity, 0 };
ZSTD_CCtx* const cctx = ZSTD_createCCtx();
@@ -1779,6 +1780,94 @@ static int basicUnitTests(U32 const seed, double compressibility)
if (!ZSTD_isError(r)) goto _output_error;
}
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(3, "test%3i : test estimation functions with default cctx params : ", testNb++);
{
// Test ZSTD_estimateCCtxSize_usingCCtxParams
{
ZSTD_CCtx_params* params = ZSTD_createCCtxParams();
size_t const cctxSizeDefault = ZSTD_estimateCCtxSize_usingCCtxParams(params);
staticCCtx = ZSTD_initStaticCCtx(staticCCtxBuffer, cctxSizeDefault);
CHECK_VAR(cSize, ZSTD_compressCCtx(staticCCtx,
compressedBuffer, compressedBufferSize,
CNBuffer, CNBuffSize, 3));
{
size_t const r = ZSTD_decompressDCtx(staticDCtx,
decodedBuffer, CNBuffSize,
compressedBuffer, cSize);
if (r != CNBuffSize) goto _output_error;
if (memcmp(decodedBuffer, CNBuffer, CNBuffSize)) goto _output_error;
}
ZSTD_freeCCtxParams(params);
}
// Test ZSTD_estimateCStreamSize_usingCCtxParams
{
ZSTD_CCtx_params* params = ZSTD_createCCtxParams();
size_t const cctxSizeDefault = ZSTD_estimateCStreamSize_usingCCtxParams(params);
staticCCtx = ZSTD_initStaticCCtx(staticCCtxBuffer, cctxSizeDefault);
CHECK_VAR(cSize, ZSTD_compressCCtx(staticCCtx,
compressedBuffer, compressedBufferSize,
CNBuffer, CNBuffSize, 3) );
{
size_t const r = ZSTD_decompressDCtx(staticDCtx,
decodedBuffer, CNBuffSize,
compressedBuffer, cSize);
if (r != CNBuffSize) goto _output_error;
if (memcmp(decodedBuffer, CNBuffer, CNBuffSize)) goto _output_error;
}
ZSTD_freeCCtxParams(params);
}
}
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(3, "test%3i : test estimation functions with maxBlockSize = 0 : ", testNb++);
{
// Test ZSTD_estimateCCtxSize_usingCCtxParams
{
ZSTD_CCtx_params* params = ZSTD_createCCtxParams();
size_t cctxSizeDefault;
CHECK_Z(ZSTD_CCtxParams_setParameter(params, ZSTD_c_maxBlockSize, 0));
cctxSizeDefault = ZSTD_estimateCCtxSize_usingCCtxParams(params);
staticCCtx = ZSTD_initStaticCCtx(staticCCtxBuffer, cctxSizeDefault);
CHECK_VAR(cSize, ZSTD_compressCCtx(staticCCtx,
compressedBuffer, compressedBufferSize,
CNBuffer, CNBuffSize, 3) );
{
size_t const r = ZSTD_decompressDCtx(staticDCtx,
decodedBuffer, CNBuffSize,
compressedBuffer, cSize);
if (r != CNBuffSize) goto _output_error;
if (memcmp(decodedBuffer, CNBuffer, CNBuffSize)) goto _output_error;
}
ZSTD_freeCCtxParams(params);
}
// Test ZSTD_estimateCStreamSize_usingCCtxParams
{
ZSTD_CCtx_params* params = ZSTD_createCCtxParams();
size_t cctxSizeDefault;
CHECK_Z(ZSTD_CCtxParams_setParameter(params, ZSTD_c_maxBlockSize, 0));
cctxSizeDefault = ZSTD_estimateCStreamSize_usingCCtxParams(params);
staticCCtx = ZSTD_initStaticCCtx(staticCCtxBuffer, cctxSizeDefault);
CHECK_VAR(cSize, ZSTD_compressCCtx(staticCCtx,
compressedBuffer, compressedBufferSize,
CNBuffer, CNBuffSize, 3) );
{
size_t const r = ZSTD_decompressDCtx(staticDCtx,
decodedBuffer, CNBuffSize,
compressedBuffer, cSize);
if (r != CNBuffSize) goto _output_error;
if (memcmp(decodedBuffer, CNBuffer, CNBuffSize)) goto _output_error;
}
ZSTD_freeCCtxParams(params);
}
}
DISPLAYLEVEL(3, "OK \n");
}
free(staticCCtxBuffer);
free(staticDCtxBuffer);
@@ -3535,7 +3624,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
DISPLAYLEVEL(3, "test%3i : testing bitwise intrinsics PR#3045: ", testNb++);
{
U32 seed_copy = seed; // need non-const seed to avoid compiler warning for FUZ_rand(&seed)
U32 seed_copy = seed; /* need non-const seed to avoid compiler warning for FUZ_rand(&seed) */
U32 rand32 = FUZ_rand(&seed_copy);
U64 rand64 = ((U64)FUZ_rand(&seed_copy) << 32) | FUZ_rand(&seed_copy);
U32 lowbit_only_32 = 1;
@@ -3543,8 +3632,8 @@ static int basicUnitTests(U32 const seed, double compressibility)
U32 highbit_only_32 = (U32)1 << 31;
U64 highbit_only_64 = (U64)1 << 63;
U32 i;
if (rand32 == 0) rand32 = 1; // CLZ and CTZ are undefined on 0
if (rand64 == 0) rand64 = 1; // CLZ and CTZ are undefined on 0
if (rand32 == 0) rand32 = 1; /* CLZ and CTZ are undefined on 0 */
if (rand64 == 0) rand64 = 1; /* CLZ and CTZ are undefined on 0 */
/* Test ZSTD_countTrailingZeros32 */
CHECK_EQ(ZSTD_countTrailingZeros32(lowbit_only_32), 0u);
tests/test-zstd-versions.py
+25 -6
View File
@@ -29,8 +29,19 @@ test_dat_src = 'README.md'
test_dat = 'test_dat'
head = 'vdevel'
dict_source = 'dict_source'
dict_files = './zstd/programs/*.c ./zstd/lib/common/*.c ./zstd/lib/compress/*.c ./zstd/lib/decompress/*.c ./zstd/lib/dictBuilder/*.c ./zstd/lib/legacy/*.c '
dict_files += './zstd/programs/*.h ./zstd/lib/common/*.h ./zstd/lib/compress/*.h ./zstd/lib/dictBuilder/*.h ./zstd/lib/legacy/*.h'
dict_globs = [
'programs/*.c',
'lib/common/*.c',
'lib/compress/*.c',
'lib/decompress/*.c',
'lib/dictBuilder/*.c',
'lib/legacy/*.c',
'programs/*.h',
'lib/common/*.h',
'lib/compress/*.h',
'lib/dictBuilder/*.h',
'lib/legacy/*.h'
]
def execute(command, print_output=False, print_error=True, param_shell=False):
@@ -85,6 +96,7 @@ def create_dict(tag, dict_source_path):
result = execute('./zstd.' + tag + ' -f --train ' + ' '.join(cFiles) + ' ' + ' '.join(hFiles) + ' -o ' + dict_name, print_output=False, param_shell=True)
if result == 0:
print(dict_name + ' created')
assert os.path.isfile(dict_name)
else:
raise RuntimeError('ERROR: creating of ' + dict_name + ' failed')
else:
@@ -103,12 +115,15 @@ def zstd(tag, args, input_file, output_file):
print("Running: '{}', input={}, output={}" .format(
' '.join(cmd), input_file, output_file
))
subprocess.check_call(cmd, stdin=i, stdout=o)
result = subprocess.run(cmd, stdin=i, stdout=o, stderr=subprocess.PIPE)
print("Stderr: {}".format(result.stderr.decode("ascii")))
result.check_returncode()
def dict_compress_sample(tag, sample):
dict_name = 'dict.' + tag
zstd(tag, ['-D', dict_name, '-1'], sample, sample + '_01_64_' + tag + '_dictio.zst')
verbose = ['-v', '-v', '-v']
zstd(tag, ['-D', dict_name, '-1'] + verbose, sample, sample + '_01_64_' + tag + '_dictio.zst')
zstd(tag, ['-D', dict_name, '-3'], sample, sample + '_03_64_' + tag + '_dictio.zst')
zstd(tag, ['-D', dict_name, '-5'], sample, sample + '_05_64_' + tag + '_dictio.zst')
zstd(tag, ['-D', dict_name, '-9'], sample, sample + '_09_64_' + tag + '_dictio.zst')
@@ -246,8 +261,12 @@ if __name__ == '__main__':
# copy *.c and *.h to a temporary directory ("dict_source")
if not os.path.isdir(dict_source_path):
os.mkdir(dict_source_path)
print('cp ' + dict_files + ' ' + dict_source_path)
execute('cp ' + dict_files + ' ' + dict_source_path, param_shell=True)
for dict_glob in dict_globs:
files = glob.glob(dict_glob, root_dir=base_dir)
for file in files:
file = os.path.join(base_dir, file)
print("copying " + file + " to " + dict_source_path)
shutil.copy(file, dict_source_path)
print('-----------------------------------------------')
print('Compress test.dat by all released zstd')
tests/zstreamtest.c
+147 -7
View File
@@ -1846,15 +1846,11 @@ static int basicUnitTests(U32 seed, double compressibility, int bigTests)
CHECK(dstBuf == NULL || checkBuf == NULL, "allocation failed");
ZSTD_CCtx_reset(zc, ZSTD_reset_session_and_parameters);
CHECK_Z(ZSTD_CCtx_reset(zc, ZSTD_reset_session_and_parameters));
/* Reference external matchfinder outside the test loop to
* check that the reference is preserved across compressions */
ZSTD_registerExternalMatchFinder(
zc,
&externalMatchState,
zstreamExternalMatchFinder
);
ZSTD_registerExternalMatchFinder(zc, &externalMatchState, zstreamExternalMatchFinder);
for (enableFallback = 0; enableFallback < 1; enableFallback++) {
size_t testCaseId;
@@ -1916,16 +1912,160 @@ static int basicUnitTests(U32 seed, double compressibility, int bigTests)
}
/* Test that reset clears the external matchfinder */
ZSTD_CCtx_reset(zc, ZSTD_reset_session_and_parameters);
CHECK_Z(ZSTD_CCtx_reset(zc, ZSTD_reset_session_and_parameters));
externalMatchState = EMF_BIG_ERROR; /* ensure zstd will fail if the matchfinder wasn't cleared */
CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_enableMatchFinderFallback, 0));
CHECK_Z(ZSTD_compress2(zc, dstBuf, dstBufSize, CNBuffer, CNBufferSize));
/* Test that registering mFinder == NULL clears the external matchfinder */
ZSTD_CCtx_reset(zc, ZSTD_reset_session_and_parameters);
ZSTD_registerExternalMatchFinder(zc, &externalMatchState, zstreamExternalMatchFinder);
externalMatchState = EMF_BIG_ERROR; /* ensure zstd will fail if the matchfinder wasn't cleared */
CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_enableMatchFinderFallback, 0));
ZSTD_registerExternalMatchFinder(zc, NULL, NULL); /* clear the external matchfinder */
CHECK_Z(ZSTD_compress2(zc, dstBuf, dstBufSize, CNBuffer, CNBufferSize));
/* Test that external matchfinder doesn't interact with older APIs */
ZSTD_CCtx_reset(zc, ZSTD_reset_session_and_parameters);
ZSTD_registerExternalMatchFinder(zc, &externalMatchState, zstreamExternalMatchFinder);
externalMatchState = EMF_BIG_ERROR; /* ensure zstd will fail if the matchfinder is used */
CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_enableMatchFinderFallback, 0));
CHECK_Z(ZSTD_compressCCtx(zc, dstBuf, dstBufSize, CNBuffer, CNBufferSize, 3));
/* Test that compression returns the correct error with LDM */
CHECK_Z(ZSTD_CCtx_reset(zc, ZSTD_reset_session_and_parameters));
{
size_t res;
ZSTD_registerExternalMatchFinder(zc, &externalMatchState, zstreamExternalMatchFinder);
CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable));
res = ZSTD_compress2(zc, dstBuf, dstBufSize, CNBuffer, CNBufferSize);
CHECK(!ZSTD_isError(res), "EMF: Should have raised an error!");
CHECK(
ZSTD_getErrorCode(res) != ZSTD_error_parameter_combination_unsupported,
"EMF: Wrong error code: %s", ZSTD_getErrorName(res)
);
}
#ifdef ZSTD_MULTITHREAD
/* Test that compression returns the correct error with nbWorkers > 0 */
CHECK_Z(ZSTD_CCtx_reset(zc, ZSTD_reset_session_and_parameters));
{
size_t res;
ZSTD_registerExternalMatchFinder(zc, &externalMatchState, zstreamExternalMatchFinder);
CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_nbWorkers, 1));
res = ZSTD_compress2(zc, dstBuf, dstBufSize, CNBuffer, CNBufferSize);
CHECK(!ZSTD_isError(res), "EMF: Should have raised an error!");
CHECK(
ZSTD_getErrorCode(res) != ZSTD_error_parameter_combination_unsupported,
"EMF: Wrong error code: %s", ZSTD_getErrorName(res)
);
}
#endif
free(dstBuf);
free(checkBuf);
}
DISPLAYLEVEL(3, "OK \n");
/* Test maxBlockSize cctx param functionality */
DISPLAYLEVEL(3, "test%3i : Testing maxBlockSize PR#3418: ", testNb++);
{
ZSTD_CCtx* cctx = ZSTD_createCCtx();
/* Quick test to make sure maxBlockSize bounds are enforced */
assert(ZSTD_isError(ZSTD_CCtx_setParameter(cctx, ZSTD_c_maxBlockSize, ZSTD_BLOCKSIZE_MAX_MIN - 1)));
assert(ZSTD_isError(ZSTD_CCtx_setParameter(cctx, ZSTD_c_maxBlockSize, ZSTD_BLOCKSIZE_MAX + 1)));
/* Test maxBlockSize < windowSize and windowSize < maxBlockSize*/
{
size_t srcSize = 2 << 10;
void* const src = CNBuffer;
size_t dstSize = ZSTD_compressBound(srcSize);
void* const dst1 = compressedBuffer;
void* const dst2 = (BYTE*)compressedBuffer + dstSize;
size_t size1, size2;
void* const checkBuf = malloc(srcSize);
memset(src, 'x', srcSize);
/* maxBlockSize = 1KB */
CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_maxBlockSize, 1u << 10));
size1 = ZSTD_compress2(cctx, dst1, dstSize, src, srcSize);
if (ZSTD_isError(size1)) goto _output_error;
CHECK_Z(ZSTD_decompress(checkBuf, srcSize, dst1, size1));
CHECK(memcmp(src, checkBuf, srcSize) != 0, "Corruption!");
/* maxBlockSize = 3KB */
CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_maxBlockSize, 3u << 10));
size2 = ZSTD_compress2(cctx, dst2, dstSize, src, srcSize);
if (ZSTD_isError(size2)) goto _output_error;
CHECK_Z(ZSTD_decompress(checkBuf, srcSize, dst2, size2));
CHECK(memcmp(src, checkBuf, srcSize) != 0, "Corruption!");
assert(size1 - size2 == 4); /* We add another RLE block with header + character */
assert(memcmp(dst1, dst2, size2) != 0); /* Compressed output should not be equal */
/* maxBlockSize = 1KB, windowLog = 10 */
CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_maxBlockSize, 1u << 10));
CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, 10));
size1 = ZSTD_compress2(cctx, dst1, dstSize, src, srcSize);
if (ZSTD_isError(size1)) goto _output_error;
CHECK_Z(ZSTD_decompress(checkBuf, srcSize, dst1, size1));
CHECK(memcmp(src, checkBuf, srcSize) != 0, "Corruption!");
/* maxBlockSize = 3KB, windowLog = 10 */
CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_maxBlockSize, 3u << 10));
CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, 10));
size2 = ZSTD_compress2(cctx, dst2, dstSize, src, srcSize);
if (ZSTD_isError(size2)) goto _output_error;
CHECK_Z(ZSTD_decompress(checkBuf, srcSize, dst2, size2));
CHECK(memcmp(src, checkBuf, srcSize) != 0, "Corruption!");
assert(size1 == size2);
assert(memcmp(dst1, dst2, size1) == 0); /* Compressed output should be equal */
free(checkBuf);
}
ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
/* Test maxBlockSize = 0 is valid */
{ size_t srcSize = 256 << 10;
void* const src = CNBuffer;
size_t dstSize = ZSTD_compressBound(srcSize);
void* const dst1 = compressedBuffer;
void* const dst2 = (BYTE*)compressedBuffer + dstSize;
size_t size1, size2;
void* const checkBuf = malloc(srcSize);
/* maxBlockSize = 0 */
CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_maxBlockSize, 0));
size1 = ZSTD_compress2(cctx, dst1, dstSize, src, srcSize);
if (ZSTD_isError(size1)) goto _output_error;
CHECK_Z(ZSTD_decompress(checkBuf, srcSize, dst1, size1));
CHECK(memcmp(src, checkBuf, srcSize) != 0, "Corruption!");
/* maxBlockSize = ZSTD_BLOCKSIZE_MAX */
CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_maxBlockSize, ZSTD_BLOCKSIZE_MAX));
size2 = ZSTD_compress2(cctx, dst2, dstSize, src, srcSize);
if (ZSTD_isError(size2)) goto _output_error;
CHECK_Z(ZSTD_decompress(checkBuf, srcSize, dst2, size2));
CHECK(memcmp(src, checkBuf, srcSize) != 0, "Corruption!");
assert(size1 == size2);
assert(memcmp(dst1, dst2, size1) == 0); /* Compressed output should be equal */
free(checkBuf);
}
ZSTD_freeCCtx(cctx);
}
DISPLAYLEVEL(3, "OK \n");
_end:
FUZ_freeDictionary(dictionary);
ZSTD_freeCStream(zc);