Merge branch 'master' of https://github.com/bimbashrestha/zstd into named_pipes

This commit is contained in:
Bimba Shrestha
2019-10-25 15:43:41 -07:00
52 changed files with 1468 additions and 691 deletions
+4 -3
View File
@@ -250,7 +250,8 @@ clean:
$(MAKE) -C $(ZSTDDIR) clean
$(MAKE) -C $(PRGDIR) clean
@$(RM) -fR $(TESTARTEFACT)
@$(RM) -f core *.o tmp* *.tmp result* *.gcda dictionary *.zst \
@$(RM) -rf tmp* # some test directories are named tmp*
@$(RM) core *.o *.tmp result* *.gcda dictionary *.zst \
$(PRGDIR)/zstd$(EXT) $(PRGDIR)/zstd32$(EXT) \
fullbench$(EXT) fullbench32$(EXT) \
fullbench-lib$(EXT) fullbench-dll$(EXT) \
@@ -264,7 +265,7 @@ clean:
#----------------------------------------------------------------------------------
#make valgrindTest is validated only for Linux, macOS, BSD, Hurd and Solaris targets
# valgrind tests are validated only for some posix platforms
#----------------------------------------------------------------------------------
ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS))
HOST_OS = POSIX
@@ -286,7 +287,7 @@ valgrindTest: zstd datagen fuzzer fullbench
endif
ifneq (,$(filter MSYS%,$(shell uname)))
ifneq (,$(filter MINGW% MSYS%,$(shell uname)))
HOST_OS = MSYS
endif
+4 -5
View File
@@ -45,7 +45,6 @@
#define NBLOOPS 6
#define TIMELOOP_S 2
#define KNUTH 2654435761U
#define MAX_MEM (1984 MB)
#define DEFAULT_CLEVEL 1
@@ -450,7 +449,7 @@ static int benchMem(unsigned benchNb,
case 31: /* ZSTD_decodeLiteralsBlock : starts literals block in dstBuff2 */
{ size_t frameHeaderSize;
g_cSize = ZSTD_compress(dstBuff, dstBuffSize, src, srcSize, cLevel);
frameHeaderSize = ZSTD_frameHeaderSize(dstBuff, ZSTD_FRAMEHEADERSIZE_PREFIX);
frameHeaderSize = ZSTD_frameHeaderSize(dstBuff, ZSTD_FRAMEHEADERSIZE_PREFIX(ZSTD_f_zstd1));
CONTROL(!ZSTD_isError(frameHeaderSize));
/* check block is compressible, hence contains a literals section */
{ blockProperties_t bp;
@@ -471,10 +470,10 @@ static int benchMem(unsigned benchNb,
const BYTE* ip = dstBuff;
const BYTE* iend;
{ size_t const cSize = ZSTD_compress(dstBuff, dstBuffSize, src, srcSize, cLevel);
CONTROL(cSize > ZSTD_FRAMEHEADERSIZE_PREFIX);
CONTROL(cSize > ZSTD_FRAMEHEADERSIZE_PREFIX(ZSTD_f_zstd1));
}
/* Skip frame Header */
{ size_t const frameHeaderSize = ZSTD_frameHeaderSize(dstBuff, ZSTD_FRAMEHEADERSIZE_PREFIX);
{ size_t const frameHeaderSize = ZSTD_frameHeaderSize(dstBuff, ZSTD_FRAMEHEADERSIZE_PREFIX(ZSTD_f_zstd1));
CONTROL(!ZSTD_isError(frameHeaderSize));
ip += frameHeaderSize;
}
@@ -722,7 +721,7 @@ static int usage_advanced(const char* exename)
DISPLAY( "\nAdvanced options :\n");
DISPLAY( " -b# : test only function # \n");
DISPLAY( " -l# : benchmark functions at that compression level (default : %i)\n", DEFAULT_CLEVEL);
DISPLAY( " --zstd : custom parameter selection. Format same as zstdcli \n");
DISPLAY( "--zstd= : custom parameter selection. Format same as zstdcli \n");
DISPLAY( " -P# : sample compressibility (default : %.1f%%)\n", COMPRESSIBILITY_DEFAULT * 100);
DISPLAY( " -B# : sample size (default : %u)\n", (unsigned)kSampleSizeDefault);
DISPLAY( " -i# : iteration loops [1-9](default : %i)\n", NBLOOPS);
+3 -2
View File
@@ -125,13 +125,14 @@ static size_t compress(uint8_t *dst, size_t capacity,
int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size)
{
size_t neededBufSize;
/* Give a random portion of src data to the producer, to use for
parameter generation. The rest will be used for (de)compression */
FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(src, size);
size = FUZZ_dataProducer_reserveDataPrefix(producer);
size_t neededBufSize;
neededBufSize = ZSTD_compressBound(size) * 5;
neededBufSize = ZSTD_compressBound(size) * 15;
/* Allocate all buffers and contexts if not already allocated */
if (neededBufSize > bufSize) {
+83 -5
View File
@@ -304,6 +304,28 @@ static int FUZ_mallocTests(unsigned seed, double compressibility, unsigned part)
#endif
static void FUZ_decodeSequences(BYTE* dst, ZSTD_Sequence* seqs, size_t seqsSize, BYTE* src, size_t size)
{
size_t i;
size_t j;
for(i = 0; i < seqsSize - 1; ++i) {
assert(dst + seqs[i].litLength + seqs[i].matchLength < dst + size);
assert(src + seqs[i].litLength + seqs[i].matchLength < src + size);
memcpy(dst, src, seqs[i].litLength);
dst += seqs[i].litLength;
src += seqs[i].litLength;
size -= seqs[i].litLength;
for (j = 0; j < seqs[i].matchLength; ++j)
dst[j] = dst[j - seqs[i].offset];
dst += seqs[i].matchLength;
src += seqs[i].matchLength;
size -= seqs[i].matchLength;
}
memcpy(dst, src, size);
}
/*=============================================
* Unit tests
=============================================*/
@@ -1365,7 +1387,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
}
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(3, "test%3i : Building cdict w/ ZSTD_dm_fullDict on a good dictionary : ", testNb++);
DISPLAYLEVEL(3, "test%3i : Building cdict w/ ZSTD_dct_fullDict on a good dictionary : ", testNb++);
{ ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize);
ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_fullDict, cParams, ZSTD_defaultCMem);
if (cdict==NULL) goto _output_error;
@@ -1373,7 +1395,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
}
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(3, "test%3i : Building cdict w/ ZSTD_dm_fullDict on a rawContent (must fail) : ", testNb++);
DISPLAYLEVEL(3, "test%3i : Building cdict w/ ZSTD_dct_fullDict on a rawContent (must fail) : ", testNb++);
{ ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize);
ZSTD_CDict* const cdict = ZSTD_createCDict_advanced((const char*)dictBuffer+1, dictSize-1, ZSTD_dlm_byRef, ZSTD_dct_fullDict, cParams, ZSTD_defaultCMem);
if (cdict!=NULL) goto _output_error;
@@ -1381,7 +1403,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
}
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(3, "test%3i : Loading rawContent starting with dict header w/ ZSTD_dm_auto should fail : ", testNb++);
DISPLAYLEVEL(3, "test%3i : Loading rawContent starting with dict header w/ ZSTD_dct_auto should fail : ", testNb++);
{
size_t ret;
MEM_writeLE32((char*)dictBuffer+2, ZSTD_MAGIC_DICTIONARY);
@@ -1395,7 +1417,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
}
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(3, "test%3i : Loading rawContent starting with dict header w/ ZSTD_dm_rawContent should pass : ", testNb++);
DISPLAYLEVEL(3, "test%3i : Loading rawContent starting with dict header w/ ZSTD_dct_rawContent should pass : ", testNb++);
{
size_t ret;
MEM_writeLE32((char*)dictBuffer+2, ZSTD_MAGIC_DICTIONARY);
@@ -1864,6 +1886,36 @@ static int basicUnitTests(U32 const seed, double compressibility)
DISPLAYLEVEL(3, "streaming OK : regenerated %u bytes \n", (unsigned)out.pos);
}
/* basic block compression */
DISPLAYLEVEL(3, "test%3i : empty magic-less format test : ", testNb++);
CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_format, ZSTD_f_zstd1_magicless) );
{ ZSTD_inBuffer in = { CNBuffer, 0, 0 };
ZSTD_outBuffer out = { compressedBuffer, ZSTD_compressBound(0), 0 };
size_t const result = ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end);
if (result != 0) goto _output_error;
if (in.pos != in.size) goto _output_error;
cSize = out.pos;
}
DISPLAYLEVEL(3, "OK (compress : %u -> %u bytes)\n", (unsigned)0, (unsigned)cSize);
DISPLAYLEVEL(3, "test%3i : decompress of empty magic-less frame : ", testNb++);
ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters);
CHECK( ZSTD_DCtx_setParameter(dctx, ZSTD_d_format, ZSTD_f_zstd1_magicless) );
/* one shot */
{ size_t const result = ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize);
if (result != 0) goto _output_error;
DISPLAYLEVEL(3, "one-shot OK, ");
}
/* streaming */
{ ZSTD_inBuffer in = { compressedBuffer, cSize, 0 };
ZSTD_outBuffer out = { decodedBuffer, CNBuffSize, 0 };
size_t const result = ZSTD_decompressStream(dctx, &out, &in);
if (result != 0) goto _output_error;
if (in.pos != in.size) goto _output_error;
if (out.pos != 0) goto _output_error;
DISPLAYLEVEL(3, "streaming OK : regenerated %u bytes \n", (unsigned)out.pos);
}
ZSTD_freeCCtx(cctx);
ZSTD_freeDCtx(dctx);
}
@@ -1960,6 +2012,33 @@ static int basicUnitTests(U32 const seed, double compressibility)
DISPLAYLEVEL(3, "OK \n");
}
DISPLAYLEVEL(3, "test%3i : ZSTD_getSequences decode from sequences test : ", testNb++);
{
size_t srcSize = 100 KB;
BYTE* src = (BYTE*)CNBuffer;
BYTE* decoded = (BYTE*)compressedBuffer;
ZSTD_CCtx* cctx = ZSTD_createCCtx();
ZSTD_Sequence* seqs = (ZSTD_Sequence*)malloc(srcSize * sizeof(ZSTD_Sequence));
size_t seqsSize;
if (seqs == NULL) goto _output_error;
assert(cctx != NULL);
/* Populate src with random data */
RDG_genBuffer(CNBuffer, srcSize, compressibility, 0., seed);
/* get the sequences */
seqsSize = ZSTD_getSequences(cctx, seqs, srcSize, src, srcSize);
/* "decode" and compare the sequences */
FUZ_decodeSequences(decoded, seqs, seqsSize, src, srcSize);
assert(!memcmp(CNBuffer, compressedBuffer, srcSize));
ZSTD_freeCCtx(cctx);
free(seqs);
}
/* Multiple blocks of zeros test */
#define LONGZEROSLENGTH 1000000 /* 1MB of zeros */
DISPLAYLEVEL(3, "test%3i : compress %u zeroes : ", testNb++, LONGZEROSLENGTH);
@@ -1972,7 +2051,6 @@ static int basicUnitTests(U32 const seed, double compressibility)
if (r != LONGZEROSLENGTH) goto _output_error; }
DISPLAYLEVEL(3, "OK \n");
/* All zeroes test (test bug #137) */
#define ZEROESLENGTH 100
DISPLAYLEVEL(3, "test%3i : compress %u zeroes : ", testNb++, ZEROESLENGTH);
+41 -22
View File
@@ -64,11 +64,12 @@ PRGDIR="$SCRIPT_DIR/../programs"
TESTDIR="$SCRIPT_DIR/../tests"
UNAME=$(uname)
isTerminal=false
detectedTerminal=false
if [ -t 0 ] && [ -t 1 ]
then
isTerminal=true
detectedTerminal=true
fi
isTerminal=${isTerminal:-$detectedTerminal}
isWindows=false
INTOVOID="/dev/null"
@@ -211,8 +212,8 @@ $ZSTD tmp -c --no-compress-literals -19 | $ZSTD -t
$ZSTD tmp -c --compress-literals -1 | $ZSTD -t
$ZSTD tmp -c --compress-literals --fast=1 | $ZSTD -t
$ZSTD tmp -c --compress-literals -19 | $ZSTD -t
$ZSTD -b --fast=1 -i1e1 tmp --compress-literals
$ZSTD -b --fast=1 -i1e1 tmp --no-compress-literals
$ZSTD -b --fast=1 -i0e1 tmp --compress-literals
$ZSTD -b --fast=1 -i0e1 tmp --no-compress-literals
println "test : file removal"
$ZSTD -f --rm tmp
@@ -239,7 +240,7 @@ rm tmp # erase source file
touch tmp.zst # create destination file
$ZSTD -f tmp && die "attempt to compress a non existing file"
test -f tmp.zst # destination file should still be present
rm tmp*
rm -rf tmp* # may also erase tmp* directory from previous failed run
println "\n===> decompression only tests "
head -c 1048576 /dev/zero > tmp
@@ -269,6 +270,28 @@ if [ "$?" -eq 139 ]; then
fi
rm tmp*
println "test : compress multiple files into an output directory, --output-dir-flat"
println henlo > tmp1
mkdir tmpInputTestDir
mkdir tmpInputTestDir/we
mkdir tmpInputTestDir/we/must
mkdir tmpInputTestDir/we/must/go
mkdir tmpInputTestDir/we/must/go/deeper
println cool > tmpInputTestDir/we/must/go/deeper/tmp2
mkdir tmpOutDir
$ZSTD tmp1 tmpInputTestDir/we/must/go/deeper/tmp2 --output-dir-flat tmpOutDir
test -f tmpOutDir/tmp1.zst
test -f tmpOutDir/tmp2.zst
println "test : decompress multiple files into an output directory, --output-dir-flat"
mkdir tmpOutDirDecomp
$ZSTD tmpOutDir -r -d --output-dir-flat tmpOutDirDecomp
test -f tmpOutDirDecomp/tmp2
test -f tmpOutDirDecomp/tmp1
rm -f tmpOutDirDecomp/*
$ZSTD tmpOutDir -r -d --output-dir-flat=tmpOutDirDecomp
test -f tmpOutDirDecomp/tmp2
test -f tmpOutDirDecomp/tmp1
rm -rf tmp*
println "\n===> Advanced compression parameters "
println "Hello world!" | $ZSTD --zstd=windowLog=21, - -o tmp.zst && die "wrong parameters not detected!"
@@ -412,7 +435,6 @@ ls -ls tmp* # check size of tmpdec (should be 2*(tmp1 + tmp2 + tmp3))
println "compress multiple files including a missing one (notHere) : "
$ZSTD -f tmp1 notHere tmp2 && die "missing file not detected!"
println "\n===> stream-size mode"
./datagen -g11000 > tmp
@@ -560,30 +582,27 @@ println "- Create dictionary with short dictID"
$ZSTD --train-fastcover=k=46,d=8,f=15,split=80 "$TESTDIR"/*.c "$PRGDIR"/*.c --dictID=1 -o tmpDict1
cmp tmpDict tmpDict1 && die "dictionaries should have different ID !"
println "- Create dictionaries with shrink-dict flag enabled"
$ZSTD --train-fastcover=steps=256,shrink "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpShrinkDict
$ZSTD --train-fastcover=steps=256,shrink=1 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpShrinkDict1
$ZSTD --train-fastcover=steps=256,shrink=5 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpShrinkDict2
$ZSTD --train-fastcover=steps=1,shrink "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpShrinkDict
$ZSTD --train-fastcover=steps=1,shrink=1 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpShrinkDict1
$ZSTD --train-fastcover=steps=1,shrink=5 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpShrinkDict2
println "- Create dictionary with size limit"
$ZSTD --train-fastcover=steps=8 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict2 --maxdict=4K
println "- Compare size of dictionary from 90% training samples with 80% training samples"
$ZSTD --train-fastcover=split=90 -r "$TESTDIR"/*.c "$PRGDIR"/*.c
$ZSTD --train-fastcover=split=80 -r "$TESTDIR"/*.c "$PRGDIR"/*.c
$ZSTD --train-fastcover=steps=1 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict2 --maxdict=4K
println "- Create dictionary using all samples for both training and testing"
$ZSTD --train-fastcover=split=100 -r "$TESTDIR"/*.c "$PRGDIR"/*.c
$ZSTD --train-fastcover=k=56,d=8,split=100 -r "$TESTDIR"/*.c "$PRGDIR"/*.c
println "- Create dictionary using f=16"
$ZSTD --train-fastcover=f=16 -r "$TESTDIR"/*.c "$PRGDIR"/*.c
$ZSTD --train-fastcover=accel=15 -r "$TESTDIR"/*.c "$PRGDIR"/*.c && die "Created dictionary using accel=15"
$ZSTD --train-fastcover=k=56,d=8,f=16 -r "$TESTDIR"/*.c "$PRGDIR"/*.c
$ZSTD --train-fastcover=k=56,d=8,accel=15 -r "$TESTDIR"/*.c "$PRGDIR"/*.c && die "Created dictionary using accel=15"
println "- Create dictionary using accel=2"
$ZSTD --train-fastcover=accel=2 -r "$TESTDIR"/*.c "$PRGDIR"/*.c
$ZSTD --train-fastcover=k=56,d=8,accel=2 -r "$TESTDIR"/*.c "$PRGDIR"/*.c
println "- Create dictionary using accel=10"
$ZSTD --train-fastcover=accel=10 -r "$TESTDIR"/*.c "$PRGDIR"/*.c
$ZSTD --train-fastcover=k=56,d=8,accel=10 -r "$TESTDIR"/*.c "$PRGDIR"/*.c
println "- Create dictionary with multithreading"
$ZSTD --train-fastcover -T4 -r "$TESTDIR"/*.c "$PRGDIR"/*.c
println "- Test -o before --train-fastcover"
rm -f tmpDict dictionary
$ZSTD -o tmpDict --train-fastcover "$TESTDIR"/*.c "$PRGDIR"/*.c
$ZSTD -o tmpDict --train-fastcover=k=56,d=8 "$TESTDIR"/*.c "$PRGDIR"/*.c
test -f tmpDict
$ZSTD --train-fastcover "$TESTDIR"/*.c "$PRGDIR"/*.c
$ZSTD --train-fastcover=k=56,d=8 "$TESTDIR"/*.c "$PRGDIR"/*.c
test -f dictionary
rm tmp* dictionary
@@ -657,10 +676,10 @@ $ZSTD -i0b0e3 tmp1
println "bench negative level"
$ZSTD -bi0 --fast tmp1
println "with recursive and quiet modes"
$ZSTD -rqi1b1e2 tmp1
$ZSTD -rqi0b1e2 tmp1
println "benchmark decompression only"
$ZSTD -f tmp1
$ZSTD -b -d -i1 tmp1.zst
$ZSTD -b -d -i0 tmp1.zst
println "\n===> zstd compatibility tests "
+1 -1
View File
@@ -46,7 +46,7 @@ static int testOrder(size_t numThreads, size_t queueSize)
POOL_ctx* const ctx = POOL_create(numThreads, queueSize);
ASSERT_TRUE(ctx);
data.i = 0;
(void)ZSTD_pthread_mutex_init(&data.mutex, NULL);
ASSERT_FALSE(ZSTD_pthread_mutex_init(&data.mutex, NULL));
{ size_t i;
for (i = 0; i < 16; ++i) {
POOL_add(ctx, &fn, &data);