ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'dev' into ahmed_file

Browse Source
This commit is contained in:
Yann Collet
2019-11-26 11:20:26 -08:00
parent 5e657aca90 d6e0a44576
commit c71bd45a3b
52 changed files with 1614 additions and 391 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/fuzz/.gitignore
+6
View File
@@ -2,7 +2,13 @@
corpora
block_decompress
block_round_trip
dictionary_decompress
dictionary_loader
dictionary_round_trip
simple_compress
simple_decompress
simple_round_trip
stream_decompress
stream_round_trip
zstd_frame_info
fuzz-*.log
tests/fuzz/dictionary_round_trip.c
+1 -1
View File
@@ -73,7 +73,7 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size)
size_t const rBufSize = size;
void* rBuf = malloc(rBufSize);
size_t cBufSize = ZSTD_compressBound(size);
size_t cBufSize = ZSTD_compressBound(size) * 2;
void *cBuf;
/* Half of the time fuzz with a 1 byte smaller output size.
* This will still succeed because we force the checksum to be disabled,
tests/fuzz/simple_round_trip.c
+1 -1
View File
@@ -48,7 +48,7 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size)
{
size_t const rBufSize = size;
void* rBuf = malloc(rBufSize);
size_t cBufSize = ZSTD_compressBound(size);
size_t cBufSize = ZSTD_compressBound(size) * 2;
void* cBuf;
/* Give a random portion of src data to the producer, to use for
tests/fuzz/zstd_helpers.c
+3
View File
@@ -96,6 +96,9 @@ void FUZZ_setRandomParameters(ZSTD_CCtx *cctx, size_t srcSize, FUZZ_dataProducer
if (FUZZ_dataProducer_uint32Range(producer, 0, 1) == 0) {
setRand(cctx, ZSTD_c_srcSizeHint, ZSTD_SRCSIZEHINT_MIN, 2 * srcSize, producer);
}
if (FUZZ_dataProducer_uint32Range(producer, 0, 1) == 0) {
setRand(cctx, ZSTD_c_targetCBlockSize, ZSTD_TARGETCBLOCKSIZE_MIN, ZSTD_TARGETCBLOCKSIZE_MAX, producer);
}
}
FUZZ_dict_t FUZZ_train(void const* src, size_t srcSize, FUZZ_dataProducer_t *producer)
tests/fuzzer.c
+62 -3
View File
@@ -490,6 +490,19 @@ static int basicUnitTests(U32 const seed, double compressibility)
}
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(3, "test%3i : compress a NULL input with each level : ", testNb++);
{ int level = -1;
ZSTD_CCtx* cctx = ZSTD_createCCtx();
if (!cctx) goto _output_error;
for (level = -1; level <= ZSTD_maxCLevel(); ++level) {
CHECK_Z( ZSTD_compress(compressedBuffer, compressedBufferSize, NULL, 0, level) );
CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, level) );
CHECK_Z( ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, NULL, 0) );
}
ZSTD_freeCCtx(cctx);
}
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(3, "test%3d : check CCtx size after compressing empty input : ", testNb++);
{ ZSTD_CCtx* const cctx = ZSTD_createCCtx();
size_t const r = ZSTD_compressCCtx(cctx, compressedBuffer, compressedBufferSize, NULL, 0, 19);
@@ -884,6 +897,28 @@ static int basicUnitTests(U32 const seed, double compressibility)
ZSTDMT_freeCCtx(mtctx);
}
DISPLAYLEVEL(3, "test%3u : compress empty string and decompress with small window log : ", testNb++);
{ ZSTD_CCtx* const cctx = ZSTD_createCCtx();
ZSTD_DCtx* const dctx = ZSTD_createDCtx();
char out[32];
if (cctx == NULL || dctx == NULL) goto _output_error;
CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_contentSizeFlag, 0) );
CHECK_VAR(cSize, ZSTD_compress2(cctx, out, sizeof(out), NULL, 0) );
DISPLAYLEVEL(3, "OK (%u bytes)\n", (unsigned)cSize);
CHECK( ZSTD_DCtx_setParameter(dctx, ZSTD_d_windowLogMax, 10) );
{ char const* outPtr = out;
ZSTD_inBuffer inBuffer = { outPtr, cSize, 0 };
ZSTD_outBuffer outBuffer = { NULL, 0, 0 };
size_t dSize;
CHECK_VAR(dSize, ZSTD_decompressStream(dctx, &outBuffer, &inBuffer) );
if (dSize != 0) goto _output_error;
}
ZSTD_freeDCtx(dctx);
ZSTD_freeCCtx(cctx);
}
DISPLAYLEVEL(3, "test%3i : compress -T2 with/without literals compression : ", testNb++)
{ ZSTD_CCtx* cctx = ZSTD_createCCtx();
size_t cSize1, cSize2;
@@ -1137,6 +1172,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
size_t* const samplesSizes = (size_t*) malloc(nbSamples * sizeof(size_t));
size_t dictSize;
U32 dictID;
size_t dictHeaderSize;
if (dictBuffer==NULL || samplesSizes==NULL) {
free(dictBuffer);
@@ -1226,6 +1262,29 @@ static int basicUnitTests(U32 const seed, double compressibility)
if (dictID==0) goto _output_error;
DISPLAYLEVEL(3, "OK : %u \n", (unsigned)dictID);
DISPLAYLEVEL(3, "test%3i : check dict header size no error : ", testNb++);
dictHeaderSize = ZDICT_getDictHeaderSize(dictBuffer, dictSize);
if (dictHeaderSize==0) goto _output_error;
DISPLAYLEVEL(3, "OK : %u \n", (unsigned)dictHeaderSize);
DISPLAYLEVEL(3, "test%3i : check dict header size correctness : ", testNb++);
{ unsigned char const dictBufferFixed[144] = { 0x37, 0xa4, 0x30, 0xec, 0x63, 0x00, 0x00, 0x00, 0x08, 0x10, 0x00, 0x1f,
0x0f, 0x00, 0x28, 0xe5, 0x03, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x80, 0x0f, 0x9e, 0x0f, 0x00, 0x00, 0x24, 0x40, 0x80, 0x00, 0x01,
0x02, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0xde, 0x08,
0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08,
0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08,
0x08, 0x08, 0x08, 0x08, 0xbc, 0xe1, 0x4b, 0x92, 0x0e, 0xb4, 0x7b, 0x18,
0x86, 0x61, 0x18, 0xc6, 0x18, 0x63, 0x8c, 0x31, 0xc6, 0x18, 0x63, 0x8c,
0x31, 0x66, 0x66, 0x66, 0x66, 0xb6, 0x6d, 0x01, 0x00, 0x00, 0x00, 0x04,
0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x20, 0x73, 0x6f, 0x64, 0x61,
0x6c, 0x65, 0x73, 0x20, 0x74, 0x6f, 0x72, 0x74, 0x6f, 0x72, 0x20, 0x65,
0x6c, 0x65, 0x69, 0x66, 0x65, 0x6e, 0x64, 0x2e, 0x20, 0x41, 0x6c, 0x69 };
dictHeaderSize = ZDICT_getDictHeaderSize(dictBufferFixed, 144);
if (dictHeaderSize != 115) goto _output_error;
}
DISPLAYLEVEL(3, "OK : %u \n", (unsigned)dictHeaderSize);
DISPLAYLEVEL(3, "test%3i : compress with dictionary : ", testNb++);
cSize = ZSTD_compress_usingDict(cctx, compressedBuffer, compressedBufferSize,
CNBuffer, CNBuffSize,
@@ -1999,14 +2058,14 @@ static int basicUnitTests(U32 const seed, double compressibility)
/* long rle test */
{ size_t sampleSize = 0;
size_t expectedCompressedSize = 39; /* block 1, 2: compressed, block 3: RLE, zstd 1.4.4 */
DISPLAYLEVEL(3, "test%3i : Long RLE test : ", testNb++);
RDG_genBuffer(CNBuffer, sampleSize, compressibility, 0., seed+1);
memset((char*)CNBuffer+sampleSize, 'B', 256 KB - 1);
sampleSize += 256 KB - 1;
RDG_genBuffer((char*)CNBuffer+sampleSize, 96 KB, compressibility, 0., seed+2);
memset((char*)CNBuffer+sampleSize, 'A', 96 KB);
sampleSize += 96 KB;
cSize = ZSTD_compress(compressedBuffer, ZSTD_compressBound(sampleSize), CNBuffer, sampleSize, 1);
if (ZSTD_isError(cSize)) goto _output_error;
if (ZSTD_isError(cSize) || cSize > expectedCompressedSize) goto _output_error;
{ CHECK_NEWV(regenSize, ZSTD_decompress(decodedBuffer, sampleSize, compressedBuffer, cSize));
if (regenSize!=sampleSize) goto _output_error; }
DISPLAYLEVEL(3, "OK \n");
tests/playTests.sh
+33 -12
View File
@@ -92,6 +92,11 @@ case "$UNAME" in
*) MD5SUM="md5sum" ;;
esac
MTIME="stat -c %Y"
case "$UNAME" in
Darwin | FreeBSD | OpenBSD) MTIME="stat -f %m" ;;
esac
DIFF="diff"
case "$UNAME" in
SunOS) DIFF="gdiff" ;;
@@ -215,20 +220,19 @@ $ZSTD tmp -c --compress-literals -19 | $ZSTD -t
$ZSTD -b --fast=1 -i0e1 tmp --compress-literals
$ZSTD -b --fast=1 -i0e1 tmp --no-compress-literals
println "test: --exclude-compressed flag"
println "\n===> --exclude-compressed flag"
rm -rf precompressedFilterTestDir
mkdir -p precompressedFilterTestDir
./datagen $size > precompressedFilterTestDir/input.5
./datagen $size > precompressedFilterTestDir/input.6
$ZSTD --exclude-compressed --long --rm -r precompressedFilterTestDir
sleep 5
./datagen $size > precompressedFilterTestDir/input.7
./datagen $size > precompressedFilterTestDir/input.8
$ZSTD --exclude-compressed --long --rm -r precompressedFilterTestDir
test ! -f precompressedFilterTestDir/input.5.zst.zst
test ! -f precompressedFilterTestDir/input.6.zst.zst
file1timestamp=`date -r precompressedFilterTestDir/input.5.zst +%s`
file2timestamp=`date -r precompressedFilterTestDir/input.7.zst +%s`
file1timestamp=`$MTIME precompressedFilterTestDir/input.5.zst`
file2timestamp=`$MTIME precompressedFilterTestDir/input.7.zst`
if [[ $file2timestamp -ge $file1timestamp ]]; then
println "Test is successful. input.5.zst is precompressed and therefore not compressed/modified again."
else
@@ -246,7 +250,7 @@ test -f precompressedFilterTestDir/input.5.zst.zst
test -f precompressedFilterTestDir/input.6.zst.zst
println "Test completed"
println "test : file removal"
println "\n===> file removal"
$ZSTD -f --rm tmp
test ! -f tmp # tmp should no longer be present
$ZSTD -f -d --rm tmp.zst
@@ -274,14 +278,17 @@ test -f tmp.zst # destination file should still be present
rm -rf tmp* # may also erase tmp* directory from previous failed run
println "\n===> decompression only tests "
head -c 1048576 /dev/zero > tmp
println "\n===> decompression only tests "
# the following test verifies that the decoder is compatible with RLE as first block
# older versions of zstd cli are not able to decode such corner case.
# As a consequence, the zstd cli do not generate them, to maintain compatibility with older versions.
dd bs=1048576 count=1 if=/dev/zero of=tmp
$ZSTD -d -o tmp1 "$TESTDIR/golden-decompression/rle-first-block.zst"
$DIFF -s tmp1 tmp
rm tmp*
println "test : compress multiple files"
println "\n===> compress multiple files"
println hello > tmp1
println world > tmp2
$ZSTD tmp1 tmp2 -o "$INTOVOID" -f
@@ -328,7 +335,22 @@ $ZSTD -f tmp1 notHere tmp2 && die "missing file not detected!"
rm tmp*
println "test : compress multiple files into an output directory, --output-dir-flat"
if [ -n "$DEVNULLRIGHTS" ]
then
# these tests requires sudo rights, which is uncommon.
# they are only triggered if DEVNULLRIGHTS macro is defined.
println "\n===> checking /dev/null permissions are unaltered "
./datagen > tmp
sudo $ZSTD tmp -o $INTOVOID # sudo rights could modify /dev/null permissions
sudo $ZSTD tmp -c > $INTOVOID
$ZSTD tmp -f -o tmp.zst
sudo $ZSTD -d tmp.zst -c > $INTOVOID
sudo $ZSTD -d tmp.zst -o $INTOVOID
ls -las $INTOVOID | grep "rw-rw-rw-"
fi
println "\n===> compress multiple files into an output directory, --output-dir-flat"
println henlo > tmp1
mkdir tmpInputTestDir
mkdir tmpInputTestDir/we
@@ -436,7 +458,6 @@ $ZSTD -dcf tmp1
println "\n===> frame concatenation "
println "hello " > hello.tmp
println "world!" > world.tmp
cat hello.tmp world.tmp > helloworld.tmp
@@ -1241,9 +1262,9 @@ rm -f tmp* dictionary
if [ "$isWindows" = false ] ; then
println "\n===> zstd fifo named pipe test "
head -c 10 /dev/zero > tmp_original
dd bs=1 count=10 if=/dev/zero of=tmp_original
mkfifo named_pipe
head -c 10 /dev/zero > named_pipe &
dd bs=1 count=10 if=/dev/zero of=named_pipe &
$ZSTD named_pipe -o tmp_compressed
$ZSTD -d -o tmp_decompressed tmp_compressed
$DIFF -s tmp_original tmp_decompressed
tests/zstreamtest.c
+27
View File
@@ -509,6 +509,33 @@ static int basicUnitTests(U32 seed, double compressibility)
}
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(3, "test%3i : NULL buffers : ", testNb++);
inBuff.src = NULL;
inBuff.size = 0;
inBuff.pos = 0;
outBuff.dst = NULL;
outBuff.size = 0;
outBuff.pos = 0;
CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) );
CHECK(inBuff.pos != inBuff.size, "Entire input should be consumed");
CHECK_Z( ZSTD_endStream(zc, &outBuff) );
outBuff.dst = (char*)(compressedBuffer);
outBuff.size = compressedBufferSize;
outBuff.pos = 0;
{ size_t const r = ZSTD_endStream(zc, &outBuff);
CHECK(r != 0, "Error or some data not flushed (ret=%zu)", r);
}
inBuff.src = outBuff.dst;
inBuff.size = outBuff.pos;
inBuff.pos = 0;
outBuff.dst = NULL;
outBuff.size = 0;
outBuff.pos = 0;
CHECK_Z( ZSTD_initDStream(zd) );
{ size_t const ret = ZSTD_decompressStream(zd, &outBuff, &inBuff);
if (ret != 0) goto _output_error;
}
DISPLAYLEVEL(3, "OK\n");
/* _srcSize compression test */
DISPLAYLEVEL(3, "test%3i : compress_srcSize %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH);
CHECK_Z( ZSTD_initCStream_srcSize(zc, 1, CNBufferSize) );