ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #217 from Cyan4973/dev

Browse Source 
v0.7.1 tag
This commit is contained in:
Yann Collet
2016-06-22 14:07:04 +02:00
committed by GitHub
parent d000042108 3755eb8fea
commit 378822521a
13 changed files with 253 additions and 186 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.travis.yml
+31 -34
View File
@@ -6,85 +6,82 @@ matrix:
# Container-based Ubuntu 12.04 LTS Server Edition 64 bit (doesn't support 32-bit includes)
- os: linux
sudo: false
env: PLATFORM="Ubuntu 12.04 container" MAKE_PARAM=travis-install
env: PLATFORM="Ubuntu 12.04 container" CMD="make travis-install"
- os: linux
sudo: false
env: PLATFORM="Ubuntu 12.04 container" MAKE_PARAM=cmaketest
env: PLATFORM="Ubuntu 12.04 container" CMD="make cmaketest"
- os: linux
sudo: false
env: PLATFORM="Ubuntu 12.04 container" MAKE_PARAM=test
env: PLATFORM="Ubuntu 12.04 container" CMD="make test"
- os: linux
sudo: false
env: PLATFORM="Ubuntu 12.04 container" MAKE_PARAM="-C programs test-zstd_nolegacy"
env: PLATFORM="Ubuntu 12.04 container" CMD="make -C programs test-zstd_nolegacy"
- os: linux
sudo: false
env: PLATFORM="Ubuntu 12.04 container" MAKE_PARAM=usan
env: PLATFORM="Ubuntu 12.04 container" CMD="make usan"
- os: linux
sudo: false
env: PLATFORM="Ubuntu 12.04 container" MAKE_PARAM=asan
env: PLATFORM="Ubuntu 12.04 container" CMD="make asan"
- os: linux
sudo: false
env: PLATFORM="Ubuntu 12.04 container" MAKE_PARAM=zlibwrapper
env: PLATFORM="Ubuntu 12.04 container" CMD="make zlibwrapper"
# Standard Ubuntu 12.04 LTS Server Edition 64 bit
- os: linux
sudo: required
env: PLATFORM="Ubuntu 12.04" MAKE_PARAM=clangtest
env: PLATFORM="Ubuntu 12.04" CMD="make libc6install clangtest"
- os: linux
sudo: required
env: PLATFORM="Ubuntu 12.04" MAKE_PARAM=gpptest
env: PLATFORM="Ubuntu 12.04" CMD="make gppinstall gpptest"
- os: linux
sudo: required
env: PLATFORM="Ubuntu 12.04" MAKE_PARAM=gnu90test
env: PLATFORM="Ubuntu 12.04" CMD="make libc6install gnu90test"
- os: linux
sudo: required
env: PLATFORM="Ubuntu 12.04" MAKE_PARAM=c99test
env: PLATFORM="Ubuntu 12.04" CMD="make libc6install c99test"
- os: linux
sudo: required
env: PLATFORM="Ubuntu 12.04" MAKE_PARAM=gnu99test
env: PLATFORM="Ubuntu 12.04" CMD="make libc6install gnu99test"
- os: linux
sudo: required
env: PLATFORM="Ubuntu 12.04" MAKE_PARAM=armtest-w-install
env: PLATFORM="Ubuntu 12.04" CMD="make arminstall armtest"
- os: linux
sudo: required
env: PLATFORM="Ubuntu 12.04" MAKE_PARAM="-C programs test32"
env: PLATFORM="Ubuntu 12.04" CMD="make libc6install && make -C programs test32"
- os: linux
sudo: required
env: PLATFORM="Ubuntu 12.04" MAKE_PARAM="-C tests versionsTest"
env: PLATFORM="Ubuntu 12.04" CMD="make -C tests versionsTest"
- os: linux
sudo: required
env: PLATFORM="Ubuntu 12.04" MAKE_PARAM=asan32
env: PLATFORM="Ubuntu 12.04" CMD="make libc6install asan32"
- os: linux
sudo: required
env: PLATFORM="Ubuntu 12.04" MAKE_PARAM="-C programs valgrindTest"
env: PLATFORM="Ubuntu 12.04" CMD="make valgrindinstall && make -C programs valgrindTest"
# Ubuntu 14.04 LTS Server Edition 64 bit
- os: linux
dist: trusty
sudo: required
env: PLATFORM="Ubuntu 14.04" MAKE_PARAM=ppctest-w-install
env: PLATFORM="Ubuntu 14.04" CMD="make gcc5install gcc5test"
- os: linux
dist: trusty
sudo: required
env: PLATFORM="Ubuntu 14.04" MAKE_PARAM=zlibwrapper
env: PLATFORM="Ubuntu 14.04" CMD="make gcc6install gcc6test"
- os: linux
dist: trusty
sudo: required
env: PLATFORM="Ubuntu 14.04" CMD="make ppcinstall ppctest"
- os: linux
dist: trusty
sudo: required
env: PLATFORM="Ubuntu 14.04" CMD="make zlibwrapper"
# OS X Mavericks
- os: osx
env: PLATFORM="OS X Mavericks" MAKE_PARAM=travis-install
env: PLATFORM="OS X Mavericks" CMD="make travis-install"
- os: osx
env: PLATFORM="OS X Mavericks" MAKE_PARAM=gnu90test
env: PLATFORM="OS X Mavericks" CMD="make gnu90test"
- os: osx
env: PLATFORM="OS X Mavericks" MAKE_PARAM=test
env: PLATFORM="OS X Mavericks" CMD="make test"
exclude:
- compiler: gcc
before_install:
- set -e
- |
if [ "$TRAVIS_OS_NAME" == "linux" ]; then
CAN_I_RUN_SUDO=$(sudo -n uptime 2>&1|grep "load"|wc -l)
echo "CAN_I_RUN_SUDO=$CAN_I_RUN_SUDO\n"
if [ ${CAN_I_RUN_SUDO} -gt 0 ]; then
sudo apt-get install -y -qq clang g++-multilib gcc-multilib valgrind
fi
fi
script:
- make $MAKE_PARAM
- sh -c "$CMD"
Makefile
+44 -24
View File
@@ -51,7 +51,7 @@ all:
zstdprogram:
$(MAKE) -C $(PRGDIR)
mv $(PRGDIR)/zstd .
cp $(PRGDIR)/zstd .
zlibwrapper:
$(MAKE) -C $(ZSTDDIR) all
@@ -87,6 +87,14 @@ travis-install:
gpptest: clean
$(MAKE) all CC=g++ CFLAGS="-O3 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Werror"
gcc5test: clean
gcc-5 -v
$(MAKE) all CC=gcc-5 MOREFLAGS="-Werror"
gcc6test: clean
gcc-6 -v
$(MAKE) all CC=gcc-6 MOREFLAGS="-Werror"
clangtest: clean
clang -v
$(MAKE) all CC=clang MOREFLAGS="-Werror -Wconversion -Wno-sign-conversion"
@@ -95,35 +103,13 @@ armtest: clean
$(MAKE) -C $(PRGDIR) datagen # use native, faster
$(MAKE) -C $(PRGDIR) test CC=arm-linux-gnueabi-gcc ZSTDRTTEST= MOREFLAGS="-Werror -static"
# for Travis CI
arminstall: clean
sudo apt-get install -y -q qemu binfmt-support qemu-user-static gcc-arm-linux-gnueabi
# for Travis CI
armtest-w-install: clean arminstall armtest
ppctest: clean
$(MAKE) -C $(PRGDIR) datagen # use native, faster
$(MAKE) -C $(PRGDIR) test CC=powerpc-linux-gnu-gcc ZSTDRTTEST= MOREFLAGS="-Werror -static"
# for Travis CI
ppcinstall: clean
# sudo apt-get update -y -q
sudo apt-get install -y -q qemu-system-ppc binfmt-support qemu-user-static gcc-powerpc-linux-gnu # doesn't work with Ubuntu 12.04
# for Travis CI
ppctest-w-install: clean ppcinstall ppctest
ppc64test: clean
$(MAKE) -C $(PRGDIR) datagen # use native, faster
$(MAKE) -C $(PRGDIR) test CC=powerpc64le-linux-gnu-gcc ZSTDRTTEST= MOREFLAGS="-Werror -static"
ppc64install: clean
sudo apt-get update -y -q
sudo apt-get install -y -q qemu-ppc64le binfmt-support qemu-user-static gcc-powerpc64le-linux-gnu
update-binfmts --displ
ppc64test-w-install: clean ppc64install ppc64test
$(MAKE) -C $(PRGDIR) test CC=powerpc-linux-gnu-gcc ZSTDRTTEST= MOREFLAGS="-m64 -Werror -static"
usan: clean
$(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=undefined"
@@ -183,3 +169,37 @@ bmix32test: clean
bmi32test: clean
CFLAGS="-O3 -mbmi -m32 -Werror" $(MAKE) -C $(PRGDIR) test
endif
#------------------------------------------------------------------------
# for Travis CI
#------------------------------------------------------------------------
libc6install:
sudo apt-get install -y -qq libc6-dev-i386
gppinstall:
sudo apt-get install -y -qq g++-multilib
gcc5install:
sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test
sudo apt-get update -y -qq
sudo apt-get install -y -qq gcc-5-multilib
gcc6install:
sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test
sudo apt-get update -y -qq
sudo apt-get install -y -qq gcc-6-multilib
arminstall: clean
sudo apt-get install -y -q qemu binfmt-support qemu-user-static gcc-arm-linux-gnueabi
ppcinstall: clean
sudo apt-get install -y -q qemu-system-ppc binfmt-support qemu-user-static gcc-powerpc-linux-gnu # doesn't work with Ubuntu 12.04
ppc64install: clean # compilation works but cannot be executed
sudo apt-get install -y -q qemu-system-ppc binfmt-support qemu-user-static gcc-powerpc-linux-gnu
update-binfmts --displ
valgrindinstall:
echo sudo apt-get install -y -qq valgrind
sudo apt-get install -y -qq valgrind
NEWS
+2
View File
@@ -1,4 +1,6 @@
v0.7.1
fixed : ZBUFF_compressEnd() called multiple times with too small `dst` buffer, reported by Christophe Chevalier
fixed : dictBuilder fails if first sample is too small, reported by Руслан Ковалёв
fixed : corruption issue, reported by cj
modified : checksum enabled by default in command line mode
lib/common/huf.h
+1 -1
View File
@@ -136,7 +136,7 @@ typedef U32 HUF_DTable;
size_t HUF_decompress4X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< single-symbol decoder */
size_t HUF_decompress4X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< double-symbols decoder */
size_t HUF_decompress4X_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize);
size_t HUF_decompress4X_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< decodes RLE and uncompressed */
size_t HUF_decompress4X_hufOnly(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< considers RLE and uncompressed as errors */
size_t HUF_decompress4X2_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< single-symbol decoder */
size_t HUF_decompress4X4_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< double-symbols decoder */
lib/common/zstd.h
+18 -11
View File
@@ -61,7 +61,7 @@ extern "C" {
***************************************/
#define ZSTD_VERSION_MAJOR 0
#define ZSTD_VERSION_MINOR 7
#define ZSTD_VERSION_RELEASE 0
#define ZSTD_VERSION_RELEASE 1
#define ZSTD_LIB_VERSION ZSTD_VERSION_MAJOR.ZSTD_VERSION_MINOR.ZSTD_VERSION_RELEASE
#define ZSTD_QUOTE(str) #str
@@ -132,7 +132,7 @@ ZSTDLIB_API size_t ZSTD_decompressDCtx(ZSTD_DCtx* ctx, void* dst, size_t dstCapa
/*! ZSTD_compress_usingDict() :
* Compression using a pre-defined Dictionary content (see dictBuilder).
* Note 1 : This function load the dictionary, resulting in a significant startup time.
* Note 2 : `dict` must remain valid and unmodified during compression operation.
* Note 2 : `dict` must remain accessible and unmodified during compression operation.
* Note 3 : `dict` can be `NULL`, in which case, it's equivalent to ZSTD_compressCCtx() */
ZSTDLIB_API size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx,
void* dst, size_t dstCapacity,
@@ -144,7 +144,7 @@ ZSTDLIB_API size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx,
* Decompression using a pre-defined Dictionary content (see dictBuilder).
* Dictionary must be identical to the one used during compression.
* Note 1 : This function load the dictionary, resulting in a significant startup time
* Note 2 : `dict` must remain valid and unmodified during compression operation.
* Note 2 : `dict` must remain accessible and unmodified during compression operation.
* Note 3 : `dict` can be `NULL`, in which case, it's equivalent to ZSTD_decompressDCtx() */
ZSTDLIB_API size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx,
void* dst, size_t dstCapacity,
@@ -192,7 +192,7 @@ ZSTDLIB_API size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx,
/* ====================================================================================
* The definitions in this section are considered experimental.
* They should never be used in association with a dynamic library, as they may change in the future.
* They should never be used with a dynamic library, as they may change in the future.
* They are provided for advanced usages.
* Use them only in association with static linking.
* ==================================================================================== */
@@ -322,15 +322,22 @@ ZSTDLIB_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapaci
It's also possible to duplicate a reference context which has already been initialized, using ZSTD_copyCCtx()
Then, consume your input using ZSTD_compressContinue().
ZSTD_compressContinue() presumes prior data is still accessible and unmodified (up to maximum distance size, see WindowLog).
The interface is synchronous, so input will be entirely consumed and produce associated compressed output.
You must ensure there is enough space in destination buffer to store compressed data under worst case scenario.
Worst case evaluation is provided by ZSTD_compressBound().
There are some important considerations to keep in mind when using this advanced function :
- ZSTD_compressContinue() has no internal buffer. It uses externally provided buffer only.
- Interface is synchronous : input will be entirely consumed and produce 1+ compressed blocks.
- Caller must ensure there is enough space in `dst` to store compressed data under worst case scenario.
Worst case evaluation is provided by ZSTD_compressBound().
ZSTD_compressContinue() doesn't guarantee recover after a failed compression.
- ZSTD_compressContinue() presumes prior input ***is still accessible and unmodified*** (up to maximum distance size, see WindowLog).
It remembers all previous contiguous blocks, plus one separated memory segment (which can itself consists of multiple contiguous blocks)
- ZSTD_compressContinue() detects that prior input has been overwritten when `src` buffer overlaps.
In which case, it will "discard" the relevant memory section from its history.
Finish a frame with ZSTD_compressEnd(), which will write the epilogue.
Without epilogue, frames will be considered unfinished (broken) by decoders.
You can then reuse ZSTD_CCtx to compress some new frame.
You can then reuse `ZSTD_CCtx` (ZSTD_compressBegin()) to compress some new frame.
*/
typedef struct {
@@ -357,7 +364,7 @@ ZSTDLIB_API size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t ds
A ZSTD_DCtx object can be re-used multiple times.
First optional operation is to retrieve frame parameters, using ZSTD_getFrameParams(), which doesn't consume the input.
It can provide the minimum size of rolling buffer required to properly decompress data,
It can provide the minimum size of rolling buffer required to properly decompress data (`windowSize`),
and optionally the final size of uncompressed content.
(Note : content size is an optional info that may not be present. 0 means : content size unknown)
Frame parameters are extracted from the beginning of compressed frame.
@@ -373,7 +380,7 @@ ZSTDLIB_API size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t ds
Then use ZSTD_nextSrcSizeToDecompress() and ZSTD_decompressContinue() alternatively.
ZSTD_nextSrcSizeToDecompress() tells how much bytes to provide as 'srcSize' to ZSTD_decompressContinue().
ZSTD_decompressContinue() requires this exact amount of bytes, or it will fail.
ZSTD_decompressContinue() needs previous data blocks during decompression, up to (1 << windowlog).
ZSTD_decompressContinue() needs previous data blocks during decompression, up to `windowSize`.
They should preferably be located contiguously, prior to current block. Alternatively, a round buffer is also possible.
@result of ZSTD_decompressContinue() is the number of bytes regenerated within 'dst' (necessarily <= dstCapacity).
lib/compress/zbuff_compress.c
+34 -20
View File
@@ -79,7 +79,7 @@ static size_t const ZBUFF_endFrameSize = ZSTD_BLOCKHEADERSIZE;
* output : ZSTD_compressBound(ZSTD_BLOCKSIZE_MAX) + ZSTD_blockHeaderSize + ZBUFF_endFrameSize : ensures it's always possible to write/flush/end a full block at best speed.
* **************************************************/
typedef enum { ZBUFFcs_init, ZBUFFcs_load, ZBUFFcs_flush } ZBUFF_cStage;
typedef enum { ZBUFFcs_init, ZBUFFcs_load, ZBUFFcs_flush, ZBUFFcs_final } ZBUFF_cStage;
/* *** Resources *** */
struct ZBUFF_CCtx_s {
@@ -162,7 +162,7 @@ size_t ZBUFF_compressInit_advanced(ZBUFF_CCtx* zbc,
zbc->inToCompress = 0;
zbc->inBuffPos = 0;
zbc->inBuffTarget = zbc->blockSize;
zbc->outBuffFlushedSize = 0;
zbc->outBuffContentSize = zbc->outBuffFlushedSize = 0;
zbc->stage = ZBUFFcs_load;
return 0; /* ready to go */
}
@@ -196,7 +196,7 @@ MEM_STATIC size_t ZBUFF_limitCopy(void* dst, size_t dstCapacity, const void* src
static size_t ZBUFF_compressContinue_generic(ZBUFF_CCtx* zbc,
void* dst, size_t* dstCapacityPtr,
const void* src, size_t* srcSizePtr,
int flush) /* aggregate : wait for full block before compressing */
int flush)
{
U32 notDone = 1;
const char* const istart = (const char*)src;
@@ -243,17 +243,20 @@ static size_t ZBUFF_compressContinue_generic(ZBUFF_CCtx* zbc,
}
case ZBUFFcs_flush:
/* flush into dst */
{ size_t const toFlush = zbc->outBuffContentSize - zbc->outBuffFlushedSize;
size_t const flushed = ZBUFF_limitCopy(op, oend-op, zbc->outBuff + zbc->outBuffFlushedSize, toFlush);
op += flushed;
zbc->outBuffFlushedSize += flushed;
if (toFlush!=flushed) { notDone = 0; break; } /* not enough space within dst to store compressed block : stop there */
zbc->outBuffContentSize = 0;
zbc->outBuffFlushedSize = 0;
if (toFlush!=flushed) { notDone = 0; break; } /* dst too small to store flushed data : stop there */
zbc->outBuffContentSize = zbc->outBuffFlushedSize = 0;
zbc->stage = ZBUFFcs_load;
break;
}
case ZBUFFcs_final:
notDone = 0; /* do nothing */
break;
default:
return ERROR(GENERIC); /* impossible */
}
@@ -291,19 +294,30 @@ size_t ZBUFF_compressEnd(ZBUFF_CCtx* zbc, void* dst, size_t* dstCapacityPtr)
BYTE* const ostart = (BYTE*)dst;
BYTE* const oend = ostart + *dstCapacityPtr;
BYTE* op = ostart;
size_t outSize = *dstCapacityPtr;
size_t epilogueSize, remaining;
ZBUFF_compressFlush(zbc, dst, &outSize); /* flush any remaining inBuff */
op += outSize;
epilogueSize = ZSTD_compressEnd(zbc->zc, zbc->outBuff + zbc->outBuffContentSize, zbc->outBuffSize - zbc->outBuffContentSize); /* epilogue into outBuff */
zbc->outBuffContentSize += epilogueSize;
outSize = oend-op;
zbc->stage = ZBUFFcs_flush;
remaining = ZBUFF_compressFlush(zbc, op, &outSize); /* attempt to flush epilogue into dst */
op += outSize;
if (!remaining) zbc->stage = ZBUFFcs_init; /* close only if nothing left to flush */
*dstCapacityPtr = op-ostart; /* tells how many bytes were written */
return remaining;
if (zbc->stage != ZBUFFcs_final) {
/* flush whatever remains */
size_t outSize = *dstCapacityPtr;
size_t const remainingToFlush = ZBUFF_compressFlush(zbc, dst, &outSize);
op += outSize;
if (remainingToFlush) {
*dstCapacityPtr = op-ostart;
return remainingToFlush + ZBUFF_endFrameSize;
}
/* create epilogue */
zbc->stage = ZBUFFcs_final;
zbc->outBuffContentSize = ZSTD_compressEnd(zbc->zc, zbc->outBuff, zbc->outBuffSize); /* epilogue into outBuff */
}
/* flush epilogue */
{ size_t const toFlush = zbc->outBuffContentSize - zbc->outBuffFlushedSize;
size_t const flushed = ZBUFF_limitCopy(op, oend-op, zbc->outBuff + zbc->outBuffFlushedSize, toFlush);
op += flushed;
zbc->outBuffFlushedSize += flushed;
*dstCapacityPtr = op-ostart;
if (toFlush==flushed) zbc->stage = ZBUFFcs_init; /* end reached */
return toFlush - flushed;
}
}
lib/compress/zstd_compress.c
+12 -11
View File
@@ -295,7 +295,7 @@ static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc,
zc->seqStore.buffer = zc->hashTable3 + h3Size;
zc->hufTable = (HUF_CElt*)zc->seqStore.buffer;
zc->flagStaticTables = 0;
zc->seqStore.buffer = ((U32*)(zc->seqStore.buffer)) + 256;
zc->seqStore.buffer = ((U32*)(zc->seqStore.buffer)) + 256; /* note : HUF_CElt* is incomplete type, size is simulated using U32 */
zc->nextToUpdate = 1;
zc->nextSrc = NULL;
@@ -313,14 +313,17 @@ static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc,
zc->seqStore.litLengthFreq = zc->seqStore.litFreq + (1<<Litbits);
zc->seqStore.matchLengthFreq = zc->seqStore.litLengthFreq + (MaxLL+1);
zc->seqStore.offCodeFreq = zc->seqStore.matchLengthFreq + (MaxML+1);
zc->seqStore.matchTable = (ZSTD_match_t*)((void*)(zc->seqStore.offCodeFreq + (MaxOff+1)));
zc->seqStore.priceTable = (ZSTD_optimal_t*)((void*)(zc->seqStore.matchTable + ZSTD_OPT_NUM+1));
zc->seqStore.buffer = zc->seqStore.offCodeFreq + (MaxOff+1);
zc->seqStore.matchTable = (ZSTD_match_t*)zc->seqStore.buffer;
zc->seqStore.buffer = zc->seqStore.matchTable + ZSTD_OPT_NUM+1;
zc->seqStore.priceTable = (ZSTD_optimal_t*)zc->seqStore.buffer;
zc->seqStore.buffer = zc->seqStore.priceTable + ZSTD_OPT_NUM+1;
zc->seqStore.litLengthSum = 0;
}
zc->seqStore.offsetStart = (U32*) (zc->seqStore.buffer);
zc->seqStore.litLengthStart = (U16*) (void*)(zc->seqStore.offsetStart + maxNbSeq);
zc->seqStore.matchLengthStart = (U16*) (void*)(zc->seqStore.litLengthStart + maxNbSeq);
zc->seqStore.offsetStart = (U32*)(zc->seqStore.buffer);
zc->seqStore.buffer = zc->seqStore.offsetStart + maxNbSeq;
zc->seqStore.litLengthStart = (U16*)zc->seqStore.buffer;
zc->seqStore.matchLengthStart = zc->seqStore.litLengthStart + maxNbSeq;
zc->seqStore.llCodeStart = (BYTE*) (zc->seqStore.matchLengthStart + maxNbSeq);
zc->seqStore.mlCodeStart = zc->seqStore.llCodeStart + maxNbSeq;
zc->seqStore.offCodeStart = zc->seqStore.mlCodeStart + maxNbSeq;
@@ -1330,8 +1333,6 @@ static void ZSTD_compressBlock_fast_extDict(ZSTD_CCtx* ctx,
}
/*-*************************************
* Binary Tree search
***************************************/
@@ -1614,7 +1615,7 @@ U32 ZSTD_insertAndFindFirstIndex (ZSTD_CCtx* zc, const BYTE* ip, U32 mls)
const U32 target = (U32)(ip - base);
U32 idx = zc->nextToUpdate;
while(idx < target) {
while(idx < target) { /* catch up */
size_t const h = ZSTD_hashPtr(base+idx, hashLog, mls);
NEXT_IN_CHAIN(idx, chainMask) = hashTable[h];
hashTable[h] = idx;
@@ -1651,7 +1652,7 @@ size_t ZSTD_HcFindBestMatch_generic (
/* HC4 match finder */
U32 matchIndex = ZSTD_insertAndFindFirstIndex (zc, ip, mls);
for ( ; (matchIndex>lowLimit) && (nbAttempts) ; nbAttempts--) {
for ( ; (matchIndex>lowLimit) & (nbAttempts>0) ; nbAttempts--) {
const BYTE* match;
size_t currentMl=0;
if ((!extDict) || matchIndex >= dictLimit) {
@@ -1665,7 +1666,7 @@ size_t ZSTD_HcFindBestMatch_generic (
}
/* save best solution */
if (currentMl > ml) { ml = currentMl; *offsetPtr = ZSTD_REP_MOVE + current - matchIndex; if (ip+currentMl == iLimit) break; /* best possible, and avoid read overflow*/ }
if (currentMl > ml) { ml = currentMl; *offsetPtr = current - matchIndex + ZSTD_REP_MOVE; if (ip+currentMl == iLimit) break; /* best possible, and avoid read overflow*/ }
if (matchIndex <= minChain) break;
matchIndex = NEXT_IN_CHAIN(matchIndex, chainMask);
lib/dictBuilder/zdict.c
+36 -33
View File
@@ -584,48 +584,51 @@ static void ZDICT_countEStats(EStats_ress_t esr,
U32* countLit, U32* offsetcodeCount, U32* matchlengthCount, U32* litlengthCount, U32* repOffsets,
const void* src, size_t srcSize)
{
const seqStore_t* seqStorePtr;
size_t cSize;
if (srcSize > ZSTD_BLOCKSIZE_MAX) srcSize = ZSTD_BLOCKSIZE_MAX; /* protection vs large samples */
ZSTD_copyCCtx(esr.zc, esr.ref);
ZSTD_compressBlock(esr.zc, esr.workPlace, ZSTD_BLOCKSIZE_MAX, src, srcSize);
seqStorePtr = ZSTD_getSeqStore(esr.zc);
cSize = ZSTD_compressBlock(esr.zc, esr.workPlace, ZSTD_BLOCKSIZE_MAX, src, srcSize);
if (ZSTD_isError(cSize)) { DISPLAYLEVEL(1, "warning : could not compress sample size %u \n", (U32)srcSize); return; }
/* literals stats */
{ const BYTE* bytePtr;
for(bytePtr = seqStorePtr->litStart; bytePtr < seqStorePtr->lit; bytePtr++)
countLit[*bytePtr]++;
}
if (cSize) { /* if == 0; block is not compressible */
const seqStore_t* seqStorePtr = ZSTD_getSeqStore(esr.zc);
/* seqStats */
{ size_t const nbSeq = (size_t)(seqStorePtr->offset - seqStorePtr->offsetStart);
ZSTD_seqToCodes(seqStorePtr, nbSeq);
{ const BYTE* codePtr = seqStorePtr->offCodeStart;
size_t u;
for (u=0; u<nbSeq; u++) offsetcodeCount[codePtr[u]]++;
/* literals stats */
{ const BYTE* bytePtr;
for(bytePtr = seqStorePtr->litStart; bytePtr < seqStorePtr->lit; bytePtr++)
countLit[*bytePtr]++;
}
{ const BYTE* codePtr = seqStorePtr->mlCodeStart;
size_t u;
for (u=0; u<nbSeq; u++) matchlengthCount[codePtr[u]]++;
/* seqStats */
{ size_t const nbSeq = (size_t)(seqStorePtr->offset - seqStorePtr->offsetStart);
ZSTD_seqToCodes(seqStorePtr, nbSeq);
{ const BYTE* codePtr = seqStorePtr->offCodeStart;
size_t u;
for (u=0; u<nbSeq; u++) offsetcodeCount[codePtr[u]]++;
}
{ const BYTE* codePtr = seqStorePtr->mlCodeStart;
size_t u;
for (u=0; u<nbSeq; u++) matchlengthCount[codePtr[u]]++;
}
{ const BYTE* codePtr = seqStorePtr->llCodeStart;
size_t u;
for (u=0; u<nbSeq; u++) litlengthCount[codePtr[u]]++;
} }
/* rep offsets */
{ const U32* const offsetPtr = seqStorePtr->offsetStart;
U32 offset1 = offsetPtr[0] - 3;
U32 offset2 = offsetPtr[1] - 3;
if (offset1 >= MAXREPOFFSET) offset1 = 0;
if (offset2 >= MAXREPOFFSET) offset2 = 0;
repOffsets[offset1] += 3;
repOffsets[offset2] += 1;
}
{ const BYTE* codePtr = seqStorePtr->llCodeStart;
size_t u;
for (u=0; u<nbSeq; u++) litlengthCount[codePtr[u]]++;
} }
/* rep offsets */
{ const U32* const offsetPtr = seqStorePtr->offsetStart;
U32 offset1 = offsetPtr[0] - 3;
U32 offset2 = offsetPtr[1] - 3;
if (offset1 >= MAXREPOFFSET) offset1 = 0;
if (offset2 >= MAXREPOFFSET) offset2 = 0;
repOffsets[offset1] += 3;
repOffsets[offset2] += 1;
}
}
/*
programs/bench.c
+1 -5
View File
@@ -42,10 +42,6 @@
/* *************************************
* Constants
***************************************/
#ifndef ZSTD_VERSION
# define ZSTD_VERSION ""
#endif
#define NBLOOPS 3
#define TIMELOOP_MICROSEC 1*1000000ULL /* 1 second */
#define ACTIVEPERIOD_MICROSEC 70*1000000ULL /* 70 seconds */
@@ -363,7 +359,7 @@ static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize,
memset(&total, 0, sizeof(total));
if (g_displayLevel == 1 && !g_additionalParam)
DISPLAY("bench %s: input %u bytes, %i iterations, %u KB blocks\n", ZSTD_VERSION, (U32)benchedSize, g_nbIterations, (U32)(g_blockSize>>10));
DISPLAY("bench %s: input %u bytes, %i iterations, %u KB blocks\n", ZSTD_VERSION_STRING, (U32)benchedSize, g_nbIterations, (U32)(g_blockSize>>10));
if (cLevelLast < cLevel) cLevelLast = cLevel;
programs/fileio.c
+1 -1
View File
@@ -413,7 +413,7 @@ static int FIO_compressFilename_dstFile(cRess_t ress,
int result;
ress.dstFile = FIO_openDstFile(dstFileName);
if (ress.dstFile==0) { fclose(ress.srcFile); return 1; }
if (ress.dstFile==0) return 1;
result = FIO_compressFilename_srcFile(ress, dstFileName, srcFileName, cLevel);
programs/playTests.sh
+14 -10
View File
@@ -22,10 +22,12 @@ roundTripTest() {
isWindows=false
ECHO="echo"
INTOVOID="/dev/null"
case "$OS" in
Windows*)
isWindows=true
ECHO="echo -e"
INTOVOID="nul"
;;
esac
@@ -53,19 +55,21 @@ $ECHO "test : null-length file roundtrip"
$ECHO -n '' | $ZSTD - --stdout | $ZSTD -d --stdout
$ECHO "test : decompress file with wrong suffix (must fail)"
$ZSTD -d tmpCompressed && die "wrong suffix error not detected!"
$ZSTD -df tmp && die "should have refused : wrong extension"
$ECHO "test : decompress into stdout"
$ZSTD -d tmpCompressed -c > tmpResult # decompression using stdout
$ZSTD --decompress tmpCompressed -c > tmpResult
$ZSTD --decompress tmpCompressed --stdout > tmpResult
if [ "$isWindows" = false ] ; then
$ZSTD -d < tmp.zst > /dev/null # combine decompression, stdin & stdout
$ZSTD -d - < tmp.zst > /dev/null
fi
$ZSTD -dc < tmp.zst > /dev/null
$ZSTD -dc - < tmp.zst > /dev/null
$ECHO "test : decompress from stdin into stdout"
$ZSTD -dc < tmp.zst > $INTOVOID # combine decompression, stdin & stdout
$ZSTD -dc - < tmp.zst > $INTOVOID
$ZSTD -d < tmp.zst > $INTOVOID # implicit stdout when stdin is used
$ZSTD -d - < tmp.zst > $INTOVOID
$ECHO "test : overwrite protection"
$ZSTD -q tmp && die "overwrite check failed!"
$ECHO "test : force overwrite"
$ZSTD -q -f tmp
$ZSTD -q --force tmp
$ZSTD -df tmp && die "should have refused : wrong extension"
$ECHO "test : file removal"
$ZSTD -f --rm tmp
ls tmp && die "tmp should no longer be present"
@@ -135,9 +139,9 @@ rm tmpSparse*
$ECHO "\n**** multiple files tests **** "
./datagen -s1 > tmp1 2> /dev/null
./datagen -s2 -g100K > tmp2 2> /dev/null
./datagen -s3 -g1M > tmp3 2> /dev/null
./datagen -s1 > tmp1 2> $INTOVOID
./datagen -s2 -g100K > tmp2 2> $INTOVOID
./datagen -s3 -g1M > tmp3 2> $INTOVOID
$ZSTD -f tmp*
$ECHO "compress tmp* : "
ls -ls tmp*
programs/zbufftest.c
+11 -5
View File
@@ -421,11 +421,17 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compres
} }
/* final frame epilogue */
{ size_t dstBuffSize = cBufferSize - cSize;
size_t const flushError = ZBUFF_compressEnd(zc, cBuffer+cSize, &dstBuffSize);
CHECK (ZBUFF_isError(flushError), "flush error : %s", ZBUFF_getErrorName(flushError));
cSize += dstBuffSize;
}
{ size_t remainingToFlush = (size_t)(-1);
while (remainingToFlush) {
size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog);
size_t dstBuffSize = MIN(cBufferSize - cSize, randomDstSize);
U32 const enoughDstSize = dstBuffSize >= remainingToFlush;
remainingToFlush = ZBUFF_compressEnd(zc, cBuffer+cSize, &dstBuffSize);
CHECK (ZBUFF_isError(remainingToFlush), "flush error : %s", ZBUFF_getErrorName(remainingToFlush));
//DISPLAY("flush %u bytes : still within context : %i \n", (U32)dstBuffSize, (int)remainingToFlush);
CHECK (enoughDstSize && remainingToFlush, "ZBUFF_compressEnd() not fully flushed, but enough space available");
cSize += dstBuffSize;
} }
crcOrig = XXH64_digest(&xxhState);
/* multi - fragments decompression test */
tests/test-zstd-speed.py
+48 -31
View File
@@ -8,9 +8,9 @@ import time
import traceback
from subprocess import Popen, PIPE
repo_url = 'https://github.com/Cyan4973/zstd.git'
default_repo_url = 'https://github.com/Cyan4973/zstd.git'
test_dir_name = 'speedTest'
email_header = '[ZSTD_speedTest]'
def log(text):
print time.strftime("%Y/%m/%d %H:%M:%S") + ' - ' + text
@@ -126,14 +126,23 @@ def benchmark_and_compare(branch, commit, resultsFileName, lastCLevel, testFileP
return text
def send_simple_email(emails, email_topic, have_mutt, have_mail):
if have_mutt:
execute('mutt -s "' + email_header + ' ' + email_topic + '" ' + emails + ' </dev/null')
elif have_mail:
execute('mail -s "' + email_header + ' ' + email_topic + '" ' + emails + ' </dev/null')
else:
log("e-mail cannot be sent (mail and mutt not found)")
def send_email(branch, commit, last_commit, emails, text, results_files, logFileName, lower_limit, have_mutt, have_mail):
with open(logFileName, "w") as myfile:
myfile.writelines(text)
myfile.close()
if have_mutt:
execute("mutt -s \"[ZSTD_speedTest] Warning for branch=" + branch + " commit=" + commit + " last_commit=" + last_commit + " speed<" + str(lower_limit) + "\" " + emails + " -a " + results_files + " < " + logFileName)
execute('mutt -s "' + email_header + ' Warning for branch=' + branch + ' commit=' + commit + ' last_commit=' + last_commit + ' speed<' + str(lower_limit) + '" ' + emails + ' -a ' + results_files + ' < ' + logFileName)
elif have_mail:
execute("mail -s \"[ZSTD_speedTest] Warning for branch=" + branch + " commit=" + commit + " last_commit=" + last_commit + " speed<" + str(lower_limit) + "\" " + emails + " < " + logFileName)
execute('mail -s "' + email_header + ' Warning for branch=' + branch + ' commit=' + commit + ' last_commit=' + last_commit + ' speed<' + str(lower_limit) + '" ' + emails + ' < ' + logFileName)
else:
log("e-mail cannot be sent (mail and mutt not found)")
@@ -183,6 +192,7 @@ if __name__ == '__main__':
parser.add_argument('testFileNames', help='file names list for speed benchmark')
parser.add_argument('emails', help='list of e-mail addresses to send warnings')
parser.add_argument('--message', help='attach an additional message to e-mail', default="")
parser.add_argument('--repoURL', help='changes default repository URL', default=default_repo_url)
parser.add_argument('--lowerLimit', type=float, help='send email if speed is lower than given limit', default=0.98)
parser.add_argument('--maxLoadAvg', type=float, help='maximum load average to start testing', default=0.75)
parser.add_argument('--lastCLevel', type=int, help='last compression level for testing', default=5)
@@ -197,7 +207,8 @@ if __name__ == '__main__':
if os.path.isfile(fileName):
testFilePaths.append(os.path.abspath(fileName))
else:
raise RuntimeError("File not found: " + fileName)
log("ERROR: File not found: " + fileName)
exit(1)
test_path = os.getcwd() + '/' + test_dir_name # /path/to/zstd/tests/speedTest
clone_path = test_path + '/' + 'zstd' # /path/to/zstd/tests/speedTest/zstd
@@ -206,19 +217,11 @@ if __name__ == '__main__':
have_mutt = does_command_exist("mutt --help");
have_mail = does_command_exist("mail -V");
if not have_mutt and not have_mail:
log("WARNING: e-mail senders mail and mutt not found")
log("ERROR: e-mail senders 'mail' or 'mutt' not found")
exit(1)
# clone ZSTD repo if needed
if not os.path.isdir(test_path):
os.mkdir(test_path)
if not os.path.isdir(clone_path):
execute.cwd = test_path
execute('git clone ' + repo_url)
if not os.path.isdir(clone_path):
raise RuntimeError("ZSTD clone not found: " + clone_path)
execute.cwd = clone_path
print "PARAMETERS:\ntest_path=%s" % test_path
print "PARAMETERS:\nrepoURL=%s" % args.repoURL
print "test_path=%s" % test_path
print "clone_path=%s" % clone_path
print "testFilePath(%s)=%s" % (len(testFilePaths), testFilePaths)
print "message=%s" % args.message
@@ -230,20 +233,34 @@ if __name__ == '__main__':
print "dry_run=%s" % args.dry_run
print "have_mutt=%s have_mail=%s" % (have_mutt, have_mail)
# clone ZSTD repo if needed
if not os.path.isdir(test_path):
os.mkdir(test_path)
if not os.path.isdir(clone_path):
execute.cwd = test_path
execute('git clone ' + args.repoURL)
if not os.path.isdir(clone_path):
log("ERROR: ZSTD clone not found: " + clone_path)
exit(1)
execute.cwd = clone_path
# check if speedTest.pid already exists
pid = str(os.getpid())
pidfile = "./speedTest.pid"
if os.path.isfile(pidfile):
log("ERROR: %s already exists, exiting" % pidfile)
exit(1)
send_simple_email(args.emails, "test-zstd-speed.py(%s) has been started" % pid, have_mutt, have_mail)
while True:
pid = str(os.getpid())
pidfile = "./speedTest.pid"
if os.path.isfile(pidfile):
log("%s already exists, exiting" % pidfile)
else:
file(pidfile, 'w').write(pid)
try:
loadavg = os.getloadavg()[0]
if (loadavg <= args.maxLoadAvg):
check_branches(args, test_path, testFilePaths, have_mutt, have_mail)
else:
log("WARNING: main loadavg=%.2f is higher than %s" % (loadavg, args.maxLoadAvg))
finally:
os.unlink(pidfile)
file(pidfile, 'w').write(pid)
try:
loadavg = os.getloadavg()[0]
if (loadavg <= args.maxLoadAvg):
check_branches(args, test_path, testFilePaths, have_mutt, have_mail)
else:
log("WARNING: main loadavg=%.2f is higher than %s" % (loadavg, args.maxLoadAvg))
finally:
os.unlink(pidfile)
log("sleep for %s seconds" % args.sleepTime)
time.sleep(args.sleepTime)