ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #295 from Cyan4973/dev

Browse Source 
v0.8.1
This commit is contained in:
Yann Collet
2016-08-18 16:14:00 +02:00
committed by GitHub
parent 908a9f8f36 a5dbf9f629
commit be6180c142
58 changed files with 4635 additions and 1572 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitignore
+5 -19
View File
@@ -18,29 +18,15 @@ zstd
*.out
*.app
# Visual C++
projects/VS2008
projects/VS2010
projects/VS2012
projects/VS2013
projects/VS2015
build/bin
# IDEA solution files
*.idea
# Test artefacts
tmp*
dictionary
# Other files
.directory
_codelite/
_zstdbench/
.clang_complete
# CMake
projects/cmake/
# Test artefacts
tmp*
dictionary
# tmp files
*.idea
*.swp
.DS_Store
.travis.yml
+47 -46
View File
@@ -6,80 +6,81 @@ matrix:
# Container-based Ubuntu 12.04 LTS Server Edition 64 bit (doesn't support 32-bit includes)
- os: linux
sudo: false
env: PLATFORM="Ubuntu 12.04 container" CMD="make travis-install"
env: PLATFORM="Ubuntu 12.04 container" CMD="make test && make clean && make travis-install"
- os: linux
sudo: false
env: PLATFORM="Ubuntu 12.04 container" CMD="make cmaketest"
- os: linux
sudo: false
env: PLATFORM="Ubuntu 12.04 container" CMD="make test"
- os: linux
sudo: false
env: PLATFORM="Ubuntu 12.04 container" CMD="make -C programs test-zstd_nolegacy"
env: PLATFORM="Ubuntu 12.04 container" CMD="make -C programs test-zstd_nolegacy && make clean && make zlibwrapper && make clean && make cmaketest"
- os: linux
sudo: false
env: PLATFORM="Ubuntu 12.04 container" CMD="make usan"
- os: linux
sudo: false
env: PLATFORM="Ubuntu 12.04 container" CMD="make asan"
- os: linux
sudo: false
env: PLATFORM="Ubuntu 12.04 container" CMD="make zlibwrapper"
# Standard Ubuntu 12.04 LTS Server Edition 64 bit
- os: linux
sudo: required
env: PLATFORM="Ubuntu 12.04" CMD="make libc6install clangtest"
env: PLATFORM="Ubuntu 12.04" CMD="make gpptest && make clean && make gnu90test && make clean && make c99test && make clean && make gnu99test && make clean && make clangtest"
addons:
apt:
packages:
- libc6-dev-i386
- g++-multilib
- os: linux
sudo: required
env: PLATFORM="Ubuntu 12.04" CMD="make gppinstall gpptest"
env: PLATFORM="Ubuntu 12.04" CMD="make armtest"
addons:
apt:
packages:
- gcc-arm-linux-gnueabi
- libc6-dev-armel-cross
- linux-libc-dev-armel-cross
- binfmt-support
- qemu
- qemu-user-static
- os: linux
sudo: required
env: PLATFORM="Ubuntu 12.04" CMD="make libc6install gnu90test"
- os: linux
sudo: required
env: PLATFORM="Ubuntu 12.04" CMD="make libc6install c99test"
- os: linux
sudo: required
env: PLATFORM="Ubuntu 12.04" CMD="make libc6install gnu99test"
- os: linux
sudo: required
env: PLATFORM="Ubuntu 12.04" CMD="make arminstall armtest"
- os: linux
sudo: required
env: PLATFORM="Ubuntu 12.04" CMD="make libc6install && make -C programs test32"
env: PLATFORM="Ubuntu 12.04" CMD="make -C programs test32"
addons:
apt:
packages:
- libc6-dev-i386
- gcc-multilib
- os: linux
sudo: required
env: PLATFORM="Ubuntu 12.04" CMD="make -C tests versionsTest"
- os: linux
sudo: required
env: PLATFORM="Ubuntu 12.04" CMD="make libc6install asan32"
env: PLATFORM="Ubuntu 12.04" CMD="make asan32"
addons:
apt:
packages:
- libc6-dev-i386
- gcc-multilib
- os: linux
sudo: required
env: PLATFORM="Ubuntu 12.04" CMD="make valgrindinstall && make -C programs valgrindTest"
env: PLATFORM="Ubuntu 12.04" CMD="make -C programs valgrindTest"
addons:
apt:
packages:
- valgrind
# Ubuntu 14.04 LTS Server Edition 64 bit
- os: linux
dist: trusty
sudo: required
env: PLATFORM="Ubuntu 14.04" CMD="make gcc5install gcc5test"
- os: linux
dist: trusty
sudo: required
env: PLATFORM="Ubuntu 14.04" CMD="make gcc6install gcc6test"
- os: linux
dist: trusty
sudo: required
env: PLATFORM="Ubuntu 14.04" CMD="make ppcinstall ppctest"
- os: linux
dist: trusty
sudo: required
env: PLATFORM="Ubuntu 14.04" CMD="make zlibwrapper"
env: PLATFORM="Ubuntu 14.04" CMD="make zlibwrapper && make clean && make gcc5test && make clean && make gcc6test && sudo apt-get install -y -q qemu-system-ppc binfmt-support qemu-user-static gcc-powerpc-linux-gnu && make clean && make ppctest"
addons:
apt:
sources:
- ubuntu-toolchain-r-test
packages:
- gcc-multilib
- gcc-5
- gcc-5-multilib
- gcc-6
- gcc-6-multilib
# OS X Mavericks
- os: osx
env: PLATFORM="OS X Mavericks" CMD="make travis-install"
- os: osx
env: PLATFORM="OS X Mavericks" CMD="make gnu90test"
- os: osx
env: PLATFORM="OS X Mavericks" CMD="make test"
env: PLATFORM="OS X Mavericks" CMD="make gnu90test && make clean && make test && make clean && make travis-install"
exclude:
- compiler: gcc
Makefile
+1 -35
View File
@@ -120,7 +120,7 @@ asan: clean
$(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=address"
msan: clean
$(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=memory" # datagen.c used to fail this test for no obvious reason
$(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=memory -fno-omit-frame-pointer" # datagen.c fails this test for no obvious reason
asan32: clean
$(MAKE) -C $(PRGDIR) test32 CC=clang MOREFLAGS="-g -fsanitize=address"
@@ -174,37 +174,3 @@ bmi32test: clean
staticAnalyze: clean
CPPFLAGS=-g scan-build --status-bugs -v $(MAKE) all
endif
#------------------------------------------------------------------------
# for Travis CI
#------------------------------------------------------------------------
libc6install:
sudo apt-get install -y -qq libc6-dev-i386
gppinstall:
sudo apt-get install -y -qq g++-multilib
gcc5install:
sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test
sudo apt-get update -y -qq
sudo apt-get install -y -qq gcc-5-multilib
gcc6install:
sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test
sudo apt-get update -y -qq
sudo apt-get install -y -qq gcc-6-multilib
arminstall: clean
sudo apt-get install -y -q qemu binfmt-support qemu-user-static gcc-arm-linux-gnueabi
ppcinstall: clean
sudo apt-get install -y -q qemu-system-ppc binfmt-support qemu-user-static gcc-powerpc-linux-gnu # doesn't work with Ubuntu 12.04
ppc64install: clean # compilation works but cannot be executed
sudo apt-get install -y -q qemu-system-ppc binfmt-support qemu-user-static gcc-powerpc-linux-gnu
update-binfmts --displ
valgrindinstall:
echo sudo apt-get install -y -qq valgrind
sudo apt-get install -y -qq valgrind
NEWS
+9
View File
@@ -1,3 +1,12 @@
v0.8.1
New streaming API
Changed : --ultra now enables levels beyond 19
Changed : -i# now selects benchmark time in second
Fixed : ZSTD_compress* can now compress > 4 GB in a single pass, reported by Nick Terrell
Fixed : speed regression on specific patterns (#272)
Fixed : support for Z_SYNC_FLUSH, by Dmitry Krot (#291)
Fixed : ICC compilation, by Przemyslaw Skibinski
v0.8.0
Improved : better speed on clang and gcc -O2, thanks to Eric Biggers
New : Build on FreeBSD and DragonFly, thanks to JrMarino
appveyor.yml
+28 -6
View File
@@ -59,29 +59,51 @@ build_script:
)
- if [%COMPILER%]==[visual] (
ECHO *** &&
ECHO *** Building Visual Studio 2008 %PLATFORM%\%CONFIGURATION% &&
ECHO *** Building Visual Studio 2008 %PLATFORM%\%CONFIGURATION% in %APPVEYOR_BUILD_FOLDER% &&
ECHO *** &&
msbuild "projects\VS2008\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v90 /t:Clean,Build /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" &&
msbuild "projects\VS2008\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v90 /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" &&
DIR projects\VS2008\bin\%PLATFORM%\%CONFIGURATION%\*.exe &&
MD5sum projects/VS2008/bin/%PLATFORM%/%CONFIGURATION%/*.exe &&
COPY projects\VS2008\bin\%PLATFORM%\%CONFIGURATION%\fuzzer.exe projects\fuzzer_VS2008_%PLATFORM%_%CONFIGURATION%.exe &&
ECHO *** &&
ECHO *** Building Visual Studio 2010 %PLATFORM%\%CONFIGURATION% &&
ECHO *** &&
msbuild "projects\VS2010\zstd.sln" %ADDITIONALPARAM% /m /verbosity:minimal /property:PlatformToolset=v100 /t:Clean,Build /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" &&
msbuild "projects\VS2010\zstd.sln" %ADDITIONALPARAM% /m /verbosity:minimal /property:PlatformToolset=v100 /p:ForceImportBeforeCppTargets=%APPVEYOR_BUILD_FOLDER%\projects\VS2010\CompileAsCpp.props /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" &&
DIR projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\*.exe &&
MD5sum projects/VS2010/bin/%PLATFORM%/%CONFIGURATION%/*.exe &&
msbuild "projects\VS2010\zstd.sln" %ADDITIONALPARAM% /m /verbosity:minimal /property:PlatformToolset=v100 /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" &&
DIR projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\*.exe &&
MD5sum projects/VS2010/bin/%PLATFORM%/%CONFIGURATION%/*.exe &&
COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\fuzzer.exe projects\fuzzer_VS2010_%PLATFORM%_%CONFIGURATION%.exe &&
ECHO *** &&
ECHO *** Building Visual Studio 2012 %PLATFORM%\%CONFIGURATION% &&
ECHO *** &&
msbuild "projects\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v110 /t:Clean,Build /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" &&
msbuild "projects\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v110 /p:ForceImportBeforeCppTargets=%APPVEYOR_BUILD_FOLDER%\projects\VS2010\CompileAsCpp.props /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" &&
DIR projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\*.exe &&
MD5sum projects/VS2010/bin/%PLATFORM%/%CONFIGURATION%/*.exe &&
msbuild "projects\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v110 /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" &&
DIR projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\*.exe &&
MD5sum projects/VS2010/bin/%PLATFORM%/%CONFIGURATION%/*.exe &&
COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\fuzzer.exe projects\fuzzer_VS2012_%PLATFORM%_%CONFIGURATION%.exe &&
ECHO *** &&
ECHO *** Building Visual Studio 2013 %PLATFORM%\%CONFIGURATION% &&
ECHO *** &&
msbuild "projects\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v120 /t:Clean,Build /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" &&
msbuild "projects\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v120 /p:ForceImportBeforeCppTargets=%APPVEYOR_BUILD_FOLDER%\projects\VS2010\CompileAsCpp.props /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" &&
DIR projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\*.exe &&
MD5sum projects/VS2010/bin/%PLATFORM%/%CONFIGURATION%/*.exe &&
msbuild "projects\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v120 /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" &&
DIR projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\*.exe &&
MD5sum projects/VS2010/bin/%PLATFORM%/%CONFIGURATION%/*.exe &&
COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\fuzzer.exe projects\fuzzer_VS2013_%PLATFORM%_%CONFIGURATION%.exe &&
ECHO *** &&
ECHO *** Building Visual Studio 2015 %PLATFORM%\%CONFIGURATION% &&
ECHO *** &&
msbuild "projects\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v140 /t:Clean,Build /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" &&
msbuild "projects\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v140 /p:ForceImportBeforeCppTargets=%APPVEYOR_BUILD_FOLDER%\projects\VS2010\CompileAsCpp.props /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" &&
DIR projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\*.exe &&
MD5sum projects/VS2010/bin/%PLATFORM%/%CONFIGURATION%/*.exe &&
msbuild "projects\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v140 /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" &&
DIR projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\*.exe &&
MD5sum projects/VS2010/bin/%PLATFORM%/%CONFIGURATION%/*.exe &&
COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\fuzzer.exe projects\fuzzer_VS2015_%PLATFORM%_%CONFIGURATION%.exe &&
COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\*.exe programs\
)
examples/.gitignore
+2
View File
@@ -3,6 +3,8 @@ simple_compression
simple_decompression
dictionary_compression
dictionary_decompression
streaming_compression
streaming_decompression
#test artefact
tmp*
examples/Makefile
+18 -3
View File
@@ -31,7 +31,8 @@ LDFLAGS+= -lzstd
default: all
all: simple_compression simple_decompression \
dictionary_compression dictionary_decompression
dictionary_compression dictionary_decompression \
streaming_compression streaming_decompression
simple_compression : simple_compression.c
$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
@@ -45,15 +46,29 @@ dictionary_compression : dictionary_compression.c
dictionary_decompression : dictionary_decompression.c
$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
streaming_compression : streaming_compression.c
$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
streaming_decompression : streaming_decompression.c
$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
clean:
@rm -f core *.o tmp* result* *.zst \
simple_compression simple_decompression \
dictionary_compression dictionary_decompression
dictionary_compression dictionary_decompression \
streaming_compression streaming_decompression
@echo Cleaning completed
test: all
cp README.md tmp
@echo starting simple compression
./simple_compression tmp
@echo starting simple_decompression
./simple_decompression tmp.zst
./streaming_decompression tmp.zst
@echo starting streaming compression
./streaming_compression tmp
./streaming_decompression tmp.zst
@echo starting dictionary compression
./dictionary_compression tmp README.md
./dictionary_decompression tmp.zst README.md
@echo tests completed
examples/dictionary_compression.c
+27 -21
View File
@@ -31,7 +31,7 @@
#include <zstd.h> // presumes zstd library is installed
static off_t fsize_X(const char *filename)
static off_t fsize_orDie(const char *filename)
{
struct stat st;
if (stat(filename, &st) == 0) return st.st_size;
@@ -40,7 +40,7 @@ static off_t fsize_X(const char *filename)
exit(1);
}
static FILE* fopen_X(const char *filename, const char *instruction)
static FILE* fopen_orDie(const char *filename, const char *instruction)
{
FILE* const inFile = fopen(filename, instruction);
if (inFile) return inFile;
@@ -49,20 +49,20 @@ static FILE* fopen_X(const char *filename, const char *instruction)
exit(2);
}
static void* malloc_X(size_t size)
static void* malloc_orDie(size_t size)
{
void* const buff = malloc(size);
if (buff) return buff;
/* error */
perror(NULL);
perror("malloc");
exit(3);
}
static void* loadFile_X(const char* fileName, size_t* size)
static void* loadFile_orDie(const char* fileName, size_t* size)
{
off_t const buffSize = fsize_X(fileName);
FILE* const inFile = fopen_X(fileName, "rb");
void* const buffer = malloc_X(buffSize);
off_t const buffSize = fsize_orDie(fileName);
FILE* const inFile = fopen_orDie(fileName, "rb");
void* const buffer = malloc_orDie(buffSize);
size_t const readSize = fread(buffer, 1, buffSize, inFile);
if (readSize != (size_t)buffSize) {
fprintf(stderr, "fread: %s : %s \n", fileName, strerror(errno));
@@ -73,9 +73,9 @@ static void* loadFile_X(const char* fileName, size_t* size)
return buffer;
}
static void saveFile_X(const char* fileName, const void* buff, size_t buffSize)
static void saveFile_orDie(const char* fileName, const void* buff, size_t buffSize)
{
FILE* const oFile = fopen_X(fileName, "wb");
FILE* const oFile = fopen_orDie(fileName, "wb");
size_t const wSize = fwrite(buff, 1, buffSize, oFile);
if (wSize != (size_t)buffSize) {
fprintf(stderr, "fwrite: %s : %s \n", fileName, strerror(errno));
@@ -89,23 +89,27 @@ static void saveFile_X(const char* fileName, const void* buff, size_t buffSize)
/* createDict() :
`dictFileName` is supposed to have been created using `zstd --train` */
static const ZSTD_CDict* createDict(const char* dictFileName)
static ZSTD_CDict* createCDict_orDie(const char* dictFileName)
{
size_t dictSize;
printf("loading dictionary %s \n", dictFileName);
void* const dictBuffer = loadFile_X(dictFileName, &dictSize);
const ZSTD_CDict* const ddict = ZSTD_createCDict(dictBuffer, dictSize, 3);
void* const dictBuffer = loadFile_orDie(dictFileName, &dictSize);
ZSTD_CDict* const cdict = ZSTD_createCDict(dictBuffer, dictSize, 3);
if (!cdict) {
fprintf(stderr, "ZSTD_createCDict error \n");
exit(7);
}
free(dictBuffer);
return ddict;
return cdict;
}
static void compress(const char* fname, const char* oname, const ZSTD_CDict* cdict)
{
size_t fSize;
void* const fBuff = loadFile_X(fname, &fSize);
void* const fBuff = loadFile_orDie(fname, &fSize);
size_t const cBuffSize = ZSTD_compressBound(fSize);
void* const cBuff = malloc_X(cBuffSize);
void* const cBuff = malloc_orDie(cBuffSize);
ZSTD_CCtx* const cctx = ZSTD_createCCtx();
size_t const cSize = ZSTD_compress_usingCDict(cctx, cBuff, cBuffSize, fBuff, fSize, cdict);
@@ -114,7 +118,7 @@ static void compress(const char* fname, const char* oname, const ZSTD_CDict* cdi
exit(7);
}
saveFile_X(oname, cBuff, cSize);
saveFile_orDie(oname, cBuff, cSize);
/* success */
printf("%25s : %6u -> %7u - %s \n", fname, (unsigned)fSize, (unsigned)cSize, oname);
@@ -125,11 +129,11 @@ static void compress(const char* fname, const char* oname, const ZSTD_CDict* cdi
}
static char* createOutFilename(const char* filename)
static char* createOutFilename_orDie(const char* filename)
{
size_t const inL = strlen(filename);
size_t const outL = inL + 5;
void* outSpace = malloc_X(outL);
void* outSpace = malloc_orDie(outL);
memset(outSpace, 0, outL);
strcat(outSpace, filename);
strcat(outSpace, ".zst");
@@ -149,15 +153,17 @@ int main(int argc, const char** argv)
/* load dictionary only once */
const char* const dictName = argv[argc-1];
const ZSTD_CDict* const dictPtr = createDict(dictName);
ZSTD_CDict* const dictPtr = createCDict_orDie(dictName);
int u;
for (u=1; u<argc-1; u++) {
const char* inFilename = argv[u];
char* const outFilename = createOutFilename(inFilename);
char* const outFilename = createOutFilename_orDie(inFilename);
compress(inFilename, outFilename, dictPtr);
free(outFilename);
}
ZSTD_freeCDict(dictPtr);
printf("All %u files compressed. \n", argc-2);
return 0;
}
examples/dictionary_decompression.c
+23 -20
View File
@@ -31,41 +31,41 @@
#include <zstd.h> // presumes zstd library is installed
static off_t fsize_X(const char *filename)
static off_t fsize_orDie(const char *filename)
{
struct stat st;
if (stat(filename, &st) == 0) return st.st_size;
/* error */
printf("stat: %s : %s \n", filename, strerror(errno));
perror(filename);
exit(1);
}
static FILE* fopen_X(const char *filename, const char *instruction)
static FILE* fopen_orDie(const char *filename, const char *instruction)
{
FILE* const inFile = fopen(filename, instruction);
if (inFile) return inFile;
/* error */
printf("fopen: %s : %s \n", filename, strerror(errno));
perror(filename);
exit(2);
}
static void* malloc_X(size_t size)
static void* malloc_orDie(size_t size)
{
void* const buff = malloc(size);
if (buff) return buff;
/* error */
printf("malloc: %s \n", strerror(errno));
perror("malloc");
exit(3);
}
static void* loadFile_X(const char* fileName, size_t* size)
static void* loadFile_orDie(const char* fileName, size_t* size)
{
off_t const buffSize = fsize_X(fileName);
FILE* const inFile = fopen_X(fileName, "rb");
void* const buffer = malloc_X(buffSize);
off_t const buffSize = fsize_orDie(fileName);
FILE* const inFile = fopen_orDie(fileName, "rb");
void* const buffer = malloc_orDie(buffSize);
size_t const readSize = fread(buffer, 1, buffSize, inFile);
if (readSize != (size_t)buffSize) {
printf("fread: %s : %s \n", fileName, strerror(errno));
fprintf(stderr, "fread: %s : %s \n", fileName, strerror(errno));
exit(4);
}
fclose(inFile);
@@ -75,12 +75,13 @@ static void* loadFile_X(const char* fileName, size_t* size)
/* createDict() :
`dictFileName` is supposed to have been created using `zstd --train` */
static const ZSTD_DDict* createDict(const char* dictFileName)
static ZSTD_DDict* createDict_orDie(const char* dictFileName)
{
size_t dictSize;
printf("loading dictionary %s \n", dictFileName);
void* const dictBuffer = loadFile_X(dictFileName, &dictSize);
const ZSTD_DDict* const ddict = ZSTD_createDDict(dictBuffer, dictSize);
void* const dictBuffer = loadFile_orDie(dictFileName, &dictSize);
ZSTD_DDict* const ddict = ZSTD_createDDict(dictBuffer, dictSize);
if (ddict==NULL) { fprintf(stderr, "ZSTD_createDDict error \n"); exit(5); }
free(dictBuffer);
return ddict;
}
@@ -89,19 +90,19 @@ static const ZSTD_DDict* createDict(const char* dictFileName)
static void decompress(const char* fname, const ZSTD_DDict* ddict)
{
size_t cSize;
void* const cBuff = loadFile_X(fname, &cSize);
void* const cBuff = loadFile_orDie(fname, &cSize);
unsigned long long const rSize = ZSTD_getDecompressedSize(cBuff, cSize);
if (rSize==0) {
printf("%s : original size unknown \n", fname);
exit(5);
fprintf(stderr, "%s : original size unknown \n", fname);
exit(6);
}
void* const rBuff = malloc_X(rSize);
void* const rBuff = malloc_orDie(rSize);
ZSTD_DCtx* const dctx = ZSTD_createDCtx();
size_t const dSize = ZSTD_decompress_usingDDict(dctx, rBuff, rSize, cBuff, cSize, ddict);
if (dSize != rSize) {
printf("error decoding %s : %s \n", fname, ZSTD_getErrorName(dSize));
fprintf(stderr, "error decoding %s : %s \n", fname, ZSTD_getErrorName(dSize));
exit(7);
}
@@ -127,10 +128,12 @@ int main(int argc, const char** argv)
/* load dictionary only once */
const char* const dictName = argv[argc-1];
const ZSTD_DDict* const dictPtr = createDict(dictName);
ZSTD_DDict* const dictPtr = createDict_orDie(dictName);
int u;
for (u=1; u<argc-1; u++) decompress(argv[u], dictPtr);
ZSTD_freeDDict(dictPtr);
printf("All %u files correctly decoded (in memory) \n", argc-2);
return 0;
}
examples/streaming_compression.c
+143
View File
@@ -0,0 +1,143 @@
/*
Streaming compression
Educational program using zstd library
Copyright (C) Yann Collet 2016
GPL v2 License
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 2 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License along
with this program; if not, write to the Free Software Foundation, Inc.,
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
You can contact the author at :
- zstd homepage : http://www.zstd.net/
*/
#include <stdlib.h> // malloc, exit
#include <stdio.h> // fprintf, perror, feof
#include <string.h> // strerror
#include <errno.h> // errno
#define ZSTD_STATIC_LINKING_ONLY // streaming API defined as "experimental" for the time being
#include <zstd.h> // presumes zstd library is installed
static void* malloc_orDie(size_t size)
{
void* const buff = malloc(size);
if (buff) return buff;
/* error */
perror("malloc:");
exit(1);
}
static FILE* fopen_orDie(const char *filename, const char *instruction)
{
FILE* const inFile = fopen(filename, instruction);
if (inFile) return inFile;
/* error */
perror(filename);
exit(3);
}
static size_t fread_orDie(void* buffer, size_t sizeToRead, FILE* file)
{
size_t const readSize = fread(buffer, 1, sizeToRead, file);
if (readSize == sizeToRead) return readSize; /* good */
if (feof(file)) return readSize; /* good, reached end of file */
/* error */
perror("fread");
exit(4);
}
static size_t fwrite_orDie(const void* buffer, size_t sizeToWrite, FILE* file)
{
size_t const writtenSize = fwrite(buffer, 1, sizeToWrite, file);
if (writtenSize == sizeToWrite) return sizeToWrite; /* good */
/* error */
perror("fwrite");
exit(5);
}
static size_t fclose_orDie(FILE* file)
{
if (!fclose(file)) return 0;
/* error */
perror("fclose");
exit(6);
}
static void compressFile_orDie(const char* fname, const char* outName, int cLevel)
{
FILE* const fin = fopen_orDie(fname, "rb");
FILE* const fout = fopen_orDie(outName, "wb");
size_t const buffInSize = ZSTD_CStreamInSize();;
void* const buffIn = malloc_orDie(buffInSize);
size_t const buffOutSize = ZSTD_CStreamOutSize();;
void* const buffOut = malloc_orDie(buffOutSize);
size_t read, toRead = buffInSize;
ZSTD_CStream* const cstream = ZSTD_createCStream();
if (cstream==NULL) { fprintf(stderr, "ZSTD_createCStream() error \n"); exit(10); }
size_t const initResult = ZSTD_initCStream(cstream, cLevel);
if (ZSTD_isError(initResult)) { fprintf(stderr, "ZSTD_initCStream() error \n"); exit(11); }
while( (read = fread_orDie(buffIn, toRead, fin)) ) {
ZSTD_inBuffer input = { buffIn, read, 0 };
while (input.pos < input.size) {
ZSTD_outBuffer output = { buffOut, buffOutSize, 0 };
toRead = ZSTD_compressStream(cstream, &output , &input);
fwrite_orDie(buffOut, output.pos, fout);
}
}
ZSTD_outBuffer output = { buffOut, buffOutSize, 0 };
size_t const remainingToFlush = ZSTD_endStream(cstream, &output);
if (remainingToFlush) { fprintf(stderr, "not fully flushed"); exit(12); }
fwrite_orDie(buffOut, output.pos, fout);
fclose_orDie(fout);
fclose_orDie(fin);
free(buffIn);
free(buffOut);
}
static const char* createOutFilename_orDie(const char* filename)
{
size_t const inL = strlen(filename);
size_t const outL = inL + 5;
void* outSpace = malloc_orDie(outL);
memset(outSpace, 0, outL);
strcat(outSpace, filename);
strcat(outSpace, ".zst");
return (const char*)outSpace;
}
int main(int argc, const char** argv)
{
const char* const exeName = argv[0];
const char* const inFilename = argv[1];
if (argc!=2) {
printf("wrong arguments\n");
printf("usage:\n");
printf("%s FILE\n", exeName);
return 1;
}
const char* const outFilename = createOutFilename_orDie(inFilename);
compressFile_orDie(inFilename, outFilename, 1);
return 0;
}
examples/streaming_decompression.c
+117
View File
@@ -0,0 +1,117 @@
/*
Streaming compression
Educational program using zstd library
Copyright (C) Yann Collet 2016
GPL v2 License
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 2 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License along
with this program; if not, write to the Free Software Foundation, Inc.,
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
You can contact the author at :
- zstd homepage : http://www.zstd.net/
*/
#include <stdlib.h> // malloc, exit
#include <stdio.h> // fprintf, perror, feof
#include <string.h> // strerror
#include <errno.h> // errno
#define ZSTD_STATIC_LINKING_ONLY // streaming API defined as "experimental" for the time being
#include <zstd.h> // presumes zstd library is installed
static void* malloc_orDie(size_t size)
{
void* const buff = malloc(size);
if (buff) return buff;
/* error */
perror("malloc:");
exit(1);
}
static FILE* fopen_orDie(const char *filename, const char *instruction)
{
FILE* const inFile = fopen(filename, instruction);
if (inFile) return inFile;
/* error */
perror(filename);
exit(3);
}
static size_t fread_orDie(void* buffer, size_t sizeToRead, FILE* file)
{
size_t const readSize = fread(buffer, 1, sizeToRead, file);
if (readSize == sizeToRead) return readSize; /* good */
if (feof(file)) return readSize; /* good, reached end of file */
/* error */
perror("fread");
exit(4);
}
static size_t fclose_orDie(FILE* file)
{
if (!fclose(file)) return 0;
/* error */
perror("fclose");
exit(6);
}
static void decompressFile_orDie(const char* fname)
{
FILE* const fin = fopen_orDie(fname, "rb");
size_t const buffInSize = ZSTD_DStreamInSize();;
void* const buffIn = malloc_orDie(buffInSize);
size_t const buffOutSize = ZSTD_DStreamOutSize();;
void* const buffOut = malloc_orDie(buffOutSize);
size_t read, toRead = buffInSize;
ZSTD_DStream* const dstream = ZSTD_createDStream();
if (dstream==NULL) { fprintf(stderr, "ZSTD_createDStream() error \n"); exit(10); }
size_t const initResult = ZSTD_initDStream(dstream);
if (ZSTD_isError(initResult)) { fprintf(stderr, "ZSTD_initDStream() error \n"); exit(11); }
while( (read = fread_orDie(buffIn, toRead, fin)) ) {
ZSTD_inBuffer input = { buffIn, read, 0 };
while (input.pos < input.size) {
ZSTD_outBuffer output = { buffOut, buffOutSize, 0 };
toRead = ZSTD_decompressStream(dstream, &output , &input);
/* note : data is just "sinked" into buffOut
a more complete example would write it to disk or stdout */
}
}
fclose_orDie(fin);
free(buffIn);
free(buffOut);
}
int main(int argc, const char** argv)
{
const char* const exeName = argv[0];
const char* const inFilename = argv[1];
if (argc!=2) {
printf("wrong arguments\n");
printf("usage:\n");
printf("%s FILE\n", exeName);
return 1;
}
decompressFile_orDie(inFilename);
printf("%s correctly decoded (in memory). \n", inFilename);
return 0;
}
images/Dspeed4.png
BIN
View File
Binary file not shown.
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 9.7 KiB

After

Width:  |  Height:  |  Size: 8.8 KiB

lib/Makefile
+1 -1
View File
@@ -46,7 +46,7 @@ PREFIX ?= /usr/local
LIBDIR ?= $(PREFIX)/lib
INCLUDEDIR=$(PREFIX)/include
CPPFLAGS= -I. -I./common -DXXH_NAMESPACE=ZSTD_
CPPFLAGS= -I. -I./common
CFLAGS ?= -O3
CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wstrict-aliasing=1 \
-Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes -Wundef
lib/README.md
+9 -5
View File
@@ -37,15 +37,18 @@ Other optional functionalities provided are :
- `legacy/` : source code to decompress previous versions of zstd, starting from `v0.1`.
This module also depends on `common/` and `decompress/` .
Note that it's required to compile the library with `ZSTD_LEGACY_SUPPORT = 1` .
Library compilation must include directive `ZSTD_LEGACY_SUPPORT = 1` .
The main API can be consulted in `legacy/zstd_legacy.h`.
Advanced API from each version can be found in its relevant header file.
For example, advanced API for version `v0.4` is in `zstd_v04.h` .
Advanced API from each version can be found in their relevant header file.
For example, advanced API for version `v0.4` is in `legacy/zstd_v04.h` .
#### Streaming API
#### Obsolete streaming API
Streaming is currently provided by `common/zbuff.h`.
Streaming is now provided within `zstd.h`.
Older streaming API is still provided within `common/zbuff.h`.
It is considered obsolete, and will be removed in a future version.
Consider migrating towards newer streaming API.
#### Miscellaneous
@@ -55,3 +58,4 @@ The other files are not source code. There are :
- LICENSE : contains the BSD license text
- Makefile : script to compile or install zstd library (static and dynamic)
- libzstd.pc.in : for pkg-config (`make install`)
- README.md : this file
lib/common/mem.h
+8 -2
View File
@@ -110,7 +110,7 @@ MEM_STATIC void MEM_check(void) { MEM_STATIC_ASSERT((sizeof(size_t)==4) || (size
#ifndef MEM_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */
# if defined(__GNUC__) && ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) )
# define MEM_FORCE_MEMORY_ACCESS 2
# elif defined(__INTEL_COMPILER) || \
# elif defined(__INTEL_COMPILER) /*|| defined(_MSC_VER)*/ || \
(defined(__GNUC__) && ( defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) || defined(__ARM_ARCH_7R__) || defined(__ARM_ARCH_7M__) || defined(__ARM_ARCH_7S__) ))
# define MEM_FORCE_MEMORY_ACCESS 1
# endif
@@ -142,7 +142,13 @@ MEM_STATIC void MEM_write64(void* memPtr, U64 value) { *(U64*)memPtr = value; }
/* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */
/* currently only defined for gcc and icc */
typedef union { U16 u16; U32 u32; U64 u64; size_t st; } __attribute__((packed)) unalign;
#if defined(_MSC_VER) || (defined(__INTEL_COMPILER) && defined(WIN32))
__pragma( pack(push, 1) )
typedef union { U16 u16; U32 u32; U64 u64; size_t st; } unalign;
__pragma( pack(pop) )
#else
typedef union { U16 u16; U32 u32; U64 u64; size_t st; } __attribute__((packed)) unalign;
#endif
MEM_STATIC U16 MEM_read16(const void* ptr) { return ((const unalign*)ptr)->u16; }
MEM_STATIC U32 MEM_read32(const void* ptr) { return ((const unalign*)ptr)->u32; }
lib/common/xxhash.c
+30 -17
View File
@@ -52,7 +52,7 @@
#ifndef XXH_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */
# if defined(__GNUC__) && ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) )
# define XXH_FORCE_MEMORY_ACCESS 2
# elif defined(__INTEL_COMPILER) || \
# elif (defined(__INTEL_COMPILER) && !defined(WIN32)) || \
(defined(__GNUC__) && ( defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) || defined(__ARM_ARCH_7R__) || defined(__ARM_ARCH_7M__) || defined(__ARM_ARCH_7S__) ))
# define XXH_FORCE_MEMORY_ACCESS 1
# endif
@@ -132,7 +132,7 @@ static void* XXH_memcpy(void* dest, const void* src, size_t size) { return memcp
***************************************/
#ifndef MEM_MODULE
# define MEM_MODULE
# if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */
# if !defined (__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) )
# include <stdint.h>
typedef uint8_t BYTE;
typedef uint16_t U16;
@@ -144,7 +144,7 @@ static void* XXH_memcpy(void* dest, const void* src, size_t size) { return memcp
typedef unsigned short U16;
typedef unsigned int U32;
typedef signed int S32;
typedef unsigned long long U64;
typedef unsigned long long U64; /* if your compiler doesn't support unsigned long long, replace by another 64-bit type here. Note that xxhash.h will also need to be updated. */
# endif
#endif
@@ -307,6 +307,20 @@ static const U64 PRIME64_5 = 2870177450012600261ULL;
XXH_PUBLIC_API unsigned XXH_versionNumber (void) { return XXH_VERSION_NUMBER; }
/* **************************
* Utils
****************************/
XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t* restrict dstState, const XXH32_state_t* restrict srcState)
{
memcpy(dstState, srcState, sizeof(*dstState));
}
XXH_PUBLIC_API void XXH64_copyState(XXH64_state_t* restrict dstState, const XXH64_state_t* restrict srcState)
{
memcpy(dstState, srcState, sizeof(*dstState));
}
/* ***************************
* Simple Hash Functions
*****************************/
@@ -545,8 +559,7 @@ XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr)
XXH_PUBLIC_API XXH_errorcode XXH32_reset(XXH32_state_t* statePtr, unsigned int seed)
{
XXH32_state_t state; /* using a local state to memcpy() in order to avoid strict-aliasing warnings */
memset(&state, 0, sizeof(state));
state.seed = seed;
memset(&state, 0, sizeof(state)-4); /* do not write into reserved, for future removal */
state.v1 = seed + PRIME32_1 + PRIME32_2;
state.v2 = seed + PRIME32_2;
state.v3 = seed + 0;
@@ -559,8 +572,7 @@ XXH_PUBLIC_API XXH_errorcode XXH32_reset(XXH32_state_t* statePtr, unsigned int s
XXH_PUBLIC_API XXH_errorcode XXH64_reset(XXH64_state_t* statePtr, unsigned long long seed)
{
XXH64_state_t state; /* using a local state to memcpy() in order to avoid strict-aliasing warnings */
memset(&state, 0, sizeof(state));
state.seed = seed;
memset(&state, 0, sizeof(state)-8); /* do not write into reserved, for future removal */
state.v1 = seed + PRIME64_1 + PRIME64_2;
state.v2 = seed + PRIME64_2;
state.v3 = seed + 0;
@@ -579,11 +591,12 @@ FORCE_INLINE XXH_errorcode XXH32_update_endian (XXH32_state_t* state, const void
if (input==NULL) return XXH_ERROR;
#endif
state->total_len += len;
state->total_len_32 += (unsigned)len;
state->large_len |= (len>=16) | (state->total_len_32>=16);
if (state->memsize + len < 16) { /* fill in tmp buffer */
XXH_memcpy((BYTE*)(state->mem32) + state->memsize, input, len);
state->memsize += (U32)len;
state->memsize += (unsigned)len;
return XXH_OK;
}
@@ -620,8 +633,8 @@ FORCE_INLINE XXH_errorcode XXH32_update_endian (XXH32_state_t* state, const void
}
if (p < bEnd) {
XXH_memcpy(state->mem32, p, bEnd-p);
state->memsize = (int)(bEnd-p);
XXH_memcpy(state->mem32, p, (size_t)(bEnd-p));
state->memsize = (unsigned)(bEnd-p);
}
return XXH_OK;
@@ -645,13 +658,13 @@ FORCE_INLINE U32 XXH32_digest_endian (const XXH32_state_t* state, XXH_endianess
const BYTE* const bEnd = (const BYTE*)(state->mem32) + state->memsize;
U32 h32;
if (state->total_len >= 16) {
if (state->large_len) {
h32 = XXH_rotl32(state->v1, 1) + XXH_rotl32(state->v2, 7) + XXH_rotl32(state->v3, 12) + XXH_rotl32(state->v4, 18);
} else {
h32 = state->seed + PRIME32_5;
h32 = state->v3 /* == seed */ + PRIME32_5;
}
h32 += (U32) state->total_len;
h32 += state->total_len_32;
while (p+4<=bEnd) {
h32 += XXH_readLE32(p, endian) * PRIME32_3;
@@ -737,8 +750,8 @@ FORCE_INLINE XXH_errorcode XXH64_update_endian (XXH64_state_t* state, const void
}
if (p < bEnd) {
XXH_memcpy(state->mem64, p, bEnd-p);
state->memsize = (int)(bEnd-p);
XXH_memcpy(state->mem64, p, (size_t)(bEnd-p));
state->memsize = (unsigned)(bEnd-p);
}
return XXH_OK;
@@ -774,7 +787,7 @@ FORCE_INLINE U64 XXH64_digest_endian (const XXH64_state_t* state, XXH_endianess
h64 = XXH64_mergeRound(h64, v3);
h64 = XXH64_mergeRound(h64, v4);
} else {
h64 = state->seed + PRIME64_5;
h64 = state->v3 + PRIME64_5;
}
h64 += (U64) state->total_len;
lib/common/xxhash.h
+71 -35
View File
@@ -71,6 +71,10 @@ XXH32 6.8 GB/s 6.0 GB/s
extern "C" {
#endif
#ifndef XXH_NAMESPACE
# define XXH_NAMESPACE ZSTD_ /* Zstandard specific */
#endif
/* ****************************
* Definitions
@@ -82,18 +86,21 @@ typedef enum { XXH_OK=0, XXH_ERROR } XXH_errorcode;
/* ****************************
* API modifier
******************************/
/*!XXH_PRIVATE_API
* Transforms all publics symbols within `xxhash.c` into private ones.
* Methodology :
* instead of : #include "xxhash.h"
* do :
/** XXH_PRIVATE_API
* This is useful if you want to include xxhash functions in `static` mode
* in order to inline them, and remove their symbol from the public list.
* Methodology :
* #define XXH_PRIVATE_API
* #include "xxhash.c" // note the .c , instead of .h
* also : don't compile and link xxhash.c separately
* #include "xxhash.h"
* `xxhash.c` is automatically included.
* It's not useful to compile and link it as a separate module anymore.
*/
#ifdef XXH_PRIVATE_API
# ifndef XXH_STATIC_LINKING_ONLY
# define XXH_STATIC_LINKING_ONLY
# endif
# if defined(__GNUC__)
# define XXH_PUBLIC_API static __attribute__((unused))
# define XXH_PUBLIC_API static __inline __attribute__((unused))
# elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
# define XXH_PUBLIC_API static inline
# elif defined(_MSC_VER)
@@ -103,17 +110,17 @@ typedef enum { XXH_OK=0, XXH_ERROR } XXH_errorcode;
# endif
#else
# define XXH_PUBLIC_API /* do nothing */
#endif
#endif /* XXH_PRIVATE_API */
/*!XXH_NAMESPACE, aka Namespace Emulation :
If you want to include _and expose_ xxHash functions from within your own library,
but also want to avoid symbol collisions with another library which also includes xxHash,
you can use XXH_NAMESPACE, to automatically prefix any public symbol from `xxhash.c`
you can use XXH_NAMESPACE, to automatically prefix any public symbol from xxhash library
with the value of XXH_NAMESPACE (so avoid to keep it NULL and avoid numeric values).
Note that no change is required within the calling program as long as it also includes `xxhash.h` :
Note that no change is required within the calling program as long as it includes `xxhash.h` :
regular symbol name will be automatically translated by this header.
*/
#ifdef XXH_NAMESPACE
@@ -132,6 +139,12 @@ regular symbol name will be automatically translated by this header.
# define XXH64_update XXH_NAME2(XXH_NAMESPACE, XXH64_update)
# define XXH32_digest XXH_NAME2(XXH_NAMESPACE, XXH32_digest)
# define XXH64_digest XXH_NAME2(XXH_NAMESPACE, XXH64_digest)
# define XXH32_copyState XXH_NAME2(XXH_NAMESPACE, XXH32_copyState)
# define XXH64_copyState XXH_NAME2(XXH_NAMESPACE, XXH64_copyState)
# define XXH32_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH32_canonicalFromHash)
# define XXH64_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH64_canonicalFromHash)
# define XXH32_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH32_hashFromCanonical)
# define XXH64_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH64_hashFromCanonical)
#endif
@@ -140,7 +153,7 @@ regular symbol name will be automatically translated by this header.
***************************************/
#define XXH_VERSION_MAJOR 0
#define XXH_VERSION_MINOR 6
#define XXH_VERSION_RELEASE 0
#define XXH_VERSION_RELEASE 2
#define XXH_VERSION_NUMBER (XXH_VERSION_MAJOR *100*100 + XXH_VERSION_MINOR *100 + XXH_VERSION_RELEASE)
XXH_PUBLIC_API unsigned XXH_versionNumber (void);
@@ -163,7 +176,7 @@ XXH32() :
XXH64() :
Calculate the 64-bits hash of sequence of length "len" stored at memory address "input".
"seed" can be used to alter the result predictably.
This function runs faster on 64-bits systems, but slower on 32-bits systems (see benchmark).
This function runs 2x faster on 64-bits systems, but slower on 32-bits systems (see benchmark).
*/
@@ -173,8 +186,7 @@ XXH64() :
typedef struct XXH32_state_s XXH32_state_t; /* incomplete type */
typedef struct XXH64_state_s XXH64_state_t; /* incomplete type */
/*! Dynamic allocation of states
Compatible with dynamic libraries */
/*! State allocation, compatible with dynamic libraries */
XXH_PUBLIC_API XXH32_state_t* XXH32_createState(void);
XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr);
@@ -193,28 +205,40 @@ XXH_PUBLIC_API XXH_errorcode XXH64_reset (XXH64_state_t* statePtr, unsigned lon
XXH_PUBLIC_API XXH_errorcode XXH64_update (XXH64_state_t* statePtr, const void* input, size_t length);
XXH_PUBLIC_API XXH64_hash_t XXH64_digest (const XXH64_state_t* statePtr);
/*!
These functions generate the xxHash of an input provided in multiple segments,
as opposed to provided as a single block.
/*
These functions generate the xxHash of an input provided in multiple segments.
Note that, for small input, they are slower than single-call functions, due to state management.
For small input, prefer `XXH32()` and `XXH64()` .
XXH state must first be allocated, using either static or dynamic method provided above.
XXH state must first be allocated, using XXH*_createState() .
Start a new hash by initializing state with a seed, using XXHnn_reset().
Start a new hash by initializing state with a seed, using XXH*_reset().
Then, feed the hash state by calling XXHnn_update() as many times as necessary.
Obviously, input must be valid, hence allocated and read accessible.
Then, feed the hash state by calling XXH*_update() as many times as necessary.
Obviously, input must be allocated and read accessible.
The function returns an error code, with 0 meaning OK, and any other value meaning there is an error.
Finally, a hash value can be produced anytime, by using XXHnn_digest().
Finally, a hash value can be produced anytime, by using XXH*_digest().
This function returns the nn-bits hash as an int or long long.
It's still possible to continue inserting input into the hash state after a digest,
and later on generate some new hashes, by calling again XXHnn_digest().
and generate some new hashes later on, by calling again XXH*_digest().
When done, free XXH state space if it was allocated dynamically.
*/
/* **************************
* Utils
****************************/
#if !(defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L)) /* ! C99 */
# define restrict /* disable restrict */
#endif
XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t* restrict dst_state, const XXH32_state_t* restrict src_state);
XXH_PUBLIC_API void XXH64_copyState(XXH64_state_t* restrict dst_state, const XXH64_state_t* restrict src_state);
/* **************************
* Canonical representation
****************************/
@@ -227,43 +251,55 @@ XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH64_canonical_t* dst, XXH64_hash_t
XXH_PUBLIC_API XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src);
XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(const XXH64_canonical_t* src);
/*! Default result type for XXH functions are primitive unsigned 32 and 64 bits.
* The canonical representation uses human-readable write convention, aka big-endian (large digits first).
* These functions allow transformation of hash result into and from its canonical format.
* This way, hash values can be written into a file / memory, and remain comparable on different systems and programs.
/* Default result type for XXH functions are primitive unsigned 32 and 64 bits.
* The canonical representation uses human-readable write convention, aka big-endian (large digits first).
* These functions allow transformation of hash result into and from its canonical format.
* This way, hash values can be written into a file / memory, and remain comparable on different systems and programs.
*/
#ifdef XXH_STATIC_LINKING_ONLY
/* This part contains definition which shall only be used with static linking.
The prototypes / types defined here are not guaranteed to remain stable.
They could change in a future version, becoming incompatible with a different version of the library */
/* ================================================================================================
This section contains definitions which are not guaranteed to remain stable.
They may change in future versions, becoming incompatible with a different version of the library.
They shall only be used with static linking.
Never use these definitions in association with dynamic linking !
=================================================================================================== */
/* These definitions are only meant to allow allocation of XXH state
statically, on stack, or in a struct for example.
Do not use members directly. */
struct XXH32_state_s {
unsigned long long total_len;
unsigned seed;
unsigned total_len_32;
unsigned large_len;
unsigned v1;
unsigned v2;
unsigned v3;
unsigned v4;
unsigned mem32[4]; /* buffer defined as U32 for alignment */
unsigned memsize;
unsigned reserved; /* never read nor write, will be removed in a future version */
}; /* typedef'd to XXH32_state_t */
struct XXH64_state_s {
unsigned long long total_len;
unsigned long long seed;
unsigned long long v1;
unsigned long long v2;
unsigned long long v3;
unsigned long long v4;
unsigned long long mem64[4]; /* buffer defined as U64 for alignment */
unsigned memsize;
unsigned reserved[2]; /* never read nor write, will be removed in a future version */
}; /* typedef'd to XXH64_state_t */
#endif
# ifdef XXH_PRIVATE_API
# include "xxhash.c" /* include xxhash functions as `static`, for inlining */
# endif
#endif /* XXH_STATIC_LINKING_ONLY */
#if defined (__cplusplus)
lib/common/zstd_internal.h
+14 -3
View File
@@ -67,9 +67,10 @@
#define ZSTD_OPT_NUM (1<<12)
#define ZSTD_DICT_MAGIC 0xEC30A437 /* v0.7+ */
#define ZSTD_REP_NUM 3 /* number of repcodes */
#define ZSTD_REP_CHECK (ZSTD_REP_NUM-0) /* number of repcodes to check by the optimal parser */
#define ZSTD_REP_MOVE (ZSTD_REP_NUM-1)
#define ZSTD_REP_NUM 3 /* number of repcodes */
#define ZSTD_REP_CHECK (ZSTD_REP_NUM) /* number of repcodes to check by the optimal parser */
#define ZSTD_REP_MOVE (ZSTD_REP_NUM-1)
#define ZSTD_REP_MOVE_OPT (ZSTD_REP_NUM)
static const U32 repStartValue[ZSTD_REP_NUM] = { 1, 4, 8 };
#define KB *(1 <<10)
@@ -154,6 +155,16 @@ MEM_STATIC void ZSTD_wildcopy(void* dst, const void* src, size_t length)
while (op < oend);
}
MEM_STATIC void ZSTD_wildcopy_e(void* dst, const void* src, void* dstEnd) /* should be faster for decoding, but strangely, not verified on all platform */
{
const BYTE* ip = (const BYTE*)src;
BYTE* op = (BYTE*)dst;
BYTE* const oend = (BYTE*)dstEnd;
do
COPY8(op, ip)
while (op < oend);
}
/*-*******************************************
* Private interfaces
lib/compress/zstd_compress.c
+329 -61
View File
@@ -32,7 +32,7 @@
*/
/* *******************************************************
/*-*******************************************************
* Compiler specifics
*********************************************************/
#ifdef _MSC_VER /* Visual Studio */
@@ -55,7 +55,7 @@
#include "mem.h"
#define XXH_STATIC_LINKING_ONLY /* XXH64_state_t */
#include "xxhash.h" /* XXH_reset, update, digest */
#define FSE_STATIC_LINKING_ONLY
#define FSE_STATIC_LINKING_ONLY /* FSE_encodeSymbol */
#include "fse.h"
#define HUF_STATIC_LINKING_ONLY
#include "huf.h"
@@ -940,8 +940,8 @@ static void ZSTD_fillHashTable (ZSTD_CCtx* zc, const void* end, const U32 mls)
FORCE_INLINE
void ZSTD_compressBlock_fast_generic(ZSTD_CCtx* cctx,
const void* src, size_t srcSize,
const U32 mls)
const void* src, size_t srcSize,
const U32 mls)
{
U32* const hashTable = cctx->hashTable;
U32 const hBits = cctx->params.cParams.hashLog;
@@ -973,7 +973,7 @@ void ZSTD_compressBlock_fast_generic(ZSTD_CCtx* cctx,
const BYTE* match = base + matchIndex;
hashTable[h] = current; /* update hash table */
if ((offset_1 > 0) & (MEM_read32(ip+1-offset_1) == MEM_read32(ip+1))) { /* note : by construction, offset_1 <= current */
if ((offset_1 > 0) & (MEM_read32(ip+1-offset_1) == MEM_read32(ip+1))) {
mLength = ZSTD_count(ip+1+4, ip+1+4-offset_1, iend) + 4;
ip++;
ZSTD_storeSeq(seqStorePtr, ip-anchor, anchor, 0, mLength-MINMATCH);
@@ -2205,8 +2205,13 @@ static ZSTD_blockCompressor ZSTD_selectBlockCompressor(ZSTD_strategy strat, int
static size_t ZSTD_compressBlock_internal(ZSTD_CCtx* zc, void* dst, size_t dstCapacity, const void* src, size_t srcSize)
{
ZSTD_blockCompressor const blockCompressor = ZSTD_selectBlockCompressor(zc->params.cParams.strategy, zc->lowLimit < zc->dictLimit);
const BYTE* const base = zc->base;
const BYTE* const istart = (const BYTE*)src;
const U32 current = (U32)(istart-base);
if (srcSize < MIN_CBLOCK_SIZE+ZSTD_blockHeaderSize+1) return 0; /* don't even attempt compression below a certain srcSize */
ZSTD_resetSeqStore(&(zc->seqStore));
if (current > zc->nextToUpdate + 384)
zc->nextToUpdate = current - MIN(192, (U32)(current - zc->nextToUpdate - 384)); /* update tree not updated after finding very long rep matches */
blockCompressor(zc, src, srcSize);
return ZSTD_compressSequences(zc, dst, dstCapacity, srcSize);
}
@@ -2244,6 +2249,21 @@ static size_t ZSTD_compress_generic (ZSTD_CCtx* cctx,
if (dstCapacity < ZSTD_blockHeaderSize + MIN_CBLOCK_SIZE) return ERROR(dstSize_tooSmall); /* not enough space to store compressed block */
if (remaining < blockSize) blockSize = remaining;
/* preemptive overflow correction */
if (cctx->lowLimit > (1<<30)) {
U32 const btplus = (cctx->params.cParams.strategy == ZSTD_btlazy2) | (cctx->params.cParams.strategy == ZSTD_btopt);
U32 const chainMask = (1 << (cctx->params.cParams.chainLog - btplus)) - 1;
U32 const newLowLimit = cctx->lowLimit & chainMask; /* preserve position % chainSize */
U32 const correction = cctx->lowLimit - newLowLimit;
ZSTD_reduceIndex(cctx, correction);
cctx->base += correction;
cctx->dictBase += correction;
cctx->lowLimit = newLowLimit;
cctx->dictLimit -= correction;
if (cctx->nextToUpdate < correction) cctx->nextToUpdate = 0;
else cctx->nextToUpdate -= correction;
}
if ((U32)(ip+blockSize - cctx->base) > cctx->loadedDictEnd + maxDist) {
/* enforce maxDist */
U32 const newLowLimit = (U32)(ip+blockSize - cctx->base) - maxDist;
@@ -2317,7 +2337,7 @@ static size_t ZSTD_writeFrameHeader(void* dst, size_t dstCapacity,
}
static size_t ZSTD_compressContinue_internal (ZSTD_CCtx* zc,
static size_t ZSTD_compressContinue_internal (ZSTD_CCtx* cctx,
void* dst, size_t dstCapacity,
const void* src, size_t srcSize,
U32 frame, U32 lastFrameChunk)
@@ -2325,53 +2345,40 @@ static size_t ZSTD_compressContinue_internal (ZSTD_CCtx* zc,
const BYTE* const ip = (const BYTE*) src;
size_t fhSize = 0;
if (zc->stage==ZSTDcs_created) return ERROR(stage_wrong); /* missing init (ZSTD_compressBegin) */
if (cctx->stage==ZSTDcs_created) return ERROR(stage_wrong); /* missing init (ZSTD_compressBegin) */
if (frame && (zc->stage==ZSTDcs_init)) {
fhSize = ZSTD_writeFrameHeader(dst, dstCapacity, zc->params, zc->frameContentSize, zc->dictID);
if (frame && (cctx->stage==ZSTDcs_init)) {
fhSize = ZSTD_writeFrameHeader(dst, dstCapacity, cctx->params, cctx->frameContentSize, cctx->dictID);
if (ZSTD_isError(fhSize)) return fhSize;
dstCapacity -= fhSize;
dst = (char*)dst + fhSize;
zc->stage = ZSTDcs_ongoing;
cctx->stage = ZSTDcs_ongoing;
}
/* Check if blocks follow each other */
if (src != zc->nextSrc) {
if (src != cctx->nextSrc) {
/* not contiguous */
ptrdiff_t const delta = zc->nextSrc - ip;
zc->lowLimit = zc->dictLimit;
zc->dictLimit = (U32)(zc->nextSrc - zc->base);
zc->dictBase = zc->base;
zc->base -= delta;
zc->nextToUpdate = zc->dictLimit;
if (zc->dictLimit - zc->lowLimit < HASH_READ_SIZE) zc->lowLimit = zc->dictLimit; /* too small extDict */
ptrdiff_t const delta = cctx->nextSrc - ip;
cctx->lowLimit = cctx->dictLimit;
cctx->dictLimit = (U32)(cctx->nextSrc - cctx->base);
cctx->dictBase = cctx->base;
cctx->base -= delta;
cctx->nextToUpdate = cctx->dictLimit;
if (cctx->dictLimit - cctx->lowLimit < HASH_READ_SIZE) cctx->lowLimit = cctx->dictLimit; /* too small extDict */
}
/* preemptive overflow correction */
if (zc->lowLimit > (1<<30)) {
U32 const btplus = (zc->params.cParams.strategy == ZSTD_btlazy2) | (zc->params.cParams.strategy == ZSTD_btopt);
U32 const chainMask = (1 << (zc->params.cParams.chainLog - btplus)) - 1;
U32 const newLowLimit = zc->lowLimit & chainMask; /* preserve position % chainSize */
U32 const correction = zc->lowLimit - newLowLimit;
ZSTD_reduceIndex(zc, correction);
zc->base += correction;
zc->dictBase += correction;
zc->lowLimit = newLowLimit;
zc->dictLimit -= correction;
if (zc->nextToUpdate < correction) zc->nextToUpdate = 0;
else zc->nextToUpdate -= correction;
/* if input and dictionary overlap : reduce dictionary (area presumed modified by input) */
if ((ip+srcSize > cctx->dictBase + cctx->lowLimit) & (ip < cctx->dictBase + cctx->dictLimit)) {
ptrdiff_t const highInputIdx = (ip + srcSize) - cctx->dictBase;
U32 const lowLimitMax = (highInputIdx > (ptrdiff_t)cctx->dictLimit) ? cctx->dictLimit : (U32)highInputIdx;
cctx->lowLimit = lowLimitMax;
}
/* if input and dictionary overlap : reduce dictionary (presumed modified by input) */
if ((ip+srcSize > zc->dictBase + zc->lowLimit) && (ip < zc->dictBase + zc->dictLimit)) {
zc->lowLimit = (U32)(ip + srcSize - zc->dictBase);
if (zc->lowLimit > zc->dictLimit) zc->lowLimit = zc->dictLimit;
}
cctx->nextSrc = ip + srcSize;
zc->nextSrc = ip + srcSize;
{ size_t const cSize = frame ?
ZSTD_compress_generic (zc, dst, dstCapacity, src, srcSize, lastFrameChunk) :
ZSTD_compressBlock_internal (zc, dst, dstCapacity, src, srcSize);
ZSTD_compress_generic (cctx, dst, dstCapacity, src, srcSize, lastFrameChunk) :
ZSTD_compressBlock_internal (cctx, dst, dstCapacity, src, srcSize);
if (ZSTD_isError(cSize)) return cSize;
return cSize + fhSize;
}
@@ -2619,22 +2626,6 @@ size_t ZSTD_compressEnd (ZSTD_CCtx* cctx,
}
/*! ZSTD_compress_usingPreparedCCtx() :
* Same as ZSTD_compress_usingDict, but using a reference context `preparedCCtx`, where dictionary has been loaded.
* It avoids reloading the dictionary each time.
* `preparedCCtx` must have been properly initialized using ZSTD_compressBegin_usingDict() or ZSTD_compressBegin_advanced().
* Requires 2 contexts : 1 for reference (preparedCCtx) which will not be modified, and 1 to run the compression operation (cctx) */
static size_t ZSTD_compress_usingPreparedCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx,
void* dst, size_t dstCapacity,
const void* src, size_t srcSize)
{
size_t const errorCode = ZSTD_copyCCtx(cctx, preparedCCtx);
if (ZSTD_isError(errorCode)) return errorCode;
return ZSTD_compressEnd(cctx, dst, dstCapacity, src, srcSize);
}
static size_t ZSTD_compress_internal (ZSTD_CCtx* cctx,
void* dst, size_t dstCapacity,
const void* src, size_t srcSize,
@@ -2730,9 +2721,7 @@ ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictSize, ZSTD_pa
ZSTD_CDict* ZSTD_createCDict(const void* dict, size_t dictSize, int compressionLevel)
{
ZSTD_customMem const allocator = { NULL, NULL, NULL };
ZSTD_parameters params;
memset(&params, 0, sizeof(params));
params.cParams = ZSTD_getCParams(compressionLevel, 0, dictSize);
ZSTD_parameters params = ZSTD_getParams(compressionLevel, 0, dictSize);
params.fParams.contentSizeFlag = 1;
return ZSTD_createCDict_advanced(dict, dictSize, params, allocator);
}
@@ -2747,14 +2736,293 @@ size_t ZSTD_freeCDict(ZSTD_CDict* cdict)
return 0;
}
/*! ZSTD_compress_usingCDict() :
* Compression using a digested Dictionary.
* Faster startup than ZSTD_compress_usingDict(), recommended when same dictionary is used multiple times.
* Note that compression level is decided during dictionary creation */
ZSTDLIB_API size_t ZSTD_compress_usingCDict(ZSTD_CCtx* cctx,
void* dst, size_t dstCapacity,
const void* src, size_t srcSize,
const ZSTD_CDict* cdict)
{
return ZSTD_compress_usingPreparedCCtx(cctx, cdict->refContext,
dst, dstCapacity,
src, srcSize);
size_t const errorCode = ZSTD_copyCCtx(cctx, cdict->refContext);
if (ZSTD_isError(errorCode)) return errorCode;
if (cdict->refContext->params.fParams.contentSizeFlag==1) {
cctx->params.fParams.contentSizeFlag = 1;
cctx->frameContentSize = srcSize;
}
return ZSTD_compressEnd(cctx, dst, dstCapacity, src, srcSize);
}
/* ******************************************************************
* Streaming
********************************************************************/
typedef enum { zcss_init, zcss_load, zcss_flush, zcss_final } ZSTD_cStreamStage;
struct ZSTD_CStream_s {
ZSTD_CCtx* zc;
char* inBuff;
size_t inBuffSize;
size_t inToCompress;
size_t inBuffPos;
size_t inBuffTarget;
size_t blockSize;
char* outBuff;
size_t outBuffSize;
size_t outBuffContentSize;
size_t outBuffFlushedSize;
ZSTD_cStreamStage stage;
U32 checksum;
U32 frameEnded;
ZSTD_customMem customMem;
}; /* typedef'd to ZSTD_CStream within "zstd.h" */
ZSTD_CStream* ZSTD_createCStream(void)
{
return ZSTD_createCStream_advanced(defaultCustomMem);
}
ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem)
{
ZSTD_CStream* zcs;
if (!customMem.customAlloc && !customMem.customFree)
customMem = defaultCustomMem;
if (!customMem.customAlloc || !customMem.customFree)
return NULL;
zcs = (ZSTD_CStream*)customMem.customAlloc(customMem.opaque, sizeof(ZSTD_CStream));
if (zcs==NULL) return NULL;
memset(zcs, 0, sizeof(ZSTD_CStream));
memcpy(&zcs->customMem, &customMem, sizeof(ZSTD_customMem));
zcs->zc = ZSTD_createCCtx_advanced(customMem);
if (zcs->zc == NULL) { ZSTD_freeCStream(zcs); return NULL; }
return zcs;
}
size_t ZSTD_freeCStream(ZSTD_CStream* zcs)
{
if (zcs==NULL) return 0; /* support free on NULL */
ZSTD_freeCCtx(zcs->zc);
if (zcs->inBuff) zcs->customMem.customFree(zcs->customMem.opaque, zcs->inBuff);
if (zcs->outBuff) zcs->customMem.customFree(zcs->customMem.opaque, zcs->outBuff);
zcs->customMem.customFree(zcs->customMem.opaque, zcs);
return 0;
}
/*====== Initialization ======*/
size_t ZSTD_CStreamInSize(void) { return ZSTD_BLOCKSIZE_ABSOLUTEMAX; }
size_t ZSTD_CStreamOutSize(void) { return ZSTD_compressBound(ZSTD_BLOCKSIZE_ABSOLUTEMAX) + ZSTD_blockHeaderSize + 4 /* 32-bits hash */ ; }
size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs,
const void* dict, size_t dictSize,
ZSTD_parameters params, unsigned long long pledgedSrcSize)
{
/* allocate buffers */
{ size_t const neededInBuffSize = (size_t)1 << params.cParams.windowLog;
if (zcs->inBuffSize < neededInBuffSize) {
zcs->inBuffSize = neededInBuffSize;
zcs->customMem.customFree(zcs->customMem.opaque, zcs->inBuff); /* should not be necessary */
zcs->inBuff = (char*)zcs->customMem.customAlloc(zcs->customMem.opaque, neededInBuffSize);
if (zcs->inBuff == NULL) return ERROR(memory_allocation);
}
zcs->blockSize = MIN(ZSTD_BLOCKSIZE_ABSOLUTEMAX, neededInBuffSize);
}
if (zcs->outBuffSize < ZSTD_compressBound(zcs->blockSize)+1) {
zcs->outBuffSize = ZSTD_compressBound(zcs->blockSize)+1;
zcs->customMem.customFree(zcs->customMem.opaque, zcs->outBuff); /* should not be necessary */
zcs->outBuff = (char*)zcs->customMem.customAlloc(zcs->customMem.opaque, zcs->outBuffSize);
if (zcs->outBuff == NULL) return ERROR(memory_allocation);
}
{ size_t const errorCode = ZSTD_compressBegin_advanced(zcs->zc, dict, dictSize, params, pledgedSrcSize);
if (ZSTD_isError(errorCode)) return errorCode; }
zcs->inToCompress = 0;
zcs->inBuffPos = 0;
zcs->inBuffTarget = zcs->blockSize;
zcs->outBuffContentSize = zcs->outBuffFlushedSize = 0;
zcs->stage = zcss_load;
zcs->checksum = params.fParams.checksumFlag > 0;
zcs->frameEnded = 0;
return 0; /* ready to go */
}
size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel)
{
ZSTD_parameters const params = ZSTD_getParams(compressionLevel, 0, dictSize);
return ZSTD_initCStream_advanced(zcs, dict, dictSize, params, 0);
}
size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel)
{
return ZSTD_initCStream_usingDict(zcs, NULL, 0, compressionLevel);
}
/*====== Compression ======*/
typedef enum { zsf_gather, zsf_flush, zsf_end } ZSTD_flush_e;
MEM_STATIC size_t ZSTD_limitCopy(void* dst, size_t dstCapacity, const void* src, size_t srcSize)
{
size_t const length = MIN(dstCapacity, srcSize);
memcpy(dst, src, length);
return length;
}
static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs,
void* dst, size_t* dstCapacityPtr,
const void* src, size_t* srcSizePtr,
ZSTD_flush_e const flush)
{
U32 someMoreWork = 1;
const char* const istart = (const char*)src;
const char* const iend = istart + *srcSizePtr;
const char* ip = istart;
char* const ostart = (char*)dst;
char* const oend = ostart + *dstCapacityPtr;
char* op = ostart;
while (someMoreWork) {
switch(zcs->stage)
{
case zcss_init: return ERROR(init_missing); /* call ZBUFF_compressInit() first ! */
case zcss_load:
/* complete inBuffer */
{ size_t const toLoad = zcs->inBuffTarget - zcs->inBuffPos;
size_t const loaded = ZSTD_limitCopy(zcs->inBuff + zcs->inBuffPos, toLoad, ip, iend-ip);
zcs->inBuffPos += loaded;
ip += loaded;
if ( (zcs->inBuffPos==zcs->inToCompress) || (!flush && (toLoad != loaded)) ) {
someMoreWork = 0; break; /* not enough input to get a full block : stop there, wait for more */
} }
/* compress current block (note : this stage cannot be stopped in the middle) */
{ void* cDst;
size_t cSize;
size_t const iSize = zcs->inBuffPos - zcs->inToCompress;
size_t oSize = oend-op;
if (oSize >= ZSTD_compressBound(iSize))
cDst = op; /* compress directly into output buffer (avoid flush stage) */
else
cDst = zcs->outBuff, oSize = zcs->outBuffSize;
cSize = (flush == zsf_end) ?
ZSTD_compressEnd(zcs->zc, cDst, oSize, zcs->inBuff + zcs->inToCompress, iSize) :
ZSTD_compressContinue(zcs->zc, cDst, oSize, zcs->inBuff + zcs->inToCompress, iSize);
if (ZSTD_isError(cSize)) return cSize;
if (flush == zsf_end) zcs->frameEnded = 1;
/* prepare next block */
zcs->inBuffTarget = zcs->inBuffPos + zcs->blockSize;
if (zcs->inBuffTarget > zcs->inBuffSize)
zcs->inBuffPos = 0, zcs->inBuffTarget = zcs->blockSize; /* note : inBuffSize >= blockSize */
zcs->inToCompress = zcs->inBuffPos;
if (cDst == op) { op += cSize; break; } /* no need to flush */
zcs->outBuffContentSize = cSize;
zcs->outBuffFlushedSize = 0;
zcs->stage = zcss_flush; /* pass-through to flush stage */
}
case zcss_flush:
{ size_t const toFlush = zcs->outBuffContentSize - zcs->outBuffFlushedSize;
size_t const flushed = ZSTD_limitCopy(op, oend-op, zcs->outBuff + zcs->outBuffFlushedSize, toFlush);
op += flushed;
zcs->outBuffFlushedSize += flushed;
if (toFlush!=flushed) { someMoreWork = 0; break; } /* dst too small to store flushed data : stop there */
zcs->outBuffContentSize = zcs->outBuffFlushedSize = 0;
zcs->stage = zcss_load;
break;
}
case zcss_final:
someMoreWork = 0; /* do nothing */
break;
default:
return ERROR(GENERIC); /* impossible */
}
}
*srcSizePtr = ip - istart;
*dstCapacityPtr = op - ostart;
if (zcs->frameEnded) return 0;
{ size_t hintInSize = zcs->inBuffTarget - zcs->inBuffPos;
if (hintInSize==0) hintInSize = zcs->blockSize;
return hintInSize;
}
}
size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuffer* input)
{
size_t sizeRead = input->size - input->pos;
size_t sizeWritten = output->size - output->pos;
size_t const result = ZSTD_compressStream_generic(zcs,
(char*)(output->dst) + output->pos, &sizeWritten,
(const char*)(input->src) + input->pos, &sizeRead, zsf_gather);
input->pos += sizeRead;
output->pos += sizeWritten;
return result;
}
/*====== Finalize ======*/
/*! ZSTD_flushStream() :
* @return : amount of data remaining to flush */
size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output)
{
size_t srcSize = 0;
size_t sizeWritten = output->size - output->pos;
size_t const result = ZSTD_compressStream_generic(zcs,
(char*)(output->dst) + output->pos, &sizeWritten,
&srcSize, &srcSize, /* use a valid src address instead of NULL */
zsf_flush);
output->pos += sizeWritten;
if (ZSTD_isError(result)) return result;
return zcs->outBuffContentSize - zcs->outBuffFlushedSize; /* remaining to flush */
}
size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output)
{
BYTE* const ostart = (BYTE*)(output->dst) + output->pos;
BYTE* const oend = (BYTE*)(output->dst) + output->size;
BYTE* op = ostart;
if (zcs->stage != zcss_final) {
/* flush whatever remains */
size_t srcSize = 0;
size_t sizeWritten = output->size - output->pos;
size_t const notEnded = ZSTD_compressStream_generic(zcs, ostart, &sizeWritten, &srcSize, &srcSize, zsf_end); /* use a valid src address instead of NULL */
size_t const remainingToFlush = zcs->outBuffContentSize - zcs->outBuffFlushedSize;
op += sizeWritten;
if (remainingToFlush) {
output->pos += sizeWritten;
return remainingToFlush + ZSTD_BLOCKHEADERSIZE /* final empty block */ + (zcs->checksum * 4);
}
/* create epilogue */
zcs->stage = zcss_final;
zcs->outBuffContentSize = !notEnded ? 0 :
ZSTD_compressEnd(zcs->zc, zcs->outBuff, zcs->outBuffSize, NULL, 0); /* write epilogue, including final empty block, into outBuff */
}
/* flush epilogue */
{ size_t const toFlush = zcs->outBuffContentSize - zcs->outBuffFlushedSize;
size_t const flushed = ZSTD_limitCopy(op, oend-op, zcs->outBuff + zcs->outBuffFlushedSize, toFlush);
op += flushed;
zcs->outBuffFlushedSize += flushed;
output->pos += op-ostart;
if (toFlush==flushed) zcs->stage = zcss_init; /* end reached */
return toFlush - flushed;
}
}
lib/compress/zstd_opt.h
+55 -48
View File
@@ -276,7 +276,7 @@ static U32 ZSTD_insertBtAndGetAllMatches (
/* save best solution */
if (currentMl > bestLength) {
bestLength = currentMl;
matches[mnum].off = ZSTD_REP_MOVE + current - matchIndex3;
matches[mnum].off = ZSTD_REP_MOVE_OPT + current - matchIndex3;
matches[mnum].len = (U32)currentMl;
mnum++;
if (currentMl > ZSTD_OPT_NUM) goto update;
@@ -321,7 +321,7 @@ static U32 ZSTD_insertBtAndGetAllMatches (
if (matchLength > bestLength) {
if (matchLength > matchEndIdx - matchIndex) matchEndIdx = matchIndex + (U32)matchLength;
bestLength = matchLength;
matches[mnum].off = ZSTD_REP_MOVE + current - matchIndex;
matches[mnum].off = ZSTD_REP_MOVE_OPT + current - matchIndex;
matches[mnum].len = (U32)matchLength;
mnum++;
if (matchLength > ZSTD_OPT_NUM) break;
@@ -452,11 +452,12 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx,
litlen = (U32)(ip - anchor);
/* check repCode */
{ U32 i;
for (i=(ip == anchor); i<ZSTD_REP_CHECK; i++) {
if ((rep[i]<(U32)(ip-prefixStart))
&& (MEM_readMINMATCH(ip, minMatch) == MEM_readMINMATCH(ip - rep[i], minMatch))) {
mlen = (U32)ZSTD_count(ip+minMatch, ip+minMatch-rep[i], iend) + minMatch;
{ U32 i, last_i = ZSTD_REP_CHECK + (ip==anchor);
for (i=(ip == anchor); i<last_i; i++) {
const S32 repCur = ((i==ZSTD_REP_MOVE_OPT) && (ip==anchor)) ? (rep[0] - 1) : rep[i];
if ( (repCur > 0) && (repCur < (S32)(ip-prefixStart))
&& (MEM_readMINMATCH(ip, minMatch) == MEM_readMINMATCH(ip - repCur, minMatch))) {
mlen = (U32)ZSTD_count(ip+minMatch, ip+minMatch-repCur, iend) + minMatch;
ZSTD_LOG_PARSER("%d: start try REP rep[%d]=%d mlen=%d\n", (int)(ip-base), i, (int)rep[i], (int)mlen);
if (mlen > sufficient_len || mlen >= ZSTD_OPT_NUM) {
best_mlen = mlen; best_off = i; cur = 0; last_pos = 1;
@@ -491,7 +492,7 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx,
best_mlen = matches[u].len;
ZSTD_LOG_PARSER("%d: start Found mlen=%d off=%d best_mlen=%d last_pos=%d\n", (int)(ip-base), matches[u].len, matches[u].off, (int)best_mlen, (int)last_pos);
while (mlen <= best_mlen) {
price = ZSTD_getPrice(seqStorePtr, litlen, anchor, matches[u].off, mlen - MINMATCH);
price = ZSTD_getPrice(seqStorePtr, litlen, anchor, matches[u].off-1, mlen - MINMATCH);
if (mlen > last_pos || price < opt[mlen].price)
SET_PRICE(mlen, mlen, matches[u].off, litlen, price); /* note : macro modifies last_pos */
mlen++;
@@ -528,26 +529,27 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx,
continue;
mlen = opt[cur].mlen;
if (opt[cur].off >= ZSTD_REP_NUM) {
if (opt[cur].off > ZSTD_REP_MOVE_OPT) {
opt[cur].rep[2] = opt[cur-mlen].rep[1];
opt[cur].rep[1] = opt[cur-mlen].rep[0];
opt[cur].rep[0] = opt[cur].off - ZSTD_REP_MOVE;
opt[cur].rep[0] = opt[cur].off - ZSTD_REP_MOVE_OPT;
ZSTD_LOG_ENCODE("%d: COPYREP_OFF cur=%d mlen=%d rep[0]=%d rep[1]=%d\n", (int)(inr-base), cur, mlen, opt[cur].rep[0], opt[cur].rep[1]);
} else {
opt[cur].rep[2] = (opt[cur].off > 1) ? opt[cur-mlen].rep[1] : opt[cur-mlen].rep[2];
opt[cur].rep[1] = (opt[cur].off > 0) ? opt[cur-mlen].rep[0] : opt[cur-mlen].rep[1];
opt[cur].rep[0] = opt[cur-mlen].rep[opt[cur].off];
opt[cur].rep[0] = ((opt[cur].off==ZSTD_REP_MOVE_OPT) && (mlen != 1)) ? (opt[cur-mlen].rep[0] - 1) : (opt[cur-mlen].rep[opt[cur].off]);
ZSTD_LOG_ENCODE("%d: COPYREP_NOR cur=%d mlen=%d rep[0]=%d rep[1]=%d\n", (int)(inr-base), cur, mlen, opt[cur].rep[0], opt[cur].rep[1]);
}
ZSTD_LOG_PARSER("%d: CURRENT_NoExt price[%d/%d]=%d off=%d mlen=%d litlen=%d rep[0]=%d rep[1]=%d\n", (int)(inr-base), cur, last_pos, opt[cur].price, opt[cur].off, opt[cur].mlen, opt[cur].litlen, opt[cur].rep[0], opt[cur].rep[1]);
best_mlen = minMatch;
{ U32 i;
for (i=(opt[cur].mlen != 1); i<ZSTD_REP_CHECK; i++) { /* check rep */
if ((opt[cur].rep[i]<(U32)(inr-prefixStart))
&& (MEM_readMINMATCH(inr, minMatch) == MEM_readMINMATCH(inr - opt[cur].rep[i], minMatch))) {
mlen = (U32)ZSTD_count(inr+minMatch, inr+minMatch - opt[cur].rep[i], iend) + minMatch;
{ U32 i, last_i = ZSTD_REP_CHECK + (mlen != 1);
for (i=(opt[cur].mlen != 1); i<last_i; i++) { /* check rep */
const S32 repCur = ((i==ZSTD_REP_MOVE_OPT) && (opt[cur].mlen != 1)) ? (opt[cur].rep[0] - 1) : opt[cur].rep[i];
if ( (repCur > 0) && (repCur < (S32)(inr-prefixStart))
&& (MEM_readMINMATCH(inr, minMatch) == MEM_readMINMATCH(inr - repCur, minMatch))) {
mlen = (U32)ZSTD_count(inr+minMatch, inr+minMatch - repCur, iend) + minMatch;
ZSTD_LOG_PARSER("%d: Found REP %d/%d mlen=%d off=%d rep=%d opt[%d].off=%d\n", (int)(inr-base), i, ZSTD_REP_NUM, mlen, i, opt[cur].rep[i], cur, opt[cur].off);
if (mlen > sufficient_len || cur + mlen >= ZSTD_OPT_NUM) {
@@ -600,12 +602,12 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx,
if (opt[cur].mlen == 1) {
litlen = opt[cur].litlen;
if (cur > litlen)
price = opt[cur - litlen].price + ZSTD_getPrice(seqStorePtr, litlen, ip+cur-litlen, matches[u].off, mlen - MINMATCH);
price = opt[cur - litlen].price + ZSTD_getPrice(seqStorePtr, litlen, ip+cur-litlen, matches[u].off-1, mlen - MINMATCH);
else
price = ZSTD_getPrice(seqStorePtr, litlen, anchor, matches[u].off, mlen - MINMATCH);
price = ZSTD_getPrice(seqStorePtr, litlen, anchor, matches[u].off-1, mlen - MINMATCH);
} else {
litlen = 0;
price = opt[cur].price + ZSTD_getPrice(seqStorePtr, 0, NULL, matches[u].off, mlen - MINMATCH);
price = opt[cur].price + ZSTD_getPrice(seqStorePtr, 0, NULL, matches[u].off-1, mlen - MINMATCH);
}
// ZSTD_LOG_PARSER("%d: Found2 mlen=%d best_mlen=%d off=%d price=%d litlen=%d\n", (int)(inr-base), mlen, best_mlen, matches[u].off, price, litlen);
@@ -652,27 +654,28 @@ _storeSequence: /* cur, last_pos, best_mlen, best_off have to be set */
litLength = (U32)(ip - anchor);
// ZSTD_LOG_ENCODE("%d/%d: ENCODE literals=%d mlen=%d off=%d rep[0]=%d rep[1]=%d\n", (int)(ip-base), (int)(iend-base), (int)(litLength), (int)mlen, (int)(offset), (int)rep[0], (int)rep[1]);
if (offset >= ZSTD_REP_NUM) {
if (offset > ZSTD_REP_MOVE_OPT) {
rep[2] = rep[1];
rep[1] = rep[0];
rep[0] = offset - ZSTD_REP_MOVE;
rep[0] = offset - ZSTD_REP_MOVE_OPT;
offset--;
} else {
if (offset != 0) {
best_off = rep[offset];
best_off = ((offset==ZSTD_REP_MOVE_OPT) && (litLength==0)) ? (rep[0] - 1) : (rep[offset]);
if (offset != 1) rep[2] = rep[1];
rep[1] = rep[0];
rep[0] = best_off;
}
if ((litLength == 0) & (offset==0)) offset = rep[1]; /* protection, but should never happen */
if ((litLength == 0) & (offset<=2)) offset--;
if ((litLength==0) & (offset==0)) { ZSTD_LOG_ENCODE("ERROR (litLength==0) & (offset==0)\n"); };
if (litLength==0) offset--;
}
ZSTD_LOG_ENCODE("%d/%d: ENCODE literals=%d mlen=%d off=%d rep[0]=%d rep[1]=%d\n", (int)(ip-base), (int)(iend-base), (int)(litLength), (int)mlen, (int)(offset), (int)rep[0], (int)rep[1]);
#if ZSTD_OPT_DEBUG >= 5
U32 ml2;
if (offset >= ZSTD_REP_NUM)
ml2 = (U32)ZSTD_count(ip, ip-(offset-ZSTD_REP_MOVE), iend);
if (offset+1 > ZSTD_REP_MOVE_OPT)
ml2 = (U32)ZSTD_count(ip, ip-(offset+1-ZSTD_REP_MOVE_OPT), iend);
else
ml2 = (U32)ZSTD_count(ip, ip-rep[0], iend);
if ((offset >= 8) && (ml2 < mlen || ml2 < minMatch)) {
@@ -748,12 +751,13 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx,
opt[0].litlen = (U32)(ip - anchor);
/* check repCode */
{ U32 i;
for (i = (ip==anchor); i<ZSTD_REP_CHECK; i++) {
const U32 repIndex = (U32)(current - rep[i]);
{ U32 i, last_i = ZSTD_REP_CHECK + (ip==anchor);
for (i = (ip==anchor); i<last_i; i++) {
const S32 repCur = ((i==ZSTD_REP_MOVE_OPT) && (ip==anchor)) ? (rep[0] - 1) : rep[i];
const U32 repIndex = (U32)(current - repCur);
const BYTE* const repBase = repIndex < dictLimit ? dictBase : base;
const BYTE* const repMatch = repBase + repIndex;
if ( (rep[i] <= current)
if ( (repCur > 0 && repCur <= (S32)current)
&& (((U32)((dictLimit-1) - repIndex) >= 3) & (repIndex>lowestIndex)) /* intentional overflow */
&& (MEM_readMINMATCH(ip, minMatch) == MEM_readMINMATCH(repMatch, minMatch)) ) {
/* repcode detected we should take it */
@@ -801,7 +805,7 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx,
ZSTD_LOG_PARSER("%d: start Found mlen=%d off=%d best_mlen=%d last_pos=%d\n", (int)(ip-base), matches[u].len, matches[u].off, (int)best_mlen, (int)last_pos);
litlen = opt[0].litlen;
while (mlen <= best_mlen) {
price = ZSTD_getPrice(seqStorePtr, litlen, anchor, matches[u].off, mlen - MINMATCH);
price = ZSTD_getPrice(seqStorePtr, litlen, anchor, matches[u].off-1, mlen - MINMATCH);
if (mlen > last_pos || price < opt[mlen].price)
SET_PRICE(mlen, mlen, matches[u].off, litlen, price);
mlen++;
@@ -836,27 +840,28 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx,
continue;
mlen = opt[cur].mlen;
if (opt[cur].off >= ZSTD_REP_NUM) {
if (opt[cur].off > ZSTD_REP_MOVE_OPT) {
opt[cur].rep[2] = opt[cur-mlen].rep[1];
opt[cur].rep[1] = opt[cur-mlen].rep[0];
opt[cur].rep[0] = opt[cur].off - ZSTD_REP_MOVE;
opt[cur].rep[0] = opt[cur].off - ZSTD_REP_MOVE_OPT;
ZSTD_LOG_ENCODE("%d: COPYREP_OFF cur=%d mlen=%d rep[0]=%d rep[1]=%d\n", (int)(inr-base), cur, mlen, opt[cur].rep[0], opt[cur].rep[1]);
} else {
opt[cur].rep[2] = (opt[cur].off > 1) ? opt[cur-mlen].rep[1] : opt[cur-mlen].rep[2];
opt[cur].rep[1] = (opt[cur].off > 0) ? opt[cur-mlen].rep[0] : opt[cur-mlen].rep[1];
opt[cur].rep[0] = opt[cur-mlen].rep[opt[cur].off];
opt[cur].rep[0] = ((opt[cur].off==ZSTD_REP_MOVE_OPT) && (mlen != 1)) ? (opt[cur-mlen].rep[0] - 1) : (opt[cur-mlen].rep[opt[cur].off]);
ZSTD_LOG_ENCODE("%d: COPYREP_NOR cur=%d mlen=%d rep[0]=%d rep[1]=%d\n", (int)(inr-base), cur, mlen, opt[cur].rep[0], opt[cur].rep[1]);
}
ZSTD_LOG_PARSER("%d: CURRENT_Ext price[%d/%d]=%d off=%d mlen=%d litlen=%d rep[0]=%d rep[1]=%d\n", (int)(inr-base), cur, last_pos, opt[cur].price, opt[cur].off, opt[cur].mlen, opt[cur].litlen, opt[cur].rep[0], opt[cur].rep[1]);
best_mlen = 0;
{ U32 i;
for (i = (opt[cur].mlen != 1); i<ZSTD_REP_CHECK; i++) {
const U32 repIndex = (U32)(current+cur - opt[cur].rep[i]);
{ U32 i, last_i = ZSTD_REP_CHECK + (mlen != 1);
for (i = (mlen != 1); i<last_i; i++) {
const S32 repCur = ((i==ZSTD_REP_MOVE_OPT) && (opt[cur].mlen != 1)) ? (opt[cur].rep[0] - 1) : opt[cur].rep[i];
const U32 repIndex = (U32)(current+cur - repCur);
const BYTE* const repBase = repIndex < dictLimit ? dictBase : base;
const BYTE* const repMatch = repBase + repIndex;
if ( (opt[cur].rep[i] <= current+cur)
if ( (repCur > 0 && repCur <= (S32)(current+cur))
&& (((U32)((dictLimit-1) - repIndex) >= 3) & (repIndex>lowestIndex)) /* intentional overflow */
&& (MEM_readMINMATCH(inr, minMatch) == MEM_readMINMATCH(repMatch, minMatch)) ) {
/* repcode detected */
@@ -914,12 +919,12 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx,
if (opt[cur].mlen == 1) {
litlen = opt[cur].litlen;
if (cur > litlen)
price = opt[cur - litlen].price + ZSTD_getPrice(seqStorePtr, litlen, ip+cur-litlen, matches[u].off, mlen - MINMATCH);
price = opt[cur - litlen].price + ZSTD_getPrice(seqStorePtr, litlen, ip+cur-litlen, matches[u].off-1, mlen - MINMATCH);
else
price = ZSTD_getPrice(seqStorePtr, litlen, anchor, matches[u].off, mlen - MINMATCH);
price = ZSTD_getPrice(seqStorePtr, litlen, anchor, matches[u].off-1, mlen - MINMATCH);
} else {
litlen = 0;
price = opt[cur].price + ZSTD_getPrice(seqStorePtr, 0, NULL, matches[u].off, mlen - MINMATCH);
price = opt[cur].price + ZSTD_getPrice(seqStorePtr, 0, NULL, matches[u].off-1, mlen - MINMATCH);
}
// ZSTD_LOG_PARSER("%d: Found2 mlen=%d best_mlen=%d off=%d price=%d litlen=%d\n", (int)(inr-base), mlen, best_mlen, matches[u].off, price, litlen);
@@ -966,27 +971,29 @@ _storeSequence: /* cur, last_pos, best_mlen, best_off have to be set */
litLength = (U32)(ip - anchor);
// ZSTD_LOG_ENCODE("%d/%d: ENCODE1 literals=%d mlen=%d off=%d rep[0]=%d rep[1]=%d\n", (int)(ip-base), (int)(iend-base), (int)(litLength), (int)mlen, (int)(offset), (int)rep[0], (int)rep[1]);
if (offset >= ZSTD_REP_NUM) {
if (offset > ZSTD_REP_MOVE_OPT) {
rep[2] = rep[1];
rep[1] = rep[0];
rep[0] = offset - ZSTD_REP_MOVE;
rep[0] = offset - ZSTD_REP_MOVE_OPT;
offset--;
} else {
if (offset != 0) {
best_off = rep[offset];
best_off = ((offset==ZSTD_REP_MOVE_OPT) && (litLength==0)) ? (rep[0] - 1) : (rep[offset]);
if (offset != 1) rep[2] = rep[1];
rep[1] = rep[0];
rep[0] = best_off;
}
if ((litLength==0) & (offset==0)) offset = rep[1]; /* protection, but should never happen */
if ((litLength==0) & (offset<=2)) offset --;
if ((litLength==0) & (offset==0)) { ZSTD_LOG_ENCODE("ERROR (litLength==0) & (offset==0)\n"); };
if (litLength==0) offset--;
}
ZSTD_LOG_ENCODE("%d/%d: ENCODE literals=%d mlen=%d off=%d rep[0]=%d rep[1]=%d\n", (int)(ip-base), (int)(iend-base), (int)(litLength), (int)mlen, (int)(offset), (int)rep[0], (int)rep[1]);
#if ZSTD_OPT_DEBUG >= 5
U32 ml2;
if (offset >= ZSTD_REP_NUM) {
best_off = offset - ZSTD_REP_MOVE;
if (offset+1 > ZSTD_REP_MOVE_OPT) {
best_off = offset+1 - ZSTD_REP_MOVE_OPT;
if (best_off > (size_t)(ip - prefixStart)) {
const BYTE* match = dictEnd - (best_off - (ip - prefixStart));
ml2 = ZSTD_count_2segments(ip, match, iend, dictEnd, prefixStart);
lib/decompress/zstd_decompress.c
+245 -10
View File
@@ -77,6 +77,7 @@
# include <intrin.h> /* For Visual 2005 */
# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
# pragma warning(disable : 4324) /* disable: C4324: padded structure */
# pragma warning(disable : 4100) /* disable: C4100: unreferenced formal parameter */
#else
# ifdef __GNUC__
# define FORCE_INLINE static inline __attribute__((always_inline))
@@ -152,7 +153,8 @@ size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx)
dctx->hufTable[0] = (HUF_DTable)((HufLog)*0x1000001);
dctx->litEntropy = dctx->fseEntropy = 0;
dctx->dictID = 0;
{ int i; for (i=0; i<ZSTD_REP_NUM; i++) dctx->rep[i] = repStartValue[i]; }
MEM_STATIC_ASSERT(sizeof(dctx->rep)==sizeof(repStartValue));
memcpy(dctx->rep, repStartValue, sizeof(repStartValue));
return 0;
}
@@ -160,15 +162,12 @@ ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem)
{
ZSTD_DCtx* dctx;
if (!customMem.customAlloc && !customMem.customFree)
customMem = defaultCustomMem;
if (!customMem.customAlloc || !customMem.customFree)
return NULL;
if (!customMem.customAlloc && !customMem.customFree) customMem = defaultCustomMem;
if (!customMem.customAlloc || !customMem.customFree) return NULL;
dctx = (ZSTD_DCtx*) customMem.customAlloc(customMem.opaque, sizeof(ZSTD_DCtx));
if (!dctx) return NULL;
memcpy(&dctx->customMem, &customMem, sizeof(ZSTD_customMem));
memcpy(&dctx->customMem, &customMem, sizeof(customMem));
ZSTD_decompressBegin(dctx);
return dctx;
}
@@ -187,8 +186,8 @@ size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx)
void ZSTD_copyDCtx(ZSTD_DCtx* dstDCtx, const ZSTD_DCtx* srcDCtx)
{
memcpy(dstDCtx, srcDCtx,
sizeof(ZSTD_DCtx) - (ZSTD_BLOCKSIZE_ABSOLUTEMAX+WILDCOPY_OVERLENGTH + ZSTD_frameHeaderSize_max)); /* no need to copy workspace */
size_t const workSpaceSize = (ZSTD_BLOCKSIZE_ABSOLUTEMAX+WILDCOPY_OVERLENGTH) + ZSTD_frameHeaderSize_max;
memcpy(dstDCtx, srcDCtx, sizeof(ZSTD_DCtx) - workSpaceSize); /* no need to copy workspace */
}
@@ -209,7 +208,7 @@ static size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize)
U32 const singleSegment = (fhd >> 5) & 1;
U32 const fcsId = fhd >> 6;
return ZSTD_frameHeaderSize_min + !singleSegment + ZSTD_did_fieldSize[dictID] + ZSTD_fcs_fieldSize[fcsId]
+ (singleSegment && !ZSTD_fcs_fieldSize[fcsId]);
+ (singleSegment && !fcsId);
}
}
@@ -1300,3 +1299,239 @@ ZSTDLIB_API size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx,
dst, dstCapacity,
src, srcSize);
}
/*=====================================
* Streaming decompression
*====================================*/
typedef enum { zdss_init, zdss_loadHeader,
zdss_read, zdss_load, zdss_flush } ZSTD_dStreamStage;
/* *** Resource management *** */
struct ZSTD_DStream_s {
ZSTD_DCtx* zd;
ZSTD_frameParams fParams;
ZSTD_dStreamStage stage;
char* inBuff;
size_t inBuffSize;
size_t inPos;
char* outBuff;
size_t outBuffSize;
size_t outStart;
size_t outEnd;
size_t blockSize;
BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX];
size_t lhSize;
ZSTD_customMem customMem;
}; /* typedef'd to ZSTD_DStream within "zstd.h" */
ZSTD_DStream* ZSTD_createDStream(void)
{
return ZSTD_createDStream_advanced(defaultCustomMem);
}
ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem)
{
ZSTD_DStream* zds;
if (!customMem.customAlloc && !customMem.customFree)
customMem = defaultCustomMem;
if (!customMem.customAlloc || !customMem.customFree)
return NULL;
zds = (ZSTD_DStream*)customMem.customAlloc(customMem.opaque, sizeof(ZSTD_DStream));
if (zds==NULL) return NULL;
memset(zds, 0, sizeof(ZSTD_DStream));
memcpy(&zds->customMem, &customMem, sizeof(ZSTD_customMem));
zds->zd = ZSTD_createDCtx_advanced(customMem);
if (zds->zd == NULL) { ZSTD_freeDStream(zds); return NULL; }
zds->stage = zdss_init;
return zds;
}
size_t ZSTD_freeDStream(ZSTD_DStream* zds)
{
if (zds==NULL) return 0; /* support free on null */
ZSTD_freeDCtx(zds->zd);
if (zds->inBuff) zds->customMem.customFree(zds->customMem.opaque, zds->inBuff);
if (zds->outBuff) zds->customMem.customFree(zds->customMem.opaque, zds->outBuff);
zds->customMem.customFree(zds->customMem.opaque, zds);
return 0;
}
/* *** Initialization *** */
size_t ZSTD_DStreamInSize(void) { return ZSTD_BLOCKSIZE_ABSOLUTEMAX + ZSTD_blockHeaderSize; }
size_t ZSTD_DStreamOutSize(void) { return ZSTD_BLOCKSIZE_ABSOLUTEMAX; }
size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize)
{
zds->stage = zdss_loadHeader;
zds->lhSize = zds->inPos = zds->outStart = zds->outEnd = 0;
return ZSTD_decompressBegin_usingDict(zds->zd, dict, dictSize);
}
size_t ZSTD_initDStream(ZSTD_DStream* zds)
{
return ZSTD_initDStream_usingDict(zds, NULL, 0);
}
/* *** Decompression *** */
MEM_STATIC size_t ZSTD_limitCopy(void* dst, size_t dstCapacity, const void* src, size_t srcSize)
{
size_t const length = MIN(dstCapacity, srcSize);
memcpy(dst, src, length);
return length;
}
size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inBuffer* input)
{
const char* const istart = (const char*)(input->src) + input->pos;
const char* const iend = (const char*)(input->src) + input->size;
const char* ip = istart;
char* const ostart = (char*)(output->dst) + output->pos;
char* const oend = (char*)(output->dst) + output->size;
char* op = ostart;
U32 someMoreWork = 1;
while (someMoreWork) {
switch(zds->stage)
{
case zdss_init :
return ERROR(init_missing);
case zdss_loadHeader :
{ size_t const hSize = ZSTD_getFrameParams(&(zds->fParams), zds->headerBuffer, zds->lhSize);
if (ZSTD_isError(hSize)) return hSize;
if (hSize != 0) { /* need more input */
size_t const toLoad = hSize - zds->lhSize; /* if hSize!=0, hSize > zds->lhSize */
if (toLoad > (size_t)(iend-ip)) { /* not enough input to load full header */
memcpy(zds->headerBuffer + zds->lhSize, ip, iend-ip);
zds->lhSize += iend-ip;
input->pos = input->size;
return (hSize - zds->lhSize) + ZSTD_blockHeaderSize; /* remaining header bytes + next block header */
}
memcpy(zds->headerBuffer + zds->lhSize, ip, toLoad); zds->lhSize = hSize; ip += toLoad;
break;
} }
/* Consume header */
{ size_t const h1Size = ZSTD_nextSrcSizeToDecompress(zds->zd); /* == ZSTD_frameHeaderSize_min */
size_t const h1Result = ZSTD_decompressContinue(zds->zd, NULL, 0, zds->headerBuffer, h1Size);
if (ZSTD_isError(h1Result)) return h1Result; /* should not happen : already checked */
if (h1Size < zds->lhSize) { /* long header */
size_t const h2Size = ZSTD_nextSrcSizeToDecompress(zds->zd);
size_t const h2Result = ZSTD_decompressContinue(zds->zd, NULL, 0, zds->headerBuffer+h1Size, h2Size);
if (ZSTD_isError(h2Result)) return h2Result;
} }
zds->fParams.windowSize = MAX(zds->fParams.windowSize, 1U << ZSTD_WINDOWLOG_ABSOLUTEMIN);
/* Frame header instruct buffer sizes */
{ size_t const blockSize = MIN(zds->fParams.windowSize, ZSTD_BLOCKSIZE_ABSOLUTEMAX);
size_t const neededOutSize = zds->fParams.windowSize + blockSize;
zds->blockSize = blockSize;
if (zds->inBuffSize < blockSize) {
zds->customMem.customFree(zds->customMem.opaque, zds->inBuff);
zds->inBuffSize = blockSize;
zds->inBuff = (char*)zds->customMem.customAlloc(zds->customMem.opaque, blockSize);
if (zds->inBuff == NULL) return ERROR(memory_allocation);
}
if (zds->outBuffSize < neededOutSize) {
zds->customMem.customFree(zds->customMem.opaque, zds->outBuff);
zds->outBuffSize = neededOutSize;
zds->outBuff = (char*)zds->customMem.customAlloc(zds->customMem.opaque, neededOutSize);
if (zds->outBuff == NULL) return ERROR(memory_allocation);
} }
zds->stage = zdss_read;
/* pass-through */
case zdss_read:
{ size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zds->zd);
if (neededInSize==0) { /* end of frame */
zds->stage = zdss_init;
someMoreWork = 0;
break;
}
if ((size_t)(iend-ip) >= neededInSize) { /* decode directly from src */
const int isSkipFrame = ZSTD_isSkipFrame(zds->zd);
size_t const decodedSize = ZSTD_decompressContinue(zds->zd,
zds->outBuff + zds->outStart, (isSkipFrame ? 0 : zds->outBuffSize - zds->outStart),
ip, neededInSize);
if (ZSTD_isError(decodedSize)) return decodedSize;
ip += neededInSize;
if (!decodedSize && !isSkipFrame) break; /* this was just a header */
zds->outEnd = zds->outStart + decodedSize;
zds->stage = zdss_flush;
break;
}
if (ip==iend) { someMoreWork = 0; break; } /* no more input */
zds->stage = zdss_load;
/* pass-through */
}
case zdss_load:
{ size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zds->zd);
size_t const toLoad = neededInSize - zds->inPos; /* should always be <= remaining space within inBuff */
size_t loadedSize;
if (toLoad > zds->inBuffSize - zds->inPos) return ERROR(corruption_detected); /* should never happen */
loadedSize = ZSTD_limitCopy(zds->inBuff + zds->inPos, toLoad, ip, iend-ip);
ip += loadedSize;
zds->inPos += loadedSize;
if (loadedSize < toLoad) { someMoreWork = 0; break; } /* not enough input, wait for more */
/* decode loaded input */
{ const int isSkipFrame = ZSTD_isSkipFrame(zds->zd);
size_t const decodedSize = ZSTD_decompressContinue(zds->zd,
zds->outBuff + zds->outStart, zds->outBuffSize - zds->outStart,
zds->inBuff, neededInSize);
if (ZSTD_isError(decodedSize)) return decodedSize;
zds->inPos = 0; /* input is consumed */
if (!decodedSize && !isSkipFrame) { zds->stage = zdss_read; break; } /* this was just a header */
zds->outEnd = zds->outStart + decodedSize;
zds->stage = zdss_flush;
/* pass-through */
} }
case zdss_flush:
{ size_t const toFlushSize = zds->outEnd - zds->outStart;
size_t const flushedSize = ZSTD_limitCopy(op, oend-op, zds->outBuff + zds->outStart, toFlushSize);
op += flushedSize;
zds->outStart += flushedSize;
if (flushedSize == toFlushSize) { /* flush completed */
zds->stage = zdss_read;
if (zds->outStart + zds->blockSize > zds->outBuffSize)
zds->outStart = zds->outEnd = 0;
break;
}
/* cannot flush everything */
someMoreWork = 0;
break;
}
default: return ERROR(GENERIC); /* impossible */
} }
/* result */
input->pos += (size_t)(ip-istart);
output->pos += (size_t)(op-ostart);
{ size_t nextSrcSizeHint = ZSTD_nextSrcSizeToDecompress(zds->zd);
if (!nextSrcSizeHint) return (zds->outEnd != zds->outStart); /* return 0 only if fully flushed too */
nextSrcSizeHint += ZSTD_blockHeaderSize * (ZSTD_nextInputType(zds->zd) == ZSTDnit_block);
if (zds->inPos > nextSrcSizeHint) return ERROR(GENERIC); /* should never happen */
nextSrcSizeHint -= zds->inPos; /* already loaded*/
return nextSrcSizeHint;
}
}
lib/dictBuilder/zdict.c
+45 -40
View File
@@ -332,12 +332,13 @@ static dictItem ZDICT_analyzePos(
} while (length >=MINMATCHLENGTH);
/* look backward */
do {
length = ZDICT_count(b + pos, b + suffix[start-1]);
if (length >= LLIMIT) length = LLIMIT-1;
lengthList[length]++;
if (length >=MINMATCHLENGTH) start--;
} while(length >= MINMATCHLENGTH);
length = MINMATCHLENGTH;
while ((length >= MINMATCHLENGTH) & (start > 0)) {
length = ZDICT_count(b + pos, b + suffix[start - 1]);
if (length >= LLIMIT) length = LLIMIT - 1;
lengthList[length]++;
if (length >= MINMATCHLENGTH) start--;
}
/* largest useful length */
memset(cumulLength, 0, sizeof(cumulLength));
@@ -610,18 +611,17 @@ static void ZDICT_countEStats(EStats_ress_t esr, ZSTD_parameters params,
{ const BYTE* codePtr = seqStorePtr->llCode;
U32 u;
for (u=0; u<nbSeq; u++) litlengthCount[codePtr[u]]++;
} }
}
/* rep offsets */
{ const seqDef* const seq = seqStorePtr->sequences;
U32 offset1 = seq[0].offset - 3;
U32 offset2 = seq[1].offset - 3;
if (offset1 >= MAXREPOFFSET) offset1 = 0;
if (offset2 >= MAXREPOFFSET) offset2 = 0;
repOffsets[offset1] += 3;
repOffsets[offset2] += 1;
}
}
if (nbSeq >= 2) { /* rep offsets */
const seqDef* const seq = seqStorePtr->sequencesStart;
U32 offset1 = seq[0].offset - 3;
U32 offset2 = seq[1].offset - 3;
if (offset1 >= MAXREPOFFSET) offset1 = 0;
if (offset2 >= MAXREPOFFSET) offset2 = 0;
repOffsets[offset1] += 3;
repOffsets[offset2] += 1;
} } }
}
/*
@@ -675,7 +675,7 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize,
short matchLengthNCount[MaxML+1];
U32 litLengthCount[MaxLL+1];
short litLengthNCount[MaxLL+1];
U32 repOffset[MAXREPOFFSET] = { 0 };
U32 repOffset[MAXREPOFFSET];
offsetCount_t bestRepOffset[ZSTD_REP_NUM+1];
EStats_ress_t esr;
ZSTD_parameters params;
@@ -683,31 +683,32 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize,
size_t pos = 0, errorCode;
size_t eSize = 0;
size_t const totalSrcSize = ZDICT_totalSampleSize(fileSizes, nbFiles);
size_t const averageSampleSize = totalSrcSize / nbFiles;
size_t const averageSampleSize = totalSrcSize / (nbFiles + !nbFiles);
BYTE* dstPtr = (BYTE*)dstBuffer;
/* init */
esr.ref = ZSTD_createCCtx();
esr.zc = ZSTD_createCCtx();
esr.workPlace = malloc(ZSTD_BLOCKSIZE_ABSOLUTEMAX);
if (!esr.ref || !esr.zc || !esr.workPlace) {
eSize = ERROR(memory_allocation);
DISPLAYLEVEL(1, "Not enough memory \n");
goto _cleanup;
}
if (offcodeMax>OFFCODE_MAX) { eSize = ERROR(dictionary_wrong); goto _cleanup; } /* too large dictionary */
for (u=0; u<256; u++) countLit[u]=1; /* any character must be described */
for (u=0; u<=offcodeMax; u++) offcodeCount[u]=1;
for (u=0; u<=MaxML; u++) matchLengthCount[u]=1;
for (u=0; u<=MaxLL; u++) litLengthCount[u]=1;
memset(repOffset, 0, sizeof(repOffset));
repOffset[1] = repOffset[4] = repOffset[8] = 1;
memset(bestRepOffset, 0, sizeof(bestRepOffset));
esr.ref = ZSTD_createCCtx();
esr.zc = ZSTD_createCCtx();
esr.workPlace = malloc(ZSTD_BLOCKSIZE_ABSOLUTEMAX);
if (!esr.ref || !esr.zc || !esr.workPlace) {
eSize = ERROR(memory_allocation);
DISPLAYLEVEL(1, "Not enough memory");
goto _cleanup;
}
if (compressionLevel==0) compressionLevel=g_compressionLevel_default;
params = ZSTD_getParams(compressionLevel, averageSampleSize, dictBufferSize);
{ size_t const beginResult = ZSTD_compressBegin_advanced(esr.ref, dictBuffer, dictBufferSize, params, 0);
if (ZSTD_isError(beginResult)) {
eSize = ERROR(GENERIC);
DISPLAYLEVEL(1, "error : ZSTD_compressBegin_advanced failed ");
DISPLAYLEVEL(1, "error : ZSTD_compressBegin_advanced failed \n");
goto _cleanup;
} }
@@ -723,7 +724,7 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize,
errorCode = HUF_buildCTable (hufTable, countLit, 255, huffLog);
if (HUF_isError(errorCode)) {
eSize = ERROR(GENERIC);
DISPLAYLEVEL(1, "HUF_buildCTable error");
DISPLAYLEVEL(1, "HUF_buildCTable error \n");
goto _cleanup;
}
huffLog = (U32)errorCode;
@@ -739,7 +740,7 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize,
errorCode = FSE_normalizeCount(offcodeNCount, Offlog, offcodeCount, total, offcodeMax);
if (FSE_isError(errorCode)) {
eSize = ERROR(GENERIC);
DISPLAYLEVEL(1, "FSE_normalizeCount error with offcodeCount");
DISPLAYLEVEL(1, "FSE_normalizeCount error with offcodeCount \n");
goto _cleanup;
}
Offlog = (U32)errorCode;
@@ -748,7 +749,7 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize,
errorCode = FSE_normalizeCount(matchLengthNCount, mlLog, matchLengthCount, total, MaxML);
if (FSE_isError(errorCode)) {
eSize = ERROR(GENERIC);
DISPLAYLEVEL(1, "FSE_normalizeCount error with matchLengthCount");
DISPLAYLEVEL(1, "FSE_normalizeCount error with matchLengthCount \n");
goto _cleanup;
}
mlLog = (U32)errorCode;
@@ -757,7 +758,7 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize,
errorCode = FSE_normalizeCount(litLengthNCount, llLog, litLengthCount, total, MaxLL);
if (FSE_isError(errorCode)) {
eSize = ERROR(GENERIC);
DISPLAYLEVEL(1, "FSE_normalizeCount error with litLengthCount");
DISPLAYLEVEL(1, "FSE_normalizeCount error with litLengthCount \n");
goto _cleanup;
}
llLog = (U32)errorCode;
@@ -767,7 +768,7 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize,
{ size_t const hhSize = HUF_writeCTable(dstPtr, maxDstSize, hufTable, 255, huffLog);
if (HUF_isError(hhSize)) {
eSize = ERROR(GENERIC);
DISPLAYLEVEL(1, "HUF_writeCTable error");
DISPLAYLEVEL(1, "HUF_writeCTable error \n");
goto _cleanup;
}
dstPtr += hhSize;
@@ -778,7 +779,7 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize,
{ size_t const ohSize = FSE_writeNCount(dstPtr, maxDstSize, offcodeNCount, OFFCODE_MAX, Offlog);
if (FSE_isError(ohSize)) {
eSize = ERROR(GENERIC);
DISPLAYLEVEL(1, "FSE_writeNCount error with offcodeNCount");
DISPLAYLEVEL(1, "FSE_writeNCount error with offcodeNCount \n");
goto _cleanup;
}
dstPtr += ohSize;
@@ -789,7 +790,7 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize,
{ size_t const mhSize = FSE_writeNCount(dstPtr, maxDstSize, matchLengthNCount, MaxML, mlLog);
if (FSE_isError(mhSize)) {
eSize = ERROR(GENERIC);
DISPLAYLEVEL(1, "FSE_writeNCount error with matchLengthNCount");
DISPLAYLEVEL(1, "FSE_writeNCount error with matchLengthNCount \n");
goto _cleanup;
}
dstPtr += mhSize;
@@ -800,7 +801,7 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize,
{ size_t const lhSize = FSE_writeNCount(dstPtr, maxDstSize, litLengthNCount, MaxLL, llLog);
if (FSE_isError(lhSize)) {
eSize = ERROR(GENERIC);
DISPLAYLEVEL(1, "FSE_writeNCount error with litlengthNCount");
DISPLAYLEVEL(1, "FSE_writeNCount error with litlengthNCount \n");
goto _cleanup;
}
dstPtr += lhSize;
@@ -810,7 +811,7 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize,
if (maxDstSize<12) {
eSize = ERROR(GENERIC);
DISPLAYLEVEL(1, "not enough space to write RepOffsets");
DISPLAYLEVEL(1, "not enough space to write RepOffsets \n");
goto _cleanup;
}
# if 0
@@ -855,10 +856,14 @@ size_t ZDICT_addEntropyTablesFromBuffer_advanced(void* dictBuffer, size_t dictCo
/* entropy tables */
DISPLAYLEVEL(2, "\r%70s\r", ""); /* clean display line */
DISPLAYLEVEL(2, "statistics ... \n");
hSize += ZDICT_analyzeEntropy((char*)dictBuffer+hSize, dictBufferCapacity-hSize,
{ size_t const eSize = ZDICT_analyzeEntropy((char*)dictBuffer+hSize, dictBufferCapacity-hSize,
compressionLevel,
samplesBuffer, samplesSizes, nbSamples,
(char*)dictBuffer + dictBufferCapacity - dictContentSize, dictContentSize);
if (ZDICT_isError(eSize)) return eSize;
hSize += eSize;
}
if (hSize + dictContentSize < dictBufferCapacity)
memmove((char*)dictBuffer + hSize, (char*)dictBuffer + dictBufferCapacity - dictContentSize, dictContentSize);
@@ -901,7 +906,7 @@ size_t ZDICT_trainFromBuffer_unsafe(
/* display best matches */
if (g_displayLevel>= 3) {
U32 const nb = 25;
U32 const nb = MIN(25, dictList[0].pos);
U32 const dictContentSize = ZDICT_dictSize(dictList);
U32 u;
DISPLAYLEVEL(3, "\n %u segments found, of total size %u \n", dictList[0].pos, dictContentSize);
@@ -920,7 +925,7 @@ size_t ZDICT_trainFromBuffer_unsafe(
/* create dictionary */
{ U32 dictContentSize = ZDICT_dictSize(dictList);
if (dictContentSize < targetDictSize/2) {
DISPLAYLEVEL(2, "! warning : created dictionary significantly smaller than requested (%u < %u) \n", dictContentSize, (U32)maxDictSize);
DISPLAYLEVEL(2, "! warning : selected content significantly smaller than requested (%u < %u) \n", dictContentSize, (U32)maxDictSize);
if (minRep > MINRATIO) {
DISPLAYLEVEL(2, "! consider increasing selectivity to produce larger dictionary (-s%u) \n", selectivity+1);
DISPLAYLEVEL(2, "! note : larger dictionaries are not necessarily better, test its efficiency on samples \n");
lib/dictBuilder/zdict.h
+16 -3
View File
@@ -38,6 +38,19 @@
extern "C" {
#endif
/*====== Export for Windows ======*/
/*!
* ZSTD_DLL_EXPORT :
* Enable exporting of functions when building a Windows DLL
*/
#if defined(_WIN32) && defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1)
# define ZDICTLIB_API __declspec(dllexport)
#else
# define ZDICTLIB_API
#endif
/*! ZDICT_trainFromBuffer() :
Train a dictionary from an array of samples.
Samples must be stored concatenated in a single flat buffer `samplesBuffer`,
@@ -50,13 +63,13 @@ extern "C" {
In general, it's recommended to provide a few thousands samples, but this can vary a lot.
It's recommended that total size of all samples be about ~x100 times the target size of dictionary.
*/
size_t ZDICT_trainFromBuffer(void* dictBuffer, size_t dictBufferCapacity,
ZDICTLIB_API size_t ZDICT_trainFromBuffer(void* dictBuffer, size_t dictBufferCapacity,
const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples);
/*====== Helper functions ======*/
unsigned ZDICT_isError(size_t errorCode);
const char* ZDICT_getErrorName(size_t errorCode);
ZDICTLIB_API unsigned ZDICT_isError(size_t errorCode);
ZDICTLIB_API const char* ZDICT_getErrorName(size_t errorCode);
lib/legacy/zstd_v01.c
+1 -1
View File
@@ -225,7 +225,7 @@ typedef signed long long S64;
#ifndef FSE_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */
# if defined(__GNUC__) && ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) )
# define FSE_FORCE_MEMORY_ACCESS 2
# elif defined(__INTEL_COMPILER) || \
# elif (defined(__INTEL_COMPILER) && !defined(WIN32)) || \
(defined(__GNUC__) && ( defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) || defined(__ARM_ARCH_7R__) || defined(__ARM_ARCH_7M__) || defined(__ARM_ARCH_7S__) ))
# define FSE_FORCE_MEMORY_ACCESS 1
# endif
lib/legacy/zstd_v02.c
+5 -149
View File
@@ -44,6 +44,10 @@ extern "C" {
/******************************************
* Compiler-specific
******************************************/
#if defined(_MSC_VER) /* Visual Studio */
# include <stdlib.h> /* _byteswap_ulong */
# include <intrin.h> /* _byteswap_* */
#endif
#if defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
# define ERR_STATIC static inline
#elif defined(_MSC_VER)
@@ -190,7 +194,7 @@ extern "C" {
#ifndef MEM_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */
# if defined(__GNUC__) && ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) )
# define MEM_FORCE_MEMORY_ACCESS 2
# elif defined(__INTEL_COMPILER) || \
# elif (defined(__INTEL_COMPILER) && !defined(WIN32)) || \
(defined(__GNUC__) && ( defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) || defined(__ARM_ARCH_7R__) || defined(__ARM_ARCH_7M__) || defined(__ARM_ARCH_7S__) ))
# define MEM_FORCE_MEMORY_ACCESS 1
# endif
@@ -214,8 +218,6 @@ MEM_STATIC U32 MEM_read32(const void* memPtr) { return *(const U32*) memPtr; }
MEM_STATIC U64 MEM_read64(const void* memPtr) { return *(const U64*) memPtr; }
MEM_STATIC void MEM_write16(void* memPtr, U16 value) { *(U16*)memPtr = value; }
MEM_STATIC void MEM_write32(void* memPtr, U32 value) { *(U32*)memPtr = value; }
MEM_STATIC void MEM_write64(void* memPtr, U64 value) { *(U64*)memPtr = value; }
#elif defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==1)
@@ -228,8 +230,6 @@ MEM_STATIC U32 MEM_read32(const void* ptr) { return ((const unalign*)ptr)->u32;
MEM_STATIC U64 MEM_read64(const void* ptr) { return ((const unalign*)ptr)->u64; }
MEM_STATIC void MEM_write16(void* memPtr, U16 value) { ((unalign*)memPtr)->u16 = value; }
MEM_STATIC void MEM_write32(void* memPtr, U32 value) { ((unalign*)memPtr)->u32 = value; }
MEM_STATIC void MEM_write64(void* memPtr, U64 value) { ((unalign*)memPtr)->u64 = value; }
#else
@@ -256,16 +256,6 @@ MEM_STATIC void MEM_write16(void* memPtr, U16 value)
memcpy(memPtr, &value, sizeof(value));
}
MEM_STATIC void MEM_write32(void* memPtr, U32 value)
{
memcpy(memPtr, &value, sizeof(value));
}
MEM_STATIC void MEM_write64(void* memPtr, U64 value)
{
memcpy(memPtr, &value, sizeof(value));
}
#endif // MEM_FORCE_MEMORY_ACCESS
@@ -305,21 +295,6 @@ MEM_STATIC U32 MEM_readLE32(const void* memPtr)
}
}
MEM_STATIC void MEM_writeLE32(void* memPtr, U32 val32)
{
if (MEM_isLittleEndian())
{
MEM_write32(memPtr, val32);
}
else
{
BYTE* p = (BYTE*)memPtr;
p[0] = (BYTE)val32;
p[1] = (BYTE)(val32>>8);
p[2] = (BYTE)(val32>>16);
p[3] = (BYTE)(val32>>24);
}
}
MEM_STATIC U64 MEM_readLE64(const void* memPtr)
{
@@ -333,25 +308,6 @@ MEM_STATIC U64 MEM_readLE64(const void* memPtr)
}
}
MEM_STATIC void MEM_writeLE64(void* memPtr, U64 val64)
{
if (MEM_isLittleEndian())
{
MEM_write64(memPtr, val64);
}
else
{
BYTE* p = (BYTE*)memPtr;
p[0] = (BYTE)val64;
p[1] = (BYTE)(val64>>8);
p[2] = (BYTE)(val64>>16);
p[3] = (BYTE)(val64>>24);
p[4] = (BYTE)(val64>>32);
p[5] = (BYTE)(val64>>40);
p[6] = (BYTE)(val64>>48);
p[7] = (BYTE)(val64>>56);
}
}
MEM_STATIC size_t MEM_readLEST(const void* memPtr)
{
@@ -361,14 +317,6 @@ MEM_STATIC size_t MEM_readLEST(const void* memPtr)
return (size_t)MEM_readLE64(memPtr);
}
MEM_STATIC void MEM_writeLEST(void* memPtr, size_t val)
{
if (MEM_32bits())
MEM_writeLE32(memPtr, (U32)val);
else
MEM_writeLE64(memPtr, (U64)val);
}
#if defined (__cplusplus)
}
#endif
@@ -2983,98 +2931,6 @@ typedef struct {
static unsigned ZSTD_isError(size_t code) { return ERR_isError(code); }
/* *************************************
* Function body to include
***************************************/
static size_t ZSTD_read_ARCH(const void* p) { size_t r; memcpy(&r, p, sizeof(r)); return r; }
MEM_STATIC unsigned ZSTD_NbCommonBytes (register size_t val)
{
if (MEM_isLittleEndian())
{
if (MEM_64bits())
{
# if defined(_MSC_VER) && defined(_WIN64) && !defined(LZ4_FORCE_SW_BITCOUNT)
unsigned long r = 0;
_BitScanForward64( &r, (U64)val );
return (int)(r>>3);
# elif defined(__GNUC__) && (__GNUC__ >= 3) && !defined(LZ4_FORCE_SW_BITCOUNT)
return (__builtin_ctzll((U64)val) >> 3);
# else
static const int DeBruijnBytePos[64] = { 0, 0, 0, 0, 0, 1, 1, 2, 0, 3, 1, 3, 1, 4, 2, 7, 0, 2, 3, 6, 1, 5, 3, 5, 1, 3, 4, 4, 2, 5, 6, 7, 7, 0, 1, 2, 3, 3, 4, 6, 2, 6, 5, 5, 3, 4, 5, 6, 7, 1, 2, 4, 6, 4, 4, 5, 7, 2, 6, 5, 7, 6, 7, 7 };
return DeBruijnBytePos[((U64)((val & -(long long)val) * 0x0218A392CDABBD3FULL)) >> 58];
# endif
}
else /* 32 bits */
{
# if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT)
unsigned long r;
_BitScanForward( &r, (U32)val );
return (int)(r>>3);
# elif defined(__GNUC__) && (__GNUC__ >= 3) && !defined(LZ4_FORCE_SW_BITCOUNT)
return (__builtin_ctz((U32)val) >> 3);
# else
static const int DeBruijnBytePos[32] = { 0, 0, 3, 0, 3, 1, 3, 0, 3, 2, 2, 1, 3, 2, 0, 1, 3, 3, 1, 2, 2, 2, 2, 0, 3, 1, 2, 0, 1, 0, 1, 1 };
return DeBruijnBytePos[((U32)((val & -(S32)val) * 0x077CB531U)) >> 27];
# endif
}
}
else /* Big Endian CPU */
{
if (MEM_32bits())
{
# if defined(_MSC_VER) && defined(_WIN64) && !defined(LZ4_FORCE_SW_BITCOUNT)
unsigned long r = 0;
_BitScanReverse64( &r, val );
return (unsigned)(r>>3);
# elif defined(__GNUC__) && (__GNUC__ >= 3) && !defined(LZ4_FORCE_SW_BITCOUNT)
return (__builtin_clzll(val) >> 3);
# else
unsigned r;
const unsigned n32 = sizeof(size_t)*4; /* calculate this way due to compiler complaining in 32-bits mode */
if (!(val>>n32)) { r=4; } else { r=0; val>>=n32; }
if (!(val>>16)) { r+=2; val>>=8; } else { val>>=24; }
r += (!val);
return r;
# endif
}
else /* 32 bits */
{
# if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT)
unsigned long r = 0;
_BitScanReverse( &r, (unsigned long)val );
return (unsigned)(r>>3);
# elif defined(__GNUC__) && (__GNUC__ >= 3) && !defined(LZ4_FORCE_SW_BITCOUNT)
return (__builtin_clz((U32)val) >> 3);
# else
unsigned r;
if (!(val>>16)) { r=2; val>>=8; } else { r=0; val>>=24; }
r += (!val);
return r;
# endif
}
}
}
MEM_STATIC size_t ZSTD_count(const BYTE* pIn, const BYTE* pMatch, const BYTE* pInLimit)
{
const BYTE* const pStart = pIn;
while ((pIn<pInLimit-(sizeof(size_t)-1)))
{
size_t diff = ZSTD_read_ARCH(pMatch) ^ ZSTD_read_ARCH(pIn);
if (!diff) { pIn+=sizeof(size_t); pMatch+=sizeof(size_t); continue; }
pIn += ZSTD_NbCommonBytes(diff);
return (size_t)(pIn - pStart);
}
if (MEM_32bits()) if ((pIn<(pInLimit-3)) && (MEM_read32(pMatch) == MEM_read32(pIn))) { pIn+=4; pMatch+=4; }
if ((pIn<(pInLimit-1)) && (MEM_read16(pMatch) == MEM_read16(pIn))) { pIn+=2; pMatch+=2; }
if ((pIn<pInLimit) && (*pMatch == *pIn)) pIn++;
return (size_t)(pIn - pStart);
}
/* *************************************************************
* Decompression section
lib/legacy/zstd_v03.c
+5 -148
View File
@@ -44,6 +44,10 @@ extern "C" {
/******************************************
* Compiler-specific
******************************************/
#if defined(_MSC_VER) /* Visual Studio */
# include <stdlib.h> /* _byteswap_ulong */
# include <intrin.h> /* _byteswap_* */
#endif
#if defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
# define ERR_STATIC static inline
#elif defined(_MSC_VER)
@@ -190,7 +194,7 @@ extern "C" {
#ifndef MEM_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */
# if defined(__GNUC__) && ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) )
# define MEM_FORCE_MEMORY_ACCESS 2
# elif defined(__INTEL_COMPILER) || \
# elif (defined(__INTEL_COMPILER) && !defined(WIN32)) || \
(defined(__GNUC__) && ( defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) || defined(__ARM_ARCH_7R__) || defined(__ARM_ARCH_7M__) || defined(__ARM_ARCH_7S__) ))
# define MEM_FORCE_MEMORY_ACCESS 1
# endif
@@ -214,8 +218,6 @@ MEM_STATIC U32 MEM_read32(const void* memPtr) { return *(const U32*) memPtr; }
MEM_STATIC U64 MEM_read64(const void* memPtr) { return *(const U64*) memPtr; }
MEM_STATIC void MEM_write16(void* memPtr, U16 value) { *(U16*)memPtr = value; }
MEM_STATIC void MEM_write32(void* memPtr, U32 value) { *(U32*)memPtr = value; }
MEM_STATIC void MEM_write64(void* memPtr, U64 value) { *(U64*)memPtr = value; }
#elif defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==1)
@@ -228,8 +230,6 @@ MEM_STATIC U32 MEM_read32(const void* ptr) { return ((const unalign*)ptr)->u32;
MEM_STATIC U64 MEM_read64(const void* ptr) { return ((const unalign*)ptr)->u64; }
MEM_STATIC void MEM_write16(void* memPtr, U16 value) { ((unalign*)memPtr)->u16 = value; }
MEM_STATIC void MEM_write32(void* memPtr, U32 value) { ((unalign*)memPtr)->u32 = value; }
MEM_STATIC void MEM_write64(void* memPtr, U64 value) { ((unalign*)memPtr)->u64 = value; }
#else
@@ -256,15 +256,6 @@ MEM_STATIC void MEM_write16(void* memPtr, U16 value)
memcpy(memPtr, &value, sizeof(value));
}
MEM_STATIC void MEM_write32(void* memPtr, U32 value)
{
memcpy(memPtr, &value, sizeof(value));
}
MEM_STATIC void MEM_write64(void* memPtr, U64 value)
{
memcpy(memPtr, &value, sizeof(value));
}
#endif // MEM_FORCE_MEMORY_ACCESS
@@ -305,22 +296,6 @@ MEM_STATIC U32 MEM_readLE32(const void* memPtr)
}
}
MEM_STATIC void MEM_writeLE32(void* memPtr, U32 val32)
{
if (MEM_isLittleEndian())
{
MEM_write32(memPtr, val32);
}
else
{
BYTE* p = (BYTE*)memPtr;
p[0] = (BYTE)val32;
p[1] = (BYTE)(val32>>8);
p[2] = (BYTE)(val32>>16);
p[3] = (BYTE)(val32>>24);
}
}
MEM_STATIC U64 MEM_readLE64(const void* memPtr)
{
if (MEM_isLittleEndian())
@@ -333,25 +308,6 @@ MEM_STATIC U64 MEM_readLE64(const void* memPtr)
}
}
MEM_STATIC void MEM_writeLE64(void* memPtr, U64 val64)
{
if (MEM_isLittleEndian())
{
MEM_write64(memPtr, val64);
}
else
{
BYTE* p = (BYTE*)memPtr;
p[0] = (BYTE)val64;
p[1] = (BYTE)(val64>>8);
p[2] = (BYTE)(val64>>16);
p[3] = (BYTE)(val64>>24);
p[4] = (BYTE)(val64>>32);
p[5] = (BYTE)(val64>>40);
p[6] = (BYTE)(val64>>48);
p[7] = (BYTE)(val64>>56);
}
}
MEM_STATIC size_t MEM_readLEST(const void* memPtr)
{
@@ -361,13 +317,6 @@ MEM_STATIC size_t MEM_readLEST(const void* memPtr)
return (size_t)MEM_readLE64(memPtr);
}
MEM_STATIC void MEM_writeLEST(void* memPtr, size_t val)
{
if (MEM_32bits())
MEM_writeLE32(memPtr, (U32)val);
else
MEM_writeLE64(memPtr, (U64)val);
}
#if defined (__cplusplus)
}
@@ -2624,98 +2573,6 @@ typedef struct {
static unsigned ZSTD_isError(size_t code) { return ERR_isError(code); }
/* *************************************
* Function body to include
***************************************/
static size_t ZSTD_read_ARCH(const void* p) { size_t r; memcpy(&r, p, sizeof(r)); return r; }
MEM_STATIC unsigned ZSTD_NbCommonBytes (register size_t val)
{
if (MEM_isLittleEndian())
{
if (MEM_64bits())
{
# if defined(_MSC_VER) && defined(_WIN64) && !defined(LZ4_FORCE_SW_BITCOUNT)
unsigned long r = 0;
_BitScanForward64( &r, (U64)val );
return (int)(r>>3);
# elif defined(__GNUC__) && (__GNUC__ >= 3) && !defined(LZ4_FORCE_SW_BITCOUNT)
return (__builtin_ctzll((U64)val) >> 3);
# else
static const int DeBruijnBytePos[64] = { 0, 0, 0, 0, 0, 1, 1, 2, 0, 3, 1, 3, 1, 4, 2, 7, 0, 2, 3, 6, 1, 5, 3, 5, 1, 3, 4, 4, 2, 5, 6, 7, 7, 0, 1, 2, 3, 3, 4, 6, 2, 6, 5, 5, 3, 4, 5, 6, 7, 1, 2, 4, 6, 4, 4, 5, 7, 2, 6, 5, 7, 6, 7, 7 };
return DeBruijnBytePos[((U64)((val & -(long long)val) * 0x0218A392CDABBD3FULL)) >> 58];
# endif
}
else /* 32 bits */
{
# if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT)
unsigned long r;
_BitScanForward( &r, (U32)val );
return (int)(r>>3);
# elif defined(__GNUC__) && (__GNUC__ >= 3) && !defined(LZ4_FORCE_SW_BITCOUNT)
return (__builtin_ctz((U32)val) >> 3);
# else
static const int DeBruijnBytePos[32] = { 0, 0, 3, 0, 3, 1, 3, 0, 3, 2, 2, 1, 3, 2, 0, 1, 3, 3, 1, 2, 2, 2, 2, 0, 3, 1, 2, 0, 1, 0, 1, 1 };
return DeBruijnBytePos[((U32)((val & -(S32)val) * 0x077CB531U)) >> 27];
# endif
}
}
else /* Big Endian CPU */
{
if (MEM_32bits())
{
# if defined(_MSC_VER) && defined(_WIN64) && !defined(LZ4_FORCE_SW_BITCOUNT)
unsigned long r = 0;
_BitScanReverse64( &r, val );
return (unsigned)(r>>3);
# elif defined(__GNUC__) && (__GNUC__ >= 3) && !defined(LZ4_FORCE_SW_BITCOUNT)
return (__builtin_clzll(val) >> 3);
# else
unsigned r;
const unsigned n32 = sizeof(size_t)*4; /* calculate this way due to compiler complaining in 32-bits mode */
if (!(val>>n32)) { r=4; } else { r=0; val>>=n32; }
if (!(val>>16)) { r+=2; val>>=8; } else { val>>=24; }
r += (!val);
return r;
# endif
}
else /* 32 bits */
{
# if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT)
unsigned long r = 0;
_BitScanReverse( &r, (unsigned long)val );
return (unsigned)(r>>3);
# elif defined(__GNUC__) && (__GNUC__ >= 3) && !defined(LZ4_FORCE_SW_BITCOUNT)
return (__builtin_clz((U32)val) >> 3);
# else
unsigned r;
if (!(val>>16)) { r=2; val>>=8; } else { r=0; val>>=24; }
r += (!val);
return r;
# endif
}
}
}
MEM_STATIC size_t ZSTD_count(const BYTE* pIn, const BYTE* pMatch, const BYTE* pInLimit)
{
const BYTE* const pStart = pIn;
while ((pIn<pInLimit-(sizeof(size_t)-1)))
{
size_t diff = ZSTD_read_ARCH(pMatch) ^ ZSTD_read_ARCH(pIn);
if (!diff) { pIn+=sizeof(size_t); pMatch+=sizeof(size_t); continue; }
pIn += ZSTD_NbCommonBytes(diff);
return (size_t)(pIn - pStart);
}
if (MEM_32bits()) if ((pIn<(pInLimit-3)) && (MEM_read32(pMatch) == MEM_read32(pIn))) { pIn+=4; pMatch+=4; }
if ((pIn<(pInLimit-1)) && (MEM_read16(pMatch) == MEM_read16(pIn))) { pIn+=2; pMatch+=2; }
if ((pIn<pInLimit) && (*pMatch == *pIn)) pIn++;
return (size_t)(pIn - pStart);
}
/* *************************************************************
* Decompression section
lib/legacy/zstd_v04.c
+5 -56
View File
@@ -87,6 +87,10 @@ extern "C" {
/******************************************
* Compiler-specific
******************************************/
#if defined(_MSC_VER) /* Visual Studio */
# include <stdlib.h> /* _byteswap_ulong */
# include <intrin.h> /* _byteswap_* */
#endif
#if defined(__GNUC__)
# define MEM_STATIC static __attribute__((unused))
#elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
@@ -140,7 +144,7 @@ extern "C" {
#ifndef MEM_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */
# if defined(__GNUC__) && ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) )
# define MEM_FORCE_MEMORY_ACCESS 2
# elif defined(__INTEL_COMPILER) || \
# elif (defined(__INTEL_COMPILER) && !defined(WIN32)) || \
(defined(__GNUC__) && ( defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) || defined(__ARM_ARCH_7R__) || defined(__ARM_ARCH_7M__) || defined(__ARM_ARCH_7S__) ))
# define MEM_FORCE_MEMORY_ACCESS 1
# endif
@@ -164,8 +168,6 @@ MEM_STATIC U32 MEM_read32(const void* memPtr) { return *(const U32*) memPtr; }
MEM_STATIC U64 MEM_read64(const void* memPtr) { return *(const U64*) memPtr; }
MEM_STATIC void MEM_write16(void* memPtr, U16 value) { *(U16*)memPtr = value; }
MEM_STATIC void MEM_write32(void* memPtr, U32 value) { *(U32*)memPtr = value; }
MEM_STATIC void MEM_write64(void* memPtr, U64 value) { *(U64*)memPtr = value; }
#elif defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==1)
@@ -178,8 +180,6 @@ MEM_STATIC U32 MEM_read32(const void* ptr) { return ((const unalign*)ptr)->u32;
MEM_STATIC U64 MEM_read64(const void* ptr) { return ((const unalign*)ptr)->u64; }
MEM_STATIC void MEM_write16(void* memPtr, U16 value) { ((unalign*)memPtr)->u16 = value; }
MEM_STATIC void MEM_write32(void* memPtr, U32 value) { ((unalign*)memPtr)->u32 = value; }
MEM_STATIC void MEM_write64(void* memPtr, U64 value) { ((unalign*)memPtr)->u64 = value; }
#else
@@ -206,16 +206,6 @@ MEM_STATIC void MEM_write16(void* memPtr, U16 value)
memcpy(memPtr, &value, sizeof(value));
}
MEM_STATIC void MEM_write32(void* memPtr, U32 value)
{
memcpy(memPtr, &value, sizeof(value));
}
MEM_STATIC void MEM_write64(void* memPtr, U64 value)
{
memcpy(memPtr, &value, sizeof(value));
}
#endif // MEM_FORCE_MEMORY_ACCESS
@@ -255,21 +245,6 @@ MEM_STATIC U32 MEM_readLE32(const void* memPtr)
}
}
MEM_STATIC void MEM_writeLE32(void* memPtr, U32 val32)
{
if (MEM_isLittleEndian())
{
MEM_write32(memPtr, val32);
}
else
{
BYTE* p = (BYTE*)memPtr;
p[0] = (BYTE)val32;
p[1] = (BYTE)(val32>>8);
p[2] = (BYTE)(val32>>16);
p[3] = (BYTE)(val32>>24);
}
}
MEM_STATIC U64 MEM_readLE64(const void* memPtr)
{
@@ -283,25 +258,6 @@ MEM_STATIC U64 MEM_readLE64(const void* memPtr)
}
}
MEM_STATIC void MEM_writeLE64(void* memPtr, U64 val64)
{
if (MEM_isLittleEndian())
{
MEM_write64(memPtr, val64);
}
else
{
BYTE* p = (BYTE*)memPtr;
p[0] = (BYTE)val64;
p[1] = (BYTE)(val64>>8);
p[2] = (BYTE)(val64>>16);
p[3] = (BYTE)(val64>>24);
p[4] = (BYTE)(val64>>32);
p[5] = (BYTE)(val64>>40);
p[6] = (BYTE)(val64>>48);
p[7] = (BYTE)(val64>>56);
}
}
MEM_STATIC size_t MEM_readLEST(const void* memPtr)
{
@@ -311,13 +267,6 @@ MEM_STATIC size_t MEM_readLEST(const void* memPtr)
return (size_t)MEM_readLE64(memPtr);
}
MEM_STATIC void MEM_writeLEST(void* memPtr, size_t val)
{
if (MEM_32bits())
MEM_writeLE32(memPtr, (U32)val);
else
MEM_writeLE64(memPtr, (U64)val);
}
#if defined (__cplusplus)
}
lib/legacy/zstd_v05.c
+1 -63
View File
@@ -140,7 +140,7 @@ extern "C" {
#ifndef MEM_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */
# if defined(__GNUC__) && ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) )
# define MEM_FORCE_MEMORY_ACCESS 2
# elif defined(__INTEL_COMPILER) || \
# elif (defined(__INTEL_COMPILER) && !defined(WIN32)) || \
(defined(__GNUC__) && ( defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) || defined(__ARM_ARCH_7R__) || defined(__ARM_ARCH_7M__) || defined(__ARM_ARCH_7S__) ))
# define MEM_FORCE_MEMORY_ACCESS 1
# endif
@@ -250,18 +250,6 @@ MEM_STATIC U32 MEM_readLE32(const void* memPtr)
}
}
MEM_STATIC void MEM_writeLE32(void* memPtr, U32 val32)
{
if (MEM_isLittleEndian()) {
MEM_write32(memPtr, val32);
} else {
BYTE* p = (BYTE*)memPtr;
p[0] = (BYTE)val32;
p[1] = (BYTE)(val32>>8);
p[2] = (BYTE)(val32>>16);
p[3] = (BYTE)(val32>>24);
}
}
MEM_STATIC U64 MEM_readLE64(const void* memPtr)
{
@@ -274,22 +262,6 @@ MEM_STATIC U64 MEM_readLE64(const void* memPtr)
}
}
MEM_STATIC void MEM_writeLE64(void* memPtr, U64 val64)
{
if (MEM_isLittleEndian()) {
MEM_write64(memPtr, val64);
} else {
BYTE* p = (BYTE*)memPtr;
p[0] = (BYTE)val64;
p[1] = (BYTE)(val64>>8);
p[2] = (BYTE)(val64>>16);
p[3] = (BYTE)(val64>>24);
p[4] = (BYTE)(val64>>32);
p[5] = (BYTE)(val64>>40);
p[6] = (BYTE)(val64>>48);
p[7] = (BYTE)(val64>>56);
}
}
MEM_STATIC size_t MEM_readLEST(const void* memPtr)
{
@@ -299,13 +271,6 @@ MEM_STATIC size_t MEM_readLEST(const void* memPtr)
return (size_t)MEM_readLE64(memPtr);
}
MEM_STATIC void MEM_writeLEST(void* memPtr, size_t val)
{
if (MEM_32bits())
MEM_writeLE32(memPtr, (U32)val);
else
MEM_writeLE64(memPtr, (U64)val);
}
#if defined (__cplusplus)
}
@@ -763,28 +728,6 @@ MEM_STATIC void ZSTDv05_wildcopy(void* dst, const void* src, size_t length)
while (op < oend);
}
MEM_STATIC unsigned ZSTDv05_highbit(U32 val)
{
# if defined(_MSC_VER) /* Visual */
unsigned long r=0;
_BitScanReverse(&r, val);
return (unsigned)r;
# elif defined(__GNUC__) && (__GNUC__ >= 3) /* GCC Intrinsic */
return 31 - __builtin_clz(val);
# else /* Software version */
static const int DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30, 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31 };
U32 v = val;
int r;
v |= v >> 1;
v |= v >> 2;
v |= v >> 4;
v |= v >> 8;
v |= v >> 16;
r = DeBruijnClz[(U32)(v * 0x07C4ACDDU) >> 27];
return r;
# endif
}
/*-*******************************************
* Private interfaces
@@ -1350,11 +1293,6 @@ MEM_STATIC void FSEv05_initDState(FSEv05_DState_t* DStatePtr, BITv05_DStream_t*
DStatePtr->table = dt + 1;
}
MEM_STATIC size_t FSEv05_getStateValue(FSEv05_DState_t* DStatePtr)
{
return DStatePtr->state;
}
MEM_STATIC BYTE FSEv05_peakSymbol(FSEv05_DState_t* DStatePtr)
{
const FSEv05_decode_t DInfo = ((const FSEv05_decode_t*)(DStatePtr->table))[DStatePtr->state];
lib/legacy/zstd_v06.c
+6 -180
View File
@@ -85,6 +85,10 @@ extern "C" {
/*-****************************************
* Compiler specifics
******************************************/
#if defined(_MSC_VER) /* Visual Studio */
# include <stdlib.h> /* _byteswap_ulong */
# include <intrin.h> /* _byteswap_* */
#endif
#if defined(__GNUC__)
# define MEM_STATIC static __attribute__((unused))
#elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
@@ -138,7 +142,7 @@ extern "C" {
#ifndef MEM_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */
# if defined(__GNUC__) && ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) )
# define MEM_FORCE_MEMORY_ACCESS 2
# elif defined(__INTEL_COMPILER) || \
# elif (defined(__INTEL_COMPILER) && !defined(WIN32)) || \
(defined(__GNUC__) && ( defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) || defined(__ARM_ARCH_7R__) || defined(__ARM_ARCH_7M__) || defined(__ARM_ARCH_7S__) ))
# define MEM_FORCE_MEMORY_ACCESS 1
# endif
@@ -160,11 +164,8 @@ Only use if no other choice to achieve best performance on target platform */
MEM_STATIC U16 MEM_read16(const void* memPtr) { return *(const U16*) memPtr; }
MEM_STATIC U32 MEM_read32(const void* memPtr) { return *(const U32*) memPtr; }
MEM_STATIC U64 MEM_read64(const void* memPtr) { return *(const U64*) memPtr; }
MEM_STATIC U64 MEM_readST(const void* memPtr) { return *(const size_t*) memPtr; }
MEM_STATIC void MEM_write16(void* memPtr, U16 value) { *(U16*)memPtr = value; }
MEM_STATIC void MEM_write32(void* memPtr, U32 value) { *(U32*)memPtr = value; }
MEM_STATIC void MEM_write64(void* memPtr, U64 value) { *(U64*)memPtr = value; }
#elif defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==1)
@@ -175,11 +176,8 @@ typedef union { U16 u16; U32 u32; U64 u64; size_t st; } __attribute__((packed))
MEM_STATIC U16 MEM_read16(const void* ptr) { return ((const unalign*)ptr)->u16; }
MEM_STATIC U32 MEM_read32(const void* ptr) { return ((const unalign*)ptr)->u32; }
MEM_STATIC U64 MEM_read64(const void* ptr) { return ((const unalign*)ptr)->u64; }
MEM_STATIC U64 MEM_readST(const void* ptr) { return ((const unalign*)ptr)->st; }
MEM_STATIC void MEM_write16(void* memPtr, U16 value) { ((unalign*)memPtr)->u16 = value; }
MEM_STATIC void MEM_write32(void* memPtr, U32 value) { ((unalign*)memPtr)->u32 = value; }
MEM_STATIC void MEM_write64(void* memPtr, U64 value) { ((unalign*)memPtr)->u64 = value; }
#else
@@ -201,25 +199,11 @@ MEM_STATIC U64 MEM_read64(const void* memPtr)
U64 val; memcpy(&val, memPtr, sizeof(val)); return val;
}
MEM_STATIC size_t MEM_readST(const void* memPtr)
{
size_t val; memcpy(&val, memPtr, sizeof(val)); return val;
}
MEM_STATIC void MEM_write16(void* memPtr, U16 value)
{
memcpy(memPtr, &value, sizeof(value));
}
MEM_STATIC void MEM_write32(void* memPtr, U32 value)
{
memcpy(memPtr, &value, sizeof(value));
}
MEM_STATIC void MEM_write64(void* memPtr, U64 value)
{
memcpy(memPtr, &value, sizeof(value));
}
#endif /* MEM_FORCE_MEMORY_ACCESS */
@@ -255,13 +239,6 @@ MEM_STATIC U64 MEM_swap64(U64 in)
#endif
}
MEM_STATIC size_t MEM_swapST(size_t in)
{
if (MEM_32bits())
return (size_t)MEM_swap32((U32)in);
else
return (size_t)MEM_swap64((U64)in);
}
/*=== Little endian r/w ===*/
@@ -294,13 +271,6 @@ MEM_STATIC U32 MEM_readLE32(const void* memPtr)
return MEM_swap32(MEM_read32(memPtr));
}
MEM_STATIC void MEM_writeLE32(void* memPtr, U32 val32)
{
if (MEM_isLittleEndian())
MEM_write32(memPtr, val32);
else
MEM_write32(memPtr, MEM_swap32(val32));
}
MEM_STATIC U64 MEM_readLE64(const void* memPtr)
{
@@ -310,13 +280,6 @@ MEM_STATIC U64 MEM_readLE64(const void* memPtr)
return MEM_swap64(MEM_read64(memPtr));
}
MEM_STATIC void MEM_writeLE64(void* memPtr, U64 val64)
{
if (MEM_isLittleEndian())
MEM_write64(memPtr, val64);
else
MEM_write64(memPtr, MEM_swap64(val64));
}
MEM_STATIC size_t MEM_readLEST(const void* memPtr)
{
@@ -326,78 +289,7 @@ MEM_STATIC size_t MEM_readLEST(const void* memPtr)
return (size_t)MEM_readLE64(memPtr);
}
MEM_STATIC void MEM_writeLEST(void* memPtr, size_t val)
{
if (MEM_32bits())
MEM_writeLE32(memPtr, (U32)val);
else
MEM_writeLE64(memPtr, (U64)val);
}
/*=== Big endian r/w ===*/
MEM_STATIC U32 MEM_readBE32(const void* memPtr)
{
if (MEM_isLittleEndian())
return MEM_swap32(MEM_read32(memPtr));
else
return MEM_read32(memPtr);
}
MEM_STATIC void MEM_writeBE32(void* memPtr, U32 val32)
{
if (MEM_isLittleEndian())
MEM_write32(memPtr, MEM_swap32(val32));
else
MEM_write32(memPtr, val32);
}
MEM_STATIC U64 MEM_readBE64(const void* memPtr)
{
if (MEM_isLittleEndian())
return MEM_swap64(MEM_read64(memPtr));
else
return MEM_read64(memPtr);
}
MEM_STATIC void MEM_writeBE64(void* memPtr, U64 val64)
{
if (MEM_isLittleEndian())
MEM_write64(memPtr, MEM_swap64(val64));
else
MEM_write64(memPtr, val64);
}
MEM_STATIC size_t MEM_readBEST(const void* memPtr)
{
if (MEM_32bits())
return (size_t)MEM_readBE32(memPtr);
else
return (size_t)MEM_readBE64(memPtr);
}
MEM_STATIC void MEM_writeBEST(void* memPtr, size_t val)
{
if (MEM_32bits())
MEM_writeBE32(memPtr, (U32)val);
else
MEM_writeBE64(memPtr, (U64)val);
}
/* function safe only for comparisons */
MEM_STATIC U32 MEM_readMINMATCH(const void* memPtr, U32 length)
{
switch (length)
{
default :
case 4 : return MEM_read32(memPtr);
case 3 : if (MEM_isLittleEndian())
return MEM_read32(memPtr)<<8;
else
return MEM_read32(memPtr)>>8;
}
}
#if defined (__cplusplus)
}
@@ -887,27 +779,6 @@ MEM_STATIC void ZSTDv06_wildcopy(void* dst, const void* src, size_t length)
while (op < oend);
}
MEM_STATIC unsigned ZSTDv06_highbit(U32 val)
{
# if defined(_MSC_VER) /* Visual */
unsigned long r=0;
_BitScanReverse(&r, val);
return (unsigned)r;
# elif defined(__GNUC__) && (__GNUC__ >= 3) /* GCC Intrinsic */
return 31 - __builtin_clz(val);
# else /* Software version */
static const int DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30, 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31 };
U32 v = val;
int r;
v |= v >> 1;
v |= v >> 2;
v |= v >> 4;
v |= v >> 8;
v |= v >> 16;
r = DeBruijnClz[(U32)(v * 0x07C4ACDDU) >> 27];
return r;
# endif
}
/*-*******************************************
@@ -926,15 +797,7 @@ typedef struct {
U32 rep[ZSTDv06_REP_INIT];
} ZSTDv06_optimal_t;
#if ZSTDv06_OPT_DEBUG == 3
#include ".debug/zstd_stats.h"
#else
typedef struct { U32 unused; } ZSTDv06_stats_t;
MEM_STATIC void ZSTDv06_statsPrint(ZSTDv06_stats_t* stats, U32 searchLength) { (void)stats; (void)searchLength; }
MEM_STATIC void ZSTDv06_statsInit(ZSTDv06_stats_t* stats) { (void)stats; }
MEM_STATIC void ZSTDv06_statsResetFreqs(ZSTDv06_stats_t* stats) { (void)stats; }
MEM_STATIC void ZSTDv06_statsUpdatePrices(ZSTDv06_stats_t* stats, size_t litLength, const BYTE* literals, size_t offset, size_t matchLength) { (void)stats; (void)litLength; (void)literals; (void)offset; (void)matchLength; }
#endif
typedef struct { U32 unused; } ZSTDv06_stats_t;
typedef struct {
void* buffer;
@@ -1251,9 +1114,6 @@ MEM_STATIC unsigned BITv06_highbit32 (register U32 val)
# endif
}
/*===== Local Constants =====*/
static const unsigned BITv06_mask[] = { 0, 1, 3, 7, 0xF, 0x1F, 0x3F, 0x7F, 0xFF, 0x1FF, 0x3FF, 0x7FF, 0xFFF, 0x1FFF, 0x3FFF, 0x7FFF, 0xFFFF, 0x1FFFF, 0x3FFFF, 0x7FFFF, 0xFFFFF, 0x1FFFFF, 0x3FFFFF, 0x7FFFFF, 0xFFFFFF, 0x1FFFFFF, 0x3FFFFFF }; /* up to 26 bits */
/*-********************************************************
@@ -1299,29 +1159,6 @@ MEM_STATIC size_t BITv06_initDStream(BITv06_DStream_t* bitD, const void* srcBuff
return srcSize;
}
MEM_STATIC size_t BITv06_getUpperBits(size_t bitContainer, U32 const start)
{
return bitContainer >> start;
}
MEM_STATIC size_t BITv06_getMiddleBits(size_t bitContainer, U32 const start, U32 const nbBits)
{
#if defined(__BMI__) && defined(__GNUC__) /* experimental */
# if defined(__x86_64__)
if (sizeof(bitContainer)==8)
return _bextr_u64(bitContainer, start, nbBits);
else
# endif
return _bextr_u32(bitContainer, start, nbBits);
#else
return (bitContainer >> start) & BITv06_mask[nbBits];
#endif
}
MEM_STATIC size_t BITv06_getLowerBits(size_t bitContainer, U32 const nbBits)
{
return bitContainer & BITv06_mask[nbBits];
}
/*! BITv06_lookBits() :
* Provides next n bits from local register.
@@ -1332,12 +1169,8 @@ MEM_STATIC size_t BITv06_getLowerBits(size_t bitContainer, U32 const nbBits)
*/
MEM_STATIC size_t BITv06_lookBits(const BITv06_DStream_t* bitD, U32 nbBits)
{
#if defined(__BMI__) && defined(__GNUC__) /* experimental; fails if bitD->bitsConsumed + nbBits > sizeof(bitD->bitContainer)*8 */
return BITv06_getMiddleBits(bitD->bitContainer, (sizeof(bitD->bitContainer)*8) - bitD->bitsConsumed - nbBits, nbBits);
#else
U32 const bitMask = sizeof(bitD->bitContainer)*8 - 1;
return ((bitD->bitContainer << (bitD->bitsConsumed & bitMask)) >> 1) >> ((bitMask-nbBits) & bitMask);
#endif
}
/*! BITv06_lookBitsFast() :
@@ -1501,8 +1334,6 @@ static void FSEv06_initDState(FSEv06_DState_t* DStatePtr, BITv06_DStream_t*
static unsigned char FSEv06_decodeSymbol(FSEv06_DState_t* DStatePtr, BITv06_DStream_t* bitD);
static unsigned FSEv06_endOfDState(const FSEv06_DState_t* DStatePtr);
/*!
Let's now decompose FSEv06_decompress_usingDTable() into its unitary components.
You will decode FSE-encoded symbols from the bitStream,
@@ -1626,11 +1457,6 @@ MEM_STATIC BYTE FSEv06_decodeSymbolFast(FSEv06_DState_t* DStatePtr, BITv06_DStre
return symbol;
}
MEM_STATIC unsigned FSEv06_endOfDState(const FSEv06_DState_t* DStatePtr)
{
return DStatePtr->state == 0;
}
#ifndef FSEv06_COMMONDEFS_ONLY
lib/legacy/zstd_v07.c
+2 -158
View File
@@ -256,10 +256,6 @@ extern "C" {
# define MEM_STATIC static /* this version may generate warnings for unused static functions; disable the relevant warning */
#endif
/* code only tested on 32 and 64 bits systems */
#define MEM_STATIC_ASSERT(c) { enum { XXH_static_assert = 1/(int)(!!(c)) }; }
MEM_STATIC void MEM_check(void) { MEM_STATIC_ASSERT((sizeof(size_t)==4) || (sizeof(size_t)==8)); }
/*-**************************************************************
* Basic Types
@@ -303,7 +299,7 @@ MEM_STATIC void MEM_check(void) { MEM_STATIC_ASSERT((sizeof(size_t)==4) || (size
#ifndef MEM_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */
# if defined(__GNUC__) && ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) )
# define MEM_FORCE_MEMORY_ACCESS 2
# elif defined(__INTEL_COMPILER) || \
# elif (defined(__INTEL_COMPILER) && !defined(WIN32)) || \
(defined(__GNUC__) && ( defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) || defined(__ARM_ARCH_7R__) || defined(__ARM_ARCH_7M__) || defined(__ARM_ARCH_7S__) ))
# define MEM_FORCE_MEMORY_ACCESS 1
# endif
@@ -325,11 +321,8 @@ Only use if no other choice to achieve best performance on target platform */
MEM_STATIC U16 MEM_read16(const void* memPtr) { return *(const U16*) memPtr; }
MEM_STATIC U32 MEM_read32(const void* memPtr) { return *(const U32*) memPtr; }
MEM_STATIC U64 MEM_read64(const void* memPtr) { return *(const U64*) memPtr; }
MEM_STATIC U64 MEM_readST(const void* memPtr) { return *(const size_t*) memPtr; }
MEM_STATIC void MEM_write16(void* memPtr, U16 value) { *(U16*)memPtr = value; }
MEM_STATIC void MEM_write32(void* memPtr, U32 value) { *(U32*)memPtr = value; }
MEM_STATIC void MEM_write64(void* memPtr, U64 value) { *(U64*)memPtr = value; }
#elif defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==1)
@@ -340,11 +333,8 @@ typedef union { U16 u16; U32 u32; U64 u64; size_t st; } __attribute__((packed))
MEM_STATIC U16 MEM_read16(const void* ptr) { return ((const unalign*)ptr)->u16; }
MEM_STATIC U32 MEM_read32(const void* ptr) { return ((const unalign*)ptr)->u32; }
MEM_STATIC U64 MEM_read64(const void* ptr) { return ((const unalign*)ptr)->u64; }
MEM_STATIC U64 MEM_readST(const void* ptr) { return ((const unalign*)ptr)->st; }
MEM_STATIC void MEM_write16(void* memPtr, U16 value) { ((unalign*)memPtr)->u16 = value; }
MEM_STATIC void MEM_write32(void* memPtr, U32 value) { ((unalign*)memPtr)->u32 = value; }
MEM_STATIC void MEM_write64(void* memPtr, U64 value) { ((unalign*)memPtr)->u64 = value; }
#else
@@ -366,26 +356,11 @@ MEM_STATIC U64 MEM_read64(const void* memPtr)
U64 val; memcpy(&val, memPtr, sizeof(val)); return val;
}
MEM_STATIC size_t MEM_readST(const void* memPtr)
{
size_t val; memcpy(&val, memPtr, sizeof(val)); return val;
}
MEM_STATIC void MEM_write16(void* memPtr, U16 value)
{
memcpy(memPtr, &value, sizeof(value));
}
MEM_STATIC void MEM_write32(void* memPtr, U32 value)
{
memcpy(memPtr, &value, sizeof(value));
}
MEM_STATIC void MEM_write64(void* memPtr, U64 value)
{
memcpy(memPtr, &value, sizeof(value));
}
#endif /* MEM_FORCE_MEMORY_ACCESS */
MEM_STATIC U32 MEM_swap32(U32 in)
@@ -420,13 +395,6 @@ MEM_STATIC U64 MEM_swap64(U64 in)
#endif
}
MEM_STATIC size_t MEM_swapST(size_t in)
{
if (MEM_32bits())
return (size_t)MEM_swap32((U32)in);
else
return (size_t)MEM_swap64((U64)in);
}
/*=== Little endian r/w ===*/
@@ -459,13 +427,6 @@ MEM_STATIC U32 MEM_readLE32(const void* memPtr)
return MEM_swap32(MEM_read32(memPtr));
}
MEM_STATIC void MEM_writeLE32(void* memPtr, U32 val32)
{
if (MEM_isLittleEndian())
MEM_write32(memPtr, val32);
else
MEM_write32(memPtr, MEM_swap32(val32));
}
MEM_STATIC U64 MEM_readLE64(const void* memPtr)
{
@@ -475,14 +436,6 @@ MEM_STATIC U64 MEM_readLE64(const void* memPtr)
return MEM_swap64(MEM_read64(memPtr));
}
MEM_STATIC void MEM_writeLE64(void* memPtr, U64 val64)
{
if (MEM_isLittleEndian())
MEM_write64(memPtr, val64);
else
MEM_write64(memPtr, MEM_swap64(val64));
}
MEM_STATIC size_t MEM_readLEST(const void* memPtr)
{
if (MEM_32bits())
@@ -491,78 +444,7 @@ MEM_STATIC size_t MEM_readLEST(const void* memPtr)
return (size_t)MEM_readLE64(memPtr);
}
MEM_STATIC void MEM_writeLEST(void* memPtr, size_t val)
{
if (MEM_32bits())
MEM_writeLE32(memPtr, (U32)val);
else
MEM_writeLE64(memPtr, (U64)val);
}
/*=== Big endian r/w ===*/
MEM_STATIC U32 MEM_readBE32(const void* memPtr)
{
if (MEM_isLittleEndian())
return MEM_swap32(MEM_read32(memPtr));
else
return MEM_read32(memPtr);
}
MEM_STATIC void MEM_writeBE32(void* memPtr, U32 val32)
{
if (MEM_isLittleEndian())
MEM_write32(memPtr, MEM_swap32(val32));
else
MEM_write32(memPtr, val32);
}
MEM_STATIC U64 MEM_readBE64(const void* memPtr)
{
if (MEM_isLittleEndian())
return MEM_swap64(MEM_read64(memPtr));
else
return MEM_read64(memPtr);
}
MEM_STATIC void MEM_writeBE64(void* memPtr, U64 val64)
{
if (MEM_isLittleEndian())
MEM_write64(memPtr, MEM_swap64(val64));
else
MEM_write64(memPtr, val64);
}
MEM_STATIC size_t MEM_readBEST(const void* memPtr)
{
if (MEM_32bits())
return (size_t)MEM_readBE32(memPtr);
else
return (size_t)MEM_readBE64(memPtr);
}
MEM_STATIC void MEM_writeBEST(void* memPtr, size_t val)
{
if (MEM_32bits())
MEM_writeBE32(memPtr, (U32)val);
else
MEM_writeBE64(memPtr, (U64)val);
}
/* function safe only for comparisons */
MEM_STATIC U32 MEM_readMINMATCH(const void* memPtr, U32 length)
{
switch (length)
{
default :
case 4 : return MEM_read32(memPtr);
case 3 : if (MEM_isLittleEndian())
return MEM_read32(memPtr)<<8;
else
return MEM_read32(memPtr)>>8;
}
}
#if defined (__cplusplus)
}
@@ -887,8 +769,6 @@ MEM_STATIC unsigned BITv07_highbit32 (register U32 val)
# endif
}
/*===== Local Constants =====*/
static const unsigned BITv07_mask[] = { 0, 1, 3, 7, 0xF, 0x1F, 0x3F, 0x7F, 0xFF, 0x1FF, 0x3FF, 0x7FF, 0xFFF, 0x1FFF, 0x3FFF, 0x7FFF, 0xFFFF, 0x1FFFF, 0x3FFFF, 0x7FFFF, 0xFFFFF, 0x1FFFFF, 0x3FFFFF, 0x7FFFFF, 0xFFFFFF, 0x1FFFFFF, 0x3FFFFFF }; /* up to 26 bits */
/*-********************************************************
@@ -934,29 +814,6 @@ MEM_STATIC size_t BITv07_initDStream(BITv07_DStream_t* bitD, const void* srcBuff
return srcSize;
}
MEM_STATIC size_t BITv07_getUpperBits(size_t bitContainer, U32 const start)
{
return bitContainer >> start;
}
MEM_STATIC size_t BITv07_getMiddleBits(size_t bitContainer, U32 const start, U32 const nbBits)
{
#if defined(__BMI__) && defined(__GNUC__) /* experimental */
# if defined(__x86_64__)
if (sizeof(bitContainer)==8)
return _bextr_u64(bitContainer, start, nbBits);
else
# endif
return _bextr_u32(bitContainer, start, nbBits);
#else
return (bitContainer >> start) & BITv07_mask[nbBits];
#endif
}
MEM_STATIC size_t BITv07_getLowerBits(size_t bitContainer, U32 const nbBits)
{
return bitContainer & BITv07_mask[nbBits];
}
/*! BITv07_lookBits() :
* Provides next n bits from local register.
@@ -967,12 +824,8 @@ MEM_STATIC size_t BITv07_getLowerBits(size_t bitContainer, U32 const nbBits)
*/
MEM_STATIC size_t BITv07_lookBits(const BITv07_DStream_t* bitD, U32 nbBits)
{
#if defined(__BMI__) && defined(__GNUC__) /* experimental; fails if bitD->bitsConsumed + nbBits > sizeof(bitD->bitContainer)*8 */
return BITv07_getMiddleBits(bitD->bitContainer, (sizeof(bitD->bitContainer)*8) - bitD->bitsConsumed - nbBits, nbBits);
#else
U32 const bitMask = sizeof(bitD->bitContainer)*8 - 1;
return ((bitD->bitContainer << (bitD->bitsConsumed & bitMask)) >> 1) >> ((bitMask-nbBits) & bitMask);
#endif
}
/*! BITv07_lookBitsFast() :
@@ -1236,7 +1089,6 @@ static void FSEv07_initDState(FSEv07_DState_t* DStatePtr, BITv07_DStream_t*
static unsigned char FSEv07_decodeSymbol(FSEv07_DState_t* DStatePtr, BITv07_DStream_t* bitD);
static unsigned FSEv07_endOfDState(const FSEv07_DState_t* DStatePtr);
/**<
Let's now decompose FSEv07_decompress_usingDTable() into its unitary components.
@@ -1356,11 +1208,6 @@ MEM_STATIC BYTE FSEv07_decodeSymbolFast(FSEv07_DState_t* DStatePtr, BITv07_DStre
return symbol;
}
MEM_STATIC unsigned FSEv07_endOfDState(const FSEv07_DState_t* DStatePtr)
{
return DStatePtr->state == 0;
}
#ifndef FSEv07_COMMONDEFS_ONLY
@@ -3269,10 +3116,6 @@ typedef struct {
} ZSTDv07_optimal_t;
struct ZSTDv07_stats_s { U32 unused; };
MEM_STATIC void ZSTDv07_statsPrint(ZSTDv07_stats_t* stats, U32 searchLength) { (void)stats; (void)searchLength; }
MEM_STATIC void ZSTDv07_statsInit(ZSTDv07_stats_t* stats) { (void)stats; }
MEM_STATIC void ZSTDv07_statsResetFreqs(ZSTDv07_stats_t* stats) { (void)stats; }
MEM_STATIC void ZSTDv07_statsUpdatePrices(ZSTDv07_stats_t* stats, size_t litLength, const BYTE* literals, size_t offset, size_t matchLength) { (void)stats; (void)litLength; (void)literals; (void)offset; (void)matchLength; }
typedef struct {
void* buffer;
@@ -3373,6 +3216,7 @@ static const ZSTDv07_customMem defaultCustomMem = { ZSTDv07_defaultAllocFunction
# include <intrin.h> /* For Visual 2005 */
# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
# pragma warning(disable : 4324) /* disable: C4324: padded structure */
# pragma warning(disable : 4100) /* disable: C4100: unreferenced formal parameter */
#else
# ifdef __GNUC__
# define FORCE_INLINE static inline __attribute__((always_inline))
lib/zstd.h
+136 -28
View File
@@ -36,7 +36,7 @@
extern "C" {
#endif
/*====== Dependency ======*/
/*====== Dependency ======*/
#include <stddef.h> /* size_t */
@@ -52,10 +52,10 @@ extern "C" {
#endif
/*====== Version ======*/
/*======= Version =======*/
#define ZSTD_VERSION_MAJOR 0
#define ZSTD_VERSION_MINOR 8
#define ZSTD_VERSION_RELEASE 0
#define ZSTD_VERSION_RELEASE 1
#define ZSTD_LIB_VERSION ZSTD_VERSION_MAJOR.ZSTD_VERSION_MINOR.ZSTD_VERSION_RELEASE
#define ZSTD_QUOTE(str) #str
@@ -84,23 +84,21 @@ ZSTDLIB_API size_t ZSTD_compress( void* dst, size_t dstCapacity,
* potentially larger than what local system can handle as a single memory segment.
* In which case, it's necessary to use streaming mode to decompress data.
* note 2 : decompressed size is an optional field, that may not be present.
* When `return==0`, consider data to decompress could have any size.
* In which case, it's necessary to use streaming mode to decompress data,
* or rely on application's implied limits.
* (For example, it may know that its own data is necessarily cut into blocks <= 16 KB).
* When `return==0`, data to decompress can have any size.
* In which case, it's necessary to use streaming mode to decompress data.
* Optionally, application may rely on its own implied limits.
* (For example, application own data could be necessarily cut into blocks <= 16 KB).
* note 3 : decompressed size could be wrong or intentionally modified !
* Always ensure result fits within application's authorized limits !
* Each application can have its own set of conditions.
* If the intention is to decompress public data compressed by zstd command line utility,
* it is recommended to support at least 8 MB for extended compatibility.
* Each application can set its own limits.
* note 4 : when `return==0`, if precise failure cause is needed, use ZSTD_getFrameParams() to know more. */
unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize);
ZSTDLIB_API unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize);
/*! ZSTD_decompress() :
`compressedSize` : must be the _exact_ size of compressed input, otherwise decompression will fail.
`dstCapacity` must be equal or larger than originalSize (see ZSTD_getDecompressedSize() ).
If originalSize is unknown, and if there is no implied application-specific limitations,
it's necessary to use streaming mode to decompress data.
it's preferable to use streaming mode to decompress data.
@return : the number of bytes decompressed into `dst` (<= `dstCapacity`),
or an errorCode if it fails (which can be tested using ZSTD_isError()) */
ZSTDLIB_API size_t ZSTD_decompress( void* dst, size_t dstCapacity,
@@ -141,7 +139,7 @@ ZSTDLIB_API size_t ZSTD_decompressDCtx(ZSTD_DCtx* ctx, void* dst, size_t dstCapa
***************************/
/*! ZSTD_compress_usingDict() :
* Compression using a predefined Dictionary (see dictBuilder/zdict.h).
* Note : This function load the dictionary, resulting in a significant startup time. */
* Note : This function load the dictionary, resulting in significant startup delay. */
ZSTDLIB_API size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx,
void* dst, size_t dstCapacity,
const void* src, size_t srcSize,
@@ -151,7 +149,7 @@ ZSTDLIB_API size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx,
/*! ZSTD_decompress_usingDict() :
* Decompression using a predefined Dictionary (see dictBuilder/zdict.h).
* Dictionary must be identical to the one used during compression.
* Note : This function load the dictionary, resulting in a significant startup time */
* Note : This function load the dictionary, resulting in significant startup delay */
ZSTDLIB_API size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx,
void* dst, size_t dstCapacity,
const void* src, size_t srcSize,
@@ -163,7 +161,7 @@ ZSTDLIB_API size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx,
****************************/
/*! ZSTD_createCDict() :
* Create a digested dictionary, ready to start compression operation without startup delay.
* `dict` can be released after creation */
* `dict` can be released after ZSTD_CDict creation */
typedef struct ZSTD_CDict_s ZSTD_CDict;
ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict(const void* dict, size_t dictSize, int compressionLevel);
ZSTDLIB_API size_t ZSTD_freeCDict(ZSTD_CDict* CDict);
@@ -232,19 +230,19 @@ static const size_t ZSTD_skippableHeaderSize = 8; /* magic number + skippable f
typedef enum { ZSTD_fast, ZSTD_dfast, ZSTD_greedy, ZSTD_lazy, ZSTD_lazy2, ZSTD_btlazy2, ZSTD_btopt } ZSTD_strategy; /*< from faster to stronger */
typedef struct {
unsigned windowLog; /*< largest match distance : larger == more compression, more memory needed during decompression */
unsigned chainLog; /*< fully searched segment : larger == more compression, slower, more memory (useless for fast) */
unsigned hashLog; /*< dispatch table : larger == faster, more memory */
unsigned searchLog; /*< nb of searches : larger == more compression, slower */
unsigned searchLength; /*< match length searched : larger == faster decompression, sometimes less compression */
unsigned targetLength; /*< acceptable match size for optimal parser (only) : larger == more compression, slower */
unsigned windowLog; /**< largest match distance : larger == more compression, more memory needed during decompression */
unsigned chainLog; /**< fully searched segment : larger == more compression, slower, more memory (useless for fast) */
unsigned hashLog; /**< dispatch table : larger == faster, more memory */
unsigned searchLog; /**< nb of searches : larger == more compression, slower */
unsigned searchLength; /**< match length searched : larger == faster decompression, sometimes less compression */
unsigned targetLength; /**< acceptable match size for optimal parser (only) : larger == more compression, slower */
ZSTD_strategy strategy;
} ZSTD_compressionParameters;
typedef struct {
unsigned contentSizeFlag; /*< 1: content size will be in frame header (if known). */
unsigned checksumFlag; /*< 1: will generate a 22-bits checksum at end of frame, to be used for error detection by decompressor */
unsigned noDictIDFlag; /*< 1: no dict ID will be saved into frame header (if dictionary compression) */
unsigned contentSizeFlag; /**< 1: content size will be in frame header (if known). */
unsigned checksumFlag; /**< 1: will generate a 22-bits checksum at end of frame, to be used for error detection by decompressor */
unsigned noDictIDFlag; /**< 1: no dict ID will be saved into frame header (if dictionary compression) */
} ZSTD_frameParameters;
typedef struct {
@@ -323,12 +321,122 @@ ZSTDLIB_API size_t ZSTD_sizeofDCtx(const ZSTD_DCtx* dctx);
/* ******************************************************************
* Buffer-less streaming functions (synchronous mode)
* Streaming
********************************************************************/
typedef struct ZSTD_inBuffer_s {
const void* src; /**< start of input buffer */
size_t size; /**< size of input buffer */
size_t pos; /**< position where reading stopped. Will be updated. Necessarily 0 <= pos <= size */
} ZSTD_inBuffer;
typedef struct ZSTD_outBuffer_s {
void* dst; /**< start of output buffer */
size_t size; /**< size of output buffer */
size_t pos; /**< position where writing stopped. Will be updated. Necessarily 0 <= pos <= size */
} ZSTD_outBuffer;
/*====== compression ======*/
/*-***********************************************************************
* Streaming compression - howto
*
* A ZSTD_CStream object is required to track streaming operation.
* Use ZSTD_createCStream() and ZSTD_freeCStream() to create/release resources.
* ZSTD_CStream objects can be reused multiple times on consecutive compression operations.
*
* Start by initializing ZSTD_CStream.
* Use ZSTD_initCStream() to start a new compression operation.
* Use ZSTD_initCStream_usingDict() for a compression which requires a dictionary.
*
* Use ZSTD_compressStream() repetitively to consume input stream.
* The function will automatically update both `pos`.
* Note that it may not consume the entire input, in which case `pos < size`,
* and it's up to the caller to present again remaining data.
* @return : a hint to preferred nb of bytes to use as input for next function call (it's just a hint, to improve latency)
* or an error code, which can be tested using ZSTD_isError().
*
* At any moment, it's possible to flush whatever data remains within buffer, using ZSTD_flushStream().
* `output->pos` will be updated.
* Note some content might still be left within internal buffer if `output->size` is too small.
* @return : nb of bytes still present within internal buffer (0 if it's empty)
* or an error code, which can be tested using ZSTD_isError().
*
* ZSTD_endStream() instructs to finish a frame.
* It will perform a flush and write frame epilogue.
* The epilogue is required for decoders to consider a frame completed.
* Similar to ZSTD_flushStream(), it may not be able to flush the full content if `output->size` is too small.
* In which case, call again ZSTD_endStream() to complete the flush.
* @return : nb of bytes still present within internal buffer (0 if it's empty)
* or an error code, which can be tested using ZSTD_isError().
*
* *******************************************************************/
typedef struct ZSTD_CStream_s ZSTD_CStream;
ZSTD_CStream* ZSTD_createCStream(void);
size_t ZSTD_freeCStream(ZSTD_CStream* zcs);
size_t ZSTD_CStreamInSize(void); /**< recommended size for input buffer */
size_t ZSTD_CStreamOutSize(void); /**< recommended size for output buffer */
size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel);
size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuffer* input);
size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output);
size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output);
/* advanced */
ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem);
size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel);
size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize,
ZSTD_parameters params, unsigned long long pledgedSrcSize);
/*====== decompression ======*/
/*-***************************************************************************
* Streaming decompression howto
*
* A ZSTD_DStream object is required to track streaming operations.
* Use ZSTD_createDStream() and ZSTD_freeDStream() to create/release resources.
* ZSTD_DStream objects can be re-init multiple times.
*
* Use ZSTD_initDStream() to start a new decompression operation,
* or ZSTD_initDStream_usingDict() if decompression requires a dictionary.
*
* Use ZSTD_decompressStream() repetitively to consume your input.
* The function will update both `pos`.
* Note that it may not consume the entire input (pos < size),
* in which case it's up to the caller to present remaining input again.
* @return : 0 when a frame is completely decoded and fully flushed,
* 1 when there is still some data left within internal buffer to flush,
* >1 when more data is expected, with value being a suggested next input size (it's just a hint, which helps latency, any size is accepted),
* or an error code, which can be tested using ZSTD_isError().
*
* *******************************************************************************/
typedef struct ZSTD_DStream_s ZSTD_DStream;
ZSTD_DStream* ZSTD_createDStream(void);
size_t ZSTD_freeDStream(ZSTD_DStream* zds);
size_t ZSTD_DStreamInSize(void); /*!< recommended size for input buffer */
size_t ZSTD_DStreamOutSize(void); /*!< recommended size for output buffer */
size_t ZSTD_initDStream(ZSTD_DStream* zds);
size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inBuffer* input);
/* advanced */
ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem);
size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize);
/* ******************************************************************
* Buffer-less and synchronous inner streaming functions
********************************************************************/
/* This is an advanced API, giving full control over buffer management, for users which need direct control over memory.
* But it's also a complex one, with a lot of restrictions (documented below).
* For an easier streaming API, look into common/zbuff.h
* which removes all restrictions by allocating and managing its own internal buffer */
* But it's also a complex one, with many restrictions (documented below).
* Prefer using normal streaming API for an easier experience */
ZSTDLIB_API size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel);
ZSTDLIB_API size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel);
programs/.gitignore
+2
View File
@@ -9,6 +9,8 @@ fuzzer
fuzzer32
zbufftest
zbufftest32
zstreamtest
zstreamtest32
datagen
paramgrill
paramgrill32
programs/Makefile
+47 -30
View File
@@ -44,7 +44,7 @@ else
ALIGN_LOOP =
endif
CPPFLAGS= -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/dictBuilder -DXXH_NAMESPACE=ZSTD_
CPPFLAGS= -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/dictBuilder
CFLAGS ?= -O3
CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wstrict-aliasing=1 \
-Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes -Wundef
@@ -54,10 +54,12 @@ FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS)
ZSTDCOMMON_FILES := $(ZSTDDIR)/common/*.c
ZSTDCOMP_FILES := $(ZSTDDIR)/compress/zstd_compress.c $(ZSTDDIR)/compress/fse_compress.c $(ZSTDDIR)/compress/huf_compress.c
ZSTDDECOMP_FILES := $(ZSTDDIR)/decompress/zstd_decompress.o $(ZSTDDIR)/decompress/huf_decompress.c
ZSTDDECOMP_FILES := $(ZSTDDIR)/decompress/huf_decompress.c
ZSTD_FILES := $(ZSTDDECOMP_FILES) $(ZSTDCOMMON_FILES) $(ZSTDCOMP_FILES)
ZBUFF_FILES := $(ZSTDDIR)/compress/zbuff_compress.c $(ZSTDDIR)/decompress/zbuff_decompress.c
ZDICT_FILES := $(ZSTDDIR)/dictBuilder/*.c
ZSTDDECOMP_O = $(ZSTDDIR)/decompress/zstd_decompress.o
ZSTDDECOMP32_O = $(ZSTDDIR)/decompress/zstd_decompress32.o
ifeq ($(ZSTD_LEGACY_SUPPORT), 0)
CPPFLAGS += -DZSTD_LEGACY_SUPPORT=0
@@ -86,21 +88,25 @@ ZSTDRTTEST= --test-large-data
default: zstd
all: zstd fullbench fuzzer zbufftest paramgrill datagen
all: zstd fullbench fuzzer zbufftest zstreamtest paramgrill datagen
all32: CFLAGS += -m32
all32: EXT := 32$(EXT)
all32: cleano32 all
all32: cleano32 zstd32 fullbench32 fuzzer32 zbufftest32 zstreamtest32
$(ZSTDDIR)/decompress/zstd_decompress.o: CFLAGS += $(ALIGN_LOOP)
zstd : $(ZSTD_FILES) $(ZSTDLEGACY_FILES) $(ZBUFF_FILES) $(ZDICT_FILES) \
$(ZSTDDECOMP_O): $(ZSTDDIR)/decompress/zstd_decompress.c
$(CC) $(ALIGN_LOOP) $(FLAGS) $^ -c -o $@
$(ZSTDDECOMP32_O): $(ZSTDDIR)/decompress/zstd_decompress.c
$(CC) -m32 $(ALIGN_LOOP) $(FLAGS) $^ -c -o $@
zstd : $(ZSTDDECOMP_O) $(ZSTD_FILES) $(ZSTDLEGACY_FILES) $(ZDICT_FILES) \
zstdcli.c fileio.c bench.c datagen.c dibio.c
$(CC) $(FLAGS) -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT) $^ -o $@$(EXT)
zstd32: CFLAGS += -m32
zstd32: EXT := 32$(EXT)
zstd32: zstd
zstd32 : $(ZSTDDECOMP32_O) $(ZSTD_FILES) $(ZSTDLEGACY_FILES) $(ZDICT_FILES) \
zstdcli.c fileio.c bench.c datagen.c dibio.c
$(CC) -m32 $(FLAGS) -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT) $^ -o $@$(EXT)
zstd_nolegacy :
$(MAKE) zstd ZSTD_LEGACY_SUPPORT=0
@@ -116,49 +122,53 @@ zstd-pgo : clean zstd
rm zstd
$(MAKE) zstd MOREFLAGS=-fprofile-use
zstd-frugal: $(ZSTD_FILES) $(ZBUFF_FILES) zstdcli.c fileio.c
zstd-frugal: $(ZSTDDECOMP_O) $(ZSTD_FILES) zstdcli.c fileio.c
$(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_LEGACY_SUPPORT=0 $^ -o zstd$(EXT)
zstd-compress: $(ZSTDCOMMON_FILES) $(ZSTDCOMP_FILES) \
$(ZSTDDIR)/compress/zbuff_compress.c zstdcli.c fileio.c
zstdcli.c fileio.c
$(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_NODECOMPRESS -DZSTD_LEGACY_SUPPORT=0 $^ -o $@$(EXT)
zstd-decompress: $(ZSTDCOMMON_FILES) $(ZSTDDECOMP_FILES) \
$(ZSTDDIR)/decompress/zbuff_decompress.c zstdcli.c fileio.c
zstdcli.c fileio.c
$(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_NOCOMPRESS -DZSTD_LEGACY_SUPPORT=0 $^ -o $@$(EXT)
zstd-small: clean
CFLAGS="-Os -s" $(MAKE) zstd-frugal
fullbench : $(ZSTD_FILES) $(ZBUFF_FILES) datagen.c fullbench.c
fullbench : $(ZSTDDECOMP_O) $(ZSTD_FILES) $(ZBUFF_FILES) datagen.c fullbench.c
$(CC) $(FLAGS) $^ -o $@$(EXT)
fullbench32 : CFLAGS += -m32
fullbench32 : EXT := 32$(EXT)
fullbench32 : fullbench
fullbench32 : $(ZSTDDECOMP32_O) $(ZSTD_FILES) $(ZBUFF_FILES) datagen.c fullbench.c
$(CC) -m32 $(FLAGS) $^ -o $@$(EXT)
fuzzer : CPPFLAGS += -I$(ZSTDDIR)/dictBuilder
fuzzer : $(ZSTD_FILES) $(ZDICT_FILES) datagen.c fuzzer.c
fuzzer : $(ZSTDDECOMP_O) $(ZSTD_FILES) $(ZDICT_FILES) datagen.c fuzzer.c
$(CC) $(FLAGS) $^ -o $@$(EXT)
fuzzer32 : CFLAGS += -m32
fuzzer32 : EXT := 32$(EXT)
fuzzer32 : fuzzer
fuzzer32 : CPPFLAGS += -I$(ZSTDDIR)/dictBuilder
fuzzer32 : $(ZSTDDECOMP32_O) $(ZSTD_FILES) $(ZDICT_FILES) datagen.c fuzzer.c
$(CC) -m32 $(FLAGS) $^ -o $@$(EXT)
zbufftest : $(ZSTD_FILES) $(ZBUFF_FILES) datagen.c zbufftest.c
zbufftest : $(ZSTDDECOMP_O) $(ZSTD_FILES) $(ZBUFF_FILES) datagen.c zbufftest.c
$(CC) $(FLAGS) $^ -o $@$(EXT)
zbufftest32 : CFLAGS += -m32
zbufftest32 : EXT := 32$(EXT)
zbufftest32 : zbufftest
zbufftest32 : $(ZSTDDECOMP32_O) $(ZSTD_FILES) $(ZBUFF_FILES) datagen.c zbufftest.c
$(CC) -m32 $(FLAGS) $^ -o $@$(EXT)
paramgrill : $(ZSTD_FILES) datagen.c paramgrill.c
zstreamtest : $(ZSTDDECOMP_O) $(ZSTD_FILES) datagen.c zstreamtest.c
$(CC) $(FLAGS) $^ -o $@$(EXT)
zstreamtest32 : $(ZSTDDECOMP32_O) $(ZSTD_FILES) datagen.c zstreamtest.c
$(CC) -m32 $(FLAGS) $^ -o $@$(EXT)
paramgrill : $(ZSTDDECOMP_O) $(ZSTD_FILES) datagen.c paramgrill.c
$(CC) $(FLAGS) $^ -lm -o $@$(EXT)
datagen : datagen.c datagencli.c
$(CC) $(FLAGS) $^ -o $@$(EXT)
roundTripCrash : $(ZSTD_FILES) roundTripCrash.c
roundTripCrash : $(ZSTDDECOMP_O) $(ZSTD_FILES) roundTripCrash.c
$(CC) $(FLAGS) $^ -o $@$(EXT)
clean:
@@ -229,9 +239,9 @@ ifneq (,$(filter $(HOST_OS),MSYS POSIX))
zstd-playTests: datagen
ZSTD=$(ZSTD) ./playTests.sh $(ZSTDRTTEST)
test: test-zstd test-fullbench test-fuzzer test-zbuff
test: test-zstd test-fullbench test-fuzzer test-zbuff test-zstream
test32: test-zstd32 test-fullbench32 test-fuzzer32 test-zbuff32
test32: test-zstd32 test-fullbench32 test-fuzzer32 test-zbuff32 test-zstream32
test-all: test test32 valgrindTest
@@ -263,4 +273,11 @@ test-zbuff: zbufftest
test-zbuff32: zbufftest32
./zbufftest32 $(ZBUFFTEST)
test-zstream: zstreamtest
./zstreamtest $(ZBUFFTEST)
test-zstream32: zstreamtest32
./zstreamtest32 $(ZBUFFTEST)
endif
programs/bench.c
+46 -38
View File
@@ -30,6 +30,7 @@
#include <stdlib.h> /* malloc, free */
#include <string.h> /* memset */
#include <stdio.h> /* fprintf, fopen, ftello64 */
#include <time.h> /* clock_t, clock, CLOCKS_PER_SEC */
#include "mem.h"
#define ZSTD_STATIC_LINKING_ONLY
@@ -69,6 +70,13 @@ static U32 g_compressibilityDefault = 50;
#define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); }
static U32 g_displayLevel = 2; /* 0 : no display; 1: errors; 2 : + result + interaction + warnings; 3 : + progression; 4 : + information */
#define DISPLAYUPDATE(l, ...) if (g_displayLevel>=l) { \
if ((clock() - g_time > refreshRate) || (g_displayLevel>=4)) \
{ g_time = clock(); DISPLAY(__VA_ARGS__); \
if (g_displayLevel>=4) fflush(stdout); } }
static const clock_t refreshRate = CLOCKS_PER_SEC * 15 / 100;
static clock_t g_time = 0;
/* *************************************
* Exceptions
@@ -101,7 +109,7 @@ void BMK_setAdditionalParam(int additionalParam) { g_additionalParam=additionalP
void BMK_SetNbIterations(unsigned nbLoops)
{
g_nbIterations = nbLoops;
DISPLAYLEVEL(2, "- %i iterations -\n", g_nbIterations);
DISPLAYLEVEL(3, "- test >= %u seconds per compression / decompression -\n", g_nbIterations);
}
void BMK_SetBlockSize(size_t blockSize)
@@ -125,14 +133,6 @@ typedef struct
size_t resSize;
} blockParam_t;
typedef struct
{
double ratio;
size_t cSize;
double cSpeed;
double dSpeed;
} benchResult_t;
#define MIN(a,b) ((a)<(b) ? (a) : (b))
#define MAX(a,b) ((a)>(b) ? (a) : (b))
@@ -140,9 +140,10 @@ typedef struct
static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
const char* displayName, int cLevel,
const size_t* fileSizes, U32 nbFiles,
const void* dictBuffer, size_t dictBufferSize, benchResult_t *result)
const void* dictBuffer, size_t dictBufferSize)
{
size_t const blockSize = (g_blockSize>=32 ? g_blockSize : srcSize) + (!srcSize) /* avoid div by 0 */ ;
size_t const avgSize = MIN(g_blockSize, (srcSize / nbFiles));
U32 const maxNbBlocks = (U32) ((srcSize + (blockSize-1)) / blockSize) + nbFiles;
blockParam_t* const blockTable = (blockParam_t*) malloc(maxNbBlocks * sizeof(blockParam_t));
size_t const maxCompressedSize = ZSTD_compressBound(srcSize) + (maxNbBlocks * 1024); /* add some room for safety */
@@ -190,13 +191,18 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
{ U64 fastestC = (U64)(-1LL), fastestD = (U64)(-1LL);
U64 const crcOrig = XXH64(srcBuffer, srcSize, 0);
UTIL_time_t coolTime;
U32 testNb;
U64 const maxTime = (g_nbIterations * TIMELOOP_MICROSEC) + 100;
U64 totalCTime=0, totalDTime=0;
U32 cCompleted=0, dCompleted=0;
# define NB_MARKS 4
const char* const marks[NB_MARKS] = { " |", " /", " =", "\\" };
U32 markNb = 0;
size_t cSize = 0;
double ratio = 0.;
UTIL_getTime(&coolTime);
DISPLAYLEVEL(2, "\r%79s\r", "");
for (testNb = 1; testNb <= (g_nbIterations + !g_nbIterations); testNb++) {
while (!cCompleted | !dCompleted) {
UTIL_time_t clockStart;
U64 clockLoop = g_nbIterations ? TIMELOOP_MICROSEC : 1;
@@ -208,16 +214,15 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
}
/* Compression */
DISPLAYLEVEL(2, "%2i-%-17.17s :%10u ->\r", testNb, displayName, (U32)srcSize);
memset(compressedBuffer, 0xE5, maxCompressedSize); /* warm up and erase result buffer */
DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->\r", marks[markNb], displayName, (U32)srcSize);
if (!cCompleted) memset(compressedBuffer, 0xE5, maxCompressedSize); /* warm up and erase result buffer */
UTIL_sleepMilli(1); /* give processor time to other processes */
UTIL_waitForNextTick(ticksPerSecond);
UTIL_getTime(&clockStart);
{ //size_t const refSrcSize = (nbBlocks == 1) ? srcSize : 0;
//ZSTD_parameters const zparams = ZSTD_getParams(cLevel, refSrcSize, dictBufferSize);
ZSTD_parameters const zparams = ZSTD_getParams(cLevel, blockSize, dictBufferSize);
if (!cCompleted) { /* still some time to do compression tests */
ZSTD_parameters const zparams = ZSTD_getParams(cLevel, avgSize, dictBufferSize);
ZSTD_customMem const cmem = { NULL, NULL, NULL };
U32 nbLoops = 0;
ZSTD_CDict* cdict = ZSTD_createCDict_advanced(dictBuffer, dictBufferSize, zparams, cmem);
@@ -237,13 +242,16 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
ZSTD_freeCDict(cdict);
{ U64 const clockSpan = UTIL_clockSpanMicro(clockStart, ticksPerSecond);
if (clockSpan < fastestC*nbLoops) fastestC = clockSpan / nbLoops;
totalCTime += clockSpan;
cCompleted = totalCTime>maxTime;
} }
cSize = 0;
{ U32 blockNb; for (blockNb=0; blockNb<nbBlocks; blockNb++) cSize += blockTable[blockNb].cSize; }
ratio = (double)srcSize / (double)cSize;
DISPLAYLEVEL(2, "%2i-%-17.17s :%10u ->%10u (%5.3f),%6.1f MB/s\r",
testNb, displayName, (U32)srcSize, (U32)cSize, ratio,
markNb = (markNb+1) % NB_MARKS;
DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.3f),%6.1f MB/s\r",
marks[markNb], displayName, (U32)srcSize, (U32)cSize, ratio,
(double)srcSize / fastestC );
(void)fastestD; (void)crcOrig; /* unused when decompression disabled */
@@ -255,7 +263,8 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
UTIL_waitForNextTick(ticksPerSecond);
UTIL_getTime(&clockStart);
{ U32 nbLoops = 0;
if (!dCompleted) {
U32 nbLoops = 0;
ZSTD_DDict* ddict = ZSTD_createDDict(dictBuffer, dictBufferSize);
if (!ddict) EXM_THROW(2, "ZSTD_createDDict() allocation failure");
do {
@@ -278,10 +287,13 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
ZSTD_freeDDict(ddict);
{ U64 const clockSpan = UTIL_clockSpanMicro(clockStart, ticksPerSecond);
if (clockSpan < fastestD*nbLoops) fastestD = clockSpan / nbLoops;
totalDTime += clockSpan;
dCompleted = totalDTime>maxTime;
} }
DISPLAYLEVEL(2, "%2i-%-17.17s :%10u ->%10u (%5.3f),%6.1f MB/s ,%6.1f MB/s\r",
testNb, displayName, (U32)srcSize, (U32)cSize, ratio,
markNb = (markNb+1) % NB_MARKS;
DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.3f),%6.1f MB/s ,%6.1f MB/s\r",
marks[markNb], displayName, (U32)srcSize, (U32)cSize, ratio,
(double)srcSize / fastestC,
(double)srcSize / fastestD );
@@ -312,10 +324,14 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
#endif
} /* for (testNb = 1; testNb <= (g_nbIterations + !g_nbIterations); testNb++) */
result->ratio = ratio;
result->cSize = cSize;
result->cSpeed = (double)srcSize / fastestC;
result->dSpeed = (double)srcSize / fastestD;
if (g_displayLevel == 1) {
double cSpeed = (double)srcSize / fastestC;
double dSpeed = (double)srcSize / fastestD;
if (g_additionalParam)
DISPLAY("-%-3i%11i (%5.3f) %6.2f MB/s %6.1f MB/s %s (param=%d)\n", cLevel, (int)cSize, ratio, cSpeed, dSpeed, displayName, g_additionalParam);
else
DISPLAY("-%-3i%11i (%5.3f) %6.2f MB/s %6.1f MB/s %s\n", cLevel, (int)cSize, ratio, cSpeed, dSpeed, displayName);
}
DISPLAYLEVEL(2, "%2i#\n", cLevel);
} /* Bench */
@@ -352,7 +368,6 @@ static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize,
const size_t* fileSizes, unsigned nbFiles,
const void* dictBuffer, size_t dictBufferSize)
{
benchResult_t result;
int l;
const char* pch = strrchr(displayName, '\\'); /* Windows */
@@ -361,10 +376,8 @@ static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize,
SET_HIGH_PRIORITY;
memset(&result, 0, sizeof(result));
if (g_displayLevel == 1 && !g_additionalParam)
DISPLAY("bench %s %s: input %u bytes, %i iterations, %u KB blocks\n", ZSTD_VERSION_STRING, ZSTD_GIT_COMMIT_STRING, (U32)benchedSize, g_nbIterations, (U32)(g_blockSize>>10));
DISPLAY("bench %s %s: input %u bytes, %u iterations, %u KB blocks\n", ZSTD_VERSION_STRING, ZSTD_GIT_COMMIT_STRING, (U32)benchedSize, g_nbIterations, (U32)(g_blockSize>>10));
if (cLevelLast < cLevel) cLevelLast = cLevel;
@@ -372,13 +385,8 @@ static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize,
BMK_benchMem(srcBuffer, benchedSize,
displayName, l,
fileSizes, nbFiles,
dictBuffer, dictBufferSize, &result);
if (g_displayLevel == 1) {
if (g_additionalParam)
DISPLAY("%-3i%11i (%5.3f) %6.2f MB/s %6.1f MB/s %s (param=%d)\n", -l, (int)result.cSize, result.ratio, result.cSpeed, result.dSpeed, displayName, g_additionalParam);
else
DISPLAY("%-3i%11i (%5.3f) %6.2f MB/s %6.1f MB/s %s\n", -l, (int)result.cSize, result.ratio, result.cSpeed, result.dSpeed, displayName);
} }
dictBuffer, dictBufferSize);
}
}
@@ -401,7 +409,7 @@ static void BMK_loadFiles(void* buffer, size_t bufferSize,
}
f = fopen(fileNamesTable[n], "rb");
if (f==NULL) EXM_THROW(10, "impossible to open file %s", fileNamesTable[n]);
DISPLAYLEVEL(2, "Loading %s... \r", fileNamesTable[n]);
DISPLAYUPDATE(2, "Loading %s... \r", fileNamesTable[n]);
if (fileSize > bufferSize-pos) fileSize = bufferSize-pos, nbFiles=n; /* buffer too small - stop after this file */
{ size_t const readSize = fread(((char*)buffer)+pos, 1, (size_t)fileSize, f);
if (readSize != (size_t)fileSize) EXM_THROW(11, "could not read %s", fileNamesTable[n]);
programs/datagen.c
+4 -2
View File
@@ -26,8 +26,10 @@
/* *************************************
* Compiler Options
***************************************/
#define _CRT_SECURE_NO_WARNINGS /* removes Visual warning on strerror() */
#if defined(_MSC_VER)
# define _CRT_SECURE_NO_WARNINGS /* removes Visual warning on strerror() */
# define _CRT_SECURE_NO_DEPRECATE /* removes VS2005 warning on strerror() */
#endif
/*-************************************
* Dependencies
programs/dibio.c
+5 -5
View File
@@ -204,11 +204,11 @@ int DiB_trainFromFiles(const char* dictFileName, unsigned maxDictSize,
if ((!fileSizes) || (!srcBuffer) || (!dictBuffer)) EXM_THROW(12, "not enough memory for DiB_trainFiles"); /* should not happen */
g_displayLevel = params.notificationLevel;
if (nbFiles < 5) {
DISPLAYLEVEL(2, "! Warning : nb of samples too low for proper processing \n");
DISPLAYLEVEL(2, "! Please provide one file per sample \n");
DISPLAYLEVEL(2, "! Avoid concatenating multiple samples into a single file \n");
DISPLAYLEVEL(2, "! otherwise, dictBuilder will be unable to find the beginning of each sample \n");
DISPLAYLEVEL(2, "! resulting in distorted statistics \n");
DISPLAYLEVEL(2, "! Warning : nb of samples too low for proper processing ! \n");
DISPLAYLEVEL(2, "! Please provide _one file per sample_. \n");
DISPLAYLEVEL(2, "! Do not concatenate samples together into a single file, \n");
DISPLAYLEVEL(2, "! as dictBuilder will be unable to find the beginning of each sample, \n");
DISPLAYLEVEL(2, "! resulting in poor dictionary quality. \n");
}
/* init */
programs/fileio.c
+49 -50
View File
@@ -22,8 +22,7 @@
- zstd homepage : http://www.zstd.net
*/
/*
Note : this is stand-alone program.
It is not part of ZSTD compression library, it is a user program of ZSTD library.
Note : this file is part of zstd command line, which is not library.
The license of ZSTD library is BSD.
The license of this file is GPLv2.
*/
@@ -41,9 +40,13 @@
/* *************************************
* Compiler Options
***************************************/
#define _POSIX_SOURCE 1 /* enable %llu on Windows */
#define _CRT_SECURE_NO_WARNINGS /* removes Visual warning on strerror() */
#ifdef _MSC_VER /* Visual */
# define _CRT_SECURE_NO_WARNINGS /* removes Visual warning on strerror() */
# pragma warning(disable : 4204) /* non-constant aggregate initializer */
#endif
#if defined(__MINGW32__) && !defined(_POSIX_SOURCE)
# define _POSIX_SOURCE 1 /* disable %llu warnings with MinGW on Windows */
#endif
/*-*************************************
* Includes
@@ -59,8 +62,6 @@
#include "fileio.h"
#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_magicNumber, ZSTD_frameHeaderSize_max */
#include "zstd.h"
#define ZBUFF_STATIC_LINKING_ONLY
#include "zbuff.h"
#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1)
# include "zstd_legacy.h" /* ZSTD_isLegacy */
@@ -173,10 +174,9 @@ static FILE* FIO_openSrcFile(const char* srcFileName)
SET_BINARY_MODE(stdin);
} else {
f = fopen(srcFileName, "rb");
if ( f==NULL ) DISPLAYLEVEL(1, "zstd: %s: %s \n", srcFileName, strerror(errno));
}
if ( f==NULL ) DISPLAYLEVEL(1, "zstd: %s: %s \n", srcFileName, strerror(errno));
return f;
}
@@ -212,9 +212,9 @@ static FILE* FIO_openDstFile(const char* dstFileName)
while ((ch!=EOF) && (ch!='\n')) ch = getchar(); /* flush rest of input line */
} } }
f = fopen( dstFileName, "wb" );
if (f==NULL) DISPLAYLEVEL(1, "zstd: %s: %s\n", dstFileName, strerror(errno));
}
if (f==NULL) DISPLAYLEVEL(1, "zstd: %s: %s\n", dstFileName, strerror(errno));
return f;
}
@@ -240,7 +240,7 @@ static size_t FIO_loadFile(void** bufferPtr, const char* fileName)
if (fileSize > MAX_DICT_SIZE) {
int seekResult;
if (fileSize > 1 GB) EXM_THROW(32, "Dictionary file %s is too large", fileName); /* avoid extreme cases */
DISPLAYLEVEL(2,"Dictionary %s is too large : using last %u bytes only \n", fileName, MAX_DICT_SIZE);
DISPLAYLEVEL(2,"Dictionary %s is too large : using last %u bytes only \n", fileName, (U32)MAX_DICT_SIZE);
seekResult = fseek(fileHandle, (long int)(fileSize-MAX_DICT_SIZE), SEEK_SET); /* use end of file */
if (seekResult != 0) EXM_THROW(33, "zstd: %s: %s", fileName, strerror(errno));
fileSize = MAX_DICT_SIZE;
@@ -265,7 +265,7 @@ typedef struct {
size_t dstBufferSize;
void* dictBuffer;
size_t dictBufferSize;
ZBUFF_CCtx* ctx;
ZSTD_CStream* cctx;
FILE* dstFile;
FILE* srcFile;
} cRess_t;
@@ -275,11 +275,11 @@ static cRess_t FIO_createCResources(const char* dictFileName)
cRess_t ress;
memset(&ress, 0, sizeof(ress));
ress.ctx = ZBUFF_createCCtx();
if (ress.ctx == NULL) EXM_THROW(30, "zstd: allocation error : can't create ZBUFF context");
ress.srcBufferSize = ZBUFF_recommendedCInSize();
ress.cctx = ZSTD_createCStream();
if (ress.cctx == NULL) EXM_THROW(30, "zstd: allocation error : can't create ZSTD_CStream");
ress.srcBufferSize = ZSTD_CStreamInSize();
ress.srcBuffer = malloc(ress.srcBufferSize);
ress.dstBufferSize = ZBUFF_recommendedCOutSize();
ress.dstBufferSize = ZSTD_CStreamOutSize();
ress.dstBuffer = malloc(ress.dstBufferSize);
if (!ress.srcBuffer || !ress.dstBuffer) EXM_THROW(31, "zstd: allocation error : not enough memory");
@@ -295,8 +295,8 @@ static void FIO_freeCResources(cRess_t ress)
free(ress.srcBuffer);
free(ress.dstBuffer);
free(ress.dictBuffer);
errorCode = ZBUFF_freeCCtx(ress.ctx);
if (ZBUFF_isError(errorCode)) EXM_THROW(38, "zstd: error : can't release ZBUFF context resource : %s", ZBUFF_getErrorName(errorCode));
errorCode = ZSTD_freeCStream(ress.cctx);
if (ZSTD_isError(errorCode)) EXM_THROW(38, "zstd: error : can't release ZSTD_CStream : %s", ZSTD_getErrorName(errorCode));
}
@@ -324,8 +324,8 @@ static int FIO_compressFilename_internal(cRess_t ress,
params.cParams.windowLog = g_maxWLog;
params.cParams = ZSTD_adjustCParams(params.cParams, fileSize, ress.dictBufferSize);
}
{ size_t const errorCode = ZBUFF_compressInit_advanced(ress.ctx, ress.dictBuffer, ress.dictBufferSize, params, fileSize);
if (ZBUFF_isError(errorCode)) EXM_THROW(21, "Error initializing compression : %s", ZBUFF_getErrorName(errorCode));
{ size_t const errorCode = ZSTD_initCStream_advanced(ress.cctx, ress.dictBuffer, ress.dictBufferSize, params, fileSize);
if (ZSTD_isError(errorCode)) EXM_THROW(21, "Error initializing compression : %s", ZSTD_getErrorName(errorCode));
} }
/* Main compression loop */
@@ -337,36 +337,35 @@ static int FIO_compressFilename_internal(cRess_t ress,
DISPLAYUPDATE(2, "\rRead : %u MB ", (U32)(readsize>>20));
/* Compress using buffered streaming */
{ size_t usedInSize = inSize;
size_t cSize = ress.dstBufferSize;
{ size_t const result = ZBUFF_compressContinue(ress.ctx, ress.dstBuffer, &cSize, ress.srcBuffer, &usedInSize);
if (ZBUFF_isError(result)) EXM_THROW(23, "Compression error : %s ", ZBUFF_getErrorName(result)); }
if (inSize != usedInSize)
{ ZSTD_inBuffer inBuff = { ress.srcBuffer, inSize, 0 };
ZSTD_outBuffer outBuff= { ress.dstBuffer, ress.dstBufferSize, 0 };
{ size_t const result = ZSTD_compressStream(ress.cctx, &outBuff, &inBuff);
if (ZSTD_isError(result)) EXM_THROW(23, "Compression error : %s ", ZSTD_getErrorName(result)); }
if (inBuff.pos != inBuff.size)
/* inBuff should be entirely consumed since buffer sizes are recommended ones */
EXM_THROW(24, "Compression error : input block not fully consumed");
/* Write cBlock */
{ size_t const sizeCheck = fwrite(ress.dstBuffer, 1, cSize, dstFile);
if (sizeCheck!=cSize) EXM_THROW(25, "Write error : cannot write compressed block into %s", dstFileName); }
compressedfilesize += cSize;
{ size_t const sizeCheck = fwrite(ress.dstBuffer, 1, outBuff.pos, dstFile);
if (sizeCheck!=outBuff.pos) EXM_THROW(25, "Write error : cannot write compressed block into %s", dstFileName); }
compressedfilesize += outBuff.pos;
}
DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%% ", (U32)(readsize>>20), (double)compressedfilesize/readsize*100);
}
/* End of Frame */
{ size_t cSize = ress.dstBufferSize;
size_t const result = ZBUFF_compressEnd(ress.ctx, ress.dstBuffer, &cSize);
{ ZSTD_outBuffer outBuff = { ress.dstBuffer, ress.dstBufferSize, 0 };
size_t const result = ZSTD_endStream(ress.cctx, &outBuff);
if (result!=0) EXM_THROW(26, "Compression error : cannot create frame end");
{ size_t const sizeCheck = fwrite(ress.dstBuffer, 1, cSize, dstFile);
if (sizeCheck!=cSize) EXM_THROW(27, "Write error : cannot write frame end into %s", dstFileName); }
compressedfilesize += cSize;
{ size_t const sizeCheck = fwrite(ress.dstBuffer, 1, outBuff.pos, dstFile);
if (sizeCheck!=outBuff.pos) EXM_THROW(27, "Write error : cannot write frame end into %s", dstFileName); }
compressedfilesize += outBuff.pos;
}
/* Status */
if (strlen(srcFileName) > 20) srcFileName += strlen(srcFileName)-20; /* display last 20 characters */
DISPLAYLEVEL(2, "\r%79s\r", "");
DISPLAYLEVEL(2,"%-20.20s :%6.2f%% (%6llu => %6llu bytes, %s) \n", srcFileName,
DISPLAYLEVEL(2,"%-20s :%6.2f%% (%6llu => %6llu bytes, %s) \n", srcFileName,
(double)compressedfilesize/(readsize+(!readsize) /* avoid div by zero */ )*100,
(unsigned long long)readsize, (unsigned long long) compressedfilesize,
dstFileName);
@@ -496,7 +495,7 @@ typedef struct {
size_t dstBufferSize;
void* dictBuffer;
size_t dictBufferSize;
ZBUFF_DCtx* dctx;
ZSTD_DStream* dctx;
FILE* dstFile;
} dRess_t;
@@ -506,11 +505,11 @@ static dRess_t FIO_createDResources(const char* dictFileName)
memset(&ress, 0, sizeof(ress));
/* Allocation */
ress.dctx = ZBUFF_createDCtx();
if (ress.dctx==NULL) EXM_THROW(60, "Can't create ZBUFF decompression context");
ress.srcBufferSize = ZBUFF_recommendedDInSize();
ress.dctx = ZSTD_createDStream();
if (ress.dctx==NULL) EXM_THROW(60, "Can't create ZSTD_DStream");
ress.srcBufferSize = ZSTD_DStreamInSize();
ress.srcBuffer = malloc(ress.srcBufferSize);
ress.dstBufferSize = ZBUFF_recommendedDOutSize();
ress.dstBufferSize = ZSTD_DStreamOutSize();
ress.dstBuffer = malloc(ress.dstBufferSize);
if (!ress.srcBuffer || !ress.dstBuffer) EXM_THROW(61, "Allocation error : not enough memory");
@@ -522,8 +521,8 @@ static dRess_t FIO_createDResources(const char* dictFileName)
static void FIO_freeDResources(dRess_t ress)
{
size_t const errorCode = ZBUFF_freeDCtx(ress.dctx);
if (ZBUFF_isError(errorCode)) EXM_THROW(69, "Error : can't free ZBUFF context resource : %s", ZBUFF_getErrorName(errorCode));
size_t const errorCode = ZSTD_freeDStream(ress.dctx);
if (ZSTD_isError(errorCode)) EXM_THROW(69, "Error : can't free ZSTD_DStream context resource : %s", ZSTD_getErrorName(errorCode));
free(ress.srcBuffer);
free(ress.dstBuffer);
free(ress.dictBuffer);
@@ -615,7 +614,7 @@ unsigned long long FIO_decompressFrame(dRess_t ress,
size_t readSize;
U32 storedSkips = 0;
ZBUFF_decompressInitDictionary(ress.dctx, ress.dictBuffer, ress.dictBufferSize);
ZSTD_initDStream_usingDict(ress.dctx, ress.dictBuffer, ress.dictBufferSize);
/* Header loading (optional, saves one loop) */
{ size_t const toLoad = 9 - alreadyLoaded; /* assumption : 9 >= alreadyLoaded */
@@ -625,18 +624,18 @@ unsigned long long FIO_decompressFrame(dRess_t ress,
/* Main decompression Loop */
while (1) {
size_t inSize=readSize, decodedSize=ress.dstBufferSize;
size_t const toRead = ZBUFF_decompressContinue(ress.dctx, ress.dstBuffer, &decodedSize, ress.srcBuffer, &inSize);
if (ZBUFF_isError(toRead)) EXM_THROW(36, "Decoding error : %s", ZBUFF_getErrorName(toRead));
readSize -= inSize;
ZSTD_inBuffer inBuff = { ress.srcBuffer, readSize, 0 };
ZSTD_outBuffer outBuff= { ress.dstBuffer, ress.dstBufferSize, 0 };
size_t const toRead = ZSTD_decompressStream(ress.dctx, &outBuff, &inBuff );
if (ZSTD_isError(toRead)) EXM_THROW(36, "Decoding error : %s", ZSTD_getErrorName(toRead));
/* Write block */
storedSkips = FIO_fwriteSparse(foutput, ress.dstBuffer, decodedSize, storedSkips);
frameSize += decodedSize;
storedSkips = FIO_fwriteSparse(foutput, ress.dstBuffer, outBuff.pos, storedSkips);
frameSize += outBuff.pos;
DISPLAYUPDATE(2, "\rDecoded : %u MB... ", (U32)(frameSize>>20) );
if (toRead == 0) break; /* end of frame */
if (readSize) EXM_THROW(37, "Decoding error : should consume entire input");
if (inBuff.size != inBuff.pos) EXM_THROW(37, "Decoding error : should consume entire input");
/* Fill input buffer */
if (toRead > ress.srcBufferSize) EXM_THROW(38, "too large block");
programs/fuzzer.c
+10
View File
@@ -132,6 +132,16 @@ static int basicUnitTests(U32 seed, double compressibility)
RDG_genBuffer(CNBuffer, CNBuffSize, compressibility, 0., seed);
/* Basic tests */
DISPLAYLEVEL(4, "test%3i : ZSTD_getErrorName : ", testNb++);
{ const char* errorString = ZSTD_getErrorName(0);
DISPLAYLEVEL(4, "OK : %s \n", errorString);
}
DISPLAYLEVEL(4, "test%3i : ZSTD_getErrorName with wrong value : ", testNb++);
{ const char* errorString = ZSTD_getErrorName(499);
DISPLAYLEVEL(4, "OK : %s \n", errorString);
}
DISPLAYLEVEL(4, "test%3i : compress %u bytes : ", testNb++, (U32)CNBuffSize);
CHECKPLUS(r, ZSTD_compress(compressedBuffer, ZSTD_compressBound(CNBuffSize),
CNBuffer, CNBuffSize, 1),
programs/playTests.sh
+1 -1
View File
@@ -47,7 +47,7 @@ $ECHO "\n**** simple tests **** "
$ZSTD -f tmp # trivial compression case, creates tmp.zst
$ZSTD -df tmp.zst # trivial decompression case (overwrites tmp)
$ECHO "test : too large compression level (must fail)"
$ZSTD -99 tmp && die "too large compression level undetected"
$ZSTD -99 -f tmp # too large compression level, automatic sized down
$ECHO "test : compress to stdout"
$ZSTD tmp -c > tmpCompressed
$ZSTD tmp --stdout > tmpCompressed # long command format
programs/util.h
+5 -5
View File
@@ -31,6 +31,9 @@ extern "C" {
/* **************************************
* Compiler Options
****************************************/
#if defined(__INTEL_COMPILER)
# pragma warning(disable : 177) /* disable: message #177: function was declared but never referenced */
#endif
#if defined(_MSC_VER)
# define _CRT_SECURE_NO_WARNINGS /* Disable some Visual warning messages for fopen, strncpy */
# define _CRT_SECURE_NO_DEPRECATE /* VS2005 */
@@ -365,13 +368,10 @@ UTIL_STATIC const char** UTIL_createFileList(const char **inputNames, unsigned i
pos += len + 1;
nbFiles++;
}
}
else
{
} else {
nbFiles += UTIL_prepareFileList(inputNames[i], &buf, &pos, &bufend);
if (buf == NULL) return NULL;
}
}
} }
if (nbFiles == 0) { free(buf); return NULL; }
programs/zstd.1
+1 -1
View File
@@ -128,7 +128,7 @@ Typical gains range from ~10% (at 64KB) to x5 better (at <1KB).
benchmark file(s) using compression level #
.TP
.B \-i#
iteration loops [1-9](default : 3), benchmark mode only
minimum evaluation time, in seconds (default : 3s), benchmark mode only
.TP
.B \-B#
cut file into independent blocks of size # (default: no block)
programs/zstdcli.c
+27 -16
View File
@@ -35,6 +35,10 @@
# define ZSTDCLI_CLEVEL_DEFAULT 3
#endif
#ifndef ZSTDCLI_CLEVEL_MAX
# define ZSTDCLI_CLEVEL_MAX 19
#endif
/*-************************************
* Includes
@@ -88,6 +92,8 @@
#define MB *(1 <<20)
#define GB *(1U<<30)
#define DEFAULT_DISPLAY_LEVEL 2
static const char* g_defaultDictName = "dictionary";
static const unsigned g_defaultMaxDictSize = 110 KB;
static const int g_defaultDictCLevel = 5;
@@ -100,7 +106,7 @@ static const unsigned g_defaultSelectivityLevel = 9;
#define DISPLAY(...) fprintf(displayOut, __VA_ARGS__)
#define DISPLAYLEVEL(l, ...) if (displayLevel>=l) { DISPLAY(__VA_ARGS__); }
static FILE* displayOut;
static unsigned displayLevel = 2; /* 0 : no display, 1: errors, 2 : + result + interaction + warnings, 3 : + progression, 4 : + information */
static unsigned displayLevel = DEFAULT_DISPLAY_LEVEL; /* 0 : no display, 1: errors, 2 : + result + interaction + warnings, 3 : + progression, 4 : + information */
/*-************************************
@@ -115,7 +121,7 @@ static int usage(const char* programName)
DISPLAY( " with no FILE, or when FILE is - , read standard input\n");
DISPLAY( "Arguments :\n");
#ifndef ZSTD_NOCOMPRESS
DISPLAY( " -# : # compression level (1-%u, default:%u) \n", ZSTD_maxCLevel(), ZSTDCLI_CLEVEL_DEFAULT);
DISPLAY( " -# : # compression level (1-%d, default:%d) \n", ZSTDCLI_CLEVEL_MAX, ZSTDCLI_CLEVEL_DEFAULT);
#endif
#ifndef ZSTD_NODECOMPRESS
DISPLAY( " -d : decompression \n");
@@ -136,14 +142,14 @@ static int usage_advanced(const char* programName)
DISPLAY( "\n");
DISPLAY( "Advanced arguments :\n");
DISPLAY( " -V : display Version number and exit\n");
DISPLAY( " -v : verbose mode\n");
DISPLAY( " -v : verbose mode; specify multiple times to increase log level (default:%d)\n", DEFAULT_DISPLAY_LEVEL);
DISPLAY( " -q : suppress warnings; specify twice to suppress errors too\n");
DISPLAY( " -c : force write to standard output, even if it is the console\n");
#ifdef UTIL_HAS_CREATEFILELIST
DISPLAY( " -r : operate recursively on directories\n");
#endif
#ifndef ZSTD_NOCOMPRESS
DISPLAY( "--ultra : enable ultra modes (requires more memory to decompress)\n");
DISPLAY( "--ultra : enable levels beyond %i, up to %i (requires more memory)\n", ZSTDCLI_CLEVEL_MAX, ZSTD_maxCLevel());
DISPLAY( "--no-dictID : don't write dictID into header (dictionary compression)\n");
DISPLAY( "--[no-]check : integrity check (default:enabled)\n");
#endif
@@ -165,7 +171,7 @@ static int usage_advanced(const char* programName)
DISPLAY( "Benchmark arguments :\n");
DISPLAY( " -b# : benchmark file(s), using # compression level (default : 1) \n");
DISPLAY( " -e# : test all compression levels from -bX to # (default: 1)\n");
DISPLAY( " -i# : iteration loops [1-9](default : 3)\n");
DISPLAY( " -i# : minimum evaluation time in seconds (default : 3s)\n");
DISPLAY( " -B# : cut file into independent blocks of size # (default: no block)\n");
#endif
return 0;
@@ -201,7 +207,7 @@ static unsigned readU32FromChar(const char** stringPtr)
#define CLEAN_RETURN(i) { operationResult = (i); goto _end; }
int main(int argCount, const char** argv)
int main(int argCount, char** argv)
{
int argNb,
bench=0,
@@ -215,7 +221,8 @@ int main(int argCount, const char** argv)
nextArgumentIsOutFileName=0,
nextArgumentIsMaxDict=0,
nextArgumentIsDictID=0,
nextArgumentIsFile=0;
nextArgumentIsFile=0,
ultra=0;
int cLevel = ZSTDCLI_CLEVEL_DEFAULT;
int cLevelLast = 1;
unsigned recursive = 0;
@@ -268,7 +275,7 @@ int main(int argCount, const char** argv)
if (!strcmp(argument, "--verbose")) { displayLevel++; continue; }
if (!strcmp(argument, "--quiet")) { displayLevel--; continue; }
if (!strcmp(argument, "--stdout")) { forceStdout=1; outFileName=stdoutmark; displayLevel-=(displayLevel==2); continue; }
if (!strcmp(argument, "--ultra")) { FIO_setMaxWLog(0); continue; }
if (!strcmp(argument, "--ultra")) { ultra=1; FIO_setMaxWLog(0); continue; }
if (!strcmp(argument, "--check")) { FIO_setChecksumFlag(2); continue; }
if (!strcmp(argument, "--no-check")) { FIO_setChecksumFlag(0); continue; }
if (!strcmp(argument, "--no-dictID")) { FIO_setDictIDFlag(0); continue; }
@@ -298,10 +305,7 @@ int main(int argCount, const char** argv)
#ifndef ZSTD_NOCOMPRESS
/* compression Level */
if ((*argument>='0') && (*argument<='9')) {
cLevel = readU32FromChar(&argument);
dictCLevel = cLevel;
if (dictCLevel > ZSTD_maxCLevel())
CLEAN_RETURN(badusage(programName));
dictCLevel = cLevel = readU32FromChar(&argument);
continue;
}
#endif
@@ -317,7 +321,7 @@ int main(int argCount, const char** argv)
case 'd': decode=1; argument++; break;
/* Force stdout, even if stdout==console */
case 'c': forceStdout=1; outFileName=stdoutmark; displayLevel-=(displayLevel==2); argument++; break;
case 'c': forceStdout=1; outFileName=stdoutmark; argument++; break;
/* Use file content as dictionary */
case 'D': nextEntryIsDictionary = 1; argument++; break;
@@ -441,8 +445,8 @@ int main(int argCount, const char** argv)
if (recursive) {
fileNamesTable = UTIL_createFileList(filenameTable, filenameIdx, &fileNamesBuf, &fileNamesNb);
if (fileNamesTable) {
unsigned i;
for (i=0; i<fileNamesNb; i++) DISPLAYLEVEL(4, "%d %s\n", i, fileNamesTable[i]);
unsigned u;
for (u=0; u<fileNamesNb; u++) DISPLAYLEVEL(4, "%u %s\n", u, fileNamesTable[u]);
free((void*)filenameTable);
filenameTable = fileNamesTable;
filenameIdx = fileNamesNb;
@@ -488,7 +492,14 @@ int main(int argCount, const char** argv)
CLEAN_RETURN(filenameIdx);
}
/* No warning message in pipe mode (stdin + stdout) or multiple mode */
/* check compression level limits */
{ int const maxCLevel = ultra ? ZSTD_maxCLevel() : ZSTDCLI_CLEVEL_MAX;
if (cLevel > maxCLevel) {
DISPLAYLEVEL(2, "Warning : compression level higher than max, reduced to %i \n", maxCLevel);
cLevel = maxCLevel;
} }
/* No warning message in pipe mode (stdin + stdout) or multi-files mode */
if (!strcmp(filenameTable[0], stdinmark) && outFileName && !strcmp(outFileName,stdoutmark) && (displayLevel==2)) displayLevel=1;
if ((filenameIdx>1) & (displayLevel==2)) displayLevel=1;
programs/zstreamtest.c
+659
View File
@@ -0,0 +1,659 @@
/*
Fuzzer test tool for zstd streaming API
Copyright (C) Yann Collet 2016
GPL v2 License
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 2 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License along
with this program; if not, write to the Free Software Foundation, Inc.,
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
You can contact the author at :
- ZSTD homepage : https://www.zstd.net/
*/
/*-************************************
* Compiler specific
**************************************/
#ifdef _MSC_VER /* Visual Studio */
# define _CRT_SECURE_NO_WARNINGS /* fgets */
# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
# pragma warning(disable : 4146) /* disable: C4146: minus unsigned expression */
#endif
/*-************************************
* Includes
**************************************/
#include <stdlib.h> /* free */
#include <stdio.h> /* fgets, sscanf */
#include <sys/timeb.h> /* timeb */
#include <string.h> /* strcmp */
#include "mem.h"
#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_maxCLevel */
#include "zstd.h" /* ZSTD_compressBound */
#include "datagen.h" /* RDG_genBuffer */
#define XXH_STATIC_LINKING_ONLY
#include "xxhash.h" /* XXH64_* */
/*-************************************
* Constants
**************************************/
#define KB *(1U<<10)
#define MB *(1U<<20)
#define GB *(1U<<30)
static const U32 nbTestsDefault = 10000;
#define COMPRESSIBLE_NOISE_LENGTH (10 MB)
#define FUZ_COMPRESSIBILITY_DEFAULT 50
static const U32 prime1 = 2654435761U;
static const U32 prime2 = 2246822519U;
/*-************************************
* Display Macros
**************************************/
#define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
#define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); }
static U32 g_displayLevel = 2;
#define DISPLAYUPDATE(l, ...) if (g_displayLevel>=l) { \
if ((FUZ_GetMilliSpan(g_displayTime) > g_refreshRate) || (g_displayLevel>=4)) \
{ g_displayTime = FUZ_GetMilliStart(); DISPLAY(__VA_ARGS__); \
if (g_displayLevel>=4) fflush(stdout); } }
static const U32 g_refreshRate = 150;
static U32 g_displayTime = 0;
static U32 g_testTime = 0;
/*-*******************************************************
* Fuzzer functions
*********************************************************/
#define MAX(a,b) ((a)>(b)?(a):(b))
static U32 FUZ_GetMilliStart(void)
{
struct timeb tb;
U32 nCount;
ftime( &tb );
nCount = (U32) (((tb.time & 0xFFFFF) * 1000) + tb.millitm);
return nCount;
}
static U32 FUZ_GetMilliSpan(U32 nTimeStart)
{
U32 const nCurrent = FUZ_GetMilliStart();
U32 nSpan = nCurrent - nTimeStart;
if (nTimeStart > nCurrent)
nSpan += 0x100000 * 1000;
return nSpan;
}
/*! FUZ_rand() :
@return : a 27 bits random value, from a 32-bits `seed`.
`seed` is also modified */
# define FUZ_rotl32(x,r) ((x << r) | (x >> (32 - r)))
unsigned int FUZ_rand(unsigned int* seedPtr)
{
U32 rand32 = *seedPtr;
rand32 *= prime1;
rand32 += prime2;
rand32 = FUZ_rotl32(rand32, 13);
*seedPtr = rand32;
return rand32 >> 5;
}
/*
static unsigned FUZ_highbit32(U32 v32)
{
unsigned nbBits = 0;
if (v32==0) return 0;
for ( ; v32 ; v32>>=1) nbBits++;
return nbBits;
}
*/
static void* allocFunction(void* opaque, size_t size)
{
void* address = malloc(size);
(void)opaque;
return address;
}
static void freeFunction(void* opaque, void* address)
{
(void)opaque;
free(address);
}
static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem customMem)
{
int testResult = 0;
size_t CNBufferSize = COMPRESSIBLE_NOISE_LENGTH;
void* CNBuffer = malloc(CNBufferSize);
size_t const skippableFrameSize = 11;
size_t const compressedBufferSize = (8 + skippableFrameSize) + ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH);
void* compressedBuffer = malloc(compressedBufferSize);
size_t const decodedBufferSize = CNBufferSize;
void* decodedBuffer = malloc(decodedBufferSize);
size_t cSize;
U32 testNb=0;
ZSTD_CStream* zc = ZSTD_createCStream_advanced(customMem);
ZSTD_DStream* zd = ZSTD_createDStream_advanced(customMem);
ZSTD_inBuffer inBuff;
ZSTD_outBuffer outBuff;
/* Create compressible test buffer */
if (!CNBuffer || !compressedBuffer || !decodedBuffer || !zc || !zd) {
DISPLAY("Not enough memory, aborting\n");
goto _output_error;
}
RDG_genBuffer(CNBuffer, CNBufferSize, compressibility, 0., seed);
/* generate skippable frame */
MEM_writeLE32(compressedBuffer, ZSTD_MAGIC_SKIPPABLE_START);
MEM_writeLE32(((char*)compressedBuffer)+4, (U32)skippableFrameSize);
cSize = skippableFrameSize + 8;
/* Basic compression test */
DISPLAYLEVEL(4, "test%3i : compress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH);
ZSTD_initCStream_usingDict(zc, CNBuffer, 128 KB, 1);
outBuff.dst = (char*)(compressedBuffer)+cSize;
outBuff.size = compressedBufferSize;
outBuff.pos = 0;
inBuff.src = CNBuffer;
inBuff.size = CNBufferSize;
inBuff.pos = 0;
{ size_t const r = ZSTD_compressStream(zc, &outBuff, &inBuff);
if (ZSTD_isError(r)) goto _output_error; }
if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */
{ size_t const r = ZSTD_endStream(zc, &outBuff);
if (r != 0) goto _output_error; } /*< error, or some data not flushed */
cSize += outBuff.pos;
DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/COMPRESSIBLE_NOISE_LENGTH*100);
/* skippable frame test */
DISPLAYLEVEL(4, "test%3i : decompress skippable frame : ", testNb++);
ZSTD_initDStream_usingDict(zd, CNBuffer, 128 KB);
inBuff.src = compressedBuffer;
inBuff.size = cSize;
inBuff.pos = 0;
outBuff.dst = decodedBuffer;
outBuff.size = CNBufferSize;
outBuff.pos = 0;
{ size_t const r = ZSTD_decompressStream(zd, &outBuff, &inBuff);
if (r != 0) goto _output_error; }
if (outBuff.pos != 0) goto _output_error; /* skippable frame len is 0 */
DISPLAYLEVEL(4, "OK \n");
/* Basic decompression test */
DISPLAYLEVEL(4, "test%3i : decompress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH);
ZSTD_initDStream_usingDict(zd, CNBuffer, 128 KB);
{ size_t const r = ZSTD_decompressStream(zd, &outBuff, &inBuff);
if (r != 0) goto _output_error; } /* should reach end of frame == 0; otherwise, some data left, or an error */
if (outBuff.pos != CNBufferSize) goto _output_error; /* should regenerate the same amount */
if (inBuff.pos != inBuff.size) goto _output_error; /* should have read the entire frame */
DISPLAYLEVEL(4, "OK \n");
/* check regenerated data is byte exact */
DISPLAYLEVEL(4, "test%3i : check decompressed result : ", testNb++);
{ size_t i;
for (i=0; i<CNBufferSize; i++) {
if (((BYTE*)decodedBuffer)[i] != ((BYTE*)CNBuffer)[i]) goto _output_error;
} }
DISPLAYLEVEL(4, "OK \n");
/* Byte-by-byte decompression test */
DISPLAYLEVEL(4, "test%3i : decompress byte-by-byte : ", testNb++);
{ size_t r = 1;
ZSTD_initDStream_usingDict(zd, CNBuffer, 128 KB);
inBuff.src = compressedBuffer;
outBuff.dst = decodedBuffer;
inBuff.pos = 0;
outBuff.pos = 0;
while (r) { /* skippable frame */
inBuff.size = inBuff.pos + 1;
outBuff.size = outBuff.pos + 1;
r = ZSTD_decompressStream(zd, &outBuff, &inBuff);
if (ZSTD_isError(r)) goto _output_error;
}
ZSTD_initDStream_usingDict(zd, CNBuffer, 128 KB);
r=1;
while (r) { /* normal frame */
inBuff.size = inBuff.pos + 1;
outBuff.size = outBuff.pos + 1;
r = ZSTD_decompressStream(zd, &outBuff, &inBuff);
if (ZSTD_isError(r)) goto _output_error;
}
}
if (outBuff.pos != CNBufferSize) goto _output_error; /* should regenerate the same amount */
if (inBuff.pos != cSize) goto _output_error; /* should have read the entire frame */
DISPLAYLEVEL(4, "OK \n");
/* check regenerated data is byte exact */
DISPLAYLEVEL(4, "test%3i : check decompressed result : ", testNb++);
{ size_t i;
for (i=0; i<CNBufferSize; i++) {
if (((BYTE*)decodedBuffer)[i] != ((BYTE*)CNBuffer)[i]) goto _output_error;;
} }
DISPLAYLEVEL(4, "OK \n");
_end:
ZSTD_freeCStream(zc);
ZSTD_freeDStream(zd);
free(CNBuffer);
free(compressedBuffer);
free(decodedBuffer);
return testResult;
_output_error:
testResult = 1;
DISPLAY("Error detected in Unit tests ! \n");
goto _end;
}
static size_t findDiff(const void* buf1, const void* buf2, size_t max)
{
const BYTE* b1 = (const BYTE*)buf1;
const BYTE* b2 = (const BYTE*)buf2;
size_t u;
for (u=0; u<max; u++) {
if (b1[u] != b2[u]) break;
}
return u;
}
static size_t FUZ_rLogLength(U32* seed, U32 logLength)
{
size_t const lengthMask = ((size_t)1 << logLength) - 1;
return (lengthMask+1) + (FUZ_rand(seed) & lengthMask);
}
static size_t FUZ_randomLength(U32* seed, U32 maxLog)
{
U32 const logLength = FUZ_rand(seed) % maxLog;
return FUZ_rLogLength(seed, logLength);
}
#define MIN(a,b) ( (a) < (b) ? (a) : (b) )
#define CHECK(cond, ...) if (cond) { DISPLAY("Error => "); DISPLAY(__VA_ARGS__); \
DISPLAY(" (seed %u, test nb %u) \n", seed, testNb); goto _output_error; }
static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibility)
{
static const U32 maxSrcLog = 24;
static const U32 maxSampleLog = 19;
BYTE* cNoiseBuffer[5];
size_t srcBufferSize = (size_t)1<<maxSrcLog;
BYTE* copyBuffer;
size_t copyBufferSize= srcBufferSize + (1<<maxSampleLog);
BYTE* cBuffer;
size_t cBufferSize = ZSTD_compressBound(srcBufferSize);
BYTE* dstBuffer;
size_t dstBufferSize = srcBufferSize;
U32 result = 0;
U32 testNb = 0;
U32 coreSeed = seed;
ZSTD_CStream* zc;
ZSTD_DStream* zd;
U32 startTime = FUZ_GetMilliStart();
/* allocations */
zc = ZSTD_createCStream();
zd = ZSTD_createDStream();
cNoiseBuffer[0] = (BYTE*)malloc (srcBufferSize);
cNoiseBuffer[1] = (BYTE*)malloc (srcBufferSize);
cNoiseBuffer[2] = (BYTE*)malloc (srcBufferSize);
cNoiseBuffer[3] = (BYTE*)malloc (srcBufferSize);
cNoiseBuffer[4] = (BYTE*)malloc (srcBufferSize);
copyBuffer= (BYTE*)malloc (copyBufferSize);
dstBuffer = (BYTE*)malloc (dstBufferSize);
cBuffer = (BYTE*)malloc (cBufferSize);
CHECK (!cNoiseBuffer[0] || !cNoiseBuffer[1] || !cNoiseBuffer[2] || !cNoiseBuffer[3] || !cNoiseBuffer[4] ||
!copyBuffer || !dstBuffer || !cBuffer || !zc || !zd,
"Not enough memory, fuzzer tests cancelled");
/* Create initial samples */
RDG_genBuffer(cNoiseBuffer[0], srcBufferSize, 0.00, 0., coreSeed); /* pure noise */
RDG_genBuffer(cNoiseBuffer[1], srcBufferSize, 0.05, 0., coreSeed); /* barely compressible */
RDG_genBuffer(cNoiseBuffer[2], srcBufferSize, compressibility, 0., coreSeed);
RDG_genBuffer(cNoiseBuffer[3], srcBufferSize, 0.95, 0., coreSeed); /* highly compressible */
RDG_genBuffer(cNoiseBuffer[4], srcBufferSize, 1.00, 0., coreSeed); /* sparse content */
memset(copyBuffer, 0x65, copyBufferSize); /* make copyBuffer considered initialized */
/* catch up testNb */
for (testNb=1; testNb < startTest; testNb++)
FUZ_rand(&coreSeed);
/* test loop */
for ( ; (testNb <= nbTests) || (FUZ_GetMilliSpan(startTime) < g_testTime) ; testNb++ ) {
U32 lseed;
const BYTE* srcBuffer;
const BYTE* dict;
size_t maxTestSize, dictSize;
size_t cSize, totalTestSize, totalGenSize;
U32 n, nbChunks;
XXH64_state_t xxhState;
U64 crcOrig;
/* init */
DISPLAYUPDATE(2, "\r%6u", testNb);
if (nbTests >= testNb) DISPLAYUPDATE(2, "/%6u ", nbTests);
FUZ_rand(&coreSeed);
lseed = coreSeed ^ prime1;
/* states full reset (unsynchronized) */
/* some issues only happen when reusing states in a specific sequence of parameters */
if ((FUZ_rand(&lseed) & 0xFF) == 131) { ZSTD_freeCStream(zc); zc = ZSTD_createCStream(); }
if ((FUZ_rand(&lseed) & 0xFF) == 132) { ZSTD_freeDStream(zd); zd = ZSTD_createDStream(); }
/* srcBuffer selection [0-4] */
{ U32 buffNb = FUZ_rand(&lseed) & 0x7F;
if (buffNb & 7) buffNb=2; /* most common : compressible (P) */
else {
buffNb >>= 3;
if (buffNb & 7) {
const U32 tnb[2] = { 1, 3 }; /* barely/highly compressible */
buffNb = tnb[buffNb >> 3];
} else {
const U32 tnb[2] = { 0, 4 }; /* not compressible / sparse */
buffNb = tnb[buffNb >> 3];
} }
srcBuffer = cNoiseBuffer[buffNb];
}
/* compression init */
{ U32 const testLog = FUZ_rand(&lseed) % maxSrcLog;
U32 const cLevel = (FUZ_rand(&lseed) % (ZSTD_maxCLevel() - (testLog/3))) + 1;
maxTestSize = FUZ_rLogLength(&lseed, testLog);
dictSize = (FUZ_rand(&lseed)==1) ? FUZ_randomLength(&lseed, maxSampleLog) : 0;
/* random dictionary selection */
{ size_t const dictStart = FUZ_rand(&lseed) % (srcBufferSize - dictSize);
dict = srcBuffer + dictStart;
}
{ ZSTD_parameters params = ZSTD_getParams(cLevel, 0, dictSize);
params.fParams.checksumFlag = FUZ_rand(&lseed) & 1;
params.fParams.noDictIDFlag = FUZ_rand(&lseed) & 1;
{ size_t const initError = ZSTD_initCStream_advanced(zc, dict, dictSize, params, 0);
CHECK (ZSTD_isError(initError),"init error : %s", ZSTD_getErrorName(initError));
} } }
/* multi-segments compression test */
XXH64_reset(&xxhState, 0);
nbChunks = (FUZ_rand(&lseed) & 127) + 2;
{ ZSTD_outBuffer outBuff = { cBuffer, cBufferSize, 0 } ;
for (n=0, cSize=0, totalTestSize=0 ; (n<nbChunks) && (totalTestSize < maxTestSize) ; n++) {
/* compress random chunk into random size dst buffer */
{ size_t const readChunkSize = FUZ_randomLength(&lseed, maxSampleLog);
size_t const srcStart = FUZ_rand(&lseed) % (srcBufferSize - readChunkSize);
size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog);
size_t const dstBuffSize = MIN(cBufferSize - cSize, randomDstSize);
ZSTD_inBuffer inBuff = { srcBuffer+srcStart, readChunkSize, 0 };
outBuff.size = outBuff.pos + dstBuffSize;
{ size_t const compressionError = ZSTD_compressStream(zc, &outBuff, &inBuff);
CHECK (ZSTD_isError(compressionError), "compression error : %s", ZSTD_getErrorName(compressionError)); }
XXH64_update(&xxhState, srcBuffer+srcStart, inBuff.pos);
memcpy(copyBuffer+totalTestSize, srcBuffer+srcStart, inBuff.pos);
totalTestSize += inBuff.pos;
}
/* random flush operation, to mess around */
if ((FUZ_rand(&lseed) & 15) == 0) {
size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog);
size_t const adjustedDstSize = MIN(cBufferSize - cSize, randomDstSize);
outBuff.size = outBuff.pos + adjustedDstSize;
{ size_t const flushError = ZSTD_flushStream(zc, &outBuff);
CHECK (ZSTD_isError(flushError), "flush error : %s", ZSTD_getErrorName(flushError));
} } }
/* final frame epilogue */
{ size_t remainingToFlush = (size_t)(-1);
while (remainingToFlush) {
size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog);
size_t const adjustedDstSize = MIN(cBufferSize - cSize, randomDstSize);
U32 const enoughDstSize = (adjustedDstSize >= remainingToFlush);
outBuff.size = outBuff.pos + adjustedDstSize;
remainingToFlush = ZSTD_endStream(zc, &outBuff);
CHECK (ZSTD_isError(remainingToFlush), "flush error : %s", ZSTD_getErrorName(remainingToFlush));
CHECK (enoughDstSize && remainingToFlush, "ZSTD_endStream() not fully flushed (%u remaining), but enough space available", (U32)remainingToFlush);
} }
crcOrig = XXH64_digest(&xxhState);
cSize = outBuff.pos;
}
/* multi - fragments decompression test */
ZSTD_initDStream_usingDict(zd, dict, dictSize);
{ size_t decompressionResult = 1;
ZSTD_inBuffer inBuff = { cBuffer, cSize, 0 };
ZSTD_outBuffer outBuff= { dstBuffer, dstBufferSize, 0 };
for (totalGenSize = 0 ; decompressionResult ; ) {
size_t const readCSrcSize = FUZ_randomLength(&lseed, maxSampleLog);
size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog);
size_t const dstBuffSize = MIN(dstBufferSize - totalGenSize, randomDstSize);
inBuff.size = inBuff.pos + readCSrcSize;
outBuff.size = inBuff.pos + dstBuffSize;
decompressionResult = ZSTD_decompressStream(zd, &outBuff, &inBuff);
CHECK (ZSTD_isError(decompressionResult), "decompression error : %s", ZSTD_getErrorName(decompressionResult));
}
CHECK (decompressionResult != 0, "frame not fully decoded");
CHECK (outBuff.pos != totalTestSize, "decompressed data : wrong size")
CHECK (inBuff.pos != cSize, "compressed data should be fully read")
{ U64 const crcDest = XXH64(dstBuffer, totalTestSize, 0);
if (crcDest!=crcOrig) findDiff(copyBuffer, dstBuffer, totalTestSize);
CHECK (crcDest!=crcOrig, "decompressed data corrupted");
} }
/*===== noisy/erroneous src decompression test =====*/
/* add some noise */
{ U32 const nbNoiseChunks = (FUZ_rand(&lseed) & 7) + 2;
U32 nn; for (nn=0; nn<nbNoiseChunks; nn++) {
size_t const randomNoiseSize = FUZ_randomLength(&lseed, maxSampleLog);
size_t const noiseSize = MIN((cSize/3) , randomNoiseSize);
size_t const noiseStart = FUZ_rand(&lseed) % (srcBufferSize - noiseSize);
size_t const cStart = FUZ_rand(&lseed) % (cSize - noiseSize);
memcpy(cBuffer+cStart, srcBuffer+noiseStart, noiseSize);
} }
/* try decompression on noisy data */
ZSTD_initDStream(zd);
{ ZSTD_inBuffer inBuff = { cBuffer, cSize, 0 };
ZSTD_outBuffer outBuff= { dstBuffer, dstBufferSize, 0 };
while (outBuff.pos < dstBufferSize) {
size_t const randomCSrcSize = FUZ_randomLength(&lseed, maxSampleLog);
size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog);
size_t const adjustedDstSize = MIN(dstBufferSize - outBuff.pos, randomDstSize);
outBuff.size = outBuff.pos + adjustedDstSize;
inBuff.size = inBuff.pos + randomCSrcSize;
{ size_t const decompressError = ZSTD_decompressStream(zd, &outBuff, &inBuff);
if (ZSTD_isError(decompressError)) break; /* error correctly detected */
} } } }
DISPLAY("\r%u fuzzer tests completed \n", testNb);
_cleanup:
ZSTD_freeCStream(zc);
ZSTD_freeDStream(zd);
free(cNoiseBuffer[0]);
free(cNoiseBuffer[1]);
free(cNoiseBuffer[2]);
free(cNoiseBuffer[3]);
free(cNoiseBuffer[4]);
free(copyBuffer);
free(cBuffer);
free(dstBuffer);
return result;
_output_error:
result = 1;
goto _cleanup;
}
/*-*******************************************************
* Command line
*********************************************************/
int FUZ_usage(const char* programName)
{
DISPLAY( "Usage :\n");
DISPLAY( " %s [args]\n", programName);
DISPLAY( "\n");
DISPLAY( "Arguments :\n");
DISPLAY( " -i# : Nb of tests (default:%u) \n", nbTestsDefault);
DISPLAY( " -s# : Select seed (default:prompt user)\n");
DISPLAY( " -t# : Select starting test number (default:0)\n");
DISPLAY( " -P# : Select compressibility in %% (default:%i%%)\n", FUZ_COMPRESSIBILITY_DEFAULT);
DISPLAY( " -v : verbose\n");
DISPLAY( " -p : pause at the end\n");
DISPLAY( " -h : display help and exit\n");
return 0;
}
int main(int argc, const char** argv)
{
U32 seed=0;
int seedset=0;
int argNb;
int nbTests = nbTestsDefault;
int testNb = 0;
int proba = FUZ_COMPRESSIBILITY_DEFAULT;
int result=0;
U32 mainPause = 0;
const char* programName = argv[0];
ZSTD_customMem customMem = { allocFunction, freeFunction, NULL };
ZSTD_customMem customNULL = { NULL, NULL, NULL };
/* Check command line */
for(argNb=1; argNb<argc; argNb++) {
const char* argument = argv[argNb];
if(!argument) continue; /* Protection if argument empty */
/* Parsing commands. Aggregated commands are allowed */
if (argument[0]=='-') {
argument++;
while (*argument!=0) {
switch(*argument)
{
case 'h':
return FUZ_usage(programName);
case 'v':
argument++;
g_displayLevel=4;
break;
case 'q':
argument++;
g_displayLevel--;
break;
case 'p': /* pause at the end */
argument++;
mainPause = 1;
break;
case 'i':
argument++;
nbTests=0; g_testTime=0;
while ((*argument>='0') && (*argument<='9')) {
nbTests *= 10;
nbTests += *argument - '0';
argument++;
}
break;
case 'T':
argument++;
nbTests=0; g_testTime=0;
while ((*argument>='0') && (*argument<='9')) {
g_testTime *= 10;
g_testTime += *argument - '0';
argument++;
}
if (*argument=='m') g_testTime *=60, argument++;
if (*argument=='n') argument++;
g_testTime *= 1000;
break;
case 's':
argument++;
seed=0;
seedset=1;
while ((*argument>='0') && (*argument<='9')) {
seed *= 10;
seed += *argument - '0';
argument++;
}
break;
case 't':
argument++;
testNb=0;
while ((*argument>='0') && (*argument<='9')) {
testNb *= 10;
testNb += *argument - '0';
argument++;
}
break;
case 'P': /* compressibility % */
argument++;
proba=0;
while ((*argument>='0') && (*argument<='9')) {
proba *= 10;
proba += *argument - '0';
argument++;
}
if (proba<0) proba=0;
if (proba>100) proba=100;
break;
default:
return FUZ_usage(programName);
}
} } } /* for(argNb=1; argNb<argc; argNb++) */
/* Get Seed */
DISPLAY("Starting zstd_buffered tester (%i-bits, %s)\n", (int)(sizeof(size_t)*8), ZSTD_VERSION_STRING);
if (!seedset) seed = FUZ_GetMilliStart() % 10000;
DISPLAY("Seed = %u\n", seed);
if (proba!=FUZ_COMPRESSIBILITY_DEFAULT) DISPLAY("Compressibility : %i%%\n", proba);
if (nbTests<=0) nbTests=1;
if (testNb==0) {
result = basicUnitTests(0, ((double)proba) / 100, customNULL); /* constant seed for predictability */
if (!result) {
DISPLAYLEVEL(4, "Unit tests using customMem :\n")
result = basicUnitTests(0, ((double)proba) / 100, customMem); /* use custom memory allocation functions */
} }
if (!result)
result = fuzzerTests(seed, nbTests, testNb, ((double)proba) / 100);
if (mainPause) {
int unused;
DISPLAY("Press Enter \n");
unused = getchar();
(void)unused;
}
return result;
}
projects/.gitignore
+12 -1
View File
@@ -1,2 +1,13 @@
build
*Copy
# Visual C++
build/
VS2005/
VS2008/
VS2010/
VS2012/
VS2013/
VS2015/
# CMake
cmake/
projects/VS2005/fullbench/fullbench.vcproj
+448
View File
@@ -0,0 +1,448 @@
<?xml version="1.0" encoding="Windows-1252"?>
<VisualStudioProject
ProjectType="Visual C++"
Version="8.00"
Name="fullbench"
ProjectGUID="{CC8F1D1B-BA2F-43E3-A71F-FA415D81AAD3}"
RootNamespace="fullbench"
Keyword="Win32Proj"
>
<Platforms>
<Platform
Name="Win32"
/>
<Platform
Name="x64"
/>
</Platforms>
<ToolFiles>
</ToolFiles>
<Configurations>
<Configuration
Name="Debug|Win32"
OutputDirectory="$(SolutionDir)bin\$(PlatformName)\$(ConfigurationName)"
IntermediateDirectory="$(PlatformName)\$(ConfigurationName)"
ConfigurationType="1"
CharacterSet="2"
>
<Tool
Name="VCPreBuildEventTool"
/>
<Tool
Name="VCCustomBuildTool"
/>
<Tool
Name="VCXMLDataGeneratorTool"
/>
<Tool
Name="VCWebServiceProxyGeneratorTool"
/>
<Tool
Name="VCMIDLTool"
/>
<Tool
Name="VCCLCompilerTool"
Optimization="0"
AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy"
PreprocessorDefinitions="WIN32;_DEBUG;_CONSOLE"
MinimalRebuild="true"
BasicRuntimeChecks="3"
RuntimeLibrary="3"
UsePrecompiledHeader="0"
WarningLevel="4"
WarnAsError="true"
DebugInformationFormat="4"
/>
<Tool
Name="VCManagedResourceCompilerTool"
/>
<Tool
Name="VCResourceCompilerTool"
/>
<Tool
Name="VCPreLinkEventTool"
/>
<Tool
Name="VCLinkerTool"
LinkIncremental="2"
GenerateDebugInformation="true"
SubSystem="1"
TargetMachine="1"
/>
<Tool
Name="VCALinkTool"
/>
<Tool
Name="VCManifestTool"
/>
<Tool
Name="VCXDCMakeTool"
/>
<Tool
Name="VCBscMakeTool"
/>
<Tool
Name="VCFxCopTool"
/>
<Tool
Name="VCAppVerifierTool"
/>
<Tool
Name="VCPostBuildEventTool"
/>
</Configuration>
<Configuration
Name="Release|Win32"
OutputDirectory="$(SolutionDir)bin\$(PlatformName)\$(ConfigurationName)"
IntermediateDirectory="$(PlatformName)\$(ConfigurationName)"
ConfigurationType="1"
CharacterSet="2"
WholeProgramOptimization="1"
>
<Tool
Name="VCPreBuildEventTool"
/>
<Tool
Name="VCCustomBuildTool"
/>
<Tool
Name="VCXMLDataGeneratorTool"
/>
<Tool
Name="VCWebServiceProxyGeneratorTool"
/>
<Tool
Name="VCMIDLTool"
/>
<Tool
Name="VCCLCompilerTool"
Optimization="2"
EnableIntrinsicFunctions="true"
OmitFramePointers="true"
AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy"
PreprocessorDefinitions="WIN32;NDEBUG;_CONSOLE"
RuntimeLibrary="0"
EnableFunctionLevelLinking="true"
UsePrecompiledHeader="0"
WarningLevel="4"
DebugInformationFormat="3"
/>
<Tool
Name="VCManagedResourceCompilerTool"
/>
<Tool
Name="VCResourceCompilerTool"
/>
<Tool
Name="VCPreLinkEventTool"
/>
<Tool
Name="VCLinkerTool"
LinkIncremental="1"
GenerateDebugInformation="true"
SubSystem="1"
OptimizeReferences="2"
EnableCOMDATFolding="2"
TargetMachine="1"
/>
<Tool
Name="VCALinkTool"
/>
<Tool
Name="VCManifestTool"
/>
<Tool
Name="VCXDCMakeTool"
/>
<Tool
Name="VCBscMakeTool"
/>
<Tool
Name="VCFxCopTool"
/>
<Tool
Name="VCAppVerifierTool"
/>
<Tool
Name="VCPostBuildEventTool"
/>
</Configuration>
<Configuration
Name="Debug|x64"
OutputDirectory="$(SolutionDir)bin\$(PlatformName)\$(ConfigurationName)"
IntermediateDirectory="$(PlatformName)\$(ConfigurationName)"
ConfigurationType="1"
CharacterSet="2"
>
<Tool
Name="VCPreBuildEventTool"
/>
<Tool
Name="VCCustomBuildTool"
/>
<Tool
Name="VCXMLDataGeneratorTool"
/>
<Tool
Name="VCWebServiceProxyGeneratorTool"
/>
<Tool
Name="VCMIDLTool"
TargetEnvironment="3"
/>
<Tool
Name="VCCLCompilerTool"
Optimization="0"
AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy"
PreprocessorDefinitions="WIN32;_DEBUG;_CONSOLE"
MinimalRebuild="true"
BasicRuntimeChecks="3"
RuntimeLibrary="3"
UsePrecompiledHeader="0"
WarningLevel="4"
WarnAsError="true"
DebugInformationFormat="3"
/>
<Tool
Name="VCManagedResourceCompilerTool"
/>
<Tool
Name="VCResourceCompilerTool"
/>
<Tool
Name="VCPreLinkEventTool"
/>
<Tool
Name="VCLinkerTool"
LinkIncremental="2"
GenerateDebugInformation="true"
SubSystem="1"
TargetMachine="17"
/>
<Tool
Name="VCALinkTool"
/>
<Tool
Name="VCManifestTool"
/>
<Tool
Name="VCXDCMakeTool"
/>
<Tool
Name="VCBscMakeTool"
/>
<Tool
Name="VCFxCopTool"
/>
<Tool
Name="VCAppVerifierTool"
/>
<Tool
Name="VCPostBuildEventTool"
/>
</Configuration>
<Configuration
Name="Release|x64"
OutputDirectory="$(SolutionDir)bin\$(PlatformName)\$(ConfigurationName)"
IntermediateDirectory="$(PlatformName)\$(ConfigurationName)"
ConfigurationType="1"
CharacterSet="2"
WholeProgramOptimization="1"
>
<Tool
Name="VCPreBuildEventTool"
/>
<Tool
Name="VCCustomBuildTool"
/>
<Tool
Name="VCXMLDataGeneratorTool"
/>
<Tool
Name="VCWebServiceProxyGeneratorTool"
/>
<Tool
Name="VCMIDLTool"
TargetEnvironment="3"
/>
<Tool
Name="VCCLCompilerTool"
Optimization="2"
EnableIntrinsicFunctions="true"
OmitFramePointers="true"
AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy"
PreprocessorDefinitions="WIN32;NDEBUG;_CONSOLE"
RuntimeLibrary="0"
EnableFunctionLevelLinking="true"
UsePrecompiledHeader="0"
WarningLevel="4"
DebugInformationFormat="3"
/>
<Tool
Name="VCManagedResourceCompilerTool"
/>
<Tool
Name="VCResourceCompilerTool"
/>
<Tool
Name="VCPreLinkEventTool"
/>
<Tool
Name="VCLinkerTool"
LinkIncremental="1"
GenerateDebugInformation="true"
SubSystem="1"
OptimizeReferences="2"
EnableCOMDATFolding="2"
TargetMachine="17"
/>
<Tool
Name="VCALinkTool"
/>
<Tool
Name="VCManifestTool"
/>
<Tool
Name="VCXDCMakeTool"
/>
<Tool
Name="VCBscMakeTool"
/>
<Tool
Name="VCFxCopTool"
/>
<Tool
Name="VCAppVerifierTool"
/>
<Tool
Name="VCPostBuildEventTool"
/>
</Configuration>
</Configurations>
<References>
</References>
<Files>
<Filter
Name="Source Files"
Filter="cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx"
UniqueIdentifier="{4FC737F1-C7A5-4376-A066-2A32D752A2FF}"
>
<File
RelativePath="..\..\..\programs\datagen.c"
>
</File>
<File
RelativePath="..\..\..\lib\common\entropy_common.c"
>
</File>
<File
RelativePath="..\..\..\lib\compress\fse_compress.c"
>
</File>
<File
RelativePath="..\..\..\lib\common\fse_decompress.c"
>
</File>
<File
RelativePath="..\..\..\lib\common\xxhash.c"
>
</File>
<File
RelativePath="..\..\..\programs\fullbench.c"
>
</File>
<File
RelativePath="..\..\..\lib\compress\huf_compress.c"
>
</File>
<File
RelativePath="..\..\..\lib\decompress\huf_decompress.c"
>
</File>
<File
RelativePath="..\..\..\lib\compress\zbuff_compress.c"
>
</File>
<File
RelativePath="..\..\..\lib\decompress\zbuff_decompress.c"
>
</File>
<File
RelativePath="..\..\..\lib\common\zstd_common.c"
>
</File>
<File
RelativePath="..\..\..\lib\compress\zstd_compress.c"
>
</File>
<File
RelativePath="..\..\..\lib\decompress\zstd_decompress.c"
>
</File>
</Filter>
<Filter
Name="Header Files"
Filter="h;hpp;hxx;hm;inl;inc;xsd"
UniqueIdentifier="{93995380-89BD-4b04-88EB-625FBE52EBFB}"
>
<File
RelativePath="..\..\..\lib\common\bitstream.h"
>
</File>
<File
RelativePath="..\..\..\lib\common\error_private.h"
>
</File>
<File
RelativePath="..\..\..\lib\common\error_public.h"
>
</File>
<File
RelativePath="..\..\..\lib\common\fse.h"
>
</File>
<File
RelativePath="..\..\..\lib\common\fse_static.h"
>
</File>
<File
RelativePath="..\..\..\lib\common\huf.h"
>
</File>
<File
RelativePath="..\..\..\lib\common\huf_static.h"
>
</File>
<File
RelativePath="..\..\..\lib\common\xxhash.h"
>
</File>
<File
RelativePath="..\..\..\lib\common\mem.h"
>
</File>
<File
RelativePath="..\..\..\lib\common\zbuff_static.h"
>
</File>
<File
RelativePath="..\..\..\lib\zstd.h"
>
</File>
<File
RelativePath="..\..\..\lib\common\zstd_internal.h"
>
</File>
<File
RelativePath="..\..\..\lib\compress\zstd_opt.h"
>
</File>
<File
RelativePath="..\..\..\lib\common\zstd_static.h"
>
</File>
</Filter>
</Files>
<Globals>
</Globals>
</VisualStudioProject>
projects/VS2005/fuzzer/fuzzer.vcproj
+460
View File
@@ -0,0 +1,460 @@
<?xml version="1.0" encoding="Windows-1252"?>
<VisualStudioProject
ProjectType="Visual C++"
Version="8.00"
Name="fuzzer"
ProjectGUID="{A62E89D2-9DDE-42BA-8F9B-9DA74889A6B0}"
RootNamespace="fuzzer"
Keyword="Win32Proj"
>
<Platforms>
<Platform
Name="Win32"
/>
<Platform
Name="x64"
/>
</Platforms>
<ToolFiles>
</ToolFiles>
<Configurations>
<Configuration
Name="Debug|Win32"
OutputDirectory="$(SolutionDir)bin\$(PlatformName)\$(ConfigurationName)"
IntermediateDirectory="$(PlatformName)\$(ConfigurationName)"
ConfigurationType="1"
CharacterSet="2"
>
<Tool
Name="VCPreBuildEventTool"
/>
<Tool
Name="VCCustomBuildTool"
/>
<Tool
Name="VCXMLDataGeneratorTool"
/>
<Tool
Name="VCWebServiceProxyGeneratorTool"
/>
<Tool
Name="VCMIDLTool"
/>
<Tool
Name="VCCLCompilerTool"
Optimization="0"
AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy"
PreprocessorDefinitions="WIN32;_DEBUG;_CONSOLE"
MinimalRebuild="true"
BasicRuntimeChecks="3"
RuntimeLibrary="3"
UsePrecompiledHeader="0"
WarningLevel="4"
WarnAsError="true"
DebugInformationFormat="4"
/>
<Tool
Name="VCManagedResourceCompilerTool"
/>
<Tool
Name="VCResourceCompilerTool"
/>
<Tool
Name="VCPreLinkEventTool"
/>
<Tool
Name="VCLinkerTool"
LinkIncremental="2"
GenerateDebugInformation="true"
SubSystem="1"
TargetMachine="1"
/>
<Tool
Name="VCALinkTool"
/>
<Tool
Name="VCManifestTool"
/>
<Tool
Name="VCXDCMakeTool"
/>
<Tool
Name="VCBscMakeTool"
/>
<Tool
Name="VCFxCopTool"
/>
<Tool
Name="VCAppVerifierTool"
/>
<Tool
Name="VCPostBuildEventTool"
/>
</Configuration>
<Configuration
Name="Release|Win32"
OutputDirectory="$(SolutionDir)bin\$(PlatformName)\$(ConfigurationName)"
IntermediateDirectory="$(PlatformName)\$(ConfigurationName)"
ConfigurationType="1"
CharacterSet="2"
WholeProgramOptimization="1"
>
<Tool
Name="VCPreBuildEventTool"
/>
<Tool
Name="VCCustomBuildTool"
/>
<Tool
Name="VCXMLDataGeneratorTool"
/>
<Tool
Name="VCWebServiceProxyGeneratorTool"
/>
<Tool
Name="VCMIDLTool"
/>
<Tool
Name="VCCLCompilerTool"
Optimization="2"
EnableIntrinsicFunctions="true"
OmitFramePointers="true"
AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy"
PreprocessorDefinitions="WIN32;NDEBUG;_CONSOLE"
RuntimeLibrary="0"
EnableFunctionLevelLinking="true"
UsePrecompiledHeader="0"
WarningLevel="4"
DebugInformationFormat="3"
/>
<Tool
Name="VCManagedResourceCompilerTool"
/>
<Tool
Name="VCResourceCompilerTool"
/>
<Tool
Name="VCPreLinkEventTool"
/>
<Tool
Name="VCLinkerTool"
LinkIncremental="1"
GenerateDebugInformation="true"
SubSystem="1"
OptimizeReferences="2"
EnableCOMDATFolding="2"
TargetMachine="1"
/>
<Tool
Name="VCALinkTool"
/>
<Tool
Name="VCManifestTool"
/>
<Tool
Name="VCXDCMakeTool"
/>
<Tool
Name="VCBscMakeTool"
/>
<Tool
Name="VCFxCopTool"
/>
<Tool
Name="VCAppVerifierTool"
/>
<Tool
Name="VCPostBuildEventTool"
/>
</Configuration>
<Configuration
Name="Debug|x64"
OutputDirectory="$(SolutionDir)bin\$(PlatformName)\$(ConfigurationName)"
IntermediateDirectory="$(PlatformName)\$(ConfigurationName)"
ConfigurationType="1"
CharacterSet="2"
>
<Tool
Name="VCPreBuildEventTool"
/>
<Tool
Name="VCCustomBuildTool"
/>
<Tool
Name="VCXMLDataGeneratorTool"
/>
<Tool
Name="VCWebServiceProxyGeneratorTool"
/>
<Tool
Name="VCMIDLTool"
TargetEnvironment="3"
/>
<Tool
Name="VCCLCompilerTool"
Optimization="0"
AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy"
PreprocessorDefinitions="WIN32;_DEBUG;_CONSOLE"
MinimalRebuild="true"
BasicRuntimeChecks="3"
RuntimeLibrary="3"
UsePrecompiledHeader="0"
WarningLevel="4"
WarnAsError="true"
DebugInformationFormat="3"
/>
<Tool
Name="VCManagedResourceCompilerTool"
/>
<Tool
Name="VCResourceCompilerTool"
/>
<Tool
Name="VCPreLinkEventTool"
/>
<Tool
Name="VCLinkerTool"
LinkIncremental="2"
GenerateDebugInformation="true"
SubSystem="1"
TargetMachine="17"
/>
<Tool
Name="VCALinkTool"
/>
<Tool
Name="VCManifestTool"
/>
<Tool
Name="VCXDCMakeTool"
/>
<Tool
Name="VCBscMakeTool"
/>
<Tool
Name="VCFxCopTool"
/>
<Tool
Name="VCAppVerifierTool"
/>
<Tool
Name="VCPostBuildEventTool"
/>
</Configuration>
<Configuration
Name="Release|x64"
OutputDirectory="$(SolutionDir)bin\$(PlatformName)\$(ConfigurationName)"
IntermediateDirectory="$(PlatformName)\$(ConfigurationName)"
ConfigurationType="1"
CharacterSet="2"
WholeProgramOptimization="1"
>
<Tool
Name="VCPreBuildEventTool"
/>
<Tool
Name="VCCustomBuildTool"
/>
<Tool
Name="VCXMLDataGeneratorTool"
/>
<Tool
Name="VCWebServiceProxyGeneratorTool"
/>
<Tool
Name="VCMIDLTool"
TargetEnvironment="3"
/>
<Tool
Name="VCCLCompilerTool"
Optimization="2"
EnableIntrinsicFunctions="true"
OmitFramePointers="true"
AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy"
PreprocessorDefinitions="WIN32;NDEBUG;_CONSOLE"
RuntimeLibrary="0"
EnableFunctionLevelLinking="true"
UsePrecompiledHeader="0"
WarningLevel="4"
DebugInformationFormat="3"
/>
<Tool
Name="VCManagedResourceCompilerTool"
/>
<Tool
Name="VCResourceCompilerTool"
/>
<Tool
Name="VCPreLinkEventTool"
/>
<Tool
Name="VCLinkerTool"
LinkIncremental="1"
GenerateDebugInformation="true"
SubSystem="1"
OptimizeReferences="2"
EnableCOMDATFolding="2"
TargetMachine="17"
/>
<Tool
Name="VCALinkTool"
/>
<Tool
Name="VCManifestTool"
/>
<Tool
Name="VCXDCMakeTool"
/>
<Tool
Name="VCBscMakeTool"
/>
<Tool
Name="VCFxCopTool"
/>
<Tool
Name="VCAppVerifierTool"
/>
<Tool
Name="VCPostBuildEventTool"
/>
</Configuration>
</Configurations>
<References>
</References>
<Files>
<Filter
Name="Source Files"
Filter="cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx"
UniqueIdentifier="{4FC737F1-C7A5-4376-A066-2A32D752A2FF}"
>
<File
RelativePath="..\..\..\programs\datagen.c"
>
</File>
<File
RelativePath="..\..\..\lib\dictBuilder\divsufsort.c"
>
</File>
<File
RelativePath="..\..\..\lib\common\entropy_common.c"
>
</File>
<File
RelativePath="..\..\..\lib\compress\fse_compress.c"
>
</File>
<File
RelativePath="..\..\..\lib\common\fse_decompress.c"
>
</File>
<File
RelativePath="..\..\..\programs\fuzzer.c"
>
</File>
<File
RelativePath="..\..\..\lib\compress\huf_compress.c"
>
</File>
<File
RelativePath="..\..\..\lib\decompress\huf_decompress.c"
>
</File>
<File
RelativePath="..\..\..\lib\common\xxhash.c"
>
</File>
<File
RelativePath="..\..\..\lib\dictBuilder\zdict.c"
>
</File>
<File
RelativePath="..\..\..\lib\common\zstd_common.c"
>
</File>
<File
RelativePath="..\..\..\lib\compress\zstd_compress.c"
>
</File>
<File
RelativePath="..\..\..\lib\decompress\zstd_decompress.c"
>
</File>
</Filter>
<Filter
Name="Header Files"
Filter="h;hpp;hxx;hm;inl;inc;xsd"
UniqueIdentifier="{93995380-89BD-4b04-88EB-625FBE52EBFB}"
>
<File
RelativePath="..\..\..\lib\common\bitstream.h"
>
</File>
<File
RelativePath="..\..\..\lib\dictBuilder\divsufsort.h"
>
</File>
<File
RelativePath="..\..\..\lib\common\error_private.h"
>
</File>
<File
RelativePath="..\..\..\lib\common\error_public.h"
>
</File>
<File
RelativePath="..\..\..\lib\common\fse.h"
>
</File>
<File
RelativePath="..\..\..\lib\common\fse_static.h"
>
</File>
<File
RelativePath="..\..\..\lib\common\huf.h"
>
</File>
<File
RelativePath="..\..\..\lib\common\huf_static.h"
>
</File>
<File
RelativePath="..\..\..\lib\common\mem.h"
>
</File>
<File
RelativePath="..\..\..\lib\common\xxhash.h"
>
</File>
<File
RelativePath="..\..\..\lib\common\zbuff_static.h"
>
</File>
<File
RelativePath="..\..\..\lib\dictBuilder\zdict.h"
>
</File>
<File
RelativePath="..\..\..\lib\dictBuilder\zdict_static.h"
>
</File>
<File
RelativePath="..\..\..\lib\zstd.h"
>
</File>
<File
RelativePath="..\..\..\lib\common\zstd_internal.h"
>
</File>
<File
RelativePath="..\..\..\lib\compress\zstd_opt.h"
>
</File>
<File
RelativePath="..\..\..\lib\common\zstd_static.h"
>
</File>
</Filter>
</Files>
<Globals>
</Globals>
</VisualStudioProject>
projects/VS2005/zstd.sln
+55
View File
@@ -0,0 +1,55 @@
Microsoft Visual Studio Solution File, Format Version 9.00
# Visual C++ Express 2005
Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "zstd", "zstd\zstd.vcproj", "{1A2AB08E-5CE7-4C5B-BE55-458157C14051}"
EndProject
Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "fuzzer", "fuzzer\fuzzer.vcproj", "{A62E89D2-9DDE-42BA-8F9B-9DA74889A6B0}"
EndProject
Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "fullbench", "fullbench\fullbench.vcproj", "{CC8F1D1B-BA2F-43E3-A71F-FA415D81AAD3}"
EndProject
Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "zstdlib", "zstdlib\zstdlib.vcproj", "{99DE2A79-7298-4004-A0ED-030D7A3796CA}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Win32 = Debug|Win32
Debug|x64 = Debug|x64
Release|Win32 = Release|Win32
Release|x64 = Release|x64
EndGlobalSection
GlobalSection(ProjectConfigurationPlatforms) = postSolution
{1A2AB08E-5CE7-4C5B-BE55-458157C14051}.Debug|Win32.ActiveCfg = Debug|Win32
{1A2AB08E-5CE7-4C5B-BE55-458157C14051}.Debug|Win32.Build.0 = Debug|Win32
{1A2AB08E-5CE7-4C5B-BE55-458157C14051}.Debug|x64.ActiveCfg = Debug|x64
{1A2AB08E-5CE7-4C5B-BE55-458157C14051}.Debug|x64.Build.0 = Debug|x64
{1A2AB08E-5CE7-4C5B-BE55-458157C14051}.Release|Win32.ActiveCfg = Release|Win32
{1A2AB08E-5CE7-4C5B-BE55-458157C14051}.Release|Win32.Build.0 = Release|Win32
{1A2AB08E-5CE7-4C5B-BE55-458157C14051}.Release|x64.ActiveCfg = Release|x64
{1A2AB08E-5CE7-4C5B-BE55-458157C14051}.Release|x64.Build.0 = Release|x64
{A62E89D2-9DDE-42BA-8F9B-9DA74889A6B0}.Debug|Win32.ActiveCfg = Debug|Win32
{A62E89D2-9DDE-42BA-8F9B-9DA74889A6B0}.Debug|Win32.Build.0 = Debug|Win32
{A62E89D2-9DDE-42BA-8F9B-9DA74889A6B0}.Debug|x64.ActiveCfg = Debug|x64
{A62E89D2-9DDE-42BA-8F9B-9DA74889A6B0}.Debug|x64.Build.0 = Debug|x64
{A62E89D2-9DDE-42BA-8F9B-9DA74889A6B0}.Release|Win32.ActiveCfg = Release|Win32
{A62E89D2-9DDE-42BA-8F9B-9DA74889A6B0}.Release|Win32.Build.0 = Release|Win32
{A62E89D2-9DDE-42BA-8F9B-9DA74889A6B0}.Release|x64.ActiveCfg = Release|x64
{A62E89D2-9DDE-42BA-8F9B-9DA74889A6B0}.Release|x64.Build.0 = Release|x64
{CC8F1D1B-BA2F-43E3-A71F-FA415D81AAD3}.Debug|Win32.ActiveCfg = Debug|Win32
{CC8F1D1B-BA2F-43E3-A71F-FA415D81AAD3}.Debug|Win32.Build.0 = Debug|Win32
{CC8F1D1B-BA2F-43E3-A71F-FA415D81AAD3}.Debug|x64.ActiveCfg = Debug|x64
{CC8F1D1B-BA2F-43E3-A71F-FA415D81AAD3}.Debug|x64.Build.0 = Debug|x64
{CC8F1D1B-BA2F-43E3-A71F-FA415D81AAD3}.Release|Win32.ActiveCfg = Release|Win32
{CC8F1D1B-BA2F-43E3-A71F-FA415D81AAD3}.Release|Win32.Build.0 = Release|Win32
{CC8F1D1B-BA2F-43E3-A71F-FA415D81AAD3}.Release|x64.ActiveCfg = Release|x64
{CC8F1D1B-BA2F-43E3-A71F-FA415D81AAD3}.Release|x64.Build.0 = Release|x64
{99DE2A79-7298-4004-A0ED-030D7A3796CA}.Debug|Win32.ActiveCfg = Debug|Win32
{99DE2A79-7298-4004-A0ED-030D7A3796CA}.Debug|Win32.Build.0 = Debug|Win32
{99DE2A79-7298-4004-A0ED-030D7A3796CA}.Debug|x64.ActiveCfg = Debug|x64
{99DE2A79-7298-4004-A0ED-030D7A3796CA}.Debug|x64.Build.0 = Debug|x64
{99DE2A79-7298-4004-A0ED-030D7A3796CA}.Release|Win32.ActiveCfg = Release|Win32
{99DE2A79-7298-4004-A0ED-030D7A3796CA}.Release|Win32.Build.0 = Release|Win32
{99DE2A79-7298-4004-A0ED-030D7A3796CA}.Release|x64.ActiveCfg = Release|x64
{99DE2A79-7298-4004-A0ED-030D7A3796CA}.Release|x64.Build.0 = Release|x64
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
EndGlobalSection
EndGlobal
projects/VS2005/zstd/zstd.vcproj
+552
View File
@@ -0,0 +1,552 @@
<?xml version="1.0" encoding="Windows-1252"?>
<VisualStudioProject
ProjectType="Visual C++"
Version="8.00"
Name="zstd"
ProjectGUID="{1A2AB08E-5CE7-4C5B-BE55-458157C14051}"
RootNamespace="zstd"
Keyword="Win32Proj"
>
<Platforms>
<Platform
Name="Win32"
/>
<Platform
Name="x64"
/>
</Platforms>
<ToolFiles>
</ToolFiles>
<Configurations>
<Configuration
Name="Debug|Win32"
OutputDirectory="$(SolutionDir)bin\$(PlatformName)\$(ConfigurationName)"
IntermediateDirectory="$(PlatformName)\$(ConfigurationName)"
ConfigurationType="1"
CharacterSet="2"
>
<Tool
Name="VCPreBuildEventTool"
/>
<Tool
Name="VCCustomBuildTool"
/>
<Tool
Name="VCXMLDataGeneratorTool"
/>
<Tool
Name="VCWebServiceProxyGeneratorTool"
/>
<Tool
Name="VCMIDLTool"
/>
<Tool
Name="VCCLCompilerTool"
Optimization="0"
AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\dictBuilder"
PreprocessorDefinitions="WIN32;_DEBUG;_CONSOLE"
MinimalRebuild="true"
BasicRuntimeChecks="3"
RuntimeLibrary="3"
UsePrecompiledHeader="0"
WarningLevel="4"
WarnAsError="true"
DebugInformationFormat="4"
/>
<Tool
Name="VCManagedResourceCompilerTool"
/>
<Tool
Name="VCResourceCompilerTool"
/>
<Tool
Name="VCPreLinkEventTool"
/>
<Tool
Name="VCLinkerTool"
AdditionalDependencies="setargv.obj"
LinkIncremental="2"
GenerateDebugInformation="true"
SubSystem="1"
TargetMachine="1"
/>
<Tool
Name="VCALinkTool"
/>
<Tool
Name="VCManifestTool"
/>
<Tool
Name="VCXDCMakeTool"
/>
<Tool
Name="VCBscMakeTool"
/>
<Tool
Name="VCFxCopTool"
/>
<Tool
Name="VCAppVerifierTool"
/>
<Tool
Name="VCPostBuildEventTool"
/>
</Configuration>
<Configuration
Name="Release|Win32"
OutputDirectory="$(SolutionDir)bin\$(PlatformName)\$(ConfigurationName)"
IntermediateDirectory="$(PlatformName)\$(ConfigurationName)"
ConfigurationType="1"
CharacterSet="2"
WholeProgramOptimization="1"
>
<Tool
Name="VCPreBuildEventTool"
/>
<Tool
Name="VCCustomBuildTool"
/>
<Tool
Name="VCXMLDataGeneratorTool"
/>
<Tool
Name="VCWebServiceProxyGeneratorTool"
/>
<Tool
Name="VCMIDLTool"
/>
<Tool
Name="VCCLCompilerTool"
Optimization="2"
EnableIntrinsicFunctions="true"
OmitFramePointers="true"
AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\dictBuilder"
PreprocessorDefinitions="WIN32;NDEBUG;_CONSOLE"
RuntimeLibrary="0"
EnableFunctionLevelLinking="true"
UsePrecompiledHeader="0"
WarningLevel="4"
DebugInformationFormat="3"
/>
<Tool
Name="VCManagedResourceCompilerTool"
/>
<Tool
Name="VCResourceCompilerTool"
/>
<Tool
Name="VCPreLinkEventTool"
/>
<Tool
Name="VCLinkerTool"
AdditionalDependencies="setargv.obj"
LinkIncremental="1"
GenerateDebugInformation="true"
SubSystem="1"
OptimizeReferences="2"
EnableCOMDATFolding="2"
TargetMachine="1"
/>
<Tool
Name="VCALinkTool"
/>
<Tool
Name="VCManifestTool"
/>
<Tool
Name="VCXDCMakeTool"
/>
<Tool
Name="VCBscMakeTool"
/>
<Tool
Name="VCFxCopTool"
/>
<Tool
Name="VCAppVerifierTool"
/>
<Tool
Name="VCPostBuildEventTool"
/>
</Configuration>
<Configuration
Name="Debug|x64"
OutputDirectory="$(SolutionDir)bin\$(PlatformName)\$(ConfigurationName)"
IntermediateDirectory="$(PlatformName)\$(ConfigurationName)"
ConfigurationType="1"
CharacterSet="2"
>
<Tool
Name="VCPreBuildEventTool"
/>
<Tool
Name="VCCustomBuildTool"
/>
<Tool
Name="VCXMLDataGeneratorTool"
/>
<Tool
Name="VCWebServiceProxyGeneratorTool"
/>
<Tool
Name="VCMIDLTool"
TargetEnvironment="3"
/>
<Tool
Name="VCCLCompilerTool"
Optimization="0"
AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\dictBuilder"
PreprocessorDefinitions="WIN32;_DEBUG;_CONSOLE"
MinimalRebuild="true"
BasicRuntimeChecks="3"
RuntimeLibrary="3"
UsePrecompiledHeader="0"
WarningLevel="4"
WarnAsError="true"
DebugInformationFormat="3"
/>
<Tool
Name="VCManagedResourceCompilerTool"
/>
<Tool
Name="VCResourceCompilerTool"
/>
<Tool
Name="VCPreLinkEventTool"
/>
<Tool
Name="VCLinkerTool"
AdditionalDependencies="setargv.obj"
LinkIncremental="2"
GenerateDebugInformation="true"
SubSystem="1"
TargetMachine="17"
/>
<Tool
Name="VCALinkTool"
/>
<Tool
Name="VCManifestTool"
/>
<Tool
Name="VCXDCMakeTool"
/>
<Tool
Name="VCBscMakeTool"
/>
<Tool
Name="VCFxCopTool"
/>
<Tool
Name="VCAppVerifierTool"
/>
<Tool
Name="VCPostBuildEventTool"
/>
</Configuration>
<Configuration
Name="Release|x64"
OutputDirectory="$(SolutionDir)bin\$(PlatformName)\$(ConfigurationName)"
IntermediateDirectory="$(PlatformName)\$(ConfigurationName)"
ConfigurationType="1"
CharacterSet="2"
WholeProgramOptimization="1"
>
<Tool
Name="VCPreBuildEventTool"
/>
<Tool
Name="VCCustomBuildTool"
/>
<Tool
Name="VCXMLDataGeneratorTool"
/>
<Tool
Name="VCWebServiceProxyGeneratorTool"
/>
<Tool
Name="VCMIDLTool"
TargetEnvironment="3"
/>
<Tool
Name="VCCLCompilerTool"
Optimization="2"
EnableIntrinsicFunctions="true"
OmitFramePointers="true"
AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\dictBuilder"
PreprocessorDefinitions="WIN32;NDEBUG;_CONSOLE"
RuntimeLibrary="0"
EnableFunctionLevelLinking="true"
UsePrecompiledHeader="0"
WarningLevel="4"
DebugInformationFormat="3"
/>
<Tool
Name="VCManagedResourceCompilerTool"
/>
<Tool
Name="VCResourceCompilerTool"
/>
<Tool
Name="VCPreLinkEventTool"
/>
<Tool
Name="VCLinkerTool"
AdditionalDependencies="setargv.obj"
LinkIncremental="1"
GenerateDebugInformation="true"
SubSystem="1"
OptimizeReferences="2"
EnableCOMDATFolding="2"
TargetMachine="17"
/>
<Tool
Name="VCALinkTool"
/>
<Tool
Name="VCManifestTool"
/>
<Tool
Name="VCXDCMakeTool"
/>
<Tool
Name="VCBscMakeTool"
/>
<Tool
Name="VCFxCopTool"
/>
<Tool
Name="VCAppVerifierTool"
/>
<Tool
Name="VCPostBuildEventTool"
/>
</Configuration>
</Configurations>
<References>
</References>
<Files>
<Filter
Name="Source Files"
Filter="cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx"
UniqueIdentifier="{4FC737F1-C7A5-4376-A066-2A32D752A2FF}"
>
<File
RelativePath="..\..\..\programs\bench.c"
>
</File>
<File
RelativePath="..\..\..\programs\datagen.c"
>
</File>
<File
RelativePath="..\..\..\programs\dibio.c"
>
</File>
<File
RelativePath="..\..\..\lib\dictBuilder\divsufsort.c"
>
</File>
<File
RelativePath="..\..\..\lib\common\entropy_common.c"
>
</File>
<File
RelativePath="..\..\..\programs\fileio.c"
>
</File>
<File
RelativePath="..\..\..\programs\legacy\fileio_legacy.c"
>
</File>
<File
RelativePath="..\..\..\lib\compress\fse_compress.c"
>
</File>
<File
RelativePath="..\..\..\lib\common\fse_decompress.c"
>
</File>
<File
RelativePath="..\..\..\lib\compress\huf_compress.c"
>
</File>
<File
RelativePath="..\..\..\lib\decompress\huf_decompress.c"
>
</File>
<File
RelativePath="..\..\..\lib\common\xxhash.c"
>
</File>
<File
RelativePath="..\..\..\lib\compress\zbuff_compress.c"
>
</File>
<File
RelativePath="..\..\..\lib\decompress\zbuff_decompress.c"
>
</File>
<File
RelativePath="..\..\..\lib\dictBuilder\zdict.c"
>
</File>
<File
RelativePath="..\..\..\lib\common\zstd_common.c"
>
</File>
<File
RelativePath="..\..\..\lib\compress\zstd_compress.c"
>
</File>
<File
RelativePath="..\..\..\lib\decompress\zstd_decompress.c"
>
</File>
<File
RelativePath="..\..\..\lib\legacy\zstd_v01.c"
>
</File>
<File
RelativePath="..\..\..\lib\legacy\zstd_v02.c"
>
</File>
<File
RelativePath="..\..\..\lib\legacy\zstd_v03.c"
>
</File>
<File
RelativePath="..\..\..\lib\legacy\zstd_v04.c"
>
</File>
<File
RelativePath="..\..\..\lib\legacy\zstd_v05.c"
>
</File>
<File
RelativePath="..\..\..\lib\legacy\zstd_v06.c"
>
</File>
<File
RelativePath="..\..\..\lib\legacy\zstd_v07.c"
>
</File>
<File
RelativePath="..\..\..\programs\zstdcli.c"
>
</File>
</Filter>
<Filter
Name="Header Files"
Filter="h;hpp;hxx;hm;inl;inc;xsd"
UniqueIdentifier="{93995380-89BD-4b04-88EB-625FBE52EBFB}"
>
<File
RelativePath="..\..\..\lib\common\bitstream.h"
>
</File>
<File
RelativePath="..\..\..\lib\dictBuilder\divsufsort.h"
>
</File>
<File
RelativePath="..\..\..\lib\common\error_private.h"
>
</File>
<File
RelativePath="..\..\..\lib\common\error_public.h"
>
</File>
<File
RelativePath="..\..\..\lib\common\fse.h"
>
</File>
<File
RelativePath="..\..\..\lib\common\fse_static.h"
>
</File>
<File
RelativePath="..\..\..\lib\common\huf.h"
>
</File>
<File
RelativePath="..\..\..\lib\common\huf_static.h"
>
</File>
<File
RelativePath="..\..\..\lib\common\mem.h"
>
</File>
<File
RelativePath="..\..\..\lib\common\xxhash.h"
>
</File>
<File
RelativePath="..\..\..\lib\common\zbuff.h"
>
</File>
<File
RelativePath="..\..\..\lib\common\zbuff_static.h"
>
</File>
<File
RelativePath="..\..\..\lib\dictBuilder\zdict.h"
>
</File>
<File
RelativePath="..\..\..\lib\dictBuilder\zdict_static.h"
>
</File>
<File
RelativePath="..\..\..\lib\zstd.h"
>
</File>
<File
RelativePath="..\..\..\lib\common\zstd_internal.h"
>
</File>
<File
RelativePath="..\..\..\lib\legacy\zstd_legacy.h"
>
</File>
<File
RelativePath="..\..\..\lib\compress\zstd_opt.h"
>
</File>
<File
RelativePath="..\..\..\lib\common\zstd_static.h"
>
</File>
<File
RelativePath="..\..\..\lib\legacy\zstd_v01.h"
>
</File>
<File
RelativePath="..\..\..\lib\legacy\zstd_v02.h"
>
</File>
<File
RelativePath="..\..\..\lib\legacy\zstd_v03.h"
>
</File>
<File
RelativePath="..\..\..\lib\legacy\zstd_v04.h"
>
</File>
<File
RelativePath="..\..\..\lib\legacy\zstd_v05.h"
>
</File>
<File
RelativePath="..\..\..\lib\legacy\zstd_v06.h"
>
</File>
<File
RelativePath="..\..\..\lib\legacy\zstd_v07.h"
>
</File>
</Filter>
</Files>
<Globals>
</Globals>
</VisualStudioProject>
projects/VS2005/zstdlib/zstdlib.vcproj
+494
View File
@@ -0,0 +1,494 @@
<?xml version="1.0" encoding="Windows-1252"?>
<VisualStudioProject
ProjectType="Visual C++"
Version="8.00"
Name="zstdlib"
ProjectGUID="{99DE2A79-7298-4004-A0ED-030D7A3796CA}"
RootNamespace="zstdlib"
Keyword="Win32Proj"
>
<Platforms>
<Platform
Name="Win32"
/>
<Platform
Name="x64"
/>
</Platforms>
<ToolFiles>
</ToolFiles>
<Configurations>
<Configuration
Name="Debug|Win32"
OutputDirectory="$(SolutionDir)bin\$(PlatformName)\$(ConfigurationName)"
IntermediateDirectory="$(PlatformName)\$(ConfigurationName)"
ConfigurationType="2"
CharacterSet="2"
>
<Tool
Name="VCPreBuildEventTool"
/>
<Tool
Name="VCCustomBuildTool"
/>
<Tool
Name="VCXMLDataGeneratorTool"
/>
<Tool
Name="VCWebServiceProxyGeneratorTool"
/>
<Tool
Name="VCMIDLTool"
/>
<Tool
Name="VCCLCompilerTool"
Optimization="0"
AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\dictBuilder"
PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_HEAPMODE=0;ZSTD_LEGACY_SUPPORT=0;WIN32;_DEBUG;_CONSOLE"
MinimalRebuild="true"
BasicRuntimeChecks="3"
RuntimeLibrary="3"
UsePrecompiledHeader="0"
WarningLevel="4"
WarnAsError="true"
DebugInformationFormat="4"
/>
<Tool
Name="VCManagedResourceCompilerTool"
/>
<Tool
Name="VCResourceCompilerTool"
/>
<Tool
Name="VCPreLinkEventTool"
/>
<Tool
Name="VCLinkerTool"
LinkIncremental="2"
GenerateDebugInformation="true"
SubSystem="1"
TargetMachine="1"
/>
<Tool
Name="VCALinkTool"
/>
<Tool
Name="VCManifestTool"
/>
<Tool
Name="VCXDCMakeTool"
/>
<Tool
Name="VCBscMakeTool"
/>
<Tool
Name="VCFxCopTool"
/>
<Tool
Name="VCAppVerifierTool"
/>
<Tool
Name="VCPostBuildEventTool"
/>
</Configuration>
<Configuration
Name="Release|Win32"
OutputDirectory="$(SolutionDir)bin\$(PlatformName)\$(ConfigurationName)"
IntermediateDirectory="$(PlatformName)\$(ConfigurationName)"
ConfigurationType="2"
CharacterSet="2"
WholeProgramOptimization="1"
>
<Tool
Name="VCPreBuildEventTool"
/>
<Tool
Name="VCCustomBuildTool"
/>
<Tool
Name="VCXMLDataGeneratorTool"
/>
<Tool
Name="VCWebServiceProxyGeneratorTool"
/>
<Tool
Name="VCMIDLTool"
/>
<Tool
Name="VCCLCompilerTool"
Optimization="2"
EnableIntrinsicFunctions="true"
OmitFramePointers="true"
AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\dictBuilder"
PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_HEAPMODE=0;ZSTD_LEGACY_SUPPORT=0;WIN32;NDEBUG;_CONSOLE"
RuntimeLibrary="0"
EnableFunctionLevelLinking="true"
UsePrecompiledHeader="0"
WarningLevel="4"
DebugInformationFormat="3"
/>
<Tool
Name="VCManagedResourceCompilerTool"
/>
<Tool
Name="VCResourceCompilerTool"
/>
<Tool
Name="VCPreLinkEventTool"
/>
<Tool
Name="VCLinkerTool"
LinkIncremental="1"
GenerateDebugInformation="true"
SubSystem="1"
OptimizeReferences="2"
EnableCOMDATFolding="2"
TargetMachine="1"
/>
<Tool
Name="VCALinkTool"
/>
<Tool
Name="VCManifestTool"
/>
<Tool
Name="VCXDCMakeTool"
/>
<Tool
Name="VCBscMakeTool"
/>
<Tool
Name="VCFxCopTool"
/>
<Tool
Name="VCAppVerifierTool"
/>
<Tool
Name="VCPostBuildEventTool"
/>
</Configuration>
<Configuration
Name="Debug|x64"
OutputDirectory="$(SolutionDir)bin\$(PlatformName)\$(ConfigurationName)"
IntermediateDirectory="$(PlatformName)\$(ConfigurationName)"
ConfigurationType="2"
CharacterSet="2"
>
<Tool
Name="VCPreBuildEventTool"
/>
<Tool
Name="VCCustomBuildTool"
/>
<Tool
Name="VCXMLDataGeneratorTool"
/>
<Tool
Name="VCWebServiceProxyGeneratorTool"
/>
<Tool
Name="VCMIDLTool"
TargetEnvironment="3"
/>
<Tool
Name="VCCLCompilerTool"
Optimization="0"
AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\dictBuilder"
PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_HEAPMODE=0;ZSTD_LEGACY_SUPPORT=0;WIN32;_DEBUG;_CONSOLE"
MinimalRebuild="true"
BasicRuntimeChecks="3"
RuntimeLibrary="3"
UsePrecompiledHeader="0"
WarningLevel="4"
WarnAsError="true"
DebugInformationFormat="3"
/>
<Tool
Name="VCManagedResourceCompilerTool"
/>
<Tool
Name="VCResourceCompilerTool"
/>
<Tool
Name="VCPreLinkEventTool"
/>
<Tool
Name="VCLinkerTool"
LinkIncremental="2"
GenerateDebugInformation="true"
SubSystem="1"
TargetMachine="17"
/>
<Tool
Name="VCALinkTool"
/>
<Tool
Name="VCManifestTool"
/>
<Tool
Name="VCXDCMakeTool"
/>
<Tool
Name="VCBscMakeTool"
/>
<Tool
Name="VCFxCopTool"
/>
<Tool
Name="VCAppVerifierTool"
/>
<Tool
Name="VCPostBuildEventTool"
/>
</Configuration>
<Configuration
Name="Release|x64"
OutputDirectory="$(SolutionDir)bin\$(PlatformName)\$(ConfigurationName)"
IntermediateDirectory="$(PlatformName)\$(ConfigurationName)"
ConfigurationType="2"
CharacterSet="2"
WholeProgramOptimization="1"
>
<Tool
Name="VCPreBuildEventTool"
/>
<Tool
Name="VCCustomBuildTool"
/>
<Tool
Name="VCXMLDataGeneratorTool"
/>
<Tool
Name="VCWebServiceProxyGeneratorTool"
/>
<Tool
Name="VCMIDLTool"
TargetEnvironment="3"
/>
<Tool
Name="VCCLCompilerTool"
Optimization="2"
EnableIntrinsicFunctions="true"
OmitFramePointers="true"
AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\dictBuilder"
PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_HEAPMODE=0;ZSTD_LEGACY_SUPPORT=0;WIN32;NDEBUG;_CONSOLE"
RuntimeLibrary="0"
EnableFunctionLevelLinking="true"
UsePrecompiledHeader="0"
WarningLevel="4"
DebugInformationFormat="3"
/>
<Tool
Name="VCManagedResourceCompilerTool"
/>
<Tool
Name="VCResourceCompilerTool"
/>
<Tool
Name="VCPreLinkEventTool"
/>
<Tool
Name="VCLinkerTool"
LinkIncremental="1"
GenerateDebugInformation="true"
SubSystem="1"
OptimizeReferences="2"
EnableCOMDATFolding="2"
TargetMachine="17"
/>
<Tool
Name="VCALinkTool"
/>
<Tool
Name="VCManifestTool"
/>
<Tool
Name="VCXDCMakeTool"
/>
<Tool
Name="VCBscMakeTool"
/>
<Tool
Name="VCFxCopTool"
/>
<Tool
Name="VCAppVerifierTool"
/>
<Tool
Name="VCPostBuildEventTool"
/>
</Configuration>
</Configurations>
<References>
</References>
<Files>
<Filter
Name="Source Files"
Filter="cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx"
UniqueIdentifier="{4FC737F1-C7A5-4376-A066-2A32D752A2FF}"
>
<File
RelativePath="..\..\..\lib\dictBuilder\divsufsort.c"
>
</File>
<File
RelativePath="..\..\..\lib\common\entropy_common.c"
>
</File>
<File
RelativePath="..\..\..\lib\compress\fse_compress.c"
>
</File>
<File
RelativePath="..\..\..\lib\common\fse_decompress.c"
>
</File>
<File
RelativePath="..\..\..\lib\compress\huf_compress.c"
>
</File>
<File
RelativePath="..\..\..\lib\decompress\huf_decompress.c"
>
</File>
<File
RelativePath="..\..\..\lib\common\xxhash.c"
>
</File>
<File
RelativePath="..\..\..\lib\compress\zbuff_compress.c"
>
</File>
<File
RelativePath="..\..\..\lib\decompress\zbuff_decompress.c"
>
</File>
<File
RelativePath="..\..\..\lib\dictBuilder\zdict.c"
>
</File>
<File
RelativePath="..\..\..\lib\common\zstd_common.c"
>
</File>
<File
RelativePath="..\..\..\lib\compress\zstd_compress.c"
>
</File>
<File
RelativePath="..\..\..\lib\decompress\zstd_decompress.c"
>
</File>
</Filter>
<Filter
Name="Header Files"
Filter="h;hpp;hxx;hm;inl;inc;xsd"
UniqueIdentifier="{93995380-89BD-4b04-88EB-625FBE52EBFB}"
>
<File
RelativePath="..\..\..\lib\common\bitstream.h"
>
</File>
<File
RelativePath="..\..\..\lib\dictBuilder\divsufsort.h"
>
</File>
<File
RelativePath="..\..\..\lib\common\error_private.h"
>
</File>
<File
RelativePath="..\..\..\lib\common\error_public.h"
>
</File>
<File
RelativePath="..\..\..\lib\common\fse.h"
>
</File>
<File
RelativePath="..\..\..\lib\common\fse_static.h"
>
</File>
<File
RelativePath="..\..\..\lib\common\huf.h"
>
</File>
<File
RelativePath="..\..\..\lib\common\huf_static.h"
>
</File>
<File
RelativePath="..\..\..\lib\common\mem.h"
>
</File>
<File
RelativePath="..\..\..\lib\common\xxhash.h"
>
</File>
<File
RelativePath="..\..\..\lib\common\zbuff.h"
>
</File>
<File
RelativePath="..\..\..\lib\common\zbuff_static.h"
>
</File>
<File
RelativePath="..\..\..\lib\dictBuilder\zdict.h"
>
</File>
<File
RelativePath="..\..\..\lib\dictBuilder\zdict_static.h"
>
</File>
<File
RelativePath="..\..\..\lib\zstd.h"
>
</File>
<File
RelativePath="..\..\..\lib\common\zstd_internal.h"
>
</File>
<File
RelativePath="..\..\..\lib\legacy\zstd_legacy.h"
>
</File>
<File
RelativePath="..\..\..\lib\compress\zstd_opt.h"
>
</File>
<File
RelativePath="..\..\..\lib\common\zstd_static.h"
>
</File>
<File
RelativePath="..\..\..\lib\legacy\zstd_v01.h"
>
</File>
<File
RelativePath="..\..\..\lib\legacy\zstd_v02.h"
>
</File>
<File
RelativePath="..\..\..\lib\legacy\zstd_v03.h"
>
</File>
<File
RelativePath="..\..\..\lib\legacy\zstd_v04.h"
>
</File>
<File
RelativePath="..\..\..\lib\legacy\zstd_v05.h"
>
</File>
</Filter>
<Filter
Name="Resource Files"
Filter="rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav"
UniqueIdentifier="{67DA6AB6-F800-4c08-8B7A-83BB121AAD01}"
>
</Filter>
</Files>
<Globals>
</Globals>
</VisualStudioProject>
projects/VS2010/CompileAsCpp.props
+8
View File
@@ -0,0 +1,8 @@
<?xml version="1.0" encoding="utf-8"?>
<Project xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<ItemDefinitionGroup>
<ClCompile>
<CompileAs>CompileAsCpp</CompileAs>
</ClCompile>
</ItemDefinitionGroup>
</Project>
projects/VS2010/zstd/zstd.vcxproj
+1 -1
View File
@@ -219,4 +219,4 @@
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
<ImportGroup Label="ExtensionTargets">
</ImportGroup>
</Project>
</Project>
tests/.gitignore
+1
View File
@@ -2,6 +2,7 @@
zstdtest
speedTest
versionsTest
namespaceTest
# Local script
startSpeedTest
tests/Makefile
+11 -6
View File
@@ -1,6 +1,7 @@
# ##########################################################################
# Zstd tests - Makefile
# Based on LZ4 version test, by Takayuki Matsuoka - 2015-2016
# Copyright (C) Yann Collet 2015-2016
# All rights reserved.
#
# GPL v2 License
#
@@ -24,18 +25,22 @@
# versionstest : Compatibility test between zstd versions stored on Github (v0.1+)
# ##########################################################################
PYTHON?= python3
TESTDIR := zstdtest
PYTHON ?= python3
TESTARTEFACT := versionsTest namespaceTest
.PHONY: default all clean versionsTest
.PHONY: default all clean namespaceTest versionsTest
default: all
all: versionsTest
all: namespaceTest versionsTest
namespaceTest:
if $(CC) namespaceTest.c ../lib/common/xxhash.c -o $@ ; then echo compilation should fail; exit 1 ; fi
$(RM) $@
versionsTest:
$(PYTHON) test-zstd-versions.py
clean:
@rm -fR $(TESTDIR)
@$(RM) -fR $(TESTARTEFACT)
@echo Cleaning completed
tests/namespaceTest.c
+38
View File
@@ -0,0 +1,38 @@
/* ##########################################################################
# namespaceTest
# ensure xxhash namespace emulation is properly triggered
# Copyright (C) Yann Collet 2016
#
# GPL v2 License
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
#
# You can contact the author at :
# - zstd homepage : http://www.zstd.net/
# ########################################################################*/
#include <stddef.h> /* size_t */
#include <string.h> /* strlen */
/* symbol definition */
extern unsigned XXH32(const void* src, size_t srcSize, unsigned seed);
int main(int argc, const char** argv)
{
const char* exename = argv[0];
unsigned result = XXH32(exename, strlen(exename), argc);
return !result;
}
tests/test-zstd-speed.py
+35 -14
View File
@@ -6,17 +6,26 @@ import string
import subprocess
import time
import traceback
import hashlib
script_version = 'v0.8.0 (2016-08-03)'
default_repo_url = 'https://github.com/Cyan4973/zstd.git'
working_dir_name = 'speedTest'
working_path = os.getcwd() + '/' + working_dir_name # /path/to/zstd/tests/speedTest
clone_path = working_path + '/' + 'zstd' # /path/to/zstd/tests/speedTest/zstd
email_header = '[ZSTD_speedTest]'
email_header = 'ZSTD_speedTest'
pid = str(os.getpid())
verbose = False
def hashfile(hasher, fname, blocksize=65536):
with open(fname, "rb") as f:
for chunk in iter(lambda: f.read(blocksize), b""):
hasher.update(chunk)
return hasher.hexdigest()
def log(text):
print(time.strftime("%Y/%m/%d %H:%M:%S") + ' - ' + text)
@@ -64,7 +73,7 @@ def send_email_with_attachments(branch, commit, last_commit, args, text, results
with open(logFileName, "w") as myfile:
myfile.writelines(text)
myfile.close()
email_topic = '%s:%s Warning for %s:%s last_commit=%s speed<%s ratio<%s' \
email_topic = '[%s:%s] Warning for %s:%s last_commit=%s speed<%s ratio<%s' \
% (email_header, pid, branch, commit, last_commit,
args.lowerLimit, args.ratioLimit)
if have_mutt:
@@ -117,7 +126,7 @@ def get_last_results(resultsFileName):
return commit, csize, cspeed, dspeed
def benchmark_and_compare(branch, commit, last_commit, args, executableName, resultsFileName,
def benchmark_and_compare(branch, commit, last_commit, args, executableName, md5sum, resultsFileName,
testFilePath, fileName, last_csize, last_cspeed, last_dspeed):
sleepTime = 30
while os.getloadavg()[0] > args.maxLoadAvg:
@@ -149,7 +158,7 @@ def benchmark_and_compare(branch, commit, last_commit, args, executableName, res
if (float(last_csize[i])/csize[i] < args.ratioLimit):
text += "WARNING: %s -%d cSize=%d last_cSize=%d diff=%.4f %s\n" % (executableName, i+1, csize[i], last_csize[i], float(last_csize[i])/csize[i], fileName)
if text:
text = args.message + ("\nmaxLoadAvg=%s load average at start=%s end=%s last_commit=%s\n" % (args.maxLoadAvg, start_load, end_load, last_commit)) + text
text = args.message + ("\nmaxLoadAvg=%s load average at start=%s end=%s last_commit=%s md5=%s\n" % (args.maxLoadAvg, start_load, end_load, last_commit, md5sum)) + text
return text
@@ -162,13 +171,13 @@ def update_config_file(branch, commit):
return last_commit
def double_check(branch, commit, args, executableName, resultsFileName, filePath, fileName):
def double_check(branch, commit, args, executableName, md5sum, resultsFileName, filePath, fileName):
last_commit, csize, cspeed, dspeed = get_last_results(resultsFileName)
if not args.dry_run:
text = benchmark_and_compare(branch, commit, last_commit, args, executableName, resultsFileName, filePath, fileName, csize, cspeed, dspeed)
text = benchmark_and_compare(branch, commit, last_commit, args, executableName, md5sum, resultsFileName, filePath, fileName, csize, cspeed, dspeed)
if text:
log("WARNING: redoing tests for branch %s: commit %s" % (branch, commit))
text = benchmark_and_compare(branch, commit, last_commit, args, executableName, resultsFileName, filePath, fileName, csize, cspeed, dspeed)
text = benchmark_and_compare(branch, commit, last_commit, args, executableName, md5sum, resultsFileName, filePath, fileName, csize, cspeed, dspeed)
return text
@@ -176,19 +185,31 @@ def test_commit(branch, commit, last_commit, args, testFilePaths, have_mutt, hav
local_branch = string.split(branch, '/')[1]
version = local_branch.rpartition('-')[2] + '_' + commit
if not args.dry_run:
execute('make -C programs clean zstd MOREFLAGS="-DZSTD_GIT_COMMIT=%s" && make -B -C programs zstd32 MOREFLAGS="-DZSTD_GIT_COMMIT=%s"' % (version, version))
execute('make -C programs clean zstd CC=clang MOREFLAGS="-Werror -Wconversion -Wno-sign-conversion -DZSTD_GIT_COMMIT=%s" && ' % version +
'mv programs/zstd programs/zstd_clang && ' +
'make -C programs clean zstd MOREFLAGS="-DZSTD_GIT_COMMIT=%s" && ' % version +
'make -B -C programs zstd32 MOREFLAGS="-DZSTD_GIT_COMMIT=%s"' % version)
md5_zstd = hashfile(hashlib.md5(), clone_path + '/programs/zstd')
md5_zstd32 = hashfile(hashlib.md5(), clone_path + '/programs/zstd32')
md5_zstd_clang = hashfile(hashlib.md5(), clone_path + '/programs/zstd_clang')
print("md5(zstd)=%s\nmd5(zstd32)=%s\nmd5(zstd32_clang)=%s" % (md5_zstd, md5_zstd32, md5_zstd_clang))
logFileName = working_path + "/log_" + branch.replace("/", "_") + ".txt"
text_to_send = []
results_files = ""
for filePath in testFilePaths:
fileName = filePath.rpartition('/')[2]
resultsFileName = working_path + "/results_" + branch.replace("/", "_") + "_" + fileName.replace(".", "_") + ".txt"
text = double_check(branch, commit, args, 'zstd', resultsFileName, filePath, fileName)
text = double_check(branch, commit, args, 'zstd', md5_zstd, resultsFileName, filePath, fileName)
if text:
text_to_send.append(text)
results_files += resultsFileName + " "
resultsFileName = working_path + "/results32_" + branch.replace("/", "_") + "_" + fileName.replace(".", "_") + ".txt"
text = double_check(branch, commit, args, 'zstd32', resultsFileName, filePath, fileName)
text = double_check(branch, commit, args, 'zstd32', md5_zstd32, resultsFileName, filePath, fileName)
if text:
text_to_send.append(text)
results_files += resultsFileName + " "
resultsFileName = working_path + "/resultsClang_" + branch.replace("/", "_") + "_" + fileName.replace(".", "_") + ".txt"
text = double_check(branch, commit, args, 'zstd_clang', md5_zstd_clang, resultsFileName, filePath, fileName)
if text:
text_to_send.append(text)
results_files += resultsFileName + " "
@@ -263,7 +284,7 @@ if __name__ == '__main__':
log("ERROR: %s already exists, exiting" % pidfile)
exit(1)
send_email(args.emails, email_header + ':%s test-zstd-speed.py has been started' % pid, args.message, have_mutt, have_mail)
send_email(args.emails, '[%s:%s] test-zstd-speed.py %s has been started' % (email_header, pid, script_version), args.message, have_mutt, have_mail)
file(pidfile, 'w').write(pid)
while True:
@@ -288,11 +309,11 @@ if __name__ == '__main__':
time.sleep(args.sleepTime)
except Exception as e:
stack = traceback.format_exc()
email_topic = '%s:%s ERROR in %s:%s' % (email_header, pid, branch, commit)
email_topic = '[%s:%s] ERROR in %s:%s' % (email_header, pid, branch, commit)
send_email(args.emails, email_topic, stack, have_mutt, have_mail)
print(stack)
time.sleep(args.sleepTime)
except KeyboardInterrupt:
os.unlink(pidfile)
send_email(args.emails, email_header + ':%s test-zstd-speed.py has been stopped' % pid, args.message, have_mutt, have_mail)
send_email(args.emails, '[%s:%s] test-zstd-speed.py %s has been stopped' % (email_header, pid, script_version), args.message, have_mutt, have_mail)
exit(0)
zlibWrapper/zstd_zlibwrapper.c
+16 -3
View File
@@ -29,14 +29,15 @@
- zstd source repository : https://github.com/Cyan4973/zstd
*/
#include <stdarg.h> /* va_list, for z_gzprintf */
#include <stdio.h> /* vsprintf */
#include <stdarg.h> /* va_list, for z_gzprintf */
#include <zlib.h>
#include "zstd_zlibwrapper.h"
#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_MAGICNUMBER */
#include "zstd.h"
#define ZBUFF_STATIC_LINKING_ONLY /* ZBUFF_createCCtx_advanced */
#include "zbuff.h"
#include "zstd_internal.h" /* defaultCustomMem */
#include "zstd_internal.h" /* defaultCustomMem */
#define Z_INFLATE_SYNC 8
@@ -226,7 +227,7 @@ ZEXTERN int ZEXPORT z_deflate OF((z_streamp strm, int flush))
if (flush == Z_FULL_FLUSH) FINISH_WITH_ERR(strm, "Z_FULL_FLUSH is not supported!");
if (flush == Z_FINISH || flush == Z_FULL_FLUSH) {
if (flush == Z_FINISH) {
size_t bytesLeft;
size_t dstCapacity = strm->avail_out;
if (zwc->bytesLeft) {
@@ -245,6 +246,18 @@ ZEXTERN int ZEXPORT z_deflate OF((z_streamp strm, int flush))
if (flush == Z_FINISH && bytesLeft == 0) return Z_STREAM_END;
zwc->bytesLeft = bytesLeft;
}
if (flush == Z_SYNC_FLUSH) {
size_t bytesLeft;
size_t dstCapacity = strm->avail_out;
bytesLeft = ZBUFF_compressFlush(zwc->zbc, strm->next_out, &dstCapacity);
LOG_WRAPPER("ZBUFF_compressFlush avail_out=%d dstCapacity=%d bytesLeft=%d\n", (int)strm->avail_out, (int)dstCapacity, (int)bytesLeft);
if (ZSTD_isError(bytesLeft)) return Z_MEM_ERROR;
strm->next_out += dstCapacity;
strm->total_out += dstCapacity;
strm->avail_out -= dstCapacity;
zwc->bytesLeft = bytesLeft;
}
return Z_OK;
}
zstd.rb
+2 -2
View File
@@ -1,8 +1,8 @@
class Zstd < Formula
desc "Zstandard - Fast real-time compression algorithm"
homepage "http://www.zstd.net/"
url "https://github.com/Cyan4973/zstd/archive/v0.7.4.tar.gz"
sha256 "35ab3a5084d0194e9ff08e702edb6f507eab1bfb8c09c913639241cec852e2b7"
url "https://github.com/Cyan4973/zstd/archive/v0.7.5.tar.gz"
sha256 "6800defac9a93ddb1d9673d62c78526800df71cf9f353456f8e488ba4de51061"
def install
system "make", "install", "PREFIX=#{prefix}"
zstd_compression_format.md
+242 -239
View File
@@ -60,8 +60,9 @@ explaining which parameter is unsupported.
Overall conventions
-----------
In this document square brackets i.e. `[` and `]` are used to indicate optional fields or parameters.
In this document:
- square brackets i.e. `[` and `]` are used to indicate optional fields or parameters.
- a naming convention for identifiers is `Mixed_Case_With_Underscores`
Definitions
-----------
@@ -106,7 +107,7 @@ The structure of a single Zstandard frame is following:
__`Magic_Number`__
4 Bytes, Little-endian format.
4 Bytes, little-endian format.
Value : 0xFD2FB527
__`Frame_Header`__
@@ -122,7 +123,7 @@ __`Content_Checksum`__
An optional 32-bit checksum, only present if `Content_Checksum_flag` is set.
The content checksum is the result
of [xxh64() hash function](https://www.xxHash.com)
of [xxh64() hash function](http://www.xxhash.org)
digesting the original (decoded) data as input, and a seed of zero.
The low 4 bytes of the checksum are stored in little endian format.
@@ -214,11 +215,11 @@ __`Dictionary_ID_flag`__
This is a 2-bits flag (`= FHD & 3`),
telling if a dictionary ID is provided within the header.
It also specifies the size of this field.
It also specifies the size of this field as `Field_Size`.
| Value | 0 | 1 | 2 | 3 |
| -------- | --- | --- | --- | --- |
|Field size| 0 | 1 | 2 | 4 |
|`Flag_Value`| 0 | 1 | 2 | 3 |
| ---------- | --- | --- | --- | --- |
|`Field_Size`| 0 | 1 | 2 | 4 |
### `Window_Descriptor`
@@ -230,22 +231,22 @@ The `Window_Descriptor` byte is optional. It is absent when `Single_Segment_flag
In this case, the maximum back-reference distance is the content size itself,
which can be any value from 1 to 2^64-1 bytes (16 EB).
| Bit numbers | 7-3 | 0-2 |
| ----------- | -------- | -------- |
| Field name | Exponent | Mantissa |
| Bit numbers | 7-3 | 0-2 |
| ----------- | ---------- | ---------- |
| Field name | `Exponent` | `Mantissa` |
Maximum distance is given by the following formulae :
```
windowLog = 10 + Exponent;
windowBase = 1 << windowLog;
windowAdd = (windowBase / 8) * Mantissa;
windowSize = windowBase + windowAdd;
Window_Size = windowBase + windowAdd;
```
The minimum window size is 1 KB.
The maximum size is `15*(1<<38)` bytes, which is 1.875 TB.
To properly decode compressed data,
a decoder will need to allocate a buffer of at least `windowSize` bytes.
a decoder will need to allocate a buffer of at least `Window_Size` bytes.
In order to preserve decoder from unreasonable memory requirements,
a decoder can refuse a compressed frame
@@ -287,7 +288,7 @@ the following ranges are reserved for future use and should not be used :
This is the original (uncompressed) size. This information is optional.
The `Field_Size` is provided according to value of `Frame_Content_Size_flag`.
The `Field_Size` can be equal to 0 (not present), 1, 2, 4 or 8 bytes.
Format is Little-endian.
Format is little-endian.
| `Field_Size` | Range |
| ------------ | ---------- |
@@ -309,7 +310,7 @@ The structure of `Data_Block` is following:
|:------------:|:------------:|:------------:|:---------------:|
| 1 bit | 2 bits | 21 bits | n bytes |
The block header uses 3-bytes.
The block header (`Last_Block`, `Block_Type`, and `Block_Size`) uses 3-bytes.
__`Last_Block`__
@@ -379,7 +380,7 @@ Skippable frames defined in this specification are compatible with [LZ4] ones.
__`Magic_Number`__
4 Bytes, Little-endian format.
4 Bytes, little-endian format.
Value : 0x184D2A5X, which means any value from 0x184D2A50 to 0x184D2A5F.
All 16 values are valid to identify a skippable frame.
@@ -387,7 +388,7 @@ __`Frame_Size`__
This is the size, in bytes, of the following `User_Data`
(without including the magic number nor the size field itself).
This field is represented using 4 Bytes, Little-endian format, unsigned 32-bits.
This field is represented using 4 Bytes, little-endian format, unsigned 32-bits.
This means `User_Data` can’t be bigger than (2^32-1) bytes.
__`User_Data`__
@@ -403,107 +404,108 @@ in order to properly allocate destination buffer.
See [`Data_Block`](#the-structure-of-data_block) for more details.
A compressed block consists of 2 sections :
- [Literals section](#literals-section)
- [Sequences section](#sequences-section)
- [`Literals_Section`](#literals_section)
- [`Sequences_Section`](#sequences_section)
### Prerequisites
To decode a compressed block, the following elements are necessary :
- Previous decoded blocks, up to a distance of `windowSize`,
- Previous decoded blocks, up to a distance of `Window_Size`,
or all previous blocks when `Single_Segment_flag` is set.
- List of "recent offsets" from previous compressed block.
- Decoding tables of previous compressed block for each symbol type
(literals, litLength, matchLength, offset).
### Literals section
### `Literals_Section`
During sequence phase, literals will be entangled with match copy operations.
All literals are regrouped in the first part of the block.
They can be decoded first, and then copied during sequence operations,
or they can be decoded on the flow, as needed by sequence commands.
| Literals section header | [Huffman Tree Description] | Stream1 | [Stream2] | [Stream3] | [Stream4] |
| ----------------------- | -------------------------- | ------- | --------- | --------- | --------- |
| `Literals_Section_Header` | [`Huffman_Tree_Description`] | Stream1 | [Stream2] | [Stream3] | [Stream4] |
| ------------------------- | ---------------------------- | ------- | --------- | --------- | --------- |
Literals can be stored uncompressed or compressed using Huffman prefix codes.
When compressed, an optional tree description can be present,
followed by 1 or 4 streams.
#### Literals section header
#### `Literals_Section_Header`
Header is in charge of describing how literals are packed.
It's a byte-aligned variable-size bitfield, ranging from 1 to 5 bytes,
using little-endian convention.
| Literals Block Type | sizes format | regenerated size | [compressed size] |
| ------------------- | ------------ | ---------------- | ----------------- |
| 2 bits | 1 - 2 bits | 5 - 20 bits | 0 - 18 bits |
| `Literals_Block_Type` | `Size_Format` | `Regenerated_Size` | [`Compressed_Size`] |
| --------------------- | ------------- | ------------------ | ----------------- |
| 2 bits | 1 - 2 bits | 5 - 20 bits | 0 - 18 bits |
In this representation, bits on the left are smallest bits.
__Literals Block Type__ :
__`Literals_Block_Type`__
This field uses 2 lowest bits of first byte, describing 4 different block types :
| Value | 0 | 1 | 2 | 3 |
| ------------------- | --- | --- | ---------- | ----------- |
| Literals Block Type | Raw | RLE | Compressed | RepeatStats |
| Value | 0 | 1 | 2 | 3 |
| --------------------- | -------------------- | -------------------- | --------------------------- | ----------------------------- |
| `Literals_Block_Type` | `Raw_Literals_Block` | `RLE_Literals_Block` | `Compressed_Literals_Block` | `Repeat_Stats_Literals_Block` |
- Raw literals block - Literals are stored uncompressed.
- RLE literals block - Literals consist of a single byte value repeated N times.
- Compressed literals block - This is a standard huffman-compressed block,
starting with a huffman tree description.
- `Raw_Literals_Block` - Literals are stored uncompressed.
- `RLE_Literals_Block` - Literals consist of a single byte value repeated N times.
- `Compressed_Literals_Block` - This is a standard Huffman-compressed block,
starting with a Huffman tree description.
See details below.
- Repeat Stats literals block - This is a huffman-compressed block,
using huffman tree _from previous huffman-compressed literals block_.
- `Repeat_Stats_Literals_Block` - This is a Huffman-compressed block,
using Huffman tree _from previous Huffman-compressed literals block_.
Huffman tree description will be skipped.
__Sizes format__ :
__`Size_Format`__
Sizes format are divided into 2 families :
`Size_Format` is divided into 2 families :
- For compressed block, it requires to decode both the compressed size
and the decompressed size. It will also decode the number of streams.
- For Raw or RLE blocks, it's enough to decode the size to regenerate.
- For `Compressed_Block`, it requires to decode both `Compressed_Size`
and `Regenerated_Size` (the decompressed size). It will also decode the number of streams.
- For `Raw_Block` and `RLE_Block` it's enough to decode `Regenerated_Size`.
For values spanning several bytes, convention is Little-endian.
For values spanning several bytes, convention is little-endian.
__Sizes format for Raw and RLE literals block__ :
__`Size_Format` for `Raw_Literals_Block` and `RLE_Literals_Block`__ :
- Value : x0 : Regenerated size uses 5 bits (0-31).
Total literal header size is 1 byte.
`size = h[0]>>3;`
- Value : 01 : Regenerated size uses 12 bits (0-4095).
Total literal header size is 2 bytes.
`size = (h[0]>>4) + (h[1]<<4);`
- Value : 11 : Regenerated size uses 20 bits (0-1048575).
Total literal header size is 3 bytes.
`size = (h[0]>>4) + (h[1]<<4) + (h[2]<<12);`
- Value : x0 : `Regenerated_Size` uses 5 bits (0-31).
`Literals_Section_Header` has 1 byte.
`Regenerated_Size = Header[0]>>3`
- Value : 01 : `Regenerated_Size` uses 12 bits (0-4095).
`Literals_Section_Header` has 2 bytes.
`Regenerated_Size = (Header[0]>>4) + (Header[1]<<4)`
- Value : 11 : `Regenerated_Size` uses 20 bits (0-1048575).
`Literals_Section_Header` has 3 bytes.
`Regenerated_Size = (Header[0]>>4) + (Header[1]<<4) + (Header[2]<<12)`
Note : it's allowed to represent a short value (ex : `13`)
using a long format, accepting the reduced compacity.
__Sizes format for Compressed literals block and Repeat Stats literals block__ :
__`Size_Format` for `Compressed_Literals_Block` and `Repeat_Stats_Literals_Block`__ :
- Value : 00 : _Single stream_.
Compressed and regenerated sizes use 10 bits (0-1023).
Total literal header size is 3 bytes.
Both `Compressed_Size` and `Regenerated_Size` use 10 bits (0-1023).
`Literals_Section_Header` has 3 bytes.
- Value : 01 : 4 streams.
Compressed and regenerated sizes use 10 bits (0-1023).
Total literal header size is 3 bytes.
Both `Compressed_Size` and `Regenerated_Size` use 10 bits (0-1023).
`Literals_Section_Header` has 3 bytes.
- Value : 10 : 4 streams.
Compressed and regenerated sizes use 14 bits (0-16383).
Total literal header size is 4 bytes.
Both `Compressed_Size` and `Regenerated_Size` use 14 bits (0-16383).
`Literals_Section_Header` has 4 bytes.
- Value : 11 : 4 streams.
Compressed and regenerated sizes use 18 bits (0-262143).
Total literal header size is 5 bytes.
Both `Compressed_Size` and `Regenerated_Size` use 18 bits (0-262143).
`Literals_Section_Header` has 5 bytes.
Compressed and regenerated size fields follow little-endian convention.
Both `Compressed_Size` and `Regenerated_Size` fields follow little-endian convention.
#### Huffman Tree description
This section is only present when literals block type is `Compressed` (`0`).
#### `Huffman_Tree_Description`
This section is only present when `Literals_Block_Type` type is `Compressed_Block` (`2`).
Prefix coding represents symbols from an a priori known alphabet
by bit sequences (codewords), one codeword for each symbol,
@@ -525,57 +527,57 @@ This specification limits maximum code length to 11 bits.
##### Representation
All literal values from zero (included) to last present one (excluded)
are represented by `weight` values, from 0 to `maxBits`.
Transformation from `weight` to `nbBits` follows this formulae :
`nbBits = weight ? maxBits + 1 - weight : 0;` .
The last symbol's weight is deduced from previously decoded ones,
are represented by `Weight` values, from 0 to `Max_Number_of_Bits`.
Transformation from `Weight` to `Number_of_Bits` follows this formulae :
`Number_of_Bits = Weight ? Max_Number_of_Bits + 1 - Weight : 0` .
The last symbol's `Weight` is deduced from previously decoded ones,
by completing to the nearest power of 2.
This power of 2 gives `maxBits`, the depth of the current tree.
This power of 2 gives `Max_Number_of_Bits`, the depth of the current tree.
__Example__ :
Let's presume the following huffman tree must be described :
Let's presume the following Huffman tree must be described :
| literal | 0 | 1 | 2 | 3 | 4 | 5 |
| ------- | --- | --- | --- | --- | --- | --- |
| nbBits | 1 | 2 | 3 | 0 | 4 | 4 |
| literal | 0 | 1 | 2 | 3 | 4 | 5 |
| ---------------- | --- | --- | --- | --- | --- | --- |
| `Number_of_Bits` | 1 | 2 | 3 | 0 | 4 | 4 |
The tree depth is 4, since its smallest element uses 4 bits.
Value `5` will not be listed, nor will values above `5`.
Values from `0` to `4` will be listed using `weight` instead of `nbBits`.
Weight formula is : `weight = nbBits ? maxBits + 1 - nbBits : 0;`
Values from `0` to `4` will be listed using `Weight` instead of `Number_of_Bits`.
Weight formula is : `Weight = Number_of_Bits ? Max_Number_of_Bits + 1 - Number_of_Bits : 0`.
It gives the following serie of weights :
| weights | 4 | 3 | 2 | 0 | 1 |
| ------- | --- | --- | --- | --- | --- |
| literal | 0 | 1 | 2 | 3 | 4 |
| `Weight` | 4 | 3 | 2 | 0 | 1 |
| -------- | --- | --- | --- | --- | --- |
| literal | 0 | 1 | 2 | 3 | 4 |
The decoder will do the inverse operation :
having collected weights of literals from `0` to `4`,
it knows the last literal, `5`, is present with a non-zero weight.
The weight of `5` can be deducted by joining to the nearest power of 2.
Sum of 2^(weight-1) (excluding 0) is :
`8 + 4 + 2 + 0 + 1 = 15`
Sum of `2^(Weight-1)` (excluding 0) is :
`8 + 4 + 2 + 0 + 1 = 15`.
Nearest power of 2 is 16.
Therefore, `maxBits = 4` and `weight[5] = 1`.
Therefore, `Max_Number_of_Bits = 4` and `Weight[5] = 1`.
##### Huffman Tree header
This is a single byte value (0-255),
which tells how to decode the list of weights.
- if headerByte >= 128 : this is a direct representation,
where each weight is written directly as a 4 bits field (0-15).
The full representation occupies `((nbSymbols+1)/2)` bytes,
meaning it uses a last full byte even if nbSymbols is odd.
`nbSymbols = headerByte - 127;`.
Note that maximum nbSymbols is 255-127 = 128.
- if `headerByte` >= 128 : this is a direct representation,
where each `Weight` is written directly as a 4 bits field (0-15).
The full representation occupies `((Number_of_Symbols+1)/2)` bytes,
meaning it uses a last full byte even if `Number_of_Symbols` is odd.
`Number_of_Symbols = headerByte - 127`.
Note that maximum `Number_of_Symbols` is 255-127 = 128.
A larger serie must necessarily use FSE compression.
- if headerByte < 128 :
- if `headerByte` < 128 :
the serie of weights is compressed by FSE.
The length of the FSE-compressed serie is `headerByte` (0-127).
##### FSE (Finite State Entropy) compression of huffman weights
##### FSE (Finite State Entropy) compression of Huffman weights
The serie of weights is compressed using FSE compression.
It's a single bitstream with 2 interleaved states,
@@ -590,7 +592,7 @@ and last symbol value is not represented.
An FSE bitstream starts by a header, describing probabilities distribution.
It will create a Decoding Table.
Table must be pre-allocated, which requires to support a maximum accuracy.
For a list of huffman weights, maximum accuracy is 7 bits.
For a list of Huffman weights, maximum accuracy is 7 bits.
FSE header is [described in relevant chapter](#fse-distribution-table--condensed-format),
and so is [FSE bitstream](#bitstream).
@@ -602,31 +604,31 @@ by tracking bitStream overflow condition.
When both states have overflowed the bitstream, end is reached.
##### Conversion from weights to huffman prefix codes
##### Conversion from weights to Huffman prefix codes
All present symbols shall now have a `weight` value.
It is possible to transform weights into nbBits, using this formula :
`nbBits = nbBits ? maxBits + 1 - weight : 0;` .
All present symbols shall now have a `Weight` value.
It is possible to transform weights into Number_of_Bits, using this formula :
`Number_of_Bits = Number_of_Bits ? Max_Number_of_Bits + 1 - Weight : 0` .
Symbols are sorted by weight. Within same weight, symbols keep natural order.
Symbols with a weight of zero are removed.
Symbols are sorted by `Weight`. Within same `Weight`, symbols keep natural order.
Symbols with a `Weight` of zero are removed.
Then, starting from lowest weight, prefix codes are distributed in order.
__Example__ :
Let's presume the following list of weights has been decoded :
| Literal | 0 | 1 | 2 | 3 | 4 | 5 |
| ------- | --- | --- | --- | --- | --- | --- |
| weight | 4 | 3 | 2 | 0 | 1 | 1 |
| Literal | 0 | 1 | 2 | 3 | 4 | 5 |
| -------- | --- | --- | --- | --- | --- | --- |
| `Weight` | 4 | 3 | 2 | 0 | 1 | 1 |
Sorted by weight and then natural order,
it gives the following distribution :
| Literal | 3 | 4 | 5 | 2 | 1 | 0 |
| ------------ | --- | --- | --- | --- | --- | ---- |
| weight | 0 | 1 | 1 | 2 | 3 | 4 |
| nb bits | 0 | 4 | 4 | 3 | 2 | 1 |
| prefix codes | N/A | 0000| 0001| 001 | 01 | 1 |
| Literal | 3 | 4 | 5 | 2 | 1 | 0 |
| ---------------- | --- | --- | --- | --- | --- | ---- |
| `Weight` | 0 | 1 | 1 | 2 | 3 | 4 |
| `Number_of_Bits` | 0 | 4 | 4 | 3 | 2 | 1 |
| prefix codes | N/A | 0000| 0001| 001 | 01 | 1 |
#### Literals bitstreams
@@ -634,10 +636,10 @@ it gives the following distribution :
##### Bitstreams sizes
As seen in a previous paragraph,
there are 2 flavors of huffman-compressed literals :
there are 2 flavors of Huffman-compressed literals :
single stream, and 4-streams.
4-streams is useful for CPU with multiple execution units and OoO operations.
4-streams is useful for CPU with multiple execution units and out-of-order operations.
Since each stream can be decoded independently,
it's possible to decode them up to 4x faster than a single stream,
presuming the CPU has enough parallelism available.
@@ -652,11 +654,11 @@ Regenerated size of each stream can be calculated by `(totalSize+3)/4`,
except for last one, which can be up to 3 bytes smaller, to reach `totalSize`.
Compressed size is provided explicitly : in the 4-streams variant,
bitstreams are preceded by 3 unsigned Little-Endian 16-bits values.
bitstreams are preceded by 3 unsigned little-endian 16-bits values.
Each value represents the compressed size of one stream, in order.
The last stream size is deducted from total compressed size
and from previously decoded stream sizes :
`stream4CSize = totalCSize - 6 - stream1CSize - stream2CSize - stream3CSize;`
`stream4CSize = totalCSize - 6 - stream1CSize - stream2CSize - stream3CSize`.
##### Bitstreams read and decode
@@ -675,7 +677,7 @@ Starting from the end,
it's possible to read the bitstream in a little-endian fashion,
keeping track of already used bits.
Reading the last `maxBits` bits,
Reading the last `Max_Number_of_Bits` bits,
it's then possible to compare extracted value to decoding table,
determining the symbol to decode and number of bits to discard.
@@ -685,7 +687,7 @@ hence reaching exactly its beginning position with _all_ bits consumed,
the decoding process is considered faulty.
### Sequences section
### `Sequences_Section`
A compressed block is a succession of _sequences_ .
A sequence is a literal copy command, followed by a match copy command.
@@ -695,7 +697,7 @@ A match copy command specifies an offset and a length.
The offset gives the position to copy from,
which can be within a previous block.
There are 3 symbol types, `literalLength`, `matchLength` and `offset`,
There are 3 symbol types, literals lengths, offsets and match lengths,
which are encoded together, interleaved in a single _bitstream_.
Each symbol is a _code_ in its own context,
@@ -707,126 +709,125 @@ The Sequences section starts by a header,
followed by optional Probability tables for each symbol type,
followed by the bitstream.
| Header | [LitLengthTable] | [OffsetTable] | [MatchLengthTable] | bitStream |
| ------ | ---------------- | ------------- | ------------------ | --------- |
| `Sequences_Section_Header` | [`Literals_Length_Table`] | [`Offset_Table`] | [`Match_Length_Table`] | bitStream |
| -------------------------- | ------------------------- | ---------------- | ---------------------- | --------- |
To decode the Sequence section, it's required to know its size.
This size is deducted from `blockSize - literalSectionSize`.
#### Sequences section header
#### `Sequences_Section_Header`
Consists in 2 items :
- Nb of Sequences
- Flags providing Symbol compression types
- `Number_of_Sequences`
- Symbol compression modes
__Nb of Sequences__
__`Number_of_Sequences`__
This is a variable size field, `nbSeqs`, using between 1 and 3 bytes.
This is a variable size field using between 1 and 3 bytes.
Let's call its first byte `byte0`.
- `if (byte0 == 0)` : there are no sequences.
The sequence section stops there.
Regenerated content is defined entirely by literals section.
- `if (byte0 < 128)` : `nbSeqs = byte0;` . Uses 1 byte.
- `if (byte0 < 255)` : `nbSeqs = ((byte0-128) << 8) + byte1;` . Uses 2 bytes.
- `if (byte0 == 255)`: `nbSeqs = byte1 + (byte2<<8) + 0x7F00;` . Uses 3 bytes.
- `if (byte0 < 128)` : `Number_of_Sequences = byte0` . Uses 1 byte.
- `if (byte0 < 255)` : `Number_of_Sequences = ((byte0-128) << 8) + byte1` . Uses 2 bytes.
- `if (byte0 == 255)`: `Number_of_Sequences = byte1 + (byte2<<8) + 0x7F00` . Uses 3 bytes.
__Symbol encoding modes__
__Symbol compression modes__
This is a single byte, defining the compression mode of each symbol type.
| BitNb | 7-6 | 5-4 | 3-2 | 1-0 |
| ------- | ------ | ------ | ------ | -------- |
|FieldName| LLType | OFType | MLType | Reserved |
|Bit number| 7-6 | 5-4 | 3-2 | 1-0 |
| -------- | ----------------------- | -------------- | -------------------- | ---------- |
|Field name| `Literals_Lengths_Mode` | `Offsets_Mode` | `Match_Lengths_Mode` | `Reserved` |
The last field, `Reserved`, must be all-zeroes.
`LLType`, `OFType` and `MLType` define the compression mode of
Literal Lengths, Offsets and Match Lengths respectively.
`Literals_Lengths_Mode`, `Offsets_Mode` and `Match_Lengths_Mode` define the compression mode of
literals lengths, offsets and match lengths respectively.
They follow the same enumeration :
| Value | 0 | 1 | 2 | 3 |
| ---------------- | ------ | --- | ---------- | ------ |
| Compression Mode | predef | RLE | Compressed | Repeat |
| Value | 0 | 1 | 2 | 3 |
| ------------------ | ----------------- | ---------- | --------------------- | ------------- |
| `Compression_Mode` | `Predefined_Mode` | `RLE_Mode` | `FSE_Compressed_Mode` | `Repeat_Mode` |
- "predef" : uses a pre-defined distribution table.
- "RLE" : it's a single code, repeated `nbSeqs` times.
- "Repeat" : re-use distribution table from previous compressed block.
- "Compressed" : standard FSE compression.
- `Predefined_Mode` : uses a predefined distribution table.
- `RLE_Mode` : it's a single code, repeated `Number_of_Sequences` times.
- `Repeat_Mode` : re-use distribution table from previous compressed block.
- `FSE_Compressed_Mode` : standard FSE compression.
A distribution table will be present.
It will be described in [next part](#distribution-tables).
#### Symbols decoding
##### Literal Lengths codes
##### Literals Length codes
Literal lengths codes are values ranging from `0` to `35` included.
Literals length codes are values ranging from `0` to `35` included.
They define lengths from 0 to 131071 bytes.
| Code | 0-15 |
| ------ | ---- |
| length | Code |
| nbBits | 0 |
| `Literals_Length_Code` | 0-15 |
| ---------------------- | ---------------------- |
| length | `Literals_Length_Code` |
| `Number_of_Bits` | 0 |
| `Literals_Length_Code` | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 |
| ---------------------- | ---- | ---- | ---- | ---- | ---- | ---- | ---- | ---- |
| Baseline | 16 | 18 | 20 | 22 | 24 | 28 | 32 | 40 |
| `Number_of_Bits` | 1 | 1 | 1 | 1 | 2 | 2 | 3 | 3 |
| Code | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 |
| -------- | ---- | ---- | ---- | ---- | ---- | ---- | ---- | ---- |
| Baseline | 16 | 18 | 20 | 22 | 24 | 28 | 32 | 40 |
| nb Bits | 1 | 1 | 1 | 1 | 2 | 2 | 3 | 3 |
| `Literals_Length_Code` | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 |
| ---------------------- | ---- | ---- | ---- | ---- | ---- | ---- | ---- | ---- |
| Baseline | 48 | 64 | 128 | 256 | 512 | 1024 | 2048 | 4096 |
| `Number_of_Bits` | 4 | 6 | 7 | 8 | 9 | 10 | 11 | 12 |
| Code | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 |
| -------- | ---- | ---- | ---- | ---- | ---- | ---- | ---- | ---- |
| Baseline | 48 | 64 | 128 | 256 | 512 | 1024 | 2048 | 4096 |
| nb Bits | 4 | 6 | 7 | 8 | 9 | 10 | 11 | 12 |
| Code | 32 | 33 | 34 | 35 |
| -------- | ---- | ---- | ---- | ---- |
| Baseline | 8192 |16384 |32768 |65536 |
| nb Bits | 13 | 14 | 15 | 16 |
| `Literals_Length_Code` | 32 | 33 | 34 | 35 |
| ---------------------- | ---- | ---- | ---- | ---- |
| Baseline | 8192 |16384 |32768 |65536 |
| `Number_of_Bits` | 13 | 14 | 15 | 16 |
__Default distribution__
When "compression mode" is "predef"",
a pre-defined distribution is used for FSE compression.
When `Compression_Mode` is `Predefined_Mode`,
a predefined distribution is used for FSE compression.
Below is its definition. It uses an accuracy of 6 bits (64 states).
```
short literalLengths_defaultDistribution[36] =
short literalsLength_defaultDistribution[36] =
{ 4, 3, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1,
2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 2, 1, 1, 1, 1, 1,
-1,-1,-1,-1 };
```
##### Match Lengths codes
##### Match Length codes
Match lengths codes are values ranging from `0` to `52` included.
Match length codes are values ranging from `0` to `52` included.
They define lengths from 3 to 131074 bytes.
| Code | 0-31 |
| ------ | -------- |
| value | Code + 3 |
| nbBits | 0 |
| `Match_Length_Code` | 0-31 |
| ------------------- | ----------------------- |
| value | `Match_Length_Code` + 3 |
| `Number_of_Bits` | 0 |
| Code | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 |
| -------- | ---- | ---- | ---- | ---- | ---- | ---- | ---- | ---- |
| Baseline | 35 | 37 | 39 | 41 | 43 | 47 | 51 | 59 |
| nb Bits | 1 | 1 | 1 | 1 | 2 | 2 | 3 | 3 |
| `Match_Length_Code` | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 |
| ------------------- | ---- | ---- | ---- | ---- | ---- | ---- | ---- | ---- |
| Baseline | 35 | 37 | 39 | 41 | 43 | 47 | 51 | 59 |
| `Number_of_Bits` | 1 | 1 | 1 | 1 | 2 | 2 | 3 | 3 |
| Code | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 |
| -------- | ---- | ---- | ---- | ---- | ---- | ---- | ---- | ---- |
| Baseline | 67 | 83 | 99 | 131 | 258 | 514 | 1026 | 2050 |
| nb Bits | 4 | 4 | 5 | 7 | 8 | 9 | 10 | 11 |
| `Match_Length_Code` | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 |
| ------------------- | ---- | ---- | ---- | ---- | ---- | ---- | ---- | ---- |
| Baseline | 67 | 83 | 99 | 131 | 258 | 514 | 1026 | 2050 |
| `Number_of_Bits` | 4 | 4 | 5 | 7 | 8 | 9 | 10 | 11 |
| Code | 48 | 49 | 50 | 51 | 52 |
| -------- | ---- | ---- | ---- | ---- | ---- |
| Baseline | 4098 | 8194 |16486 |32770 |65538 |
| nb Bits | 12 | 13 | 14 | 15 | 16 |
| `Match_Length_Code` | 48 | 49 | 50 | 51 | 52 |
| ------------------- | ---- | ---- | ---- | ---- | ---- |
| Baseline | 4098 | 8194 |16486 |32770 |65538 |
| `Number_of_Bits` | 12 | 13 | 14 | 15 | 16 |
__Default distribution__
When "compression mode" is defined as "predef",
a pre-defined distribution is used for FSE compression.
When `Compression_Mode` is defined as `Predefined_Mode`,
a predefined distribution is used for FSE compression.
Here is its definition. It uses an accuracy of 6 bits (64 states).
```
@@ -839,31 +840,32 @@ short matchLengths_defaultDistribution[53] =
##### Offset codes
Offset codes are values ranging from `0` to `N`,
with `N` being limited by maximum backreference distance.
Offset codes are values ranging from `0` to `N`.
A decoder is free to limit its maximum `N` supported.
Recommendation is to support at least up to `22`.
For information, at the time of this writing.
the reference decoder supports a maximum `N` value of `28` in 64-bits mode.
An offset code is also the nb of additional bits to read,
and can be translated into an `OFValue` using the following formulae :
An offset code is also the number of additional bits to read,
and can be translated into an `Offset_Value` using the following formulae :
```
OFValue = (1 << offsetCode) + readNBits(offsetCode);
if (OFValue > 3) offset = OFValue - 3;
Offset_Value = (1 << offsetCode) + readNBits(offsetCode);
if (Offset_Value > 3) offset = Offset_Value - 3;
```
It means that maximum `Offset_Value` is `2^(N+1))-1` and it supports back-reference distance up to 2^(N+1))-4
but is limited by [maximum back-reference distance](#window_descriptor).
OFValue from 1 to 3 are special : they define "repeat codes",
Offset_Value from 1 to 3 are special : they define "repeat codes",
which means one of the previous offsets will be repeated.
They are sorted in recency order, with 1 meaning the most recent one.
See [Repeat offsets](#repeat-offsets) paragraph.
__Default distribution__
When "compression mode" is defined as "predef",
a pre-defined distribution is used for FSE compression.
When `Compression_Mode` is defined as `Predefined_Mode`,
a predefined distribution is used for FSE compression.
Here is its definition. It uses an accuracy of 5 bits (32 states),
and supports a maximum `N` of 28, allowing offset values up to 536,870,908 .
@@ -881,38 +883,38 @@ short offsetCodes_defaultDistribution[53] =
Following the header, up to 3 distribution tables can be described.
When present, they are in this order :
- Literal lengthes
- Literals lengths
- Offsets
- Match Lengthes
- Match Lengths
The content to decode depends on their respective encoding mode :
- Predef : no content. Use pre-defined distribution table.
- RLE : 1 byte. This is the only code to use across the whole compressed block.
- FSE : A distribution table is present.
- Repeat mode : no content. Re-use distribution from previous compressed block.
- `Predefined_Mode` : no content. Use predefined distribution table.
- `RLE_Mode` : 1 byte. This is the only code to use across the whole compressed block.
- `FSE_Compressed_Mode` : A distribution table is present.
- `Repeat_Mode` : no content. Re-use distribution from previous compressed block.
##### FSE distribution table : condensed format
An FSE distribution table describes the probabilities of all symbols
from `0` to the last present one (included)
on a normalized scale of `1 << AccuracyLog` .
on a normalized scale of `1 << Accuracy_Log` .
It's a bitstream which is read forward, in little-endian fashion.
It's not necessary to know its exact size,
since it will be discovered and reported by the decoding process.
The bitstream starts by reporting on which scale it operates.
`AccuracyLog = low4bits + 5;`
Note that maximum `AccuracyLog` for literal and match lengthes is `9`,
`Accuracy_Log = low4bits + 5`.
Note that maximum `Accuracy_Log` for literal and match length is `9`,
and for offsets it is `8`. Higher values are considered errors.
Then follow each symbol value, from `0` to last present one.
The nb of bits used by each field is variable.
The number of bits used by each field is variable.
It depends on :
- Remaining probabilities + 1 :
__example__ :
Presuming an AccuracyLog of 8,
Presuming an `Accuracy_Log` of 8,
and presuming 100 probabilities points have already been distributed,
the decoder may read any value from `0` to `255 - 100 + 1 == 156` (included).
Therefore, it must read `log2sup(156) == 8` bits.
@@ -926,17 +928,17 @@ It depends on :
values from 99 to 156 use 8 bits.
This is achieved through this scheme :
| Value read | Value decoded | nb Bits used |
| ---------- | ------------- | ------------ |
| 0 - 98 | 0 - 98 | 7 |
| 99 - 127 | 99 - 127 | 8 |
| 128 - 226 | 0 - 98 | 7 |
| 227 - 255 | 128 - 156 | 8 |
| Value read | Value decoded | Number of bits used |
| ---------- | ------------- | ------------------- |
| 0 - 98 | 0 - 98 | 7 |
| 99 - 127 | 99 - 127 | 8 |
| 128 - 226 | 0 - 98 | 7 |
| 227 - 255 | 128 - 156 | 8 |
Symbols probabilities are read one by one, in order.
Probability is obtained from Value decoded by following formulae :
`Proba = value - 1;`
`Proba = value - 1`
It means value `0` becomes negative probability `-1`.
`-1` is a special probability, which means `less than 1`.
@@ -951,9 +953,9 @@ This repeat flag tells how many probabilities of zeroes follow the current one.
It provides a number ranging from 0 to 3.
If it is a 3, another 2-bits repeat flag follows, and so on.
When last symbol reaches cumulated total of `1 << AccuracyLog`,
When last symbol reaches cumulated total of `1 << Accuracy_Log`,
decoding is complete.
If the last symbol makes cumulated total go above `1 << AccuracyLog`,
If the last symbol makes cumulated total go above `1 << Accuracy_Log`,
distribution is considered corrupted.
Then the decoder can tell how many bytes were used in this process,
@@ -968,14 +970,14 @@ to create a unique decoding table.
It follows the following build rule :
The table has a size of `tableSize = 1 << AccuracyLog;`.
The table has a size of `tableSize = 1 << Accuracy_Log`.
Each cell describes the symbol decoded,
and instructions to get the next state.
Symbols are scanned in their natural order for `less than 1` probabilities.
Symbols with this probability are being attributed a single cell,
starting from the end of the table.
These symbols define a full state reset, reading `AccuracyLog` bits.
These symbols define a full state reset, reading `Accuracy_Log` bits.
All remaining symbols are sorted in their natural order.
Starting from symbol `0` and table position `0`,
@@ -1004,7 +1006,7 @@ It receives 5 state values. States are sorted in natural order.
Next power of 2 is 8.
Space of probabilities is divided into 8 equal parts.
Presuming the AccuracyLog is 7, it defines 128 states.
Presuming the `Accuracy_Log` is 7, it defines 128 states.
Divided by 8, each share is 16 large.
In order to reach 8, 8-5=3 lowest states will count "double",
@@ -1013,13 +1015,13 @@ requiring one more bit in the process.
Numbering starts from higher states using less bits.
| state order | 0 | 1 | 2 | 3 | 4 |
| ----------- | ----- | ----- | ------ | ---- | ----- |
| width | 32 | 32 | 32 | 16 | 16 |
| nb Bits | 5 | 5 | 5 | 4 | 4 |
| range nb | 2 | 4 | 6 | 0 | 1 |
| baseline | 32 | 64 | 96 | 0 | 16 |
| range | 32-63 | 64-95 | 96-127 | 0-15 | 16-31 |
| state order | 0 | 1 | 2 | 3 | 4 |
| ---------------- | ----- | ----- | ------ | ---- | ----- |
| width | 32 | 32 | 32 | 16 | 16 |
| `Number_of_Bits` | 5 | 5 | 5 | 4 | 4 |
| range number | 2 | 4 | 6 | 0 | 1 |
| baseline | 32 | 64 | 96 | 0 | 16 |
| range | 32-63 | 64-95 | 96-127 | 0-15 | 16-31 |
Next state is determined from current state
by reading the required number of bits, and adding the specified baseline.
@@ -1043,9 +1045,9 @@ The bitstream starts with initial state values,
each using the required number of bits in their respective _accuracy_,
decoded previously from their normalized distribution.
It starts by `Literal Length State`,
followed by `Offset State`,
and finally `Match Length State`.
It starts by `Literals_Length_State`,
followed by `Offset_State`,
and finally `Match_Length_State`.
Reminder : always keep in mind that all values are read _backward_.
@@ -1055,9 +1057,9 @@ A state gives a code.
A code provides a baseline and number of bits to add.
See [Symbol Decoding] section for details on each symbol.
Decoding starts by reading the nb of bits required to decode offset.
Decoding starts by reading the number of bits required to decode offset.
It then does the same for match length,
and then for literal length.
and then for literals length.
Offset / matchLength / litLength define a sequence.
It starts by inserting the number of literals defined by `litLength`,
@@ -1065,11 +1067,11 @@ then continue by copying `matchLength` bytes from `currentPos - offset`.
The next operation is to update states.
Using rules pre-calculated in the decoding tables,
`Literal Length State` is updated,
followed by `Match Length State`,
and then `Offset State`.
`Literals_Length_State` is updated,
followed by `Match_Length_State`,
and then `Offset_State`.
This operation will be repeated `NbSeqs` times.
This operation will be repeated `Number_of_Sequences` times.
At the end, the bitstream shall be entirely consumed,
otherwise bitstream is considered corrupted.
@@ -1080,7 +1082,7 @@ otherwise bitstream is considered corrupted.
As seen in [Offset Codes], the first 3 values define a repeated offset.
They are sorted in recency order, with 1 meaning "most recent one".
There is an exception though, when current sequence's literal length is `0`.
There is an exception though, when current sequence's literals length is `0`.
In which case, repcodes are "pushed by one",
so 1 becomes 2, 2 becomes 3,
and 3 becomes "offset_1 - 1_byte".
@@ -1111,31 +1113,32 @@ But dictionaries created by `zstd --train` follow a format, described here.
__Pre-requisites__ : a dictionary has a known length,
defined either by a buffer limit, or a file size.
| Header | DictID | Stats | Content |
| ------ | ------ | ----- | ------- |
| `Magic_Number` | `Dictionary_ID` | `Entropy_Tables` | `Content` |
| -------------- | --------------- | ---------------- | --------- |
__Header__ : 4 bytes ID, value 0xEC30A437, Little-Endian format
__`Magic_Number`__ : 4 bytes ID, value 0xEC30A437, little-endian format
__Dict_ID__ : 4 bytes, stored in Little-Endian format.
DictID can be any value, except 0 (which means no DictID).
__`Dictionary_ID`__ : 4 bytes, stored in little-endian format.
`Dictionary_ID` can be any value, except 0 (which means no `Dictionary_ID`).
It's used by decoders to check if they use the correct dictionary.
_Reserved ranges :_
_Reserved ranges :_
If the frame is going to be distributed in a private environment,
any dictionary ID can be used.
any `Dictionary_ID` can be used.
However, for public distribution of compressed frames,
some ranges are reserved for future use :
the following ranges are reserved for future use and should not be used :
- low range : 1 - 32767 : reserved
- high range : >= (2^31) : reserved
- low range : 1 - 32767
- high range : >= (2^31)
__Stats__ : Entropy tables, following the same format as a [compressed blocks].
__`Entropy_Tables`__ : following the same format as a [compressed blocks].
They are stored in following order :
Huffman tables for literals, FSE table for offset,
FSE table for matchLenth, and FSE table for litLength.
It's finally followed by 3 offset values, populating recent offsets,
stored in order, 4-bytes little-endian each, for a total of 12 bytes.
__Content__ : Where the actual dictionary content is.
__`Content`__ : Where the actual dictionary content is.
Content size depends on Dictionary size.
[compressed blocks]: #the-format-of-compressed_block
@@ -1144,6 +1147,6 @@ __Content__ : Where the actual dictionary content is.
Version changes
---------------
- 0.2.0 : numerous format adjustments for zstd v0.8
- 0.1.2 : limit huffman tree depth to 11 bits
- 0.1.2 : limit Huffman tree depth to 11 bits
- 0.1.1 : reserved dictID ranges
- 0.1.0 : initial release