From 447212d07c1981c49f8c1724e1a46d40e6d1039e Mon Sep 17 00:00:00 2001 From: inikep Date: Wed, 28 Sep 2016 12:23:07 +0200 Subject: [PATCH 01/92] RES files for zstd 1.1.0 --- build/VS2010/zstd/generate_res/zstd32.res | Bin 948 -> 948 bytes build/VS2010/zstd/generate_res/zstd64.res | Bin 948 -> 948 bytes 2 files changed, 0 insertions(+), 0 deletions(-) diff --git a/build/VS2010/zstd/generate_res/zstd32.res b/build/VS2010/zstd/generate_res/zstd32.res index 362d9c22df2cd8fcfd98134449f1eff466c293e6..75705d06802ba59bc017e31c22f8c6169ec8a645 100644 GIT binary patch delta 47 qcmdnOzJ-0l5q?GnMg|ao(i<-uGO`*n=rI^<=47&mGAH{orvm`UcnAal delta 47 ocmdnOzJ-0l5q<^+Mg}Cj@v Date: Wed, 28 Sep 2016 13:23:11 +0200 Subject: [PATCH 02/92] updated .gitignore --- .gitignore | 2 +- zlibWrapper/.gitignore | 26 +++++--------------------- 2 files changed, 6 insertions(+), 22 deletions(-) diff --git a/.gitignore b/.gitignore index 751b674e8..c939f1228 100644 --- a/.gitignore +++ b/.gitignore @@ -21,7 +21,7 @@ zstd # Test artefacts tmp* -dictionary +dictionary* # Other files .directory diff --git a/zlibWrapper/.gitignore b/zlibWrapper/.gitignore index f197c8cc6..1fd8f416c 100644 --- a/zlibWrapper/.gitignore +++ b/zlibWrapper/.gitignore @@ -1,26 +1,10 @@ -# Object files -*.o -*.ko - -# Libraries -*.lib -*.a - -# Shared objects (inc. Windows DLLs) -*.dll -*.so -*.so.* -*.dylib - -# Executables -*.exe -*.out -*.app - # Default result files _* -example -example_zstd +example.* +example_zstd.* +fitblk.* +fitblk_zstd.* +zwrapbench.* foo.gz # Misc files From ba49de38cfadb874b62e8106c77752ddb02f07af Mon Sep 17 00:00:00 2001 From: inikep Date: Mon, 3 Oct 2016 14:19:30 +0200 Subject: [PATCH 03/92] zlibWrapper: updated README.md --- zlibWrapper/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/zlibWrapper/README.md b/zlibWrapper/README.md index 427cdbe09..5fefac1d3 100644 --- a/zlibWrapper/README.md +++ b/zlibWrapper/README.md @@ -71,7 +71,7 @@ The script used for compilation can be found at [zlibWrapper/Makefile](Makefile) The zstd distribution contains a tool called `zwrapbench` which can measure speed and ratio of zlib, zstd, and the wrapper. The benchmark is conducted using given filenames or synthetic data if filenames are not provided. -The files are read into memory and joined together. +The files are read into memory and processed independently. It makes benchmark more precise as it eliminates I/O overhead. Many filenames can be supplied as multiple parameters, parameters with wildcards or names of directories can be used as parameters with the -r option. One can select compression levels starting from `-b` and ending with `-e`. The `-i` parameter selects minimal time used for each of tested levels. @@ -119,7 +119,7 @@ In our example (the last 2 lines) it gives 4% better compression speed and 5% be #### Compatibility issues -After enabling zstd compression not all native zlib functions are supported. When calling unsupported methods they put error message into strm->msg and return Z_STREAM_ERROR. +After enabling zstd compression not all native zlib functions are supported. When calling unsupported methods they put error message into `strm->msg` and return Z_STREAM_ERROR. Supported methods: - deflateInit From 23cec875cda890ec965bee15de4797a59550d973 Mon Sep 17 00:00:00 2001 From: inikep Date: Wed, 5 Oct 2016 11:56:22 +0200 Subject: [PATCH 04/92] fixed Makefile targets: zstd-small, zstd-decompress, zstd-compress --- .travis.yml | 2 +- programs/Makefile | 27 ++++++++++++++++----------- programs/zstdcli.c | 5 ++++- 3 files changed, 21 insertions(+), 13 deletions(-) diff --git a/.travis.yml b/.travis.yml index ff6ab0f79..5e499ab9f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -44,7 +44,7 @@ matrix: - qemu-user-static - os: linux sudo: required - env: PLATFORM="Ubuntu 12.04" CMD="make -C tests versionsTest" + env: PLATFORM="Ubuntu 12.04" CMD="make -C programs zstd-small && make -C programs zstd-decompress && make -C programs zstd-compress && make -C tests versionsTest" - os: linux sudo: required env: PLATFORM="Ubuntu 12.04" CMD="make asan32" diff --git a/programs/Makefile b/programs/Makefile index 6e78d0ea9..ed26f0700 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -70,7 +70,7 @@ VOID = /dev/null endif -.PHONY: default all clean install uninstall +.PHONY: default all clean clean_decomp_o install uninstall default: zstd @@ -92,7 +92,7 @@ zstd32 : $(ZSTDDECOMP32_O) $(ZSTD_FILES) $(ZSTDLEGACY_FILES) $(ZDICT_FILES) \ $(CC) -m32 $(FLAGS) -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT) $^ $(RES32_FILE) -o $@$(EXT) -zstd_nolegacy : +zstd_nolegacy : clean_decomp_o $(MAKE) zstd ZSTD_LEGACY_SUPPORT=0 zstd-pgo : MOREFLAGS = -fprofile-generate @@ -108,18 +108,19 @@ zstd-pgo : clean zstd $(MAKE) zstd MOREFLAGS=-fprofile-use zstd-frugal: $(ZSTDDECOMP_O) $(ZSTD_FILES) zstdcli.c fileio.c - $(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_LEGACY_SUPPORT=0 $^ -o zstd$(EXT) + $(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT $^ -o zstd$(EXT) -zstd-compress: $(ZSTDCOMMON_FILES) $(ZSTDCOMP_FILES) \ - zstdcli.c fileio.c - $(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_NODECOMPRESS -DZSTD_LEGACY_SUPPORT=0 $^ -o $@$(EXT) +zstd-small: clean_decomp_o + ZSTD_LEGACY_SUPPORT=0 CFLAGS="-Os -s" $(MAKE) zstd-frugal -zstd-decompress: $(ZSTDCOMMON_FILES) $(ZSTDDECOMP_FILES) \ - zstdcli.c fileio.c - $(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_NOCOMPRESS -DZSTD_LEGACY_SUPPORT=0 $^ -o $@$(EXT) +zstd-decompress-clean: $(ZSTDDECOMP_O) $(ZSTDCOMMON_FILES) $(ZSTDDECOMP_FILES) zstdcli.c fileio.c + $(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_NOCOMPRESS $^ -o zstd-decompress$(EXT) -zstd-small: clean - CFLAGS="-Os -s" $(MAKE) zstd-frugal +zstd-decompress: clean_decomp_o + ZSTD_LEGACY_SUPPORT=0 $(MAKE) zstd-decompress-clean + +zstd-compress: $(ZSTDCOMMON_FILES) $(ZSTDCOMP_FILES) zstdcli.c fileio.c + $(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_NODECOMPRESS $^ -o $@$(EXT) clean: @@ -130,6 +131,10 @@ clean: *.gcda default.profraw @echo Cleaning completed +clean_decomp_o: + @$(RM) $(ZSTDDECOMP_O) + @$(RM) $(ZSTDDECOMP32_O) + #---------------------------------------------------------------------------------- #make install is validated only for Linux, OSX, kFreeBSD, Hurd and some BSD targets diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 6d1d9f649..fe97f965c 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -231,7 +231,8 @@ int main(int argCount, const char* argv[]) /* init */ (void)recursive; (void)cLevelLast; /* not used when ZSTD_NOBENCH set */ (void)dictCLevel; (void)dictSelect; (void)dictID; /* not used when ZSTD_NODICT set */ - (void)decode; (void)cLevel; /* not used when ZSTD_NOCOMPRESS set */ + (void)decode; (void)cLevel; (void)testmode;/* not used when ZSTD_NOCOMPRESS set */ + (void)ultra; /* not used when ZSTD_NODECOMPRESS set */ if (filenameTable==NULL) { DISPLAY("zstd: %s \n", strerror(errno)); exit(1); } filenameTable[0] = stdinmark; displayOut = stderr; @@ -490,12 +491,14 @@ int main(int argCount, const char* argv[]) CLEAN_RETURN(filenameIdx); } +#ifndef ZSTD_NOCOMPRESS /* check compression level limits */ { int const maxCLevel = ultra ? ZSTD_maxCLevel() : ZSTDCLI_CLEVEL_MAX; if (cLevel > maxCLevel) { DISPLAYLEVEL(2, "Warning : compression level higher than max, reduced to %i \n", maxCLevel); cLevel = maxCLevel; } } +#endif /* No warning message in pipe mode (stdin + stdout) or multi-files mode */ if (!strcmp(filenameTable[0], stdinmark) && outFileName && !strcmp(outFileName,stdoutmark) && (displayLevel==2)) displayLevel=1; From 0fa190b5ebe424d9c1177ed06f2e49624d914837 Mon Sep 17 00:00:00 2001 From: inikep Date: Wed, 5 Oct 2016 13:41:37 +0200 Subject: [PATCH 05/92] .travis.yml: added "make clean" before versionsTest --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 5e499ab9f..972c1b4ba 100644 --- a/.travis.yml +++ b/.travis.yml @@ -44,7 +44,7 @@ matrix: - qemu-user-static - os: linux sudo: required - env: PLATFORM="Ubuntu 12.04" CMD="make -C programs zstd-small && make -C programs zstd-decompress && make -C programs zstd-compress && make -C tests versionsTest" + env: PLATFORM="Ubuntu 12.04" CMD="make -C programs zstd-small && make -C programs zstd-decompress && make -C programs zstd-compress && make clean && make -C tests versionsTest" - os: linux sudo: required env: PLATFORM="Ubuntu 12.04" CMD="make asan32" From 82057aa7ecb59a5f1b1acd5de95109d8ccc85e17 Mon Sep 17 00:00:00 2001 From: inikep Date: Thu, 6 Oct 2016 13:23:52 +0200 Subject: [PATCH 06/92] zstd.h: updated comments --- .travis.yml | 2 +- lib/zstd.h | 137 ++++++++++++++++++++++++++-------------------------- 2 files changed, 69 insertions(+), 70 deletions(-) diff --git a/.travis.yml b/.travis.yml index 972c1b4ba..9e2f58613 100644 --- a/.travis.yml +++ b/.travis.yml @@ -44,7 +44,7 @@ matrix: - qemu-user-static - os: linux sudo: required - env: PLATFORM="Ubuntu 12.04" CMD="make -C programs zstd-small && make -C programs zstd-decompress && make -C programs zstd-compress && make clean && make -C tests versionsTest" + env: PLATFORM="Ubuntu 12.04" CMD="make -C programs zstd-small && make -C programs zstd-decompress && make -C programs zstd-compress && make -C programs clean && make -C tests versionsTest" - os: linux sudo: required env: PLATFORM="Ubuntu 12.04" CMD="make asan32" diff --git a/lib/zstd.h b/lib/zstd.h index dd3f5df4c..f7c77af7b 100644 --- a/lib/zstd.h +++ b/lib/zstd.h @@ -1,4 +1,4 @@ -/** +/* * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. * All rights reserved. * @@ -14,12 +14,12 @@ extern "C" { #endif -/*====== Dependency ======*/ +/* ====== Dependency ======*/ #include /* size_t */ -/*====== Export for Windows ======*/ -/*! +/* ====== Export for Windows ======*/ +/* * ZSTD_DLL_EXPORT : * Enable exporting of functions when building a Windows DLL */ @@ -30,7 +30,7 @@ extern "C" { #endif -/*======= Version =======*/ +/* ======= Version =======*/ #define ZSTD_VERSION_MAJOR 1 #define ZSTD_VERSION_MINOR 1 #define ZSTD_VERSION_RELEASE 0 @@ -44,7 +44,7 @@ extern "C" { ZSTDLIB_API unsigned ZSTD_versionNumber (void); -/* ************************************* +/*************************************** * Simple API ***************************************/ /*! ZSTD_compress() : @@ -91,29 +91,29 @@ ZSTDLIB_API unsigned ZSTD_isError(size_t code); /*!< tells if a `siz ZSTDLIB_API const char* ZSTD_getErrorName(size_t code); /*!< provides readable string from an error code */ -/*-************************************* +/*************************************** * Explicit memory management ***************************************/ -/** Compression context */ +/*= Compression context */ typedef struct ZSTD_CCtx_s ZSTD_CCtx; ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx(void); ZSTDLIB_API size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx); -/** ZSTD_compressCCtx() : +/*! ZSTD_compressCCtx() : Same as ZSTD_compress(), requires an allocated ZSTD_CCtx (see ZSTD_createCCtx()) */ ZSTDLIB_API size_t ZSTD_compressCCtx(ZSTD_CCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, int compressionLevel); -/** Decompression context */ +/*= Decompression context */ typedef struct ZSTD_DCtx_s ZSTD_DCtx; ZSTDLIB_API ZSTD_DCtx* ZSTD_createDCtx(void); ZSTDLIB_API size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx); -/** ZSTD_decompressDCtx() : +/*! ZSTD_decompressDCtx() : * Same as ZSTD_decompress(), requires an allocated ZSTD_DCtx (see ZSTD_createDCtx()) */ ZSTDLIB_API size_t ZSTD_decompressDCtx(ZSTD_DCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); -/*-************************ +/************************** * Simple dictionary API ***************************/ /*! ZSTD_compress_usingDict() : @@ -135,7 +135,7 @@ ZSTDLIB_API size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx, const void* dict,size_t dictSize); -/*-************************** +/**************************** * Fast Dictionary API ****************************/ /*! ZSTD_createCDict() : @@ -170,7 +170,7 @@ ZSTDLIB_API size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict); -/*-************************** +/**************************** * Streaming ****************************/ @@ -187,9 +187,8 @@ typedef struct ZSTD_outBuffer_s { } ZSTD_outBuffer; -/*====== streaming compression ======*/ -/*-*********************************************************************** +/************************************************************************* * Streaming compression - howto * * A ZSTD_CStream object is required to track streaming operation. @@ -225,22 +224,21 @@ typedef struct ZSTD_outBuffer_s { * * *******************************************************************/ +/*===== Streaming compression functions ======*/ typedef struct ZSTD_CStream_s ZSTD_CStream; ZSTDLIB_API ZSTD_CStream* ZSTD_createCStream(void); ZSTDLIB_API size_t ZSTD_freeCStream(ZSTD_CStream* zcs); - -ZSTDLIB_API size_t ZSTD_CStreamInSize(void); /**< recommended size for input buffer */ -ZSTDLIB_API size_t ZSTD_CStreamOutSize(void); /**< recommended size for output buffer. Guarantee to successfully flush at least one complete compressed block in all circumstances. */ - ZSTDLIB_API size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel); ZSTDLIB_API size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuffer* input); ZSTDLIB_API size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output); ZSTDLIB_API size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output); +ZSTDLIB_API size_t ZSTD_CStreamInSize(void); /**< recommended size for input buffer */ +ZSTDLIB_API size_t ZSTD_CStreamOutSize(void); /**< recommended size for output buffer. Guarantee to successfully flush at least one complete compressed block in all circumstances. */ -/*====== decompression ======*/ -/*-*************************************************************************** + +/***************************************************************************** * Streaming decompression howto * * A ZSTD_DStream object is required to track streaming operations. @@ -262,28 +260,29 @@ ZSTDLIB_API size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output); * The return value is a suggested next input size (just an hint, to help latency). * *******************************************************************************/ +/*===== Streaming decompression functions =====*/ typedef struct ZSTD_DStream_s ZSTD_DStream; ZSTDLIB_API ZSTD_DStream* ZSTD_createDStream(void); ZSTDLIB_API size_t ZSTD_freeDStream(ZSTD_DStream* zds); +ZSTDLIB_API size_t ZSTD_initDStream(ZSTD_DStream* zds); +ZSTDLIB_API size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inBuffer* input); ZSTDLIB_API size_t ZSTD_DStreamInSize(void); /*!< recommended size for input buffer */ ZSTDLIB_API size_t ZSTD_DStreamOutSize(void); /*!< recommended size for output buffer. Guarantee to successfully flush at least one complete block in all circumstances. */ -ZSTDLIB_API size_t ZSTD_initDStream(ZSTD_DStream* zds); -ZSTDLIB_API size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inBuffer* input); - #ifdef ZSTD_STATIC_LINKING_ONLY -/* ==================================================================================== +/**************************************************************************************** + * START OF EXPERIMENTAL FUNCTIONS * The definitions in this section are considered experimental. * They should never be used with a dynamic library, as they may change in the future. * They are provided for advanced usages. * Use them only in association with static linking. - * ==================================================================================== */ + * ***************************************************************************************/ -/*--- Constants ---*/ +/* --- Constants ---*/ #define ZSTD_MAGICNUMBER 0xFD2FB528 /* v0.8 */ #define ZSTD_MAGIC_SKIPPABLE_START 0x184D2A50U @@ -334,13 +333,13 @@ typedef struct { ZSTD_frameParameters fParams; } ZSTD_parameters; -/* custom memory allocation functions */ +/*= Custom memory allocation functions */ typedef void* (*ZSTD_allocFunction) (void* opaque, size_t size); typedef void (*ZSTD_freeFunction) (void* opaque, void* address); typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; void* opaque; } ZSTD_customMem; -/*-************************************* +/*************************************** * Advanced compression functions ***************************************/ /*! ZSTD_estimateCCtxSize() : @@ -412,12 +411,11 @@ ZSTDLIB_API size_t ZSTD_sizeof_DCtx(const ZSTD_DCtx* dctx); ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict); -/* ****************************************************************** +/******************************************************************** * Advanced Streaming functions ********************************************************************/ -/*====== compression ======*/ - +/*===== Advanced Streaming compression functions =====*/ ZSTDLIB_API ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem); ZSTDLIB_API size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel); ZSTDLIB_API size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize, @@ -426,10 +424,8 @@ ZSTDLIB_API size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledg ZSTDLIB_API size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs); -/*====== decompression ======*/ - +/*===== Advanced Streaming decompression functions =====*/ typedef enum { ZSTDdsp_maxWindowSize } ZSTD_DStreamParameter_e; - ZSTDLIB_API ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem); ZSTDLIB_API size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize); ZSTDLIB_API size_t ZSTD_setDStreamParameter(ZSTD_DStream* zds, ZSTD_DStreamParameter_e paramType, unsigned paramValue); @@ -437,22 +433,17 @@ ZSTDLIB_API size_t ZSTD_resetDStream(ZSTD_DStream* zds); /**< re-use decompress ZSTDLIB_API size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds); -/* ****************************************************************** +/********************************************************************* * Buffer-less and synchronous inner streaming functions -********************************************************************/ -/* This is an advanced API, giving full control over buffer management, for users which need direct control over memory. +* +* This is an advanced API, giving full control over buffer management, for users which need direct control over memory. * But it's also a complex one, with many restrictions (documented below). -* Prefer using normal streaming API for an easier experience */ +* Prefer using normal streaming API for an easier experience +********************************************************************* */ -ZSTDLIB_API size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel); -ZSTDLIB_API size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel); -ZSTDLIB_API size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize); -ZSTDLIB_API size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize); +/** + Buffer-less streaming compression (synchronous mode) -ZSTDLIB_API size_t ZSTD_compressContinue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); -ZSTDLIB_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); - -/* A ZSTD_CCtx object is required to track streaming operations. Use ZSTD_createCCtx() / ZSTD_freeCCtx() to manage resource. ZSTD_CCtx object can be re-used multiple times within successive compression operations. @@ -481,26 +472,17 @@ ZSTDLIB_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapaci You can then reuse `ZSTD_CCtx` (ZSTD_compressBegin()) to compress some new frame. */ -typedef struct { - unsigned long long frameContentSize; - unsigned windowSize; - unsigned dictID; - unsigned checksumFlag; -} ZSTD_frameParams; +/*===== Buffer-less streaming compression functions =====*/ +ZSTDLIB_API size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel); +ZSTDLIB_API size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel); +ZSTDLIB_API size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize); +ZSTDLIB_API size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize); +ZSTDLIB_API size_t ZSTD_compressContinue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); +ZSTDLIB_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); -ZSTDLIB_API size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t srcSize); /**< doesn't consume input, see details below */ -ZSTDLIB_API size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx); -ZSTDLIB_API size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t dictSize); -ZSTDLIB_API void ZSTD_copyDCtx(ZSTD_DCtx* dctx, const ZSTD_DCtx* preparedDCtx); -ZSTDLIB_API size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx); -ZSTDLIB_API size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); - -typedef enum { ZSTDnit_frameHeader, ZSTDnit_blockHeader, ZSTDnit_block, ZSTDnit_lastBlock, ZSTDnit_checksum, ZSTDnit_skippableFrame } ZSTD_nextInputType_e; -ZSTDLIB_API ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx); - -/* +/** Buffer-less streaming decompression (synchronous mode) A ZSTD_DCtx object is required to track streaming operations. @@ -557,11 +539,27 @@ ZSTDLIB_API ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx); It also returns Frame Size as fparamsPtr->frameContentSize. */ +typedef struct { + unsigned long long frameContentSize; + unsigned windowSize; + unsigned dictID; + unsigned checksumFlag; +} ZSTD_frameParams; -/* ************************************** -* Block functions -****************************************/ -/*! Block functions produce and decode raw zstd blocks, without frame metadata. +/*===== Buffer-less streaming decompression functions =====*/ +ZSTDLIB_API size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t srcSize); /**< doesn't consume input, see details below */ +ZSTDLIB_API size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx); +ZSTDLIB_API size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t dictSize); +ZSTDLIB_API void ZSTD_copyDCtx(ZSTD_DCtx* dctx, const ZSTD_DCtx* preparedDCtx); +ZSTDLIB_API size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx); +ZSTDLIB_API size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); +typedef enum { ZSTDnit_frameHeader, ZSTDnit_blockHeader, ZSTDnit_block, ZSTDnit_lastBlock, ZSTDnit_checksum, ZSTDnit_skippableFrame } ZSTD_nextInputType_e; +ZSTDLIB_API ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx); + +/** + Block functions + + Block functions produce and decode raw zstd blocks, without frame metadata. Frame metadata cost is typically ~18 bytes, which can be non-negligible for very small blocks (< 100 bytes). User will have to take in charge required information to regenerate data, such as compressed and content sizes. @@ -585,6 +583,7 @@ ZSTDLIB_API ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx); */ #define ZSTD_BLOCKSIZE_ABSOLUTEMAX (128 * 1024) /* define, for static allocation */ +/*===== Raw zstd block functions =====*/ ZSTDLIB_API size_t ZSTD_getBlockSizeMax(ZSTD_CCtx* cctx); ZSTDLIB_API size_t ZSTD_compressBlock (ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); ZSTDLIB_API size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); From ba1db376acfed644b813231d819bd12b4b4c1c6e Mon Sep 17 00:00:00 2001 From: inikep Date: Thu, 6 Oct 2016 14:22:48 +0200 Subject: [PATCH 07/92] fixed clang 3.5 warnings --- lib/zstd.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/zstd.h b/lib/zstd.h index f7c77af7b..be2bd952b 100644 --- a/lib/zstd.h +++ b/lib/zstd.h @@ -188,7 +188,7 @@ typedef struct ZSTD_outBuffer_s { -/************************************************************************* +/*-*********************************************************************** * Streaming compression - howto * * A ZSTD_CStream object is required to track streaming operation. @@ -238,7 +238,7 @@ ZSTDLIB_API size_t ZSTD_CStreamOutSize(void); /**< recommended size for output -/***************************************************************************** +/*-*************************************************************************** * Streaming decompression howto * * A ZSTD_DStream object is required to track streaming operations. @@ -482,7 +482,7 @@ ZSTDLIB_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapaci -/** +/*- Buffer-less streaming decompression (synchronous mode) A ZSTD_DCtx object is required to track streaming operations. From 2d2613399a8ea583269b4a2f68aef889fb511d14 Mon Sep 17 00:00:00 2001 From: inikep Date: Thu, 6 Oct 2016 16:28:21 +0200 Subject: [PATCH 08/92] zstd.h: added the Introduction section --- lib/zstd.h | 40 +++++++++++++++++++++++++++++++++++----- 1 file changed, 35 insertions(+), 5 deletions(-) diff --git a/lib/zstd.h b/lib/zstd.h index be2bd952b..2c1407306 100644 --- a/lib/zstd.h +++ b/lib/zstd.h @@ -30,7 +30,29 @@ extern "C" { #endif -/* ======= Version =======*/ +/******************************************************************************************************* + Introduction + + Zstd, short for Zstandard, is a fast lossless compression algorithm, targeting real-time compression scenarios + at zlib-level and better compression ratios. The zstd compression library provides in-memory compression and + decompression functions. The library supports compression levels from 1 up to ZSTD_maxCLevel() which is 22. + Levels from 20 to 22 should be used with caution as they require about 300-1300 MB for compression. + Compression can be done in: + - a single step (described as Simple API) + - a single step, reusing a context (described as Explicit memory management) + - repeated calls of the compression function (described as Streaming compression) + The compression ratio achievable on small data can be highly improved using compression with a dictionary in: + - a single step (described as Simple dictionary API) + - a single step, reusing a dictionary (described as Fast Dictionary API) + + Advanced and experimantal functions can be accessed using #define ZSTD_STATIC_LINKING_ONLY before including zstd.h. + These APIs shall never be used with a dynamic library. + They are not "stable", their definition may change in the future. Only static linking is allowed. +*********************************************************************************************************/ + +/*------ Version ------*/ +ZSTDLIB_API unsigned ZSTD_versionNumber (void); /**< returns version number of ZSTD */ + #define ZSTD_VERSION_MAJOR 1 #define ZSTD_VERSION_MINOR 1 #define ZSTD_VERSION_RELEASE 0 @@ -41,7 +63,6 @@ extern "C" { #define ZSTD_VERSION_STRING ZSTD_EXPAND_AND_QUOTE(ZSTD_LIB_VERSION) #define ZSTD_VERSION_NUMBER (ZSTD_VERSION_MAJOR *100*100 + ZSTD_VERSION_MINOR *100 + ZSTD_VERSION_RELEASE) -ZSTDLIB_API unsigned ZSTD_versionNumber (void); /*************************************** @@ -138,11 +159,15 @@ ZSTDLIB_API size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx, /**************************** * Fast Dictionary API ****************************/ +typedef struct ZSTD_CDict_s ZSTD_CDict; + /*! ZSTD_createCDict() : * Create a digested dictionary, ready to start compression operation without startup delay. * `dict` can be released after ZSTD_CDict creation */ -typedef struct ZSTD_CDict_s ZSTD_CDict; ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict(const void* dict, size_t dictSize, int compressionLevel); + +/*! ZSTD_freeCDict() : +* Function frees memory allocated with ZSTD_createCDict() */ ZSTDLIB_API size_t ZSTD_freeCDict(ZSTD_CDict* CDict); /*! ZSTD_compress_usingCDict() : @@ -154,11 +179,16 @@ ZSTDLIB_API size_t ZSTD_compress_usingCDict(ZSTD_CCtx* cctx, const void* src, size_t srcSize, const ZSTD_CDict* cdict); + +typedef struct ZSTD_DDict_s ZSTD_DDict; + /*! ZSTD_createDDict() : * Create a digested dictionary, ready to start decompression operation without startup delay. * `dict` can be released after creation */ -typedef struct ZSTD_DDict_s ZSTD_DDict; ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict(const void* dict, size_t dictSize); + +/*! ZSTD_freeDDict() : +* Function frees memory allocated with ZSTD_createDDict() */ ZSTDLIB_API size_t ZSTD_freeDDict(ZSTD_DDict* ddict); /*! ZSTD_decompress_usingDDict() : @@ -275,7 +305,7 @@ ZSTDLIB_API size_t ZSTD_DStreamOutSize(void); /*!< recommended size for output #ifdef ZSTD_STATIC_LINKING_ONLY /**************************************************************************************** - * START OF EXPERIMENTAL FUNCTIONS + * START OF ADVANCED AND EXPERIMENTAL FUNCTIONS * The definitions in this section are considered experimental. * They should never be used with a dynamic library, as they may change in the future. * They are provided for advanced usages. From 5fa71396e0f019467f5852a3f7391d8306d576f0 Mon Sep 17 00:00:00 2001 From: Nick Terrell Date: Thu, 6 Oct 2016 12:45:47 -0700 Subject: [PATCH 09/92] [pzstd] Fix typo in readme --- contrib/pzstd/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/pzstd/README.md b/contrib/pzstd/README.md index 05ceb5599..3fe7b0b9d 100644 --- a/contrib/pzstd/README.md +++ b/contrib/pzstd/README.md @@ -10,7 +10,7 @@ When decompressing files compressed with Zstandard, PZstandard does IO in one th ## Usage -PZstandard supports the same command line interface as Zstandard, but also provies the `-p` option to specify the number of threads. +PZstandard supports the same command line interface as Zstandard, but also provides the `-p` option to specify the number of threads. Dictionary mode is not currently supported. Basic usage From b0f6d73002eff829e60c8344ac7bb2bac06564cb Mon Sep 17 00:00:00 2001 From: Nick Terrell Date: Thu, 6 Oct 2016 12:58:23 -0700 Subject: [PATCH 10/92] [pzstd] Remove empty else statement --- contrib/pzstd/Pzstd.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/contrib/pzstd/Pzstd.cpp b/contrib/pzstd/Pzstd.cpp index e0826b9d8..59a2496c4 100644 --- a/contrib/pzstd/Pzstd.cpp +++ b/contrib/pzstd/Pzstd.cpp @@ -183,8 +183,6 @@ int pzstdMain(const Options &options) { std::fprintf(stderr, "pzstd: %s: %s.\n", input.c_str(), errorHolder.getError().c_str()); } - } else { - } }); // Open the input file From 0dd8071e2e96c0ce4a536a4e97cef3ddca6d0a9f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A1draig=20Brady?= Date: Thu, 6 Oct 2016 15:13:10 +0100 Subject: [PATCH 11/92] pzstd: fix test failure on headless build Remove this test as pass/fail status is dependent on whether I/O is connected to a tty. It currrently passes on a tty but fails on a package build system for example. --- contrib/pzstd/test/OptionsTest.cpp | 6 ------ 1 file changed, 6 deletions(-) diff --git a/contrib/pzstd/test/OptionsTest.cpp b/contrib/pzstd/test/OptionsTest.cpp index e7d4b2b3e..b3efe2b7e 100644 --- a/contrib/pzstd/test/OptionsTest.cpp +++ b/contrib/pzstd/test/OptionsTest.cpp @@ -180,12 +180,6 @@ TEST(Options, GetOutputFile) { EXPECT_SUCCESS(options.parse(args.size(), args.data())); EXPECT_EQ("x.zst", options.getOutputFile(options.inputFiles[0])); } - { - Options options; - auto args = makeArray("-o-"); - EXPECT_FAILURE(options.parse(args.size(), args.data())); - EXPECT_EQ("-", options.getOutputFile(options.inputFiles[0])); - } { Options options; auto args = makeArray("x", "y", "-o", nullOutput); From 87629978d30fb262ba55214e6a92cc7608fe0062 Mon Sep 17 00:00:00 2001 From: Nick Terrell Date: Thu, 6 Oct 2016 15:51:58 -0700 Subject: [PATCH 12/92] [pzstd] Fix latent bug in WorkQueue --- contrib/pzstd/utils/WorkQueue.h | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/contrib/pzstd/utils/WorkQueue.h b/contrib/pzstd/utils/WorkQueue.h index 538213500..c46e6cbcf 100644 --- a/contrib/pzstd/utils/WorkQueue.h +++ b/contrib/pzstd/utils/WorkQueue.h @@ -28,6 +28,7 @@ class WorkQueue { std::mutex mutex_; std::condition_variable readerCv_; std::condition_variable writerCv_; + std::condition_variable finishCv_; std::queue queue_; bool done_; @@ -124,19 +125,14 @@ class WorkQueue { } readerCv_.notify_all(); writerCv_.notify_all(); + finishCv_.notify_all(); } /// Blocks until `finish()` has been called (but the queue may not be empty). void waitUntilFinished() { std::unique_lock lock(mutex_); while (!done_) { - readerCv_.wait(lock); - // If we were woken by a push, we need to wake a thread waiting on pop(). - if (!done_) { - lock.unlock(); - readerCv_.notify_one(); - lock.lock(); - } + finishCv_.wait(lock); } } }; From 18b51b99c07f5474cb431a934ac2585551ea6e12 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 11 Oct 2016 08:21:09 -0700 Subject: [PATCH 13/92] sync fse --- lib/common/fse.h | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/lib/common/fse.h b/lib/common/fse.h index 720d54b11..cecb1aef7 100644 --- a/lib/common/fse.h +++ b/lib/common/fse.h @@ -503,6 +503,7 @@ MEM_STATIC void FSE_flushCState(BIT_CStream_t* bitC, const FSE_CState_t* statePt BIT_flushBits(bitC); } + /* ====== Decompression ====== */ typedef struct { @@ -581,14 +582,19 @@ MEM_STATIC unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr) * Increasing memory usage improves compression ratio * Reduced memory usage can improve speed, due to cache effect * Recommended max value is 14, for 16KB, which nicely fits into Intel x86 L1 cache */ -#define FSE_MAX_MEMORY_USAGE 14 -#define FSE_DEFAULT_MEMORY_USAGE 13 +#ifndef FSE_MAX_MEMORY_USAGE +# define FSE_MAX_MEMORY_USAGE 14 +#endif +#ifndef FSE_DEFAULT_MEMORY_USAGE +# define FSE_DEFAULT_MEMORY_USAGE 13 +#endif /*!FSE_MAX_SYMBOL_VALUE : * Maximum symbol value authorized. * Required for proper stack allocation */ -#define FSE_MAX_SYMBOL_VALUE 255 - +#ifndef FSE_MAX_SYMBOL_VALUE +# define FSE_MAX_SYMBOL_VALUE 255 +#endif /* ************************************************************** * template functions type & suffix From 3e82f2b61add5954301fa819047856d6a12e0dac Mon Sep 17 00:00:00 2001 From: Nick Terrell Date: Tue, 11 Oct 2016 15:27:44 -0700 Subject: [PATCH 14/92] Remove references to GPLv2 license License headers added to Makefiles were taken from `zstd/Makefile`. --- examples/Makefile | 29 +++++++---------------------- programs/fileio.c | 7 ------- programs/zstdcli.c | 7 ------- tests/Makefile | 25 +++++-------------------- 4 files changed, 12 insertions(+), 56 deletions(-) diff --git a/examples/Makefile b/examples/Makefile index 54602dfe0..8ce6a2589 100644 --- a/examples/Makefile +++ b/examples/Makefile @@ -1,26 +1,11 @@ -# ########################################################################## -# ZSTD educational examples - Makefile -# Copyright (C) Yann Collet 2016 +# ################################################################ +# Copyright (c) 2016-present, Yann Collet, Facebook, Inc. +# All rights reserved. # -# GPL v2 License -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -# -# You can contact the author at : -# - zstd homepage : http://www.zstd.net/ -# ########################################################################## +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. An additional grant +# of patent rights can be found in the PATENTS file in the same directory. +# ################################################################ # This Makefile presumes libzstd is installed, using `sudo make install` diff --git a/programs/fileio.c b/programs/fileio.c index 56f22fe75..d57ffedd2 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -8,13 +8,6 @@ */ -/* - Note : this file is part of zstd command line, which is not library. - The license of ZSTD library is BSD. - The license of this file is GPLv2. -*/ - - /* ************************************* * Tuning options ***************************************/ diff --git a/programs/zstdcli.c b/programs/zstdcli.c index fe97f965c..f64909d98 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -8,13 +8,6 @@ */ -/* - Note : this is a user program, not part of libzstd. - The license of libzstd is BSD. - The license of this command line program is GPLv2. -*/ - - /*-************************************ * Tuning parameters **************************************/ diff --git a/tests/Makefile b/tests/Makefile index 3ce9f317e..309b5d402 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -1,25 +1,10 @@ # ########################################################################## -# ZSTD tests - Makefile -# Copyright (C) Yann Collet 2015-2016 +# Copyright (c) 2016-present, Yann Collet, Facebook, Inc. +# All rights reserved. # -# GPL v2 License -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -# -# You can contact the author at : -# - zstd homepage : http://www.zstd.net/ +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. An additional grant +# of patent rights can be found in the PATENTS file in the same directory. # ########################################################################## # datagen : Synthetic and parametrable data generator, for tests # fullbench : Precisely measure speed for each zstd inner functions From a17fd7312af6517b0bc63d339c106761b258f628 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 11 Oct 2016 16:41:09 -0700 Subject: [PATCH 15/92] changed error_public.h into zstd_errors.h --- lib/common/error_private.h | 2 +- lib/common/{error_public.h => zstd_errors.h} | 6 +++--- tests/fuzzer.c | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) rename lib/common/{error_public.h => zstd_errors.h} (94%) diff --git a/lib/common/error_private.h b/lib/common/error_private.h index d27e15af8..3e99b87e7 100644 --- a/lib/common/error_private.h +++ b/lib/common/error_private.h @@ -21,7 +21,7 @@ extern "C" { * Dependencies ******************************************/ #include /* size_t */ -#include "error_public.h" /* enum list */ +#include "zstd_errors.h" /* enum list */ /* **************************************** diff --git a/lib/common/error_public.h b/lib/common/zstd_errors.h similarity index 94% rename from lib/common/error_public.h rename to lib/common/zstd_errors.h index d46abd2c7..9335eaab0 100644 --- a/lib/common/error_public.h +++ b/lib/common/zstd_errors.h @@ -7,8 +7,8 @@ * of patent rights can be found in the PATENTS file in the same directory. */ -#ifndef ERROR_PUBLIC_H_MODULE -#define ERROR_PUBLIC_H_MODULE +#ifndef ZSTD_ERRORS_H_398273423 +#define ZSTD_ERRORS_H_398273423 #if defined (__cplusplus) extern "C" { @@ -56,4 +56,4 @@ const char* ZSTD_getErrorString(ZSTD_ErrorCode code); } #endif -#endif /* ERROR_PUBLIC_H_MODULE */ +#endif /* ZSTD_ERRORS_H_398273423 */ diff --git a/tests/fuzzer.c b/tests/fuzzer.c index ae8450e40..22f034d13 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -27,7 +27,7 @@ #include /* clock_t */ #define ZSTD_STATIC_LINKING_ONLY /* ZSTD_compressContinue, ZSTD_compressBlock */ #include "zstd.h" /* ZSTD_VERSION_STRING */ -#include "error_public.h" /* ZSTD_getErrorCode */ +#include "zstd_errors.h" /* ZSTD_getErrorCode */ #include "zdict.h" /* ZDICT_trainFromBuffer */ #include "datagen.h" /* RDG_genBuffer */ #include "mem.h" From 14efab827bd41e574848792a8ace8236a4fa807f Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 11 Oct 2016 16:51:29 -0700 Subject: [PATCH 16/92] added zstd_errors.h to include installation --- lib/Makefile | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/Makefile b/lib/Makefile index 4fb8ed9d2..c3c993e3c 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -97,7 +97,8 @@ install: libzstd libzstd.pc @cp -a libzstd.pc $(DESTDIR)$(LIBDIR)/pkgconfig/ @install -m 644 libzstd.a $(DESTDIR)$(LIBDIR)/libzstd.a @install -m 644 zstd.h $(DESTDIR)$(INCLUDEDIR)/zstd.h - @install -m 644 common/zbuff.h $(DESTDIR)$(INCLUDEDIR)/zbuff.h + @install -m 644 zstd_errors.h $(DESTDIR)$(INCLUDEDIR)/zstd_errors.h + @install -m 644 common/zbuff.h $(DESTDIR)$(INCLUDEDIR)/zbuff.h # Deprecated streaming functions @install -m 644 dictBuilder/zdict.h $(DESTDIR)$(INCLUDEDIR)/zdict.h @echo zstd static and shared library installed @@ -108,7 +109,8 @@ uninstall: $(RM) $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT_VER) $(RM) $(DESTDIR)$(LIBDIR)/libzstd.a $(RM) $(DESTDIR)$(INCLUDEDIR)/zstd.h - $(RM) $(DESTDIR)$(INCLUDEDIR)/zbuff.h + $(RM) $(DESTDIR)$(INCLUDEDIR)/zstd_errors.h + $(RM) $(DESTDIR)$(INCLUDEDIR)/zbuff.h # Deprecated streaming functions $(RM) $(DESTDIR)$(INCLUDEDIR)/zdict.h @echo zstd libraries successfully uninstalled From ef2357d0d3edc936dc04b38e3f5d85285907bd14 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 11 Oct 2016 17:24:50 -0700 Subject: [PATCH 17/92] created error_private.c, so that a single list of error strings get included --- lib/common/error_private.c | 42 ++++++++++++++++++++++++++++++++++++++ lib/common/error_private.h | 30 +-------------------------- 2 files changed, 43 insertions(+), 29 deletions(-) create mode 100644 lib/common/error_private.c diff --git a/lib/common/error_private.c b/lib/common/error_private.c new file mode 100644 index 000000000..839f77434 --- /dev/null +++ b/lib/common/error_private.c @@ -0,0 +1,42 @@ +/** + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ + +/* The purpose of this file is to have a single list of error strings embedded in binary */ + +#include "error_private.h" + +const char* ERR_getErrorString(ERR_enum code) +{ + static const char* const notErrorCode = "Unspecified error code"; + switch( code ) + { + case PREFIX(no_error): return "No error detected"; + case PREFIX(GENERIC): return "Error (generic)"; + case PREFIX(prefix_unknown): return "Unknown frame descriptor"; + case PREFIX(version_unsupported): return "Version not supported"; + case PREFIX(parameter_unknown): return "Unknown parameter type"; + case PREFIX(frameParameter_unsupported): return "Unsupported frame parameter"; + case PREFIX(frameParameter_unsupportedBy32bits): return "Frame parameter unsupported in 32-bits mode"; + case PREFIX(compressionParameter_unsupported): return "Compression parameter is out of bound"; + case PREFIX(init_missing): return "Context should be init first"; + case PREFIX(memory_allocation): return "Allocation error : not enough memory"; + case PREFIX(stage_wrong): return "Operation not authorized at current processing stage"; + case PREFIX(dstSize_tooSmall): return "Destination buffer is too small"; + case PREFIX(srcSize_wrong): return "Src size incorrect"; + case PREFIX(corruption_detected): return "Corrupted block detected"; + case PREFIX(checksum_wrong): return "Restored data doesn't match checksum"; + case PREFIX(tableLog_tooLarge): return "tableLog requires too much memory : unsupported"; + case PREFIX(maxSymbolValue_tooLarge): return "Unsupported max Symbol Value : too large"; + case PREFIX(maxSymbolValue_tooSmall): return "Specified maxSymbolValue is too small"; + case PREFIX(dictionary_corrupted): return "Dictionary is corrupted"; + case PREFIX(dictionary_wrong): return "Dictionary mismatch"; + case PREFIX(maxCode): + default: return notErrorCode; + } +} diff --git a/lib/common/error_private.h b/lib/common/error_private.h index 3e99b87e7..1bc2e4954 100644 --- a/lib/common/error_private.h +++ b/lib/common/error_private.h @@ -62,35 +62,7 @@ ERR_STATIC ERR_enum ERR_getErrorCode(size_t code) { if (!ERR_isError(code)) retu * Error Strings ******************************************/ -ERR_STATIC const char* ERR_getErrorString(ERR_enum code) -{ - static const char* notErrorCode = "Unspecified error code"; - switch( code ) - { - case PREFIX(no_error): return "No error detected"; - case PREFIX(GENERIC): return "Error (generic)"; - case PREFIX(prefix_unknown): return "Unknown frame descriptor"; - case PREFIX(version_unsupported): return "Version not supported"; - case PREFIX(parameter_unknown): return "Unknown parameter type"; - case PREFIX(frameParameter_unsupported): return "Unsupported frame parameter"; - case PREFIX(frameParameter_unsupportedBy32bits): return "Frame parameter unsupported in 32-bits mode"; - case PREFIX(compressionParameter_unsupported): return "Compression parameter is out of bound"; - case PREFIX(init_missing): return "Context should be init first"; - case PREFIX(memory_allocation): return "Allocation error : not enough memory"; - case PREFIX(stage_wrong): return "Operation not authorized at current processing stage"; - case PREFIX(dstSize_tooSmall): return "Destination buffer is too small"; - case PREFIX(srcSize_wrong): return "Src size incorrect"; - case PREFIX(corruption_detected): return "Corrupted block detected"; - case PREFIX(checksum_wrong): return "Restored data doesn't match checksum"; - case PREFIX(tableLog_tooLarge): return "tableLog requires too much memory : unsupported"; - case PREFIX(maxSymbolValue_tooLarge): return "Unsupported max Symbol Value : too large"; - case PREFIX(maxSymbolValue_tooSmall): return "Specified maxSymbolValue is too small"; - case PREFIX(dictionary_corrupted): return "Dictionary is corrupted"; - case PREFIX(dictionary_wrong): return "Dictionary mismatch"; - case PREFIX(maxCode): - default: return notErrorCode; - } -} +const char* ERR_getErrorString(ERR_enum code); /* error_private.c */ ERR_STATIC const char* ERR_getErrorName(size_t code) { From f52cd03e736dba2acfde57e2d94650a4ff7fba73 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 11 Oct 2016 17:29:27 -0700 Subject: [PATCH 18/92] bumped version number --- lib/zstd.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/zstd.h b/lib/zstd.h index dd3f5df4c..c2f5f5d4c 100644 --- a/lib/zstd.h +++ b/lib/zstd.h @@ -33,7 +33,7 @@ extern "C" { /*======= Version =======*/ #define ZSTD_VERSION_MAJOR 1 #define ZSTD_VERSION_MINOR 1 -#define ZSTD_VERSION_RELEASE 0 +#define ZSTD_VERSION_RELEASE 1 #define ZSTD_LIB_VERSION ZSTD_VERSION_MAJOR.ZSTD_VERSION_MINOR.ZSTD_VERSION_RELEASE #define ZSTD_QUOTE(str) #str From 71585843998459e636f639165ff8a1ca9d161b62 Mon Sep 17 00:00:00 2001 From: Nick Terrell Date: Mon, 10 Oct 2016 16:19:21 -0700 Subject: [PATCH 19/92] Fix ZSTD_execSequence() edge case --- lib/decompress/zstd_decompress.c | 5 +++++ lib/legacy/zstd_v04.c | 5 +++++ lib/legacy/zstd_v05.c | 5 +++++ lib/legacy/zstd_v06.c | 5 +++++ lib/legacy/zstd_v07.c | 5 +++++ 5 files changed, 25 insertions(+) diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index 47b5f42c7..88869d82a 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -878,7 +878,12 @@ size_t ZSTD_execSequence(BYTE* op, op = oLitEnd + length1; sequence.matchLength -= length1; match = base; + if (op > oend_w) { + memmove(op, match, sequence.matchLength); + return sequenceLength; + } } } + /* Requirement: op <= oend_w */ /* match within prefix */ if (sequence.offset < 8) { diff --git a/lib/legacy/zstd_v04.c b/lib/legacy/zstd_v04.c index c9dcb94e0..05e40aac5 100644 --- a/lib/legacy/zstd_v04.c +++ b/lib/legacy/zstd_v04.c @@ -3107,8 +3107,13 @@ static size_t ZSTD_execSequence(BYTE* op, op = oLitEnd + length1; sequence.matchLength -= length1; match = base; + if (op > oend_8) { + memmove(op, match, sequence.matchLength); + return sequenceLength; + } } } + /* Requirement: op <= oend_8 */ /* match within prefix */ if (sequence.offset < 8) diff --git a/lib/legacy/zstd_v05.c b/lib/legacy/zstd_v05.c index 5027e2b8b..96ffceb97 100644 --- a/lib/legacy/zstd_v05.c +++ b/lib/legacy/zstd_v05.c @@ -3312,7 +3312,12 @@ static size_t ZSTDv05_execSequence(BYTE* op, op = oLitEnd + length1; sequence.matchLength -= length1; match = base; + if (op > oend_8) { + memmove(op, match, sequence.matchLength); + return sequenceLength; + } } } + /* Requirement: op <= oend_8 */ /* match within prefix */ if (sequence.offset < 8) { diff --git a/lib/legacy/zstd_v06.c b/lib/legacy/zstd_v06.c index d9e89f806..96a84d3e8 100644 --- a/lib/legacy/zstd_v06.c +++ b/lib/legacy/zstd_v06.c @@ -3466,7 +3466,12 @@ size_t ZSTDv06_execSequence(BYTE* op, op = oLitEnd + length1; sequence.matchLength -= length1; match = base; + if (op > oend_8) { + memmove(op, match, sequence.matchLength); + return sequenceLength; + } } } + /* Requirement: op <= oend_8 */ /* match within prefix */ if (sequence.offset < 8) { diff --git a/lib/legacy/zstd_v07.c b/lib/legacy/zstd_v07.c index f4c8073f9..62285238a 100644 --- a/lib/legacy/zstd_v07.c +++ b/lib/legacy/zstd_v07.c @@ -3690,7 +3690,12 @@ size_t ZSTDv07_execSequence(BYTE* op, op = oLitEnd + length1; sequence.matchLength -= length1; match = base; + if (op > oend_w) { + memmove(op, match, sequence.matchLength); + return sequenceLength; + } } } + /* Requirement: op <= oend_w */ /* match within prefix */ if (sequence.offset < 8) { From 8b70d012f0b1043f966aa132a45c0b327a053fba Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 12 Oct 2016 10:23:53 -0700 Subject: [PATCH 20/92] fix cmake --- build/cmake/lib/CMakeLists.txt | 3 ++- lib/Makefile | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/build/cmake/lib/CMakeLists.txt b/build/cmake/lib/CMakeLists.txt index c984145ba..f970fe7e9 100644 --- a/build/cmake/lib/CMakeLists.txt +++ b/build/cmake/lib/CMakeLists.txt @@ -59,6 +59,7 @@ MESSAGE("ZSTD VERSION ${LIBVER_MAJOR}.${LIBVER_MINOR}.${LIBVER_RELEASE}") SET(Sources ${LIBRARY_DIR}/common/entropy_common.c ${LIBRARY_DIR}/common/zstd_common.c + ${LIBRARY_DIR}/common/error_private.c ${LIBRARY_DIR}/common/xxhash.c ${LIBRARY_DIR}/common/fse_decompress.c ${LIBRARY_DIR}/compress/fse_compress.c @@ -74,7 +75,7 @@ SET(Sources SET(Headers ${LIBRARY_DIR}/common/bitstream.h ${LIBRARY_DIR}/common/error_private.h - ${LIBRARY_DIR}/common/error_public.h + ${LIBRARY_DIR}/common/zstd_errors.h ${LIBRARY_DIR}/common/fse.h ${LIBRARY_DIR}/common/huf.h ${LIBRARY_DIR}/common/mem.h diff --git a/lib/Makefile b/lib/Makefile index c3c993e3c..5df8c3cb4 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -97,7 +97,7 @@ install: libzstd libzstd.pc @cp -a libzstd.pc $(DESTDIR)$(LIBDIR)/pkgconfig/ @install -m 644 libzstd.a $(DESTDIR)$(LIBDIR)/libzstd.a @install -m 644 zstd.h $(DESTDIR)$(INCLUDEDIR)/zstd.h - @install -m 644 zstd_errors.h $(DESTDIR)$(INCLUDEDIR)/zstd_errors.h + @install -m 644 common/zstd_errors.h $(DESTDIR)$(INCLUDEDIR)/zstd_errors.h @install -m 644 common/zbuff.h $(DESTDIR)$(INCLUDEDIR)/zbuff.h # Deprecated streaming functions @install -m 644 dictBuilder/zdict.h $(DESTDIR)$(INCLUDEDIR)/zdict.h @echo zstd static and shared library installed From e19111c42f19f4d98b230faad79bd7763f5e1e49 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 12 Oct 2016 11:09:36 -0700 Subject: [PATCH 21/92] make creates libzstd binaries (#415) --- Makefile | 15 +++++++++------ lib/Makefile | 18 +++++++++--------- programs/Makefile | 12 ++++++------ 3 files changed, 24 insertions(+), 21 deletions(-) diff --git a/Makefile b/Makefile index ac0c583f4..ae7189993 100644 --- a/Makefile +++ b/Makefile @@ -22,15 +22,18 @@ endif .PHONY: default all zlibwrapper zstd clean install uninstall travis-install test clangtest gpptest armtest usan asan uasan -default: zstd +default: libzstd zstd all: $(MAKE) -C $(ZSTDDIR) $@ $(MAKE) -C $(PRGDIR) $@ zstd32 $(MAKE) -C $(TESTDIR) $@ all32 +libzstd: + @$(MAKE) -C $(ZSTDDIR) + zstd: - $(MAKE) -C $(PRGDIR) + @$(MAKE) -C $(PRGDIR) cp $(PRGDIR)/zstd . zlibwrapper: @@ -54,12 +57,12 @@ clean: ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU FreeBSD DragonFly NetBSD)) HOST_OS = POSIX install: - $(MAKE) -C $(ZSTDDIR) $@ - $(MAKE) -C $(PRGDIR) $@ + @$(MAKE) -C $(ZSTDDIR) $@ + @$(MAKE) -C $(PRGDIR) $@ uninstall: - $(MAKE) -C $(ZSTDDIR) $@ - $(MAKE) -C $(PRGDIR) $@ + @$(MAKE) -C $(ZSTDDIR) $@ + @$(MAKE) -C $(PRGDIR) $@ travis-install: $(MAKE) install PREFIX=~/install_test_dir diff --git a/lib/Makefile b/lib/Makefile index 5df8c3cb4..1117b491c 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -103,15 +103,15 @@ install: libzstd libzstd.pc @echo zstd static and shared library installed uninstall: - $(RM) $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT) - $(RM) $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT_MAJOR) - $(RM) $(DESTDIR)$(LIBDIR)/pkgconfig/libzstd.pc - $(RM) $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT_VER) - $(RM) $(DESTDIR)$(LIBDIR)/libzstd.a - $(RM) $(DESTDIR)$(INCLUDEDIR)/zstd.h - $(RM) $(DESTDIR)$(INCLUDEDIR)/zstd_errors.h - $(RM) $(DESTDIR)$(INCLUDEDIR)/zbuff.h # Deprecated streaming functions - $(RM) $(DESTDIR)$(INCLUDEDIR)/zdict.h + @$(RM) $(DESTDIR)$(LIBDIR)/libzstd.a + @$(RM) $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT) + @$(RM) $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT_MAJOR) + @$(RM) $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT_VER) + @$(RM) $(DESTDIR)$(LIBDIR)/pkgconfig/libzstd.pc + @$(RM) $(DESTDIR)$(INCLUDEDIR)/zstd.h + @$(RM) $(DESTDIR)$(INCLUDEDIR)/zstd_errors.h + @$(RM) $(DESTDIR)$(INCLUDEDIR)/zbuff.h # Deprecated streaming functions + @$(RM) $(DESTDIR)$(INCLUDEDIR)/zdict.h @echo zstd libraries successfully uninstalled endif diff --git a/programs/Makefile b/programs/Makefile index ed26f0700..06da7c44b 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -153,11 +153,11 @@ install: zstd @echo zstd installation completed uninstall: - $(RM) $(DESTDIR)$(BINDIR)/zstdcat - $(RM) $(DESTDIR)$(BINDIR)/unzstd - $(RM) $(DESTDIR)$(BINDIR)/zstd$(EXT) - $(RM) $(DESTDIR)$(MANDIR)/zstdcat.1 - $(RM) $(DESTDIR)$(MANDIR)/unzstd.1 - $(RM) $(DESTDIR)$(MANDIR)/zstd.1 + @$(RM) $(DESTDIR)$(BINDIR)/zstdcat + @$(RM) $(DESTDIR)$(BINDIR)/unzstd + @$(RM) $(DESTDIR)$(BINDIR)/zstd$(EXT) + @$(RM) $(DESTDIR)$(MANDIR)/zstdcat.1 + @$(RM) $(DESTDIR)$(MANDIR)/unzstd.1 + @$(RM) $(DESTDIR)$(MANDIR)/zstd.1 @echo zstd programs successfully uninstalled endif From e405c7a126af23fccd5660ecccf65e46b1160bff Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 12 Oct 2016 15:29:22 -0700 Subject: [PATCH 22/92] updated visual projects --- Makefile | 6 +++--- NEWS | 2 ++ build/VS2005/fullbench/fullbench.vcproj | 6 +++++- build/VS2005/fuzzer/fuzzer.vcproj | 6 +++++- build/VS2005/zstd/zstd.vcproj | 6 +++++- build/VS2005/zstdlib/zstdlib.vcproj | 6 +++++- build/VS2008/fullbench/fullbench.vcproj | 6 +++++- build/VS2008/fuzzer/fuzzer.vcproj | 6 +++++- build/VS2008/zstd/zstd.vcproj | 6 +++++- build/VS2008/zstdlib/zstdlib.vcproj | 6 +++++- build/VS2010/fullbench/fullbench.vcxproj | 2 ++ build/VS2010/fuzzer/fuzzer.vcxproj | 2 ++ build/VS2010/zstd/zstd.vcxproj | 4 +++- build/VS2010/zstdlib/zstdlib.vcxproj | 5 ++++- 14 files changed, 56 insertions(+), 13 deletions(-) diff --git a/Makefile b/Makefile index ae7189993..20ae31e44 100644 --- a/Makefile +++ b/Makefile @@ -51,9 +51,9 @@ clean: @echo Cleaning completed -#---------------------------------------------------------------------------------- -#make install is validated only for Linux, OSX, kFreeBSD, Hurd and some BSD targets -#---------------------------------------------------------------------------------- +#------------------------------------------------------------------------------ +# make install is validated only for Linux, OSX, Hurd and some BSD targets +#------------------------------------------------------------------------------ ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU FreeBSD DragonFly NetBSD)) HOST_OS = POSIX install: diff --git a/NEWS b/NEWS index ad56d17a8..ab6b26b42 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,5 @@ +v1.1.1 + v1.1.0 New : contrib/pzstd, parallel version of zstd, by Nick Terrell added : NetBSD install target (#338) diff --git a/build/VS2005/fullbench/fullbench.vcproj b/build/VS2005/fullbench/fullbench.vcproj index 17af9c819..c28d1f696 100644 --- a/build/VS2005/fullbench/fullbench.vcproj +++ b/build/VS2005/fullbench/fullbench.vcproj @@ -335,6 +335,10 @@ RelativePath="..\..\..\lib\common\entropy_common.c" > + + @@ -394,7 +398,7 @@ > + + @@ -398,7 +402,7 @@ > + + @@ -450,7 +454,7 @@ > + + @@ -426,7 +430,7 @@ > + + @@ -395,7 +399,7 @@ > + + @@ -399,7 +403,7 @@ > + + @@ -451,7 +455,7 @@ > + + @@ -427,7 +431,7 @@ > + @@ -175,6 +176,7 @@ + diff --git a/build/VS2010/fuzzer/fuzzer.vcxproj b/build/VS2010/fuzzer/fuzzer.vcxproj index a436b0389..020e521aa 100644 --- a/build/VS2010/fuzzer/fuzzer.vcxproj +++ b/build/VS2010/fuzzer/fuzzer.vcxproj @@ -157,6 +157,7 @@ + @@ -176,6 +177,7 @@ + diff --git a/build/VS2010/zstd/zstd.vcxproj b/build/VS2010/zstd/zstd.vcxproj index eb7e7b504..181bbe6df 100644 --- a/build/VS2010/zstd/zstd.vcxproj +++ b/build/VS2010/zstd/zstd.vcxproj @@ -20,6 +20,7 @@ + @@ -54,6 +55,7 @@ + @@ -220,4 +222,4 @@ - \ No newline at end of file + diff --git a/build/VS2010/zstdlib/zstdlib.vcxproj b/build/VS2010/zstdlib/zstdlib.vcxproj index b97808dd0..d32b4861f 100644 --- a/build/VS2010/zstdlib/zstdlib.vcxproj +++ b/build/VS2010/zstdlib/zstdlib.vcxproj @@ -20,6 +20,7 @@ + @@ -39,9 +40,11 @@ + + - + From 5d919e7ac349b9efaaf340f54a22cffa496884b8 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 12 Oct 2016 17:28:59 -0700 Subject: [PATCH 23/92] added ZSTD_error_frameParameter_windowTooLarge (#403) --- lib/common/error_private.c | 1 + lib/common/zstd_errors.h | 1 + lib/decompress/zstd_decompress.c | 4 ++-- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/common/error_private.c b/lib/common/error_private.c index 839f77434..a0fa1724a 100644 --- a/lib/common/error_private.c +++ b/lib/common/error_private.c @@ -23,6 +23,7 @@ const char* ERR_getErrorString(ERR_enum code) case PREFIX(parameter_unknown): return "Unknown parameter type"; case PREFIX(frameParameter_unsupported): return "Unsupported frame parameter"; case PREFIX(frameParameter_unsupportedBy32bits): return "Frame parameter unsupported in 32-bits mode"; + case PREFIX(frameParameter_windowTooLarge): return "Frame requires too much memory for decoding"; case PREFIX(compressionParameter_unsupported): return "Compression parameter is out of bound"; case PREFIX(init_missing): return "Context should be init first"; case PREFIX(memory_allocation): return "Allocation error : not enough memory"; diff --git a/lib/common/zstd_errors.h b/lib/common/zstd_errors.h index 9335eaab0..50dc4f720 100644 --- a/lib/common/zstd_errors.h +++ b/lib/common/zstd_errors.h @@ -29,6 +29,7 @@ typedef enum { ZSTD_error_parameter_unknown, ZSTD_error_frameParameter_unsupported, ZSTD_error_frameParameter_unsupportedBy32bits, + ZSTD_error_frameParameter_windowTooLarge, ZSTD_error_compressionParameter_unsupported, ZSTD_error_init_missing, ZSTD_error_memory_allocation, diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index 88869d82a..d157e0051 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -248,7 +248,7 @@ size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t if (!singleSegment) { BYTE const wlByte = ip[pos++]; U32 const windowLog = (wlByte >> 3) + ZSTD_WINDOWLOG_ABSOLUTEMIN; - if (windowLog > ZSTD_WINDOWLOG_MAX) return ERROR(frameParameter_unsupported); + if (windowLog > ZSTD_WINDOWLOG_MAX) return ERROR(frameParameter_windowTooLarge); /* avoids issue with 1 << windowLog */ windowSize = (1U << windowLog); windowSize += (windowSize >> 3) * (wlByte&7); } @@ -270,7 +270,7 @@ size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t case 3 : frameContentSize = MEM_readLE64(ip+pos); break; } if (!windowSize) windowSize = (U32)frameContentSize; - if (windowSize > windowSizeMax) return ERROR(frameParameter_unsupported); + if (windowSize > windowSizeMax) return ERROR(frameParameter_windowTooLarge); fparamsPtr->frameContentSize = frameContentSize; fparamsPtr->windowSize = windowSize; fparamsPtr->dictID = dictID; From 3b9cdf922072a0fe4996fb3db1c4ed828605c693 Mon Sep 17 00:00:00 2001 From: Nick Terrell Date: Wed, 12 Oct 2016 20:54:42 -0700 Subject: [PATCH 24/92] Fix ubsan failures (pass NULL to memcpy) --- lib/compress/zstd_compress.c | 6 ++++-- lib/decompress/zstd_decompress.c | 4 +++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 94f4b5a25..e53ecc5fd 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -2235,7 +2235,7 @@ static size_t ZSTD_compress_generic (ZSTD_CCtx* cctx, BYTE* op = ostart; U32 const maxDist = 1 << cctx->params.cParams.windowLog; - if (cctx->params.fParams.checksumFlag) + if (cctx->params.fParams.checksumFlag && srcSize) XXH64_update(&cctx->xxhState, src, srcSize); while (remaining) { @@ -2688,7 +2688,9 @@ ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictSize, ZSTD_pa return NULL; } - memcpy(dictContent, dict, dictSize); + if (dictSize) { + memcpy(dictContent, dict, dictSize); + } { size_t const errorCode = ZSTD_compressBegin_advanced(cctx, dictContent, dictSize, params, 0); if (ZSTD_isError(errorCode)) { ZSTD_free(dictContent, customMem); diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index d157e0051..bb4336ff2 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -1402,7 +1402,9 @@ ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictSize, ZSTD_cu return NULL; } - memcpy(dictContent, dict, dictSize); + if (dictSize) { + memcpy(dictContent, dict, dictSize); + } { size_t const errorCode = ZSTD_decompressBegin_usingDict(dctx, dictContent, dictSize); if (ZSTD_isError(errorCode)) { ZSTD_free(dictContent, customMem); From d4cda27b63ec07f1b9f93a63581868c65204eb92 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 14 Oct 2016 13:13:13 -0700 Subject: [PATCH 25/92] new command -M#, to limit memory usage during decompression (#403) --- NEWS | 1 + lib/decompress/zstd_decompress.c | 4 ++-- programs/fileio.c | 4 ++++ programs/fileio.h | 2 +- programs/zstdcli.c | 29 +++++++++++++++++++---------- tests/playTests.sh | 2 ++ 6 files changed, 29 insertions(+), 13 deletions(-) diff --git a/NEWS b/NEWS index ab6b26b42..1b65b508c 100644 --- a/NEWS +++ b/NEWS @@ -1,4 +1,5 @@ v1.1.1 +New : command -M, to limit allowed memory consumption v1.1.0 New : contrib/pzstd, parallel version of zstd, by Nick Terrell diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index d157e0051..e6c409990 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -1573,7 +1573,7 @@ size_t ZSTD_setDStreamParameter(ZSTD_DStream* zds, switch(paramType) { default : return ERROR(parameter_unknown); - case ZSTDdsp_maxWindowSize : zds->maxWindowSize = paramValue; break; + case ZSTDdsp_maxWindowSize : zds->maxWindowSize = paramValue ? paramValue : (U32)(-1); break; } return 0; } @@ -1654,7 +1654,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB } } zds->fParams.windowSize = MAX(zds->fParams.windowSize, 1U << ZSTD_WINDOWLOG_ABSOLUTEMIN); - if (zds->fParams.windowSize > zds->maxWindowSize) return ERROR(frameParameter_unsupported); + if (zds->fParams.windowSize > zds->maxWindowSize) return ERROR(frameParameter_windowTooLarge); /* Adapt buffer sizes to frame header instructions */ { size_t const blockSize = MIN(zds->fParams.windowSize, ZSTD_BLOCKSIZE_ABSOLUTEMAX); diff --git a/programs/fileio.c b/programs/fileio.c index d57ffedd2..c4c308e03 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -120,6 +120,9 @@ static U32 g_checksumFlag = 1; void FIO_setChecksumFlag(unsigned checksumFlag) { g_checksumFlag = checksumFlag; } static U32 g_removeSrcFile = 0; void FIO_setRemoveSrcFile(unsigned flag) { g_removeSrcFile = (flag>0); } +static U32 g_memLimit = 0; +void FIO_setMemLimit(unsigned memLimit) { g_memLimit = memLimit; } + /*-************************************* @@ -480,6 +483,7 @@ static dRess_t FIO_createDResources(const char* dictFileName) /* Allocation */ ress.dctx = ZSTD_createDStream(); if (ress.dctx==NULL) EXM_THROW(60, "Can't create ZSTD_DStream"); + ZSTD_setDStreamParameter(ress.dctx, ZSTDdsp_maxWindowSize, g_memLimit); ress.srcBufferSize = ZSTD_DStreamInSize(); ress.srcBuffer = malloc(ress.srcBufferSize); ress.dstBufferSize = ZSTD_DStreamOutSize(); diff --git a/programs/fileio.h b/programs/fileio.h index 1e89aec27..60a7e0de8 100644 --- a/programs/fileio.h +++ b/programs/fileio.h @@ -15,7 +15,6 @@ extern "C" { #endif - /* ************************************* * Special i/o constants **************************************/ @@ -37,6 +36,7 @@ void FIO_setSparseWrite(unsigned sparse); /**< 0: no sparse; 1: disable on stdo void FIO_setDictIDFlag(unsigned dictIDFlag); void FIO_setChecksumFlag(unsigned checksumFlag); void FIO_setRemoveSrcFile(unsigned flag); +void FIO_setMemLimit(unsigned memLimit); /*-************************************* diff --git a/programs/zstdcli.c b/programs/zstdcli.c index f64909d98..3a284c31d 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -135,6 +135,7 @@ static int usage_advanced(const char* programName) DISPLAY( "--test : test compressed file integrity \n"); DISPLAY( "--[no-]sparse : sparse mode (default:enabled on file, disabled on stdout)\n"); #endif + DISPLAY( " -M# : Set a memory usage limit for decompression \n"); DISPLAY( "-- : All arguments after \"--\" are treated as files \n"); #ifndef ZSTD_NODICT DISPLAY( "\n"); @@ -172,14 +173,19 @@ static void waitEnter(void) } /*! readU32FromChar() : - @return : unsigned integer value reach from input in `char` format + @return : unsigned integer value read from input in `char` format + allows and interprets K, KB, KiB, M, MB and MiB suffix. Will also modify `*stringPtr`, advancing it to position where it stopped reading. - Note : this function can overflow if digit string > MAX_UINT */ + Note : function result can overflow if digit string > MAX_UINT */ static unsigned readU32FromChar(const char** stringPtr) { unsigned result = 0; while ((**stringPtr >='0') && (**stringPtr <='9')) result *= 10, result += **stringPtr - '0', (*stringPtr)++ ; + if (toupper(**stringPtr)=='K') result <<= 10, (*stringPtr)++ ; + else if (toupper(**stringPtr)=='M') result <<= 20, (*stringPtr)++ ; + if (toupper(**stringPtr)=='i') (*stringPtr)++; + if (toupper(**stringPtr)=='B') (*stringPtr)++; return result; } @@ -206,6 +212,7 @@ int main(int argCount, const char* argv[]) int cLevel = ZSTDCLI_CLEVEL_DEFAULT; int cLevelLast = 1; unsigned recursive = 0; + unsigned memLimit = 0; const char** filenameTable = (const char**)malloc(argCount * sizeof(const char*)); /* argCount >= 1 */ unsigned filenameIdx = 0; const char* programName = argv[0]; @@ -224,8 +231,8 @@ int main(int argCount, const char* argv[]) /* init */ (void)recursive; (void)cLevelLast; /* not used when ZSTD_NOBENCH set */ (void)dictCLevel; (void)dictSelect; (void)dictID; /* not used when ZSTD_NODICT set */ - (void)decode; (void)cLevel; (void)testmode;/* not used when ZSTD_NOCOMPRESS set */ - (void)ultra; /* not used when ZSTD_NODECOMPRESS set */ + (void)decode; (void)cLevel; (void)testmode; /* not used when ZSTD_NOCOMPRESS set */ + (void)ultra; (void)memLimit; /* not used when ZSTD_NODECOMPRESS set */ if (filenameTable==NULL) { DISPLAY("zstd: %s \n", strerror(errno)); exit(1); } filenameTable[0] = stdinmark; displayOut = stderr; @@ -331,6 +338,12 @@ int main(int argCount, const char* argv[]) /* destination file name */ case 'o': nextArgumentIsOutFileName=1; lastCommand=1; argument++; break; + /* limit decompression memory */ + case 'M': + argument++; + memLimit = readU32FromChar(&argument); + break; + #ifdef UTIL_HAS_CREATEFILELIST /* recursive */ case 'r': recursive=1; argument++; break; @@ -359,10 +372,7 @@ int main(int argCount, const char* argv[]) /* cut input into blocks (benchmark only) */ case 'B': argument++; - { size_t bSize = readU32FromChar(&argument); - if (toupper(*argument)=='K') bSize<<=10, argument++; /* allows using KB notation */ - if (toupper(*argument)=='M') bSize<<=20, argument++; - if (toupper(*argument)=='B') argument++; + { size_t const bSize = readU32FromChar(&argument); BMK_setNotificationLevel(displayLevel); BMK_SetBlockSize(bSize); } @@ -395,8 +405,6 @@ int main(int argCount, const char* argv[]) nextArgumentIsMaxDict = 0; lastCommand = 0; maxDictSize = readU32FromChar(&argument); - if (toupper(*argument)=='K') maxDictSize <<= 10; - if (toupper(*argument)=='M') maxDictSize <<= 20; continue; } @@ -511,6 +519,7 @@ int main(int argCount, const char* argv[]) } else { /* decompression */ #ifndef ZSTD_NODECOMPRESS if (testmode) { outFileName=nulmark; FIO_setRemoveSrcFile(0); } /* test mode */ + FIO_setMemLimit(memLimit); if (filenameIdx==1 && outFileName) operationResult = FIO_decompressFilename(outFileName, filenameTable[0], dictFileName); else diff --git a/tests/playTests.sh b/tests/playTests.sh index d94d8fab9..efb0f0cbd 100755 --- a/tests/playTests.sh +++ b/tests/playTests.sh @@ -81,6 +81,8 @@ $ZSTD -dc < tmp.zst > $INTOVOID # combine decompression, stdin & stdout $ZSTD -dc - < tmp.zst > $INTOVOID $ZSTD -d < tmp.zst > $INTOVOID # implicit stdout when stdin is used $ZSTD -d - < tmp.zst > $INTOVOID +$ECHO "test : impose memory limitation (must fail)" +$ZSTD -d -f tmp.zst -M2K -c > $INTOVOID && die "decompression needs more memory than allowed" $ECHO "test : overwrite protection" $ZSTD -q tmp && die "overwrite check failed!" $ECHO "test : force overwrite" From 1122349ac2d8d022031f4a4b1ec002317603916d Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 14 Oct 2016 14:07:11 -0700 Subject: [PATCH 26/92] added long comment --memlimit= --- programs/zstdcli.c | 13 ++++++++++--- tests/playTests.sh | 1 + 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 3a284c31d..9a38ede7a 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -247,16 +247,16 @@ int main(int argCount, const char* argv[]) if (!strcmp(programName, ZSTD_CAT)) { decode=1; forceStdout=1; displayLevel=1; outFileName=stdoutmark; } /* command switches */ - for(argNb=1; argNb $INTOVOID # implicit stdout when stdin is used $ZSTD -d - < tmp.zst > $INTOVOID $ECHO "test : impose memory limitation (must fail)" $ZSTD -d -f tmp.zst -M2K -c > $INTOVOID && die "decompression needs more memory than allowed" +$ZSTD -d -f tmp.zst --memlimit=2K -c > $INTOVOID && die "decompression needs more memory than allowed" # long command $ECHO "test : overwrite protection" $ZSTD -q tmp && die "overwrite check failed!" $ECHO "test : force overwrite" From d7b120ab5cb34224c978ea2674b7c2b7517e3435 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 14 Oct 2016 14:22:32 -0700 Subject: [PATCH 27/92] added long commands --memory= and --memlimit-decompress= --- programs/zstdcli.c | 10 ++++++++++ tests/playTests.sh | 2 ++ 2 files changed, 12 insertions(+) diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 9a38ede7a..da3786d60 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -281,6 +281,16 @@ int main(int argCount, const char* argv[]) memLimit = readU32FromChar(&argument); continue; } + if (!strncmp(argument, "--memory=", strlen("--memory="))) { + argument += strlen("--memory="); + memLimit = readU32FromChar(&argument); + continue; + } + if (!strncmp(argument, "--memlimit-decompress=", strlen("--memlimit-decompress="))) { + argument += strlen("--memlimit-decompress="); + memLimit = readU32FromChar(&argument); + continue; + } /* '-' means stdin/stdout */ if (!strcmp(argument, "-")){ diff --git a/tests/playTests.sh b/tests/playTests.sh index df9b167f5..c5a58d628 100755 --- a/tests/playTests.sh +++ b/tests/playTests.sh @@ -84,6 +84,8 @@ $ZSTD -d - < tmp.zst > $INTOVOID $ECHO "test : impose memory limitation (must fail)" $ZSTD -d -f tmp.zst -M2K -c > $INTOVOID && die "decompression needs more memory than allowed" $ZSTD -d -f tmp.zst --memlimit=2K -c > $INTOVOID && die "decompression needs more memory than allowed" # long command +$ZSTD -d -f tmp.zst --memory=2K -c > $INTOVOID && die "decompression needs more memory than allowed" # long command +$ZSTD -d -f tmp.zst --memlimit-decompress=2K -c > $INTOVOID && die "decompression needs more memory than allowed" # long command $ECHO "test : overwrite protection" $ZSTD -q tmp && die "overwrite check failed!" $ECHO "test : force overwrite" From 70077bc9bb7dcc835fd3bb57ce7ca4bc64009056 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 14 Oct 2016 14:41:17 -0700 Subject: [PATCH 28/92] refactor for long commands --- programs/zstdcli.c | 26 +++++++++++--------------- 1 file changed, 11 insertions(+), 15 deletions(-) diff --git a/programs/zstdcli.c b/programs/zstdcli.c index da3786d60..2df410431 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -189,6 +189,14 @@ static unsigned readU32FromChar(const char** stringPtr) return result; } +static unsigned longCommandWArg(const char** stringPtr, const char* longCommand) +{ + size_t const comSize = strlen(longCommand); + unsigned const result = !strncmp(*stringPtr, longCommand, comSize); + if (result) *stringPtr += comSize; + return result; +} + #define CLEAN_RETURN(i) { operationResult = (i); goto _end; } @@ -276,21 +284,9 @@ int main(int argCount, const char* argv[]) if (!strcmp(argument, "--rm")) { FIO_setRemoveSrcFile(1); continue; } /* long commands with arguments */ - if (!strncmp(argument, "--memlimit=", strlen("--memlimit="))) { - argument += strlen("--memlimit="); - memLimit = readU32FromChar(&argument); - continue; - } - if (!strncmp(argument, "--memory=", strlen("--memory="))) { - argument += strlen("--memory="); - memLimit = readU32FromChar(&argument); - continue; - } - if (!strncmp(argument, "--memlimit-decompress=", strlen("--memlimit-decompress="))) { - argument += strlen("--memlimit-decompress="); - memLimit = readU32FromChar(&argument); - continue; - } + if (longCommandWArg(&argument, "--memlimit=")) { memLimit = readU32FromChar(&argument); continue; } + if (longCommandWArg(&argument, "--memory=")) { memLimit = readU32FromChar(&argument); continue; } + if (longCommandWArg(&argument, "--memlimit-decompress=")) { memLimit = readU32FromChar(&argument); continue; } /* '-' means stdin/stdout */ if (!strcmp(argument, "-")){ From 8b4e84249b1f6eba4ace7de3335cc2fdd34cd651 Mon Sep 17 00:00:00 2001 From: Nick Terrell Date: Thu, 6 Oct 2016 18:17:08 -0700 Subject: [PATCH 29/92] [pzstd] Fix Makefile --- .gitignore | 1 + .travis.yml | 4 +- appveyor.yml | 5 +- contrib/pzstd/Makefile | 280 ++++++++++++++++++++++-------- contrib/pzstd/test/Makefile | 48 ----- contrib/pzstd/utils/test/Makefile | 42 ----- 6 files changed, 209 insertions(+), 171 deletions(-) delete mode 100644 contrib/pzstd/test/Makefile delete mode 100644 contrib/pzstd/utils/test/Makefile diff --git a/.gitignore b/.gitignore index c939f1228..220a1e0eb 100644 --- a/.gitignore +++ b/.gitignore @@ -32,3 +32,4 @@ _zstdbench/ *.swp .DS_Store googletest/ +*.d diff --git a/.travis.yml b/.travis.yml index 972c1b4ba..f328f480d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -22,7 +22,7 @@ matrix: packages: - gcc-4.8 - g++-4.8 - env: PLATFORM="Ubuntu 12.04 container" CMD="make zlibwrapper && make clean && make -C tests test-zstd_nolegacy && make clean && make clean && make cmaketest && make clean && make -C contrib/pzstd pzstd && make -C contrib/pzstd googletest && make -C contrib/pzstd test && make -C contrib/pzstd clean" + env: PLATFORM="Ubuntu 12.04 container" CMD="make zlibwrapper && make clean && make -C tests test-zstd_nolegacy && make clean && make clean && make cmaketest && make clean && make -C contrib/pzstd googletest && make -C contrib/pzstd all && make -C contrib/pzstd check && make -C contrib/pzstd clean" - os: linux sudo: false env: PLATFORM="Ubuntu 12.04 container" CMD="make usan" @@ -69,7 +69,7 @@ matrix: sudo: required install: - export CXX="g++-4.8" CC="gcc-4.8" - env: PLATFORM="Ubuntu 14.04" CMD="make gpptest && make clean && make gnu90test && make clean && make c99test && make clean && make gnu99test && make clean && make clangtest && make clean && make -C contrib/pzstd pzstd32 && make -C contrib/pzstd googletest32 && make -C contrib/pzstd test32 && make -C contrib/pzstd clean" + env: PLATFORM="Ubuntu 14.04" CMD="make gpptest && make clean && make gnu90test && make clean && make c99test && make clean && make gnu99test && make clean && make clangtest && make clean && make -C contrib/pzstd googletest32 && make -C contrib/pzstd all32 && make -C contrib/pzstd check && make -C contrib/pzstd clean" addons: apt: packages: diff --git a/appveyor.yml b/appveyor.yml index 6345c7b39..fbdc30c40 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -50,10 +50,9 @@ build_script: ECHO *** && ECHO *** Building pzstd for %PLATFORM% && ECHO *** && - ECHO make -C contrib\pzstd pzstd && - make -C contrib\pzstd pzstd && make -C contrib\pzstd googletest-mingw64 && - make -C contrib\pzstd test && + make -C contrib\pzstd all && + make -C contrib\pzstd check && make -C contrib\pzstd clean ) - if [%COMPILER%]==[gcc] ( diff --git a/contrib/pzstd/Makefile b/contrib/pzstd/Makefile index e30be0bed..b998c1fff 100644 --- a/contrib/pzstd/Makefile +++ b/contrib/pzstd/Makefile @@ -7,20 +7,69 @@ # of patent rights can be found in the PATENTS file in the same directory. # ########################################################################## +# Standard variables for installation +DESTDIR ?= +PREFIX ?= /usr/local +BINDIR := $(DESTDIR)$(PREFIX)/bin + ZSTDDIR = ../../lib PROGDIR = ../../programs -CPPFLAGS = -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(PROGDIR) -I. -CXXFLAGS ?= -O3 -CXXFLAGS += -std=c++11 -CXXFLAGS += $(MOREFLAGS) -FLAGS = $(CPPFLAGS) $(CXXFLAGS) $(LDFLAGS) +# External program to use to run tests, e.g. qemu or valgrind +TESTPROG ?= +# Flags to pass to the tests +TESTFLAGS ?= + +# We use gcc/clang to generate the header dependencies of files +DEPFLAGS = -MMD -MP -MF $*.Td +POSTCOMPILE = mv -f $*.Td $*.d + +# CFLAGS, CXXFLAGS, and LDFLAGS are for the users to override +CFLAGS ?= -O3 -Wall -Wextra +CXXFLAGS ?= -O3 -Wall -Wextra -pedantic -std=c++11 +LDFLAGS ?= + +# Googletest default flags +PZSTD_INC = -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(PROGDIR) -I. +GTEST_INC = -isystem googletest/googletest/include + +PZSTD_CCXXFLAGS = $(PZSTD_INC) $(GTEST_INC) +PZSTD_CFLAGS = $(PZSTD_CCXXFLAGS) +PZSTD_CXXFLAGS = $(PZSTD_CCXXFLAGS) +PZSTD_LDFLAGS = +EXTRA_FLAGS = +ALL_CFLAGS = $(EXTRA_FLAGS) $(CPPFLAGS) $(PZSTD_CPPFLAGS) $(CFLAGS) $(PZSTD_CFLAGS) +ALL_CXXFLAGS = $(EXTRA_FLAGS) $(CPPFLAGS) $(PZSTD_CPPFLAGS) $(CXXFLAGS) $(PZSTD_CXXFLAGS) +ALL_LDFLAGS = $(EXTRA_FLAGS) $(LDFLAGS) $(PZSTD_LDFLAGS) -ZSTDCOMMON_FILES := $(ZSTDDIR)/common/*.c -ZSTDCOMP_FILES := $(ZSTDDIR)/compress/zstd_compress.c $(ZSTDDIR)/compress/fse_compress.c $(ZSTDDIR)/compress/huf_compress.c -ZSTDDECOMP_FILES := $(ZSTDDIR)/decompress/huf_decompress.c -ZSTD_FILES := $(ZSTDDECOMP_FILES) $(ZSTDCOMMON_FILES) $(ZSTDCOMP_FILES) +# gtest libraries need to go before "-lpthread" because they depend on it. +GTEST_LIB = -L googletest/build/googlemock/gtest +LIBS = $(GTEST_LIB) -lpthread + +# Compilation commands +LD_COMMAND = $(CXX) $^ $(ALL_LDFLAGS) $(LIBS) -o $@ +CC_COMMAND = $(CC) $(DEPFLAGS) $(ALL_CFLAGS) -c $< -o $@ +CXX_COMMAND = $(CXX) $(DEPFLAGS) $(ALL_CXXFLAGS) -c $< -o $@ + +# Get a list of all zstd files so we rebuild the static library when we need to +ZSTDCOMMON_FILES := $(wildcard $(ZSTDDIR)/common/*.c) \ + $(wildcard $(ZSTDDIR)/common/*.h) +ZSTDCOMP_FILES := $(wildcard $(ZSTDDIR)/compress/*.c) \ + $(wildcard $(ZSTDDIR)/compress/*.h) +ZSTDDECOMP_FILES := $(wildcard $(ZSTDDIR)/decompress/*.c) \ + $(wildcard $(ZSTDDIR)/decompress/*.h) +ZSTDPROG_FILES := $(wildcard $(PROGDIR)/*.c) \ + $(wildcard $(PROGDIR)/*.h) +ZSTD_FILES := $(wildcard $(ZSTDDIR)/*.h) \ + $(ZSTDDECOMP_FILES) $(ZSTDCOMMON_FILES) $(ZSTDCOMP_FILES) \ + $(ZSTDPROG_FILES) + +# List all the pzstd source files so we can determine their dependencies +PZSTD_SRCS := $(wildcard *.cpp) +PZSTD_TESTS := $(wildcard test/*.cpp) +UTILS_TESTS := $(wildcard utils/test/*.cpp) +ALL_SRCS := $(PZSTD_SRCS) $(PZSTD_TESTS) $(UTILS_TESTS) # Define *.exe as extension for Windows systems @@ -30,89 +79,168 @@ else EXT = endif -.PHONY: default all test clean test32 googletest googletest32 +# Standard targets +.PHONY: default +default: all -default: pzstd +.PHONY: check +check: + $(TESTPROG) ./utils/test/BufferTest$(EXT) $(TESTFLAGS) + $(TESTPROG) ./utils/test/RangeTest$(EXT) $(TESTFLAGS) + $(TESTPROG) ./utils/test/ScopeGuardTest$(EXT) $(TESTFLAGS) + $(TESTPROG) ./utils/test/ThreadPoolTest$(EXT) $(TESTFLAGS) + $(TESTPROG) ./utils/test/WorkQueueTest$(EXT) $(TESTFLAGS) + $(TESTPROG) ./test/OptionsTest$(EXT) $(TESTFLAGS) + $(TESTPROG) ./test/PzstdTest$(EXT) $(TESTFLAGS) -all: pzstd +.PHONY: install +install: PZSTD_CPPFLAGS += -DNDEBUG +install: pzstd$(EXT) + install -d -m 755 $(BINDIR)/ + install -m 755 pzstd$(EXT) $(BINDIR)/pzstd$(EXT) + +.PHONY: uninstall +uninstall: + $(RM) $(BINDIR)/pzstd$(EXT) + +# Targets for many different builds +.PHONY: all +all: PZSTD_CPPFLAGS += -DNDEBUG +all: pzstd$(EXT) tests roundtrip + +.PHONY: debug +debug: EXTRA_FLAGS += -g +debug: pzstd$(EXT) tests roundtrip + +.PHONY: tsan +tsan: PZSTD_CCXXFLAGS += -fsanitize=thread -fPIC +tsan: PZSTD_LDFLAGS += -fsanitize=thread -pie +tsan: debug + +.PHONY: asan +asan: EXTRA_FLAGS += -fsanitize=address +asan: debug + +.PHONY: ubsan +ubsan: EXTRA_FLAGS += -fsanitize=undefined +ubsan: debug + +.PHONY: all32 +all32: EXTRA_FLAGS += -m32 +all32: all + +.PHONY: debug32 +debug32: EXTRA_FLAGS += -m32 +debug32: debug + +.PHONY: asan32 +asan32: EXTRA_FLAGS += -m32 +asan32: asan + +.PHONY: tsan32 +tsan32: EXTRA_FLAGS += -m32 +tsan32: tsan + +.PHONY: ubsan32 +ubsan32: EXTRA_FLAGS += -m32 +ubsan32: ubsan + +# Run long round trip tests +.PHONY: roundtripcheck +roundtripcheck: roundtrip check + $(TESTPROG) ./test/RoundTripTest$(EXT) $(TESTFLAGS) + +# Build the main binary +pzstd$(EXT): main.o Options.o Pzstd.o SkippableFrame.o $(ZSTDDIR)/libzstd.a + $(LD_COMMAND) + +# Target that depends on all the tests +.PHONY: tests +tests: EXTRA_FLAGS += -Wno-deprecated-declarations +tests: $(patsubst %,%$(EXT),$(basename $(PZSTD_TESTS) $(UTILS_TESTS))) + +# Build the round trip tests +.PHONY: roundtrip +roundtrip: EXTRA_FLAGS += -Wno-deprecated-declarations +roundtrip: test/RoundTripTest$(EXT) + +# Use the static library that zstd builds for simplicity and +# so we get the compiler options correct +$(ZSTDDIR)/libzstd.a: $(ZSTD_FILES) + $(MAKE) -C $(ZSTDDIR) libzstd CFLAGS="$(ALL_CFLAGS)" LDFLAGS="$(ALL_LDFLAGS)" -libzstd.a: $(ZSTD_FILES) - $(MAKE) -C $(ZSTDDIR) libzstd - @cp $(ZSTDDIR)/libzstd.a . +# Rules to build the tests +test/RoundTripTest$(EXT): test/RoundTripTest.o $(PROGDIR)/datagen.o Options.o \ + Pzstd.o SkippableFrame.o $(ZSTDDIR)/libzstd.a + $(LD_COMMAND) -Pzstd.o: Pzstd.h Pzstd.cpp ErrorHolder.h utils/*.h - $(CXX) $(FLAGS) -c Pzstd.cpp -o $@ +test/%Test$(EXT): GTEST_LIB += -lgtest -lgtest_main +test/%Test$(EXT): test/%Test.o $(PROGDIR)/datagen.o Options.o Pzstd.o \ + SkippableFrame.o $(ZSTDDIR)/libzstd.a + $(LD_COMMAND) -SkippableFrame.o: SkippableFrame.h SkippableFrame.cpp utils/*.h - $(CXX) $(FLAGS) -c SkippableFrame.cpp -o $@ +utils/test/%Test$(EXT): GTEST_LIB += -lgtest -lgtest_main +utils/test/%Test$(EXT): utils/test/%Test.o + $(LD_COMMAND) -Options.o: Options.h Options.cpp - $(CXX) $(FLAGS) -c Options.cpp -o $@ -main.o: main.cpp *.h utils/*.h - $(CXX) $(FLAGS) -c main.cpp -o $@ - -pzstd: Pzstd.o SkippableFrame.o Options.o main.o libzstd.a - $(CXX) $(FLAGS) $^ -o $@$(EXT) -lpthread - -libzstd32.a: $(ZSTD_FILES) - $(MAKE) -C $(ZSTDDIR) libzstd MOREFLAGS="-m32" - @cp $(ZSTDDIR)/libzstd.a libzstd32.a - -Pzstd32.o: Pzstd.h Pzstd.cpp ErrorHolder.h utils/*.h - $(CXX) -m32 $(FLAGS) -c Pzstd.cpp -o $@ - -SkippableFrame32.o: SkippableFrame.h SkippableFrame.cpp utils/*.h - $(CXX) -m32 $(FLAGS) -c SkippableFrame.cpp -o $@ - -Options32.o: Options.h Options.cpp - $(CXX) -m32 $(FLAGS) -c Options.cpp -o $@ - -main32.o: main.cpp *.h utils/*.h - $(CXX) -m32 $(FLAGS) -c main.cpp -o $@ - -pzstd32: Pzstd32.o SkippableFrame32.o Options32.o main32.o libzstd32.a - $(CXX) -m32 $(FLAGS) $^ -o $@$(EXT) -lpthread +GTEST_CMAKEFLAGS = +# Install googletest +.PHONY: googletest +googletest: PZSTD_CCXXFLAGS += -fPIC googletest: @$(RM) -rf googletest @git clone https://github.com/google/googletest @mkdir -p googletest/build - @cd googletest/build && cmake .. && make + @cd googletest/build && cmake $(GTEST_CMAKEFLAGS) -DCMAKE_CXX_FLAGS="$(ALL_CXXFLAGS)" .. && $(MAKE) -googletest32: - @$(RM) -rf googletest - @git clone https://github.com/google/googletest - @mkdir -p googletest/build - @cd googletest/build && cmake .. -DCMAKE_CXX_FLAGS=-m32 && make - -googletest-mingw64: - $(RM) -rf googletest - git clone https://github.com/google/googletest - mkdir -p googletest/build - cd googletest/build && cmake -G "MSYS Makefiles" .. && $(MAKE) - -test: - $(MAKE) libzstd.a - $(MAKE) pzstd MOREFLAGS="-Wall -Wextra -pedantic -Werror" - $(MAKE) -C utils/test clean - $(MAKE) -C utils/test test MOREFLAGS="-Wall -Wextra -pedantic -Werror" - $(MAKE) -C test clean - $(MAKE) -C test test MOREFLAGS="-Wall -Wextra -pedantic -Werror" - -test32: - $(MAKE) libzstd.a MOREFLAGS="-m32" - $(MAKE) pzstd MOREFLAGS="-m32 -Wall -Wextra -pedantic -Werror" - $(MAKE) -C utils/test clean - $(MAKE) -C utils/test test MOREFLAGS="-m32 -Wall -Wextra -pedantic -Werror" - $(MAKE) -C test clean - $(MAKE) -C test test MOREFLAGS="-m32 -Wall -Wextra -pedantic -Werror" +.PHONY: googletest32 +googletest32: PZSTD_CCXXFLAGS += -m32 +googletest32: googletest +.PHONY: googletest-mingw64 +googletest-mingw64: GTEST_CMAKEFLAGS += -G "MSYS Makefiles" +googletest-mingw64: googletest +.PHONY: clean clean: + $(RM) -f *.o pzstd$(EXT) *.Td *.d + $(RM) -f test/*.o test/*Test$(EXT) test/*.Td test/*.d + $(RM) -f utils/test/*.o utils/test/*Test$(EXT) utils/test/*.Td utils/test/*.d + $(RM) -f $(PROGDIR)/*.o $(PROGDIR)/*.Td $(PROGDIR)/*.d $(MAKE) -C $(ZSTDDIR) clean - $(MAKE) -C utils/test clean - $(MAKE) -C test clean - @$(RM) -rf libzstd.a *.o pzstd$(EXT) pzstd32$(EXT) @echo Cleaning completed + + +# Cancel implicit rules +%.o: %.c +%.o: %.cpp + +# Object file rules +%.o: %.c + $(CC_COMMAND) + $(POSTCOMPILE) + +$(PROGDIR)/%.o: $(PROGDIR)/%.c + $(CC_COMMAND) + $(POSTCOMPILE) + +%.o: %.cpp + $(CXX_COMMAND) + $(POSTCOMPILE) + +test/%.o: test/%.cpp + $(CXX_COMMAND) + $(POSTCOMPILE) + +utils/test/%.o: utils/test/%.cpp + $(CXX_COMMAND) + $(POSTCOMPILE) + +# Dependency file stuff +.PRECIOUS: %.d test/%.d utils/test/%.d + +# Include rules that specify header file dependencies +-include $(patsubst %,%.d,$(basename $(ALL_SRCS))) diff --git a/contrib/pzstd/test/Makefile b/contrib/pzstd/test/Makefile deleted file mode 100644 index 4f6ba9997..000000000 --- a/contrib/pzstd/test/Makefile +++ /dev/null @@ -1,48 +0,0 @@ -# ########################################################################## -# Copyright (c) 2016-present, Facebook, Inc. -# All rights reserved. -# -# This source code is licensed under the BSD-style license found in the -# LICENSE file in the root directory of this source tree. An additional grant -# of patent rights can be found in the PATENTS file in the same directory. -# ########################################################################## - -# Define *.exe as extension for Windows systems -ifneq (,$(filter Windows%,$(OS))) -EXT =.exe -else -EXT = -endif - -PZSTDDIR = .. -PROGDIR = ../../../programs -ZSTDDIR = ../../../lib - -# Set GTEST_INC and GTEST_LIB to work with your install of gtest -GTEST_INC ?= -isystem $(PZSTDDIR)/googletest/googletest/include -GTEST_LIB ?= -L $(PZSTDDIR)/googletest/build/googlemock/gtest -GTEST_FLAGS = $(GTEST_INC) $(GTEST_LIB) -CPPFLAGS = -I$(PZSTDDIR) -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(PROGDIR) -I. - -CXXFLAGS ?= -O3 -CXXFLAGS += -std=c++11 -Wno-deprecated-declarations -CXXFLAGS += $(MOREFLAGS) -FLAGS = $(CPPFLAGS) $(CXXFLAGS) $(LDFLAGS) - -datagen.o: $(PROGDIR)/datagen.* - $(CC) $(CPPFLAGS) -O3 $(MOREFLAGS) $(LDFLAGS) -Wno-long-long -Wno-variadic-macros $(PROGDIR)/datagen.c -c -o $@ - -%: %.cpp *.h datagen.o - $(CXX) $(FLAGS) $@.cpp datagen.o $(PZSTDDIR)/Pzstd.o $(PZSTDDIR)/SkippableFrame.o $(PZSTDDIR)/Options.o $(PZSTDDIR)/libzstd.a -o $@$(EXT) $(GTEST_FLAGS) -lgtest -lgtest_main -lpthread - -.PHONY: test clean - -test: OptionsTest PzstdTest - @./OptionsTest$(EXT) - @./PzstdTest$(EXT) - -roundtrip: RoundTripTest - @./RoundTripTest$(EXT) - -clean: - @rm -f datagen.o OptionsTest PzstdTest RoundTripTest diff --git a/contrib/pzstd/utils/test/Makefile b/contrib/pzstd/utils/test/Makefile deleted file mode 100644 index b9ea73e32..000000000 --- a/contrib/pzstd/utils/test/Makefile +++ /dev/null @@ -1,42 +0,0 @@ -# ########################################################################## -# Copyright (c) 2016-present, Facebook, Inc. -# All rights reserved. -# -# This source code is licensed under the BSD-style license found in the -# LICENSE file in the root directory of this source tree. An additional grant -# of patent rights can be found in the PATENTS file in the same directory. -# ########################################################################## - -# Define *.exe as extension for Windows systems -ifneq (,$(filter Windows%,$(OS))) -EXT =.exe -else -EXT = -endif - -PZSTDDIR = ../.. - -# Set GTEST_INC and GTEST_LIB to work with your install of gtest -GTEST_INC ?= -isystem $(PZSTDDIR)/googletest/googletest/include -GTEST_LIB ?= -L $(PZSTDDIR)/googletest/build/googlemock/gtest - -CPPFLAGS = -I$(PZSTDDIR) $(GTEST_INC) $(GTEST_LIB) -CXXFLAGS ?= -O3 -CXXFLAGS += -std=c++11 -CXXFLAGS += $(MOREFLAGS) -FLAGS = $(CPPFLAGS) $(CXXFLAGS) $(LDFLAGS) - -%: %.cpp - $(CXX) $(FLAGS) $^ -o $@$(EXT) -lgtest -lgtest_main -lpthread - -.PHONY: test clean - -test: BufferTest RangeTest ScopeGuardTest ThreadPoolTest WorkQueueTest - @./BufferTest$(EXT) - @./RangeTest$(EXT) - @./ScopeGuardTest$(EXT) - @./ThreadPoolTest$(EXT) - @./WorkQueueTest$(EXT) - -clean: - @rm -f BufferTest RangeTest ScopeGuardTest ThreadPoolTest WorkQueueTest From 96e0702c00c1ec9a9888a2d113b38d926681d3e0 Mon Sep 17 00:00:00 2001 From: Nick Terrell Date: Thu, 6 Oct 2016 21:31:16 -0700 Subject: [PATCH 30/92] [pzstd] Print the correct width ints --- contrib/pzstd/Pzstd.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/contrib/pzstd/Pzstd.cpp b/contrib/pzstd/Pzstd.cpp index 59a2496c4..68f5bb977 100644 --- a/contrib/pzstd/Pzstd.cpp +++ b/contrib/pzstd/Pzstd.cpp @@ -15,6 +15,7 @@ #include "utils/WorkQueue.h" #include +#include #include #include #include @@ -104,11 +105,12 @@ static std::uint64_t handleOneInput(const Options &options, if (!options.decompress) { double ratio = static_cast(bytesWritten) / static_cast(bytesRead + !bytesRead); - std::fprintf(stderr, "%-20s :%6.2f%% (%6llu => %6llu bytes, %s)\n", + std::fprintf(stderr, "%-20s :%6.2f%% (%6" PRIu64 " => %6" PRIu64 + " bytes, %s)\n", inputFileName.c_str(), ratio * 100, bytesRead, bytesWritten, outputFileName.c_str()); } else { - std::fprintf(stderr, "%-20s: %llu bytes \n", + std::fprintf(stderr, "%-20s: %" PRIu64 " bytes \n", inputFileName.c_str(),bytesWritten); } } From 4cb5e90a5c7ce0162c2b8dbf2e055e3951fd75fb Mon Sep 17 00:00:00 2001 From: Nick Terrell Date: Thu, 6 Oct 2016 21:32:06 -0700 Subject: [PATCH 31/92] [pzstd] Add asan and tsan tests to travis gcc-6 tsan is buggy. It fails to use the correct linker. It is also broken with `-pie` with linux kernels newer than 4.1, but previous versions require `-pie`... --- .travis.yml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/.travis.yml b/.travis.yml index f328f480d..0c8960725 100644 --- a/.travis.yml +++ b/.travis.yml @@ -55,6 +55,20 @@ matrix: packages: - libc6-dev-i386 - gcc-multilib + - os: linux + sudo: required + install: + - export CXX="g++-6" CC="gcc-6" + - export LDFLAGS="-fuse-ld=gold" + - export TESTFLAGS='--gtest_filter=-*ExtremelyLarge*' + env: PLATFORM="Ubuntu 12.04" CMD='cd contrib/pzstd && make googletest && make tsan && make check && make clean && make asan && make check && make clean && cd ../..' + addons: + apt: + sources: + - ubuntu-toolchain-r-test + packages: + - gcc-6 + - g++-6 # Ubuntu 14.04 LTS Server Edition 64 bit - os: linux dist: trusty From 9b603ee284878039c076e1cc88215e4749d126dc Mon Sep 17 00:00:00 2001 From: Nick Terrell Date: Fri, 7 Oct 2016 15:04:34 -0700 Subject: [PATCH 32/92] [pzstd] Run the reading thread separately --- contrib/pzstd/Pzstd.cpp | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/contrib/pzstd/Pzstd.cpp b/contrib/pzstd/Pzstd.cpp index 68f5bb977..db9b8c85b 100644 --- a/contrib/pzstd/Pzstd.cpp +++ b/contrib/pzstd/Pzstd.cpp @@ -62,19 +62,18 @@ static std::uint64_t handleOneInput(const Options &options, ErrorHolder &errorHolder) { auto inputSize = fileSizeOrZero(inputFile); // WorkQueue outlives ThreadPool so in the case of error we are certain - // we don't accidently try to call push() on it after it is destroyed. + // we don't accidently try to call push() on it after it is destroyed WorkQueue> outs{options.numThreads + 1}; std::uint64_t bytesRead; std::uint64_t bytesWritten; { - // Initialize the thread pool with numThreads + 1 - // We add one because the read thread spends most of its time waiting. - // This also sets the minimum number of threads to 2, so the algorithm - // doesn't deadlock. - ThreadPool executor(options.numThreads + 1); + // Initialize the (de)compression thread pool with numThreads + ThreadPool executor(options.numThreads); + // Run the reader thread on an extra thread + ThreadPool readExecutor(1); if (!options.decompress) { // Add a job that reads the input and starts all the compression jobs - executor.add( + readExecutor.add( [&errorHolder, &outs, &executor, inputFd, inputSize, &options, &bytesRead] { bytesRead = asyncCompressChunks( @@ -91,7 +90,7 @@ static std::uint64_t handleOneInput(const Options &options, options.verbosity); } else { // Add a job that reads the input and starts all the decompression jobs - executor.add([&errorHolder, &outs, &executor, inputFd, &bytesRead] { + readExecutor.add([&errorHolder, &outs, &executor, inputFd, &bytesRead] { bytesRead = asyncDecompressFrames(errorHolder, outs, executor, inputFd); }); // Start writing From 48294b57c359bc0cfc6560df89f34507658bdc3d Mon Sep 17 00:00:00 2001 From: Nick Terrell Date: Wed, 12 Oct 2016 15:18:16 -0700 Subject: [PATCH 33/92] [pzstd] Put ErrorHolder into SharedState --- contrib/pzstd/Pzstd.cpp | 74 +++++++++++++++++++++-------------------- contrib/pzstd/Pzstd.h | 18 ++++++---- 2 files changed, 50 insertions(+), 42 deletions(-) diff --git a/contrib/pzstd/Pzstd.cpp b/contrib/pzstd/Pzstd.cpp index db9b8c85b..70c0515b3 100644 --- a/contrib/pzstd/Pzstd.cpp +++ b/contrib/pzstd/Pzstd.cpp @@ -59,7 +59,7 @@ static std::uint64_t handleOneInput(const Options &options, FILE* inputFd, const std::string &outputFile, FILE* outputFd, - ErrorHolder &errorHolder) { + SharedState& state) { auto inputSize = fileSizeOrZero(inputFile); // WorkQueue outlives ThreadPool so in the case of error we are certain // we don't accidently try to call push() on it after it is destroyed @@ -74,10 +74,9 @@ static std::uint64_t handleOneInput(const Options &options, if (!options.decompress) { // Add a job that reads the input and starts all the compression jobs readExecutor.add( - [&errorHolder, &outs, &executor, inputFd, inputSize, &options, - &bytesRead] { + [&state, &outs, &executor, inputFd, inputSize, &options, &bytesRead] { bytesRead = asyncCompressChunks( - errorHolder, + state, outs, executor, inputFd, @@ -86,19 +85,19 @@ static std::uint64_t handleOneInput(const Options &options, options.determineParameters()); }); // Start writing - bytesWritten = writeFile(errorHolder, outs, outputFd, options.decompress, + bytesWritten = writeFile(state, outs, outputFd, options.decompress, options.verbosity); } else { // Add a job that reads the input and starts all the decompression jobs - readExecutor.add([&errorHolder, &outs, &executor, inputFd, &bytesRead] { - bytesRead = asyncDecompressFrames(errorHolder, outs, executor, inputFd); + readExecutor.add([&state, &outs, &executor, inputFd, &bytesRead] { + bytesRead = asyncDecompressFrames(state, outs, executor, inputFd); }); // Start writing - bytesWritten = writeFile(errorHolder, outs, outputFd, options.decompress, + bytesWritten = writeFile(state, outs, outputFd, options.decompress, options.verbosity); } } - if (options.verbosity > 1 && !errorHolder.hasError()) { + if (options.verbosity > 1 && !state.errorHolder.hasError()) { std::string inputFileName = inputFile == "-" ? "stdin" : inputFile; std::string outputFileName = outputFile == "-" ? "stdout" : outputFile; if (!options.decompress) { @@ -176,53 +175,53 @@ int pzstdMain(const Options &options) { int returnCode = 0; for (const auto& input : options.inputFiles) { // Setup the error holder - ErrorHolder errorHolder; + SharedState state; auto printErrorGuard = makeScopeGuard([&] { - if (errorHolder.hasError()) { + if (state.errorHolder.hasError()) { returnCode = 1; if (options.verbosity > 0) { std::fprintf(stderr, "pzstd: %s: %s.\n", input.c_str(), - errorHolder.getError().c_str()); + state.errorHolder.getError().c_str()); } } }); // Open the input file - auto inputFd = openInputFile(input, errorHolder); + auto inputFd = openInputFile(input, state.errorHolder); if (inputFd == nullptr) { continue; } auto closeInputGuard = makeScopeGuard([&] { std::fclose(inputFd); }); // Open the output file auto outputFile = options.getOutputFile(input); - if (!errorHolder.check(outputFile != "", + if (!state.errorHolder.check(outputFile != "", "Input file does not have extension .zst")) { continue; } - auto outputFd = openOutputFile(options, outputFile, errorHolder); + auto outputFd = openOutputFile(options, outputFile, state.errorHolder); if (outputFd == nullptr) { continue; } auto closeOutputGuard = makeScopeGuard([&] { std::fclose(outputFd); }); // (de)compress the file - handleOneInput(options, input, inputFd, outputFile, outputFd, errorHolder); - if (errorHolder.hasError()) { + handleOneInput(options, input, inputFd, outputFile, outputFd, state); + if (state.errorHolder.hasError()) { continue; } // Delete the input file if necessary if (!options.keepSource) { // Be sure that we are done and have written everything before we delete - if (!errorHolder.check(std::fclose(inputFd) == 0, + if (!state.errorHolder.check(std::fclose(inputFd) == 0, "Failed to close input file")) { continue; } closeInputGuard.dismiss(); - if (!errorHolder.check(std::fclose(outputFd) == 0, + if (!state.errorHolder.check(std::fclose(outputFd) == 0, "Failed to close output file")) { continue; } closeOutputGuard.dismiss(); if (std::remove(input.c_str()) != 0) { - errorHolder.setError("Failed to remove input file"); + state.errorHolder.setError("Failed to remove input file"); continue; } } @@ -268,18 +267,19 @@ Buffer split(Buffer& buffer, ZSTD_outBuffer& outBuffer) { /** * Stream chunks of input from `in`, compress it, and stream it out to `out`. * - * @param errorHolder Used to report errors and check if an error occured + * @param state The shared state * @param in Queue that we `pop()` input buffers from * @param out Queue that we `push()` compressed output buffers to * @param maxInputSize An upper bound on the size of the input * @param parameters The zstd parameters to use for compression */ static void compress( - ErrorHolder& errorHolder, + SharedState& state, std::shared_ptr in, std::shared_ptr out, size_t maxInputSize, ZSTD_parameters parameters) { + auto& errorHolder = state.errorHolder; auto guard = makeScopeGuard([&] { out->finish(); }); // Initialize the CCtx std::unique_ptr ctx( @@ -395,7 +395,7 @@ readData(BufferWorkQueue& queue, size_t chunkSize, size_t size, FILE* fd, } std::uint64_t asyncCompressChunks( - ErrorHolder& errorHolder, + SharedState& state, WorkQueue>& chunks, ThreadPool& executor, FILE* fd, @@ -409,23 +409,23 @@ std::uint64_t asyncCompressChunks( // independently. size_t step = calculateStep(size, numThreads, params); auto status = FileStatus::Continue; - while (status == FileStatus::Continue && !errorHolder.hasError()) { + while (status == FileStatus::Continue && !state.errorHolder.hasError()) { // Make a new input queue that we will put the chunk's input data into. auto in = std::make_shared(); auto inGuard = makeScopeGuard([&] { in->finish(); }); // Make a new output queue that compress will put the compressed data into. auto out = std::make_shared(); // Start compression in the thread pool - executor.add([&errorHolder, in, out, step, params] { + executor.add([&state, in, out, step, params] { return compress( - errorHolder, std::move(in), std::move(out), step, params); + state, std::move(in), std::move(out), step, params); }); // Pass the output queue to the writer thread. chunks.push(std::move(out)); // Fill the input queue for the compression job we just started status = readData(*in, ZSTD_CStreamInSize(), step, fd, &bytesRead); } - errorHolder.check(status != FileStatus::Error, "Error reading input"); + state.errorHolder.check(status != FileStatus::Error, "Error reading input"); return bytesRead; } @@ -433,15 +433,16 @@ std::uint64_t asyncCompressChunks( * Decompress a frame, whose data is streamed into `in`, and stream the output * to `out`. * - * @param errorHolder Used to report errors and check if an error occured + * @param state The shared state * @param in Queue that we `pop()` input buffers from. It contains * exactly one compressed frame. * @param out Queue that we `push()` decompressed output buffers to */ static void decompress( - ErrorHolder& errorHolder, + SharedState& state, std::shared_ptr in, std::shared_ptr out) { + auto& errorHolder = state.errorHolder; auto guard = makeScopeGuard([&] { out->finish(); }); // Initialize the DCtx std::unique_ptr ctx( @@ -508,7 +509,7 @@ static void decompress( } std::uint64_t asyncDecompressFrames( - ErrorHolder& errorHolder, + SharedState& state, WorkQueue>& frames, ThreadPool& executor, FILE* fd) { @@ -521,7 +522,7 @@ std::uint64_t asyncDecompressFrames( // Otherwise, we will decompress using only one decompression task. const size_t chunkSize = ZSTD_DStreamInSize(); auto status = FileStatus::Continue; - while (status == FileStatus::Continue && !errorHolder.hasError()) { + while (status == FileStatus::Continue && !state.errorHolder.hasError()) { // Make a new input queue that we will put the frames's bytes into. auto in = std::make_shared(); auto inGuard = makeScopeGuard([&] { in->finish(); }); @@ -550,15 +551,15 @@ std::uint64_t asyncDecompressFrames( out->setMaxSize(64); } // Start decompression in the thread pool - executor.add([&errorHolder, in, out] { - return decompress(errorHolder, std::move(in), std::move(out)); + executor.add([&state, in, out] { + return decompress(state, std::move(in), std::move(out)); }); // Pass the output queue to the writer thread frames.push(std::move(out)); if (frameSize == 0) { // We hit a non SkippableFrame ==> not compressed by pzstd or corrupted // Pass the rest of the source to this decompression task - while (status == FileStatus::Continue && !errorHolder.hasError()) { + while (status == FileStatus::Continue && !state.errorHolder.hasError()) { status = readData(*in, chunkSize, chunkSize, fd, &totalBytesRead); } break; @@ -566,7 +567,7 @@ std::uint64_t asyncDecompressFrames( // Fill the input queue for the decompression job we just started status = readData(*in, chunkSize, frameSize, fd, &totalBytesRead); } - errorHolder.check(status != FileStatus::Error, "Error reading input"); + state.errorHolder.check(status != FileStatus::Error, "Error reading input"); return totalBytesRead; } @@ -598,11 +599,12 @@ void updateWritten(int verbosity, std::uint64_t bytesWritten) { } std::uint64_t writeFile( - ErrorHolder& errorHolder, + SharedState& state, WorkQueue>& outs, FILE* outputFd, bool decompress, int verbosity) { + auto& errorHolder = state.errorHolder; auto lineClearGuard = makeScopeGuard([verbosity] { if (verbosity > 1) { std::fprintf(stderr, "\r%79s\r", ""); diff --git a/contrib/pzstd/Pzstd.h b/contrib/pzstd/Pzstd.h index fe44ccfde..469c20cd4 100644 --- a/contrib/pzstd/Pzstd.h +++ b/contrib/pzstd/Pzstd.h @@ -12,6 +12,7 @@ #include "Options.h" #include "utils/Buffer.h" #include "utils/Range.h" +#include "utils/ResourcePool.h" #include "utils/ThreadPool.h" #include "utils/WorkQueue.h" #define ZSTD_STATIC_LINKING_ONLY @@ -32,12 +33,17 @@ namespace pzstd { */ int pzstdMain(const Options& options); +class SharedState { + public: + ErrorHolder errorHolder; +}; + /** * Streams input from `fd`, breaks input up into chunks, and compresses each * chunk independently. Output of each chunk gets streamed to a queue, and * the output queues get put into `chunks` in order. * - * @param errorHolder Used to report errors and coordinate early shutdown + * @param state The shared state * @param chunks Each compression jobs output queue gets `pushed()` here * as soon as it is available * @param executor The thread pool to run compression jobs in @@ -48,7 +54,7 @@ int pzstdMain(const Options& options); * @returns The number of bytes read from the file */ std::uint64_t asyncCompressChunks( - ErrorHolder& errorHolder, + SharedState& state, WorkQueue>& chunks, ThreadPool& executor, FILE* fd, @@ -62,7 +68,7 @@ std::uint64_t asyncCompressChunks( * decompression job. Output of each frame gets streamed to a queue, and * the output queues get put into `frames` in order. * - * @param errorHolder Used to report errors and coordinate early shutdown + * @param state The shared state * @param frames Each decompression jobs output queue gets `pushed()` here * as soon as it is available * @param executor The thread pool to run compression jobs in @@ -70,7 +76,7 @@ std::uint64_t asyncCompressChunks( * @returns The number of bytes read from the file */ std::uint64_t asyncDecompressFrames( - ErrorHolder& errorHolder, + SharedState& state, WorkQueue>& frames, ThreadPool& executor, FILE* fd); @@ -79,7 +85,7 @@ std::uint64_t asyncDecompressFrames( * Streams input in from each queue in `outs` in order, and writes the data to * `outputFd`. * - * @param errorHolder Used to report errors and coordinate early exit + * @param state The shared state * @param outs A queue of output queues, one for each * (de)compression job. * @param outputFd The file descriptor to write to @@ -88,7 +94,7 @@ std::uint64_t asyncDecompressFrames( * @returns The number of bytes written */ std::uint64_t writeFile( - ErrorHolder& errorHolder, + SharedState& state, WorkQueue>& outs, FILE* outputFd, bool decompress, From e9e151ce31835accd17ed5dcdf87ba18c558606d Mon Sep 17 00:00:00 2001 From: Nick Terrell Date: Wed, 12 Oct 2016 17:23:38 -0700 Subject: [PATCH 34/92] [pzstd] Reuse ZSTD_{C,D}Stream --- contrib/pzstd/Makefile | 1 + contrib/pzstd/Pzstd.cpp | 22 ++--- contrib/pzstd/Pzstd.h | 39 ++++++++ contrib/pzstd/utils/ResourcePool.h | 96 +++++++++++++++++++ contrib/pzstd/utils/test/ResourcePoolTest.cpp | 72 ++++++++++++++ 5 files changed, 217 insertions(+), 13 deletions(-) create mode 100644 contrib/pzstd/utils/ResourcePool.h create mode 100644 contrib/pzstd/utils/test/ResourcePoolTest.cpp diff --git a/contrib/pzstd/Makefile b/contrib/pzstd/Makefile index b998c1fff..4f63887d3 100644 --- a/contrib/pzstd/Makefile +++ b/contrib/pzstd/Makefile @@ -87,6 +87,7 @@ default: all check: $(TESTPROG) ./utils/test/BufferTest$(EXT) $(TESTFLAGS) $(TESTPROG) ./utils/test/RangeTest$(EXT) $(TESTFLAGS) + $(TESTPROG) ./utils/test/ResourcePoolTest$(EXT) $(TESTFLAGS) $(TESTPROG) ./utils/test/ScopeGuardTest$(EXT) $(TESTFLAGS) $(TESTPROG) ./utils/test/ThreadPoolTest$(EXT) $(TESTFLAGS) $(TESTPROG) ./utils/test/WorkQueueTest$(EXT) $(TESTFLAGS) diff --git a/contrib/pzstd/Pzstd.cpp b/contrib/pzstd/Pzstd.cpp index 70c0515b3..6b3d27b3e 100644 --- a/contrib/pzstd/Pzstd.cpp +++ b/contrib/pzstd/Pzstd.cpp @@ -173,9 +173,9 @@ static FILE *openOutputFile(const Options &options, int pzstdMain(const Options &options) { int returnCode = 0; + SharedState state(options.decompress, options.determineParameters()); for (const auto& input : options.inputFiles) { - // Setup the error holder - SharedState state; + // Setup the shared state auto printErrorGuard = makeScopeGuard([&] { if (state.errorHolder.hasError()) { returnCode = 1; @@ -271,24 +271,21 @@ Buffer split(Buffer& buffer, ZSTD_outBuffer& outBuffer) { * @param in Queue that we `pop()` input buffers from * @param out Queue that we `push()` compressed output buffers to * @param maxInputSize An upper bound on the size of the input - * @param parameters The zstd parameters to use for compression */ static void compress( SharedState& state, std::shared_ptr in, std::shared_ptr out, - size_t maxInputSize, - ZSTD_parameters parameters) { + size_t maxInputSize) { auto& errorHolder = state.errorHolder; auto guard = makeScopeGuard([&] { out->finish(); }); // Initialize the CCtx - std::unique_ptr ctx( - ZSTD_createCStream(), ZSTD_freeCStream); + auto ctx = state.cStreamPool->get(); if (!errorHolder.check(ctx != nullptr, "Failed to allocate ZSTD_CStream")) { return; } { - auto err = ZSTD_initCStream_advanced(ctx.get(), nullptr, 0, parameters, 0); + auto err = ZSTD_resetCStream(ctx.get(), 0); if (!errorHolder.check(!ZSTD_isError(err), ZSTD_getErrorName(err))) { return; } @@ -416,9 +413,9 @@ std::uint64_t asyncCompressChunks( // Make a new output queue that compress will put the compressed data into. auto out = std::make_shared(); // Start compression in the thread pool - executor.add([&state, in, out, step, params] { + executor.add([&state, in, out, step] { return compress( - state, std::move(in), std::move(out), step, params); + state, std::move(in), std::move(out), step); }); // Pass the output queue to the writer thread. chunks.push(std::move(out)); @@ -445,13 +442,12 @@ static void decompress( auto& errorHolder = state.errorHolder; auto guard = makeScopeGuard([&] { out->finish(); }); // Initialize the DCtx - std::unique_ptr ctx( - ZSTD_createDStream(), ZSTD_freeDStream); + auto ctx = state.dStreamPool->get(); if (!errorHolder.check(ctx != nullptr, "Failed to allocate ZSTD_DStream")) { return; } { - auto err = ZSTD_initDStream(ctx.get()); + auto err = ZSTD_resetDStream(ctx.get()); if (!errorHolder.check(!ZSTD_isError(err), ZSTD_getErrorName(err))) { return; } diff --git a/contrib/pzstd/Pzstd.h b/contrib/pzstd/Pzstd.h index 469c20cd4..b02fe7b19 100644 --- a/contrib/pzstd/Pzstd.h +++ b/contrib/pzstd/Pzstd.h @@ -35,7 +35,46 @@ int pzstdMain(const Options& options); class SharedState { public: + SharedState(bool decompress, ZSTD_parameters parameters) { + if (!decompress) { + cStreamPool.reset(new ResourcePool{ + [parameters]() -> ZSTD_CStream* { + auto zcs = ZSTD_createCStream(); + if (zcs) { + auto err = ZSTD_initCStream_advanced( + zcs, nullptr, 0, parameters, 0); + if (ZSTD_isError(err)) { + ZSTD_freeCStream(zcs); + return nullptr; + } + } + return zcs; + }, + [](ZSTD_CStream *zcs) { + ZSTD_freeCStream(zcs); + }}); + } else { + dStreamPool.reset(new ResourcePool{ + []() -> ZSTD_DStream* { + auto zds = ZSTD_createDStream(); + if (zds) { + auto err = ZSTD_initDStream(zds); + if (ZSTD_isError(err)) { + ZSTD_freeDStream(zds); + return nullptr; + } + } + return zds; + }, + [](ZSTD_DStream *zds) { + ZSTD_freeDStream(zds); + }}); + } + } + ErrorHolder errorHolder; + std::unique_ptr> cStreamPool; + std::unique_ptr> dStreamPool; }; /** diff --git a/contrib/pzstd/utils/ResourcePool.h b/contrib/pzstd/utils/ResourcePool.h new file mode 100644 index 000000000..ed011306b --- /dev/null +++ b/contrib/pzstd/utils/ResourcePool.h @@ -0,0 +1,96 @@ +/** + * Copyright (c) 2016-present, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ +#pragma once + +#include +#include +#include +#include +#include + +namespace pzstd { + +/** + * An unbounded pool of resources. + * A `ResourcePool` requires a factory function that takes allocates `T*` and + * a free function that frees a `T*`. + * Calling `ResourcePool::get()` will give you a new `ResourcePool::UniquePtr` + * to a `T`, and when it goes out of scope the resource will be returned to the + * pool. + * The `ResourcePool` *must* survive longer than any resources it hands out. + * Remember that `ResourcePool` hands out mutable `T`s, so make sure to clean + * up the resource before or after every use. + */ +template +class ResourcePool { + public: + class Deleter; + using Factory = std::function; + using Free = std::function; + using UniquePtr = std::unique_ptr; + + private: + std::mutex mutex_; + Factory factory_; + Free free_; + std::vector resources_; + unsigned inUse_; + + public: + /** + * Creates a `ResourcePool`. + * + * @param factory The function to use to create new resources. + * @param free The function to use to free resources created by `factory`. + */ + ResourcePool(Factory factory, Free free) + : factory_(std::move(factory)), free_(std::move(free)), inUse_(0) {} + + /** + * @returns A unique pointer to a resource. The resource is null iff + * there are no avaiable resources and `factory()` returns null. + */ + UniquePtr get() { + std::lock_guard lock(mutex_); + if (!resources_.empty()) { + UniquePtr resource{resources_.back(), Deleter{*this}}; + resources_.pop_back(); + ++inUse_; + return resource; + } + UniquePtr resource{factory_(), Deleter{*this}}; + ++inUse_; + return resource; + } + + ~ResourcePool() noexcept { + assert(inUse_ == 0); + for (const auto resource : resources_) { + free_(resource); + } + } + + class Deleter { + ResourcePool *pool_; + public: + explicit Deleter(ResourcePool &pool) : pool_(&pool) {} + + void operator() (T *resource) { + std::lock_guard lock(pool_->mutex_); + // Make sure we don't put null resources into the pool + if (resource) { + pool_->resources_.push_back(resource); + } + assert(pool_->inUse_ > 0); + --pool_->inUse_; + } + }; +}; + +} diff --git a/contrib/pzstd/utils/test/ResourcePoolTest.cpp b/contrib/pzstd/utils/test/ResourcePoolTest.cpp new file mode 100644 index 000000000..a6a86b345 --- /dev/null +++ b/contrib/pzstd/utils/test/ResourcePoolTest.cpp @@ -0,0 +1,72 @@ +/** + * Copyright (c) 2016-present, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ +#include "utils/ResourcePool.h" + +#include +#include +#include + +using namespace pzstd; + +TEST(ResourcePool, FullTest) { + unsigned numCreated = 0; + unsigned numDeleted = 0; + { + ResourcePool pool( + [&numCreated] { ++numCreated; return new int{5}; }, + [&numDeleted](int *x) { ++numDeleted; delete x; }); + + { + auto i = pool.get(); + EXPECT_EQ(5, *i); + *i = 6; + } + { + auto i = pool.get(); + EXPECT_EQ(6, *i); + auto j = pool.get(); + EXPECT_EQ(5, *j); + *j = 7; + } + { + auto i = pool.get(); + EXPECT_EQ(6, *i); + auto j = pool.get(); + EXPECT_EQ(7, *j); + } + } + EXPECT_EQ(2, numCreated); + EXPECT_EQ(numCreated, numDeleted); +} + +TEST(ResourcePool, ThreadSafe) { + std::atomic numCreated{0}; + std::atomic numDeleted{0}; + { + ResourcePool pool( + [&numCreated] { ++numCreated; return new int{0}; }, + [&numDeleted](int *x) { ++numDeleted; delete x; }); + auto push = [&pool] { + for (int i = 0; i < 100; ++i) { + auto x = pool.get(); + ++*x; + } + }; + std::thread t1{push}; + std::thread t2{push}; + t1.join(); + t2.join(); + + auto x = pool.get(); + auto y = pool.get(); + EXPECT_EQ(200, *x + *y); + } + EXPECT_GE(2, numCreated); + EXPECT_EQ(numCreated, numDeleted); +} From baa152e56e5f56e8991615998b3550b0a736d5cb Mon Sep 17 00:00:00 2001 From: Nick Terrell Date: Wed, 12 Oct 2016 19:02:27 -0700 Subject: [PATCH 35/92] [pzstd] Add Logger class --- contrib/pzstd/Logging.h | 72 +++++++++++++++++++++++++++++++++++++++++ contrib/pzstd/Pzstd.cpp | 66 +++++++++++++------------------------ contrib/pzstd/Pzstd.h | 11 ++++--- 3 files changed, 100 insertions(+), 49 deletions(-) create mode 100644 contrib/pzstd/Logging.h diff --git a/contrib/pzstd/Logging.h b/contrib/pzstd/Logging.h new file mode 100644 index 000000000..76c982ab2 --- /dev/null +++ b/contrib/pzstd/Logging.h @@ -0,0 +1,72 @@ +/** + * Copyright (c) 2016-present, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ +#pragma once + +#include +#include + +namespace pzstd { + +constexpr int ERROR = 1; +constexpr int INFO = 2; +constexpr int DEBUG = 3; +constexpr int VERBOSE = 4; + +class Logger { + std::mutex mutex_; + FILE* out_; + const int level_; + + using Clock = std::chrono::system_clock; + Clock::time_point lastUpdate_; + std::chrono::milliseconds refreshRate_; + + public: + explicit Logger(int level, FILE* out = stderr) + : out_(out), level_(level), lastUpdate_(Clock::now()), + refreshRate_(150) {} + + + bool logsAt(int level) { + return level <= level_; + } + + template + void operator()(int level, const char *fmt, Args... args) { + if (level > level_) { + return; + } + std::lock_guard lock(mutex_); + std::fprintf(out_, fmt, args...); + } + + template + void update(int level, const char *fmt, Args... args) { + if (level > level_) { + return; + } + std::lock_guard lock(mutex_); + auto now = Clock::now(); + if (now - lastUpdate_ > refreshRate_) { + lastUpdate_ = now; + std::fprintf(out_, "\r"); + std::fprintf(out_, fmt, args...); + } + } + + void clear(int level) { + if (level > level_) { + return; + } + std::lock_guard lock(mutex_); + std::fprintf(out_, "\r%79s\r", ""); + } +}; + +} diff --git a/contrib/pzstd/Pzstd.cpp b/contrib/pzstd/Pzstd.cpp index 6b3d27b3e..c5b4ce4cb 100644 --- a/contrib/pzstd/Pzstd.cpp +++ b/contrib/pzstd/Pzstd.cpp @@ -85,30 +85,28 @@ static std::uint64_t handleOneInput(const Options &options, options.determineParameters()); }); // Start writing - bytesWritten = writeFile(state, outs, outputFd, options.decompress, - options.verbosity); + bytesWritten = writeFile(state, outs, outputFd, options.decompress); } else { // Add a job that reads the input and starts all the decompression jobs readExecutor.add([&state, &outs, &executor, inputFd, &bytesRead] { bytesRead = asyncDecompressFrames(state, outs, executor, inputFd); }); // Start writing - bytesWritten = writeFile(state, outs, outputFd, options.decompress, - options.verbosity); + bytesWritten = writeFile(state, outs, outputFd, options.decompress); } } - if (options.verbosity > 1 && !state.errorHolder.hasError()) { + if (!state.errorHolder.hasError()) { std::string inputFileName = inputFile == "-" ? "stdin" : inputFile; std::string outputFileName = outputFile == "-" ? "stdout" : outputFile; if (!options.decompress) { double ratio = static_cast(bytesWritten) / static_cast(bytesRead + !bytesRead); - std::fprintf(stderr, "%-20s :%6.2f%% (%6" PRIu64 " => %6" PRIu64 + state.log(INFO, "%-20s :%6.2f%% (%6" PRIu64 " => %6" PRIu64 " bytes, %s)\n", inputFileName.c_str(), ratio * 100, bytesRead, bytesWritten, outputFileName.c_str()); } else { - std::fprintf(stderr, "%-20s: %" PRIu64 " bytes \n", + state.log(INFO, "%-20s: %" PRIu64 " bytes \n", inputFileName.c_str(),bytesWritten); } } @@ -138,7 +136,7 @@ static FILE *openInputFile(const std::string &inputFile, static FILE *openOutputFile(const Options &options, const std::string &outputFile, - ErrorHolder &errorHolder) { + SharedState& state) { if (outputFile == "-") { SET_BINARY_MODE(stdout); return stdout; @@ -148,41 +146,39 @@ static FILE *openOutputFile(const Options &options, auto outputFd = std::fopen(outputFile.c_str(), "rb"); if (outputFd != nullptr) { std::fclose(outputFd); - if (options.verbosity <= 1) { - errorHolder.setError("Output file exists"); + if (!state.log.logsAt(INFO)) { + state.errorHolder.setError("Output file exists"); return nullptr; } - std::fprintf( - stderr, + state.log( + INFO, "pzstd: %s already exists; do you wish to overwrite (y/n) ? ", outputFile.c_str()); int c = getchar(); if (c != 'y' && c != 'Y') { - errorHolder.setError("Not overwritten"); + state.errorHolder.setError("Not overwritten"); return nullptr; } } } auto outputFd = std::fopen(outputFile.c_str(), "wb"); - if (!errorHolder.check( + if (!state.errorHolder.check( outputFd != nullptr, "Failed to open output file")) { - return 0; + return nullptr; } return outputFd; } int pzstdMain(const Options &options) { int returnCode = 0; - SharedState state(options.decompress, options.determineParameters()); + SharedState state(options); for (const auto& input : options.inputFiles) { // Setup the shared state auto printErrorGuard = makeScopeGuard([&] { if (state.errorHolder.hasError()) { returnCode = 1; - if (options.verbosity > 0) { - std::fprintf(stderr, "pzstd: %s: %s.\n", input.c_str(), - state.errorHolder.getError().c_str()); - } + state.log(ERROR, "pzstd: %s: %s.\n", input.c_str(), + state.errorHolder.getError().c_str()); } }); // Open the input file @@ -197,7 +193,7 @@ int pzstdMain(const Options &options) { "Input file does not have extension .zst")) { continue; } - auto outputFd = openOutputFile(options, outputFile, state.errorHolder); + auto outputFd = openOutputFile(options, outputFile, state); if (outputFd == nullptr) { continue; } @@ -578,33 +574,14 @@ static bool writeData(ByteRange data, FILE* fd) { return true; } -void updateWritten(int verbosity, std::uint64_t bytesWritten) { - if (verbosity <= 1) { - return; - } - using Clock = std::chrono::system_clock; - static Clock::time_point then; - constexpr std::chrono::milliseconds refreshRate{150}; - - auto now = Clock::now(); - if (now - then > refreshRate) { - then = now; - std::fprintf(stderr, "\rWritten: %u MB ", - static_cast(bytesWritten >> 20)); - } -} - std::uint64_t writeFile( SharedState& state, WorkQueue>& outs, FILE* outputFd, - bool decompress, - int verbosity) { + bool decompress) { auto& errorHolder = state.errorHolder; - auto lineClearGuard = makeScopeGuard([verbosity] { - if (verbosity > 1) { - std::fprintf(stderr, "\r%79s\r", ""); - } + auto lineClearGuard = makeScopeGuard([&state] { + state.log.clear(INFO); }); std::uint64_t bytesWritten = 0; std::shared_ptr out; @@ -630,7 +607,8 @@ std::uint64_t writeFile( return bytesWritten; } bytesWritten += buffer.size(); - updateWritten(verbosity, bytesWritten); + state.log.update(INFO, "Written: %u MB ", + static_cast(bytesWritten >> 20)); } } return bytesWritten; diff --git a/contrib/pzstd/Pzstd.h b/contrib/pzstd/Pzstd.h index b02fe7b19..9fb2c4884 100644 --- a/contrib/pzstd/Pzstd.h +++ b/contrib/pzstd/Pzstd.h @@ -9,6 +9,7 @@ #pragma once #include "ErrorHolder.h" +#include "Logging.h" #include "Options.h" #include "utils/Buffer.h" #include "utils/Range.h" @@ -35,8 +36,9 @@ int pzstdMain(const Options& options); class SharedState { public: - SharedState(bool decompress, ZSTD_parameters parameters) { - if (!decompress) { + SharedState(const Options& options) : log(options.verbosity) { + if (!options.decompress) { + auto parameters = options.determineParameters(); cStreamPool.reset(new ResourcePool{ [parameters]() -> ZSTD_CStream* { auto zcs = ZSTD_createCStream(); @@ -72,6 +74,7 @@ class SharedState { } } + Logger log; ErrorHolder errorHolder; std::unique_ptr> cStreamPool; std::unique_ptr> dStreamPool; @@ -129,13 +132,11 @@ std::uint64_t asyncDecompressFrames( * (de)compression job. * @param outputFd The file descriptor to write to * @param decompress Are we decompressing? - * @param verbosity The verbosity level to log at * @returns The number of bytes written */ std::uint64_t writeFile( SharedState& state, WorkQueue>& outs, FILE* outputFd, - bool decompress, - int verbosity); + bool decompress); } From 8c6c686d0ae9c25f249b41ea83a8f7909ccc0a38 Mon Sep 17 00:00:00 2001 From: Nick Terrell Date: Thu, 13 Oct 2016 12:03:02 -0700 Subject: [PATCH 36/92] [pzstd] Fix lantent bug in WorkQueue::push() --- contrib/pzstd/utils/ThreadPool.h | 2 +- contrib/pzstd/utils/WorkQueue.h | 3 ++- contrib/pzstd/utils/test/WorkQueueTest.cpp | 23 +++++++++++++++++----- 3 files changed, 21 insertions(+), 7 deletions(-) diff --git a/contrib/pzstd/utils/ThreadPool.h b/contrib/pzstd/utils/ThreadPool.h index a1d1fc0b9..99b3ecfa5 100644 --- a/contrib/pzstd/utils/ThreadPool.h +++ b/contrib/pzstd/utils/ThreadPool.h @@ -27,7 +27,7 @@ class ThreadPool { explicit ThreadPool(std::size_t numThreads) { threads_.reserve(numThreads); for (std::size_t i = 0; i < numThreads; ++i) { - threads_.emplace_back([&] { + threads_.emplace_back([this] { std::function task; while (tasks_.pop(task)) { task(); diff --git a/contrib/pzstd/utils/WorkQueue.h b/contrib/pzstd/utils/WorkQueue.h index c46e6cbcf..780e5360f 100644 --- a/contrib/pzstd/utils/WorkQueue.h +++ b/contrib/pzstd/utils/WorkQueue.h @@ -54,12 +54,13 @@ class WorkQueue { /** * Push an item onto the work queue. Notify a single thread that work is * available. If `finish()` has been called, do nothing and return false. + * If `push()` returns false, then `item` has not been moved from. * * @param item Item to push onto the queue. * @returns True upon success, false if `finish()` has been called. An * item was pushed iff `push()` returns true. */ - bool push(T item) { + bool push(T&& item) { { std::unique_lock lock(mutex_); while (full() && !done_) { diff --git a/contrib/pzstd/utils/test/WorkQueueTest.cpp b/contrib/pzstd/utils/test/WorkQueueTest.cpp index ebf375a84..7f58ccb3f 100644 --- a/contrib/pzstd/utils/test/WorkQueueTest.cpp +++ b/contrib/pzstd/utils/test/WorkQueueTest.cpp @@ -10,6 +10,7 @@ #include "utils/WorkQueue.h" #include +#include #include #include #include @@ -64,7 +65,7 @@ TEST(WorkQueue, SPSC) { const int max = 100; for (int i = 0; i < 10; ++i) { - queue.push(i); + queue.push(int{i}); } std::thread thread([ &queue, max ] { @@ -80,7 +81,7 @@ TEST(WorkQueue, SPSC) { std::this_thread::yield(); for (int i = 10; i < max; ++i) { - queue.push(i); + queue.push(int{i}); } queue.finish(); @@ -97,7 +98,7 @@ TEST(WorkQueue, SPMC) { } for (int i = 0; i < 50; ++i) { - queue.push(i); + queue.push(int{i}); } queue.finish(); @@ -126,7 +127,7 @@ TEST(WorkQueue, MPMC) { pusherThreads.emplace_back( [ &queue, min, max ] { for (int i = min; i < max; ++i) { - queue.push(i); + queue.push(int{i}); } }); } @@ -212,7 +213,7 @@ TEST(WorkQueue, BoundedSizeMPMC) { pusherThreads.emplace_back( [ &queue, min, max ] { for (int i = min; i < max; ++i) { - queue.push(i); + queue.push(int{i}); } }); } @@ -231,6 +232,18 @@ TEST(WorkQueue, BoundedSizeMPMC) { } } +TEST(WorkQueue, FailedPush) { + WorkQueue> queue; + std::unique_ptr x(new int{5}); + EXPECT_TRUE(queue.push(std::move(x))); + EXPECT_EQ(nullptr, x); + queue.finish(); + x.reset(new int{6}); + EXPECT_FALSE(queue.push(std::move(x))); + EXPECT_NE(nullptr, x); + EXPECT_EQ(6, *x); +} + TEST(BufferWorkQueue, SizeCalculatedCorrectly) { { BufferWorkQueue queue; From 2b361cf2f1b5b461e93357eb4a1cafed6e2bcfeb Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 14 Oct 2016 16:03:34 -0700 Subject: [PATCH 37/92] minor opt --- NEWS | 2 +- lib/compress/zstd_compress.c | 2 +- lib/compress/zstd_opt.h | 6 +++--- lib/dictBuilder/zdict.c | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/NEWS b/NEWS index 1b65b508c..a9454b40b 100644 --- a/NEWS +++ b/NEWS @@ -1,5 +1,5 @@ v1.1.1 -New : command -M, to limit allowed memory consumption +New : command -M#, --memory=, --memlimit=, --memlimit-decompress= to limit allowed memory consumption v1.1.0 New : contrib/pzstd, parallel version of zstd, by Nick Terrell diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index e53ecc5fd..e71873b03 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -1458,7 +1458,7 @@ static U32 ZSTD_insertBt1(ZSTD_CCtx* zc, const BYTE* const ip, const U32 mls, co const U32 dictLimit = zc->dictLimit; const BYTE* const dictEnd = dictBase + dictLimit; const BYTE* const prefixStart = base + dictLimit; - const BYTE* match = base + matchIndex; + const BYTE* match; const U32 current = (U32)(ip-base); const U32 btLow = btMask >= current ? 0 : current - btMask; U32* smallerPtr = bt + 2*(current&btMask); diff --git a/lib/compress/zstd_opt.h b/lib/compress/zstd_opt.h index cb5872908..cea67056a 100644 --- a/lib/compress/zstd_opt.h +++ b/lib/compress/zstd_opt.h @@ -401,7 +401,7 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx, ZSTD_rescaleFreqs(seqStorePtr); ip += (ip==prefixStart); { U32 i; for (i=0; irep[i]; } - inr = ip; + //inr = ip; /* Match Loop */ while (ip < ilimit) { @@ -657,7 +657,7 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx, ctx->nextToUpdate3 = ctx->nextToUpdate; ZSTD_rescaleFreqs(seqStorePtr); ip += (ip==prefixStart); - inr = ip; + //inr = ip; /* Match Loop */ while (ip < ilimit) { @@ -666,7 +666,7 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx, U32 current = (U32)(ip-base); memset(opt, 0, sizeof(ZSTD_optimal_t)); last_pos = 0; - inr = ip; + //inr = ip; opt[0].litlen = (U32)(ip - anchor); /* check repCode */ diff --git a/lib/dictBuilder/zdict.c b/lib/dictBuilder/zdict.c index 47a82af14..a88e70520 100644 --- a/lib/dictBuilder/zdict.c +++ b/lib/dictBuilder/zdict.c @@ -811,7 +811,7 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize, MEM_writeLE32(dstPtr+4, repStartValue[1]); MEM_writeLE32(dstPtr+8, repStartValue[2]); #endif - dstPtr += 12; + //dstPtr += 12; eSize += 12; _cleanup: From ccfcc643da9e9f1dbf957731d2d4fc1f3150a98f Mon Sep 17 00:00:00 2001 From: Nick Terrell Date: Mon, 17 Oct 2016 11:28:02 -0700 Subject: [PATCH 38/92] Check if dict is empty before reading first byte --- lib/common/entropy_common.c | 4 +++- lib/legacy/zstd_v01.c | 5 ++++- lib/legacy/zstd_v02.c | 4 +++- lib/legacy/zstd_v03.c | 4 +++- lib/legacy/zstd_v04.c | 4 +++- lib/legacy/zstd_v05.c | 4 +++- lib/legacy/zstd_v06.c | 4 +++- lib/legacy/zstd_v07.c | 4 +++- 8 files changed, 25 insertions(+), 8 deletions(-) diff --git a/lib/common/entropy_common.c b/lib/common/entropy_common.c index acd966999..6625a8057 100644 --- a/lib/common/entropy_common.c +++ b/lib/common/entropy_common.c @@ -168,9 +168,11 @@ size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats, { U32 weightTotal; const BYTE* ip = (const BYTE*) src; - size_t iSize = ip[0]; + size_t iSize; size_t oSize; + if (!srcSize) return ERROR(srcSize_wrong); + iSize = ip[0]; /* memset(huffWeight, 0, hwSize); *//* is not necessary, even though some analyzer complain ... */ if (iSize >= 128) { /* special header */ diff --git a/lib/legacy/zstd_v01.c b/lib/legacy/zstd_v01.c index fe9c5ccdd..c115fd822 100644 --- a/lib/legacy/zstd_v01.c +++ b/lib/legacy/zstd_v01.c @@ -958,13 +958,16 @@ static size_t HUF_readDTable (U16* DTable, const void* src, size_t srcSize) U32 weightTotal; U32 maxBits; const BYTE* ip = (const BYTE*) src; - size_t iSize = ip[0]; + size_t iSize; size_t oSize; U32 n; U32 nextRankStart; void* ptr = DTable+1; HUF_DElt* const dt = (HUF_DElt*)ptr; + if (!srcSize) return (size_t)-FSE_ERROR_srcSize_wrong; + iSize = ip[0]; + FSE_STATIC_ASSERT(sizeof(HUF_DElt) == sizeof(U16)); /* if compilation fails here, assertion is false */ //memset(huffWeight, 0, sizeof(huffWeight)); /* should not be necessary, but some analyzer complain ... */ if (iSize >= 128) /* special header */ diff --git a/lib/legacy/zstd_v02.c b/lib/legacy/zstd_v02.c index de1592e18..cd1ce647e 100644 --- a/lib/legacy/zstd_v02.c +++ b/lib/legacy/zstd_v02.c @@ -1607,10 +1607,12 @@ static size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats, U32 weightTotal; U32 tableLog; const BYTE* ip = (const BYTE*) src; - size_t iSize = ip[0]; + size_t iSize; size_t oSize; U32 n; + if (!srcSize) return ERROR(srcSize_wrong); + iSize = ip[0]; //memset(huffWeight, 0, hwSize); /* is not necessary, even though some analyzer complain ... */ if (iSize >= 128) /* special header */ diff --git a/lib/legacy/zstd_v03.c b/lib/legacy/zstd_v03.c index caad331d3..2de41f2bd 100644 --- a/lib/legacy/zstd_v03.c +++ b/lib/legacy/zstd_v03.c @@ -1604,10 +1604,12 @@ static size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats, U32 weightTotal; U32 tableLog; const BYTE* ip = (const BYTE*) src; - size_t iSize = ip[0]; + size_t iSize; size_t oSize; U32 n; + if (!srcSize) return ERROR(srcSize_wrong); + iSize = ip[0]; //memset(huffWeight, 0, hwSize); /* is not necessary, even though some analyzer complain ... */ if (iSize >= 128) /* special header */ diff --git a/lib/legacy/zstd_v04.c b/lib/legacy/zstd_v04.c index 05e40aac5..599778b87 100644 --- a/lib/legacy/zstd_v04.c +++ b/lib/legacy/zstd_v04.c @@ -1896,10 +1896,12 @@ static size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats, U32 weightTotal; U32 tableLog; const BYTE* ip = (const BYTE*) src; - size_t iSize = ip[0]; + size_t iSize; size_t oSize; U32 n; + if (!srcSize) return ERROR(srcSize_wrong); + iSize = ip[0]; //memset(huffWeight, 0, hwSize); /* is not necessary, even though some analyzer complain ... */ if (iSize >= 128) /* special header */ diff --git a/lib/legacy/zstd_v05.c b/lib/legacy/zstd_v05.c index 96ffceb97..dcaa2ac02 100644 --- a/lib/legacy/zstd_v05.c +++ b/lib/legacy/zstd_v05.c @@ -1873,10 +1873,12 @@ static size_t HUFv05_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats, U32 weightTotal; U32 tableLog; const BYTE* ip = (const BYTE*) src; - size_t iSize = ip[0]; + size_t iSize; size_t oSize; U32 n; + if (!srcSize) return ERROR(srcSize_wrong); + iSize = ip[0]; //memset(huffWeight, 0, hwSize); /* is not necessary, even though some analyzer complain ... */ if (iSize >= 128) { /* special header */ diff --git a/lib/legacy/zstd_v06.c b/lib/legacy/zstd_v06.c index 96a84d3e8..cce629343 100644 --- a/lib/legacy/zstd_v06.c +++ b/lib/legacy/zstd_v06.c @@ -1932,9 +1932,11 @@ MEM_STATIC size_t HUFv06_readStats(BYTE* huffWeight, size_t hwSize, U32* rankSta { U32 weightTotal; const BYTE* ip = (const BYTE*) src; - size_t iSize = ip[0]; + size_t iSize; size_t oSize; + if (!srcSize) return ERROR(srcSize_wrong); + iSize = ip[0]; //memset(huffWeight, 0, hwSize); /* is not necessary, even though some analyzer complain ... */ if (iSize >= 128) { /* special header */ diff --git a/lib/legacy/zstd_v07.c b/lib/legacy/zstd_v07.c index 62285238a..a1f1911a4 100644 --- a/lib/legacy/zstd_v07.c +++ b/lib/legacy/zstd_v07.c @@ -1382,9 +1382,11 @@ size_t HUFv07_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats, { U32 weightTotal; const BYTE* ip = (const BYTE*) src; - size_t iSize = ip[0]; + size_t iSize; size_t oSize; + if (!srcSize) return ERROR(srcSize_wrong); + iSize = ip[0]; //memset(huffWeight, 0, hwSize); /* is not necessary, even though some analyzer complain ... */ if (iSize >= 128) { /* special header */ From 4db751668fbf72cd23205fb1ecd582149a0db210 Mon Sep 17 00:00:00 2001 From: Nick Terrell Date: Mon, 17 Oct 2016 15:49:50 -0700 Subject: [PATCH 39/92] Fix buffer overrun in ZSTD_loadEntropy() The table log set by `FSE_readNCount()` was not checked in `ZSTD_loadEntropy()`. This caused `FSE_buildDTable(dctx->MLTable, ...)` to overwrite the beginning of `dctx->hufTable`. The benchmarks look good, there is no obvious performance regression: > ./zstds/zstd.opt.0 -i10 -b1 -e5 ~/bench/silesia.tar 1#silesia.tar : 211988480 -> 73656930 (2.878), 268.2 MB/s , 701.0 MB/s 2#silesia.tar : 211988480 -> 70162842 (3.021), 199.5 MB/s , 666.9 MB/s 3#silesia.tar : 211988480 -> 66997986 (3.164), 154.9 MB/s , 655.6 MB/s 4#silesia.tar : 211988480 -> 66002591 (3.212), 128.9 MB/s , 648.4 MB/s 5#silesia.tar : 211988480 -> 65008480 (3.261), 98.4 MB/s , 633.4 MB/s > ./zstds/zstd.opt.2 -i10 -b1 -e5 ~/bench/silesia.tar 1#silesia.tar : 211988480 -> 73656930 (2.878), 266.1 MB/s , 703.7 MB/s 2#silesia.tar : 211988480 -> 70162842 (3.021), 199.0 MB/s , 666.6 MB/s 3#silesia.tar : 211988480 -> 66997986 (3.164), 156.2 MB/s , 656.2 MB/s 4#silesia.tar : 211988480 -> 66002591 (3.212), 133.2 MB/s , 647.4 MB/s 5#silesia.tar : 211988480 -> 65008480 (3.261), 96.3 MB/s , 633.3 MB/s --- lib/decompress/zstd_decompress.c | 9 ++++++--- lib/legacy/zstd_v05.c | 9 ++++++--- lib/legacy/zstd_v06.c | 9 ++++++--- lib/legacy/zstd_v07.c | 9 ++++++--- 4 files changed, 24 insertions(+), 12 deletions(-) diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index 990a64910..83d01f3ad 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -1315,25 +1315,28 @@ static size_t ZSTD_loadEntropy(ZSTD_DCtx* dctx, const void* const dict, size_t c } { short offcodeNCount[MaxOff+1]; - U32 offcodeMaxValue=MaxOff, offcodeLog=OffFSELog; + U32 offcodeMaxValue=MaxOff, offcodeLog; size_t const offcodeHeaderSize = FSE_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dictPtr, dictEnd-dictPtr); if (FSE_isError(offcodeHeaderSize)) return ERROR(dictionary_corrupted); + if (offcodeLog > OffFSELog) return ERROR(dictionary_corrupted); CHECK_E(FSE_buildDTable(dctx->OFTable, offcodeNCount, offcodeMaxValue, offcodeLog), dictionary_corrupted); dictPtr += offcodeHeaderSize; } { short matchlengthNCount[MaxML+1]; - unsigned matchlengthMaxValue = MaxML, matchlengthLog = MLFSELog; + unsigned matchlengthMaxValue = MaxML, matchlengthLog; size_t const matchlengthHeaderSize = FSE_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dictPtr, dictEnd-dictPtr); if (FSE_isError(matchlengthHeaderSize)) return ERROR(dictionary_corrupted); + if (matchlengthLog > MLFSELog) return ERROR(dictionary_corrupted); CHECK_E(FSE_buildDTable(dctx->MLTable, matchlengthNCount, matchlengthMaxValue, matchlengthLog), dictionary_corrupted); dictPtr += matchlengthHeaderSize; } { short litlengthNCount[MaxLL+1]; - unsigned litlengthMaxValue = MaxLL, litlengthLog = LLFSELog; + unsigned litlengthMaxValue = MaxLL, litlengthLog; size_t const litlengthHeaderSize = FSE_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dictPtr, dictEnd-dictPtr); if (FSE_isError(litlengthHeaderSize)) return ERROR(dictionary_corrupted); + if (litlengthLog > LLFSELog) return ERROR(dictionary_corrupted); CHECK_E(FSE_buildDTable(dctx->LLTable, litlengthNCount, litlengthMaxValue, litlengthLog), dictionary_corrupted); dictPtr += litlengthHeaderSize; } diff --git a/lib/legacy/zstd_v05.c b/lib/legacy/zstd_v05.c index dcaa2ac02..7446b8457 100644 --- a/lib/legacy/zstd_v05.c +++ b/lib/legacy/zstd_v05.c @@ -3667,11 +3667,11 @@ static size_t ZSTDv05_loadEntropy(ZSTDv05_DCtx* dctx, const void* dict, size_t d { size_t hSize, offcodeHeaderSize, matchlengthHeaderSize, errorCode, litlengthHeaderSize; short offcodeNCount[MaxOff+1]; - U32 offcodeMaxValue=MaxOff, offcodeLog=OffFSEv05Log; + U32 offcodeMaxValue=MaxOff, offcodeLog; short matchlengthNCount[MaxML+1]; - unsigned matchlengthMaxValue = MaxML, matchlengthLog = MLFSEv05Log; + unsigned matchlengthMaxValue = MaxML, matchlengthLog; short litlengthNCount[MaxLL+1]; - unsigned litlengthMaxValue = MaxLL, litlengthLog = LLFSEv05Log; + unsigned litlengthMaxValue = MaxLL, litlengthLog; hSize = HUFv05_readDTableX4(dctx->hufTableX4, dict, dictSize); if (HUFv05_isError(hSize)) return ERROR(dictionary_corrupted); @@ -3680,6 +3680,7 @@ static size_t ZSTDv05_loadEntropy(ZSTDv05_DCtx* dctx, const void* dict, size_t d offcodeHeaderSize = FSEv05_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dict, dictSize); if (FSEv05_isError(offcodeHeaderSize)) return ERROR(dictionary_corrupted); + if (offcodeLog > OffFSEv05Log) return ERROR(dictionary_corrupted); errorCode = FSEv05_buildDTable(dctx->OffTable, offcodeNCount, offcodeMaxValue, offcodeLog); if (FSEv05_isError(errorCode)) return ERROR(dictionary_corrupted); dict = (const char*)dict + offcodeHeaderSize; @@ -3687,12 +3688,14 @@ static size_t ZSTDv05_loadEntropy(ZSTDv05_DCtx* dctx, const void* dict, size_t d matchlengthHeaderSize = FSEv05_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dict, dictSize); if (FSEv05_isError(matchlengthHeaderSize)) return ERROR(dictionary_corrupted); + if (matchlengthLog > MLFSEv05Log) return ERROR(dictionary_corrupted); errorCode = FSEv05_buildDTable(dctx->MLTable, matchlengthNCount, matchlengthMaxValue, matchlengthLog); if (FSEv05_isError(errorCode)) return ERROR(dictionary_corrupted); dict = (const char*)dict + matchlengthHeaderSize; dictSize -= matchlengthHeaderSize; litlengthHeaderSize = FSEv05_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dict, dictSize); + if (litlengthLog > LLFSEv05Log) return ERROR(dictionary_corrupted); if (FSEv05_isError(litlengthHeaderSize)) return ERROR(dictionary_corrupted); errorCode = FSEv05_buildDTable(dctx->LLTable, litlengthNCount, litlengthMaxValue, litlengthLog); if (FSEv05_isError(errorCode)) return ERROR(dictionary_corrupted); diff --git a/lib/legacy/zstd_v06.c b/lib/legacy/zstd_v06.c index cce629343..403a3a0b3 100644 --- a/lib/legacy/zstd_v06.c +++ b/lib/legacy/zstd_v06.c @@ -3829,9 +3829,10 @@ static size_t ZSTDv06_loadEntropy(ZSTDv06_DCtx* dctx, const void* dict, size_t d dictSize -= hSize; { short offcodeNCount[MaxOff+1]; - U32 offcodeMaxValue=MaxOff, offcodeLog=OffFSELog; + U32 offcodeMaxValue=MaxOff, offcodeLog; offcodeHeaderSize = FSEv06_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dict, dictSize); if (FSEv06_isError(offcodeHeaderSize)) return ERROR(dictionary_corrupted); + if (offcodeLog > OffFSELog) return ERROR(dictionary_corrupted); { size_t const errorCode = FSEv06_buildDTable(dctx->OffTable, offcodeNCount, offcodeMaxValue, offcodeLog); if (FSEv06_isError(errorCode)) return ERROR(dictionary_corrupted); } dict = (const char*)dict + offcodeHeaderSize; @@ -3839,9 +3840,10 @@ static size_t ZSTDv06_loadEntropy(ZSTDv06_DCtx* dctx, const void* dict, size_t d } { short matchlengthNCount[MaxML+1]; - unsigned matchlengthMaxValue = MaxML, matchlengthLog = MLFSELog; + unsigned matchlengthMaxValue = MaxML, matchlengthLog; matchlengthHeaderSize = FSEv06_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dict, dictSize); if (FSEv06_isError(matchlengthHeaderSize)) return ERROR(dictionary_corrupted); + if (matchlengthLog > MLFSELog) return ERROR(dictionary_corrupted); { size_t const errorCode = FSEv06_buildDTable(dctx->MLTable, matchlengthNCount, matchlengthMaxValue, matchlengthLog); if (FSEv06_isError(errorCode)) return ERROR(dictionary_corrupted); } dict = (const char*)dict + matchlengthHeaderSize; @@ -3849,9 +3851,10 @@ static size_t ZSTDv06_loadEntropy(ZSTDv06_DCtx* dctx, const void* dict, size_t d } { short litlengthNCount[MaxLL+1]; - unsigned litlengthMaxValue = MaxLL, litlengthLog = LLFSELog; + unsigned litlengthMaxValue = MaxLL, litlengthLog; litlengthHeaderSize = FSEv06_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dict, dictSize); if (FSEv06_isError(litlengthHeaderSize)) return ERROR(dictionary_corrupted); + if (litlengthLog > LLFSELog) return ERROR(dictionary_corrupted); { size_t const errorCode = FSEv06_buildDTable(dctx->LLTable, litlengthNCount, litlengthMaxValue, litlengthLog); if (FSEv06_isError(errorCode)) return ERROR(dictionary_corrupted); } } diff --git a/lib/legacy/zstd_v07.c b/lib/legacy/zstd_v07.c index a1f1911a4..95572b93b 100644 --- a/lib/legacy/zstd_v07.c +++ b/lib/legacy/zstd_v07.c @@ -4104,27 +4104,30 @@ static size_t ZSTDv07_loadEntropy(ZSTDv07_DCtx* dctx, const void* const dict, si } { short offcodeNCount[MaxOff+1]; - U32 offcodeMaxValue=MaxOff, offcodeLog=OffFSELog; + U32 offcodeMaxValue=MaxOff, offcodeLog; size_t const offcodeHeaderSize = FSEv07_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dictPtr, dictEnd-dictPtr); if (FSEv07_isError(offcodeHeaderSize)) return ERROR(dictionary_corrupted); + if (offcodeLog > OffFSELog) return ERROR(dictionary_corrupted); { size_t const errorCode = FSEv07_buildDTable(dctx->OffTable, offcodeNCount, offcodeMaxValue, offcodeLog); if (FSEv07_isError(errorCode)) return ERROR(dictionary_corrupted); } dictPtr += offcodeHeaderSize; } { short matchlengthNCount[MaxML+1]; - unsigned matchlengthMaxValue = MaxML, matchlengthLog = MLFSELog; + unsigned matchlengthMaxValue = MaxML, matchlengthLog; size_t const matchlengthHeaderSize = FSEv07_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dictPtr, dictEnd-dictPtr); if (FSEv07_isError(matchlengthHeaderSize)) return ERROR(dictionary_corrupted); + if (matchlengthLog > MLFSELog) return ERROR(dictionary_corrupted); { size_t const errorCode = FSEv07_buildDTable(dctx->MLTable, matchlengthNCount, matchlengthMaxValue, matchlengthLog); if (FSEv07_isError(errorCode)) return ERROR(dictionary_corrupted); } dictPtr += matchlengthHeaderSize; } { short litlengthNCount[MaxLL+1]; - unsigned litlengthMaxValue = MaxLL, litlengthLog = LLFSELog; + unsigned litlengthMaxValue = MaxLL, litlengthLog; size_t const litlengthHeaderSize = FSEv07_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dictPtr, dictEnd-dictPtr); if (FSEv07_isError(litlengthHeaderSize)) return ERROR(dictionary_corrupted); + if (litlengthLog > LLFSELog) return ERROR(dictionary_corrupted); { size_t const errorCode = FSEv07_buildDTable(dctx->LLTable, litlengthNCount, litlengthMaxValue, litlengthLog); if (FSEv07_isError(errorCode)) return ERROR(dictionary_corrupted); } dictPtr += litlengthHeaderSize; From bfd943ace5ff23d12501a744d719cb0e306513b4 Mon Sep 17 00:00:00 2001 From: Nick Terrell Date: Mon, 17 Oct 2016 16:55:52 -0700 Subject: [PATCH 40/92] Fix buffer overrun in ZSTD_loadDictEntropyStats() The table log set by `FSE_readNCount()` was not checked in `ZSTD_loadDictEntropyStats()`. This caused `FSE_buildCTable()` to stack/heap overflow in a few places. The benchmarks look good, there is no obvious compression performance regression: > ./zstds/zstd.opt.0 -i10 -b1 -e10 ~/bench/silesia.tar 1#silesia.tar : 211988480 -> 73656930 (2.878), 271.6 MB/s , 716.8 MB/s 2#silesia.tar : 211988480 -> 70162842 (3.021), 204.8 MB/s , 671.1 MB/s 3#silesia.tar : 211988480 -> 66997986 (3.164), 156.8 MB/s , 658.6 MB/s 4#silesia.tar : 211988480 -> 66002591 (3.212), 136.4 MB/s , 665.3 MB/s 5#silesia.tar : 211988480 -> 65008480 (3.261), 98.9 MB/s , 647.0 MB/s 6#silesia.tar : 211988480 -> 62979643 (3.366), 65.2 MB/s , 670.4 MB/s 7#silesia.tar : 211988480 -> 61974560 (3.421), 44.9 MB/s , 688.2 MB/s 8#silesia.tar : 211988480 -> 61028308 (3.474), 32.4 MB/s , 711.9 MB/s 9#silesia.tar : 211988480 -> 60416751 (3.509), 21.1 MB/s , 718.1 MB/s 10#silesia.tar : 211988480 -> 60174239 (3.523), 22.2 MB/s , 721.8 MB/s > ./compress_zstds/zstd.opt.1 -i10 -b1 -e10 ~/bench/silesia.tar 1#silesia.tar : 211988480 -> 73656930 (2.878), 273.8 MB/s , 722.0 MB/s 2#silesia.tar : 211988480 -> 70162842 (3.021), 203.2 MB/s , 666.6 MB/s 3#silesia.tar : 211988480 -> 66997986 (3.164), 157.4 MB/s , 666.5 MB/s 4#silesia.tar : 211988480 -> 66002591 (3.212), 132.1 MB/s , 661.9 MB/s 5#silesia.tar : 211988480 -> 65008480 (3.261), 96.8 MB/s , 641.6 MB/s 6#silesia.tar : 211988480 -> 62979643 (3.366), 63.1 MB/s , 677.0 MB/s 7#silesia.tar : 211988480 -> 61974560 (3.421), 44.3 MB/s , 678.2 MB/s 8#silesia.tar : 211988480 -> 61028308 (3.474), 33.1 MB/s , 708.9 MB/s 9#silesia.tar : 211988480 -> 60416751 (3.509), 21.5 MB/s , 710.1 MB/s 10#silesia.tar : 211988480 -> 60174239 (3.523), 21.9 MB/s , 723.9 MB/s --- lib/compress/zstd_compress.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index e71873b03..c44ef3d41 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -2471,25 +2471,28 @@ static size_t ZSTD_loadDictEntropyStats(ZSTD_CCtx* cctx, const void* dict, size_ } { short offcodeNCount[MaxOff+1]; - unsigned offcodeMaxValue = MaxOff, offcodeLog = OffFSELog; + unsigned offcodeMaxValue = MaxOff, offcodeLog; size_t const offcodeHeaderSize = FSE_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dictPtr, dictEnd-dictPtr); if (FSE_isError(offcodeHeaderSize)) return ERROR(dictionary_corrupted); + if (offcodeLog > OffFSELog) return ERROR(dictionary_corrupted); CHECK_E (FSE_buildCTable(cctx->offcodeCTable, offcodeNCount, offcodeMaxValue, offcodeLog), dictionary_corrupted); dictPtr += offcodeHeaderSize; } { short matchlengthNCount[MaxML+1]; - unsigned matchlengthMaxValue = MaxML, matchlengthLog = MLFSELog; + unsigned matchlengthMaxValue = MaxML, matchlengthLog; size_t const matchlengthHeaderSize = FSE_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dictPtr, dictEnd-dictPtr); if (FSE_isError(matchlengthHeaderSize)) return ERROR(dictionary_corrupted); + if (matchlengthLog > MLFSELog) return ERROR(dictionary_corrupted); CHECK_E (FSE_buildCTable(cctx->matchlengthCTable, matchlengthNCount, matchlengthMaxValue, matchlengthLog), dictionary_corrupted); dictPtr += matchlengthHeaderSize; } { short litlengthNCount[MaxLL+1]; - unsigned litlengthMaxValue = MaxLL, litlengthLog = LLFSELog; + unsigned litlengthMaxValue = MaxLL, litlengthLog; size_t const litlengthHeaderSize = FSE_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dictPtr, dictEnd-dictPtr); if (FSE_isError(litlengthHeaderSize)) return ERROR(dictionary_corrupted); + if (litlengthLog > LLFSELog) return ERROR(dictionary_corrupted); CHECK_E(FSE_buildCTable(cctx->litlengthCTable, litlengthNCount, litlengthMaxValue, litlengthLog), dictionary_corrupted); dictPtr += litlengthHeaderSize; } From 06573e17be39c6f308fc09a40efab58773902a3e Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Mon, 17 Oct 2016 17:28:28 -0700 Subject: [PATCH 41/92] fixed minor corner case --- lib/decompress/zstd_decompress.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index 990a64910..00b069e9f 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -807,7 +807,8 @@ static seq_t ZSTD_decodeSequence(seqState_t* seqState) if (ofCode <= 1) { offset += (llCode==0); if (offset) { - size_t const temp = (offset==3) ? seqState->prevOffset[0] - 1 : seqState->prevOffset[offset]; + size_t temp = (offset==3) ? seqState->prevOffset[0] - 1 : seqState->prevOffset[offset]; + temp += !temp; /* 0 is not valid; input is corrupted; force offset to 1 */ if (offset != 1) seqState->prevOffset[2] = seqState->prevOffset[1]; seqState->prevOffset[1] = seqState->prevOffset[0]; seqState->prevOffset[0] = offset = temp; From 33fdd099bbaef46d4a0c94783cca21649824b2fc Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Mon, 17 Oct 2016 17:48:48 -0700 Subject: [PATCH 42/92] fix command line interpretation --- programs/zstdcli.c | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 2df410431..9aa97cdd2 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -25,7 +25,6 @@ **************************************/ #include "util.h" /* Compiler options, UTIL_HAS_CREATEFILELIST */ #include /* strcmp, strlen */ -#include /* toupper */ #include /* errno */ #include "fileio.h" #ifndef ZSTD_NOBENCH @@ -182,10 +181,13 @@ static unsigned readU32FromChar(const char** stringPtr) unsigned result = 0; while ((**stringPtr >='0') && (**stringPtr <='9')) result *= 10, result += **stringPtr - '0', (*stringPtr)++ ; - if (toupper(**stringPtr)=='K') result <<= 10, (*stringPtr)++ ; - else if (toupper(**stringPtr)=='M') result <<= 20, (*stringPtr)++ ; - if (toupper(**stringPtr)=='i') (*stringPtr)++; - if (toupper(**stringPtr)=='B') (*stringPtr)++; + if ((**stringPtr=='K') || (**stringPtr=='M')) { + result <<= 10; + if (**stringPtr=='M') result <<= 10; + (*stringPtr)++ ; + if (**stringPtr=='i') (*stringPtr)++; + if (**stringPtr=='B') (*stringPtr)++; + } return result; } From fd9808704799f74a30246754ebf239ac1e8cafc9 Mon Sep 17 00:00:00 2001 From: Nick Terrell Date: Mon, 17 Oct 2016 18:16:57 -0700 Subject: [PATCH 43/92] Fix stack buffer overflow in HUF_readCTable() If `w ==0` on line 153, then `CTable[n].nbBits == tableLog + 1`. Then `nbPerRank[CTable[n].nbBits]` and `valPerRank[CTable[n].nbBits]` are stack buffer overflows. --- lib/compress/huf_compress.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/compress/huf_compress.c b/lib/compress/huf_compress.c index b7d3d77a2..41de7449a 100644 --- a/lib/compress/huf_compress.c +++ b/lib/compress/huf_compress.c @@ -155,8 +155,8 @@ size_t HUF_readCTable (HUF_CElt* CTable, U32 maxSymbolValue, const void* src, si } } /* fill val */ - { U16 nbPerRank[HUF_TABLELOG_MAX+1] = {0}; - U16 valPerRank[HUF_TABLELOG_MAX+1] = {0}; + { U16 nbPerRank[HUF_TABLELOG_MAX+2] = {0}; + U16 valPerRank[HUF_TABLELOG_MAX+2] = {0}; { U32 n; for (n=0; n Date: Tue, 18 Oct 2016 11:27:52 -0700 Subject: [PATCH 44/92] fix edge case --- lib/compress/huf_compress.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/lib/compress/huf_compress.c b/lib/compress/huf_compress.c index b7d3d77a2..78784aa36 100644 --- a/lib/compress/huf_compress.c +++ b/lib/compress/huf_compress.c @@ -155,13 +155,14 @@ size_t HUF_readCTable (HUF_CElt* CTable, U32 maxSymbolValue, const void* src, si } } /* fill val */ - { U16 nbPerRank[HUF_TABLELOG_MAX+1] = {0}; - U16 valPerRank[HUF_TABLELOG_MAX+1] = {0}; + { U16 nbPerRank[HUF_TABLELOG_MAX+2] = {0}; /* support w=0=>n=tableLog+1 */ + U16 valPerRank[HUF_TABLELOG_MAX+2] = {0}; { U32 n; for (n=0; n0; n--) { - valPerRank[n] = min; /* get starting value within each rank */ + U32 n; for (n=tableLog; n>0; n--) { /* start at n=tablelog <-> w=1 */ + valPerRank[n] = min; /* get starting value within each rank */ min += nbPerRank[n]; min >>= 1; } } From 83d7bdee4b4695e6036b9c7889572df0a96dbd56 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 18 Oct 2016 12:25:43 -0700 Subject: [PATCH 45/92] fixed invalid invocation of dictionary in legacy decoder v0.5 --- lib/legacy/zstd_v05.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/lib/legacy/zstd_v05.c b/lib/legacy/zstd_v05.c index 96ffceb97..7e0690174 100644 --- a/lib/legacy/zstd_v05.c +++ b/lib/legacy/zstd_v05.c @@ -3063,7 +3063,7 @@ size_t ZSTDv05_decodeLiteralsBlock(ZSTDv05_DCtx* dctx, size_t ZSTDv05_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* dumpsLengthPtr, FSEv05_DTable* DTableLL, FSEv05_DTable* DTableML, FSEv05_DTable* DTableOffb, - const void* src, size_t srcSize) + const void* src, size_t srcSize, U32 flagStaticTable) { const BYTE* const istart = (const BYTE* const)src; const BYTE* ip = istart; @@ -3118,6 +3118,7 @@ size_t ZSTDv05_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* dumps FSEv05_buildDTable_raw(DTableLL, LLbits); break; case FSEv05_ENCODING_STATIC: + if (!flagStaticTable) return ERROR(corruption_detected); break; case FSEv05_ENCODING_DYNAMIC : default : /* impossible */ @@ -3141,6 +3142,7 @@ size_t ZSTDv05_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* dumps FSEv05_buildDTable_raw(DTableOffb, Offbits); break; case FSEv05_ENCODING_STATIC: + if (!flagStaticTable) return ERROR(corruption_detected); break; case FSEv05_ENCODING_DYNAMIC : default : /* impossible */ @@ -3164,6 +3166,7 @@ size_t ZSTDv05_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* dumps FSEv05_buildDTable_raw(DTableML, MLbits); break; case FSEv05_ENCODING_STATIC: + if (!flagStaticTable) return ERROR(corruption_detected); break; case FSEv05_ENCODING_DYNAMIC : default : /* impossible */ @@ -3376,7 +3379,7 @@ static size_t ZSTDv05_decompressSequences( /* Build Decoding Tables */ errorCode = ZSTDv05_decodeSeqHeaders(&nbSeq, &dumps, &dumpsLength, DTableLL, DTableML, DTableOffb, - ip, seqSize); + ip, seqSize, dctx->flagStaticTables); if (ZSTDv05_isError(errorCode)) return errorCode; ip += errorCode; @@ -3385,6 +3388,7 @@ static size_t ZSTDv05_decompressSequences( seq_t sequence; seqState_t seqState; + dctx->flagStaticTables = 1; memset(&sequence, 0, sizeof(sequence)); sequence.offset = REPCODE_STARTVALUE; seqState.dumps = dumps; From 9313c8d95302a3c0c0fe3252b47aa7d3c4b6a045 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 18 Oct 2016 13:36:15 -0700 Subject: [PATCH 46/92] no repeat mode in legacy v0.5 --- lib/legacy/zstd_v05.c | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/legacy/zstd_v05.c b/lib/legacy/zstd_v05.c index 7e0690174..4c1c8c5e4 100644 --- a/lib/legacy/zstd_v05.c +++ b/lib/legacy/zstd_v05.c @@ -3388,7 +3388,6 @@ static size_t ZSTDv05_decompressSequences( seq_t sequence; seqState_t seqState; - dctx->flagStaticTables = 1; memset(&sequence, 0, sizeof(sequence)); sequence.offset = REPCODE_STARTVALUE; seqState.dumps = dumps; From f7906d595500c04951643eb98c8f26405b01973f Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 18 Oct 2016 13:48:32 -0700 Subject: [PATCH 47/92] detect early impossible decompression scenario in legacy decoder v0.5 --- NEWS | 1 + lib/legacy/zstd_v05.c | 7 ++++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/NEWS b/NEWS index a9454b40b..570d66799 100644 --- a/NEWS +++ b/NEWS @@ -1,5 +1,6 @@ v1.1.1 New : command -M#, --memory=, --memlimit=, --memlimit-decompress= to limit allowed memory consumption +Changed : zstd_errors.h is now part of include installation v1.1.0 New : contrib/pzstd, parallel version of zstd, by Nick Terrell diff --git a/lib/legacy/zstd_v05.c b/lib/legacy/zstd_v05.c index 4c1c8c5e4..3cb5bb4af 100644 --- a/lib/legacy/zstd_v05.c +++ b/lib/legacy/zstd_v05.c @@ -2032,13 +2032,14 @@ size_t HUFv05_decompress1X2_usingDTable( { BYTE* op = (BYTE*)dst; BYTE* const oend = op + dstSize; - size_t errorCode; const U32 dtLog = DTable[0]; const void* dtPtr = DTable; const HUFv05_DEltX2* const dt = ((const HUFv05_DEltX2*)dtPtr)+1; BITv05_DStream_t bitD; - errorCode = BITv05_initDStream(&bitD, cSrc, cSrcSize); - if (HUFv05_isError(errorCode)) return errorCode; + + if (dstSize <= cSrcSize) return ERROR(dstSize_tooSmall); + { size_t const errorCode = BITv05_initDStream(&bitD, cSrc, cSrcSize); + if (HUFv05_isError(errorCode)) return errorCode; } HUFv05_decodeStreamX2(op, &bitD, oend, dt, dtLog); From f45b157d956e083c16a5c2b1f5793476a1df9081 Mon Sep 17 00:00:00 2001 From: Nick Terrell Date: Tue, 18 Oct 2016 14:22:49 -0700 Subject: [PATCH 48/92] Backport fix from commit 9e8b09a Fixes uninitialized memory reads. Full commit hash: 9e8b09a7bd42dd06ee62b33aff215fbb52708d7b --- lib/legacy/zstd_v05.c | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/legacy/zstd_v05.c b/lib/legacy/zstd_v05.c index 7446b8457..a8045aea0 100644 --- a/lib/legacy/zstd_v05.c +++ b/lib/legacy/zstd_v05.c @@ -2967,6 +2967,7 @@ size_t ZSTDv05_decodeLiteralsBlock(ZSTDv05_DCtx* dctx, break; } if (litSize > BLOCKSIZE) return ERROR(corruption_detected); + if (litCSize + lhSize > srcSize) return ERROR(corruption_detected); if (HUFv05_isError(singleStream ? HUFv05_decompress1X2(dctx->litBuffer, litSize, istart+lhSize, litCSize) : From 7b06ad7a05eabf1e03a9eeff4d6fea7fe63eb0f1 Mon Sep 17 00:00:00 2001 From: Nick Terrell Date: Tue, 18 Oct 2016 14:52:34 -0700 Subject: [PATCH 49/92] Backport fix from commit 125d817 This fixes a read of unitialized memory. Full commit hash: 125d81774fe87a2bc18023d999d8e510678c38fb. --- lib/legacy/zstd_v05.c | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/legacy/zstd_v05.c b/lib/legacy/zstd_v05.c index a8045aea0..06dba0667 100644 --- a/lib/legacy/zstd_v05.c +++ b/lib/legacy/zstd_v05.c @@ -2944,6 +2944,7 @@ size_t ZSTDv05_decodeLiteralsBlock(ZSTDv05_DCtx* dctx, { size_t litSize, litCSize, singleStream=0; U32 lhSize = ((istart[0]) >> 4) & 3; + if (srcSize < 5) return ERROR(corruption_detected); /* srcSize >= MIN_CBLOCK_SIZE == 3; here we need up to 5 for case 3 */ switch(lhSize) { case 0: case 1: default: /* note : default is impossible, since lhSize into [0..3] */ From a7a4690b0a097dbb29b1f91823c86c0f9e0f21c9 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 18 Oct 2016 16:01:03 -0700 Subject: [PATCH 50/92] use implicit rules to compile zstd_decompress.c --- programs/Makefile | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/programs/Makefile b/programs/Makefile index 06da7c44b..0ed7f3d07 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -76,17 +76,18 @@ default: zstd all: zstd - +$(ZSTDDECOMP_O): CPPFLAGS += -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT) +$(ZSTDDECOMP_O): CFLAGS += $(ALIGN_LOOP) $(ZSTDDECOMP_O): $(ZSTDDIR)/decompress/zstd_decompress.c - $(CC) $(ALIGN_LOOP) $(FLAGS) -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT) $^ -c -o $@ - -$(ZSTDDECOMP32_O): $(ZSTDDIR)/decompress/zstd_decompress.c - $(CC) -m32 $(ALIGN_LOOP) $(FLAGS) -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT) $^ -c -o $@ zstd : $(ZSTDDECOMP_O) $(ZSTD_FILES) $(ZSTDLEGACY_FILES) $(ZDICT_FILES) \ zstdcli.c fileio.c bench.c datagen.c dibio.c $(CC) $(FLAGS) -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT) $^ $(RES_FILE) -o $@$(EXT) + +$(ZSTDDECOMP32_O): $(ZSTDDIR)/decompress/zstd_decompress.c + $(CC) -m32 $(ALIGN_LOOP) $(FLAGS) -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT) $^ -c -o $@ + zstd32 : $(ZSTDDECOMP32_O) $(ZSTD_FILES) $(ZSTDLEGACY_FILES) $(ZDICT_FILES) \ zstdcli.c fileio.c bench.c datagen.c dibio.c $(CC) -m32 $(FLAGS) -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT) $^ $(RES32_FILE) -o $@$(EXT) From 52c1bf93fe677dc45f243ba05bf7e3a20aa9196c Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 18 Oct 2016 16:34:58 -0700 Subject: [PATCH 51/92] improved dicitonary segment merge --- lib/dictBuilder/zdict.c | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/lib/dictBuilder/zdict.c b/lib/dictBuilder/zdict.c index a88e70520..b3f20b12b 100644 --- a/lib/dictBuilder/zdict.c +++ b/lib/dictBuilder/zdict.c @@ -371,21 +371,22 @@ static dictItem ZDICT_analyzePos( static U32 ZDICT_checkMerge(dictItem* table, dictItem elt, U32 eltNbToSkip) { const U32 tableSize = table->pos; - const U32 max = elt.pos + (elt.length-1); + const U32 eltEnd = elt.pos + elt.length; /* tail overlap */ U32 u; for (u=1; u elt.pos) && (table[u].pos < max)) { /* overlap */ + if ((table[u].pos > elt.pos) && (table[u].pos <= eltEnd)) { /* overlap, existing > new */ /* append */ U32 addedLength = table[u].pos - elt.pos; table[u].length += addedLength; table[u].pos = elt.pos; table[u].savings += elt.savings * addedLength / elt.length; /* rough approx */ - table[u].savings += elt.length / 8; /* rough approx */ + table[u].savings += elt.length / 8; /* rough approx bonus */ elt = table[u]; + /* sort : improve rank */ while ((u>1) && (table[u-1].savings < elt.savings)) - table[u] = table[u-1], u--; + table[u] = table[u-1], u--; table[u] = elt; return u; } } @@ -393,14 +394,15 @@ static U32 ZDICT_checkMerge(dictItem* table, dictItem elt, U32 eltNbToSkip) /* front overlap */ for (u=1; u elt.pos) && (table[u].pos < elt.pos)) { /* overlap */ + if ((table[u].pos + table[u].length >= elt.pos) && (table[u].pos < elt.pos)) { /* overlap, existing < new */ /* append */ - int addedLength = (elt.pos + elt.length) - (table[u].pos + table[u].length); - table[u].savings += elt.length / 8; /* rough approx */ - if (addedLength > 0) { /* otherwise, already included */ + int addedLength = (int)eltEnd - (table[u].pos + table[u].length); + table[u].savings += elt.length / 8; /* rough approx bonus */ + if (addedLength > 0) { /* otherwise, elt fully included into existing */ table[u].length += addedLength; table[u].savings += elt.savings * addedLength / elt.length; /* rough approx */ } + /* sort : improve rank */ elt = table[u]; while ((u>1) && (table[u-1].savings < elt.savings)) table[u] = table[u-1], u--; From bb68062c590dbd46905907dd2a63a658040a79d4 Mon Sep 17 00:00:00 2001 From: Nick Terrell Date: Tue, 18 Oct 2016 16:08:52 -0700 Subject: [PATCH 52/92] Unitialized memory read in ZSTD_decodeSeqHeaders() Caused by two things: 1. Not checking that `ip` is in range except for the first byte. 2. `ZSTDv0{5,6}_decodeLiteralsBlock()` could return a value larger than `srcSize`. --- lib/decompress/zstd_decompress.c | 7 +++++-- lib/legacy/zstd_v01.c | 1 + lib/legacy/zstd_v05.c | 9 ++++++++- lib/legacy/zstd_v06.c | 8 ++++++-- lib/legacy/zstd_v07.c | 7 +++++-- 5 files changed, 25 insertions(+), 7 deletions(-) diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index 83d01f3ad..de5f46eef 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -710,10 +710,13 @@ size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeqPtr, { int nbSeq = *ip++; if (!nbSeq) { *nbSeqPtr=0; return 1; } if (nbSeq > 0x7F) { - if (nbSeq == 0xFF) + if (nbSeq == 0xFF) { + if (ip+2 > iend) return ERROR(srcSize_wrong); nbSeq = MEM_readLE16(ip) + LONGNBSEQ, ip+=2; - else + } else { + if (ip >= iend) return ERROR(srcSize_wrong); nbSeq = ((nbSeq-0x80)<<8) + *ip++; + } } *nbSeqPtr = nbSeq; } diff --git a/lib/legacy/zstd_v01.c b/lib/legacy/zstd_v01.c index c115fd822..217f34aa3 100644 --- a/lib/legacy/zstd_v01.c +++ b/lib/legacy/zstd_v01.c @@ -1536,6 +1536,7 @@ size_t ZSTDv01_decodeLiteralsBlock(void* ctx, { size_t rleSize = litbp.origSize; if (rleSize>maxDstSize) return ERROR(dstSize_tooSmall); + if (!srcSize) return ERROR(srcSize_wrong); memset(oend - rleSize, *ip, rleSize); *litStart = oend - rleSize; *litSize = rleSize; diff --git a/lib/legacy/zstd_v05.c b/lib/legacy/zstd_v05.c index 06dba0667..0b46f0895 100644 --- a/lib/legacy/zstd_v05.c +++ b/lib/legacy/zstd_v05.c @@ -2994,6 +2994,7 @@ size_t ZSTDv05_decodeLiteralsBlock(ZSTDv05_DCtx* dctx, lhSize=3; litSize = ((istart[0] & 15) << 6) + (istart[1] >> 2); litCSize = ((istart[1] & 3) << 8) + istart[2]; + if (litCSize + litSize > srcSize) return ERROR(corruption_detected); errorCode = HUFv05_decompress1X4_usingDTable(dctx->litBuffer, litSize, istart+lhSize, litCSize, dctx->hufTableX4); if (HUFv05_isError(errorCode)) return ERROR(corruption_detected); @@ -3050,6 +3051,7 @@ size_t ZSTDv05_decodeLiteralsBlock(ZSTDv05_DCtx* dctx, break; case 3: litSize = ((istart[0] & 15) << 16) + (istart[1] << 8) + istart[2]; + if (srcSize<4) return ERROR(corruption_detected); /* srcSize >= MIN_CBLOCK_SIZE == 3; here we need lhSize+1 = 4 */ break; } if (litSize > BLOCKSIZE) return ERROR(corruption_detected); @@ -3083,17 +3085,22 @@ size_t ZSTDv05_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* dumps /* SeqHead */ *nbSeq = *ip++; if (*nbSeq==0) return 1; - if (*nbSeq >= 128) + if (*nbSeq >= 128) { + if (ip >= iend) return ERROR(srcSize_wrong); *nbSeq = ((nbSeq[0]-128)<<8) + *ip++; + } + if (ip >= iend) return ERROR(srcSize_wrong); LLtype = *ip >> 6; Offtype = (*ip >> 4) & 3; MLtype = (*ip >> 2) & 3; if (*ip & 2) { + if (ip+3 > iend) return ERROR(srcSize_wrong); dumpsLength = ip[2]; dumpsLength += ip[1] << 8; ip += 3; } else { + if (ip+2 > iend) return ERROR(srcSize_wrong); dumpsLength = ip[1]; dumpsLength += (ip[0] & 1) << 8; ip += 2; diff --git a/lib/legacy/zstd_v06.c b/lib/legacy/zstd_v06.c index 403a3a0b3..659b0e8ea 100644 --- a/lib/legacy/zstd_v06.c +++ b/lib/legacy/zstd_v06.c @@ -3185,6 +3185,7 @@ size_t ZSTDv06_decodeLiteralsBlock(ZSTDv06_DCtx* dctx, lhSize=3; litSize = ((istart[0] & 15) << 6) + (istart[1] >> 2); litCSize = ((istart[1] & 3) << 8) + istart[2]; + if (litCSize + litSize > srcSize) return ERROR(corruption_detected); { size_t const errorCode = HUFv06_decompress1X4_usingDTable(dctx->litBuffer, litSize, istart+lhSize, litCSize, dctx->hufTableX4); if (HUFv06_isError(errorCode)) return ERROR(corruption_detected); @@ -3304,10 +3305,13 @@ size_t ZSTDv06_decodeSeqHeaders(int* nbSeqPtr, { int nbSeq = *ip++; if (!nbSeq) { *nbSeqPtr=0; return 1; } if (nbSeq > 0x7F) { - if (nbSeq == 0xFF) + if (nbSeq == 0xFF) { + if (ip+2 > iend) return ERROR(srcSize_wrong); nbSeq = MEM_readLE16(ip) + LONGNBSEQ, ip+=2; - else + } else { + if (ip >= iend) return ERROR(srcSize_wrong); nbSeq = ((nbSeq-0x80)<<8) + *ip++; + } } *nbSeqPtr = nbSeq; } diff --git a/lib/legacy/zstd_v07.c b/lib/legacy/zstd_v07.c index 95572b93b..c8f223af7 100644 --- a/lib/legacy/zstd_v07.c +++ b/lib/legacy/zstd_v07.c @@ -3531,10 +3531,13 @@ size_t ZSTDv07_decodeSeqHeaders(int* nbSeqPtr, { int nbSeq = *ip++; if (!nbSeq) { *nbSeqPtr=0; return 1; } if (nbSeq > 0x7F) { - if (nbSeq == 0xFF) + if (nbSeq == 0xFF) { + if (ip+2 > iend) return ERROR(srcSize_wrong); nbSeq = MEM_readLE16(ip) + LONGNBSEQ, ip+=2; - else + } else { + if (ip >= iend) return ERROR(srcSize_wrong); nbSeq = ((nbSeq-0x80)<<8) + *ip++; + } } *nbSeqPtr = nbSeq; } From d760529a058e38bb7a24b5be31060d4b2a020f6f Mon Sep 17 00:00:00 2001 From: Nick Terrell Date: Wed, 19 Oct 2016 11:19:54 -0700 Subject: [PATCH 53/92] Fix stack buffer overrun when weightTotal == 0 If `weightTotal == 0`, then `BIT_highbit32(weightTotal)` is undefined behavior in the case that it calls `__builtin_clz()`. If `tableLog == HUF_TABLELOG_ABSOLUTEMAX` then we will access one byte beyond the end of the buffer. --- lib/common/entropy_common.c | 1 + lib/legacy/zstd_v01.c | 1 + lib/legacy/zstd_v02.c | 1 + lib/legacy/zstd_v03.c | 1 + lib/legacy/zstd_v04.c | 1 + lib/legacy/zstd_v05.c | 1 + lib/legacy/zstd_v06.c | 1 + lib/legacy/zstd_v07.c | 1 + 8 files changed, 8 insertions(+) diff --git a/lib/common/entropy_common.c b/lib/common/entropy_common.c index 6625a8057..18bba0e87 100644 --- a/lib/common/entropy_common.c +++ b/lib/common/entropy_common.c @@ -200,6 +200,7 @@ size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats, rankStats[huffWeight[n]]++; weightTotal += (1 << huffWeight[n]) >> 1; } } + if (weightTotal == 0) return ERROR(corruption_detected); /* get last non-null symbol weight (implied, total must be 2^n) */ { U32 const tableLog = BIT_highbit32(weightTotal) + 1; diff --git a/lib/legacy/zstd_v01.c b/lib/legacy/zstd_v01.c index 217f34aa3..5c36c2108 100644 --- a/lib/legacy/zstd_v01.c +++ b/lib/legacy/zstd_v01.c @@ -1008,6 +1008,7 @@ static size_t HUF_readDTable (U16* DTable, const void* src, size_t srcSize) rankVal[huffWeight[n]]++; weightTotal += (1 << huffWeight[n]) >> 1; } + if (weightTotal == 0) return (size_t)-FSE_ERROR_corruptionDetected; /* get last non-null symbol weight (implied, total must be 2^n) */ maxBits = FSE_highbit32(weightTotal) + 1; diff --git a/lib/legacy/zstd_v02.c b/lib/legacy/zstd_v02.c index cd1ce647e..24498fedf 100644 --- a/lib/legacy/zstd_v02.c +++ b/lib/legacy/zstd_v02.c @@ -1654,6 +1654,7 @@ static size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats, rankStats[huffWeight[n]]++; weightTotal += (1 << huffWeight[n]) >> 1; } + if (weightTotal == 0) return ERROR(corruption_detected); /* get last non-null symbol weight (implied, total must be 2^n) */ tableLog = BIT_highbit32(weightTotal) + 1; diff --git a/lib/legacy/zstd_v03.c b/lib/legacy/zstd_v03.c index 2de41f2bd..a3bd1da23 100644 --- a/lib/legacy/zstd_v03.c +++ b/lib/legacy/zstd_v03.c @@ -1651,6 +1651,7 @@ static size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats, rankStats[huffWeight[n]]++; weightTotal += (1 << huffWeight[n]) >> 1; } + if (weightTotal == 0) return ERROR(corruption_detected); /* get last non-null symbol weight (implied, total must be 2^n) */ tableLog = BIT_highbit32(weightTotal) + 1; diff --git a/lib/legacy/zstd_v04.c b/lib/legacy/zstd_v04.c index 599778b87..60479cb79 100644 --- a/lib/legacy/zstd_v04.c +++ b/lib/legacy/zstd_v04.c @@ -1943,6 +1943,7 @@ static size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats, rankStats[huffWeight[n]]++; weightTotal += (1 << huffWeight[n]) >> 1; } + if (weightTotal == 0) return ERROR(corruption_detected); /* get last non-null symbol weight (implied, total must be 2^n) */ tableLog = BIT_highbit32(weightTotal) + 1; diff --git a/lib/legacy/zstd_v05.c b/lib/legacy/zstd_v05.c index 0b46f0895..2fde052c2 100644 --- a/lib/legacy/zstd_v05.c +++ b/lib/legacy/zstd_v05.c @@ -1912,6 +1912,7 @@ static size_t HUFv05_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats, rankStats[huffWeight[n]]++; weightTotal += (1 << huffWeight[n]) >> 1; } + if (weightTotal == 0) return ERROR(corruption_detected); /* get last non-null symbol weight (implied, total must be 2^n) */ tableLog = BITv05_highbit32(weightTotal) + 1; diff --git a/lib/legacy/zstd_v06.c b/lib/legacy/zstd_v06.c index 659b0e8ea..3d6512846 100644 --- a/lib/legacy/zstd_v06.c +++ b/lib/legacy/zstd_v06.c @@ -1971,6 +1971,7 @@ MEM_STATIC size_t HUFv06_readStats(BYTE* huffWeight, size_t hwSize, U32* rankSta rankStats[huffWeight[n]]++; weightTotal += (1 << huffWeight[n]) >> 1; } } + if (weightTotal == 0) return ERROR(corruption_detected); /* get last non-null symbol weight (implied, total must be 2^n) */ { U32 const tableLog = BITv06_highbit32(weightTotal) + 1; diff --git a/lib/legacy/zstd_v07.c b/lib/legacy/zstd_v07.c index c8f223af7..ace64ede4 100644 --- a/lib/legacy/zstd_v07.c +++ b/lib/legacy/zstd_v07.c @@ -1421,6 +1421,7 @@ size_t HUFv07_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats, rankStats[huffWeight[n]]++; weightTotal += (1 << huffWeight[n]) >> 1; } } + if (weightTotal == 0) return ERROR(corruption_detected); /* get last non-null symbol weight (implied, total must be 2^n) */ { U32 const tableLog = BITv07_highbit32(weightTotal) + 1; From d365ae34978d9fd3ab83749c84d2634f1d5e86a8 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Thu, 20 Oct 2016 11:49:02 +0200 Subject: [PATCH 54/92] zstd_opt.h: small improvement in compression ratio --- lib/compress/zstd_opt.h | 29 +++++++++++++---------------- 1 file changed, 13 insertions(+), 16 deletions(-) diff --git a/lib/compress/zstd_opt.h b/lib/compress/zstd_opt.h index cea67056a..8a436df42 100644 --- a/lib/compress/zstd_opt.h +++ b/lib/compress/zstd_opt.h @@ -1,5 +1,5 @@ /** - * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * Copyright (c) 2016-present, Przemyslaw Skibinski, Yann Collet, Facebook, Inc. * All rights reserved. * * This source code is licensed under the BSD-style license found in the @@ -401,7 +401,6 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx, ZSTD_rescaleFreqs(seqStorePtr); ip += (ip==prefixStart); { U32 i; for (i=0; irep[i]; } - //inr = ip; /* Match Loop */ while (ip < ilimit) { @@ -511,20 +510,20 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx, best_off = i - (opt[cur].mlen != 1); - if (opt[cur].mlen == 1) { - litlen = opt[cur].litlen; - if (cur > litlen) { - price = opt[cur - litlen].price + ZSTD_getPrice(seqStorePtr, litlen, inr-litlen, best_off, mlen - MINMATCH); - } else - price = ZSTD_getPrice(seqStorePtr, litlen, anchor, best_off, mlen - MINMATCH); - } else { - litlen = 0; - price = opt[cur].price + ZSTD_getPrice(seqStorePtr, 0, NULL, best_off, mlen - MINMATCH); - } + if (mlen > best_mlen) best_mlen = mlen; - if (mlen > best_mlen) best_mlen = mlen; + do { + if (opt[cur].mlen == 1) { + litlen = opt[cur].litlen; + if (cur > litlen) { + price = opt[cur - litlen].price + ZSTD_getPrice(seqStorePtr, litlen, inr-litlen, best_off, mlen - MINMATCH); + } else + price = ZSTD_getPrice(seqStorePtr, litlen, anchor, best_off, mlen - MINMATCH); + } else { + litlen = 0; + price = opt[cur].price + ZSTD_getPrice(seqStorePtr, 0, NULL, best_off, mlen - MINMATCH); + } - do { if (cur + mlen > last_pos || price <= opt[cur + mlen].price) SET_PRICE(cur + mlen, mlen, i, litlen, price); mlen--; @@ -657,7 +656,6 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx, ctx->nextToUpdate3 = ctx->nextToUpdate; ZSTD_rescaleFreqs(seqStorePtr); ip += (ip==prefixStart); - //inr = ip; /* Match Loop */ while (ip < ilimit) { @@ -666,7 +664,6 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx, U32 current = (U32)(ip-base); memset(opt, 0, sizeof(ZSTD_optimal_t)); last_pos = 0; - //inr = ip; opt[0].litlen = (U32)(ip - anchor); /* check repCode */ From da3bd8b6de77364c95656f1c8cc92ca27305a864 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Thu, 20 Oct 2016 20:11:00 -0700 Subject: [PATCH 55/92] protect ZSTD_decodeFrameHeader() from invalid usage, as suggested by @spaskob --- lib/decompress/zstd_decompress.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index 00b069e9f..886a2a477 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -301,14 +301,16 @@ unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize) /** ZSTD_decodeFrameHeader() : -* `srcSize` must be the size provided by ZSTD_frameHeaderSize(). +* `headerSize` must be the size provided by ZSTD_frameHeaderSize(). * @return : 0 if success, or an error code, which can be tested using ZSTD_isError() */ -static size_t ZSTD_decodeFrameHeader(ZSTD_DCtx* dctx, const void* src, size_t srcSize) +static size_t ZSTD_decodeFrameHeader(ZSTD_DCtx* dctx, const void* src, size_t headerSize) { - size_t const result = ZSTD_getFrameParams(&(dctx->fParams), src, srcSize); + size_t const result = ZSTD_getFrameParams(&(dctx->fParams), src, headerSize); + if (ZSTD_isError(result)) return result; /* invalid header */ + if (result>0) return ERROR(srcSize_wrong); /* headerSize too small */ if (dctx->fParams.dictID && (dctx->dictID != dctx->fParams.dictID)) return ERROR(dictionary_wrong); if (dctx->fParams.checksumFlag) XXH64_reset(&dctx->xxhState, 0); - return result; + return 0; } From 4732074a71cf80811aee608dbd24368a854719c2 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Fri, 21 Oct 2016 11:19:00 +0200 Subject: [PATCH 56/92] improved ZSTD_compressBlock_opt_extDict_generic --- lib/compress/zstd_opt.h | 35 ++++++++++++++++------------------- 1 file changed, 16 insertions(+), 19 deletions(-) diff --git a/lib/compress/zstd_opt.h b/lib/compress/zstd_opt.h index 8a436df42..5e4cb7c27 100644 --- a/lib/compress/zstd_opt.h +++ b/lib/compress/zstd_opt.h @@ -16,6 +16,7 @@ #define ZSTD_FREQ_DIV 5 +#define ZSTD_MAX_PRICE (1<<30) /*-************************************* * Price functions for optimal parser @@ -171,7 +172,7 @@ MEM_STATIC void ZSTD_updatePrice(seqStore_t* seqStorePtr, U32 litLength, const B #define SET_PRICE(pos, mlen_, offset_, litlen_, price_) \ { \ - while (last_pos < pos) { opt[last_pos+1].price = 1<<30; last_pos++; } \ + while (last_pos < pos) { opt[last_pos+1].price = ZSTD_MAX_PRICE; last_pos++; } \ opt[pos].mlen = mlen_; \ opt[pos].off = offset_; \ opt[pos].litlen = litlen_; \ @@ -495,7 +496,7 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx, opt[cur].rep[0] = ((opt[cur].off==ZSTD_REP_MOVE_OPT) && (mlen != 1)) ? (opt[cur-mlen].rep[0] - 1) : (opt[cur-mlen].rep[opt[cur].off]); } - best_mlen = minMatch; + best_mlen = minMatch; { U32 i, last_i = ZSTD_REP_CHECK + (mlen != 1); for (i=(opt[cur].mlen != 1); i best_mlen) best_mlen = mlen; do { @@ -762,8 +762,7 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx, opt[cur].rep[0] = ((opt[cur].off==ZSTD_REP_MOVE_OPT) && (mlen != 1)) ? (opt[cur-mlen].rep[0] - 1) : (opt[cur-mlen].rep[opt[cur].off]); } - best_mlen = 0; - + best_mlen = minMatch; { U32 i, last_i = ZSTD_REP_CHECK + (mlen != 1); for (i = (mlen != 1); i litlen) { - price = opt[cur - litlen].price + ZSTD_getPrice(seqStorePtr, litlen, inr-litlen, best_off, mlen - MINMATCH); - } else - price = ZSTD_getPrice(seqStorePtr, litlen, anchor, best_off, mlen - MINMATCH); - } else { - litlen = 0; - price = opt[cur].price + ZSTD_getPrice(seqStorePtr, 0, NULL, best_off, mlen - MINMATCH); - } - - best_mlen = mlen; + if (mlen > best_mlen) best_mlen = mlen; do { + if (opt[cur].mlen == 1) { + litlen = opt[cur].litlen; + if (cur > litlen) { + price = opt[cur - litlen].price + ZSTD_getPrice(seqStorePtr, litlen, inr-litlen, best_off, mlen - MINMATCH); + } else + price = ZSTD_getPrice(seqStorePtr, litlen, anchor, best_off, mlen - MINMATCH); + } else { + litlen = 0; + price = opt[cur].price + ZSTD_getPrice(seqStorePtr, 0, NULL, best_off, mlen - MINMATCH); + } + if (cur + mlen > last_pos || price <= opt[cur + mlen].price) SET_PRICE(cur + mlen, mlen, i, litlen, price); mlen--; @@ -812,8 +811,6 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx, goto _storeSequence; } - best_mlen = (best_mlen > minMatch) ? best_mlen : minMatch; - /* set prices using matches at position = cur */ for (u = 0; u < match_num; u++) { mlen = (u>0) ? matches[u-1].len+1 : best_mlen; From ae1cb3b3d07024618269b89e3421d828adfd34d9 Mon Sep 17 00:00:00 2001 From: Nick Terrell Date: Thu, 20 Oct 2016 16:45:10 -0700 Subject: [PATCH 57/92] Fix small bug in ZSTD_execSequence() `memmove(op, match, sequence.matchLength)` is not the desired behavior. Overlap is allowed, and handled as if we did `*op++ = *match++`, which is not how `memmove()` handles overlap. Only triggered if both of the following conditions are met: * The match spans extDict & currentPrefixSegment * `oLitEnd <= oend_w < oLitEnd + length1 < oMatchEnd <= oend`. These two conditions imply that the block is less than 15 bytes long. This bug isn't triggered by the streaming API, because it allocates enough space for the window size + the block size, so there cannot be a match that is within 8 bytes of the end and overlaps with itself. It cannot be triggered by the block decompression API because all of the decompressed data is in the currentPrefixSegment. Introduced by commit 71585843998459e636f639165ff8a1ca9d161b62 --- lib/decompress/zstd_decompress.c | 2 +- lib/legacy/zstd_v04.c | 2 +- lib/legacy/zstd_v05.c | 4 ++-- lib/legacy/zstd_v06.c | 2 +- lib/legacy/zstd_v07.c | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index 00b069e9f..90c76faeb 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -880,7 +880,7 @@ size_t ZSTD_execSequence(BYTE* op, sequence.matchLength -= length1; match = base; if (op > oend_w) { - memmove(op, match, sequence.matchLength); + while (op < oMatchEnd) *op++ = *match++; return sequenceLength; } } } diff --git a/lib/legacy/zstd_v04.c b/lib/legacy/zstd_v04.c index 05e40aac5..a2332d933 100644 --- a/lib/legacy/zstd_v04.c +++ b/lib/legacy/zstd_v04.c @@ -3108,7 +3108,7 @@ static size_t ZSTD_execSequence(BYTE* op, sequence.matchLength -= length1; match = base; if (op > oend_8) { - memmove(op, match, sequence.matchLength); + while (op < oMatchEnd) *op++ = *match++; return sequenceLength; } } diff --git a/lib/legacy/zstd_v05.c b/lib/legacy/zstd_v05.c index 3cb5bb4af..0de28c983 100644 --- a/lib/legacy/zstd_v05.c +++ b/lib/legacy/zstd_v05.c @@ -2036,7 +2036,7 @@ size_t HUFv05_decompress1X2_usingDTable( const void* dtPtr = DTable; const HUFv05_DEltX2* const dt = ((const HUFv05_DEltX2*)dtPtr)+1; BITv05_DStream_t bitD; - + if (dstSize <= cSrcSize) return ERROR(dstSize_tooSmall); { size_t const errorCode = BITv05_initDStream(&bitD, cSrc, cSrcSize); if (HUFv05_isError(errorCode)) return errorCode; } @@ -3317,7 +3317,7 @@ static size_t ZSTDv05_execSequence(BYTE* op, sequence.matchLength -= length1; match = base; if (op > oend_8) { - memmove(op, match, sequence.matchLength); + while (op < oMatchEnd) *op++ = *match++; return sequenceLength; } } } diff --git a/lib/legacy/zstd_v06.c b/lib/legacy/zstd_v06.c index 96a84d3e8..b1ff4be4a 100644 --- a/lib/legacy/zstd_v06.c +++ b/lib/legacy/zstd_v06.c @@ -3467,7 +3467,7 @@ size_t ZSTDv06_execSequence(BYTE* op, sequence.matchLength -= length1; match = base; if (op > oend_8) { - memmove(op, match, sequence.matchLength); + while (op < oMatchEnd) *op++ = *match++; return sequenceLength; } } } diff --git a/lib/legacy/zstd_v07.c b/lib/legacy/zstd_v07.c index 62285238a..c679b97f2 100644 --- a/lib/legacy/zstd_v07.c +++ b/lib/legacy/zstd_v07.c @@ -3691,7 +3691,7 @@ size_t ZSTDv07_execSequence(BYTE* op, sequence.matchLength -= length1; match = base; if (op > oend_w) { - memmove(op, match, sequence.matchLength); + while (op < oMatchEnd) *op++ = *match++; return sequenceLength; } } } From 3ee94a7600ad0d628b1c43777de2ea7bb31f3ed0 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Mon, 24 Oct 2016 15:58:07 +0200 Subject: [PATCH 58/92] zstd_compression_format.md moved to doc/ --- .../zstd_compression_format.md | 0 lib/compress/zstd_compress.c | 2 +- lib/decompress/zstd_decompress.c | 2 +- zlibWrapper/Makefile | 21 ++++++++++--------- 4 files changed, 13 insertions(+), 12 deletions(-) rename zstd_compression_format.md => doc/zstd_compression_format.md (100%) diff --git a/zstd_compression_format.md b/doc/zstd_compression_format.md similarity index 100% rename from zstd_compression_format.md rename to doc/zstd_compression_format.md diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index e71873b03..21a1ffbe5 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -377,7 +377,7 @@ static void ZSTD_reduceIndex (ZSTD_CCtx* zc, const U32 reducerValue) * Block entropic compression *********************************************************/ -/* See zstd_compression_format.md for detailed format description */ +/* See doc/zstd_compression_format.md for detailed format description */ size_t ZSTD_noCompressBlock (void* dst, size_t dstCapacity, const void* src, size_t srcSize) { diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index 00b069e9f..456d7dd39 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -191,7 +191,7 @@ static void ZSTD_refDCtx(ZSTD_DCtx* dstDCtx, const ZSTD_DCtx* srcDCtx) * Decompression section ***************************************************************/ -/* See compression format details in : zstd_compression_format.md */ +/* See compression format details in : doc/zstd_compression_format.md */ /** ZSTD_frameHeaderSize() : * srcSize must be >= ZSTD_frameHeaderSize_prefix. diff --git a/zlibWrapper/Makefile b/zlibWrapper/Makefile index 69c976fa5..0e4ca9e6f 100644 --- a/zlibWrapper/Makefile +++ b/zlibWrapper/Makefile @@ -14,6 +14,7 @@ ZSTDLIBRARY = $(ZSTDLIBDIR)/libzstd.a ZLIBWRAPPER_PATH = . EXAMPLE_PATH = examples PROGRAMS_PATH = ../programs +TEST_FILE = ../doc/zstd_compression_format.md CC ?= gcc CFLAGS ?= -O3 CFLAGS += $(LOC) -I$(PROGRAMS_PATH) -I$(ZSTDLIBDIR) -I$(ZSTDLIBDIR)/common -I$(ZLIBWRAPPER_PATH) -std=gnu99 @@ -27,11 +28,11 @@ all: clean fitblk example zwrapbench test: example fitblk example_zstd fitblk_zstd zwrapbench ./example ./example_zstd - ./fitblk 10240 <../zstd_compression_format.md - ./fitblk 40960 <../zstd_compression_format.md - ./fitblk_zstd 10240 <../zstd_compression_format.md - ./fitblk_zstd 40960 <../zstd_compression_format.md - ./zwrapbench -qb3B1K ../zstd_compression_format.md + ./fitblk 10240 <$(TEST_FILE) + ./fitblk 40960 <$(TEST_FILE) + ./fitblk_zstd 10240 <$(TEST_FILE) + ./fitblk_zstd 40960 <$(TEST_FILE) + ./zwrapbench -qb3B1K $(TEST_FILE) ./zwrapbench -rqb1e5 ../lib ../programs ../tests #valgrindTest: ZSTDLIBRARY = $(ZSTDLIBDIR)/libzstd.so @@ -40,11 +41,11 @@ valgrindTest: clean example fitblk example_zstd fitblk_zstd zwrapbench @echo "\n ---- valgrind tests ----" $(VALGRIND) ./example $(VALGRIND) ./example_zstd - $(VALGRIND) ./fitblk 10240 <../zstd_compression_format.md - $(VALGRIND) ./fitblk 40960 <../zstd_compression_format.md - $(VALGRIND) ./fitblk_zstd 10240 <../zstd_compression_format.md - $(VALGRIND) ./fitblk_zstd 40960 <../zstd_compression_format.md - $(VALGRIND) ./zwrapbench -qb3B1K ../zstd_compression_format.md + $(VALGRIND) ./fitblk 10240 <$(TEST_FILE) + $(VALGRIND) ./fitblk 40960 <$(TEST_FILE) + $(VALGRIND) ./fitblk_zstd 10240 <$(TEST_FILE) + $(VALGRIND) ./fitblk_zstd 40960 <$(TEST_FILE) + $(VALGRIND) ./zwrapbench -qb3B1K $(TEST_FILE) $(VALGRIND) ./zwrapbench -rqb1e5 ../lib ../programs ../tests .c.o: From 984b66cd72295799a39c1bdf99bb040351161392 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Mon, 24 Oct 2016 15:59:51 +0200 Subject: [PATCH 59/92] added contrib\gen_html --- contrib/gen_html/Makefile | 36 ++++++ contrib/gen_html/README.md | 25 ++++ contrib/gen_html/gen_html.cpp | 216 ++++++++++++++++++++++++++++++++++ lib/zstd.h | 14 +-- 4 files changed, 284 insertions(+), 7 deletions(-) create mode 100644 contrib/gen_html/Makefile create mode 100644 contrib/gen_html/README.md create mode 100644 contrib/gen_html/gen_html.cpp diff --git a/contrib/gen_html/Makefile b/contrib/gen_html/Makefile new file mode 100644 index 000000000..c68e560a1 --- /dev/null +++ b/contrib/gen_html/Makefile @@ -0,0 +1,36 @@ +# ########################################################################## +# Copyright (c) 2016-present, Facebook, Inc. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. An additional grant +# of patent rights can be found in the PATENTS file in the same directory. +# ########################################################################## + + +CFLAGS ?= -O3 +CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wstrict-aliasing=1 -Wswitch-enum -Wno-comment +CFLAGS += $(MOREFLAGS) +FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) + + + +# Define *.exe as extension for Windows systems +ifneq (,$(filter Windows%,$(OS))) +EXT =.exe +else +EXT = +endif + + +.PHONY: default gen_html + +default: gen_html + +gen_html: gen_html.cpp + $(CXX) $(FLAGS) $^ -o $@$(EXT) + + +clean: + @$(RM) gen_html$(EXT) + @echo Cleaning completed diff --git a/contrib/gen_html/README.md b/contrib/gen_html/README.md new file mode 100644 index 000000000..37851f617 --- /dev/null +++ b/contrib/gen_html/README.md @@ -0,0 +1,25 @@ +gen_html - a program for automatic generation of zstd manual +============================================================ + +#### Introduction + +This simple C++ program generates a single-page HTML manual from `zstd.h`. + +The format of recognized comment blocks is following: +- comments of type `/*!` mean: this is a function declaration; switch comments with declarations +- comments of type `/**` and `/*-` mean: this is a comment; use a `

` header for the first line +- comments of type `/*=` and `/**=` mean: use a `

` header and show also all functions until first empty line +- comments of type `/*X` where `X` is different from above-mentioned are ignored + +Moreover: +- `ZSTDLIB_API` is removed to improve readability +- `typedef`s are detected and included even if uncommented +- comments of type `/**<` and `/*!<` are detected and only function declaration is highlighted (bold) + + +#### Usage + +The program requires 3 parameters: +``` +gen_html [zstd_version] [input_file] [output_html] +``` diff --git a/contrib/gen_html/gen_html.cpp b/contrib/gen_html/gen_html.cpp new file mode 100644 index 000000000..2157829f7 --- /dev/null +++ b/contrib/gen_html/gen_html.cpp @@ -0,0 +1,216 @@ +/* + * Copyright (c) 2016-present, Przemyslaw Skibinski, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ + +#include +#include +#include +#include +using namespace std; + + +/* trim string at the beginning and at the end */ +void trim(string& s, string characters) +{ + size_t p = s.find_first_not_of(characters); + s.erase(0, p); + + p = s.find_last_not_of(characters); + if (string::npos != p) + s.erase(p+1); +} + + +/* trim C++ style comments */ +void trim_comments(string &s) +{ + size_t spos, epos; + + spos = s.find("/*"); + epos = s.find("*/"); + s = s.substr(spos+3, epos-(spos+3)); +} + + +/* get lines until a given terminator */ +vector get_lines(vector& input, int& linenum, string terminator) +{ + vector out; + string line; + size_t epos; + + while ((size_t)linenum < input.size()) { + line = input[linenum]; + + if (terminator.empty() && line.empty()) { linenum--; break; } + + epos = line.find(terminator); + if (!terminator.empty() && epos!=string::npos) { + out.push_back(line); + break; + } + out.push_back(line); + linenum++; + } + return out; +} + + +/* print line with ZSTDLIB_API removed and C++ comments not bold */ +void print_line(stringstream &sout, string line) +{ + size_t spos; + + if (line.substr(0,12) == "ZSTDLIB_API ") line = line.substr(12); + spos = line.find("/*"); + if (spos!=string::npos) { + sout << line.substr(0, spos); + sout << "" << line.substr(spos) << "" << endl; + } else { + // fprintf(stderr, "lines=%s\n", line.c_str()); + sout << line << endl; + } +} + + +int main(int argc, char *argv[]) { + char exclam; + int linenum, chapter = 1; + vector input, lines, comments, chapters; + string line, version; + size_t spos, l; + stringstream sout; + ifstream istream; + ofstream ostream; + + if (argc < 4) { + cout << "usage: " << argv[0] << " [zstd_version] [input_file] [output_html]" << endl; + exit(0); + } + + version = "zstd " + string(argv[1]) + " Manual"; + + istream.open(argv[2], ifstream::in); + if (!istream.is_open()) { + cout << "Error opening file " << argv[2] << endl; + exit(0); + } + + ostream.open(argv[3], ifstream::out); + if (!ostream.is_open()) { + cout << "Error opening file " << argv[3] << endl; + exit(0); + } + + while (getline(istream, line)) { + input.push_back(line); + } + + for (linenum=0; (size_t)linenum < input.size(); linenum++) { + line = input[linenum]; + + /* typedefs are detected and included even if uncommented */ + if (line.substr(0,7) == "typedef" && line.find("{")!=string::npos) { + lines = get_lines(input, linenum, "}"); + sout << "
";
+            for (l=0; l

" << endl; + continue; + } + + /* comments of type /**< and /*!< are detected and only function declaration is highlighted (bold) */ + if ((line.find("/**<")!=string::npos || line.find("/*!<")!=string::npos) && line.find("*/")!=string::npos) { + sout << "
";
+            print_line(sout, line);
+            sout << "

" << endl; + continue; + } + + /* comments of type /*= and /**= mean: use a

header and show also all functions until first empty line */ + if ((line.substr(0,3) == "/*=" || line.substr(0,4) == "/**=") && line.find("*/")!=string::npos) { + trim_comments(line); + trim(line, "= "); + sout << "

" << line << "

";
+            lines = get_lines(input, ++linenum, "");
+            for (l=0; l

" << endl; + continue; + } + + spos = line.find("/*!"); + if (spos==string::npos) + spos = line.find("/**"); + if (spos==string::npos) + spos = line.find("/*-"); + + if (spos==string::npos) + continue; + + exclam = line[spos+2]; + comments = get_lines(input, linenum, "*/"); + if (!comments.empty()) comments[0] = line.substr(spos+3); + if (!comments.empty()) comments[comments.size()-1] = comments[comments.size()-1].substr(0, comments[comments.size()-1].find("*/")); + for (l=0; l"; + for (l=0; l

"; + for (l=0; l
" << endl << endl; + } else { /* comments of type /** and /*- mean: this is a comment; use a

header for the first line */ + if (comments.empty()) continue; + + trim(comments[0], " "); + sout << "

" << comments[0] << "

";
+            chapters.push_back(comments[0]);
+            chapter++;
+
+            for (l=1; l 1)
+                sout << "
" << endl << endl; + else + sout << "" << endl << endl; + } + } + + ostream << "\n\n\n" << version << "\n\n" << endl; + ostream << "

" << version << "

\n"; + + ostream << "
\n

Contents

\n
    \n"; + for (size_t i=0; i" << chapters[i].c_str() << "\n"; + ostream << "
\n
\n"; + + ostream << sout.str(); + ostream << "" << endl << "" << endl; + + return 0; +} \ No newline at end of file diff --git a/lib/zstd.h b/lib/zstd.h index 8039cc7db..1b6665c2b 100644 --- a/lib/zstd.h +++ b/lib/zstd.h @@ -43,7 +43,7 @@ extern "C" { - repeated calls of the compression function (described as Streaming compression) The compression ratio achievable on small data can be highly improved using compression with a dictionary in: - a single step (described as Simple dictionary API) - - a single step, reusing a dictionary (described as Fast Dictionary API) + - a single step, reusing a dictionary (described as Fast dictionary API) Advanced and experimantal functions can be accessed using #define ZSTD_STATIC_LINKING_ONLY before including zstd.h. These APIs shall never be used with a dynamic library. @@ -157,7 +157,7 @@ ZSTDLIB_API size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx, /**************************** -* Fast Dictionary API +* Fast dictionary API ****************************/ typedef struct ZSTD_CDict_s ZSTD_CDict; @@ -219,7 +219,7 @@ typedef struct ZSTD_outBuffer_s { /*-*********************************************************************** -* Streaming compression - howto +* Streaming compression - HowTo * * A ZSTD_CStream object is required to track streaming operation. * Use ZSTD_createCStream() and ZSTD_freeCStream() to create/release resources. @@ -269,7 +269,7 @@ ZSTDLIB_API size_t ZSTD_CStreamOutSize(void); /**< recommended size for output /*-*************************************************************************** -* Streaming decompression howto +* Streaming decompression - HowTo * * A ZSTD_DStream object is required to track streaming operations. * Use ZSTD_createDStream() and ZSTD_freeDStream() to create/release resources. @@ -339,7 +339,7 @@ static const size_t ZSTD_frameHeaderSize_max = ZSTD_FRAMEHEADERSIZE_MAX; static const size_t ZSTD_skippableHeaderSize = 8; /* magic number + skippable frame length */ -/*--- Types ---*/ +/*--- Advanced types ---*/ typedef enum { ZSTD_fast, ZSTD_dfast, ZSTD_greedy, ZSTD_lazy, ZSTD_lazy2, ZSTD_btlazy2, ZSTD_btopt } ZSTD_strategy; /* from faster to stronger */ typedef struct { @@ -422,7 +422,7 @@ ZSTDLIB_API size_t ZSTD_compress_advanced (ZSTD_CCtx* ctx, ZSTD_parameters params); -/*--- Advanced Decompression functions ---*/ +/*--- Advanced decompression functions ---*/ /*! ZSTD_estimateDCtxSize() : * Gives the potential amount of memory allocated to create a ZSTD_DCtx */ @@ -442,7 +442,7 @@ ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict); /******************************************************************** -* Advanced Streaming functions +* Advanced streaming functions ********************************************************************/ /*===== Advanced Streaming compression functions =====*/ From 86d9424c81e2febfbf5e885b0cc621fa4b7858d1 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Mon, 24 Oct 2016 16:07:53 +0200 Subject: [PATCH 60/92] added doc\zstd_manual.html --- contrib/gen_html/README.md | 8 +- doc/zstd_manual.html | 528 +++++++++++++++++++++++++++++++++++++ 2 files changed, 535 insertions(+), 1 deletion(-) create mode 100644 doc/zstd_manual.html diff --git a/contrib/gen_html/README.md b/contrib/gen_html/README.md index 37851f617..63a4caa25 100644 --- a/contrib/gen_html/README.md +++ b/contrib/gen_html/README.md @@ -13,7 +13,7 @@ The format of recognized comment blocks is following: Moreover: - `ZSTDLIB_API` is removed to improve readability -- `typedef`s are detected and included even if uncommented +- `typedef` are detected and included even if uncommented - comments of type `/**<` and `/*!<` are detected and only function declaration is highlighted (bold) @@ -23,3 +23,9 @@ The program requires 3 parameters: ``` gen_html [zstd_version] [input_file] [output_html] ``` + +To compile program and generate zstd manual we have used: +``` +make +./gen_html.exe 1.1.1 ../../lib/zstd.h zstd_manual.html +``` diff --git a/doc/zstd_manual.html b/doc/zstd_manual.html new file mode 100644 index 000000000..aac5af9c2 --- /dev/null +++ b/doc/zstd_manual.html @@ -0,0 +1,528 @@ + + + +zstd 1.1.1 Manual + + +

zstd 1.1.1 Manual

+
+

Contents

+
    +
  1. Introduction
    2. +
  2. Version
    3. +
  3. Simple API
    4. +
  4. Explicit memory management
    5. +
  5. Simple dictionary API
    6. +
  6. Fast dictionary API
    7. +
  7. Streaming
    8. +
  8. Streaming compression - HowTo
    9. +
  9. Streaming decompression - HowTo
    10. +
  10. START OF ADVANCED AND EXPERIMENTAL FUNCTIONS
    11. +
  11. Advanced types
    12. +
  12. Advanced compression functions
    13. +
  13. Advanced decompression functions
    14. +
  14. Advanced streaming functions
    15. +
  15. Buffer-less and synchronous inner streaming functions
    16. +
  16. Buffer-less streaming compression (synchronous mode)
    17. +
  17. Buffer-less streaming decompression (synchronous mode)
    18. +
  18. Block functions
    19. +
+
+

Introduction

+  Zstd, short for Zstandard, is a fast lossless compression algorithm, targeting real-time compression scenarios
+  at zlib-level and better compression ratios. The zstd compression library provides in-memory compression and
+  decompression functions. The library supports compression levels from 1 up to ZSTD_maxCLevel() which is 22.
+  Levels from 20 to 22 should be used with caution as they require about 300-1300 MB for compression.
+  Compression can be done in:
+    - a single step (described as Simple API)
+    - a single step, reusing a context (described as Explicit memory management)
+    - repeated calls of the compression function (described as Streaming compression)
+  The compression ratio achievable on small data can be highly improved using compression with a dictionary in:
+    - a single step (described as Simple dictionary API)
+    - a single step, reusing a dictionary (described as Fast dictionary API)
+
+  Advanced and experimantal functions can be accessed using #define ZSTD_STATIC_LINKING_ONLY before including zstd.h.
+  These APIs shall never be used with a dynamic library. 
+  They are not "stable", their definition may change in the future. Only static linking is allowed.
+
+ +

Version


+
+
unsigned ZSTD_versionNumber (void);  /**< returns version number of ZSTD */
+


+
Simple API

+
+
size_t ZSTD_compress( void* dst, size_t dstCapacity,
+                            const void* src, size_t srcSize,
+                                  int compressionLevel);
+
    Compresses `src` content as a single zstd compressed frame into already allocated `dst`.
+    Hint : compression runs faster if `dstCapacity` >=  `ZSTD_compressBound(srcSize)`.
+    @return : compressed size written into `dst` (<= `dstCapacity),
+              or an error code if it fails (which can be tested using ZSTD_isError()) 
+


+
+
size_t ZSTD_decompress( void* dst, size_t dstCapacity,
+                              const void* src, size_t compressedSize);
+
    `compressedSize` : must be the _exact_ size of a single compressed frame.
+    `dstCapacity` is an upper bound of originalSize.
+    If user cannot imply a maximum upper bound, it's better to use streaming mode to decompress data.
+    @return : the number of bytes decompressed into `dst` (<= `dstCapacity`),
+              or an errorCode if it fails (which can be tested using ZSTD_isError()) 
+


+
+
unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize);
+
   'src' is the start of a zstd compressed frame.
+   @return : content size to be decompressed, as a 64-bits value _if known_, 0 otherwise.
+    note 1 : decompressed size is an optional field, that may not be present, especially in streaming mode.
+             When `return==0`, data to decompress could be any size.
+             In which case, it's necessary to use streaming mode to decompress data.
+             Optionally, application can still use ZSTD_decompress() while relying on implied limits.
+             (For example, data may be necessarily cut into blocks <= 16 KB).
+    note 2 : decompressed size is always present when compression is done with ZSTD_compress()
+    note 3 : decompressed size can be very large (64-bits value),
+             potentially larger than what local system can handle as a single memory segment.
+             In which case, it's necessary to use streaming mode to decompress data.
+    note 4 : If source is untrusted, decompressed size could be wrong or intentionally modified.
+             Always ensure result fits within application's authorized limits.
+             Each application can set its own limits.
+    note 5 : when `return==0`, if precise failure cause is needed, use ZSTD_getFrameParams() to know more. 
+


+
+
Helper functions
int         ZSTD_maxCLevel(void);               /*!< maximum compression level available */
+size_t      ZSTD_compressBound(size_t srcSize); /*!< maximum compressed size in worst case scenario */
+unsigned    ZSTD_isError(size_t code);          /*!< tells if a `size_t` function result is an error code */
+const char* ZSTD_getErrorName(size_t code);     /*!< provides readable string from an error code */
+


+
Explicit memory management

+
+
Compression context
typedef struct ZSTD_CCtx_s ZSTD_CCtx;
+ZSTD_CCtx* ZSTD_createCCtx(void);
+size_t     ZSTD_freeCCtx(ZSTD_CCtx* cctx);
+


+
size_t ZSTD_compressCCtx(ZSTD_CCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, int compressionLevel);
+
    Same as ZSTD_compress(), requires an allocated ZSTD_CCtx (see ZSTD_createCCtx()) 
+


+
+
Decompression context
typedef struct ZSTD_DCtx_s ZSTD_DCtx;
+ZSTD_DCtx* ZSTD_createDCtx(void);
+size_t     ZSTD_freeDCtx(ZSTD_DCtx* dctx);
+


+
size_t ZSTD_decompressDCtx(ZSTD_DCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
+
   Same as ZSTD_decompress(), requires an allocated ZSTD_DCtx (see ZSTD_createDCtx()) 
+


+
+
Simple dictionary API

+
+
size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx,
+                                           void* dst, size_t dstCapacity,
+                                     const void* src, size_t srcSize,
+                                     const void* dict,size_t dictSize,
+                                           int compressionLevel);
+
   Compression using a predefined Dictionary (see dictBuilder/zdict.h).
+   Note : This function load the dictionary, resulting in significant startup delay. 
+


+
+
size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx,
+                                             void* dst, size_t dstCapacity,
+                                       const void* src, size_t srcSize,
+                                       const void* dict,size_t dictSize);
+
   Decompression using a predefined Dictionary (see dictBuilder/zdict.h).
+   Dictionary must be identical to the one used during compression.
+   Note : This function load the dictionary, resulting in significant startup delay 
+


+
+
Fast dictionary API

+
+
ZSTD_CDict* ZSTD_createCDict(const void* dict, size_t dictSize, int compressionLevel);
+
   Create a digested dictionary, ready to start compression operation without startup delay.
+   `dict` can be released after ZSTD_CDict creation 
+


+
+
size_t      ZSTD_freeCDict(ZSTD_CDict* CDict);
+
   Function frees memory allocated with ZSTD_createCDict() 
+


+
+
size_t ZSTD_compress_usingCDict(ZSTD_CCtx* cctx,
+                                            void* dst, size_t dstCapacity,
+                                      const void* src, size_t srcSize,
+                                      const ZSTD_CDict* cdict);
+
   Compression using a digested Dictionary.
+   Faster startup than ZSTD_compress_usingDict(), recommended when same dictionary is used multiple times.
+   Note that compression level is decided during dictionary creation 
+


+
+
ZSTD_DDict* ZSTD_createDDict(const void* dict, size_t dictSize);
+
   Create a digested dictionary, ready to start decompression operation without startup delay.
+   `dict` can be released after creation 
+


+
+
size_t      ZSTD_freeDDict(ZSTD_DDict* ddict);
+
   Function frees memory allocated with ZSTD_createDDict() 
+


+
+
size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx,
+                                              void* dst, size_t dstCapacity,
+                                        const void* src, size_t srcSize,
+                                        const ZSTD_DDict* ddict);
+
   Decompression using a digested Dictionary
+   Faster startup than ZSTD_decompress_usingDict(), recommended when same dictionary is used multiple times. 
+


+
+
Streaming

+
+
typedef struct ZSTD_inBuffer_s {
+  const void* src;    /**< start of input buffer */
+  size_t size;        /**< size of input buffer */
+  size_t pos;         /**< position where reading stopped. Will be updated. Necessarily 0 <= pos <= size */
+} ZSTD_inBuffer;
+


+
typedef struct ZSTD_outBuffer_s {
+  void*  dst;         /**< start of output buffer */
+  size_t size;        /**< size of output buffer */
+  size_t pos;         /**< position where writing stopped. Will be updated. Necessarily 0 <= pos <= size */
+} ZSTD_outBuffer;
+


+
Streaming compression - HowTo
+  A ZSTD_CStream object is required to track streaming operation.
+  Use ZSTD_createCStream() and ZSTD_freeCStream() to create/release resources.
+  ZSTD_CStream objects can be reused multiple times on consecutive compression operations.
+
+  Start by initializing ZSTD_CStream.
+  Use ZSTD_initCStream() to start a new compression operation.
+  Use ZSTD_initCStream_usingDict() for a compression which requires a dictionary.
+
+  Use ZSTD_compressStream() repetitively to consume input stream.
+  The function will automatically update both `pos` fields.
+  Note that it may not consume the entire input, in which case `pos < size`,
+  and it's up to the caller to present again remaining data.
+  @return : a size hint, preferred nb of bytes to use as input for next function call
+           (it's just a hint, to help latency a little, any other value will work fine)
+           (note : the size hint is guaranteed to be <= ZSTD_CStreamInSize() )
+            or an error code, which can be tested using ZSTD_isError().
+
+  At any moment, it's possible to flush whatever data remains within buffer, using ZSTD_flushStream().
+  `output->pos` will be updated.
+  Note some content might still be left within internal buffer if `output->size` is too small.
+  @return : nb of bytes still present within internal buffer (0 if it's empty)
+            or an error code, which can be tested using ZSTD_isError().
+
+  ZSTD_endStream() instructs to finish a frame.
+  It will perform a flush and write frame epilogue.
+  The epilogue is required for decoders to consider a frame completed.
+  Similar to ZSTD_flushStream(), it may not be able to flush the full content if `output->size` is too small.
+  In which case, call again ZSTD_endStream() to complete the flush.
+  @return : nb of bytes still present within internal buffer (0 if it's empty)
+            or an error code, which can be tested using ZSTD_isError().
+
+ 
+

+
+
Streaming compression functions
typedef struct ZSTD_CStream_s ZSTD_CStream;
+ZSTD_CStream* ZSTD_createCStream(void);
+size_t ZSTD_freeCStream(ZSTD_CStream* zcs);
+size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel);
+size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuffer* input);
+size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output);
+size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output);
+


+
size_t ZSTD_CStreamInSize(void);    /**< recommended size for input buffer */
+


+
size_t ZSTD_CStreamOutSize(void);   /**< recommended size for output buffer. Guarantee to successfully flush at least one complete compressed block in all circumstances. */
+


+
Streaming decompression - HowTo
+  A ZSTD_DStream object is required to track streaming operations.
+  Use ZSTD_createDStream() and ZSTD_freeDStream() to create/release resources.
+  ZSTD_DStream objects can be re-used multiple times.
+
+  Use ZSTD_initDStream() to start a new decompression operation,
+   or ZSTD_initDStream_usingDict() if decompression requires a dictionary.
+   @return : recommended first input size
+
+  Use ZSTD_decompressStream() repetitively to consume your input.
+  The function will update both `pos` fields.
+  If `input.pos < input.size`, some input has not been consumed.
+  It's up to the caller to present again remaining data.
+  If `output.pos < output.size`, decoder has flushed everything it could.
+  @return : 0 when a frame is completely decoded and fully flushed,
+            an error code, which can be tested using ZSTD_isError(),
+            any other value > 0, which means there is still some work to do to complete the frame.
+            The return value is a suggested next input size (just an hint, to help latency).
+ 
+

+
+
Streaming decompression functions
typedef struct ZSTD_DStream_s ZSTD_DStream;
+ZSTD_DStream* ZSTD_createDStream(void);
+size_t ZSTD_freeDStream(ZSTD_DStream* zds);
+size_t ZSTD_initDStream(ZSTD_DStream* zds);
+size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inBuffer* input);
+


+
size_t ZSTD_DStreamInSize(void);    /*!< recommended size for input buffer */
+


+
size_t ZSTD_DStreamOutSize(void);   /*!< recommended size for output buffer. Guarantee to successfully flush at least one complete block in all circumstances. */
+


+
START OF ADVANCED AND EXPERIMENTAL FUNCTIONS
 The definitions in this section are considered experimental.
+ They should never be used with a dynamic library, as they may change in the future.
+ They are provided for advanced usages.
+ Use them only in association with static linking.
+ 
+

+
+
Advanced types

+
+
typedef enum { ZSTD_fast, ZSTD_dfast, ZSTD_greedy, ZSTD_lazy, ZSTD_lazy2, ZSTD_btlazy2, ZSTD_btopt } ZSTD_strategy;   /* from faster to stronger */
+


+
typedef struct {
+    unsigned windowLog;      /**< largest match distance : larger == more compression, more memory needed during decompression */
+    unsigned chainLog;       /**< fully searched segment : larger == more compression, slower, more memory (useless for fast) */
+    unsigned hashLog;        /**< dispatch table : larger == faster, more memory */
+    unsigned searchLog;      /**< nb of searches : larger == more compression, slower */
+    unsigned searchLength;   /**< match length searched : larger == faster decompression, sometimes less compression */
+    unsigned targetLength;   /**< acceptable match size for optimal parser (only) : larger == more compression, slower */
+    ZSTD_strategy strategy;
+} ZSTD_compressionParameters;
+


+
typedef struct {
+    unsigned contentSizeFlag; /**< 1: content size will be in frame header (if known). */
+    unsigned checksumFlag;    /**< 1: will generate a 22-bits checksum at end of frame, to be used for error detection by decompressor */
+    unsigned noDictIDFlag;    /**< 1: no dict ID will be saved into frame header (if dictionary compression) */
+} ZSTD_frameParameters;
+


+
typedef struct {
+    ZSTD_compressionParameters cParams;
+    ZSTD_frameParameters fParams;
+} ZSTD_parameters;
+


+
Custom memory allocation functions
typedef void* (*ZSTD_allocFunction) (void* opaque, size_t size);
+typedef void  (*ZSTD_freeFunction) (void* opaque, void* address);
+typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; void* opaque; } ZSTD_customMem;
+


+
Advanced compression functions

+
+
size_t ZSTD_estimateCCtxSize(ZSTD_compressionParameters cParams);
+
  Gives the amount of memory allocated for a ZSTD_CCtx given a set of compression parameters.
+  `frameContentSize` is an optional parameter, provide `0` if unknown 
+


+
+
ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem);
+
  Create a ZSTD compression context using external alloc and free functions 
+


+
+
size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx);
+
  Gives the amount of memory used by a given ZSTD_CCtx 
+


+
+
ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictSize,
+                                                  ZSTD_parameters params, ZSTD_customMem customMem);
+
  Create a ZSTD_CDict using external alloc and free, and customized compression parameters 
+


+
+
size_t ZSTD_sizeof_CDict(const ZSTD_CDict* cdict);
+
  Gives the amount of memory used by a given ZSTD_sizeof_CDict 
+


+
+
ZSTD_parameters ZSTD_getParams(int compressionLevel, unsigned long long srcSize, size_t dictSize);
+
   same as ZSTD_getCParams(), but @return a full `ZSTD_parameters` object instead of a `ZSTD_compressionParameters`.
+   All fields of `ZSTD_frameParameters` are set to default (0) 
+


+
+
ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, unsigned long long srcSize, size_t dictSize);
+
   @return ZSTD_compressionParameters structure for a selected compression level and srcSize.
+   `srcSize` value is optional, select 0 if not known 
+


+
+
size_t ZSTD_checkCParams(ZSTD_compressionParameters params);
+
   Ensure param values remain within authorized range 
+


+
+
ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, unsigned long long srcSize, size_t dictSize);
+
   optimize params for a given `srcSize` and `dictSize`.
+   both values are optional, select `0` if unknown. 
+


+
+
size_t ZSTD_compress_advanced (ZSTD_CCtx* ctx,
+                                           void* dst, size_t dstCapacity,
+                                     const void* src, size_t srcSize,
+                                     const void* dict,size_t dictSize,
+                                           ZSTD_parameters params);
+
   Same as ZSTD_compress_usingDict(), with fine-tune control of each compression parameter 
+


+
+
Advanced decompression functions

+
+
size_t ZSTD_estimateDCtxSize(void);
+
  Gives the potential amount of memory allocated to create a ZSTD_DCtx 
+


+
+
ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem);
+
  Create a ZSTD decompression context using external alloc and free functions 
+


+
+
size_t ZSTD_sizeof_DCtx(const ZSTD_DCtx* dctx);
+
  Gives the amount of memory used by a given ZSTD_DCtx 
+


+
+
size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
+
  Gives the amount of memory used by a given ZSTD_DDict 
+


+
+
Advanced streaming functions

+
+
Advanced Streaming compression functions
ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem);
+size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel);
+size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize,
+                                             ZSTD_parameters params, unsigned long long pledgedSrcSize);  /**< pledgedSrcSize is optional and can be zero == unknown */
+size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize);  /**< re-use compression parameters from previous init; saves dictionary loading */
+size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs);
+


+
Advanced Streaming decompression functions
typedef enum { ZSTDdsp_maxWindowSize } ZSTD_DStreamParameter_e;
+ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem);
+size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize);
+size_t ZSTD_setDStreamParameter(ZSTD_DStream* zds, ZSTD_DStreamParameter_e paramType, unsigned paramValue);
+size_t ZSTD_resetDStream(ZSTD_DStream* zds);  /**< re-use decompression parameters from previous init; saves dictionary loading */
+size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds);
+


+
Buffer-less and synchronous inner streaming functions
+  This is an advanced API, giving full control over buffer management, for users which need direct control over memory.
+  But it's also a complex one, with many restrictions (documented below).
+  Prefer using normal streaming API for an easier experience 
+ 
+

+
+
Buffer-less streaming compression (synchronous mode)
+  A ZSTD_CCtx object is required to track streaming operations.
+  Use ZSTD_createCCtx() / ZSTD_freeCCtx() to manage resource.
+  ZSTD_CCtx object can be re-used multiple times within successive compression operations.
+
+  Start by initializing a context.
+  Use ZSTD_compressBegin(), or ZSTD_compressBegin_usingDict() for dictionary compression,
+  or ZSTD_compressBegin_advanced(), for finer parameter control.
+  It's also possible to duplicate a reference context which has already been initialized, using ZSTD_copyCCtx()
+
+  Then, consume your input using ZSTD_compressContinue().
+  There are some important considerations to keep in mind when using this advanced function :
+  - ZSTD_compressContinue() has no internal buffer. It uses externally provided buffer only.
+  - Interface is synchronous : input is consumed entirely and produce 1+ (or more) compressed blocks.
+  - Caller must ensure there is enough space in `dst` to store compressed data under worst case scenario.
+    Worst case evaluation is provided by ZSTD_compressBound().
+    ZSTD_compressContinue() doesn't guarantee recover after a failed compression.
+  - ZSTD_compressContinue() presumes prior input ***is still accessible and unmodified*** (up to maximum distance size, see WindowLog).
+    It remembers all previous contiguous blocks, plus one separated memory segment (which can itself consists of multiple contiguous blocks)
+  - ZSTD_compressContinue() detects that prior input has been overwritten when `src` buffer overlaps.
+    In which case, it will "discard" the relevant memory section from its history.
+
+  Finish a frame with ZSTD_compressEnd(), which will write the last block(s) and optional checksum.
+  It's possible to use a NULL,0 src content, in which case, it will write a final empty block to end the frame,
+  Without last block mark, frames will be considered unfinished (broken) by decoders.
+
+  You can then reuse `ZSTD_CCtx` (ZSTD_compressBegin()) to compress some new frame.
+

+
+
Buffer-less streaming compression functions
size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel);
+size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel);
+size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize);
+size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize);
+size_t ZSTD_compressContinue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
+size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
+


+
Buffer-less streaming decompression (synchronous mode)
+  A ZSTD_DCtx object is required to track streaming operations.
+  Use ZSTD_createDCtx() / ZSTD_freeDCtx() to manage it.
+  A ZSTD_DCtx object can be re-used multiple times.
+
+  First typical operation is to retrieve frame parameters, using ZSTD_getFrameParams().
+  It fills a ZSTD_frameParams structure which provide important information to correctly decode the frame,
+  such as the minimum rolling buffer size to allocate to decompress data (`windowSize`),
+  and the dictionary ID used.
+  (Note : content size is optional, it may not be present. 0 means : content size unknown).
+  Note that these values could be wrong, either because of data malformation, or because an attacker is spoofing deliberate false information.
+  As a consequence, check that values remain within valid application range, especially `windowSize`, before allocation.
+  Each application can set its own limit, depending on local restrictions. For extended interoperability, it is recommended to support at least 8 MB.
+  Frame parameters are extracted from the beginning of the compressed frame.
+  Data fragment must be large enough to ensure successful decoding, typically `ZSTD_frameHeaderSize_max` bytes.
+  @result : 0 : successful decoding, the `ZSTD_frameParams` structure is correctly filled.
+           >0 : `srcSize` is too small, please provide at least @result bytes on next attempt.
+           errorCode, which can be tested using ZSTD_isError().
+
+  Start decompression, with ZSTD_decompressBegin() or ZSTD_decompressBegin_usingDict().
+  Alternatively, you can copy a prepared context, using ZSTD_copyDCtx().
+
+  Then use ZSTD_nextSrcSizeToDecompress() and ZSTD_decompressContinue() alternatively.
+  ZSTD_nextSrcSizeToDecompress() tells how many bytes to provide as 'srcSize' to ZSTD_decompressContinue().
+  ZSTD_decompressContinue() requires this _exact_ amount of bytes, or it will fail.
+
+  @result of ZSTD_decompressContinue() is the number of bytes regenerated within 'dst' (necessarily <= dstCapacity).
+  It can be zero, which is not an error; it just means ZSTD_decompressContinue() has decoded some metadata item.
+  It can also be an error code, which can be tested with ZSTD_isError().
+
+  ZSTD_decompressContinue() needs previous data blocks during decompression, up to `windowSize`.
+  They should preferably be located contiguously, prior to current block.
+  Alternatively, a round buffer of sufficient size is also possible. Sufficient size is determined by frame parameters.
+  ZSTD_decompressContinue() is very sensitive to contiguity,
+  if 2 blocks don't follow each other, make sure that either the compressor breaks contiguity at the same place,
+  or that previous contiguous segment is large enough to properly handle maximum back-reference.
+
+  A frame is fully decoded when ZSTD_nextSrcSizeToDecompress() returns zero.
+  Context can then be reset to start a new decompression.
+
+  Note : it's possible to know if next input to present is a header or a block, using ZSTD_nextInputType().
+  This information is not required to properly decode a frame.
+
+  == Special case : skippable frames ==
+
+  Skippable frames allow integration of user-defined data into a flow of concatenated frames.
+  Skippable frames will be ignored (skipped) by a decompressor. The format of skippable frames is as follows :
+  a) Skippable frame ID - 4 Bytes, Little endian format, any value from 0x184D2A50 to 0x184D2A5F
+  b) Frame Size - 4 Bytes, Little endian format, unsigned 32-bits
+  c) Frame Content - any content (User Data) of length equal to Frame Size
+  For skippable frames ZSTD_decompressContinue() always returns 0.
+  For skippable frames ZSTD_getFrameParams() returns fparamsPtr->windowLog==0 what means that a frame is skippable.
+  It also returns Frame Size as fparamsPtr->frameContentSize.
+

+
+
typedef struct {
+    unsigned long long frameContentSize;
+    unsigned windowSize;
+    unsigned dictID;
+    unsigned checksumFlag;
+} ZSTD_frameParams;
+


+
Buffer-less streaming decompression functions
size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t srcSize);   /**< doesn't consume input, see details below */
+size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx);
+size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t dictSize);
+void   ZSTD_copyDCtx(ZSTD_DCtx* dctx, const ZSTD_DCtx* preparedDCtx);
+size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx);
+size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
+typedef enum { ZSTDnit_frameHeader, ZSTDnit_blockHeader, ZSTDnit_block, ZSTDnit_lastBlock, ZSTDnit_checksum, ZSTDnit_skippableFrame } ZSTD_nextInputType_e;
+ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx);
+


+
Block functions
+    Block functions produce and decode raw zstd blocks, without frame metadata.
+    Frame metadata cost is typically ~18 bytes, which can be non-negligible for very small blocks (< 100 bytes).
+    User will have to take in charge required information to regenerate data, such as compressed and content sizes.
+
+    A few rules to respect :
+    - Compressing and decompressing require a context structure
+      + Use ZSTD_createCCtx() and ZSTD_createDCtx()
+    - It is necessary to init context before starting
+      + compression : ZSTD_compressBegin()
+      + decompression : ZSTD_decompressBegin()
+      + variants _usingDict() are also allowed
+      + copyCCtx() and copyDCtx() work too
+    - Block size is limited, it must be <= ZSTD_getBlockSizeMax()
+      + If you need to compress more, cut data into multiple blocks
+      + Consider using the regular ZSTD_compress() instead, as frame metadata costs become negligible when source size is large.
+    - When a block is considered not compressible enough, ZSTD_compressBlock() result will be zero.
+      In which case, nothing is produced into `dst`.
+      + User must test for such outcome and deal directly with uncompressed data
+      + ZSTD_decompressBlock() doesn't accept uncompressed data as input !!!
+      + In case of multiple successive blocks, decoder must be informed of uncompressed block existence to follow proper history.
+        Use ZSTD_insertBlock() in such a case.
+

+
+
Raw zstd block functions
size_t ZSTD_getBlockSizeMax(ZSTD_CCtx* cctx);
+size_t ZSTD_compressBlock  (ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
+size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
+size_t ZSTD_insertBlock(ZSTD_DCtx* dctx, const void* blockStart, size_t blockSize);  /**< insert block into `dctx` history. Useful for uncompressed blocks */
+


+
+

From f9c9af3c2e84bb61841ff41aa8e5afbd2ca83d7d Mon Sep 17 00:00:00 2001
From: Nick Terrell 
Date: Wed, 19 Oct 2016 17:22:08 -0700
Subject: [PATCH 61/92] Reject dictionaries with incomplete entropy tables

If a dictionary specifies that a symbol has probability zero in its
`matchLength`, `literalLength`, or `offset` FSE table, but the symbol
appears when compressing input, the compressor fails.

Ensure that dictionaries support all `matchLength`, and `literalLength`
codes.  They must also support all of the `offset` codes required to
represent every possible offset that can appear in the first block.
---
 lib/compress/zstd_compress.c | 31 +++++++++++++++++++++++++++++--
 1 file changed, 29 insertions(+), 2 deletions(-)

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index c44ef3d41..fb12898e8 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -2448,6 +2448,20 @@ static size_t ZSTD_loadDictionaryContent(ZSTD_CCtx* zc, const void* src, size_t
 }
 
 
+/* Dictionaries that assign zero probability to symbols that show up causes problems
+   when FSE encoding.  Refuse dictionaries that assign zero probability to symbols
+   that we may encounter during compression.
+   NOTE: This behavior is not standard and could be improved in the future. */
+static size_t ZSTD_checkDictNCount(short* normalizedCounter, unsigned dictMaxSymbolValue, unsigned maxSymbolValue) {
+    U32 s;
+    if (dictMaxSymbolValue < maxSymbolValue) return ERROR(dictionary_corrupted);
+    for (s = 0; s <= maxSymbolValue; ++s) {
+        if (normalizedCounter[s] == 0) return ERROR(dictionary_corrupted);
+    }
+    return 0;
+}
+
+
 /* Dictionary format :
      Magic == ZSTD_DICT_MAGIC (4 bytes)
      HUF_writeCTable(256)
@@ -2464,17 +2478,19 @@ static size_t ZSTD_loadDictEntropyStats(ZSTD_CCtx* cctx, const void* dict, size_
 {
     const BYTE* dictPtr = (const BYTE*)dict;
     const BYTE* const dictEnd = dictPtr + dictSize;
+    short offcodeNCount[MaxOff+1];
+    unsigned offcodeMaxValue = MaxOff;
 
     {   size_t const hufHeaderSize = HUF_readCTable(cctx->hufTable, 255, dict, dictSize);
         if (HUF_isError(hufHeaderSize)) return ERROR(dictionary_corrupted);
         dictPtr += hufHeaderSize;
     }
 
-    {   short offcodeNCount[MaxOff+1];
-        unsigned offcodeMaxValue = MaxOff, offcodeLog;
+    {   unsigned offcodeLog;
         size_t const offcodeHeaderSize = FSE_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dictPtr, dictEnd-dictPtr);
         if (FSE_isError(offcodeHeaderSize)) return ERROR(dictionary_corrupted);
         if (offcodeLog > OffFSELog) return ERROR(dictionary_corrupted);
+        /* Defer checking offcodeMaxValue because we need to know the size of the dictionary content */
         CHECK_E (FSE_buildCTable(cctx->offcodeCTable, offcodeNCount, offcodeMaxValue, offcodeLog), dictionary_corrupted);
         dictPtr += offcodeHeaderSize;
     }
@@ -2484,6 +2500,8 @@ static size_t ZSTD_loadDictEntropyStats(ZSTD_CCtx* cctx, const void* dict, size_
         size_t const matchlengthHeaderSize = FSE_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dictPtr, dictEnd-dictPtr);
         if (FSE_isError(matchlengthHeaderSize)) return ERROR(dictionary_corrupted);
         if (matchlengthLog > MLFSELog) return ERROR(dictionary_corrupted);
+        /* Every match length code must have non-zero probability */
+        CHECK_F (ZSTD_checkDictNCount(matchlengthNCount, matchlengthMaxValue, MaxML));
         CHECK_E (FSE_buildCTable(cctx->matchlengthCTable, matchlengthNCount, matchlengthMaxValue, matchlengthLog), dictionary_corrupted);
         dictPtr += matchlengthHeaderSize;
     }
@@ -2493,6 +2511,8 @@ static size_t ZSTD_loadDictEntropyStats(ZSTD_CCtx* cctx, const void* dict, size_
         size_t const litlengthHeaderSize = FSE_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dictPtr, dictEnd-dictPtr);
         if (FSE_isError(litlengthHeaderSize)) return ERROR(dictionary_corrupted);
         if (litlengthLog > LLFSELog) return ERROR(dictionary_corrupted);
+        /* Every literal length code must have non-zero probability */
+        CHECK_F (ZSTD_checkDictNCount(litlengthNCount, litlengthMaxValue, MaxLL));
         CHECK_E(FSE_buildCTable(cctx->litlengthCTable, litlengthNCount, litlengthMaxValue, litlengthLog), dictionary_corrupted);
         dictPtr += litlengthHeaderSize;
     }
@@ -2503,6 +2523,13 @@ static size_t ZSTD_loadDictEntropyStats(ZSTD_CCtx* cctx, const void* dict, size_
     cctx->rep[2] = MEM_readLE32(dictPtr+8); if (cctx->rep[2] >= dictSize) return ERROR(dictionary_corrupted);
     dictPtr += 12;
 
+    {   size_t const maxOffset = (dictEnd - dictPtr) + 128 KB; /* The maximum offset that must be supported */
+        /* Calculate minimum offset code required to represent maxOffset */
+        unsigned const offcodeMax = ZSTD_highbit32(maxOffset);
+        /* Every possible supported offset <= dictContentSize + 128 KB must be representable */
+        CHECK_F (ZSTD_checkDictNCount(offcodeNCount, offcodeMaxValue, MIN(offcodeMax, MaxOff)));
+    }
+
     cctx->flagStaticTables = 1;
     return dictPtr - (const BYTE*)dict;
 }

From b2c39a22b028cd4d7967e233e2bf8e1d0bdc4a44 Mon Sep 17 00:00:00 2001
From: Nick Terrell 
Date: Mon, 24 Oct 2016 14:11:27 -0700
Subject: [PATCH 62/92] Fix compiler narrowing warning

---
 lib/compress/zstd_compress.c | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 1d4450aa5..6922aab6c 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -2523,9 +2523,12 @@ static size_t ZSTD_loadDictEntropyStats(ZSTD_CCtx* cctx, const void* dict, size_
     cctx->rep[2] = MEM_readLE32(dictPtr+8); if (cctx->rep[2] >= dictSize) return ERROR(dictionary_corrupted);
     dictPtr += 12;
 
-    {   size_t const maxOffset = (dictEnd - dictPtr) + 128 KB; /* The maximum offset that must be supported */
-        /* Calculate minimum offset code required to represent maxOffset */
-        unsigned const offcodeMax = ZSTD_highbit32(maxOffset);
+    {   U32 offcodeMax = MaxOff;
+        if ((size_t)(dictEnd - dictPtr) <= ((U32)-1) - 128 KB) {
+            U32 const maxOffset = (U32)(dictEnd - dictPtr) + 128 KB; /* The maximum offset that must be supported */
+            /* Calculate minimum offset code required to represent maxOffset */
+            offcodeMax = ZSTD_highbit32(maxOffset);
+        }
         /* Every possible supported offset <= dictContentSize + 128 KB must be representable */
         CHECK_F (ZSTD_checkDictNCount(offcodeNCount, offcodeMaxValue, MIN(offcodeMax, MaxOff)));
     }

From 37d130031d63a21d99344fad22bbc417e255537e Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Mon, 24 Oct 2016 17:22:12 -0700
Subject: [PATCH 63/92] updated comments on context re-use

---
 NEWS       |  2 ++
 lib/zstd.h | 29 +++++++++++++++++++----------
 2 files changed, 21 insertions(+), 10 deletions(-)

diff --git a/NEWS b/NEWS
index 570d66799..27bf47a92 100644
--- a/NEWS
+++ b/NEWS
@@ -1,6 +1,8 @@
 v1.1.1
 New : command -M#, --memory=, --memlimit=, --memlimit-decompress= to limit allowed memory consumption
+New : doc/zstd_manual.html, by Przemyslaw Skibinski
 Changed : zstd_errors.h is now part of include installation
+Fixed : zstd-small, zstd-compress and zstd-decompress compilation targets
 
 v1.1.0
 New : contrib/pzstd, parallel version of zstd, by Nick Terrell
diff --git a/lib/zstd.h b/lib/zstd.h
index 1b6665c2b..20c8c7658 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -33,20 +33,20 @@ extern "C" {
 /*******************************************************************************************************
   Introduction
 
-  Zstd, short for Zstandard, is a fast lossless compression algorithm, targeting real-time compression scenarios
+  zstd, short for Zstandard, is a fast lossless compression algorithm, targeting real-time compression scenarios
   at zlib-level and better compression ratios. The zstd compression library provides in-memory compression and
   decompression functions. The library supports compression levels from 1 up to ZSTD_maxCLevel() which is 22.
-  Levels from 20 to 22 should be used with caution as they require about 300-1300 MB for compression.
+  Levels >= 20, labelled `--ultra`, should be used with caution, as they require more memory.
   Compression can be done in:
     - a single step (described as Simple API)
     - a single step, reusing a context (described as Explicit memory management)
-    - repeated calls of the compression function (described as Streaming compression)
+    - unbounded multiple steps (described as Streaming compression)
   The compression ratio achievable on small data can be highly improved using compression with a dictionary in:
     - a single step (described as Simple dictionary API)
     - a single step, reusing a dictionary (described as Fast dictionary API)
 
-  Advanced and experimantal functions can be accessed using #define ZSTD_STATIC_LINKING_ONLY before including zstd.h.
-  These APIs shall never be used with a dynamic library. 
+  Advanced experimental functions can be accessed using #define ZSTD_STATIC_LINKING_ONLY before including zstd.h.
+  These APIs shall never be used with a dynamic library.
   They are not "stable", their definition may change in the future. Only static linking is allowed.
 *********************************************************************************************************/
 
@@ -115,7 +115,11 @@ ZSTDLIB_API const char* ZSTD_getErrorName(size_t code);     /*!< provides readab
 /***************************************
 *  Explicit memory management
 ***************************************/
-/*= Compression context */
+/*= Compression context
+*   When compressing many messages / blocks,
+*   it is recommended to allocate a context just once, and re-use it for each successive compression operation.
+*   This will make the situation much easier for the system's memory.
+*   Use one context per thread for parallel execution in multi-threaded environments. */
 typedef struct ZSTD_CCtx_s ZSTD_CCtx;
 ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx(void);
 ZSTDLIB_API size_t     ZSTD_freeCCtx(ZSTD_CCtx* cctx);
@@ -162,12 +166,14 @@ ZSTDLIB_API size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx,
 typedef struct ZSTD_CDict_s ZSTD_CDict;
 
 /*! ZSTD_createCDict() :
-*   Create a digested dictionary, ready to start compression operation without startup delay.
+*   When compressing multiple messages / blocks with the same dictionary, it's recommended to load it just once.
+*   ZSTD_createCDict() will create a digested dictionary, ready to start future compression operations without startup delay.
+*   ZSTD_CDict can be created once and used by multiple threads concurrently, as its usage is read-only.
 *   `dict` can be released after ZSTD_CDict creation */
 ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict(const void* dict, size_t dictSize, int compressionLevel);
 
 /*! ZSTD_freeCDict() :
-*   Function frees memory allocated with ZSTD_createCDict() */
+*   Function frees memory allocated by ZSTD_createCDict() */
 ZSTDLIB_API size_t      ZSTD_freeCDict(ZSTD_CDict* CDict);
 
 /*! ZSTD_compress_usingCDict() :
@@ -224,8 +230,11 @@ typedef struct ZSTD_outBuffer_s {
 *  A ZSTD_CStream object is required to track streaming operation.
 *  Use ZSTD_createCStream() and ZSTD_freeCStream() to create/release resources.
 *  ZSTD_CStream objects can be reused multiple times on consecutive compression operations.
+*  It is recommended to re-use ZSTD_CStream in situations where many streaming operations will be achieved consecutively,
+*  since it will play nicer with system's memory, by re-using already allocated memory.
+*  Use one separate ZSTD_CStream per thread for parallel execution.
 *
-*  Start by initializing ZSTD_CStream.
+*  Start a new compression by initializing ZSTD_CStream.
 *  Use ZSTD_initCStream() to start a new compression operation.
 *  Use ZSTD_initCStream_usingDict() for a compression which requires a dictionary.
 *
@@ -468,7 +477,7 @@ ZSTDLIB_API size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds);
 *
 *  This is an advanced API, giving full control over buffer management, for users which need direct control over memory.
 *  But it's also a complex one, with many restrictions (documented below).
-*  Prefer using normal streaming API for an easier experience 
+*  Prefer using normal streaming API for an easier experience
 ********************************************************************* */
 
 /**

From 5fc901d1cef634941287d291812fc06cec579f66 Mon Sep 17 00:00:00 2001
From: Przemyslaw Skibinski 
Date: Tue, 25 Oct 2016 10:05:20 +0200
Subject: [PATCH 64/92] images/ moved to doc/images/

---
 README.md                            |   6 +++---
 {images => doc/images}/Cspeed4.png   | Bin
 {images => doc/images}/DCspeed5.png  | Bin
 {images => doc/images}/Dspeed4.png   | Bin
 {images => doc/images}/smallData.png | Bin
 5 files changed, 3 insertions(+), 3 deletions(-)
 rename {images => doc/images}/Cspeed4.png (100%)
 rename {images => doc/images}/DCspeed5.png (100%)
 rename {images => doc/images}/Dspeed4.png (100%)
 rename {images => doc/images}/smallData.png (100%)

diff --git a/README.md b/README.md
index 53609a146..0ab1b667b 100644
--- a/README.md
+++ b/README.md
@@ -38,10 +38,10 @@ The following tests were run on a Core i7-3930K CPU @ 4.5GHz, using [lzbench], a
 
 Compression Speed vs Ratio | Decompression Speed
 ---------------------------|--------------------
-![Compression Speed vs Ratio](images/Cspeed4.png "Compression Speed vs Ratio") | ![Decompression Speed](images/Dspeed4.png "Decompression Speed")
+![Compression Speed vs Ratio](doc/images/Cspeed4.png "Compression Speed vs Ratio") | ![Decompression Speed](doc/images/Dspeed4.png "Decompression Speed")
 
 Several algorithms can produce higher compression ratios, but at slower speeds, falling outside of the graph.
-For a larger picture including very slow modes, [click on this link](images/DCspeed5.png) .
+For a larger picture including very slow modes, [click on this link](doc/images/DCspeed5.png) .
 
 
 ### The case for Small Data compression
@@ -52,7 +52,7 @@ This problem is common to many compression algorithms. The reason is, compressio
 
 To solve this situation, Zstd offers a __training mode__, which can be used to tune the algorithm for a selected type of data, by providing it with a few samples. The result of the training is stored in a file called "dictionary", which can be loaded before compression and decompression. Using this dictionary, the compression ratio achievable on small data improves dramatically:
 
-![Compressing Small Data](images/smallData.png "Compressing Small Data")
+![Compressing Small Data](doc/images/smallData.png "Compressing Small Data")
 
 These compression gains are achieved while simultaneously providing faster compression and decompression speeds.
 
diff --git a/images/Cspeed4.png b/doc/images/Cspeed4.png
similarity index 100%
rename from images/Cspeed4.png
rename to doc/images/Cspeed4.png
diff --git a/images/DCspeed5.png b/doc/images/DCspeed5.png
similarity index 100%
rename from images/DCspeed5.png
rename to doc/images/DCspeed5.png
diff --git a/images/Dspeed4.png b/doc/images/Dspeed4.png
similarity index 100%
rename from images/Dspeed4.png
rename to doc/images/Dspeed4.png
diff --git a/images/smallData.png b/doc/images/smallData.png
similarity index 100%
rename from images/smallData.png
rename to doc/images/smallData.png

From 347903cfb5a476c63a3edb6d52813030ecf471d2 Mon Sep 17 00:00:00 2001
From: Przemyslaw Skibinski 
Date: Tue, 25 Oct 2016 10:34:41 +0200
Subject: [PATCH 65/92] Windows resources moved to programs/windres/

---
 build/VS2010/zstd/generate_res/zstd32.res | Bin 948 -> 0 bytes
 build/VS2010/zstd/generate_res/zstd64.res | Bin 948 -> 0 bytes
 build/VS2010/zstd/zstd.rc                 |   6 +++---
 programs/Makefile                         |   4 ++--
 programs/windres/zstd32.res               | Bin 0 -> 1044 bytes
 programs/windres/zstd64.res               | Bin 0 -> 1044 bytes
 6 files changed, 5 insertions(+), 5 deletions(-)
 delete mode 100644 build/VS2010/zstd/generate_res/zstd32.res
 delete mode 100644 build/VS2010/zstd/generate_res/zstd64.res
 create mode 100644 programs/windres/zstd32.res
 create mode 100644 programs/windres/zstd64.res

diff --git a/build/VS2010/zstd/generate_res/zstd32.res b/build/VS2010/zstd/generate_res/zstd32.res
deleted file mode 100644
index 75705d06802ba59bc017e31c22f8c6169ec8a645..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 948
zcmeZaWMlw=dCUw95EcugUQuyTGDr}LBbXT&Y#10ArZ7VUK1Th3Ncry4i
z_%XyYcry4gxH0&HmF;DD{}1G9hz$@HNWvaOFfcH%Ft9K%LD?`WfeEB8n4yHBh#`|9
zk0G7GjUkgEhar{0lOc~GjUk_bfx!l>#(=?u!GOV%0Yp177%(s}STOJ~FfuqZ4+84S5#Gj$l)kn9d)NM$Hufaw7#Fl5kU0AY~-j2Ji>7{TFC0#=vDkjRjO
zrxo>7*ZK3pnlF{P-S3b@L@=0NC&G1g-!uOCD;||3>gd{|6__NFlaD1GiWj>
zFc>iyFc>lzGw4EDW(+zE3JkbHl7Ybr>_&g6`JfmCxe*qJAQwYo1Djh-7`PZ183GuJ
z81flX7)lwE8A_0R$G{K;_8%x-Qy3B%iWpKD7#Iw|;fJaRJrqIV2g0B*bYTFcuQ0HF
xn5z&ei-AFffrWvAA%vj_9LL2BISh#?X@HYu0}CimfZ~Ro0XZjvavd{RHvlVcd5{1A

diff --git a/build/VS2010/zstd/generate_res/zstd64.res b/build/VS2010/zstd/generate_res/zstd64.res
deleted file mode 100644
index 85de277c7f3145bcbcd9df4fbcbccd766163ccdb..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 948
zcmYdkV`Kn1Th3Ncry4i
z_%XyYcry4gxH0&HmF;DD{}1G9hz$@HNWvaOFfcH%Ft9K%LD?`WfeEB8n4yHBh#`|9
zk0G7GjUkgEhar{0lOc~GjUk_bfx!l>#(=?u!GOV%0Yp177%(s}STOJ~FfuqZ4+84S5#Gj$l)kn9d)NM$Hufaw7#Fl5kU0AY~-j2Ji>7{TFC0#=vDkjRjO
zrxo>7*ZK3pnlF{P-S3b@L@=0NC&G1g-!uOCD;||3>gd{|6__NFlaD1GiWj>
zFc>iyFc>lzGw4EDW(+zE3JkbHl7Ybr>_&g6`JfmCxe*qJAQwYo1Djh-7`PZ183GuJ
z81flX7)lwE8A_0R$G{K;_8%x-Qy3B%iWpKD7#Iw|;fJaRJrqIV2g0B*bYTFcuQ0HF
wn5z&ei-AFffrWvAA%vj_9LL2BISh#?X@HYu0}CimfZ~Q3B`1P%9Wz)r0BmY{C;$Ke

diff --git a/build/VS2010/zstd/zstd.rc b/build/VS2010/zstd/zstd.rc
index 464b2f143..f5e404730 100644
--- a/build/VS2010/zstd/zstd.rc
+++ b/build/VS2010/zstd/zstd.rc
@@ -32,11 +32,11 @@ BEGIN
     BEGIN
         BLOCK "040904B0"
         BEGIN
-            VALUE "CompanyName", "Yann Collet"
-            VALUE "FileDescription", "Fast and efficient compression algorithm"
+            VALUE "CompanyName", "Yann Collet, Facebook, Inc."
+            VALUE "FileDescription", "Zstandard - Fast and efficient compression algorithm"
             VALUE "FileVersion", ZSTD_VERSION_STRING
             VALUE "InternalName", "zstd.exe"
-            VALUE "LegalCopyright", "Copyright (C) 2013-2016, Yann Collet"
+            VALUE "LegalCopyright", "Copyright (c) 2013-present, Yann Collet, Facebook, Inc."
             VALUE "OriginalFilename", "zstd.exe"
             VALUE "ProductName", "Zstandard"
             VALUE "ProductVersion", ZSTD_VERSION_STRING
diff --git a/programs/Makefile b/programs/Makefile
index 0ed7f3d07..013f54354 100644
--- a/programs/Makefile
+++ b/programs/Makefile
@@ -57,8 +57,8 @@ endif
 ifneq (,$(filter Windows%,$(OS)))
 EXT =.exe
 VOID = nul
-RES64_FILE = ..\build\VS2010\zstd\generate_res\zstd64.res
-RES32_FILE = ..\build\VS2010\zstd\generate_res\zstd32.res
+RES64_FILE = windres\zstd64.res
+RES32_FILE = windres\zstd32.res
 ifneq (,$(filter x86_64%,$(shell $(CC) -dumpmachine)))
     RES_FILE = $(RES64_FILE)
 else
diff --git a/programs/windres/zstd32.res b/programs/windres/zstd32.res
new file mode 100644
index 0000000000000000000000000000000000000000..aec8fcf24e4f15e1915af725ba5b957fa3e7b50a
GIT binary patch
literal 1044
zcmeZaWMlw=f6NRF5EcugUQuyTGDr}LcQ7+B*f20Kd|`$NfYdoK7#x7`U?d}h00RSq
z1Or0@OawwRg3UF+BF@Rez~I5az!1T}zz_p98>Aa>!5^fIVK2-3e;`*wYygoUmoYNfgJl^QSQuCsn4oMJwTB6$BbcFt
zp@<=qA&()Q!Hpr4A%`KA!IL46A&nuQfq`KLSd9UL34;NHB?E|dVlV)^Q-*<&!I>eS
zA(x?mA(0`Ep_0LmA(0`MAr)+VBv>quK>@5L2W)BygARiNgBwF4Lo!1uLlQ$iLq0<`
zSPW!eGJ_ri1H&W+QwB!3J6sr28HyQ_!R{+yC}GHC$Y;m{n;*qc%uoWhBZVQ6p@<=c
zL4iRRYC1?p0V=1!kjjw8kj9Y7kj#+Dkjjw9P{N?VkPLNq5!n1
z7%~`g!FK5|upzlAj3JewhykVtq`;6tj{$-i7>pP=85qIgR{~a-$B@X7gA_7VV0Wi5
z=rN=+R6xUO3WEg$BZCh^DnmM0H7HaI7%IW8NN31kC}Cg#ha#$&0)qxaGJ__A0)r8Q
z0fQlfF@r8c0RzNG5PyO~o7{M?0{g=s>IP8i0Qm!!K0v;L#56Ylm@sfLFfs%%6fxv8
zq%f2+Br}vCg#ZHsX529_7=XhVRS$YdL&6vo)(i|T44||d29A4}t3as;l4=Aa>!5^fIVK2-3e;`*wYygoUmoYNfgJl^QSQuCsn4oMJwTB6$BbcFt
zp@<=qA&()Q!Hpr4A%`KA!IL46A&nuQfq`KLSd9UL34;NHB?E|dVlV)^Q-*<&!I>eS
zA(x?mA(0`Ep_0LmA(0`MAr)+VBv>quK>@5L2W)BygARiNgBwF4Lo!1uLlQ$iLq0<`
zSPW!eGJ_ri1H&W+QwB!3J6sr28HyQ_!R{+yC}GHC$Y;m{n;*qc%uoWhBZVQ6p@<=c
zL4iRRYC1?p0V=1!kjjw8kj9Y7kj#+Dkjjw9P{N?VkPLNq5!n1
z7%~`g!FK5|upzlAj3JewhykVtq`;6tj{$-i7>pP=85qIgR{~a-$B@X7gA_7VV0Wi5
z=rN=+R6xUO3WEg$BZCh^DnmM0H7HaI7%IW8NN31kC}Cg#ha#$&0)qxaGJ__A0)r8Q
z0fQlfF@r8c0RzNG5PyO~o7{M?0{g=s>IP8i0Qm!!K0v;L#56Ylm@sfLFfs%%6fxv8
zq%f2+Br}vCg#ZHsX529_7=XhVRS$YdL&6vo)(i|T44||d29A4}t3as;l4=
Date: Tue, 25 Oct 2016 12:25:07 +0200
Subject: [PATCH 66/92] added ZSTD_btopt2 strategy

---
 lib/compress/zstd_compress.c | 67 +++++++++++++++++++++++-------------
 lib/compress/zstd_opt.h      | 40 +++++++++++----------
 lib/zstd.h                   |  2 +-
 tests/paramgrill.c           |  7 ++--
 4 files changed, 70 insertions(+), 46 deletions(-)

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 21a1ffbe5..f444a7ea4 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -137,7 +137,7 @@ size_t ZSTD_checkCParams(ZSTD_compressionParameters cParams)
       U32 const searchLengthMax = (cParams.strategy == ZSTD_fast) ? ZSTD_SEARCHLENGTH_MAX : ZSTD_SEARCHLENGTH_MAX-1;
       CLAMPCHECK(cParams.searchLength, searchLengthMin, searchLengthMax); }
     CLAMPCHECK(cParams.targetLength, ZSTD_TARGETLENGTH_MIN, ZSTD_TARGETLENGTH_MAX);
-    if ((U32)(cParams.strategy) > (U32)ZSTD_btopt) return ERROR(compressionParameter_unsupported);
+    if ((U32)(cParams.strategy) > (U32)ZSTD_btopt2) return ERROR(compressionParameter_unsupported);
     return 0;
 }
 
@@ -160,7 +160,7 @@ ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, u
             if (cPar.windowLog > srcLog) cPar.windowLog = srcLog;
     }   }
     if (cPar.hashLog > cPar.windowLog) cPar.hashLog = cPar.windowLog;
-    {   U32 const btPlus = (cPar.strategy == ZSTD_btlazy2) | (cPar.strategy == ZSTD_btopt);
+    {   U32 const btPlus = (cPar.strategy == ZSTD_btlazy2) | (cPar.strategy == ZSTD_btopt) | (cPar.strategy == ZSTD_btopt2);
         U32 const maxChainLog = cPar.windowLog+btPlus;
         if (cPar.chainLog > maxChainLog) cPar.chainLog = maxChainLog; }   /* <= ZSTD_CHAINLOG_MAX */
 
@@ -186,7 +186,7 @@ size_t ZSTD_estimateCCtxSize(ZSTD_compressionParameters cParams)
     size_t const optSpace = ((MaxML+1) + (MaxLL+1) + (MaxOff+1) + (1<workSpaceSize < neededSpace) {
                 ZSTD_free(zc->workSpace, zc->customMem);
                 zc->workSpace = ZSTD_malloc(neededSpace, zc->customMem);
@@ -276,7 +276,7 @@ static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc,
         zc->frameContentSize = frameContentSize;
         { int i; for (i=0; irep[i] = repStartValue[i]; }
 
-        if (params.cParams.strategy == ZSTD_btopt) {
+        if ((params.cParams.strategy == ZSTD_btopt) || (params.cParams.strategy == ZSTD_btopt2)) {
             zc->seqStore.litFreq = (U32*)ptr;
             zc->seqStore.litLengthFreq = zc->seqStore.litFreq + (1<seqStore.matchLengthFreq = zc->seqStore.litLengthFreq + (MaxLL+1);
@@ -2170,7 +2170,17 @@ static void ZSTD_compressBlock_btlazy2_extDict(ZSTD_CCtx* ctx, const void* src,
 static void ZSTD_compressBlock_btopt(ZSTD_CCtx* ctx, const void* src, size_t srcSize)
 {
 #ifdef ZSTD_OPT_H_91842398743
-    ZSTD_compressBlock_opt_generic(ctx, src, srcSize);
+    ZSTD_compressBlock_opt_generic(ctx, src, srcSize, 0);
+#else
+    (void)ctx; (void)src; (void)srcSize;
+    return;
+#endif
+}
+
+static void ZSTD_compressBlock_btopt2(ZSTD_CCtx* ctx, const void* src, size_t srcSize)
+{
+#ifdef ZSTD_OPT_H_91842398743
+    ZSTD_compressBlock_opt_generic(ctx, src, srcSize, 1);
 #else
     (void)ctx; (void)src; (void)srcSize;
     return;
@@ -2180,7 +2190,17 @@ static void ZSTD_compressBlock_btopt(ZSTD_CCtx* ctx, const void* src, size_t src
 static void ZSTD_compressBlock_btopt_extDict(ZSTD_CCtx* ctx, const void* src, size_t srcSize)
 {
 #ifdef ZSTD_OPT_H_91842398743
-    ZSTD_compressBlock_opt_extDict_generic(ctx, src, srcSize);
+    ZSTD_compressBlock_opt_extDict_generic(ctx, src, srcSize, 0);
+#else
+    (void)ctx; (void)src; (void)srcSize;
+    return;
+#endif
+}
+
+static void ZSTD_compressBlock_btopt2_extDict(ZSTD_CCtx* ctx, const void* src, size_t srcSize)
+{
+#ifdef ZSTD_OPT_H_91842398743
+    ZSTD_compressBlock_opt_extDict_generic(ctx, src, srcSize, 1);
 #else
     (void)ctx; (void)src; (void)srcSize;
     return;
@@ -2192,9 +2212,9 @@ typedef void (*ZSTD_blockCompressor) (ZSTD_CCtx* ctx, const void* src, size_t sr
 
 static ZSTD_blockCompressor ZSTD_selectBlockCompressor(ZSTD_strategy strat, int extDict)
 {
-    static const ZSTD_blockCompressor blockCompressor[2][7] = {
-        { ZSTD_compressBlock_fast, ZSTD_compressBlock_doubleFast, ZSTD_compressBlock_greedy, ZSTD_compressBlock_lazy, ZSTD_compressBlock_lazy2, ZSTD_compressBlock_btlazy2, ZSTD_compressBlock_btopt },
-        { ZSTD_compressBlock_fast_extDict, ZSTD_compressBlock_doubleFast_extDict, ZSTD_compressBlock_greedy_extDict, ZSTD_compressBlock_lazy_extDict,ZSTD_compressBlock_lazy2_extDict, ZSTD_compressBlock_btlazy2_extDict, ZSTD_compressBlock_btopt_extDict }
+    static const ZSTD_blockCompressor blockCompressor[2][8] = {
+        { ZSTD_compressBlock_fast, ZSTD_compressBlock_doubleFast, ZSTD_compressBlock_greedy, ZSTD_compressBlock_lazy, ZSTD_compressBlock_lazy2, ZSTD_compressBlock_btlazy2, ZSTD_compressBlock_btopt, ZSTD_compressBlock_btopt2 },
+        { ZSTD_compressBlock_fast_extDict, ZSTD_compressBlock_doubleFast_extDict, ZSTD_compressBlock_greedy_extDict, ZSTD_compressBlock_lazy_extDict,ZSTD_compressBlock_lazy2_extDict, ZSTD_compressBlock_btlazy2_extDict, ZSTD_compressBlock_btopt_extDict, ZSTD_compressBlock_btopt2_extDict }
     };
 
     return blockCompressor[extDict][(U32)strat];
@@ -2247,7 +2267,7 @@ static size_t ZSTD_compress_generic (ZSTD_CCtx* cctx,
 
         /* preemptive overflow correction */
         if (cctx->lowLimit > (1<<30)) {
-            U32 const btplus = (cctx->params.cParams.strategy == ZSTD_btlazy2) | (cctx->params.cParams.strategy == ZSTD_btopt);
+            U32 const btplus = (cctx->params.cParams.strategy == ZSTD_btlazy2) | (cctx->params.cParams.strategy == ZSTD_btopt) | (cctx->params.cParams.strategy == ZSTD_btopt2);
             U32 const chainMask = (1 << (cctx->params.cParams.chainLog - btplus)) - 1;
             U32 const supLog = MAX(cctx->params.cParams.chainLog, 17 /* blockSize */);
             U32 const newLowLimit = (cctx->lowLimit & chainMask) + (1 << supLog);   /* preserve position % chainSize, ensure current-repcode doesn't underflow */
@@ -2436,6 +2456,7 @@ static size_t ZSTD_loadDictionaryContent(ZSTD_CCtx* zc, const void* src, size_t
 
     case ZSTD_btlazy2:
     case ZSTD_btopt:
+    case ZSTD_btopt2:
         ZSTD_updateTree(zc, iend-HASH_READ_SIZE, iend, 1 << zc->params.cParams.searchLog, zc->params.cParams.searchLength);
         break;
 
@@ -3066,9 +3087,9 @@ static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEV
     { 23, 21, 22,  4,  5, 24, ZSTD_btopt   },  /* level 17 */
     { 23, 23, 22,  6,  5, 32, ZSTD_btopt   },  /* level 18 */
     { 23, 23, 22,  6,  3, 48, ZSTD_btopt   },  /* level 19 */
-    { 25, 25, 23,  7,  3, 64, ZSTD_btopt   },  /* level 20 */
-    { 26, 26, 23,  7,  3,256, ZSTD_btopt   },  /* level 21 */
-    { 27, 27, 25,  9,  3,512, ZSTD_btopt   },  /* level 22 */
+    { 25, 25, 23,  7,  3, 64, ZSTD_btopt2  },  /* level 20 */
+    { 26, 26, 23,  7,  3,256, ZSTD_btopt2  },  /* level 21 */
+    { 27, 27, 25,  9,  3,512, ZSTD_btopt2  },  /* level 22 */
 },
 {   /* for srcSize <= 256 KB */
     /* W,  C,  H,  S,  L,  T, strat */
@@ -3092,9 +3113,9 @@ static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEV
     { 18, 19, 18,  8,  3, 64, ZSTD_btopt   },  /* level 17.*/
     { 18, 19, 18,  9,  3,128, ZSTD_btopt   },  /* level 18.*/
     { 18, 19, 18, 10,  3,256, ZSTD_btopt   },  /* level 19.*/
-    { 18, 19, 18, 11,  3,512, ZSTD_btopt   },  /* level 20.*/
-    { 18, 19, 18, 12,  3,512, ZSTD_btopt   },  /* level 21.*/
-    { 18, 19, 18, 13,  3,512, ZSTD_btopt   },  /* level 22.*/
+    { 18, 19, 18, 11,  3,512, ZSTD_btopt2  },  /* level 20.*/
+    { 18, 19, 18, 12,  3,512, ZSTD_btopt2  },  /* level 21.*/
+    { 18, 19, 18, 13,  3,512, ZSTD_btopt2  },  /* level 22.*/
 },
 {   /* for srcSize <= 128 KB */
     /* W,  C,  H,  S,  L,  T, strat */
@@ -3118,9 +3139,9 @@ static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEV
     { 17, 18, 17,  7,  3, 64, ZSTD_btopt   },  /* level 17.*/
     { 17, 18, 17,  7,  3,256, ZSTD_btopt   },  /* level 18.*/
     { 17, 18, 17,  8,  3,256, ZSTD_btopt   },  /* level 19.*/
-    { 17, 18, 17,  9,  3,256, ZSTD_btopt   },  /* level 20.*/
-    { 17, 18, 17, 10,  3,256, ZSTD_btopt   },  /* level 21.*/
-    { 17, 18, 17, 11,  3,512, ZSTD_btopt   },  /* level 22.*/
+    { 17, 18, 17,  9,  3,256, ZSTD_btopt2  },  /* level 20.*/
+    { 17, 18, 17, 10,  3,256, ZSTD_btopt2  },  /* level 21.*/
+    { 17, 18, 17, 11,  3,512, ZSTD_btopt2  },  /* level 22.*/
 },
 {   /* for srcSize <= 16 KB */
     /* W,  C,  H,  S,  L,  T, strat */
@@ -3144,9 +3165,9 @@ static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEV
     { 14, 15, 15,  6,  3,128, ZSTD_btopt   },  /* level 17.*/
     { 14, 15, 15,  6,  3,256, ZSTD_btopt   },  /* level 18.*/
     { 14, 15, 15,  7,  3,256, ZSTD_btopt   },  /* level 19.*/
-    { 14, 15, 15,  8,  3,256, ZSTD_btopt   },  /* level 20.*/
-    { 14, 15, 15,  9,  3,256, ZSTD_btopt   },  /* level 21.*/
-    { 14, 15, 15, 10,  3,256, ZSTD_btopt   },  /* level 22.*/
+    { 14, 15, 15,  8,  3,256, ZSTD_btopt2  },  /* level 20.*/
+    { 14, 15, 15,  9,  3,256, ZSTD_btopt2  },  /* level 21.*/
+    { 14, 15, 15, 10,  3,256, ZSTD_btopt2  },  /* level 22.*/
 },
 };
 
diff --git a/lib/compress/zstd_opt.h b/lib/compress/zstd_opt.h
index 5e4cb7c27..90d511c7e 100644
--- a/lib/compress/zstd_opt.h
+++ b/lib/compress/zstd_opt.h
@@ -121,12 +121,14 @@ FORCE_INLINE U32 ZSTD_getLiteralPrice(seqStore_t* ssPtr, U32 litLength, const BY
 }
 
 
-FORCE_INLINE U32 ZSTD_getPrice(seqStore_t* seqStorePtr, U32 litLength, const BYTE* literals, U32 offset, U32 matchLength)
+FORCE_INLINE U32 ZSTD_getPrice(seqStore_t* seqStorePtr, U32 litLength, const BYTE* literals, U32 offset, U32 matchLength, const int ultra)
 {
     /* offset */
     BYTE const offCode = (BYTE)ZSTD_highbit32(offset+1);
     U32 price = offCode + seqStorePtr->log2offCodeSum - ZSTD_highbit32(seqStorePtr->offCodeFreq[offCode]+1);
 
+    if (!ultra && offCode >= 20) price += (offCode-19)*2;
+
     /* match Length */
     {   const BYTE ML_deltaCode = 36;
         const BYTE mlCode = (matchLength>127) ? (BYTE)ZSTD_highbit32(matchLength) + ML_deltaCode : ML_Code[matchLength];
@@ -376,7 +378,7 @@ static U32 ZSTD_BtGetAllMatches_selectMLS_extDict (
 *********************************/
 FORCE_INLINE
 void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx,
-                                    const void* src, size_t srcSize)
+                                    const void* src, size_t srcSize, const int ultra)
 {
     seqStore_t* seqStorePtr = &(ctx->seqStore);
     const BYTE* const istart = (const BYTE*)src;
@@ -424,7 +426,7 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx,
                     }
                     best_off = i - (ip == anchor);
                     do {
-                        price = ZSTD_getPrice(seqStorePtr, litlen, anchor, best_off, mlen - MINMATCH);
+                        price = ZSTD_getPrice(seqStorePtr, litlen, anchor, best_off, mlen - MINMATCH, ultra);
                         if (mlen > last_pos || price < opt[mlen].price)
                             SET_PRICE(mlen, mlen, i, litlen, price);   /* note : macro modifies last_pos */
                         mlen--;
@@ -449,7 +451,7 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx,
             mlen = (u>0) ? matches[u-1].len+1 : best_mlen;
             best_mlen = matches[u].len;
             while (mlen <= best_mlen) {
-                price = ZSTD_getPrice(seqStorePtr, litlen, anchor, matches[u].off-1, mlen - MINMATCH);
+                price = ZSTD_getPrice(seqStorePtr, litlen, anchor, matches[u].off-1, mlen - MINMATCH, ultra);
                 if (mlen > last_pos || price < opt[mlen].price)
                     SET_PRICE(mlen, mlen, matches[u].off, litlen, price);   /* note : macro modifies last_pos */
                 mlen++;
@@ -516,12 +518,12 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx,
                            if (opt[cur].mlen == 1) {
                                 litlen = opt[cur].litlen;
                                 if (cur > litlen) {
-                                    price = opt[cur - litlen].price + ZSTD_getPrice(seqStorePtr, litlen, inr-litlen, best_off, mlen - MINMATCH);
+                                    price = opt[cur - litlen].price + ZSTD_getPrice(seqStorePtr, litlen, inr-litlen, best_off, mlen - MINMATCH, ultra);
                                 } else
-                                    price = ZSTD_getPrice(seqStorePtr, litlen, anchor, best_off, mlen - MINMATCH);
+                                    price = ZSTD_getPrice(seqStorePtr, litlen, anchor, best_off, mlen - MINMATCH, ultra);
                             } else {
                                 litlen = 0;
-                                price = opt[cur].price + ZSTD_getPrice(seqStorePtr, 0, NULL, best_off, mlen - MINMATCH);
+                                price = opt[cur].price + ZSTD_getPrice(seqStorePtr, 0, NULL, best_off, mlen - MINMATCH, ultra);
                             }
 
                             if (cur + mlen > last_pos || price <= opt[cur + mlen].price)
@@ -548,12 +550,12 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx,
                     if (opt[cur].mlen == 1) {
                         litlen = opt[cur].litlen;
                         if (cur > litlen)
-                            price = opt[cur - litlen].price + ZSTD_getPrice(seqStorePtr, litlen, ip+cur-litlen, matches[u].off-1, mlen - MINMATCH);
+                            price = opt[cur - litlen].price + ZSTD_getPrice(seqStorePtr, litlen, ip+cur-litlen, matches[u].off-1, mlen - MINMATCH, ultra);
                         else
-                            price = ZSTD_getPrice(seqStorePtr, litlen, anchor, matches[u].off-1, mlen - MINMATCH);
+                            price = ZSTD_getPrice(seqStorePtr, litlen, anchor, matches[u].off-1, mlen - MINMATCH, ultra);
                     } else {
                         litlen = 0;
-                        price = opt[cur].price + ZSTD_getPrice(seqStorePtr, 0, NULL, matches[u].off-1, mlen - MINMATCH);
+                        price = opt[cur].price + ZSTD_getPrice(seqStorePtr, 0, NULL, matches[u].off-1, mlen - MINMATCH, ultra);
                     }
 
                     if (cur + mlen > last_pos || (price < opt[cur + mlen].price))
@@ -625,7 +627,7 @@ _storeSequence:   /* cur, last_pos, best_mlen, best_off have to be set */
 
 FORCE_INLINE
 void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx,
-                                     const void* src, size_t srcSize)
+                                     const void* src, size_t srcSize, const int ultra)
 {
     seqStore_t* seqStorePtr = &(ctx->seqStore);
     const BYTE* const istart = (const BYTE*)src;
@@ -688,7 +690,7 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx,
                     best_off = i - (ip==anchor);
                     litlen = opt[0].litlen;
                     do {
-                        price = ZSTD_getPrice(seqStorePtr, litlen, anchor, best_off, mlen - MINMATCH);
+                        price = ZSTD_getPrice(seqStorePtr, litlen, anchor, best_off, mlen - MINMATCH, ultra);
                         if (mlen > last_pos || price < opt[mlen].price)
                             SET_PRICE(mlen, mlen, i, litlen, price);   /* note : macro modifies last_pos */
                         mlen--;
@@ -718,7 +720,7 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx,
             best_mlen = matches[u].len;
             litlen = opt[0].litlen;
             while (mlen <= best_mlen) {
-                price = ZSTD_getPrice(seqStorePtr, litlen, anchor, matches[u].off-1, mlen - MINMATCH);
+                price = ZSTD_getPrice(seqStorePtr, litlen, anchor, matches[u].off-1, mlen - MINMATCH, ultra);
                 if (mlen > last_pos || price < opt[mlen].price)
                     SET_PRICE(mlen, mlen, matches[u].off, litlen, price);
                 mlen++;
@@ -788,12 +790,12 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx,
                             if (opt[cur].mlen == 1) {
                                 litlen = opt[cur].litlen;
                                 if (cur > litlen) {
-                                    price = opt[cur - litlen].price + ZSTD_getPrice(seqStorePtr, litlen, inr-litlen, best_off, mlen - MINMATCH);
+                                    price = opt[cur - litlen].price + ZSTD_getPrice(seqStorePtr, litlen, inr-litlen, best_off, mlen - MINMATCH, ultra);
                                 } else
-                                    price = ZSTD_getPrice(seqStorePtr, litlen, anchor, best_off, mlen - MINMATCH);
+                                    price = ZSTD_getPrice(seqStorePtr, litlen, anchor, best_off, mlen - MINMATCH, ultra);
                             } else {
                                 litlen = 0;
-                                price = opt[cur].price + ZSTD_getPrice(seqStorePtr, 0, NULL, best_off, mlen - MINMATCH);
+                                price = opt[cur].price + ZSTD_getPrice(seqStorePtr, 0, NULL, best_off, mlen - MINMATCH, ultra);
                             }
 
                             if (cur + mlen > last_pos || price <= opt[cur + mlen].price)
@@ -820,12 +822,12 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx,
                     if (opt[cur].mlen == 1) {
                         litlen = opt[cur].litlen;
                         if (cur > litlen)
-                            price = opt[cur - litlen].price + ZSTD_getPrice(seqStorePtr, litlen, ip+cur-litlen, matches[u].off-1, mlen - MINMATCH);
+                            price = opt[cur - litlen].price + ZSTD_getPrice(seqStorePtr, litlen, ip+cur-litlen, matches[u].off-1, mlen - MINMATCH, ultra);
                         else
-                            price = ZSTD_getPrice(seqStorePtr, litlen, anchor, matches[u].off-1, mlen - MINMATCH);
+                            price = ZSTD_getPrice(seqStorePtr, litlen, anchor, matches[u].off-1, mlen - MINMATCH, ultra);
                     } else {
                         litlen = 0;
-                        price = opt[cur].price + ZSTD_getPrice(seqStorePtr, 0, NULL, matches[u].off-1, mlen - MINMATCH);
+                        price = opt[cur].price + ZSTD_getPrice(seqStorePtr, 0, NULL, matches[u].off-1, mlen - MINMATCH, ultra);
                     }
 
                     if (cur + mlen > last_pos || (price < opt[cur + mlen].price))
diff --git a/lib/zstd.h b/lib/zstd.h
index 1b6665c2b..471cd7436 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -340,7 +340,7 @@ static const size_t ZSTD_skippableHeaderSize = 8;  /* magic number + skippable f
 
 
 /*--- Advanced types ---*/
-typedef enum { ZSTD_fast, ZSTD_dfast, ZSTD_greedy, ZSTD_lazy, ZSTD_lazy2, ZSTD_btlazy2, ZSTD_btopt } ZSTD_strategy;   /* from faster to stronger */
+typedef enum { ZSTD_fast, ZSTD_dfast, ZSTD_greedy, ZSTD_lazy, ZSTD_lazy2, ZSTD_btlazy2, ZSTD_btopt, ZSTD_btopt2 } ZSTD_strategy;   /* from faster to stronger */
 
 typedef struct {
     unsigned windowLog;      /**< largest match distance : larger == more compression, more memory needed during decompression */
diff --git a/tests/paramgrill.c b/tests/paramgrill.c
index f253fa39c..5eabcba2b 100644
--- a/tests/paramgrill.c
+++ b/tests/paramgrill.c
@@ -302,7 +302,8 @@ const char* g_stratName[] = { "ZSTD_fast   ",
                               "ZSTD_lazy   ",
                               "ZSTD_lazy2  ",
                               "ZSTD_btlazy2",
-                              "ZSTD_btopt  " };
+                              "ZSTD_btopt  ",
+                              "ZSTD_btopt2 "};
 
 static void BMK_printWinner(FILE* f, U32 cLevel, BMK_result_t result, ZSTD_compressionParameters params, size_t srcSize)
 {
@@ -442,7 +443,7 @@ static ZSTD_compressionParameters* sanitizeParams(ZSTD_compressionParameters par
         g_params.chainLog = 0, g_params.searchLog = 0;
     if (params.strategy == ZSTD_dfast)
         g_params.searchLog = 0;
-    if (params.strategy != ZSTD_btopt )
+    if (params.strategy != ZSTD_btopt && params.strategy != ZSTD_btopt2)
         g_params.targetLength = 0;
     return &g_params;
 }
@@ -548,7 +549,7 @@ static ZSTD_compressionParameters randomParams(void)
         p.windowLog  = FUZ_rand(&g_rand) % (ZSTD_WINDOWLOG_MAX+1 - ZSTD_WINDOWLOG_MIN) + ZSTD_WINDOWLOG_MIN;
         p.searchLength=FUZ_rand(&g_rand) % (ZSTD_SEARCHLENGTH_MAX+1 - ZSTD_SEARCHLENGTH_MIN) + ZSTD_SEARCHLENGTH_MIN;
         p.targetLength=FUZ_rand(&g_rand) % (ZSTD_TARGETLENGTH_MAX+1 - ZSTD_TARGETLENGTH_MIN) + ZSTD_TARGETLENGTH_MIN;
-        p.strategy   = (ZSTD_strategy) (FUZ_rand(&g_rand) % (ZSTD_btopt +1));
+        p.strategy   = (ZSTD_strategy) (FUZ_rand(&g_rand) % (ZSTD_btopt2 +1));
         validated = !ZSTD_isError(ZSTD_checkCParams(p));
     }
     return p;

From 9516234e67dc71321b984bd49919887ccb5a990f Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Tue, 25 Oct 2016 16:19:52 -0700
Subject: [PATCH 67/92] first sketch for ZSTD_initCStream_usingCDict()

---
 NEWS                         |  1 +
 lib/compress/zstd_compress.c | 30 +++++++++++++++++++++++++-----
 lib/zstd.h                   |  2 ++
 tests/zstreamtest.c          | 26 ++++++++++++++++++++++++--
 4 files changed, 52 insertions(+), 7 deletions(-)

diff --git a/NEWS b/NEWS
index 27bf47a92..2e6118648 100644
--- a/NEWS
+++ b/NEWS
@@ -1,6 +1,7 @@
 v1.1.1
 New : command -M#, --memory=, --memlimit=, --memlimit-decompress= to limit allowed memory consumption
 New : doc/zstd_manual.html, by Przemyslaw Skibinski
+Improved : slightly better compression ratio at --ultra levels (>= 20)
 Changed : zstd_errors.h is now part of include installation
 Fixed : zstd-small, zstd-compress and zstd-decompress compilation targets
 
diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 3dbaff56b..5409c233a 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -122,6 +122,11 @@ const seqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx)   /* hidden interface *
     return &(ctx->seqStore);
 }
 
+static ZSTD_parameters ZSTD_getParamsFromCCtx(const ZSTD_CCtx* cctx)
+{
+    return cctx->params;
+}
+
 
 /** ZSTD_checkParams() :
     ensure param values remain within authorized range.
@@ -2779,6 +2784,10 @@ size_t ZSTD_freeCDict(ZSTD_CDict* cdict)
     }
 }
 
+static ZSTD_parameters ZSTD_getParamsFromCDict(const ZSTD_CDict* cdict) {
+    return ZSTD_getParamsFromCCtx(cdict->refContext);
+}
+
 size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict, U64 pledgedSrcSize)
 {
     if (cdict->dictContentSize) CHECK_F(ZSTD_copyCCtx(cctx, cdict->refContext, pledgedSrcSize))
@@ -2815,7 +2824,8 @@ typedef enum { zcss_init, zcss_load, zcss_flush, zcss_final } ZSTD_cStreamStage;
 
 struct ZSTD_CStream_s {
     ZSTD_CCtx* cctx;
-    ZSTD_CDict* cdict;
+    ZSTD_CDict* cdictLocal;
+    const ZSTD_CDict* cdict;
     char*  inBuff;
     size_t inBuffSize;
     size_t inToCompress;
@@ -2858,7 +2868,7 @@ size_t ZSTD_freeCStream(ZSTD_CStream* zcs)
     if (zcs==NULL) return 0;   /* support free on NULL */
     {   ZSTD_customMem const cMem = zcs->customMem;
         ZSTD_freeCCtx(zcs->cctx);
-        ZSTD_freeCDict(zcs->cdict);
+        ZSTD_freeCDict(zcs->cdictLocal);
         ZSTD_free(zcs->inBuff, cMem);
         ZSTD_free(zcs->outBuff, cMem);
         ZSTD_free(zcs, cMem);
@@ -2906,15 +2916,25 @@ size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs,
         if (zcs->outBuff == NULL) return ERROR(memory_allocation);
     }
 
-    ZSTD_freeCDict(zcs->cdict);
-    zcs->cdict = ZSTD_createCDict_advanced(dict, dictSize, params, zcs->customMem);
-    if (zcs->cdict == NULL) return ERROR(memory_allocation);
+    ZSTD_freeCDict(zcs->cdictLocal);
+    zcs->cdictLocal = ZSTD_createCDict_advanced(dict, dictSize, params, zcs->customMem);
+    if (zcs->cdictLocal == NULL) return ERROR(memory_allocation);
+    zcs->cdict = zcs->cdictLocal;
 
     zcs->checksum = params.fParams.checksumFlag > 0;
 
     return ZSTD_resetCStream(zcs, pledgedSrcSize);
 }
 
+/* note : cdict must outlive compression session */
+size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict)
+{
+    ZSTD_parameters const params = ZSTD_getParamsFromCDict(cdict);
+    size_t const initError =  ZSTD_initCStream_advanced(zcs, NULL, 0, params, 0);
+    zcs->cdict = cdict;
+    return initError;
+}
+
 size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel)
 {
     ZSTD_parameters const params = ZSTD_getParams(compressionLevel, 0, dictSize);
diff --git a/lib/zstd.h b/lib/zstd.h
index e0d4ce99f..89b6c8274 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -459,6 +459,7 @@ ZSTDLIB_API ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem);
 ZSTDLIB_API size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel);
 ZSTDLIB_API size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize,
                                              ZSTD_parameters params, unsigned long long pledgedSrcSize);  /**< pledgedSrcSize is optional and can be zero == unknown */
+ZSTDLIB_API size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict);  /**< note : cdict will just be referenced, and must outlive compression session */
 ZSTDLIB_API size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize);  /**< re-use compression parameters from previous init; saves dictionary loading */
 ZSTDLIB_API size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs);
 
@@ -468,6 +469,7 @@ typedef enum { ZSTDdsp_maxWindowSize } ZSTD_DStreamParameter_e;
 ZSTDLIB_API ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem);
 ZSTDLIB_API size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize);
 ZSTDLIB_API size_t ZSTD_setDStreamParameter(ZSTD_DStream* zds, ZSTD_DStreamParameter_e paramType, unsigned paramValue);
+ZSTDLIB_API size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDict* ddict);  /**< note : ddict will just be referenced, and must outlive decompression session */
 ZSTDLIB_API size_t ZSTD_resetDStream(ZSTD_DStream* zds);  /**< re-use decompression parameters from previous init; saves dictionary loading */
 ZSTDLIB_API size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds);
 
diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c
index 8486013c2..963078d8a 100644
--- a/tests/zstreamtest.c
+++ b/tests/zstreamtest.c
@@ -77,7 +77,7 @@ static clock_t FUZ_GetClockSpan(clock_t clockStart)
 /*! FUZ_rand() :
     @return : a 27 bits random value, from a 32-bits `seed`.
     `seed` is also modified */
-#  define FUZ_rotl32(x,r) ((x << r) | (x >> (32 - r)))
+#define FUZ_rotl32(x,r) ((x << r) | (x >> (32 - r)))
 unsigned int FUZ_rand(unsigned int* seedPtr)
 {
     U32 rand32 = *seedPtr;
@@ -281,6 +281,28 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
     }
     DISPLAYLEVEL(4, "OK \n");
 
+    /* CDict scenario */
+    DISPLAYLEVEL(4, "test%3i : digested dictionary : ", testNb++);
+    {   ZSTD_CDict* const cdict = ZSTD_createCDict(CNBuffer, 128 KB, 1);
+        size_t const initError = ZSTD_initCStream_usingCDict(zc, cdict);
+        if (ZSTD_isError(initError)) goto _output_error;
+        cSize = 0;
+        outBuff.dst = compressedBuffer;
+        outBuff.size = compressedBufferSize;
+        outBuff.pos = 0;
+        inBuff.src = CNBuffer;
+        inBuff.size = CNBufferSize;
+        inBuff.pos = 0;
+        { size_t const r = ZSTD_compressStream(zc, &outBuff, &inBuff);
+          if (ZSTD_isError(r)) goto _output_error; }
+        if (inBuff.pos != inBuff.size) goto _output_error;   /* entire input should be consumed */
+        { size_t const r = ZSTD_endStream(zc, &outBuff);
+          if (r != 0) goto _output_error; }  /* error, or some data not flushed */
+        cSize = outBuff.pos;
+        ZSTD_freeCDict(cdict);
+        DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/CNBufferSize*100);
+    }
+
     DISPLAYLEVEL(4, "test%3i : check CStream size : ", testNb++);
     { size_t const s = ZSTD_sizeof_CStream(zc);
       if (ZSTD_isError(s)) goto _output_error;
@@ -511,7 +533,7 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compres
 
         /* multi - fragments decompression test */
         if (!dictSize /* don't reset if dictionary : could be different */ && (FUZ_rand(&lseed) & 1)) {
-            CHECK( ZSTD_isError(ZSTD_resetDStream(zd)), "ZSTD_resetDStream failed");
+            CHECK (ZSTD_isError(ZSTD_resetDStream(zd)), "ZSTD_resetDStream failed");
         } else
             ZSTD_initDStream_usingDict(zd, dict, dictSize);
         {   size_t decompressionResult = 1;

From 335ad5d4d4310f224aaa8cd75b55c371de891641 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Tue, 25 Oct 2016 17:47:02 -0700
Subject: [PATCH 68/92] added ZSTD_initDStream_usingDDict() . slightly
 optimized ZSTD_initDStream() when no dictionary . fixed ZSTD_sizeof_CStream()
 .

---
 lib/compress/zstd_compress.c     |  2 +-
 lib/decompress/zstd_decompress.c | 64 ++++++++++++++++++++------------
 tests/zstreamtest.c              | 19 ++++++++++
 3 files changed, 60 insertions(+), 25 deletions(-)

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 5409c233a..85a5ff22d 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -2949,7 +2949,7 @@ size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel)
 size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs)
 {
     if (zcs==NULL) return 0;   /* support sizeof on NULL */
-    return sizeof(zcs) + ZSTD_sizeof_CCtx(zcs->cctx) + ZSTD_sizeof_CDict(zcs->cdict) + zcs->outBuffSize + zcs->inBuffSize;
+    return sizeof(zcs) + ZSTD_sizeof_CCtx(zcs->cctx) + ZSTD_sizeof_CDict(zcs->cdictLocal) + zcs->outBuffSize + zcs->inBuffSize;
 }
 
 /*======   Compression   ======*/
diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c
index 2a1232eee..f3ff4ebff 100644
--- a/lib/decompress/zstd_decompress.c
+++ b/lib/decompress/zstd_decompress.c
@@ -170,20 +170,22 @@ void ZSTD_copyDCtx(ZSTD_DCtx* dstDCtx, const ZSTD_DCtx* srcDCtx)
 static void ZSTD_refDCtx(ZSTD_DCtx* dstDCtx, const ZSTD_DCtx* srcDCtx)
 {
     ZSTD_decompressBegin(dstDCtx);  /* init */
-    dstDCtx->dictEnd = srcDCtx->dictEnd;
-    dstDCtx->vBase = srcDCtx->vBase;
-    dstDCtx->base = srcDCtx->base;
-    dstDCtx->previousDstEnd = srcDCtx->previousDstEnd;
-    dstDCtx->dictID = srcDCtx->dictID;
-    dstDCtx->litEntropy = srcDCtx->litEntropy;
-    dstDCtx->fseEntropy = srcDCtx->fseEntropy;
-    dstDCtx->LLTptr = srcDCtx->LLTable;
-    dstDCtx->MLTptr = srcDCtx->MLTable;
-    dstDCtx->OFTptr = srcDCtx->OFTable;
-    dstDCtx->HUFptr = srcDCtx->hufTable;
-    dstDCtx->rep[0] = srcDCtx->rep[0];
-    dstDCtx->rep[1] = srcDCtx->rep[1];
-    dstDCtx->rep[2] = srcDCtx->rep[2];
+    if (srcDCtx) {   /* support refDCtx on NULL */
+        dstDCtx->dictEnd = srcDCtx->dictEnd;
+        dstDCtx->vBase = srcDCtx->vBase;
+        dstDCtx->base = srcDCtx->base;
+        dstDCtx->previousDstEnd = srcDCtx->previousDstEnd;
+        dstDCtx->dictID = srcDCtx->dictID;
+        dstDCtx->litEntropy = srcDCtx->litEntropy;
+        dstDCtx->fseEntropy = srcDCtx->fseEntropy;
+        dstDCtx->LLTptr = srcDCtx->LLTable;
+        dstDCtx->MLTptr = srcDCtx->MLTable;
+        dstDCtx->OFTptr = srcDCtx->OFTable;
+        dstDCtx->HUFptr = srcDCtx->hufTable;
+        dstDCtx->rep[0] = srcDCtx->rep[0];
+        dstDCtx->rep[1] = srcDCtx->rep[1];
+        dstDCtx->rep[2] = srcDCtx->rep[2];
+    }
 }
 
 
@@ -1483,7 +1485,8 @@ typedef enum { zdss_init, zdss_loadHeader,
 /* *** Resource management *** */
 struct ZSTD_DStream_s {
     ZSTD_DCtx* dctx;
-    ZSTD_DDict* ddict;
+    ZSTD_DDict* ddictLocal;
+    const ZSTD_DDict* ddict;
     ZSTD_frameParams fParams;
     ZSTD_dStreamStage stage;
     char*  inBuff;
@@ -1533,7 +1536,7 @@ size_t ZSTD_freeDStream(ZSTD_DStream* zds)
     if (zds==NULL) return 0;   /* support free on null */
     {   ZSTD_customMem const cMem = zds->customMem;
         ZSTD_freeDCtx(zds->dctx);
-        ZSTD_freeDDict(zds->ddict);
+        ZSTD_freeDDict(zds->ddictLocal);
         ZSTD_free(zds->inBuff, cMem);
         ZSTD_free(zds->outBuff, cMem);
 #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1)
@@ -1555,9 +1558,12 @@ size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t di
 {
     zds->stage = zdss_loadHeader;
     zds->lhSize = zds->inPos = zds->outStart = zds->outEnd = 0;
-    ZSTD_freeDDict(zds->ddict);
-    zds->ddict = ZSTD_createDDict(dict, dictSize);
-    if (zds->ddict == NULL) return ERROR(memory_allocation);
+    ZSTD_freeDDict(zds->ddictLocal);
+    if (dict) {
+        zds->ddictLocal = ZSTD_createDDict(dict, dictSize);
+        if (zds->ddictLocal == NULL) return ERROR(memory_allocation);
+    } else zds->ddictLocal = NULL;
+    zds->ddict = zds->ddictLocal;
     zds->legacyVersion = 0;
     zds->hostageByte = 0;
     return ZSTD_frameHeaderSize_prefix;
@@ -1568,9 +1574,15 @@ size_t ZSTD_initDStream(ZSTD_DStream* zds)
     return ZSTD_initDStream_usingDict(zds, NULL, 0);
 }
 
+size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDict* ddict)  /**< note : ddict will just be referenced, and must outlive decompression session */
+{
+    size_t const initResult = ZSTD_initDStream(zds);
+    zds->ddict = ddict;
+    return initResult;
+}
+
 size_t ZSTD_resetDStream(ZSTD_DStream* zds)
 {
-    if (zds->ddict == NULL) return ERROR(stage_wrong);  /* must be init at least once */
     zds->stage = zdss_loadHeader;
     zds->lhSize = zds->inPos = zds->outStart = zds->outEnd = 0;
     zds->legacyVersion = 0;
@@ -1593,7 +1605,7 @@ size_t ZSTD_setDStreamParameter(ZSTD_DStream* zds,
 size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds)
 {
     if (zds==NULL) return 0;   /* support sizeof on NULL */
-    return sizeof(*zds) + ZSTD_sizeof_DCtx(zds->dctx) + ZSTD_sizeof_DDict(zds->ddict) + zds->inBuffSize + zds->outBuffSize;
+    return sizeof(*zds) + ZSTD_sizeof_DCtx(zds->dctx) + ZSTD_sizeof_DDict(zds->ddictLocal) + zds->inBuffSize + zds->outBuffSize;
 }
 
 
@@ -1634,15 +1646,17 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
 #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1)
                 {   U32 const legacyVersion = ZSTD_isLegacy(istart, iend-istart);
                     if (legacyVersion) {
+                        const void* const dict = zds->ddict ? zds->ddict->dict : NULL;
+                        size_t const dictSize = zds->ddict ? zds->ddict->dictSize : 0;
                         CHECK_F(ZSTD_initLegacyStream(&zds->legacyContext, zds->previousLegacyVersion, legacyVersion,
-                                                       zds->ddict->dict, zds->ddict->dictSize));
+                                                       dict, dictSize));
                         zds->legacyVersion = zds->previousLegacyVersion = legacyVersion;
                         return ZSTD_decompressLegacyStream(zds->legacyContext, zds->legacyVersion, output, input);
                     } else {
                         return hSize; /* error */
                 }   }
 #else
-                    return hSize;
+                return hSize;
 #endif
                 if (hSize != 0) {   /* need more input */
                     size_t const toLoad = hSize - zds->lhSize;   /* if hSize!=0, hSize > zds->lhSize */
@@ -1657,7 +1671,9 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
             }   }
 
             /* Consume header */
-            ZSTD_refDCtx(zds->dctx, zds->ddict->refContext);
+            {   const ZSTD_DCtx* refContext = zds->ddict ? zds->ddict->refContext : NULL;
+                ZSTD_refDCtx(zds->dctx, refContext);
+            }
             {   size_t const h1Size = ZSTD_nextSrcSizeToDecompress(zds->dctx);  /* == ZSTD_frameHeaderSize_prefix */
                 CHECK_F(ZSTD_decompressContinue(zds->dctx, NULL, 0, zds->headerBuffer, h1Size));
                 {   size_t const h2Size = ZSTD_nextSrcSizeToDecompress(zds->dctx);
diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c
index 963078d8a..38b6a76cc 100644
--- a/tests/zstreamtest.c
+++ b/tests/zstreamtest.c
@@ -309,6 +309,25 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
       DISPLAYLEVEL(4, "OK (%u bytes) \n", (U32)s);
     }
 
+    /* DDict scenario */
+    DISPLAYLEVEL(4, "test%3i : decompress %u bytes with digested dictionary : ", testNb++, (U32)CNBufferSize);
+    {   ZSTD_DDict* const ddict = ZSTD_createDDict(CNBuffer, 128 KB);
+        size_t const initError = ZSTD_initDStream_usingDDict(zd, ddict);
+        if (ZSTD_isError(initError)) goto _output_error;
+        inBuff.src = compressedBuffer;
+        inBuff.size = cSize;
+        inBuff.pos = 0;
+        outBuff.dst = decodedBuffer;
+        outBuff.size = CNBufferSize;
+        outBuff.pos = 0;
+        { size_t const r = ZSTD_decompressStream(zd, &outBuff, &inBuff);
+          if (r != 0) goto _output_error; }  /* should reach end of frame == 0; otherwise, some data left, or an error */
+        if (outBuff.pos != CNBufferSize) goto _output_error;   /* should regenerate the same amount */
+        if (inBuff.pos != inBuff.size) goto _output_error;   /* should have read the entire frame */
+        ZSTD_freeDDict(ddict);
+        DISPLAYLEVEL(4, "OK \n");
+    }
+
     /* test ZSTD_setDStreamParameter() resilience */
     DISPLAYLEVEL(4, "test%3i : wrong parameter for ZSTD_setDStreamParameter(): ", testNb++);
     { size_t const r = ZSTD_setDStreamParameter(zd, (ZSTD_DStreamParameter_e)999, 1);  /* large limit */

From eb7873a048dee0c9462746550daa9d0a502b6154 Mon Sep 17 00:00:00 2001
From: Nick Terrell 
Date: Fri, 21 Oct 2016 16:55:26 -0700
Subject: [PATCH 69/92] ZSTD_execSequence() accepts match in last 7 bytes

The zstd reference compressor will not emit a match in the last 7
bytes of a block.  The decompressor will also not accept a match
in the last 7 bytes.  This patch makes the decompressor accept a
match in the last 7 bytes.
---
 lib/common/zstd_internal.h       | 10 +++++++
 lib/decompress/zstd_decompress.c | 50 +++++++++++++++++++++++++++++++-
 2 files changed, 59 insertions(+), 1 deletion(-)

diff --git a/lib/common/zstd_internal.h b/lib/common/zstd_internal.h
index f40e00aab..d889c840f 100644
--- a/lib/common/zstd_internal.h
+++ b/lib/common/zstd_internal.h
@@ -31,6 +31,16 @@
 #  endif /* __STDC_VERSION__ */
 #endif
 
+#ifdef _MSC_VER
+#  define FORCE_NOINLINE static __declspec(noinline)
+#else
+#  ifdef __GNUC__
+#    define FORCE_NOINLINE static __attribute__((__noinline__))
+#  else
+#    define FORCE_NOINLINE static
+#  endif
+#endif
+
 
 /*-*************************************
 *  Dependencies
diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c
index 00b069e9f..045513afd 100644
--- a/lib/decompress/zstd_decompress.c
+++ b/lib/decompress/zstd_decompress.c
@@ -840,6 +840,53 @@ static seq_t ZSTD_decodeSequence(seqState_t* seqState)
 }
 
 
+FORCE_NOINLINE
+size_t ZSTD_execSequenceLast7(BYTE* op,
+                              BYTE* const oend, seq_t sequence,
+                              const BYTE** litPtr, const BYTE* const litLimit_w,
+                              const BYTE* const base, const BYTE* const vBase, const BYTE* const dictEnd)
+{
+    BYTE* const oLitEnd = op + sequence.litLength;
+    size_t const sequenceLength = sequence.litLength + sequence.matchLength;
+    BYTE* const oMatchEnd = op + sequenceLength;   /* risk : address space overflow (32-bits) */
+    BYTE* const oend_w = oend - WILDCOPY_OVERLENGTH;
+    const BYTE* const iLitEnd = *litPtr + sequence.litLength;
+    const BYTE* match = oLitEnd - sequence.offset;
+
+    /* check */
+    if (oMatchEnd>oend) return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of WILDCOPY_OVERLENGTH from oend */
+    if (iLitEnd > litLimit_w) return ERROR(corruption_detected);   /* over-read beyond lit buffer */
+    if (oLitEnd <= oend_w) return ERROR(GENERIC);   /* Precondition */
+
+    /* copy literals */
+    if (op < oend_w) {
+        ZSTD_wildcopy(op, *litPtr, oend_w - op);
+        *litPtr += oend_w - op;
+        op = oend_w;
+    }
+    while (op < oLitEnd) *op++ = *(*litPtr)++;
+
+    /* copy Match */
+    if (sequence.offset > (size_t)(oLitEnd - base)) {
+        /* offset beyond prefix */
+        if (sequence.offset > (size_t)(oLitEnd - vBase)) return ERROR(corruption_detected);
+        match = dictEnd - (base-match);
+        if (match + sequence.matchLength <= dictEnd) {
+            memmove(oLitEnd, match, sequence.matchLength);
+            return sequenceLength;
+        }
+        /* span extDict & currentPrefixSegment */
+        {   size_t const length1 = dictEnd - match;
+            memmove(oLitEnd, match, length1);
+            op = oLitEnd + length1;
+            sequence.matchLength -= length1;
+            match = base;
+    }   }
+    while (op < oMatchEnd) *op++ = *match++;
+    return sequenceLength;
+}
+
+
 FORCE_INLINE
 size_t ZSTD_execSequence(BYTE* op,
                                 BYTE* const oend, seq_t sequence,
@@ -854,8 +901,9 @@ size_t ZSTD_execSequence(BYTE* op,
     const BYTE* match = oLitEnd - sequence.offset;
 
     /* check */
-    if ((oLitEnd>oend_w) | (oMatchEnd>oend)) return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of WILDCOPY_OVERLENGTH from oend */
+    if (oMatchEnd>oend) return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of WILDCOPY_OVERLENGTH from oend */
     if (iLitEnd > litLimit_w) return ERROR(corruption_detected);   /* over-read beyond lit buffer */
+    if (oLitEnd>oend_w) return ZSTD_execSequenceLast7(op, oend, sequence, litPtr, litLimit_w, base, vBase, dictEnd);
 
     /* copy Literals */
     ZSTD_copy8(op, *litPtr);

From ac4310d303f363df6b0ccea32cedbb6f6a893f00 Mon Sep 17 00:00:00 2001
From: Nick Terrell 
Date: Wed, 26 Oct 2016 00:09:39 -0700
Subject: [PATCH 70/92] [pzstd] Fix test mode for streaming input

---
 contrib/pzstd/Options.cpp | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/contrib/pzstd/Options.cpp b/contrib/pzstd/Options.cpp
index ece8c0782..18c069eae 100644
--- a/contrib/pzstd/Options.cpp
+++ b/contrib/pzstd/Options.cpp
@@ -303,6 +303,12 @@ Options::Status Options::parse(int argc, const char **argv) {
     } // while (*options != 0);
   }   // for (int i = 1; i < argc; ++i);
 
+  // Set options for test mode
+  if (test) {
+    outputFile = nullOutput;
+    keepSource = true;
+  }
+
   // Input file defaults to standard input if not provided.
   if (localInputFiles.empty()) {
     localInputFiles.emplace_back(kStdIn);
@@ -399,11 +405,6 @@ Options::Status Options::parse(int argc, const char **argv) {
     verbosity = 1;
   }
 
-  // Set options for test mode
-  if (test) {
-    outputFile = nullOutput;
-    keepSource = true;
-  }
   return Status::Success;
 }
 

From 82613d72e4df2449ebc1e67f81e29ef8a52c8a8a Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Wed, 26 Oct 2016 18:10:43 -0700
Subject: [PATCH 71/92] added example multiple_streaming_compression

---
 examples/.gitignore                       |   1 +
 examples/Makefile                         |  12 +-
 examples/README.md                        |   5 +
 examples/multiple_streaming_compression.c | 163 ++++++++++++++++++++++
 4 files changed, 179 insertions(+), 2 deletions(-)
 create mode 100644 examples/multiple_streaming_compression.c

diff --git a/examples/.gitignore b/examples/.gitignore
index 1c98e1884..0711813d3 100644
--- a/examples/.gitignore
+++ b/examples/.gitignore
@@ -5,6 +5,7 @@ dictionary_compression
 dictionary_decompression
 streaming_compression
 streaming_decompression
+multiple_streaming_compression
 
 #test artefact
 tmp*
diff --git a/examples/Makefile b/examples/Makefile
index 8ce6a2589..741022869 100644
--- a/examples/Makefile
+++ b/examples/Makefile
@@ -17,7 +17,8 @@ default: all
 
 all: simple_compression simple_decompression \
 	dictionary_compression dictionary_decompression \
-	streaming_compression streaming_decompression
+	streaming_compression streaming_decompression \
+	multiple_streaming_compression
 
 simple_compression : simple_compression.c
 	$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
@@ -34,6 +35,9 @@ dictionary_decompression : dictionary_decompression.c
 streaming_compression : streaming_compression.c
 	$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
 
+multiple_streaming_compression : multiple_streaming_compression.c
+	$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
+
 streaming_decompression : streaming_decompression.c
 	$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
 
@@ -41,7 +45,8 @@ clean:
 	@rm -f core *.o tmp* result* *.zst \
         simple_compression simple_decompression \
         dictionary_compression dictionary_decompression \
-        streaming_compression streaming_decompression
+        streaming_compression streaming_decompression \
+		multiple_streaming_compression
 	@echo Cleaning completed
 
 test: all
@@ -54,7 +59,10 @@ test: all
 	@echo starting streaming compression
 	./streaming_compression tmp
 	./streaming_decompression tmp.zst > /dev/null
+	@echo starting multiple streaming compression
+	./multiple_streaming_compression *.c
 	@echo starting dictionary compression
 	./dictionary_compression tmp2 tmp README.md
 	./dictionary_decompression tmp2.zst tmp.zst README.md
+	$(RM) tmp* *.zst
 	@echo tests completed
diff --git a/examples/README.md b/examples/README.md
index ba132f6c3..8a40443ea 100644
--- a/examples/README.md
+++ b/examples/README.md
@@ -15,6 +15,11 @@ Zstandard library : usage examples
   Compress a single file.
   Introduces usage of : `ZSTD_compressStream()`
 
+- [Multiple Streaming compression](multiple_streaming_compression.c) :
+  Compress multiple files in a single command line.
+  Introduces memory usage preservation technique,
+  reducing impact of malloc()/free() and memset() by re-using existing resources.
+
 - [Streaming decompression](streaming_decompression.c) :
   Decompress a single file compressed by zstd.
   Compatible with both simple and streaming compression.
diff --git a/examples/multiple_streaming_compression.c b/examples/multiple_streaming_compression.c
new file mode 100644
index 000000000..81541f590
--- /dev/null
+++ b/examples/multiple_streaming_compression.c
@@ -0,0 +1,163 @@
+/**
+ * Copyright 2016-present, Yann Collet, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the license found in the
+ * LICENSE-examples file in the root directory of this source tree.
+ */
+
+
+/* The objective of this example is to show of to compress multiple successive files
+*  while preserving memory management.
+*  All structures and buffers will be created only once,
+*  and shared across all compression operations */
+
+#include     // malloc, exit
+#include      // fprintf, perror, feof
+#include     // strerror
+#include      // errno
+#define ZSTD_STATIC_LINKING_ONLY  // streaming API defined as "experimental" for the time being
+#include       // presumes zstd library is installed
+
+
+static void* malloc_orDie(size_t size)
+{
+    void* const buff = malloc(size);
+    if (buff) return buff;
+    /* error */
+    perror("malloc:");
+    exit(1);
+}
+
+static FILE* fopen_orDie(const char *filename, const char *instruction)
+{
+    FILE* const inFile = fopen(filename, instruction);
+    if (inFile) return inFile;
+    /* error */
+    perror(filename);
+    exit(3);
+}
+
+static size_t fread_orDie(void* buffer, size_t sizeToRead, FILE* file)
+{
+    size_t const readSize = fread(buffer, 1, sizeToRead, file);
+    if (readSize == sizeToRead) return readSize;   /* good */
+    if (feof(file)) return readSize;   /* good, reached end of file */
+    /* error */
+    perror("fread");
+    exit(4);
+}
+
+static size_t fwrite_orDie(const void* buffer, size_t sizeToWrite, FILE* file)
+{
+    size_t const writtenSize = fwrite(buffer, 1, sizeToWrite, file);
+    if (writtenSize == sizeToWrite) return sizeToWrite;   /* good */
+    /* error */
+    perror("fwrite");
+    exit(5);
+}
+
+static size_t fclose_orDie(FILE* file)
+{
+    if (!fclose(file)) return 0;
+    /* error */
+    perror("fclose");
+    exit(6);
+}
+
+
+typedef struct {
+    void* buffIn;
+    void* buffOut;
+    size_t buffInSize;
+    size_t buffOutSize;
+    ZSTD_CStream* cstream;
+} resources ;
+
+static resources createResources_orDie()
+{
+    resources ress;
+    ress.buffInSize = ZSTD_CStreamInSize();   /* can always read one full block */
+    ress.buffOutSize= ZSTD_CStreamOutSize();  /* can always flush a full block */
+    ress.buffIn = malloc_orDie(ress.buffInSize);
+    ress.buffOut= malloc_orDie(ress.buffOutSize);
+    ress.cstream = ZSTD_createCStream();
+    if (ress.cstream==NULL) { fprintf(stderr, "ZSTD_createCStream() error \n"); exit(10); }
+    return ress;
+}
+
+static void freeResources(resources ress)
+{
+    ZSTD_freeCStream(ress.cstream);
+    free(ress.buffIn);
+    free(ress.buffOut);
+}
+
+
+static void compressFile_orDie(resources ress, const char* fname, const char* outName, int cLevel)
+{
+    FILE* const fin  = fopen_orDie(fname, "rb");
+    FILE* const fout = fopen_orDie(outName, "wb");
+
+    size_t const initResult = ZSTD_initCStream(ress.cstream, cLevel);
+    if (ZSTD_isError(initResult)) { fprintf(stderr, "ZSTD_initCStream() error : %s \n", ZSTD_getErrorName(initResult)); exit(11); }
+
+    size_t read, toRead = ress.buffInSize;
+    while( (read = fread_orDie(ress.buffIn, toRead, fin)) ) {
+        ZSTD_inBuffer input = { ress.buffIn, read, 0 };
+        while (input.pos < input.size) {
+            ZSTD_outBuffer output = { ress.buffOut, ress.buffOutSize, 0 };
+            toRead = ZSTD_compressStream(ress.cstream, &output , &input);   /* toRead is guaranteed to be <= ZSTD_CStreamInSize() */
+            if (ZSTD_isError(toRead)) { fprintf(stderr, "ZSTD_compressStream() error : %s \n", ZSTD_getErrorName(toRead)); exit(12); }
+            if (toRead > ress.buffInSize) toRead = ress.buffInSize;   /* Safely handle when `buffInSize` is manually changed to a smaller value */
+            fwrite_orDie(ress.buffOut, output.pos, fout);
+        }
+    }
+
+    ZSTD_outBuffer output = { ress.buffOut, ress.buffOutSize, 0 };
+    size_t const remainingToFlush = ZSTD_endStream(ress.cstream, &output);   /* close frame */
+    if (remainingToFlush) { fprintf(stderr, "not fully flushed"); exit(13); }
+    fwrite_orDie(ress.buffOut, output.pos, fout);
+
+    fclose_orDie(fout);
+    fclose_orDie(fin);
+
+    /* success */
+    printf("%-30s -> %-30s \n", fname, outName);
+}
+
+
+int main(int argc, const char** argv)
+{
+    const char* const exeName = argv[0];
+
+    if (argc<2) {
+        printf("wrong arguments\n");
+        printf("usage:\n");
+        printf("%s FILE(s)\n", exeName);
+        return 1;
+    }
+
+    resources ress = createResources_orDie();
+    void* ofnBuffer = NULL;
+    size_t ofnbSize = 0;
+
+    int argNb;
+    for (argNb = 1; argNb < argc; argNb++) {
+        const char* const ifn = argv[argNb];
+        size_t const ifnSize = strlen(ifn);
+        size_t const ofnSize = ifnSize + 5;
+        if (ofnbSize <= ofnSize) {
+            ofnbSize = ofnSize + 16;
+            free(ofnBuffer);
+            ofnBuffer = malloc_orDie(ofnbSize);
+        }
+        memset(ofnBuffer, 0, ofnSize);
+        strcat(ofnBuffer, ifn);
+        strcat(ofnBuffer, ".zst");
+        compressFile_orDie(ress, ifn, ofnBuffer, 7);
+    }
+
+    freeResources(ress);
+    return 0;
+}

From cf08958d9ba966c3cf046eac5f203b1ba24bbdb4 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Wed, 26 Oct 2016 18:37:29 -0700
Subject: [PATCH 72/92] print one global summary (instead of one per file)

---
 NEWS                                      | 2 ++
 examples/multiple_streaming_compression.c | 6 +++---
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/NEWS b/NEWS
index 2e6118648..2c2585f00 100644
--- a/NEWS
+++ b/NEWS
@@ -2,6 +2,8 @@ v1.1.1
 New : command -M#, --memory=, --memlimit=, --memlimit-decompress= to limit allowed memory consumption
 New : doc/zstd_manual.html, by Przemyslaw Skibinski
 Improved : slightly better compression ratio at --ultra levels (>= 20)
+Added : ZSTD_initCStream_usingCDict(), ZSTD_initDStream_usingDDict() (experimental section)
+Added : example/multiple_streaming_compression
 Changed : zstd_errors.h is now part of include installation
 Fixed : zstd-small, zstd-compress and zstd-decompress compilation targets
 
diff --git a/examples/multiple_streaming_compression.c b/examples/multiple_streaming_compression.c
index 81541f590..c002a8872 100644
--- a/examples/multiple_streaming_compression.c
+++ b/examples/multiple_streaming_compression.c
@@ -121,9 +121,6 @@ static void compressFile_orDie(resources ress, const char* fname, const char* ou
 
     fclose_orDie(fout);
     fclose_orDie(fin);
-
-    /* success */
-    printf("%-30s -> %-30s \n", fname, outName);
 }
 
 
@@ -159,5 +156,8 @@ int main(int argc, const char** argv)
     }
 
     freeResources(ress);
+    /* success */
+    printf("compressed %i files \n", argc-1);
+
     return 0;
 }

From bfae317ae3b3f768a648cb48d59d9ceed88a968d Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Wed, 26 Oct 2016 18:44:59 -0700
Subject: [PATCH 73/92] minor const added

---
 examples/multiple_streaming_compression.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/examples/multiple_streaming_compression.c b/examples/multiple_streaming_compression.c
index c002a8872..61699104c 100644
--- a/examples/multiple_streaming_compression.c
+++ b/examples/multiple_streaming_compression.c
@@ -135,7 +135,7 @@ int main(int argc, const char** argv)
         return 1;
     }
 
-    resources ress = createResources_orDie();
+    resources const ress = createResources_orDie();
     void* ofnBuffer = NULL;
     size_t ofnbSize = 0;
 

From ee5b72582389835d02c4c94e51cc998f9fce540a Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Thu, 27 Oct 2016 14:20:55 -0700
Subject: [PATCH 74/92] ZSTD_initCStream() optimization : do not allocate a
 CDict when no dictionary used

---
 lib/compress/zstd_compress.c | 31 +++++++++++++++++++------------
 lib/zstd.h                   |  2 +-
 2 files changed, 20 insertions(+), 13 deletions(-)

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 85a5ff22d..e7f7d9911 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -2489,13 +2489,13 @@ static size_t ZSTD_checkDictNCount(short* normalizedCounter, unsigned dictMaxSym
 
 
 /* Dictionary format :
-     Magic == ZSTD_DICT_MAGIC (4 bytes)
-     HUF_writeCTable(256)
-     FSE_writeNCount(off)
-     FSE_writeNCount(ml)
-     FSE_writeNCount(ll)
-     RepOffsets
-     Dictionary content
+    Magic == ZSTD_DICT_MAGIC (4 bytes)
+    HUF_writeCTable(256)
+    FSE_writeNCount(off)
+    FSE_writeNCount(ml)
+    FSE_writeNCount(ll)
+    RepOffsets
+    Dictionary content
 */
 /*! ZSTD_loadDictEntropyStats() :
     @return : size read from dictionary
@@ -2839,6 +2839,7 @@ struct ZSTD_CStream_s {
     ZSTD_cStreamStage stage;
     U32    checksum;
     U32    frameEnded;
+    ZSTD_parameters params;
     ZSTD_customMem customMem;
 };   /* typedef'd to ZSTD_CStream within "zstd.h" */
 
@@ -2884,7 +2885,10 @@ size_t ZSTD_CStreamOutSize(void) { return ZSTD_compressBound(ZSTD_BLOCKSIZE_ABSO
 
 size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize)
 {
-    CHECK_F(ZSTD_compressBegin_usingCDict(zcs->cctx, zcs->cdict, pledgedSrcSize));
+    if (zcs->inBuffSize==0) return ERROR(stage_wrong);   /* zcs has not been init at least once */
+
+    if (zcs->cdict) CHECK_F(ZSTD_compressBegin_usingCDict(zcs->cctx, zcs->cdict, pledgedSrcSize))
+    else CHECK_F(ZSTD_compressBegin_advanced(zcs->cctx, NULL, 0, zcs->params, pledgedSrcSize));
 
     zcs->inToCompress = 0;
     zcs->inBuffPos = 0;
@@ -2916,12 +2920,15 @@ size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs,
         if (zcs->outBuff == NULL) return ERROR(memory_allocation);
     }
 
-    ZSTD_freeCDict(zcs->cdictLocal);
-    zcs->cdictLocal = ZSTD_createCDict_advanced(dict, dictSize, params, zcs->customMem);
-    if (zcs->cdictLocal == NULL) return ERROR(memory_allocation);
-    zcs->cdict = zcs->cdictLocal;
+    if (dict) {
+        ZSTD_freeCDict(zcs->cdictLocal);
+        zcs->cdictLocal = ZSTD_createCDict_advanced(dict, dictSize, params, zcs->customMem);
+        if (zcs->cdictLocal == NULL) return ERROR(memory_allocation);
+        zcs->cdict = zcs->cdictLocal;
+    } else zcs->cdict = NULL;
 
     zcs->checksum = params.fParams.checksumFlag > 0;
+    zcs->params = params;
 
     return ZSTD_resetCStream(zcs, pledgedSrcSize);
 }
diff --git a/lib/zstd.h b/lib/zstd.h
index 89b6c8274..ea5caf148 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -460,7 +460,7 @@ ZSTDLIB_API size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dic
 ZSTDLIB_API size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize,
                                              ZSTD_parameters params, unsigned long long pledgedSrcSize);  /**< pledgedSrcSize is optional and can be zero == unknown */
 ZSTDLIB_API size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict);  /**< note : cdict will just be referenced, and must outlive compression session */
-ZSTDLIB_API size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize);  /**< re-use compression parameters from previous init; saves dictionary loading */
+ZSTDLIB_API size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize);  /**< re-use compression parameters from previous init; skip dictionary loading stage; zcs must be init at least once before */
 ZSTDLIB_API size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs);
 
 

From 10bfd0c0d5c584c5015c5a3a34436f7025fac325 Mon Sep 17 00:00:00 2001
From: Nick Terrell 
Date: Thu, 27 Oct 2016 16:19:54 -0700
Subject: [PATCH 75/92] Fix ZSTD_execSequence() performance regression

Commit ae1cb3b3d07024618269b89e3421d828adfd34d9 caused the regression.
It is an instruction alignment issue, because if it is `U64 i` instead
of `U32 i`, the regression returns.  This patch fixes the regression
in gcc, but only gets some of the clang performance back.

Benchmarks:
Run on `silesia.tar`.  I only show levels 1-5 because the performance
regression was uniform across all levels.  I did one run on levels
1-19 and it looked good.

| Build | Level | Before | While | After |
|-------|-------|-------:|------:|------:|
| gcc   |     1 |  931.4 | 904.4 | 932.8 |
| gcc   |     2 |  849.1 | 822.6 | 851.2 |
| gcc   |     3 |  815.6 | 790.6 | 818.9 |
| gcc   |     4 |  794.1 | 770.7 | 798.0 |
| gcc   |     5 |  785.7 | 760.7 | 788.8 |
| clang |     1 |  705.5 | 683.2 | 693.8 |
| clang |     2 |  670.0 | 649.2 | 660.7 |
| clang |     3 |  659.6 | 639.8 | 651.4 |
| clang |     4 |  652.5 | 634.7 | 645.9 |
| clang |     5 |  646.9 | 625.5 | 637.7 |
---
 lib/decompress/zstd_decompress.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c
index f3ff4ebff..ed68d888b 100644
--- a/lib/decompress/zstd_decompress.c
+++ b/lib/decompress/zstd_decompress.c
@@ -887,7 +887,8 @@ size_t ZSTD_execSequence(BYTE* op,
             sequence.matchLength -= length1;
             match = base;
             if (op > oend_w) {
-              while (op < oMatchEnd) *op++ = *match++;
+              U32 i;
+              for (i = 0; i < sequence.matchLength; ++i) op[i] = match[i];
               return sequenceLength;
             }
     }   }

From 3575fcf28e699a90e1ed56e4e5a7f16b34b2a17f Mon Sep 17 00:00:00 2001
From: Przemyslaw Skibinski 
Date: Fri, 28 Oct 2016 11:54:47 +0200
Subject: [PATCH 76/92] updated build\VS2010\zstd\generate_res\verrsrc.h

---
 build/VS2010/zstd/generate_res/verrsrc.h | 166 +----------------------
 1 file changed, 1 insertion(+), 165 deletions(-)

diff --git a/build/VS2010/zstd/generate_res/verrsrc.h b/build/VS2010/zstd/generate_res/verrsrc.h
index 37e48d306..e282add02 100644
--- a/build/VS2010/zstd/generate_res/verrsrc.h
+++ b/build/VS2010/zstd/generate_res/verrsrc.h
@@ -1,172 +1,8 @@
-#include 
+/* minimal set of defines required to generate zstd.res from zstd.rc */
 
-/*****************************************************************************\
-*                                                                             *
-* verrsrc.h -   Version Resource definitions                                  *
-*                                                                             *
-*               Include file declaring version resources in rc files          *
-*                                                                             *
-*               Copyright (c) Microsoft Corporation. All rights reserved.     *
-*                                                                             *
-\*****************************************************************************/
-
-#pragma region Application Family
-#if WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_APP)
-
-/* ----- Symbols ----- */
-#define VS_FILE_INFO            RT_VERSION
 #define VS_VERSION_INFO         1
-#define VS_USER_DEFINED         100
 
-/* ----- VS_VERSION.dwFileFlags ----- */
-#ifndef _MAC
-#define VS_FFI_SIGNATURE        0xFEEF04BDL
-#else
-#define VS_FFI_SIGNATURE        0xBD04EFFEL
-#endif
-#define VS_FFI_STRUCVERSION     0x00010000L
 #define VS_FFI_FILEFLAGSMASK    0x0000003FL
-
-/* ----- VS_VERSION.dwFileFlags ----- */
-#define VS_FF_DEBUG             0x00000001L
-#define VS_FF_PRERELEASE        0x00000002L
-#define VS_FF_PATCHED           0x00000004L
-#define VS_FF_PRIVATEBUILD      0x00000008L
-#define VS_FF_INFOINFERRED      0x00000010L
-#define VS_FF_SPECIALBUILD      0x00000020L
-
-/* ----- VS_VERSION.dwFileOS ----- */
-#define VOS_UNKNOWN             0x00000000L
-#define VOS_DOS                 0x00010000L
-#define VOS_OS216               0x00020000L
-#define VOS_OS232               0x00030000L
-#define VOS_NT                  0x00040000L
-#define VOS_WINCE               0x00050000L
-
-#define VOS__BASE               0x00000000L
-#define VOS__WINDOWS16          0x00000001L
-#define VOS__PM16               0x00000002L
-#define VOS__PM32               0x00000003L
-#define VOS__WINDOWS32          0x00000004L
-
-#define VOS_DOS_WINDOWS16       0x00010001L
-#define VOS_DOS_WINDOWS32       0x00010004L
-#define VOS_OS216_PM16          0x00020002L
-#define VOS_OS232_PM32          0x00030003L
 #define VOS_NT_WINDOWS32        0x00040004L
-
-/* ----- VS_VERSION.dwFileType ----- */
-#define VFT_UNKNOWN             0x00000000L
-#define VFT_APP                 0x00000001L
 #define VFT_DLL                 0x00000002L
-#define VFT_DRV                 0x00000003L
-#define VFT_FONT                0x00000004L
-#define VFT_VXD                 0x00000005L
-#define VFT_STATIC_LIB          0x00000007L
-
-/* ----- VS_VERSION.dwFileSubtype for VFT_WINDOWS_DRV ----- */
 #define VFT2_UNKNOWN            0x00000000L
-#define VFT2_DRV_PRINTER        0x00000001L
-#define VFT2_DRV_KEYBOARD       0x00000002L
-#define VFT2_DRV_LANGUAGE       0x00000003L
-#define VFT2_DRV_DISPLAY        0x00000004L
-#define VFT2_DRV_MOUSE          0x00000005L
-#define VFT2_DRV_NETWORK        0x00000006L
-#define VFT2_DRV_SYSTEM         0x00000007L
-#define VFT2_DRV_INSTALLABLE    0x00000008L
-#define VFT2_DRV_SOUND          0x00000009L
-#define VFT2_DRV_COMM           0x0000000AL
-#define VFT2_DRV_INPUTMETHOD    0x0000000BL
-#define VFT2_DRV_VERSIONED_PRINTER    0x0000000CL
-
-/* ----- VS_VERSION.dwFileSubtype for VFT_WINDOWS_FONT ----- */
-#define VFT2_FONT_RASTER        0x00000001L
-#define VFT2_FONT_VECTOR        0x00000002L
-#define VFT2_FONT_TRUETYPE      0x00000003L
-
-#endif /* WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_APP) */
-#pragma endregion
-
-#pragma region Desktop Family
-#if WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_DESKTOP)
-
-/* ----- VerFindFile() flags ----- */
-#define VFFF_ISSHAREDFILE       0x0001
-
-#define VFF_CURNEDEST           0x0001
-#define VFF_FILEINUSE           0x0002
-#define VFF_BUFFTOOSMALL        0x0004
-
-/* ----- VerInstallFile() flags ----- */
-#define VIFF_FORCEINSTALL       0x0001
-#define VIFF_DONTDELETEOLD      0x0002
-
-#define VIF_TEMPFILE            0x00000001L
-#define VIF_MISMATCH            0x00000002L
-#define VIF_SRCOLD              0x00000004L
-
-#define VIF_DIFFLANG            0x00000008L
-#define VIF_DIFFCODEPG          0x00000010L
-#define VIF_DIFFTYPE            0x00000020L
-
-#define VIF_WRITEPROT           0x00000040L
-#define VIF_FILEINUSE           0x00000080L
-#define VIF_OUTOFSPACE          0x00000100L
-#define VIF_ACCESSVIOLATION     0x00000200L
-#define VIF_SHARINGVIOLATION    0x00000400L
-#define VIF_CANNOTCREATE        0x00000800L
-#define VIF_CANNOTDELETE        0x00001000L
-#define VIF_CANNOTRENAME        0x00002000L
-#define VIF_CANNOTDELETECUR     0x00004000L
-#define VIF_OUTOFMEMORY         0x00008000L
-
-#define VIF_CANNOTREADSRC       0x00010000L
-#define VIF_CANNOTREADDST       0x00020000L
-
-#define VIF_BUFFTOOSMALL        0x00040000L
-#define VIF_CANNOTLOADLZ32      0x00080000L
-#define VIF_CANNOTLOADCABINET   0x00100000L
-
-#ifndef RC_INVOKED              /* RC doesn't need to see the rest of this */
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-    
-/* 
-    FILE_VER_GET_... flags are for use by 
-    GetFileVersionInfoSizeEx
-    GetFileVersionInfoExW
-*/
-#define FILE_VER_GET_LOCALISED  0x01
-#define FILE_VER_GET_NEUTRAL    0x02
-#define FILE_VER_GET_PREFETCHED 0x04
-
-/* ----- Types and structures ----- */
-
-typedef struct tagVS_FIXEDFILEINFO
-{
-    DWORD   dwSignature;            /* e.g. 0xfeef04bd */
-    DWORD   dwStrucVersion;         /* e.g. 0x00000042 = "0.42" */
-    DWORD   dwFileVersionMS;        /* e.g. 0x00030075 = "3.75" */
-    DWORD   dwFileVersionLS;        /* e.g. 0x00000031 = "0.31" */
-    DWORD   dwProductVersionMS;     /* e.g. 0x00030010 = "3.10" */
-    DWORD   dwProductVersionLS;     /* e.g. 0x00000031 = "0.31" */
-    DWORD   dwFileFlagsMask;        /* = 0x3F for version "0.42" */
-    DWORD   dwFileFlags;            /* e.g. VFF_DEBUG | VFF_PRERELEASE */
-    DWORD   dwFileOS;               /* e.g. VOS_DOS_WINDOWS16 */
-    DWORD   dwFileType;             /* e.g. VFT_DRIVER */
-    DWORD   dwFileSubtype;          /* e.g. VFT2_DRV_KEYBOARD */
-    DWORD   dwFileDateMS;           /* e.g. 0 */
-    DWORD   dwFileDateLS;           /* e.g. 0 */
-} VS_FIXEDFILEINFO;
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif  /* !RC_INVOKED */
-
-#endif /* WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_DESKTOP) */
-#pragma endregion
-

From 86dc7223e9688b008c759785e1e01ef05e788dd4 Mon Sep 17 00:00:00 2001
From: Przemyslaw Skibinski 
Date: Fri, 28 Oct 2016 12:20:02 +0200
Subject: [PATCH 77/92] generate_res.bat: automatic detection if windres.exe is
 available

---
 build/VS2010/zstd/generate_res/generate_res.bat | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/build/VS2010/zstd/generate_res/generate_res.bat b/build/VS2010/zstd/generate_res/generate_res.bat
index b552dcc30..8c3996aa1 100644
--- a/build/VS2010/zstd/generate_res/generate_res.bat
+++ b/build/VS2010/zstd/generate_res/generate_res.bat
@@ -1,3 +1,10 @@
+@echo off
 REM http://stackoverflow.com/questions/708238/how-do-i-add-an-icon-to-a-mingw-gcc-compiled-executable
-REM copy "c:\Program Files (x86)\Windows Kits\8.1\Include\um\verrsrc.h" .
-windres -I ..\..\..\..\lib -O coff -I . -i ..\zstd.rc -o zstd.res
+
+where /q windres.exe
+IF ERRORLEVEL 1 (
+    ECHO The windres.exe is missing. Ensure it is installed and placed in your PATH.
+    EXIT /B
+) ELSE (
+    windres.exe -I ..\..\..\..\lib -O coff -I . -i ..\zstd.rc -o zstd.res
+)

From 76d66dffd02ae9f11263127d333eadc12ccdc99a Mon Sep 17 00:00:00 2001
From: Przemyslaw Skibinski 
Date: Fri, 28 Oct 2016 13:16:06 +0200
Subject: [PATCH 78/92] generate_res moved to programs/windres

---
 .../VS2010/zstd/generate_res/generate_res.bat | 10 ----
 programs/Makefile                             |  4 +-
 .../windres}/verrsrc.h                        |  0
 programs/windres/zstd.rc                      | 51 +++++++++++++++++++
 4 files changed, 54 insertions(+), 11 deletions(-)
 delete mode 100644 build/VS2010/zstd/generate_res/generate_res.bat
 rename {build/VS2010/zstd/generate_res => programs/windres}/verrsrc.h (100%)
 create mode 100644 programs/windres/zstd.rc

diff --git a/build/VS2010/zstd/generate_res/generate_res.bat b/build/VS2010/zstd/generate_res/generate_res.bat
deleted file mode 100644
index 8c3996aa1..000000000
--- a/build/VS2010/zstd/generate_res/generate_res.bat
+++ /dev/null
@@ -1,10 +0,0 @@
-@echo off
-REM http://stackoverflow.com/questions/708238/how-do-i-add-an-icon-to-a-mingw-gcc-compiled-executable
-
-where /q windres.exe
-IF ERRORLEVEL 1 (
-    ECHO The windres.exe is missing. Ensure it is installed and placed in your PATH.
-    EXIT /B
-) ELSE (
-    windres.exe -I ..\..\..\..\lib -O coff -I . -i ..\zstd.rc -o zstd.res
-)
diff --git a/programs/Makefile b/programs/Makefile
index 013f54354..abfb3be40 100644
--- a/programs/Makefile
+++ b/programs/Makefile
@@ -70,7 +70,7 @@ VOID = /dev/null
 endif
 
 
-.PHONY: default all clean clean_decomp_o install uninstall
+.PHONY: default all clean clean_decomp_o install uninstall generate_res
 
 default: zstd
 
@@ -123,6 +123,8 @@ zstd-decompress: clean_decomp_o
 zstd-compress: $(ZSTDCOMMON_FILES) $(ZSTDCOMP_FILES) zstdcli.c fileio.c
 	$(CC)      $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_NODECOMPRESS $^ -o $@$(EXT)
 
+generate_res:
+	windres\generate_res.bat
 
 clean:
 	$(MAKE) -C ../lib clean
diff --git a/build/VS2010/zstd/generate_res/verrsrc.h b/programs/windres/verrsrc.h
similarity index 100%
rename from build/VS2010/zstd/generate_res/verrsrc.h
rename to programs/windres/verrsrc.h
diff --git a/programs/windres/zstd.rc b/programs/windres/zstd.rc
new file mode 100644
index 000000000..f5e404730
--- /dev/null
+++ b/programs/windres/zstd.rc
@@ -0,0 +1,51 @@
+// Microsoft Visual C++ generated resource script.
+//
+
+#include "zstd.h" /* ZSTD_VERSION_STRING */
+#define APSTUDIO_READONLY_SYMBOLS
+#include "verrsrc.h"
+#undef APSTUDIO_READONLY_SYMBOLS
+
+
+#if !defined(AFX_RESOURCE_DLL) || defined(AFX_TARG_ENU)
+LANGUAGE 9, 1
+
+/////////////////////////////////////////////////////////////////////////////
+//
+// Version
+//
+
+VS_VERSION_INFO     VERSIONINFO
+  FILEVERSION       ZSTD_VERSION_MAJOR,ZSTD_VERSION_MINOR,ZSTD_VERSION_RELEASE,0
+  PRODUCTVERSION    ZSTD_VERSION_MAJOR,ZSTD_VERSION_MINOR,ZSTD_VERSION_RELEASE,0
+ FILEFLAGSMASK VS_FFI_FILEFLAGSMASK
+#ifdef _DEBUG
+ FILEFLAGS VS_FF_DEBUG
+#else
+ FILEFLAGS 0x0L
+#endif
+ FILEOS VOS_NT_WINDOWS32
+ FILETYPE VFT_DLL
+ FILESUBTYPE VFT2_UNKNOWN
+BEGIN
+    BLOCK "StringFileInfo"
+    BEGIN
+        BLOCK "040904B0"
+        BEGIN
+            VALUE "CompanyName", "Yann Collet, Facebook, Inc."
+            VALUE "FileDescription", "Zstandard - Fast and efficient compression algorithm"
+            VALUE "FileVersion", ZSTD_VERSION_STRING
+            VALUE "InternalName", "zstd.exe"
+            VALUE "LegalCopyright", "Copyright (c) 2013-present, Yann Collet, Facebook, Inc."
+            VALUE "OriginalFilename", "zstd.exe"
+            VALUE "ProductName", "Zstandard"
+            VALUE "ProductVersion", ZSTD_VERSION_STRING
+        END
+    END
+    BLOCK "VarFileInfo"
+    BEGIN
+        VALUE "Translation", 0x0409, 1200
+    END
+END
+
+#endif

From 589a8f27b74abd23d21523c7b54da095c3f86dc1 Mon Sep 17 00:00:00 2001
From: Przemyslaw Skibinski 
Date: Fri, 28 Oct 2016 13:17:35 +0200
Subject: [PATCH 79/92] generate_res.bat moved to programs/windres

---
 programs/windres/generate_res.bat | 11 +++++++++++
 1 file changed, 11 insertions(+)
 create mode 100644 programs/windres/generate_res.bat

diff --git a/programs/windres/generate_res.bat b/programs/windres/generate_res.bat
new file mode 100644
index 000000000..76e47fa0a
--- /dev/null
+++ b/programs/windres/generate_res.bat
@@ -0,0 +1,11 @@
+@echo off
+REM http://stackoverflow.com/questions/708238/how-do-i-add-an-icon-to-a-mingw-gcc-compiled-executable
+
+where /q windres.exe
+IF ERRORLEVEL 1 (
+    ECHO The windres.exe is missing. Ensure it is installed and placed in your PATH.
+    EXIT /B
+) ELSE (
+    windres.exe -I ..\lib -I windres -i windres\zstd.rc -O coff -F pe-x86-64 -o windres\zstd64.res
+    windres.exe -I ..\lib -I windres -i windres\zstd.rc -O coff -F pe-i386 -o windres\zstd32.res
+)

From bd0bd0fb6506a32787d690671dea2145978505b3 Mon Sep 17 00:00:00 2001
From: Przemyslaw Skibinski 
Date: Fri, 28 Oct 2016 18:15:34 +0200
Subject: [PATCH 80/92] generate RES files during each compilation

---
 programs/Makefile | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/programs/Makefile b/programs/Makefile
index abfb3be40..f5e625f06 100644
--- a/programs/Makefile
+++ b/programs/Makefile
@@ -82,6 +82,9 @@ $(ZSTDDECOMP_O): $(ZSTDDIR)/decompress/zstd_decompress.c
 
 zstd  : $(ZSTDDECOMP_O) $(ZSTD_FILES) $(ZSTDLEGACY_FILES) $(ZDICT_FILES) \
         zstdcli.c fileio.c bench.c datagen.c dibio.c
+ifneq (,$(filter Windows%,$(OS)))
+	windres\generate_res.bat
+endif
 	$(CC)      $(FLAGS) -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT) $^ $(RES_FILE) -o $@$(EXT)
 
 
@@ -90,6 +93,9 @@ $(ZSTDDECOMP32_O): $(ZSTDDIR)/decompress/zstd_decompress.c
 
 zstd32 : $(ZSTDDECOMP32_O) $(ZSTD_FILES) $(ZSTDLEGACY_FILES) $(ZDICT_FILES) \
         zstdcli.c fileio.c bench.c datagen.c dibio.c
+ifneq (,$(filter Windows%,$(OS)))
+	windres\generate_res.bat
+endif
 	$(CC)  -m32 $(FLAGS) -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT) $^ $(RES32_FILE) -o $@$(EXT)
 
 

From 53e7f5c2e4adba6f913abf6be76e336daa05a35a Mon Sep 17 00:00:00 2001
From: Przemyslaw Skibinski 
Date: Fri, 28 Oct 2016 19:24:16 +0200
Subject: [PATCH 81/92] test-zstd-speed.py: converted to python3 and added
 "--timeout"

---
 tests/test-zstd-speed.py | 49 +++++++++++++++++++++++++---------------
 1 file changed, 31 insertions(+), 18 deletions(-)

diff --git a/tests/test-zstd-speed.py b/tests/test-zstd-speed.py
index 56b4d46b9..a556b0337 100755
--- a/tests/test-zstd-speed.py
+++ b/tests/test-zstd-speed.py
@@ -1,4 +1,4 @@
-#! /usr/bin/env python
+#! /usr/bin/env python3
 
 #
 # Copyright (c) 2016-present, Przemyslaw Skibinski, Yann Collet, Facebook, Inc.
@@ -21,7 +21,7 @@ import time
 import traceback
 import hashlib
 
-script_version = 'v1.0.1 (2016-09-15)'
+script_version = 'v1.1.1 (2016-10-28)'
 default_repo_url = 'https://github.com/facebook/zstd.git'
 working_dir_name = 'speedTest'
 working_path = os.getcwd() + '/' + working_dir_name     # /path/to/zstd/tests/speedTest
@@ -31,7 +31,7 @@ pid = str(os.getpid())
 verbose = False
 clang_version = "unknown"
 gcc_version = "unknown"
-
+args = None
 
 
 def hashfile(hasher, fname, blocksize=65536):
@@ -48,17 +48,20 @@ def log(text):
 def execute(command, print_command=True, print_output=False, print_error=True, param_shell=True):
     if print_command:
         log("> " + command)
-    popen = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
-                             shell=param_shell, cwd=execute.cwd)
-    stdout = popen.communicate()[0]
-    stdout_lines = stdout.splitlines()
+    popen = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=param_shell, cwd=execute.cwd)
+    stdout_lines, stderr_lines = popen.communicate(timeout=args.timeout)
+    stderr_lines = stderr_lines.decode("utf-8")
+    stdout_lines = stdout_lines.decode("utf-8")
     if print_output:
-        print('\n'.join(stdout_lines))
+        if stdout_lines:
+            print(stdout_lines)
+        if stderr_lines:
+            print(stderr_lines)
     if popen.returncode is not None and popen.returncode != 0:
-        if not print_output and print_error:
-            print('\n'.join(stdout_lines))
-        raise RuntimeError('\n'.join(stdout_lines))
-    return stdout_lines
+        if stderr_lines and not print_output and print_error:
+            print(stderr_lines)
+        raise RuntimeError(stdout_lines + stderr_lines)
+    return (stdout_lines + stderr_lines).splitlines()
 execute.cwd = None
 
 
@@ -183,8 +186,10 @@ def update_config_file(branch, commit):
     last_commit = None
     commitFileName = working_path + "/commit_" + branch.replace("/", "_") + ".txt"
     if os.path.isfile(commitFileName):
-        last_commit = file(commitFileName, 'r').read()
-    file(commitFileName, 'w').write(commit)
+        with open(commitFileName, 'r') as infile:
+            last_commit = infile.read()
+    with open(commitFileName, 'w') as outfile:
+        outfile.write(commit)
     return last_commit
 
 
@@ -199,7 +204,7 @@ def double_check(branch, commit, args, executableName, md5sum, compilerVersion,
 
 
 def test_commit(branch, commit, last_commit, args, testFilePaths, have_mutt, have_mail):
-    local_branch = string.split(branch, '/')[1]
+    local_branch = branch.split('/')[1]
     version = local_branch.rpartition('-')[2] + '_' + commit
     if not args.dry_run:
         execute('make -C programs clean zstd CC=clang MOREFLAGS="-Werror -Wconversion -Wno-sign-conversion -DZSTD_GIT_COMMIT=%s" && ' % version +
@@ -255,6 +260,7 @@ if __name__ == '__main__':
     parser.add_argument('--maxLoadAvg', type=float, help='maximum load average to start testing', default=0.75)
     parser.add_argument('--lastCLevel', type=int, help='last compression level for testing', default=5)
     parser.add_argument('--sleepTime', '-s', type=int, help='frequency of repository checking in seconds', default=300)
+    parser.add_argument('--timeout', '-t', type=int, help='timeout for executing shell commands', default=600)
     parser.add_argument('--dry-run', dest='dry_run', action='store_true', help='not build', default=False)
     parser.add_argument('--verbose', '-v', action='store_true', help='more verbose logs', default=False)
     args = parser.parse_args()
@@ -301,6 +307,7 @@ if __name__ == '__main__':
         print("ratioLimit=%s" % args.ratioLimit)
         print("lastCLevel=%s" % args.lastCLevel)
         print("sleepTime=%s" % args.sleepTime)
+        print("timeout=%s" % args.timeout)
         print("dry_run=%s" % args.dry_run)
         print("verbose=%s" % args.verbose)
         print("have_mutt=%s have_mail=%s" % (have_mutt, have_mail))
@@ -323,10 +330,18 @@ if __name__ == '__main__':
         exit(1)
 
     send_email(args.emails, '[%s:%s] test-zstd-speed.py %s has been started' % (email_header, pid, script_version), args.message, have_mutt, have_mail)
-    file(pidfile, 'w').write(pid)
+    with open(pidfile, 'w') as the_file:
+        the_file.write(pid)
 
+    branch = ""
+    commit = ""
+    first_time = True
     while True:
         try:
+            if first_time:
+                first_time = False
+            else:
+                time.sleep(args.sleepTime)
             loadavg = os.getloadavg()[0]
             if (loadavg <= args.maxLoadAvg):
                 branches = git_get_branches()
@@ -344,13 +359,11 @@ if __name__ == '__main__':
                 log("WARNING: main loadavg=%.2f is higher than %s" % (loadavg, args.maxLoadAvg))
             if verbose:
                 log("sleep for %s seconds" % args.sleepTime)
-            time.sleep(args.sleepTime)
         except Exception as e:
             stack = traceback.format_exc()
             email_topic = '[%s:%s] ERROR in %s:%s' % (email_header, pid, branch, commit)
             send_email(args.emails, email_topic, stack, have_mutt, have_mail)
             print(stack)
-            time.sleep(args.sleepTime)
         except KeyboardInterrupt:
             os.unlink(pidfile)
             send_email(args.emails, '[%s:%s] test-zstd-speed.py %s has been stopped' % (email_header, pid, script_version), args.message, have_mutt, have_mail)

From 81c334b02415d15e4facfd88e3ad9a29c91624dc Mon Sep 17 00:00:00 2001
From: Przemyslaw Skibinski 
Date: Fri, 28 Oct 2016 20:40:21 +0200
Subject: [PATCH 82/92] test-zstd-speed.py: default timeout changed to 1800
 seconds

---
 tests/test-zstd-speed.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test-zstd-speed.py b/tests/test-zstd-speed.py
index a556b0337..23d4f477c 100755
--- a/tests/test-zstd-speed.py
+++ b/tests/test-zstd-speed.py
@@ -260,7 +260,7 @@ if __name__ == '__main__':
     parser.add_argument('--maxLoadAvg', type=float, help='maximum load average to start testing', default=0.75)
     parser.add_argument('--lastCLevel', type=int, help='last compression level for testing', default=5)
     parser.add_argument('--sleepTime', '-s', type=int, help='frequency of repository checking in seconds', default=300)
-    parser.add_argument('--timeout', '-t', type=int, help='timeout for executing shell commands', default=600)
+    parser.add_argument('--timeout', '-t', type=int, help='timeout for executing shell commands', default=1800)
     parser.add_argument('--dry-run', dest='dry_run', action='store_true', help='not build', default=False)
     parser.add_argument('--verbose', '-v', action='store_true', help='more verbose logs', default=False)
     args = parser.parse_args()

From 22de81e87e7281bb62c9225000b82616dc2686d6 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Fri, 28 Oct 2016 13:58:31 -0700
Subject: [PATCH 83/92] updated man page

---
 NEWS                               |  1 +
 examples/simple_decompression.c    |  2 +-
 examples/streaming_decompression.c |  5 +-
 programs/zstd.1                    | 89 ++++++++++++++++++++++++++----
 programs/zstdcli.c                 | 41 +++++++-------
 5 files changed, 107 insertions(+), 31 deletions(-)

diff --git a/NEWS b/NEWS
index 2c2585f00..1b0c3d6c1 100644
--- a/NEWS
+++ b/NEWS
@@ -5,6 +5,7 @@ Improved : slightly better compression ratio at --ultra levels (>= 20)
 Added : ZSTD_initCStream_usingCDict(), ZSTD_initDStream_usingDDict() (experimental section)
 Added : example/multiple_streaming_compression
 Changed : zstd_errors.h is now part of include installation
+Updated man page
 Fixed : zstd-small, zstd-compress and zstd-decompress compilation targets
 
 v1.1.0
diff --git a/examples/simple_decompression.c b/examples/simple_decompression.c
index 907ee3387..62a881fcb 100644
--- a/examples/simple_decompression.c
+++ b/examples/simple_decompression.c
@@ -65,7 +65,7 @@ static void decompress(const char* fname)
     void* const cBuff = loadFile_X(fname, &cSize);
     unsigned long long const rSize = ZSTD_getDecompressedSize(cBuff, cSize);
     if (rSize==0) {
-        printf("%s : original size unknown \n", fname);
+        printf("%s : original size unknown. Use streaming decompression instead. \n", fname);
         exit(5);
     }
     void* const rBuff = malloc_X((size_t)rSize);
diff --git a/examples/streaming_decompression.c b/examples/streaming_decompression.c
index 1ba1a3d81..400aa673d 100644
--- a/examples/streaming_decompression.c
+++ b/examples/streaming_decompression.c
@@ -71,9 +71,12 @@ static void decompressFile_orDie(const char* fname)
 
     ZSTD_DStream* const dstream = ZSTD_createDStream();
     if (dstream==NULL) { fprintf(stderr, "ZSTD_createDStream() error \n"); exit(10); }
+
+    /* In more complex scenarios, a file may consist of multiple appended frames (ex : pzstd).
+    *  The following example decompresses only the first frame.
+    *  It is compatible with other provided streaming examples */
     size_t const initResult = ZSTD_initDStream(dstream);
     if (ZSTD_isError(initResult)) { fprintf(stderr, "ZSTD_initDStream() error : %s \n", ZSTD_getErrorName(initResult)); exit(11); }
-
     size_t read, toRead = initResult;
     while ( (read = fread_orDie(buffIn, toRead, fin)) ) {
         ZSTD_inBuffer input = { buffIn, read, 0 };
diff --git a/programs/zstd.1 b/programs/zstd.1
index c262a0c6a..812883717 100644
--- a/programs/zstd.1
+++ b/programs/zstd.1
@@ -35,22 +35,85 @@ It also features a very fast decoder, with speed > 500 MB/s per core.
 \fBzstd\fR command line is generally similar to gzip, but features the following differences :
  - Source files are preserved by default
    It's possible to remove them automatically by using \fB--rm\fR command
- - By default, when compressing a single file, \fBzstd\fR displays progress notifications and result summary.
+ - When compressing a single file, \fBzstd\fR displays progress notifications and result summary by default.
      Use \fB-q\fR to turn them off
 
 
 
 .SH OPTIONS
+
+.
+.SS "Integer suffixes and special values"
+In most places where an integer argument is expected,
+an optional suffix is supported to easily indicate large integers.
+There must be no space between the integer and the suffix.
+.TP
+.B KiB
+Multiply the integer by 1,024 (2^10).
+.BR Ki ,
+.BR K ,
+and
+.B KB
+are accepted as synonyms for
+.BR KiB .
+.TP
+.B MiB
+Multiply the integer by 1,048,576 (2^20).
+.BR Mi ,
+.BR M ,
+and
+.B MB
+are accepted as synonyms for
+.BR MiB .
+
+.
+.SS "Operation mode"
+If multiple operation mode options are given,
+the last one takes effect.
+.TP
+.BR \-z ", " \-\-compress
+Compress.
+This is the default operation mode when no operation mode option
+is specified and no other operation mode is implied from
+the command name (for example,
+.B unzstd
+implies
+.BR \-\-decompress ).
+.TP
+.BR \-d ", " \-\-decompress ", " \-\-uncompress
+Decompress.
+.TP
+.BR \-t ", " \-\-test
+Test the integrity of compressed
+.IR files .
+This option is equivalent to
+.B "\-\-decompress \-\-stdout"
+except that the decompressed data is discarded instead of being
+written to standard output.
+No files are created or removed.
+.TP
+.B \-b#
+ benchmark file(s) using compression level #
+.TP
+.B \--train FILEs
+ use FILEs as training set to create a dictionary. The training set should contain a lot of small files (> 100).
+
+.
+.SS "Operation modifiers"
 .TP
 .B \-#
- # compression level [1-22] (default:3)
+ # compression level [1-19] (default:3)
 .TP
-.BR \-d ", " --decompress
- decompression
+.BR \--ultra
+ unlocks high compression levels 20+ (maximum 22), using a lot more memory
 .TP
 .B \-D file
  use `file` as Dictionary to compress or decompress FILE(s)
 .TP
+.BR \--no-dictID
+ do not store dictionary ID within frame header (dictionary compression).
+ the decoder will have to rely on implicit knowledge about which dictionary to use, it won't be able to check if it's correct.
+.TP
 .B \-o file
  save result into `file` (only possible with a single INPUT-FILE)
 .TP
@@ -60,6 +123,11 @@ It also features a very fast decoder, with speed > 500 MB/s per core.
 .BR \-c ", " --stdout
  force write to standard output, even if it is the console
 .TP
+.BR \--[no-]sparse
+ enable / disable sparse FS support, to make files with many zeroes smaller on disk.
+ default : enabled when output is into a file, and disabled when output is stdout.
+ This setting overrides default and can force sparse mode over stdout.
+.TP
 .BR \--rm
  remove source file(s) after successful compression or decompression
 .TP
@@ -83,8 +151,8 @@ It also features a very fast decoder, with speed > 500 MB/s per core.
  suppress warnings, interactivity and notifications.
  specify twice to suppress errors too.
 .TP
-.BR \-C ", " --check
- add integrity check computed from uncompressed data
+.BR \-C ", " --[no-]check
+ add integrity check computed from uncompressed data (default : enabled)
 .TP
 .BR \-t ", " --test
  Test the integrity of compressed files. This option is equivalent to \fB--decompress --stdout > /dev/null\fR.
@@ -103,10 +171,8 @@ It will improve compression ratio of small files.
 Typical gains range from ~10% (at 64KB) to x5 better (at <1KB).
 .TP
 .B \--train FILEs
- use FILEs as training set to create a dictionary.
- The training set should contain a lot of small files (> 100).
- and weight typically 100x the target dictionary size
- (for example, 10 MB for a 100 KB dictionary)
+ use FILEs as training set to create a dictionary. The training set should contain a lot of small files (> 100),
+and weight typically 100x the target dictionary size (for example, 10 MB for a 100 KB dictionary)
 .TP
 .B \-o file
  dictionary saved into `file` (default: dictionary)
@@ -131,6 +197,9 @@ Typical gains range from ~10% (at 64KB) to x5 better (at <1KB).
 .B \-b#
  benchmark file(s) using compression level #
 .TP
+.B \-e#
+ benchmark file(s) using multiple compression levels, from -b# to -e# (included).
+.TP
 .B \-i#
  minimum evaluation time, in seconds (default : 3s), benchmark mode only
 .TP
diff --git a/programs/zstdcli.c b/programs/zstdcli.c
index 9aa97cdd2..c9d2cf5db 100644
--- a/programs/zstdcli.c
+++ b/programs/zstdcli.c
@@ -199,26 +199,24 @@ static unsigned longCommandWArg(const char** stringPtr, const char* longCommand)
     return result;
 }
 
+typedef enum { zom_compress, zom_decompress, zom_test, zom_bench, zom_train } zstd_operation_mode;
 
 #define CLEAN_RETURN(i) { operationResult = (i); goto _end; }
 
 int main(int argCount, const char* argv[])
 {
     int argNb,
-        bench=0,
-        decode=0,
-        testmode=0,
         forceStdout=0,
         main_pause=0,
         nextEntryIsDictionary=0,
         operationResult=0,
-        dictBuild=0,
         nextArgumentIsOutFileName=0,
         nextArgumentIsMaxDict=0,
         nextArgumentIsDictID=0,
         nextArgumentIsFile=0,
         ultra=0,
         lastCommand = 0;
+    zstd_operation_mode operation = zom_compress;
     int cLevel = ZSTDCLI_CLEVEL_DEFAULT;
     int cLevelLast = 1;
     unsigned recursive = 0;
@@ -241,7 +239,7 @@ int main(int argCount, const char* argv[])
     /* init */
     (void)recursive; (void)cLevelLast;    /* not used when ZSTD_NOBENCH set */
     (void)dictCLevel; (void)dictSelect; (void)dictID;  /* not used when ZSTD_NODICT set */
-    (void)decode; (void)cLevel; (void)testmode; /* not used when ZSTD_NOCOMPRESS set */
+    (void)cLevel; /* not used when ZSTD_NOCOMPRESS set */
     (void)ultra; (void)memLimit;   /* not used when ZSTD_NODECOMPRESS set */
     if (filenameTable==NULL) { DISPLAY("zstd: %s \n", strerror(errno)); exit(1); }
     filenameTable[0] = stdinmark;
@@ -253,8 +251,8 @@ int main(int argCount, const char* argv[])
     }
 
     /* preset behaviors */
-    if (!strcmp(programName, ZSTD_UNZSTD)) decode=1;
-    if (!strcmp(programName, ZSTD_CAT)) { decode=1; forceStdout=1; displayLevel=1; outFileName=stdoutmark; }
+    if (!strcmp(programName, ZSTD_UNZSTD)) operation=zom_decompress;
+    if (!strcmp(programName, ZSTD_CAT)) { operation=zom_decompress; forceStdout=1; displayLevel=1; outFileName=stdoutmark; }
 
     /* command switches */
     for (argNb=1; argNb
Date: Fri, 28 Oct 2016 14:24:15 -0700
Subject: [PATCH 84/92] [pzstd] Move -I flags to PZSTD_CPPFLAGS

---
 contrib/pzstd/Makefile | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/contrib/pzstd/Makefile b/contrib/pzstd/Makefile
index 4f63887d3..2de50416c 100644
--- a/contrib/pzstd/Makefile
+++ b/contrib/pzstd/Makefile
@@ -24,16 +24,18 @@ TESTFLAGS ?=
 DEPFLAGS = -MMD -MP -MF $*.Td
 POSTCOMPILE = mv -f $*.Td $*.d
 
-# CFLAGS, CXXFLAGS, and LDFLAGS are for the users to override
+# CFLAGS, CXXFLAGS, CPPFLAGS, and LDFLAGS are for the users to override
 CFLAGS   ?= -O3 -Wall -Wextra
 CXXFLAGS ?= -O3 -Wall -Wextra -pedantic -std=c++11
+CPPFLAGS ?=
 LDFLAGS  ?=
 
-# Googletest default flags
+# Include flags
 PZSTD_INC  = -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(PROGDIR) -I.
 GTEST_INC  = -isystem googletest/googletest/include
 
-PZSTD_CCXXFLAGS = $(PZSTD_INC) $(GTEST_INC)
+PZSTD_CPPFLAGS  = $(PZSTD_INC) $(GTEST_INC)
+PZSTD_CCXXFLAGS =
 PZSTD_CFLAGS    = $(PZSTD_CCXXFLAGS)
 PZSTD_CXXFLAGS  = $(PZSTD_CCXXFLAGS)
 PZSTD_LDFLAGS   =

From b9550d6d43d2487a4b5184f5c94afb565f19ab97 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Fri, 28 Oct 2016 14:43:24 -0700
Subject: [PATCH 85/92] fixed benchmark mode typo . added benchmark mode tests

---
 programs/zstdcli.c |  2 +-
 tests/playTests.sh | 11 +++++++++++
 2 files changed, 12 insertions(+), 1 deletion(-)

diff --git a/programs/zstdcli.c b/programs/zstdcli.c
index c9d2cf5db..db4d6acc6 100644
--- a/programs/zstdcli.c
+++ b/programs/zstdcli.c
@@ -369,7 +369,7 @@ int main(int argCount, const char* argv[])
 
 #ifndef ZSTD_NOBENCH
                         /* Benchmark */
-                    case 'b': operation=zom_train; argument++; break;
+                    case 'b': operation=zom_bench; argument++; break;
 
                         /* range bench (benchmark only) */
                     case 'e':
diff --git a/tests/playTests.sh b/tests/playTests.sh
index c5a58d628..ad70538a3 100755
--- a/tests/playTests.sh
+++ b/tests/playTests.sh
@@ -258,6 +258,17 @@ $ZSTD -t --rm tmp1.zst
 ls -ls tmp1.zst  # check file is still present
 
 
+$ECHO "\n**** benchmark mode tests **** "
+
+$ECHO "bench one file"
+./datagen > tmp1
+$ZSTD -bi1 tmp1
+$ECHO "bench multiple levels"
+$ZSTD -i1b1e3 tmp1
+$ECHO "with recursive and quiet modes"
+$ZSTD -rqi1b1e3 tmp1
+
+
 $ECHO "\n**** zstd round-trip tests **** "
 
 roundTripTest

From 589f01176134f30684a58ac4fb2ecce6fd02407a Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Fri, 28 Oct 2016 15:17:38 -0700
Subject: [PATCH 86/92] changed ZBUFF_* by ZSTD_*Stream() within fullbench, in
 anticipation of future deprecation of ZBUFF_*.

---
 tests/Makefile    |  4 +-
 tests/fullbench.c | 93 ++++++++++++++++++++++++++---------------------
 2 files changed, 53 insertions(+), 44 deletions(-)

diff --git a/tests/Makefile b/tests/Makefile
index 309b5d402..ecff18290 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -79,10 +79,10 @@ zstd32:
 zstd_nolegacy:
 	$(MAKE) -C $(PRGDIR) $@
 
-fullbench  : $(ZSTD_FILES) $(ZBUFF_FILES) $(PRGDIR)/datagen.c fullbench.c
+fullbench  : $(ZSTD_FILES) $(PRGDIR)/datagen.c fullbench.c
 	$(CC)      $(FLAGS) $^ -o $@$(EXT)
 
-fullbench32 : $(ZSTD_FILES) $(ZBUFF_FILES) $(PRGDIR)/datagen.c fullbench.c
+fullbench32 : $(ZSTD_FILES) $(PRGDIR)/datagen.c fullbench.c
 	$(CC)  -m32  $(FLAGS) $^ -o $@$(EXT)
 
 fuzzer  : CPPFLAGS += -I$(ZSTDDIR)/dictBuilder
diff --git a/tests/fullbench.c b/tests/fullbench.c
index 670b51681..ffc32f9bb 100644
--- a/tests/fullbench.c
+++ b/tests/fullbench.c
@@ -22,7 +22,6 @@
 #include "zstd.h"            /* ZSTD_VERSION_STRING */
 #define FSE_STATIC_LINKING_ONLY   /* FSE_DTABLE_SIZE_U32 */
 #include "fse.h"
-#include "zbuff.h"
 #include "datagen.h"
 
 
@@ -130,29 +129,39 @@ size_t local_ZSTD_decodeSeqHeaders(void* dst, size_t dstSize, void* buff2, const
 }
 
 
-static ZBUFF_CCtx* g_zbcc = NULL;
-size_t local_ZBUFF_compress(void* dst, size_t dstCapacity, void* buff2, const void* src, size_t srcSize)
+static ZSTD_CStream* g_cstream= NULL;
+size_t local_ZSTD_compressStream(void* dst, size_t dstCapacity, void* buff2, const void* src, size_t srcSize)
 {
-    size_t compressedSize;
-    size_t srcRead = srcSize, dstWritten = dstCapacity;
+    ZSTD_outBuffer buffOut;
+    ZSTD_inBuffer buffIn;
     (void)buff2;
-    ZBUFF_compressInit(g_zbcc, 1);
-    ZBUFF_compressContinue(g_zbcc, dst, &dstWritten, src, &srcRead);
-    compressedSize = dstWritten;
-    dstWritten = dstCapacity-compressedSize;
-    ZBUFF_compressEnd(g_zbcc, ((char*)dst)+compressedSize, &dstWritten);
-    compressedSize += dstWritten;
-    return compressedSize;
+    ZSTD_initCStream(g_cstream, 1);
+    buffOut.dst = dst;
+    buffOut.size = dstCapacity;
+    buffOut.pos = 0;
+    buffIn.src = src;
+    buffIn.size = srcSize;
+    buffIn.pos = 0;
+    ZSTD_compressStream(g_cstream, &buffOut, &buffIn);
+    ZSTD_endStream(g_cstream, &buffOut);
+    return buffOut.pos;
 }
 
-static ZBUFF_DCtx* g_zbdc = NULL;
-static size_t local_ZBUFF_decompress(void* dst, size_t dstCapacity, void* buff2, const void* src, size_t srcSize)
+static ZSTD_DStream* g_dstream= NULL;
+static size_t local_ZSTD_decompressStream(void* dst, size_t dstCapacity, void* buff2, const void* src, size_t srcSize)
 {
-    size_t srcRead = g_cSize, dstWritten = dstCapacity;
+    ZSTD_outBuffer buffOut;
+    ZSTD_inBuffer buffIn;
     (void)src; (void)srcSize;
-    ZBUFF_decompressInit(g_zbdc);
-    ZBUFF_decompressContinue(g_zbdc, dst, &dstWritten, buff2, &srcRead);
-    return dstWritten;
+    ZSTD_initDStream(g_dstream);
+    buffOut.dst = dst;
+    buffOut.size = dstCapacity;
+    buffOut.pos = 0;
+    buffIn.src = buff2;
+    buffIn.size = g_cSize;
+    buffIn.pos = 0;
+    ZSTD_decompressStream(g_dstream, &buffOut, &buffIn);
+    return buffOut.pos;
 }
 
 static ZSTD_CCtx* g_zcc = NULL;
@@ -220,10 +229,10 @@ static size_t benchMem(const void* src, size_t srcSize, U32 benchNb)
         benchFunction = local_ZSTD_decodeSeqHeaders; benchName = "ZSTD_decodeSeqHeaders";
         break;
     case 41:
-        benchFunction = local_ZBUFF_compress; benchName = "ZBUFF_compressContinue";
+        benchFunction = local_ZSTD_compressStream; benchName = "ZSTD_compressStream";
         break;
     case 42:
-        benchFunction = local_ZBUFF_decompress; benchName = "ZBUFF_decompressContinue";
+        benchFunction = local_ZSTD_decompressStream; benchName = "ZSTD_decompressStream";
         break;
     default :
         return 0;
@@ -296,10 +305,10 @@ static size_t benchMem(const void* src, size_t srcSize, U32 benchNb)
             break;
         }
     case 41 :
-        if (g_zbcc==NULL) g_zbcc = ZBUFF_createCCtx();
+        if (g_cstream==NULL) g_cstream = ZSTD_createCStream();
         break;
     case 42 :
-        if (g_zbdc==NULL) g_zbdc = ZBUFF_createDCtx();
+        if (g_dstream==NULL) g_dstream = ZSTD_createDStream();
         g_cSize = ZSTD_compress(buff2, dstBuffSize, src, srcSize, 1);
         break;
 
@@ -311,27 +320,27 @@ static size_t benchMem(const void* src, size_t srcSize, U32 benchNb)
 
     { size_t i; for (i=0; i %s !! \n", benchName, ZSTD_getErrorName(benchResult)); exit(1); }
-        }
-        averageTime = (((double)BMK_clockSpan(clockStart)) / CLOCKS_PER_SEC) / nbRounds;
-        if (averageTime < bestTime) bestTime = averageTime;
-        DISPLAY("%2i- %-30.30s : %7.1f MB/s  (%9u)\r", loopNb, benchName, (double)srcSize / (1 MB) / bestTime, (U32)benchResult);
-    }}
+            clockStart = clock();
+            while (clock() == clockStart);
+            clockStart = clock();
+            for (nbRounds=0; BMK_clockSpan(clockStart) < timeLoop; nbRounds++) {
+                benchResult = benchFunction(dstBuff, dstBuffSize, buff2, src, srcSize);
+                if (ZSTD_isError(benchResult)) { DISPLAY("ERROR ! %s() => %s !! \n", benchName, ZSTD_getErrorName(benchResult)); exit(1); }
+            }
+            averageTime = (((double)BMK_clockSpan(clockStart)) / CLOCKS_PER_SEC) / nbRounds;
+            if (averageTime < bestTime) bestTime = averageTime;
+            DISPLAY("%2i- %-30.30s : %7.1f MB/s  (%9u)\r", loopNb, benchName, (double)srcSize / (1 MB) / bestTime, (U32)benchResult);
+    }   }
     DISPLAY("%2u\n", benchNb);
 
 _cleanOut:
@@ -466,7 +475,7 @@ int main(int argc, const char** argv)
                 switch(argument[0])
                 {
                     /* Display help on usage */
-                case 'h' :
+                case 'h':
                 case 'H': return usage_advanced(exename);
 
                     /* Pause at the end (hidden option) */

From 31e660e7aa5dc7c3137e7b3635cb5adb5dbf2ac0 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Sat, 29 Oct 2016 03:56:45 -0700
Subject: [PATCH 87/92] more accurate default maximum window size

---
 lib/decompress/zstd_decompress.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c
index 15f98989a..4c47930cd 100644
--- a/lib/decompress/zstd_decompress.c
+++ b/lib/decompress/zstd_decompress.c
@@ -34,7 +34,7 @@
 *  Frames requiring more memory will be rejected.
 */
 #ifndef ZSTD_MAXWINDOWSIZE_DEFAULT
-#  define ZSTD_MAXWINDOWSIZE_DEFAULT (257 << 20)   /* 257 MB */
+#  define ZSTD_MAXWINDOWSIZE_DEFAULT ((1 << ZSTD_WINDOWLOG_MAX) + 1)   /* defined within zstd.h */
 #endif
 
 
@@ -111,7 +111,7 @@ struct ZSTD_DCtx_s
     BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX];
 };  /* typedef'd to ZSTD_DCtx within "zstd.h" */
 
-size_t ZSTD_sizeof_DCtx (const ZSTD_DCtx* dctx) { if (dctx==NULL) return 0; return sizeof(ZSTD_DCtx); }  /* support sizeof on NULL */
+size_t ZSTD_sizeof_DCtx (const ZSTD_DCtx* dctx) { return (dctx==NULL) ? 0 : sizeof(ZSTD_DCtx); }
 
 size_t ZSTD_estimateDCtxSize(void) { return sizeof(ZSTD_DCtx); }
 

From 1fd5b45c6d9c18e02e481734d892287781eea554 Mon Sep 17 00:00:00 2001
From: Przemyslaw Skibinski 
Date: Mon, 31 Oct 2016 10:44:44 +0100
Subject: [PATCH 88/92] updated doc/zstd_manual.html

---
 doc/zstd_manual.html | 33 ++++++++++++++++++---------------
 1 file changed, 18 insertions(+), 15 deletions(-)

diff --git a/doc/zstd_manual.html b/doc/zstd_manual.html
index aac5af9c2..2470a4555 100644
--- a/doc/zstd_manual.html
+++ b/doc/zstd_manual.html
@@ -29,20 +29,20 @@
 
 

 
Introduction
-  Zstd, short for Zstandard, is a fast lossless compression algorithm, targeting real-time compression scenarios
+  zstd, short for Zstandard, is a fast lossless compression algorithm, targeting real-time compression scenarios
   at zlib-level and better compression ratios. The zstd compression library provides in-memory compression and
   decompression functions. The library supports compression levels from 1 up to ZSTD_maxCLevel() which is 22.
-  Levels from 20 to 22 should be used with caution as they require about 300-1300 MB for compression.
+  Levels >= 20, labelled `--ultra`, should be used with caution, as they require more memory.
   Compression can be done in:
     - a single step (described as Simple API)
     - a single step, reusing a context (described as Explicit memory management)
-    - repeated calls of the compression function (described as Streaming compression)
+    - unbounded multiple steps (described as Streaming compression)
   The compression ratio achievable on small data can be highly improved using compression with a dictionary in:
     - a single step (described as Simple dictionary API)
     - a single step, reusing a dictionary (described as Fast dictionary API)
 
-  Advanced and experimantal functions can be accessed using #define ZSTD_STATIC_LINKING_ONLY before including zstd.h.
-  These APIs shall never be used with a dynamic library. 
+  Advanced experimental functions can be accessed using #define ZSTD_STATIC_LINKING_ONLY before including zstd.h.
+  These APIs shall never be used with a dynamic library.
   They are not "stable", their definition may change in the future. Only static linking is allowed.
 

 
@@ -95,10 +95,6 @@ const char* ZSTD_getErrorName(size_t code);     /*!< provides readable strin
 

 
Explicit memory management

 
-
Compression context
typedef struct ZSTD_CCtx_s ZSTD_CCtx;
-ZSTD_CCtx* ZSTD_createCCtx(void);
-size_t     ZSTD_freeCCtx(ZSTD_CCtx* cctx);
-


 
size_t ZSTD_compressCCtx(ZSTD_CCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, int compressionLevel);
 
    Same as ZSTD_compress(), requires an allocated ZSTD_CCtx (see ZSTD_createCCtx()) 
 


@@ -134,12 +130,14 @@ size_t     ZSTD_freeDCtx(ZSTD_DCtx* dctx);
 
Fast dictionary API

 
 
ZSTD_CDict* ZSTD_createCDict(const void* dict, size_t dictSize, int compressionLevel);
-
   Create a digested dictionary, ready to start compression operation without startup delay.
+
   When compressing multiple messages / blocks with the same dictionary, it's recommended to load it just once.
+   ZSTD_createCDict() will create a digested dictionary, ready to start future compression operations without startup delay.
+   ZSTD_CDict can be created once and used by multiple threads concurrently, as its usage is read-only.
    `dict` can be released after ZSTD_CDict creation 
 


 
 
size_t      ZSTD_freeCDict(ZSTD_CDict* CDict);
-
   Function frees memory allocated with ZSTD_createCDict() 
+
   Function frees memory allocated by ZSTD_createCDict() 
 


 
 
size_t ZSTD_compress_usingCDict(ZSTD_CCtx* cctx,
@@ -186,8 +184,11 @@ size_t     ZSTD_freeDCtx(ZSTD_DCtx* dctx);
   A ZSTD_CStream object is required to track streaming operation.
   Use ZSTD_createCStream() and ZSTD_freeCStream() to create/release resources.
   ZSTD_CStream objects can be reused multiple times on consecutive compression operations.
+  It is recommended to re-use ZSTD_CStream in situations where many streaming operations will be achieved consecutively,
+  since it will play nicer with system's memory, by re-using already allocated memory.
+  Use one separate ZSTD_CStream per thread for parallel execution.
 
-  Start by initializing ZSTD_CStream.
+  Start a new compression by initializing ZSTD_CStream.
   Use ZSTD_initCStream() to start a new compression operation.
   Use ZSTD_initCStream_usingDict() for a compression which requires a dictionary.
 
@@ -269,7 +270,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
 
 
Advanced types

 
-
typedef enum { ZSTD_fast, ZSTD_dfast, ZSTD_greedy, ZSTD_lazy, ZSTD_lazy2, ZSTD_btlazy2, ZSTD_btopt } ZSTD_strategy;   /* from faster to stronger */
+
typedef enum { ZSTD_fast, ZSTD_dfast, ZSTD_greedy, ZSTD_lazy, ZSTD_lazy2, ZSTD_btlazy2, ZSTD_btopt, ZSTD_btopt2 } ZSTD_strategy;   /* from faster to stronger */
 


 
typedef struct {
     unsigned windowLog;      /**< largest match distance : larger == more compression, more memory needed during decompression */
@@ -371,20 +372,22 @@ typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; v
 size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel);
 size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize,
                                              ZSTD_parameters params, unsigned long long pledgedSrcSize);  /**< pledgedSrcSize is optional and can be zero == unknown */
-size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize);  /**< re-use compression parameters from previous init; saves dictionary loading */
+size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict);  /**< note : cdict will just be referenced, and must outlive compression session */
+size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize);  /**< re-use compression parameters from previous init; skip dictionary loading stage; zcs must be init at least once before */
 size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs);
 


 
Advanced Streaming decompression functions
typedef enum { ZSTDdsp_maxWindowSize } ZSTD_DStreamParameter_e;
 ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem);
 size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize);
 size_t ZSTD_setDStreamParameter(ZSTD_DStream* zds, ZSTD_DStreamParameter_e paramType, unsigned paramValue);
+size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDict* ddict);  /**< note : ddict will just be referenced, and must outlive decompression session */
 size_t ZSTD_resetDStream(ZSTD_DStream* zds);  /**< re-use decompression parameters from previous init; saves dictionary loading */
 size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds);
 


 
Buffer-less and synchronous inner streaming functions
   This is an advanced API, giving full control over buffer management, for users which need direct control over memory.
   But it's also a complex one, with many restrictions (documented below).
-  Prefer using normal streaming API for an easier experience 
+  Prefer using normal streaming API for an easier experience
  
 

 

From f882e132cfad2579cfc700af9df85dea11469f1c Mon Sep 17 00:00:00 2001
From: Przemyslaw Skibinski 
Date: Mon, 31 Oct 2016 10:51:58 +0100
Subject: [PATCH 89/92] added gen-zstd-manual.sh

---
 contrib/gen_html/gen-zstd-manual.sh | 9 +++++++++
 contrib/gen_html/gen_html.cpp       | 8 ++++----
 2 files changed, 13 insertions(+), 4 deletions(-)
 create mode 100755 contrib/gen_html/gen-zstd-manual.sh

diff --git a/contrib/gen_html/gen-zstd-manual.sh b/contrib/gen_html/gen-zstd-manual.sh
new file mode 100755
index 000000000..57a8b6ea5
--- /dev/null
+++ b/contrib/gen_html/gen-zstd-manual.sh
@@ -0,0 +1,9 @@
+#!/bin/sh
+
+LIBVER_MAJOR_SCRIPT=`sed -n '/define ZSTD_VERSION_MAJOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < ../../lib/zstd.h`
+LIBVER_MINOR_SCRIPT=`sed -n '/define ZSTD_VERSION_MINOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < ../../lib/zstd.h`
+LIBVER_PATCH_SCRIPT=`sed -n '/define ZSTD_VERSION_RELEASE/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < ../../lib/zstd.h`
+LIBVER_SCRIPT=$LIBVER_MAJOR_SCRIPT.$LIBVER_MINOR_SCRIPT.$LIBVER_PATCH_SCRIPT
+
+echo ZSTD_VERSION=$LIBVER_SCRIPT
+./gen_html $LIBVER_SCRIPT ../../lib/zstd.h ./zstd_manual.html
diff --git a/contrib/gen_html/gen_html.cpp b/contrib/gen_html/gen_html.cpp
index 2157829f7..d3ab265a6 100644
--- a/contrib/gen_html/gen_html.cpp
+++ b/contrib/gen_html/gen_html.cpp
@@ -90,7 +90,7 @@ int main(int argc, char *argv[]) {
 
     if (argc < 4) {
         cout << "usage: " << argv[0] << " [zstd_version] [input_file] [output_html]" << endl;
-        exit(0);
+        return 1;
     }
 
     version = "zstd " + string(argv[1]) + " Manual";
@@ -98,14 +98,14 @@ int main(int argc, char *argv[]) {
     istream.open(argv[2], ifstream::in);
     if (!istream.is_open()) {
         cout << "Error opening file " << argv[2] << endl;
-        exit(0);
+        return 1;
     }
 
     ostream.open(argv[3], ifstream::out);
     if (!ostream.is_open()) {
         cout << "Error opening file " << argv[3] << endl;
-        exit(0);
-    }
+        return 1;
+   }
 
     while (getline(istream, line)) {
         input.push_back(line);

From fe9239891867c5fd1ca25f06bdb39eddc6a04c45 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Mon, 31 Oct 2016 15:48:58 -0700
Subject: [PATCH 90/92] updated README, mentioning .zst suffix

---
 README.md | 27 ++++++++++++++-------------
 1 file changed, 14 insertions(+), 13 deletions(-)

diff --git a/README.md b/README.md
index 0ab1b667b..c1b2bda4f 100644
--- a/README.md
+++ b/README.md
@@ -1,7 +1,8 @@
- **Zstd**, short for Zstandard, is a fast lossless compression algorithm,
+ __Zstandard__, or `zstd` as short version, is a fast lossless compression algorithm,
  targeting real-time compression scenarios at zlib-level and better compression ratios.
 
-It is provided as an open-source BSD-licensed **C** library.
+It is provided as an open-source BSD-licensed **C** library,
+and a command line utility producing and decoding `.zst` compressed files.
 For other programming languages,
 you can consult a list of known ports on [Zstandard homepage](http://www.zstd.net/#other-languages).
 
@@ -16,17 +17,17 @@ As a reference, several fast compression algorithms were tested and compared on
 [Silesia compression corpus]: http://sun.aei.polsl.pl/~sdeor/index.php?page=silesia
 
 
-|Name             | Ratio | C.speed | D.speed |
-|-----------------|-------|--------:|--------:|
-|                 |       |   MB/s  |  MB/s   |
-|**zstd 0.8.2 -1**|**2.877**|**330**| **940** |
-| [zlib] 1.2.8 -1 | 2.730 |    95   |   360   |
-| brotli 0.4 -0   | 2.708 |   320   |   375   |
-| QuickLZ 1.5     | 2.237 |   510   |   605   |
-| LZO 2.09        | 2.106 |   610   |   870   |
-| [LZ4] r131      | 2.101 |   620   |  3100   |
-| Snappy 1.1.3    | 2.091 |   480   |  1600   |
-| LZF 3.6         | 2.077 |   375   |   790   |
+| Name                    | Ratio | C.speed | D.speed |
+|-------------------------|-------|--------:|--------:|
+|                         |       |   MB/s  |  MB/s   |
+|**zstd 0.8.2 -1**      |**2.877**| **330** | **940** |
+| [zlib] 1.2.8 deflate -1 | 2.730 |    95   |   360   |
+| brotli 0.4 -0           | 2.708 |   320   |   375   |
+| QuickLZ 1.5             | 2.237 |   510   |   605   |
+| LZO 2.09                | 2.106 |   610   |   870   |
+| [LZ4] r131              | 2.101 |   620   |  3100   |
+| Snappy 1.1.3            | 2.091 |   480   |  1600   |
+| LZF 3.6                 | 2.077 |   375   |   790   |
 
 [zlib]:http://www.zlib.net/
 [LZ4]: http://www.lz4.org/

From 3d197d4c8c66671fc8ef7468abc4d983b03fbdc3 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Mon, 31 Oct 2016 18:10:32 -0700
Subject: [PATCH 91/92] updated man page

---
 README.md       |  2 +-
 programs/zstd.1 | 75 +++++++++++++++++++++++++++++++++++++++++++------
 2 files changed, 67 insertions(+), 10 deletions(-)

diff --git a/README.md b/README.md
index c1b2bda4f..694bfd53f 100644
--- a/README.md
+++ b/README.md
@@ -20,7 +20,7 @@ As a reference, several fast compression algorithms were tested and compared on
 | Name                    | Ratio | C.speed | D.speed |
 |-------------------------|-------|--------:|--------:|
 |                         |       |   MB/s  |  MB/s   |
-|**zstd 0.8.2 -1**      |**2.877**| **330** | **940** |
+| **zstd 0.8.2 -1**     |**2.877**| **330** | **940** |
 | [zlib] 1.2.8 deflate -1 | 2.730 |    95   |   360   |
 | brotli 0.4 -0           | 2.708 |   320   |   375   |
 | QuickLZ 1.5             | 2.237 |   510   |   605   |
diff --git a/programs/zstd.1 b/programs/zstd.1
index 812883717..9d91b5100 100644
--- a/programs/zstd.1
+++ b/programs/zstd.1
@@ -27,16 +27,70 @@ is equivalent to
 
 .SH DESCRIPTION
 .PP
-\fBzstd\fR is a fast lossless compression algorithm.
+\fBzstd\fR is a fast lossless compression algorithm
+and data compression tool,
+with command line syntax similar to \fB gzip (1) \fR and \fB xz (1) \fR .
 It is based on the \fBLZ77\fR family, with further FSE & huff0 entropy stages.
-\fBzstd\fR offers configurable compression speed, with fast modes at > 200 MB/s per core.
-It also features a very fast decoder, with speed > 500 MB/s per core.
+\fBzstd\fR offers highly configurable compression speed,
+with fast modes at > 200 MB/s per core,
+and strong modes nearing lzma compression ratios.
+It also features a very fast decoder, with speeds > 500 MB/s per core.
 
-\fBzstd\fR command line is generally similar to gzip, but features the following differences :
- - Source files are preserved by default
-   It's possible to remove them automatically by using \fB--rm\fR command
+\fBzstd\fR command line syntax is generally similar to gzip,
+but features the following differences :
+ - Source files are preserved by default.
+   It's possible to remove them automatically by using \fB--rm\fR command.
  - When compressing a single file, \fBzstd\fR displays progress notifications and result summary by default.
-     Use \fB-q\fR to turn them off
+   Use \fB-q\fR to turn them off
+
+.PP
+.B zstd
+compresses or decompresses each
+.I file
+according to the selected operation mode.
+If no
+.I files
+are given or
+.I file
+is
+.BR \- ,
+.B zstd
+reads from standard input and writes the processed data
+to standard output.
+.B zstd
+will refuse (display an error and skip the
+.IR file )
+to write compressed data to standard output if it is a terminal.
+Similarly,
+.B zstd
+will refuse to read compressed data
+from standard input if it is a terminal.
+
+.PP
+Unless
+.B \-\-stdout
+is specified,
+.I files
+are written to a new file whose name is derived from the source
+.I file
+name:
+.IP \(bu 3
+When compressing, the suffix
+.B .zst
+is appended to the source filename to get the target filename.
+.IP \(bu 3
+When decompressing, the
+.B .zst
+suffix is removed from the filename to get the target filename.
+
+.SS "Concatenation with .zst files"
+It is possible to concatenate
+.B .zst
+files as is.
+.B zstd
+will decompress such files as if they were a single
+.B .zst
+file.
 
 
 
@@ -112,7 +166,8 @@ No files are created or removed.
 .TP
 .BR \--no-dictID
  do not store dictionary ID within frame header (dictionary compression).
- the decoder will have to rely on implicit knowledge about which dictionary to use, it won't be able to check if it's correct.
+ The decoder will have to rely on implicit knowledge about which dictionary to use,
+it won't be able to check if it's correct.
 .TP
 .B \-o file
  save result into `file` (only possible with a single INPUT-FILE)
@@ -125,6 +180,8 @@ No files are created or removed.
 .TP
 .BR \--[no-]sparse
  enable / disable sparse FS support, to make files with many zeroes smaller on disk.
+ Creating sparse files may save disk space and speed up the decompression
+by reducing the amount of disk I/O.
  default : enabled when output is into a file, and disabled when output is stdout.
  This setting overrides default and can force sparse mode over stdout.
 .TP
@@ -162,7 +219,7 @@ No files are created or removed.
  All arguments after -- are treated as files
 
 
-.SH DICTIONARY
+.SH DICTIONARY BUILDER
 .PP
 \fBzstd\fR offers \fIdictionary\fR compression, useful for very small files and messages.
 It's possible to train \fBzstd\fR with some samples, the result of which is saved into a file called `dictionary`.

From 4c0b44f8ced84c4c8edfa07b564d31e4fa3e8885 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Tue, 1 Nov 2016 11:13:22 -0700
Subject: [PATCH 92/92] minor display improvement in fuzzer when nb of tests is
 very large

---
 NEWS                | 7 ++++---
 tests/fuzzer.c      | 2 +-
 tests/zstreamtest.c | 4 ++--
 3 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/NEWS b/NEWS
index 1b0c3d6c1..7710a07aa 100644
--- a/NEWS
+++ b/NEWS
@@ -2,9 +2,10 @@ v1.1.1
 New : command -M#, --memory=, --memlimit=, --memlimit-decompress= to limit allowed memory consumption
 New : doc/zstd_manual.html, by Przemyslaw Skibinski
 Improved : slightly better compression ratio at --ultra levels (>= 20)
-Added : ZSTD_initCStream_usingCDict(), ZSTD_initDStream_usingDDict() (experimental section)
-Added : example/multiple_streaming_compression
-Changed : zstd_errors.h is now part of include installation
+Improved : better memory usage when using streaming compression API, thanks to @Rogier-5 report
+Added : API : ZSTD_initCStream_usingCDict(), ZSTD_initDStream_usingDDict() (experimental section)
+Added : example/multiple_streaming_compression.c
+Changed : zstd_errors.h is now installed within /include (and replaces errors_public.h)
 Updated man page
 Fixed : zstd-small, zstd-compress and zstd-decompress compilation targets
 
diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index 22f034d13..f8110498d 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -523,7 +523,7 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxD
 
         /* notification */
         if (nbTests >= testNb) { DISPLAYUPDATE(2, "\r%6u/%6u    ", testNb, nbTests); }
-        else { DISPLAYUPDATE(2, "\r%6u      ", testNb); }
+        else { DISPLAYUPDATE(2, "\r%6u          ", testNb); }
 
         FUZ_rand(&coreSeed);
         { U32 const prime1 = 2654435761U; lseed = coreSeed ^ prime1; }
diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c
index 38b6a76cc..aa119e636 100644
--- a/tests/zstreamtest.c
+++ b/tests/zstreamtest.c
@@ -452,8 +452,8 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compres
         size_t maxTestSize;
 
         /* init */
-        DISPLAYUPDATE(2, "\r%6u", testNb);
-        if (nbTests >= testNb) DISPLAYUPDATE(2, "/%6u   ", nbTests);
+        if (nbTests >= testNb) { DISPLAYUPDATE(2, "\r%6u/%6u    ", testNb, nbTests); }
+        else { DISPLAYUPDATE(2, "\r%6u          ", testNb); }
         FUZ_rand(&coreSeed);
         lseed = coreSeed ^ prime1;