diff --git a/build/cmake/lib/CMakeLists.txt b/build/cmake/lib/CMakeLists.txt
index d22ddeada..abcf4ffe2 100644
--- a/build/cmake/lib/CMakeLists.txt
+++ b/build/cmake/lib/CMakeLists.txt
@@ -130,20 +130,20 @@ ELSE ()
SET(LIBRARY_BASE_NAME libzstd)
ENDIF (MSVC)
-# Define static and shared library names
-SET(STATIC_LIBRARY_OUTPUT_NAME ${LIBRARY_BASE_NAME})
-SET(SHARED_LIBRARY_OUTPUT_NAME ${LIBRARY_BASE_NAME}.${LIBVER_MAJOR}.${LIBVER_MINOR}.${LIBVER_RELEASE})
-
IF (MSVC)
IF (CMAKE_SIZEOF_VOID_P MATCHES "8")
- SET(STATIC_LIBRARY_OUTPUT_NAME ${STATIC_LIBRARY_OUTPUT_NAME}_x64)
- SET(SHARED_LIBRARY_OUTPUT_NAME ${SHARED_LIBRARY_OUTPUT_NAME}_x64)
+ SET(LIBRARY_ARCH_SUFFIX "_x64")
ELSE ()
- SET(STATIC_LIBRARY_OUTPUT_NAME ${STATIC_LIBRARY_OUTPUT_NAME}_x86)
- SET(SHARED_LIBRARY_OUTPUT_NAME ${SHARED_LIBRARY_OUTPUT_NAME}_x86)
+ SET(LIBRARY_ARCH_SUFFIX "_x86")
ENDIF (CMAKE_SIZEOF_VOID_P MATCHES "8")
+ELSE ()
+ SET(LIBRARY_ARCH_SUFFIX "")
ENDIF (MSVC)
+# Define static and shared library names
+SET(STATIC_LIBRARY_OUTPUT_NAME ${LIBRARY_BASE_NAME}${LIBRARY_ARCH_SUFFIX} CACHE STRING "Static library output name")
+SET(SHARED_LIBRARY_OUTPUT_NAME ${LIBRARY_BASE_NAME}.${LIBVER_MAJOR}.${LIBVER_MINOR}.${LIBVER_RELEASE}${LIBRARY_ARCH_SUFFIX} CACHE STRING "Shared library output name")
+
SET_TARGET_PROPERTIES(
libzstd_static
PROPERTIES
diff --git a/contrib/gen_html/.gitignore b/contrib/gen_html/.gitignore
new file mode 100644
index 000000000..ccf28c1d6
--- /dev/null
+++ b/contrib/gen_html/.gitignore
@@ -0,0 +1,2 @@
+# make artefact
+gen_html
diff --git a/contrib/gen_html/zstd_manual.html b/contrib/gen_html/zstd_manual.html
new file mode 100644
index 000000000..d6465c0b4
--- /dev/null
+++ b/contrib/gen_html/zstd_manual.html
@@ -0,0 +1,564 @@
+
+
+
+zstd 1.1.2 Manual
+
+
+zstd 1.1.2 Manual
+
+Contents
+
+- Introduction
+- Version
+- Simple API
+- Explicit memory management
+- Simple dictionary API
+- Fast dictionary API
+- Streaming
+- Streaming compression - HowTo
+- Streaming decompression - HowTo
+- START OF ADVANCED AND EXPERIMENTAL FUNCTIONS
+- Advanced types
+- Advanced compression functions
+- Advanced decompression functions
+- Advanced streaming functions
+- Buffer-less and synchronous inner streaming functions
+- Buffer-less streaming compression (synchronous mode)
+- Buffer-less streaming decompression (synchronous mode)
+- Block functions
+
+
+Introduction
+ zstd, short for Zstandard, is a fast lossless compression algorithm, targeting real-time compression scenarios
+ at zlib-level and better compression ratios. The zstd compression library provides in-memory compression and
+ decompression functions. The library supports compression levels from 1 up to ZSTD_maxCLevel() which is 22.
+ Levels >= 20, labelled `--ultra`, should be used with caution, as they require more memory.
+ Compression can be done in:
+ - a single step (described as Simple API)
+ - a single step, reusing a context (described as Explicit memory management)
+ - unbounded multiple steps (described as Streaming compression)
+ The compression ratio achievable on small data can be highly improved using compression with a dictionary in:
+ - a single step (described as Simple dictionary API)
+ - a single step, reusing a dictionary (described as Fast dictionary API)
+
+ Advanced experimental functions can be accessed using #define ZSTD_STATIC_LINKING_ONLY before including zstd.h.
+ These APIs shall never be used with a dynamic library.
+ They are not "stable", their definition may change in the future. Only static linking is allowed.
+
+
+Version
+
+unsigned ZSTD_versionNumber(void); /**< library version number; to be used when checking dll version */
+
+Simple API
+
+size_t ZSTD_compress( void* dst, size_t dstCapacity,
+ const void* src, size_t srcSize,
+ int compressionLevel);
+ Compresses `src` content as a single zstd compressed frame into already allocated `dst`.
+ Hint : compression runs faster if `dstCapacity` >= `ZSTD_compressBound(srcSize)`.
+ @return : compressed size written into `dst` (<= `dstCapacity),
+ or an error code if it fails (which can be tested using ZSTD_isError()).
+
+
+size_t ZSTD_decompress( void* dst, size_t dstCapacity,
+ const void* src, size_t compressedSize);
+ `compressedSize` : must be the _exact_ size of a single compressed frame.
+ `dstCapacity` is an upper bound of originalSize.
+ If user cannot imply a maximum upper bound, it's better to use streaming mode to decompress data.
+ @return : the number of bytes decompressed into `dst` (<= `dstCapacity`),
+ or an errorCode if it fails (which can be tested using ZSTD_isError()).
+
+
+unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize);
+ 'src' is the start of a zstd compressed frame.
+ @return : content size to be decompressed, as a 64-bits value _if known_, 0 otherwise.
+ note 1 : decompressed size is an optional field, that may not be present, especially in streaming mode.
+ When `return==0`, data to decompress could be any size.
+ In which case, it's necessary to use streaming mode to decompress data.
+ Optionally, application can still use ZSTD_decompress() while relying on implied limits.
+ (For example, data may be necessarily cut into blocks <= 16 KB).
+ note 2 : decompressed size is always present when compression is done with ZSTD_compress()
+ note 3 : decompressed size can be very large (64-bits value),
+ potentially larger than what local system can handle as a single memory segment.
+ In which case, it's necessary to use streaming mode to decompress data.
+ note 4 : If source is untrusted, decompressed size could be wrong or intentionally modified.
+ Always ensure result fits within application's authorized limits.
+ Each application can set its own limits.
+ note 5 : when `return==0`, if precise failure cause is needed, use ZSTD_getFrameParams() to know more.
+
+
+Helper functions
int ZSTD_maxCLevel(void); /*!< maximum compression level available */
+size_t ZSTD_compressBound(size_t srcSize); /*!< maximum compressed size in worst case scenario */
+unsigned ZSTD_isError(size_t code); /*!< tells if a `size_t` function result is an error code */
+const char* ZSTD_getErrorName(size_t code); /*!< provides readable string from an error code */
+
+Explicit memory management
+
+size_t ZSTD_compressCCtx(ZSTD_CCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, int compressionLevel);
+ Same as ZSTD_compress(), requires an allocated ZSTD_CCtx (see ZSTD_createCCtx()).
+
+
+Decompression context
typedef struct ZSTD_DCtx_s ZSTD_DCtx;
+ZSTD_DCtx* ZSTD_createDCtx(void);
+size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx);
+
+size_t ZSTD_decompressDCtx(ZSTD_DCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
+ Same as ZSTD_decompress(), requires an allocated ZSTD_DCtx (see ZSTD_createDCtx()).
+
+
+Simple dictionary API
+
+size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx,
+ void* dst, size_t dstCapacity,
+ const void* src, size_t srcSize,
+ const void* dict,size_t dictSize,
+ int compressionLevel);
+ Compression using a predefined Dictionary (see dictBuilder/zdict.h).
+ Note : This function loads the dictionary, resulting in significant startup delay.
+ Note : When `dict == NULL || dictSize < 8` no dictionary is used.
+
+
+size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx,
+ void* dst, size_t dstCapacity,
+ const void* src, size_t srcSize,
+ const void* dict,size_t dictSize);
+ Decompression using a predefined Dictionary (see dictBuilder/zdict.h).
+ Dictionary must be identical to the one used during compression.
+ Note : This function loads the dictionary, resulting in significant startup delay.
+ Note : When `dict == NULL || dictSize < 8` no dictionary is used.
+
+
+Fast dictionary API
+
+ZSTD_CDict* ZSTD_createCDict(const void* dict, size_t dictSize, int compressionLevel);
+ When compressing multiple messages / blocks with the same dictionary, it's recommended to load it just once.
+ ZSTD_createCDict() will create a digested dictionary, ready to start future compression operations without startup delay.
+ ZSTD_CDict can be created once and used by multiple threads concurrently, as its usage is read-only.
+ `dict` can be released after ZSTD_CDict creation.
+
+
+size_t ZSTD_freeCDict(ZSTD_CDict* CDict);
+ Function frees memory allocated by ZSTD_createCDict().
+
+
+size_t ZSTD_compress_usingCDict(ZSTD_CCtx* cctx,
+ void* dst, size_t dstCapacity,
+ const void* src, size_t srcSize,
+ const ZSTD_CDict* cdict);
+ Compression using a digested Dictionary.
+ Faster startup than ZSTD_compress_usingDict(), recommended when same dictionary is used multiple times.
+ Note that compression level is decided during dictionary creation.
+
+
+ZSTD_DDict* ZSTD_createDDict(const void* dict, size_t dictSize);
+ Create a digested dictionary, ready to start decompression operation without startup delay.
+ `dict` can be released after creation.
+
+
+size_t ZSTD_freeDDict(ZSTD_DDict* ddict);
+ Function frees memory allocated with ZSTD_createDDict()
+
+
+size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx,
+ void* dst, size_t dstCapacity,
+ const void* src, size_t srcSize,
+ const ZSTD_DDict* ddict);
+ Decompression using a digested Dictionary.
+ Faster startup than ZSTD_decompress_usingDict(), recommended when same dictionary is used multiple times.
+
+
+Streaming
+
+typedef struct ZSTD_inBuffer_s {
+ const void* src; /**< start of input buffer */
+ size_t size; /**< size of input buffer */
+ size_t pos; /**< position where reading stopped. Will be updated. Necessarily 0 <= pos <= size */
+} ZSTD_inBuffer;
+
+typedef struct ZSTD_outBuffer_s {
+ void* dst; /**< start of output buffer */
+ size_t size; /**< size of output buffer */
+ size_t pos; /**< position where writing stopped. Will be updated. Necessarily 0 <= pos <= size */
+} ZSTD_outBuffer;
+
+Streaming compression - HowTo
+ A ZSTD_CStream object is required to track streaming operation.
+ Use ZSTD_createCStream() and ZSTD_freeCStream() to create/release resources.
+ ZSTD_CStream objects can be reused multiple times on consecutive compression operations.
+ It is recommended to re-use ZSTD_CStream in situations where many streaming operations will be achieved consecutively,
+ since it will play nicer with system's memory, by re-using already allocated memory.
+ Use one separate ZSTD_CStream per thread for parallel execution.
+
+ Start a new compression by initializing ZSTD_CStream.
+ Use ZSTD_initCStream() to start a new compression operation.
+ Use ZSTD_initCStream_usingDict() for a compression which requires a dictionary.
+
+ Use ZSTD_compressStream() repetitively to consume input stream.
+ The function will automatically update both `pos` fields.
+ Note that it may not consume the entire input, in which case `pos < size`,
+ and it's up to the caller to present again remaining data.
+ @return : a size hint, preferred nb of bytes to use as input for next function call
+ (it's just a hint, to help latency a little, any other value will work fine)
+ (note : the size hint is guaranteed to be <= ZSTD_CStreamInSize() )
+ or an error code, which can be tested using ZSTD_isError().
+
+ At any moment, it's possible to flush whatever data remains within buffer, using ZSTD_flushStream().
+ `output->pos` will be updated.
+ Note some content might still be left within internal buffer if `output->size` is too small.
+ @return : nb of bytes still present within internal buffer (0 if it's empty)
+ or an error code, which can be tested using ZSTD_isError().
+
+ ZSTD_endStream() instructs to finish a frame.
+ It will perform a flush and write frame epilogue.
+ The epilogue is required for decoders to consider a frame completed.
+ Similar to ZSTD_flushStream(), it may not be able to flush the full content if `output->size` is too small.
+ In which case, call again ZSTD_endStream() to complete the flush.
+ @return : nb of bytes still present within internal buffer (0 if it's empty)
+ or an error code, which can be tested using ZSTD_isError().
+
+
+
+
+Streaming compression functions
typedef struct ZSTD_CStream_s ZSTD_CStream;
+ZSTD_CStream* ZSTD_createCStream(void);
+size_t ZSTD_freeCStream(ZSTD_CStream* zcs);
+size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel);
+size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuffer* input);
+size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output);
+size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output);
+
+size_t ZSTD_CStreamInSize(void); /**< recommended size for input buffer */
+
+size_t ZSTD_CStreamOutSize(void); /**< recommended size for output buffer. Guarantee to successfully flush at least one complete compressed block in all circumstances. */
+
+Streaming decompression - HowTo
+ A ZSTD_DStream object is required to track streaming operations.
+ Use ZSTD_createDStream() and ZSTD_freeDStream() to create/release resources.
+ ZSTD_DStream objects can be re-used multiple times.
+
+ Use ZSTD_initDStream() to start a new decompression operation,
+ or ZSTD_initDStream_usingDict() if decompression requires a dictionary.
+ @return : recommended first input size
+
+ Use ZSTD_decompressStream() repetitively to consume your input.
+ The function will update both `pos` fields.
+ If `input.pos < input.size`, some input has not been consumed.
+ It's up to the caller to present again remaining data.
+ If `output.pos < output.size`, decoder has flushed everything it could.
+ @return : 0 when a frame is completely decoded and fully flushed,
+ an error code, which can be tested using ZSTD_isError(),
+ any other value > 0, which means there is still some decoding to do to complete current frame.
+ The return value is a suggested next input size (a hint to improve latency) that will never load more than the current frame.
+
+
+
+Streaming decompression functions
typedef struct ZSTD_DStream_s ZSTD_DStream;
+ZSTD_DStream* ZSTD_createDStream(void);
+size_t ZSTD_freeDStream(ZSTD_DStream* zds);
+size_t ZSTD_initDStream(ZSTD_DStream* zds);
+size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inBuffer* input);
+
+size_t ZSTD_DStreamInSize(void); /*!< recommended size for input buffer */
+
+size_t ZSTD_DStreamOutSize(void); /*!< recommended size for output buffer. Guarantee to successfully flush at least one complete block in all circumstances. */
+
+START OF ADVANCED AND EXPERIMENTAL FUNCTIONS
The definitions in this section are considered experimental.
+ They should never be used with a dynamic library, as they may change in the future.
+ They are provided for advanced usages.
+ Use them only in association with static linking.
+
+
+
+Advanced types
+
+typedef enum { ZSTD_fast, ZSTD_dfast, ZSTD_greedy, ZSTD_lazy, ZSTD_lazy2, ZSTD_btlazy2, ZSTD_btopt, ZSTD_btopt2 } ZSTD_strategy; /* from faster to stronger */
+
+typedef struct {
+ unsigned windowLog; /**< largest match distance : larger == more compression, more memory needed during decompression */
+ unsigned chainLog; /**< fully searched segment : larger == more compression, slower, more memory (useless for fast) */
+ unsigned hashLog; /**< dispatch table : larger == faster, more memory */
+ unsigned searchLog; /**< nb of searches : larger == more compression, slower */
+ unsigned searchLength; /**< match length searched : larger == faster decompression, sometimes less compression */
+ unsigned targetLength; /**< acceptable match size for optimal parser (only) : larger == more compression, slower */
+ ZSTD_strategy strategy;
+} ZSTD_compressionParameters;
+
+typedef struct {
+ unsigned contentSizeFlag; /**< 1: content size will be in frame header (if known). */
+ unsigned checksumFlag; /**< 1: will generate a 22-bits checksum at end of frame, to be used for error detection by decompressor */
+ unsigned noDictIDFlag; /**< 1: no dict ID will be saved into frame header (if dictionary compression) */
+} ZSTD_frameParameters;
+
+typedef struct {
+ ZSTD_compressionParameters cParams;
+ ZSTD_frameParameters fParams;
+} ZSTD_parameters;
+
+Custom memory allocation functions
typedef void* (*ZSTD_allocFunction) (void* opaque, size_t size);
+typedef void (*ZSTD_freeFunction) (void* opaque, void* address);
+typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; void* opaque; } ZSTD_customMem;
+
+Advanced compression functions
+
+size_t ZSTD_estimateCCtxSize(ZSTD_compressionParameters cParams);
+ Gives the amount of memory allocated for a ZSTD_CCtx given a set of compression parameters.
+ `frameContentSize` is an optional parameter, provide `0` if unknown
+
+
+ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem);
+ Create a ZSTD compression context using external alloc and free functions
+
+
+size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx);
+ Gives the amount of memory used by a given ZSTD_CCtx
+
+
+ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictSize,
+ ZSTD_parameters params, ZSTD_customMem customMem);
+ Create a ZSTD_CDict using external alloc and free, and customized compression parameters
+
+
+size_t ZSTD_sizeof_CDict(const ZSTD_CDict* cdict);
+ Gives the amount of memory used by a given ZSTD_sizeof_CDict
+
+
+ZSTD_parameters ZSTD_getParams(int compressionLevel, unsigned long long srcSize, size_t dictSize);
+ same as ZSTD_getCParams(), but @return a full `ZSTD_parameters` object instead of a `ZSTD_compressionParameters`.
+ All fields of `ZSTD_frameParameters` are set to default (0)
+
+
+ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, unsigned long long srcSize, size_t dictSize);
+ @return ZSTD_compressionParameters structure for a selected compression level and srcSize.
+ `srcSize` value is optional, select 0 if not known
+
+
+size_t ZSTD_checkCParams(ZSTD_compressionParameters params);
+ Ensure param values remain within authorized range
+
+
+ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, unsigned long long srcSize, size_t dictSize);
+ optimize params for a given `srcSize` and `dictSize`.
+ both values are optional, select `0` if unknown.
+
+
+size_t ZSTD_compress_advanced (ZSTD_CCtx* ctx,
+ void* dst, size_t dstCapacity,
+ const void* src, size_t srcSize,
+ const void* dict,size_t dictSize,
+ ZSTD_parameters params);
+ Same as ZSTD_compress_usingDict(), with fine-tune control of each compression parameter
+
+
+Advanced decompression functions
+
+unsigned ZSTD_isFrame(const void* buffer, size_t size);
+ Tells if the content of `buffer` starts with a valid Frame Identifier.
+ Note : Frame Identifier is 4 bytes. If `size < 4`, @return will always be 0.
+ Note 2 : Legacy Frame Identifiers are considered valid only if Legacy Support is enabled.
+ Note 3 : Skippable Frame Identifiers are considered valid.
+
+
+size_t ZSTD_estimateDCtxSize(void);
+ Gives the potential amount of memory allocated to create a ZSTD_DCtx
+
+
+ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem);
+ Create a ZSTD decompression context using external alloc and free functions
+
+
+size_t ZSTD_sizeof_DCtx(const ZSTD_DCtx* dctx);
+ Gives the amount of memory used by a given ZSTD_DCtx
+
+
+size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
+ Gives the amount of memory used by a given ZSTD_DDict
+
+
+unsigned ZSTD_getDictID_fromDict(const void* dict, size_t dictSize);
+ Provides the dictID stored within dictionary.
+ if @return == 0, the dictionary is not conformant with Zstandard specification.
+ It can still be loaded, but as a content-only dictionary.
+
+
+unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict);
+ Provides the dictID of the dictionary loaded into `ddict`.
+ If @return == 0, the dictionary is not conformant to Zstandard specification, or empty.
+ Non-conformant dictionaries can still be loaded, but as content-only dictionaries.
+
+
+unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize);
+ Provides the dictID required to decompressed the frame stored within `src`.
+ If @return == 0, the dictID could not be decoded.
+ This could for one of the following reasons :
+ - The frame does not require a dictionary to be decoded (most common case).
+ - The frame was built with dictID intentionally removed. Whatever dictionary is necessary is a hidden information.
+ Note : this use case also happens when using a non-conformant dictionary.
+ - `srcSize` is too small, and as a result, the frame header could not be decoded (only possible if `srcSize < ZSTD_FRAMEHEADERSIZE_MAX`).
+ - This is not a Zstandard frame.
+ When identifying the exact failure cause, it's possible to used ZSTD_getFrameParams(), which will provide a more precise error code.
+
+
+Advanced streaming functions
+
+Advanced Streaming compression functions
ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem);
+size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel);
+size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize,
+ ZSTD_parameters params, unsigned long long pledgedSrcSize); /**< pledgedSrcSize is optional and can be zero == unknown */
+size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict); /**< note : cdict will just be referenced, and must outlive compression session */
+size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize); /**< re-use compression parameters from previous init; skip dictionary loading stage; zcs must be init at least once before */
+size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs);
+
+Advanced Streaming decompression functions
typedef enum { ZSTDdsp_maxWindowSize } ZSTD_DStreamParameter_e;
+ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem);
+size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize);
+size_t ZSTD_setDStreamParameter(ZSTD_DStream* zds, ZSTD_DStreamParameter_e paramType, unsigned paramValue);
+size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDict* ddict); /**< note : ddict will just be referenced, and must outlive decompression session */
+size_t ZSTD_resetDStream(ZSTD_DStream* zds); /**< re-use decompression parameters from previous init; saves dictionary loading */
+size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds);
+
+Buffer-less and synchronous inner streaming functions
+ This is an advanced API, giving full control over buffer management, for users which need direct control over memory.
+ But it's also a complex one, with many restrictions (documented below).
+ Prefer using normal streaming API for an easier experience
+
+
+
+Buffer-less streaming compression (synchronous mode)
+ A ZSTD_CCtx object is required to track streaming operations.
+ Use ZSTD_createCCtx() / ZSTD_freeCCtx() to manage resource.
+ ZSTD_CCtx object can be re-used multiple times within successive compression operations.
+
+ Start by initializing a context.
+ Use ZSTD_compressBegin(), or ZSTD_compressBegin_usingDict() for dictionary compression,
+ or ZSTD_compressBegin_advanced(), for finer parameter control.
+ It's also possible to duplicate a reference context which has already been initialized, using ZSTD_copyCCtx()
+
+ Then, consume your input using ZSTD_compressContinue().
+ There are some important considerations to keep in mind when using this advanced function :
+ - ZSTD_compressContinue() has no internal buffer. It uses externally provided buffer only.
+ - Interface is synchronous : input is consumed entirely and produce 1+ (or more) compressed blocks.
+ - Caller must ensure there is enough space in `dst` to store compressed data under worst case scenario.
+ Worst case evaluation is provided by ZSTD_compressBound().
+ ZSTD_compressContinue() doesn't guarantee recover after a failed compression.
+ - ZSTD_compressContinue() presumes prior input ***is still accessible and unmodified*** (up to maximum distance size, see WindowLog).
+ It remembers all previous contiguous blocks, plus one separated memory segment (which can itself consists of multiple contiguous blocks)
+ - ZSTD_compressContinue() detects that prior input has been overwritten when `src` buffer overlaps.
+ In which case, it will "discard" the relevant memory section from its history.
+
+ Finish a frame with ZSTD_compressEnd(), which will write the last block(s) and optional checksum.
+ It's possible to use a NULL,0 src content, in which case, it will write a final empty block to end the frame,
+ Without last block mark, frames will be considered unfinished (broken) by decoders.
+
+ You can then reuse `ZSTD_CCtx` (ZSTD_compressBegin()) to compress some new frame.
+
+
+Buffer-less streaming compression functions
size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel);
+size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel);
+size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize);
+size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize);
+size_t ZSTD_compressContinue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
+size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
+
+Buffer-less streaming decompression (synchronous mode)
+ A ZSTD_DCtx object is required to track streaming operations.
+ Use ZSTD_createDCtx() / ZSTD_freeDCtx() to manage it.
+ A ZSTD_DCtx object can be re-used multiple times.
+
+ First typical operation is to retrieve frame parameters, using ZSTD_getFrameParams().
+ It fills a ZSTD_frameParams structure which provide important information to correctly decode the frame,
+ such as the minimum rolling buffer size to allocate to decompress data (`windowSize`),
+ and the dictionary ID used.
+ (Note : content size is optional, it may not be present. 0 means : content size unknown).
+ Note that these values could be wrong, either because of data malformation, or because an attacker is spoofing deliberate false information.
+ As a consequence, check that values remain within valid application range, especially `windowSize`, before allocation.
+ Each application can set its own limit, depending on local restrictions. For extended interoperability, it is recommended to support at least 8 MB.
+ Frame parameters are extracted from the beginning of the compressed frame.
+ Data fragment must be large enough to ensure successful decoding, typically `ZSTD_frameHeaderSize_max` bytes.
+ @result : 0 : successful decoding, the `ZSTD_frameParams` structure is correctly filled.
+ >0 : `srcSize` is too small, please provide at least @result bytes on next attempt.
+ errorCode, which can be tested using ZSTD_isError().
+
+ Start decompression, with ZSTD_decompressBegin() or ZSTD_decompressBegin_usingDict().
+ Alternatively, you can copy a prepared context, using ZSTD_copyDCtx().
+
+ Then use ZSTD_nextSrcSizeToDecompress() and ZSTD_decompressContinue() alternatively.
+ ZSTD_nextSrcSizeToDecompress() tells how many bytes to provide as 'srcSize' to ZSTD_decompressContinue().
+ ZSTD_decompressContinue() requires this _exact_ amount of bytes, or it will fail.
+
+ @result of ZSTD_decompressContinue() is the number of bytes regenerated within 'dst' (necessarily <= dstCapacity).
+ It can be zero, which is not an error; it just means ZSTD_decompressContinue() has decoded some metadata item.
+ It can also be an error code, which can be tested with ZSTD_isError().
+
+ ZSTD_decompressContinue() needs previous data blocks during decompression, up to `windowSize`.
+ They should preferably be located contiguously, prior to current block.
+ Alternatively, a round buffer of sufficient size is also possible. Sufficient size is determined by frame parameters.
+ ZSTD_decompressContinue() is very sensitive to contiguity,
+ if 2 blocks don't follow each other, make sure that either the compressor breaks contiguity at the same place,
+ or that previous contiguous segment is large enough to properly handle maximum back-reference.
+
+ A frame is fully decoded when ZSTD_nextSrcSizeToDecompress() returns zero.
+ Context can then be reset to start a new decompression.
+
+ Note : it's possible to know if next input to present is a header or a block, using ZSTD_nextInputType().
+ This information is not required to properly decode a frame.
+
+ == Special case : skippable frames ==
+
+ Skippable frames allow integration of user-defined data into a flow of concatenated frames.
+ Skippable frames will be ignored (skipped) by a decompressor. The format of skippable frames is as follows :
+ a) Skippable frame ID - 4 Bytes, Little endian format, any value from 0x184D2A50 to 0x184D2A5F
+ b) Frame Size - 4 Bytes, Little endian format, unsigned 32-bits
+ c) Frame Content - any content (User Data) of length equal to Frame Size
+ For skippable frames ZSTD_decompressContinue() always returns 0.
+ For skippable frames ZSTD_getFrameParams() returns fparamsPtr->windowLog==0 what means that a frame is skippable.
+ It also returns Frame Size as fparamsPtr->frameContentSize.
+
+
+typedef struct {
+ unsigned long long frameContentSize;
+ unsigned windowSize;
+ unsigned dictID;
+ unsigned checksumFlag;
+} ZSTD_frameParams;
+
+Buffer-less streaming decompression functions
size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t srcSize); /**< doesn't consume input, see details below */
+size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx);
+size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t dictSize);
+void ZSTD_copyDCtx(ZSTD_DCtx* dctx, const ZSTD_DCtx* preparedDCtx);
+size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx);
+size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
+typedef enum { ZSTDnit_frameHeader, ZSTDnit_blockHeader, ZSTDnit_block, ZSTDnit_lastBlock, ZSTDnit_checksum, ZSTDnit_skippableFrame } ZSTD_nextInputType_e;
+ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx);
+
+Block functions
+ Block functions produce and decode raw zstd blocks, without frame metadata.
+ Frame metadata cost is typically ~18 bytes, which can be non-negligible for very small blocks (< 100 bytes).
+ User will have to take in charge required information to regenerate data, such as compressed and content sizes.
+
+ A few rules to respect :
+ - Compressing and decompressing require a context structure
+ + Use ZSTD_createCCtx() and ZSTD_createDCtx()
+ - It is necessary to init context before starting
+ + compression : ZSTD_compressBegin()
+ + decompression : ZSTD_decompressBegin()
+ + variants _usingDict() are also allowed
+ + copyCCtx() and copyDCtx() work too
+ - Block size is limited, it must be <= ZSTD_getBlockSizeMax()
+ + If you need to compress more, cut data into multiple blocks
+ + Consider using the regular ZSTD_compress() instead, as frame metadata costs become negligible when source size is large.
+ - When a block is considered not compressible enough, ZSTD_compressBlock() result will be zero.
+ In which case, nothing is produced into `dst`.
+ + User must test for such outcome and deal directly with uncompressed data
+ + ZSTD_decompressBlock() doesn't accept uncompressed data as input !!!
+ + In case of multiple successive blocks, decoder must be informed of uncompressed block existence to follow proper history.
+ Use ZSTD_insertBlock() in such a case.
+
+
+Raw zstd block functions
size_t ZSTD_getBlockSizeMax(ZSTD_CCtx* cctx);
+size_t ZSTD_compressBlock (ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
+size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
+size_t ZSTD_insertBlock(ZSTD_DCtx* dctx, const void* blockStart, size_t blockSize); /**< insert block into `dctx` history. Useful for uncompressed blocks */
+
+
+
diff --git a/lib/deprecated/zbuff.h b/lib/deprecated/zbuff.h
index 8e55c39a5..4956aaaa4 100644
--- a/lib/deprecated/zbuff.h
+++ b/lib/deprecated/zbuff.h
@@ -15,17 +15,19 @@
* See 'lib/README.md'.
*****************************************************************/
-#ifndef ZSTD_BUFFERED_H_23987
-#define ZSTD_BUFFERED_H_23987
#if defined (__cplusplus)
extern "C" {
#endif
+#ifndef ZSTD_BUFFERED_H_23987
+#define ZSTD_BUFFERED_H_23987
+
/* *************************************
* Dependencies
***************************************/
#include /* size_t */
+#include /* ZSTD_CStream, ZSTD_DStream */
/* ***************************************************************
@@ -48,7 +50,7 @@ extern "C" {
#ifdef ZBUFF_DISABLE_DEPRECATE_WARNINGS
# define ZBUFF_DEPRECATED(message) /* disable deprecation warnings */
#else
-# if (defined(__GNUC__) && (__GNUC__ >= 4)) || defined(__clang__)
+# if (defined(__GNUC__) && (__GNUC__ >= 5)) || defined(__clang__)
# define ZBUFF_DEPRECATED(message) __attribute__((deprecated(message)))
# elif defined(__GNUC__) && (__GNUC__ >= 3)
# define ZBUFF_DEPRECATED(message) __attribute__((deprecated))
@@ -70,7 +72,7 @@ extern "C" {
* ZBUFF and ZSTD are 100% interoperable,
* frames created by one can be decoded by the other one */
-typedef struct ZBUFF_CCtx_s ZBUFF_CCtx;
+typedef ZSTD_CStream ZBUFF_CCtx;
ZBUFF_DEPRECATED("use ZSTD_createCStream") ZBUFF_CCtx* ZBUFF_createCCtx(void);
ZBUFF_DEPRECATED("use ZSTD_freeCStream") size_t ZBUFF_freeCCtx(ZBUFF_CCtx* cctx);
@@ -122,7 +124,7 @@ ZBUFF_DEPRECATED("use ZSTD_endStream") size_t ZBUFF_compressEnd(ZBUFF_CCtx*
* **************************************************/
-typedef struct ZBUFF_DCtx_s ZBUFF_DCtx;
+typedef ZSTD_DStream ZBUFF_DCtx;
ZBUFF_DEPRECATED("use ZSTD_createDStream") ZBUFF_DCtx* ZBUFF_createDCtx(void);
ZBUFF_DEPRECATED("use ZSTD_freeDStream") size_t ZBUFF_freeDCtx(ZBUFF_DCtx* dctx);
@@ -172,9 +174,14 @@ ZBUFF_DEPRECATED("use ZSTD_CStreamOutSize") size_t ZBUFF_recommendedCOutSize(voi
ZBUFF_DEPRECATED("use ZSTD_DStreamInSize") size_t ZBUFF_recommendedDInSize(void);
ZBUFF_DEPRECATED("use ZSTD_DStreamOutSize") size_t ZBUFF_recommendedDOutSize(void);
+#endif /* ZSTD_BUFFERED_H_23987 */
+
#ifdef ZBUFF_STATIC_LINKING_ONLY
+#ifndef ZBUFF_STATIC_H_30298098432
+#define ZBUFF_STATIC_H_30298098432
+
/* ====================================================================================
* The definitions in this section are considered experimental.
* They should never be used in association with a dynamic library, as they may change in the future.
@@ -203,11 +210,11 @@ ZBUFF_DEPRECATED("use ZSTD_initDStream_usingDict") size_t ZBUFF_compressInit_adv
ZSTD_parameters params, unsigned long long pledgedSrcSize);
-#endif /* ZBUFF_STATIC_LINKING_ONLY */
+#endif /* ZBUFF_STATIC_H_30298098432 */
+
+#endif /* ZBUFF_STATIC_LINKING_ONLY */
#if defined (__cplusplus)
}
#endif
-
-#endif /* ZSTD_BUFFERED_H_23987 */
diff --git a/lib/deprecated/zbuff_compress.c b/lib/deprecated/zbuff_compress.c
index 5095b43e6..5a37a0027 100644
--- a/lib/deprecated/zbuff_compress.c
+++ b/lib/deprecated/zbuff_compress.c
@@ -12,19 +12,10 @@
/* *************************************
* Dependencies
***************************************/
-#include
-#include "error_private.h"
-#include "zstd_internal.h" /* MIN, ZSTD_BLOCKHEADERSIZE, defaultCustomMem */
#define ZBUFF_STATIC_LINKING_ONLY
#include "zbuff.h"
-/* *************************************
-* Constants
-***************************************/
-static size_t const ZBUFF_endFrameSize = ZSTD_BLOCKHEADERSIZE;
-
-
/*-***********************************************************
* Streaming compression
*
@@ -58,59 +49,19 @@ static size_t const ZBUFF_endFrameSize = ZSTD_BLOCKHEADERSIZE;
* output : ZSTD_compressBound(ZSTD_BLOCKSIZE_MAX) + ZSTD_blockHeaderSize + ZBUFF_endFrameSize : ensures it's always possible to write/flush/end a full block at best speed.
* ***********************************************************/
-typedef enum { ZBUFFcs_init, ZBUFFcs_load, ZBUFFcs_flush, ZBUFFcs_final } ZBUFF_cStage;
-
-/* *** Resources *** */
-struct ZBUFF_CCtx_s {
- ZSTD_CCtx* zc;
- char* inBuff;
- size_t inBuffSize;
- size_t inToCompress;
- size_t inBuffPos;
- size_t inBuffTarget;
- size_t blockSize;
- char* outBuff;
- size_t outBuffSize;
- size_t outBuffContentSize;
- size_t outBuffFlushedSize;
- ZBUFF_cStage stage;
- U32 checksum;
- U32 frameEnded;
- ZSTD_customMem customMem;
-}; /* typedef'd tp ZBUFF_CCtx within "zbuff.h" */
-
ZBUFF_CCtx* ZBUFF_createCCtx(void)
{
- return ZBUFF_createCCtx_advanced(defaultCustomMem);
+ return ZSTD_createCStream();
}
ZBUFF_CCtx* ZBUFF_createCCtx_advanced(ZSTD_customMem customMem)
{
- ZBUFF_CCtx* zbc;
-
- if (!customMem.customAlloc && !customMem.customFree)
- customMem = defaultCustomMem;
-
- if (!customMem.customAlloc || !customMem.customFree)
- return NULL;
-
- zbc = (ZBUFF_CCtx*)customMem.customAlloc(customMem.opaque, sizeof(ZBUFF_CCtx));
- if (zbc==NULL) return NULL;
- memset(zbc, 0, sizeof(ZBUFF_CCtx));
- memcpy(&zbc->customMem, &customMem, sizeof(ZSTD_customMem));
- zbc->zc = ZSTD_createCCtx_advanced(customMem);
- if (zbc->zc == NULL) { ZBUFF_freeCCtx(zbc); return NULL; }
- return zbc;
+ return ZSTD_createCStream_advanced(customMem);
}
size_t ZBUFF_freeCCtx(ZBUFF_CCtx* zbc)
{
- if (zbc==NULL) return 0; /* support free on NULL */
- ZSTD_freeCCtx(zbc->zc);
- if (zbc->inBuff) zbc->customMem.customFree(zbc->customMem.opaque, zbc->inBuff);
- if (zbc->outBuff) zbc->customMem.customFree(zbc->customMem.opaque, zbc->outBuff);
- zbc->customMem.customFree(zbc->customMem.opaque, zbc);
- return 0;
+ return ZSTD_freeCStream(zbc);
}
@@ -120,148 +71,40 @@ size_t ZBUFF_compressInit_advanced(ZBUFF_CCtx* zbc,
const void* dict, size_t dictSize,
ZSTD_parameters params, unsigned long long pledgedSrcSize)
{
- /* allocate buffers */
- { size_t const neededInBuffSize = (size_t)1 << params.cParams.windowLog;
- if (zbc->inBuffSize < neededInBuffSize) {
- zbc->inBuffSize = neededInBuffSize;
- zbc->customMem.customFree(zbc->customMem.opaque, zbc->inBuff); /* should not be necessary */
- zbc->inBuff = (char*)zbc->customMem.customAlloc(zbc->customMem.opaque, neededInBuffSize);
- if (zbc->inBuff == NULL) return ERROR(memory_allocation);
- }
- zbc->blockSize = MIN(ZSTD_BLOCKSIZE_ABSOLUTEMAX, neededInBuffSize);
- }
- if (zbc->outBuffSize < ZSTD_compressBound(zbc->blockSize)+1) {
- zbc->outBuffSize = ZSTD_compressBound(zbc->blockSize)+1;
- zbc->customMem.customFree(zbc->customMem.opaque, zbc->outBuff); /* should not be necessary */
- zbc->outBuff = (char*)zbc->customMem.customAlloc(zbc->customMem.opaque, zbc->outBuffSize);
- if (zbc->outBuff == NULL) return ERROR(memory_allocation);
- }
-
- { size_t const errorCode = ZSTD_compressBegin_advanced(zbc->zc, dict, dictSize, params, pledgedSrcSize);
- if (ZSTD_isError(errorCode)) return errorCode; }
-
- zbc->inToCompress = 0;
- zbc->inBuffPos = 0;
- zbc->inBuffTarget = zbc->blockSize;
- zbc->outBuffContentSize = zbc->outBuffFlushedSize = 0;
- zbc->stage = ZBUFFcs_load;
- zbc->checksum = params.fParams.checksumFlag > 0;
- zbc->frameEnded = 0;
- return 0; /* ready to go */
+ return ZSTD_initCStream_advanced(zbc, dict, dictSize, params, pledgedSrcSize);
}
size_t ZBUFF_compressInitDictionary(ZBUFF_CCtx* zbc, const void* dict, size_t dictSize, int compressionLevel)
{
- ZSTD_parameters const params = ZSTD_getParams(compressionLevel, 0, dictSize);
- return ZBUFF_compressInit_advanced(zbc, dict, dictSize, params, 0);
+ return ZSTD_initCStream_usingDict(zbc, dict, dictSize, compressionLevel);
}
size_t ZBUFF_compressInit(ZBUFF_CCtx* zbc, int compressionLevel)
{
- return ZBUFF_compressInitDictionary(zbc, NULL, 0, compressionLevel);
+ return ZSTD_initCStream(zbc, compressionLevel);
}
-
-/* internal util function */
-MEM_STATIC size_t ZBUFF_limitCopy(void* dst, size_t dstCapacity, const void* src, size_t srcSize)
-{
- size_t const length = MIN(dstCapacity, srcSize);
- memcpy(dst, src, length);
- return length;
-}
-
-
/* ====== Compression ====== */
-typedef enum { zbf_gather, zbf_flush, zbf_end } ZBUFF_flush_e;
-
-static size_t ZBUFF_compressContinue_generic(ZBUFF_CCtx* zbc,
- void* dst, size_t* dstCapacityPtr,
- const void* src, size_t* srcSizePtr,
- ZBUFF_flush_e const flush)
-{
- U32 someMoreWork = 1;
- const char* const istart = (const char*)src;
- const char* const iend = istart + *srcSizePtr;
- const char* ip = istart;
- char* const ostart = (char*)dst;
- char* const oend = ostart + *dstCapacityPtr;
- char* op = ostart;
-
- while (someMoreWork) {
- switch(zbc->stage)
- {
- case ZBUFFcs_init: return ERROR(init_missing); /* call ZBUFF_compressInit() first ! */
-
- case ZBUFFcs_load:
- /* complete inBuffer */
- { size_t const toLoad = zbc->inBuffTarget - zbc->inBuffPos;
- size_t const loaded = ZBUFF_limitCopy(zbc->inBuff + zbc->inBuffPos, toLoad, ip, iend-ip);
- zbc->inBuffPos += loaded;
- ip += loaded;
- if ( (zbc->inBuffPos==zbc->inToCompress) || (!flush && (toLoad != loaded)) ) {
- someMoreWork = 0; break; /* not enough input to get a full block : stop there, wait for more */
- } }
- /* compress current block (note : this stage cannot be stopped in the middle) */
- { void* cDst;
- size_t cSize;
- size_t const iSize = zbc->inBuffPos - zbc->inToCompress;
- size_t oSize = oend-op;
- if (oSize >= ZSTD_compressBound(iSize))
- cDst = op; /* compress directly into output buffer (avoid flush stage) */
- else
- cDst = zbc->outBuff, oSize = zbc->outBuffSize;
- cSize = (flush == zbf_end) ?
- ZSTD_compressEnd(zbc->zc, cDst, oSize, zbc->inBuff + zbc->inToCompress, iSize) :
- ZSTD_compressContinue(zbc->zc, cDst, oSize, zbc->inBuff + zbc->inToCompress, iSize);
- if (ZSTD_isError(cSize)) return cSize;
- if (flush == zbf_end) zbc->frameEnded = 1;
- /* prepare next block */
- zbc->inBuffTarget = zbc->inBuffPos + zbc->blockSize;
- if (zbc->inBuffTarget > zbc->inBuffSize)
- zbc->inBuffPos = 0, zbc->inBuffTarget = zbc->blockSize; /* note : inBuffSize >= blockSize */
- zbc->inToCompress = zbc->inBuffPos;
- if (cDst == op) { op += cSize; break; } /* no need to flush */
- zbc->outBuffContentSize = cSize;
- zbc->outBuffFlushedSize = 0;
- zbc->stage = ZBUFFcs_flush; /* continue to flush stage */
- }
-
- case ZBUFFcs_flush:
- { size_t const toFlush = zbc->outBuffContentSize - zbc->outBuffFlushedSize;
- size_t const flushed = ZBUFF_limitCopy(op, oend-op, zbc->outBuff + zbc->outBuffFlushedSize, toFlush);
- op += flushed;
- zbc->outBuffFlushedSize += flushed;
- if (toFlush!=flushed) { someMoreWork = 0; break; } /* dst too small to store flushed data : stop there */
- zbc->outBuffContentSize = zbc->outBuffFlushedSize = 0;
- zbc->stage = ZBUFFcs_load;
- break;
- }
-
- case ZBUFFcs_final:
- someMoreWork = 0; /* do nothing */
- break;
-
- default:
- return ERROR(GENERIC); /* impossible */
- }
- }
-
- *srcSizePtr = ip - istart;
- *dstCapacityPtr = op - ostart;
- if (zbc->frameEnded) return 0;
- { size_t hintInSize = zbc->inBuffTarget - zbc->inBuffPos;
- if (hintInSize==0) hintInSize = zbc->blockSize;
- return hintInSize;
- }
-}
size_t ZBUFF_compressContinue(ZBUFF_CCtx* zbc,
void* dst, size_t* dstCapacityPtr,
const void* src, size_t* srcSizePtr)
{
- return ZBUFF_compressContinue_generic(zbc, dst, dstCapacityPtr, src, srcSizePtr, zbf_gather);
+ size_t result;
+ ZSTD_outBuffer outBuff;
+ ZSTD_inBuffer inBuff;
+ outBuff.dst = dst;
+ outBuff.pos = 0;
+ outBuff.size = *dstCapacityPtr;
+ inBuff.src = src;
+ inBuff.pos = 0;
+ inBuff.size = *srcSizePtr;
+ result = ZSTD_compressStream(zbc, &outBuff, &inBuff);
+ *dstCapacityPtr = outBuff.pos;
+ *srcSizePtr = inBuff.pos;
+ return result;
}
@@ -270,44 +113,27 @@ size_t ZBUFF_compressContinue(ZBUFF_CCtx* zbc,
size_t ZBUFF_compressFlush(ZBUFF_CCtx* zbc, void* dst, size_t* dstCapacityPtr)
{
- size_t srcSize = 0;
- ZBUFF_compressContinue_generic(zbc, dst, dstCapacityPtr, &srcSize, &srcSize, zbf_flush); /* use a valid src address instead of NULL */
- return zbc->outBuffContentSize - zbc->outBuffFlushedSize;
+ size_t result;
+ ZSTD_outBuffer outBuff;
+ outBuff.dst = dst;
+ outBuff.pos = 0;
+ outBuff.size = *dstCapacityPtr;
+ result = ZSTD_flushStream(zbc, &outBuff);
+ *dstCapacityPtr = outBuff.pos;
+ return result;
}
size_t ZBUFF_compressEnd(ZBUFF_CCtx* zbc, void* dst, size_t* dstCapacityPtr)
{
- BYTE* const ostart = (BYTE*)dst;
- BYTE* const oend = ostart + *dstCapacityPtr;
- BYTE* op = ostart;
-
- if (zbc->stage != ZBUFFcs_final) {
- /* flush whatever remains */
- size_t outSize = *dstCapacityPtr;
- size_t srcSize = 0;
- size_t const notEnded = ZBUFF_compressContinue_generic(zbc, dst, &outSize, &srcSize, &srcSize, zbf_end); /* use a valid address instead of NULL */
- size_t const remainingToFlush = zbc->outBuffContentSize - zbc->outBuffFlushedSize;
- op += outSize;
- if (remainingToFlush) {
- *dstCapacityPtr = op-ostart;
- return remainingToFlush + ZBUFF_endFrameSize + (zbc->checksum * 4);
- }
- /* create epilogue */
- zbc->stage = ZBUFFcs_final;
- zbc->outBuffContentSize = !notEnded ? 0 :
- ZSTD_compressEnd(zbc->zc, zbc->outBuff, zbc->outBuffSize, NULL, 0); /* write epilogue into outBuff */
- }
-
- /* flush epilogue */
- { size_t const toFlush = zbc->outBuffContentSize - zbc->outBuffFlushedSize;
- size_t const flushed = ZBUFF_limitCopy(op, oend-op, zbc->outBuff + zbc->outBuffFlushedSize, toFlush);
- op += flushed;
- zbc->outBuffFlushedSize += flushed;
- *dstCapacityPtr = op-ostart;
- if (toFlush==flushed) zbc->stage = ZBUFFcs_init; /* end reached */
- return toFlush - flushed;
- }
+ size_t result;
+ ZSTD_outBuffer outBuff;
+ outBuff.dst = dst;
+ outBuff.pos = 0;
+ outBuff.size = *dstCapacityPtr;
+ result = ZSTD_endStream(zbc, &outBuff);
+ *dstCapacityPtr = outBuff.pos;
+ return result;
}
@@ -315,5 +141,5 @@ size_t ZBUFF_compressEnd(ZBUFF_CCtx* zbc, void* dst, size_t* dstCapacityPtr)
/* *************************************
* Tool functions
***************************************/
-size_t ZBUFF_recommendedCInSize(void) { return ZSTD_BLOCKSIZE_ABSOLUTEMAX; }
-size_t ZBUFF_recommendedCOutSize(void) { return ZSTD_compressBound(ZSTD_BLOCKSIZE_ABSOLUTEMAX) + ZSTD_blockHeaderSize + ZBUFF_endFrameSize; }
+size_t ZBUFF_recommendedCInSize(void) { return ZSTD_CStreamInSize(); }
+size_t ZBUFF_recommendedCOutSize(void) { return ZSTD_CStreamOutSize(); }
diff --git a/lib/deprecated/zbuff_decompress.c b/lib/deprecated/zbuff_decompress.c
index b20ee9705..d9c155e08 100644
--- a/lib/deprecated/zbuff_decompress.c
+++ b/lib/deprecated/zbuff_decompress.c
@@ -12,68 +12,23 @@
/* *************************************
* Dependencies
***************************************/
-#include
-#include "error_private.h"
-#include "zstd_internal.h" /* MIN, ZSTD_blockHeaderSize, ZSTD_BLOCKSIZE_MAX */
#define ZBUFF_STATIC_LINKING_ONLY
#include "zbuff.h"
-typedef enum { ZBUFFds_init, ZBUFFds_loadHeader,
- ZBUFFds_read, ZBUFFds_load, ZBUFFds_flush } ZBUFF_dStage;
-
-/* *** Resource management *** */
-struct ZBUFF_DCtx_s {
- ZSTD_DCtx* zd;
- ZSTD_frameParams fParams;
- ZBUFF_dStage stage;
- char* inBuff;
- size_t inBuffSize;
- size_t inPos;
- char* outBuff;
- size_t outBuffSize;
- size_t outStart;
- size_t outEnd;
- size_t blockSize;
- BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX];
- size_t lhSize;
- ZSTD_customMem customMem;
-}; /* typedef'd to ZBUFF_DCtx within "zbuff.h" */
-
-
ZBUFF_DCtx* ZBUFF_createDCtx(void)
{
- return ZBUFF_createDCtx_advanced(defaultCustomMem);
+ return ZSTD_createDStream();
}
ZBUFF_DCtx* ZBUFF_createDCtx_advanced(ZSTD_customMem customMem)
{
- ZBUFF_DCtx* zbd;
-
- if (!customMem.customAlloc && !customMem.customFree)
- customMem = defaultCustomMem;
-
- if (!customMem.customAlloc || !customMem.customFree)
- return NULL;
-
- zbd = (ZBUFF_DCtx*)customMem.customAlloc(customMem.opaque, sizeof(ZBUFF_DCtx));
- if (zbd==NULL) return NULL;
- memset(zbd, 0, sizeof(ZBUFF_DCtx));
- memcpy(&zbd->customMem, &customMem, sizeof(ZSTD_customMem));
- zbd->zd = ZSTD_createDCtx_advanced(customMem);
- if (zbd->zd == NULL) { ZBUFF_freeDCtx(zbd); return NULL; }
- zbd->stage = ZBUFFds_init;
- return zbd;
+ return ZSTD_createDStream_advanced(customMem);
}
size_t ZBUFF_freeDCtx(ZBUFF_DCtx* zbd)
{
- if (zbd==NULL) return 0; /* support free on null */
- ZSTD_freeDCtx(zbd->zd);
- if (zbd->inBuff) zbd->customMem.customFree(zbd->customMem.opaque, zbd->inBuff);
- if (zbd->outBuff) zbd->customMem.customFree(zbd->customMem.opaque, zbd->outBuff);
- zbd->customMem.customFree(zbd->customMem.opaque, zbd);
- return 0;
+ return ZSTD_freeDStream(zbd);
}
@@ -81,23 +36,12 @@ size_t ZBUFF_freeDCtx(ZBUFF_DCtx* zbd)
size_t ZBUFF_decompressInitDictionary(ZBUFF_DCtx* zbd, const void* dict, size_t dictSize)
{
- zbd->stage = ZBUFFds_loadHeader;
- zbd->lhSize = zbd->inPos = zbd->outStart = zbd->outEnd = 0;
- return ZSTD_decompressBegin_usingDict(zbd->zd, dict, dictSize);
+ return ZSTD_initDStream_usingDict(zbd, dict, dictSize);
}
size_t ZBUFF_decompressInit(ZBUFF_DCtx* zbd)
{
- return ZBUFF_decompressInitDictionary(zbd, NULL, 0);
-}
-
-
-/* internal util function */
-MEM_STATIC size_t ZBUFF_limitCopy(void* dst, size_t dstCapacity, const void* src, size_t srcSize)
-{
- size_t const length = MIN(dstCapacity, srcSize);
- memcpy(dst, src, length);
- return length;
+ return ZSTD_initDStream(zbd);
}
@@ -107,146 +51,24 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbd,
void* dst, size_t* dstCapacityPtr,
const void* src, size_t* srcSizePtr)
{
- const char* const istart = (const char*)src;
- const char* const iend = istart + *srcSizePtr;
- const char* ip = istart;
- char* const ostart = (char*)dst;
- char* const oend = ostart + *dstCapacityPtr;
- char* op = ostart;
- U32 someMoreWork = 1;
-
- while (someMoreWork) {
- switch(zbd->stage)
- {
- case ZBUFFds_init :
- return ERROR(init_missing);
-
- case ZBUFFds_loadHeader :
- { size_t const hSize = ZSTD_getFrameParams(&(zbd->fParams), zbd->headerBuffer, zbd->lhSize);
- if (ZSTD_isError(hSize)) return hSize;
- if (hSize != 0) { /* need more input */
- size_t const toLoad = hSize - zbd->lhSize; /* if hSize!=0, hSize > zbd->lhSize */
- if (toLoad > (size_t)(iend-ip)) { /* not enough input to load full header */
- memcpy(zbd->headerBuffer + zbd->lhSize, ip, iend-ip);
- zbd->lhSize += iend-ip;
- *dstCapacityPtr = 0;
- return (hSize - zbd->lhSize) + ZSTD_blockHeaderSize; /* remaining header bytes + next block header */
- }
- memcpy(zbd->headerBuffer + zbd->lhSize, ip, toLoad); zbd->lhSize = hSize; ip += toLoad;
- break;
- } }
-
- /* Consume header */
- { size_t const h1Size = ZSTD_nextSrcSizeToDecompress(zbd->zd); /* == ZSTD_frameHeaderSize_min */
- size_t const h1Result = ZSTD_decompressContinue(zbd->zd, NULL, 0, zbd->headerBuffer, h1Size);
- if (ZSTD_isError(h1Result)) return h1Result; /* should not happen : already checked */
- if (h1Size < zbd->lhSize) { /* long header */
- size_t const h2Size = ZSTD_nextSrcSizeToDecompress(zbd->zd);
- size_t const h2Result = ZSTD_decompressContinue(zbd->zd, NULL, 0, zbd->headerBuffer+h1Size, h2Size);
- if (ZSTD_isError(h2Result)) return h2Result;
- } }
-
- zbd->fParams.windowSize = MAX(zbd->fParams.windowSize, 1U << ZSTD_WINDOWLOG_ABSOLUTEMIN);
-
- /* Frame header instruct buffer sizes */
- { size_t const blockSize = MIN(zbd->fParams.windowSize, ZSTD_BLOCKSIZE_ABSOLUTEMAX);
- size_t const neededOutSize = zbd->fParams.windowSize + blockSize;
- zbd->blockSize = blockSize;
- if (zbd->inBuffSize < blockSize) {
- zbd->customMem.customFree(zbd->customMem.opaque, zbd->inBuff);
- zbd->inBuffSize = blockSize;
- zbd->inBuff = (char*)zbd->customMem.customAlloc(zbd->customMem.opaque, blockSize);
- if (zbd->inBuff == NULL) return ERROR(memory_allocation);
- }
- if (zbd->outBuffSize < neededOutSize) {
- zbd->customMem.customFree(zbd->customMem.opaque, zbd->outBuff);
- zbd->outBuffSize = neededOutSize;
- zbd->outBuff = (char*)zbd->customMem.customAlloc(zbd->customMem.opaque, neededOutSize);
- if (zbd->outBuff == NULL) return ERROR(memory_allocation);
- } }
- zbd->stage = ZBUFFds_read;
- /* pass-through */
-
- case ZBUFFds_read:
- { size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zbd->zd);
- if (neededInSize==0) { /* end of frame */
- zbd->stage = ZBUFFds_init;
- someMoreWork = 0;
- break;
- }
- if ((size_t)(iend-ip) >= neededInSize) { /* decode directly from src */
- const int isSkipFrame = ZSTD_isSkipFrame(zbd->zd);
- size_t const decodedSize = ZSTD_decompressContinue(zbd->zd,
- zbd->outBuff + zbd->outStart, (isSkipFrame ? 0 : zbd->outBuffSize - zbd->outStart),
- ip, neededInSize);
- if (ZSTD_isError(decodedSize)) return decodedSize;
- ip += neededInSize;
- if (!decodedSize && !isSkipFrame) break; /* this was just a header */
- zbd->outEnd = zbd->outStart + decodedSize;
- zbd->stage = ZBUFFds_flush;
- break;
- }
- if (ip==iend) { someMoreWork = 0; break; } /* no more input */
- zbd->stage = ZBUFFds_load;
- /* pass-through */
- }
-
- case ZBUFFds_load:
- { size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zbd->zd);
- size_t const toLoad = neededInSize - zbd->inPos; /* should always be <= remaining space within inBuff */
- size_t loadedSize;
- if (toLoad > zbd->inBuffSize - zbd->inPos) return ERROR(corruption_detected); /* should never happen */
- loadedSize = ZBUFF_limitCopy(zbd->inBuff + zbd->inPos, toLoad, ip, iend-ip);
- ip += loadedSize;
- zbd->inPos += loadedSize;
- if (loadedSize < toLoad) { someMoreWork = 0; break; } /* not enough input, wait for more */
-
- /* decode loaded input */
- { const int isSkipFrame = ZSTD_isSkipFrame(zbd->zd);
- size_t const decodedSize = ZSTD_decompressContinue(zbd->zd,
- zbd->outBuff + zbd->outStart, zbd->outBuffSize - zbd->outStart,
- zbd->inBuff, neededInSize);
- if (ZSTD_isError(decodedSize)) return decodedSize;
- zbd->inPos = 0; /* input is consumed */
- if (!decodedSize && !isSkipFrame) { zbd->stage = ZBUFFds_read; break; } /* this was just a header */
- zbd->outEnd = zbd->outStart + decodedSize;
- zbd->stage = ZBUFFds_flush;
- /* pass-through */
- } }
-
- case ZBUFFds_flush:
- { size_t const toFlushSize = zbd->outEnd - zbd->outStart;
- size_t const flushedSize = ZBUFF_limitCopy(op, oend-op, zbd->outBuff + zbd->outStart, toFlushSize);
- op += flushedSize;
- zbd->outStart += flushedSize;
- if (flushedSize == toFlushSize) { /* flush completed */
- zbd->stage = ZBUFFds_read;
- if (zbd->outStart + zbd->blockSize > zbd->outBuffSize)
- zbd->outStart = zbd->outEnd = 0;
- break;
- }
- /* cannot flush everything */
- someMoreWork = 0;
- break;
- }
- default: return ERROR(GENERIC); /* impossible */
- } }
-
- /* result */
- *srcSizePtr = ip-istart;
- *dstCapacityPtr = op-ostart;
- { size_t nextSrcSizeHint = ZSTD_nextSrcSizeToDecompress(zbd->zd);
- if (!nextSrcSizeHint) return (zbd->outEnd != zbd->outStart); /* return 0 only if fully flushed too */
- nextSrcSizeHint += ZSTD_blockHeaderSize * (ZSTD_nextInputType(zbd->zd) == ZSTDnit_block);
- if (zbd->inPos > nextSrcSizeHint) return ERROR(GENERIC); /* should never happen */
- nextSrcSizeHint -= zbd->inPos; /* already loaded*/
- return nextSrcSizeHint;
- }
+ ZSTD_outBuffer outBuff;
+ ZSTD_inBuffer inBuff;
+ size_t result;
+ outBuff.dst = dst;
+ outBuff.pos = 0;
+ outBuff.size = *dstCapacityPtr;
+ inBuff.src = src;
+ inBuff.pos = 0;
+ inBuff.size = *srcSizePtr;
+ result = ZSTD_decompressStream(zbd, &outBuff, &inBuff);
+ *dstCapacityPtr = outBuff.pos;
+ *srcSizePtr = inBuff.pos;
+ return result;
}
/* *************************************
* Tool functions
***************************************/
-size_t ZBUFF_recommendedDInSize(void) { return ZSTD_BLOCKSIZE_ABSOLUTEMAX + ZSTD_blockHeaderSize /* block header size*/ ; }
-size_t ZBUFF_recommendedDOutSize(void) { return ZSTD_BLOCKSIZE_ABSOLUTEMAX; }
+size_t ZBUFF_recommendedDInSize(void) { return ZSTD_DStreamInSize(); }
+size_t ZBUFF_recommendedDOutSize(void) { return ZSTD_DStreamOutSize(); }
diff --git a/lib/zstd.h b/lib/zstd.h
index a10a9eaab..3989cc307 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -63,7 +63,7 @@ extern "C" {
#define ZSTD_VERSION_STRING ZSTD_EXPAND_AND_QUOTE(ZSTD_LIB_VERSION)
#define ZSTD_VERSION_NUMBER (ZSTD_VERSION_MAJOR *100*100 + ZSTD_VERSION_MINOR *100 + ZSTD_VERSION_RELEASE)
-ZSTDLIB_API unsigned ZSTD_versionNumber (void);
+ZSTDLIB_API unsigned ZSTD_versionNumber(void); /**< library version number; to be used when checking dll version */
/***************************************
@@ -117,9 +117,9 @@ ZSTDLIB_API const char* ZSTD_getErrorName(size_t code); /*!< provides readab
* Explicit memory management
***************************************/
/*= Compression context
-* When compressing many messages / blocks,
+* When compressing many times,
* it is recommended to allocate a context just once, and re-use it for each successive compression operation.
-* This will make the situation much easier for the system's memory.
+* This will make workload friendlier for system's memory.
* Use one context per thread for parallel execution in multi-threaded environments. */
typedef struct ZSTD_CCtx_s ZSTD_CCtx;
ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx(void);
diff --git a/programs/Makefile b/programs/Makefile
index 393944920..e7ff3e5a0 100644
--- a/programs/Makefile
+++ b/programs/Makefile
@@ -125,8 +125,15 @@ zstd-compress: $(ZSTDCOMMON_FILES) $(ZSTDCOMP_FILES) zstdcli.c fileio.c
$(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_NODECOMPRESS $^ -o $@$(EXT)
gzstd: clean_decomp_o
- echo "int main(){}" | $(CC) -o have_zlib -x c - -lz && echo found zlib || echo did not found zlib
- if [ -s have_zlib ]; then echo building gzstd && rm have_zlib$(EXT) && CPPFLAGS=-DZSTD_GZDECOMPRESS LDFLAGS="-lz" $(MAKE) zstd; else echo building plain zstd && $(MAKE) zstd; fi
+ @echo "int main(){}" | $(CC) -o have_zlib -x c - -lz && echo found zlib || echo did not found zlib
+ @if [ -s have_zlib ]; then \
+ echo building gzstd with .gz decompression support \
+ && rm have_zlib$(EXT) \
+ && CPPFLAGS=-DZSTD_GZDECOMPRESS LDFLAGS="-lz" $(MAKE) zstd; \
+ else \
+ echo "WARNING : no zlib, building gzstd with only .zst files support : NO .gz SUPPORT !!!" \
+ && $(MAKE) zstd; \
+ fi
generate_res:
windres\generate_res.bat
diff --git a/programs/bench.c b/programs/bench.c
index a66f4b345..9104ea89c 100644
--- a/programs/bench.c
+++ b/programs/bench.c
@@ -74,7 +74,7 @@ static clock_t g_time = 0;
DEBUGOUTPUT("Error defined at %s, line %i : \n", __FILE__, __LINE__); \
DISPLAYLEVEL(1, "Error %i : ", error); \
DISPLAYLEVEL(1, __VA_ARGS__); \
- DISPLAYLEVEL(1, "\n"); \
+ DISPLAYLEVEL(1, " \n"); \
exit(error); \
}
@@ -84,7 +84,8 @@ static clock_t g_time = 0;
***************************************/
static U32 g_nbSeconds = NBSECONDS;
static size_t g_blockSize = 0;
-int g_additionalParam = 0;
+static int g_additionalParam = 0;
+static U32 g_decodeOnly = 0;
void BMK_setNotificationLevel(unsigned level) { g_displayLevel=level; }
@@ -102,18 +103,18 @@ void BMK_SetBlockSize(size_t blockSize)
DISPLAYLEVEL(2, "using blocks of size %u KB \n", (U32)(blockSize>>10));
}
+void BMK_setDecodeOnly(unsigned decodeFlag) { g_decodeOnly = (decodeFlag>0); }
/* ********************************************************
* Bench functions
**********************************************************/
-typedef struct
-{
- const char* srcPtr;
+typedef struct {
+ const void* srcPtr;
size_t srcSize;
- char* cPtr;
+ void* cPtr;
size_t cRoom;
size_t cSize;
- char* resPtr;
+ void* resPtr;
size_t resSize;
} blockParam_t;
@@ -126,15 +127,18 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
const size_t* fileSizes, U32 nbFiles,
const void* dictBuffer, size_t dictBufferSize)
{
- size_t const blockSize = (g_blockSize>=32 ? g_blockSize : srcSize) + (!srcSize) /* avoid div by 0 */ ;
+ size_t const blockSize = ((g_blockSize>=32 && !g_decodeOnly) ? g_blockSize : srcSize) + (!srcSize) /* avoid div by 0 */ ;
size_t const avgSize = MIN(g_blockSize, (srcSize / nbFiles));
U32 const maxNbBlocks = (U32) ((srcSize + (blockSize-1)) / blockSize) + nbFiles;
blockParam_t* const blockTable = (blockParam_t*) malloc(maxNbBlocks * sizeof(blockParam_t));
size_t const maxCompressedSize = ZSTD_compressBound(srcSize) + (maxNbBlocks * 1024); /* add some room for safety */
void* const compressedBuffer = malloc(maxCompressedSize);
- void* const resultBuffer = malloc(srcSize);
+ void* resultBuffer = malloc(srcSize);
ZSTD_CCtx* const ctx = ZSTD_createCCtx();
ZSTD_DCtx* const dctx = ZSTD_createDCtx();
+ size_t const loadedCompressedSize = srcSize;
+ size_t cSize = 0;
+ double ratio = 0.;
U32 nbBlocks;
UTIL_time_t ticksPerSecond;
@@ -146,6 +150,27 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
if (strlen(displayName)>17) displayName += strlen(displayName)-17; /* can only display 17 characters */
UTIL_initTimer(&ticksPerSecond);
+ if (g_decodeOnly) {
+ const char* srcPtr = (const char*) srcBuffer;
+ U64 dSize64 = 0;
+ U32 fileNb;
+ for (fileNb=0; fileNb decodedSize) EXM_THROW(32, "original size is too large");
+ if (decodedSize==0) EXM_THROW(32, "Impossible to determine original size ");
+ free(resultBuffer);
+ resultBuffer = malloc(decodedSize);
+ if (!resultBuffer) EXM_THROW(33, "not enough memory");
+ cSize = srcSize;
+ srcSize = decodedSize;
+ ratio = (double)srcSize / (double)cSize;
+ } }
+
/* Init blockTable data */
{ const char* srcPtr = (const char*)srcBuffer;
char* cPtr = (char*)compressedBuffer;
@@ -153,15 +178,17 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
U32 fileNb;
for (nbBlocks=0, fileNb=0; fileNb ACTIVEPERIOD_MICROSEC) {
@@ -197,53 +221,58 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
UTIL_getTime(&coolTime);
}
- /* Compression */
- DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->\r", marks[markNb], displayName, (U32)srcSize);
- if (!cCompleted) memset(compressedBuffer, 0xE5, maxCompressedSize); /* warm up and erase result buffer */
+ if (!g_decodeOnly) {
+ /* Compression */
+ DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->\r", marks[markNb], displayName, (U32)srcSize);
+ if (!cCompleted) memset(compressedBuffer, 0xE5, maxCompressedSize); /* warm up and erase result buffer */
- UTIL_sleepMilli(1); /* give processor time to other processes */
- UTIL_waitForNextTick(ticksPerSecond);
- UTIL_getTime(&clockStart);
+ UTIL_sleepMilli(1); /* give processor time to other processes */
+ UTIL_waitForNextTick(ticksPerSecond);
+ UTIL_getTime(&clockStart);
- if (!cCompleted) { /* still some time to do compression tests */
- ZSTD_parameters const zparams = ZSTD_getParams(cLevel, avgSize, dictBufferSize);
- ZSTD_customMem const cmem = { NULL, NULL, NULL };
- U32 nbLoops = 0;
- ZSTD_CDict* cdict = ZSTD_createCDict_advanced(dictBuffer, dictBufferSize, zparams, cmem);
- if (cdict==NULL) EXM_THROW(1, "ZSTD_createCDict_advanced() allocation failure");
- do {
- U32 blockNb;
- size_t rSize;
- for (blockNb=0; blockNb= maxTime);
- } }
+ nbLoops++;
+ } while (UTIL_clockSpanMicro(clockStart, ticksPerSecond) < clockLoop);
+ ZSTD_freeCDict(cdict);
+ { U64 const clockSpan = UTIL_clockSpanMicro(clockStart, ticksPerSecond);
+ if (clockSpan < fastestC*nbLoops) fastestC = clockSpan / nbLoops;
+ totalCTime += clockSpan;
+ cCompleted = (totalCTime >= maxTime);
+ } }
- cSize = 0;
- { U32 blockNb; for (blockNb=0; blockNb%10u (%5.3f),%6.1f MB/s\r",
- marks[markNb], displayName, (U32)srcSize, (U32)cSize, ratio,
- (double)srcSize / fastestC );
+ cSize = 0;
+ { U32 blockNb; for (blockNb=0; blockNb%10u (%5.3f),%6.1f MB/s\r",
+ marks[markNb], displayName, (U32)srcSize, (U32)cSize, ratio,
+ (double)srcSize / fastestC );
+ } else { /* g_decodeOnly */
+ memcpy(compressedBuffer, srcBuffer, loadedCompressedSize);
+ }
(void)fastestD; (void)crcOrig; /* unused when decompression disabled */
#if 1
@@ -255,14 +284,15 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
UTIL_getTime(&clockStart);
if (!dCompleted) {
+ U64 clockLoop = g_nbSeconds ? TIMELOOP_MICROSEC : 1;
U32 nbLoops = 0;
- ZSTD_DDict* ddict = ZSTD_createDDict(dictBuffer, dictBufferSize);
+ ZSTD_DDict* const ddict = ZSTD_createDDict(dictBuffer, dictBufferSize);
if (!ddict) EXM_THROW(2, "ZSTD_createDDict() allocation failure");
do {
U32 blockNb;
for (blockNb=0; blockNb ZSTD_maxCLevel()) cLevel = ZSTD_maxCLevel();
if (cLevelLast > ZSTD_maxCLevel()) cLevelLast = ZSTD_maxCLevel();
if (cLevelLast < cLevel) cLevelLast = cLevel;
- if (cLevelLast > cLevel) DISPLAYLEVEL(2, "Benchmarking levels from %d to %d\n", cLevel, cLevelLast);
+ if (cLevelLast > cLevel) DISPLAYLEVEL(2, "Benchmarking levels from %d to %d\n", cLevel, cLevelLast);
if (nbFiles == 0)
BMK_syntheticTest(cLevel, cLevelLast, compressibility);
diff --git a/programs/bench.h b/programs/bench.h
index 1e3e3812b..7009dc2f6 100644
--- a/programs/bench.h
+++ b/programs/bench.h
@@ -11,7 +11,7 @@
#ifndef BENCH_H_121279284357
#define BENCH_H_121279284357
-#include
+#include /* size_t */
int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles,
const char* dictFileName, int cLevel, int cLevelLast);
@@ -21,5 +21,6 @@ void BMK_SetNbSeconds(unsigned nbLoops);
void BMK_SetBlockSize(size_t blockSize);
void BMK_setAdditionalParam(int additionalParam);
void BMK_setNotificationLevel(unsigned level);
+void BMK_setDecodeOnly(unsigned decodeFlag);
- #endif /* BENCH_H_121279284357 */
+#endif /* BENCH_H_121279284357 */
diff --git a/programs/zstdcli.c b/programs/zstdcli.c
index c7c682bb3..7ffb0bb62 100644
--- a/programs/zstdcli.c
+++ b/programs/zstdcli.c
@@ -334,7 +334,11 @@ int main(int argCount, const char* argv[])
case 'z': operation=zom_compress; argument++; break;
/* Decoding */
- case 'd': operation=zom_decompress; argument++; break;
+ case 'd':
+#ifndef ZSTD_NOBENCH
+ if (operation==zom_bench) { BMK_setDecodeOnly(1); argument++; break; } /* benchmark decode (hidden option) */
+#endif
+ operation=zom_decompress; argument++; break;
/* Force stdout, even if stdout==console */
case 'c': forceStdout=1; outFileName=stdoutmark; argument++; break;
diff --git a/tests/playTests.sh b/tests/playTests.sh
index dfca1de87..6179265e7 100755
--- a/tests/playTests.sh
+++ b/tests/playTests.sh
@@ -31,10 +31,10 @@ case "$OS" in
;;
esac
-MD5SUM="md5sum"
-if [[ "$OSTYPE" == "darwin"* ]]; then
- MD5SUM="md5 -r"
-fi
+case "$OSTYPE" in
+ darwin*) MD5SUM="md5 -r" ;;
+ *) MD5SUM="md5sum" ;;
+esac
$ECHO "\nStarting playTests.sh isWindows=$isWindows ZSTD='$ZSTD'"
@@ -228,11 +228,10 @@ cp ../programs/*.h dirTestDict
$MD5SUM dirTestDict/* > tmph1
$ZSTD -f --rm dirTestDict/* -D tmpDictC
$ZSTD -d --rm dirTestDict/*.zst -D tmpDictC # note : use internal checksum by default
-if [[ "$OSTYPE" == "darwin"* ]]; then
- $ECHO "md5sum -c not supported on OS-X : test skipped" # not compatible with OS-X's md5
-else
- $MD5SUM -c tmph1
-fi
+case "$OSTYPE" in
+ darwin*) $ECHO "md5sum -c not supported on OS-X : test skipped" ;; # not compatible with OS-X's md5
+ *) $MD5SUM -c tmph1 ;;
+esac
rm -rf dirTestDict
rm tmp*