diff --git a/.travis.yml b/.travis.yml index 972c1b4ba..9e2f58613 100644 --- a/.travis.yml +++ b/.travis.yml @@ -44,7 +44,7 @@ matrix: - qemu-user-static - os: linux sudo: required - env: PLATFORM="Ubuntu 12.04" CMD="make -C programs zstd-small && make -C programs zstd-decompress && make -C programs zstd-compress && make clean && make -C tests versionsTest" + env: PLATFORM="Ubuntu 12.04" CMD="make -C programs zstd-small && make -C programs zstd-decompress && make -C programs zstd-compress && make -C programs clean && make -C tests versionsTest" - os: linux sudo: required env: PLATFORM="Ubuntu 12.04" CMD="make asan32" diff --git a/lib/zstd.h b/lib/zstd.h index dd3f5df4c..2c1407306 100644 --- a/lib/zstd.h +++ b/lib/zstd.h @@ -1,4 +1,4 @@ -/** +/* * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. * All rights reserved. * @@ -14,12 +14,12 @@ extern "C" { #endif -/*====== Dependency ======*/ +/* ====== Dependency ======*/ #include /* size_t */ -/*====== Export for Windows ======*/ -/*! +/* ====== Export for Windows ======*/ +/* * ZSTD_DLL_EXPORT : * Enable exporting of functions when building a Windows DLL */ @@ -30,7 +30,29 @@ extern "C" { #endif -/*======= Version =======*/ +/******************************************************************************************************* + Introduction + + Zstd, short for Zstandard, is a fast lossless compression algorithm, targeting real-time compression scenarios + at zlib-level and better compression ratios. The zstd compression library provides in-memory compression and + decompression functions. The library supports compression levels from 1 up to ZSTD_maxCLevel() which is 22. + Levels from 20 to 22 should be used with caution as they require about 300-1300 MB for compression. + Compression can be done in: + - a single step (described as Simple API) + - a single step, reusing a context (described as Explicit memory management) + - repeated calls of the compression function (described as Streaming compression) + The compression ratio achievable on small data can be highly improved using compression with a dictionary in: + - a single step (described as Simple dictionary API) + - a single step, reusing a dictionary (described as Fast Dictionary API) + + Advanced and experimantal functions can be accessed using #define ZSTD_STATIC_LINKING_ONLY before including zstd.h. + These APIs shall never be used with a dynamic library. + They are not "stable", their definition may change in the future. Only static linking is allowed. +*********************************************************************************************************/ + +/*------ Version ------*/ +ZSTDLIB_API unsigned ZSTD_versionNumber (void); /**< returns version number of ZSTD */ + #define ZSTD_VERSION_MAJOR 1 #define ZSTD_VERSION_MINOR 1 #define ZSTD_VERSION_RELEASE 0 @@ -41,10 +63,9 @@ extern "C" { #define ZSTD_VERSION_STRING ZSTD_EXPAND_AND_QUOTE(ZSTD_LIB_VERSION) #define ZSTD_VERSION_NUMBER (ZSTD_VERSION_MAJOR *100*100 + ZSTD_VERSION_MINOR *100 + ZSTD_VERSION_RELEASE) -ZSTDLIB_API unsigned ZSTD_versionNumber (void); -/* ************************************* +/*************************************** * Simple API ***************************************/ /*! ZSTD_compress() : @@ -91,29 +112,29 @@ ZSTDLIB_API unsigned ZSTD_isError(size_t code); /*!< tells if a `siz ZSTDLIB_API const char* ZSTD_getErrorName(size_t code); /*!< provides readable string from an error code */ -/*-************************************* +/*************************************** * Explicit memory management ***************************************/ -/** Compression context */ +/*= Compression context */ typedef struct ZSTD_CCtx_s ZSTD_CCtx; ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx(void); ZSTDLIB_API size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx); -/** ZSTD_compressCCtx() : +/*! ZSTD_compressCCtx() : Same as ZSTD_compress(), requires an allocated ZSTD_CCtx (see ZSTD_createCCtx()) */ ZSTDLIB_API size_t ZSTD_compressCCtx(ZSTD_CCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, int compressionLevel); -/** Decompression context */ +/*= Decompression context */ typedef struct ZSTD_DCtx_s ZSTD_DCtx; ZSTDLIB_API ZSTD_DCtx* ZSTD_createDCtx(void); ZSTDLIB_API size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx); -/** ZSTD_decompressDCtx() : +/*! ZSTD_decompressDCtx() : * Same as ZSTD_decompress(), requires an allocated ZSTD_DCtx (see ZSTD_createDCtx()) */ ZSTDLIB_API size_t ZSTD_decompressDCtx(ZSTD_DCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); -/*-************************ +/************************** * Simple dictionary API ***************************/ /*! ZSTD_compress_usingDict() : @@ -135,14 +156,18 @@ ZSTDLIB_API size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx, const void* dict,size_t dictSize); -/*-************************** +/**************************** * Fast Dictionary API ****************************/ +typedef struct ZSTD_CDict_s ZSTD_CDict; + /*! ZSTD_createCDict() : * Create a digested dictionary, ready to start compression operation without startup delay. * `dict` can be released after ZSTD_CDict creation */ -typedef struct ZSTD_CDict_s ZSTD_CDict; ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict(const void* dict, size_t dictSize, int compressionLevel); + +/*! ZSTD_freeCDict() : +* Function frees memory allocated with ZSTD_createCDict() */ ZSTDLIB_API size_t ZSTD_freeCDict(ZSTD_CDict* CDict); /*! ZSTD_compress_usingCDict() : @@ -154,11 +179,16 @@ ZSTDLIB_API size_t ZSTD_compress_usingCDict(ZSTD_CCtx* cctx, const void* src, size_t srcSize, const ZSTD_CDict* cdict); + +typedef struct ZSTD_DDict_s ZSTD_DDict; + /*! ZSTD_createDDict() : * Create a digested dictionary, ready to start decompression operation without startup delay. * `dict` can be released after creation */ -typedef struct ZSTD_DDict_s ZSTD_DDict; ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict(const void* dict, size_t dictSize); + +/*! ZSTD_freeDDict() : +* Function frees memory allocated with ZSTD_createDDict() */ ZSTDLIB_API size_t ZSTD_freeDDict(ZSTD_DDict* ddict); /*! ZSTD_decompress_usingDDict() : @@ -170,7 +200,7 @@ ZSTDLIB_API size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict); -/*-************************** +/**************************** * Streaming ****************************/ @@ -187,7 +217,6 @@ typedef struct ZSTD_outBuffer_s { } ZSTD_outBuffer; -/*====== streaming compression ======*/ /*-*********************************************************************** * Streaming compression - howto @@ -225,20 +254,19 @@ typedef struct ZSTD_outBuffer_s { * * *******************************************************************/ +/*===== Streaming compression functions ======*/ typedef struct ZSTD_CStream_s ZSTD_CStream; ZSTDLIB_API ZSTD_CStream* ZSTD_createCStream(void); ZSTDLIB_API size_t ZSTD_freeCStream(ZSTD_CStream* zcs); - -ZSTDLIB_API size_t ZSTD_CStreamInSize(void); /**< recommended size for input buffer */ -ZSTDLIB_API size_t ZSTD_CStreamOutSize(void); /**< recommended size for output buffer. Guarantee to successfully flush at least one complete compressed block in all circumstances. */ - ZSTDLIB_API size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel); ZSTDLIB_API size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuffer* input); ZSTDLIB_API size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output); ZSTDLIB_API size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output); +ZSTDLIB_API size_t ZSTD_CStreamInSize(void); /**< recommended size for input buffer */ +ZSTDLIB_API size_t ZSTD_CStreamOutSize(void); /**< recommended size for output buffer. Guarantee to successfully flush at least one complete compressed block in all circumstances. */ + -/*====== decompression ======*/ /*-*************************************************************************** * Streaming decompression howto @@ -262,28 +290,29 @@ ZSTDLIB_API size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output); * The return value is a suggested next input size (just an hint, to help latency). * *******************************************************************************/ +/*===== Streaming decompression functions =====*/ typedef struct ZSTD_DStream_s ZSTD_DStream; ZSTDLIB_API ZSTD_DStream* ZSTD_createDStream(void); ZSTDLIB_API size_t ZSTD_freeDStream(ZSTD_DStream* zds); +ZSTDLIB_API size_t ZSTD_initDStream(ZSTD_DStream* zds); +ZSTDLIB_API size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inBuffer* input); ZSTDLIB_API size_t ZSTD_DStreamInSize(void); /*!< recommended size for input buffer */ ZSTDLIB_API size_t ZSTD_DStreamOutSize(void); /*!< recommended size for output buffer. Guarantee to successfully flush at least one complete block in all circumstances. */ -ZSTDLIB_API size_t ZSTD_initDStream(ZSTD_DStream* zds); -ZSTDLIB_API size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inBuffer* input); - #ifdef ZSTD_STATIC_LINKING_ONLY -/* ==================================================================================== +/**************************************************************************************** + * START OF ADVANCED AND EXPERIMENTAL FUNCTIONS * The definitions in this section are considered experimental. * They should never be used with a dynamic library, as they may change in the future. * They are provided for advanced usages. * Use them only in association with static linking. - * ==================================================================================== */ + * ***************************************************************************************/ -/*--- Constants ---*/ +/* --- Constants ---*/ #define ZSTD_MAGICNUMBER 0xFD2FB528 /* v0.8 */ #define ZSTD_MAGIC_SKIPPABLE_START 0x184D2A50U @@ -334,13 +363,13 @@ typedef struct { ZSTD_frameParameters fParams; } ZSTD_parameters; -/* custom memory allocation functions */ +/*= Custom memory allocation functions */ typedef void* (*ZSTD_allocFunction) (void* opaque, size_t size); typedef void (*ZSTD_freeFunction) (void* opaque, void* address); typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; void* opaque; } ZSTD_customMem; -/*-************************************* +/*************************************** * Advanced compression functions ***************************************/ /*! ZSTD_estimateCCtxSize() : @@ -412,12 +441,11 @@ ZSTDLIB_API size_t ZSTD_sizeof_DCtx(const ZSTD_DCtx* dctx); ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict); -/* ****************************************************************** +/******************************************************************** * Advanced Streaming functions ********************************************************************/ -/*====== compression ======*/ - +/*===== Advanced Streaming compression functions =====*/ ZSTDLIB_API ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem); ZSTDLIB_API size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel); ZSTDLIB_API size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize, @@ -426,10 +454,8 @@ ZSTDLIB_API size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledg ZSTDLIB_API size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs); -/*====== decompression ======*/ - +/*===== Advanced Streaming decompression functions =====*/ typedef enum { ZSTDdsp_maxWindowSize } ZSTD_DStreamParameter_e; - ZSTDLIB_API ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem); ZSTDLIB_API size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize); ZSTDLIB_API size_t ZSTD_setDStreamParameter(ZSTD_DStream* zds, ZSTD_DStreamParameter_e paramType, unsigned paramValue); @@ -437,22 +463,17 @@ ZSTDLIB_API size_t ZSTD_resetDStream(ZSTD_DStream* zds); /**< re-use decompress ZSTDLIB_API size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds); -/* ****************************************************************** +/********************************************************************* * Buffer-less and synchronous inner streaming functions -********************************************************************/ -/* This is an advanced API, giving full control over buffer management, for users which need direct control over memory. +* +* This is an advanced API, giving full control over buffer management, for users which need direct control over memory. * But it's also a complex one, with many restrictions (documented below). -* Prefer using normal streaming API for an easier experience */ +* Prefer using normal streaming API for an easier experience +********************************************************************* */ -ZSTDLIB_API size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel); -ZSTDLIB_API size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel); -ZSTDLIB_API size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize); -ZSTDLIB_API size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize); +/** + Buffer-less streaming compression (synchronous mode) -ZSTDLIB_API size_t ZSTD_compressContinue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); -ZSTDLIB_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); - -/* A ZSTD_CCtx object is required to track streaming operations. Use ZSTD_createCCtx() / ZSTD_freeCCtx() to manage resource. ZSTD_CCtx object can be re-used multiple times within successive compression operations. @@ -481,26 +502,17 @@ ZSTDLIB_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapaci You can then reuse `ZSTD_CCtx` (ZSTD_compressBegin()) to compress some new frame. */ -typedef struct { - unsigned long long frameContentSize; - unsigned windowSize; - unsigned dictID; - unsigned checksumFlag; -} ZSTD_frameParams; +/*===== Buffer-less streaming compression functions =====*/ +ZSTDLIB_API size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel); +ZSTDLIB_API size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel); +ZSTDLIB_API size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize); +ZSTDLIB_API size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize); +ZSTDLIB_API size_t ZSTD_compressContinue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); +ZSTDLIB_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); -ZSTDLIB_API size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t srcSize); /**< doesn't consume input, see details below */ -ZSTDLIB_API size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx); -ZSTDLIB_API size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t dictSize); -ZSTDLIB_API void ZSTD_copyDCtx(ZSTD_DCtx* dctx, const ZSTD_DCtx* preparedDCtx); -ZSTDLIB_API size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx); -ZSTDLIB_API size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); - -typedef enum { ZSTDnit_frameHeader, ZSTDnit_blockHeader, ZSTDnit_block, ZSTDnit_lastBlock, ZSTDnit_checksum, ZSTDnit_skippableFrame } ZSTD_nextInputType_e; -ZSTDLIB_API ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx); - -/* +/*- Buffer-less streaming decompression (synchronous mode) A ZSTD_DCtx object is required to track streaming operations. @@ -557,11 +569,27 @@ ZSTDLIB_API ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx); It also returns Frame Size as fparamsPtr->frameContentSize. */ +typedef struct { + unsigned long long frameContentSize; + unsigned windowSize; + unsigned dictID; + unsigned checksumFlag; +} ZSTD_frameParams; -/* ************************************** -* Block functions -****************************************/ -/*! Block functions produce and decode raw zstd blocks, without frame metadata. +/*===== Buffer-less streaming decompression functions =====*/ +ZSTDLIB_API size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t srcSize); /**< doesn't consume input, see details below */ +ZSTDLIB_API size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx); +ZSTDLIB_API size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t dictSize); +ZSTDLIB_API void ZSTD_copyDCtx(ZSTD_DCtx* dctx, const ZSTD_DCtx* preparedDCtx); +ZSTDLIB_API size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx); +ZSTDLIB_API size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); +typedef enum { ZSTDnit_frameHeader, ZSTDnit_blockHeader, ZSTDnit_block, ZSTDnit_lastBlock, ZSTDnit_checksum, ZSTDnit_skippableFrame } ZSTD_nextInputType_e; +ZSTDLIB_API ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx); + +/** + Block functions + + Block functions produce and decode raw zstd blocks, without frame metadata. Frame metadata cost is typically ~18 bytes, which can be non-negligible for very small blocks (< 100 bytes). User will have to take in charge required information to regenerate data, such as compressed and content sizes. @@ -585,6 +613,7 @@ ZSTDLIB_API ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx); */ #define ZSTD_BLOCKSIZE_ABSOLUTEMAX (128 * 1024) /* define, for static allocation */ +/*===== Raw zstd block functions =====*/ ZSTDLIB_API size_t ZSTD_getBlockSizeMax(ZSTD_CCtx* cctx); ZSTDLIB_API size_t ZSTD_compressBlock (ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); ZSTDLIB_API size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);