From 9516234e67dc71321b984bd49919887ccb5a990f Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 25 Oct 2016 16:19:52 -0700 Subject: [PATCH] first sketch for ZSTD_initCStream_usingCDict() --- NEWS | 1 + lib/compress/zstd_compress.c | 30 +++++++++++++++++++++++++----- lib/zstd.h | 2 ++ tests/zstreamtest.c | 26 ++++++++++++++++++++++++-- 4 files changed, 52 insertions(+), 7 deletions(-) diff --git a/NEWS b/NEWS index 27bf47a92..2e6118648 100644 --- a/NEWS +++ b/NEWS @@ -1,6 +1,7 @@ v1.1.1 New : command -M#, --memory=, --memlimit=, --memlimit-decompress= to limit allowed memory consumption New : doc/zstd_manual.html, by Przemyslaw Skibinski +Improved : slightly better compression ratio at --ultra levels (>= 20) Changed : zstd_errors.h is now part of include installation Fixed : zstd-small, zstd-compress and zstd-decompress compilation targets diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 3dbaff56b..5409c233a 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -122,6 +122,11 @@ const seqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx) /* hidden interface * return &(ctx->seqStore); } +static ZSTD_parameters ZSTD_getParamsFromCCtx(const ZSTD_CCtx* cctx) +{ + return cctx->params; +} + /** ZSTD_checkParams() : ensure param values remain within authorized range. @@ -2779,6 +2784,10 @@ size_t ZSTD_freeCDict(ZSTD_CDict* cdict) } } +static ZSTD_parameters ZSTD_getParamsFromCDict(const ZSTD_CDict* cdict) { + return ZSTD_getParamsFromCCtx(cdict->refContext); +} + size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict, U64 pledgedSrcSize) { if (cdict->dictContentSize) CHECK_F(ZSTD_copyCCtx(cctx, cdict->refContext, pledgedSrcSize)) @@ -2815,7 +2824,8 @@ typedef enum { zcss_init, zcss_load, zcss_flush, zcss_final } ZSTD_cStreamStage; struct ZSTD_CStream_s { ZSTD_CCtx* cctx; - ZSTD_CDict* cdict; + ZSTD_CDict* cdictLocal; + const ZSTD_CDict* cdict; char* inBuff; size_t inBuffSize; size_t inToCompress; @@ -2858,7 +2868,7 @@ size_t ZSTD_freeCStream(ZSTD_CStream* zcs) if (zcs==NULL) return 0; /* support free on NULL */ { ZSTD_customMem const cMem = zcs->customMem; ZSTD_freeCCtx(zcs->cctx); - ZSTD_freeCDict(zcs->cdict); + ZSTD_freeCDict(zcs->cdictLocal); ZSTD_free(zcs->inBuff, cMem); ZSTD_free(zcs->outBuff, cMem); ZSTD_free(zcs, cMem); @@ -2906,15 +2916,25 @@ size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, if (zcs->outBuff == NULL) return ERROR(memory_allocation); } - ZSTD_freeCDict(zcs->cdict); - zcs->cdict = ZSTD_createCDict_advanced(dict, dictSize, params, zcs->customMem); - if (zcs->cdict == NULL) return ERROR(memory_allocation); + ZSTD_freeCDict(zcs->cdictLocal); + zcs->cdictLocal = ZSTD_createCDict_advanced(dict, dictSize, params, zcs->customMem); + if (zcs->cdictLocal == NULL) return ERROR(memory_allocation); + zcs->cdict = zcs->cdictLocal; zcs->checksum = params.fParams.checksumFlag > 0; return ZSTD_resetCStream(zcs, pledgedSrcSize); } +/* note : cdict must outlive compression session */ +size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict) +{ + ZSTD_parameters const params = ZSTD_getParamsFromCDict(cdict); + size_t const initError = ZSTD_initCStream_advanced(zcs, NULL, 0, params, 0); + zcs->cdict = cdict; + return initError; +} + size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel) { ZSTD_parameters const params = ZSTD_getParams(compressionLevel, 0, dictSize); diff --git a/lib/zstd.h b/lib/zstd.h index e0d4ce99f..89b6c8274 100644 --- a/lib/zstd.h +++ b/lib/zstd.h @@ -459,6 +459,7 @@ ZSTDLIB_API ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem); ZSTDLIB_API size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel); ZSTDLIB_API size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize); /**< pledgedSrcSize is optional and can be zero == unknown */ +ZSTDLIB_API size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict); /**< note : cdict will just be referenced, and must outlive compression session */ ZSTDLIB_API size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize); /**< re-use compression parameters from previous init; saves dictionary loading */ ZSTDLIB_API size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs); @@ -468,6 +469,7 @@ typedef enum { ZSTDdsp_maxWindowSize } ZSTD_DStreamParameter_e; ZSTDLIB_API ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem); ZSTDLIB_API size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize); ZSTDLIB_API size_t ZSTD_setDStreamParameter(ZSTD_DStream* zds, ZSTD_DStreamParameter_e paramType, unsigned paramValue); +ZSTDLIB_API size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDict* ddict); /**< note : ddict will just be referenced, and must outlive decompression session */ ZSTDLIB_API size_t ZSTD_resetDStream(ZSTD_DStream* zds); /**< re-use decompression parameters from previous init; saves dictionary loading */ ZSTDLIB_API size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds); diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c index 8486013c2..963078d8a 100644 --- a/tests/zstreamtest.c +++ b/tests/zstreamtest.c @@ -77,7 +77,7 @@ static clock_t FUZ_GetClockSpan(clock_t clockStart) /*! FUZ_rand() : @return : a 27 bits random value, from a 32-bits `seed`. `seed` is also modified */ -# define FUZ_rotl32(x,r) ((x << r) | (x >> (32 - r))) +#define FUZ_rotl32(x,r) ((x << r) | (x >> (32 - r))) unsigned int FUZ_rand(unsigned int* seedPtr) { U32 rand32 = *seedPtr; @@ -281,6 +281,28 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo } DISPLAYLEVEL(4, "OK \n"); + /* CDict scenario */ + DISPLAYLEVEL(4, "test%3i : digested dictionary : ", testNb++); + { ZSTD_CDict* const cdict = ZSTD_createCDict(CNBuffer, 128 KB, 1); + size_t const initError = ZSTD_initCStream_usingCDict(zc, cdict); + if (ZSTD_isError(initError)) goto _output_error; + cSize = 0; + outBuff.dst = compressedBuffer; + outBuff.size = compressedBufferSize; + outBuff.pos = 0; + inBuff.src = CNBuffer; + inBuff.size = CNBufferSize; + inBuff.pos = 0; + { size_t const r = ZSTD_compressStream(zc, &outBuff, &inBuff); + if (ZSTD_isError(r)) goto _output_error; } + if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */ + { size_t const r = ZSTD_endStream(zc, &outBuff); + if (r != 0) goto _output_error; } /* error, or some data not flushed */ + cSize = outBuff.pos; + ZSTD_freeCDict(cdict); + DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/CNBufferSize*100); + } + DISPLAYLEVEL(4, "test%3i : check CStream size : ", testNb++); { size_t const s = ZSTD_sizeof_CStream(zc); if (ZSTD_isError(s)) goto _output_error; @@ -511,7 +533,7 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compres /* multi - fragments decompression test */ if (!dictSize /* don't reset if dictionary : could be different */ && (FUZ_rand(&lseed) & 1)) { - CHECK( ZSTD_isError(ZSTD_resetDStream(zd)), "ZSTD_resetDStream failed"); + CHECK (ZSTD_isError(ZSTD_resetDStream(zd)), "ZSTD_resetDStream failed"); } else ZSTD_initDStream_usingDict(zd, dict, dictSize); { size_t decompressionResult = 1;