ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

moved a few structures from zstd_internal.h to zstd_compress.h

Browse Source 
which is a more precise scope
This commit is contained in:
Yann Collet
2017-11-07 16:03:14 -08:00
parent aec56a52fb
commit 8b6aecf2cb
4 changed files with 164 additions and 152 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/common/zstd_internal.h
+11 -142
View File
@@ -11,6 +11,10 @@
#ifndef ZSTD_CCOMMON_H_MODULE #ifndef ZSTD_CCOMMON_H_MODULE
#define ZSTD_CCOMMON_H_MODULE #define ZSTD_CCOMMON_H_MODULE
/* this module contains definitions which must be identical
* across compression, decompression and dictBuilder.
* It also contains a few functions useful to at least 2 of them
* and which benefit from being inlined */
/*-************************************* /*-*************************************
* Dependencies * Dependencies
@@ -191,17 +195,14 @@ MEM_STATIC void ZSTD_wildcopy_e(void* dst, const void* src, void* dstEnd) /* s
/*-******************************************* /*-*******************************************
* Private interfaces * Private declarations
*********************************************/ *********************************************/
typedef struct ZSTD_stats_s ZSTD_stats_t;
typedef struct seqDef_s { typedef struct seqDef_s {
U32 offset; U32 offset;
U16 litLength; U16 litLength;
U16 matchLength; U16 matchLength;
} seqDef; } seqDef;
typedef struct { typedef struct {
seqDef* sequencesStart; seqDef* sequencesStart;
seqDef* sequences; seqDef* sequences;
@@ -216,100 +217,8 @@ typedef struct {
U32 repToConfirm[ZSTD_REP_NUM]; U32 repToConfirm[ZSTD_REP_NUM];
} seqStore_t; } seqStore_t;
typedef struct { const seqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx); /* compress & dictBuilder */
U32 off; void ZSTD_seqToCodes(const seqStore_t* seqStorePtr); /* compress, dictBuilder, decodeCorpus (shouldn't get its definition from here) */
U32 len;
} ZSTD_match_t;
typedef struct {
U32 price;
U32 off;
U32 mlen;
U32 litlen;
U32 rep[ZSTD_REP_NUM];
} ZSTD_optimal_t;
typedef struct {
U32* litFreq;
U32* litLengthFreq;
U32* matchLengthFreq;
U32* offCodeFreq;
ZSTD_match_t* matchTable;
ZSTD_optimal_t* priceTable;
U32 matchLengthSum;
U32 matchSum;
U32 litLengthSum;
U32 litSum;
U32 offCodeSum;
U32 log2matchLengthSum;
U32 log2matchSum;
U32 log2litLengthSum;
U32 log2litSum;
U32 log2offCodeSum;
U32 factor;
U32 staticPrices;
U32 cachedPrice;
U32 cachedLitLength;
const BYTE* cachedLiterals;
} optState_t;
typedef struct {
U32 offset;
U32 checksum;
} ldmEntry_t;
typedef struct {
ldmEntry_t* hashTable;
BYTE* bucketOffsets; /* Next position in bucket to insert entry */
U64 hashPower; /* Used to compute the rolling hash.
* Depends on ldmParams.minMatchLength */
} ldmState_t;
typedef struct {
U32 enableLdm; /* 1 if enable long distance matching */
U32 hashLog; /* Log size of hashTable */
U32 bucketSizeLog; /* Log bucket size for collision resolution, at most 8 */
U32 minMatchLength; /* Minimum match length */
U32 hashEveryLog; /* Log number of entries to skip */
} ldmParams_t;
typedef struct {
U32 hufCTable[HUF_CTABLE_SIZE_U32(255)];
FSE_CTable offcodeCTable[FSE_CTABLE_SIZE_U32(OffFSELog, MaxOff)];
FSE_CTable matchlengthCTable[FSE_CTABLE_SIZE_U32(MLFSELog, MaxML)];
FSE_CTable litlengthCTable[FSE_CTABLE_SIZE_U32(LLFSELog, MaxLL)];
U32 workspace[HUF_WORKSPACE_SIZE_U32];
HUF_repeat hufCTable_repeatMode;
FSE_repeat offcode_repeatMode;
FSE_repeat matchlength_repeatMode;
FSE_repeat litlength_repeatMode;
} ZSTD_entropyCTables_t;
struct ZSTD_CCtx_params_s {
ZSTD_format_e format;
ZSTD_compressionParameters cParams;
ZSTD_frameParameters fParams;
int compressionLevel;
U32 forceWindow; /* force back-references to respect limit of
* 1<<wLog, even for dictionary */
/* Multithreading: used to pass parameters to mtctx */
U32 nbThreads;
unsigned jobSize;
unsigned overlapSizeLog;
/* Long distance matching parameters */
ldmParams_t ldmParams;
/* For use with createCCtxParams() and freeCCtxParams() only */
ZSTD_customMem customMem;
}; /* typedef'd to ZSTD_CCtx_params within "zstd.h" */
const seqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx);
void ZSTD_seqToCodes(const seqStore_t* seqStorePtr);
/* custom memory allocation functions */ /* custom memory allocation functions */
void* ZSTD_malloc(size_t size, ZSTD_customMem customMem); void* ZSTD_malloc(size_t size, ZSTD_customMem customMem);
@@ -317,9 +226,7 @@ void* ZSTD_calloc(size_t size, ZSTD_customMem customMem);
void ZSTD_free(void* ptr, ZSTD_customMem customMem); void ZSTD_free(void* ptr, ZSTD_customMem customMem);
/*====== common function ======*/ MEM_STATIC U32 ZSTD_highbit32(U32 val) /* compress, dictBuilder, decodeCorpus */
MEM_STATIC U32 ZSTD_highbit32(U32 val)
{ {
assert(val != 0); assert(val != 0);
{ {
@@ -345,52 +252,13 @@ MEM_STATIC U32 ZSTD_highbit32(U32 val)
} }
/* hidden functions */
/* ZSTD_invalidateRepCodes() : /* ZSTD_invalidateRepCodes() :
* ensures next compression will not use repcodes from previous block. * ensures next compression will not use repcodes from previous block.
* Note : only works with regular variant; * Note : only works with regular variant;
* do not use with extDict variant ! */ * do not use with extDict variant ! */
void ZSTD_invalidateRepCodes(ZSTD_CCtx* cctx); void ZSTD_invalidateRepCodes(ZSTD_CCtx* cctx); /* zstdmt, adaptive_compression (shouldn't get this definition from here) */
/*! ZSTD_initCStream_internal() :
* Private use only. Init streaming operation.
* expects params to be valid.
* must receive dict, or cdict, or none, but not both.
* @return : 0, or an error code */
size_t ZSTD_initCStream_internal(ZSTD_CStream* zcs,
const void* dict, size_t dictSize,
const ZSTD_CDict* cdict,
ZSTD_CCtx_params params, unsigned long long pledgedSrcSize);
/*! ZSTD_compressStream_generic() :
* Private use only. To be called from zstdmt_compress.c in single-thread mode. */
size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs,
ZSTD_outBuffer* output,
ZSTD_inBuffer* input,
ZSTD_EndDirective const flushMode);
/*! ZSTD_getCParamsFromCDict() :
* as the name implies */
ZSTD_compressionParameters ZSTD_getCParamsFromCDict(const ZSTD_CDict* cdict);
/* ZSTD_compressBegin_advanced_internal() :
* Private use only. To be called from zstdmt_compress.c. */
size_t ZSTD_compressBegin_advanced_internal(ZSTD_CCtx* cctx,
const void* dict, size_t dictSize,
ZSTD_dictMode_e dictMode,
ZSTD_CCtx_params params,
unsigned long long pledgedSrcSize);
/* ZSTD_compress_advanced_internal() :
* Private use only. To be called from zstdmt_compress.c. */
size_t ZSTD_compress_advanced_internal(ZSTD_CCtx* cctx,
void* dst, size_t dstCapacity,
const void* src, size_t srcSize,
const void* dict,size_t dictSize,
ZSTD_CCtx_params params);
typedef struct { typedef struct {
blockType_e blockType; blockType_e blockType;
U32 lastBlock; U32 lastBlock;
@@ -398,7 +266,8 @@ typedef struct {
} blockProperties_t; } blockProperties_t;
/*! ZSTD_getcBlockSize() : /*! ZSTD_getcBlockSize() :
* Provides the size of compressed block from block header `src` */ * Provides the size of compressed block from block header `src`
* Used by: decompress, fullbench (does not get its definition from here) */
size_t ZSTD_getcBlockSize(const void* src, size_t srcSize, size_t ZSTD_getcBlockSize(const void* src, size_t srcSize,
blockProperties_t* bpPtr); blockProperties_t* bpPtr);
lib/compress/zstd_compress.h
+143 -3
View File
@@ -8,6 +8,9 @@
* You may select, at your option, one of the above-listed licenses. * You may select, at your option, one of the above-listed licenses.
*/ */
/* This header contains definitions
* that shall only be used by modules from within lib/compress.
*/
#ifndef ZSTD_COMPRESS_H #ifndef ZSTD_COMPRESS_H
#define ZSTD_COMPRESS_H #define ZSTD_COMPRESS_H
@@ -43,6 +46,98 @@ typedef struct ZSTD_prefixDict_s {
ZSTD_dictMode_e dictMode; ZSTD_dictMode_e dictMode;
} ZSTD_prefixDict; } ZSTD_prefixDict;
typedef struct {
U32 hufCTable[HUF_CTABLE_SIZE_U32(255)];
FSE_CTable offcodeCTable[FSE_CTABLE_SIZE_U32(OffFSELog, MaxOff)];
FSE_CTable matchlengthCTable[FSE_CTABLE_SIZE_U32(MLFSELog, MaxML)];
FSE_CTable litlengthCTable[FSE_CTABLE_SIZE_U32(LLFSELog, MaxLL)];
U32 workspace[HUF_WORKSPACE_SIZE_U32];
HUF_repeat hufCTable_repeatMode;
FSE_repeat offcode_repeatMode;
FSE_repeat matchlength_repeatMode;
FSE_repeat litlength_repeatMode;
} ZSTD_entropyCTables_t;
typedef struct {
U32 off;
U32 len;
} ZSTD_match_t;
typedef struct {
U32 price;
U32 off;
U32 mlen;
U32 litlen;
U32 rep[ZSTD_REP_NUM];
} ZSTD_optimal_t;
typedef struct {
U32* litFreq;
U32* litLengthFreq;
U32* matchLengthFreq;
U32* offCodeFreq;
ZSTD_match_t* matchTable;
ZSTD_optimal_t* priceTable;
U32 matchLengthSum;
U32 matchSum;
U32 litLengthSum;
U32 litSum;
U32 offCodeSum;
U32 log2matchLengthSum;
U32 log2matchSum;
U32 log2litLengthSum;
U32 log2litSum;
U32 log2offCodeSum;
U32 factor;
U32 staticPrices;
U32 cachedPrice;
U32 cachedLitLength;
const BYTE* cachedLiterals;
} optState_t;
typedef struct {
U32 offset;
U32 checksum;
} ldmEntry_t;
typedef struct {
ldmEntry_t* hashTable;
BYTE* bucketOffsets; /* Next position in bucket to insert entry */
U64 hashPower; /* Used to compute the rolling hash.
* Depends on ldmParams.minMatchLength */
} ldmState_t;
typedef struct {
U32 enableLdm; /* 1 if enable long distance matching */
U32 hashLog; /* Log size of hashTable */
U32 bucketSizeLog; /* Log bucket size for collision resolution, at most 8 */
U32 minMatchLength; /* Minimum match length */
U32 hashEveryLog; /* Log number of entries to skip */
} ldmParams_t;
struct ZSTD_CCtx_params_s {
ZSTD_format_e format;
ZSTD_compressionParameters cParams;
ZSTD_frameParameters fParams;
int compressionLevel;
U32 forceWindow; /* force back-references to respect limit of
* 1<<wLog, even for dictionary */
/* Multithreading: used to pass parameters to mtctx */
U32 nbThreads;
unsigned jobSize;
unsigned overlapSizeLog;
/* Long distance matching parameters */
ldmParams_t ldmParams;
/* For use with createCCtxParams() and freeCCtxParams() only */
ZSTD_customMem customMem;
}; /* typedef'd to ZSTD_CCtx_params within "zstd.h" */
struct ZSTD_CCtx_s { struct ZSTD_CCtx_s {
const BYTE* nextSrc; /* next block here to continue on current prefix */ const BYTE* nextSrc; /* next block here to continue on current prefix */
const BYTE* base; /* All regular indexes relative to this position */ const BYTE* base; /* All regular indexes relative to this position */
@@ -118,9 +213,9 @@ static const BYTE ML_Code[128] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11
42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42 }; 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42 };
/*! ZSTD_storeSeq() : /*! ZSTD_storeSeq() :
Store a sequence (literal length, literals, offset code and match length code) into seqStore_t. * Store a sequence (literal length, literals, offset code and match length code) into seqStore_t.
`offsetCode` : distance to match, or 0 == repCode. * `offsetCode` : distance to match + 3 (values 1-3 are repCodes).
`matchCode` : matchLength - MINMATCH * `matchCode` : matchLength - MINMATCH
*/ */
MEM_STATIC void ZSTD_storeSeq(seqStore_t* seqStorePtr, size_t litLength, const void* literals, U32 offsetCode, size_t matchCode) MEM_STATIC void ZSTD_storeSeq(seqStore_t* seqStorePtr, size_t litLength, const void* literals, U32 offsetCode, size_t matchCode)
{ {
@@ -139,6 +234,7 @@ MEM_STATIC void ZSTD_storeSeq(seqStore_t* seqStorePtr, size_t litLength, const v
/* literal Length */ /* literal Length */
if (litLength>0xFFFF) { if (litLength>0xFFFF) {
assert(seqStorePtr->longLengthID == 0); /* there can only be a single long length */
seqStorePtr->longLengthID = 1; seqStorePtr->longLengthID = 1;
seqStorePtr->longLengthPos = (U32)(seqStorePtr->sequences - seqStorePtr->sequencesStart); seqStorePtr->longLengthPos = (U32)(seqStorePtr->sequences - seqStorePtr->sequencesStart);
} }
@@ -149,6 +245,7 @@ MEM_STATIC void ZSTD_storeSeq(seqStore_t* seqStorePtr, size_t litLength, const v
/* match Length */ /* match Length */
if (matchCode>0xFFFF) { if (matchCode>0xFFFF) {
assert(seqStorePtr->longLengthID == 0); /* there can only be a single long length */
seqStorePtr->longLengthID = 2; seqStorePtr->longLengthID = 2;
seqStorePtr->longLengthPos = (U32)(seqStorePtr->sequences - seqStorePtr->sequencesStart); seqStorePtr->longLengthPos = (U32)(seqStorePtr->sequences - seqStorePtr->sequencesStart);
} }
@@ -304,4 +401,47 @@ MEM_STATIC size_t ZSTD_hashPtr(const void* p, U32 hBits, U32 mls)
} }
#endif #endif
/* ==============================================================
* Private declarations
* These prototypes shall only be called from within lib/compress
* ============================================================== */
/*! ZSTD_initCStream_internal() :
* Private use only. Init streaming operation.
* expects params to be valid.
* must receive dict, or cdict, or none, but not both.
* @return : 0, or an error code */
size_t ZSTD_initCStream_internal(ZSTD_CStream* zcs,
const void* dict, size_t dictSize,
const ZSTD_CDict* cdict,
ZSTD_CCtx_params params, unsigned long long pledgedSrcSize);
/*! ZSTD_compressStream_generic() :
* Private use only. To be called from zstdmt_compress.c in single-thread mode. */
size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs,
ZSTD_outBuffer* output,
ZSTD_inBuffer* input,
ZSTD_EndDirective const flushMode);
/*! ZSTD_getCParamsFromCDict() :
* as the name implies */
ZSTD_compressionParameters ZSTD_getCParamsFromCDict(const ZSTD_CDict* cdict);
/* ZSTD_compressBegin_advanced_internal() :
* Private use only. To be called from zstdmt_compress.c. */
size_t ZSTD_compressBegin_advanced_internal(ZSTD_CCtx* cctx,
const void* dict, size_t dictSize,
ZSTD_dictMode_e dictMode,
ZSTD_CCtx_params params,
unsigned long long pledgedSrcSize);
/* ZSTD_compress_advanced_internal() :
* Private use only. To be called from zstdmt_compress.c. */
size_t ZSTD_compress_advanced_internal(ZSTD_CCtx* cctx,
void* dst, size_t dstCapacity,
const void* src, size_t srcSize,
const void* dict,size_t dictSize,
ZSTD_CCtx_params params);
#endif /* ZSTD_COMPRESS_H */ #endif /* ZSTD_COMPRESS_H */
lib/compress/zstd_opt.c
+9 -6
View File
@@ -25,19 +25,19 @@ static void ZSTD_setLog2Prices(optState_t* optPtr)
optPtr->log2litLengthSum = ZSTD_highbit32(optPtr->litLengthSum+1); optPtr->log2litLengthSum = ZSTD_highbit32(optPtr->litLengthSum+1);
optPtr->log2litSum = ZSTD_highbit32(optPtr->litSum+1); optPtr->log2litSum = ZSTD_highbit32(optPtr->litSum+1);
optPtr->log2offCodeSum = ZSTD_highbit32(optPtr->offCodeSum+1); optPtr->log2offCodeSum = ZSTD_highbit32(optPtr->offCodeSum+1);
optPtr->factor = 1 + ((optPtr->litSum>>5) / optPtr->litLengthSum) + ((optPtr->litSum<<1) / (optPtr->litSum + optPtr->matchSum)); optPtr->factor = 1 + ((optPtr->litSum>>5) / optPtr->litLengthSum)
+ ((optPtr->litSum<<1) / (optPtr->litSum + optPtr->matchSum));
} }
static void ZSTD_rescaleFreqs(optState_t* optPtr, const BYTE* src, size_t srcSize) static void ZSTD_rescaleFreqs(optState_t* optPtr, const BYTE* src, size_t srcSize)
{ {
unsigned u;
optPtr->cachedLiterals = NULL; optPtr->cachedLiterals = NULL;
optPtr->cachedPrice = optPtr->cachedLitLength = 0; optPtr->cachedPrice = optPtr->cachedLitLength = 0;
optPtr->staticPrices = 0; optPtr->staticPrices = 0;
if (optPtr->litLengthSum == 0) { if (optPtr->litLengthSum == 0) { /* first init */
unsigned u;
if (srcSize <= 1024) optPtr->staticPrices = 1; if (srcSize <= 1024) optPtr->staticPrices = 1;
assert(optPtr->litFreq!=NULL); assert(optPtr->litFreq!=NULL);
@@ -50,10 +50,10 @@ static void ZSTD_rescaleFreqs(optState_t* optPtr, const BYTE* src, size_t srcSiz
optPtr->litLengthSum = MaxLL+1; optPtr->litLengthSum = MaxLL+1;
optPtr->matchLengthSum = MaxML+1; optPtr->matchLengthSum = MaxML+1;
optPtr->offCodeSum = (MaxOff+1); optPtr->offCodeSum = (MaxOff+1);
optPtr->matchSum = (ZSTD_LITFREQ_ADD<<Litbits); optPtr->matchSum = (ZSTD_LITFREQ_ADD << Litbits);
for (u=0; u<=MaxLit; u++) { for (u=0; u<=MaxLit; u++) {
optPtr->litFreq[u] = 1 + (optPtr->litFreq[u]>>ZSTD_FREQ_DIV); optPtr->litFreq[u] = 1 + (optPtr->litFreq[u] >> ZSTD_FREQ_DIV);
optPtr->litSum += optPtr->litFreq[u]; optPtr->litSum += optPtr->litFreq[u];
} }
for (u=0; u<=MaxLL; u++) for (u=0; u<=MaxLL; u++)
@@ -62,7 +62,10 @@ static void ZSTD_rescaleFreqs(optState_t* optPtr, const BYTE* src, size_t srcSiz
optPtr->matchLengthFreq[u] = 1; optPtr->matchLengthFreq[u] = 1;
for (u=0; u<=MaxOff; u++) for (u=0; u<=MaxOff; u++)
optPtr->offCodeFreq[u] = 1; optPtr->offCodeFreq[u] = 1;
} else { } else {
unsigned u;
optPtr->matchLengthSum = 0; optPtr->matchLengthSum = 0;
optPtr->litLengthSum = 0; optPtr->litLengthSum = 0;
optPtr->offCodeSum = 0; optPtr->offCodeSum = 0;
lib/compress/zstdmt_compress.c
+1 -1
View File
@@ -24,7 +24,7 @@
#include <string.h> /* memcpy, memset */ #include <string.h> /* memcpy, memset */
#include "pool.h" /* threadpool */ #include "pool.h" /* threadpool */
#include "threading.h" /* mutex */ #include "threading.h" /* mutex */
#include "zstd_internal.h" /* MIN, ERROR, ZSTD_*, ZSTD_highbit32 */ #include "zstd_compress.h" /* MIN, ERROR, ZSTD_*, ZSTD_highbit32 */
#include "zstdmt_compress.h" #include "zstdmt_compress.h"