ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[linux-kernel] clang-format lib/zstd/

Browse Source
This commit is contained in:
Nick Terrell
2017-05-17 14:40:46 -07:00
parent 588579f3a1
commit 67101c6c77
15 changed files with 5241 additions and 4655 deletions
Download Patch File Download Diff File Expand all files Collapse all files
contrib/linux-kernel/lib/zstd/bitstream.h
+40 -48
View File
@@ -50,9 +50,8 @@
/*-**************************************** /*-****************************************
* Dependencies * Dependencies
******************************************/ ******************************************/
#include "mem.h" /* unaligned access routines */
#include "error_private.h" /* error codes and messages */ #include "error_private.h" /* error codes and messages */
#include "mem.h" /* unaligned access routines */
/*========================================= /*=========================================
* Target specific * Target specific
@@ -68,8 +67,7 @@
* A critical property of these streams is that they encode and decode in **reverse** direction. * A critical property of these streams is that they encode and decode in **reverse** direction.
* So the first bit sequence you add will be the last to be read, like a LIFO stack. * So the first bit sequence you add will be the last to be read, like a LIFO stack.
*/ */
typedef struct typedef struct {
{
size_t bitContainer; size_t bitContainer;
int bitPos; int bitPos;
char *startPtr; char *startPtr;
@@ -99,22 +97,22 @@ ZSTD_STATIC size_t BIT_closeCStream(BIT_CStream_t* bitC);
* If data couldn't fit into `dstBuffer`, it will return a 0 ( == not storable) * If data couldn't fit into `dstBuffer`, it will return a 0 ( == not storable)
*/ */
/*-******************************************** /*-********************************************
* bitStream decoding API (read backward) * bitStream decoding API (read backward)
**********************************************/ **********************************************/
typedef struct typedef struct {
{
size_t bitContainer; size_t bitContainer;
unsigned bitsConsumed; unsigned bitsConsumed;
const char *ptr; const char *ptr;
const char *start; const char *start;
} BIT_DStream_t; } BIT_DStream_t;
typedef enum { BIT_DStream_unfinished = 0, typedef enum {
BIT_DStream_unfinished = 0,
BIT_DStream_endOfBuffer = 1, BIT_DStream_endOfBuffer = 1,
BIT_DStream_completed = 2, BIT_DStream_completed = 2,
BIT_DStream_overflow = 3 } BIT_DStream_status; /* result of BIT_reloadDStream() */ BIT_DStream_overflow = 3
} BIT_DStream_status; /* result of BIT_reloadDStream() */
/* 1,2,4,8 would be better for bitmap combinations, but slows down performance a bit ... :( */ /* 1,2,4,8 would be better for bitmap combinations, but slows down performance a bit ... :( */
ZSTD_STATIC size_t BIT_initDStream(BIT_DStream_t *bitD, const void *srcBuffer, size_t srcSize); ZSTD_STATIC size_t BIT_initDStream(BIT_DStream_t *bitD, const void *srcBuffer, size_t srcSize);
@@ -122,7 +120,6 @@ ZSTD_STATIC size_t BIT_readBits(BIT_DStream_t* bitD, unsigned nbBits);
ZSTD_STATIC BIT_DStream_status BIT_reloadDStream(BIT_DStream_t *bitD); ZSTD_STATIC BIT_DStream_status BIT_reloadDStream(BIT_DStream_t *bitD);
ZSTD_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t *bitD); ZSTD_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t *bitD);
/* Start by invoking BIT_initDStream(). /* Start by invoking BIT_initDStream().
* A chunk of the bitStream is then stored into a local register. * A chunk of the bitStream is then stored into a local register.
* Local register size is 64-bits on 64-bits systems, 32-bits on 32-bits systems (size_t). * Local register size is 64-bits on 64-bits systems, 32-bits on 32-bits systems (size_t).
@@ -133,7 +130,6 @@ ZSTD_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* bitD);
* Checking if DStream has reached its end can be performed with BIT_endOfDStream(). * Checking if DStream has reached its end can be performed with BIT_endOfDStream().
*/ */
/*-**************************************** /*-****************************************
* unsafe API * unsafe API
******************************************/ ******************************************/
@@ -146,19 +142,15 @@ ZSTD_STATIC void BIT_flushBitsFast(BIT_CStream_t* bitC);
ZSTD_STATIC size_t BIT_readBitsFast(BIT_DStream_t *bitD, unsigned nbBits); ZSTD_STATIC size_t BIT_readBitsFast(BIT_DStream_t *bitD, unsigned nbBits);
/* faster, but works only if nbBits >= 1 */ /* faster, but works only if nbBits >= 1 */
/*-************************************************************** /*-**************************************************************
* Internal functions * Internal functions
****************************************************************/ ****************************************************************/
ZSTD_STATIC unsigned BIT_highbit32 (register U32 val) ZSTD_STATIC unsigned BIT_highbit32(register U32 val) { return 31 - __builtin_clz(val); }
{
return 31 - __builtin_clz(val);
}
/*===== Local Constants =====*/ /*===== Local Constants =====*/
static const unsigned BIT_mask[] = { 0, 1, 3, 7, 0xF, 0x1F, 0x3F, 0x7F, 0xFF, 0x1FF, 0x3FF, 0x7FF, 0xFFF, 0x1FFF, 0x3FFF, 0x7FFF, 0xFFFF, 0x1FFFF, 0x3FFFF, 0x7FFFF, 0xFFFFF, 0x1FFFFF, 0x3FFFFF, 0x7FFFFF, 0xFFFFFF, 0x1FFFFFF, 0x3FFFFFF }; /* up to 26 bits */ static const unsigned BIT_mask[] = {0, 1, 3, 7, 0xF, 0x1F, 0x3F, 0x7F, 0xFF,
0x1FF, 0x3FF, 0x7FF, 0xFFF, 0x1FFF, 0x3FFF, 0x7FFF, 0xFFFF, 0x1FFFF,
0x3FFFF, 0x7FFFF, 0xFFFFF, 0x1FFFFF, 0x3FFFFF, 0x7FFFFF, 0xFFFFFF, 0x1FFFFFF, 0x3FFFFFF}; /* up to 26 bits */
/*-************************************************************** /*-**************************************************************
* bitStream encoding * bitStream encoding
@@ -174,7 +166,8 @@ ZSTD_STATIC size_t BIT_initCStream(BIT_CStream_t* bitC, void* startPtr, size_t d
bitC->startPtr = (char *)startPtr; bitC->startPtr = (char *)startPtr;
bitC->ptr = bitC->startPtr; bitC->ptr = bitC->startPtr;
bitC->endPtr = bitC->startPtr + dstCapacity - sizeof(bitC->ptr); bitC->endPtr = bitC->startPtr + dstCapacity - sizeof(bitC->ptr);
if (dstCapacity <= sizeof(bitC->ptr)) return ERROR(dstSize_tooSmall); if (dstCapacity <= sizeof(bitC->ptr))
return ERROR(dstSize_tooSmall);
return 0; return 0;
} }
@@ -214,7 +207,8 @@ ZSTD_STATIC void BIT_flushBits(BIT_CStream_t* bitC)
size_t const nbBytes = bitC->bitPos >> 3; size_t const nbBytes = bitC->bitPos >> 3;
ZSTD_writeLEST(bitC->ptr, bitC->bitContainer); ZSTD_writeLEST(bitC->ptr, bitC->bitContainer);
bitC->ptr += nbBytes; bitC->ptr += nbBytes;
if (bitC->ptr > bitC->endPtr) bitC->ptr = bitC->endPtr; if (bitC->ptr > bitC->endPtr)
bitC->ptr = bitC->endPtr;
bitC->bitPos &= 7; bitC->bitPos &= 7;
bitC->bitContainer >>= nbBytes * 8; /* if bitPos >= sizeof(bitContainer)*8 --> undefined behavior */ bitC->bitContainer >>= nbBytes * 8; /* if bitPos >= sizeof(bitContainer)*8 --> undefined behavior */
} }
@@ -227,12 +221,12 @@ ZSTD_STATIC size_t BIT_closeCStream(BIT_CStream_t* bitC)
BIT_addBitsFast(bitC, 1, 1); /* endMark */ BIT_addBitsFast(bitC, 1, 1); /* endMark */
BIT_flushBits(bitC); BIT_flushBits(bitC);
if (bitC->ptr >= bitC->endPtr) return 0; /* doesn't fit within authorized budget : cancel */ if (bitC->ptr >= bitC->endPtr)
return 0; /* doesn't fit within authorized budget : cancel */
return (bitC->ptr - bitC->startPtr) + (bitC->bitPos > 0); return (bitC->ptr - bitC->startPtr) + (bitC->bitPos > 0);
} }
/*-******************************************************** /*-********************************************************
* bitStream decoding * bitStream decoding
**********************************************************/ **********************************************************/
@@ -244,21 +238,26 @@ ZSTD_STATIC size_t BIT_closeCStream(BIT_CStream_t* bitC)
*/ */
ZSTD_STATIC size_t BIT_initDStream(BIT_DStream_t *bitD, const void *srcBuffer, size_t srcSize) ZSTD_STATIC size_t BIT_initDStream(BIT_DStream_t *bitD, const void *srcBuffer, size_t srcSize)
{ {
if (srcSize < 1) { memset(bitD, 0, sizeof(*bitD)); return ERROR(srcSize_wrong); } if (srcSize < 1) {
memset(bitD, 0, sizeof(*bitD));
return ERROR(srcSize_wrong);
}
if (srcSize >= sizeof(bitD->bitContainer)) { /* normal case */ if (srcSize >= sizeof(bitD->bitContainer)) { /* normal case */
bitD->start = (const char *)srcBuffer; bitD->start = (const char *)srcBuffer;
bitD->ptr = (const char *)srcBuffer + srcSize - sizeof(bitD->bitContainer); bitD->ptr = (const char *)srcBuffer + srcSize - sizeof(bitD->bitContainer);
bitD->bitContainer = ZSTD_readLEST(bitD->ptr); bitD->bitContainer = ZSTD_readLEST(bitD->ptr);
{ BYTE const lastByte = ((const BYTE*)srcBuffer)[srcSize-1]; {
BYTE const lastByte = ((const BYTE *)srcBuffer)[srcSize - 1];
bitD->bitsConsumed = lastByte ? 8 - BIT_highbit32(lastByte) : 0; /* ensures bitsConsumed is always set */ bitD->bitsConsumed = lastByte ? 8 - BIT_highbit32(lastByte) : 0; /* ensures bitsConsumed is always set */
if (lastByte == 0) return ERROR(GENERIC); /* endMark not present */ } if (lastByte == 0)
return ERROR(GENERIC); /* endMark not present */
}
} else { } else {
bitD->start = (const char *)srcBuffer; bitD->start = (const char *)srcBuffer;
bitD->ptr = bitD->start; bitD->ptr = bitD->start;
bitD->bitContainer = *(const BYTE *)(bitD->start); bitD->bitContainer = *(const BYTE *)(bitD->start);
switch(srcSize) switch (srcSize) {
{
case 7: bitD->bitContainer += (size_t)(((const BYTE *)(srcBuffer))[6]) << (sizeof(bitD->bitContainer) * 8 - 16); case 7: bitD->bitContainer += (size_t)(((const BYTE *)(srcBuffer))[6]) << (sizeof(bitD->bitContainer) * 8 - 16);
case 6: bitD->bitContainer += (size_t)(((const BYTE *)(srcBuffer))[5]) << (sizeof(bitD->bitContainer) * 8 - 24); case 6: bitD->bitContainer += (size_t)(((const BYTE *)(srcBuffer))[5]) << (sizeof(bitD->bitContainer) * 8 - 24);
case 5: bitD->bitContainer += (size_t)(((const BYTE *)(srcBuffer))[4]) << (sizeof(bitD->bitContainer) * 8 - 32); case 5: bitD->bitContainer += (size_t)(((const BYTE *)(srcBuffer))[4]) << (sizeof(bitD->bitContainer) * 8 - 32);
@@ -267,29 +266,23 @@ ZSTD_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, s
case 2: bitD->bitContainer += (size_t)(((const BYTE *)(srcBuffer))[1]) << 8; case 2: bitD->bitContainer += (size_t)(((const BYTE *)(srcBuffer))[1]) << 8;
default:; default:;
} }
{ BYTE const lastByte = ((const BYTE*)srcBuffer)[srcSize-1]; {
BYTE const lastByte = ((const BYTE *)srcBuffer)[srcSize - 1];
bitD->bitsConsumed = lastByte ? 8 - BIT_highbit32(lastByte) : 0; bitD->bitsConsumed = lastByte ? 8 - BIT_highbit32(lastByte) : 0;
if (lastByte == 0) return ERROR(GENERIC); /* endMark not present */ } if (lastByte == 0)
return ERROR(GENERIC); /* endMark not present */
}
bitD->bitsConsumed += (U32)(sizeof(bitD->bitContainer) - srcSize) * 8; bitD->bitsConsumed += (U32)(sizeof(bitD->bitContainer) - srcSize) * 8;
} }
return srcSize; return srcSize;
} }
ZSTD_STATIC size_t BIT_getUpperBits(size_t bitContainer, U32 const start) ZSTD_STATIC size_t BIT_getUpperBits(size_t bitContainer, U32 const start) { return bitContainer >> start; }
{
return bitContainer >> start;
}
ZSTD_STATIC size_t BIT_getMiddleBits(size_t bitContainer, U32 const start, U32 const nbBits) ZSTD_STATIC size_t BIT_getMiddleBits(size_t bitContainer, U32 const start, U32 const nbBits) { return (bitContainer >> start) & BIT_mask[nbBits]; }
{
return (bitContainer >> start) & BIT_mask[nbBits];
}
ZSTD_STATIC size_t BIT_getLowerBits(size_t bitContainer, U32 const nbBits) ZSTD_STATIC size_t BIT_getLowerBits(size_t bitContainer, U32 const nbBits) { return bitContainer & BIT_mask[nbBits]; }
{
return bitContainer & BIT_mask[nbBits];
}
/*! BIT_lookBits() : /*! BIT_lookBits() :
* Provides next n bits from local register. * Provides next n bits from local register.
@@ -312,10 +305,7 @@ ZSTD_STATIC size_t BIT_lookBitsFast(const BIT_DStream_t* bitD, U32 nbBits)
return (bitD->bitContainer << (bitD->bitsConsumed & bitMask)) >> (((bitMask + 1) - nbBits) & bitMask); return (bitD->bitContainer << (bitD->bitsConsumed & bitMask)) >> (((bitMask + 1) - nbBits) & bitMask);
} }
ZSTD_STATIC void BIT_skipBits(BIT_DStream_t* bitD, U32 nbBits) ZSTD_STATIC void BIT_skipBits(BIT_DStream_t *bitD, U32 nbBits) { bitD->bitsConsumed += nbBits; }
{
bitD->bitsConsumed += nbBits;
}
/*! BIT_readBits() : /*! BIT_readBits() :
* Read (consume) next n bits from local register and update. * Read (consume) next n bits from local register and update.
@@ -355,10 +345,12 @@ ZSTD_STATIC BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD)
return BIT_DStream_unfinished; return BIT_DStream_unfinished;
} }
if (bitD->ptr == bitD->start) { if (bitD->ptr == bitD->start) {
if (bitD->bitsConsumed < sizeof(bitD->bitContainer)*8) return BIT_DStream_endOfBuffer; if (bitD->bitsConsumed < sizeof(bitD->bitContainer) * 8)
return BIT_DStream_endOfBuffer;
return BIT_DStream_completed; return BIT_DStream_completed;
} }
{ U32 nbBytes = bitD->bitsConsumed >> 3; {
U32 nbBytes = bitD->bitsConsumed >> 3;
BIT_DStream_status result = BIT_DStream_unfinished; BIT_DStream_status result = BIT_DStream_unfinished;
if (bitD->ptr - nbBytes < bitD->start) { if (bitD->ptr - nbBytes < bitD->start) {
nbBytes = (U32)(bitD->ptr - bitD->start); /* ptr > start */ nbBytes = (U32)(bitD->ptr - bitD->start); /* ptr > start */
contrib/linux-kernel/lib/zstd/compress.c
+847 -686
View File
@@ -14,17 +14,16 @@
* ("BSD"). * ("BSD").
*/ */
/*-************************************* /*-*************************************
* Dependencies * Dependencies
***************************************/ ***************************************/
#include "fse.h"
#include "huf.h"
#include "mem.h"
#include "zstd_internal.h" /* includes zstd.h */
#include <linux/kernel.h> #include <linux/kernel.h>
#include <linux/module.h> #include <linux/module.h>
#include <linux/string.h> /* memset */ #include <linux/string.h> /* memset */
#include "mem.h"
#include "fse.h"
#include "huf.h"
#include "zstd_internal.h" /* includes zstd.h */
/*-************************************* /*-*************************************
* Constants * Constants
@@ -33,14 +32,15 @@ static const U32 g_searchStrength = 8; /* control skip over incompressible dat
#define HASH_READ_SIZE 8 #define HASH_READ_SIZE 8
typedef enum { ZSTDcs_created = 0, ZSTDcs_init, ZSTDcs_ongoing, ZSTDcs_ending } ZSTD_compressionStage_e; typedef enum { ZSTDcs_created = 0, ZSTDcs_init, ZSTDcs_ongoing, ZSTDcs_ending } ZSTD_compressionStage_e;
/*-************************************* /*-*************************************
* Helper functions * Helper functions
***************************************/ ***************************************/
#define ZSTD_STATIC_ASSERT(c) { enum { ZSTD_static_assert = 1/(int)(!!(c)) }; } #define ZSTD_STATIC_ASSERT(c) \
{ \
enum { ZSTD_static_assert = 1 / (int)(!!(c)) }; \
}
size_t ZSTD_compressBound(size_t srcSize) { return FSE_compressBound(srcSize) + 12; } size_t ZSTD_compressBound(size_t srcSize) { return FSE_compressBound(srcSize) + 12; }
/*-************************************* /*-*************************************
* Sequence storage * Sequence storage
***************************************/ ***************************************/
@@ -51,7 +51,6 @@ static void ZSTD_resetSeqStore(seqStore_t* ssPtr)
ssPtr->longLengthID = 0; ssPtr->longLengthID = 0;
} }
/*-************************************* /*-*************************************
* Context memory management * Context memory management
***************************************/ ***************************************/
@@ -92,7 +91,8 @@ struct ZSTD_CCtx_s {
unsigned tmpCounters[HUF_WORKSPACE_SIZE_U32]; unsigned tmpCounters[HUF_WORKSPACE_SIZE_U32];
}; };
size_t ZSTD_CCtxWorkspaceBound(ZSTD_compressionParameters cParams) { size_t ZSTD_CCtxWorkspaceBound(ZSTD_compressionParameters cParams)
{
size_t const blockSize = MIN(ZSTD_BLOCKSIZE_ABSOLUTEMAX, (size_t)1 << cParams.windowLog); size_t const blockSize = MIN(ZSTD_BLOCKSIZE_ABSOLUTEMAX, (size_t)1 << cParams.windowLog);
U32 const divider = (cParams.searchLength == 3) ? 3 : 4; U32 const divider = (cParams.searchLength == 3) ? 3 : 4;
size_t const maxNbSeq = blockSize / divider; size_t const maxNbSeq = blockSize / divider;
@@ -102,8 +102,10 @@ size_t ZSTD_CCtxWorkspaceBound(ZSTD_compressionParameters cParams) {
U32 const hashLog3 = (cParams.searchLength > 3) ? 0 : MIN(ZSTD_HASHLOG3_MAX, cParams.windowLog); U32 const hashLog3 = (cParams.searchLength > 3) ? 0 : MIN(ZSTD_HASHLOG3_MAX, cParams.windowLog);
size_t const h3Size = ((size_t)1) << hashLog3; size_t const h3Size = ((size_t)1) << hashLog3;
size_t const tableSpace = (chainSize + hSize + h3Size) * sizeof(U32); size_t const tableSpace = (chainSize + hSize + h3Size) * sizeof(U32);
size_t const optSpace = ((MaxML+1) + (MaxLL+1) + (MaxOff+1) + (1<<Litbits))*sizeof(U32) + (ZSTD_OPT_NUM+1)*(sizeof(ZSTD_match_t) + sizeof(ZSTD_optimal_t)); size_t const optSpace =
size_t const workspaceSize = tableSpace + (256*sizeof(U32)) /* huffTable */ + tokenSpace + (((cParams.strategy == ZSTD_btopt) || (cParams.strategy == ZSTD_btopt2)) ? optSpace : 0); ((MaxML + 1) + (MaxLL + 1) + (MaxOff + 1) + (1 << Litbits)) * sizeof(U32) + (ZSTD_OPT_NUM + 1) * (sizeof(ZSTD_match_t) + sizeof(ZSTD_optimal_t));
size_t const workspaceSize = tableSpace + (256 * sizeof(U32)) /* huffTable */ + tokenSpace +
(((cParams.strategy == ZSTD_btopt) || (cParams.strategy == ZSTD_btopt2)) ? optSpace : 0);
return ZSTD_ALIGN(sizeof(ZSTD_stack)) + ZSTD_ALIGN(sizeof(ZSTD_CCtx)) + ZSTD_ALIGN(workspaceSize); return ZSTD_ALIGN(sizeof(ZSTD_stack)) + ZSTD_ALIGN(sizeof(ZSTD_CCtx)) + ZSTD_ALIGN(workspaceSize);
} }
@@ -111,9 +113,11 @@ size_t ZSTD_CCtxWorkspaceBound(ZSTD_compressionParameters cParams) {
static ZSTD_CCtx *ZSTD_createCCtx_advanced(ZSTD_customMem customMem) static ZSTD_CCtx *ZSTD_createCCtx_advanced(ZSTD_customMem customMem)
{ {
ZSTD_CCtx *cctx; ZSTD_CCtx *cctx;
if (!customMem.customAlloc || !customMem.customFree) return NULL; if (!customMem.customAlloc || !customMem.customFree)
return NULL;
cctx = (ZSTD_CCtx *)ZSTD_malloc(sizeof(ZSTD_CCtx), customMem); cctx = (ZSTD_CCtx *)ZSTD_malloc(sizeof(ZSTD_CCtx), customMem);
if (!cctx) return NULL; if (!cctx)
return NULL;
memset(cctx, 0, sizeof(ZSTD_CCtx)); memset(cctx, 0, sizeof(ZSTD_CCtx));
cctx->customMem = customMem; cctx->customMem = customMem;
return cctx; return cctx;
@@ -131,40 +135,38 @@ ZSTD_CCtx* ZSTD_initCCtx(void* workspace, size_t workspaceSize)
size_t ZSTD_freeCCtx(ZSTD_CCtx *cctx) size_t ZSTD_freeCCtx(ZSTD_CCtx *cctx)
{ {
if (cctx==NULL) return 0; /* support free on NULL */ if (cctx == NULL)
return 0; /* support free on NULL */
ZSTD_free(cctx->workSpace, cctx->customMem); ZSTD_free(cctx->workSpace, cctx->customMem);
ZSTD_free(cctx, cctx->customMem); ZSTD_free(cctx, cctx->customMem);
return 0; /* reserved as a potential error code in the future */ return 0; /* reserved as a potential error code in the future */
} }
const seqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx) /* hidden interface */ const seqStore_t *ZSTD_getSeqStore(const ZSTD_CCtx *ctx) /* hidden interface */ { return &(ctx->seqStore); }
{
return &(ctx->seqStore);
}
static ZSTD_parameters ZSTD_getParamsFromCCtx(const ZSTD_CCtx* cctx)
{
return cctx->params;
}
static ZSTD_parameters ZSTD_getParamsFromCCtx(const ZSTD_CCtx *cctx) { return cctx->params; }
/** ZSTD_checkParams() : /** ZSTD_checkParams() :
ensure param values remain within authorized range. ensure param values remain within authorized range.
@return : 0, or an error code if one value is beyond authorized range */ @return : 0, or an error code if one value is beyond authorized range */
size_t ZSTD_checkCParams(ZSTD_compressionParameters cParams) size_t ZSTD_checkCParams(ZSTD_compressionParameters cParams)
{ {
# define CLAMPCHECK(val,min,max) { if ((val<min) | (val>max)) return ERROR(compressionParameter_unsupported); } #define CLAMPCHECK(val, min, max) \
{ \
if ((val < min) | (val > max)) \
return ERROR(compressionParameter_unsupported); \
}
CLAMPCHECK(cParams.windowLog, ZSTD_WINDOWLOG_MIN, ZSTD_WINDOWLOG_MAX); CLAMPCHECK(cParams.windowLog, ZSTD_WINDOWLOG_MIN, ZSTD_WINDOWLOG_MAX);
CLAMPCHECK(cParams.chainLog, ZSTD_CHAINLOG_MIN, ZSTD_CHAINLOG_MAX); CLAMPCHECK(cParams.chainLog, ZSTD_CHAINLOG_MIN, ZSTD_CHAINLOG_MAX);
CLAMPCHECK(cParams.hashLog, ZSTD_HASHLOG_MIN, ZSTD_HASHLOG_MAX); CLAMPCHECK(cParams.hashLog, ZSTD_HASHLOG_MIN, ZSTD_HASHLOG_MAX);
CLAMPCHECK(cParams.searchLog, ZSTD_SEARCHLOG_MIN, ZSTD_SEARCHLOG_MAX); CLAMPCHECK(cParams.searchLog, ZSTD_SEARCHLOG_MIN, ZSTD_SEARCHLOG_MAX);
CLAMPCHECK(cParams.searchLength, ZSTD_SEARCHLENGTH_MIN, ZSTD_SEARCHLENGTH_MAX); CLAMPCHECK(cParams.searchLength, ZSTD_SEARCHLENGTH_MIN, ZSTD_SEARCHLENGTH_MAX);
CLAMPCHECK(cParams.targetLength, ZSTD_TARGETLENGTH_MIN, ZSTD_TARGETLENGTH_MAX); CLAMPCHECK(cParams.targetLength, ZSTD_TARGETLENGTH_MIN, ZSTD_TARGETLENGTH_MAX);
if ((U32)(cParams.strategy) > (U32)ZSTD_btopt2) return ERROR(compressionParameter_unsupported); if ((U32)(cParams.strategy) > (U32)ZSTD_btopt2)
return ERROR(compressionParameter_unsupported);
return 0; return 0;
} }
/** ZSTD_cycleLog() : /** ZSTD_cycleLog() :
* condition for correct operation : hashLog > 1 */ * condition for correct operation : hashLog > 1 */
static U32 ZSTD_cycleLog(U32 hashLog, ZSTD_strategy strat) static U32 ZSTD_cycleLog(U32 hashLog, ZSTD_strategy strat)
@@ -181,32 +183,37 @@ static U32 ZSTD_cycleLog(U32 hashLog, ZSTD_strategy strat)
Note : cPar is considered validated at this stage. Use ZSTD_checkParams() to ensure that. */ Note : cPar is considered validated at this stage. Use ZSTD_checkParams() to ensure that. */
ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, unsigned long long srcSize, size_t dictSize) ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, unsigned long long srcSize, size_t dictSize)
{ {
if (srcSize+dictSize == 0) return cPar; /* no size information available : no adjustment */ if (srcSize + dictSize == 0)
return cPar; /* no size information available : no adjustment */
/* resize params, to use less memory when necessary */ /* resize params, to use less memory when necessary */
{ U32 const minSrcSize = (srcSize==0) ? 500 : 0; {
U32 const minSrcSize = (srcSize == 0) ? 500 : 0;
U64 const rSize = srcSize + dictSize + minSrcSize; U64 const rSize = srcSize + dictSize + minSrcSize;
if (rSize < ((U64)1 << ZSTD_WINDOWLOG_MAX)) { if (rSize < ((U64)1 << ZSTD_WINDOWLOG_MAX)) {
U32 const srcLog = MAX(ZSTD_HASHLOG_MIN, ZSTD_highbit32((U32)(rSize)-1) + 1); U32 const srcLog = MAX(ZSTD_HASHLOG_MIN, ZSTD_highbit32((U32)(rSize)-1) + 1);
if (cPar.windowLog > srcLog) cPar.windowLog = srcLog; if (cPar.windowLog > srcLog)
} } cPar.windowLog = srcLog;
if (cPar.hashLog > cPar.windowLog) cPar.hashLog = cPar.windowLog; }
{ U32 const cycleLog = ZSTD_cycleLog(cPar.chainLog, cPar.strategy); }
if (cycleLog > cPar.windowLog) cPar.chainLog -= (cycleLog - cPar.windowLog); if (cPar.hashLog > cPar.windowLog)
cPar.hashLog = cPar.windowLog;
{
U32 const cycleLog = ZSTD_cycleLog(cPar.chainLog, cPar.strategy);
if (cycleLog > cPar.windowLog)
cPar.chainLog -= (cycleLog - cPar.windowLog);
} }
if (cPar.windowLog < ZSTD_WINDOWLOG_ABSOLUTEMIN) cPar.windowLog = ZSTD_WINDOWLOG_ABSOLUTEMIN; /* required for frame header */ if (cPar.windowLog < ZSTD_WINDOWLOG_ABSOLUTEMIN)
cPar.windowLog = ZSTD_WINDOWLOG_ABSOLUTEMIN; /* required for frame header */
return cPar; return cPar;
} }
static U32 ZSTD_equivalentParams(ZSTD_parameters param1, ZSTD_parameters param2) static U32 ZSTD_equivalentParams(ZSTD_parameters param1, ZSTD_parameters param2)
{ {
return (param1.cParams.hashLog == param2.cParams.hashLog) return (param1.cParams.hashLog == param2.cParams.hashLog) & (param1.cParams.chainLog == param2.cParams.chainLog) &
& (param1.cParams.chainLog == param2.cParams.chainLog) (param1.cParams.strategy == param2.cParams.strategy) & ((param1.cParams.searchLength == 3) == (param2.cParams.searchLength == 3));
& (param1.cParams.strategy == param2.cParams.strategy)
& ((param1.cParams.searchLength==3) == (param2.cParams.searchLength==3));
} }
/*! ZSTD_continueCCtx() : /*! ZSTD_continueCCtx() :
@@ -222,7 +229,11 @@ static size_t ZSTD_continueCCtx(ZSTD_CCtx* cctx, ZSTD_parameters params, U64 fra
cctx->stage = ZSTDcs_init; cctx->stage = ZSTDcs_init;
cctx->dictID = 0; cctx->dictID = 0;
cctx->loadedDictEnd = 0; cctx->loadedDictEnd = 0;
{ int i; for (i=0; i<ZSTD_REP_NUM; i++) cctx->rep[i] = repStartValue[i]; } {
int i;
for (i = 0; i < ZSTD_REP_NUM; i++)
cctx->rep[i] = repStartValue[i];
}
cctx->seqStore.litLengthSum = 0; /* force reset of btopt stats */ cctx->seqStore.litLengthSum = 0; /* force reset of btopt stats */
xxh64_reset(&cctx->xxhState, 0); xxh64_reset(&cctx->xxhState, 0);
return 0; return 0;
@@ -232,9 +243,7 @@ typedef enum { ZSTDcrp_continue, ZSTDcrp_noMemset, ZSTDcrp_fullReset } ZSTD_comp
/*! ZSTD_resetCCtx_advanced() : /*! ZSTD_resetCCtx_advanced() :
note : `params` must be validated */ note : `params` must be validated */
static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc, static size_t ZSTD_resetCCtx_advanced(ZSTD_CCtx *zc, ZSTD_parameters params, U64 frameContentSize, ZSTD_compResetPolicy_e const crp)
ZSTD_parameters params, U64 frameContentSize,
ZSTD_compResetPolicy_e const crp)
{ {
if (crp == ZSTDcrp_continue) if (crp == ZSTDcrp_continue)
if (ZSTD_equivalentParams(params, zc->params)) { if (ZSTD_equivalentParams(params, zc->params)) {
@@ -243,7 +252,8 @@ static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc,
return ZSTD_continueCCtx(zc, params, frameContentSize); return ZSTD_continueCCtx(zc, params, frameContentSize);
} }
{ size_t const blockSize = MIN(ZSTD_BLOCKSIZE_ABSOLUTEMAX, (size_t)1 << params.cParams.windowLog); {
size_t const blockSize = MIN(ZSTD_BLOCKSIZE_ABSOLUTEMAX, (size_t)1 << params.cParams.windowLog);
U32 const divider = (params.cParams.searchLength == 3) ? 3 : 4; U32 const divider = (params.cParams.searchLength == 3) ? 3 : 4;
size_t const maxNbSeq = blockSize / divider; size_t const maxNbSeq = blockSize / divider;
size_t const tokenSpace = blockSize + 11 * maxNbSeq; size_t const tokenSpace = blockSize + 11 * maxNbSeq;
@@ -255,18 +265,22 @@ static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc,
void *ptr; void *ptr;
/* Check if workSpace is large enough, alloc a new one if needed */ /* Check if workSpace is large enough, alloc a new one if needed */
{ size_t const optSpace = ((MaxML+1) + (MaxLL+1) + (MaxOff+1) + (1<<Litbits))*sizeof(U32) {
+ (ZSTD_OPT_NUM+1)*(sizeof(ZSTD_match_t) + sizeof(ZSTD_optimal_t)); size_t const optSpace = ((MaxML + 1) + (MaxLL + 1) + (MaxOff + 1) + (1 << Litbits)) * sizeof(U32) +
size_t const neededSpace = tableSpace + (256*sizeof(U32)) /* huffTable */ + tokenSpace (ZSTD_OPT_NUM + 1) * (sizeof(ZSTD_match_t) + sizeof(ZSTD_optimal_t));
+ (((params.cParams.strategy == ZSTD_btopt) || (params.cParams.strategy == ZSTD_btopt2)) ? optSpace : 0); size_t const neededSpace = tableSpace + (256 * sizeof(U32)) /* huffTable */ + tokenSpace +
(((params.cParams.strategy == ZSTD_btopt) || (params.cParams.strategy == ZSTD_btopt2)) ? optSpace : 0);
if (zc->workSpaceSize < neededSpace) { if (zc->workSpaceSize < neededSpace) {
ZSTD_free(zc->workSpace, zc->customMem); ZSTD_free(zc->workSpace, zc->customMem);
zc->workSpace = ZSTD_malloc(neededSpace, zc->customMem); zc->workSpace = ZSTD_malloc(neededSpace, zc->customMem);
if (zc->workSpace == NULL) return ERROR(memory_allocation); if (zc->workSpace == NULL)
return ERROR(memory_allocation);
zc->workSpaceSize = neededSpace; zc->workSpaceSize = neededSpace;
} } }
}
if (crp!=ZSTDcrp_noMemset) memset(zc->workSpace, 0, tableSpace); /* reset tables only */ if (crp != ZSTDcrp_noMemset)
memset(zc->workSpace, 0, tableSpace); /* reset tables only */
xxh64_reset(&zc->xxhState, 0); xxh64_reset(&zc->xxhState, 0);
zc->hashLog3 = hashLog3; zc->hashLog3 = hashLog3;
zc->hashTable = (U32 *)(zc->workSpace); zc->hashTable = (U32 *)(zc->workSpace);
@@ -287,7 +301,11 @@ static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc,
zc->params = params; zc->params = params;
zc->blockSize = blockSize; zc->blockSize = blockSize;
zc->frameContentSize = frameContentSize; zc->frameContentSize = frameContentSize;
{ int i; for (i=0; i<ZSTD_REP_NUM; i++) zc->rep[i] = repStartValue[i]; } {
int i;
for (i = 0; i < ZSTD_REP_NUM; i++)
zc->rep[i] = repStartValue[i];
}
if ((params.cParams.strategy == ZSTD_btopt) || (params.cParams.strategy == ZSTD_btopt2)) { if ((params.cParams.strategy == ZSTD_btopt) || (params.cParams.strategy == ZSTD_btopt2)) {
zc->seqStore.litFreq = (U32 *)ptr; zc->seqStore.litFreq = (U32 *)ptr;
@@ -320,9 +338,11 @@ static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc,
* ensures next compression will not use repcodes from previous block. * ensures next compression will not use repcodes from previous block.
* Note : only works with regular variant; * Note : only works with regular variant;
* do not use with extDict variant ! */ * do not use with extDict variant ! */
void ZSTD_invalidateRepCodes(ZSTD_CCtx* cctx) { void ZSTD_invalidateRepCodes(ZSTD_CCtx *cctx)
{
int i; int i;
for (i=0; i<ZSTD_REP_NUM; i++) cctx->rep[i] = 0; for (i = 0; i < ZSTD_REP_NUM; i++)
cctx->rep[i] = 0;
} }
/*! ZSTD_copyCCtx() : /*! ZSTD_copyCCtx() :
@@ -331,17 +351,19 @@ void ZSTD_invalidateRepCodes(ZSTD_CCtx* cctx) {
* @return : 0, or an error code */ * @return : 0, or an error code */
size_t ZSTD_copyCCtx(ZSTD_CCtx *dstCCtx, const ZSTD_CCtx *srcCCtx, unsigned long long pledgedSrcSize) size_t ZSTD_copyCCtx(ZSTD_CCtx *dstCCtx, const ZSTD_CCtx *srcCCtx, unsigned long long pledgedSrcSize)
{ {
if (srcCCtx->stage!=ZSTDcs_init) return ERROR(stage_wrong); if (srcCCtx->stage != ZSTDcs_init)
return ERROR(stage_wrong);
memcpy(&dstCCtx->customMem, &srcCCtx->customMem, sizeof(ZSTD_customMem)); memcpy(&dstCCtx->customMem, &srcCCtx->customMem, sizeof(ZSTD_customMem));
{ ZSTD_parameters params = srcCCtx->params; {
ZSTD_parameters params = srcCCtx->params;
params.fParams.contentSizeFlag = (pledgedSrcSize > 0); params.fParams.contentSizeFlag = (pledgedSrcSize > 0);
ZSTD_resetCCtx_advanced(dstCCtx, params, pledgedSrcSize, ZSTDcrp_noMemset); ZSTD_resetCCtx_advanced(dstCCtx, params, pledgedSrcSize, ZSTDcrp_noMemset);
} }
/* copy tables */ /* copy tables */
{ size_t const chainSize = (srcCCtx->params.cParams.strategy == ZSTD_fast) ? 0 : (1 << srcCCtx->params.cParams.chainLog); {
size_t const chainSize = (srcCCtx->params.cParams.strategy == ZSTD_fast) ? 0 : (1 << srcCCtx->params.cParams.chainLog);
size_t const hSize = ((size_t)1) << srcCCtx->params.cParams.hashLog; size_t const hSize = ((size_t)1) << srcCCtx->params.cParams.hashLog;
size_t const h3Size = (size_t)1 << srcCCtx->hashLog3; size_t const h3Size = (size_t)1 << srcCCtx->hashLog3;
size_t const tableSpace = (chainSize + hSize + h3Size) * sizeof(U32); size_t const tableSpace = (chainSize + hSize + h3Size) * sizeof(U32);
@@ -374,15 +396,16 @@ size_t ZSTD_copyCCtx(ZSTD_CCtx* dstCCtx, const ZSTD_CCtx* srcCCtx, unsigned long
return 0; return 0;
} }
/*! ZSTD_reduceTable() : /*! ZSTD_reduceTable() :
* reduce table indexes by `reducerValue` */ * reduce table indexes by `reducerValue` */
static void ZSTD_reduceTable(U32 *const table, U32 const size, U32 const reducerValue) static void ZSTD_reduceTable(U32 *const table, U32 const size, U32 const reducerValue)
{ {
U32 u; U32 u;
for (u = 0; u < size; u++) { for (u = 0; u < size; u++) {
if (table[u] < reducerValue) table[u] = 0; if (table[u] < reducerValue)
else table[u] -= reducerValue; table[u] = 0;
else
table[u] -= reducerValue;
} }
} }
@@ -390,16 +413,21 @@ static void ZSTD_reduceTable (U32* const table, U32 const size, U32 const reduce
* rescale all indexes to avoid future overflow (indexes are U32) */ * rescale all indexes to avoid future overflow (indexes are U32) */
static void ZSTD_reduceIndex(ZSTD_CCtx *zc, const U32 reducerValue) static void ZSTD_reduceIndex(ZSTD_CCtx *zc, const U32 reducerValue)
{ {
{ U32 const hSize = 1 << zc->params.cParams.hashLog; {
ZSTD_reduceTable(zc->hashTable, hSize, reducerValue); } U32 const hSize = 1 << zc->params.cParams.hashLog;
ZSTD_reduceTable(zc->hashTable, hSize, reducerValue);
{ U32 const chainSize = (zc->params.cParams.strategy == ZSTD_fast) ? 0 : (1 << zc->params.cParams.chainLog);
ZSTD_reduceTable(zc->chainTable, chainSize, reducerValue); }
{ U32 const h3Size = (zc->hashLog3) ? 1 << zc->hashLog3 : 0;
ZSTD_reduceTable(zc->hashTable3, h3Size, reducerValue); }
} }
{
U32 const chainSize = (zc->params.cParams.strategy == ZSTD_fast) ? 0 : (1 << zc->params.cParams.chainLog);
ZSTD_reduceTable(zc->chainTable, chainSize, reducerValue);
}
{
U32 const h3Size = (zc->hashLog3) ? 1 << zc->hashLog3 : 0;
ZSTD_reduceTable(zc->hashTable3, h3Size, reducerValue);
}
}
/*-******************************************************* /*-*******************************************************
* Block entropic compression * Block entropic compression
@@ -409,32 +437,26 @@ static void ZSTD_reduceIndex (ZSTD_CCtx* zc, const U32 reducerValue)
size_t ZSTD_noCompressBlock(void *dst, size_t dstCapacity, const void *src, size_t srcSize) size_t ZSTD_noCompressBlock(void *dst, size_t dstCapacity, const void *src, size_t srcSize)
{ {
if (srcSize + ZSTD_blockHeaderSize > dstCapacity) return ERROR(dstSize_tooSmall); if (srcSize + ZSTD_blockHeaderSize > dstCapacity)
return ERROR(dstSize_tooSmall);
memcpy((BYTE *)dst + ZSTD_blockHeaderSize, src, srcSize); memcpy((BYTE *)dst + ZSTD_blockHeaderSize, src, srcSize);
ZSTD_writeLE24(dst, (U32)(srcSize << 2) + (U32)bt_raw); ZSTD_writeLE24(dst, (U32)(srcSize << 2) + (U32)bt_raw);
return ZSTD_blockHeaderSize + srcSize; return ZSTD_blockHeaderSize + srcSize;
} }
static size_t ZSTD_noCompressLiterals(void *dst, size_t dstCapacity, const void *src, size_t srcSize) static size_t ZSTD_noCompressLiterals(void *dst, size_t dstCapacity, const void *src, size_t srcSize)
{ {
BYTE *const ostart = (BYTE * const)dst; BYTE *const ostart = (BYTE * const)dst;
U32 const flSize = 1 + (srcSize > 31) + (srcSize > 4095); U32 const flSize = 1 + (srcSize > 31) + (srcSize > 4095);
if (srcSize + flSize > dstCapacity) return ERROR(dstSize_tooSmall); if (srcSize + flSize > dstCapacity)
return ERROR(dstSize_tooSmall);
switch(flSize) switch (flSize) {
{ case 1: /* 2 - 1 - 5 */ ostart[0] = (BYTE)((U32)set_basic + (srcSize << 3)); break;
case 1: /* 2 - 1 - 5 */ case 2: /* 2 - 2 - 12 */ ZSTD_writeLE16(ostart, (U16)((U32)set_basic + (1 << 2) + (srcSize << 4))); break;
ostart[0] = (BYTE)((U32)set_basic + (srcSize<<3));
break;
case 2: /* 2 - 2 - 12 */
ZSTD_writeLE16(ostart, (U16)((U32)set_basic + (1<<2) + (srcSize<<4)));
break;
default: /*note : should not be necessary : flSize is within {1,2,3} */ default: /*note : should not be necessary : flSize is within {1,2,3} */
case 3: /* 2 - 2 - 20 */ case 3: /* 2 - 2 - 20 */ ZSTD_writeLE32(ostart, (U32)((U32)set_basic + (3 << 2) + (srcSize << 4))); break;
ZSTD_writeLE32(ostart, (U32)((U32)set_basic + (3<<2) + (srcSize<<4)));
break;
} }
memcpy(ostart + flSize, src, srcSize); memcpy(ostart + flSize, src, srcSize);
@@ -448,30 +470,20 @@ static size_t ZSTD_compressRleLiteralsBlock (void* dst, size_t dstCapacity, cons
(void)dstCapacity; /* dstCapacity already guaranteed to be >=4, hence large enough */ (void)dstCapacity; /* dstCapacity already guaranteed to be >=4, hence large enough */
switch(flSize) switch (flSize) {
{ case 1: /* 2 - 1 - 5 */ ostart[0] = (BYTE)((U32)set_rle + (srcSize << 3)); break;
case 1: /* 2 - 1 - 5 */ case 2: /* 2 - 2 - 12 */ ZSTD_writeLE16(ostart, (U16)((U32)set_rle + (1 << 2) + (srcSize << 4))); break;
ostart[0] = (BYTE)((U32)set_rle + (srcSize<<3));
break;
case 2: /* 2 - 2 - 12 */
ZSTD_writeLE16(ostart, (U16)((U32)set_rle + (1<<2) + (srcSize<<4)));
break;
default: /*note : should not be necessary : flSize is necessarily within {1,2,3} */ default: /*note : should not be necessary : flSize is necessarily within {1,2,3} */
case 3: /* 2 - 2 - 20 */ case 3: /* 2 - 2 - 20 */ ZSTD_writeLE32(ostart, (U32)((U32)set_rle + (3 << 2) + (srcSize << 4))); break;
ZSTD_writeLE32(ostart, (U32)((U32)set_rle + (3<<2) + (srcSize<<4)));
break;
} }
ostart[flSize] = *(const BYTE *)src; ostart[flSize] = *(const BYTE *)src;
return flSize + 1; return flSize + 1;
} }
static size_t ZSTD_minGain(size_t srcSize) { return (srcSize >> 6) + 2; } static size_t ZSTD_minGain(size_t srcSize) { return (srcSize >> 6) + 2; }
static size_t ZSTD_compressLiterals (ZSTD_CCtx* zc, static size_t ZSTD_compressLiterals(ZSTD_CCtx *zc, void *dst, size_t dstCapacity, const void *src, size_t srcSize)
void* dst, size_t dstCapacity,
const void* src, size_t srcSize)
{ {
size_t const minGain = ZSTD_minGain(srcSize); size_t const minGain = ZSTD_minGain(srcSize);
size_t const lhSize = 3 + (srcSize >= 1 KB) + (srcSize >= 16 KB); size_t const lhSize = 3 + (srcSize >= 1 KB) + (srcSize >= 16 KB);
@@ -480,21 +492,31 @@ static size_t ZSTD_compressLiterals (ZSTD_CCtx* zc,
symbolEncodingType_e hType = set_compressed; symbolEncodingType_e hType = set_compressed;
size_t cLitSize; size_t cLitSize;
/* small ? don't even attempt compression (speed opt) */ /* small ? don't even attempt compression (speed opt) */
#define LITERAL_NOENTROPY 63 #define LITERAL_NOENTROPY 63
{ size_t const minLitSize = zc->flagStaticHufTable == HUF_repeat_valid ? 6 : LITERAL_NOENTROPY; {
if (srcSize <= minLitSize) return ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize); size_t const minLitSize = zc->flagStaticHufTable == HUF_repeat_valid ? 6 : LITERAL_NOENTROPY;
if (srcSize <= minLitSize)
return ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize);
} }
if (dstCapacity < lhSize+1) return ERROR(dstSize_tooSmall); /* not enough space for compression */ if (dstCapacity < lhSize + 1)
{ HUF_repeat repeat = zc->flagStaticHufTable; return ERROR(dstSize_tooSmall); /* not enough space for compression */
{
HUF_repeat repeat = zc->flagStaticHufTable;
int const preferRepeat = zc->params.cParams.strategy < ZSTD_lazy ? srcSize <= 1024 : 0; int const preferRepeat = zc->params.cParams.strategy < ZSTD_lazy ? srcSize <= 1024 : 0;
if (repeat == HUF_repeat_valid && lhSize == 3) singleStream = 1; if (repeat == HUF_repeat_valid && lhSize == 3)
cLitSize = singleStream ? HUF_compress1X_repeat(ostart+lhSize, dstCapacity-lhSize, src, srcSize, 255, 11, zc->tmpCounters, sizeof(zc->tmpCounters), zc->hufTable, &repeat, preferRepeat) singleStream = 1;
: HUF_compress4X_repeat(ostart+lhSize, dstCapacity-lhSize, src, srcSize, 255, 11, zc->tmpCounters, sizeof(zc->tmpCounters), zc->hufTable, &repeat, preferRepeat); cLitSize = singleStream ? HUF_compress1X_repeat(ostart + lhSize, dstCapacity - lhSize, src, srcSize, 255, 11, zc->tmpCounters,
if (repeat != HUF_repeat_none) { hType = set_repeat; } /* reused the existing table */ sizeof(zc->tmpCounters), zc->hufTable, &repeat, preferRepeat)
else { zc->flagStaticHufTable = HUF_repeat_check; } /* now have a table to reuse */ : HUF_compress4X_repeat(ostart + lhSize, dstCapacity - lhSize, src, srcSize, 255, 11, zc->tmpCounters,
sizeof(zc->tmpCounters), zc->hufTable, &repeat, preferRepeat);
if (repeat != HUF_repeat_none) {
hType = set_repeat;
} /* reused the existing table */
else {
zc->flagStaticHufTable = HUF_repeat_check;
} /* now have a table to reuse */
} }
if ((cLitSize == 0) | (cLitSize >= srcSize - minGain)) { if ((cLitSize == 0) | (cLitSize >= srcSize - minGain)) {
@@ -507,21 +529,23 @@ static size_t ZSTD_compressLiterals (ZSTD_CCtx* zc,
} }
/* Build header */ /* Build header */
switch(lhSize) switch (lhSize) {
{
case 3: /* 2 - 2 - 10 - 10 */ case 3: /* 2 - 2 - 10 - 10 */
{ U32 const lhc = hType + ((!singleStream) << 2) + ((U32)srcSize<<4) + ((U32)cLitSize<<14); {
U32 const lhc = hType + ((!singleStream) << 2) + ((U32)srcSize << 4) + ((U32)cLitSize << 14);
ZSTD_writeLE24(ostart, lhc); ZSTD_writeLE24(ostart, lhc);
break; break;
} }
case 4: /* 2 - 2 - 14 - 14 */ case 4: /* 2 - 2 - 14 - 14 */
{ U32 const lhc = hType + (2 << 2) + ((U32)srcSize<<4) + ((U32)cLitSize<<18); {
U32 const lhc = hType + (2 << 2) + ((U32)srcSize << 4) + ((U32)cLitSize << 18);
ZSTD_writeLE32(ostart, lhc); ZSTD_writeLE32(ostart, lhc);
break; break;
} }
default: /* should not be necessary, lhSize is only {3,4,5} */ default: /* should not be necessary, lhSize is only {3,4,5} */
case 5: /* 2 - 2 - 18 - 18 */ case 5: /* 2 - 2 - 18 - 18 */
{ U32 const lhc = hType + (3 << 2) + ((U32)srcSize<<4) + ((U32)cLitSize<<22); {
U32 const lhc = hType + (3 << 2) + ((U32)srcSize << 4) + ((U32)cLitSize << 22);
ZSTD_writeLE32(ostart, lhc); ZSTD_writeLE32(ostart, lhc);
ostart[4] = (BYTE)(cLitSize >> 10); ostart[4] = (BYTE)(cLitSize >> 10);
break; break;
@@ -530,24 +554,15 @@ static size_t ZSTD_compressLiterals (ZSTD_CCtx* zc,
return lhSize + cLitSize; return lhSize + cLitSize;
} }
static const BYTE LL_Code[64] = { 0, 1, 2, 3, 4, 5, 6, 7, static const BYTE LL_Code[64] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 16, 17, 17, 18, 18,
8, 9, 10, 11, 12, 13, 14, 15, 19, 19, 20, 20, 20, 20, 21, 21, 21, 21, 22, 22, 22, 22, 22, 22, 22, 22, 23, 23, 23, 23,
16, 16, 17, 17, 18, 18, 19, 19, 23, 23, 23, 23, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24};
20, 20, 20, 20, 21, 21, 21, 21,
22, 22, 22, 22, 22, 22, 22, 22,
23, 23, 23, 23, 23, 23, 23, 23,
24, 24, 24, 24, 24, 24, 24, 24,
24, 24, 24, 24, 24, 24, 24, 24 };
static const BYTE ML_Code[128] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31,
32, 32, 33, 33, 34, 34, 35, 35, 36, 36, 36, 36, 37, 37, 37, 37,
38, 38, 38, 38, 38, 38, 38, 38, 39, 39, 39, 39, 39, 39, 39, 39,
40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40,
41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41,
42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42,
42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42 };
static const BYTE ML_Code[128] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25,
26, 27, 28, 29, 30, 31, 32, 32, 33, 33, 34, 34, 35, 35, 36, 36, 36, 36, 37, 37, 37, 37, 38, 38, 38, 38,
38, 38, 38, 38, 39, 39, 39, 39, 39, 39, 39, 39, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40,
40, 40, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 42, 42, 42, 42, 42, 42, 42, 42,
42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42};
void ZSTD_seqToCodes(const seqStore_t *seqStorePtr) void ZSTD_seqToCodes(const seqStore_t *seqStorePtr)
{ {
@@ -572,9 +587,7 @@ void ZSTD_seqToCodes(const seqStore_t* seqStorePtr)
mlCodeTable[seqStorePtr->longLengthPos] = MaxML; mlCodeTable[seqStorePtr->longLengthPos] = MaxML;
} }
ZSTD_STATIC size_t ZSTD_compressSequences (ZSTD_CCtx* zc, ZSTD_STATIC size_t ZSTD_compressSequences(ZSTD_CCtx *zc, void *dst, size_t dstCapacity, size_t srcSize)
void* dst, size_t dstCapacity,
size_t srcSize)
{ {
const int longOffsets = zc->params.cParams.windowLog > STREAM_ACCUMULATOR_MIN; const int longOffsets = zc->params.cParams.windowLog > STREAM_ACCUMULATOR_MIN;
const seqStore_t *seqStorePtr = &(zc->seqStore); const seqStore_t *seqStorePtr = &(zc->seqStore);
@@ -596,19 +609,26 @@ ZSTD_STATIC size_t ZSTD_compressSequences (ZSTD_CCtx* zc,
BYTE scratchBuffer[1 << MAX(MLFSELog, LLFSELog)]; BYTE scratchBuffer[1 << MAX(MLFSELog, LLFSELog)];
/* Compress literals */ /* Compress literals */
{ const BYTE* const literals = seqStorePtr->litStart; {
const BYTE *const literals = seqStorePtr->litStart;
size_t const litSize = seqStorePtr->lit - literals; size_t const litSize = seqStorePtr->lit - literals;
size_t const cSize = ZSTD_compressLiterals(zc, op, dstCapacity, literals, litSize); size_t const cSize = ZSTD_compressLiterals(zc, op, dstCapacity, literals, litSize);
if (ZSTD_isError(cSize)) return cSize; if (ZSTD_isError(cSize))
return cSize;
op += cSize; op += cSize;
} }
/* Sequences Header */ /* Sequences Header */
if ((oend-op) < 3 /*max nbSeq Size*/ + 1 /*seqHead */) return ERROR(dstSize_tooSmall); if ((oend - op) < 3 /*max nbSeq Size*/ + 1 /*seqHead */)
if (nbSeq < 0x7F) *op++ = (BYTE)nbSeq; return ERROR(dstSize_tooSmall);
else if (nbSeq < LONGNBSEQ) op[0] = (BYTE)((nbSeq>>8) + 0x80), op[1] = (BYTE)nbSeq, op+=2; if (nbSeq < 0x7F)
else op[0]=0xFF, ZSTD_writeLE16(op+1, (U16)(nbSeq - LONGNBSEQ)), op+=3; *op++ = (BYTE)nbSeq;
if (nbSeq==0) goto _check_compressibility; else if (nbSeq < LONGNBSEQ)
op[0] = (BYTE)((nbSeq >> 8) + 0x80), op[1] = (BYTE)nbSeq, op += 2;
else
op[0] = 0xFF, ZSTD_writeLE16(op + 1, (U16)(nbSeq - LONGNBSEQ)), op += 3;
if (nbSeq == 0)
goto _check_compressibility;
/* seqHead : flags for FSE encoding type */ /* seqHead : flags for FSE encoding type */
seqHead = op++; seqHead = op++;
@@ -620,7 +640,8 @@ ZSTD_STATIC size_t ZSTD_compressSequences (ZSTD_CCtx* zc,
ZSTD_seqToCodes(seqStorePtr); ZSTD_seqToCodes(seqStorePtr);
/* CTable for Literal Lengths */ /* CTable for Literal Lengths */
{ U32 max = MaxLL; {
U32 max = MaxLL;
size_t const mostFrequent = FSE_countFast_wksp(count, &max, llCodeTable, nbSeq, zc->tmpCounters); size_t const mostFrequent = FSE_countFast_wksp(count, &max, llCodeTable, nbSeq, zc->tmpCounters);
if ((mostFrequent == nbSeq) && (nbSeq > 2)) { if ((mostFrequent == nbSeq) && (nbSeq > 2)) {
*op++ = llCodeTable[0]; *op++ = llCodeTable[0];
@@ -634,17 +655,25 @@ ZSTD_STATIC size_t ZSTD_compressSequences (ZSTD_CCtx* zc,
} else { } else {
size_t nbSeq_1 = nbSeq; size_t nbSeq_1 = nbSeq;
const U32 tableLog = FSE_optimalTableLog(LLFSELog, nbSeq, max); const U32 tableLog = FSE_optimalTableLog(LLFSELog, nbSeq, max);
if (count[llCodeTable[nbSeq-1]]>1) { count[llCodeTable[nbSeq-1]]--; nbSeq_1--; } if (count[llCodeTable[nbSeq - 1]] > 1) {
count[llCodeTable[nbSeq - 1]]--;
nbSeq_1--;
}
FSE_normalizeCount(norm, tableLog, count, nbSeq_1, max); FSE_normalizeCount(norm, tableLog, count, nbSeq_1, max);
{ size_t const NCountSize = FSE_writeNCount(op, oend-op, norm, max, tableLog); /* overflow protected */ {
if (FSE_isError(NCountSize)) return NCountSize; size_t const NCountSize = FSE_writeNCount(op, oend - op, norm, max, tableLog); /* overflow protected */
op += NCountSize; } if (FSE_isError(NCountSize))
return NCountSize;
op += NCountSize;
}
FSE_buildCTable_wksp(CTable_LitLength, norm, max, tableLog, scratchBuffer, sizeof(scratchBuffer)); FSE_buildCTable_wksp(CTable_LitLength, norm, max, tableLog, scratchBuffer, sizeof(scratchBuffer));
LLtype = set_compressed; LLtype = set_compressed;
} } }
}
/* CTable for Offsets */ /* CTable for Offsets */
{ U32 max = MaxOff; {
U32 max = MaxOff;
size_t const mostFrequent = FSE_countFast_wksp(count, &max, ofCodeTable, nbSeq, zc->tmpCounters); size_t const mostFrequent = FSE_countFast_wksp(count, &max, ofCodeTable, nbSeq, zc->tmpCounters);
if ((mostFrequent == nbSeq) && (nbSeq > 2)) { if ((mostFrequent == nbSeq) && (nbSeq > 2)) {
*op++ = ofCodeTable[0]; *op++ = ofCodeTable[0];
@@ -658,17 +687,25 @@ ZSTD_STATIC size_t ZSTD_compressSequences (ZSTD_CCtx* zc,
} else { } else {
size_t nbSeq_1 = nbSeq; size_t nbSeq_1 = nbSeq;
const U32 tableLog = FSE_optimalTableLog(OffFSELog, nbSeq, max); const U32 tableLog = FSE_optimalTableLog(OffFSELog, nbSeq, max);
if (count[ofCodeTable[nbSeq-1]]>1) { count[ofCodeTable[nbSeq-1]]--; nbSeq_1--; } if (count[ofCodeTable[nbSeq - 1]] > 1) {
count[ofCodeTable[nbSeq - 1]]--;
nbSeq_1--;
}
FSE_normalizeCount(norm, tableLog, count, nbSeq_1, max); FSE_normalizeCount(norm, tableLog, count, nbSeq_1, max);
{ size_t const NCountSize = FSE_writeNCount(op, oend-op, norm, max, tableLog); /* overflow protected */ {
if (FSE_isError(NCountSize)) return NCountSize; size_t const NCountSize = FSE_writeNCount(op, oend - op, norm, max, tableLog); /* overflow protected */
op += NCountSize; } if (FSE_isError(NCountSize))
return NCountSize;
op += NCountSize;
}
FSE_buildCTable_wksp(CTable_OffsetBits, norm, max, tableLog, scratchBuffer, sizeof(scratchBuffer)); FSE_buildCTable_wksp(CTable_OffsetBits, norm, max, tableLog, scratchBuffer, sizeof(scratchBuffer));
Offtype = set_compressed; Offtype = set_compressed;
} } }
}
/* CTable for MatchLengths */ /* CTable for MatchLengths */
{ U32 max = MaxML; {
U32 max = MaxML;
size_t const mostFrequent = FSE_countFast_wksp(count, &max, mlCodeTable, nbSeq, zc->tmpCounters); size_t const mostFrequent = FSE_countFast_wksp(count, &max, mlCodeTable, nbSeq, zc->tmpCounters);
if ((mostFrequent == nbSeq) && (nbSeq > 2)) { if ((mostFrequent == nbSeq) && (nbSeq > 2)) {
*op++ = *mlCodeTable; *op++ = *mlCodeTable;
@@ -682,20 +719,28 @@ ZSTD_STATIC size_t ZSTD_compressSequences (ZSTD_CCtx* zc,
} else { } else {
size_t nbSeq_1 = nbSeq; size_t nbSeq_1 = nbSeq;
const U32 tableLog = FSE_optimalTableLog(MLFSELog, nbSeq, max); const U32 tableLog = FSE_optimalTableLog(MLFSELog, nbSeq, max);
if (count[mlCodeTable[nbSeq-1]]>1) { count[mlCodeTable[nbSeq-1]]--; nbSeq_1--; } if (count[mlCodeTable[nbSeq - 1]] > 1) {
count[mlCodeTable[nbSeq - 1]]--;
nbSeq_1--;
}
FSE_normalizeCount(norm, tableLog, count, nbSeq_1, max); FSE_normalizeCount(norm, tableLog, count, nbSeq_1, max);
{ size_t const NCountSize = FSE_writeNCount(op, oend-op, norm, max, tableLog); /* overflow protected */ {
if (FSE_isError(NCountSize)) return NCountSize; size_t const NCountSize = FSE_writeNCount(op, oend - op, norm, max, tableLog); /* overflow protected */
op += NCountSize; } if (FSE_isError(NCountSize))
return NCountSize;
op += NCountSize;
}
FSE_buildCTable_wksp(CTable_MatchLength, norm, max, tableLog, scratchBuffer, sizeof(scratchBuffer)); FSE_buildCTable_wksp(CTable_MatchLength, norm, max, tableLog, scratchBuffer, sizeof(scratchBuffer));
MLtype = set_compressed; MLtype = set_compressed;
} } }
}
*seqHead = (BYTE)((LLtype << 6) + (Offtype << 4) + (MLtype << 2)); *seqHead = (BYTE)((LLtype << 6) + (Offtype << 4) + (MLtype << 2));
zc->flagStaticTables = 0; zc->flagStaticTables = 0;
/* Encoding Sequences */ /* Encoding Sequences */
{ BIT_CStream_t blockStream; {
BIT_CStream_t blockStream;
FSE_CState_t stateMatchLength; FSE_CState_t stateMatchLength;
FSE_CState_t stateOffsetBits; FSE_CState_t stateOffsetBits;
FSE_CState_t stateLitLength; FSE_CState_t stateLitLength;
@@ -707,9 +752,11 @@ ZSTD_STATIC size_t ZSTD_compressSequences (ZSTD_CCtx* zc,
FSE_initCState2(&stateOffsetBits, CTable_OffsetBits, ofCodeTable[nbSeq - 1]); FSE_initCState2(&stateOffsetBits, CTable_OffsetBits, ofCodeTable[nbSeq - 1]);
FSE_initCState2(&stateLitLength, CTable_LitLength, llCodeTable[nbSeq - 1]); FSE_initCState2(&stateLitLength, CTable_LitLength, llCodeTable[nbSeq - 1]);
BIT_addBits(&blockStream, sequences[nbSeq - 1].litLength, LL_bits[llCodeTable[nbSeq - 1]]); BIT_addBits(&blockStream, sequences[nbSeq - 1].litLength, LL_bits[llCodeTable[nbSeq - 1]]);
if (ZSTD_32bits()) BIT_flushBits(&blockStream); if (ZSTD_32bits())
BIT_flushBits(&blockStream);
BIT_addBits(&blockStream, sequences[nbSeq - 1].matchLength, ML_bits[mlCodeTable[nbSeq - 1]]); BIT_addBits(&blockStream, sequences[nbSeq - 1].matchLength, ML_bits[mlCodeTable[nbSeq - 1]]);
if (ZSTD_32bits()) BIT_flushBits(&blockStream); if (ZSTD_32bits())
BIT_flushBits(&blockStream);
if (longOffsets) { if (longOffsets) {
U32 const ofBits = ofCodeTable[nbSeq - 1]; U32 const ofBits = ofCodeTable[nbSeq - 1];
int const extraBits = ofBits - MIN(ofBits, STREAM_ACCUMULATOR_MIN - 1); int const extraBits = ofBits - MIN(ofBits, STREAM_ACCUMULATOR_MIN - 1);
@@ -717,14 +764,14 @@ ZSTD_STATIC size_t ZSTD_compressSequences (ZSTD_CCtx* zc,
BIT_addBits(&blockStream, sequences[nbSeq - 1].offset, extraBits); BIT_addBits(&blockStream, sequences[nbSeq - 1].offset, extraBits);
BIT_flushBits(&blockStream); BIT_flushBits(&blockStream);
} }
BIT_addBits(&blockStream, sequences[nbSeq-1].offset >> extraBits, BIT_addBits(&blockStream, sequences[nbSeq - 1].offset >> extraBits, ofBits - extraBits);
ofBits - extraBits);
} else { } else {
BIT_addBits(&blockStream, sequences[nbSeq - 1].offset, ofCodeTable[nbSeq - 1]); BIT_addBits(&blockStream, sequences[nbSeq - 1].offset, ofCodeTable[nbSeq - 1]);
} }
BIT_flushBits(&blockStream); BIT_flushBits(&blockStream);
{ size_t n; {
size_t n;
for (n = nbSeq - 2; n < nbSeq; n--) { /* intentional underflow */ for (n = nbSeq - 2; n < nbSeq; n--) { /* intentional underflow */
BYTE const llCode = llCodeTable[n]; BYTE const llCode = llCodeTable[n];
BYTE const ofCode = ofCodeTable[n]; BYTE const ofCode = ofCodeTable[n];
@@ -735,48 +782,59 @@ ZSTD_STATIC size_t ZSTD_compressSequences (ZSTD_CCtx* zc,
/* (7)*/ /* (7)*/ /* (7)*/ /* (7)*/
FSE_encodeSymbol(&blockStream, &stateOffsetBits, ofCode); /* 15 */ /* 15 */ FSE_encodeSymbol(&blockStream, &stateOffsetBits, ofCode); /* 15 */ /* 15 */
FSE_encodeSymbol(&blockStream, &stateMatchLength, mlCode); /* 24 */ /* 24 */ FSE_encodeSymbol(&blockStream, &stateMatchLength, mlCode); /* 24 */ /* 24 */
if (ZSTD_32bits()) BIT_flushBits(&blockStream); /* (7)*/ if (ZSTD_32bits())
BIT_flushBits(&blockStream); /* (7)*/
FSE_encodeSymbol(&blockStream, &stateLitLength, llCode); /* 16 */ /* 33 */ FSE_encodeSymbol(&blockStream, &stateLitLength, llCode); /* 16 */ /* 33 */
if (ZSTD_32bits() || (ofBits + mlBits + llBits >= 64 - 7 - (LLFSELog + MLFSELog + OffFSELog))) if (ZSTD_32bits() || (ofBits + mlBits + llBits >= 64 - 7 - (LLFSELog + MLFSELog + OffFSELog)))
BIT_flushBits(&blockStream); /* (7)*/ BIT_flushBits(&blockStream); /* (7)*/
BIT_addBits(&blockStream, sequences[n].litLength, llBits); BIT_addBits(&blockStream, sequences[n].litLength, llBits);
if (ZSTD_32bits() && ((llBits+mlBits)>24)) BIT_flushBits(&blockStream); if (ZSTD_32bits() && ((llBits + mlBits) > 24))
BIT_flushBits(&blockStream);
BIT_addBits(&blockStream, sequences[n].matchLength, mlBits); BIT_addBits(&blockStream, sequences[n].matchLength, mlBits);
if (ZSTD_32bits()) BIT_flushBits(&blockStream); /* (7)*/ if (ZSTD_32bits())
BIT_flushBits(&blockStream); /* (7)*/
if (longOffsets) { if (longOffsets) {
int const extraBits = ofBits - MIN(ofBits, STREAM_ACCUMULATOR_MIN - 1); int const extraBits = ofBits - MIN(ofBits, STREAM_ACCUMULATOR_MIN - 1);
if (extraBits) { if (extraBits) {
BIT_addBits(&blockStream, sequences[n].offset, extraBits); BIT_addBits(&blockStream, sequences[n].offset, extraBits);
BIT_flushBits(&blockStream); /* (7)*/ BIT_flushBits(&blockStream); /* (7)*/
} }
BIT_addBits(&blockStream, sequences[n].offset >> extraBits, BIT_addBits(&blockStream, sequences[n].offset >> extraBits, ofBits - extraBits); /* 31 */
ofBits - extraBits); /* 31 */
} else { } else {
BIT_addBits(&blockStream, sequences[n].offset, ofBits); /* 31 */ BIT_addBits(&blockStream, sequences[n].offset, ofBits); /* 31 */
} }
BIT_flushBits(&blockStream); /* (7)*/ BIT_flushBits(&blockStream); /* (7)*/
} } }
}
FSE_flushCState(&blockStream, &stateMatchLength); FSE_flushCState(&blockStream, &stateMatchLength);
FSE_flushCState(&blockStream, &stateOffsetBits); FSE_flushCState(&blockStream, &stateOffsetBits);
FSE_flushCState(&blockStream, &stateLitLength); FSE_flushCState(&blockStream, &stateLitLength);
{ size_t const streamSize = BIT_closeCStream(&blockStream); {
if (streamSize==0) return ERROR(dstSize_tooSmall); /* not enough space */ size_t const streamSize = BIT_closeCStream(&blockStream);
if (streamSize == 0)
return ERROR(dstSize_tooSmall); /* not enough space */
op += streamSize; op += streamSize;
} } }
}
/* check compressibility */ /* check compressibility */
_check_compressibility: _check_compressibility : {
{ size_t const minGain = ZSTD_minGain(srcSize); size_t const minGain = ZSTD_minGain(srcSize);
size_t const maxCSize = srcSize - minGain; size_t const maxCSize = srcSize - minGain;
if ((size_t)(op - ostart) >= maxCSize) { if ((size_t)(op - ostart) >= maxCSize) {
zc->flagStaticHufTable = HUF_repeat_none; zc->flagStaticHufTable = HUF_repeat_none;
return 0; return 0;
} } }
}
/* confirm repcodes */ /* confirm repcodes */
{ int i; for (i=0; i<ZSTD_REP_NUM; i++) zc->rep[i] = zc->repToConfirm[i]; } {
int i;
for (i = 0; i < ZSTD_REP_NUM; i++)
zc->rep[i] = zc->repToConfirm[i];
}
return op - ostart; return op - ostart;
} }
@@ -793,20 +851,25 @@ ZSTD_STATIC void ZSTD_storeSeq(seqStore_t* seqStorePtr, size_t litLength, const
seqStorePtr->lit += litLength; seqStorePtr->lit += litLength;
/* literal Length */ /* literal Length */
if (litLength>0xFFFF) { seqStorePtr->longLengthID = 1; seqStorePtr->longLengthPos = (U32)(seqStorePtr->sequences - seqStorePtr->sequencesStart); } if (litLength > 0xFFFF) {
seqStorePtr->longLengthID = 1;
seqStorePtr->longLengthPos = (U32)(seqStorePtr->sequences - seqStorePtr->sequencesStart);
}
seqStorePtr->sequences[0].litLength = (U16)litLength; seqStorePtr->sequences[0].litLength = (U16)litLength;
/* match offset */ /* match offset */
seqStorePtr->sequences[0].offset = offsetCode + 1; seqStorePtr->sequences[0].offset = offsetCode + 1;
/* match Length */ /* match Length */
if (matchCode>0xFFFF) { seqStorePtr->longLengthID = 2; seqStorePtr->longLengthPos = (U32)(seqStorePtr->sequences - seqStorePtr->sequencesStart); } if (matchCode > 0xFFFF) {
seqStorePtr->longLengthID = 2;
seqStorePtr->longLengthPos = (U32)(seqStorePtr->sequences - seqStorePtr->sequencesStart);
}
seqStorePtr->sequences[0].matchLength = (U16)matchCode; seqStorePtr->sequences[0].matchLength = (U16)matchCode;
seqStorePtr->sequences++; seqStorePtr->sequences++;
} }
/*-************************************* /*-*************************************
* Match length counter * Match length counter
***************************************/ ***************************************/
@@ -823,9 +886,9 @@ static unsigned ZSTD_NbCommonBytes (register size_t val)
return (__builtin_clzll(val) >> 3); return (__builtin_clzll(val) >> 3);
} else { /* 32 bits */ } else { /* 32 bits */
return (__builtin_clz((U32)val) >> 3); return (__builtin_clz((U32)val) >> 3);
} }
} }
}
}
static size_t ZSTD_count(const BYTE *pIn, const BYTE *pMatch, const BYTE *const pInLimit) static size_t ZSTD_count(const BYTE *pIn, const BYTE *pMatch, const BYTE *const pInLimit)
{ {
@@ -834,13 +897,25 @@ static size_t ZSTD_count(const BYTE* pIn, const BYTE* pMatch, const BYTE* const
while (pIn < pInLoopLimit) { while (pIn < pInLoopLimit) {
size_t const diff = ZSTD_readST(pMatch) ^ ZSTD_readST(pIn); size_t const diff = ZSTD_readST(pMatch) ^ ZSTD_readST(pIn);
if (!diff) { pIn+=sizeof(size_t); pMatch+=sizeof(size_t); continue; } if (!diff) {
pIn += sizeof(size_t);
pMatch += sizeof(size_t);
continue;
}
pIn += ZSTD_NbCommonBytes(diff); pIn += ZSTD_NbCommonBytes(diff);
return (size_t)(pIn - pStart); return (size_t)(pIn - pStart);
} }
if (ZSTD_64bits()) if ((pIn<(pInLimit-3)) && (ZSTD_read32(pMatch) == ZSTD_read32(pIn))) { pIn+=4; pMatch+=4; } if (ZSTD_64bits())
if ((pIn<(pInLimit-1)) && (ZSTD_read16(pMatch) == ZSTD_read16(pIn))) { pIn+=2; pMatch+=2; } if ((pIn < (pInLimit - 3)) && (ZSTD_read32(pMatch) == ZSTD_read32(pIn))) {
if ((pIn<pInLimit) && (*pMatch == *pIn)) pIn++; pIn += 4;
pMatch += 4;
}
if ((pIn < (pInLimit - 1)) && (ZSTD_read16(pMatch) == ZSTD_read16(pIn))) {
pIn += 2;
pMatch += 2;
}
if ((pIn < pInLimit) && (*pMatch == *pIn))
pIn++;
return (size_t)(pIn - pStart); return (size_t)(pIn - pStart);
} }
@@ -852,11 +927,11 @@ static size_t ZSTD_count_2segments(const BYTE* ip, const BYTE* match, const BYTE
{ {
const BYTE *const vEnd = MIN(ip + (mEnd - match), iEnd); const BYTE *const vEnd = MIN(ip + (mEnd - match), iEnd);
size_t const matchLength = ZSTD_count(ip, match, vEnd); size_t const matchLength = ZSTD_count(ip, match, vEnd);
if (match + matchLength != mEnd) return matchLength; if (match + matchLength != mEnd)
return matchLength;
return matchLength + ZSTD_count(ip + matchLength, iStart, iEnd); return matchLength + ZSTD_count(ip + matchLength, iStart, iEnd);
} }
/*-************************************* /*-*************************************
* Hashes * Hashes
***************************************/ ***************************************/
@@ -886,8 +961,7 @@ static size_t ZSTD_hash8Ptr(const void* p, U32 h) { return ZSTD_hash8(ZSTD_readL
static size_t ZSTD_hashPtr(const void *p, U32 hBits, U32 mls) static size_t ZSTD_hashPtr(const void *p, U32 hBits, U32 mls)
{ {
switch(mls) switch (mls) {
{
// case 3: return ZSTD_hash3Ptr(p, hBits); // case 3: return ZSTD_hash3Ptr(p, hBits);
default: default:
case 4: return ZSTD_hash4Ptr(p, hBits); case 4: return ZSTD_hash4Ptr(p, hBits);
@@ -898,7 +972,6 @@ static size_t ZSTD_hashPtr(const void* p, U32 hBits, U32 mls)
} }
} }
/*-************************************* /*-*************************************
* Fast Scan * Fast Scan
***************************************/ ***************************************/
@@ -917,11 +990,8 @@ static void ZSTD_fillHashTable (ZSTD_CCtx* zc, const void* end, const U32 mls)
} }
} }
FORCE_INLINE FORCE_INLINE
void ZSTD_compressBlock_fast_generic(ZSTD_CCtx* cctx, void ZSTD_compressBlock_fast_generic(ZSTD_CCtx *cctx, const void *src, size_t srcSize, const U32 mls)
const void* src, size_t srcSize,
const U32 mls)
{ {
U32 *const hashTable = cctx->hashTable; U32 *const hashTable = cctx->hashTable;
U32 const hBits = cctx->params.cParams.hashLog; U32 const hBits = cctx->params.cParams.hashLog;
@@ -939,9 +1009,12 @@ void ZSTD_compressBlock_fast_generic(ZSTD_CCtx* cctx,
/* init */ /* init */
ip += (ip == lowest); ip += (ip == lowest);
{ U32 const maxRep = (U32)(ip-lowest); {
if (offset_2 > maxRep) offsetSaved = offset_2, offset_2 = 0; U32 const maxRep = (U32)(ip - lowest);
if (offset_1 > maxRep) offsetSaved = offset_1, offset_1 = 0; if (offset_2 > maxRep)
offsetSaved = offset_2, offset_2 = 0;
if (offset_1 > maxRep)
offsetSaved = offset_1, offset_1 = 0;
} }
/* Main Search Loop */ /* Main Search Loop */
@@ -965,7 +1038,11 @@ void ZSTD_compressBlock_fast_generic(ZSTD_CCtx* cctx,
} }
mLength = ZSTD_count(ip + 4, match + 4, iend) + 4; mLength = ZSTD_count(ip + 4, match + 4, iend) + 4;
offset = (U32)(ip - match); offset = (U32)(ip - match);
while (((ip>anchor) & (match>lowest)) && (ip[-1] == match[-1])) { ip--; match--; mLength++; } /* catch up */ while (((ip > anchor) & (match > lowest)) && (ip[-1] == match[-1])) {
ip--;
match--;
mLength++;
} /* catch up */
offset_2 = offset_1; offset_2 = offset_1;
offset_1 = offset; offset_1 = offset;
@@ -981,53 +1058,48 @@ void ZSTD_compressBlock_fast_generic(ZSTD_CCtx* cctx,
hashTable[ZSTD_hashPtr(base + curr + 2, hBits, mls)] = curr + 2; /* here because curr+2 could be > iend-8 */ hashTable[ZSTD_hashPtr(base + curr + 2, hBits, mls)] = curr + 2; /* here because curr+2 could be > iend-8 */
hashTable[ZSTD_hashPtr(ip - 2, hBits, mls)] = (U32)(ip - 2 - base); hashTable[ZSTD_hashPtr(ip - 2, hBits, mls)] = (U32)(ip - 2 - base);
/* check immediate repcode */ /* check immediate repcode */
while ( (ip <= ilimit) while ((ip <= ilimit) && ((offset_2 > 0) & (ZSTD_read32(ip) == ZSTD_read32(ip - offset_2)))) {
&& ( (offset_2>0)
& (ZSTD_read32(ip) == ZSTD_read32(ip - offset_2)) )) {
/* store sequence */ /* store sequence */
size_t const rLength = ZSTD_count(ip + 4, ip + 4 - offset_2, iend) + 4; size_t const rLength = ZSTD_count(ip + 4, ip + 4 - offset_2, iend) + 4;
{ U32 const tmpOff = offset_2; offset_2 = offset_1; offset_1 = tmpOff; } /* swap offset_2 <=> offset_1 */ {
U32 const tmpOff = offset_2;
offset_2 = offset_1;
offset_1 = tmpOff;
} /* swap offset_2 <=> offset_1 */
hashTable[ZSTD_hashPtr(ip, hBits, mls)] = (U32)(ip - base); hashTable[ZSTD_hashPtr(ip, hBits, mls)] = (U32)(ip - base);
ZSTD_storeSeq(seqStorePtr, 0, anchor, 0, rLength - MINMATCH); ZSTD_storeSeq(seqStorePtr, 0, anchor, 0, rLength - MINMATCH);
ip += rLength; ip += rLength;
anchor = ip; anchor = ip;
continue; /* faster when present ... (?) */ continue; /* faster when present ... (?) */
} } } }
}
}
/* save reps for next block */ /* save reps for next block */
cctx->repToConfirm[0] = offset_1 ? offset_1 : offsetSaved; cctx->repToConfirm[0] = offset_1 ? offset_1 : offsetSaved;
cctx->repToConfirm[1] = offset_2 ? offset_2 : offsetSaved; cctx->repToConfirm[1] = offset_2 ? offset_2 : offsetSaved;
/* Last Literals */ /* Last Literals */
{ size_t const lastLLSize = iend - anchor; {
size_t const lastLLSize = iend - anchor;
memcpy(seqStorePtr->lit, anchor, lastLLSize); memcpy(seqStorePtr->lit, anchor, lastLLSize);
seqStorePtr->lit += lastLLSize; seqStorePtr->lit += lastLLSize;
} }
} }
static void ZSTD_compressBlock_fast(ZSTD_CCtx *ctx, const void *src, size_t srcSize)
static void ZSTD_compressBlock_fast(ZSTD_CCtx* ctx,
const void* src, size_t srcSize)
{ {
const U32 mls = ctx->params.cParams.searchLength; const U32 mls = ctx->params.cParams.searchLength;
switch(mls) switch (mls) {
{
default: /* includes case 3 */ default: /* includes case 3 */
case 4 : case 4: ZSTD_compressBlock_fast_generic(ctx, src, srcSize, 4); return;
ZSTD_compressBlock_fast_generic(ctx, src, srcSize, 4); return; case 5: ZSTD_compressBlock_fast_generic(ctx, src, srcSize, 5); return;
case 5 : case 6: ZSTD_compressBlock_fast_generic(ctx, src, srcSize, 6); return;
ZSTD_compressBlock_fast_generic(ctx, src, srcSize, 5); return; case 7: ZSTD_compressBlock_fast_generic(ctx, src, srcSize, 7); return;
case 6 :
ZSTD_compressBlock_fast_generic(ctx, src, srcSize, 6); return;
case 7 :
ZSTD_compressBlock_fast_generic(ctx, src, srcSize, 7); return;
} }
} }
static void ZSTD_compressBlock_fast_extDict_generic(ZSTD_CCtx *ctx, const void *src, size_t srcSize, const U32 mls)
static void ZSTD_compressBlock_fast_extDict_generic(ZSTD_CCtx* ctx,
const void* src, size_t srcSize,
const U32 mls)
{ {
U32 *hashTable = ctx->hashTable; U32 *hashTable = ctx->hashTable;
const U32 hBits = ctx->params.cParams.hashLog; const U32 hBits = ctx->params.cParams.hashLog;
@@ -1059,28 +1131,33 @@ static void ZSTD_compressBlock_fast_extDict_generic(ZSTD_CCtx* ctx,
size_t mLength; size_t mLength;
hashTable[h] = curr; /* update hash table */ hashTable[h] = curr; /* update hash table */
if ( (((U32)((dictLimit-1) - repIndex) >= 3) /* intentional underflow */ & (repIndex > lowestIndex)) if ((((U32)((dictLimit - 1) - repIndex) >= 3) /* intentional underflow */ & (repIndex > lowestIndex)) &&
&& (ZSTD_read32(repMatch) == ZSTD_read32(ip+1)) ) { (ZSTD_read32(repMatch) == ZSTD_read32(ip + 1))) {
const BYTE *repMatchEnd = repIndex < dictLimit ? dictEnd : iend; const BYTE *repMatchEnd = repIndex < dictLimit ? dictEnd : iend;
mLength = ZSTD_count_2segments(ip + 1 + EQUAL_READ32, repMatch + EQUAL_READ32, iend, repMatchEnd, lowPrefixPtr) + EQUAL_READ32; mLength = ZSTD_count_2segments(ip + 1 + EQUAL_READ32, repMatch + EQUAL_READ32, iend, repMatchEnd, lowPrefixPtr) + EQUAL_READ32;
ip++; ip++;
ZSTD_storeSeq(seqStorePtr, ip - anchor, anchor, 0, mLength - MINMATCH); ZSTD_storeSeq(seqStorePtr, ip - anchor, anchor, 0, mLength - MINMATCH);
} else { } else {
if ( (matchIndex < lowestIndex) || if ((matchIndex < lowestIndex) || (ZSTD_read32(match) != ZSTD_read32(ip))) {
(ZSTD_read32(match) != ZSTD_read32(ip)) ) {
ip += ((ip - anchor) >> g_searchStrength) + 1; ip += ((ip - anchor) >> g_searchStrength) + 1;
continue; continue;
} }
{ const BYTE* matchEnd = matchIndex < dictLimit ? dictEnd : iend; {
const BYTE *matchEnd = matchIndex < dictLimit ? dictEnd : iend;
const BYTE *lowMatchPtr = matchIndex < dictLimit ? dictStart : lowPrefixPtr; const BYTE *lowMatchPtr = matchIndex < dictLimit ? dictStart : lowPrefixPtr;
U32 offset; U32 offset;
mLength = ZSTD_count_2segments(ip + EQUAL_READ32, match + EQUAL_READ32, iend, matchEnd, lowPrefixPtr) + EQUAL_READ32; mLength = ZSTD_count_2segments(ip + EQUAL_READ32, match + EQUAL_READ32, iend, matchEnd, lowPrefixPtr) + EQUAL_READ32;
while (((ip>anchor) & (match>lowMatchPtr)) && (ip[-1] == match[-1])) { ip--; match--; mLength++; } /* catch up */ while (((ip > anchor) & (match > lowMatchPtr)) && (ip[-1] == match[-1])) {
ip--;
match--;
mLength++;
} /* catch up */
offset = curr - matchIndex; offset = curr - matchIndex;
offset_2 = offset_1; offset_2 = offset_1;
offset_1 = offset; offset_1 = offset;
ZSTD_storeSeq(seqStorePtr, ip - anchor, anchor, offset + ZSTD_REP_MOVE, mLength - MINMATCH); ZSTD_storeSeq(seqStorePtr, ip - anchor, anchor, offset + ZSTD_REP_MOVE, mLength - MINMATCH);
} } }
}
/* found a match : store it */ /* found a match : store it */
ip += mLength; ip += mLength;
@@ -1098,8 +1175,11 @@ static void ZSTD_compressBlock_fast_extDict_generic(ZSTD_CCtx* ctx,
if ((((U32)((dictLimit - 1) - repIndex2) >= 3) & (repIndex2 > lowestIndex)) /* intentional overflow */ if ((((U32)((dictLimit - 1) - repIndex2) >= 3) & (repIndex2 > lowestIndex)) /* intentional overflow */
&& (ZSTD_read32(repMatch2) == ZSTD_read32(ip))) { && (ZSTD_read32(repMatch2) == ZSTD_read32(ip))) {
const BYTE *const repEnd2 = repIndex2 < dictLimit ? dictEnd : iend; const BYTE *const repEnd2 = repIndex2 < dictLimit ? dictEnd : iend;
size_t repLength2 = ZSTD_count_2segments(ip+EQUAL_READ32, repMatch2+EQUAL_READ32, iend, repEnd2, lowPrefixPtr) + EQUAL_READ32; size_t repLength2 =
U32 tmpOffset = offset_2; offset_2 = offset_1; offset_1 = tmpOffset; /* swap offset_2 <=> offset_1 */ ZSTD_count_2segments(ip + EQUAL_READ32, repMatch2 + EQUAL_READ32, iend, repEnd2, lowPrefixPtr) + EQUAL_READ32;
U32 tmpOffset = offset_2;
offset_2 = offset_1;
offset_1 = tmpOffset; /* swap offset_2 <=> offset_1 */
ZSTD_storeSeq(seqStorePtr, 0, anchor, 0, repLength2 - MINMATCH); ZSTD_storeSeq(seqStorePtr, 0, anchor, 0, repLength2 - MINMATCH);
hashTable[ZSTD_hashPtr(ip, hBits, mls)] = curr2; hashTable[ZSTD_hashPtr(ip, hBits, mls)] = curr2;
ip += repLength2; ip += repLength2;
@@ -1107,38 +1187,34 @@ static void ZSTD_compressBlock_fast_extDict_generic(ZSTD_CCtx* ctx,
continue; continue;
} }
break; break;
} } } }
}
}
/* save reps for next block */ /* save reps for next block */
ctx->repToConfirm[0] = offset_1; ctx->repToConfirm[1] = offset_2; ctx->repToConfirm[0] = offset_1;
ctx->repToConfirm[1] = offset_2;
/* Last Literals */ /* Last Literals */
{ size_t const lastLLSize = iend - anchor; {
size_t const lastLLSize = iend - anchor;
memcpy(seqStorePtr->lit, anchor, lastLLSize); memcpy(seqStorePtr->lit, anchor, lastLLSize);
seqStorePtr->lit += lastLLSize; seqStorePtr->lit += lastLLSize;
} }
} }
static void ZSTD_compressBlock_fast_extDict(ZSTD_CCtx *ctx, const void *src, size_t srcSize)
static void ZSTD_compressBlock_fast_extDict(ZSTD_CCtx* ctx,
const void* src, size_t srcSize)
{ {
U32 const mls = ctx->params.cParams.searchLength; U32 const mls = ctx->params.cParams.searchLength;
switch(mls) switch (mls) {
{
default: /* includes case 3 */ default: /* includes case 3 */
case 4 : case 4: ZSTD_compressBlock_fast_extDict_generic(ctx, src, srcSize, 4); return;
ZSTD_compressBlock_fast_extDict_generic(ctx, src, srcSize, 4); return; case 5: ZSTD_compressBlock_fast_extDict_generic(ctx, src, srcSize, 5); return;
case 5 : case 6: ZSTD_compressBlock_fast_extDict_generic(ctx, src, srcSize, 6); return;
ZSTD_compressBlock_fast_extDict_generic(ctx, src, srcSize, 5); return; case 7: ZSTD_compressBlock_fast_extDict_generic(ctx, src, srcSize, 7); return;
case 6 :
ZSTD_compressBlock_fast_extDict_generic(ctx, src, srcSize, 6); return;
case 7 :
ZSTD_compressBlock_fast_extDict_generic(ctx, src, srcSize, 7); return;
} }
} }
/*-************************************* /*-*************************************
* Double Fast * Double Fast
***************************************/ ***************************************/
@@ -1160,11 +1236,8 @@ static void ZSTD_fillDoubleHashTable (ZSTD_CCtx* cctx, const void* end, const U3
} }
} }
FORCE_INLINE FORCE_INLINE
void ZSTD_compressBlock_doubleFast_generic(ZSTD_CCtx* cctx, void ZSTD_compressBlock_doubleFast_generic(ZSTD_CCtx *cctx, const void *src, size_t srcSize, const U32 mls)
const void* src, size_t srcSize,
const U32 mls)
{ {
U32 *const hashLong = cctx->hashTable; U32 *const hashLong = cctx->hashTable;
const U32 hBitsL = cctx->params.cParams.hashLog; const U32 hBitsL = cctx->params.cParams.hashLog;
@@ -1184,9 +1257,12 @@ void ZSTD_compressBlock_doubleFast_generic(ZSTD_CCtx* cctx,
/* init */ /* init */
ip += (ip == lowest); ip += (ip == lowest);
{ U32 const maxRep = (U32)(ip-lowest); {
if (offset_2 > maxRep) offsetSaved = offset_2, offset_2 = 0; U32 const maxRep = (U32)(ip - lowest);
if (offset_1 > maxRep) offsetSaved = offset_1, offset_1 = 0; if (offset_2 > maxRep)
offsetSaved = offset_2, offset_2 = 0;
if (offset_1 > maxRep)
offsetSaved = offset_1, offset_1 = 0;
} }
/* Main Search Loop */ /* Main Search Loop */
@@ -1210,7 +1286,11 @@ void ZSTD_compressBlock_doubleFast_generic(ZSTD_CCtx* cctx,
if ((matchIndexL > lowestIndex) && (ZSTD_read64(matchLong) == ZSTD_read64(ip))) { if ((matchIndexL > lowestIndex) && (ZSTD_read64(matchLong) == ZSTD_read64(ip))) {
mLength = ZSTD_count(ip + 8, matchLong + 8, iend) + 8; mLength = ZSTD_count(ip + 8, matchLong + 8, iend) + 8;
offset = (U32)(ip - matchLong); offset = (U32)(ip - matchLong);
while (((ip>anchor) & (matchLong>lowest)) && (ip[-1] == matchLong[-1])) { ip--; matchLong--; mLength++; } /* catch up */ while (((ip > anchor) & (matchLong > lowest)) && (ip[-1] == matchLong[-1])) {
ip--;
matchLong--;
mLength++;
} /* catch up */
} else if ((matchIndexS > lowestIndex) && (ZSTD_read32(match) == ZSTD_read32(ip))) { } else if ((matchIndexS > lowestIndex) && (ZSTD_read32(match) == ZSTD_read32(ip))) {
size_t const h3 = ZSTD_hashPtr(ip + 1, hBitsL, 8); size_t const h3 = ZSTD_hashPtr(ip + 1, hBitsL, 8);
U32 const matchIndex3 = hashLong[h3]; U32 const matchIndex3 = hashLong[h3];
@@ -1220,11 +1300,19 @@ void ZSTD_compressBlock_doubleFast_generic(ZSTD_CCtx* cctx,
mLength = ZSTD_count(ip + 9, match3 + 8, iend) + 8; mLength = ZSTD_count(ip + 9, match3 + 8, iend) + 8;
ip++; ip++;
offset = (U32)(ip - match3); offset = (U32)(ip - match3);
while (((ip>anchor) & (match3>lowest)) && (ip[-1] == match3[-1])) { ip--; match3--; mLength++; } /* catch up */ while (((ip > anchor) & (match3 > lowest)) && (ip[-1] == match3[-1])) {
ip--;
match3--;
mLength++;
} /* catch up */
} else { } else {
mLength = ZSTD_count(ip + 4, match + 4, iend) + 4; mLength = ZSTD_count(ip + 4, match + 4, iend) + 4;
offset = (U32)(ip - match); offset = (U32)(ip - match);
while (((ip>anchor) & (match>lowest)) && (ip[-1] == match[-1])) { ip--; match--; mLength++; } /* catch up */ while (((ip > anchor) & (match > lowest)) && (ip[-1] == match[-1])) {
ip--;
match--;
mLength++;
} /* catch up */
} }
} else { } else {
ip += ((ip - anchor) >> g_searchStrength) + 1; ip += ((ip - anchor) >> g_searchStrength) + 1;
@@ -1243,59 +1331,54 @@ void ZSTD_compressBlock_doubleFast_generic(ZSTD_CCtx* cctx,
if (ip <= ilimit) { if (ip <= ilimit) {
/* Fill Table */ /* Fill Table */
hashLong[ZSTD_hashPtr(base+curr+2, hBitsL, 8)] = hashLong[ZSTD_hashPtr(base + curr + 2, hBitsL, 8)] = hashSmall[ZSTD_hashPtr(base + curr + 2, hBitsS, mls)] =
hashSmall[ZSTD_hashPtr(base+curr+2, hBitsS, mls)] = curr+2; /* here because curr+2 could be > iend-8 */ curr + 2; /* here because curr+2 could be > iend-8 */
hashLong[ZSTD_hashPtr(ip-2, hBitsL, 8)] = hashLong[ZSTD_hashPtr(ip - 2, hBitsL, 8)] = hashSmall[ZSTD_hashPtr(ip - 2, hBitsS, mls)] = (U32)(ip - 2 - base);
hashSmall[ZSTD_hashPtr(ip-2, hBitsS, mls)] = (U32)(ip-2-base);
/* check immediate repcode */ /* check immediate repcode */
while ( (ip <= ilimit) while ((ip <= ilimit) && ((offset_2 > 0) & (ZSTD_read32(ip) == ZSTD_read32(ip - offset_2)))) {
&& ( (offset_2>0)
& (ZSTD_read32(ip) == ZSTD_read32(ip - offset_2)) )) {
/* store sequence */ /* store sequence */
size_t const rLength = ZSTD_count(ip + 4, ip + 4 - offset_2, iend) + 4; size_t const rLength = ZSTD_count(ip + 4, ip + 4 - offset_2, iend) + 4;
{ U32 const tmpOff = offset_2; offset_2 = offset_1; offset_1 = tmpOff; } /* swap offset_2 <=> offset_1 */ {
U32 const tmpOff = offset_2;
offset_2 = offset_1;
offset_1 = tmpOff;
} /* swap offset_2 <=> offset_1 */
hashSmall[ZSTD_hashPtr(ip, hBitsS, mls)] = (U32)(ip - base); hashSmall[ZSTD_hashPtr(ip, hBitsS, mls)] = (U32)(ip - base);
hashLong[ZSTD_hashPtr(ip, hBitsL, 8)] = (U32)(ip - base); hashLong[ZSTD_hashPtr(ip, hBitsL, 8)] = (U32)(ip - base);
ZSTD_storeSeq(seqStorePtr, 0, anchor, 0, rLength - MINMATCH); ZSTD_storeSeq(seqStorePtr, 0, anchor, 0, rLength - MINMATCH);
ip += rLength; ip += rLength;
anchor = ip; anchor = ip;
continue; /* faster when present ... (?) */ continue; /* faster when present ... (?) */
} } } }
}
}
/* save reps for next block */ /* save reps for next block */
cctx->repToConfirm[0] = offset_1 ? offset_1 : offsetSaved; cctx->repToConfirm[0] = offset_1 ? offset_1 : offsetSaved;
cctx->repToConfirm[1] = offset_2 ? offset_2 : offsetSaved; cctx->repToConfirm[1] = offset_2 ? offset_2 : offsetSaved;
/* Last Literals */ /* Last Literals */
{ size_t const lastLLSize = iend - anchor; {
size_t const lastLLSize = iend - anchor;
memcpy(seqStorePtr->lit, anchor, lastLLSize); memcpy(seqStorePtr->lit, anchor, lastLLSize);
seqStorePtr->lit += lastLLSize; seqStorePtr->lit += lastLLSize;
} }
} }
static void ZSTD_compressBlock_doubleFast(ZSTD_CCtx *ctx, const void *src, size_t srcSize) static void ZSTD_compressBlock_doubleFast(ZSTD_CCtx *ctx, const void *src, size_t srcSize)
{ {
const U32 mls = ctx->params.cParams.searchLength; const U32 mls = ctx->params.cParams.searchLength;
switch(mls) switch (mls) {
{
default: /* includes case 3 */ default: /* includes case 3 */
case 4 : case 4: ZSTD_compressBlock_doubleFast_generic(ctx, src, srcSize, 4); return;
ZSTD_compressBlock_doubleFast_generic(ctx, src, srcSize, 4); return; case 5: ZSTD_compressBlock_doubleFast_generic(ctx, src, srcSize, 5); return;
case 5 : case 6: ZSTD_compressBlock_doubleFast_generic(ctx, src, srcSize, 6); return;
ZSTD_compressBlock_doubleFast_generic(ctx, src, srcSize, 5); return; case 7: ZSTD_compressBlock_doubleFast_generic(ctx, src, srcSize, 7); return;
case 6 :
ZSTD_compressBlock_doubleFast_generic(ctx, src, srcSize, 6); return;
case 7 :
ZSTD_compressBlock_doubleFast_generic(ctx, src, srcSize, 7); return;
} }
} }
static void ZSTD_compressBlock_doubleFast_extDict_generic(ZSTD_CCtx *ctx, const void *src, size_t srcSize, const U32 mls)
static void ZSTD_compressBlock_doubleFast_extDict_generic(ZSTD_CCtx* ctx,
const void* src, size_t srcSize,
const U32 mls)
{ {
U32 *const hashLong = ctx->hashTable; U32 *const hashLong = ctx->hashTable;
U32 const hBitsL = ctx->params.cParams.hashLog; U32 const hBitsL = ctx->params.cParams.hashLog;
@@ -1335,8 +1418,8 @@ static void ZSTD_compressBlock_doubleFast_extDict_generic(ZSTD_CCtx* ctx,
size_t mLength; size_t mLength;
hashSmall[hSmall] = hashLong[hLong] = curr; /* update hash table */ hashSmall[hSmall] = hashLong[hLong] = curr; /* update hash table */
if ( (((U32)((dictLimit-1) - repIndex) >= 3) /* intentional underflow */ & (repIndex > lowestIndex)) if ((((U32)((dictLimit - 1) - repIndex) >= 3) /* intentional underflow */ & (repIndex > lowestIndex)) &&
&& (ZSTD_read32(repMatch) == ZSTD_read32(ip+1)) ) { (ZSTD_read32(repMatch) == ZSTD_read32(ip + 1))) {
const BYTE *repMatchEnd = repIndex < dictLimit ? dictEnd : iend; const BYTE *repMatchEnd = repIndex < dictLimit ? dictEnd : iend;
mLength = ZSTD_count_2segments(ip + 1 + 4, repMatch + 4, iend, repMatchEnd, lowPrefixPtr) + 4; mLength = ZSTD_count_2segments(ip + 1 + 4, repMatch + 4, iend, repMatchEnd, lowPrefixPtr) + 4;
ip++; ip++;
@@ -1348,7 +1431,11 @@ static void ZSTD_compressBlock_doubleFast_extDict_generic(ZSTD_CCtx* ctx,
U32 offset; U32 offset;
mLength = ZSTD_count_2segments(ip + 8, matchLong + 8, iend, matchEnd, lowPrefixPtr) + 8; mLength = ZSTD_count_2segments(ip + 8, matchLong + 8, iend, matchEnd, lowPrefixPtr) + 8;
offset = curr - matchLongIndex; offset = curr - matchLongIndex;
while (((ip>anchor) & (matchLong>lowMatchPtr)) && (ip[-1] == matchLong[-1])) { ip--; matchLong--; mLength++; } /* catch up */ while (((ip > anchor) & (matchLong > lowMatchPtr)) && (ip[-1] == matchLong[-1])) {
ip--;
matchLong--;
mLength++;
} /* catch up */
offset_2 = offset_1; offset_2 = offset_1;
offset_1 = offset; offset_1 = offset;
ZSTD_storeSeq(seqStorePtr, ip - anchor, anchor, offset + ZSTD_REP_MOVE, mLength - MINMATCH); ZSTD_storeSeq(seqStorePtr, ip - anchor, anchor, offset + ZSTD_REP_MOVE, mLength - MINMATCH);
@@ -1366,13 +1453,21 @@ static void ZSTD_compressBlock_doubleFast_extDict_generic(ZSTD_CCtx* ctx,
mLength = ZSTD_count_2segments(ip + 9, match3 + 8, iend, matchEnd, lowPrefixPtr) + 8; mLength = ZSTD_count_2segments(ip + 9, match3 + 8, iend, matchEnd, lowPrefixPtr) + 8;
ip++; ip++;
offset = curr + 1 - matchIndex3; offset = curr + 1 - matchIndex3;
while (((ip>anchor) & (match3>lowMatchPtr)) && (ip[-1] == match3[-1])) { ip--; match3--; mLength++; } /* catch up */ while (((ip > anchor) & (match3 > lowMatchPtr)) && (ip[-1] == match3[-1])) {
ip--;
match3--;
mLength++;
} /* catch up */
} else { } else {
const BYTE *matchEnd = matchIndex < dictLimit ? dictEnd : iend; const BYTE *matchEnd = matchIndex < dictLimit ? dictEnd : iend;
const BYTE *lowMatchPtr = matchIndex < dictLimit ? dictStart : lowPrefixPtr; const BYTE *lowMatchPtr = matchIndex < dictLimit ? dictStart : lowPrefixPtr;
mLength = ZSTD_count_2segments(ip + 4, match + 4, iend, matchEnd, lowPrefixPtr) + 4; mLength = ZSTD_count_2segments(ip + 4, match + 4, iend, matchEnd, lowPrefixPtr) + 4;
offset = curr - matchIndex; offset = curr - matchIndex;
while (((ip>anchor) & (match>lowMatchPtr)) && (ip[-1] == match[-1])) { ip--; match--; mLength++; } /* catch up */ while (((ip > anchor) & (match > lowMatchPtr)) && (ip[-1] == match[-1])) {
ip--;
match--;
mLength++;
} /* catch up */
} }
offset_2 = offset_1; offset_2 = offset_1;
offset_1 = offset; offset_1 = offset;
@@ -1381,7 +1476,8 @@ static void ZSTD_compressBlock_doubleFast_extDict_generic(ZSTD_CCtx* ctx,
} else { } else {
ip += ((ip - anchor) >> g_searchStrength) + 1; ip += ((ip - anchor) >> g_searchStrength) + 1;
continue; continue;
} } }
}
/* found a match : store it */ /* found a match : store it */
ip += mLength; ip += mLength;
@@ -1401,8 +1497,11 @@ static void ZSTD_compressBlock_doubleFast_extDict_generic(ZSTD_CCtx* ctx,
if ((((U32)((dictLimit - 1) - repIndex2) >= 3) & (repIndex2 > lowestIndex)) /* intentional overflow */ if ((((U32)((dictLimit - 1) - repIndex2) >= 3) & (repIndex2 > lowestIndex)) /* intentional overflow */
&& (ZSTD_read32(repMatch2) == ZSTD_read32(ip))) { && (ZSTD_read32(repMatch2) == ZSTD_read32(ip))) {
const BYTE *const repEnd2 = repIndex2 < dictLimit ? dictEnd : iend; const BYTE *const repEnd2 = repIndex2 < dictLimit ? dictEnd : iend;
size_t const repLength2 = ZSTD_count_2segments(ip+EQUAL_READ32, repMatch2+EQUAL_READ32, iend, repEnd2, lowPrefixPtr) + EQUAL_READ32; size_t const repLength2 =
U32 tmpOffset = offset_2; offset_2 = offset_1; offset_1 = tmpOffset; /* swap offset_2 <=> offset_1 */ ZSTD_count_2segments(ip + EQUAL_READ32, repMatch2 + EQUAL_READ32, iend, repEnd2, lowPrefixPtr) + EQUAL_READ32;
U32 tmpOffset = offset_2;
offset_2 = offset_1;
offset_1 = tmpOffset; /* swap offset_2 <=> offset_1 */
ZSTD_storeSeq(seqStorePtr, 0, anchor, 0, repLength2 - MINMATCH); ZSTD_storeSeq(seqStorePtr, 0, anchor, 0, repLength2 - MINMATCH);
hashSmall[ZSTD_hashPtr(ip, hBitsS, mls)] = curr2; hashSmall[ZSTD_hashPtr(ip, hBitsS, mls)] = curr2;
hashLong[ZSTD_hashPtr(ip, hBitsL, 8)] = curr2; hashLong[ZSTD_hashPtr(ip, hBitsL, 8)] = curr2;
@@ -1411,46 +1510,41 @@ static void ZSTD_compressBlock_doubleFast_extDict_generic(ZSTD_CCtx* ctx,
continue; continue;
} }
break; break;
} } } }
}
}
/* save reps for next block */ /* save reps for next block */
ctx->repToConfirm[0] = offset_1; ctx->repToConfirm[1] = offset_2; ctx->repToConfirm[0] = offset_1;
ctx->repToConfirm[1] = offset_2;
/* Last Literals */ /* Last Literals */
{ size_t const lastLLSize = iend - anchor; {
size_t const lastLLSize = iend - anchor;
memcpy(seqStorePtr->lit, anchor, lastLLSize); memcpy(seqStorePtr->lit, anchor, lastLLSize);
seqStorePtr->lit += lastLLSize; seqStorePtr->lit += lastLLSize;
} }
} }
static void ZSTD_compressBlock_doubleFast_extDict(ZSTD_CCtx *ctx, const void *src, size_t srcSize)
static void ZSTD_compressBlock_doubleFast_extDict(ZSTD_CCtx* ctx,
const void* src, size_t srcSize)
{ {
U32 const mls = ctx->params.cParams.searchLength; U32 const mls = ctx->params.cParams.searchLength;
switch(mls) switch (mls) {
{
default: /* includes case 3 */ default: /* includes case 3 */
case 4 : case 4: ZSTD_compressBlock_doubleFast_extDict_generic(ctx, src, srcSize, 4); return;
ZSTD_compressBlock_doubleFast_extDict_generic(ctx, src, srcSize, 4); return; case 5: ZSTD_compressBlock_doubleFast_extDict_generic(ctx, src, srcSize, 5); return;
case 5 : case 6: ZSTD_compressBlock_doubleFast_extDict_generic(ctx, src, srcSize, 6); return;
ZSTD_compressBlock_doubleFast_extDict_generic(ctx, src, srcSize, 5); return; case 7: ZSTD_compressBlock_doubleFast_extDict_generic(ctx, src, srcSize, 7); return;
case 6 :
ZSTD_compressBlock_doubleFast_extDict_generic(ctx, src, srcSize, 6); return;
case 7 :
ZSTD_compressBlock_doubleFast_extDict_generic(ctx, src, srcSize, 7); return;
} }
} }
/*-************************************* /*-*************************************
* Binary Tree search * Binary Tree search
***************************************/ ***************************************/
/** ZSTD_insertBt1() : add one or multiple positions to tree. /** ZSTD_insertBt1() : add one or multiple positions to tree.
* ip : assumed <= iend-8 . * ip : assumed <= iend-8 .
* @return : nb of positions added */ * @return : nb of positions added */
static U32 ZSTD_insertBt1(ZSTD_CCtx* zc, const BYTE* const ip, const U32 mls, const BYTE* const iend, U32 nbCompares, static U32 ZSTD_insertBt1(ZSTD_CCtx *zc, const BYTE *const ip, const U32 mls, const BYTE *const iend, U32 nbCompares, U32 extDict)
U32 extDict)
{ {
U32 *const hashTable = zc->hashTable; U32 *const hashTable = zc->hashTable;
U32 const hashLog = zc->params.cParams.hashLog; U32 const hashLog = zc->params.cParams.hashLog;
@@ -1505,30 +1599,34 @@ static U32 ZSTD_insertBt1(ZSTD_CCtx* zc, const BYTE* const ip, const U32 mls, co
/* match is smaller than curr */ /* match is smaller than curr */
*smallerPtr = matchIndex; /* update smaller idx */ *smallerPtr = matchIndex; /* update smaller idx */
commonLengthSmaller = matchLength; /* all smaller will now have at least this guaranteed common length */ commonLengthSmaller = matchLength; /* all smaller will now have at least this guaranteed common length */
if (matchIndex <= btLow) { smallerPtr=&dummy32; break; } /* beyond tree size, stop the search */ if (matchIndex <= btLow) {
smallerPtr = &dummy32;
break;
} /* beyond tree size, stop the search */
smallerPtr = nextPtr + 1; /* new "smaller" => larger of match */ smallerPtr = nextPtr + 1; /* new "smaller" => larger of match */
matchIndex = nextPtr[1]; /* new matchIndex larger than previous (closer to curr) */ matchIndex = nextPtr[1]; /* new matchIndex larger than previous (closer to curr) */
} else { } else {
/* match is larger than curr */ /* match is larger than curr */
*largerPtr = matchIndex; *largerPtr = matchIndex;
commonLengthLarger = matchLength; commonLengthLarger = matchLength;
if (matchIndex <= btLow) { largerPtr=&dummy32; break; } /* beyond tree size, stop the search */ if (matchIndex <= btLow) {
largerPtr = &dummy32;
break;
} /* beyond tree size, stop the search */
largerPtr = nextPtr; largerPtr = nextPtr;
matchIndex = nextPtr[0]; matchIndex = nextPtr[0];
} } }
}
*smallerPtr = *largerPtr = 0; *smallerPtr = *largerPtr = 0;
if (bestLength > 384) return MIN(192, (U32)(bestLength - 384)); /* speed optimization */ if (bestLength > 384)
if (matchEndIdx > curr + 8) return matchEndIdx - curr - 8; return MIN(192, (U32)(bestLength - 384)); /* speed optimization */
if (matchEndIdx > curr + 8)
return matchEndIdx - curr - 8;
return 1; return 1;
} }
static size_t ZSTD_insertBtAndFindBestMatch(ZSTD_CCtx *zc, const BYTE *const ip, const BYTE *const iend, size_t *offsetPtr, U32 nbCompares, const U32 mls,
static size_t ZSTD_insertBtAndFindBestMatch (
ZSTD_CCtx* zc,
const BYTE* const ip, const BYTE* const iend,
size_t* offsetPtr,
U32 nbCompares, const U32 mls,
U32 extDict) U32 extDict)
{ {
U32 *const hashTable = zc->hashTable; U32 *const hashTable = zc->hashTable;
@@ -1584,17 +1682,24 @@ static size_t ZSTD_insertBtAndFindBestMatch (
/* match is smaller than curr */ /* match is smaller than curr */
*smallerPtr = matchIndex; /* update smaller idx */ *smallerPtr = matchIndex; /* update smaller idx */
commonLengthSmaller = matchLength; /* all smaller will now have at least this guaranteed common length */ commonLengthSmaller = matchLength; /* all smaller will now have at least this guaranteed common length */
if (matchIndex <= btLow) { smallerPtr=&dummy32; break; } /* beyond tree size, stop the search */ if (matchIndex <= btLow) {
smallerPtr = &dummy32;
break;
} /* beyond tree size, stop the search */
smallerPtr = nextPtr + 1; /* new "smaller" => larger of match */ smallerPtr = nextPtr + 1; /* new "smaller" => larger of match */
matchIndex = nextPtr[1]; /* new matchIndex larger than previous (closer to curr) */ matchIndex = nextPtr[1]; /* new matchIndex larger than previous (closer to curr) */
} else { } else {
/* match is larger than curr */ /* match is larger than curr */
*largerPtr = matchIndex; *largerPtr = matchIndex;
commonLengthLarger = matchLength; commonLengthLarger = matchLength;
if (matchIndex <= btLow) { largerPtr=&dummy32; break; } /* beyond tree size, stop the search */ if (matchIndex <= btLow) {
largerPtr = &dummy32;
break;
} /* beyond tree size, stop the search */
largerPtr = nextPtr; largerPtr = nextPtr;
matchIndex = nextPtr[0]; matchIndex = nextPtr[0];
} } }
}
*smallerPtr = *largerPtr = 0; *smallerPtr = *largerPtr = 0;
@@ -1602,7 +1707,6 @@ static size_t ZSTD_insertBtAndFindBestMatch (
return bestLength; return bestLength;
} }
static void ZSTD_updateTree(ZSTD_CCtx *zc, const BYTE *const ip, const BYTE *const iend, const U32 nbCompares, const U32 mls) static void ZSTD_updateTree(ZSTD_CCtx *zc, const BYTE *const ip, const BYTE *const iend, const U32 nbCompares, const U32 mls)
{ {
const BYTE *const base = zc->base; const BYTE *const base = zc->base;
@@ -1614,26 +1718,18 @@ static void ZSTD_updateTree(ZSTD_CCtx* zc, const BYTE* const ip, const BYTE* con
} }
/** ZSTD_BtFindBestMatch() : Tree updater, providing best match */ /** ZSTD_BtFindBestMatch() : Tree updater, providing best match */
static size_t ZSTD_BtFindBestMatch ( static size_t ZSTD_BtFindBestMatch(ZSTD_CCtx *zc, const BYTE *const ip, const BYTE *const iLimit, size_t *offsetPtr, const U32 maxNbAttempts, const U32 mls)
ZSTD_CCtx* zc,
const BYTE* const ip, const BYTE* const iLimit,
size_t* offsetPtr,
const U32 maxNbAttempts, const U32 mls)
{ {
if (ip < zc->base + zc->nextToUpdate) return 0; /* skipped area */ if (ip < zc->base + zc->nextToUpdate)
return 0; /* skipped area */
ZSTD_updateTree(zc, ip, iLimit, maxNbAttempts, mls); ZSTD_updateTree(zc, ip, iLimit, maxNbAttempts, mls);
return ZSTD_insertBtAndFindBestMatch(zc, ip, iLimit, offsetPtr, maxNbAttempts, mls, 0); return ZSTD_insertBtAndFindBestMatch(zc, ip, iLimit, offsetPtr, maxNbAttempts, mls, 0);
} }
static size_t ZSTD_BtFindBestMatch_selectMLS(ZSTD_CCtx *zc, /* Index table will be updated */
static size_t ZSTD_BtFindBestMatch_selectMLS ( const BYTE *ip, const BYTE *const iLimit, size_t *offsetPtr, const U32 maxNbAttempts, const U32 matchLengthSearch)
ZSTD_CCtx* zc, /* Index table will be updated */
const BYTE* ip, const BYTE* const iLimit,
size_t* offsetPtr,
const U32 maxNbAttempts, const U32 matchLengthSearch)
{
switch(matchLengthSearch)
{ {
switch (matchLengthSearch) {
default: /* includes case 3 */ default: /* includes case 3 */
case 4: return ZSTD_BtFindBestMatch(zc, ip, iLimit, offsetPtr, maxNbAttempts, 4); case 4: return ZSTD_BtFindBestMatch(zc, ip, iLimit, offsetPtr, maxNbAttempts, 4);
case 5: return ZSTD_BtFindBestMatch(zc, ip, iLimit, offsetPtr, maxNbAttempts, 5); case 5: return ZSTD_BtFindBestMatch(zc, ip, iLimit, offsetPtr, maxNbAttempts, 5);
@@ -1642,38 +1738,31 @@ static size_t ZSTD_BtFindBestMatch_selectMLS (
} }
} }
static void ZSTD_updateTree_extDict(ZSTD_CCtx *zc, const BYTE *const ip, const BYTE *const iend, const U32 nbCompares, const U32 mls) static void ZSTD_updateTree_extDict(ZSTD_CCtx *zc, const BYTE *const ip, const BYTE *const iend, const U32 nbCompares, const U32 mls)
{ {
const BYTE *const base = zc->base; const BYTE *const base = zc->base;
const U32 target = (U32)(ip - base); const U32 target = (U32)(ip - base);
U32 idx = zc->nextToUpdate; U32 idx = zc->nextToUpdate;
while (idx < target) idx += ZSTD_insertBt1(zc, base+idx, mls, iend, nbCompares, 1); while (idx < target)
idx += ZSTD_insertBt1(zc, base + idx, mls, iend, nbCompares, 1);
} }
/** Tree updater, providing best match */ /** Tree updater, providing best match */
static size_t ZSTD_BtFindBestMatch_extDict ( static size_t ZSTD_BtFindBestMatch_extDict(ZSTD_CCtx *zc, const BYTE *const ip, const BYTE *const iLimit, size_t *offsetPtr, const U32 maxNbAttempts,
ZSTD_CCtx* zc, const U32 mls)
const BYTE* const ip, const BYTE* const iLimit,
size_t* offsetPtr,
const U32 maxNbAttempts, const U32 mls)
{ {
if (ip < zc->base + zc->nextToUpdate) return 0; /* skipped area */ if (ip < zc->base + zc->nextToUpdate)
return 0; /* skipped area */
ZSTD_updateTree_extDict(zc, ip, iLimit, maxNbAttempts, mls); ZSTD_updateTree_extDict(zc, ip, iLimit, maxNbAttempts, mls);
return ZSTD_insertBtAndFindBestMatch(zc, ip, iLimit, offsetPtr, maxNbAttempts, mls, 1); return ZSTD_insertBtAndFindBestMatch(zc, ip, iLimit, offsetPtr, maxNbAttempts, mls, 1);
} }
static size_t ZSTD_BtFindBestMatch_selectMLS_extDict(ZSTD_CCtx *zc, /* Index table will be updated */
static size_t ZSTD_BtFindBestMatch_selectMLS_extDict ( const BYTE *ip, const BYTE *const iLimit, size_t *offsetPtr, const U32 maxNbAttempts,
ZSTD_CCtx* zc, /* Index table will be updated */ const U32 matchLengthSearch)
const BYTE* ip, const BYTE* const iLimit,
size_t* offsetPtr,
const U32 maxNbAttempts, const U32 matchLengthSearch)
{
switch(matchLengthSearch)
{ {
switch (matchLengthSearch) {
default: /* includes case 3 */ default: /* includes case 3 */
case 4: return ZSTD_BtFindBestMatch_extDict(zc, ip, iLimit, offsetPtr, maxNbAttempts, 4); case 4: return ZSTD_BtFindBestMatch_extDict(zc, ip, iLimit, offsetPtr, maxNbAttempts, 4);
case 5: return ZSTD_BtFindBestMatch_extDict(zc, ip, iLimit, offsetPtr, maxNbAttempts, 5); case 5: return ZSTD_BtFindBestMatch_extDict(zc, ip, iLimit, offsetPtr, maxNbAttempts, 5);
@@ -1682,8 +1771,6 @@ static size_t ZSTD_BtFindBestMatch_selectMLS_extDict (
} }
} }
/* ********************************* /* *********************************
* Hash Chain * Hash Chain
***********************************/ ***********************************/
@@ -1713,14 +1800,10 @@ U32 ZSTD_insertAndFindFirstIndex (ZSTD_CCtx* zc, const BYTE* ip, U32 mls)
return hashTable[ZSTD_hashPtr(ip, hashLog, mls)]; return hashTable[ZSTD_hashPtr(ip, hashLog, mls)];
} }
FORCE_INLINE /* inlining is important to hardwire a hot branch (template emulation) */ FORCE_INLINE /* inlining is important to hardwire a hot branch (template emulation) */
size_t ZSTD_HcFindBestMatch_generic ( size_t
ZSTD_CCtx* zc, /* Index table will be updated */ ZSTD_HcFindBestMatch_generic(ZSTD_CCtx *zc, /* Index table will be updated */
const BYTE* const ip, const BYTE* const iLimit, const BYTE *const ip, const BYTE *const iLimit, size_t *offsetPtr, const U32 maxNbAttempts, const U32 mls, const U32 extDict)
size_t* offsetPtr,
const U32 maxNbAttempts, const U32 mls, const U32 extDict)
{ {
U32 *const chainTable = zc->chainTable; U32 *const chainTable = zc->chainTable;
const U32 chainSize = (1 << zc->params.cParams.chainLog); const U32 chainSize = (1 << zc->params.cParams.chainLog);
@@ -1753,24 +1836,25 @@ size_t ZSTD_HcFindBestMatch_generic (
} }
/* save best solution */ /* save best solution */
if (currMl > ml) { ml = currMl; *offsetPtr = curr - matchIndex + ZSTD_REP_MOVE; if (ip+currMl == iLimit) break; /* best possible, and avoid read overflow*/ } if (currMl > ml) {
ml = currMl;
*offsetPtr = curr - matchIndex + ZSTD_REP_MOVE;
if (ip + currMl == iLimit)
break; /* best possible, and avoid read overflow*/
}
if (matchIndex <= minChain) break; if (matchIndex <= minChain)
break;
matchIndex = NEXT_IN_CHAIN(matchIndex, chainMask); matchIndex = NEXT_IN_CHAIN(matchIndex, chainMask);
} }
return ml; return ml;
} }
FORCE_INLINE size_t ZSTD_HcFindBestMatch_selectMLS(ZSTD_CCtx *zc, const BYTE *ip, const BYTE *const iLimit, size_t *offsetPtr, const U32 maxNbAttempts,
FORCE_INLINE size_t ZSTD_HcFindBestMatch_selectMLS ( const U32 matchLengthSearch)
ZSTD_CCtx* zc,
const BYTE* ip, const BYTE* const iLimit,
size_t* offsetPtr,
const U32 maxNbAttempts, const U32 matchLengthSearch)
{
switch(matchLengthSearch)
{ {
switch (matchLengthSearch) {
default: /* includes case 3 */ default: /* includes case 3 */
case 4: return ZSTD_HcFindBestMatch_generic(zc, ip, iLimit, offsetPtr, maxNbAttempts, 4, 0); case 4: return ZSTD_HcFindBestMatch_generic(zc, ip, iLimit, offsetPtr, maxNbAttempts, 4, 0);
case 5: return ZSTD_HcFindBestMatch_generic(zc, ip, iLimit, offsetPtr, maxNbAttempts, 5, 0); case 5: return ZSTD_HcFindBestMatch_generic(zc, ip, iLimit, offsetPtr, maxNbAttempts, 5, 0);
@@ -1779,15 +1863,10 @@ FORCE_INLINE size_t ZSTD_HcFindBestMatch_selectMLS (
} }
} }
FORCE_INLINE size_t ZSTD_HcFindBestMatch_extDict_selectMLS(ZSTD_CCtx *zc, const BYTE *ip, const BYTE *const iLimit, size_t *offsetPtr, const U32 maxNbAttempts,
FORCE_INLINE size_t ZSTD_HcFindBestMatch_extDict_selectMLS ( const U32 matchLengthSearch)
ZSTD_CCtx* zc,
const BYTE* ip, const BYTE* const iLimit,
size_t* offsetPtr,
const U32 maxNbAttempts, const U32 matchLengthSearch)
{
switch(matchLengthSearch)
{ {
switch (matchLengthSearch) {
default: /* includes case 3 */ default: /* includes case 3 */
case 4: return ZSTD_HcFindBestMatch_generic(zc, ip, iLimit, offsetPtr, maxNbAttempts, 4, 1); case 4: return ZSTD_HcFindBestMatch_generic(zc, ip, iLimit, offsetPtr, maxNbAttempts, 4, 1);
case 5: return ZSTD_HcFindBestMatch_generic(zc, ip, iLimit, offsetPtr, maxNbAttempts, 5, 1); case 5: return ZSTD_HcFindBestMatch_generic(zc, ip, iLimit, offsetPtr, maxNbAttempts, 5, 1);
@@ -1796,14 +1875,11 @@ FORCE_INLINE size_t ZSTD_HcFindBestMatch_extDict_selectMLS (
} }
} }
/* ******************************* /* *******************************
* Common parser - lazy strategy * Common parser - lazy strategy
*********************************/ *********************************/
FORCE_INLINE FORCE_INLINE
void ZSTD_compressBlock_lazy_generic(ZSTD_CCtx* ctx, void ZSTD_compressBlock_lazy_generic(ZSTD_CCtx *ctx, const void *src, size_t srcSize, const U32 searchMethod, const U32 depth)
const void* src, size_t srcSize,
const U32 searchMethod, const U32 depth)
{ {
seqStore_t *seqStorePtr = &(ctx->seqStore); seqStore_t *seqStorePtr = &(ctx->seqStore);
const BYTE *const istart = (const BYTE *)src; const BYTE *const istart = (const BYTE *)src;
@@ -1816,18 +1892,19 @@ void ZSTD_compressBlock_lazy_generic(ZSTD_CCtx* ctx,
U32 const maxSearches = 1 << ctx->params.cParams.searchLog; U32 const maxSearches = 1 << ctx->params.cParams.searchLog;
U32 const mls = ctx->params.cParams.searchLength; U32 const mls = ctx->params.cParams.searchLength;
typedef size_t (*searchMax_f)(ZSTD_CCtx* zc, const BYTE* ip, const BYTE* iLimit, typedef size_t (*searchMax_f)(ZSTD_CCtx * zc, const BYTE *ip, const BYTE *iLimit, size_t *offsetPtr, U32 maxNbAttempts, U32 matchLengthSearch);
size_t* offsetPtr,
U32 maxNbAttempts, U32 matchLengthSearch);
searchMax_f const searchMax = searchMethod ? ZSTD_BtFindBestMatch_selectMLS : ZSTD_HcFindBestMatch_selectMLS; searchMax_f const searchMax = searchMethod ? ZSTD_BtFindBestMatch_selectMLS : ZSTD_HcFindBestMatch_selectMLS;
U32 offset_1 = ctx->rep[0], offset_2 = ctx->rep[1], savedOffset = 0; U32 offset_1 = ctx->rep[0], offset_2 = ctx->rep[1], savedOffset = 0;
/* init */ /* init */
ip += (ip == base); ip += (ip == base);
ctx->nextToUpdate3 = ctx->nextToUpdate; ctx->nextToUpdate3 = ctx->nextToUpdate;
{ U32 const maxRep = (U32)(ip-base); {
if (offset_2 > maxRep) savedOffset = offset_2, offset_2 = 0; U32 const maxRep = (U32)(ip - base);
if (offset_1 > maxRep) savedOffset = offset_1, offset_1 = 0; if (offset_2 > maxRep)
savedOffset = offset_2, offset_2 = 0;
if (offset_1 > maxRep)
savedOffset = offset_1, offset_1 = 0;
} }
/* Match Loop */ /* Match Loop */
@@ -1840,11 +1917,13 @@ void ZSTD_compressBlock_lazy_generic(ZSTD_CCtx* ctx,
if ((offset_1 > 0) & (ZSTD_read32(ip + 1) == ZSTD_read32(ip + 1 - offset_1))) { if ((offset_1 > 0) & (ZSTD_read32(ip + 1) == ZSTD_read32(ip + 1 - offset_1))) {
/* repcode : we take it */ /* repcode : we take it */
matchLength = ZSTD_count(ip + 1 + EQUAL_READ32, ip + 1 + EQUAL_READ32 - offset_1, iend) + EQUAL_READ32; matchLength = ZSTD_count(ip + 1 + EQUAL_READ32, ip + 1 + EQUAL_READ32 - offset_1, iend) + EQUAL_READ32;
if (depth==0) goto _storeSequence; if (depth == 0)
goto _storeSequence;
} }
/* first search (depth 0) */ /* first search (depth 0) */
{ size_t offsetFound = 99999999; {
size_t offsetFound = 99999999;
size_t const ml2 = searchMax(ctx, ip, iend, &offsetFound, maxSearches, mls); size_t const ml2 = searchMax(ctx, ip, iend, &offsetFound, maxSearches, mls);
if (ml2 > matchLength) if (ml2 > matchLength)
matchLength = ml2, start = ip, offset = offsetFound; matchLength = ml2, start = ip, offset = offsetFound;
@@ -1866,14 +1945,16 @@ void ZSTD_compressBlock_lazy_generic(ZSTD_CCtx* ctx,
if ((mlRep >= EQUAL_READ32) && (gain2 > gain1)) if ((mlRep >= EQUAL_READ32) && (gain2 > gain1))
matchLength = mlRep, offset = 0, start = ip; matchLength = mlRep, offset = 0, start = ip;
} }
{ size_t offset2=99999999; {
size_t offset2 = 99999999;
size_t const ml2 = searchMax(ctx, ip, iend, &offset2, maxSearches, mls); size_t const ml2 = searchMax(ctx, ip, iend, &offset2, maxSearches, mls);
int const gain2 = (int)(ml2 * 4 - ZSTD_highbit32((U32)offset2 + 1)); /* raw approx */ int const gain2 = (int)(ml2 * 4 - ZSTD_highbit32((U32)offset2 + 1)); /* raw approx */
int const gain1 = (int)(matchLength * 4 - ZSTD_highbit32((U32)offset + 1) + 4); int const gain1 = (int)(matchLength * 4 - ZSTD_highbit32((U32)offset + 1) + 4);
if ((ml2 >= EQUAL_READ32) && (gain2 > gain1)) { if ((ml2 >= EQUAL_READ32) && (gain2 > gain1)) {
matchLength = ml2, offset = offset2, start = ip; matchLength = ml2, offset = offset2, start = ip;
continue; /* search a better one */ continue; /* search a better one */
} } }
}
/* let's find an even better one */ /* let's find an even better one */
if ((depth == 2) && (ip < ilimit)) { if ((depth == 2) && (ip < ilimit)) {
@@ -1885,81 +1966,75 @@ void ZSTD_compressBlock_lazy_generic(ZSTD_CCtx* ctx,
if ((ml2 >= EQUAL_READ32) && (gain2 > gain1)) if ((ml2 >= EQUAL_READ32) && (gain2 > gain1))
matchLength = ml2, offset = 0, start = ip; matchLength = ml2, offset = 0, start = ip;
} }
{ size_t offset2=99999999; {
size_t offset2 = 99999999;
size_t const ml2 = searchMax(ctx, ip, iend, &offset2, maxSearches, mls); size_t const ml2 = searchMax(ctx, ip, iend, &offset2, maxSearches, mls);
int const gain2 = (int)(ml2 * 4 - ZSTD_highbit32((U32)offset2 + 1)); /* raw approx */ int const gain2 = (int)(ml2 * 4 - ZSTD_highbit32((U32)offset2 + 1)); /* raw approx */
int const gain1 = (int)(matchLength * 4 - ZSTD_highbit32((U32)offset + 1) + 7); int const gain1 = (int)(matchLength * 4 - ZSTD_highbit32((U32)offset + 1) + 7);
if ((ml2 >= EQUAL_READ32) && (gain2 > gain1)) { if ((ml2 >= EQUAL_READ32) && (gain2 > gain1)) {
matchLength = ml2, offset = offset2, start = ip; matchLength = ml2, offset = offset2, start = ip;
continue; continue;
} } } }
}
}
break; /* nothing found : store previous solution */ break; /* nothing found : store previous solution */
} }
/* catch up */ /* catch up */
if (offset) { if (offset) {
while ((start>anchor) && (start>base+offset-ZSTD_REP_MOVE) && (start[-1] == start[-1-offset+ZSTD_REP_MOVE])) /* only search for offset within prefix */ while ((start > anchor) && (start > base + offset - ZSTD_REP_MOVE) &&
{ start--; matchLength++; } (start[-1] == start[-1 - offset + ZSTD_REP_MOVE])) /* only search for offset within prefix */
offset_2 = offset_1; offset_1 = (U32)(offset - ZSTD_REP_MOVE); {
start--;
matchLength++;
}
offset_2 = offset_1;
offset_1 = (U32)(offset - ZSTD_REP_MOVE);
} }
/* store sequence */ /* store sequence */
_storeSequence: _storeSequence : {
{ size_t const litLength = start - anchor; size_t const litLength = start - anchor;
ZSTD_storeSeq(seqStorePtr, litLength, anchor, (U32)offset, matchLength - MINMATCH); ZSTD_storeSeq(seqStorePtr, litLength, anchor, (U32)offset, matchLength - MINMATCH);
anchor = ip = start + matchLength; anchor = ip = start + matchLength;
} }
/* check immediate repcode */ /* check immediate repcode */
while ( (ip <= ilimit) while ((ip <= ilimit) && ((offset_2 > 0) & (ZSTD_read32(ip) == ZSTD_read32(ip - offset_2)))) {
&& ((offset_2>0)
& (ZSTD_read32(ip) == ZSTD_read32(ip - offset_2)) )) {
/* store sequence */ /* store sequence */
matchLength = ZSTD_count(ip + EQUAL_READ32, ip + EQUAL_READ32 - offset_2, iend) + EQUAL_READ32; matchLength = ZSTD_count(ip + EQUAL_READ32, ip + EQUAL_READ32 - offset_2, iend) + EQUAL_READ32;
offset = offset_2; offset_2 = offset_1; offset_1 = (U32)offset; /* swap repcodes */ offset = offset_2;
offset_2 = offset_1;
offset_1 = (U32)offset; /* swap repcodes */
ZSTD_storeSeq(seqStorePtr, 0, anchor, 0, matchLength - MINMATCH); ZSTD_storeSeq(seqStorePtr, 0, anchor, 0, matchLength - MINMATCH);
ip += matchLength; ip += matchLength;
anchor = ip; anchor = ip;
continue; /* faster when present ... (?) */ continue; /* faster when present ... (?) */
} } }
}
/* Save reps for next block */ /* Save reps for next block */
ctx->repToConfirm[0] = offset_1 ? offset_1 : savedOffset; ctx->repToConfirm[0] = offset_1 ? offset_1 : savedOffset;
ctx->repToConfirm[1] = offset_2 ? offset_2 : savedOffset; ctx->repToConfirm[1] = offset_2 ? offset_2 : savedOffset;
/* Last Literals */ /* Last Literals */
{ size_t const lastLLSize = iend - anchor; {
size_t const lastLLSize = iend - anchor;
memcpy(seqStorePtr->lit, anchor, lastLLSize); memcpy(seqStorePtr->lit, anchor, lastLLSize);
seqStorePtr->lit += lastLLSize; seqStorePtr->lit += lastLLSize;
} }
} }
static void ZSTD_compressBlock_btlazy2(ZSTD_CCtx *ctx, const void *src, size_t srcSize) { ZSTD_compressBlock_lazy_generic(ctx, src, srcSize, 1, 2); }
static void ZSTD_compressBlock_btlazy2(ZSTD_CCtx* ctx, const void* src, size_t srcSize) static void ZSTD_compressBlock_lazy2(ZSTD_CCtx *ctx, const void *src, size_t srcSize) { ZSTD_compressBlock_lazy_generic(ctx, src, srcSize, 0, 2); }
{
ZSTD_compressBlock_lazy_generic(ctx, src, srcSize, 1, 2);
}
static void ZSTD_compressBlock_lazy2(ZSTD_CCtx* ctx, const void* src, size_t srcSize) static void ZSTD_compressBlock_lazy(ZSTD_CCtx *ctx, const void *src, size_t srcSize) { ZSTD_compressBlock_lazy_generic(ctx, src, srcSize, 0, 1); }
{
ZSTD_compressBlock_lazy_generic(ctx, src, srcSize, 0, 2);
}
static void ZSTD_compressBlock_lazy(ZSTD_CCtx* ctx, const void* src, size_t srcSize)
{
ZSTD_compressBlock_lazy_generic(ctx, src, srcSize, 0, 1);
}
static void ZSTD_compressBlock_greedy(ZSTD_CCtx* ctx, const void* src, size_t srcSize)
{
ZSTD_compressBlock_lazy_generic(ctx, src, srcSize, 0, 0);
}
static void ZSTD_compressBlock_greedy(ZSTD_CCtx *ctx, const void *src, size_t srcSize) { ZSTD_compressBlock_lazy_generic(ctx, src, srcSize, 0, 0); }
FORCE_INLINE FORCE_INLINE
void ZSTD_compressBlock_lazy_extDict_generic(ZSTD_CCtx* ctx, void ZSTD_compressBlock_lazy_extDict_generic(ZSTD_CCtx *ctx, const void *src, size_t srcSize, const U32 searchMethod, const U32 depth)
const void* src, size_t srcSize,
const U32 searchMethod, const U32 depth)
{ {
seqStore_t *seqStorePtr = &(ctx->seqStore); seqStore_t *seqStorePtr = &(ctx->seqStore);
const BYTE *const istart = (const BYTE *)src; const BYTE *const istart = (const BYTE *)src;
@@ -1978,9 +2053,7 @@ void ZSTD_compressBlock_lazy_extDict_generic(ZSTD_CCtx* ctx,
const U32 maxSearches = 1 << ctx->params.cParams.searchLog; const U32 maxSearches = 1 << ctx->params.cParams.searchLog;
const U32 mls = ctx->params.cParams.searchLength; const U32 mls = ctx->params.cParams.searchLength;
typedef size_t (*searchMax_f)(ZSTD_CCtx* zc, const BYTE* ip, const BYTE* iLimit, typedef size_t (*searchMax_f)(ZSTD_CCtx * zc, const BYTE *ip, const BYTE *iLimit, size_t *offsetPtr, U32 maxNbAttempts, U32 matchLengthSearch);
size_t* offsetPtr,
U32 maxNbAttempts, U32 matchLengthSearch);
searchMax_f searchMax = searchMethod ? ZSTD_BtFindBestMatch_selectMLS_extDict : ZSTD_HcFindBestMatch_extDict_selectMLS; searchMax_f searchMax = searchMethod ? ZSTD_BtFindBestMatch_selectMLS_extDict : ZSTD_HcFindBestMatch_extDict_selectMLS;
U32 offset_1 = ctx->rep[0], offset_2 = ctx->rep[1]; U32 offset_1 = ctx->rep[0], offset_2 = ctx->rep[1];
@@ -1997,19 +2070,24 @@ void ZSTD_compressBlock_lazy_extDict_generic(ZSTD_CCtx* ctx,
U32 curr = (U32)(ip - base); U32 curr = (U32)(ip - base);
/* check repCode */ /* check repCode */
{ const U32 repIndex = (U32)(curr+1 - offset_1); {
const U32 repIndex = (U32)(curr + 1 - offset_1);
const BYTE *const repBase = repIndex < dictLimit ? dictBase : base; const BYTE *const repBase = repIndex < dictLimit ? dictBase : base;
const BYTE *const repMatch = repBase + repIndex; const BYTE *const repMatch = repBase + repIndex;
if (((U32)((dictLimit - 1) - repIndex) >= 3) & (repIndex > lowestIndex)) /* intentional overflow */ if (((U32)((dictLimit - 1) - repIndex) >= 3) & (repIndex > lowestIndex)) /* intentional overflow */
if (ZSTD_read32(ip + 1) == ZSTD_read32(repMatch)) { if (ZSTD_read32(ip + 1) == ZSTD_read32(repMatch)) {
/* repcode detected we should take it */ /* repcode detected we should take it */
const BYTE *const repEnd = repIndex < dictLimit ? dictEnd : iend; const BYTE *const repEnd = repIndex < dictLimit ? dictEnd : iend;
matchLength = ZSTD_count_2segments(ip+1+EQUAL_READ32, repMatch+EQUAL_READ32, iend, repEnd, prefixStart) + EQUAL_READ32; matchLength =
if (depth==0) goto _storeSequence; ZSTD_count_2segments(ip + 1 + EQUAL_READ32, repMatch + EQUAL_READ32, iend, repEnd, prefixStart) + EQUAL_READ32;
} } if (depth == 0)
goto _storeSequence;
}
}
/* first search (depth 0) */ /* first search (depth 0) */
{ size_t offsetFound = 99999999; {
size_t offsetFound = 99999999;
size_t const ml2 = searchMax(ctx, ip, iend, &offsetFound, maxSearches, mls); size_t const ml2 = searchMax(ctx, ip, iend, &offsetFound, maxSearches, mls);
if (ml2 > matchLength) if (ml2 > matchLength)
matchLength = ml2, start = ip, offset = offsetFound; matchLength = ml2, start = ip, offset = offsetFound;
@@ -2034,22 +2112,27 @@ void ZSTD_compressBlock_lazy_extDict_generic(ZSTD_CCtx* ctx,
if (ZSTD_read32(ip) == ZSTD_read32(repMatch)) { if (ZSTD_read32(ip) == ZSTD_read32(repMatch)) {
/* repcode detected */ /* repcode detected */
const BYTE *const repEnd = repIndex < dictLimit ? dictEnd : iend; const BYTE *const repEnd = repIndex < dictLimit ? dictEnd : iend;
size_t const repLength = ZSTD_count_2segments(ip+EQUAL_READ32, repMatch+EQUAL_READ32, iend, repEnd, prefixStart) + EQUAL_READ32; size_t const repLength =
ZSTD_count_2segments(ip + EQUAL_READ32, repMatch + EQUAL_READ32, iend, repEnd, prefixStart) +
EQUAL_READ32;
int const gain2 = (int)(repLength * 3); int const gain2 = (int)(repLength * 3);
int const gain1 = (int)(matchLength * 3 - ZSTD_highbit32((U32)offset + 1) + 1); int const gain1 = (int)(matchLength * 3 - ZSTD_highbit32((U32)offset + 1) + 1);
if ((repLength >= EQUAL_READ32) && (gain2 > gain1)) if ((repLength >= EQUAL_READ32) && (gain2 > gain1))
matchLength = repLength, offset = 0, start = ip; matchLength = repLength, offset = 0, start = ip;
} } }
}
/* search match, depth 1 */ /* search match, depth 1 */
{ size_t offset2=99999999; {
size_t offset2 = 99999999;
size_t const ml2 = searchMax(ctx, ip, iend, &offset2, maxSearches, mls); size_t const ml2 = searchMax(ctx, ip, iend, &offset2, maxSearches, mls);
int const gain2 = (int)(ml2 * 4 - ZSTD_highbit32((U32)offset2 + 1)); /* raw approx */ int const gain2 = (int)(ml2 * 4 - ZSTD_highbit32((U32)offset2 + 1)); /* raw approx */
int const gain1 = (int)(matchLength * 4 - ZSTD_highbit32((U32)offset + 1) + 4); int const gain1 = (int)(matchLength * 4 - ZSTD_highbit32((U32)offset + 1) + 4);
if ((ml2 >= EQUAL_READ32) && (gain2 > gain1)) { if ((ml2 >= EQUAL_READ32) && (gain2 > gain1)) {
matchLength = ml2, offset = offset2, start = ip; matchLength = ml2, offset = offset2, start = ip;
continue; /* search a better one */ continue; /* search a better one */
} } }
}
/* let's find an even better one */ /* let's find an even better one */
if ((depth == 2) && (ip < ilimit)) { if ((depth == 2) && (ip < ilimit)) {
@@ -2064,22 +2147,28 @@ void ZSTD_compressBlock_lazy_extDict_generic(ZSTD_CCtx* ctx,
if (ZSTD_read32(ip) == ZSTD_read32(repMatch)) { if (ZSTD_read32(ip) == ZSTD_read32(repMatch)) {
/* repcode detected */ /* repcode detected */
const BYTE *const repEnd = repIndex < dictLimit ? dictEnd : iend; const BYTE *const repEnd = repIndex < dictLimit ? dictEnd : iend;
size_t repLength = ZSTD_count_2segments(ip+EQUAL_READ32, repMatch+EQUAL_READ32, iend, repEnd, prefixStart) + EQUAL_READ32; size_t repLength = ZSTD_count_2segments(ip + EQUAL_READ32, repMatch + EQUAL_READ32, iend,
repEnd, prefixStart) +
EQUAL_READ32;
int gain2 = (int)(repLength * 4); int gain2 = (int)(repLength * 4);
int gain1 = (int)(matchLength * 4 - ZSTD_highbit32((U32)offset + 1) + 1); int gain1 = (int)(matchLength * 4 - ZSTD_highbit32((U32)offset + 1) + 1);
if ((repLength >= EQUAL_READ32) && (gain2 > gain1)) if ((repLength >= EQUAL_READ32) && (gain2 > gain1))
matchLength = repLength, offset = 0, start = ip; matchLength = repLength, offset = 0, start = ip;
} } }
}
/* search match, depth 2 */ /* search match, depth 2 */
{ size_t offset2=99999999; {
size_t offset2 = 99999999;
size_t const ml2 = searchMax(ctx, ip, iend, &offset2, maxSearches, mls); size_t const ml2 = searchMax(ctx, ip, iend, &offset2, maxSearches, mls);
int const gain2 = (int)(ml2 * 4 - ZSTD_highbit32((U32)offset2 + 1)); /* raw approx */ int const gain2 = (int)(ml2 * 4 - ZSTD_highbit32((U32)offset2 + 1)); /* raw approx */
int const gain1 = (int)(matchLength * 4 - ZSTD_highbit32((U32)offset + 1) + 7); int const gain1 = (int)(matchLength * 4 - ZSTD_highbit32((U32)offset + 1) + 7);
if ((ml2 >= EQUAL_READ32) && (gain2 > gain1)) { if ((ml2 >= EQUAL_READ32) && (gain2 > gain1)) {
matchLength = ml2, offset = offset2, start = ip; matchLength = ml2, offset = offset2, start = ip;
continue; continue;
} } } }
}
}
break; /* nothing found : store previous solution */ break; /* nothing found : store previous solution */
} }
@@ -2088,13 +2177,18 @@ void ZSTD_compressBlock_lazy_extDict_generic(ZSTD_CCtx* ctx,
U32 const matchIndex = (U32)((start - base) - (offset - ZSTD_REP_MOVE)); U32 const matchIndex = (U32)((start - base) - (offset - ZSTD_REP_MOVE));
const BYTE *match = (matchIndex < dictLimit) ? dictBase + matchIndex : base + matchIndex; const BYTE *match = (matchIndex < dictLimit) ? dictBase + matchIndex : base + matchIndex;
const BYTE *const mStart = (matchIndex < dictLimit) ? dictStart : prefixStart; const BYTE *const mStart = (matchIndex < dictLimit) ? dictStart : prefixStart;
while ((start>anchor) && (match>mStart) && (start[-1] == match[-1])) { start--; match--; matchLength++; } /* catch up */ while ((start > anchor) && (match > mStart) && (start[-1] == match[-1])) {
offset_2 = offset_1; offset_1 = (U32)(offset - ZSTD_REP_MOVE); start--;
match--;
matchLength++;
} /* catch up */
offset_2 = offset_1;
offset_1 = (U32)(offset - ZSTD_REP_MOVE);
} }
/* store sequence */ /* store sequence */
_storeSequence: _storeSequence : {
{ size_t const litLength = start - anchor; size_t const litLength = start - anchor;
ZSTD_storeSeq(seqStorePtr, litLength, anchor, (U32)offset, matchLength - MINMATCH); ZSTD_storeSeq(seqStorePtr, litLength, anchor, (U32)offset, matchLength - MINMATCH);
anchor = ip = start + matchLength; anchor = ip = start + matchLength;
} }
@@ -2108,31 +2202,33 @@ _storeSequence:
if (ZSTD_read32(ip) == ZSTD_read32(repMatch)) { if (ZSTD_read32(ip) == ZSTD_read32(repMatch)) {
/* repcode detected we should take it */ /* repcode detected we should take it */
const BYTE *const repEnd = repIndex < dictLimit ? dictEnd : iend; const BYTE *const repEnd = repIndex < dictLimit ? dictEnd : iend;
matchLength = ZSTD_count_2segments(ip+EQUAL_READ32, repMatch+EQUAL_READ32, iend, repEnd, prefixStart) + EQUAL_READ32; matchLength =
offset = offset_2; offset_2 = offset_1; offset_1 = (U32)offset; /* swap offset history */ ZSTD_count_2segments(ip + EQUAL_READ32, repMatch + EQUAL_READ32, iend, repEnd, prefixStart) + EQUAL_READ32;
offset = offset_2;
offset_2 = offset_1;
offset_1 = (U32)offset; /* swap offset history */
ZSTD_storeSeq(seqStorePtr, 0, anchor, 0, matchLength - MINMATCH); ZSTD_storeSeq(seqStorePtr, 0, anchor, 0, matchLength - MINMATCH);
ip += matchLength; ip += matchLength;
anchor = ip; anchor = ip;
continue; /* faster when present ... (?) */ continue; /* faster when present ... (?) */
} }
break; break;
} } }
}
/* Save reps for next block */ /* Save reps for next block */
ctx->repToConfirm[0] = offset_1; ctx->repToConfirm[1] = offset_2; ctx->repToConfirm[0] = offset_1;
ctx->repToConfirm[1] = offset_2;
/* Last Literals */ /* Last Literals */
{ size_t const lastLLSize = iend - anchor; {
size_t const lastLLSize = iend - anchor;
memcpy(seqStorePtr->lit, anchor, lastLLSize); memcpy(seqStorePtr->lit, anchor, lastLLSize);
seqStorePtr->lit += lastLLSize; seqStorePtr->lit += lastLLSize;
} }
} }
void ZSTD_compressBlock_greedy_extDict(ZSTD_CCtx *ctx, const void *src, size_t srcSize) { ZSTD_compressBlock_lazy_extDict_generic(ctx, src, srcSize, 0, 0); }
void ZSTD_compressBlock_greedy_extDict(ZSTD_CCtx* ctx, const void* src, size_t srcSize)
{
ZSTD_compressBlock_lazy_extDict_generic(ctx, src, srcSize, 0, 0);
}
static void ZSTD_compressBlock_lazy_extDict(ZSTD_CCtx *ctx, const void *src, size_t srcSize) static void ZSTD_compressBlock_lazy_extDict(ZSTD_CCtx *ctx, const void *src, size_t srcSize)
{ {
@@ -2149,7 +2245,6 @@ static void ZSTD_compressBlock_btlazy2_extDict(ZSTD_CCtx* ctx, const void* src,
ZSTD_compressBlock_lazy_extDict_generic(ctx, src, srcSize, 1, 2); ZSTD_compressBlock_lazy_extDict_generic(ctx, src, srcSize, 1, 2);
} }
/* The optimal parser */ /* The optimal parser */
#include "zstd_opt.h" #include "zstd_opt.h"
@@ -2158,7 +2253,9 @@ static void ZSTD_compressBlock_btopt(ZSTD_CCtx* ctx, const void* src, size_t src
#ifdef ZSTD_OPT_H_91842398743 #ifdef ZSTD_OPT_H_91842398743
ZSTD_compressBlock_opt_generic(ctx, src, srcSize, 0); ZSTD_compressBlock_opt_generic(ctx, src, srcSize, 0);
#else #else
(void)ctx; (void)src; (void)srcSize; (void)ctx;
(void)src;
(void)srcSize;
return; return;
#endif #endif
} }
@@ -2168,7 +2265,9 @@ static void ZSTD_compressBlock_btopt2(ZSTD_CCtx* ctx, const void* src, size_t sr
#ifdef ZSTD_OPT_H_91842398743 #ifdef ZSTD_OPT_H_91842398743
ZSTD_compressBlock_opt_generic(ctx, src, srcSize, 1); ZSTD_compressBlock_opt_generic(ctx, src, srcSize, 1);
#else #else
(void)ctx; (void)src; (void)srcSize; (void)ctx;
(void)src;
(void)srcSize;
return; return;
#endif #endif
} }
@@ -2178,7 +2277,9 @@ static void ZSTD_compressBlock_btopt_extDict(ZSTD_CCtx* ctx, const void* src, si
#ifdef ZSTD_OPT_H_91842398743 #ifdef ZSTD_OPT_H_91842398743
ZSTD_compressBlock_opt_extDict_generic(ctx, src, srcSize, 0); ZSTD_compressBlock_opt_extDict_generic(ctx, src, srcSize, 0);
#else #else
(void)ctx; (void)src; (void)srcSize; (void)ctx;
(void)src;
(void)srcSize;
return; return;
#endif #endif
} }
@@ -2188,32 +2289,34 @@ static void ZSTD_compressBlock_btopt2_extDict(ZSTD_CCtx* ctx, const void* src, s
#ifdef ZSTD_OPT_H_91842398743 #ifdef ZSTD_OPT_H_91842398743
ZSTD_compressBlock_opt_extDict_generic(ctx, src, srcSize, 1); ZSTD_compressBlock_opt_extDict_generic(ctx, src, srcSize, 1);
#else #else
(void)ctx; (void)src; (void)srcSize; (void)ctx;
(void)src;
(void)srcSize;
return; return;
#endif #endif
} }
typedef void (*ZSTD_blockCompressor)(ZSTD_CCtx *ctx, const void *src, size_t srcSize); typedef void (*ZSTD_blockCompressor)(ZSTD_CCtx *ctx, const void *src, size_t srcSize);
static ZSTD_blockCompressor ZSTD_selectBlockCompressor(ZSTD_strategy strat, int extDict) static ZSTD_blockCompressor ZSTD_selectBlockCompressor(ZSTD_strategy strat, int extDict)
{ {
static const ZSTD_blockCompressor blockCompressor[2][8] = { static const ZSTD_blockCompressor blockCompressor[2][8] = {
{ ZSTD_compressBlock_fast, ZSTD_compressBlock_doubleFast, ZSTD_compressBlock_greedy, ZSTD_compressBlock_lazy, ZSTD_compressBlock_lazy2, ZSTD_compressBlock_btlazy2, ZSTD_compressBlock_btopt, ZSTD_compressBlock_btopt2 }, {ZSTD_compressBlock_fast, ZSTD_compressBlock_doubleFast, ZSTD_compressBlock_greedy, ZSTD_compressBlock_lazy, ZSTD_compressBlock_lazy2,
{ ZSTD_compressBlock_fast_extDict, ZSTD_compressBlock_doubleFast_extDict, ZSTD_compressBlock_greedy_extDict, ZSTD_compressBlock_lazy_extDict,ZSTD_compressBlock_lazy2_extDict, ZSTD_compressBlock_btlazy2_extDict, ZSTD_compressBlock_btopt_extDict, ZSTD_compressBlock_btopt2_extDict } ZSTD_compressBlock_btlazy2, ZSTD_compressBlock_btopt, ZSTD_compressBlock_btopt2},
}; {ZSTD_compressBlock_fast_extDict, ZSTD_compressBlock_doubleFast_extDict, ZSTD_compressBlock_greedy_extDict, ZSTD_compressBlock_lazy_extDict,
ZSTD_compressBlock_lazy2_extDict, ZSTD_compressBlock_btlazy2_extDict, ZSTD_compressBlock_btopt_extDict, ZSTD_compressBlock_btopt2_extDict}};
return blockCompressor[extDict][(U32)strat]; return blockCompressor[extDict][(U32)strat];
} }
static size_t ZSTD_compressBlock_internal(ZSTD_CCtx *zc, void *dst, size_t dstCapacity, const void *src, size_t srcSize) static size_t ZSTD_compressBlock_internal(ZSTD_CCtx *zc, void *dst, size_t dstCapacity, const void *src, size_t srcSize)
{ {
ZSTD_blockCompressor const blockCompressor = ZSTD_selectBlockCompressor(zc->params.cParams.strategy, zc->lowLimit < zc->dictLimit); ZSTD_blockCompressor const blockCompressor = ZSTD_selectBlockCompressor(zc->params.cParams.strategy, zc->lowLimit < zc->dictLimit);
const BYTE *const base = zc->base; const BYTE *const base = zc->base;
const BYTE *const istart = (const BYTE *)src; const BYTE *const istart = (const BYTE *)src;
const U32 curr = (U32)(istart - base); const U32 curr = (U32)(istart - base);
if (srcSize < MIN_CBLOCK_SIZE+ZSTD_blockHeaderSize+1) return 0; /* don't even attempt compression below a certain srcSize */ if (srcSize < MIN_CBLOCK_SIZE + ZSTD_blockHeaderSize + 1)
return 0; /* don't even attempt compression below a certain srcSize */
ZSTD_resetSeqStore(&(zc->seqStore)); ZSTD_resetSeqStore(&(zc->seqStore));
if (curr > zc->nextToUpdate + 384) if (curr > zc->nextToUpdate + 384)
zc->nextToUpdate = curr - MIN(192, (U32)(curr - zc->nextToUpdate - 384)); /* update tree not updated after finding very long rep matches */ zc->nextToUpdate = curr - MIN(192, (U32)(curr - zc->nextToUpdate - 384)); /* update tree not updated after finding very long rep matches */
@@ -2221,7 +2324,6 @@ static size_t ZSTD_compressBlock_internal(ZSTD_CCtx* zc, void* dst, size_t dstCa
return ZSTD_compressSequences(zc, dst, dstCapacity, srcSize); return ZSTD_compressSequences(zc, dst, dstCapacity, srcSize);
} }
/*! ZSTD_compress_generic() : /*! ZSTD_compress_generic() :
* Compress a chunk of data into one or multiple blocks. * Compress a chunk of data into one or multiple blocks.
* All blocks will be terminated, all input will be consumed. * All blocks will be terminated, all input will be consumed.
@@ -2229,10 +2331,7 @@ static size_t ZSTD_compressBlock_internal(ZSTD_CCtx* zc, void* dst, size_t dstCa
* Frame is supposed already started (header already produced) * Frame is supposed already started (header already produced)
* @return : compressed size, or an error code * @return : compressed size, or an error code
*/ */
static size_t ZSTD_compress_generic (ZSTD_CCtx* cctx, static size_t ZSTD_compress_generic(ZSTD_CCtx *cctx, void *dst, size_t dstCapacity, const void *src, size_t srcSize, U32 lastFrameChunk)
void* dst, size_t dstCapacity,
const void* src, size_t srcSize,
U32 lastFrameChunk)
{ {
size_t blockSize = cctx->blockSize; size_t blockSize = cctx->blockSize;
size_t remaining = srcSize; size_t remaining = srcSize;
@@ -2248,8 +2347,10 @@ static size_t ZSTD_compress_generic (ZSTD_CCtx* cctx,
U32 const lastBlock = lastFrameChunk & (blockSize >= remaining); U32 const lastBlock = lastFrameChunk & (blockSize >= remaining);
size_t cSize; size_t cSize;
if (dstCapacity < ZSTD_blockHeaderSize + MIN_CBLOCK_SIZE) return ERROR(dstSize_tooSmall); /* not enough space to store compressed block */ if (dstCapacity < ZSTD_blockHeaderSize + MIN_CBLOCK_SIZE)
if (remaining < blockSize) blockSize = remaining; return ERROR(dstSize_tooSmall); /* not enough space to store compressed block */
if (remaining < blockSize)
blockSize = remaining;
/* preemptive overflow correction */ /* preemptive overflow correction */
if (cctx->lowLimit > (3U << 29)) { if (cctx->lowLimit > (3U << 29)) {
@@ -2263,23 +2364,29 @@ static size_t ZSTD_compress_generic (ZSTD_CCtx* cctx,
cctx->dictBase += correction; cctx->dictBase += correction;
cctx->lowLimit -= correction; cctx->lowLimit -= correction;
cctx->dictLimit -= correction; cctx->dictLimit -= correction;
if (cctx->nextToUpdate < correction) cctx->nextToUpdate = 0; if (cctx->nextToUpdate < correction)
else cctx->nextToUpdate -= correction; cctx->nextToUpdate = 0;
else
cctx->nextToUpdate -= correction;
} }
if ((U32)(ip + blockSize - cctx->base) > cctx->loadedDictEnd + maxDist) { if ((U32)(ip + blockSize - cctx->base) > cctx->loadedDictEnd + maxDist) {
/* enforce maxDist */ /* enforce maxDist */
U32 const newLowLimit = (U32)(ip + blockSize - cctx->base) - maxDist; U32 const newLowLimit = (U32)(ip + blockSize - cctx->base) - maxDist;
if (cctx->lowLimit < newLowLimit) cctx->lowLimit = newLowLimit; if (cctx->lowLimit < newLowLimit)
if (cctx->dictLimit < cctx->lowLimit) cctx->dictLimit = cctx->lowLimit; cctx->lowLimit = newLowLimit;
if (cctx->dictLimit < cctx->lowLimit)
cctx->dictLimit = cctx->lowLimit;
} }
cSize = ZSTD_compressBlock_internal(cctx, op + ZSTD_blockHeaderSize, dstCapacity - ZSTD_blockHeaderSize, ip, blockSize); cSize = ZSTD_compressBlock_internal(cctx, op + ZSTD_blockHeaderSize, dstCapacity - ZSTD_blockHeaderSize, ip, blockSize);
if (ZSTD_isError(cSize)) return cSize; if (ZSTD_isError(cSize))
return cSize;
if (cSize == 0) { /* block is not compressible */ if (cSize == 0) { /* block is not compressible */
U32 const cBlockHeader24 = lastBlock + (((U32)bt_raw) << 1) + (U32)(blockSize << 3); U32 const cBlockHeader24 = lastBlock + (((U32)bt_raw) << 1) + (U32)(blockSize << 3);
if (blockSize + ZSTD_blockHeaderSize > dstCapacity) return ERROR(dstSize_tooSmall); if (blockSize + ZSTD_blockHeaderSize > dstCapacity)
return ERROR(dstSize_tooSmall);
ZSTD_writeLE32(op, cBlockHeader24); /* no pb, 4th byte will be overwritten */ ZSTD_writeLE32(op, cBlockHeader24); /* no pb, 4th byte will be overwritten */
memcpy(op + ZSTD_blockHeaderSize, ip, blockSize); memcpy(op + ZSTD_blockHeaderSize, ip, blockSize);
cSize = ZSTD_blockHeaderSize + blockSize; cSize = ZSTD_blockHeaderSize + blockSize;
@@ -2295,63 +2402,83 @@ static size_t ZSTD_compress_generic (ZSTD_CCtx* cctx,
op += cSize; op += cSize;
} }
if (lastFrameChunk && (op>ostart)) cctx->stage = ZSTDcs_ending; if (lastFrameChunk && (op > ostart))
cctx->stage = ZSTDcs_ending;
return op - ostart; return op - ostart;
} }
static size_t ZSTD_writeFrameHeader(void *dst, size_t dstCapacity, ZSTD_parameters params, U64 pledgedSrcSize, U32 dictID)
static size_t ZSTD_writeFrameHeader(void* dst, size_t dstCapacity, {
ZSTD_parameters params, U64 pledgedSrcSize, U32 dictID) BYTE *const op = (BYTE *)dst;
{ BYTE* const op = (BYTE*)dst;
U32 const dictIDSizeCode = (dictID > 0) + (dictID >= 256) + (dictID >= 65536); /* 0-3 */ U32 const dictIDSizeCode = (dictID > 0) + (dictID >= 256) + (dictID >= 65536); /* 0-3 */
U32 const checksumFlag = params.fParams.checksumFlag > 0; U32 const checksumFlag = params.fParams.checksumFlag > 0;
U32 const windowSize = 1U << params.cParams.windowLog; U32 const windowSize = 1U << params.cParams.windowLog;
U32 const singleSegment = params.fParams.contentSizeFlag && (windowSize >= pledgedSrcSize); U32 const singleSegment = params.fParams.contentSizeFlag && (windowSize >= pledgedSrcSize);
BYTE const windowLogByte = (BYTE)((params.cParams.windowLog - ZSTD_WINDOWLOG_ABSOLUTEMIN) << 3); BYTE const windowLogByte = (BYTE)((params.cParams.windowLog - ZSTD_WINDOWLOG_ABSOLUTEMIN) << 3);
U32 const fcsCode = params.fParams.contentSizeFlag ? U32 const fcsCode = params.fParams.contentSizeFlag ? (pledgedSrcSize >= 256) + (pledgedSrcSize >= 65536 + 256) + (pledgedSrcSize >= 0xFFFFFFFFU)
(pledgedSrcSize>=256) + (pledgedSrcSize>=65536+256) + (pledgedSrcSize>=0xFFFFFFFFU) : /* 0-3 */ : /* 0-3 */
0; 0;
BYTE const frameHeaderDecriptionByte = (BYTE)(dictIDSizeCode + (checksumFlag << 2) + (singleSegment << 5) + (fcsCode << 6)); BYTE const frameHeaderDecriptionByte = (BYTE)(dictIDSizeCode + (checksumFlag << 2) + (singleSegment << 5) + (fcsCode << 6));
size_t pos; size_t pos;
if (dstCapacity < ZSTD_frameHeaderSize_max) return ERROR(dstSize_tooSmall); if (dstCapacity < ZSTD_frameHeaderSize_max)
return ERROR(dstSize_tooSmall);
ZSTD_writeLE32(dst, ZSTD_MAGICNUMBER); ZSTD_writeLE32(dst, ZSTD_MAGICNUMBER);
op[4] = frameHeaderDecriptionByte; pos=5; op[4] = frameHeaderDecriptionByte;
if (!singleSegment) op[pos++] = windowLogByte; pos = 5;
switch(dictIDSizeCode) if (!singleSegment)
{ op[pos++] = windowLogByte;
switch (dictIDSizeCode) {
default: /* impossible */ default: /* impossible */
case 0: break; case 0: break;
case 1 : op[pos] = (BYTE)(dictID); pos++; break; case 1:
case 2 : ZSTD_writeLE16(op+pos, (U16)dictID); pos+=2; break; op[pos] = (BYTE)(dictID);
case 3 : ZSTD_writeLE32(op+pos, dictID); pos+=4; break; pos++;
break;
case 2:
ZSTD_writeLE16(op + pos, (U16)dictID);
pos += 2;
break;
case 3:
ZSTD_writeLE32(op + pos, dictID);
pos += 4;
break;
} }
switch(fcsCode) switch (fcsCode) {
{
default: /* impossible */ default: /* impossible */
case 0 : if (singleSegment) op[pos++] = (BYTE)(pledgedSrcSize); break; case 0:
case 1 : ZSTD_writeLE16(op+pos, (U16)(pledgedSrcSize-256)); pos+=2; break; if (singleSegment)
case 2 : ZSTD_writeLE32(op+pos, (U32)(pledgedSrcSize)); pos+=4; break; op[pos++] = (BYTE)(pledgedSrcSize);
case 3 : ZSTD_writeLE64(op+pos, (U64)(pledgedSrcSize)); pos+=8; break; break;
case 1:
ZSTD_writeLE16(op + pos, (U16)(pledgedSrcSize - 256));
pos += 2;
break;
case 2:
ZSTD_writeLE32(op + pos, (U32)(pledgedSrcSize));
pos += 4;
break;
case 3:
ZSTD_writeLE64(op + pos, (U64)(pledgedSrcSize));
pos += 8;
break;
} }
return pos; return pos;
} }
static size_t ZSTD_compressContinue_internal(ZSTD_CCtx *cctx, void *dst, size_t dstCapacity, const void *src, size_t srcSize, U32 frame, U32 lastFrameChunk)
static size_t ZSTD_compressContinue_internal (ZSTD_CCtx* cctx,
void* dst, size_t dstCapacity,
const void* src, size_t srcSize,
U32 frame, U32 lastFrameChunk)
{ {
const BYTE *const ip = (const BYTE *)src; const BYTE *const ip = (const BYTE *)src;
size_t fhSize = 0; size_t fhSize = 0;
if (cctx->stage==ZSTDcs_created) return ERROR(stage_wrong); /* missing init (ZSTD_compressBegin) */ if (cctx->stage == ZSTDcs_created)
return ERROR(stage_wrong); /* missing init (ZSTD_compressBegin) */
if (frame && (cctx->stage == ZSTDcs_init)) { if (frame && (cctx->stage == ZSTDcs_init)) {
fhSize = ZSTD_writeFrameHeader(dst, dstCapacity, cctx->params, cctx->frameContentSize, cctx->dictID); fhSize = ZSTD_writeFrameHeader(dst, dstCapacity, cctx->params, cctx->frameContentSize, cctx->dictID);
if (ZSTD_isError(fhSize)) return fhSize; if (ZSTD_isError(fhSize))
return fhSize;
dstCapacity -= fhSize; dstCapacity -= fhSize;
dst = (char *)dst + fhSize; dst = (char *)dst + fhSize;
cctx->stage = ZSTDcs_ongoing; cctx->stage = ZSTDcs_ongoing;
@@ -2366,7 +2493,8 @@ static size_t ZSTD_compressContinue_internal (ZSTD_CCtx* cctx,
cctx->dictBase = cctx->base; cctx->dictBase = cctx->base;
cctx->base -= delta; cctx->base -= delta;
cctx->nextToUpdate = cctx->dictLimit; cctx->nextToUpdate = cctx->dictLimit;
if (cctx->dictLimit - cctx->lowLimit < HASH_READ_SIZE) cctx->lowLimit = cctx->dictLimit; /* too small extDict */ if (cctx->dictLimit - cctx->lowLimit < HASH_READ_SIZE)
cctx->lowLimit = cctx->dictLimit; /* too small extDict */
} }
/* if input and dictionary overlap : reduce dictionary (area presumed modified by input) */ /* if input and dictionary overlap : reduce dictionary (area presumed modified by input) */
@@ -2379,33 +2507,27 @@ static size_t ZSTD_compressContinue_internal (ZSTD_CCtx* cctx,
cctx->nextSrc = ip + srcSize; cctx->nextSrc = ip + srcSize;
if (srcSize) { if (srcSize) {
size_t const cSize = frame ? size_t const cSize = frame ? ZSTD_compress_generic(cctx, dst, dstCapacity, src, srcSize, lastFrameChunk)
ZSTD_compress_generic (cctx, dst, dstCapacity, src, srcSize, lastFrameChunk) : : ZSTD_compressBlock_internal(cctx, dst, dstCapacity, src, srcSize);
ZSTD_compressBlock_internal (cctx, dst, dstCapacity, src, srcSize); if (ZSTD_isError(cSize))
if (ZSTD_isError(cSize)) return cSize; return cSize;
return cSize + fhSize; return cSize + fhSize;
} else } else
return fhSize; return fhSize;
} }
size_t ZSTD_compressContinue(ZSTD_CCtx *cctx, void *dst, size_t dstCapacity, const void *src, size_t srcSize)
size_t ZSTD_compressContinue (ZSTD_CCtx* cctx,
void* dst, size_t dstCapacity,
const void* src, size_t srcSize)
{ {
return ZSTD_compressContinue_internal(cctx, dst, dstCapacity, src, srcSize, 1, 0); return ZSTD_compressContinue_internal(cctx, dst, dstCapacity, src, srcSize, 1, 0);
} }
size_t ZSTD_getBlockSizeMax(ZSTD_CCtx *cctx) { return MIN(ZSTD_BLOCKSIZE_ABSOLUTEMAX, 1 << cctx->params.cParams.windowLog); }
size_t ZSTD_getBlockSizeMax(ZSTD_CCtx* cctx)
{
return MIN (ZSTD_BLOCKSIZE_ABSOLUTEMAX, 1 << cctx->params.cParams.windowLog);
}
size_t ZSTD_compressBlock(ZSTD_CCtx *cctx, void *dst, size_t dstCapacity, const void *src, size_t srcSize) size_t ZSTD_compressBlock(ZSTD_CCtx *cctx, void *dst, size_t dstCapacity, const void *src, size_t srcSize)
{ {
size_t const blockSizeMax = ZSTD_getBlockSizeMax(cctx); size_t const blockSizeMax = ZSTD_getBlockSizeMax(cctx);
if (srcSize > blockSizeMax) return ERROR(srcSize_wrong); if (srcSize > blockSizeMax)
return ERROR(srcSize_wrong);
return ZSTD_compressContinue_internal(cctx, dst, dstCapacity, src, srcSize, 0, 0); return ZSTD_compressContinue_internal(cctx, dst, dstCapacity, src, srcSize, 0, 0);
} }
@@ -2426,17 +2548,13 @@ static size_t ZSTD_loadDictionaryContent(ZSTD_CCtx* zc, const void* src, size_t
zc->loadedDictEnd = zc->forceWindow ? 0 : (U32)(iend - zc->base); zc->loadedDictEnd = zc->forceWindow ? 0 : (U32)(iend - zc->base);
zc->nextSrc = iend; zc->nextSrc = iend;
if (srcSize <= HASH_READ_SIZE) return 0; if (srcSize <= HASH_READ_SIZE)
return 0;
switch(zc->params.cParams.strategy) switch (zc->params.cParams.strategy) {
{ case ZSTD_fast: ZSTD_fillHashTable(zc, iend, zc->params.cParams.searchLength); break;
case ZSTD_fast:
ZSTD_fillHashTable (zc, iend, zc->params.cParams.searchLength);
break;
case ZSTD_dfast: case ZSTD_dfast: ZSTD_fillDoubleHashTable(zc, iend, zc->params.cParams.searchLength); break;
ZSTD_fillDoubleHashTable (zc, iend, zc->params.cParams.searchLength);
break;
case ZSTD_greedy: case ZSTD_greedy:
case ZSTD_lazy: case ZSTD_lazy:
@@ -2460,21 +2578,22 @@ static size_t ZSTD_loadDictionaryContent(ZSTD_CCtx* zc, const void* src, size_t
return 0; return 0;
} }
/* Dictionaries that assign zero probability to symbols that show up causes problems /* Dictionaries that assign zero probability to symbols that show up causes problems
when FSE encoding. Refuse dictionaries that assign zero probability to symbols when FSE encoding. Refuse dictionaries that assign zero probability to symbols
that we may encounter during compression. that we may encounter during compression.
NOTE: This behavior is not standard and could be improved in the future. */ NOTE: This behavior is not standard and could be improved in the future. */
static size_t ZSTD_checkDictNCount(short* normalizedCounter, unsigned dictMaxSymbolValue, unsigned maxSymbolValue) { static size_t ZSTD_checkDictNCount(short *normalizedCounter, unsigned dictMaxSymbolValue, unsigned maxSymbolValue)
{
U32 s; U32 s;
if (dictMaxSymbolValue < maxSymbolValue) return ERROR(dictionary_corrupted); if (dictMaxSymbolValue < maxSymbolValue)
return ERROR(dictionary_corrupted);
for (s = 0; s <= maxSymbolValue; ++s) { for (s = 0; s <= maxSymbolValue; ++s) {
if (normalizedCounter[s] == 0) return ERROR(dictionary_corrupted); if (normalizedCounter[s] == 0)
return ERROR(dictionary_corrupted);
} }
return 0; return 0;
} }
/* Dictionary format : /* Dictionary format :
* See : * See :
* https://github.com/facebook/zstd/blob/master/doc/zstd_compression_format.md#dictionary-format * https://github.com/facebook/zstd/blob/master/doc/zstd_compression_format.md#dictionary-format
@@ -2496,49 +2615,66 @@ static size_t ZSTD_loadZstdDictionary(ZSTD_CCtx* cctx, const void* dict, size_t
cctx->dictID = cctx->params.fParams.noDictIDFlag ? 0 : ZSTD_readLE32(dictPtr); cctx->dictID = cctx->params.fParams.noDictIDFlag ? 0 : ZSTD_readLE32(dictPtr);
dictPtr += 4; dictPtr += 4;
{ size_t const hufHeaderSize = HUF_readCTable(cctx->hufTable, 255, dictPtr, dictEnd-dictPtr); {
if (HUF_isError(hufHeaderSize)) return ERROR(dictionary_corrupted); size_t const hufHeaderSize = HUF_readCTable(cctx->hufTable, 255, dictPtr, dictEnd - dictPtr);
if (HUF_isError(hufHeaderSize))
return ERROR(dictionary_corrupted);
dictPtr += hufHeaderSize; dictPtr += hufHeaderSize;
} }
{ unsigned offcodeLog; {
unsigned offcodeLog;
size_t const offcodeHeaderSize = FSE_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dictPtr, dictEnd - dictPtr); size_t const offcodeHeaderSize = FSE_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dictPtr, dictEnd - dictPtr);
if (FSE_isError(offcodeHeaderSize)) return ERROR(dictionary_corrupted); if (FSE_isError(offcodeHeaderSize))
if (offcodeLog > OffFSELog) return ERROR(dictionary_corrupted); return ERROR(dictionary_corrupted);
if (offcodeLog > OffFSELog)
return ERROR(dictionary_corrupted);
/* Defer checking offcodeMaxValue because we need to know the size of the dictionary content */ /* Defer checking offcodeMaxValue because we need to know the size of the dictionary content */
CHECK_E (FSE_buildCTable_wksp(cctx->offcodeCTable, offcodeNCount, offcodeMaxValue, offcodeLog, scratchBuffer, sizeof(scratchBuffer)), dictionary_corrupted); CHECK_E(FSE_buildCTable_wksp(cctx->offcodeCTable, offcodeNCount, offcodeMaxValue, offcodeLog, scratchBuffer, sizeof(scratchBuffer)),
dictionary_corrupted);
dictPtr += offcodeHeaderSize; dictPtr += offcodeHeaderSize;
} }
{ short matchlengthNCount[MaxML+1]; {
short matchlengthNCount[MaxML + 1];
unsigned matchlengthMaxValue = MaxML, matchlengthLog; unsigned matchlengthMaxValue = MaxML, matchlengthLog;
size_t const matchlengthHeaderSize = FSE_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dictPtr, dictEnd - dictPtr); size_t const matchlengthHeaderSize = FSE_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dictPtr, dictEnd - dictPtr);
if (FSE_isError(matchlengthHeaderSize)) return ERROR(dictionary_corrupted); if (FSE_isError(matchlengthHeaderSize))
if (matchlengthLog > MLFSELog) return ERROR(dictionary_corrupted); return ERROR(dictionary_corrupted);
if (matchlengthLog > MLFSELog)
return ERROR(dictionary_corrupted);
/* Every match length code must have non-zero probability */ /* Every match length code must have non-zero probability */
CHECK_F(ZSTD_checkDictNCount(matchlengthNCount, matchlengthMaxValue, MaxML)); CHECK_F(ZSTD_checkDictNCount(matchlengthNCount, matchlengthMaxValue, MaxML));
CHECK_E (FSE_buildCTable_wksp(cctx->matchlengthCTable, matchlengthNCount, matchlengthMaxValue, matchlengthLog, scratchBuffer, sizeof(scratchBuffer)), dictionary_corrupted); CHECK_E(
FSE_buildCTable_wksp(cctx->matchlengthCTable, matchlengthNCount, matchlengthMaxValue, matchlengthLog, scratchBuffer, sizeof(scratchBuffer)),
dictionary_corrupted);
dictPtr += matchlengthHeaderSize; dictPtr += matchlengthHeaderSize;
} }
{ short litlengthNCount[MaxLL+1]; {
short litlengthNCount[MaxLL + 1];
unsigned litlengthMaxValue = MaxLL, litlengthLog; unsigned litlengthMaxValue = MaxLL, litlengthLog;
size_t const litlengthHeaderSize = FSE_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dictPtr, dictEnd - dictPtr); size_t const litlengthHeaderSize = FSE_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dictPtr, dictEnd - dictPtr);
if (FSE_isError(litlengthHeaderSize)) return ERROR(dictionary_corrupted); if (FSE_isError(litlengthHeaderSize))
if (litlengthLog > LLFSELog) return ERROR(dictionary_corrupted); return ERROR(dictionary_corrupted);
if (litlengthLog > LLFSELog)
return ERROR(dictionary_corrupted);
/* Every literal length code must have non-zero probability */ /* Every literal length code must have non-zero probability */
CHECK_F(ZSTD_checkDictNCount(litlengthNCount, litlengthMaxValue, MaxLL)); CHECK_F(ZSTD_checkDictNCount(litlengthNCount, litlengthMaxValue, MaxLL));
CHECK_E(FSE_buildCTable_wksp(cctx->litlengthCTable, litlengthNCount, litlengthMaxValue, litlengthLog, scratchBuffer, sizeof(scratchBuffer)), dictionary_corrupted); CHECK_E(FSE_buildCTable_wksp(cctx->litlengthCTable, litlengthNCount, litlengthMaxValue, litlengthLog, scratchBuffer, sizeof(scratchBuffer)),
dictionary_corrupted);
dictPtr += litlengthHeaderSize; dictPtr += litlengthHeaderSize;
} }
if (dictPtr+12 > dictEnd) return ERROR(dictionary_corrupted); if (dictPtr + 12 > dictEnd)
return ERROR(dictionary_corrupted);
cctx->rep[0] = ZSTD_readLE32(dictPtr + 0); cctx->rep[0] = ZSTD_readLE32(dictPtr + 0);
cctx->rep[1] = ZSTD_readLE32(dictPtr + 4); cctx->rep[1] = ZSTD_readLE32(dictPtr + 4);
cctx->rep[2] = ZSTD_readLE32(dictPtr + 8); cctx->rep[2] = ZSTD_readLE32(dictPtr + 8);
dictPtr += 12; dictPtr += 12;
{ size_t const dictContentSize = (size_t)(dictEnd - dictPtr); {
size_t const dictContentSize = (size_t)(dictEnd - dictPtr);
U32 offcodeMax = MaxOff; U32 offcodeMax = MaxOff;
if (dictContentSize <= ((U32)-1) - 128 KB) { if (dictContentSize <= ((U32)-1) - 128 KB) {
U32 const maxOffset = (U32)dictContentSize + 128 KB; /* The maximum offset that must be supported */ U32 const maxOffset = (U32)dictContentSize + 128 KB; /* The maximum offset that must be supported */
@@ -2547,11 +2683,15 @@ static size_t ZSTD_loadZstdDictionary(ZSTD_CCtx* cctx, const void* dict, size_t
/* All offset values <= dictContentSize + 128 KB must be representable */ /* All offset values <= dictContentSize + 128 KB must be representable */
CHECK_F(ZSTD_checkDictNCount(offcodeNCount, offcodeMaxValue, MIN(offcodeMax, MaxOff))); CHECK_F(ZSTD_checkDictNCount(offcodeNCount, offcodeMaxValue, MIN(offcodeMax, MaxOff)));
/* All repCodes must be <= dictContentSize and != 0*/ /* All repCodes must be <= dictContentSize and != 0*/
{ U32 u; {
U32 u;
for (u = 0; u < 3; u++) { for (u = 0; u < 3; u++) {
if (cctx->rep[u] == 0) return ERROR(dictionary_corrupted); if (cctx->rep[u] == 0)
if (cctx->rep[u] > dictContentSize) return ERROR(dictionary_corrupted); return ERROR(dictionary_corrupted);
} } if (cctx->rep[u] > dictContentSize)
return ERROR(dictionary_corrupted);
}
}
cctx->flagStaticTables = 1; cctx->flagStaticTables = 1;
cctx->flagStaticHufTable = HUF_repeat_valid; cctx->flagStaticHufTable = HUF_repeat_valid;
@@ -2563,7 +2703,8 @@ static size_t ZSTD_loadZstdDictionary(ZSTD_CCtx* cctx, const void* dict, size_t
* @return : 0, or an error code */ * @return : 0, or an error code */
static size_t ZSTD_compress_insertDictionary(ZSTD_CCtx *cctx, const void *dict, size_t dictSize) static size_t ZSTD_compress_insertDictionary(ZSTD_CCtx *cctx, const void *dict, size_t dictSize)
{ {
if ((dict==NULL) || (dictSize<=8)) return 0; if ((dict == NULL) || (dictSize <= 8))
return 0;
/* dict as pure content */ /* dict as pure content */
if ((ZSTD_readLE32(dict) != ZSTD_DICT_MAGIC) || (cctx->forceRawDict)) if ((ZSTD_readLE32(dict) != ZSTD_DICT_MAGIC) || (cctx->forceRawDict))
@@ -2575,40 +2716,29 @@ static size_t ZSTD_compress_insertDictionary(ZSTD_CCtx* cctx, const void* dict,
/*! ZSTD_compressBegin_internal() : /*! ZSTD_compressBegin_internal() :
* @return : 0, or an error code */ * @return : 0, or an error code */
static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* cctx, static size_t ZSTD_compressBegin_internal(ZSTD_CCtx *cctx, const void *dict, size_t dictSize, ZSTD_parameters params, U64 pledgedSrcSize)
const void* dict, size_t dictSize,
ZSTD_parameters params, U64 pledgedSrcSize)
{ {
ZSTD_compResetPolicy_e const crp = dictSize ? ZSTDcrp_fullReset : ZSTDcrp_continue; ZSTD_compResetPolicy_e const crp = dictSize ? ZSTDcrp_fullReset : ZSTDcrp_continue;
CHECK_F(ZSTD_resetCCtx_advanced(cctx, params, pledgedSrcSize, crp)); CHECK_F(ZSTD_resetCCtx_advanced(cctx, params, pledgedSrcSize, crp));
return ZSTD_compress_insertDictionary(cctx, dict, dictSize); return ZSTD_compress_insertDictionary(cctx, dict, dictSize);
} }
/*! ZSTD_compressBegin_advanced() : /*! ZSTD_compressBegin_advanced() :
* @return : 0, or an error code */ * @return : 0, or an error code */
size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, size_t ZSTD_compressBegin_advanced(ZSTD_CCtx *cctx, const void *dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize)
const void* dict, size_t dictSize,
ZSTD_parameters params, unsigned long long pledgedSrcSize)
{ {
/* compression parameters verification and optimization */ /* compression parameters verification and optimization */
CHECK_F(ZSTD_checkCParams(params.cParams)); CHECK_F(ZSTD_checkCParams(params.cParams));
return ZSTD_compressBegin_internal(cctx, dict, dictSize, params, pledgedSrcSize); return ZSTD_compressBegin_internal(cctx, dict, dictSize, params, pledgedSrcSize);
} }
size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx *cctx, const void *dict, size_t dictSize, int compressionLevel) size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx *cctx, const void *dict, size_t dictSize, int compressionLevel)
{ {
ZSTD_parameters const params = ZSTD_getParams(compressionLevel, 0, dictSize); ZSTD_parameters const params = ZSTD_getParams(compressionLevel, 0, dictSize);
return ZSTD_compressBegin_internal(cctx, dict, dictSize, params, 0); return ZSTD_compressBegin_internal(cctx, dict, dictSize, params, 0);
} }
size_t ZSTD_compressBegin(ZSTD_CCtx *cctx, int compressionLevel) { return ZSTD_compressBegin_usingDict(cctx, NULL, 0, compressionLevel); }
size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel)
{
return ZSTD_compressBegin_usingDict(cctx, NULL, 0, compressionLevel);
}
/*! ZSTD_writeEpilogue() : /*! ZSTD_writeEpilogue() :
* Ends a frame. * Ends a frame.
@@ -2619,12 +2749,14 @@ static size_t ZSTD_writeEpilogue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity)
BYTE *op = ostart; BYTE *op = ostart;
size_t fhSize = 0; size_t fhSize = 0;
if (cctx->stage == ZSTDcs_created) return ERROR(stage_wrong); /* init missing */ if (cctx->stage == ZSTDcs_created)
return ERROR(stage_wrong); /* init missing */
/* special case : empty frame */ /* special case : empty frame */
if (cctx->stage == ZSTDcs_init) { if (cctx->stage == ZSTDcs_init) {
fhSize = ZSTD_writeFrameHeader(dst, dstCapacity, cctx->params, 0, 0); fhSize = ZSTD_writeFrameHeader(dst, dstCapacity, cctx->params, 0, 0);
if (ZSTD_isError(fhSize)) return fhSize; if (ZSTD_isError(fhSize))
return fhSize;
dstCapacity -= fhSize; dstCapacity -= fhSize;
op += fhSize; op += fhSize;
cctx->stage = ZSTDcs_ongoing; cctx->stage = ZSTDcs_ongoing;
@@ -2633,7 +2765,8 @@ static size_t ZSTD_writeEpilogue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity)
if (cctx->stage != ZSTDcs_ending) { if (cctx->stage != ZSTDcs_ending) {
/* write one last empty block, make it the "last" block */ /* write one last empty block, make it the "last" block */
U32 const cBlockHeader24 = 1 /* last block */ + (((U32)bt_raw) << 1) + 0; U32 const cBlockHeader24 = 1 /* last block */ + (((U32)bt_raw) << 1) + 0;
if (dstCapacity<4) return ERROR(dstSize_tooSmall); if (dstCapacity < 4)
return ERROR(dstSize_tooSmall);
ZSTD_writeLE32(op, cBlockHeader24); ZSTD_writeLE32(op, cBlockHeader24);
op += ZSTD_blockHeaderSize; op += ZSTD_blockHeaderSize;
dstCapacity -= ZSTD_blockHeaderSize; dstCapacity -= ZSTD_blockHeaderSize;
@@ -2641,7 +2774,8 @@ static size_t ZSTD_writeEpilogue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity)
if (cctx->params.fParams.checksumFlag) { if (cctx->params.fParams.checksumFlag) {
U32 const checksum = (U32)xxh64_digest(&cctx->xxhState); U32 const checksum = (U32)xxh64_digest(&cctx->xxhState);
if (dstCapacity<4) return ERROR(dstSize_tooSmall); if (dstCapacity < 4)
return ERROR(dstSize_tooSmall);
ZSTD_writeLE32(op, checksum); ZSTD_writeLE32(op, checksum);
op += 4; op += 4;
} }
@@ -2650,42 +2784,36 @@ static size_t ZSTD_writeEpilogue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity)
return op - ostart; return op - ostart;
} }
size_t ZSTD_compressEnd(ZSTD_CCtx *cctx, void *dst, size_t dstCapacity, const void *src, size_t srcSize)
size_t ZSTD_compressEnd (ZSTD_CCtx* cctx,
void* dst, size_t dstCapacity,
const void* src, size_t srcSize)
{ {
size_t endResult; size_t endResult;
size_t const cSize = ZSTD_compressContinue_internal(cctx, dst, dstCapacity, src, srcSize, 1, 1); size_t const cSize = ZSTD_compressContinue_internal(cctx, dst, dstCapacity, src, srcSize, 1, 1);
if (ZSTD_isError(cSize)) return cSize; if (ZSTD_isError(cSize))
return cSize;
endResult = ZSTD_writeEpilogue(cctx, (char *)dst + cSize, dstCapacity - cSize); endResult = ZSTD_writeEpilogue(cctx, (char *)dst + cSize, dstCapacity - cSize);
if (ZSTD_isError(endResult)) return endResult; if (ZSTD_isError(endResult))
return endResult;
return cSize + endResult; return cSize + endResult;
} }
static size_t ZSTD_compress_internal(ZSTD_CCtx *cctx, void *dst, size_t dstCapacity, const void *src, size_t srcSize, const void *dict, size_t dictSize,
static size_t ZSTD_compress_internal (ZSTD_CCtx* cctx,
void* dst, size_t dstCapacity,
const void* src, size_t srcSize,
const void* dict,size_t dictSize,
ZSTD_parameters params) ZSTD_parameters params)
{ {
CHECK_F(ZSTD_compressBegin_internal(cctx, dict, dictSize, params, srcSize)); CHECK_F(ZSTD_compressBegin_internal(cctx, dict, dictSize, params, srcSize));
return ZSTD_compressEnd(cctx, dst, dstCapacity, src, srcSize); return ZSTD_compressEnd(cctx, dst, dstCapacity, src, srcSize);
} }
size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, const void* dict, size_t dictSize, ZSTD_parameters params) size_t ZSTD_compress_usingDict(ZSTD_CCtx *ctx, void *dst, size_t dstCapacity, const void *src, size_t srcSize, const void *dict, size_t dictSize,
ZSTD_parameters params)
{ {
return ZSTD_compress_internal(ctx, dst, dstCapacity, src, srcSize, dict, dictSize, params); return ZSTD_compress_internal(ctx, dst, dstCapacity, src, srcSize, dict, dictSize, params);
} }
size_t ZSTD_compressCCtx(ZSTD_CCtx *ctx, void *dst, size_t dstCapacity, const void *src, size_t srcSize, ZSTD_parameters params) size_t ZSTD_compressCCtx(ZSTD_CCtx *ctx, void *dst, size_t dstCapacity, const void *src, size_t srcSize, ZSTD_parameters params)
{ {
return ZSTD_compress_internal(ctx, dst, dstCapacity, src, srcSize, NULL, 0, params); return ZSTD_compress_internal(ctx, dst, dstCapacity, src, srcSize, NULL, 0, params);
} }
/* ===== Dictionary API ===== */ /* ===== Dictionary API ===== */
struct ZSTD_CDict_s { struct ZSTD_CDict_s {
@@ -2695,17 +2823,15 @@ struct ZSTD_CDict_s {
ZSTD_CCtx *refContext; ZSTD_CCtx *refContext;
}; /* typedef'd tp ZSTD_CDict within "zstd.h" */ }; /* typedef'd tp ZSTD_CDict within "zstd.h" */
size_t ZSTD_CDictWorkspaceBound(ZSTD_compressionParameters cParams) size_t ZSTD_CDictWorkspaceBound(ZSTD_compressionParameters cParams) { return ZSTD_CCtxWorkspaceBound(cParams) + ZSTD_ALIGN(sizeof(ZSTD_CDict)); }
{
return ZSTD_CCtxWorkspaceBound(cParams) + ZSTD_ALIGN(sizeof(ZSTD_CDict));
}
static ZSTD_CDict* ZSTD_createCDict_advanced(const void* dictBuffer, size_t dictSize, unsigned byReference, static ZSTD_CDict *ZSTD_createCDict_advanced(const void *dictBuffer, size_t dictSize, unsigned byReference, ZSTD_parameters params, ZSTD_customMem customMem)
ZSTD_parameters params, ZSTD_customMem customMem)
{ {
if (!customMem.customAlloc || !customMem.customFree) return NULL; if (!customMem.customAlloc || !customMem.customFree)
return NULL;
{ ZSTD_CDict* const cdict = (ZSTD_CDict*) ZSTD_malloc(sizeof(ZSTD_CDict), customMem); {
ZSTD_CDict *const cdict = (ZSTD_CDict *)ZSTD_malloc(sizeof(ZSTD_CDict), customMem);
ZSTD_CCtx *const cctx = ZSTD_createCCtx_advanced(customMem); ZSTD_CCtx *const cctx = ZSTD_createCCtx_advanced(customMem);
if (!cdict || !cctx) { if (!cdict || !cctx) {
@@ -2719,19 +2845,25 @@ static ZSTD_CDict* ZSTD_createCDict_advanced(const void* dictBuffer, size_t dict
cdict->dictContent = dictBuffer; cdict->dictContent = dictBuffer;
} else { } else {
void *const internalBuffer = ZSTD_malloc(dictSize, customMem); void *const internalBuffer = ZSTD_malloc(dictSize, customMem);
if (!internalBuffer) { ZSTD_free(cctx, customMem); ZSTD_free(cdict, customMem); return NULL; } if (!internalBuffer) {
ZSTD_free(cctx, customMem);
ZSTD_free(cdict, customMem);
return NULL;
}
memcpy(internalBuffer, dictBuffer, dictSize); memcpy(internalBuffer, dictBuffer, dictSize);
cdict->dictBuffer = internalBuffer; cdict->dictBuffer = internalBuffer;
cdict->dictContent = internalBuffer; cdict->dictContent = internalBuffer;
} }
{ size_t const errorCode = ZSTD_compressBegin_advanced(cctx, cdict->dictContent, dictSize, params, 0); {
size_t const errorCode = ZSTD_compressBegin_advanced(cctx, cdict->dictContent, dictSize, params, 0);
if (ZSTD_isError(errorCode)) { if (ZSTD_isError(errorCode)) {
ZSTD_free(cdict->dictBuffer, customMem); ZSTD_free(cdict->dictBuffer, customMem);
ZSTD_free(cdict, customMem); ZSTD_free(cdict, customMem);
ZSTD_freeCCtx(cctx); ZSTD_freeCCtx(cctx);
return NULL; return NULL;
} } }
}
cdict->refContext = cctx; cdict->refContext = cctx;
cdict->dictContentSize = dictSize; cdict->dictContentSize = dictSize;
@@ -2747,8 +2879,10 @@ ZSTD_CDict* ZSTD_initCDict(const void* dict, size_t dictSize, ZSTD_parameters pa
size_t ZSTD_freeCDict(ZSTD_CDict *cdict) size_t ZSTD_freeCDict(ZSTD_CDict *cdict)
{ {
if (cdict==NULL) return 0; /* support free on NULL */ if (cdict == NULL)
{ ZSTD_customMem const cMem = cdict->refContext->customMem; return 0; /* support free on NULL */
{
ZSTD_customMem const cMem = cdict->refContext->customMem;
ZSTD_freeCCtx(cdict->refContext); ZSTD_freeCCtx(cdict->refContext);
ZSTD_free(cdict->dictBuffer, cMem); ZSTD_free(cdict->dictBuffer, cMem);
ZSTD_free(cdict, cMem); ZSTD_free(cdict, cMem);
@@ -2756,13 +2890,12 @@ size_t ZSTD_freeCDict(ZSTD_CDict* cdict)
} }
} }
static ZSTD_parameters ZSTD_getParamsFromCDict(const ZSTD_CDict* cdict) { static ZSTD_parameters ZSTD_getParamsFromCDict(const ZSTD_CDict *cdict) { return ZSTD_getParamsFromCCtx(cdict->refContext); }
return ZSTD_getParamsFromCCtx(cdict->refContext);
}
size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx *cctx, const ZSTD_CDict *cdict, unsigned long long pledgedSrcSize) size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx *cctx, const ZSTD_CDict *cdict, unsigned long long pledgedSrcSize)
{ {
if (cdict->dictContentSize) CHECK_F(ZSTD_copyCCtx(cctx, cdict->refContext, pledgedSrcSize)) if (cdict->dictContentSize)
CHECK_F(ZSTD_copyCCtx(cctx, cdict->refContext, pledgedSrcSize))
else { else {
ZSTD_parameters params = cdict->refContext->params; ZSTD_parameters params = cdict->refContext->params;
params.fParams.contentSizeFlag = (pledgedSrcSize > 0); params.fParams.contentSizeFlag = (pledgedSrcSize > 0);
@@ -2775,10 +2908,7 @@ size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict, u
* Compression using a digested Dictionary. * Compression using a digested Dictionary.
* Faster startup than ZSTD_compress_usingDict(), recommended when same dictionary is used multiple times. * Faster startup than ZSTD_compress_usingDict(), recommended when same dictionary is used multiple times.
* Note that compression level is decided during dictionary creation */ * Note that compression level is decided during dictionary creation */
size_t ZSTD_compress_usingCDict(ZSTD_CCtx* cctx, size_t ZSTD_compress_usingCDict(ZSTD_CCtx *cctx, void *dst, size_t dstCapacity, const void *src, size_t srcSize, const ZSTD_CDict *cdict)
void* dst, size_t dstCapacity,
const void* src, size_t srcSize,
const ZSTD_CDict* cdict)
{ {
CHECK_F(ZSTD_compressBegin_usingCDict(cctx, cdict, srcSize)); CHECK_F(ZSTD_compressBegin_usingCDict(cctx, cdict, srcSize));
@@ -2792,8 +2922,6 @@ size_t ZSTD_compress_usingCDict(ZSTD_CCtx* cctx,
return ZSTD_compressEnd(cctx, dst, dstCapacity, src, srcSize); return ZSTD_compressEnd(cctx, dst, dstCapacity, src, srcSize);
} }
/* ****************************************************************** /* ******************************************************************
* Streaming * Streaming
********************************************************************/ ********************************************************************/
@@ -2836,21 +2964,28 @@ ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem)
{ {
ZSTD_CStream *zcs; ZSTD_CStream *zcs;
if (!customMem.customAlloc || !customMem.customFree) return NULL; if (!customMem.customAlloc || !customMem.customFree)
return NULL;
zcs = (ZSTD_CStream *)ZSTD_malloc(sizeof(ZSTD_CStream), customMem); zcs = (ZSTD_CStream *)ZSTD_malloc(sizeof(ZSTD_CStream), customMem);
if (zcs==NULL) return NULL; if (zcs == NULL)
return NULL;
memset(zcs, 0, sizeof(ZSTD_CStream)); memset(zcs, 0, sizeof(ZSTD_CStream));
memcpy(&zcs->customMem, &customMem, sizeof(ZSTD_customMem)); memcpy(&zcs->customMem, &customMem, sizeof(ZSTD_customMem));
zcs->cctx = ZSTD_createCCtx_advanced(customMem); zcs->cctx = ZSTD_createCCtx_advanced(customMem);
if (zcs->cctx == NULL) { ZSTD_freeCStream(zcs); return NULL; } if (zcs->cctx == NULL) {
ZSTD_freeCStream(zcs);
return NULL;
}
return zcs; return zcs;
} }
size_t ZSTD_freeCStream(ZSTD_CStream *zcs) size_t ZSTD_freeCStream(ZSTD_CStream *zcs)
{ {
if (zcs==NULL) return 0; /* support free on NULL */ if (zcs == NULL)
{ ZSTD_customMem const cMem = zcs->customMem; return 0; /* support free on NULL */
{
ZSTD_customMem const cMem = zcs->customMem;
ZSTD_freeCCtx(zcs->cctx); ZSTD_freeCCtx(zcs->cctx);
zcs->cctx = NULL; zcs->cctx = NULL;
ZSTD_freeCDict(zcs->cdictLocal); ZSTD_freeCDict(zcs->cdictLocal);
@@ -2864,7 +2999,6 @@ size_t ZSTD_freeCStream(ZSTD_CStream* zcs)
} }
} }
/*====== Initialization ======*/ /*====== Initialization ======*/
size_t ZSTD_CStreamInSize(void) { return ZSTD_BLOCKSIZE_ABSOLUTEMAX; } size_t ZSTD_CStreamInSize(void) { return ZSTD_BLOCKSIZE_ABSOLUTEMAX; }
@@ -2872,10 +3006,13 @@ size_t ZSTD_CStreamOutSize(void) { return ZSTD_compressBound(ZSTD_BLOCKSIZE_ABSO
static size_t ZSTD_resetCStream_internal(ZSTD_CStream *zcs, unsigned long long pledgedSrcSize) static size_t ZSTD_resetCStream_internal(ZSTD_CStream *zcs, unsigned long long pledgedSrcSize)
{ {
if (zcs->inBuffSize==0) return ERROR(stage_wrong); /* zcs has not been init at least once => can't reset */ if (zcs->inBuffSize == 0)
return ERROR(stage_wrong); /* zcs has not been init at least once => can't reset */
if (zcs->cdict) CHECK_F(ZSTD_compressBegin_usingCDict(zcs->cctx, zcs->cdict, pledgedSrcSize)) if (zcs->cdict)
else CHECK_F(ZSTD_compressBegin_advanced(zcs->cctx, NULL, 0, zcs->params, pledgedSrcSize)); CHECK_F(ZSTD_compressBegin_usingCDict(zcs->cctx, zcs->cdict, pledgedSrcSize))
else
CHECK_F(ZSTD_compressBegin_advanced(zcs->cctx, NULL, 0, zcs->params, pledgedSrcSize));
zcs->inToCompress = 0; zcs->inToCompress = 0;
zcs->inBuffPos = 0; zcs->inBuffPos = 0;
@@ -2896,17 +3033,17 @@ size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize)
return ZSTD_resetCStream_internal(zcs, pledgedSrcSize); return ZSTD_resetCStream_internal(zcs, pledgedSrcSize);
} }
static size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, static size_t ZSTD_initCStream_advanced(ZSTD_CStream *zcs, const void *dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize)
const void* dict, size_t dictSize,
ZSTD_parameters params, unsigned long long pledgedSrcSize)
{ {
/* allocate buffers */ /* allocate buffers */
{ size_t const neededInBuffSize = (size_t)1 << params.cParams.windowLog; {
size_t const neededInBuffSize = (size_t)1 << params.cParams.windowLog;
if (zcs->inBuffSize < neededInBuffSize) { if (zcs->inBuffSize < neededInBuffSize) {
zcs->inBuffSize = neededInBuffSize; zcs->inBuffSize = neededInBuffSize;
ZSTD_free(zcs->inBuff, zcs->customMem); ZSTD_free(zcs->inBuff, zcs->customMem);
zcs->inBuff = (char *)ZSTD_malloc(neededInBuffSize, zcs->customMem); zcs->inBuff = (char *)ZSTD_malloc(neededInBuffSize, zcs->customMem);
if (zcs->inBuff == NULL) return ERROR(memory_allocation); if (zcs->inBuff == NULL)
return ERROR(memory_allocation);
} }
zcs->blockSize = MIN(ZSTD_BLOCKSIZE_ABSOLUTEMAX, neededInBuffSize); zcs->blockSize = MIN(ZSTD_BLOCKSIZE_ABSOLUTEMAX, neededInBuffSize);
} }
@@ -2914,15 +3051,18 @@ static size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs,
zcs->outBuffSize = ZSTD_compressBound(zcs->blockSize) + 1; zcs->outBuffSize = ZSTD_compressBound(zcs->blockSize) + 1;
ZSTD_free(zcs->outBuff, zcs->customMem); ZSTD_free(zcs->outBuff, zcs->customMem);
zcs->outBuff = (char *)ZSTD_malloc(zcs->outBuffSize, zcs->customMem); zcs->outBuff = (char *)ZSTD_malloc(zcs->outBuffSize, zcs->customMem);
if (zcs->outBuff == NULL) return ERROR(memory_allocation); if (zcs->outBuff == NULL)
return ERROR(memory_allocation);
} }
if (dict && dictSize >= 8) { if (dict && dictSize >= 8) {
ZSTD_freeCDict(zcs->cdictLocal); ZSTD_freeCDict(zcs->cdictLocal);
zcs->cdictLocal = ZSTD_createCDict_advanced(dict, dictSize, 0, params, zcs->customMem); zcs->cdictLocal = ZSTD_createCDict_advanced(dict, dictSize, 0, params, zcs->customMem);
if (zcs->cdictLocal == NULL) return ERROR(memory_allocation); if (zcs->cdictLocal == NULL)
return ERROR(memory_allocation);
zcs->cdict = zcs->cdictLocal; zcs->cdict = zcs->cdictLocal;
} else zcs->cdict = NULL; } else
zcs->cdict = NULL;
zcs->checksum = params.fParams.checksumFlag > 0; zcs->checksum = params.fParams.checksumFlag > 0;
zcs->params = params; zcs->params = params;
@@ -2936,7 +3076,9 @@ ZSTD_CStream* ZSTD_initCStream(ZSTD_parameters params, unsigned long long pledge
ZSTD_CStream *const zcs = ZSTD_createCStream_advanced(stackMem); ZSTD_CStream *const zcs = ZSTD_createCStream_advanced(stackMem);
if (zcs) { if (zcs) {
size_t const code = ZSTD_initCStream_advanced(zcs, NULL, 0, params, pledgedSrcSize); size_t const code = ZSTD_initCStream_advanced(zcs, NULL, 0, params, pledgedSrcSize);
if (ZSTD_isError(code)) { return NULL; } if (ZSTD_isError(code)) {
return NULL;
}
} }
return zcs; return zcs;
} }
@@ -2965,10 +3107,7 @@ ZSTD_STATIC size_t ZSTD_limitCopy(void* dst, size_t dstCapacity, const void* src
return length; return length;
} }
static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs, static size_t ZSTD_compressStream_generic(ZSTD_CStream *zcs, void *dst, size_t *dstCapacityPtr, const void *src, size_t *srcSizePtr, ZSTD_flush_e const flush)
void* dst, size_t* dstCapacityPtr,
const void* src, size_t* srcSizePtr,
ZSTD_flush_e const flush)
{ {
U32 someMoreWork = 1; U32 someMoreWork = 1;
const char *const istart = (const char *)src; const char *const istart = (const char *)src;
@@ -2979,21 +3118,25 @@ static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs,
char *op = ostart; char *op = ostart;
while (someMoreWork) { while (someMoreWork) {
switch(zcs->stage) switch (zcs->stage) {
{ case zcss_init:
case zcss_init: return ERROR(init_missing); /* call ZBUFF_compressInit() first ! */ return ERROR(init_missing); /* call ZBUFF_compressInit() first ! */
case zcss_load: case zcss_load:
/* complete inBuffer */ /* complete inBuffer */
{ size_t const toLoad = zcs->inBuffTarget - zcs->inBuffPos; {
size_t const toLoad = zcs->inBuffTarget - zcs->inBuffPos;
size_t const loaded = ZSTD_limitCopy(zcs->inBuff + zcs->inBuffPos, toLoad, ip, iend - ip); size_t const loaded = ZSTD_limitCopy(zcs->inBuff + zcs->inBuffPos, toLoad, ip, iend - ip);
zcs->inBuffPos += loaded; zcs->inBuffPos += loaded;
ip += loaded; ip += loaded;
if ((zcs->inBuffPos == zcs->inToCompress) || (!flush && (toLoad != loaded))) { if ((zcs->inBuffPos == zcs->inToCompress) || (!flush && (toLoad != loaded))) {
someMoreWork = 0; break; /* not enough input to get a full block : stop there, wait for more */ someMoreWork = 0;
} } break; /* not enough input to get a full block : stop there, wait for more */
}
}
/* compress curr block (note : this stage cannot be stopped in the middle) */ /* compress curr block (note : this stage cannot be stopped in the middle) */
{ void* cDst; {
void *cDst;
size_t cSize; size_t cSize;
size_t const iSize = zcs->inBuffPos - zcs->inToCompress; size_t const iSize = zcs->inBuffPos - zcs->inToCompress;
size_t oSize = oend - op; size_t oSize = oend - op;
@@ -3001,28 +3144,35 @@ static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs,
cDst = op; /* compress directly into output buffer (avoid flush stage) */ cDst = op; /* compress directly into output buffer (avoid flush stage) */
else else
cDst = zcs->outBuff, oSize = zcs->outBuffSize; cDst = zcs->outBuff, oSize = zcs->outBuffSize;
cSize = (flush == zsf_end) ? cSize = (flush == zsf_end) ? ZSTD_compressEnd(zcs->cctx, cDst, oSize, zcs->inBuff + zcs->inToCompress, iSize)
ZSTD_compressEnd(zcs->cctx, cDst, oSize, zcs->inBuff + zcs->inToCompress, iSize) : : ZSTD_compressContinue(zcs->cctx, cDst, oSize, zcs->inBuff + zcs->inToCompress, iSize);
ZSTD_compressContinue(zcs->cctx, cDst, oSize, zcs->inBuff + zcs->inToCompress, iSize); if (ZSTD_isError(cSize))
if (ZSTD_isError(cSize)) return cSize; return cSize;
if (flush == zsf_end) zcs->frameEnded = 1; if (flush == zsf_end)
zcs->frameEnded = 1;
/* prepare next block */ /* prepare next block */
zcs->inBuffTarget = zcs->inBuffPos + zcs->blockSize; zcs->inBuffTarget = zcs->inBuffPos + zcs->blockSize;
if (zcs->inBuffTarget > zcs->inBuffSize) if (zcs->inBuffTarget > zcs->inBuffSize)
zcs->inBuffPos = 0, zcs->inBuffTarget = zcs->blockSize; /* note : inBuffSize >= blockSize */ zcs->inBuffPos = 0, zcs->inBuffTarget = zcs->blockSize; /* note : inBuffSize >= blockSize */
zcs->inToCompress = zcs->inBuffPos; zcs->inToCompress = zcs->inBuffPos;
if (cDst == op) { op += cSize; break; } /* no need to flush */ if (cDst == op) {
op += cSize;
break;
} /* no need to flush */
zcs->outBuffContentSize = cSize; zcs->outBuffContentSize = cSize;
zcs->outBuffFlushedSize = 0; zcs->outBuffFlushedSize = 0;
zcs->stage = zcss_flush; /* pass-through to flush stage */ zcs->stage = zcss_flush; /* pass-through to flush stage */
} }
case zcss_flush: case zcss_flush: {
{ size_t const toFlush = zcs->outBuffContentSize - zcs->outBuffFlushedSize; size_t const toFlush = zcs->outBuffContentSize - zcs->outBuffFlushedSize;
size_t const flushed = ZSTD_limitCopy(op, oend - op, zcs->outBuff + zcs->outBuffFlushedSize, toFlush); size_t const flushed = ZSTD_limitCopy(op, oend - op, zcs->outBuff + zcs->outBuffFlushedSize, toFlush);
op += flushed; op += flushed;
zcs->outBuffFlushedSize += flushed; zcs->outBuffFlushedSize += flushed;
if (toFlush!=flushed) { someMoreWork = 0; break; } /* dst too small to store flushed data : stop there */ if (toFlush != flushed) {
someMoreWork = 0;
break;
} /* dst too small to store flushed data : stop there */
zcs->outBuffContentSize = zcs->outBuffFlushedSize = 0; zcs->outBuffContentSize = zcs->outBuffFlushedSize = 0;
zcs->stage = zcss_load; zcs->stage = zcss_load;
break; break;
@@ -3040,9 +3190,12 @@ static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs,
*srcSizePtr = ip - istart; *srcSizePtr = ip - istart;
*dstCapacityPtr = op - ostart; *dstCapacityPtr = op - ostart;
zcs->inputProcessed += *srcSizePtr; zcs->inputProcessed += *srcSizePtr;
if (zcs->frameEnded) return 0; if (zcs->frameEnded)
{ size_t hintInSize = zcs->inBuffTarget - zcs->inBuffPos; return 0;
if (hintInSize==0) hintInSize = zcs->blockSize; {
size_t hintInSize = zcs->inBuffTarget - zcs->inBuffPos;
if (hintInSize == 0)
hintInSize = zcs->blockSize;
return hintInSize; return hintInSize;
} }
} }
@@ -3051,15 +3204,13 @@ size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuf
{ {
size_t sizeRead = input->size - input->pos; size_t sizeRead = input->size - input->pos;
size_t sizeWritten = output->size - output->pos; size_t sizeWritten = output->size - output->pos;
size_t const result = ZSTD_compressStream_generic(zcs, size_t const result =
(char*)(output->dst) + output->pos, &sizeWritten, ZSTD_compressStream_generic(zcs, (char *)(output->dst) + output->pos, &sizeWritten, (const char *)(input->src) + input->pos, &sizeRead, zsf_gather);
(const char*)(input->src) + input->pos, &sizeRead, zsf_gather);
input->pos += sizeRead; input->pos += sizeRead;
output->pos += sizeWritten; output->pos += sizeWritten;
return result; return result;
} }
/*====== Finalize ======*/ /*====== Finalize ======*/
/*! ZSTD_flushStream() : /*! ZSTD_flushStream() :
@@ -3068,16 +3219,15 @@ size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output)
{ {
size_t srcSize = 0; size_t srcSize = 0;
size_t sizeWritten = output->size - output->pos; size_t sizeWritten = output->size - output->pos;
size_t const result = ZSTD_compressStream_generic(zcs, size_t const result = ZSTD_compressStream_generic(zcs, (char *)(output->dst) + output->pos, &sizeWritten, &srcSize,
(char*)(output->dst) + output->pos, &sizeWritten, &srcSize, /* use a valid src address instead of NULL */
&srcSize, &srcSize, /* use a valid src address instead of NULL */
zsf_flush); zsf_flush);
output->pos += sizeWritten; output->pos += sizeWritten;
if (ZSTD_isError(result)) return result; if (ZSTD_isError(result))
return result;
return zcs->outBuffContentSize - zcs->outBuffFlushedSize; /* remaining to flush */ return zcs->outBuffContentSize - zcs->outBuffFlushedSize; /* remaining to flush */
} }
size_t ZSTD_endStream(ZSTD_CStream *zcs, ZSTD_outBuffer *output) size_t ZSTD_endStream(ZSTD_CStream *zcs, ZSTD_outBuffer *output)
{ {
BYTE *const ostart = (BYTE *)(output->dst) + output->pos; BYTE *const ostart = (BYTE *)(output->dst) + output->pos;
@@ -3091,7 +3241,8 @@ size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output)
/* flush whatever remains */ /* flush whatever remains */
size_t srcSize = 0; size_t srcSize = 0;
size_t sizeWritten = output->size - output->pos; size_t sizeWritten = output->size - output->pos;
size_t const notEnded = ZSTD_compressStream_generic(zcs, ostart, &sizeWritten, &srcSize, &srcSize, zsf_end); /* use a valid src address instead of NULL */ size_t const notEnded =
ZSTD_compressStream_generic(zcs, ostart, &sizeWritten, &srcSize, &srcSize, zsf_end); /* use a valid src address instead of NULL */
size_t const remainingToFlush = zcs->outBuffContentSize - zcs->outBuffFlushedSize; size_t const remainingToFlush = zcs->outBuffContentSize - zcs->outBuffFlushedSize;
op += sizeWritten; op += sizeWritten;
if (remainingToFlush) { if (remainingToFlush) {
@@ -3100,23 +3251,23 @@ size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output)
} }
/* create epilogue */ /* create epilogue */
zcs->stage = zcss_final; zcs->stage = zcss_final;
zcs->outBuffContentSize = !notEnded ? 0 : zcs->outBuffContentSize = !notEnded ? 0 : ZSTD_compressEnd(zcs->cctx, zcs->outBuff, zcs->outBuffSize, NULL,
ZSTD_compressEnd(zcs->cctx, zcs->outBuff, zcs->outBuffSize, NULL, 0); /* write epilogue, including final empty block, into outBuff */ 0); /* write epilogue, including final empty block, into outBuff */
} }
/* flush epilogue */ /* flush epilogue */
{ size_t const toFlush = zcs->outBuffContentSize - zcs->outBuffFlushedSize; {
size_t const toFlush = zcs->outBuffContentSize - zcs->outBuffFlushedSize;
size_t const flushed = ZSTD_limitCopy(op, oend - op, zcs->outBuff + zcs->outBuffFlushedSize, toFlush); size_t const flushed = ZSTD_limitCopy(op, oend - op, zcs->outBuff + zcs->outBuffFlushedSize, toFlush);
op += flushed; op += flushed;
zcs->outBuffFlushedSize += flushed; zcs->outBuffFlushedSize += flushed;
output->pos += op - ostart; output->pos += op - ostart;
if (toFlush==flushed) zcs->stage = zcss_init; /* end reached */ if (toFlush == flushed)
zcs->stage = zcss_init; /* end reached */
return toFlush - flushed; return toFlush - flushed;
} }
} }
/*-===== Pre-defined compression levels =====-*/ /*-===== Pre-defined compression levels =====-*/
#define ZSTD_DEFAULT_CLEVEL 1 #define ZSTD_DEFAULT_CLEVEL 1
@@ -3124,7 +3275,8 @@ size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output)
int ZSTD_maxCLevel(void) { return ZSTD_MAX_CLEVEL; } int ZSTD_maxCLevel(void) { return ZSTD_MAX_CLEVEL; }
static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEVEL + 1] = { static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEVEL + 1] = {
{ /* "default" */ {
/* "default" */
/* W, C, H, S, L, TL, strat */ /* W, C, H, S, L, TL, strat */
{18, 12, 12, 1, 7, 16, ZSTD_fast}, /* level 0 - never used */ {18, 12, 12, 1, 7, 16, ZSTD_fast}, /* level 0 - never used */
{19, 13, 14, 1, 7, 16, ZSTD_fast}, /* level 1 */ {19, 13, 14, 1, 7, 16, ZSTD_fast}, /* level 1 */
@@ -3150,7 +3302,8 @@ static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEV
{26, 26, 23, 7, 3, 256, ZSTD_btopt2}, /* level 21 */ {26, 26, 23, 7, 3, 256, ZSTD_btopt2}, /* level 21 */
{27, 27, 25, 9, 3, 512, ZSTD_btopt2}, /* level 22 */ {27, 27, 25, 9, 3, 512, ZSTD_btopt2}, /* level 22 */
}, },
{ /* for srcSize <= 256 KB */ {
/* for srcSize <= 256 KB */
/* W, C, H, S, L, T, strat */ /* W, C, H, S, L, T, strat */
{0, 0, 0, 0, 0, 0, ZSTD_fast}, /* level 0 - not used */ {0, 0, 0, 0, 0, 0, ZSTD_fast}, /* level 0 - not used */
{18, 13, 14, 1, 6, 8, ZSTD_fast}, /* level 1 */ {18, 13, 14, 1, 6, 8, ZSTD_fast}, /* level 1 */
@@ -3176,7 +3329,8 @@ static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEV
{18, 19, 18, 12, 3, 512, ZSTD_btopt2}, /* level 21.*/ {18, 19, 18, 12, 3, 512, ZSTD_btopt2}, /* level 21.*/
{18, 19, 18, 13, 3, 512, ZSTD_btopt2}, /* level 22.*/ {18, 19, 18, 13, 3, 512, ZSTD_btopt2}, /* level 22.*/
}, },
{ /* for srcSize <= 128 KB */ {
/* for srcSize <= 128 KB */
/* W, C, H, S, L, T, strat */ /* W, C, H, S, L, T, strat */
{17, 12, 12, 1, 7, 8, ZSTD_fast}, /* level 0 - not used */ {17, 12, 12, 1, 7, 8, ZSTD_fast}, /* level 0 - not used */
{17, 12, 13, 1, 6, 8, ZSTD_fast}, /* level 1 */ {17, 12, 13, 1, 6, 8, ZSTD_fast}, /* level 1 */
@@ -3202,7 +3356,8 @@ static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEV
{17, 18, 17, 10, 3, 256, ZSTD_btopt2}, /* level 21.*/ {17, 18, 17, 10, 3, 256, ZSTD_btopt2}, /* level 21.*/
{17, 18, 17, 11, 3, 512, ZSTD_btopt2}, /* level 22.*/ {17, 18, 17, 11, 3, 512, ZSTD_btopt2}, /* level 22.*/
}, },
{ /* for srcSize <= 16 KB */ {
/* for srcSize <= 16 KB */
/* W, C, H, S, L, T, strat */ /* W, C, H, S, L, T, strat */
{14, 12, 12, 1, 7, 6, ZSTD_fast}, /* level 0 - not used */ {14, 12, 12, 1, 7, 6, ZSTD_fast}, /* level 0 - not used */
{14, 14, 14, 1, 6, 6, ZSTD_fast}, /* level 1 */ {14, 14, 14, 1, 6, 6, ZSTD_fast}, /* level 1 */
@@ -3239,13 +3394,18 @@ ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, unsigned long l
size_t const addedSize = srcSize ? 0 : 500; size_t const addedSize = srcSize ? 0 : 500;
U64 const rSize = srcSize + dictSize ? srcSize + dictSize + addedSize : (U64)-1; U64 const rSize = srcSize + dictSize ? srcSize + dictSize + addedSize : (U64)-1;
U32 const tableID = (rSize <= 256 KB) + (rSize <= 128 KB) + (rSize <= 16 KB); /* intentional underflow for srcSizeHint == 0 */ U32 const tableID = (rSize <= 256 KB) + (rSize <= 128 KB) + (rSize <= 16 KB); /* intentional underflow for srcSizeHint == 0 */
if (compressionLevel <= 0) compressionLevel = ZSTD_DEFAULT_CLEVEL; /* 0 == default; no negative compressionLevel yet */ if (compressionLevel <= 0)
if (compressionLevel > ZSTD_MAX_CLEVEL) compressionLevel = ZSTD_MAX_CLEVEL; compressionLevel = ZSTD_DEFAULT_CLEVEL; /* 0 == default; no negative compressionLevel yet */
if (compressionLevel > ZSTD_MAX_CLEVEL)
compressionLevel = ZSTD_MAX_CLEVEL;
cp = ZSTD_defaultCParameters[tableID][compressionLevel]; cp = ZSTD_defaultCParameters[tableID][compressionLevel];
if (ZSTD_32bits()) { /* auto-correction, for 32-bits mode */ if (ZSTD_32bits()) { /* auto-correction, for 32-bits mode */
if (cp.windowLog > ZSTD_WINDOWLOG_MAX) cp.windowLog = ZSTD_WINDOWLOG_MAX; if (cp.windowLog > ZSTD_WINDOWLOG_MAX)
if (cp.chainLog > ZSTD_CHAINLOG_MAX) cp.chainLog = ZSTD_CHAINLOG_MAX; cp.windowLog = ZSTD_WINDOWLOG_MAX;
if (cp.hashLog > ZSTD_HASHLOG_MAX) cp.hashLog = ZSTD_HASHLOG_MAX; if (cp.chainLog > ZSTD_CHAINLOG_MAX)
cp.chainLog = ZSTD_CHAINLOG_MAX;
if (cp.hashLog > ZSTD_HASHLOG_MAX)
cp.hashLog = ZSTD_HASHLOG_MAX;
} }
cp = ZSTD_adjustCParams(cp, srcSize, dictSize); cp = ZSTD_adjustCParams(cp, srcSize, dictSize);
return cp; return cp;
@@ -3254,7 +3414,8 @@ ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, unsigned long l
/*! ZSTD_getParams() : /*! ZSTD_getParams() :
* same as ZSTD_getCParams(), but @return a `ZSTD_parameters` object (instead of `ZSTD_compressionParameters`). * same as ZSTD_getCParams(), but @return a `ZSTD_parameters` object (instead of `ZSTD_compressionParameters`).
* All fields of `ZSTD_frameParameters` are set to default (0) */ * All fields of `ZSTD_frameParameters` are set to default (0) */
ZSTD_parameters ZSTD_getParams(int compressionLevel, unsigned long long srcSize, size_t dictSize) { ZSTD_parameters ZSTD_getParams(int compressionLevel, unsigned long long srcSize, size_t dictSize)
{
ZSTD_parameters params; ZSTD_parameters params;
ZSTD_compressionParameters const cParams = ZSTD_getCParams(compressionLevel, srcSize, dictSize); ZSTD_compressionParameters const cParams = ZSTD_getCParams(compressionLevel, srcSize, dictSize);
memset(&params, 0, sizeof(params)); memset(&params, 0, sizeof(params));
contrib/linux-kernel/lib/zstd/decompress.c
+648 -496
View File
@@ -14,7 +14,6 @@
* ("BSD"). * ("BSD").
*/ */
/* *************************************************************** /* ***************************************************************
* Tuning parameters * Tuning parameters
*****************************************************************/ *****************************************************************/
@@ -27,17 +26,16 @@
#define ZSTD_MAXWINDOWSIZE_DEFAULT ((1 << ZSTD_WINDOWLOG_MAX) + 1) /* defined within zstd.h */ #define ZSTD_MAXWINDOWSIZE_DEFAULT ((1 << ZSTD_WINDOWLOG_MAX) + 1) /* defined within zstd.h */
#endif #endif
/*-******************************************************* /*-*******************************************************
* Dependencies * Dependencies
*********************************************************/ *********************************************************/
#include "fse.h"
#include "huf.h"
#include "mem.h" /* low level memory routines */
#include "zstd_internal.h"
#include <linux/kernel.h> #include <linux/kernel.h>
#include <linux/module.h> #include <linux/module.h>
#include <linux/string.h> /* memcpy, memmove, memset */ #include <linux/string.h> /* memcpy, memmove, memset */
#include "mem.h" /* low level memory routines */
#include "fse.h"
#include "huf.h"
#include "zstd_internal.h"
#define ZSTD_PREFETCH(ptr) __builtin_prefetch(ptr, 0, 0) #define ZSTD_PREFETCH(ptr) __builtin_prefetch(ptr, 0, 0)
@@ -48,20 +46,24 @@
#define FSE_isError ERR_isError #define FSE_isError ERR_isError
#define HUF_isError ERR_isError #define HUF_isError ERR_isError
/*_******************************************************* /*_*******************************************************
* Memory operations * Memory operations
**********************************************************/ **********************************************************/
static void ZSTD_copy4(void *dst, const void *src) { memcpy(dst, src, 4); } static void ZSTD_copy4(void *dst, const void *src) { memcpy(dst, src, 4); }
/*-************************************************************* /*-*************************************************************
* Context management * Context management
***************************************************************/ ***************************************************************/
typedef enum { ZSTDds_getFrameHeaderSize, ZSTDds_decodeFrameHeader, typedef enum {
ZSTDds_decodeBlockHeader, ZSTDds_decompressBlock, ZSTDds_getFrameHeaderSize,
ZSTDds_decompressLastBlock, ZSTDds_checkChecksum, ZSTDds_decodeFrameHeader,
ZSTDds_decodeSkippableHeader, ZSTDds_skipFrame } ZSTD_dStage; ZSTDds_decodeBlockHeader,
ZSTDds_decompressBlock,
ZSTDds_decompressLastBlock,
ZSTDds_checkChecksum,
ZSTDds_decodeSkippableHeader,
ZSTDds_skipFrame
} ZSTD_dStage;
typedef struct { typedef struct {
FSE_DTable LLTable[FSE_DTABLE_SIZE_U32(LLFSELog)]; FSE_DTable LLTable[FSE_DTABLE_SIZE_U32(LLFSELog)];
@@ -71,8 +73,7 @@ typedef struct {
U32 rep[ZSTD_REP_NUM]; U32 rep[ZSTD_REP_NUM];
} ZSTD_entropyTables_t; } ZSTD_entropyTables_t;
struct ZSTD_DCtx_s struct ZSTD_DCtx_s {
{
const FSE_DTable *LLTptr; const FSE_DTable *LLTptr;
const FSE_DTable *MLTptr; const FSE_DTable *MLTptr;
const FSE_DTable *OFTptr; const FSE_DTable *OFTptr;
@@ -99,10 +100,7 @@ struct ZSTD_DCtx_s
BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX]; BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX];
}; /* typedef'd to ZSTD_DCtx within "zstd.h" */ }; /* typedef'd to ZSTD_DCtx within "zstd.h" */
size_t ZSTD_DCtxWorkspaceBound(void) size_t ZSTD_DCtxWorkspaceBound(void) { return ZSTD_ALIGN(sizeof(ZSTD_stack)) + ZSTD_ALIGN(sizeof(ZSTD_DCtx)); }
{
return ZSTD_ALIGN(sizeof(ZSTD_stack)) + ZSTD_ALIGN(sizeof(ZSTD_DCtx));
}
size_t ZSTD_decompressBegin(ZSTD_DCtx *dctx) size_t ZSTD_decompressBegin(ZSTD_DCtx *dctx)
{ {
@@ -128,10 +126,12 @@ ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem)
{ {
ZSTD_DCtx *dctx; ZSTD_DCtx *dctx;
if (!customMem.customAlloc || !customMem.customFree) return NULL; if (!customMem.customAlloc || !customMem.customFree)
return NULL;
dctx = (ZSTD_DCtx *)ZSTD_malloc(sizeof(ZSTD_DCtx), customMem); dctx = (ZSTD_DCtx *)ZSTD_malloc(sizeof(ZSTD_DCtx), customMem);
if (!dctx) return NULL; if (!dctx)
return NULL;
memcpy(&dctx->customMem, &customMem, sizeof(customMem)); memcpy(&dctx->customMem, &customMem, sizeof(customMem));
ZSTD_decompressBegin(dctx); ZSTD_decompressBegin(dctx);
return dctx; return dctx;
@@ -145,7 +145,8 @@ ZSTD_DCtx* ZSTD_initDCtx(void* workspace, size_t workspaceSize)
size_t ZSTD_freeDCtx(ZSTD_DCtx *dctx) size_t ZSTD_freeDCtx(ZSTD_DCtx *dctx)
{ {
if (dctx==NULL) return 0; /* support free on NULL */ if (dctx == NULL)
return 0; /* support free on NULL */
ZSTD_free(dctx, dctx->customMem); ZSTD_free(dctx, dctx->customMem);
return 0; /* reserved as a potential error code in the future */ return 0; /* reserved as a potential error code in the future */
} }
@@ -158,7 +159,6 @@ void ZSTD_copyDCtx(ZSTD_DCtx* dstDCtx, const ZSTD_DCtx* srcDCtx)
static void ZSTD_refDDict(ZSTD_DCtx *dstDCtx, const ZSTD_DDict *ddict); static void ZSTD_refDDict(ZSTD_DCtx *dstDCtx, const ZSTD_DDict *ddict);
/*-************************************************************* /*-*************************************************************
* Decompression section * Decompression section
***************************************************************/ ***************************************************************/
@@ -170,31 +170,34 @@ static void ZSTD_refDDict(ZSTD_DCtx* dstDCtx, const ZSTD_DDict* ddict);
* Note 3 : Skippable Frame Identifiers are considered valid. */ * Note 3 : Skippable Frame Identifiers are considered valid. */
unsigned ZSTD_isFrame(const void *buffer, size_t size) unsigned ZSTD_isFrame(const void *buffer, size_t size)
{ {
if (size < 4) return 0; if (size < 4)
{ U32 const magic = ZSTD_readLE32(buffer); return 0;
if (magic == ZSTD_MAGICNUMBER) return 1; {
if ((magic & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) return 1; U32 const magic = ZSTD_readLE32(buffer);
if (magic == ZSTD_MAGICNUMBER)
return 1;
if ((magic & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START)
return 1;
} }
return 0; return 0;
} }
/** ZSTD_frameHeaderSize() : /** ZSTD_frameHeaderSize() :
* srcSize must be >= ZSTD_frameHeaderSize_prefix. * srcSize must be >= ZSTD_frameHeaderSize_prefix.
* @return : size of the Frame Header */ * @return : size of the Frame Header */
static size_t ZSTD_frameHeaderSize(const void *src, size_t srcSize) static size_t ZSTD_frameHeaderSize(const void *src, size_t srcSize)
{ {
if (srcSize < ZSTD_frameHeaderSize_prefix) return ERROR(srcSize_wrong); if (srcSize < ZSTD_frameHeaderSize_prefix)
{ BYTE const fhd = ((const BYTE*)src)[4]; return ERROR(srcSize_wrong);
{
BYTE const fhd = ((const BYTE *)src)[4];
U32 const dictID = fhd & 3; U32 const dictID = fhd & 3;
U32 const singleSegment = (fhd >> 5) & 1; U32 const singleSegment = (fhd >> 5) & 1;
U32 const fcsId = fhd >> 6; U32 const fcsId = fhd >> 6;
return ZSTD_frameHeaderSize_prefix + !singleSegment + ZSTD_did_fieldSize[dictID] + ZSTD_fcs_fieldSize[fcsId] return ZSTD_frameHeaderSize_prefix + !singleSegment + ZSTD_did_fieldSize[dictID] + ZSTD_fcs_fieldSize[fcsId] + (singleSegment && !fcsId);
+ (singleSegment && !fcsId);
} }
} }
/** ZSTD_getFrameParams() : /** ZSTD_getFrameParams() :
* decode Frame Header, or require larger `srcSize`. * decode Frame Header, or require larger `srcSize`.
* @return : 0, `fparamsPtr` is correctly filled, * @return : 0, `fparamsPtr` is correctly filled,
@@ -204,10 +207,12 @@ size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t
{ {
const BYTE *ip = (const BYTE *)src; const BYTE *ip = (const BYTE *)src;
if (srcSize < ZSTD_frameHeaderSize_prefix) return ZSTD_frameHeaderSize_prefix; if (srcSize < ZSTD_frameHeaderSize_prefix)
return ZSTD_frameHeaderSize_prefix;
if (ZSTD_readLE32(src) != ZSTD_MAGICNUMBER) { if (ZSTD_readLE32(src) != ZSTD_MAGICNUMBER) {
if ((ZSTD_readLE32(src) & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) { if ((ZSTD_readLE32(src) & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) {
if (srcSize < ZSTD_skippableHeaderSize) return ZSTD_skippableHeaderSize; /* magic number + skippable frame length */ if (srcSize < ZSTD_skippableHeaderSize)
return ZSTD_skippableHeaderSize; /* magic number + skippable frame length */
memset(fparamsPtr, 0, sizeof(*fparamsPtr)); memset(fparamsPtr, 0, sizeof(*fparamsPtr));
fparamsPtr->frameContentSize = ZSTD_readLE32((const char *)src + 4); fparamsPtr->frameContentSize = ZSTD_readLE32((const char *)src + 4);
fparamsPtr->windowSize = 0; /* windowSize==0 means a frame is skippable */ fparamsPtr->windowSize = 0; /* windowSize==0 means a frame is skippable */
@@ -217,10 +222,14 @@ size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t
} }
/* ensure there is enough `srcSize` to fully read/decode frame header */ /* ensure there is enough `srcSize` to fully read/decode frame header */
{ size_t const fhsize = ZSTD_frameHeaderSize(src, srcSize); {
if (srcSize < fhsize) return fhsize; } size_t const fhsize = ZSTD_frameHeaderSize(src, srcSize);
if (srcSize < fhsize)
return fhsize;
}
{ BYTE const fhdByte = ip[4]; {
BYTE const fhdByte = ip[4];
size_t pos = 5; size_t pos = 5;
U32 const dictIDSizeCode = fhdByte & 3; U32 const dictIDSizeCode = fhdByte & 3;
U32 const checksumFlag = (fhdByte >> 2) & 1; U32 const checksumFlag = (fhdByte >> 2) & 1;
@@ -230,33 +239,47 @@ size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t
U32 windowSize = 0; U32 windowSize = 0;
U32 dictID = 0; U32 dictID = 0;
U64 frameContentSize = 0; U64 frameContentSize = 0;
if ((fhdByte & 0x08) != 0) return ERROR(frameParameter_unsupported); /* reserved bits, which must be zero */ if ((fhdByte & 0x08) != 0)
return ERROR(frameParameter_unsupported); /* reserved bits, which must be zero */
if (!singleSegment) { if (!singleSegment) {
BYTE const wlByte = ip[pos++]; BYTE const wlByte = ip[pos++];
U32 const windowLog = (wlByte >> 3) + ZSTD_WINDOWLOG_ABSOLUTEMIN; U32 const windowLog = (wlByte >> 3) + ZSTD_WINDOWLOG_ABSOLUTEMIN;
if (windowLog > ZSTD_WINDOWLOG_MAX) return ERROR(frameParameter_windowTooLarge); /* avoids issue with 1 << windowLog */ if (windowLog > ZSTD_WINDOWLOG_MAX)
return ERROR(frameParameter_windowTooLarge); /* avoids issue with 1 << windowLog */
windowSize = (1U << windowLog); windowSize = (1U << windowLog);
windowSize += (windowSize >> 3) * (wlByte & 7); windowSize += (windowSize >> 3) * (wlByte & 7);
} }
switch(dictIDSizeCode) switch (dictIDSizeCode) {
{
default: /* impossible */ default: /* impossible */
case 0: break; case 0: break;
case 1 : dictID = ip[pos]; pos++; break; case 1:
case 2 : dictID = ZSTD_readLE16(ip+pos); pos+=2; break; dictID = ip[pos];
case 3 : dictID = ZSTD_readLE32(ip+pos); pos+=4; break; pos++;
break;
case 2:
dictID = ZSTD_readLE16(ip + pos);
pos += 2;
break;
case 3:
dictID = ZSTD_readLE32(ip + pos);
pos += 4;
break;
} }
switch(fcsID) switch (fcsID) {
{
default: /* impossible */ default: /* impossible */
case 0 : if (singleSegment) frameContentSize = ip[pos]; break; case 0:
if (singleSegment)
frameContentSize = ip[pos];
break;
case 1: frameContentSize = ZSTD_readLE16(ip + pos) + 256; break; case 1: frameContentSize = ZSTD_readLE16(ip + pos) + 256; break;
case 2: frameContentSize = ZSTD_readLE32(ip + pos); break; case 2: frameContentSize = ZSTD_readLE32(ip + pos); break;
case 3: frameContentSize = ZSTD_readLE64(ip + pos); break; case 3: frameContentSize = ZSTD_readLE64(ip + pos); break;
} }
if (!windowSize) windowSize = (U32)frameContentSize; if (!windowSize)
if (windowSize > windowSizeMax) return ERROR(frameParameter_windowTooLarge); windowSize = (U32)frameContentSize;
if (windowSize > windowSizeMax)
return ERROR(frameParameter_windowTooLarge);
fparamsPtr->frameContentSize = frameContentSize; fparamsPtr->frameContentSize = frameContentSize;
fparamsPtr->windowSize = windowSize; fparamsPtr->windowSize = windowSize;
fparamsPtr->dictID = dictID; fparamsPtr->dictID = dictID;
@@ -274,7 +297,8 @@ unsigned long long ZSTD_getFrameContentSize(const void *src, size_t srcSize)
{ {
{ {
ZSTD_frameParams fParams; ZSTD_frameParams fParams;
if (ZSTD_getFrameParams(&fParams, src, srcSize) != 0) return ZSTD_CONTENTSIZE_ERROR; if (ZSTD_getFrameParams(&fParams, src, srcSize) != 0)
return ZSTD_CONTENTSIZE_ERROR;
if (fParams.windowSize == 0) { if (fParams.windowSize == 0) {
/* Either skippable or empty frame, size == 0 either way */ /* Either skippable or empty frame, size == 0 either way */
return 0; return 0;
@@ -302,8 +326,7 @@ unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize)
size_t skippableSize; size_t skippableSize;
if (srcSize < ZSTD_skippableHeaderSize) if (srcSize < ZSTD_skippableHeaderSize)
return ERROR(srcSize_wrong); return ERROR(srcSize_wrong);
skippableSize = ZSTD_readLE32((const BYTE *)src + 4) + skippableSize = ZSTD_readLE32((const BYTE *)src + 4) + ZSTD_skippableHeaderSize;
ZSTD_skippableHeaderSize;
if (srcSize < skippableSize) { if (srcSize < skippableSize) {
return ZSTD_CONTENTSIZE_ERROR; return ZSTD_CONTENTSIZE_ERROR;
} }
@@ -315,10 +338,12 @@ unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize)
{ {
unsigned long long const ret = ZSTD_getFrameContentSize(src, srcSize); unsigned long long const ret = ZSTD_getFrameContentSize(src, srcSize);
if (ret >= ZSTD_CONTENTSIZE_ERROR) return ret; if (ret >= ZSTD_CONTENTSIZE_ERROR)
return ret;
/* check for overflow */ /* check for overflow */
if (totalDstSize + ret < totalDstSize) return ZSTD_CONTENTSIZE_ERROR; if (totalDstSize + ret < totalDstSize)
return ZSTD_CONTENTSIZE_ERROR;
totalDstSize += ret; totalDstSize += ret;
} }
{ {
@@ -346,16 +371,18 @@ unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize)
static size_t ZSTD_decodeFrameHeader(ZSTD_DCtx *dctx, const void *src, size_t headerSize) static size_t ZSTD_decodeFrameHeader(ZSTD_DCtx *dctx, const void *src, size_t headerSize)
{ {
size_t const result = ZSTD_getFrameParams(&(dctx->fParams), src, headerSize); size_t const result = ZSTD_getFrameParams(&(dctx->fParams), src, headerSize);
if (ZSTD_isError(result)) return result; /* invalid header */ if (ZSTD_isError(result))
if (result>0) return ERROR(srcSize_wrong); /* headerSize too small */ return result; /* invalid header */
if (dctx->fParams.dictID && (dctx->dictID != dctx->fParams.dictID)) return ERROR(dictionary_wrong); if (result > 0)
if (dctx->fParams.checksumFlag) xxh64_reset(&dctx->xxhState, 0); return ERROR(srcSize_wrong); /* headerSize too small */
if (dctx->fParams.dictID && (dctx->dictID != dctx->fParams.dictID))
return ERROR(dictionary_wrong);
if (dctx->fParams.checksumFlag)
xxh64_reset(&dctx->xxhState, 0);
return 0; return 0;
} }
typedef struct {
typedef struct
{
blockType_e blockType; blockType_e blockType;
U32 lastBlock; U32 lastBlock;
U32 origSize; U32 origSize;
@@ -365,59 +392,68 @@ typedef struct
* Provides the size of compressed block from block header `src` */ * Provides the size of compressed block from block header `src` */
size_t ZSTD_getcBlockSize(const void *src, size_t srcSize, blockProperties_t *bpPtr) size_t ZSTD_getcBlockSize(const void *src, size_t srcSize, blockProperties_t *bpPtr)
{ {
if (srcSize < ZSTD_blockHeaderSize) return ERROR(srcSize_wrong); if (srcSize < ZSTD_blockHeaderSize)
{ U32 const cBlockHeader = ZSTD_readLE24(src); return ERROR(srcSize_wrong);
{
U32 const cBlockHeader = ZSTD_readLE24(src);
U32 const cSize = cBlockHeader >> 3; U32 const cSize = cBlockHeader >> 3;
bpPtr->lastBlock = cBlockHeader & 1; bpPtr->lastBlock = cBlockHeader & 1;
bpPtr->blockType = (blockType_e)((cBlockHeader >> 1) & 3); bpPtr->blockType = (blockType_e)((cBlockHeader >> 1) & 3);
bpPtr->origSize = cSize; /* only useful for RLE */ bpPtr->origSize = cSize; /* only useful for RLE */
if (bpPtr->blockType == bt_rle) return 1; if (bpPtr->blockType == bt_rle)
if (bpPtr->blockType == bt_reserved) return ERROR(corruption_detected); return 1;
if (bpPtr->blockType == bt_reserved)
return ERROR(corruption_detected);
return cSize; return cSize;
} }
} }
static size_t ZSTD_copyRawBlock(void *dst, size_t dstCapacity, const void *src, size_t srcSize) static size_t ZSTD_copyRawBlock(void *dst, size_t dstCapacity, const void *src, size_t srcSize)
{ {
if (srcSize > dstCapacity) return ERROR(dstSize_tooSmall); if (srcSize > dstCapacity)
return ERROR(dstSize_tooSmall);
memcpy(dst, src, srcSize); memcpy(dst, src, srcSize);
return srcSize; return srcSize;
} }
static size_t ZSTD_setRleBlock(void *dst, size_t dstCapacity, const void *src, size_t srcSize, size_t regenSize) static size_t ZSTD_setRleBlock(void *dst, size_t dstCapacity, const void *src, size_t srcSize, size_t regenSize)
{ {
if (srcSize != 1) return ERROR(srcSize_wrong); if (srcSize != 1)
if (regenSize > dstCapacity) return ERROR(dstSize_tooSmall); return ERROR(srcSize_wrong);
if (regenSize > dstCapacity)
return ERROR(dstSize_tooSmall);
memset(dst, *(const BYTE *)src, regenSize); memset(dst, *(const BYTE *)src, regenSize);
return regenSize; return regenSize;
} }
/*! ZSTD_decodeLiteralsBlock() : /*! ZSTD_decodeLiteralsBlock() :
@return : nb of bytes read from src (< srcSize ) */ @return : nb of bytes read from src (< srcSize ) */
size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx, size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx *dctx, const void *src, size_t srcSize) /* note : srcSize < BLOCKSIZE */
const void* src, size_t srcSize) /* note : srcSize < BLOCKSIZE */
{ {
if (srcSize < MIN_CBLOCK_SIZE) return ERROR(corruption_detected); if (srcSize < MIN_CBLOCK_SIZE)
return ERROR(corruption_detected);
{ const BYTE* const istart = (const BYTE*) src; {
const BYTE *const istart = (const BYTE *)src;
symbolEncodingType_e const litEncType = (symbolEncodingType_e)(istart[0] & 3); symbolEncodingType_e const litEncType = (symbolEncodingType_e)(istart[0] & 3);
switch(litEncType) switch (litEncType) {
{
case set_repeat: case set_repeat:
if (dctx->litEntropy==0) return ERROR(dictionary_corrupted); if (dctx->litEntropy == 0)
return ERROR(dictionary_corrupted);
/* fall-through */ /* fall-through */
case set_compressed: case set_compressed:
if (srcSize < 5) return ERROR(corruption_detected); /* srcSize >= MIN_CBLOCK_SIZE == 3; here we need up to 5 for case 3 */ if (srcSize < 5)
{ size_t lhSize, litSize, litCSize; return ERROR(corruption_detected); /* srcSize >= MIN_CBLOCK_SIZE == 3; here we need up to 5 for case 3 */
{
size_t lhSize, litSize, litCSize;
U32 singleStream = 0; U32 singleStream = 0;
U32 const lhlCode = (istart[0] >> 2) & 3; U32 const lhlCode = (istart[0] >> 2) & 3;
U32 const lhc = ZSTD_readLE32(istart); U32 const lhc = ZSTD_readLE32(istart);
switch(lhlCode) switch (lhlCode) {
{ case 0:
case 0: case 1: default: /* note : default is impossible, since lhlCode into [0..3] */ case 1:
default: /* note : default is impossible, since lhlCode into [0..3] */
/* 2 - 2 - 10 - 10 */ /* 2 - 2 - 10 - 10 */
singleStream = !lhlCode; singleStream = !lhlCode;
lhSize = 3; lhSize = 3;
@@ -437,32 +473,36 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx,
litCSize = (lhc >> 22) + (istart[4] << 10); litCSize = (lhc >> 22) + (istart[4] << 10);
break; break;
} }
if (litSize > ZSTD_BLOCKSIZE_ABSOLUTEMAX) return ERROR(corruption_detected); if (litSize > ZSTD_BLOCKSIZE_ABSOLUTEMAX)
if (litCSize + lhSize > srcSize) return ERROR(corruption_detected); return ERROR(corruption_detected);
if (litCSize + lhSize > srcSize)
return ERROR(corruption_detected);
if (HUF_isError((litEncType==set_repeat) ? if (HUF_isError(
( singleStream ? (litEncType == set_repeat)
HUF_decompress1X_usingDTable(dctx->litBuffer, litSize, istart+lhSize, litCSize, dctx->HUFptr) : ? (singleStream ? HUF_decompress1X_usingDTable(dctx->litBuffer, litSize, istart + lhSize, litCSize, dctx->HUFptr)
HUF_decompress4X_usingDTable(dctx->litBuffer, litSize, istart+lhSize, litCSize, dctx->HUFptr) ) : : HUF_decompress4X_usingDTable(dctx->litBuffer, litSize, istart + lhSize, litCSize, dctx->HUFptr))
( singleStream ? : (singleStream
HUF_decompress1X2_DCtx(dctx->entropy.hufTable, dctx->litBuffer, litSize, istart+lhSize, litCSize) : ? HUF_decompress1X2_DCtx(dctx->entropy.hufTable, dctx->litBuffer, litSize, istart + lhSize, litCSize)
HUF_decompress4X_hufOnly (dctx->entropy.hufTable, dctx->litBuffer, litSize, istart+lhSize, litCSize)) )) : HUF_decompress4X_hufOnly(dctx->entropy.hufTable, dctx->litBuffer, litSize, istart + lhSize, litCSize))))
return ERROR(corruption_detected); return ERROR(corruption_detected);
dctx->litPtr = dctx->litBuffer; dctx->litPtr = dctx->litBuffer;
dctx->litSize = litSize; dctx->litSize = litSize;
dctx->litEntropy = 1; dctx->litEntropy = 1;
if (litEncType==set_compressed) dctx->HUFptr = dctx->entropy.hufTable; if (litEncType == set_compressed)
dctx->HUFptr = dctx->entropy.hufTable;
memset(dctx->litBuffer + dctx->litSize, 0, WILDCOPY_OVERLENGTH); memset(dctx->litBuffer + dctx->litSize, 0, WILDCOPY_OVERLENGTH);
return litCSize + lhSize; return litCSize + lhSize;
} }
case set_basic: case set_basic: {
{ size_t litSize, lhSize; size_t litSize, lhSize;
U32 const lhlCode = ((istart[0]) >> 2) & 3; U32 const lhlCode = ((istart[0]) >> 2) & 3;
switch(lhlCode) switch (lhlCode) {
{ case 0:
case 0: case 2: default: /* note : default is impossible, since lhlCode into [0..3] */ case 2:
default: /* note : default is impossible, since lhlCode into [0..3] */
lhSize = 1; lhSize = 1;
litSize = istart[0] >> 3; litSize = istart[0] >> 3;
break; break;
@@ -477,7 +517,8 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx,
} }
if (lhSize + litSize + WILDCOPY_OVERLENGTH > srcSize) { /* risk reading beyond src buffer with wildcopy */ if (lhSize + litSize + WILDCOPY_OVERLENGTH > srcSize) { /* risk reading beyond src buffer with wildcopy */
if (litSize+lhSize > srcSize) return ERROR(corruption_detected); if (litSize + lhSize > srcSize)
return ERROR(corruption_detected);
memcpy(dctx->litBuffer, istart + lhSize, litSize); memcpy(dctx->litBuffer, istart + lhSize, litSize);
dctx->litPtr = dctx->litBuffer; dctx->litPtr = dctx->litBuffer;
dctx->litSize = litSize; dctx->litSize = litSize;
@@ -490,12 +531,13 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx,
return lhSize + litSize; return lhSize + litSize;
} }
case set_rle: case set_rle: {
{ U32 const lhlCode = ((istart[0]) >> 2) & 3; U32 const lhlCode = ((istart[0]) >> 2) & 3;
size_t litSize, lhSize; size_t litSize, lhSize;
switch(lhlCode) switch (lhlCode) {
{ case 0:
case 0: case 2: default: /* note : default is impossible, since lhlCode into [0..3] */ case 2:
default: /* note : default is impossible, since lhlCode into [0..3] */
lhSize = 1; lhSize = 1;
litSize = istart[0] >> 3; litSize = istart[0] >> 3;
break; break;
@@ -506,10 +548,12 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx,
case 3: case 3:
lhSize = 3; lhSize = 3;
litSize = ZSTD_readLE24(istart) >> 4; litSize = ZSTD_readLE24(istart) >> 4;
if (srcSize<4) return ERROR(corruption_detected); /* srcSize >= MIN_CBLOCK_SIZE == 3; here we need lhSize+1 = 4 */ if (srcSize < 4)
return ERROR(corruption_detected); /* srcSize >= MIN_CBLOCK_SIZE == 3; here we need lhSize+1 = 4 */
break; break;
} }
if (litSize > ZSTD_BLOCKSIZE_ABSOLUTEMAX) return ERROR(corruption_detected); if (litSize > ZSTD_BLOCKSIZE_ABSOLUTEMAX)
return ERROR(corruption_detected);
memset(dctx->litBuffer, istart[lhSize], litSize + WILDCOPY_OVERLENGTH); memset(dctx->litBuffer, istart[lhSize], litSize + WILDCOPY_OVERLENGTH);
dctx->litPtr = dctx->litBuffer; dctx->litPtr = dctx->litBuffer;
dctx->litSize = litSize; dctx->litSize = litSize;
@@ -521,7 +565,6 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx,
} }
} }
typedef union { typedef union {
FSE_decode_t realData; FSE_decode_t realData;
U32 alignedBy4; U32 alignedBy4;
@@ -703,58 +746,65 @@ static const FSE_decode_t4 OF_defaultDTable[(1<<OF_DEFAULTNORMLOG)+1] = {
@return : nb bytes read from src, @return : nb bytes read from src,
or an error code if it fails, testable with ZSTD_isError() or an error code if it fails, testable with ZSTD_isError()
*/ */
static size_t ZSTD_buildSeqTable(FSE_DTable* DTableSpace, const FSE_DTable** DTablePtr, static size_t ZSTD_buildSeqTable(FSE_DTable *DTableSpace, const FSE_DTable **DTablePtr, symbolEncodingType_e type, U32 max, U32 maxLog, const void *src,
symbolEncodingType_e type, U32 max, U32 maxLog, size_t srcSize, const FSE_decode_t4 *defaultTable, U32 flagRepeatTable)
const void* src, size_t srcSize,
const FSE_decode_t4* defaultTable, U32 flagRepeatTable)
{ {
const void *const tmpPtr = defaultTable; /* bypass strict aliasing */ const void *const tmpPtr = defaultTable; /* bypass strict aliasing */
switch(type) switch (type) {
{
case set_rle: case set_rle:
if (!srcSize) return ERROR(srcSize_wrong); if (!srcSize)
if ( (*(const BYTE*)src) > max) return ERROR(corruption_detected); return ERROR(srcSize_wrong);
if ((*(const BYTE *)src) > max)
return ERROR(corruption_detected);
FSE_buildDTable_rle(DTableSpace, *(const BYTE *)src); FSE_buildDTable_rle(DTableSpace, *(const BYTE *)src);
*DTablePtr = DTableSpace; *DTablePtr = DTableSpace;
return 1; return 1;
case set_basic : case set_basic: *DTablePtr = (const FSE_DTable *)tmpPtr; return 0;
*DTablePtr = (const FSE_DTable*)tmpPtr;
return 0;
case set_repeat: case set_repeat:
if (!flagRepeatTable) return ERROR(corruption_detected); if (!flagRepeatTable)
return ERROR(corruption_detected);
return 0; return 0;
default: /* impossible */ default: /* impossible */
case set_compressed : case set_compressed: {
{ U32 tableLog; U32 tableLog;
S16 norm[MaxSeq + 1]; S16 norm[MaxSeq + 1];
size_t const headerSize = FSE_readNCount(norm, &max, &tableLog, src, srcSize); size_t const headerSize = FSE_readNCount(norm, &max, &tableLog, src, srcSize);
if (FSE_isError(headerSize)) return ERROR(corruption_detected); if (FSE_isError(headerSize))
if (tableLog > maxLog) return ERROR(corruption_detected); return ERROR(corruption_detected);
if (tableLog > maxLog)
return ERROR(corruption_detected);
FSE_buildDTable(DTableSpace, norm, max, tableLog); FSE_buildDTable(DTableSpace, norm, max, tableLog);
*DTablePtr = DTableSpace; *DTablePtr = DTableSpace;
return headerSize; return headerSize;
} } }
}
} }
size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeqPtr, size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx *dctx, int *nbSeqPtr, const void *src, size_t srcSize)
const void* src, size_t srcSize)
{ {
const BYTE *const istart = (const BYTE *const)src; const BYTE *const istart = (const BYTE *const)src;
const BYTE *const iend = istart + srcSize; const BYTE *const iend = istart + srcSize;
const BYTE *ip = istart; const BYTE *ip = istart;
/* check */ /* check */
if (srcSize < MIN_SEQUENCES_SIZE) return ERROR(srcSize_wrong); if (srcSize < MIN_SEQUENCES_SIZE)
return ERROR(srcSize_wrong);
/* SeqHead */ /* SeqHead */
{ int nbSeq = *ip++; {
if (!nbSeq) { *nbSeqPtr=0; return 1; } int nbSeq = *ip++;
if (!nbSeq) {
*nbSeqPtr = 0;
return 1;
}
if (nbSeq > 0x7F) { if (nbSeq > 0x7F) {
if (nbSeq == 0xFF) { if (nbSeq == 0xFF) {
if (ip+2 > iend) return ERROR(srcSize_wrong); if (ip + 2 > iend)
return ERROR(srcSize_wrong);
nbSeq = ZSTD_readLE16(ip) + LONGNBSEQ, ip += 2; nbSeq = ZSTD_readLE16(ip) + LONGNBSEQ, ip += 2;
} else { } else {
if (ip >= iend) return ERROR(srcSize_wrong); if (ip >= iend)
return ERROR(srcSize_wrong);
nbSeq = ((nbSeq - 0x80) << 8) + *ip++; nbSeq = ((nbSeq - 0x80) << 8) + *ip++;
} }
} }
@@ -762,29 +812,34 @@ size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeqPtr,
} }
/* FSE table descriptors */ /* FSE table descriptors */
if (ip+4 > iend) return ERROR(srcSize_wrong); /* minimum possible size */ if (ip + 4 > iend)
{ symbolEncodingType_e const LLtype = (symbolEncodingType_e)(*ip >> 6); return ERROR(srcSize_wrong); /* minimum possible size */
{
symbolEncodingType_e const LLtype = (symbolEncodingType_e)(*ip >> 6);
symbolEncodingType_e const OFtype = (symbolEncodingType_e)((*ip >> 4) & 3); symbolEncodingType_e const OFtype = (symbolEncodingType_e)((*ip >> 4) & 3);
symbolEncodingType_e const MLtype = (symbolEncodingType_e)((*ip >> 2) & 3); symbolEncodingType_e const MLtype = (symbolEncodingType_e)((*ip >> 2) & 3);
ip++; ip++;
/* Build DTables */ /* Build DTables */
{ size_t const llhSize = ZSTD_buildSeqTable(dctx->entropy.LLTable, &dctx->LLTptr, {
LLtype, MaxLL, LLFSELog, size_t const llhSize = ZSTD_buildSeqTable(dctx->entropy.LLTable, &dctx->LLTptr, LLtype, MaxLL, LLFSELog, ip, iend - ip,
ip, iend-ip, LL_defaultDTable, dctx->fseEntropy); LL_defaultDTable, dctx->fseEntropy);
if (ZSTD_isError(llhSize)) return ERROR(corruption_detected); if (ZSTD_isError(llhSize))
return ERROR(corruption_detected);
ip += llhSize; ip += llhSize;
} }
{ size_t const ofhSize = ZSTD_buildSeqTable(dctx->entropy.OFTable, &dctx->OFTptr, {
OFtype, MaxOff, OffFSELog, size_t const ofhSize = ZSTD_buildSeqTable(dctx->entropy.OFTable, &dctx->OFTptr, OFtype, MaxOff, OffFSELog, ip, iend - ip,
ip, iend-ip, OF_defaultDTable, dctx->fseEntropy); OF_defaultDTable, dctx->fseEntropy);
if (ZSTD_isError(ofhSize)) return ERROR(corruption_detected); if (ZSTD_isError(ofhSize))
return ERROR(corruption_detected);
ip += ofhSize; ip += ofhSize;
} }
{ size_t const mlhSize = ZSTD_buildSeqTable(dctx->entropy.MLTable, &dctx->MLTptr, {
MLtype, MaxML, MLFSELog, size_t const mlhSize = ZSTD_buildSeqTable(dctx->entropy.MLTable, &dctx->MLTptr, MLtype, MaxML, MLFSELog, ip, iend - ip,
ip, iend-ip, ML_defaultDTable, dctx->fseEntropy); ML_defaultDTable, dctx->fseEntropy);
if (ZSTD_isError(mlhSize)) return ERROR(corruption_detected); if (ZSTD_isError(mlhSize))
return ERROR(corruption_detected);
ip += mlhSize; ip += mlhSize;
} }
} }
@@ -792,7 +847,6 @@ size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeqPtr,
return ip - istart; return ip - istart;
} }
typedef struct { typedef struct {
size_t litLength; size_t litLength;
size_t matchLength; size_t matchLength;
@@ -811,12 +865,9 @@ typedef struct {
uPtrDiff gotoDict; uPtrDiff gotoDict;
} seqState_t; } seqState_t;
FORCE_NOINLINE FORCE_NOINLINE
size_t ZSTD_execSequenceLast7(BYTE* op, size_t ZSTD_execSequenceLast7(BYTE *op, BYTE *const oend, seq_t sequence, const BYTE **litPtr, const BYTE *const litLimit, const BYTE *const base,
BYTE* const oend, seq_t sequence, const BYTE *const vBase, const BYTE *const dictEnd)
const BYTE** litPtr, const BYTE* const litLimit,
const BYTE* const base, const BYTE* const vBase, const BYTE* const dictEnd)
{ {
BYTE *const oLitEnd = op + sequence.litLength; BYTE *const oLitEnd = op + sequence.litLength;
size_t const sequenceLength = sequence.litLength + sequence.matchLength; size_t const sequenceLength = sequence.litLength + sequence.matchLength;
@@ -826,9 +877,12 @@ size_t ZSTD_execSequenceLast7(BYTE* op,
const BYTE *match = oLitEnd - sequence.offset; const BYTE *match = oLitEnd - sequence.offset;
/* check */ /* check */
if (oMatchEnd>oend) return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of WILDCOPY_OVERLENGTH from oend */ if (oMatchEnd > oend)
if (iLitEnd > litLimit) return ERROR(corruption_detected); /* over-read beyond lit buffer */ return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of WILDCOPY_OVERLENGTH from oend */
if (oLitEnd <= oend_w) return ERROR(GENERIC); /* Precondition */ if (iLitEnd > litLimit)
return ERROR(corruption_detected); /* over-read beyond lit buffer */
if (oLitEnd <= oend_w)
return ERROR(GENERIC); /* Precondition */
/* copy literals */ /* copy literals */
if (op < oend_w) { if (op < oend_w) {
@@ -836,31 +890,33 @@ size_t ZSTD_execSequenceLast7(BYTE* op,
*litPtr += oend_w - op; *litPtr += oend_w - op;
op = oend_w; op = oend_w;
} }
while (op < oLitEnd) *op++ = *(*litPtr)++; while (op < oLitEnd)
*op++ = *(*litPtr)++;
/* copy Match */ /* copy Match */
if (sequence.offset > (size_t)(oLitEnd - base)) { if (sequence.offset > (size_t)(oLitEnd - base)) {
/* offset beyond prefix */ /* offset beyond prefix */
if (sequence.offset > (size_t)(oLitEnd - vBase)) return ERROR(corruption_detected); if (sequence.offset > (size_t)(oLitEnd - vBase))
return ERROR(corruption_detected);
match = dictEnd - (base - match); match = dictEnd - (base - match);
if (match + sequence.matchLength <= dictEnd) { if (match + sequence.matchLength <= dictEnd) {
memmove(oLitEnd, match, sequence.matchLength); memmove(oLitEnd, match, sequence.matchLength);
return sequenceLength; return sequenceLength;
} }
/* span extDict & currPrefixSegment */ /* span extDict & currPrefixSegment */
{ size_t const length1 = dictEnd - match; {
size_t const length1 = dictEnd - match;
memmove(oLitEnd, match, length1); memmove(oLitEnd, match, length1);
op = oLitEnd + length1; op = oLitEnd + length1;
sequence.matchLength -= length1; sequence.matchLength -= length1;
match = base; match = base;
} } }
while (op < oMatchEnd) *op++ = *match++; }
while (op < oMatchEnd)
*op++ = *match++;
return sequenceLength; return sequenceLength;
} }
static seq_t ZSTD_decodeSequence(seqState_t *seqState) static seq_t ZSTD_decodeSequence(seqState_t *seqState)
{ {
seq_t seq; seq_t seq;
@@ -874,30 +930,26 @@ static seq_t ZSTD_decodeSequence(seqState_t* seqState)
U32 const ofBits = ofCode; U32 const ofBits = ofCode;
U32 const totalBits = llBits + mlBits + ofBits; U32 const totalBits = llBits + mlBits + ofBits;
static const U32 LL_base[MaxLL+1] = { static const U32 LL_base[MaxLL + 1] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 18,
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 20, 22, 24, 28, 32, 40, 48, 64, 0x80, 0x100, 0x200, 0x400, 0x800, 0x1000, 0x2000, 0x4000, 0x8000, 0x10000};
16, 18, 20, 22, 24, 28, 32, 40, 48, 64, 0x80, 0x100, 0x200, 0x400, 0x800, 0x1000,
0x2000, 0x4000, 0x8000, 0x10000 };
static const U32 ML_base[MaxML+1] = { static const U32 ML_base[MaxML + 1] = {3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20,
3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 37, 39, 41,
19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 43, 47, 51, 59, 67, 83, 99, 0x83, 0x103, 0x203, 0x403, 0x803, 0x1003, 0x2003, 0x4003, 0x8003, 0x10003};
35, 37, 39, 41, 43, 47, 51, 59, 67, 83, 99, 0x83, 0x103, 0x203, 0x403, 0x803,
0x1003, 0x2003, 0x4003, 0x8003, 0x10003 };
static const U32 OF_base[MaxOff+1] = { static const U32 OF_base[MaxOff + 1] = {0, 1, 1, 5, 0xD, 0x1D, 0x3D, 0x7D, 0xFD, 0x1FD,
0, 1, 1, 5, 0xD, 0x1D, 0x3D, 0x7D, 0x3FD, 0x7FD, 0xFFD, 0x1FFD, 0x3FFD, 0x7FFD, 0xFFFD, 0x1FFFD, 0x3FFFD, 0x7FFFD,
0xFD, 0x1FD, 0x3FD, 0x7FD, 0xFFD, 0x1FFD, 0x3FFD, 0x7FFD, 0xFFFFD, 0x1FFFFD, 0x3FFFFD, 0x7FFFFD, 0xFFFFFD, 0x1FFFFFD, 0x3FFFFFD, 0x7FFFFFD, 0xFFFFFFD};
0xFFFD, 0x1FFFD, 0x3FFFD, 0x7FFFD, 0xFFFFD, 0x1FFFFD, 0x3FFFFD, 0x7FFFFD,
0xFFFFFD, 0x1FFFFFD, 0x3FFFFFD, 0x7FFFFFD, 0xFFFFFFD };
/* sequence */ /* sequence */
{ size_t offset; {
size_t offset;
if (!ofCode) if (!ofCode)
offset = 0; offset = 0;
else { else {
offset = OF_base[ofCode] + BIT_readBitsFast(&seqState->DStream, ofBits); /* <= (ZSTD_WINDOWLOG_MAX-1) bits */ offset = OF_base[ofCode] + BIT_readBitsFast(&seqState->DStream, ofBits); /* <= (ZSTD_WINDOWLOG_MAX-1) bits */
if (ZSTD_32bits()) BIT_reloadDStream(&seqState->DStream); if (ZSTD_32bits())
BIT_reloadDStream(&seqState->DStream);
} }
if (ofCode <= 1) { if (ofCode <= 1) {
@@ -905,7 +957,8 @@ static seq_t ZSTD_decodeSequence(seqState_t* seqState)
if (offset) { if (offset) {
size_t temp = (offset == 3) ? seqState->prevOffset[0] - 1 : seqState->prevOffset[offset]; size_t temp = (offset == 3) ? seqState->prevOffset[0] - 1 : seqState->prevOffset[offset];
temp += !temp; /* 0 is not valid; input is corrupted; force offset to 1 */ temp += !temp; /* 0 is not valid; input is corrupted; force offset to 1 */
if (offset != 1) seqState->prevOffset[2] = seqState->prevOffset[1]; if (offset != 1)
seqState->prevOffset[2] = seqState->prevOffset[1];
seqState->prevOffset[1] = seqState->prevOffset[0]; seqState->prevOffset[1] = seqState->prevOffset[0];
seqState->prevOffset[0] = offset = temp; seqState->prevOffset[0] = offset = temp;
} else { } else {
@@ -920,27 +973,26 @@ static seq_t ZSTD_decodeSequence(seqState_t* seqState)
} }
seq.matchLength = ML_base[mlCode] + ((mlCode > 31) ? BIT_readBitsFast(&seqState->DStream, mlBits) : 0); /* <= 16 bits */ seq.matchLength = ML_base[mlCode] + ((mlCode > 31) ? BIT_readBitsFast(&seqState->DStream, mlBits) : 0); /* <= 16 bits */
if (ZSTD_32bits() && (mlBits+llBits>24)) BIT_reloadDStream(&seqState->DStream); if (ZSTD_32bits() && (mlBits + llBits > 24))
BIT_reloadDStream(&seqState->DStream);
seq.litLength = LL_base[llCode] + ((llCode > 15) ? BIT_readBitsFast(&seqState->DStream, llBits) : 0); /* <= 16 bits */ seq.litLength = LL_base[llCode] + ((llCode > 15) ? BIT_readBitsFast(&seqState->DStream, llBits) : 0); /* <= 16 bits */
if (ZSTD_32bits() || if (ZSTD_32bits() || (totalBits > 64 - 7 - (LLFSELog + MLFSELog + OffFSELog)))
(totalBits > 64 - 7 - (LLFSELog+MLFSELog+OffFSELog)) ) BIT_reloadDStream(&seqState->DStream); BIT_reloadDStream(&seqState->DStream);
/* ANS state update */ /* ANS state update */
FSE_updateState(&seqState->stateLL, &seqState->DStream); /* <= 9 bits */ FSE_updateState(&seqState->stateLL, &seqState->DStream); /* <= 9 bits */
FSE_updateState(&seqState->stateML, &seqState->DStream); /* <= 9 bits */ FSE_updateState(&seqState->stateML, &seqState->DStream); /* <= 9 bits */
if (ZSTD_32bits()) BIT_reloadDStream(&seqState->DStream); /* <= 18 bits */ if (ZSTD_32bits())
BIT_reloadDStream(&seqState->DStream); /* <= 18 bits */
FSE_updateState(&seqState->stateOffb, &seqState->DStream); /* <= 8 bits */ FSE_updateState(&seqState->stateOffb, &seqState->DStream); /* <= 8 bits */
return seq; return seq;
} }
FORCE_INLINE FORCE_INLINE
size_t ZSTD_execSequence(BYTE* op, size_t ZSTD_execSequence(BYTE *op, BYTE *const oend, seq_t sequence, const BYTE **litPtr, const BYTE *const litLimit, const BYTE *const base,
BYTE* const oend, seq_t sequence, const BYTE *const vBase, const BYTE *const dictEnd)
const BYTE** litPtr, const BYTE* const litLimit,
const BYTE* const base, const BYTE* const vBase, const BYTE* const dictEnd)
{ {
BYTE *const oLitEnd = op + sequence.litLength; BYTE *const oLitEnd = op + sequence.litLength;
size_t const sequenceLength = sequence.litLength + sequence.matchLength; size_t const sequenceLength = sequence.litLength + sequence.matchLength;
@@ -950,38 +1002,46 @@ size_t ZSTD_execSequence(BYTE* op,
const BYTE *match = oLitEnd - sequence.offset; const BYTE *match = oLitEnd - sequence.offset;
/* check */ /* check */
if (oMatchEnd>oend) return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of WILDCOPY_OVERLENGTH from oend */ if (oMatchEnd > oend)
if (iLitEnd > litLimit) return ERROR(corruption_detected); /* over-read beyond lit buffer */ return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of WILDCOPY_OVERLENGTH from oend */
if (oLitEnd>oend_w) return ZSTD_execSequenceLast7(op, oend, sequence, litPtr, litLimit, base, vBase, dictEnd); if (iLitEnd > litLimit)
return ERROR(corruption_detected); /* over-read beyond lit buffer */
if (oLitEnd > oend_w)
return ZSTD_execSequenceLast7(op, oend, sequence, litPtr, litLimit, base, vBase, dictEnd);
/* copy Literals */ /* copy Literals */
ZSTD_copy8(op, *litPtr); ZSTD_copy8(op, *litPtr);
if (sequence.litLength > 8) if (sequence.litLength > 8)
ZSTD_wildcopy(op+8, (*litPtr)+8, sequence.litLength - 8); /* note : since oLitEnd <= oend-WILDCOPY_OVERLENGTH, no risk of overwrite beyond oend */ ZSTD_wildcopy(op + 8, (*litPtr) + 8,
sequence.litLength - 8); /* note : since oLitEnd <= oend-WILDCOPY_OVERLENGTH, no risk of overwrite beyond oend */
op = oLitEnd; op = oLitEnd;
*litPtr = iLitEnd; /* update for next sequence */ *litPtr = iLitEnd; /* update for next sequence */
/* copy Match */ /* copy Match */
if (sequence.offset > (size_t)(oLitEnd - base)) { if (sequence.offset > (size_t)(oLitEnd - base)) {
/* offset beyond prefix */ /* offset beyond prefix */
if (sequence.offset > (size_t)(oLitEnd - vBase)) return ERROR(corruption_detected); if (sequence.offset > (size_t)(oLitEnd - vBase))
return ERROR(corruption_detected);
match = dictEnd + (match - base); match = dictEnd + (match - base);
if (match + sequence.matchLength <= dictEnd) { if (match + sequence.matchLength <= dictEnd) {
memmove(oLitEnd, match, sequence.matchLength); memmove(oLitEnd, match, sequence.matchLength);
return sequenceLength; return sequenceLength;
} }
/* span extDict & currPrefixSegment */ /* span extDict & currPrefixSegment */
{ size_t const length1 = dictEnd - match; {
size_t const length1 = dictEnd - match;
memmove(oLitEnd, match, length1); memmove(oLitEnd, match, length1);
op = oLitEnd + length1; op = oLitEnd + length1;
sequence.matchLength -= length1; sequence.matchLength -= length1;
match = base; match = base;
if (op > oend_w || sequence.matchLength < MINMATCH) { if (op > oend_w || sequence.matchLength < MINMATCH) {
U32 i; U32 i;
for (i = 0; i < sequence.matchLength; ++i) op[i] = match[i]; for (i = 0; i < sequence.matchLength; ++i)
op[i] = match[i];
return sequenceLength; return sequenceLength;
} }
} } }
}
/* Requirement: op <= oend_w && sequence.matchLength >= MINMATCH */ /* Requirement: op <= oend_w && sequence.matchLength >= MINMATCH */
/* match within prefix */ /* match within prefix */
@@ -1000,7 +1060,8 @@ size_t ZSTD_execSequence(BYTE* op,
} else { } else {
ZSTD_copy8(op, match); ZSTD_copy8(op, match);
} }
op += 8; match += 8; op += 8;
match += 8;
if (oMatchEnd > oend - (16 - MINMATCH)) { if (oMatchEnd > oend - (16 - MINMATCH)) {
if (op < oend_w) { if (op < oend_w) {
@@ -1008,18 +1069,15 @@ size_t ZSTD_execSequence(BYTE* op,
match += oend_w - op; match += oend_w - op;
op = oend_w; op = oend_w;
} }
while (op < oMatchEnd) *op++ = *match++; while (op < oMatchEnd)
*op++ = *match++;
} else { } else {
ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength - 8); /* works even if matchLength < 8 */ ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength - 8); /* works even if matchLength < 8 */
} }
return sequenceLength; return sequenceLength;
} }
static size_t ZSTD_decompressSequences(ZSTD_DCtx *dctx, void *dst, size_t maxDstSize, const void *seqStart, size_t seqSize)
static size_t ZSTD_decompressSequences(
ZSTD_DCtx* dctx,
void* dst, size_t maxDstSize,
const void* seqStart, size_t seqSize)
{ {
const BYTE *ip = (const BYTE *)seqStart; const BYTE *ip = (const BYTE *)seqStart;
const BYTE *const iend = ip + seqSize; const BYTE *const iend = ip + seqSize;
@@ -1034,8 +1092,10 @@ static size_t ZSTD_decompressSequences(
int nbSeq; int nbSeq;
/* Build Decoding Tables */ /* Build Decoding Tables */
{ size_t const seqHSize = ZSTD_decodeSeqHeaders(dctx, &nbSeq, ip, seqSize); {
if (ZSTD_isError(seqHSize)) return seqHSize; size_t const seqHSize = ZSTD_decodeSeqHeaders(dctx, &nbSeq, ip, seqSize);
if (ZSTD_isError(seqHSize))
return seqHSize;
ip += seqHSize; ip += seqHSize;
} }
@@ -1043,7 +1103,11 @@ static size_t ZSTD_decompressSequences(
if (nbSeq) { if (nbSeq) {
seqState_t seqState; seqState_t seqState;
dctx->fseEntropy = 1; dctx->fseEntropy = 1;
{ U32 i; for (i=0; i<ZSTD_REP_NUM; i++) seqState.prevOffset[i] = dctx->entropy.rep[i]; } {
U32 i;
for (i = 0; i < ZSTD_REP_NUM; i++)
seqState.prevOffset[i] = dctx->entropy.rep[i];
}
CHECK_E(BIT_initDStream(&seqState.DStream, ip, iend - ip), corruption_detected); CHECK_E(BIT_initDStream(&seqState.DStream, ip, iend - ip), corruption_detected);
FSE_initDState(&seqState.stateLL, &seqState.DStream, dctx->LLTptr); FSE_initDState(&seqState.stateLL, &seqState.DStream, dctx->LLTptr);
FSE_initDState(&seqState.stateOffb, &seqState.DStream, dctx->OFTptr); FSE_initDState(&seqState.stateOffb, &seqState.DStream, dctx->OFTptr);
@@ -1051,21 +1115,31 @@ static size_t ZSTD_decompressSequences(
for (; (BIT_reloadDStream(&(seqState.DStream)) <= BIT_DStream_completed) && nbSeq;) { for (; (BIT_reloadDStream(&(seqState.DStream)) <= BIT_DStream_completed) && nbSeq;) {
nbSeq--; nbSeq--;
{ seq_t const sequence = ZSTD_decodeSequence(&seqState); {
seq_t const sequence = ZSTD_decodeSequence(&seqState);
size_t const oneSeqSize = ZSTD_execSequence(op, oend, sequence, &litPtr, litEnd, base, vBase, dictEnd); size_t const oneSeqSize = ZSTD_execSequence(op, oend, sequence, &litPtr, litEnd, base, vBase, dictEnd);
if (ZSTD_isError(oneSeqSize)) return oneSeqSize; if (ZSTD_isError(oneSeqSize))
return oneSeqSize;
op += oneSeqSize; op += oneSeqSize;
} } }
}
/* check if reached exact end */ /* check if reached exact end */
if (nbSeq) return ERROR(corruption_detected); if (nbSeq)
return ERROR(corruption_detected);
/* save reps for next block */ /* save reps for next block */
{ U32 i; for (i=0; i<ZSTD_REP_NUM; i++) dctx->entropy.rep[i] = (U32)(seqState.prevOffset[i]); } {
U32 i;
for (i = 0; i < ZSTD_REP_NUM; i++)
dctx->entropy.rep[i] = (U32)(seqState.prevOffset[i]);
}
} }
/* last literal segment */ /* last literal segment */
{ size_t const lastLLSize = litEnd - litPtr; {
if (lastLLSize > (size_t)(oend-op)) return ERROR(dstSize_tooSmall); size_t const lastLLSize = litEnd - litPtr;
if (lastLLSize > (size_t)(oend - op))
return ERROR(dstSize_tooSmall);
memcpy(op, litPtr, lastLLSize); memcpy(op, litPtr, lastLLSize);
op += lastLLSize; op += lastLLSize;
} }
@@ -1073,7 +1147,6 @@ static size_t ZSTD_decompressSequences(
return op - ostart; return op - ostart;
} }
FORCE_INLINE seq_t ZSTD_decodeSequenceLong_generic(seqState_t *seqState, int const longOffsets) FORCE_INLINE seq_t ZSTD_decodeSequenceLong_generic(seqState_t *seqState, int const longOffsets)
{ {
seq_t seq; seq_t seq;
@@ -1087,36 +1160,34 @@ FORCE_INLINE seq_t ZSTD_decodeSequenceLong_generic(seqState_t* seqState, int con
U32 const ofBits = ofCode; U32 const ofBits = ofCode;
U32 const totalBits = llBits + mlBits + ofBits; U32 const totalBits = llBits + mlBits + ofBits;
static const U32 LL_base[MaxLL+1] = { static const U32 LL_base[MaxLL + 1] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 18,
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 20, 22, 24, 28, 32, 40, 48, 64, 0x80, 0x100, 0x200, 0x400, 0x800, 0x1000, 0x2000, 0x4000, 0x8000, 0x10000};
16, 18, 20, 22, 24, 28, 32, 40, 48, 64, 0x80, 0x100, 0x200, 0x400, 0x800, 0x1000,
0x2000, 0x4000, 0x8000, 0x10000 };
static const U32 ML_base[MaxML+1] = { static const U32 ML_base[MaxML + 1] = {3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20,
3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 37, 39, 41,
19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 43, 47, 51, 59, 67, 83, 99, 0x83, 0x103, 0x203, 0x403, 0x803, 0x1003, 0x2003, 0x4003, 0x8003, 0x10003};
35, 37, 39, 41, 43, 47, 51, 59, 67, 83, 99, 0x83, 0x103, 0x203, 0x403, 0x803,
0x1003, 0x2003, 0x4003, 0x8003, 0x10003 };
static const U32 OF_base[MaxOff+1] = { static const U32 OF_base[MaxOff + 1] = {0, 1, 1, 5, 0xD, 0x1D, 0x3D, 0x7D, 0xFD, 0x1FD,
0, 1, 1, 5, 0xD, 0x1D, 0x3D, 0x7D, 0x3FD, 0x7FD, 0xFFD, 0x1FFD, 0x3FFD, 0x7FFD, 0xFFFD, 0x1FFFD, 0x3FFFD, 0x7FFFD,
0xFD, 0x1FD, 0x3FD, 0x7FD, 0xFFD, 0x1FFD, 0x3FFD, 0x7FFD, 0xFFFFD, 0x1FFFFD, 0x3FFFFD, 0x7FFFFD, 0xFFFFFD, 0x1FFFFFD, 0x3FFFFFD, 0x7FFFFFD, 0xFFFFFFD};
0xFFFD, 0x1FFFD, 0x3FFFD, 0x7FFFD, 0xFFFFD, 0x1FFFFD, 0x3FFFFD, 0x7FFFFD,
0xFFFFFD, 0x1FFFFFD, 0x3FFFFFD, 0x7FFFFFD, 0xFFFFFFD };
/* sequence */ /* sequence */
{ size_t offset; {
size_t offset;
if (!ofCode) if (!ofCode)
offset = 0; offset = 0;
else { else {
if (longOffsets) { if (longOffsets) {
int const extraBits = ofBits - MIN(ofBits, STREAM_ACCUMULATOR_MIN); int const extraBits = ofBits - MIN(ofBits, STREAM_ACCUMULATOR_MIN);
offset = OF_base[ofCode] + (BIT_readBitsFast(&seqState->DStream, ofBits - extraBits) << extraBits); offset = OF_base[ofCode] + (BIT_readBitsFast(&seqState->DStream, ofBits - extraBits) << extraBits);
if (ZSTD_32bits() || extraBits) BIT_reloadDStream(&seqState->DStream); if (ZSTD_32bits() || extraBits)
if (extraBits) offset += BIT_readBitsFast(&seqState->DStream, extraBits); BIT_reloadDStream(&seqState->DStream);
if (extraBits)
offset += BIT_readBitsFast(&seqState->DStream, extraBits);
} else { } else {
offset = OF_base[ofCode] + BIT_readBitsFast(&seqState->DStream, ofBits); /* <= (ZSTD_WINDOWLOG_MAX-1) bits */ offset = OF_base[ofCode] + BIT_readBitsFast(&seqState->DStream, ofBits); /* <= (ZSTD_WINDOWLOG_MAX-1) bits */
if (ZSTD_32bits()) BIT_reloadDStream(&seqState->DStream); if (ZSTD_32bits())
BIT_reloadDStream(&seqState->DStream);
} }
} }
@@ -1125,7 +1196,8 @@ FORCE_INLINE seq_t ZSTD_decodeSequenceLong_generic(seqState_t* seqState, int con
if (offset) { if (offset) {
size_t temp = (offset == 3) ? seqState->prevOffset[0] - 1 : seqState->prevOffset[offset]; size_t temp = (offset == 3) ? seqState->prevOffset[0] - 1 : seqState->prevOffset[offset];
temp += !temp; /* 0 is not valid; input is corrupted; force offset to 1 */ temp += !temp; /* 0 is not valid; input is corrupted; force offset to 1 */
if (offset != 1) seqState->prevOffset[2] = seqState->prevOffset[1]; if (offset != 1)
seqState->prevOffset[2] = seqState->prevOffset[1];
seqState->prevOffset[1] = seqState->prevOffset[0]; seqState->prevOffset[1] = seqState->prevOffset[0];
seqState->prevOffset[0] = offset = temp; seqState->prevOffset[0] = offset = temp;
} else { } else {
@@ -1140,28 +1212,33 @@ FORCE_INLINE seq_t ZSTD_decodeSequenceLong_generic(seqState_t* seqState, int con
} }
seq.matchLength = ML_base[mlCode] + ((mlCode > 31) ? BIT_readBitsFast(&seqState->DStream, mlBits) : 0); /* <= 16 bits */ seq.matchLength = ML_base[mlCode] + ((mlCode > 31) ? BIT_readBitsFast(&seqState->DStream, mlBits) : 0); /* <= 16 bits */
if (ZSTD_32bits() && (mlBits+llBits>24)) BIT_reloadDStream(&seqState->DStream); if (ZSTD_32bits() && (mlBits + llBits > 24))
BIT_reloadDStream(&seqState->DStream);
seq.litLength = LL_base[llCode] + ((llCode > 15) ? BIT_readBitsFast(&seqState->DStream, llBits) : 0); /* <= 16 bits */ seq.litLength = LL_base[llCode] + ((llCode > 15) ? BIT_readBitsFast(&seqState->DStream, llBits) : 0); /* <= 16 bits */
if (ZSTD_32bits() || if (ZSTD_32bits() || (totalBits > 64 - 7 - (LLFSELog + MLFSELog + OffFSELog)))
(totalBits > 64 - 7 - (LLFSELog+MLFSELog+OffFSELog)) ) BIT_reloadDStream(&seqState->DStream); BIT_reloadDStream(&seqState->DStream);
{ size_t const pos = seqState->pos + seq.litLength; {
size_t const pos = seqState->pos + seq.litLength;
seq.match = seqState->base + pos - seq.offset; /* single memory segment */ seq.match = seqState->base + pos - seq.offset; /* single memory segment */
if (seq.offset > pos) seq.match += seqState->gotoDict; /* separate memory segment */ if (seq.offset > pos)
seq.match += seqState->gotoDict; /* separate memory segment */
seqState->pos = pos + seq.matchLength; seqState->pos = pos + seq.matchLength;
} }
/* ANS state update */ /* ANS state update */
FSE_updateState(&seqState->stateLL, &seqState->DStream); /* <= 9 bits */ FSE_updateState(&seqState->stateLL, &seqState->DStream); /* <= 9 bits */
FSE_updateState(&seqState->stateML, &seqState->DStream); /* <= 9 bits */ FSE_updateState(&seqState->stateML, &seqState->DStream); /* <= 9 bits */
if (ZSTD_32bits()) BIT_reloadDStream(&seqState->DStream); /* <= 18 bits */ if (ZSTD_32bits())
BIT_reloadDStream(&seqState->DStream); /* <= 18 bits */
FSE_updateState(&seqState->stateOffb, &seqState->DStream); /* <= 8 bits */ FSE_updateState(&seqState->stateOffb, &seqState->DStream); /* <= 8 bits */
return seq; return seq;
} }
static seq_t ZSTD_decodeSequenceLong(seqState_t* seqState, unsigned const windowSize) { static seq_t ZSTD_decodeSequenceLong(seqState_t *seqState, unsigned const windowSize)
{
if (ZSTD_highbit32(windowSize) > STREAM_ACCUMULATOR_MIN) { if (ZSTD_highbit32(windowSize) > STREAM_ACCUMULATOR_MIN) {
return ZSTD_decodeSequenceLong_generic(seqState, 1); return ZSTD_decodeSequenceLong_generic(seqState, 1);
} else { } else {
@@ -1170,10 +1247,8 @@ static seq_t ZSTD_decodeSequenceLong(seqState_t* seqState, unsigned const window
} }
FORCE_INLINE FORCE_INLINE
size_t ZSTD_execSequenceLong(BYTE* op, size_t ZSTD_execSequenceLong(BYTE *op, BYTE *const oend, seq_t sequence, const BYTE **litPtr, const BYTE *const litLimit, const BYTE *const base,
BYTE* const oend, seq_t sequence, const BYTE *const vBase, const BYTE *const dictEnd)
const BYTE** litPtr, const BYTE* const litLimit,
const BYTE* const base, const BYTE* const vBase, const BYTE* const dictEnd)
{ {
BYTE *const oLitEnd = op + sequence.litLength; BYTE *const oLitEnd = op + sequence.litLength;
size_t const sequenceLength = sequence.litLength + sequence.matchLength; size_t const sequenceLength = sequence.litLength + sequence.matchLength;
@@ -1183,37 +1258,45 @@ size_t ZSTD_execSequenceLong(BYTE* op,
const BYTE *match = sequence.match; const BYTE *match = sequence.match;
/* check */ /* check */
if (oMatchEnd>oend) return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of WILDCOPY_OVERLENGTH from oend */ if (oMatchEnd > oend)
if (iLitEnd > litLimit) return ERROR(corruption_detected); /* over-read beyond lit buffer */ return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of WILDCOPY_OVERLENGTH from oend */
if (oLitEnd>oend_w) return ZSTD_execSequenceLast7(op, oend, sequence, litPtr, litLimit, base, vBase, dictEnd); if (iLitEnd > litLimit)
return ERROR(corruption_detected); /* over-read beyond lit buffer */
if (oLitEnd > oend_w)
return ZSTD_execSequenceLast7(op, oend, sequence, litPtr, litLimit, base, vBase, dictEnd);
/* copy Literals */ /* copy Literals */
ZSTD_copy8(op, *litPtr); ZSTD_copy8(op, *litPtr);
if (sequence.litLength > 8) if (sequence.litLength > 8)
ZSTD_wildcopy(op+8, (*litPtr)+8, sequence.litLength - 8); /* note : since oLitEnd <= oend-WILDCOPY_OVERLENGTH, no risk of overwrite beyond oend */ ZSTD_wildcopy(op + 8, (*litPtr) + 8,
sequence.litLength - 8); /* note : since oLitEnd <= oend-WILDCOPY_OVERLENGTH, no risk of overwrite beyond oend */
op = oLitEnd; op = oLitEnd;
*litPtr = iLitEnd; /* update for next sequence */ *litPtr = iLitEnd; /* update for next sequence */
/* copy Match */ /* copy Match */
if (sequence.offset > (size_t)(oLitEnd - base)) { if (sequence.offset > (size_t)(oLitEnd - base)) {
/* offset beyond prefix */ /* offset beyond prefix */
if (sequence.offset > (size_t)(oLitEnd - vBase)) return ERROR(corruption_detected); if (sequence.offset > (size_t)(oLitEnd - vBase))
return ERROR(corruption_detected);
if (match + sequence.matchLength <= dictEnd) { if (match + sequence.matchLength <= dictEnd) {
memmove(oLitEnd, match, sequence.matchLength); memmove(oLitEnd, match, sequence.matchLength);
return sequenceLength; return sequenceLength;
} }
/* span extDict & currPrefixSegment */ /* span extDict & currPrefixSegment */
{ size_t const length1 = dictEnd - match; {
size_t const length1 = dictEnd - match;
memmove(oLitEnd, match, length1); memmove(oLitEnd, match, length1);
op = oLitEnd + length1; op = oLitEnd + length1;
sequence.matchLength -= length1; sequence.matchLength -= length1;
match = base; match = base;
if (op > oend_w || sequence.matchLength < MINMATCH) { if (op > oend_w || sequence.matchLength < MINMATCH) {
U32 i; U32 i;
for (i = 0; i < sequence.matchLength; ++i) op[i] = match[i]; for (i = 0; i < sequence.matchLength; ++i)
op[i] = match[i];
return sequenceLength; return sequenceLength;
} }
} } }
}
/* Requirement: op <= oend_w && sequence.matchLength >= MINMATCH */ /* Requirement: op <= oend_w && sequence.matchLength >= MINMATCH */
/* match within prefix */ /* match within prefix */
@@ -1232,7 +1315,8 @@ size_t ZSTD_execSequenceLong(BYTE* op,
} else { } else {
ZSTD_copy8(op, match); ZSTD_copy8(op, match);
} }
op += 8; match += 8; op += 8;
match += 8;
if (oMatchEnd > oend - (16 - MINMATCH)) { if (oMatchEnd > oend - (16 - MINMATCH)) {
if (op < oend_w) { if (op < oend_w) {
@@ -1240,17 +1324,15 @@ size_t ZSTD_execSequenceLong(BYTE* op,
match += oend_w - op; match += oend_w - op;
op = oend_w; op = oend_w;
} }
while (op < oMatchEnd) *op++ = *match++; while (op < oMatchEnd)
*op++ = *match++;
} else { } else {
ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength - 8); /* works even if matchLength < 8 */ ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength - 8); /* works even if matchLength < 8 */
} }
return sequenceLength; return sequenceLength;
} }
static size_t ZSTD_decompressSequencesLong( static size_t ZSTD_decompressSequencesLong(ZSTD_DCtx *dctx, void *dst, size_t maxDstSize, const void *seqStart, size_t seqSize)
ZSTD_DCtx* dctx,
void* dst, size_t maxDstSize,
const void* seqStart, size_t seqSize)
{ {
const BYTE *ip = (const BYTE *)seqStart; const BYTE *ip = (const BYTE *)seqStart;
const BYTE *const iend = ip + seqSize; const BYTE *const iend = ip + seqSize;
@@ -1266,8 +1348,10 @@ static size_t ZSTD_decompressSequencesLong(
int nbSeq; int nbSeq;
/* Build Decoding Tables */ /* Build Decoding Tables */
{ size_t const seqHSize = ZSTD_decodeSeqHeaders(dctx, &nbSeq, ip, seqSize); {
if (ZSTD_isError(seqHSize)) return seqHSize; size_t const seqHSize = ZSTD_decodeSeqHeaders(dctx, &nbSeq, ip, seqSize);
if (ZSTD_isError(seqHSize))
return seqHSize;
ip += seqHSize; ip += seqHSize;
} }
@@ -1281,7 +1365,11 @@ static size_t ZSTD_decompressSequencesLong(
seqState_t seqState; seqState_t seqState;
int seqNb; int seqNb;
dctx->fseEntropy = 1; dctx->fseEntropy = 1;
{ U32 i; for (i=0; i<ZSTD_REP_NUM; i++) seqState.prevOffset[i] = dctx->entropy.rep[i]; } {
U32 i;
for (i = 0; i < ZSTD_REP_NUM; i++)
seqState.prevOffset[i] = dctx->entropy.rep[i];
}
seqState.base = base; seqState.base = base;
seqState.pos = (size_t)(op - base); seqState.pos = (size_t)(op - base);
seqState.gotoDict = (uPtrDiff)dictEnd - (uPtrDiff)base; /* cast to avoid undefined behaviour */ seqState.gotoDict = (uPtrDiff)dictEnd - (uPtrDiff)base; /* cast to avoid undefined behaviour */
@@ -1294,34 +1382,45 @@ static size_t ZSTD_decompressSequencesLong(
for (seqNb = 0; (BIT_reloadDStream(&seqState.DStream) <= BIT_DStream_completed) && seqNb < seqAdvance; seqNb++) { for (seqNb = 0; (BIT_reloadDStream(&seqState.DStream) <= BIT_DStream_completed) && seqNb < seqAdvance; seqNb++) {
sequences[seqNb] = ZSTD_decodeSequenceLong(&seqState, windowSize); sequences[seqNb] = ZSTD_decodeSequenceLong(&seqState, windowSize);
} }
if (seqNb<seqAdvance) return ERROR(corruption_detected); if (seqNb < seqAdvance)
return ERROR(corruption_detected);
/* decode and decompress */ /* decode and decompress */
for (; (BIT_reloadDStream(&(seqState.DStream)) <= BIT_DStream_completed) && seqNb < nbSeq; seqNb++) { for (; (BIT_reloadDStream(&(seqState.DStream)) <= BIT_DStream_completed) && seqNb < nbSeq; seqNb++) {
seq_t const sequence = ZSTD_decodeSequenceLong(&seqState, windowSize); seq_t const sequence = ZSTD_decodeSequenceLong(&seqState, windowSize);
size_t const oneSeqSize = ZSTD_execSequenceLong(op, oend, sequences[(seqNb-ADVANCED_SEQS) & STOSEQ_MASK], &litPtr, litEnd, base, vBase, dictEnd); size_t const oneSeqSize =
if (ZSTD_isError(oneSeqSize)) return oneSeqSize; ZSTD_execSequenceLong(op, oend, sequences[(seqNb - ADVANCED_SEQS) & STOSEQ_MASK], &litPtr, litEnd, base, vBase, dictEnd);
if (ZSTD_isError(oneSeqSize))
return oneSeqSize;
ZSTD_PREFETCH(sequence.match); ZSTD_PREFETCH(sequence.match);
sequences[seqNb & STOSEQ_MASK] = sequence; sequences[seqNb & STOSEQ_MASK] = sequence;
op += oneSeqSize; op += oneSeqSize;
} }
if (seqNb<nbSeq) return ERROR(corruption_detected); if (seqNb < nbSeq)
return ERROR(corruption_detected);
/* finish queue */ /* finish queue */
seqNb -= seqAdvance; seqNb -= seqAdvance;
for (; seqNb < nbSeq; seqNb++) { for (; seqNb < nbSeq; seqNb++) {
size_t const oneSeqSize = ZSTD_execSequenceLong(op, oend, sequences[seqNb & STOSEQ_MASK], &litPtr, litEnd, base, vBase, dictEnd); size_t const oneSeqSize = ZSTD_execSequenceLong(op, oend, sequences[seqNb & STOSEQ_MASK], &litPtr, litEnd, base, vBase, dictEnd);
if (ZSTD_isError(oneSeqSize)) return oneSeqSize; if (ZSTD_isError(oneSeqSize))
return oneSeqSize;
op += oneSeqSize; op += oneSeqSize;
} }
/* save reps for next block */ /* save reps for next block */
{ U32 i; for (i=0; i<ZSTD_REP_NUM; i++) dctx->entropy.rep[i] = (U32)(seqState.prevOffset[i]); } {
U32 i;
for (i = 0; i < ZSTD_REP_NUM; i++)
dctx->entropy.rep[i] = (U32)(seqState.prevOffset[i]);
}
} }
/* last literal segment */ /* last literal segment */
{ size_t const lastLLSize = litEnd - litPtr; {
if (lastLLSize > (size_t)(oend-op)) return ERROR(dstSize_tooSmall); size_t const lastLLSize = litEnd - litPtr;
if (lastLLSize > (size_t)(oend - op))
return ERROR(dstSize_tooSmall);
memcpy(op, litPtr, lastLLSize); memcpy(op, litPtr, lastLLSize);
op += lastLLSize; op += lastLLSize;
} }
@@ -1329,18 +1428,18 @@ static size_t ZSTD_decompressSequencesLong(
return op - ostart; return op - ostart;
} }
static size_t ZSTD_decompressBlock_internal(ZSTD_DCtx *dctx, void *dst, size_t dstCapacity, const void *src, size_t srcSize)
static size_t ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx,
void* dst, size_t dstCapacity,
const void* src, size_t srcSize)
{ /* blockType == blockCompressed */ { /* blockType == blockCompressed */
const BYTE *ip = (const BYTE *)src; const BYTE *ip = (const BYTE *)src;
if (srcSize >= ZSTD_BLOCKSIZE_ABSOLUTEMAX) return ERROR(srcSize_wrong); if (srcSize >= ZSTD_BLOCKSIZE_ABSOLUTEMAX)
return ERROR(srcSize_wrong);
/* Decode literals section */ /* Decode literals section */
{ size_t const litCSize = ZSTD_decodeLiteralsBlock(dctx, src, srcSize); {
if (ZSTD_isError(litCSize)) return litCSize; size_t const litCSize = ZSTD_decodeLiteralsBlock(dctx, src, srcSize);
if (ZSTD_isError(litCSize))
return litCSize;
ip += litCSize; ip += litCSize;
srcSize -= litCSize; srcSize -= litCSize;
} }
@@ -1353,7 +1452,6 @@ static size_t ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx,
return ZSTD_decompressSequences(dctx, dst, dstCapacity, ip, srcSize); return ZSTD_decompressSequences(dctx, dst, dstCapacity, ip, srcSize);
} }
static void ZSTD_checkContinuity(ZSTD_DCtx *dctx, const void *dst) static void ZSTD_checkContinuity(ZSTD_DCtx *dctx, const void *dst)
{ {
if (dst != dctx->previousDstEnd) { /* not contiguous */ if (dst != dctx->previousDstEnd) { /* not contiguous */
@@ -1364,9 +1462,7 @@ static void ZSTD_checkContinuity(ZSTD_DCtx* dctx, const void* dst)
} }
} }
size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, size_t ZSTD_decompressBlock(ZSTD_DCtx *dctx, void *dst, size_t dstCapacity, const void *src, size_t srcSize)
void* dst, size_t dstCapacity,
const void* src, size_t srcSize)
{ {
size_t dSize; size_t dSize;
ZSTD_checkContinuity(dctx, dst); ZSTD_checkContinuity(dctx, dst);
@@ -1375,7 +1471,6 @@ size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx,
return dSize; return dSize;
} }
/** ZSTD_insertBlock() : /** ZSTD_insertBlock() :
insert `src` block into `dctx` history. Useful to track uncompressed blocks. */ insert `src` block into `dctx` history. Useful to track uncompressed blocks. */
size_t ZSTD_insertBlock(ZSTD_DCtx *dctx, const void *blockStart, size_t blockSize) size_t ZSTD_insertBlock(ZSTD_DCtx *dctx, const void *blockStart, size_t blockSize)
@@ -1385,10 +1480,10 @@ size_t ZSTD_insertBlock(ZSTD_DCtx* dctx, const void* blockStart, size_t blockSiz
return blockSize; return blockSize;
} }
size_t ZSTD_generateNxBytes(void *dst, size_t dstCapacity, BYTE byte, size_t length) size_t ZSTD_generateNxBytes(void *dst, size_t dstCapacity, BYTE byte, size_t length)
{ {
if (length > dstCapacity) return ERROR(dstSize_tooSmall); if (length > dstCapacity)
return ERROR(dstSize_tooSmall);
memset(dst, byte, length); memset(dst, byte, length);
return length; return length;
} }
@@ -1400,8 +1495,7 @@ size_t ZSTD_generateNxBytes(void* dst, size_t dstCapacity, BYTE byte, size_t len
* @return : the compressed size of the frame starting at `src` */ * @return : the compressed size of the frame starting at `src` */
size_t ZSTD_findFrameCompressedSize(const void *src, size_t srcSize) size_t ZSTD_findFrameCompressedSize(const void *src, size_t srcSize)
{ {
if (srcSize >= ZSTD_skippableHeaderSize && if (srcSize >= ZSTD_skippableHeaderSize && (ZSTD_readLE32(src) & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) {
(ZSTD_readLE32(src) & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) {
return ZSTD_skippableHeaderSize + ZSTD_readLE32((const BYTE *)src + 4); return ZSTD_skippableHeaderSize + ZSTD_readLE32((const BYTE *)src + 4);
} else { } else {
const BYTE *ip = (const BYTE *)src; const BYTE *ip = (const BYTE *)src;
@@ -1410,12 +1504,16 @@ size_t ZSTD_findFrameCompressedSize(const void *src, size_t srcSize)
ZSTD_frameParams fParams; ZSTD_frameParams fParams;
size_t const headerSize = ZSTD_frameHeaderSize(ip, remainingSize); size_t const headerSize = ZSTD_frameHeaderSize(ip, remainingSize);
if (ZSTD_isError(headerSize)) return headerSize; if (ZSTD_isError(headerSize))
return headerSize;
/* Frame Header */ /* Frame Header */
{ size_t const ret = ZSTD_getFrameParams(&fParams, ip, remainingSize); {
if (ZSTD_isError(ret)) return ret; size_t const ret = ZSTD_getFrameParams(&fParams, ip, remainingSize);
if (ret > 0) return ERROR(srcSize_wrong); if (ZSTD_isError(ret))
return ret;
if (ret > 0)
return ERROR(srcSize_wrong);
} }
ip += headerSize; ip += headerSize;
@@ -1425,18 +1523,22 @@ size_t ZSTD_findFrameCompressedSize(const void *src, size_t srcSize)
while (1) { while (1) {
blockProperties_t blockProperties; blockProperties_t blockProperties;
size_t const cBlockSize = ZSTD_getcBlockSize(ip, remainingSize, &blockProperties); size_t const cBlockSize = ZSTD_getcBlockSize(ip, remainingSize, &blockProperties);
if (ZSTD_isError(cBlockSize)) return cBlockSize; if (ZSTD_isError(cBlockSize))
return cBlockSize;
if (ZSTD_blockHeaderSize + cBlockSize > remainingSize) return ERROR(srcSize_wrong); if (ZSTD_blockHeaderSize + cBlockSize > remainingSize)
return ERROR(srcSize_wrong);
ip += ZSTD_blockHeaderSize + cBlockSize; ip += ZSTD_blockHeaderSize + cBlockSize;
remainingSize -= ZSTD_blockHeaderSize + cBlockSize; remainingSize -= ZSTD_blockHeaderSize + cBlockSize;
if (blockProperties.lastBlock) break; if (blockProperties.lastBlock)
break;
} }
if (fParams.checksumFlag) { /* Frame content checksum */ if (fParams.checksumFlag) { /* Frame content checksum */
if (remainingSize < 4) return ERROR(srcSize_wrong); if (remainingSize < 4)
return ERROR(srcSize_wrong);
ip += 4; ip += 4;
remainingSize -= 4; remainingSize -= 4;
} }
@@ -1447,9 +1549,7 @@ size_t ZSTD_findFrameCompressedSize(const void *src, size_t srcSize)
/*! ZSTD_decompressFrame() : /*! ZSTD_decompressFrame() :
* @dctx must be properly initialized */ * @dctx must be properly initialized */
static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx, static size_t ZSTD_decompressFrame(ZSTD_DCtx *dctx, void *dst, size_t dstCapacity, const void **srcPtr, size_t *srcSizePtr)
void* dst, size_t dstCapacity,
const void** srcPtr, size_t *srcSizePtr)
{ {
const BYTE *ip = (const BYTE *)(*srcPtr); const BYTE *ip = (const BYTE *)(*srcPtr);
BYTE *const ostart = (BYTE * const)dst; BYTE *const ostart = (BYTE * const)dst;
@@ -1458,14 +1558,19 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx,
size_t remainingSize = *srcSizePtr; size_t remainingSize = *srcSizePtr;
/* check */ /* check */
if (remainingSize < ZSTD_frameHeaderSize_min+ZSTD_blockHeaderSize) return ERROR(srcSize_wrong); if (remainingSize < ZSTD_frameHeaderSize_min + ZSTD_blockHeaderSize)
return ERROR(srcSize_wrong);
/* Frame Header */ /* Frame Header */
{ size_t const frameHeaderSize = ZSTD_frameHeaderSize(ip, ZSTD_frameHeaderSize_prefix); {
if (ZSTD_isError(frameHeaderSize)) return frameHeaderSize; size_t const frameHeaderSize = ZSTD_frameHeaderSize(ip, ZSTD_frameHeaderSize_prefix);
if (remainingSize < frameHeaderSize+ZSTD_blockHeaderSize) return ERROR(srcSize_wrong); if (ZSTD_isError(frameHeaderSize))
return frameHeaderSize;
if (remainingSize < frameHeaderSize + ZSTD_blockHeaderSize)
return ERROR(srcSize_wrong);
CHECK_F(ZSTD_decodeFrameHeader(dctx, ip, frameHeaderSize)); CHECK_F(ZSTD_decodeFrameHeader(dctx, ip, frameHeaderSize));
ip += frameHeaderSize; remainingSize -= frameHeaderSize; ip += frameHeaderSize;
remainingSize -= frameHeaderSize;
} }
/* Loop on each block */ /* Loop on each block */
@@ -1473,42 +1578,41 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx,
size_t decodedSize; size_t decodedSize;
blockProperties_t blockProperties; blockProperties_t blockProperties;
size_t const cBlockSize = ZSTD_getcBlockSize(ip, remainingSize, &blockProperties); size_t const cBlockSize = ZSTD_getcBlockSize(ip, remainingSize, &blockProperties);
if (ZSTD_isError(cBlockSize)) return cBlockSize; if (ZSTD_isError(cBlockSize))
return cBlockSize;
ip += ZSTD_blockHeaderSize; ip += ZSTD_blockHeaderSize;
remainingSize -= ZSTD_blockHeaderSize; remainingSize -= ZSTD_blockHeaderSize;
if (cBlockSize > remainingSize) return ERROR(srcSize_wrong); if (cBlockSize > remainingSize)
return ERROR(srcSize_wrong);
switch(blockProperties.blockType) switch (blockProperties.blockType) {
{ case bt_compressed: decodedSize = ZSTD_decompressBlock_internal(dctx, op, oend - op, ip, cBlockSize); break;
case bt_compressed: case bt_raw: decodedSize = ZSTD_copyRawBlock(op, oend - op, ip, cBlockSize); break;
decodedSize = ZSTD_decompressBlock_internal(dctx, op, oend-op, ip, cBlockSize); case bt_rle: decodedSize = ZSTD_generateNxBytes(op, oend - op, *ip, blockProperties.origSize); break;
break;
case bt_raw :
decodedSize = ZSTD_copyRawBlock(op, oend-op, ip, cBlockSize);
break;
case bt_rle :
decodedSize = ZSTD_generateNxBytes(op, oend-op, *ip, blockProperties.origSize);
break;
case bt_reserved: case bt_reserved:
default: default: return ERROR(corruption_detected);
return ERROR(corruption_detected);
} }
if (ZSTD_isError(decodedSize)) return decodedSize; if (ZSTD_isError(decodedSize))
if (dctx->fParams.checksumFlag) xxh64_update(&dctx->xxhState, op, decodedSize); return decodedSize;
if (dctx->fParams.checksumFlag)
xxh64_update(&dctx->xxhState, op, decodedSize);
op += decodedSize; op += decodedSize;
ip += cBlockSize; ip += cBlockSize;
remainingSize -= cBlockSize; remainingSize -= cBlockSize;
if (blockProperties.lastBlock) break; if (blockProperties.lastBlock)
break;
} }
if (dctx->fParams.checksumFlag) { /* Frame content checksum verification */ if (dctx->fParams.checksumFlag) { /* Frame content checksum verification */
U32 const checkCalc = (U32)xxh64_digest(&dctx->xxhState); U32 const checkCalc = (U32)xxh64_digest(&dctx->xxhState);
U32 checkRead; U32 checkRead;
if (remainingSize<4) return ERROR(checksum_wrong); if (remainingSize < 4)
return ERROR(checksum_wrong);
checkRead = ZSTD_readLE32(ip); checkRead = ZSTD_readLE32(ip);
if (checkRead != checkCalc) return ERROR(checksum_wrong); if (checkRead != checkCalc)
return ERROR(checksum_wrong);
ip += 4; ip += 4;
remainingSize -= 4; remainingSize -= 4;
} }
@@ -1522,10 +1626,7 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx,
static const void *ZSTD_DDictDictContent(const ZSTD_DDict *ddict); static const void *ZSTD_DDictDictContent(const ZSTD_DDict *ddict);
static size_t ZSTD_DDictDictSize(const ZSTD_DDict *ddict); static size_t ZSTD_DDictDictSize(const ZSTD_DDict *ddict);
static size_t ZSTD_decompressMultiFrame(ZSTD_DCtx* dctx, static size_t ZSTD_decompressMultiFrame(ZSTD_DCtx *dctx, void *dst, size_t dstCapacity, const void *src, size_t srcSize, const void *dict, size_t dictSize,
void* dst, size_t dstCapacity,
const void* src, size_t srcSize,
const void *dict, size_t dictSize,
const ZSTD_DDict *ddict) const ZSTD_DDict *ddict)
{ {
void *const dststart = dst; void *const dststart = dst;
@@ -1549,8 +1650,7 @@ static size_t ZSTD_decompressMultiFrame(ZSTD_DCtx* dctx,
size_t skippableSize; size_t skippableSize;
if (srcSize < ZSTD_skippableHeaderSize) if (srcSize < ZSTD_skippableHeaderSize)
return ERROR(srcSize_wrong); return ERROR(srcSize_wrong);
skippableSize = ZSTD_readLE32((const BYTE *)src + 4) + skippableSize = ZSTD_readLE32((const BYTE *)src + 4) + ZSTD_skippableHeaderSize;
ZSTD_skippableHeaderSize;
if (srcSize < skippableSize) { if (srcSize < skippableSize) {
return ERROR(srcSize_wrong); return ERROR(srcSize_wrong);
} }
@@ -1573,9 +1673,10 @@ static size_t ZSTD_decompressMultiFrame(ZSTD_DCtx* dctx,
} }
ZSTD_checkContinuity(dctx, dst); ZSTD_checkContinuity(dctx, dst);
{ const size_t res = ZSTD_decompressFrame(dctx, dst, dstCapacity, {
&src, &srcSize); const size_t res = ZSTD_decompressFrame(dctx, dst, dstCapacity, &src, &srcSize);
if (ZSTD_isError(res)) return res; if (ZSTD_isError(res))
return res;
/* don't need to bounds check this, ZSTD_decompressFrame will have /* don't need to bounds check this, ZSTD_decompressFrame will have
* already */ * already */
dst = (BYTE *)dst + res; dst = (BYTE *)dst + res;
@@ -1583,7 +1684,8 @@ static size_t ZSTD_decompressMultiFrame(ZSTD_DCtx* dctx,
} }
} }
if (srcSize) return ERROR(srcSize_wrong); /* input not entirely consumed */ if (srcSize)
return ERROR(srcSize_wrong); /* input not entirely consumed */
return (BYTE *)dst - (BYTE *)dststart; return (BYTE *)dst - (BYTE *)dststart;
} }
@@ -1593,37 +1695,29 @@ size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity,
return ZSTD_decompressMultiFrame(dctx, dst, dstCapacity, src, srcSize, dict, dictSize, NULL); return ZSTD_decompressMultiFrame(dctx, dst, dstCapacity, src, srcSize, dict, dictSize, NULL);
} }
size_t ZSTD_decompressDCtx(ZSTD_DCtx *dctx, void *dst, size_t dstCapacity, const void *src, size_t srcSize) size_t ZSTD_decompressDCtx(ZSTD_DCtx *dctx, void *dst, size_t dstCapacity, const void *src, size_t srcSize)
{ {
return ZSTD_decompress_usingDict(dctx, dst, dstCapacity, src, srcSize, NULL, 0); return ZSTD_decompress_usingDict(dctx, dst, dstCapacity, src, srcSize, NULL, 0);
} }
/*-************************************** /*-**************************************
* Advanced Streaming Decompression API * Advanced Streaming Decompression API
* Bufferless and synchronous * Bufferless and synchronous
****************************************/ ****************************************/
size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx *dctx) { return dctx->expected; } size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx *dctx) { return dctx->expected; }
ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx) { ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx *dctx)
switch(dctx->stage)
{ {
switch (dctx->stage) {
default: /* should not happen */ default: /* should not happen */
case ZSTDds_getFrameHeaderSize: case ZSTDds_getFrameHeaderSize:
case ZSTDds_decodeFrameHeader: case ZSTDds_decodeFrameHeader: return ZSTDnit_frameHeader;
return ZSTDnit_frameHeader; case ZSTDds_decodeBlockHeader: return ZSTDnit_blockHeader;
case ZSTDds_decodeBlockHeader: case ZSTDds_decompressBlock: return ZSTDnit_block;
return ZSTDnit_blockHeader; case ZSTDds_decompressLastBlock: return ZSTDnit_lastBlock;
case ZSTDds_decompressBlock: case ZSTDds_checkChecksum: return ZSTDnit_checksum;
return ZSTDnit_block;
case ZSTDds_decompressLastBlock:
return ZSTDnit_lastBlock;
case ZSTDds_checkChecksum:
return ZSTDnit_checksum;
case ZSTDds_decodeSkippableHeader: case ZSTDds_decodeSkippableHeader:
case ZSTDds_skipFrame: case ZSTDds_skipFrame: return ZSTDnit_skippableFrame;
return ZSTDnit_skippableFrame;
} }
} }
@@ -1635,13 +1729,15 @@ int ZSTD_isSkipFrame(ZSTD_DCtx* dctx) { return dctx->stage == ZSTDds_skipFrame;
size_t ZSTD_decompressContinue(ZSTD_DCtx *dctx, void *dst, size_t dstCapacity, const void *src, size_t srcSize) size_t ZSTD_decompressContinue(ZSTD_DCtx *dctx, void *dst, size_t dstCapacity, const void *src, size_t srcSize)
{ {
/* Sanity check */ /* Sanity check */
if (srcSize != dctx->expected) return ERROR(srcSize_wrong); if (srcSize != dctx->expected)
if (dstCapacity) ZSTD_checkContinuity(dctx, dst); return ERROR(srcSize_wrong);
if (dstCapacity)
ZSTD_checkContinuity(dctx, dst);
switch (dctx->stage) switch (dctx->stage) {
{
case ZSTDds_getFrameHeaderSize: case ZSTDds_getFrameHeaderSize:
if (srcSize != ZSTD_frameHeaderSize_prefix) return ERROR(srcSize_wrong); /* impossible */ if (srcSize != ZSTD_frameHeaderSize_prefix)
return ERROR(srcSize_wrong); /* impossible */
if ((ZSTD_readLE32(src) & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) { /* skippable frame */ if ((ZSTD_readLE32(src) & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) { /* skippable frame */
memcpy(dctx->headerBuffer, src, ZSTD_frameHeaderSize_prefix); memcpy(dctx->headerBuffer, src, ZSTD_frameHeaderSize_prefix);
dctx->expected = ZSTD_skippableHeaderSize - ZSTD_frameHeaderSize_prefix; /* magic number + skippable frame length */ dctx->expected = ZSTD_skippableHeaderSize - ZSTD_frameHeaderSize_prefix; /* magic number + skippable frame length */
@@ -1649,7 +1745,8 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c
return 0; return 0;
} }
dctx->headerSize = ZSTD_frameHeaderSize(src, ZSTD_frameHeaderSize_prefix); dctx->headerSize = ZSTD_frameHeaderSize(src, ZSTD_frameHeaderSize_prefix);
if (ZSTD_isError(dctx->headerSize)) return dctx->headerSize; if (ZSTD_isError(dctx->headerSize))
return dctx->headerSize;
memcpy(dctx->headerBuffer, src, ZSTD_frameHeaderSize_prefix); memcpy(dctx->headerBuffer, src, ZSTD_frameHeaderSize_prefix);
if (dctx->headerSize > ZSTD_frameHeaderSize_prefix) { if (dctx->headerSize > ZSTD_frameHeaderSize_prefix) {
dctx->expected = dctx->headerSize - ZSTD_frameHeaderSize_prefix; dctx->expected = dctx->headerSize - ZSTD_frameHeaderSize_prefix;
@@ -1665,10 +1762,11 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c
dctx->stage = ZSTDds_decodeBlockHeader; dctx->stage = ZSTDds_decodeBlockHeader;
return 0; return 0;
case ZSTDds_decodeBlockHeader: case ZSTDds_decodeBlockHeader: {
{ blockProperties_t bp; blockProperties_t bp;
size_t const cBlockSize = ZSTD_getcBlockSize(src, ZSTD_blockHeaderSize, &bp); size_t const cBlockSize = ZSTD_getcBlockSize(src, ZSTD_blockHeaderSize, &bp);
if (ZSTD_isError(cBlockSize)) return cBlockSize; if (ZSTD_isError(cBlockSize))
return cBlockSize;
dctx->expected = cBlockSize; dctx->expected = cBlockSize;
dctx->bType = bp.blockType; dctx->bType = bp.blockType;
dctx->rleSize = bp.origSize; dctx->rleSize = bp.origSize;
@@ -1692,25 +1790,19 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c
return 0; return 0;
} }
case ZSTDds_decompressLastBlock: case ZSTDds_decompressLastBlock:
case ZSTDds_decompressBlock: case ZSTDds_decompressBlock: {
{ size_t rSize; size_t rSize;
switch(dctx->bType) switch (dctx->bType) {
{ case bt_compressed: rSize = ZSTD_decompressBlock_internal(dctx, dst, dstCapacity, src, srcSize); break;
case bt_compressed: case bt_raw: rSize = ZSTD_copyRawBlock(dst, dstCapacity, src, srcSize); break;
rSize = ZSTD_decompressBlock_internal(dctx, dst, dstCapacity, src, srcSize); case bt_rle: rSize = ZSTD_setRleBlock(dst, dstCapacity, src, srcSize, dctx->rleSize); break;
break;
case bt_raw :
rSize = ZSTD_copyRawBlock(dst, dstCapacity, src, srcSize);
break;
case bt_rle :
rSize = ZSTD_setRleBlock(dst, dstCapacity, src, srcSize, dctx->rleSize);
break;
case bt_reserved: /* should never happen */ case bt_reserved: /* should never happen */
default: default: return ERROR(corruption_detected);
return ERROR(corruption_detected);
} }
if (ZSTD_isError(rSize)) return rSize; if (ZSTD_isError(rSize))
if (dctx->fParams.checksumFlag) xxh64_update(&dctx->xxhState, dst, rSize); return rSize;
if (dctx->fParams.checksumFlag)
xxh64_update(&dctx->xxhState, dst, rSize);
if (dctx->stage == ZSTDds_decompressLastBlock) { /* end of frame */ if (dctx->stage == ZSTDds_decompressLastBlock) { /* end of frame */
if (dctx->fParams.checksumFlag) { /* another round for frame checksum */ if (dctx->fParams.checksumFlag) { /* another round for frame checksum */
@@ -1727,22 +1819,23 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c
} }
return rSize; return rSize;
} }
case ZSTDds_checkChecksum: case ZSTDds_checkChecksum: {
{ U32 const h32 = (U32)xxh64_digest(&dctx->xxhState); U32 const h32 = (U32)xxh64_digest(&dctx->xxhState);
U32 const check32 = ZSTD_readLE32(src); /* srcSize == 4, guaranteed by dctx->expected */ U32 const check32 = ZSTD_readLE32(src); /* srcSize == 4, guaranteed by dctx->expected */
if (check32 != h32) return ERROR(checksum_wrong); if (check32 != h32)
return ERROR(checksum_wrong);
dctx->expected = 0; dctx->expected = 0;
dctx->stage = ZSTDds_getFrameHeaderSize; dctx->stage = ZSTDds_getFrameHeaderSize;
return 0; return 0;
} }
case ZSTDds_decodeSkippableHeader: case ZSTDds_decodeSkippableHeader: {
{ memcpy(dctx->headerBuffer + ZSTD_frameHeaderSize_prefix, src, dctx->expected); memcpy(dctx->headerBuffer + ZSTD_frameHeaderSize_prefix, src, dctx->expected);
dctx->expected = ZSTD_readLE32(dctx->headerBuffer + 4); dctx->expected = ZSTD_readLE32(dctx->headerBuffer + 4);
dctx->stage = ZSTDds_skipFrame; dctx->stage = ZSTDds_skipFrame;
return 0; return 0;
} }
case ZSTDds_skipFrame: case ZSTDds_skipFrame: {
{ dctx->expected = 0; dctx->expected = 0;
dctx->stage = ZSTDds_getFrameHeaderSize; dctx->stage = ZSTDds_getFrameHeaderSize;
return 0; return 0;
} }
@@ -1751,7 +1844,6 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c
} }
} }
static size_t ZSTD_refDictContent(ZSTD_DCtx *dctx, const void *dict, size_t dictSize) static size_t ZSTD_refDictContent(ZSTD_DCtx *dctx, const void *dict, size_t dictSize)
{ {
dctx->dictEnd = dctx->previousDstEnd; dctx->dictEnd = dctx->previousDstEnd;
@@ -1769,66 +1861,87 @@ static size_t ZSTD_loadEntropy(ZSTD_entropyTables_t* entropy, const void* const
const BYTE *dictPtr = (const BYTE *)dict; const BYTE *dictPtr = (const BYTE *)dict;
const BYTE *const dictEnd = dictPtr + dictSize; const BYTE *const dictEnd = dictPtr + dictSize;
if (dictSize <= 8) return ERROR(dictionary_corrupted); if (dictSize <= 8)
return ERROR(dictionary_corrupted);
dictPtr += 8; /* skip header = magic + dictID */ dictPtr += 8; /* skip header = magic + dictID */
{
{ size_t const hSize = HUF_readDTableX4(entropy->hufTable, dictPtr, dictEnd-dictPtr); size_t const hSize = HUF_readDTableX4(entropy->hufTable, dictPtr, dictEnd - dictPtr);
if (HUF_isError(hSize)) return ERROR(dictionary_corrupted); if (HUF_isError(hSize))
return ERROR(dictionary_corrupted);
dictPtr += hSize; dictPtr += hSize;
} }
{ short offcodeNCount[MaxOff+1]; {
short offcodeNCount[MaxOff + 1];
U32 offcodeMaxValue = MaxOff, offcodeLog; U32 offcodeMaxValue = MaxOff, offcodeLog;
size_t const offcodeHeaderSize = FSE_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dictPtr, dictEnd - dictPtr); size_t const offcodeHeaderSize = FSE_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dictPtr, dictEnd - dictPtr);
if (FSE_isError(offcodeHeaderSize)) return ERROR(dictionary_corrupted); if (FSE_isError(offcodeHeaderSize))
if (offcodeLog > OffFSELog) return ERROR(dictionary_corrupted); return ERROR(dictionary_corrupted);
if (offcodeLog > OffFSELog)
return ERROR(dictionary_corrupted);
CHECK_E(FSE_buildDTable(entropy->OFTable, offcodeNCount, offcodeMaxValue, offcodeLog), dictionary_corrupted); CHECK_E(FSE_buildDTable(entropy->OFTable, offcodeNCount, offcodeMaxValue, offcodeLog), dictionary_corrupted);
dictPtr += offcodeHeaderSize; dictPtr += offcodeHeaderSize;
} }
{ short matchlengthNCount[MaxML+1]; {
short matchlengthNCount[MaxML + 1];
unsigned matchlengthMaxValue = MaxML, matchlengthLog; unsigned matchlengthMaxValue = MaxML, matchlengthLog;
size_t const matchlengthHeaderSize = FSE_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dictPtr, dictEnd - dictPtr); size_t const matchlengthHeaderSize = FSE_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dictPtr, dictEnd - dictPtr);
if (FSE_isError(matchlengthHeaderSize)) return ERROR(dictionary_corrupted); if (FSE_isError(matchlengthHeaderSize))
if (matchlengthLog > MLFSELog) return ERROR(dictionary_corrupted); return ERROR(dictionary_corrupted);
if (matchlengthLog > MLFSELog)
return ERROR(dictionary_corrupted);
CHECK_E(FSE_buildDTable(entropy->MLTable, matchlengthNCount, matchlengthMaxValue, matchlengthLog), dictionary_corrupted); CHECK_E(FSE_buildDTable(entropy->MLTable, matchlengthNCount, matchlengthMaxValue, matchlengthLog), dictionary_corrupted);
dictPtr += matchlengthHeaderSize; dictPtr += matchlengthHeaderSize;
} }
{ short litlengthNCount[MaxLL+1]; {
short litlengthNCount[MaxLL + 1];
unsigned litlengthMaxValue = MaxLL, litlengthLog; unsigned litlengthMaxValue = MaxLL, litlengthLog;
size_t const litlengthHeaderSize = FSE_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dictPtr, dictEnd - dictPtr); size_t const litlengthHeaderSize = FSE_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dictPtr, dictEnd - dictPtr);
if (FSE_isError(litlengthHeaderSize)) return ERROR(dictionary_corrupted); if (FSE_isError(litlengthHeaderSize))
if (litlengthLog > LLFSELog) return ERROR(dictionary_corrupted); return ERROR(dictionary_corrupted);
if (litlengthLog > LLFSELog)
return ERROR(dictionary_corrupted);
CHECK_E(FSE_buildDTable(entropy->LLTable, litlengthNCount, litlengthMaxValue, litlengthLog), dictionary_corrupted); CHECK_E(FSE_buildDTable(entropy->LLTable, litlengthNCount, litlengthMaxValue, litlengthLog), dictionary_corrupted);
dictPtr += litlengthHeaderSize; dictPtr += litlengthHeaderSize;
} }
if (dictPtr+12 > dictEnd) return ERROR(dictionary_corrupted); if (dictPtr + 12 > dictEnd)
{ int i; return ERROR(dictionary_corrupted);
{
int i;
size_t const dictContentSize = (size_t)(dictEnd - (dictPtr + 12)); size_t const dictContentSize = (size_t)(dictEnd - (dictPtr + 12));
for (i = 0; i < 3; i++) { for (i = 0; i < 3; i++) {
U32 const rep = ZSTD_readLE32(dictPtr); dictPtr += 4; U32 const rep = ZSTD_readLE32(dictPtr);
if (rep==0 || rep >= dictContentSize) return ERROR(dictionary_corrupted); dictPtr += 4;
if (rep == 0 || rep >= dictContentSize)
return ERROR(dictionary_corrupted);
entropy->rep[i] = rep; entropy->rep[i] = rep;
} } }
}
return dictPtr - (const BYTE *)dict; return dictPtr - (const BYTE *)dict;
} }
static size_t ZSTD_decompress_insertDictionary(ZSTD_DCtx *dctx, const void *dict, size_t dictSize) static size_t ZSTD_decompress_insertDictionary(ZSTD_DCtx *dctx, const void *dict, size_t dictSize)
{ {
if (dictSize < 8) return ZSTD_refDictContent(dctx, dict, dictSize); if (dictSize < 8)
{ U32 const magic = ZSTD_readLE32(dict); return ZSTD_refDictContent(dctx, dict, dictSize);
{
U32 const magic = ZSTD_readLE32(dict);
if (magic != ZSTD_DICT_MAGIC) { if (magic != ZSTD_DICT_MAGIC) {
return ZSTD_refDictContent(dctx, dict, dictSize); /* pure content mode */ return ZSTD_refDictContent(dctx, dict, dictSize); /* pure content mode */
} } }
}
dctx->dictID = ZSTD_readLE32((const char *)dict + 4); dctx->dictID = ZSTD_readLE32((const char *)dict + 4);
/* load entropy tables */ /* load entropy tables */
{ size_t const eSize = ZSTD_loadEntropy(&dctx->entropy, dict, dictSize); {
if (ZSTD_isError(eSize)) return ERROR(dictionary_corrupted); size_t const eSize = ZSTD_loadEntropy(&dctx->entropy, dict, dictSize);
if (ZSTD_isError(eSize))
return ERROR(dictionary_corrupted);
dict = (const char *)dict + eSize; dict = (const char *)dict + eSize;
dictSize -= eSize; dictSize -= eSize;
} }
@@ -1841,11 +1954,11 @@ static size_t ZSTD_decompress_insertDictionary(ZSTD_DCtx* dctx, const void* dict
size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx *dctx, const void *dict, size_t dictSize) size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx *dctx, const void *dict, size_t dictSize)
{ {
CHECK_F(ZSTD_decompressBegin(dctx)); CHECK_F(ZSTD_decompressBegin(dctx));
if (dict && dictSize) CHECK_E(ZSTD_decompress_insertDictionary(dctx, dict, dictSize), dictionary_corrupted); if (dict && dictSize)
CHECK_E(ZSTD_decompress_insertDictionary(dctx, dict, dictSize), dictionary_corrupted);
return 0; return 0;
} }
/* ====== ZSTD_DDict ====== */ /* ====== ZSTD_DDict ====== */
struct ZSTD_DDict_s { struct ZSTD_DDict_s {
@@ -1858,20 +1971,11 @@ struct ZSTD_DDict_s {
ZSTD_customMem cMem; ZSTD_customMem cMem;
}; /* typedef'd to ZSTD_DDict within "zstd.h" */ }; /* typedef'd to ZSTD_DDict within "zstd.h" */
size_t ZSTD_DDictWorkspaceBound(void) size_t ZSTD_DDictWorkspaceBound(void) { return ZSTD_ALIGN(sizeof(ZSTD_stack)) + ZSTD_ALIGN(sizeof(ZSTD_DDict)); }
{
return ZSTD_ALIGN(sizeof(ZSTD_stack)) + ZSTD_ALIGN(sizeof(ZSTD_DDict));
}
static const void* ZSTD_DDictDictContent(const ZSTD_DDict* ddict) static const void *ZSTD_DDictDictContent(const ZSTD_DDict *ddict) { return ddict->dictContent; }
{
return ddict->dictContent;
}
static size_t ZSTD_DDictDictSize(const ZSTD_DDict* ddict) static size_t ZSTD_DDictDictSize(const ZSTD_DDict *ddict) { return ddict->dictSize; }
{
return ddict->dictSize;
}
static void ZSTD_refDDict(ZSTD_DCtx *dstDCtx, const ZSTD_DDict *ddict) static void ZSTD_refDDict(ZSTD_DCtx *dstDCtx, const ZSTD_DDict *ddict)
{ {
@@ -1903,9 +2007,12 @@ static size_t ZSTD_loadEntropy_inDDict(ZSTD_DDict* ddict)
{ {
ddict->dictID = 0; ddict->dictID = 0;
ddict->entropyPresent = 0; ddict->entropyPresent = 0;
if (ddict->dictSize < 8) return 0; if (ddict->dictSize < 8)
{ U32 const magic = ZSTD_readLE32(ddict->dictContent); return 0;
if (magic != ZSTD_DICT_MAGIC) return 0; /* pure content mode */ {
U32 const magic = ZSTD_readLE32(ddict->dictContent);
if (magic != ZSTD_DICT_MAGIC)
return 0; /* pure content mode */
} }
ddict->dictID = ZSTD_readLE32((const char *)ddict->dictContent + 4); ddict->dictID = ZSTD_readLE32((const char *)ddict->dictContent + 4);
@@ -1915,13 +2022,15 @@ static size_t ZSTD_loadEntropy_inDDict(ZSTD_DDict* ddict)
return 0; return 0;
} }
static ZSTD_DDict *ZSTD_createDDict_advanced(const void *dict, size_t dictSize, unsigned byReference, ZSTD_customMem customMem) static ZSTD_DDict *ZSTD_createDDict_advanced(const void *dict, size_t dictSize, unsigned byReference, ZSTD_customMem customMem)
{ {
if (!customMem.customAlloc || !customMem.customFree) return NULL; if (!customMem.customAlloc || !customMem.customFree)
return NULL;
{ ZSTD_DDict* const ddict = (ZSTD_DDict*) ZSTD_malloc(sizeof(ZSTD_DDict), customMem); {
if (!ddict) return NULL; ZSTD_DDict *const ddict = (ZSTD_DDict *)ZSTD_malloc(sizeof(ZSTD_DDict), customMem);
if (!ddict)
return NULL;
ddict->cMem = customMem; ddict->cMem = customMem;
if ((byReference) || (!dict) || (!dictSize)) { if ((byReference) || (!dict) || (!dictSize)) {
@@ -1929,7 +2038,10 @@ static ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictSize,
ddict->dictContent = dict; ddict->dictContent = dict;
} else { } else {
void *const internalBuffer = ZSTD_malloc(dictSize, customMem); void *const internalBuffer = ZSTD_malloc(dictSize, customMem);
if (!internalBuffer) { ZSTD_freeDDict(ddict); return NULL; } if (!internalBuffer) {
ZSTD_freeDDict(ddict);
return NULL;
}
memcpy(internalBuffer, dict, dictSize); memcpy(internalBuffer, dict, dictSize);
ddict->dictBuffer = internalBuffer; ddict->dictBuffer = internalBuffer;
ddict->dictContent = internalBuffer; ddict->dictContent = internalBuffer;
@@ -1937,11 +2049,13 @@ static ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictSize,
ddict->dictSize = dictSize; ddict->dictSize = dictSize;
ddict->entropy.hufTable[0] = (HUF_DTable)((HufLog)*0x1000001); /* cover both little and big endian */ ddict->entropy.hufTable[0] = (HUF_DTable)((HufLog)*0x1000001); /* cover both little and big endian */
/* parse dictionary content */ /* parse dictionary content */
{ size_t const errorCode = ZSTD_loadEntropy_inDDict(ddict); {
size_t const errorCode = ZSTD_loadEntropy_inDDict(ddict);
if (ZSTD_isError(errorCode)) { if (ZSTD_isError(errorCode)) {
ZSTD_freeDDict(ddict); ZSTD_freeDDict(ddict);
return NULL; return NULL;
} } }
}
return ddict; return ddict;
} }
@@ -1957,11 +2071,12 @@ ZSTD_DDict* ZSTD_initDDict(const void* dict, size_t dictSize, void* workspace, s
return ZSTD_createDDict_advanced(dict, dictSize, 1, stackMem); return ZSTD_createDDict_advanced(dict, dictSize, 1, stackMem);
} }
size_t ZSTD_freeDDict(ZSTD_DDict *ddict) size_t ZSTD_freeDDict(ZSTD_DDict *ddict)
{ {
if (ddict==NULL) return 0; /* support free on NULL */ if (ddict == NULL)
{ ZSTD_customMem const cMem = ddict->cMem; return 0; /* support free on NULL */
{
ZSTD_customMem const cMem = ddict->cMem;
ZSTD_free(ddict->dictBuffer, cMem); ZSTD_free(ddict->dictBuffer, cMem);
ZSTD_free(ddict, cMem); ZSTD_free(ddict, cMem);
return 0; return 0;
@@ -1974,8 +2089,10 @@ size_t ZSTD_freeDDict(ZSTD_DDict* ddict)
* It can still be loaded, but as a content-only dictionary. */ * It can still be loaded, but as a content-only dictionary. */
unsigned ZSTD_getDictID_fromDict(const void *dict, size_t dictSize) unsigned ZSTD_getDictID_fromDict(const void *dict, size_t dictSize)
{ {
if (dictSize < 8) return 0; if (dictSize < 8)
if (ZSTD_readLE32(dict) != ZSTD_DICT_MAGIC) return 0; return 0;
if (ZSTD_readLE32(dict) != ZSTD_DICT_MAGIC)
return 0;
return ZSTD_readLE32((const char *)dict + 4); return ZSTD_readLE32((const char *)dict + 4);
} }
@@ -1985,7 +2102,8 @@ unsigned ZSTD_getDictID_fromDict(const void* dict, size_t dictSize)
* Non-conformant dictionaries can still be loaded, but as content-only dictionaries. */ * Non-conformant dictionaries can still be loaded, but as content-only dictionaries. */
unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict *ddict) unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict *ddict)
{ {
if (ddict==NULL) return 0; if (ddict == NULL)
return 0;
return ZSTD_getDictID_fromDict(ddict->dictContent, ddict->dictSize); return ZSTD_getDictID_fromDict(ddict->dictContent, ddict->dictSize);
} }
@@ -2003,32 +2121,25 @@ unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize)
{ {
ZSTD_frameParams zfp = {0, 0, 0, 0}; ZSTD_frameParams zfp = {0, 0, 0, 0};
size_t const hError = ZSTD_getFrameParams(&zfp, src, srcSize); size_t const hError = ZSTD_getFrameParams(&zfp, src, srcSize);
if (ZSTD_isError(hError)) return 0; if (ZSTD_isError(hError))
return 0;
return zfp.dictID; return zfp.dictID;
} }
/*! ZSTD_decompress_usingDDict() : /*! ZSTD_decompress_usingDDict() :
* Decompression using a pre-digested Dictionary * Decompression using a pre-digested Dictionary
* Use dictionary without significant overhead. */ * Use dictionary without significant overhead. */
size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx, size_t ZSTD_decompress_usingDDict(ZSTD_DCtx *dctx, void *dst, size_t dstCapacity, const void *src, size_t srcSize, const ZSTD_DDict *ddict)
void* dst, size_t dstCapacity,
const void* src, size_t srcSize,
const ZSTD_DDict* ddict)
{ {
/* pass content and size in case legacy frames are encountered */ /* pass content and size in case legacy frames are encountered */
return ZSTD_decompressMultiFrame(dctx, dst, dstCapacity, src, srcSize, return ZSTD_decompressMultiFrame(dctx, dst, dstCapacity, src, srcSize, NULL, 0, ddict);
NULL, 0,
ddict);
} }
/*===================================== /*=====================================
* Streaming decompression * Streaming decompression
*====================================*/ *====================================*/
typedef enum { zdss_init, zdss_loadHeader, typedef enum { zdss_init, zdss_loadHeader, zdss_read, zdss_load, zdss_flush } ZSTD_dStreamStage;
zdss_read, zdss_load, zdss_flush } ZSTD_dStreamStage;
/* *** Resource management *** */ /* *** Resource management *** */
struct ZSTD_DStream_s { struct ZSTD_DStream_s {
@@ -2055,7 +2166,8 @@ struct ZSTD_DStream_s {
U32 hostageByte; U32 hostageByte;
}; /* typedef'd to ZSTD_DStream within "zstd.h" */ }; /* typedef'd to ZSTD_DStream within "zstd.h" */
size_t ZSTD_DStreamWorkspaceBound(size_t maxWindowSize) { size_t ZSTD_DStreamWorkspaceBound(size_t maxWindowSize)
{
size_t const blockSize = MIN(maxWindowSize, ZSTD_BLOCKSIZE_ABSOLUTEMAX); size_t const blockSize = MIN(maxWindowSize, ZSTD_BLOCKSIZE_ABSOLUTEMAX);
size_t const inBuffSize = blockSize; size_t const inBuffSize = blockSize;
size_t const outBuffSize = maxWindowSize + blockSize + WILDCOPY_OVERLENGTH * 2; size_t const outBuffSize = maxWindowSize + blockSize + WILDCOPY_OVERLENGTH * 2;
@@ -2066,14 +2178,19 @@ static ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem)
{ {
ZSTD_DStream *zds; ZSTD_DStream *zds;
if (!customMem.customAlloc || !customMem.customFree) return NULL; if (!customMem.customAlloc || !customMem.customFree)
return NULL;
zds = (ZSTD_DStream *)ZSTD_malloc(sizeof(ZSTD_DStream), customMem); zds = (ZSTD_DStream *)ZSTD_malloc(sizeof(ZSTD_DStream), customMem);
if (zds==NULL) return NULL; if (zds == NULL)
return NULL;
memset(zds, 0, sizeof(ZSTD_DStream)); memset(zds, 0, sizeof(ZSTD_DStream));
memcpy(&zds->customMem, &customMem, sizeof(ZSTD_customMem)); memcpy(&zds->customMem, &customMem, sizeof(ZSTD_customMem));
zds->dctx = ZSTD_createDCtx_advanced(customMem); zds->dctx = ZSTD_createDCtx_advanced(customMem);
if (zds->dctx == NULL) { ZSTD_freeDStream(zds); return NULL; } if (zds->dctx == NULL) {
ZSTD_freeDStream(zds);
return NULL;
}
zds->stage = zdss_init; zds->stage = zdss_init;
zds->maxWindowSize = ZSTD_MAXWINDOWSIZE_DEFAULT; zds->maxWindowSize = ZSTD_MAXWINDOWSIZE_DEFAULT;
return zds; return zds;
@@ -2083,7 +2200,9 @@ ZSTD_DStream* ZSTD_initDStream(size_t maxWindowSize, void* workspace, size_t wor
{ {
ZSTD_customMem const stackMem = ZSTD_initStack(workspace, workspaceSize); ZSTD_customMem const stackMem = ZSTD_initStack(workspace, workspaceSize);
ZSTD_DStream *zds = ZSTD_createDStream_advanced(stackMem); ZSTD_DStream *zds = ZSTD_createDStream_advanced(stackMem);
if (!zds) { return NULL; } if (!zds) {
return NULL;
}
zds->maxWindowSize = maxWindowSize; zds->maxWindowSize = maxWindowSize;
zds->stage = zdss_loadHeader; zds->stage = zdss_loadHeader;
@@ -2107,8 +2226,10 @@ ZSTD_DStream* ZSTD_initDStream_usingDDict(size_t maxWindowSize, const ZSTD_DDict
size_t ZSTD_freeDStream(ZSTD_DStream *zds) size_t ZSTD_freeDStream(ZSTD_DStream *zds)
{ {
if (zds==NULL) return 0; /* support free on null */ if (zds == NULL)
{ ZSTD_customMem const cMem = zds->customMem; return 0; /* support free on null */
{
ZSTD_customMem const cMem = zds->customMem;
ZSTD_freeDCtx(zds->dctx); ZSTD_freeDCtx(zds->dctx);
zds->dctx = NULL; zds->dctx = NULL;
ZSTD_freeDDict(zds->ddictLocal); ZSTD_freeDDict(zds->ddictLocal);
@@ -2122,7 +2243,6 @@ size_t ZSTD_freeDStream(ZSTD_DStream* zds)
} }
} }
/* *** Initialization *** */ /* *** Initialization *** */
size_t ZSTD_DStreamInSize(void) { return ZSTD_BLOCKSIZE_ABSOLUTEMAX + ZSTD_blockHeaderSize; } size_t ZSTD_DStreamInSize(void) { return ZSTD_BLOCKSIZE_ABSOLUTEMAX + ZSTD_blockHeaderSize; }
@@ -2146,7 +2266,6 @@ ZSTD_STATIC size_t ZSTD_limitCopy(void* dst, size_t dstCapacity, const void* src
return length; return length;
} }
size_t ZSTD_decompressStream(ZSTD_DStream *zds, ZSTD_outBuffer *output, ZSTD_inBuffer *input) size_t ZSTD_decompressStream(ZSTD_DStream *zds, ZSTD_outBuffer *output, ZSTD_inBuffer *input)
{ {
const char *const istart = (const char *)(input->src) + input->pos; const char *const istart = (const char *)(input->src) + input->pos;
@@ -2158,14 +2277,13 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
U32 someMoreWork = 1; U32 someMoreWork = 1;
while (someMoreWork) { while (someMoreWork) {
switch(zds->stage) switch (zds->stage) {
{
case zdss_init: case zdss_init:
ZSTD_resetDStream(zds); /* transparent reset on starting decoding a new frame */ ZSTD_resetDStream(zds); /* transparent reset on starting decoding a new frame */
/* fall-through */ /* fall-through */
case zdss_loadHeader : case zdss_loadHeader: {
{ size_t const hSize = ZSTD_getFrameParams(&zds->fParams, zds->headerBuffer, zds->lhSize); size_t const hSize = ZSTD_getFrameParams(&zds->fParams, zds->headerBuffer, zds->lhSize);
if (ZSTD_isError(hSize)) if (ZSTD_isError(hSize))
return hSize; return hSize;
if (hSize != 0) { /* need more input */ if (hSize != 0) { /* need more input */
@@ -2174,11 +2292,15 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
memcpy(zds->headerBuffer + zds->lhSize, ip, iend - ip); memcpy(zds->headerBuffer + zds->lhSize, ip, iend - ip);
zds->lhSize += iend - ip; zds->lhSize += iend - ip;
input->pos = input->size; input->pos = input->size;
return (MAX(ZSTD_frameHeaderSize_min, hSize) - zds->lhSize) + ZSTD_blockHeaderSize; /* remaining header bytes + next block header */ return (MAX(ZSTD_frameHeaderSize_min, hSize) - zds->lhSize) +
ZSTD_blockHeaderSize; /* remaining header bytes + next block header */
} }
memcpy(zds->headerBuffer + zds->lhSize, ip, toLoad); zds->lhSize = hSize; ip += toLoad; memcpy(zds->headerBuffer + zds->lhSize, ip, toLoad);
zds->lhSize = hSize;
ip += toLoad;
break; break;
} } }
}
/* check for single-pass mode opportunity */ /* check for single-pass mode opportunity */
if (zds->fParams.frameContentSize && zds->fParams.windowSize /* skippable frame if == 0 */ if (zds->fParams.frameContentSize && zds->fParams.windowSize /* skippable frame if == 0 */
@@ -2186,47 +2308,57 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
size_t const cSize = ZSTD_findFrameCompressedSize(istart, iend - istart); size_t const cSize = ZSTD_findFrameCompressedSize(istart, iend - istart);
if (cSize <= (size_t)(iend - istart)) { if (cSize <= (size_t)(iend - istart)) {
size_t const decompressedSize = ZSTD_decompress_usingDDict(zds->dctx, op, oend - op, istart, cSize, zds->ddict); size_t const decompressedSize = ZSTD_decompress_usingDDict(zds->dctx, op, oend - op, istart, cSize, zds->ddict);
if (ZSTD_isError(decompressedSize)) return decompressedSize; if (ZSTD_isError(decompressedSize))
return decompressedSize;
ip = istart + cSize; ip = istart + cSize;
op += decompressedSize; op += decompressedSize;
zds->dctx->expected = 0; zds->dctx->expected = 0;
zds->stage = zdss_init; zds->stage = zdss_init;
someMoreWork = 0; someMoreWork = 0;
break; break;
} } }
}
/* Consume header */ /* Consume header */
ZSTD_refDDict(zds->dctx, zds->ddict); ZSTD_refDDict(zds->dctx, zds->ddict);
{ size_t const h1Size = ZSTD_nextSrcSizeToDecompress(zds->dctx); /* == ZSTD_frameHeaderSize_prefix */ {
size_t const h1Size = ZSTD_nextSrcSizeToDecompress(zds->dctx); /* == ZSTD_frameHeaderSize_prefix */
CHECK_F(ZSTD_decompressContinue(zds->dctx, NULL, 0, zds->headerBuffer, h1Size)); CHECK_F(ZSTD_decompressContinue(zds->dctx, NULL, 0, zds->headerBuffer, h1Size));
{ size_t const h2Size = ZSTD_nextSrcSizeToDecompress(zds->dctx); {
size_t const h2Size = ZSTD_nextSrcSizeToDecompress(zds->dctx);
CHECK_F(ZSTD_decompressContinue(zds->dctx, NULL, 0, zds->headerBuffer + h1Size, h2Size)); CHECK_F(ZSTD_decompressContinue(zds->dctx, NULL, 0, zds->headerBuffer + h1Size, h2Size));
} } }
}
zds->fParams.windowSize = MAX(zds->fParams.windowSize, 1U << ZSTD_WINDOWLOG_ABSOLUTEMIN); zds->fParams.windowSize = MAX(zds->fParams.windowSize, 1U << ZSTD_WINDOWLOG_ABSOLUTEMIN);
if (zds->fParams.windowSize > zds->maxWindowSize) return ERROR(frameParameter_windowTooLarge); if (zds->fParams.windowSize > zds->maxWindowSize)
return ERROR(frameParameter_windowTooLarge);
/* Adapt buffer sizes to frame header instructions */ /* Adapt buffer sizes to frame header instructions */
{ size_t const blockSize = MIN(zds->fParams.windowSize, ZSTD_BLOCKSIZE_ABSOLUTEMAX); {
size_t const blockSize = MIN(zds->fParams.windowSize, ZSTD_BLOCKSIZE_ABSOLUTEMAX);
size_t const neededOutSize = zds->fParams.windowSize + blockSize + WILDCOPY_OVERLENGTH * 2; size_t const neededOutSize = zds->fParams.windowSize + blockSize + WILDCOPY_OVERLENGTH * 2;
zds->blockSize = blockSize; zds->blockSize = blockSize;
if (zds->inBuffSize < blockSize) { if (zds->inBuffSize < blockSize) {
ZSTD_free(zds->inBuff, zds->customMem); ZSTD_free(zds->inBuff, zds->customMem);
zds->inBuffSize = blockSize; zds->inBuffSize = blockSize;
zds->inBuff = (char *)ZSTD_malloc(blockSize, zds->customMem); zds->inBuff = (char *)ZSTD_malloc(blockSize, zds->customMem);
if (zds->inBuff == NULL) return ERROR(memory_allocation); if (zds->inBuff == NULL)
return ERROR(memory_allocation);
} }
if (zds->outBuffSize < neededOutSize) { if (zds->outBuffSize < neededOutSize) {
ZSTD_free(zds->outBuff, zds->customMem); ZSTD_free(zds->outBuff, zds->customMem);
zds->outBuffSize = neededOutSize; zds->outBuffSize = neededOutSize;
zds->outBuff = (char *)ZSTD_malloc(neededOutSize, zds->customMem); zds->outBuff = (char *)ZSTD_malloc(neededOutSize, zds->customMem);
if (zds->outBuff == NULL) return ERROR(memory_allocation); if (zds->outBuff == NULL)
} } return ERROR(memory_allocation);
}
}
zds->stage = zdss_read; zds->stage = zdss_read;
/* pass-through */ /* pass-through */
case zdss_read: case zdss_read: {
{ size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zds->dctx); size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zds->dctx);
if (neededInSize == 0) { /* end of frame */ if (neededInSize == 0) { /* end of frame */
zds->stage = zdss_init; zds->stage = zdss_init;
someMoreWork = 0; someMoreWork = 0;
@@ -2234,46 +2366,59 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
} }
if ((size_t)(iend - ip) >= neededInSize) { /* decode directly from src */ if ((size_t)(iend - ip) >= neededInSize) { /* decode directly from src */
const int isSkipFrame = ZSTD_isSkipFrame(zds->dctx); const int isSkipFrame = ZSTD_isSkipFrame(zds->dctx);
size_t const decodedSize = ZSTD_decompressContinue(zds->dctx, size_t const decodedSize = ZSTD_decompressContinue(zds->dctx, zds->outBuff + zds->outStart,
zds->outBuff + zds->outStart, (isSkipFrame ? 0 : zds->outBuffSize - zds->outStart), (isSkipFrame ? 0 : zds->outBuffSize - zds->outStart), ip, neededInSize);
ip, neededInSize); if (ZSTD_isError(decodedSize))
if (ZSTD_isError(decodedSize)) return decodedSize; return decodedSize;
ip += neededInSize; ip += neededInSize;
if (!decodedSize && !isSkipFrame) break; /* this was just a header */ if (!decodedSize && !isSkipFrame)
break; /* this was just a header */
zds->outEnd = zds->outStart + decodedSize; zds->outEnd = zds->outStart + decodedSize;
zds->stage = zdss_flush; zds->stage = zdss_flush;
break; break;
} }
if (ip==iend) { someMoreWork = 0; break; } /* no more input */ if (ip == iend) {
someMoreWork = 0;
break;
} /* no more input */
zds->stage = zdss_load; zds->stage = zdss_load;
/* pass-through */ /* pass-through */
} }
case zdss_load: case zdss_load: {
{ size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zds->dctx); size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zds->dctx);
size_t const toLoad = neededInSize - zds->inPos; /* should always be <= remaining space within inBuff */ size_t const toLoad = neededInSize - zds->inPos; /* should always be <= remaining space within inBuff */
size_t loadedSize; size_t loadedSize;
if (toLoad > zds->inBuffSize - zds->inPos) return ERROR(corruption_detected); /* should never happen */ if (toLoad > zds->inBuffSize - zds->inPos)
return ERROR(corruption_detected); /* should never happen */
loadedSize = ZSTD_limitCopy(zds->inBuff + zds->inPos, toLoad, ip, iend - ip); loadedSize = ZSTD_limitCopy(zds->inBuff + zds->inPos, toLoad, ip, iend - ip);
ip += loadedSize; ip += loadedSize;
zds->inPos += loadedSize; zds->inPos += loadedSize;
if (loadedSize < toLoad) { someMoreWork = 0; break; } /* not enough input, wait for more */ if (loadedSize < toLoad) {
someMoreWork = 0;
break;
} /* not enough input, wait for more */
/* decode loaded input */ /* decode loaded input */
{ const int isSkipFrame = ZSTD_isSkipFrame(zds->dctx); {
size_t const decodedSize = ZSTD_decompressContinue(zds->dctx, const int isSkipFrame = ZSTD_isSkipFrame(zds->dctx);
zds->outBuff + zds->outStart, zds->outBuffSize - zds->outStart, size_t const decodedSize = ZSTD_decompressContinue(zds->dctx, zds->outBuff + zds->outStart, zds->outBuffSize - zds->outStart,
zds->inBuff, neededInSize); zds->inBuff, neededInSize);
if (ZSTD_isError(decodedSize)) return decodedSize; if (ZSTD_isError(decodedSize))
return decodedSize;
zds->inPos = 0; /* input is consumed */ zds->inPos = 0; /* input is consumed */
if (!decodedSize && !isSkipFrame) { zds->stage = zdss_read; break; } /* this was just a header */ if (!decodedSize && !isSkipFrame) {
zds->stage = zdss_read;
break;
} /* this was just a header */
zds->outEnd = zds->outStart + decodedSize; zds->outEnd = zds->outStart + decodedSize;
zds->stage = zdss_flush; zds->stage = zdss_flush;
/* pass-through */ /* pass-through */
} } }
}
case zdss_flush: case zdss_flush: {
{ size_t const toFlushSize = zds->outEnd - zds->outStart; size_t const toFlushSize = zds->outEnd - zds->outStart;
size_t const flushedSize = ZSTD_limitCopy(op, oend - op, zds->outBuff + zds->outStart, toFlushSize); size_t const flushedSize = ZSTD_limitCopy(op, oend - op, zds->outBuff + zds->outStart, toFlushSize);
op += flushedSize; op += flushedSize;
zds->outStart += flushedSize; zds->outStart += flushedSize;
@@ -2287,17 +2432,23 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
someMoreWork = 0; someMoreWork = 0;
break; break;
} }
default: return ERROR(GENERIC); /* impossible */ default:
} } return ERROR(GENERIC); /* impossible */
}
}
/* result */ /* result */
input->pos += (size_t)(ip - istart); input->pos += (size_t)(ip - istart);
output->pos += (size_t)(op - ostart); output->pos += (size_t)(op - ostart);
{ size_t nextSrcSizeHint = ZSTD_nextSrcSizeToDecompress(zds->dctx); {
size_t nextSrcSizeHint = ZSTD_nextSrcSizeToDecompress(zds->dctx);
if (!nextSrcSizeHint) { /* frame fully decoded */ if (!nextSrcSizeHint) { /* frame fully decoded */
if (zds->outEnd == zds->outStart) { /* output fully flushed */ if (zds->outEnd == zds->outStart) { /* output fully flushed */
if (zds->hostageByte) { if (zds->hostageByte) {
if (input->pos >= input->size) { zds->stage = zdss_read; return 1; } /* can't release hostage (not present) */ if (input->pos >= input->size) {
zds->stage = zdss_read;
return 1;
} /* can't release hostage (not present) */
input->pos++; /* release hostage */ input->pos++; /* release hostage */
} }
return 0; return 0;
@@ -2309,7 +2460,8 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
return 1; return 1;
} }
nextSrcSizeHint += ZSTD_blockHeaderSize * (ZSTD_nextInputType(zds->dctx) == ZSTDnit_block); /* preload header of next block */ nextSrcSizeHint += ZSTD_blockHeaderSize * (ZSTD_nextInputType(zds->dctx) == ZSTDnit_block); /* preload header of next block */
if (zds->inPos > nextSrcSizeHint) return ERROR(GENERIC); /* should never happen */ if (zds->inPos > nextSrcSizeHint)
return ERROR(GENERIC); /* should never happen */
nextSrcSizeHint -= zds->inPos; /* already loaded*/ nextSrcSizeHint -= zds->inPos; /* already loaded*/
return nextSrcSizeHint; return nextSrcSizeHint;
} }
contrib/linux-kernel/lib/zstd/entropy_common.c
+61 -39
View File
@@ -40,27 +40,23 @@
/* ************************************* /* *************************************
* Dependencies * Dependencies
***************************************/ ***************************************/
#include "mem.h"
#include "error_private.h" /* ERR_*, ERROR */ #include "error_private.h" /* ERR_*, ERROR */
#include "fse.h" #include "fse.h"
#include "huf.h" #include "huf.h"
#include "mem.h"
/*=== Version ===*/ /*=== Version ===*/
unsigned FSE_versionNumber(void) { return FSE_VERSION_NUMBER; } unsigned FSE_versionNumber(void) { return FSE_VERSION_NUMBER; }
/*=== Error Management ===*/ /*=== Error Management ===*/
unsigned FSE_isError(size_t code) { return ERR_isError(code); } unsigned FSE_isError(size_t code) { return ERR_isError(code); }
unsigned HUF_isError(size_t code) { return ERR_isError(code); } unsigned HUF_isError(size_t code) { return ERR_isError(code); }
/*-************************************************************** /*-**************************************************************
* FSE NCount encoding-decoding * FSE NCount encoding-decoding
****************************************************************/ ****************************************************************/
size_t FSE_readNCount (short* normalizedCounter, unsigned* maxSVPtr, unsigned* tableLogPtr, size_t FSE_readNCount(short *normalizedCounter, unsigned *maxSVPtr, unsigned *tableLogPtr, const void *headerBuffer, size_t hbSize)
const void* headerBuffer, size_t hbSize)
{ {
const BYTE *const istart = (const BYTE *)headerBuffer; const BYTE *const istart = (const BYTE *)headerBuffer;
const BYTE *const iend = istart + hbSize; const BYTE *const iend = istart + hbSize;
@@ -73,10 +69,12 @@ size_t FSE_readNCount (short* normalizedCounter, unsigned* maxSVPtr, unsigned* t
unsigned charnum = 0; unsigned charnum = 0;
int previous0 = 0; int previous0 = 0;
if (hbSize < 4) return ERROR(srcSize_wrong); if (hbSize < 4)
return ERROR(srcSize_wrong);
bitStream = ZSTD_readLE32(ip); bitStream = ZSTD_readLE32(ip);
nbBits = (bitStream & 0xF) + FSE_MIN_TABLELOG; /* extract tableLog */ nbBits = (bitStream & 0xF) + FSE_MIN_TABLELOG; /* extract tableLog */
if (nbBits > FSE_TABLELOG_ABSOLUTE_MAX) return ERROR(tableLog_tooLarge); if (nbBits > FSE_TABLELOG_ABSOLUTE_MAX)
return ERROR(tableLog_tooLarge);
bitStream >>= 4; bitStream >>= 4;
bitCount = 4; bitCount = 4;
*tableLogPtr = nbBits; *tableLogPtr = nbBits;
@@ -95,7 +93,8 @@ size_t FSE_readNCount (short* normalizedCounter, unsigned* maxSVPtr, unsigned* t
} else { } else {
bitStream >>= 16; bitStream >>= 16;
bitCount += 16; bitCount += 16;
} } }
}
while ((bitStream & 3) == 3) { while ((bitStream & 3) == 3) {
n0 += 3; n0 += 3;
bitStream >>= 2; bitStream >>= 2;
@@ -103,16 +102,20 @@ size_t FSE_readNCount (short* normalizedCounter, unsigned* maxSVPtr, unsigned* t
} }
n0 += bitStream & 3; n0 += bitStream & 3;
bitCount += 2; bitCount += 2;
if (n0 > *maxSVPtr) return ERROR(maxSymbolValue_tooSmall); if (n0 > *maxSVPtr)
while (charnum < n0) normalizedCounter[charnum++] = 0; return ERROR(maxSymbolValue_tooSmall);
while (charnum < n0)
normalizedCounter[charnum++] = 0;
if ((ip <= iend - 7) || (ip + (bitCount >> 3) <= iend - 4)) { if ((ip <= iend - 7) || (ip + (bitCount >> 3) <= iend - 4)) {
ip += bitCount >> 3; ip += bitCount >> 3;
bitCount &= 7; bitCount &= 7;
bitStream = ZSTD_readLE32(ip) >> bitCount; bitStream = ZSTD_readLE32(ip) >> bitCount;
} else { } else {
bitStream >>= 2; bitStream >>= 2;
} } }
{ int const max = (2*threshold-1) - remaining; }
{
int const max = (2 * threshold - 1) - remaining;
int count; int count;
if ((bitStream & (threshold - 1)) < (U32)max) { if ((bitStream & (threshold - 1)) < (U32)max) {
@@ -120,7 +123,8 @@ size_t FSE_readNCount (short* normalizedCounter, unsigned* maxSVPtr, unsigned* t
bitCount += nbBits - 1; bitCount += nbBits - 1;
} else { } else {
count = bitStream & (2 * threshold - 1); count = bitStream & (2 * threshold - 1);
if (count >= threshold) count -= max; if (count >= threshold)
count -= max;
bitCount += nbBits; bitCount += nbBits;
} }
@@ -141,16 +145,18 @@ size_t FSE_readNCount (short* normalizedCounter, unsigned* maxSVPtr, unsigned* t
ip = iend - 4; ip = iend - 4;
} }
bitStream = ZSTD_readLE32(ip) >> (bitCount & 31); bitStream = ZSTD_readLE32(ip) >> (bitCount & 31);
} } /* while ((remaining>1) & (charnum<=*maxSVPtr)) */ }
if (remaining != 1) return ERROR(corruption_detected); } /* while ((remaining>1) & (charnum<=*maxSVPtr)) */
if (bitCount > 32) return ERROR(corruption_detected); if (remaining != 1)
return ERROR(corruption_detected);
if (bitCount > 32)
return ERROR(corruption_detected);
*maxSVPtr = charnum - 1; *maxSVPtr = charnum - 1;
ip += (bitCount + 7) >> 3; ip += (bitCount + 7) >> 3;
return ip - istart; return ip - istart;
} }
/*! HUF_readStats() : /*! HUF_readStats() :
Read compact Huffman tree, saved by HUF_writeCTable(). Read compact Huffman tree, saved by HUF_writeCTable().
`huffWeight` is destination buffer. `huffWeight` is destination buffer.
@@ -158,63 +164,79 @@ size_t FSE_readNCount (short* normalizedCounter, unsigned* maxSVPtr, unsigned* t
@return : size read from `src` , or an error Code . @return : size read from `src` , or an error Code .
Note : Needed by HUF_readCTable() and HUF_readDTableX?() . Note : Needed by HUF_readCTable() and HUF_readDTableX?() .
*/ */
size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats, size_t HUF_readStats(BYTE *huffWeight, size_t hwSize, U32 *rankStats, U32 *nbSymbolsPtr, U32 *tableLogPtr, const void *src, size_t srcSize)
U32* nbSymbolsPtr, U32* tableLogPtr,
const void* src, size_t srcSize)
{ {
U32 weightTotal; U32 weightTotal;
const BYTE *ip = (const BYTE *)src; const BYTE *ip = (const BYTE *)src;
size_t iSize; size_t iSize;
size_t oSize; size_t oSize;
if (!srcSize) return ERROR(srcSize_wrong); if (!srcSize)
return ERROR(srcSize_wrong);
iSize = ip[0]; iSize = ip[0];
/* memset(huffWeight, 0, hwSize); */ /* is not necessary, even though some analyzer complain ... */ /* memset(huffWeight, 0, hwSize); */ /* is not necessary, even though some analyzer complain ... */
if (iSize >= 128) { /* special header */ if (iSize >= 128) { /* special header */
oSize = iSize - 127; oSize = iSize - 127;
iSize = ((oSize + 1) / 2); iSize = ((oSize + 1) / 2);
if (iSize+1 > srcSize) return ERROR(srcSize_wrong); if (iSize + 1 > srcSize)
if (oSize >= hwSize) return ERROR(corruption_detected); return ERROR(srcSize_wrong);
if (oSize >= hwSize)
return ERROR(corruption_detected);
ip += 1; ip += 1;
{ U32 n; {
U32 n;
for (n = 0; n < oSize; n += 2) { for (n = 0; n < oSize; n += 2) {
huffWeight[n] = ip[n / 2] >> 4; huffWeight[n] = ip[n / 2] >> 4;
huffWeight[n + 1] = ip[n / 2] & 15; huffWeight[n + 1] = ip[n / 2] & 15;
} } } }
else { /* header compressed with FSE (normal case) */ }
} else { /* header compressed with FSE (normal case) */
FSE_DTable fseWorkspace[FSE_DTABLE_SIZE_U32(6)]; /* 6 is max possible tableLog for HUF header (maybe even 5, to be tested) */ FSE_DTable fseWorkspace[FSE_DTABLE_SIZE_U32(6)]; /* 6 is max possible tableLog for HUF header (maybe even 5, to be tested) */
if (iSize+1 > srcSize) return ERROR(srcSize_wrong); if (iSize + 1 > srcSize)
return ERROR(srcSize_wrong);
oSize = FSE_decompress_wksp(huffWeight, hwSize - 1, ip + 1, iSize, fseWorkspace, 6); /* max (hwSize-1) values decoded, as last one is implied */ oSize = FSE_decompress_wksp(huffWeight, hwSize - 1, ip + 1, iSize, fseWorkspace, 6); /* max (hwSize-1) values decoded, as last one is implied */
if (FSE_isError(oSize)) return oSize; if (FSE_isError(oSize))
return oSize;
} }
/* collect weight stats */ /* collect weight stats */
memset(rankStats, 0, (HUF_TABLELOG_MAX + 1) * sizeof(U32)); memset(rankStats, 0, (HUF_TABLELOG_MAX + 1) * sizeof(U32));
weightTotal = 0; weightTotal = 0;
{ U32 n; for (n=0; n<oSize; n++) { {
if (huffWeight[n] >= HUF_TABLELOG_MAX) return ERROR(corruption_detected); U32 n;
for (n = 0; n < oSize; n++) {
if (huffWeight[n] >= HUF_TABLELOG_MAX)
return ERROR(corruption_detected);
rankStats[huffWeight[n]]++; rankStats[huffWeight[n]]++;
weightTotal += (1 << huffWeight[n]) >> 1; weightTotal += (1 << huffWeight[n]) >> 1;
} } }
if (weightTotal == 0) return ERROR(corruption_detected); }
if (weightTotal == 0)
return ERROR(corruption_detected);
/* get last non-null symbol weight (implied, total must be 2^n) */ /* get last non-null symbol weight (implied, total must be 2^n) */
{ U32 const tableLog = BIT_highbit32(weightTotal) + 1; {
if (tableLog > HUF_TABLELOG_MAX) return ERROR(corruption_detected); U32 const tableLog = BIT_highbit32(weightTotal) + 1;
if (tableLog > HUF_TABLELOG_MAX)
return ERROR(corruption_detected);
*tableLogPtr = tableLog; *tableLogPtr = tableLog;
/* determine last weight */ /* determine last weight */
{ U32 const total = 1 << tableLog; {
U32 const total = 1 << tableLog;
U32 const rest = total - weightTotal; U32 const rest = total - weightTotal;
U32 const verif = 1 << BIT_highbit32(rest); U32 const verif = 1 << BIT_highbit32(rest);
U32 const lastWeight = BIT_highbit32(rest) + 1; U32 const lastWeight = BIT_highbit32(rest) + 1;
if (verif != rest) return ERROR(corruption_detected); /* last value must be a clean power of 2 */ if (verif != rest)
return ERROR(corruption_detected); /* last value must be a clean power of 2 */
huffWeight[oSize] = (BYTE)lastWeight; huffWeight[oSize] = (BYTE)lastWeight;
rankStats[lastWeight]++; rankStats[lastWeight]++;
} } }
}
/* check tree construction validity */ /* check tree construction validity */
if ((rankStats[1] < 2) || (rankStats[1] & 1)) return ERROR(corruption_detected); /* by construction : at least 2 elts of rank 1, must be even */ if ((rankStats[1] < 2) || (rankStats[1] & 1))
return ERROR(corruption_detected); /* by construction : at least 2 elts of rank 1, must be even */
/* results */ /* results */
*nbSymbolsPtr = (U32)(oSize + 1); *nbSymbolsPtr = (U32)(oSize + 1);
contrib/linux-kernel/lib/zstd/error_private.h
+6 -4
View File
@@ -25,20 +25,17 @@
#include <linux/types.h> /* size_t */ #include <linux/types.h> /* size_t */
#include <linux/zstd.h> /* enum list */ #include <linux/zstd.h> /* enum list */
/* **************************************** /* ****************************************
* Compiler-specific * Compiler-specific
******************************************/ ******************************************/
#define ERR_STATIC static __attribute__((unused)) #define ERR_STATIC static __attribute__((unused))
/*-**************************************** /*-****************************************
* Customization (error_public.h) * Customization (error_public.h)
******************************************/ ******************************************/
typedef ZSTD_ErrorCode ERR_enum; typedef ZSTD_ErrorCode ERR_enum;
#define PREFIX(name) ZSTD_error_##name #define PREFIX(name) ZSTD_error_##name
/*-**************************************** /*-****************************************
* Error codes handling * Error codes handling
******************************************/ ******************************************/
@@ -46,6 +43,11 @@ typedef ZSTD_ErrorCode ERR_enum;
ERR_STATIC unsigned ERR_isError(size_t code) { return (code > ERROR(maxCode)); } ERR_STATIC unsigned ERR_isError(size_t code) { return (code > ERROR(maxCode)); }
ERR_STATIC ERR_enum ERR_getErrorCode(size_t code) { if (!ERR_isError(code)) return (ERR_enum)0; return (ERR_enum) (0-code); } ERR_STATIC ERR_enum ERR_getErrorCode(size_t code)
{
if (!ERR_isError(code))
return (ERR_enum)0;
return (ERR_enum)(0 - code);
}
#endif /* ERROR_H_MODULE */ #endif /* ERROR_H_MODULE */
contrib/linux-kernel/lib/zstd/fse.h
+9 -33
View File
@@ -40,13 +40,11 @@
#ifndef FSE_H #ifndef FSE_H
#define FSE_H #define FSE_H
/*-***************************************** /*-*****************************************
* Dependencies * Dependencies
******************************************/ ******************************************/
#include <linux/types.h> /* size_t, ptrdiff_t */ #include <linux/types.h> /* size_t, ptrdiff_t */
/*-***************************************** /*-*****************************************
* FSE_PUBLIC_API : control library symbols visibility * FSE_PUBLIC_API : control library symbols visibility
******************************************/ ******************************************/
@@ -73,7 +71,6 @@ FSE_PUBLIC_API size_t FSE_compressBound(size_t size); /* maximum compresse
/* Error Management */ /* Error Management */
FSE_PUBLIC_API unsigned FSE_isError(size_t code); /* tells if a return value is an error code */ FSE_PUBLIC_API unsigned FSE_isError(size_t code); /* tells if a return value is an error code */
/*-***************************************** /*-*****************************************
* FSE detailed API * FSE detailed API
******************************************/ ******************************************/
@@ -120,7 +117,6 @@ FSE_PUBLIC_API size_t FSE_NCountWriteBound(unsigned maxSymbolValue, unsigned tab
or an errorCode, which can be tested using FSE_isError(). */ or an errorCode, which can be tested using FSE_isError(). */
FSE_PUBLIC_API size_t FSE_writeNCount(void *buffer, size_t bufferSize, const short *normalizedCounter, unsigned maxSymbolValue, unsigned tableLog); FSE_PUBLIC_API size_t FSE_writeNCount(void *buffer, size_t bufferSize, const short *normalizedCounter, unsigned maxSymbolValue, unsigned tableLog);
/*! Constructor and Destructor of FSE_CTable. /*! Constructor and Destructor of FSE_CTable.
Note that FSE_CTable size depends on 'tableLog' and 'maxSymbolValue' */ Note that FSE_CTable size depends on 'tableLog' and 'maxSymbolValue' */
typedef unsigned FSE_CTable; /* don't allocate that. It's only meant to be more restrictive than void* */ typedef unsigned FSE_CTable; /* don't allocate that. It's only meant to be more restrictive than void* */
@@ -175,7 +171,6 @@ If it returns '0', compressed data could not fit into 'dst'.
If there is an error, the function will return an ErrorCode (which can be tested using FSE_isError()). If there is an error, the function will return an ErrorCode (which can be tested using FSE_isError()).
*/ */
/* *** DECOMPRESSION *** */ /* *** DECOMPRESSION *** */
/*! FSE_readNCount(): /*! FSE_readNCount():
@@ -229,11 +224,9 @@ FSE_decompress_usingDTable() result will tell how many bytes were regenerated (<
If there is an error, the function will return an error code, which can be tested using FSE_isError(). (ex: dst buffer too small) If there is an error, the function will return an error code, which can be tested using FSE_isError(). (ex: dst buffer too small)
*/ */
/* *** Dependency *** */ /* *** Dependency *** */
#include "bitstream.h" #include "bitstream.h"
/* ***************************************** /* *****************************************
* Static allocation * Static allocation
*******************************************/ *******************************************/
@@ -246,7 +239,6 @@ If there is an error, the function will return an error code, which can be teste
#define FSE_CTABLE_SIZE_U32(maxTableLog, maxSymbolValue) (1 + (1 << (maxTableLog - 1)) + ((maxSymbolValue + 1) * 2)) #define FSE_CTABLE_SIZE_U32(maxTableLog, maxSymbolValue) (1 + (1 << (maxTableLog - 1)) + ((maxSymbolValue + 1) * 2))
#define FSE_DTABLE_SIZE_U32(maxTableLog) (1 + (1 << maxTableLog)) #define FSE_DTABLE_SIZE_U32(maxTableLog) (1 + (1 << maxTableLog))
/* ***************************************** /* *****************************************
* FSE advanced API * FSE advanced API
*******************************************/ *******************************************/
@@ -254,8 +246,7 @@ If there is an error, the function will return an error code, which can be teste
* Same as FSE_count(), but using an externally provided scratch buffer. * Same as FSE_count(), but using an externally provided scratch buffer.
* `workSpace` size must be table of >= `1024` unsigned * `workSpace` size must be table of >= `1024` unsigned
*/ */
size_t FSE_count_wksp(unsigned* count, unsigned* maxSymbolValuePtr, size_t FSE_count_wksp(unsigned *count, unsigned *maxSymbolValuePtr, const void *source, size_t sourceSize, unsigned *workSpace);
const void* source, size_t sourceSize, unsigned* workSpace);
/* FSE_countFast_wksp() : /* FSE_countFast_wksp() :
* Same as FSE_countFast(), but using an externally provided scratch buffer. * Same as FSE_countFast(), but using an externally provided scratch buffer.
@@ -269,8 +260,6 @@ size_t FSE_countFast_wksp(unsigned* count, unsigned* maxSymbolValuePtr, const vo
*/ */
size_t FSE_count_simple(unsigned *count, unsigned *maxSymbolValuePtr, const void *src, size_t srcSize); size_t FSE_count_simple(unsigned *count, unsigned *maxSymbolValuePtr, const void *src, size_t srcSize);
unsigned FSE_optimalTableLog_internal(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue, unsigned minus); unsigned FSE_optimalTableLog_internal(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue, unsigned minus);
/**< same as FSE_optimalTableLog(), which used `minus==2` */ /**< same as FSE_optimalTableLog(), which used `minus==2` */
@@ -278,8 +267,10 @@ unsigned FSE_optimalTableLog_internal(unsigned maxTableLog, size_t srcSize, unsi
* Same as FSE_compress2(), but using an externally allocated scratch buffer (`workSpace`). * Same as FSE_compress2(), but using an externally allocated scratch buffer (`workSpace`).
* FSE_WKSP_SIZE_U32() provides the minimum size required for `workSpace` as a table of FSE_CTable. * FSE_WKSP_SIZE_U32() provides the minimum size required for `workSpace` as a table of FSE_CTable.
*/ */
#define FSE_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) ( FSE_CTABLE_SIZE_U32(maxTableLog, maxSymbolValue) + ((maxTableLog > 12) ? (1 << (maxTableLog - 2)) : 1024) ) #define FSE_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) \
size_t FSE_compress_wksp (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize); (FSE_CTABLE_SIZE_U32(maxTableLog, maxSymbolValue) + ((maxTableLog > 12) ? (1 << (maxTableLog - 2)) : 1024))
size_t FSE_compress_wksp(void *dst, size_t dstSize, const void *src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void *workSpace,
size_t wkspSize);
size_t FSE_buildCTable_raw(FSE_CTable *ct, unsigned nbBits); size_t FSE_buildCTable_raw(FSE_CTable *ct, unsigned nbBits);
/**< build a fake FSE_CTable, designed for a flat distribution, where each symbol uses nbBits */ /**< build a fake FSE_CTable, designed for a flat distribution, where each symbol uses nbBits */
@@ -302,7 +293,6 @@ size_t FSE_buildDTable_rle (FSE_DTable* dt, unsigned char symbolValue);
size_t FSE_decompress_wksp(void *dst, size_t dstCapacity, const void *cSrc, size_t cSrcSize, FSE_DTable *workSpace, unsigned maxLog); size_t FSE_decompress_wksp(void *dst, size_t dstCapacity, const void *cSrc, size_t cSrcSize, FSE_DTable *workSpace, unsigned maxLog);
/**< same as FSE_decompress(), using an externally allocated `workSpace` produced with `FSE_DTABLE_SIZE_U32(maxLog)` */ /**< same as FSE_decompress(), using an externally allocated `workSpace` produced with `FSE_DTABLE_SIZE_U32(maxLog)` */
/* ***************************************** /* *****************************************
* FSE symbol compression API * FSE symbol compression API
*******************************************/ *******************************************/
@@ -366,7 +356,6 @@ If there is an error, it returns an errorCode (which can be tested using FSE_isE
size_t size = BIT_closeCStream(&bitStream); size_t size = BIT_closeCStream(&bitStream);
*/ */
/* ***************************************** /* *****************************************
* FSE symbol decompression API * FSE symbol decompression API
*******************************************/ *******************************************/
@@ -375,7 +364,6 @@ typedef struct {
const void *table; /* precise table may vary, depending on U16 */ const void *table; /* precise table may vary, depending on U16 */
} FSE_DState_t; } FSE_DState_t;
static void FSE_initDState(FSE_DState_t *DStatePtr, BIT_DStream_t *bitD, const FSE_DTable *dt); static void FSE_initDState(FSE_DState_t *DStatePtr, BIT_DStream_t *bitD, const FSE_DTable *dt);
static unsigned char FSE_decodeSymbol(FSE_DState_t *DStatePtr, BIT_DStream_t *bitD); static unsigned char FSE_decodeSymbol(FSE_DState_t *DStatePtr, BIT_DStream_t *bitD);
@@ -431,14 +419,12 @@ Check also the states. There might be some symbols left there, if some high prob
FSE_endOfDState(&DState); FSE_endOfDState(&DState);
*/ */
/* ***************************************** /* *****************************************
* FSE unsafe API * FSE unsafe API
*******************************************/ *******************************************/
static unsigned char FSE_decodeSymbolFast(FSE_DState_t *DStatePtr, BIT_DStream_t *bitD); static unsigned char FSE_decodeSymbolFast(FSE_DState_t *DStatePtr, BIT_DStream_t *bitD);
/* faster, but works only if nbBits is always >= 1 (otherwise, result will be corrupted) */ /* faster, but works only if nbBits is always >= 1 (otherwise, result will be corrupted) */
/* ***************************************** /* *****************************************
* Implementation of inlined functions * Implementation of inlined functions
*******************************************/ *******************************************/
@@ -458,14 +444,14 @@ ZSTD_STATIC void FSE_initCState(FSE_CState_t* statePtr, const FSE_CTable* ct)
statePtr->stateLog = tableLog; statePtr->stateLog = tableLog;
} }
/*! FSE_initCState2() : /*! FSE_initCState2() :
* Same as FSE_initCState(), but the first symbol to include (which will be the last to be read) * Same as FSE_initCState(), but the first symbol to include (which will be the last to be read)
* uses the smallest state value possible, saving the cost of this symbol */ * uses the smallest state value possible, saving the cost of this symbol */
ZSTD_STATIC void FSE_initCState2(FSE_CState_t *statePtr, const FSE_CTable *ct, U32 symbol) ZSTD_STATIC void FSE_initCState2(FSE_CState_t *statePtr, const FSE_CTable *ct, U32 symbol)
{ {
FSE_initCState(statePtr, ct); FSE_initCState(statePtr, ct);
{ const FSE_symbolCompressionTransform symbolTT = ((const FSE_symbolCompressionTransform*)(statePtr->symbolTT))[symbol]; {
const FSE_symbolCompressionTransform symbolTT = ((const FSE_symbolCompressionTransform *)(statePtr->symbolTT))[symbol];
const U16 *stateTable = (const U16 *)(statePtr->stateTable); const U16 *stateTable = (const U16 *)(statePtr->stateTable);
U32 nbBitsOut = (U32)((symbolTT.deltaNbBits + (1 << 15)) >> 16); U32 nbBitsOut = (U32)((symbolTT.deltaNbBits + (1 << 15)) >> 16);
statePtr->value = (nbBitsOut << 16) - symbolTT.deltaNbBits; statePtr->value = (nbBitsOut << 16) - symbolTT.deltaNbBits;
@@ -488,7 +474,6 @@ ZSTD_STATIC void FSE_flushCState(BIT_CStream_t* bitC, const FSE_CState_t* stateP
BIT_flushBits(bitC); BIT_flushBits(bitC);
} }
/* ====== Decompression ====== */ /* ====== Decompression ====== */
typedef struct { typedef struct {
@@ -496,8 +481,7 @@ typedef struct {
U16 fastMode; U16 fastMode;
} FSE_DTableHeader; /* sizeof U32 */ } FSE_DTableHeader; /* sizeof U32 */
typedef struct typedef struct {
{
unsigned short newState; unsigned short newState;
unsigned char symbol; unsigned char symbol;
unsigned char nbBits; unsigned char nbBits;
@@ -550,12 +534,7 @@ ZSTD_STATIC BYTE FSE_decodeSymbolFast(FSE_DState_t* DStatePtr, BIT_DStream_t* bi
return symbol; return symbol;
} }
ZSTD_STATIC unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr) ZSTD_STATIC unsigned FSE_endOfDState(const FSE_DState_t *DStatePtr) { return DStatePtr->state == 0; }
{
return DStatePtr->state == 0;
}
#ifndef FSE_COMMONDEFS_ONLY #ifndef FSE_COMMONDEFS_ONLY
@@ -588,10 +567,8 @@ ZSTD_STATIC unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr)
#define FSE_FUNCTION_EXTENSION #define FSE_FUNCTION_EXTENSION
#define FSE_DECODE_TYPE FSE_decode_t #define FSE_DECODE_TYPE FSE_decode_t
#endif /* !FSE_COMMONDEFS_ONLY */ #endif /* !FSE_COMMONDEFS_ONLY */
/* *************************************************************** /* ***************************************************************
* Constants * Constants
*****************************************************************/ *****************************************************************/
@@ -608,5 +585,4 @@ ZSTD_STATIC unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr)
#define FSE_TABLESTEP(tableSize) ((tableSize >> 1) + (tableSize >> 3) + 3) #define FSE_TABLESTEP(tableSize) ((tableSize >> 1) + (tableSize >> 3) + 3)
#endif /* FSE_H */ #endif /* FSE_H */
contrib/linux-kernel/lib/zstd/fse_compress.c
+214 -132
View File
@@ -42,21 +42,21 @@
****************************************************************/ ****************************************************************/
#define FORCE_INLINE static __always_inline #define FORCE_INLINE static __always_inline
/* ************************************************************** /* **************************************************************
* Includes * Includes
****************************************************************/ ****************************************************************/
#include <linux/compiler.h>
#include <linux/string.h> /* memcpy, memset */
#include "bitstream.h" #include "bitstream.h"
#include "fse.h" #include "fse.h"
#include <linux/compiler.h>
#include <linux/string.h> /* memcpy, memset */
/* ************************************************************** /* **************************************************************
* Error Management * Error Management
****************************************************************/ ****************************************************************/
#define FSE_STATIC_ASSERT(c) { enum { FSE_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */ #define FSE_STATIC_ASSERT(c) \
{ \
enum { FSE_static_assert = 1 / (int)(!!(c)) }; \
} /* use only *after* variable declarations */
/* ************************************************************** /* **************************************************************
* Templates * Templates
@@ -80,7 +80,6 @@
#define FSE_FUNCTION_NAME(X, Y) FSE_CAT(X, Y) #define FSE_FUNCTION_NAME(X, Y) FSE_CAT(X, Y)
#define FSE_TYPE_NAME(X, Y) FSE_CAT(X, Y) #define FSE_TYPE_NAME(X, Y) FSE_CAT(X, Y)
/* Function templates */ /* Function templates */
/* FSE_buildCTable_wksp() : /* FSE_buildCTable_wksp() :
@@ -103,7 +102,8 @@ size_t FSE_buildCTable_wksp(FSE_CTable* ct, const short* normalizedCounter, unsi
U32 highThreshold = tableSize - 1; U32 highThreshold = tableSize - 1;
/* CTable header */ /* CTable header */
if (((size_t)1 << tableLog) * sizeof(FSE_FUNCTION_TYPE) > wkspSize) return ERROR(tableLog_tooLarge); if (((size_t)1 << tableLog) * sizeof(FSE_FUNCTION_TYPE) > wkspSize)
return ERROR(tableLog_tooLarge);
tableU16[-2] = (U16)tableLog; tableU16[-2] = (U16)tableLog;
tableU16[-1] = (U16)maxSymbolValue; tableU16[-1] = (U16)maxSymbolValue;
@@ -111,7 +111,8 @@ size_t FSE_buildCTable_wksp(FSE_CTable* ct, const short* normalizedCounter, unsi
* http://fastcompression.blogspot.fr/2014/02/fse-distributing-symbol-values.html */ * http://fastcompression.blogspot.fr/2014/02/fse-distributing-symbol-values.html */
/* symbol start positions */ /* symbol start positions */
{ U32 u; {
U32 u;
cumul[0] = 0; cumul[0] = 0;
for (u = 1; u <= maxSymbolValue + 1; u++) { for (u = 1; u <= maxSymbolValue + 1; u++) {
if (normalizedCounter[u - 1] == -1) { /* Low proba symbol */ if (normalizedCounter[u - 1] == -1) { /* Low proba symbol */
@@ -119,36 +120,44 @@ size_t FSE_buildCTable_wksp(FSE_CTable* ct, const short* normalizedCounter, unsi
tableSymbol[highThreshold--] = (FSE_FUNCTION_TYPE)(u - 1); tableSymbol[highThreshold--] = (FSE_FUNCTION_TYPE)(u - 1);
} else { } else {
cumul[u] = cumul[u - 1] + normalizedCounter[u - 1]; cumul[u] = cumul[u - 1] + normalizedCounter[u - 1];
} } }
}
cumul[maxSymbolValue + 1] = tableSize + 1; cumul[maxSymbolValue + 1] = tableSize + 1;
} }
/* Spread symbols */ /* Spread symbols */
{ U32 position = 0; {
U32 position = 0;
U32 symbol; U32 symbol;
for (symbol = 0; symbol <= maxSymbolValue; symbol++) { for (symbol = 0; symbol <= maxSymbolValue; symbol++) {
int nbOccurences; int nbOccurences;
for (nbOccurences = 0; nbOccurences < normalizedCounter[symbol]; nbOccurences++) { for (nbOccurences = 0; nbOccurences < normalizedCounter[symbol]; nbOccurences++) {
tableSymbol[position] = (FSE_FUNCTION_TYPE)symbol; tableSymbol[position] = (FSE_FUNCTION_TYPE)symbol;
position = (position + step) & tableMask; position = (position + step) & tableMask;
while (position > highThreshold) position = (position + step) & tableMask; /* Low proba area */ while (position > highThreshold)
} } position = (position + step) & tableMask; /* Low proba area */
}
}
if (position!=0) return ERROR(GENERIC); /* Must have gone through all positions */ if (position != 0)
return ERROR(GENERIC); /* Must have gone through all positions */
} }
/* Build table */ /* Build table */
{ U32 u; for (u=0; u<tableSize; u++) { {
U32 u;
for (u = 0; u < tableSize; u++) {
FSE_FUNCTION_TYPE s = tableSymbol[u]; /* note : static analyzer may not understand tableSymbol is properly initialized */ FSE_FUNCTION_TYPE s = tableSymbol[u]; /* note : static analyzer may not understand tableSymbol is properly initialized */
tableU16[cumul[s]++] = (U16)(tableSize + u); /* TableU16 : sorted by symbol order; gives next state value */ tableU16[cumul[s]++] = (U16)(tableSize + u); /* TableU16 : sorted by symbol order; gives next state value */
} } }
}
/* Build Symbol Transformation Table */ /* Build Symbol Transformation Table */
{ unsigned total = 0; {
unsigned total = 0;
unsigned s; unsigned s;
for (s = 0; s <= maxSymbolValue; s++) { for (s = 0; s <= maxSymbolValue; s++) {
switch (normalizedCounter[s]) switch (normalizedCounter[s]) {
{
case 0: break; case 0: break;
case -1: case -1:
@@ -157,19 +166,20 @@ size_t FSE_buildCTable_wksp(FSE_CTable* ct, const short* normalizedCounter, unsi
symbolTT[s].deltaFindState = total - 1; symbolTT[s].deltaFindState = total - 1;
total++; total++;
break; break;
default : default: {
{
U32 const maxBitsOut = tableLog - BIT_highbit32(normalizedCounter[s] - 1); U32 const maxBitsOut = tableLog - BIT_highbit32(normalizedCounter[s] - 1);
U32 const minStatePlus = normalizedCounter[s] << maxBitsOut; U32 const minStatePlus = normalizedCounter[s] << maxBitsOut;
symbolTT[s].deltaNbBits = (maxBitsOut << 16) - minStatePlus; symbolTT[s].deltaNbBits = (maxBitsOut << 16) - minStatePlus;
symbolTT[s].deltaFindState = total - normalizedCounter[s]; symbolTT[s].deltaFindState = total - normalizedCounter[s];
total += normalizedCounter[s]; total += normalizedCounter[s];
} } } } }
}
}
}
return 0; return 0;
} }
#ifndef FSE_COMMONDEFS_ONLY #ifndef FSE_COMMONDEFS_ONLY
/*-************************************************************** /*-**************************************************************
@@ -181,8 +191,7 @@ size_t FSE_NCountWriteBound(unsigned maxSymbolValue, unsigned tableLog)
return maxSymbolValue ? maxHeaderSize : FSE_NCOUNTBOUND; /* maxSymbolValue==0 ? use default */ return maxSymbolValue ? maxHeaderSize : FSE_NCOUNTBOUND; /* maxSymbolValue==0 ? use default */
} }
static size_t FSE_writeNCount_generic (void* header, size_t headerBufferSize, static size_t FSE_writeNCount_generic(void *header, size_t headerBufferSize, const short *normalizedCounter, unsigned maxSymbolValue, unsigned tableLog,
const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog,
unsigned writeIsSafe) unsigned writeIsSafe)
{ {
BYTE *const ostart = (BYTE *)header; BYTE *const ostart = (BYTE *)header;
@@ -211,11 +220,13 @@ static size_t FSE_writeNCount_generic (void* header, size_t headerBufferSize,
while (remaining > 1) { /* stops at 1 */ while (remaining > 1) { /* stops at 1 */
if (previous0) { if (previous0) {
unsigned start = charnum; unsigned start = charnum;
while (!normalizedCounter[charnum]) charnum++; while (!normalizedCounter[charnum])
charnum++;
while (charnum >= start + 24) { while (charnum >= start + 24) {
start += 24; start += 24;
bitStream += 0xFFFFU << bitCount; bitStream += 0xFFFFU << bitCount;
if ((!writeIsSafe) && (out > oend-2)) return ERROR(dstSize_tooSmall); /* Buffer overflow */ if ((!writeIsSafe) && (out > oend - 2))
return ERROR(dstSize_tooSmall); /* Buffer overflow */
out[0] = (BYTE)bitStream; out[0] = (BYTE)bitStream;
out[1] = (BYTE)(bitStream >> 8); out[1] = (BYTE)(bitStream >> 8);
out += 2; out += 2;
@@ -229,50 +240,61 @@ static size_t FSE_writeNCount_generic (void* header, size_t headerBufferSize,
bitStream += (charnum - start) << bitCount; bitStream += (charnum - start) << bitCount;
bitCount += 2; bitCount += 2;
if (bitCount > 16) { if (bitCount > 16) {
if ((!writeIsSafe) && (out > oend - 2)) return ERROR(dstSize_tooSmall); /* Buffer overflow */ if ((!writeIsSafe) && (out > oend - 2))
return ERROR(dstSize_tooSmall); /* Buffer overflow */
out[0] = (BYTE)bitStream; out[0] = (BYTE)bitStream;
out[1] = (BYTE)(bitStream >> 8); out[1] = (BYTE)(bitStream >> 8);
out += 2; out += 2;
bitStream >>= 16; bitStream >>= 16;
bitCount -= 16; bitCount -= 16;
} } }
{ int count = normalizedCounter[charnum++]; }
{
int count = normalizedCounter[charnum++];
int const max = (2 * threshold - 1) - remaining; int const max = (2 * threshold - 1) - remaining;
remaining -= count < 0 ? -count : count; remaining -= count < 0 ? -count : count;
count++; /* +1 for extra accuracy */ count++; /* +1 for extra accuracy */
if (count>=threshold) count += max; /* [0..max[ [max..threshold[ (...) [threshold+max 2*threshold[ */ if (count >= threshold)
count += max; /* [0..max[ [max..threshold[ (...) [threshold+max 2*threshold[ */
bitStream += count << bitCount; bitStream += count << bitCount;
bitCount += nbBits; bitCount += nbBits;
bitCount -= (count < max); bitCount -= (count < max);
previous0 = (count == 1); previous0 = (count == 1);
if (remaining<1) return ERROR(GENERIC); if (remaining < 1)
while (remaining<threshold) nbBits--, threshold>>=1; return ERROR(GENERIC);
while (remaining < threshold)
nbBits--, threshold >>= 1;
} }
if (bitCount > 16) { if (bitCount > 16) {
if ((!writeIsSafe) && (out > oend - 2)) return ERROR(dstSize_tooSmall); /* Buffer overflow */ if ((!writeIsSafe) && (out > oend - 2))
return ERROR(dstSize_tooSmall); /* Buffer overflow */
out[0] = (BYTE)bitStream; out[0] = (BYTE)bitStream;
out[1] = (BYTE)(bitStream >> 8); out[1] = (BYTE)(bitStream >> 8);
out += 2; out += 2;
bitStream >>= 16; bitStream >>= 16;
bitCount -= 16; bitCount -= 16;
} } }
}
/* flush remaining bitStream */ /* flush remaining bitStream */
if ((!writeIsSafe) && (out > oend - 2)) return ERROR(dstSize_tooSmall); /* Buffer overflow */ if ((!writeIsSafe) && (out > oend - 2))
return ERROR(dstSize_tooSmall); /* Buffer overflow */
out[0] = (BYTE)bitStream; out[0] = (BYTE)bitStream;
out[1] = (BYTE)(bitStream >> 8); out[1] = (BYTE)(bitStream >> 8);
out += (bitCount + 7) / 8; out += (bitCount + 7) / 8;
if (charnum > maxSymbolValue + 1) return ERROR(GENERIC); if (charnum > maxSymbolValue + 1)
return ERROR(GENERIC);
return (out - ostart); return (out - ostart);
} }
size_t FSE_writeNCount(void *buffer, size_t bufferSize, const short *normalizedCounter, unsigned maxSymbolValue, unsigned tableLog) size_t FSE_writeNCount(void *buffer, size_t bufferSize, const short *normalizedCounter, unsigned maxSymbolValue, unsigned tableLog)
{ {
if (tableLog > FSE_MAX_TABLELOG) return ERROR(tableLog_tooLarge); /* Unsupported */ if (tableLog > FSE_MAX_TABLELOG)
if (tableLog < FSE_MIN_TABLELOG) return ERROR(GENERIC); /* Unsupported */ return ERROR(tableLog_tooLarge); /* Unsupported */
if (tableLog < FSE_MIN_TABLELOG)
return ERROR(GENERIC); /* Unsupported */
if (bufferSize < FSE_NCountWriteBound(maxSymbolValue, tableLog)) if (bufferSize < FSE_NCountWriteBound(maxSymbolValue, tableLog))
return FSE_writeNCount_generic(buffer, bufferSize, normalizedCounter, maxSymbolValue, tableLog, 0); return FSE_writeNCount_generic(buffer, bufferSize, normalizedCounter, maxSymbolValue, tableLog, 0);
@@ -280,8 +302,6 @@ size_t FSE_writeNCount (void* buffer, size_t bufferSize, const short* normalized
return FSE_writeNCount_generic(buffer, bufferSize, normalizedCounter, maxSymbolValue, tableLog, 1); return FSE_writeNCount_generic(buffer, bufferSize, normalizedCounter, maxSymbolValue, tableLog, 1);
} }
/*-************************************************************** /*-**************************************************************
* Counting histogram * Counting histogram
****************************************************************/ ****************************************************************/
@@ -292,8 +312,7 @@ size_t FSE_writeNCount (void* buffer, size_t bufferSize, const short* normalized
For this reason, prefer using a table `count` with 256 elements. For this reason, prefer using a table `count` with 256 elements.
@return : count of most numerous element @return : count of most numerous element
*/ */
size_t FSE_count_simple(unsigned* count, unsigned* maxSymbolValuePtr, size_t FSE_count_simple(unsigned *count, unsigned *maxSymbolValuePtr, const void *src, size_t srcSize)
const void* src, size_t srcSize)
{ {
const BYTE *ip = (const BYTE *)src; const BYTE *ip = (const BYTE *)src;
const BYTE *const end = ip + srcSize; const BYTE *const end = ip + srcSize;
@@ -301,26 +320,33 @@ size_t FSE_count_simple(unsigned* count, unsigned* maxSymbolValuePtr,
unsigned max = 0; unsigned max = 0;
memset(count, 0, (maxSymbolValue + 1) * sizeof(*count)); memset(count, 0, (maxSymbolValue + 1) * sizeof(*count));
if (srcSize==0) { *maxSymbolValuePtr = 0; return 0; } if (srcSize == 0) {
*maxSymbolValuePtr = 0;
return 0;
}
while (ip<end) count[*ip++]++; while (ip < end)
count[*ip++]++;
while (!count[maxSymbolValue]) maxSymbolValue--; while (!count[maxSymbolValue])
maxSymbolValue--;
*maxSymbolValuePtr = maxSymbolValue; *maxSymbolValuePtr = maxSymbolValue;
{ U32 s; for (s=0; s<=maxSymbolValue; s++) if (count[s] > max) max = count[s]; } {
U32 s;
for (s = 0; s <= maxSymbolValue; s++)
if (count[s] > max)
max = count[s];
}
return (size_t)max; return (size_t)max;
} }
/* FSE_count_parallel_wksp() : /* FSE_count_parallel_wksp() :
* Same as FSE_count_parallel(), but using an externally provided scratch buffer. * Same as FSE_count_parallel(), but using an externally provided scratch buffer.
* `workSpace` size must be a minimum of `1024 * sizeof(unsigned)`` */ * `workSpace` size must be a minimum of `1024 * sizeof(unsigned)`` */
static size_t FSE_count_parallel_wksp( static size_t FSE_count_parallel_wksp(unsigned *count, unsigned *maxSymbolValuePtr, const void *source, size_t sourceSize, unsigned checkMax,
unsigned* count, unsigned* maxSymbolValuePtr, unsigned *const workSpace)
const void* source, size_t sourceSize,
unsigned checkMax, unsigned* const workSpace)
{ {
const BYTE *ip = (const BYTE *)source; const BYTE *ip = (const BYTE *)source;
const BYTE *const iend = ip + sourceSize; const BYTE *const iend = ip + sourceSize;
@@ -339,27 +365,38 @@ static size_t FSE_count_parallel_wksp(
*maxSymbolValuePtr = 0; *maxSymbolValuePtr = 0;
return 0; return 0;
} }
if (!maxSymbolValue) maxSymbolValue = 255; /* 0 == default */ if (!maxSymbolValue)
maxSymbolValue = 255; /* 0 == default */
/* by stripes of 16 bytes */ /* by stripes of 16 bytes */
{ U32 cached = ZSTD_read32(ip); ip += 4; {
U32 cached = ZSTD_read32(ip);
ip += 4;
while (ip < iend - 15) { while (ip < iend - 15) {
U32 c = cached; cached = ZSTD_read32(ip); ip += 4; U32 c = cached;
cached = ZSTD_read32(ip);
ip += 4;
Counting1[(BYTE)c]++; Counting1[(BYTE)c]++;
Counting2[(BYTE)(c >> 8)]++; Counting2[(BYTE)(c >> 8)]++;
Counting3[(BYTE)(c >> 16)]++; Counting3[(BYTE)(c >> 16)]++;
Counting4[c >> 24]++; Counting4[c >> 24]++;
c = cached; cached = ZSTD_read32(ip); ip += 4; c = cached;
cached = ZSTD_read32(ip);
ip += 4;
Counting1[(BYTE)c]++; Counting1[(BYTE)c]++;
Counting2[(BYTE)(c >> 8)]++; Counting2[(BYTE)(c >> 8)]++;
Counting3[(BYTE)(c >> 16)]++; Counting3[(BYTE)(c >> 16)]++;
Counting4[c >> 24]++; Counting4[c >> 24]++;
c = cached; cached = ZSTD_read32(ip); ip += 4; c = cached;
cached = ZSTD_read32(ip);
ip += 4;
Counting1[(BYTE)c]++; Counting1[(BYTE)c]++;
Counting2[(BYTE)(c >> 8)]++; Counting2[(BYTE)(c >> 8)]++;
Counting3[(BYTE)(c >> 16)]++; Counting3[(BYTE)(c >> 16)]++;
Counting4[c >> 24]++; Counting4[c >> 24]++;
c = cached; cached = ZSTD_read32(ip); ip += 4; c = cached;
cached = ZSTD_read32(ip);
ip += 4;
Counting1[(BYTE)c]++; Counting1[(BYTE)c]++;
Counting2[(BYTE)(c >> 8)]++; Counting2[(BYTE)(c >> 8)]++;
Counting3[(BYTE)(c >> 16)]++; Counting3[(BYTE)(c >> 16)]++;
@@ -369,20 +406,29 @@ static size_t FSE_count_parallel_wksp(
} }
/* finish last symbols */ /* finish last symbols */
while (ip<iend) Counting1[*ip++]++; while (ip < iend)
Counting1[*ip++]++;
if (checkMax) { /* verify stats will fit into destination table */ if (checkMax) { /* verify stats will fit into destination table */
U32 s; for (s=255; s>maxSymbolValue; s--) { U32 s;
for (s = 255; s > maxSymbolValue; s--) {
Counting1[s] += Counting2[s] + Counting3[s] + Counting4[s]; Counting1[s] += Counting2[s] + Counting3[s] + Counting4[s];
if (Counting1[s]) return ERROR(maxSymbolValue_tooSmall); if (Counting1[s])
} } return ERROR(maxSymbolValue_tooSmall);
}
}
{ U32 s; for (s=0; s<=maxSymbolValue; s++) { {
U32 s;
for (s = 0; s <= maxSymbolValue; s++) {
count[s] = Counting1[s] + Counting2[s] + Counting3[s] + Counting4[s]; count[s] = Counting1[s] + Counting2[s] + Counting3[s] + Counting4[s];
if (count[s] > max) max = count[s]; if (count[s] > max)
} } max = count[s];
}
}
while (!count[maxSymbolValue]) maxSymbolValue--; while (!count[maxSymbolValue])
maxSymbolValue--;
*maxSymbolValuePtr = maxSymbolValue; *maxSymbolValuePtr = maxSymbolValue;
return (size_t)max; return (size_t)max;
} }
@@ -390,18 +436,17 @@ static size_t FSE_count_parallel_wksp(
/* FSE_countFast_wksp() : /* FSE_countFast_wksp() :
* Same as FSE_countFast(), but using an externally provided scratch buffer. * Same as FSE_countFast(), but using an externally provided scratch buffer.
* `workSpace` size must be table of >= `1024` unsigned */ * `workSpace` size must be table of >= `1024` unsigned */
size_t FSE_countFast_wksp(unsigned* count, unsigned* maxSymbolValuePtr, size_t FSE_countFast_wksp(unsigned *count, unsigned *maxSymbolValuePtr, const void *source, size_t sourceSize, unsigned *workSpace)
const void* source, size_t sourceSize, unsigned* workSpace)
{ {
if (sourceSize < 1500) return FSE_count_simple(count, maxSymbolValuePtr, source, sourceSize); if (sourceSize < 1500)
return FSE_count_simple(count, maxSymbolValuePtr, source, sourceSize);
return FSE_count_parallel_wksp(count, maxSymbolValuePtr, source, sourceSize, 0, workSpace); return FSE_count_parallel_wksp(count, maxSymbolValuePtr, source, sourceSize, 0, workSpace);
} }
/* FSE_count_wksp() : /* FSE_count_wksp() :
* Same as FSE_count(), but using an externally provided scratch buffer. * Same as FSE_count(), but using an externally provided scratch buffer.
* `workSpace` size must be table of >= `1024` unsigned */ * `workSpace` size must be table of >= `1024` unsigned */
size_t FSE_count_wksp(unsigned* count, unsigned* maxSymbolValuePtr, size_t FSE_count_wksp(unsigned *count, unsigned *maxSymbolValuePtr, const void *source, size_t sourceSize, unsigned *workSpace)
const void* source, size_t sourceSize, unsigned* workSpace)
{ {
if (*maxSymbolValuePtr < 255) if (*maxSymbolValuePtr < 255)
return FSE_count_parallel_wksp(count, maxSymbolValuePtr, source, sourceSize, 1, workSpace); return FSE_count_parallel_wksp(count, maxSymbolValuePtr, source, sourceSize, 1, workSpace);
@@ -409,7 +454,6 @@ size_t FSE_count_wksp(unsigned* count, unsigned* maxSymbolValuePtr,
return FSE_countFast_wksp(count, maxSymbolValuePtr, source, sourceSize, workSpace); return FSE_countFast_wksp(count, maxSymbolValuePtr, source, sourceSize, workSpace);
} }
/*-************************************************************** /*-**************************************************************
* FSE Compression Code * FSE Compression Code
****************************************************************/ ****************************************************************/
@@ -423,7 +467,8 @@ Allocation is manual (C standard does not support variable-size structures).
*/ */
size_t FSE_sizeof_CTable(unsigned maxSymbolValue, unsigned tableLog) size_t FSE_sizeof_CTable(unsigned maxSymbolValue, unsigned tableLog)
{ {
if (tableLog > FSE_MAX_TABLELOG) return ERROR(tableLog_tooLarge); if (tableLog > FSE_MAX_TABLELOG)
return ERROR(tableLog_tooLarge);
return FSE_CTABLE_SIZE_U32(tableLog, maxSymbolValue) * sizeof(U32); return FSE_CTABLE_SIZE_U32(tableLog, maxSymbolValue) * sizeof(U32);
} }
@@ -441,11 +486,16 @@ unsigned FSE_optimalTableLog_internal(unsigned maxTableLog, size_t srcSize, unsi
U32 maxBitsSrc = BIT_highbit32((U32)(srcSize - 1)) - minus; U32 maxBitsSrc = BIT_highbit32((U32)(srcSize - 1)) - minus;
U32 tableLog = maxTableLog; U32 tableLog = maxTableLog;
U32 minBits = FSE_minTableLog(srcSize, maxSymbolValue); U32 minBits = FSE_minTableLog(srcSize, maxSymbolValue);
if (tableLog==0) tableLog = FSE_DEFAULT_TABLELOG; if (tableLog == 0)
if (maxBitsSrc < tableLog) tableLog = maxBitsSrc; /* Accuracy can be reduced */ tableLog = FSE_DEFAULT_TABLELOG;
if (minBits > tableLog) tableLog = minBits; /* Need a minimum to safely represent all symbol values */ if (maxBitsSrc < tableLog)
if (tableLog < FSE_MIN_TABLELOG) tableLog = FSE_MIN_TABLELOG; tableLog = maxBitsSrc; /* Accuracy can be reduced */
if (tableLog > FSE_MAX_TABLELOG) tableLog = FSE_MAX_TABLELOG; if (minBits > tableLog)
tableLog = minBits; /* Need a minimum to safely represent all symbol values */
if (tableLog < FSE_MIN_TABLELOG)
tableLog = FSE_MIN_TABLELOG;
if (tableLog > FSE_MAX_TABLELOG)
tableLog = FSE_MAX_TABLELOG;
return tableLog; return tableLog;
} }
@@ -454,7 +504,6 @@ unsigned FSE_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxS
return FSE_optimalTableLog_internal(maxTableLog, srcSize, maxSymbolValue, 2); return FSE_optimalTableLog_internal(maxTableLog, srcSize, maxSymbolValue, 2);
} }
/* Secondary normalization method. /* Secondary normalization method.
To be used when primary method fails. */ To be used when primary method fails. */
@@ -500,7 +549,8 @@ static size_t FSE_normalizeM2(short* norm, U32 tableLog, const unsigned* count,
distributed++; distributed++;
total -= count[s]; total -= count[s];
continue; continue;
} } }
}
ToDistribute = (1 << tableLog) - distributed; ToDistribute = (1 << tableLog) - distributed;
} }
@@ -510,7 +560,8 @@ static size_t FSE_normalizeM2(short* norm, U32 tableLog, const unsigned* count,
find max, then give all remaining points to max */ find max, then give all remaining points to max */
U32 maxV = 0, maxC = 0; U32 maxV = 0, maxC = 0;
for (s = 0; s <= maxSymbolValue; s++) for (s = 0; s <= maxSymbolValue; s++)
if (count[s] > maxC) maxV=s, maxC=count[s]; if (count[s] > maxC)
maxV = s, maxC = count[s];
norm[maxV] += (short)ToDistribute; norm[maxV] += (short)ToDistribute;
return 0; return 0;
} }
@@ -518,11 +569,13 @@ static size_t FSE_normalizeM2(short* norm, U32 tableLog, const unsigned* count,
if (total == 0) { if (total == 0) {
/* all of the symbols were low enough for the lowOne or lowThreshold */ /* all of the symbols were low enough for the lowOne or lowThreshold */
for (s = 0; ToDistribute > 0; s = (s + 1) % (maxSymbolValue + 1)) for (s = 0; ToDistribute > 0; s = (s + 1) % (maxSymbolValue + 1))
if (norm[s] > 0) ToDistribute--, norm[s]++; if (norm[s] > 0)
ToDistribute--, norm[s]++;
return 0; return 0;
} }
{ U64 const vStepLog = 62 - tableLog; {
U64 const vStepLog = 62 - tableLog;
U64 const mid = (1ULL << (vStepLog - 1)) - 1; U64 const mid = (1ULL << (vStepLog - 1)) - 1;
U64 const rStep = ((((U64)1 << vStepLog) * ToDistribute) + mid) / total; /* scale on remaining */ U64 const rStep = ((((U64)1 << vStepLog) * ToDistribute) + mid) / total; /* scale on remaining */
U64 tmpTotal = mid; U64 tmpTotal = mid;
@@ -536,23 +589,27 @@ static size_t FSE_normalizeM2(short* norm, U32 tableLog, const unsigned* count,
return ERROR(GENERIC); return ERROR(GENERIC);
norm[s] = (short)weight; norm[s] = (short)weight;
tmpTotal = end; tmpTotal = end;
} } } }
}
}
return 0; return 0;
} }
size_t FSE_normalizeCount(short *normalizedCounter, unsigned tableLog, const unsigned *count, size_t total, unsigned maxSymbolValue)
size_t FSE_normalizeCount (short* normalizedCounter, unsigned tableLog,
const unsigned* count, size_t total,
unsigned maxSymbolValue)
{ {
/* Sanity checks */ /* Sanity checks */
if (tableLog==0) tableLog = FSE_DEFAULT_TABLELOG; if (tableLog == 0)
if (tableLog < FSE_MIN_TABLELOG) return ERROR(GENERIC); /* Unsupported size */ tableLog = FSE_DEFAULT_TABLELOG;
if (tableLog > FSE_MAX_TABLELOG) return ERROR(tableLog_tooLarge); /* Unsupported size */ if (tableLog < FSE_MIN_TABLELOG)
if (tableLog < FSE_minTableLog(total, maxSymbolValue)) return ERROR(GENERIC); /* Too small tableLog, compression potentially impossible */ return ERROR(GENERIC); /* Unsupported size */
if (tableLog > FSE_MAX_TABLELOG)
return ERROR(tableLog_tooLarge); /* Unsupported size */
if (tableLog < FSE_minTableLog(total, maxSymbolValue))
return ERROR(GENERIC); /* Too small tableLog, compression potentially impossible */
{ U32 const rtbTable[] = { 0, 473195, 504333, 520860, 550000, 700000, 750000, 830000 }; {
U32 const rtbTable[] = {0, 473195, 504333, 520860, 550000, 700000, 750000, 830000};
U64 const scale = 62 - tableLog; U64 const scale = 62 - tableLog;
U64 const step = ((U64)1 << 62) / total; /* <== here, one division ! */ U64 const step = ((U64)1 << 62) / total; /* <== here, one division ! */
U64 const vStep = 1ULL << (scale - 20); U64 const vStep = 1ULL << (scale - 20);
@@ -563,8 +620,12 @@ size_t FSE_normalizeCount (short* normalizedCounter, unsigned tableLog,
U32 lowThreshold = (U32)(total >> tableLog); U32 lowThreshold = (U32)(total >> tableLog);
for (s = 0; s <= maxSymbolValue; s++) { for (s = 0; s <= maxSymbolValue; s++) {
if (count[s] == total) return 0; /* rle special case */ if (count[s] == total)
if (count[s] == 0) { normalizedCounter[s]=0; continue; } return 0; /* rle special case */
if (count[s] == 0) {
normalizedCounter[s] = 0;
continue;
}
if (count[s] <= lowThreshold) { if (count[s] <= lowThreshold) {
normalizedCounter[s] = -1; normalizedCounter[s] = -1;
stillToDistribute--; stillToDistribute--;
@@ -574,22 +635,24 @@ size_t FSE_normalizeCount (short* normalizedCounter, unsigned tableLog,
U64 restToBeat = vStep * rtbTable[proba]; U64 restToBeat = vStep * rtbTable[proba];
proba += (count[s] * step) - ((U64)proba << scale) > restToBeat; proba += (count[s] * step) - ((U64)proba << scale) > restToBeat;
} }
if (proba > largestP) largestP=proba, largest=s; if (proba > largestP)
largestP = proba, largest = s;
normalizedCounter[s] = proba; normalizedCounter[s] = proba;
stillToDistribute -= proba; stillToDistribute -= proba;
} } }
}
if (-stillToDistribute >= (normalizedCounter[largest] >> 1)) { if (-stillToDistribute >= (normalizedCounter[largest] >> 1)) {
/* corner case, need another normalization method */ /* corner case, need another normalization method */
size_t const errorCode = FSE_normalizeM2(normalizedCounter, tableLog, count, total, maxSymbolValue); size_t const errorCode = FSE_normalizeM2(normalizedCounter, tableLog, count, total, maxSymbolValue);
if (FSE_isError(errorCode)) return errorCode; if (FSE_isError(errorCode))
} return errorCode;
else normalizedCounter[largest] += (short)stillToDistribute; } else
normalizedCounter[largest] += (short)stillToDistribute;
} }
return tableLog; return tableLog;
} }
/* fake FSE_CTable, for raw (uncompressed) input */ /* fake FSE_CTable, for raw (uncompressed) input */
size_t FSE_buildCTable_raw(FSE_CTable *ct, unsigned nbBits) size_t FSE_buildCTable_raw(FSE_CTable *ct, unsigned nbBits)
{ {
@@ -603,7 +666,8 @@ size_t FSE_buildCTable_raw (FSE_CTable* ct, unsigned nbBits)
unsigned s; unsigned s;
/* Sanity checks */ /* Sanity checks */
if (nbBits < 1) return ERROR(GENERIC); /* min size */ if (nbBits < 1)
return ERROR(GENERIC); /* min size */
/* header */ /* header */
tableU16[-2] = (U16)nbBits; tableU16[-2] = (U16)nbBits;
@@ -614,11 +678,13 @@ size_t FSE_buildCTable_raw (FSE_CTable* ct, unsigned nbBits)
tableU16[s] = (U16)(tableSize + s); tableU16[s] = (U16)(tableSize + s);
/* Build Symbol Transformation Table */ /* Build Symbol Transformation Table */
{ const U32 deltaNbBits = (nbBits << 16) - (1 << nbBits); {
const U32 deltaNbBits = (nbBits << 16) - (1 << nbBits);
for (s = 0; s <= maxSymbolValue; s++) { for (s = 0; s <= maxSymbolValue; s++) {
symbolTT[s].deltaNbBits = deltaNbBits; symbolTT[s].deltaNbBits = deltaNbBits;
symbolTT[s].deltaFindState = s - 1; symbolTT[s].deltaFindState = s - 1;
} } }
}
return 0; return 0;
} }
@@ -646,10 +712,7 @@ size_t FSE_buildCTable_rle (FSE_CTable* ct, BYTE symbolValue)
return 0; return 0;
} }
static size_t FSE_compress_usingCTable_generic(void *dst, size_t dstSize, const void *src, size_t srcSize, const FSE_CTable *ct, const unsigned fast)
static size_t FSE_compress_usingCTable_generic (void* dst, size_t dstSize,
const void* src, size_t srcSize,
const FSE_CTable* ct, const unsigned fast)
{ {
const BYTE *const istart = (const BYTE *)src; const BYTE *const istart = (const BYTE *)src;
const BYTE *const iend = istart + srcSize; const BYTE *const iend = istart + srcSize;
@@ -659,9 +722,13 @@ static size_t FSE_compress_usingCTable_generic (void* dst, size_t dstSize,
FSE_CState_t CState1, CState2; FSE_CState_t CState1, CState2;
/* init */ /* init */
if (srcSize <= 2) return 0; if (srcSize <= 2)
{ size_t const initError = BIT_initCStream(&bitC, dst, dstSize); return 0;
if (FSE_isError(initError)) return 0; /* not enough space available to write a bitstream */ } {
size_t const initError = BIT_initCStream(&bitC, dst, dstSize);
if (FSE_isError(initError))
return 0; /* not enough space available to write a bitstream */
}
#define FSE_FLUSHBITS(s) (fast ? BIT_flushBitsFast(s) : BIT_flushBits(s)) #define FSE_FLUSHBITS(s) (fast ? BIT_flushBitsFast(s) : BIT_flushBits(s))
@@ -706,9 +773,7 @@ static size_t FSE_compress_usingCTable_generic (void* dst, size_t dstSize,
return BIT_closeCStream(&bitC); return BIT_closeCStream(&bitC);
} }
size_t FSE_compress_usingCTable (void* dst, size_t dstSize, size_t FSE_compress_usingCTable(void *dst, size_t dstSize, const void *src, size_t srcSize, const FSE_CTable *ct)
const void* src, size_t srcSize,
const FSE_CTable* ct)
{ {
unsigned const fast = (dstSize >= FSE_BLOCKBOUND(srcSize)); unsigned const fast = (dstSize >= FSE_BLOCKBOUND(srcSize));
@@ -718,17 +783,23 @@ size_t FSE_compress_usingCTable (void* dst, size_t dstSize,
return FSE_compress_usingCTable_generic(dst, dstSize, src, srcSize, ct, 0); return FSE_compress_usingCTable_generic(dst, dstSize, src, srcSize, ct, 0);
} }
size_t FSE_compressBound(size_t size) { return FSE_COMPRESSBOUND(size); } size_t FSE_compressBound(size_t size) { return FSE_COMPRESSBOUND(size); }
#define CHECK_V_F(e, f) size_t const e = f; if (ERR_isError(e)) return f #define CHECK_V_F(e, f) \
#define CHECK_F(f) { CHECK_V_F(_var_err__, f); } size_t const e = f; \
if (ERR_isError(e)) \
return f
#define CHECK_F(f) \
{ \
CHECK_V_F(_var_err__, f); \
}
/* FSE_compress_wksp() : /* FSE_compress_wksp() :
* Same as FSE_compress2(), but using an externally allocated scratch buffer (`workSpace`). * Same as FSE_compress2(), but using an externally allocated scratch buffer (`workSpace`).
* `wkspSize` size must be `(1<<tableLog)`. * `wkspSize` size must be `(1<<tableLog)`.
*/ */
size_t FSE_compress_wksp (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize) size_t FSE_compress_wksp(void *dst, size_t dstSize, const void *src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void *workSpace,
size_t wkspSize)
{ {
BYTE *const ostart = (BYTE *)dst; BYTE *const ostart = (BYTE *)dst;
BYTE *op = ostart; BYTE *op = ostart;
@@ -742,38 +813,49 @@ size_t FSE_compress_wksp (void* dst, size_t dstSize, const void* src, size_t src
size_t const scratchBufferSize = wkspSize - (CTableSize * sizeof(FSE_CTable)); size_t const scratchBufferSize = wkspSize - (CTableSize * sizeof(FSE_CTable));
/* init conditions */ /* init conditions */
if (wkspSize < FSE_WKSP_SIZE_U32(tableLog, maxSymbolValue)) return ERROR(tableLog_tooLarge); if (wkspSize < FSE_WKSP_SIZE_U32(tableLog, maxSymbolValue))
if (srcSize <= 1) return 0; /* Not compressible */ return ERROR(tableLog_tooLarge);
if (!maxSymbolValue) maxSymbolValue = FSE_MAX_SYMBOL_VALUE; if (srcSize <= 1)
if (!tableLog) tableLog = FSE_DEFAULT_TABLELOG; return 0; /* Not compressible */
if (!maxSymbolValue)
maxSymbolValue = FSE_MAX_SYMBOL_VALUE;
if (!tableLog)
tableLog = FSE_DEFAULT_TABLELOG;
/* Scan input and build symbol stats */ /* Scan input and build symbol stats */
{ CHECK_V_F(maxCount, FSE_count_wksp(count, &maxSymbolValue, src, srcSize, (unsigned*)scratchBuffer) ); {
if (maxCount == srcSize) return 1; /* only a single symbol in src : rle */ CHECK_V_F(maxCount, FSE_count_wksp(count, &maxSymbolValue, src, srcSize, (unsigned *)scratchBuffer));
if (maxCount == 1) return 0; /* each symbol present maximum once => not compressible */ if (maxCount == srcSize)
if (maxCount < (srcSize >> 7)) return 0; /* Heuristic : not compressible enough */ return 1; /* only a single symbol in src : rle */
if (maxCount == 1)
return 0; /* each symbol present maximum once => not compressible */
if (maxCount < (srcSize >> 7))
return 0; /* Heuristic : not compressible enough */
} }
tableLog = FSE_optimalTableLog(tableLog, srcSize, maxSymbolValue); tableLog = FSE_optimalTableLog(tableLog, srcSize, maxSymbolValue);
CHECK_F(FSE_normalizeCount(norm, tableLog, count, srcSize, maxSymbolValue)); CHECK_F(FSE_normalizeCount(norm, tableLog, count, srcSize, maxSymbolValue));
/* Write table description header */ /* Write table description header */
{ CHECK_V_F(nc_err, FSE_writeNCount(op, oend-op, norm, maxSymbolValue, tableLog) ); {
CHECK_V_F(nc_err, FSE_writeNCount(op, oend - op, norm, maxSymbolValue, tableLog));
op += nc_err; op += nc_err;
} }
/* Compress */ /* Compress */
CHECK_F(FSE_buildCTable_wksp(CTable, norm, maxSymbolValue, tableLog, scratchBuffer, scratchBufferSize)); CHECK_F(FSE_buildCTable_wksp(CTable, norm, maxSymbolValue, tableLog, scratchBuffer, scratchBufferSize));
{ CHECK_V_F(cSize, FSE_compress_usingCTable(op, oend - op, src, srcSize, CTable) ); {
if (cSize == 0) return 0; /* not enough space for compressed data */ CHECK_V_F(cSize, FSE_compress_usingCTable(op, oend - op, src, srcSize, CTable));
if (cSize == 0)
return 0; /* not enough space for compressed data */
op += cSize; op += cSize;
} }
/* check compressibility */ /* check compressibility */
if ( (size_t)(op-ostart) >= srcSize-1 ) return 0; if ((size_t)(op - ostart) >= srcSize - 1)
return 0;
return op - ostart; return op - ostart;
} }
#endif /* FSE_COMMONDEFS_ONLY */ #endif /* FSE_COMMONDEFS_ONLY */
contrib/linux-kernel/lib/zstd/fse_decompress.c
+62 -42
View File
@@ -37,31 +37,35 @@
* - Source repository : https://github.com/Cyan4973/FiniteStateEntropy * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy
*/ */
/* ************************************************************** /* **************************************************************
* Compiler specifics * Compiler specifics
****************************************************************/ ****************************************************************/
#define FORCE_INLINE static __always_inline #define FORCE_INLINE static __always_inline
/* ************************************************************** /* **************************************************************
* Includes * Includes
****************************************************************/ ****************************************************************/
#include <linux/compiler.h>
#include <linux/string.h> /* memcpy, memset */
#include "bitstream.h" #include "bitstream.h"
#include "fse.h" #include "fse.h"
#include <linux/compiler.h>
#include <linux/string.h> /* memcpy, memset */
/* ************************************************************** /* **************************************************************
* Error Management * Error Management
****************************************************************/ ****************************************************************/
#define FSE_isError ERR_isError #define FSE_isError ERR_isError
#define FSE_STATIC_ASSERT(c) { enum { FSE_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */ #define FSE_STATIC_ASSERT(c) \
{ \
enum { FSE_static_assert = 1 / (int)(!!(c)) }; \
} /* use only *after* variable declarations */
/* check and forward error code */ /* check and forward error code */
#define CHECK_F(f) { size_t const e = f; if (FSE_isError(e)) return e; } #define CHECK_F(f) \
{ \
size_t const e = f; \
if (FSE_isError(e)) \
return e; \
}
/* ************************************************************** /* **************************************************************
* Templates * Templates
@@ -85,7 +89,6 @@
#define FSE_FUNCTION_NAME(X, Y) FSE_CAT(X, Y) #define FSE_FUNCTION_NAME(X, Y) FSE_CAT(X, Y)
#define FSE_TYPE_NAME(X, Y) FSE_CAT(X, Y) #define FSE_TYPE_NAME(X, Y) FSE_CAT(X, Y)
/* Function templates */ /* Function templates */
size_t FSE_buildDTable(FSE_DTable *dt, const short *normalizedCounter, unsigned maxSymbolValue, unsigned tableLog) size_t FSE_buildDTable(FSE_DTable *dt, const short *normalizedCounter, unsigned maxSymbolValue, unsigned tableLog)
@@ -99,28 +102,36 @@ size_t FSE_buildDTable(FSE_DTable* dt, const short* normalizedCounter, unsigned
U32 highThreshold = tableSize - 1; U32 highThreshold = tableSize - 1;
/* Sanity Checks */ /* Sanity Checks */
if (maxSymbolValue > FSE_MAX_SYMBOL_VALUE) return ERROR(maxSymbolValue_tooLarge); if (maxSymbolValue > FSE_MAX_SYMBOL_VALUE)
if (tableLog > FSE_MAX_TABLELOG) return ERROR(tableLog_tooLarge); return ERROR(maxSymbolValue_tooLarge);
if (tableLog > FSE_MAX_TABLELOG)
return ERROR(tableLog_tooLarge);
/* Init, lay down lowprob symbols */ /* Init, lay down lowprob symbols */
{ FSE_DTableHeader DTableH; {
FSE_DTableHeader DTableH;
DTableH.tableLog = (U16)tableLog; DTableH.tableLog = (U16)tableLog;
DTableH.fastMode = 1; DTableH.fastMode = 1;
{ S16 const largeLimit= (S16)(1 << (tableLog-1)); {
S16 const largeLimit = (S16)(1 << (tableLog - 1));
U32 s; U32 s;
for (s = 0; s < maxSV1; s++) { for (s = 0; s < maxSV1; s++) {
if (normalizedCounter[s] == -1) { if (normalizedCounter[s] == -1) {
tableDecode[highThreshold--].symbol = (FSE_FUNCTION_TYPE)s; tableDecode[highThreshold--].symbol = (FSE_FUNCTION_TYPE)s;
symbolNext[s] = 1; symbolNext[s] = 1;
} else { } else {
if (normalizedCounter[s] >= largeLimit) DTableH.fastMode=0; if (normalizedCounter[s] >= largeLimit)
DTableH.fastMode = 0;
symbolNext[s] = normalizedCounter[s]; symbolNext[s] = normalizedCounter[s];
} } } }
}
}
memcpy(dt, &DTableH, sizeof(DTableH)); memcpy(dt, &DTableH, sizeof(DTableH));
} }
/* Spread symbols */ /* Spread symbols */
{ U32 const tableMask = tableSize-1; {
U32 const tableMask = tableSize - 1;
U32 const step = FSE_TABLESTEP(tableSize); U32 const step = FSE_TABLESTEP(tableSize);
U32 s, position = 0; U32 s, position = 0;
for (s = 0; s < maxSV1; s++) { for (s = 0; s < maxSV1; s++) {
@@ -128,24 +139,28 @@ size_t FSE_buildDTable(FSE_DTable* dt, const short* normalizedCounter, unsigned
for (i = 0; i < normalizedCounter[s]; i++) { for (i = 0; i < normalizedCounter[s]; i++) {
tableDecode[position].symbol = (FSE_FUNCTION_TYPE)s; tableDecode[position].symbol = (FSE_FUNCTION_TYPE)s;
position = (position + step) & tableMask; position = (position + step) & tableMask;
while (position > highThreshold) position = (position + step) & tableMask; /* lowprob area */ while (position > highThreshold)
} } position = (position + step) & tableMask; /* lowprob area */
if (position!=0) return ERROR(GENERIC); /* position must reach all cells once, otherwise normalizedCounter is incorrect */ }
}
if (position != 0)
return ERROR(GENERIC); /* position must reach all cells once, otherwise normalizedCounter is incorrect */
} }
/* Build Decoding table */ /* Build Decoding table */
{ U32 u; {
U32 u;
for (u = 0; u < tableSize; u++) { for (u = 0; u < tableSize; u++) {
FSE_FUNCTION_TYPE const symbol = (FSE_FUNCTION_TYPE)(tableDecode[u].symbol); FSE_FUNCTION_TYPE const symbol = (FSE_FUNCTION_TYPE)(tableDecode[u].symbol);
U16 nextState = symbolNext[symbol]++; U16 nextState = symbolNext[symbol]++;
tableDecode[u].nbBits = (BYTE)(tableLog - BIT_highbit32((U32)nextState)); tableDecode[u].nbBits = (BYTE)(tableLog - BIT_highbit32((U32)nextState));
tableDecode[u].newState = (U16)((nextState << tableDecode[u].nbBits) - tableSize); tableDecode[u].newState = (U16)((nextState << tableDecode[u].nbBits) - tableSize);
} } }
}
return 0; return 0;
} }
#ifndef FSE_COMMONDEFS_ONLY #ifndef FSE_COMMONDEFS_ONLY
/*-******************************************************* /*-*******************************************************
@@ -168,7 +183,6 @@ size_t FSE_buildDTable_rle (FSE_DTable* dt, BYTE symbolValue)
return 0; return 0;
} }
size_t FSE_buildDTable_raw(FSE_DTable *dt, unsigned nbBits) size_t FSE_buildDTable_raw(FSE_DTable *dt, unsigned nbBits)
{ {
void *ptr = dt; void *ptr = dt;
@@ -181,7 +195,8 @@ size_t FSE_buildDTable_raw (FSE_DTable* dt, unsigned nbBits)
unsigned s; unsigned s;
/* Sanity checks */ /* Sanity checks */
if (nbBits < 1) return ERROR(GENERIC); /* min size */ if (nbBits < 1)
return ERROR(GENERIC); /* min size */
/* Build Decoding Table */ /* Build Decoding Table */
DTableH->tableLog = (U16)nbBits; DTableH->tableLog = (U16)nbBits;
@@ -195,10 +210,8 @@ size_t FSE_buildDTable_raw (FSE_DTable* dt, unsigned nbBits)
return 0; return 0;
} }
FORCE_INLINE size_t FSE_decompress_usingDTable_generic( FORCE_INLINE size_t FSE_decompress_usingDTable_generic(void *dst, size_t maxDstSize, const void *cSrc, size_t cSrcSize, const FSE_DTable *dt,
void* dst, size_t maxDstSize, const unsigned fast)
const void* cSrc, size_t cSrcSize,
const FSE_DTable* dt, const unsigned fast)
{ {
BYTE *const ostart = (BYTE *)dst; BYTE *const ostart = (BYTE *)dst;
BYTE *op = ostart; BYTE *op = ostart;
@@ -227,7 +240,12 @@ FORCE_INLINE size_t FSE_decompress_usingDTable_generic(
op[1] = FSE_GETSYMBOL(&state2); op[1] = FSE_GETSYMBOL(&state2);
if (FSE_MAX_TABLELOG * 4 + 7 > sizeof(bitD.bitContainer) * 8) /* This test must be static */ if (FSE_MAX_TABLELOG * 4 + 7 > sizeof(bitD.bitContainer) * 8) /* This test must be static */
{ if (BIT_reloadDStream(&bitD) > BIT_DStream_unfinished) { op+=2; break; } } {
if (BIT_reloadDStream(&bitD) > BIT_DStream_unfinished) {
op += 2;
break;
}
}
op[2] = FSE_GETSYMBOL(&state1); op[2] = FSE_GETSYMBOL(&state1);
@@ -240,38 +258,38 @@ FORCE_INLINE size_t FSE_decompress_usingDTable_generic(
/* tail */ /* tail */
/* note : BIT_reloadDStream(&bitD) >= FSE_DStream_partiallyFilled; Ends at exactly BIT_DStream_completed */ /* note : BIT_reloadDStream(&bitD) >= FSE_DStream_partiallyFilled; Ends at exactly BIT_DStream_completed */
while (1) { while (1) {
if (op>(omax-2)) return ERROR(dstSize_tooSmall); if (op > (omax - 2))
return ERROR(dstSize_tooSmall);
*op++ = FSE_GETSYMBOL(&state1); *op++ = FSE_GETSYMBOL(&state1);
if (BIT_reloadDStream(&bitD) == BIT_DStream_overflow) { if (BIT_reloadDStream(&bitD) == BIT_DStream_overflow) {
*op++ = FSE_GETSYMBOL(&state2); *op++ = FSE_GETSYMBOL(&state2);
break; break;
} }
if (op>(omax-2)) return ERROR(dstSize_tooSmall); if (op > (omax - 2))
return ERROR(dstSize_tooSmall);
*op++ = FSE_GETSYMBOL(&state2); *op++ = FSE_GETSYMBOL(&state2);
if (BIT_reloadDStream(&bitD) == BIT_DStream_overflow) { if (BIT_reloadDStream(&bitD) == BIT_DStream_overflow) {
*op++ = FSE_GETSYMBOL(&state1); *op++ = FSE_GETSYMBOL(&state1);
break; break;
} } }
}
return op - ostart; return op - ostart;
} }
size_t FSE_decompress_usingDTable(void *dst, size_t originalSize, const void *cSrc, size_t cSrcSize, const FSE_DTable *dt)
size_t FSE_decompress_usingDTable(void* dst, size_t originalSize,
const void* cSrc, size_t cSrcSize,
const FSE_DTable* dt)
{ {
const void *ptr = dt; const void *ptr = dt;
const FSE_DTableHeader *DTableH = (const FSE_DTableHeader *)ptr; const FSE_DTableHeader *DTableH = (const FSE_DTableHeader *)ptr;
const U32 fastMode = DTableH->fastMode; const U32 fastMode = DTableH->fastMode;
/* select fast mode (static) */ /* select fast mode (static) */
if (fastMode) return FSE_decompress_usingDTable_generic(dst, originalSize, cSrc, cSrcSize, dt, 1); if (fastMode)
return FSE_decompress_usingDTable_generic(dst, originalSize, cSrc, cSrcSize, dt, 1);
return FSE_decompress_usingDTable_generic(dst, originalSize, cSrc, cSrcSize, dt, 0); return FSE_decompress_usingDTable_generic(dst, originalSize, cSrc, cSrcSize, dt, 0);
} }
size_t FSE_decompress_wksp(void *dst, size_t dstCapacity, const void *cSrc, size_t cSrcSize, FSE_DTable *workSpace, unsigned maxLog) size_t FSE_decompress_wksp(void *dst, size_t dstCapacity, const void *cSrc, size_t cSrcSize, FSE_DTable *workSpace, unsigned maxLog)
{ {
const BYTE *const istart = (const BYTE *)cSrc; const BYTE *const istart = (const BYTE *)cSrc;
@@ -282,9 +300,12 @@ size_t FSE_decompress_wksp(void* dst, size_t dstCapacity, const void* cSrc, size
/* normal FSE decoding mode */ /* normal FSE decoding mode */
size_t const NCountLength = FSE_readNCount(counting, &maxSymbolValue, &tableLog, istart, cSrcSize); size_t const NCountLength = FSE_readNCount(counting, &maxSymbolValue, &tableLog, istart, cSrcSize);
if (FSE_isError(NCountLength)) return NCountLength; if (FSE_isError(NCountLength))
//if (NCountLength >= cSrcSize) return ERROR(srcSize_wrong); /* too small input size; supposed to be already checked in NCountLength, only remaining case : NCountLength==cSrcSize */ return NCountLength;
if (tableLog > maxLog) return ERROR(tableLog_tooLarge); // if (NCountLength >= cSrcSize) return ERROR(srcSize_wrong); /* too small input size; supposed to be already checked in NCountLength, only remaining
// case : NCountLength==cSrcSize */
if (tableLog > maxLog)
return ERROR(tableLog_tooLarge);
ip += NCountLength; ip += NCountLength;
cSrcSize -= NCountLength; cSrcSize -= NCountLength;
@@ -293,5 +314,4 @@ size_t FSE_decompress_wksp(void* dst, size_t dstCapacity, const void* cSrc, size
return FSE_decompress_usingDTable(dst, dstCapacity, ip, cSrcSize, workSpace); /* always return, even if it is an error code */ return FSE_decompress_usingDTable(dst, dstCapacity, ip, cSrcSize, workSpace); /* always return, even if it is an error code */
} }
#endif /* FSE_COMMONDEFS_ONLY */ #endif /* FSE_COMMONDEFS_ONLY */
contrib/linux-kernel/lib/zstd/huf.h
+19 -25
View File
@@ -40,11 +40,9 @@
#ifndef HUF_H_298734234 #ifndef HUF_H_298734234
#define HUF_H_298734234 #define HUF_H_298734234
/* *** Dependencies *** */ /* *** Dependencies *** */
#include <linux/types.h> /* size_t */ #include <linux/types.h> /* size_t */
/* *** Tool functions *** */ /* *** Tool functions *** */
#define HUF_BLOCKSIZE_MAX (128 * 1024) /**< maximum input size for a single block compressed with HUF_compress */ #define HUF_BLOCKSIZE_MAX (128 * 1024) /**< maximum input size for a single block compressed with HUF_compress */
size_t HUF_compressBound(size_t size); /**< maximum compressed size (worst case) */ size_t HUF_compressBound(size_t size); /**< maximum compressed size (worst case) */
@@ -52,19 +50,16 @@ size_t HUF_compressBound(size_t size); /**< maximum compressed size (worst
/* Error Management */ /* Error Management */
unsigned HUF_isError(size_t code); /**< tells if a return value is an error code */ unsigned HUF_isError(size_t code); /**< tells if a return value is an error code */
/* *** Advanced function *** */ /* *** Advanced function *** */
/** HUF_compress4X_wksp() : /** HUF_compress4X_wksp() :
* Same as HUF_compress2(), but uses externally allocated `workSpace`, which must be a table of >= 1024 unsigned */ * Same as HUF_compress2(), but uses externally allocated `workSpace`, which must be a table of >= 1024 unsigned */
size_t HUF_compress4X_wksp (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize); /**< `workSpace` must be a table of at least HUF_WORKSPACE_SIZE_U32 unsigned */ size_t HUF_compress4X_wksp(void *dst, size_t dstSize, const void *src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void *workSpace,
size_t wkspSize); /**< `workSpace` must be a table of at least HUF_WORKSPACE_SIZE_U32 unsigned */
/* *** Dependencies *** */ /* *** Dependencies *** */
#include "mem.h" /* U32 */ #include "mem.h" /* U32 */
/* *** Constants *** */ /* *** Constants *** */
#define HUF_TABLELOG_MAX 12 /* max configured tableLog (for static allocation); can be modified up to HUF_ABSOLUTEMAX_TABLELOG */ #define HUF_TABLELOG_MAX 12 /* max configured tableLog (for static allocation); can be modified up to HUF_ABSOLUTEMAX_TABLELOG */
#define HUF_TABLELOG_DEFAULT 11 /* tableLog by default, when not specified */ #define HUF_TABLELOG_DEFAULT 11 /* tableLog by default, when not specified */
@@ -75,7 +70,6 @@ size_t HUF_compress4X_wksp (void* dst, size_t dstSize, const void* src, size_t s
#error "HUF_TABLELOG_MAX is too large !" #error "HUF_TABLELOG_MAX is too large !"
#endif #endif
/* **************************************** /* ****************************************
* Static allocation * Static allocation
******************************************/ ******************************************/
@@ -93,25 +87,22 @@ size_t HUF_compress4X_wksp (void* dst, size_t dstSize, const void* src, size_t s
/* static allocation of HUF's DTable */ /* static allocation of HUF's DTable */
typedef U32 HUF_DTable; typedef U32 HUF_DTable;
#define HUF_DTABLE_SIZE(maxTableLog) (1 + (1 << (maxTableLog))) #define HUF_DTABLE_SIZE(maxTableLog) (1 + (1 << (maxTableLog)))
#define HUF_CREATE_STATIC_DTABLEX2(DTable, maxTableLog) \ #define HUF_CREATE_STATIC_DTABLEX2(DTable, maxTableLog) HUF_DTable DTable[HUF_DTABLE_SIZE((maxTableLog)-1)] = {((U32)((maxTableLog)-1) * 0x01000001)}
HUF_DTable DTable[HUF_DTABLE_SIZE((maxTableLog)-1)] = { ((U32)((maxTableLog)-1) * 0x01000001) } #define HUF_CREATE_STATIC_DTABLEX4(DTable, maxTableLog) HUF_DTable DTable[HUF_DTABLE_SIZE(maxTableLog)] = {((U32)(maxTableLog)*0x01000001)}
#define HUF_CREATE_STATIC_DTABLEX4(DTable, maxTableLog) \
HUF_DTable DTable[HUF_DTABLE_SIZE(maxTableLog)] = { ((U32)(maxTableLog) * 0x01000001) }
/* The workspace must have alignment at least 4 and be at least this large */ /* The workspace must have alignment at least 4 and be at least this large */
#define HUF_WORKSPACE_SIZE (6 << 10) #define HUF_WORKSPACE_SIZE (6 << 10)
#define HUF_WORKSPACE_SIZE_U32 (HUF_WORKSPACE_SIZE / sizeof(U32)) #define HUF_WORKSPACE_SIZE_U32 (HUF_WORKSPACE_SIZE / sizeof(U32))
/* **************************************** /* ****************************************
* Advanced decompression functions * Advanced decompression functions
******************************************/ ******************************************/
size_t HUF_decompress4X_DCtx(HUF_DTable *dctx, void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize); /**< decodes RLE and uncompressed */ size_t HUF_decompress4X_DCtx(HUF_DTable *dctx, void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize); /**< decodes RLE and uncompressed */
size_t HUF_decompress4X_hufOnly(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< considers RLE and uncompressed as errors */ size_t HUF_decompress4X_hufOnly(HUF_DTable *dctx, void *dst, size_t dstSize, const void *cSrc,
size_t cSrcSize); /**< considers RLE and uncompressed as errors */
size_t HUF_decompress4X2_DCtx(HUF_DTable *dctx, void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize); /**< single-symbol decoder */ size_t HUF_decompress4X2_DCtx(HUF_DTable *dctx, void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize); /**< single-symbol decoder */
size_t HUF_decompress4X4_DCtx(HUF_DTable *dctx, void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize); /**< double-symbols decoder */ size_t HUF_decompress4X4_DCtx(HUF_DTable *dctx, void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize); /**< double-symbols decoder */
/* **************************************** /* ****************************************
* HUF detailed API * HUF detailed API
******************************************/ ******************************************/
@@ -135,7 +126,8 @@ size_t HUF_compress4X_usingCTable(void* dst, size_t dstSize, const void* src, si
typedef enum { typedef enum {
HUF_repeat_none, /**< Cannot use the previous table */ HUF_repeat_none, /**< Cannot use the previous table */
HUF_repeat_check, /**< Can use the previous table but it must be checked. Note : The previous table must have been constructed by HUF_compress{1, 4}X_repeat */ HUF_repeat_check, /**< Can use the previous table but it must be checked. Note : The previous table must have been constructed by HUF_compress{1,
4}X_repeat */
HUF_repeat_valid /**< Can use the previous table and it is asumed to be valid */ HUF_repeat_valid /**< Can use the previous table and it is asumed to be valid */
} HUF_repeat; } HUF_repeat;
/** HUF_compress4X_repeat() : /** HUF_compress4X_repeat() :
@@ -143,7 +135,9 @@ typedef enum {
* If it uses hufTable it does not modify hufTable or repeat. * If it uses hufTable it does not modify hufTable or repeat.
* If it doesn't, it sets *repeat = HUF_repeat_none, and it sets hufTable to the table used. * If it doesn't, it sets *repeat = HUF_repeat_none, and it sets hufTable to the table used.
* If preferRepeat then the old table will always be used if valid. */ * If preferRepeat then the old table will always be used if valid. */
size_t HUF_compress4X_repeat(void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize, HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat); /**< `workSpace` must be a table of at least HUF_WORKSPACE_SIZE_U32 unsigned */ size_t HUF_compress4X_repeat(void *dst, size_t dstSize, const void *src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void *workSpace,
size_t wkspSize, HUF_CElt *hufTable, HUF_repeat *repeat,
int preferRepeat); /**< `workSpace` must be a table of at least HUF_WORKSPACE_SIZE_U32 unsigned */
/** HUF_buildCTable_wksp() : /** HUF_buildCTable_wksp() :
* Same as HUF_buildCTable(), but using externally allocated scratch buffer. * Same as HUF_buildCTable(), but using externally allocated scratch buffer.
@@ -156,15 +150,12 @@ size_t HUF_buildCTable_wksp (HUF_CElt* tree, const U32* count, U32 maxSymbolValu
`huffWeight` is destination buffer. `huffWeight` is destination buffer.
@return : size read from `src` , or an error Code . @return : size read from `src` , or an error Code .
Note : Needed by HUF_readCTable() and HUF_readDTableXn() . */ Note : Needed by HUF_readCTable() and HUF_readDTableXn() . */
size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats, size_t HUF_readStats(BYTE *huffWeight, size_t hwSize, U32 *rankStats, U32 *nbSymbolsPtr, U32 *tableLogPtr, const void *src, size_t srcSize);
U32* nbSymbolsPtr, U32* tableLogPtr,
const void* src, size_t srcSize);
/** HUF_readCTable() : /** HUF_readCTable() :
* Loading a CTable saved with HUF_writeCTable() */ * Loading a CTable saved with HUF_writeCTable() */
size_t HUF_readCTable(HUF_CElt *CTable, unsigned maxSymbolValue, const void *src, size_t srcSize); size_t HUF_readCTable(HUF_CElt *CTable, unsigned maxSymbolValue, const void *src, size_t srcSize);
/* /*
HUF_decompress() does the following: HUF_decompress() does the following:
1. select the decompression algorithm (X2, X4) based on pre-computed heuristics 1. select the decompression algorithm (X2, X4) based on pre-computed heuristics
@@ -186,23 +177,26 @@ size_t HUF_decompress4X_usingDTable(void* dst, size_t maxDstSize, const void* cS
size_t HUF_decompress4X2_usingDTable(void *dst, size_t maxDstSize, const void *cSrc, size_t cSrcSize, const HUF_DTable *DTable); size_t HUF_decompress4X2_usingDTable(void *dst, size_t maxDstSize, const void *cSrc, size_t cSrcSize, const HUF_DTable *DTable);
size_t HUF_decompress4X4_usingDTable(void *dst, size_t maxDstSize, const void *cSrc, size_t cSrcSize, const HUF_DTable *DTable); size_t HUF_decompress4X4_usingDTable(void *dst, size_t maxDstSize, const void *cSrc, size_t cSrcSize, const HUF_DTable *DTable);
/* single stream variants */ /* single stream variants */
size_t HUF_compress1X_wksp (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize); /**< `workSpace` must be a table of at least HUF_WORKSPACE_SIZE_U32 unsigned */ size_t HUF_compress1X_wksp(void *dst, size_t dstSize, const void *src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void *workSpace,
size_t wkspSize); /**< `workSpace` must be a table of at least HUF_WORKSPACE_SIZE_U32 unsigned */
size_t HUF_compress1X_usingCTable(void *dst, size_t dstSize, const void *src, size_t srcSize, const HUF_CElt *CTable); size_t HUF_compress1X_usingCTable(void *dst, size_t dstSize, const void *src, size_t srcSize, const HUF_CElt *CTable);
/** HUF_compress1X_repeat() : /** HUF_compress1X_repeat() :
* Same as HUF_compress1X_wksp(), but considers using hufTable if *repeat != HUF_repeat_none. * Same as HUF_compress1X_wksp(), but considers using hufTable if *repeat != HUF_repeat_none.
* If it uses hufTable it does not modify hufTable or repeat. * If it uses hufTable it does not modify hufTable or repeat.
* If it doesn't, it sets *repeat = HUF_repeat_none, and it sets hufTable to the table used. * If it doesn't, it sets *repeat = HUF_repeat_none, and it sets hufTable to the table used.
* If preferRepeat then the old table will always be used if valid. */ * If preferRepeat then the old table will always be used if valid. */
size_t HUF_compress1X_repeat(void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize, HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat); /**< `workSpace` must be a table of at least HUF_WORKSPACE_SIZE_U32 unsigned */ size_t HUF_compress1X_repeat(void *dst, size_t dstSize, const void *src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void *workSpace,
size_t wkspSize, HUF_CElt *hufTable, HUF_repeat *repeat,
int preferRepeat); /**< `workSpace` must be a table of at least HUF_WORKSPACE_SIZE_U32 unsigned */
size_t HUF_decompress1X_DCtx(HUF_DTable *dctx, void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize); size_t HUF_decompress1X_DCtx(HUF_DTable *dctx, void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize);
size_t HUF_decompress1X2_DCtx(HUF_DTable *dctx, void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize); /**< single-symbol decoder */ size_t HUF_decompress1X2_DCtx(HUF_DTable *dctx, void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize); /**< single-symbol decoder */
size_t HUF_decompress1X4_DCtx(HUF_DTable *dctx, void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize); /**< double-symbols decoder */ size_t HUF_decompress1X4_DCtx(HUF_DTable *dctx, void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize); /**< double-symbols decoder */
size_t HUF_decompress1X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable); /**< automatic selection of sing or double symbol decoder, based on DTable */ size_t HUF_decompress1X_usingDTable(void *dst, size_t maxDstSize, const void *cSrc, size_t cSrcSize,
const HUF_DTable *DTable); /**< automatic selection of sing or double symbol decoder, based on DTable */
size_t HUF_decompress1X2_usingDTable(void *dst, size_t maxDstSize, const void *cSrc, size_t cSrcSize, const HUF_DTable *DTable); size_t HUF_decompress1X2_usingDTable(void *dst, size_t maxDstSize, const void *cSrc, size_t cSrcSize, const HUF_DTable *DTable);
size_t HUF_decompress1X4_usingDTable(void *dst, size_t maxDstSize, const void *cSrc, size_t cSrcSize, const HUF_DTable *DTable); size_t HUF_decompress1X4_usingDTable(void *dst, size_t maxDstSize, const void *cSrc, size_t cSrcSize, const HUF_DTable *DTable);
contrib/linux-kernel/lib/zstd/huf_compress.c
+233 -150
View File
@@ -37,23 +37,29 @@
* - Source repository : https://github.com/Cyan4973/FiniteStateEntropy * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy
*/ */
/* ************************************************************** /* **************************************************************
* Includes * Includes
****************************************************************/ ****************************************************************/
#include <linux/string.h> /* memcpy, memset */
#include "bitstream.h" #include "bitstream.h"
#include "fse.h" /* header compression */ #include "fse.h" /* header compression */
#include "huf.h" #include "huf.h"
#include <linux/string.h> /* memcpy, memset */
/* ************************************************************** /* **************************************************************
* Error Management * Error Management
****************************************************************/ ****************************************************************/
#define HUF_STATIC_ASSERT(c) { enum { HUF_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */ #define HUF_STATIC_ASSERT(c) \
#define CHECK_V_F(e, f) size_t const e = f; if (ERR_isError(e)) return f { \
#define CHECK_F(f) { CHECK_V_F(_var_err__, f); } enum { HUF_static_assert = 1 / (int)(!!(c)) }; \
} /* use only *after* variable declarations */
#define CHECK_V_F(e, f) \
size_t const e = f; \
if (ERR_isError(e)) \
return f
#define CHECK_F(f) \
{ \
CHECK_V_F(_var_err__, f); \
}
/* ************************************************************** /* **************************************************************
* Utils * Utils
@@ -63,7 +69,6 @@ unsigned HUF_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxS
return FSE_optimalTableLog_internal(maxTableLog, srcSize, maxSymbolValue, 1); return FSE_optimalTableLog_internal(maxTableLog, srcSize, maxSymbolValue, 1);
} }
/* ******************************************************* /* *******************************************************
* HUF : Huffman block compression * HUF : Huffman block compression
*********************************************************/ *********************************************************/
@@ -89,33 +94,39 @@ size_t HUF_compressWeights (void* dst, size_t dstSize, const void* weightTable,
S16 norm[HUF_TABLELOG_MAX + 1]; S16 norm[HUF_TABLELOG_MAX + 1];
/* init conditions */ /* init conditions */
if (wtSize <= 1) return 0; /* Not compressible */ if (wtSize <= 1)
return 0; /* Not compressible */
/* Scan input and build symbol stats */ /* Scan input and build symbol stats */
{ CHECK_V_F(maxCount, FSE_count_simple(count, &maxSymbolValue, weightTable, wtSize) ); {
if (maxCount == wtSize) return 1; /* only a single symbol in src : rle */ CHECK_V_F(maxCount, FSE_count_simple(count, &maxSymbolValue, weightTable, wtSize));
if (maxCount == 1) return 0; /* each symbol present maximum once => not compressible */ if (maxCount == wtSize)
return 1; /* only a single symbol in src : rle */
if (maxCount == 1)
return 0; /* each symbol present maximum once => not compressible */
} }
tableLog = FSE_optimalTableLog(tableLog, wtSize, maxSymbolValue); tableLog = FSE_optimalTableLog(tableLog, wtSize, maxSymbolValue);
CHECK_F(FSE_normalizeCount(norm, tableLog, count, wtSize, maxSymbolValue)); CHECK_F(FSE_normalizeCount(norm, tableLog, count, wtSize, maxSymbolValue));
/* Write table description header */ /* Write table description header */
{ CHECK_V_F(hSize, FSE_writeNCount(op, oend-op, norm, maxSymbolValue, tableLog) ); {
CHECK_V_F(hSize, FSE_writeNCount(op, oend - op, norm, maxSymbolValue, tableLog));
op += hSize; op += hSize;
} }
/* Compress */ /* Compress */
CHECK_F(FSE_buildCTable_wksp(CTable, norm, maxSymbolValue, tableLog, scratchBuffer, sizeof(scratchBuffer))); CHECK_F(FSE_buildCTable_wksp(CTable, norm, maxSymbolValue, tableLog, scratchBuffer, sizeof(scratchBuffer)));
{ CHECK_V_F(cSize, FSE_compress_usingCTable(op, oend - op, weightTable, wtSize, CTable) ); {
if (cSize == 0) return 0; /* not enough space for compressed data */ CHECK_V_F(cSize, FSE_compress_usingCTable(op, oend - op, weightTable, wtSize, CTable));
if (cSize == 0)
return 0; /* not enough space for compressed data */
op += cSize; op += cSize;
} }
return op - ostart; return op - ostart;
} }
struct HUF_CElt_s { struct HUF_CElt_s {
U16 val; U16 val;
BYTE nbBits; BYTE nbBits;
@@ -124,8 +135,7 @@ struct HUF_CElt_s {
/*! HUF_writeCTable() : /*! HUF_writeCTable() :
`CTable` : Huffman tree to save, using huf representation. `CTable` : Huffman tree to save, using huf representation.
@return : size of saved CTable */ @return : size of saved CTable */
size_t HUF_writeCTable (void* dst, size_t maxDstSize, size_t HUF_writeCTable(void *dst, size_t maxDstSize, const HUF_CElt *CTable, U32 maxSymbolValue, U32 huffLog)
const HUF_CElt* CTable, U32 maxSymbolValue, U32 huffLog)
{ {
BYTE bitsToWeight[HUF_TABLELOG_MAX + 1]; /* precomputed conversion table */ BYTE bitsToWeight[HUF_TABLELOG_MAX + 1]; /* precomputed conversion table */
BYTE huffWeight[HUF_SYMBOLVALUE_MAX]; BYTE huffWeight[HUF_SYMBOLVALUE_MAX];
@@ -133,7 +143,8 @@ size_t HUF_writeCTable (void* dst, size_t maxDstSize,
U32 n; U32 n;
/* check conditions */ /* check conditions */
if (maxSymbolValue > HUF_SYMBOLVALUE_MAX) return ERROR(maxSymbolValue_tooLarge); if (maxSymbolValue > HUF_SYMBOLVALUE_MAX)
return ERROR(maxSymbolValue_tooLarge);
/* convert to weight */ /* convert to weight */
bitsToWeight[0] = 0; bitsToWeight[0] = 0;
@@ -143,15 +154,19 @@ size_t HUF_writeCTable (void* dst, size_t maxDstSize,
huffWeight[n] = bitsToWeight[CTable[n].nbBits]; huffWeight[n] = bitsToWeight[CTable[n].nbBits];
/* attempt weights compression by FSE */ /* attempt weights compression by FSE */
{ CHECK_V_F(hSize, HUF_compressWeights(op+1, maxDstSize-1, huffWeight, maxSymbolValue) ); {
CHECK_V_F(hSize, HUF_compressWeights(op + 1, maxDstSize - 1, huffWeight, maxSymbolValue));
if ((hSize > 1) & (hSize < maxSymbolValue / 2)) { /* FSE compressed */ if ((hSize > 1) & (hSize < maxSymbolValue / 2)) { /* FSE compressed */
op[0] = (BYTE)hSize; op[0] = (BYTE)hSize;
return hSize + 1; return hSize + 1;
} } }
}
/* write raw values as 4-bits (max : 15) */ /* write raw values as 4-bits (max : 15) */
if (maxSymbolValue > (256-128)) return ERROR(GENERIC); /* should not happen : likely means source cannot be compressed */ if (maxSymbolValue > (256 - 128))
if (((maxSymbolValue+1)/2) + 1 > maxDstSize) return ERROR(dstSize_tooSmall); /* not enough space within dst buffer */ return ERROR(GENERIC); /* should not happen : likely means source cannot be compressed */
if (((maxSymbolValue + 1) / 2) + 1 > maxDstSize)
return ERROR(dstSize_tooSmall); /* not enough space within dst buffer */
op[0] = (BYTE)(128 /*special case*/ + (maxSymbolValue - 1)); op[0] = (BYTE)(128 /*special case*/ + (maxSymbolValue - 1));
huffWeight[maxSymbolValue] = 0; /* to be sure it doesn't cause msan issue in final combination */ huffWeight[maxSymbolValue] = 0; /* to be sure it doesn't cause msan issue in final combination */
for (n = 0; n < maxSymbolValue; n += 2) for (n = 0; n < maxSymbolValue; n += 2)
@@ -159,7 +174,6 @@ size_t HUF_writeCTable (void* dst, size_t maxDstSize,
return ((maxSymbolValue + 1) / 2) + 1; return ((maxSymbolValue + 1) / 2) + 1;
} }
size_t HUF_readCTable(HUF_CElt *CTable, U32 maxSymbolValue, const void *src, size_t srcSize) size_t HUF_readCTable(HUF_CElt *CTable, U32 maxSymbolValue, const void *src, size_t srcSize)
{ {
BYTE huffWeight[HUF_SYMBOLVALUE_MAX + 1]; /* init not required, even though some static analyzer may complain */ BYTE huffWeight[HUF_SYMBOLVALUE_MAX + 1]; /* init not required, even though some static analyzer may complain */
@@ -171,43 +185,61 @@ size_t HUF_readCTable (HUF_CElt* CTable, U32 maxSymbolValue, const void* src, si
CHECK_V_F(readSize, HUF_readStats(huffWeight, HUF_SYMBOLVALUE_MAX + 1, rankVal, &nbSymbols, &tableLog, src, srcSize)); CHECK_V_F(readSize, HUF_readStats(huffWeight, HUF_SYMBOLVALUE_MAX + 1, rankVal, &nbSymbols, &tableLog, src, srcSize));
/* check result */ /* check result */
if (tableLog > HUF_TABLELOG_MAX) return ERROR(tableLog_tooLarge); if (tableLog > HUF_TABLELOG_MAX)
if (nbSymbols > maxSymbolValue+1) return ERROR(maxSymbolValue_tooSmall); return ERROR(tableLog_tooLarge);
if (nbSymbols > maxSymbolValue + 1)
return ERROR(maxSymbolValue_tooSmall);
/* Prepare base value per rank */ /* Prepare base value per rank */
{ U32 n, nextRankStart = 0; {
U32 n, nextRankStart = 0;
for (n = 1; n <= tableLog; n++) { for (n = 1; n <= tableLog; n++) {
U32 curr = nextRankStart; U32 curr = nextRankStart;
nextRankStart += (rankVal[n] << (n - 1)); nextRankStart += (rankVal[n] << (n - 1));
rankVal[n] = curr; rankVal[n] = curr;
} } }
}
/* fill nbBits */ /* fill nbBits */
{ U32 n; for (n=0; n<nbSymbols; n++) { {
U32 n;
for (n = 0; n < nbSymbols; n++) {
const U32 w = huffWeight[n]; const U32 w = huffWeight[n];
CTable[n].nbBits = (BYTE)(tableLog + 1 - w); CTable[n].nbBits = (BYTE)(tableLog + 1 - w);
} } }
}
/* fill val */ /* fill val */
{ U16 nbPerRank[HUF_TABLELOG_MAX+2] = {0}; /* support w=0=>n=tableLog+1 */ {
U16 nbPerRank[HUF_TABLELOG_MAX + 2] = {0}; /* support w=0=>n=tableLog+1 */
U16 valPerRank[HUF_TABLELOG_MAX + 2] = {0}; U16 valPerRank[HUF_TABLELOG_MAX + 2] = {0};
{ U32 n; for (n=0; n<nbSymbols; n++) nbPerRank[CTable[n].nbBits]++; } {
U32 n;
for (n = 0; n < nbSymbols; n++)
nbPerRank[CTable[n].nbBits]++;
}
/* determine stating value per rank */ /* determine stating value per rank */
valPerRank[tableLog + 1] = 0; /* for w==0 */ valPerRank[tableLog + 1] = 0; /* for w==0 */
{ U16 min = 0; {
U32 n; for (n=tableLog; n>0; n--) { /* start at n=tablelog <-> w=1 */ U16 min = 0;
U32 n;
for (n = tableLog; n > 0; n--) { /* start at n=tablelog <-> w=1 */
valPerRank[n] = min; /* get starting value within each rank */ valPerRank[n] = min; /* get starting value within each rank */
min += nbPerRank[n]; min += nbPerRank[n];
min >>= 1; min >>= 1;
} } }
}
/* assign value within rank, symbol order */ /* assign value within rank, symbol order */
{ U32 n; for (n=0; n<=maxSymbolValue; n++) CTable[n].val = valPerRank[CTable[n].nbBits]++; } {
U32 n;
for (n = 0; n <= maxSymbolValue; n++)
CTable[n].val = valPerRank[CTable[n].nbBits]++;
}
} }
return readSize; return readSize;
} }
typedef struct nodeElt_s { typedef struct nodeElt_s {
U32 count; U32 count;
U16 parent; U16 parent;
@@ -218,10 +250,12 @@ typedef struct nodeElt_s {
static U32 HUF_setMaxHeight(nodeElt *huffNode, U32 lastNonNull, U32 maxNbBits) static U32 HUF_setMaxHeight(nodeElt *huffNode, U32 lastNonNull, U32 maxNbBits)
{ {
const U32 largestBits = huffNode[lastNonNull].nbBits; const U32 largestBits = huffNode[lastNonNull].nbBits;
if (largestBits <= maxNbBits) return largestBits; /* early exit : no elt > maxNbBits */ if (largestBits <= maxNbBits)
return largestBits; /* early exit : no elt > maxNbBits */
/* there are several too large elements (at least >= 2) */ /* there are several too large elements (at least >= 2) */
{ int totalCost = 0; {
int totalCost = 0;
const U32 baseCost = 1 << (largestBits - maxNbBits); const U32 baseCost = 1 << (largestBits - maxNbBits);
U32 n = lastNonNull; U32 n = lastNonNull;
@@ -230,38 +264,50 @@ static U32 HUF_setMaxHeight(nodeElt* huffNode, U32 lastNonNull, U32 maxNbBits)
huffNode[n].nbBits = (BYTE)maxNbBits; huffNode[n].nbBits = (BYTE)maxNbBits;
n--; n--;
} /* n stops at huffNode[n].nbBits <= maxNbBits */ } /* n stops at huffNode[n].nbBits <= maxNbBits */
while (huffNode[n].nbBits == maxNbBits) n--; /* n end at index of smallest symbol using < maxNbBits */ while (huffNode[n].nbBits == maxNbBits)
n--; /* n end at index of smallest symbol using < maxNbBits */
/* renorm totalCost */ /* renorm totalCost */
totalCost >>= (largestBits - maxNbBits); /* note : totalCost is necessarily a multiple of baseCost */ totalCost >>= (largestBits - maxNbBits); /* note : totalCost is necessarily a multiple of baseCost */
/* repay normalized cost */ /* repay normalized cost */
{ U32 const noSymbol = 0xF0F0F0F0; {
U32 const noSymbol = 0xF0F0F0F0;
U32 rankLast[HUF_TABLELOG_MAX + 2]; U32 rankLast[HUF_TABLELOG_MAX + 2];
int pos; int pos;
/* Get pos of last (smallest) symbol per rank */ /* Get pos of last (smallest) symbol per rank */
memset(rankLast, 0xF0, sizeof(rankLast)); memset(rankLast, 0xF0, sizeof(rankLast));
{ U32 currNbBits = maxNbBits; {
U32 currNbBits = maxNbBits;
for (pos = n; pos >= 0; pos--) { for (pos = n; pos >= 0; pos--) {
if (huffNode[pos].nbBits >= currNbBits) continue; if (huffNode[pos].nbBits >= currNbBits)
continue;
currNbBits = huffNode[pos].nbBits; /* < maxNbBits */ currNbBits = huffNode[pos].nbBits; /* < maxNbBits */
rankLast[maxNbBits - currNbBits] = pos; rankLast[maxNbBits - currNbBits] = pos;
} } }
}
while (totalCost > 0) { while (totalCost > 0) {
U32 nBitsToDecrease = BIT_highbit32(totalCost) + 1; U32 nBitsToDecrease = BIT_highbit32(totalCost) + 1;
for (; nBitsToDecrease > 1; nBitsToDecrease--) { for (; nBitsToDecrease > 1; nBitsToDecrease--) {
U32 highPos = rankLast[nBitsToDecrease]; U32 highPos = rankLast[nBitsToDecrease];
U32 lowPos = rankLast[nBitsToDecrease - 1]; U32 lowPos = rankLast[nBitsToDecrease - 1];
if (highPos == noSymbol) continue; if (highPos == noSymbol)
if (lowPos == noSymbol) break; continue;
{ U32 const highTotal = huffNode[highPos].count; if (lowPos == noSymbol)
break;
{
U32 const highTotal = huffNode[highPos].count;
U32 const lowTotal = 2 * huffNode[lowPos].count; U32 const lowTotal = 2 * huffNode[lowPos].count;
if (highTotal <= lowTotal) break; if (highTotal <= lowTotal)
} } break;
}
}
/* only triggered when no more rank 1 symbol left => find closest one (note : there is necessarily at least one !) */ /* only triggered when no more rank 1 symbol left => find closest one (note : there is necessarily at least one !) */
while ((nBitsToDecrease<=HUF_TABLELOG_MAX) && (rankLast[nBitsToDecrease] == noSymbol)) /* HUF_MAX_TABLELOG test just to please gcc 5+; but it should not be necessary */ while (
(nBitsToDecrease <= HUF_TABLELOG_MAX) &&
(rankLast[nBitsToDecrease] == noSymbol)) /* HUF_MAX_TABLELOG test just to please gcc 5+; but it should not be necessary */
nBitsToDecrease++; nBitsToDecrease++;
totalCost -= 1 << (nBitsToDecrease - 1); totalCost -= 1 << (nBitsToDecrease - 1);
if (rankLast[nBitsToDecrease - 1] == noSymbol) if (rankLast[nBitsToDecrease - 1] == noSymbol)
@@ -273,11 +319,14 @@ static U32 HUF_setMaxHeight(nodeElt* huffNode, U32 lastNonNull, U32 maxNbBits)
rankLast[nBitsToDecrease]--; rankLast[nBitsToDecrease]--;
if (huffNode[rankLast[nBitsToDecrease]].nbBits != maxNbBits - nBitsToDecrease) if (huffNode[rankLast[nBitsToDecrease]].nbBits != maxNbBits - nBitsToDecrease)
rankLast[nBitsToDecrease] = noSymbol; /* this rank is now empty */ rankLast[nBitsToDecrease] = noSymbol; /* this rank is now empty */
} } /* while (totalCost > 0) */ }
} /* while (totalCost > 0) */
while (totalCost < 0) { /* Sometimes, cost correction overshoot */ while (totalCost < 0) { /* Sometimes, cost correction overshoot */
if (rankLast[1] == noSymbol) { /* special case : no rank 1 symbol (using maxNbBits-1); let's create one from largest rank 0 (using maxNbBits) */ if (rankLast[1] == noSymbol) { /* special case : no rank 1 symbol (using maxNbBits-1); let's create one from largest rank 0
while (huffNode[n].nbBits == maxNbBits) n--; (using maxNbBits) */
while (huffNode[n].nbBits == maxNbBits)
n--;
huffNode[n + 1].nbBits--; huffNode[n + 1].nbBits--;
rankLast[1] = n + 1; rankLast[1] = n + 1;
totalCost++; totalCost++;
@@ -286,12 +335,13 @@ static U32 HUF_setMaxHeight(nodeElt* huffNode, U32 lastNonNull, U32 maxNbBits)
huffNode[rankLast[1] + 1].nbBits--; huffNode[rankLast[1] + 1].nbBits--;
rankLast[1]++; rankLast[1]++;
totalCost++; totalCost++;
} } } /* there are several too large elements (at least >= 2) */ }
}
} /* there are several too large elements (at least >= 2) */
return maxNbBits; return maxNbBits;
} }
typedef struct { typedef struct {
U32 base; U32 base;
U32 curr; U32 curr;
@@ -307,19 +357,21 @@ static void HUF_sort(nodeElt* huffNode, const U32* count, U32 maxSymbolValue)
U32 r = BIT_highbit32(count[n] + 1); U32 r = BIT_highbit32(count[n] + 1);
rank[r].base++; rank[r].base++;
} }
for (n=30; n>0; n--) rank[n-1].base += rank[n].base; for (n = 30; n > 0; n--)
for (n=0; n<32; n++) rank[n].curr = rank[n].base; rank[n - 1].base += rank[n].base;
for (n = 0; n < 32; n++)
rank[n].curr = rank[n].base;
for (n = 0; n <= maxSymbolValue; n++) { for (n = 0; n <= maxSymbolValue; n++) {
U32 const c = count[n]; U32 const c = count[n];
U32 const r = BIT_highbit32(c + 1) + 1; U32 const r = BIT_highbit32(c + 1) + 1;
U32 pos = rank[r].curr++; U32 pos = rank[r].curr++;
while ((pos > rank[r].base) && (c > huffNode[pos-1].count)) huffNode[pos]=huffNode[pos-1], pos--; while ((pos > rank[r].base) && (c > huffNode[pos - 1].count))
huffNode[pos] = huffNode[pos - 1], pos--;
huffNode[pos].count = c; huffNode[pos].count = c;
huffNode[pos].byte = (BYTE)n; huffNode[pos].byte = (BYTE)n;
} }
} }
/** HUF_buildCTable_wksp() : /** HUF_buildCTable_wksp() :
* Same as HUF_buildCTable(), but using externally allocated scratch buffer. * Same as HUF_buildCTable(), but using externally allocated scratch buffer.
* `workSpace` must be aligned on 4-bytes boundaries, and be at least as large as a table of 1024 unsigned. * `workSpace` must be aligned on 4-bytes boundaries, and be at least as large as a table of 1024 unsigned.
@@ -336,9 +388,12 @@ size_t HUF_buildCTable_wksp (HUF_CElt* tree, const U32* count, U32 maxSymbolValu
U32 nodeRoot; U32 nodeRoot;
/* safety checks */ /* safety checks */
if (wkspSize < sizeof(huffNodeTable)) return ERROR(GENERIC); /* workSpace is not large enough */ if (wkspSize < sizeof(huffNodeTable))
if (maxNbBits == 0) maxNbBits = HUF_TABLELOG_DEFAULT; return ERROR(GENERIC); /* workSpace is not large enough */
if (maxSymbolValue > HUF_SYMBOLVALUE_MAX) return ERROR(GENERIC); if (maxNbBits == 0)
maxNbBits = HUF_TABLELOG_DEFAULT;
if (maxSymbolValue > HUF_SYMBOLVALUE_MAX)
return ERROR(GENERIC);
memset(huffNode0, 0, sizeof(huffNodeTable)); memset(huffNode0, 0, sizeof(huffNodeTable));
/* sort, decreasing order */ /* sort, decreasing order */
@@ -346,12 +401,17 @@ size_t HUF_buildCTable_wksp (HUF_CElt* tree, const U32* count, U32 maxSymbolValu
/* init for parents */ /* init for parents */
nonNullRank = maxSymbolValue; nonNullRank = maxSymbolValue;
while(huffNode[nonNullRank].count == 0) nonNullRank--; while (huffNode[nonNullRank].count == 0)
lowS = nonNullRank; nodeRoot = nodeNb + lowS - 1; lowN = nodeNb; nonNullRank--;
lowS = nonNullRank;
nodeRoot = nodeNb + lowS - 1;
lowN = nodeNb;
huffNode[nodeNb].count = huffNode[lowS].count + huffNode[lowS - 1].count; huffNode[nodeNb].count = huffNode[lowS].count + huffNode[lowS - 1].count;
huffNode[lowS].parent = huffNode[lowS - 1].parent = nodeNb; huffNode[lowS].parent = huffNode[lowS - 1].parent = nodeNb;
nodeNb++; lowS-=2; nodeNb++;
for (n=nodeNb; n<=nodeRoot; n++) huffNode[n].count = (U32)(1U<<30); lowS -= 2;
for (n = nodeNb; n <= nodeRoot; n++)
huffNode[n].count = (U32)(1U << 30);
huffNode0[0].count = (U32)(1U << 31); /* fake entry, strong barrier */ huffNode0[0].count = (U32)(1U << 31); /* fake entry, strong barrier */
/* create parents */ /* create parents */
@@ -374,18 +434,22 @@ size_t HUF_buildCTable_wksp (HUF_CElt* tree, const U32* count, U32 maxSymbolValu
maxNbBits = HUF_setMaxHeight(huffNode, nonNullRank, maxNbBits); maxNbBits = HUF_setMaxHeight(huffNode, nonNullRank, maxNbBits);
/* fill result into tree (val, nbBits) */ /* fill result into tree (val, nbBits) */
{ U16 nbPerRank[HUF_TABLELOG_MAX+1] = {0}; {
U16 nbPerRank[HUF_TABLELOG_MAX + 1] = {0};
U16 valPerRank[HUF_TABLELOG_MAX + 1] = {0}; U16 valPerRank[HUF_TABLELOG_MAX + 1] = {0};
if (maxNbBits > HUF_TABLELOG_MAX) return ERROR(GENERIC); /* check fit into table */ if (maxNbBits > HUF_TABLELOG_MAX)
return ERROR(GENERIC); /* check fit into table */
for (n = 0; n <= nonNullRank; n++) for (n = 0; n <= nonNullRank; n++)
nbPerRank[huffNode[n].nbBits]++; nbPerRank[huffNode[n].nbBits]++;
/* determine stating value per rank */ /* determine stating value per rank */
{ U16 min = 0; {
U16 min = 0;
for (n = maxNbBits; n > 0; n--) { for (n = maxNbBits; n > 0; n--) {
valPerRank[n] = min; /* get starting value within each rank */ valPerRank[n] = min; /* get starting value within each rank */
min += nbPerRank[n]; min += nbPerRank[n];
min >>= 1; min >>= 1;
} } }
}
for (n = 0; n <= maxSymbolValue; n++) for (n = 0; n <= maxSymbolValue; n++)
tree[huffNode[n].byte].nbBits = huffNode[n].nbBits; /* push nbBits per symbol, symbol order */ tree[huffNode[n].byte].nbBits = huffNode[n].nbBits; /* push nbBits per symbol, symbol order */
for (n = 0; n <= maxSymbolValue; n++) for (n = 0; n <= maxSymbolValue; n++)
@@ -405,7 +469,8 @@ static size_t HUF_estimateCompressedSize(HUF_CElt* CTable, const unsigned* count
return nbBits >> 3; return nbBits >> 3;
} }
static int HUF_validateCTable(const HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue) { static int HUF_validateCTable(const HUF_CElt *CTable, const unsigned *count, unsigned maxSymbolValue)
{
int bad = 0; int bad = 0;
int s; int s;
for (s = 0; s <= (int)maxSymbolValue; ++s) { for (s = 0; s <= (int)maxSymbolValue; ++s) {
@@ -424,10 +489,12 @@ size_t HUF_compressBound(size_t size) { return HUF_COMPRESSBOUND(size); }
#define HUF_FLUSHBITS(s) (fast ? BIT_flushBitsFast(s) : BIT_flushBits(s)) #define HUF_FLUSHBITS(s) (fast ? BIT_flushBitsFast(s) : BIT_flushBits(s))
#define HUF_FLUSHBITS_1(stream) \ #define HUF_FLUSHBITS_1(stream) \
if (sizeof((stream)->bitContainer)*8 < HUF_TABLELOG_MAX*2+7) HUF_FLUSHBITS(stream) if (sizeof((stream)->bitContainer) * 8 < HUF_TABLELOG_MAX * 2 + 7) \
HUF_FLUSHBITS(stream)
#define HUF_FLUSHBITS_2(stream) \ #define HUF_FLUSHBITS_2(stream) \
if (sizeof((stream)->bitContainer)*8 < HUF_TABLELOG_MAX*4+7) HUF_FLUSHBITS(stream) if (sizeof((stream)->bitContainer) * 8 < HUF_TABLELOG_MAX * 4 + 7) \
HUF_FLUSHBITS(stream)
size_t HUF_compress1X_usingCTable(void *dst, size_t dstSize, const void *src, size_t srcSize, const HUF_CElt *CTable) size_t HUF_compress1X_usingCTable(void *dst, size_t dstSize, const void *src, size_t srcSize, const HUF_CElt *CTable)
{ {
@@ -440,19 +507,19 @@ size_t HUF_compress1X_usingCTable(void* dst, size_t dstSize, const void* src, si
BIT_CStream_t bitC; BIT_CStream_t bitC;
/* init */ /* init */
if (dstSize < 8) return 0; /* not enough space to compress */ if (dstSize < 8)
{ size_t const initErr = BIT_initCStream(&bitC, op, oend-op); return 0; /* not enough space to compress */
if (HUF_isError(initErr)) return 0; } {
size_t const initErr = BIT_initCStream(&bitC, op, oend - op);
if (HUF_isError(initErr))
return 0;
}
n = srcSize & ~3; /* join to mod 4 */ n = srcSize & ~3; /* join to mod 4 */
switch (srcSize & 3) switch (srcSize & 3) {
{ case 3: HUF_encodeSymbol(&bitC, ip[n + 2], CTable); HUF_FLUSHBITS_2(&bitC);
case 3 : HUF_encodeSymbol(&bitC, ip[n+ 2], CTable); case 2: HUF_encodeSymbol(&bitC, ip[n + 1], CTable); HUF_FLUSHBITS_1(&bitC);
HUF_FLUSHBITS_2(&bitC); case 1: HUF_encodeSymbol(&bitC, ip[n + 0], CTable); HUF_FLUSHBITS(&bitC);
case 2 : HUF_encodeSymbol(&bitC, ip[n+ 1], CTable);
HUF_FLUSHBITS_1(&bitC);
case 1 : HUF_encodeSymbol(&bitC, ip[n+ 0], CTable);
HUF_FLUSHBITS(&bitC);
case 0: case 0:
default:; default:;
} }
@@ -471,7 +538,6 @@ size_t HUF_compress1X_usingCTable(void* dst, size_t dstSize, const void* src, si
return BIT_closeCStream(&bitC); return BIT_closeCStream(&bitC);
} }
size_t HUF_compress4X_usingCTable(void *dst, size_t dstSize, const void *src, size_t srcSize, const HUF_CElt *CTable) size_t HUF_compress4X_usingCTable(void *dst, size_t dstSize, const void *src, size_t srcSize, const HUF_CElt *CTable)
{ {
size_t const segmentSize = (srcSize + 3) / 4; /* first 3 segments */ size_t const segmentSize = (srcSize + 3) / 4; /* first 3 segments */
@@ -481,65 +547,71 @@ size_t HUF_compress4X_usingCTable(void* dst, size_t dstSize, const void* src, si
BYTE *const oend = ostart + dstSize; BYTE *const oend = ostart + dstSize;
BYTE *op = ostart; BYTE *op = ostart;
if (dstSize < 6 + 1 + 1 + 1 + 8) return 0; /* minimum space to compress successfully */ if (dstSize < 6 + 1 + 1 + 1 + 8)
if (srcSize < 12) return 0; /* no saving possible : too small input */ return 0; /* minimum space to compress successfully */
if (srcSize < 12)
return 0; /* no saving possible : too small input */
op += 6; /* jumpTable */ op += 6; /* jumpTable */
{ CHECK_V_F(cSize, HUF_compress1X_usingCTable(op, oend-op, ip, segmentSize, CTable) ); {
if (cSize==0) return 0; CHECK_V_F(cSize, HUF_compress1X_usingCTable(op, oend - op, ip, segmentSize, CTable));
if (cSize == 0)
return 0;
ZSTD_writeLE16(ostart, (U16)cSize); ZSTD_writeLE16(ostart, (U16)cSize);
op += cSize; op += cSize;
} }
ip += segmentSize; ip += segmentSize;
{ CHECK_V_F(cSize, HUF_compress1X_usingCTable(op, oend-op, ip, segmentSize, CTable) ); {
if (cSize==0) return 0; CHECK_V_F(cSize, HUF_compress1X_usingCTable(op, oend - op, ip, segmentSize, CTable));
if (cSize == 0)
return 0;
ZSTD_writeLE16(ostart + 2, (U16)cSize); ZSTD_writeLE16(ostart + 2, (U16)cSize);
op += cSize; op += cSize;
} }
ip += segmentSize; ip += segmentSize;
{ CHECK_V_F(cSize, HUF_compress1X_usingCTable(op, oend-op, ip, segmentSize, CTable) ); {
if (cSize==0) return 0; CHECK_V_F(cSize, HUF_compress1X_usingCTable(op, oend - op, ip, segmentSize, CTable));
if (cSize == 0)
return 0;
ZSTD_writeLE16(ostart + 4, (U16)cSize); ZSTD_writeLE16(ostart + 4, (U16)cSize);
op += cSize; op += cSize;
} }
ip += segmentSize; ip += segmentSize;
{ CHECK_V_F(cSize, HUF_compress1X_usingCTable(op, oend-op, ip, iend-ip, CTable) ); {
if (cSize==0) return 0; CHECK_V_F(cSize, HUF_compress1X_usingCTable(op, oend - op, ip, iend - ip, CTable));
if (cSize == 0)
return 0;
op += cSize; op += cSize;
} }
return op - ostart; return op - ostart;
} }
static size_t HUF_compressCTable_internal(BYTE *const ostart, BYTE *op, BYTE *const oend, const void *src, size_t srcSize, unsigned singleStream,
static size_t HUF_compressCTable_internal( const HUF_CElt *CTable)
BYTE* const ostart, BYTE* op, BYTE* const oend,
const void* src, size_t srcSize,
unsigned singleStream, const HUF_CElt* CTable)
{ {
size_t const cSize = singleStream ? size_t const cSize =
HUF_compress1X_usingCTable(op, oend - op, src, srcSize, CTable) : singleStream ? HUF_compress1X_usingCTable(op, oend - op, src, srcSize, CTable) : HUF_compress4X_usingCTable(op, oend - op, src, srcSize, CTable);
HUF_compress4X_usingCTable(op, oend - op, src, srcSize, CTable); if (HUF_isError(cSize)) {
if (HUF_isError(cSize)) { return cSize; } return cSize;
if (cSize==0) { return 0; } /* uncompressible */ }
if (cSize == 0) {
return 0;
} /* uncompressible */
op += cSize; op += cSize;
/* check compressibility */ /* check compressibility */
if ((size_t)(op-ostart) >= srcSize-1) { return 0; } if ((size_t)(op - ostart) >= srcSize - 1) {
return 0;
}
return op - ostart; return op - ostart;
} }
/* `workSpace` must a table of at least 1024 unsigned */ /* `workSpace` must a table of at least 1024 unsigned */
static size_t HUF_compress_internal ( static size_t HUF_compress_internal(void *dst, size_t dstSize, const void *src, size_t srcSize, unsigned maxSymbolValue, unsigned huffLog,
void* dst, size_t dstSize, unsigned singleStream, void *workSpace, size_t wkspSize, HUF_CElt *oldHufTable, HUF_repeat *repeat, int preferRepeat)
const void* src, size_t srcSize,
unsigned maxSymbolValue, unsigned huffLog,
unsigned singleStream,
void* workSpace, size_t wkspSize,
HUF_CElt* oldHufTable, HUF_repeat* repeat, int preferRepeat)
{ {
BYTE *const ostart = (BYTE *)dst; BYTE *const ostart = (BYTE *)dst;
BYTE *const oend = ostart + dstSize; BYTE *const oend = ostart + dstSize;
@@ -551,13 +623,20 @@ static size_t HUF_compress_internal (
size_t const CTableSize = sizeof(HUF_CElt) * (HUF_SYMBOLVALUE_MAX + 1); size_t const CTableSize = sizeof(HUF_CElt) * (HUF_SYMBOLVALUE_MAX + 1);
/* checks & inits */ /* checks & inits */
if (wkspSize < sizeof(huffNodeTable) + countSize + CTableSize) return ERROR(GENERIC); if (wkspSize < sizeof(huffNodeTable) + countSize + CTableSize)
if (!srcSize) return 0; /* Uncompressed (note : 1 means rle, so first byte must be correct) */ return ERROR(GENERIC);
if (!dstSize) return 0; /* cannot fit within dst budget */ if (!srcSize)
if (srcSize > HUF_BLOCKSIZE_MAX) return ERROR(srcSize_wrong); /* curr block size limit */ return 0; /* Uncompressed (note : 1 means rle, so first byte must be correct) */
if (huffLog > HUF_TABLELOG_MAX) return ERROR(tableLog_tooLarge); if (!dstSize)
if (!maxSymbolValue) maxSymbolValue = HUF_SYMBOLVALUE_MAX; return 0; /* cannot fit within dst budget */
if (!huffLog) huffLog = HUF_TABLELOG_DEFAULT; if (srcSize > HUF_BLOCKSIZE_MAX)
return ERROR(srcSize_wrong); /* curr block size limit */
if (huffLog > HUF_TABLELOG_MAX)
return ERROR(tableLog_tooLarge);
if (!maxSymbolValue)
maxSymbolValue = HUF_SYMBOLVALUE_MAX;
if (!huffLog)
huffLog = HUF_TABLELOG_DEFAULT;
count = (U32 *)workSpace; count = (U32 *)workSpace;
workSpace = (BYTE *)workSpace + countSize; workSpace = (BYTE *)workSpace + countSize;
@@ -572,9 +651,14 @@ static size_t HUF_compress_internal (
} }
/* Scan input and build symbol stats */ /* Scan input and build symbol stats */
{ CHECK_V_F(largest, FSE_count_wksp (count, &maxSymbolValue, (const BYTE*)src, srcSize, (U32*)workSpace) ); {
if (largest == srcSize) { *ostart = ((const BYTE*)src)[0]; return 1; } /* single symbol, rle */ CHECK_V_F(largest, FSE_count_wksp(count, &maxSymbolValue, (const BYTE *)src, srcSize, (U32 *)workSpace));
if (largest <= (srcSize >> 7)+1) return 0; /* Fast heuristic : not compressible enough */ if (largest == srcSize) {
*ostart = ((const BYTE *)src)[0];
return 1;
} /* single symbol, rle */
if (largest <= (srcSize >> 7) + 1)
return 0; /* Fast heuristic : not compressible enough */
} }
/* Check validity of previous table */ /* Check validity of previous table */
@@ -588,14 +672,16 @@ static size_t HUF_compress_internal (
/* Build Huffman Tree */ /* Build Huffman Tree */
huffLog = HUF_optimalTableLog(huffLog, srcSize, maxSymbolValue); huffLog = HUF_optimalTableLog(huffLog, srcSize, maxSymbolValue);
{ CHECK_V_F(maxBits, HUF_buildCTable_wksp (CTable, count, maxSymbolValue, huffLog, workSpace, wkspSize) ); {
CHECK_V_F(maxBits, HUF_buildCTable_wksp(CTable, count, maxSymbolValue, huffLog, workSpace, wkspSize));
huffLog = (U32)maxBits; huffLog = (U32)maxBits;
/* Zero the unused symbols so we can check it for validity */ /* Zero the unused symbols so we can check it for validity */
memset(CTable + maxSymbolValue + 1, 0, CTableSize - (maxSymbolValue + 1) * sizeof(HUF_CElt)); memset(CTable + maxSymbolValue + 1, 0, CTableSize - (maxSymbolValue + 1) * sizeof(HUF_CElt));
} }
/* Write table description header */ /* Write table description header */
{ CHECK_V_F(hSize, HUF_writeCTable (op, dstSize, CTable, maxSymbolValue, huffLog) ); {
CHECK_V_F(hSize, HUF_writeCTable(op, dstSize, CTable, maxSymbolValue, huffLog));
/* Check if using the previous table will be beneficial */ /* Check if using the previous table will be beneficial */
if (repeat && *repeat != HUF_repeat_none) { if (repeat && *repeat != HUF_repeat_none) {
size_t const oldSize = HUF_estimateCompressedSize(oldHufTable, count, maxSymbolValue); size_t const oldSize = HUF_estimateCompressedSize(oldHufTable, count, maxSymbolValue);
@@ -605,45 +691,42 @@ static size_t HUF_compress_internal (
} }
} }
/* Use the new table */ /* Use the new table */
if (hSize + 12ul >= srcSize) { return 0; } if (hSize + 12ul >= srcSize) {
return 0;
}
op += hSize; op += hSize;
if (repeat) { *repeat = HUF_repeat_none; } if (repeat) {
if (oldHufTable) { memcpy(oldHufTable, CTable, CTableSize); } /* Save the new table */ *repeat = HUF_repeat_none;
}
if (oldHufTable) {
memcpy(oldHufTable, CTable, CTableSize);
} /* Save the new table */
} }
return HUF_compressCTable_internal(ostart, op, oend, src, srcSize, singleStream, CTable); return HUF_compressCTable_internal(ostart, op, oend, src, srcSize, singleStream, CTable);
} }
size_t HUF_compress1X_wksp(void *dst, size_t dstSize, const void *src, size_t srcSize, unsigned maxSymbolValue, unsigned huffLog, void *workSpace,
size_t HUF_compress1X_wksp (void* dst, size_t dstSize, size_t wkspSize)
const void* src, size_t srcSize,
unsigned maxSymbolValue, unsigned huffLog,
void* workSpace, size_t wkspSize)
{ {
return HUF_compress_internal(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, 1 /* single stream */, workSpace, wkspSize, NULL, NULL, 0); return HUF_compress_internal(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, 1 /* single stream */, workSpace, wkspSize, NULL, NULL, 0);
} }
size_t HUF_compress1X_repeat (void* dst, size_t dstSize, size_t HUF_compress1X_repeat(void *dst, size_t dstSize, const void *src, size_t srcSize, unsigned maxSymbolValue, unsigned huffLog, void *workSpace,
const void* src, size_t srcSize, size_t wkspSize, HUF_CElt *hufTable, HUF_repeat *repeat, int preferRepeat)
unsigned maxSymbolValue, unsigned huffLog,
void* workSpace, size_t wkspSize,
HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat)
{ {
return HUF_compress_internal(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, 1 /* single stream */, workSpace, wkspSize, hufTable, repeat, preferRepeat); return HUF_compress_internal(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, 1 /* single stream */, workSpace, wkspSize, hufTable, repeat,
preferRepeat);
} }
size_t HUF_compress4X_wksp (void* dst, size_t dstSize, size_t HUF_compress4X_wksp(void *dst, size_t dstSize, const void *src, size_t srcSize, unsigned maxSymbolValue, unsigned huffLog, void *workSpace,
const void* src, size_t srcSize, size_t wkspSize)
unsigned maxSymbolValue, unsigned huffLog,
void* workSpace, size_t wkspSize)
{ {
return HUF_compress_internal(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, 0 /* 4 streams */, workSpace, wkspSize, NULL, NULL, 0); return HUF_compress_internal(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, 0 /* 4 streams */, workSpace, wkspSize, NULL, NULL, 0);
} }
size_t HUF_compress4X_repeat (void* dst, size_t dstSize, size_t HUF_compress4X_repeat(void *dst, size_t dstSize, const void *src, size_t srcSize, unsigned maxSymbolValue, unsigned huffLog, void *workSpace,
const void* src, size_t srcSize, size_t wkspSize, HUF_CElt *hufTable, HUF_repeat *repeat, int preferRepeat)
unsigned maxSymbolValue, unsigned huffLog,
void* workSpace, size_t wkspSize,
HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat)
{ {
return HUF_compress_internal(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, 0 /* 4 streams */, workSpace, wkspSize, hufTable, repeat, preferRepeat); return HUF_compress_internal(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, 0 /* 4 streams */, workSpace, wkspSize, hufTable, repeat,
preferRepeat);
} }
contrib/linux-kernel/lib/zstd/huf_decompress.c
+271 -190
View File
@@ -42,28 +42,33 @@
****************************************************************/ ****************************************************************/
#define FORCE_INLINE static __always_inline #define FORCE_INLINE static __always_inline
/* ************************************************************** /* **************************************************************
* Dependencies * Dependencies
****************************************************************/ ****************************************************************/
#include <linux/compiler.h>
#include <linux/string.h> /* memcpy, memset */
#include "bitstream.h" /* BIT_* */ #include "bitstream.h" /* BIT_* */
#include "fse.h" /* header compression */ #include "fse.h" /* header compression */
#include "huf.h" #include "huf.h"
#include <linux/compiler.h>
#include <linux/string.h> /* memcpy, memset */
/* ************************************************************** /* **************************************************************
* Error Management * Error Management
****************************************************************/ ****************************************************************/
#define HUF_STATIC_ASSERT(c) { enum { HUF_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */ #define HUF_STATIC_ASSERT(c) \
{ \
enum { HUF_static_assert = 1 / (int)(!!(c)) }; \
} /* use only *after* variable declarations */
/*-***************************/ /*-***************************/
/* generic DTableDesc */ /* generic DTableDesc */
/*-***************************/ /*-***************************/
typedef struct { BYTE maxTableLog; BYTE tableType; BYTE tableLog; BYTE reserved; } DTableDesc; typedef struct {
BYTE maxTableLog;
BYTE tableType;
BYTE tableLog;
BYTE reserved;
} DTableDesc;
static DTableDesc HUF_getDTableDesc(const HUF_DTable *table) static DTableDesc HUF_getDTableDesc(const HUF_DTable *table)
{ {
@@ -72,12 +77,14 @@ static DTableDesc HUF_getDTableDesc(const HUF_DTable* table)
return dtd; return dtd;
} }
/*-***************************/ /*-***************************/
/* single-symbol decoding */ /* single-symbol decoding */
/*-***************************/ /*-***************************/
typedef struct { BYTE byte; BYTE nbBits; } HUF_DEltX2; /* single-symbol decoding */ typedef struct {
BYTE byte;
BYTE nbBits;
} HUF_DEltX2; /* single-symbol decoding */
size_t HUF_readDTableX2(HUF_DTable *DTable, const void *src, size_t srcSize) size_t HUF_readDTableX2(HUF_DTable *DTable, const void *src, size_t srcSize)
{ {
@@ -93,41 +100,48 @@ size_t HUF_readDTableX2 (HUF_DTable* DTable, const void* src, size_t srcSize)
/* memset(huffWeight, 0, sizeof(huffWeight)); */ /* is not necessary, even though some analyzer complain ... */ /* memset(huffWeight, 0, sizeof(huffWeight)); */ /* is not necessary, even though some analyzer complain ... */
iSize = HUF_readStats(huffWeight, HUF_SYMBOLVALUE_MAX + 1, rankVal, &nbSymbols, &tableLog, src, srcSize); iSize = HUF_readStats(huffWeight, HUF_SYMBOLVALUE_MAX + 1, rankVal, &nbSymbols, &tableLog, src, srcSize);
if (HUF_isError(iSize)) return iSize; if (HUF_isError(iSize))
return iSize;
/* Table header */ /* Table header */
{ DTableDesc dtd = HUF_getDTableDesc(DTable); {
if (tableLog > (U32)(dtd.maxTableLog+1)) return ERROR(tableLog_tooLarge); /* DTable too small, Huffman tree cannot fit in */ DTableDesc dtd = HUF_getDTableDesc(DTable);
if (tableLog > (U32)(dtd.maxTableLog + 1))
return ERROR(tableLog_tooLarge); /* DTable too small, Huffman tree cannot fit in */
dtd.tableType = 0; dtd.tableType = 0;
dtd.tableLog = (BYTE)tableLog; dtd.tableLog = (BYTE)tableLog;
memcpy(DTable, &dtd, sizeof(dtd)); memcpy(DTable, &dtd, sizeof(dtd));
} }
/* Calculate starting value for each rank */ /* Calculate starting value for each rank */
{ U32 n, nextRankStart = 0; {
U32 n, nextRankStart = 0;
for (n = 1; n < tableLog + 1; n++) { for (n = 1; n < tableLog + 1; n++) {
U32 const curr = nextRankStart; U32 const curr = nextRankStart;
nextRankStart += (rankVal[n] << (n - 1)); nextRankStart += (rankVal[n] << (n - 1));
rankVal[n] = curr; rankVal[n] = curr;
} } }
}
/* fill DTable */ /* fill DTable */
{ U32 n; {
U32 n;
for (n = 0; n < nbSymbols; n++) { for (n = 0; n < nbSymbols; n++) {
U32 const w = huffWeight[n]; U32 const w = huffWeight[n];
U32 const length = (1 << w) >> 1; U32 const length = (1 << w) >> 1;
U32 u; U32 u;
HUF_DEltX2 D; HUF_DEltX2 D;
D.byte = (BYTE)n; D.nbBits = (BYTE)(tableLog + 1 - w); D.byte = (BYTE)n;
D.nbBits = (BYTE)(tableLog + 1 - w);
for (u = rankVal[w]; u < rankVal[w] + length; u++) for (u = rankVal[w]; u < rankVal[w] + length; u++)
dt[u] = D; dt[u] = D;
rankVal[w] += length; rankVal[w] += length;
} } }
}
return iSize; return iSize;
} }
static BYTE HUF_decodeSymbolX2(BIT_DStream_t *Dstream, const HUF_DEltX2 *dt, const U32 dtLog) static BYTE HUF_decodeSymbolX2(BIT_DStream_t *Dstream, const HUF_DEltX2 *dt, const U32 dtLog)
{ {
size_t const val = BIT_lookBitsFast(Dstream, dtLog); /* note : dtLog >= 1 */ size_t const val = BIT_lookBitsFast(Dstream, dtLog); /* note : dtLog >= 1 */
@@ -136,8 +150,7 @@ static BYTE HUF_decodeSymbolX2(BIT_DStream_t* Dstream, const HUF_DEltX2* dt, con
return c; return c;
} }
#define HUF_DECODE_SYMBOLX2_0(ptr, DStreamPtr) \ #define HUF_DECODE_SYMBOLX2_0(ptr, DStreamPtr) *ptr++ = HUF_decodeSymbolX2(DStreamPtr, dt, dtLog)
*ptr++ = HUF_decodeSymbolX2(DStreamPtr, dt, dtLog)
#define HUF_DECODE_SYMBOLX2_1(ptr, DStreamPtr) \ #define HUF_DECODE_SYMBOLX2_1(ptr, DStreamPtr) \
if (ZSTD_64bits() || (HUF_TABLELOG_MAX <= 12)) \ if (ZSTD_64bits() || (HUF_TABLELOG_MAX <= 12)) \
@@ -170,10 +183,7 @@ FORCE_INLINE size_t HUF_decodeStreamX2(BYTE* p, BIT_DStream_t* const bitDPtr, BY
return pEnd - pStart; return pEnd - pStart;
} }
static size_t HUF_decompress1X2_usingDTable_internal( static size_t HUF_decompress1X2_usingDTable_internal(void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize, const HUF_DTable *DTable)
void* dst, size_t dstSize,
const void* cSrc, size_t cSrcSize,
const HUF_DTable* DTable)
{ {
BYTE *op = (BYTE *)dst; BYTE *op = (BYTE *)dst;
BYTE *const oend = op + dstSize; BYTE *const oend = op + dstSize;
@@ -183,24 +193,26 @@ static size_t HUF_decompress1X2_usingDTable_internal(
DTableDesc const dtd = HUF_getDTableDesc(DTable); DTableDesc const dtd = HUF_getDTableDesc(DTable);
U32 const dtLog = dtd.tableLog; U32 const dtLog = dtd.tableLog;
{ size_t const errorCode = BIT_initDStream(&bitD, cSrc, cSrcSize); {
if (HUF_isError(errorCode)) return errorCode; } size_t const errorCode = BIT_initDStream(&bitD, cSrc, cSrcSize);
if (HUF_isError(errorCode))
return errorCode;
}
HUF_decodeStreamX2(op, &bitD, oend, dt, dtLog); HUF_decodeStreamX2(op, &bitD, oend, dt, dtLog);
/* check */ /* check */
if (!BIT_endOfDStream(&bitD)) return ERROR(corruption_detected); if (!BIT_endOfDStream(&bitD))
return ERROR(corruption_detected);
return dstSize; return dstSize;
} }
size_t HUF_decompress1X2_usingDTable( size_t HUF_decompress1X2_usingDTable(void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize, const HUF_DTable *DTable)
void* dst, size_t dstSize,
const void* cSrc, size_t cSrcSize,
const HUF_DTable* DTable)
{ {
DTableDesc dtd = HUF_getDTableDesc(DTable); DTableDesc dtd = HUF_getDTableDesc(DTable);
if (dtd.tableType != 0) return ERROR(GENERIC); if (dtd.tableType != 0)
return ERROR(GENERIC);
return HUF_decompress1X2_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable); return HUF_decompress1X2_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable);
} }
@@ -209,23 +221,24 @@ size_t HUF_decompress1X2_DCtx (HUF_DTable* DCtx, void* dst, size_t dstSize, cons
const BYTE *ip = (const BYTE *)cSrc; const BYTE *ip = (const BYTE *)cSrc;
size_t const hSize = HUF_readDTableX2(DCtx, cSrc, cSrcSize); size_t const hSize = HUF_readDTableX2(DCtx, cSrc, cSrcSize);
if (HUF_isError(hSize)) return hSize; if (HUF_isError(hSize))
if (hSize >= cSrcSize) return ERROR(srcSize_wrong); return hSize;
ip += hSize; cSrcSize -= hSize; if (hSize >= cSrcSize)
return ERROR(srcSize_wrong);
ip += hSize;
cSrcSize -= hSize;
return HUF_decompress1X2_usingDTable_internal(dst, dstSize, ip, cSrcSize, DCtx); return HUF_decompress1X2_usingDTable_internal(dst, dstSize, ip, cSrcSize, DCtx);
} }
static size_t HUF_decompress4X2_usingDTable_internal(void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize, const HUF_DTable *DTable)
static size_t HUF_decompress4X2_usingDTable_internal(
void* dst, size_t dstSize,
const void* cSrc, size_t cSrcSize,
const HUF_DTable* DTable)
{ {
/* Check */ /* Check */
if (cSrcSize < 10) return ERROR(corruption_detected); /* strict minimum : jump table + 1 byte per stream */ if (cSrcSize < 10)
return ERROR(corruption_detected); /* strict minimum : jump table + 1 byte per stream */
{ const BYTE* const istart = (const BYTE*) cSrc; {
const BYTE *const istart = (const BYTE *)cSrc;
BYTE *const ostart = (BYTE *)dst; BYTE *const ostart = (BYTE *)dst;
BYTE *const oend = ostart + dstSize; BYTE *const oend = ostart + dstSize;
const void *const dtPtr = DTable + 1; const void *const dtPtr = DTable + 1;
@@ -256,15 +269,28 @@ static size_t HUF_decompress4X2_usingDTable_internal(
DTableDesc const dtd = HUF_getDTableDesc(DTable); DTableDesc const dtd = HUF_getDTableDesc(DTable);
U32 const dtLog = dtd.tableLog; U32 const dtLog = dtd.tableLog;
if (length4 > cSrcSize) return ERROR(corruption_detected); /* overflow */ if (length4 > cSrcSize)
{ size_t const errorCode = BIT_initDStream(&bitD1, istart1, length1); return ERROR(corruption_detected); /* overflow */
if (HUF_isError(errorCode)) return errorCode; } {
{ size_t const errorCode = BIT_initDStream(&bitD2, istart2, length2); size_t const errorCode = BIT_initDStream(&bitD1, istart1, length1);
if (HUF_isError(errorCode)) return errorCode; } if (HUF_isError(errorCode))
{ size_t const errorCode = BIT_initDStream(&bitD3, istart3, length3); return errorCode;
if (HUF_isError(errorCode)) return errorCode; } }
{ size_t const errorCode = BIT_initDStream(&bitD4, istart4, length4); {
if (HUF_isError(errorCode)) return errorCode; } size_t const errorCode = BIT_initDStream(&bitD2, istart2, length2);
if (HUF_isError(errorCode))
return errorCode;
}
{
size_t const errorCode = BIT_initDStream(&bitD3, istart3, length3);
if (HUF_isError(errorCode))
return errorCode;
}
{
size_t const errorCode = BIT_initDStream(&bitD4, istart4, length4);
if (HUF_isError(errorCode))
return errorCode;
}
/* 16-32 symbols per loop (4-8 symbols per stream) */ /* 16-32 symbols per loop (4-8 symbols per stream) */
endSignal = BIT_reloadDStream(&bitD1) | BIT_reloadDStream(&bitD2) | BIT_reloadDStream(&bitD3) | BIT_reloadDStream(&bitD4); endSignal = BIT_reloadDStream(&bitD1) | BIT_reloadDStream(&bitD2) | BIT_reloadDStream(&bitD3) | BIT_reloadDStream(&bitD4);
@@ -289,9 +315,12 @@ static size_t HUF_decompress4X2_usingDTable_internal(
} }
/* check corruption */ /* check corruption */
if (op1 > opStart2) return ERROR(corruption_detected); if (op1 > opStart2)
if (op2 > opStart3) return ERROR(corruption_detected); return ERROR(corruption_detected);
if (op3 > opStart4) return ERROR(corruption_detected); if (op2 > opStart3)
return ERROR(corruption_detected);
if (op3 > opStart4)
return ERROR(corruption_detected);
/* note : op4 supposed already verified within main loop */ /* note : op4 supposed already verified within main loop */
/* finish bitStreams one by one */ /* finish bitStreams one by one */
@@ -302,33 +331,33 @@ static size_t HUF_decompress4X2_usingDTable_internal(
/* check */ /* check */
endSignal = BIT_endOfDStream(&bitD1) & BIT_endOfDStream(&bitD2) & BIT_endOfDStream(&bitD3) & BIT_endOfDStream(&bitD4); endSignal = BIT_endOfDStream(&bitD1) & BIT_endOfDStream(&bitD2) & BIT_endOfDStream(&bitD3) & BIT_endOfDStream(&bitD4);
if (!endSignal) return ERROR(corruption_detected); if (!endSignal)
return ERROR(corruption_detected);
/* decoded size */ /* decoded size */
return dstSize; return dstSize;
} }
} }
size_t HUF_decompress4X2_usingDTable(void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize, const HUF_DTable *DTable)
size_t HUF_decompress4X2_usingDTable(
void* dst, size_t dstSize,
const void* cSrc, size_t cSrcSize,
const HUF_DTable* DTable)
{ {
DTableDesc dtd = HUF_getDTableDesc(DTable); DTableDesc dtd = HUF_getDTableDesc(DTable);
if (dtd.tableType != 0) return ERROR(GENERIC); if (dtd.tableType != 0)
return ERROR(GENERIC);
return HUF_decompress4X2_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable); return HUF_decompress4X2_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable);
} }
size_t HUF_decompress4X2_DCtx(HUF_DTable *dctx, void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize) size_t HUF_decompress4X2_DCtx(HUF_DTable *dctx, void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize)
{ {
const BYTE *ip = (const BYTE *)cSrc; const BYTE *ip = (const BYTE *)cSrc;
size_t const hSize = HUF_readDTableX2(dctx, cSrc, cSrcSize); size_t const hSize = HUF_readDTableX2(dctx, cSrc, cSrcSize);
if (HUF_isError(hSize)) return hSize; if (HUF_isError(hSize))
if (hSize >= cSrcSize) return ERROR(srcSize_wrong); return hSize;
ip += hSize; cSrcSize -= hSize; if (hSize >= cSrcSize)
return ERROR(srcSize_wrong);
ip += hSize;
cSrcSize -= hSize;
return HUF_decompress4X2_usingDTable_internal(dst, dstSize, ip, cSrcSize, dctx); return HUF_decompress4X2_usingDTable_internal(dst, dstSize, ip, cSrcSize, dctx);
} }
@@ -336,16 +365,21 @@ size_t HUF_decompress4X2_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, cons
/* *************************/ /* *************************/
/* double-symbols decoding */ /* double-symbols decoding */
/* *************************/ /* *************************/
typedef struct { U16 sequence; BYTE nbBits; BYTE length; } HUF_DEltX4; /* double-symbols decoding */ typedef struct {
U16 sequence;
BYTE nbBits;
BYTE length;
} HUF_DEltX4; /* double-symbols decoding */
typedef struct { BYTE symbol; BYTE weight; } sortedSymbol_t; typedef struct {
BYTE symbol;
BYTE weight;
} sortedSymbol_t;
/* HUF_fillDTableX4Level2() : /* HUF_fillDTableX4Level2() :
* `rankValOrigin` must be a table of at least (HUF_TABLELOG_MAX + 1) U32 */ * `rankValOrigin` must be a table of at least (HUF_TABLELOG_MAX + 1) U32 */
static void HUF_fillDTableX4Level2(HUF_DEltX4* DTable, U32 sizeLog, const U32 consumed, static void HUF_fillDTableX4Level2(HUF_DEltX4 *DTable, U32 sizeLog, const U32 consumed, const U32 *rankValOrigin, const int minWeight,
const U32* rankValOrigin, const int minWeight, const sortedSymbol_t *sortedSymbols, const U32 sortedListSize, U32 nbBitsBaseline, U16 baseSeq)
const sortedSymbol_t* sortedSymbols, const U32 sortedListSize,
U32 nbBitsBaseline, U16 baseSeq)
{ {
HUF_DEltX4 DElt; HUF_DEltX4 DElt;
U32 rankVal[HUF_TABLELOG_MAX + 1]; U32 rankVal[HUF_TABLELOG_MAX + 1];
@@ -364,7 +398,9 @@ static void HUF_fillDTableX4Level2(HUF_DEltX4* DTable, U32 sizeLog, const U32 co
} }
/* fill DTable */ /* fill DTable */
{ U32 s; for (s=0; s<sortedListSize; s++) { /* note : sortedSymbols already skipped */ {
U32 s;
for (s = 0; s < sortedListSize; s++) { /* note : sortedSymbols already skipped */
const U32 symbol = sortedSymbols[s].symbol; const U32 symbol = sortedSymbols[s].symbol;
const U32 weight = sortedSymbols[s].weight; const U32 weight = sortedSymbols[s].weight;
const U32 nbBits = nbBitsBaseline - weight; const U32 nbBits = nbBitsBaseline - weight;
@@ -376,18 +412,19 @@ static void HUF_fillDTableX4Level2(HUF_DEltX4* DTable, U32 sizeLog, const U32 co
ZSTD_writeLE16(&(DElt.sequence), (U16)(baseSeq + (symbol << 8))); ZSTD_writeLE16(&(DElt.sequence), (U16)(baseSeq + (symbol << 8)));
DElt.nbBits = (BYTE)(nbBits + consumed); DElt.nbBits = (BYTE)(nbBits + consumed);
DElt.length = 2; DElt.length = 2;
do { DTable[i++] = DElt; } while (i<end); /* since length >= 1 */ do {
DTable[i++] = DElt;
} while (i < end); /* since length >= 1 */
rankVal[weight] += length; rankVal[weight] += length;
} } }
}
} }
typedef U32 rankVal_t[HUF_TABLELOG_MAX][HUF_TABLELOG_MAX + 1]; typedef U32 rankVal_t[HUF_TABLELOG_MAX][HUF_TABLELOG_MAX + 1];
static void HUF_fillDTableX4(HUF_DEltX4* DTable, const U32 targetLog, static void HUF_fillDTableX4(HUF_DEltX4 *DTable, const U32 targetLog, const sortedSymbol_t *sortedList, const U32 sortedListSize, const U32 *rankStart,
const sortedSymbol_t* sortedList, const U32 sortedListSize, rankVal_t rankValOrigin, const U32 maxWeight, const U32 nbBitsBaseline)
const U32* rankStart, rankVal_t rankValOrigin, const U32 maxWeight,
const U32 nbBitsBaseline)
{ {
U32 rankVal[HUF_TABLELOG_MAX + 1]; U32 rankVal[HUF_TABLELOG_MAX + 1];
const int scaleLog = nbBitsBaseline - targetLog; /* note : targetLog >= srcLog, hence scaleLog <= 1 */ const int scaleLog = nbBitsBaseline - targetLog; /* note : targetLog >= srcLog, hence scaleLog <= 1 */
@@ -407,21 +444,23 @@ static void HUF_fillDTableX4(HUF_DEltX4* DTable, const U32 targetLog,
if (targetLog - nbBits >= minBits) { /* enough room for a second symbol */ if (targetLog - nbBits >= minBits) { /* enough room for a second symbol */
U32 sortedRank; U32 sortedRank;
int minWeight = nbBits + scaleLog; int minWeight = nbBits + scaleLog;
if (minWeight < 1) minWeight = 1; if (minWeight < 1)
minWeight = 1;
sortedRank = rankStart[minWeight]; sortedRank = rankStart[minWeight];
HUF_fillDTableX4Level2(DTable+start, targetLog-nbBits, nbBits, HUF_fillDTableX4Level2(DTable + start, targetLog - nbBits, nbBits, rankValOrigin[nbBits], minWeight, sortedList + sortedRank,
rankValOrigin[nbBits], minWeight, sortedListSize - sortedRank, nbBitsBaseline, symbol);
sortedList+sortedRank, sortedListSize-sortedRank,
nbBitsBaseline, symbol);
} else { } else {
HUF_DEltX4 DElt; HUF_DEltX4 DElt;
ZSTD_writeLE16(&(DElt.sequence), symbol); ZSTD_writeLE16(&(DElt.sequence), symbol);
DElt.nbBits = (BYTE)(nbBits); DElt.nbBits = (BYTE)(nbBits);
DElt.length = 1; DElt.length = 1;
{ U32 const end = start + length; {
U32 const end = start + length;
U32 u; U32 u;
for (u = start; u < end; u++) DTable[u] = DElt; for (u = start; u < end; u++)
} } DTable[u] = DElt;
}
}
rankVal[weight] += length; rankVal[weight] += length;
} }
} }
@@ -442,20 +481,25 @@ size_t HUF_readDTableX4 (HUF_DTable* DTable, const void* src, size_t srcSize)
HUF_DEltX4 *const dt = (HUF_DEltX4 *)dtPtr; HUF_DEltX4 *const dt = (HUF_DEltX4 *)dtPtr;
HUF_STATIC_ASSERT(sizeof(HUF_DEltX4) == sizeof(HUF_DTable)); /* if compiler fails here, assertion is wrong */ HUF_STATIC_ASSERT(sizeof(HUF_DEltX4) == sizeof(HUF_DTable)); /* if compiler fails here, assertion is wrong */
if (maxTableLog > HUF_TABLELOG_MAX) return ERROR(tableLog_tooLarge); if (maxTableLog > HUF_TABLELOG_MAX)
return ERROR(tableLog_tooLarge);
/* memset(weightList, 0, sizeof(weightList)); */ /* is not necessary, even though some analyzer complain ... */ /* memset(weightList, 0, sizeof(weightList)); */ /* is not necessary, even though some analyzer complain ... */
iSize = HUF_readStats(weightList, HUF_SYMBOLVALUE_MAX + 1, rankStats, &nbSymbols, &tableLog, src, srcSize); iSize = HUF_readStats(weightList, HUF_SYMBOLVALUE_MAX + 1, rankStats, &nbSymbols, &tableLog, src, srcSize);
if (HUF_isError(iSize)) return iSize; if (HUF_isError(iSize))
return iSize;
/* check result */ /* check result */
if (tableLog > maxTableLog) return ERROR(tableLog_tooLarge); /* DTable can't fit code depth */ if (tableLog > maxTableLog)
return ERROR(tableLog_tooLarge); /* DTable can't fit code depth */
/* find maxWeight */ /* find maxWeight */
for (maxW = tableLog; rankStats[maxW]==0; maxW--) {} /* necessarily finds a solution before 0 */ for (maxW = tableLog; rankStats[maxW] == 0; maxW--) {
} /* necessarily finds a solution before 0 */
/* Get start index of each weight */ /* Get start index of each weight */
{ U32 w, nextRankStart = 0; {
U32 w, nextRankStart = 0;
for (w = 1; w < maxW + 1; w++) { for (w = 1; w < maxW + 1; w++) {
U32 curr = nextRankStart; U32 curr = nextRankStart;
nextRankStart += rankStats[w]; nextRankStart += rankStats[w];
@@ -466,7 +510,8 @@ size_t HUF_readDTableX4 (HUF_DTable* DTable, const void* src, size_t srcSize)
} }
/* sort symbols by weight */ /* sort symbols by weight */
{ U32 s; {
U32 s;
for (s = 0; s < nbSymbols; s++) { for (s = 0; s < nbSymbols; s++) {
U32 const w = weightList[s]; U32 const w = weightList[s];
U32 const r = rankStart[w]++; U32 const r = rankStart[w]++;
@@ -477,28 +522,32 @@ size_t HUF_readDTableX4 (HUF_DTable* DTable, const void* src, size_t srcSize)
} }
/* Build rankVal */ /* Build rankVal */
{ U32* const rankVal0 = rankVal[0]; {
{ int const rescale = (maxTableLog-tableLog) - 1; /* tableLog <= maxTableLog */ U32 *const rankVal0 = rankVal[0];
{
int const rescale = (maxTableLog - tableLog) - 1; /* tableLog <= maxTableLog */
U32 nextRankVal = 0; U32 nextRankVal = 0;
U32 w; U32 w;
for (w = 1; w < maxW + 1; w++) { for (w = 1; w < maxW + 1; w++) {
U32 curr = nextRankVal; U32 curr = nextRankVal;
nextRankVal += rankStats[w] << (w + rescale); nextRankVal += rankStats[w] << (w + rescale);
rankVal0[w] = curr; rankVal0[w] = curr;
} } }
{ U32 const minBits = tableLog+1 - maxW; }
{
U32 const minBits = tableLog + 1 - maxW;
U32 consumed; U32 consumed;
for (consumed = minBits; consumed < maxTableLog - minBits + 1; consumed++) { for (consumed = minBits; consumed < maxTableLog - minBits + 1; consumed++) {
U32 *const rankValPtr = rankVal[consumed]; U32 *const rankValPtr = rankVal[consumed];
U32 w; U32 w;
for (w = 1; w < maxW + 1; w++) { for (w = 1; w < maxW + 1; w++) {
rankValPtr[w] = rankVal0[w] >> consumed; rankValPtr[w] = rankVal0[w] >> consumed;
} } } } }
}
}
}
HUF_fillDTableX4(dt, maxTableLog, HUF_fillDTableX4(dt, maxTableLog, sortedSymbol, sizeOfSort, rankStart0, rankVal, maxW, tableLog + 1);
sortedSymbol, sizeOfSort,
rankStart0, rankVal, maxW,
tableLog+1);
dtd.tableLog = (BYTE)maxTableLog; dtd.tableLog = (BYTE)maxTableLog;
dtd.tableType = 1; dtd.tableType = 1;
@@ -506,7 +555,6 @@ size_t HUF_readDTableX4 (HUF_DTable* DTable, const void* src, size_t srcSize)
return iSize; return iSize;
} }
static U32 HUF_decodeSymbolX4(void *op, BIT_DStream_t *DStream, const HUF_DEltX4 *dt, const U32 dtLog) static U32 HUF_decodeSymbolX4(void *op, BIT_DStream_t *DStream, const HUF_DEltX4 *dt, const U32 dtLog)
{ {
size_t const val = BIT_lookBitsFast(DStream, dtLog); /* note : dtLog >= 1 */ size_t const val = BIT_lookBitsFast(DStream, dtLog); /* note : dtLog >= 1 */
@@ -519,19 +567,21 @@ static U32 HUF_decodeLastSymbolX4(void* op, BIT_DStream_t* DStream, const HUF_DE
{ {
size_t const val = BIT_lookBitsFast(DStream, dtLog); /* note : dtLog >= 1 */ size_t const val = BIT_lookBitsFast(DStream, dtLog); /* note : dtLog >= 1 */
memcpy(op, dt + val, 1); memcpy(op, dt + val, 1);
if (dt[val].length==1) BIT_skipBits(DStream, dt[val].nbBits); if (dt[val].length == 1)
BIT_skipBits(DStream, dt[val].nbBits);
else { else {
if (DStream->bitsConsumed < (sizeof(DStream->bitContainer) * 8)) { if (DStream->bitsConsumed < (sizeof(DStream->bitContainer) * 8)) {
BIT_skipBits(DStream, dt[val].nbBits); BIT_skipBits(DStream, dt[val].nbBits);
if (DStream->bitsConsumed > (sizeof(DStream->bitContainer) * 8)) if (DStream->bitsConsumed > (sizeof(DStream->bitContainer) * 8))
DStream->bitsConsumed = (sizeof(DStream->bitContainer)*8); /* ugly hack; works only because it's the last symbol. Note : can't easily extract nbBits from just this symbol */ DStream->bitsConsumed =
} } (sizeof(DStream->bitContainer) *
8); /* ugly hack; works only because it's the last symbol. Note : can't easily extract nbBits from just this symbol */
}
}
return 1; return 1;
} }
#define HUF_DECODE_SYMBOLX4_0(ptr, DStreamPtr) ptr += HUF_decodeSymbolX4(ptr, DStreamPtr, dt, dtLog)
#define HUF_DECODE_SYMBOLX4_0(ptr, DStreamPtr) \
ptr += HUF_decodeSymbolX4(ptr, DStreamPtr, dt, dtLog)
#define HUF_DECODE_SYMBOLX4_1(ptr, DStreamPtr) \ #define HUF_DECODE_SYMBOLX4_1(ptr, DStreamPtr) \
if (ZSTD_64bits() || (HUF_TABLELOG_MAX <= 12)) \ if (ZSTD_64bits() || (HUF_TABLELOG_MAX <= 12)) \
@@ -566,21 +616,20 @@ FORCE_INLINE size_t HUF_decodeStreamX4(BYTE* p, BIT_DStream_t* bitDPtr, BYTE* co
return p - pStart; return p - pStart;
} }
static size_t HUF_decompress1X4_usingDTable_internal(void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize, const HUF_DTable *DTable)
static size_t HUF_decompress1X4_usingDTable_internal(
void* dst, size_t dstSize,
const void* cSrc, size_t cSrcSize,
const HUF_DTable* DTable)
{ {
BIT_DStream_t bitD; BIT_DStream_t bitD;
/* Init */ /* Init */
{ size_t const errorCode = BIT_initDStream(&bitD, cSrc, cSrcSize); {
if (HUF_isError(errorCode)) return errorCode; size_t const errorCode = BIT_initDStream(&bitD, cSrc, cSrcSize);
if (HUF_isError(errorCode))
return errorCode;
} }
/* decode */ /* decode */
{ BYTE* const ostart = (BYTE*) dst; {
BYTE *const ostart = (BYTE *)dst;
BYTE *const oend = ostart + dstSize; BYTE *const oend = ostart + dstSize;
const void *const dtPtr = DTable + 1; /* force compiler to not use strict-aliasing */ const void *const dtPtr = DTable + 1; /* force compiler to not use strict-aliasing */
const HUF_DEltX4 *const dt = (const HUF_DEltX4 *)dtPtr; const HUF_DEltX4 *const dt = (const HUF_DEltX4 *)dtPtr;
@@ -589,19 +638,18 @@ static size_t HUF_decompress1X4_usingDTable_internal(
} }
/* check */ /* check */
if (!BIT_endOfDStream(&bitD)) return ERROR(corruption_detected); if (!BIT_endOfDStream(&bitD))
return ERROR(corruption_detected);
/* decoded size */ /* decoded size */
return dstSize; return dstSize;
} }
size_t HUF_decompress1X4_usingDTable( size_t HUF_decompress1X4_usingDTable(void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize, const HUF_DTable *DTable)
void* dst, size_t dstSize,
const void* cSrc, size_t cSrcSize,
const HUF_DTable* DTable)
{ {
DTableDesc dtd = HUF_getDTableDesc(DTable); DTableDesc dtd = HUF_getDTableDesc(DTable);
if (dtd.tableType != 1) return ERROR(GENERIC); if (dtd.tableType != 1)
return ERROR(GENERIC);
return HUF_decompress1X4_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable); return HUF_decompress1X4_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable);
} }
@@ -610,21 +658,23 @@ size_t HUF_decompress1X4_DCtx (HUF_DTable* DCtx, void* dst, size_t dstSize, cons
const BYTE *ip = (const BYTE *)cSrc; const BYTE *ip = (const BYTE *)cSrc;
size_t const hSize = HUF_readDTableX4(DCtx, cSrc, cSrcSize); size_t const hSize = HUF_readDTableX4(DCtx, cSrc, cSrcSize);
if (HUF_isError(hSize)) return hSize; if (HUF_isError(hSize))
if (hSize >= cSrcSize) return ERROR(srcSize_wrong); return hSize;
ip += hSize; cSrcSize -= hSize; if (hSize >= cSrcSize)
return ERROR(srcSize_wrong);
ip += hSize;
cSrcSize -= hSize;
return HUF_decompress1X4_usingDTable_internal(dst, dstSize, ip, cSrcSize, DCtx); return HUF_decompress1X4_usingDTable_internal(dst, dstSize, ip, cSrcSize, DCtx);
} }
static size_t HUF_decompress4X4_usingDTable_internal( static size_t HUF_decompress4X4_usingDTable_internal(void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize, const HUF_DTable *DTable)
void* dst, size_t dstSize,
const void* cSrc, size_t cSrcSize,
const HUF_DTable* DTable)
{ {
if (cSrcSize < 10) return ERROR(corruption_detected); /* strict minimum : jump table + 1 byte per stream */ if (cSrcSize < 10)
return ERROR(corruption_detected); /* strict minimum : jump table + 1 byte per stream */
{ const BYTE* const istart = (const BYTE*) cSrc; {
const BYTE *const istart = (const BYTE *)cSrc;
BYTE *const ostart = (BYTE *)dst; BYTE *const ostart = (BYTE *)dst;
BYTE *const oend = ostart + dstSize; BYTE *const oend = ostart + dstSize;
const void *const dtPtr = DTable + 1; const void *const dtPtr = DTable + 1;
@@ -655,15 +705,28 @@ static size_t HUF_decompress4X4_usingDTable_internal(
DTableDesc const dtd = HUF_getDTableDesc(DTable); DTableDesc const dtd = HUF_getDTableDesc(DTable);
U32 const dtLog = dtd.tableLog; U32 const dtLog = dtd.tableLog;
if (length4 > cSrcSize) return ERROR(corruption_detected); /* overflow */ if (length4 > cSrcSize)
{ size_t const errorCode = BIT_initDStream(&bitD1, istart1, length1); return ERROR(corruption_detected); /* overflow */
if (HUF_isError(errorCode)) return errorCode; } {
{ size_t const errorCode = BIT_initDStream(&bitD2, istart2, length2); size_t const errorCode = BIT_initDStream(&bitD1, istart1, length1);
if (HUF_isError(errorCode)) return errorCode; } if (HUF_isError(errorCode))
{ size_t const errorCode = BIT_initDStream(&bitD3, istart3, length3); return errorCode;
if (HUF_isError(errorCode)) return errorCode; } }
{ size_t const errorCode = BIT_initDStream(&bitD4, istart4, length4); {
if (HUF_isError(errorCode)) return errorCode; } size_t const errorCode = BIT_initDStream(&bitD2, istart2, length2);
if (HUF_isError(errorCode))
return errorCode;
}
{
size_t const errorCode = BIT_initDStream(&bitD3, istart3, length3);
if (HUF_isError(errorCode))
return errorCode;
}
{
size_t const errorCode = BIT_initDStream(&bitD4, istart4, length4);
if (HUF_isError(errorCode))
return errorCode;
}
/* 16-32 symbols per loop (4-8 symbols per stream) */ /* 16-32 symbols per loop (4-8 symbols per stream) */
endSignal = BIT_reloadDStream(&bitD1) | BIT_reloadDStream(&bitD2) | BIT_reloadDStream(&bitD3) | BIT_reloadDStream(&bitD4); endSignal = BIT_reloadDStream(&bitD1) | BIT_reloadDStream(&bitD2) | BIT_reloadDStream(&bitD3) | BIT_reloadDStream(&bitD4);
@@ -689,9 +752,12 @@ static size_t HUF_decompress4X4_usingDTable_internal(
} }
/* check corruption */ /* check corruption */
if (op1 > opStart2) return ERROR(corruption_detected); if (op1 > opStart2)
if (op2 > opStart3) return ERROR(corruption_detected); return ERROR(corruption_detected);
if (op3 > opStart4) return ERROR(corruption_detected); if (op2 > opStart3)
return ERROR(corruption_detected);
if (op3 > opStart4)
return ERROR(corruption_detected);
/* note : op4 already verified within main loop */ /* note : op4 already verified within main loop */
/* finish bitStreams one by one */ /* finish bitStreams one by one */
@@ -701,65 +767,63 @@ static size_t HUF_decompress4X4_usingDTable_internal(
HUF_decodeStreamX4(op4, &bitD4, oend, dt, dtLog); HUF_decodeStreamX4(op4, &bitD4, oend, dt, dtLog);
/* check */ /* check */
{ U32 const endCheck = BIT_endOfDStream(&bitD1) & BIT_endOfDStream(&bitD2) & BIT_endOfDStream(&bitD3) & BIT_endOfDStream(&bitD4); {
if (!endCheck) return ERROR(corruption_detected); } U32 const endCheck = BIT_endOfDStream(&bitD1) & BIT_endOfDStream(&bitD2) & BIT_endOfDStream(&bitD3) & BIT_endOfDStream(&bitD4);
if (!endCheck)
return ERROR(corruption_detected);
}
/* decoded size */ /* decoded size */
return dstSize; return dstSize;
} }
} }
size_t HUF_decompress4X4_usingDTable(void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize, const HUF_DTable *DTable)
size_t HUF_decompress4X4_usingDTable(
void* dst, size_t dstSize,
const void* cSrc, size_t cSrcSize,
const HUF_DTable* DTable)
{ {
DTableDesc dtd = HUF_getDTableDesc(DTable); DTableDesc dtd = HUF_getDTableDesc(DTable);
if (dtd.tableType != 1) return ERROR(GENERIC); if (dtd.tableType != 1)
return ERROR(GENERIC);
return HUF_decompress4X4_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable); return HUF_decompress4X4_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable);
} }
size_t HUF_decompress4X4_DCtx(HUF_DTable *dctx, void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize) size_t HUF_decompress4X4_DCtx(HUF_DTable *dctx, void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize)
{ {
const BYTE *ip = (const BYTE *)cSrc; const BYTE *ip = (const BYTE *)cSrc;
size_t hSize = HUF_readDTableX4(dctx, cSrc, cSrcSize); size_t hSize = HUF_readDTableX4(dctx, cSrc, cSrcSize);
if (HUF_isError(hSize)) return hSize; if (HUF_isError(hSize))
if (hSize >= cSrcSize) return ERROR(srcSize_wrong); return hSize;
ip += hSize; cSrcSize -= hSize; if (hSize >= cSrcSize)
return ERROR(srcSize_wrong);
ip += hSize;
cSrcSize -= hSize;
return HUF_decompress4X4_usingDTable_internal(dst, dstSize, ip, cSrcSize, dctx); return HUF_decompress4X4_usingDTable_internal(dst, dstSize, ip, cSrcSize, dctx);
} }
/* ********************************/ /* ********************************/
/* Generic decompression selector */ /* Generic decompression selector */
/* ********************************/ /* ********************************/
size_t HUF_decompress1X_usingDTable(void* dst, size_t maxDstSize, size_t HUF_decompress1X_usingDTable(void *dst, size_t maxDstSize, const void *cSrc, size_t cSrcSize, const HUF_DTable *DTable)
const void* cSrc, size_t cSrcSize,
const HUF_DTable* DTable)
{ {
DTableDesc const dtd = HUF_getDTableDesc(DTable); DTableDesc const dtd = HUF_getDTableDesc(DTable);
return dtd.tableType ? HUF_decompress1X4_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable) : return dtd.tableType ? HUF_decompress1X4_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable)
HUF_decompress1X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable); : HUF_decompress1X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable);
} }
size_t HUF_decompress4X_usingDTable(void* dst, size_t maxDstSize, size_t HUF_decompress4X_usingDTable(void *dst, size_t maxDstSize, const void *cSrc, size_t cSrcSize, const HUF_DTable *DTable)
const void* cSrc, size_t cSrcSize,
const HUF_DTable* DTable)
{ {
DTableDesc const dtd = HUF_getDTableDesc(DTable); DTableDesc const dtd = HUF_getDTableDesc(DTable);
return dtd.tableType ? HUF_decompress4X4_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable) : return dtd.tableType ? HUF_decompress4X4_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable)
HUF_decompress4X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable); : HUF_decompress4X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable);
} }
typedef struct {
typedef struct { U32 tableTime; U32 decode256Time; } algo_time_t; U32 tableTime;
static const algo_time_t algoTime[16 /* Quantization */][3 /* single, double, quad */] = U32 decode256Time;
{ } algo_time_t;
static const algo_time_t algoTime[16 /* Quantization */][3 /* single, double, quad */] = {
/* single, double, quad */ /* single, double, quad */
{{0, 0}, {1, 1}, {2, 2}}, /* Q==0 : impossible */ {{0, 0}, {1, 1}, {2, 2}}, /* Q==0 : impossible */
{{0, 0}, {1, 1}, {2, 2}}, /* Q==1 : impossible */ {{0, 0}, {1, 1}, {2, 2}}, /* Q==1 : impossible */
@@ -796,45 +860,62 @@ U32 HUF_selectDecoder (size_t dstSize, size_t cSrcSize)
return DTime1 < DTime0; return DTime1 < DTime0;
} }
typedef size_t (*decompressionAlgo)(void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize); typedef size_t (*decompressionAlgo)(void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize);
size_t HUF_decompress4X_DCtx(HUF_DTable *dctx, void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize) size_t HUF_decompress4X_DCtx(HUF_DTable *dctx, void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize)
{ {
/* validation checks */ /* validation checks */
if (dstSize == 0) return ERROR(dstSize_tooSmall); if (dstSize == 0)
if (cSrcSize > dstSize) return ERROR(corruption_detected); /* invalid */ return ERROR(dstSize_tooSmall);
if (cSrcSize == dstSize) { memcpy(dst, cSrc, dstSize); return dstSize; } /* not compressed */ if (cSrcSize > dstSize)
if (cSrcSize == 1) { memset(dst, *(const BYTE*)cSrc, dstSize); return dstSize; } /* RLE */ return ERROR(corruption_detected); /* invalid */
if (cSrcSize == dstSize) {
memcpy(dst, cSrc, dstSize);
return dstSize;
} /* not compressed */
if (cSrcSize == 1) {
memset(dst, *(const BYTE *)cSrc, dstSize);
return dstSize;
} /* RLE */
{ U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize); {
return algoNb ? HUF_decompress4X4_DCtx(dctx, dst, dstSize, cSrc, cSrcSize) : U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize);
HUF_decompress4X2_DCtx(dctx, dst, dstSize, cSrc, cSrcSize) ; return algoNb ? HUF_decompress4X4_DCtx(dctx, dst, dstSize, cSrc, cSrcSize) : HUF_decompress4X2_DCtx(dctx, dst, dstSize, cSrc, cSrcSize);
} }
} }
size_t HUF_decompress4X_hufOnly(HUF_DTable *dctx, void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize) size_t HUF_decompress4X_hufOnly(HUF_DTable *dctx, void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize)
{ {
/* validation checks */ /* validation checks */
if (dstSize == 0) return ERROR(dstSize_tooSmall); if (dstSize == 0)
if ((cSrcSize >= dstSize) || (cSrcSize <= 1)) return ERROR(corruption_detected); /* invalid */ return ERROR(dstSize_tooSmall);
if ((cSrcSize >= dstSize) || (cSrcSize <= 1))
return ERROR(corruption_detected); /* invalid */
{ U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize); {
return algoNb ? HUF_decompress4X4_DCtx(dctx, dst, dstSize, cSrc, cSrcSize) : U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize);
HUF_decompress4X2_DCtx(dctx, dst, dstSize, cSrc, cSrcSize) ; return algoNb ? HUF_decompress4X4_DCtx(dctx, dst, dstSize, cSrc, cSrcSize) : HUF_decompress4X2_DCtx(dctx, dst, dstSize, cSrc, cSrcSize);
} }
} }
size_t HUF_decompress1X_DCtx(HUF_DTable *dctx, void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize) size_t HUF_decompress1X_DCtx(HUF_DTable *dctx, void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize)
{ {
/* validation checks */ /* validation checks */
if (dstSize == 0) return ERROR(dstSize_tooSmall); if (dstSize == 0)
if (cSrcSize > dstSize) return ERROR(corruption_detected); /* invalid */ return ERROR(dstSize_tooSmall);
if (cSrcSize == dstSize) { memcpy(dst, cSrc, dstSize); return dstSize; } /* not compressed */ if (cSrcSize > dstSize)
if (cSrcSize == 1) { memset(dst, *(const BYTE*)cSrc, dstSize); return dstSize; } /* RLE */ return ERROR(corruption_detected); /* invalid */
if (cSrcSize == dstSize) {
memcpy(dst, cSrc, dstSize);
return dstSize;
} /* not compressed */
if (cSrcSize == 1) {
memset(dst, *(const BYTE *)cSrc, dstSize);
return dstSize;
} /* RLE */
{ U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize); {
return algoNb ? HUF_decompress1X4_DCtx(dctx, dst, dstSize, cSrc, cSrcSize) : U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize);
HUF_decompress1X2_DCtx(dctx, dst, dstSize, cSrc, cSrcSize) ; return algoNb ? HUF_decompress1X4_DCtx(dctx, dst, dstSize, cSrc, cSrcSize) : HUF_decompress1X2_DCtx(dctx, dst, dstSize, cSrc, cSrcSize);
} }
} }
contrib/linux-kernel/lib/zstd/mem.h
+23 -84
View File
@@ -21,16 +21,14 @@
* Dependencies * Dependencies
******************************************/ ******************************************/
#include <asm/unaligned.h> #include <asm/unaligned.h>
#include <linux/types.h> /* size_t, ptrdiff_t */
#include <linux/string.h> /* memcpy */ #include <linux/string.h> /* memcpy */
#include <linux/types.h> /* size_t, ptrdiff_t */
/*-**************************************** /*-****************************************
* Compiler specifics * Compiler specifics
******************************************/ ******************************************/
#define ZSTD_STATIC static __inline __attribute__((unused)) #define ZSTD_STATIC static __inline __attribute__((unused))
/*-************************************************************** /*-**************************************************************
* Basic Types * Basic Types
*****************************************************************/ *****************************************************************/
@@ -44,7 +42,6 @@ typedef int64_t S64;
typedef ptrdiff_t iPtrDiff; typedef ptrdiff_t iPtrDiff;
typedef uintptr_t uPtrDiff; typedef uintptr_t uPtrDiff;
/*-************************************************************** /*-**************************************************************
* Memory I/O * Memory I/O
*****************************************************************/ *****************************************************************/
@@ -57,62 +54,29 @@ ZSTD_STATIC unsigned ZSTD_64bits(void) { return sizeof(size_t)==8; }
#define ZSTD_LITTLE_ENDIAN 0 #define ZSTD_LITTLE_ENDIAN 0
#endif #endif
ZSTD_STATIC unsigned ZSTD_isLittleEndian(void) ZSTD_STATIC unsigned ZSTD_isLittleEndian(void) { return ZSTD_LITTLE_ENDIAN; }
{
return ZSTD_LITTLE_ENDIAN;
}
ZSTD_STATIC U16 ZSTD_read16(const void* memPtr) ZSTD_STATIC U16 ZSTD_read16(const void *memPtr) { return get_unaligned((const U16 *)memPtr); }
{
return get_unaligned((const U16*)memPtr);
}
ZSTD_STATIC U32 ZSTD_read32(const void* memPtr) ZSTD_STATIC U32 ZSTD_read32(const void *memPtr) { return get_unaligned((const U32 *)memPtr); }
{
return get_unaligned((const U32*)memPtr);
}
ZSTD_STATIC U64 ZSTD_read64(const void* memPtr) ZSTD_STATIC U64 ZSTD_read64(const void *memPtr) { return get_unaligned((const U64 *)memPtr); }
{
return get_unaligned((const U64*)memPtr);
}
ZSTD_STATIC size_t ZSTD_readST(const void* memPtr) ZSTD_STATIC size_t ZSTD_readST(const void *memPtr) { return get_unaligned((const size_t *)memPtr); }
{
return get_unaligned((const size_t*)memPtr);
}
ZSTD_STATIC void ZSTD_write16(void* memPtr, U16 value) ZSTD_STATIC void ZSTD_write16(void *memPtr, U16 value) { put_unaligned(value, (U16 *)memPtr); }
{
put_unaligned(value, (U16*)memPtr);
}
ZSTD_STATIC void ZSTD_write32(void* memPtr, U32 value) ZSTD_STATIC void ZSTD_write32(void *memPtr, U32 value) { put_unaligned(value, (U32 *)memPtr); }
{
put_unaligned(value, (U32*)memPtr);
}
ZSTD_STATIC void ZSTD_write64(void* memPtr, U64 value) ZSTD_STATIC void ZSTD_write64(void *memPtr, U64 value) { put_unaligned(value, (U64 *)memPtr); }
{
put_unaligned(value, (U64*)memPtr);
}
/*=== Little endian r/w ===*/ /*=== Little endian r/w ===*/
ZSTD_STATIC U16 ZSTD_readLE16(const void* memPtr) ZSTD_STATIC U16 ZSTD_readLE16(const void *memPtr) { return get_unaligned_le16(memPtr); }
{
return get_unaligned_le16(memPtr);
}
ZSTD_STATIC void ZSTD_writeLE16(void* memPtr, U16 val) ZSTD_STATIC void ZSTD_writeLE16(void *memPtr, U16 val) { put_unaligned_le16(val, memPtr); }
{
put_unaligned_le16(val, memPtr);
}
ZSTD_STATIC U32 ZSTD_readLE24(const void* memPtr) ZSTD_STATIC U32 ZSTD_readLE24(const void *memPtr) { return ZSTD_readLE16(memPtr) + (((const BYTE *)memPtr)[2] << 16); }
{
return ZSTD_readLE16(memPtr) + (((const BYTE*)memPtr)[2] << 16);
}
ZSTD_STATIC void ZSTD_writeLE24(void *memPtr, U32 val) ZSTD_STATIC void ZSTD_writeLE24(void *memPtr, U32 val)
{ {
@@ -120,25 +84,13 @@ ZSTD_STATIC void ZSTD_writeLE24(void* memPtr, U32 val)
((BYTE *)memPtr)[2] = (BYTE)(val >> 16); ((BYTE *)memPtr)[2] = (BYTE)(val >> 16);
} }
ZSTD_STATIC U32 ZSTD_readLE32(const void* memPtr) ZSTD_STATIC U32 ZSTD_readLE32(const void *memPtr) { return get_unaligned_le32(memPtr); }
{
return get_unaligned_le32(memPtr);
}
ZSTD_STATIC void ZSTD_writeLE32(void* memPtr, U32 val32) ZSTD_STATIC void ZSTD_writeLE32(void *memPtr, U32 val32) { put_unaligned_le32(val32, memPtr); }
{
put_unaligned_le32(val32, memPtr);
}
ZSTD_STATIC U64 ZSTD_readLE64(const void* memPtr) ZSTD_STATIC U64 ZSTD_readLE64(const void *memPtr) { return get_unaligned_le64(memPtr); }
{
return get_unaligned_le64(memPtr);
}
ZSTD_STATIC void ZSTD_writeLE64(void* memPtr, U64 val64) ZSTD_STATIC void ZSTD_writeLE64(void *memPtr, U64 val64) { put_unaligned_le64(val64, memPtr); }
{
put_unaligned_le64(val64, memPtr);
}
ZSTD_STATIC size_t ZSTD_readLEST(const void *memPtr) ZSTD_STATIC size_t ZSTD_readLEST(const void *memPtr)
{ {
@@ -158,25 +110,13 @@ ZSTD_STATIC void ZSTD_writeLEST(void* memPtr, size_t val)
/*=== Big endian r/w ===*/ /*=== Big endian r/w ===*/
ZSTD_STATIC U32 ZSTD_readBE32(const void* memPtr) ZSTD_STATIC U32 ZSTD_readBE32(const void *memPtr) { return get_unaligned_be32(memPtr); }
{
return get_unaligned_be32(memPtr);
}
ZSTD_STATIC void ZSTD_writeBE32(void* memPtr, U32 val32) ZSTD_STATIC void ZSTD_writeBE32(void *memPtr, U32 val32) { put_unaligned_be32(val32, memPtr); }
{
put_unaligned_be32(val32, memPtr);
}
ZSTD_STATIC U64 ZSTD_readBE64(const void* memPtr) ZSTD_STATIC U64 ZSTD_readBE64(const void *memPtr) { return get_unaligned_be64(memPtr); }
{
return get_unaligned_be64(memPtr);
}
ZSTD_STATIC void ZSTD_writeBE64(void* memPtr, U64 val64) ZSTD_STATIC void ZSTD_writeBE64(void *memPtr, U64 val64) { put_unaligned_be64(val64, memPtr); }
{
put_unaligned_be64(val64, memPtr);
}
ZSTD_STATIC size_t ZSTD_readBEST(const void *memPtr) ZSTD_STATIC size_t ZSTD_readBEST(const void *memPtr)
{ {
@@ -194,15 +134,14 @@ ZSTD_STATIC void ZSTD_writeBEST(void* memPtr, size_t val)
ZSTD_writeBE64(memPtr, (U64)val); ZSTD_writeBE64(memPtr, (U64)val);
} }
/* function safe only for comparisons */ /* function safe only for comparisons */
ZSTD_STATIC U32 ZSTD_readMINMATCH(const void *memPtr, U32 length) ZSTD_STATIC U32 ZSTD_readMINMATCH(const void *memPtr, U32 length)
{ {
switch (length) switch (length) {
{
default: default:
case 4: return ZSTD_read32(memPtr); case 4: return ZSTD_read32(memPtr);
case 3 : if (ZSTD_isLittleEndian()) case 3:
if (ZSTD_isLittleEndian())
return ZSTD_read32(memPtr) << 8; return ZSTD_read32(memPtr) << 8;
else else
return ZSTD_read32(memPtr) >> 8; return ZSTD_read32(memPtr) >> 8;
contrib/linux-kernel/lib/zstd/zstd_common.c
+11 -12
View File
@@ -14,8 +14,6 @@
* ("BSD"). * ("BSD").
*/ */
/*-************************************* /*-*************************************
* Dependencies * Dependencies
***************************************/ ***************************************/
@@ -23,18 +21,19 @@
#include "zstd_internal.h" /* declaration of ZSTD_isError, ZSTD_getErrorName, ZSTD_getErrorCode, ZSTD_getErrorString, ZSTD_versionNumber */ #include "zstd_internal.h" /* declaration of ZSTD_isError, ZSTD_getErrorName, ZSTD_getErrorCode, ZSTD_getErrorString, ZSTD_versionNumber */
#include <linux/kernel.h> #include <linux/kernel.h>
/*=************************************************************** /*=**************************************************************
* Custom allocator * Custom allocator
****************************************************************/ ****************************************************************/
#define stack_push(stack, size) ({ \ #define stack_push(stack, size) \
({ \
void *const ptr = ZSTD_PTR_ALIGN((stack)->ptr); \ void *const ptr = ZSTD_PTR_ALIGN((stack)->ptr); \
(stack)->ptr = (char *)ptr + (size); \ (stack)->ptr = (char *)ptr + (size); \
(stack)->ptr <= (stack)->end ? ptr : NULL; \ (stack)->ptr <= (stack)->end ? ptr : NULL; \
}) })
ZSTD_customMem ZSTD_initStack(void* workspace, size_t workspaceSize) { ZSTD_customMem ZSTD_initStack(void *workspace, size_t workspaceSize)
{
ZSTD_customMem stackMem = {ZSTD_stackAlloc, ZSTD_stackFree, workspace}; ZSTD_customMem stackMem = {ZSTD_stackAlloc, ZSTD_stackFree, workspace};
ZSTD_stack *stack = (ZSTD_stack *)workspace; ZSTD_stack *stack = (ZSTD_stack *)workspace;
/* Verify preconditions */ /* Verify preconditions */
@@ -49,25 +48,25 @@ ZSTD_customMem ZSTD_initStack(void* workspace, size_t workspaceSize) {
return stackMem; return stackMem;
} }
void* ZSTD_stackAllocAll(void* opaque, size_t* size) { void *ZSTD_stackAllocAll(void *opaque, size_t *size)
{
ZSTD_stack *stack = (ZSTD_stack *)opaque; ZSTD_stack *stack = (ZSTD_stack *)opaque;
*size = stack->end - ZSTD_PTR_ALIGN(stack->ptr); *size = stack->end - ZSTD_PTR_ALIGN(stack->ptr);
return stack_push(stack, *size); return stack_push(stack, *size);
} }
void* ZSTD_stackAlloc(void* opaque, size_t size) { void *ZSTD_stackAlloc(void *opaque, size_t size)
{
ZSTD_stack *stack = (ZSTD_stack *)opaque; ZSTD_stack *stack = (ZSTD_stack *)opaque;
return stack_push(stack, size); return stack_push(stack, size);
} }
void ZSTD_stackFree(void* opaque, void* address) { void ZSTD_stackFree(void *opaque, void *address)
{
(void)opaque; (void)opaque;
(void)address; (void)address;
} }
void* ZSTD_malloc(size_t size, ZSTD_customMem customMem) void *ZSTD_malloc(size_t size, ZSTD_customMem customMem) { return customMem.customAlloc(customMem.opaque, size); }
{
return customMem.customAlloc(customMem.opaque, size);
}
void ZSTD_free(void *ptr, ZSTD_customMem customMem) void ZSTD_free(void *ptr, ZSTD_customMem customMem)
{ {
contrib/linux-kernel/lib/zstd/zstd_internal.h
+33 -36
View File
@@ -23,26 +23,33 @@
#define FORCE_INLINE static __always_inline #define FORCE_INLINE static __always_inline
#define FORCE_NOINLINE static noinline #define FORCE_NOINLINE static noinline
/*-************************************* /*-*************************************
* Dependencies * Dependencies
***************************************/ ***************************************/
#include "error_private.h"
#include "mem.h"
#include <linux/compiler.h> #include <linux/compiler.h>
#include <linux/kernel.h> #include <linux/kernel.h>
#include <linux/xxhash.h> #include <linux/xxhash.h>
#include <linux/zstd.h> #include <linux/zstd.h>
#include "mem.h"
#include "error_private.h"
/*-************************************* /*-*************************************
* shared macros * shared macros
***************************************/ ***************************************/
#define MIN(a, b) ((a) < (b) ? (a) : (b)) #define MIN(a, b) ((a) < (b) ? (a) : (b))
#define MAX(a, b) ((a) > (b) ? (a) : (b)) #define MAX(a, b) ((a) > (b) ? (a) : (b))
#define CHECK_F(f) { size_t const errcod = f; if (ERR_isError(errcod)) return errcod; } /* check and Forward error code */ #define CHECK_F(f) \
#define CHECK_E(f, e) { size_t const errcod = f; if (ERR_isError(errcod)) return ERROR(e); } /* check and send Error code */ { \
size_t const errcod = f; \
if (ERR_isError(errcod)) \
return errcod; \
} /* check and Forward error code */
#define CHECK_E(f, e) \
{ \
size_t const errcod = f; \
if (ERR_isError(errcod)) \
return ERROR(e); \
} /* check and send Error code */
/*-************************************* /*-*************************************
* Common constants * Common constants
@@ -96,37 +103,32 @@ typedef enum { set_basic, set_rle, set_compressed, set_repeat } symbolEncodingTy
#define LLFSELog 9 #define LLFSELog 9
#define OffFSELog 8 #define OffFSELog 8
static const U32 LL_bits[MaxLL+1] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, static const U32 LL_bits[MaxLL + 1] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 3, 3, 4, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16};
1, 1, 1, 1, 2, 2, 3, 3, 4, 6, 7, 8, 9,10,11,12, static const S16 LL_defaultNorm[MaxLL + 1] = {4, 3, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 2, 1, 1, 1, 1, 1, -1, -1, -1, -1};
13,14,15,16 };
static const S16 LL_defaultNorm[MaxLL+1] = { 4, 3, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1,
2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 2, 1, 1, 1, 1, 1,
-1,-1,-1,-1 };
#define LL_DEFAULTNORMLOG 6 /* for static allocation */ #define LL_DEFAULTNORMLOG 6 /* for static allocation */
static const U32 LL_defaultNormLog = LL_DEFAULTNORMLOG; static const U32 LL_defaultNormLog = LL_DEFAULTNORMLOG;
static const U32 ML_bits[MaxML+1] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, static const U32 ML_bits[MaxML + 1] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 3, 3, 4, 4, 5, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16};
1, 1, 1, 1, 2, 2, 3, 3, 4, 4, 5, 7, 8, 9,10,11, static const S16 ML_defaultNorm[MaxML + 1] = {1, 4, 3, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
12,13,14,15,16 }; 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, -1, -1, -1, -1, -1, -1, -1};
static const S16 ML_defaultNorm[MaxML+1] = { 1, 4, 3, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,-1,-1,
-1,-1,-1,-1,-1 };
#define ML_DEFAULTNORMLOG 6 /* for static allocation */ #define ML_DEFAULTNORMLOG 6 /* for static allocation */
static const U32 ML_defaultNormLog = ML_DEFAULTNORMLOG; static const U32 ML_defaultNormLog = ML_DEFAULTNORMLOG;
static const S16 OF_defaultNorm[MaxOff+1] = { 1, 1, 1, 1, 1, 1, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1, static const S16 OF_defaultNorm[MaxOff + 1] = {1, 1, 1, 1, 1, 1, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, -1, -1, -1, -1, -1};
1, 1, 1, 1, 1, 1, 1, 1,-1,-1,-1,-1,-1 };
#define OF_DEFAULTNORMLOG 5 /* for static allocation */ #define OF_DEFAULTNORMLOG 5 /* for static allocation */
static const U32 OF_defaultNormLog = OF_DEFAULTNORMLOG; static const U32 OF_defaultNormLog = OF_DEFAULTNORMLOG;
/*-******************************************* /*-*******************************************
* Shared functions to include for inlining * Shared functions to include for inlining
*********************************************/ *********************************************/
static void ZSTD_copy8(void *dst, const void *src) { memcpy(dst, src, 8); } static void ZSTD_copy8(void *dst, const void *src) { memcpy(dst, src, 8); }
#define COPY8(d,s) { ZSTD_copy8(d,s); d+=8; s+=8; } #define COPY8(d, s) \
{ \
ZSTD_copy8(d, s); \
d += 8; \
s += 8; \
}
/*! ZSTD_wildcopy() : /*! ZSTD_wildcopy() :
* custom version of memcpy(), can copy up to 7 bytes too many (8 bytes if length==0) */ * custom version of memcpy(), can copy up to 7 bytes too many (8 bytes if length==0) */
@@ -151,7 +153,6 @@ ZSTD_STATIC void ZSTD_wildcopy_e(void* dst, const void* src, void* dstEnd) /*
while (op < oend); while (op < oend);
} }
/*-******************************************* /*-*******************************************
* Private interfaces * Private interfaces
*********************************************/ *********************************************/
@@ -170,14 +171,12 @@ typedef struct {
U32 rep[ZSTD_REP_NUM]; U32 rep[ZSTD_REP_NUM];
} ZSTD_optimal_t; } ZSTD_optimal_t;
typedef struct seqDef_s { typedef struct seqDef_s {
U32 offset; U32 offset;
U16 litLength; U16 litLength;
U16 matchLength; U16 matchLength;
} seqDef; } seqDef;
typedef struct { typedef struct {
seqDef *sequencesStart; seqDef *sequencesStart;
seqDef *sequences; seqDef *sequences;
@@ -219,7 +218,11 @@ int ZSTD_isSkipFrame(ZSTD_DCtx* dctx);
/*= Custom memory allocation functions */ /*= Custom memory allocation functions */
typedef void *(*ZSTD_allocFunction)(void *opaque, size_t size); typedef void *(*ZSTD_allocFunction)(void *opaque, size_t size);
typedef void (*ZSTD_freeFunction)(void *opaque, void *address); typedef void (*ZSTD_freeFunction)(void *opaque, void *address);
typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; void* opaque; } ZSTD_customMem; typedef struct {
ZSTD_allocFunction customAlloc;
ZSTD_freeFunction customFree;
void *opaque;
} ZSTD_customMem;
void *ZSTD_malloc(size_t size, ZSTD_customMem customMem); void *ZSTD_malloc(size_t size, ZSTD_customMem customMem);
void ZSTD_free(void *ptr, ZSTD_customMem customMem); void ZSTD_free(void *ptr, ZSTD_customMem customMem);
@@ -240,14 +243,9 @@ void* ZSTD_stackAllocAll(void* opaque, size_t* size);
void *ZSTD_stackAlloc(void *opaque, size_t size); void *ZSTD_stackAlloc(void *opaque, size_t size);
void ZSTD_stackFree(void *opaque, void *address); void ZSTD_stackFree(void *opaque, void *address);
/*====== common function ======*/ /*====== common function ======*/
ZSTD_STATIC U32 ZSTD_highbit32(U32 val) ZSTD_STATIC U32 ZSTD_highbit32(U32 val) { return 31 - __builtin_clz(val); }
{
return 31 - __builtin_clz(val);
}
/* hidden functions */ /* hidden functions */
@@ -264,5 +262,4 @@ size_t ZSTD_freeDDict(ZSTD_DDict* cdict);
size_t ZSTD_freeCStream(ZSTD_CStream *zcs); size_t ZSTD_freeCStream(ZSTD_CStream *zcs);
size_t ZSTD_freeDStream(ZSTD_DStream *zds); size_t ZSTD_freeDStream(ZSTD_DStream *zds);
#endif /* ZSTD_CCOMMON_H_MODULE */ #endif /* ZSTD_CCOMMON_H_MODULE */
contrib/linux-kernel/lib/zstd/zstd_opt.h
+206 -120
View File
@@ -14,14 +14,11 @@
* ("BSD"). * ("BSD").
*/ */
/* Note : this file is intended to be included within zstd_compress.c */ /* Note : this file is intended to be included within zstd_compress.c */
#ifndef ZSTD_OPT_H_91842398743 #ifndef ZSTD_OPT_H_91842398743
#define ZSTD_OPT_H_91842398743 #define ZSTD_OPT_H_91842398743
#define ZSTD_LITFREQ_ADD 2 #define ZSTD_LITFREQ_ADD 2
#define ZSTD_FREQ_DIV 4 #define ZSTD_FREQ_DIV 4
#define ZSTD_MAX_PRICE (1 << 30) #define ZSTD_MAX_PRICE (1 << 30)
@@ -38,7 +35,6 @@ FORCE_INLINE void ZSTD_setLog2Prices(seqStore_t* ssPtr)
ssPtr->factor = 1 + ((ssPtr->litSum >> 5) / ssPtr->litLengthSum) + ((ssPtr->litSum << 1) / (ssPtr->litSum + ssPtr->matchSum)); ssPtr->factor = 1 + ((ssPtr->litSum >> 5) / ssPtr->litLengthSum) + ((ssPtr->litSum << 1) / (ssPtr->litSum + ssPtr->matchSum));
} }
ZSTD_STATIC void ZSTD_rescaleFreqs(seqStore_t *ssPtr, const BYTE *src, size_t srcSize) ZSTD_STATIC void ZSTD_rescaleFreqs(seqStore_t *ssPtr, const BYTE *src, size_t srcSize)
{ {
unsigned u; unsigned u;
@@ -48,7 +44,8 @@ ZSTD_STATIC void ZSTD_rescaleFreqs(seqStore_t* ssPtr, const BYTE* src, size_t sr
ssPtr->staticPrices = 0; ssPtr->staticPrices = 0;
if (ssPtr->litLengthSum == 0) { if (ssPtr->litLengthSum == 0) {
if (srcSize <= 1024) ssPtr->staticPrices = 1; if (srcSize <= 1024)
ssPtr->staticPrices = 1;
for (u = 0; u <= MaxLit; u++) for (u = 0; u <= MaxLit; u++)
ssPtr->litFreq[u] = 0; ssPtr->litFreq[u] = 0;
@@ -101,7 +98,6 @@ ZSTD_STATIC void ZSTD_rescaleFreqs(seqStore_t* ssPtr, const BYTE* src, size_t sr
ZSTD_setLog2Prices(ssPtr); ZSTD_setLog2Prices(ssPtr);
} }
FORCE_INLINE U32 ZSTD_getLiteralPrice(seqStore_t *ssPtr, U32 litLength, const BYTE *literals) FORCE_INLINE U32 ZSTD_getLiteralPrice(seqStore_t *ssPtr, U32 litLength, const BYTE *literals)
{ {
U32 price, u; U32 price, u;
@@ -134,7 +130,8 @@ FORCE_INLINE U32 ZSTD_getLiteralPrice(seqStore_t* ssPtr, U32 litLength, const BY
} }
/* literal Length */ /* literal Length */
{ const BYTE LL_deltaCode = 19; {
const BYTE LL_deltaCode = 19;
const BYTE llCode = (litLength > 63) ? (BYTE)ZSTD_highbit32(litLength) + LL_deltaCode : LL_Code[litLength]; const BYTE llCode = (litLength > 63) ? (BYTE)ZSTD_highbit32(litLength) + LL_deltaCode : LL_Code[litLength];
price += LL_bits[llCode] + ssPtr->log2litLengthSum - ZSTD_highbit32(ssPtr->litLengthFreq[llCode] + 1); price += LL_bits[llCode] + ssPtr->log2litLengthSum - ZSTD_highbit32(ssPtr->litLengthFreq[llCode] + 1);
} }
@@ -142,7 +139,6 @@ FORCE_INLINE U32 ZSTD_getLiteralPrice(seqStore_t* ssPtr, U32 litLength, const BY
return price; return price;
} }
FORCE_INLINE U32 ZSTD_getPrice(seqStore_t *seqStorePtr, U32 litLength, const BYTE *literals, U32 offset, U32 matchLength, const int ultra) FORCE_INLINE U32 ZSTD_getPrice(seqStore_t *seqStorePtr, U32 litLength, const BYTE *literals, U32 offset, U32 matchLength, const int ultra)
{ {
/* offset */ /* offset */
@@ -153,10 +149,12 @@ FORCE_INLINE U32 ZSTD_getPrice(seqStore_t* seqStorePtr, U32 litLength, const BYT
return ZSTD_getLiteralPrice(seqStorePtr, litLength, literals) + ZSTD_highbit32((U32)matchLength + 1) + 16 + offCode; return ZSTD_getLiteralPrice(seqStorePtr, litLength, literals) + ZSTD_highbit32((U32)matchLength + 1) + 16 + offCode;
price = offCode + seqStorePtr->log2offCodeSum - ZSTD_highbit32(seqStorePtr->offCodeFreq[offCode] + 1); price = offCode + seqStorePtr->log2offCodeSum - ZSTD_highbit32(seqStorePtr->offCodeFreq[offCode] + 1);
if (!ultra && offCode >= 20) price += (offCode-19)*2; if (!ultra && offCode >= 20)
price += (offCode - 19) * 2;
/* match Length */ /* match Length */
{ const BYTE ML_deltaCode = 36; {
const BYTE ML_deltaCode = 36;
const BYTE mlCode = (matchLength > 127) ? (BYTE)ZSTD_highbit32(matchLength) + ML_deltaCode : ML_Code[matchLength]; const BYTE mlCode = (matchLength > 127) ? (BYTE)ZSTD_highbit32(matchLength) + ML_deltaCode : ML_Code[matchLength];
price += ML_bits[mlCode] + seqStorePtr->log2matchLengthSum - ZSTD_highbit32(seqStorePtr->matchLengthFreq[mlCode] + 1); price += ML_bits[mlCode] + seqStorePtr->log2matchLengthSum - ZSTD_highbit32(seqStorePtr->matchLengthFreq[mlCode] + 1);
} }
@@ -164,7 +162,6 @@ FORCE_INLINE U32 ZSTD_getPrice(seqStore_t* seqStorePtr, U32 litLength, const BYT
return price + ZSTD_getLiteralPrice(seqStorePtr, litLength, literals) + seqStorePtr->factor; return price + ZSTD_getLiteralPrice(seqStorePtr, litLength, literals) + seqStorePtr->factor;
} }
ZSTD_STATIC void ZSTD_updatePrice(seqStore_t *seqStorePtr, U32 litLength, const BYTE *literals, U32 offset, U32 matchLength) ZSTD_STATIC void ZSTD_updatePrice(seqStore_t *seqStorePtr, U32 litLength, const BYTE *literals, U32 offset, U32 matchLength)
{ {
U32 u; U32 u;
@@ -175,20 +172,23 @@ ZSTD_STATIC void ZSTD_updatePrice(seqStore_t* seqStorePtr, U32 litLength, const
seqStorePtr->litFreq[literals[u]] += ZSTD_LITFREQ_ADD; seqStorePtr->litFreq[literals[u]] += ZSTD_LITFREQ_ADD;
/* literal Length */ /* literal Length */
{ const BYTE LL_deltaCode = 19; {
const BYTE LL_deltaCode = 19;
const BYTE llCode = (litLength > 63) ? (BYTE)ZSTD_highbit32(litLength) + LL_deltaCode : LL_Code[litLength]; const BYTE llCode = (litLength > 63) ? (BYTE)ZSTD_highbit32(litLength) + LL_deltaCode : LL_Code[litLength];
seqStorePtr->litLengthFreq[llCode]++; seqStorePtr->litLengthFreq[llCode]++;
seqStorePtr->litLengthSum++; seqStorePtr->litLengthSum++;
} }
/* match offset */ /* match offset */
{ BYTE const offCode = (BYTE)ZSTD_highbit32(offset+1); {
BYTE const offCode = (BYTE)ZSTD_highbit32(offset + 1);
seqStorePtr->offCodeSum++; seqStorePtr->offCodeSum++;
seqStorePtr->offCodeFreq[offCode]++; seqStorePtr->offCodeFreq[offCode]++;
} }
/* match Length */ /* match Length */
{ const BYTE ML_deltaCode = 36; {
const BYTE ML_deltaCode = 36;
const BYTE mlCode = (matchLength > 127) ? (BYTE)ZSTD_highbit32(matchLength) + ML_deltaCode : ML_Code[matchLength]; const BYTE mlCode = (matchLength > 127) ? (BYTE)ZSTD_highbit32(matchLength) + ML_deltaCode : ML_Code[matchLength];
seqStorePtr->matchLengthFreq[mlCode]++; seqStorePtr->matchLengthFreq[mlCode]++;
seqStorePtr->matchLengthSum++; seqStorePtr->matchLengthSum++;
@@ -197,18 +197,18 @@ ZSTD_STATIC void ZSTD_updatePrice(seqStore_t* seqStorePtr, U32 litLength, const
ZSTD_setLog2Prices(seqStorePtr); ZSTD_setLog2Prices(seqStorePtr);
} }
#define SET_PRICE(pos, mlen_, offset_, litlen_, price_) \ #define SET_PRICE(pos, mlen_, offset_, litlen_, price_) \
{ \ { \
while (last_pos < pos) { opt[last_pos+1].price = ZSTD_MAX_PRICE; last_pos++; } \ while (last_pos < pos) { \
opt[last_pos + 1].price = ZSTD_MAX_PRICE; \
last_pos++; \
} \
opt[pos].mlen = mlen_; \ opt[pos].mlen = mlen_; \
opt[pos].off = offset_; \ opt[pos].off = offset_; \
opt[pos].litlen = litlen_; \ opt[pos].litlen = litlen_; \
opt[pos].price = price_; \ opt[pos].price = price_; \
} }
/* Update hashTable3 up to ip (excluded) /* Update hashTable3 up to ip (excluded)
Assumption : always within prefix (i.e. not within extDict) */ Assumption : always within prefix (i.e. not within extDict) */
FORCE_INLINE FORCE_INLINE
@@ -229,15 +229,11 @@ U32 ZSTD_insertAndFindFirstIndexHash3 (ZSTD_CCtx* zc, const BYTE* ip)
return hashTable3[hash3]; return hashTable3[hash3];
} }
/*-************************************* /*-*************************************
* Binary Tree search * Binary Tree search
***************************************/ ***************************************/
static U32 ZSTD_insertBtAndGetAllMatches ( static U32 ZSTD_insertBtAndGetAllMatches(ZSTD_CCtx *zc, const BYTE *const ip, const BYTE *const iLimit, U32 nbCompares, const U32 mls, U32 extDict,
ZSTD_CCtx* zc, ZSTD_match_t *matches, const U32 minMatchLen)
const BYTE* const ip, const BYTE* const iLimit,
U32 nbCompares, const U32 mls,
U32 extDict, ZSTD_match_t* matches, const U32 minMatchLen)
{ {
const BYTE *const base = zc->base; const BYTE *const base = zc->base;
const U32 curr = (U32)(ip - base); const U32 curr = (U32)(ip - base);
@@ -271,10 +267,12 @@ static U32 ZSTD_insertBtAndGetAllMatches (
size_t currMl = 0; size_t currMl = 0;
if ((!extDict) || matchIndex3 >= dictLimit) { if ((!extDict) || matchIndex3 >= dictLimit) {
match = base + matchIndex3; match = base + matchIndex3;
if (match[bestLength] == ip[bestLength]) currMl = ZSTD_count(ip, match, iLimit); if (match[bestLength] == ip[bestLength])
currMl = ZSTD_count(ip, match, iLimit);
} else { } else {
match = dictBase + matchIndex3; match = dictBase + matchIndex3;
if (ZSTD_readMINMATCH(match, MINMATCH) == ZSTD_readMINMATCH(ip, MINMATCH)) /* assumption : matchIndex3 <= dictLimit-4 (by table construction) */ if (ZSTD_readMINMATCH(match, MINMATCH) ==
ZSTD_readMINMATCH(ip, MINMATCH)) /* assumption : matchIndex3 <= dictLimit-4 (by table construction) */
currMl = ZSTD_count_2segments(ip + MINMATCH, match + MINMATCH, iLimit, dictEnd, prefixStart) + MINMATCH; currMl = ZSTD_count_2segments(ip + MINMATCH, match + MINMATCH, iLimit, dictEnd, prefixStart) + MINMATCH;
} }
@@ -284,8 +282,10 @@ static U32 ZSTD_insertBtAndGetAllMatches (
matches[mnum].off = ZSTD_REP_MOVE_OPT + curr - matchIndex3; matches[mnum].off = ZSTD_REP_MOVE_OPT + curr - matchIndex3;
matches[mnum].len = (U32)currMl; matches[mnum].len = (U32)currMl;
mnum++; mnum++;
if (currMl > ZSTD_OPT_NUM) goto update; if (currMl > ZSTD_OPT_NUM)
if (ip+currMl == iLimit) goto update; /* best possible, and avoid read overflow*/ goto update;
if (ip + currMl == iLimit)
goto update; /* best possible, and avoid read overflow*/
} }
} }
} }
@@ -310,12 +310,14 @@ static U32 ZSTD_insertBtAndGetAllMatches (
} }
if (matchLength > bestLength) { if (matchLength > bestLength) {
if (matchLength > matchEndIdx - matchIndex) matchEndIdx = matchIndex + (U32)matchLength; if (matchLength > matchEndIdx - matchIndex)
matchEndIdx = matchIndex + (U32)matchLength;
bestLength = matchLength; bestLength = matchLength;
matches[mnum].off = ZSTD_REP_MOVE_OPT + curr - matchIndex; matches[mnum].off = ZSTD_REP_MOVE_OPT + curr - matchIndex;
matches[mnum].len = (U32)matchLength; matches[mnum].len = (U32)matchLength;
mnum++; mnum++;
if (matchLength > ZSTD_OPT_NUM) break; if (matchLength > ZSTD_OPT_NUM)
break;
if (ip + matchLength == iLimit) /* equal : no way to know if inf or sup */ if (ip + matchLength == iLimit) /* equal : no way to know if inf or sup */
break; /* drop, to guarantee consistency (miss a little bit of compression) */ break; /* drop, to guarantee consistency (miss a little bit of compression) */
} }
@@ -324,17 +326,24 @@ static U32 ZSTD_insertBtAndGetAllMatches (
/* match is smaller than curr */ /* match is smaller than curr */
*smallerPtr = matchIndex; /* update smaller idx */ *smallerPtr = matchIndex; /* update smaller idx */
commonLengthSmaller = matchLength; /* all smaller will now have at least this guaranteed common length */ commonLengthSmaller = matchLength; /* all smaller will now have at least this guaranteed common length */
if (matchIndex <= btLow) { smallerPtr=&dummy32; break; } /* beyond tree size, stop the search */ if (matchIndex <= btLow) {
smallerPtr = &dummy32;
break;
} /* beyond tree size, stop the search */
smallerPtr = nextPtr + 1; /* new "smaller" => larger of match */ smallerPtr = nextPtr + 1; /* new "smaller" => larger of match */
matchIndex = nextPtr[1]; /* new matchIndex larger than previous (closer to curr) */ matchIndex = nextPtr[1]; /* new matchIndex larger than previous (closer to curr) */
} else { } else {
/* match is larger than curr */ /* match is larger than curr */
*largerPtr = matchIndex; *largerPtr = matchIndex;
commonLengthLarger = matchLength; commonLengthLarger = matchLength;
if (matchIndex <= btLow) { largerPtr=&dummy32; break; } /* beyond tree size, stop the search */ if (matchIndex <= btLow) {
largerPtr = &dummy32;
break;
} /* beyond tree size, stop the search */
largerPtr = nextPtr; largerPtr = nextPtr;
matchIndex = nextPtr[0]; matchIndex = nextPtr[0];
} } }
}
*smallerPtr = *largerPtr = 0; *smallerPtr = *largerPtr = 0;
@@ -343,26 +352,21 @@ update:
return mnum; return mnum;
} }
/** Tree updater, providing best match */ /** Tree updater, providing best match */
static U32 ZSTD_BtGetAllMatches ( static U32 ZSTD_BtGetAllMatches(ZSTD_CCtx *zc, const BYTE *const ip, const BYTE *const iLimit, const U32 maxNbAttempts, const U32 mls, ZSTD_match_t *matches,
ZSTD_CCtx* zc, const U32 minMatchLen)
const BYTE* const ip, const BYTE* const iLimit,
const U32 maxNbAttempts, const U32 mls, ZSTD_match_t* matches, const U32 minMatchLen)
{ {
if (ip < zc->base + zc->nextToUpdate) return 0; /* skipped area */ if (ip < zc->base + zc->nextToUpdate)
return 0; /* skipped area */
ZSTD_updateTree(zc, ip, iLimit, maxNbAttempts, mls); ZSTD_updateTree(zc, ip, iLimit, maxNbAttempts, mls);
return ZSTD_insertBtAndGetAllMatches(zc, ip, iLimit, maxNbAttempts, mls, 0, matches, minMatchLen); return ZSTD_insertBtAndGetAllMatches(zc, ip, iLimit, maxNbAttempts, mls, 0, matches, minMatchLen);
} }
static U32 ZSTD_BtGetAllMatches_selectMLS(ZSTD_CCtx *zc, /* Index table will be updated */
static U32 ZSTD_BtGetAllMatches_selectMLS ( const BYTE *ip, const BYTE *const iHighLimit, const U32 maxNbAttempts, const U32 matchLengthSearch,
ZSTD_CCtx* zc, /* Index table will be updated */ ZSTD_match_t *matches, const U32 minMatchLen)
const BYTE* ip, const BYTE* const iHighLimit,
const U32 maxNbAttempts, const U32 matchLengthSearch, ZSTD_match_t* matches, const U32 minMatchLen)
{
switch(matchLengthSearch)
{ {
switch (matchLengthSearch) {
case 3: return ZSTD_BtGetAllMatches(zc, ip, iHighLimit, maxNbAttempts, 3, matches, minMatchLen); case 3: return ZSTD_BtGetAllMatches(zc, ip, iHighLimit, maxNbAttempts, 3, matches, minMatchLen);
default: default:
case 4: return ZSTD_BtGetAllMatches(zc, ip, iHighLimit, maxNbAttempts, 4, matches, minMatchLen); case 4: return ZSTD_BtGetAllMatches(zc, ip, iHighLimit, maxNbAttempts, 4, matches, minMatchLen);
@@ -373,24 +377,20 @@ static U32 ZSTD_BtGetAllMatches_selectMLS (
} }
/** Tree updater, providing best match */ /** Tree updater, providing best match */
static U32 ZSTD_BtGetAllMatches_extDict ( static U32 ZSTD_BtGetAllMatches_extDict(ZSTD_CCtx *zc, const BYTE *const ip, const BYTE *const iLimit, const U32 maxNbAttempts, const U32 mls,
ZSTD_CCtx* zc, ZSTD_match_t *matches, const U32 minMatchLen)
const BYTE* const ip, const BYTE* const iLimit,
const U32 maxNbAttempts, const U32 mls, ZSTD_match_t* matches, const U32 minMatchLen)
{ {
if (ip < zc->base + zc->nextToUpdate) return 0; /* skipped area */ if (ip < zc->base + zc->nextToUpdate)
return 0; /* skipped area */
ZSTD_updateTree_extDict(zc, ip, iLimit, maxNbAttempts, mls); ZSTD_updateTree_extDict(zc, ip, iLimit, maxNbAttempts, mls);
return ZSTD_insertBtAndGetAllMatches(zc, ip, iLimit, maxNbAttempts, mls, 1, matches, minMatchLen); return ZSTD_insertBtAndGetAllMatches(zc, ip, iLimit, maxNbAttempts, mls, 1, matches, minMatchLen);
} }
static U32 ZSTD_BtGetAllMatches_selectMLS_extDict(ZSTD_CCtx *zc, /* Index table will be updated */
static U32 ZSTD_BtGetAllMatches_selectMLS_extDict ( const BYTE *ip, const BYTE *const iHighLimit, const U32 maxNbAttempts, const U32 matchLengthSearch,
ZSTD_CCtx* zc, /* Index table will be updated */ ZSTD_match_t *matches, const U32 minMatchLen)
const BYTE* ip, const BYTE* const iHighLimit,
const U32 maxNbAttempts, const U32 matchLengthSearch, ZSTD_match_t* matches, const U32 minMatchLen)
{
switch(matchLengthSearch)
{ {
switch (matchLengthSearch) {
case 3: return ZSTD_BtGetAllMatches_extDict(zc, ip, iHighLimit, maxNbAttempts, 3, matches, minMatchLen); case 3: return ZSTD_BtGetAllMatches_extDict(zc, ip, iHighLimit, maxNbAttempts, 3, matches, minMatchLen);
default: default:
case 4: return ZSTD_BtGetAllMatches_extDict(zc, ip, iHighLimit, maxNbAttempts, 4, matches, minMatchLen); case 4: return ZSTD_BtGetAllMatches_extDict(zc, ip, iHighLimit, maxNbAttempts, 4, matches, minMatchLen);
@@ -400,13 +400,11 @@ static U32 ZSTD_BtGetAllMatches_selectMLS_extDict (
} }
} }
/*-******************************* /*-*******************************
* Optimal parser * Optimal parser
*********************************/ *********************************/
FORCE_INLINE FORCE_INLINE
void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx, void ZSTD_compressBlock_opt_generic(ZSTD_CCtx *ctx, const void *src, size_t srcSize, const int ultra)
const void* src, size_t srcSize, const int ultra)
{ {
seqStore_t *seqStorePtr = &(ctx->seqStore); seqStore_t *seqStorePtr = &(ctx->seqStore);
const BYTE *const istart = (const BYTE *)src; const BYTE *const istart = (const BYTE *)src;
@@ -431,7 +429,11 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx,
ctx->nextToUpdate3 = ctx->nextToUpdate; ctx->nextToUpdate3 = ctx->nextToUpdate;
ZSTD_rescaleFreqs(seqStorePtr, (const BYTE *)src, srcSize); ZSTD_rescaleFreqs(seqStorePtr, (const BYTE *)src, srcSize);
ip += (ip == prefixStart); ip += (ip == prefixStart);
{ U32 i; for (i=0; i<ZSTD_REP_NUM; i++) rep[i]=ctx->rep[i]; } {
U32 i;
for (i = 0; i < ZSTD_REP_NUM; i++)
rep[i] = ctx->rep[i];
}
/* Match Loop */ /* Match Loop */
while (ip < ilimit) { while (ip < ilimit) {
@@ -442,14 +444,18 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx,
litlen = (U32)(ip - anchor); litlen = (U32)(ip - anchor);
/* check repCode */ /* check repCode */
{ U32 i, last_i = ZSTD_REP_CHECK + (ip==anchor); {
U32 i, last_i = ZSTD_REP_CHECK + (ip == anchor);
for (i = (ip == anchor); i < last_i; i++) { for (i = (ip == anchor); i < last_i; i++) {
const S32 repCur = (i == ZSTD_REP_MOVE_OPT) ? (rep[0] - 1) : rep[i]; const S32 repCur = (i == ZSTD_REP_MOVE_OPT) ? (rep[0] - 1) : rep[i];
if ( (repCur > 0) && (repCur < (S32)(ip-prefixStart)) if ((repCur > 0) && (repCur < (S32)(ip - prefixStart)) &&
&& (ZSTD_readMINMATCH(ip, minMatch) == ZSTD_readMINMATCH(ip - repCur, minMatch))) { (ZSTD_readMINMATCH(ip, minMatch) == ZSTD_readMINMATCH(ip - repCur, minMatch))) {
mlen = (U32)ZSTD_count(ip + minMatch, ip + minMatch - repCur, iend) + minMatch; mlen = (U32)ZSTD_count(ip + minMatch, ip + minMatch - repCur, iend) + minMatch;
if (mlen > sufficient_len || mlen >= ZSTD_OPT_NUM) { if (mlen > sufficient_len || mlen >= ZSTD_OPT_NUM) {
best_mlen = mlen; best_off = i; cur = 0; last_pos = 1; best_mlen = mlen;
best_off = i;
cur = 0;
last_pos = 1;
goto _storeSequence; goto _storeSequence;
} }
best_off = i - (ip == anchor); best_off = i - (ip == anchor);
@@ -459,11 +465,16 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx,
SET_PRICE(mlen, mlen, i, litlen, price); /* note : macro modifies last_pos */ SET_PRICE(mlen, mlen, i, litlen, price); /* note : macro modifies last_pos */
mlen--; mlen--;
} while (mlen >= minMatch); } while (mlen >= minMatch);
} } } }
}
}
match_num = ZSTD_BtGetAllMatches_selectMLS(ctx, ip, iend, maxSearches, mls, matches, minMatch); match_num = ZSTD_BtGetAllMatches_selectMLS(ctx, ip, iend, maxSearches, mls, matches, minMatch);
if (!last_pos && !match_num) { ip++; continue; } if (!last_pos && !match_num) {
ip++;
continue;
}
if (match_num && (matches[match_num - 1].len > sufficient_len || matches[match_num - 1].len >= ZSTD_OPT_NUM)) { if (match_num && (matches[match_num - 1].len > sufficient_len || matches[match_num - 1].len >= ZSTD_OPT_NUM)) {
best_mlen = matches[match_num - 1].len; best_mlen = matches[match_num - 1].len;
@@ -483,12 +494,20 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx,
if (mlen > last_pos || price < opt[mlen].price) if (mlen > last_pos || price < opt[mlen].price)
SET_PRICE(mlen, mlen, matches[u].off, litlen, price); /* note : macro modifies last_pos */ SET_PRICE(mlen, mlen, matches[u].off, litlen, price); /* note : macro modifies last_pos */
mlen++; mlen++;
} } }
}
if (last_pos < minMatch) { ip++; continue; } if (last_pos < minMatch) {
ip++;
continue;
}
/* initialize opt[0] */ /* initialize opt[0] */
{ U32 i ; for (i=0; i<ZSTD_REP_NUM; i++) opt[0].rep[i] = rep[i]; } {
U32 i;
for (i = 0; i < ZSTD_REP_NUM; i++)
opt[0].rep[i] = rep[i];
}
opt[0].mlen = 1; opt[0].mlen = 1;
opt[0].litlen = litlen; opt[0].litlen = litlen;
@@ -510,7 +529,8 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx,
if (cur > last_pos || price <= opt[cur].price) if (cur > last_pos || price <= opt[cur].price)
SET_PRICE(cur, 1, 0, litlen, price); SET_PRICE(cur, 1, 0, litlen, price);
if (cur == last_pos) break; if (cur == last_pos)
break;
if (inr > ilimit) /* last match must start at a minimum distance of 8 from oend */ if (inr > ilimit) /* last match must start at a minimum distance of 8 from oend */
continue; continue;
@@ -523,30 +543,36 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx,
} else { } else {
opt[cur].rep[2] = (opt[cur].off > 1) ? opt[cur - mlen].rep[1] : opt[cur - mlen].rep[2]; opt[cur].rep[2] = (opt[cur].off > 1) ? opt[cur - mlen].rep[1] : opt[cur - mlen].rep[2];
opt[cur].rep[1] = (opt[cur].off > 0) ? opt[cur - mlen].rep[0] : opt[cur - mlen].rep[1]; opt[cur].rep[1] = (opt[cur].off > 0) ? opt[cur - mlen].rep[0] : opt[cur - mlen].rep[1];
opt[cur].rep[0] = ((opt[cur].off==ZSTD_REP_MOVE_OPT) && (mlen != 1)) ? (opt[cur-mlen].rep[0] - 1) : (opt[cur-mlen].rep[opt[cur].off]); opt[cur].rep[0] =
((opt[cur].off == ZSTD_REP_MOVE_OPT) && (mlen != 1)) ? (opt[cur - mlen].rep[0] - 1) : (opt[cur - mlen].rep[opt[cur].off]);
} }
best_mlen = minMatch; best_mlen = minMatch;
{ U32 i, last_i = ZSTD_REP_CHECK + (mlen != 1); {
U32 i, last_i = ZSTD_REP_CHECK + (mlen != 1);
for (i = (opt[cur].mlen != 1); i < last_i; i++) { /* check rep */ for (i = (opt[cur].mlen != 1); i < last_i; i++) { /* check rep */
const S32 repCur = (i == ZSTD_REP_MOVE_OPT) ? (opt[cur].rep[0] - 1) : opt[cur].rep[i]; const S32 repCur = (i == ZSTD_REP_MOVE_OPT) ? (opt[cur].rep[0] - 1) : opt[cur].rep[i];
if ( (repCur > 0) && (repCur < (S32)(inr-prefixStart)) if ((repCur > 0) && (repCur < (S32)(inr - prefixStart)) &&
&& (ZSTD_readMINMATCH(inr, minMatch) == ZSTD_readMINMATCH(inr - repCur, minMatch))) { (ZSTD_readMINMATCH(inr, minMatch) == ZSTD_readMINMATCH(inr - repCur, minMatch))) {
mlen = (U32)ZSTD_count(inr + minMatch, inr + minMatch - repCur, iend) + minMatch; mlen = (U32)ZSTD_count(inr + minMatch, inr + minMatch - repCur, iend) + minMatch;
if (mlen > sufficient_len || cur + mlen >= ZSTD_OPT_NUM) { if (mlen > sufficient_len || cur + mlen >= ZSTD_OPT_NUM) {
best_mlen = mlen; best_off = i; last_pos = cur + 1; best_mlen = mlen;
best_off = i;
last_pos = cur + 1;
goto _storeSequence; goto _storeSequence;
} }
best_off = i - (opt[cur].mlen != 1); best_off = i - (opt[cur].mlen != 1);
if (mlen > best_mlen) best_mlen = mlen; if (mlen > best_mlen)
best_mlen = mlen;
do { do {
if (opt[cur].mlen == 1) { if (opt[cur].mlen == 1) {
litlen = opt[cur].litlen; litlen = opt[cur].litlen;
if (cur > litlen) { if (cur > litlen) {
price = opt[cur - litlen].price + ZSTD_getPrice(seqStorePtr, litlen, inr-litlen, best_off, mlen - MINMATCH, ultra); price = opt[cur - litlen].price + ZSTD_getPrice(seqStorePtr, litlen, inr - litlen,
best_off, mlen - MINMATCH, ultra);
} else } else
price = ZSTD_getPrice(seqStorePtr, litlen, anchor, best_off, mlen - MINMATCH, ultra); price = ZSTD_getPrice(seqStorePtr, litlen, anchor, best_off, mlen - MINMATCH, ultra);
} else { } else {
@@ -558,7 +584,9 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx,
SET_PRICE(cur + mlen, mlen, i, litlen, price); SET_PRICE(cur + mlen, mlen, i, litlen, price);
mlen--; mlen--;
} while (mlen >= minMatch); } while (mlen >= minMatch);
} } } }
}
}
match_num = ZSTD_BtGetAllMatches_selectMLS(ctx, inr, iend, maxSearches, mls, matches, best_mlen); match_num = ZSTD_BtGetAllMatches_selectMLS(ctx, inr, iend, maxSearches, mls, matches, best_mlen);
@@ -578,7 +606,8 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx,
if (opt[cur].mlen == 1) { if (opt[cur].mlen == 1) {
litlen = opt[cur].litlen; litlen = opt[cur].litlen;
if (cur > litlen) if (cur > litlen)
price = opt[cur - litlen].price + ZSTD_getPrice(seqStorePtr, litlen, ip+cur-litlen, matches[u].off-1, mlen - MINMATCH, ultra); price = opt[cur - litlen].price + ZSTD_getPrice(seqStorePtr, litlen, ip + cur - litlen,
matches[u].off - 1, mlen - MINMATCH, ultra);
else else
price = ZSTD_getPrice(seqStorePtr, litlen, anchor, matches[u].off - 1, mlen - MINMATCH, ultra); price = ZSTD_getPrice(seqStorePtr, litlen, anchor, matches[u].off - 1, mlen - MINMATCH, ultra);
} else { } else {
@@ -590,7 +619,9 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx,
SET_PRICE(cur + mlen, mlen, matches[u].off, litlen, price); SET_PRICE(cur + mlen, mlen, matches[u].off, litlen, price);
mlen++; mlen++;
} } } }
}
}
best_mlen = opt[last_pos].mlen; best_mlen = opt[last_pos].mlen;
best_off = opt[last_pos].off; best_off = opt[last_pos].off;
@@ -607,7 +638,8 @@ _storeSequence: /* cur, last_pos, best_mlen, best_off have to be set */
opt[cur].off = best_off; opt[cur].off = best_off;
best_mlen = mlen; best_mlen = mlen;
best_off = offset; best_off = offset;
if (mlen > cur) break; if (mlen > cur)
break;
cur -= mlen; cur -= mlen;
} }
@@ -617,7 +649,11 @@ _storeSequence: /* cur, last_pos, best_mlen, best_off have to be set */
for (cur = 0; cur < last_pos;) { for (cur = 0; cur < last_pos;) {
mlen = opt[cur].mlen; mlen = opt[cur].mlen;
if (mlen == 1) { ip++; cur++; continue; } if (mlen == 1) {
ip++;
cur++;
continue;
}
offset = opt[cur].off; offset = opt[cur].off;
cur += mlen; cur += mlen;
litLength = (U32)(ip - anchor); litLength = (U32)(ip - anchor);
@@ -630,32 +666,38 @@ _storeSequence: /* cur, last_pos, best_mlen, best_off have to be set */
} else { } else {
if (offset != 0) { if (offset != 0) {
best_off = (offset == ZSTD_REP_MOVE_OPT) ? (rep[0] - 1) : (rep[offset]); best_off = (offset == ZSTD_REP_MOVE_OPT) ? (rep[0] - 1) : (rep[offset]);
if (offset != 1) rep[2] = rep[1]; if (offset != 1)
rep[2] = rep[1];
rep[1] = rep[0]; rep[1] = rep[0];
rep[0] = best_off; rep[0] = best_off;
} }
if (litLength==0) offset--; if (litLength == 0)
offset--;
} }
ZSTD_updatePrice(seqStorePtr, litLength, anchor, offset, mlen - MINMATCH); ZSTD_updatePrice(seqStorePtr, litLength, anchor, offset, mlen - MINMATCH);
ZSTD_storeSeq(seqStorePtr, litLength, anchor, offset, mlen - MINMATCH); ZSTD_storeSeq(seqStorePtr, litLength, anchor, offset, mlen - MINMATCH);
anchor = ip = ip + mlen; anchor = ip = ip + mlen;
} } /* for (cur=0; cur < last_pos; ) */ }
} /* for (cur=0; cur < last_pos; ) */
/* Save reps for next block */ /* Save reps for next block */
{ int i; for (i=0; i<ZSTD_REP_NUM; i++) ctx->repToConfirm[i] = rep[i]; } {
int i;
for (i = 0; i < ZSTD_REP_NUM; i++)
ctx->repToConfirm[i] = rep[i];
}
/* Last Literals */ /* Last Literals */
{ size_t const lastLLSize = iend - anchor; {
size_t const lastLLSize = iend - anchor;
memcpy(seqStorePtr->lit, anchor, lastLLSize); memcpy(seqStorePtr->lit, anchor, lastLLSize);
seqStorePtr->lit += lastLLSize; seqStorePtr->lit += lastLLSize;
} }
} }
FORCE_INLINE FORCE_INLINE
void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx, void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx *ctx, const void *src, size_t srcSize, const int ultra)
const void* src, size_t srcSize, const int ultra)
{ {
seqStore_t *seqStorePtr = &(ctx->seqStore); seqStore_t *seqStorePtr = &(ctx->seqStore);
const BYTE *const istart = (const BYTE *)src; const BYTE *const istart = (const BYTE *)src;
@@ -681,7 +723,11 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx,
/* init */ /* init */
U32 offset, rep[ZSTD_REP_NUM]; U32 offset, rep[ZSTD_REP_NUM];
{ U32 i; for (i=0; i<ZSTD_REP_NUM; i++) rep[i]=ctx->rep[i]; } {
U32 i;
for (i = 0; i < ZSTD_REP_NUM; i++)
rep[i] = ctx->rep[i];
}
ctx->nextToUpdate3 = ctx->nextToUpdate; ctx->nextToUpdate3 = ctx->nextToUpdate;
ZSTD_rescaleFreqs(seqStorePtr, (const BYTE *)src, srcSize); ZSTD_rescaleFreqs(seqStorePtr, (const BYTE *)src, srcSize);
@@ -697,21 +743,25 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx,
opt[0].litlen = (U32)(ip - anchor); opt[0].litlen = (U32)(ip - anchor);
/* check repCode */ /* check repCode */
{ U32 i, last_i = ZSTD_REP_CHECK + (ip==anchor); {
U32 i, last_i = ZSTD_REP_CHECK + (ip == anchor);
for (i = (ip == anchor); i < last_i; i++) { for (i = (ip == anchor); i < last_i; i++) {
const S32 repCur = (i == ZSTD_REP_MOVE_OPT) ? (rep[0] - 1) : rep[i]; const S32 repCur = (i == ZSTD_REP_MOVE_OPT) ? (rep[0] - 1) : rep[i];
const U32 repIndex = (U32)(curr - repCur); const U32 repIndex = (U32)(curr - repCur);
const BYTE *const repBase = repIndex < dictLimit ? dictBase : base; const BYTE *const repBase = repIndex < dictLimit ? dictBase : base;
const BYTE *const repMatch = repBase + repIndex; const BYTE *const repMatch = repBase + repIndex;
if ( (repCur > 0 && repCur <= (S32)curr) if ((repCur > 0 && repCur <= (S32)curr) &&
&& (((U32)((dictLimit-1) - repIndex) >= 3) & (repIndex>lowestIndex)) /* intentional overflow */ (((U32)((dictLimit - 1) - repIndex) >= 3) & (repIndex > lowestIndex)) /* intentional overflow */
&& (ZSTD_readMINMATCH(ip, minMatch) == ZSTD_readMINMATCH(repMatch, minMatch))) { && (ZSTD_readMINMATCH(ip, minMatch) == ZSTD_readMINMATCH(repMatch, minMatch))) {
/* repcode detected we should take it */ /* repcode detected we should take it */
const BYTE *const repEnd = repIndex < dictLimit ? dictEnd : iend; const BYTE *const repEnd = repIndex < dictLimit ? dictEnd : iend;
mlen = (U32)ZSTD_count_2segments(ip + minMatch, repMatch + minMatch, iend, repEnd, prefixStart) + minMatch; mlen = (U32)ZSTD_count_2segments(ip + minMatch, repMatch + minMatch, iend, repEnd, prefixStart) + minMatch;
if (mlen > sufficient_len || mlen >= ZSTD_OPT_NUM) { if (mlen > sufficient_len || mlen >= ZSTD_OPT_NUM) {
best_mlen = mlen; best_off = i; cur = 0; last_pos = 1; best_mlen = mlen;
best_off = i;
cur = 0;
last_pos = 1;
goto _storeSequence; goto _storeSequence;
} }
@@ -723,13 +773,22 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx,
SET_PRICE(mlen, mlen, i, litlen, price); /* note : macro modifies last_pos */ SET_PRICE(mlen, mlen, i, litlen, price); /* note : macro modifies last_pos */
mlen--; mlen--;
} while (mlen >= minMatch); } while (mlen >= minMatch);
} } } }
}
}
match_num = ZSTD_BtGetAllMatches_selectMLS_extDict(ctx, ip, iend, maxSearches, mls, matches, minMatch); /* first search (depth 0) */ match_num = ZSTD_BtGetAllMatches_selectMLS_extDict(ctx, ip, iend, maxSearches, mls, matches, minMatch); /* first search (depth 0) */
if (!last_pos && !match_num) { ip++; continue; } if (!last_pos && !match_num) {
ip++;
continue;
}
{ U32 i; for (i=0; i<ZSTD_REP_NUM; i++) opt[0].rep[i] = rep[i]; } {
U32 i;
for (i = 0; i < ZSTD_REP_NUM; i++)
opt[0].rep[i] = rep[i];
}
opt[0].mlen = 1; opt[0].mlen = 1;
if (match_num && (matches[match_num - 1].len > sufficient_len || matches[match_num - 1].len >= ZSTD_OPT_NUM)) { if (match_num && (matches[match_num - 1].len > sufficient_len || matches[match_num - 1].len >= ZSTD_OPT_NUM)) {
@@ -752,10 +811,12 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx,
if (mlen > last_pos || price < opt[mlen].price) if (mlen > last_pos || price < opt[mlen].price)
SET_PRICE(mlen, mlen, matches[u].off, litlen, price); SET_PRICE(mlen, mlen, matches[u].off, litlen, price);
mlen++; mlen++;
} } }
}
if (last_pos < minMatch) { if (last_pos < minMatch) {
ip++; continue; ip++;
continue;
} }
/* check further positions */ /* check further positions */
@@ -776,7 +837,8 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx,
if (cur > last_pos || price <= opt[cur].price) if (cur > last_pos || price <= opt[cur].price)
SET_PRICE(cur, 1, 0, litlen, price); SET_PRICE(cur, 1, 0, litlen, price);
if (cur == last_pos) break; if (cur == last_pos)
break;
if (inr > ilimit) /* last match must start at a minimum distance of 8 from oend */ if (inr > ilimit) /* last match must start at a minimum distance of 8 from oend */
continue; continue;
@@ -789,36 +851,42 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx,
} else { } else {
opt[cur].rep[2] = (opt[cur].off > 1) ? opt[cur - mlen].rep[1] : opt[cur - mlen].rep[2]; opt[cur].rep[2] = (opt[cur].off > 1) ? opt[cur - mlen].rep[1] : opt[cur - mlen].rep[2];
opt[cur].rep[1] = (opt[cur].off > 0) ? opt[cur - mlen].rep[0] : opt[cur - mlen].rep[1]; opt[cur].rep[1] = (opt[cur].off > 0) ? opt[cur - mlen].rep[0] : opt[cur - mlen].rep[1];
opt[cur].rep[0] = ((opt[cur].off==ZSTD_REP_MOVE_OPT) && (mlen != 1)) ? (opt[cur-mlen].rep[0] - 1) : (opt[cur-mlen].rep[opt[cur].off]); opt[cur].rep[0] =
((opt[cur].off == ZSTD_REP_MOVE_OPT) && (mlen != 1)) ? (opt[cur - mlen].rep[0] - 1) : (opt[cur - mlen].rep[opt[cur].off]);
} }
best_mlen = minMatch; best_mlen = minMatch;
{ U32 i, last_i = ZSTD_REP_CHECK + (mlen != 1); {
U32 i, last_i = ZSTD_REP_CHECK + (mlen != 1);
for (i = (mlen != 1); i < last_i; i++) { for (i = (mlen != 1); i < last_i; i++) {
const S32 repCur = (i == ZSTD_REP_MOVE_OPT) ? (opt[cur].rep[0] - 1) : opt[cur].rep[i]; const S32 repCur = (i == ZSTD_REP_MOVE_OPT) ? (opt[cur].rep[0] - 1) : opt[cur].rep[i];
const U32 repIndex = (U32)(curr + cur - repCur); const U32 repIndex = (U32)(curr + cur - repCur);
const BYTE *const repBase = repIndex < dictLimit ? dictBase : base; const BYTE *const repBase = repIndex < dictLimit ? dictBase : base;
const BYTE *const repMatch = repBase + repIndex; const BYTE *const repMatch = repBase + repIndex;
if ( (repCur > 0 && repCur <= (S32)(curr+cur)) if ((repCur > 0 && repCur <= (S32)(curr + cur)) &&
&& (((U32)((dictLimit-1) - repIndex) >= 3) & (repIndex>lowestIndex)) /* intentional overflow */ (((U32)((dictLimit - 1) - repIndex) >= 3) & (repIndex > lowestIndex)) /* intentional overflow */
&& (ZSTD_readMINMATCH(inr, minMatch) == ZSTD_readMINMATCH(repMatch, minMatch))) { && (ZSTD_readMINMATCH(inr, minMatch) == ZSTD_readMINMATCH(repMatch, minMatch))) {
/* repcode detected */ /* repcode detected */
const BYTE *const repEnd = repIndex < dictLimit ? dictEnd : iend; const BYTE *const repEnd = repIndex < dictLimit ? dictEnd : iend;
mlen = (U32)ZSTD_count_2segments(inr + minMatch, repMatch + minMatch, iend, repEnd, prefixStart) + minMatch; mlen = (U32)ZSTD_count_2segments(inr + minMatch, repMatch + minMatch, iend, repEnd, prefixStart) + minMatch;
if (mlen > sufficient_len || cur + mlen >= ZSTD_OPT_NUM) { if (mlen > sufficient_len || cur + mlen >= ZSTD_OPT_NUM) {
best_mlen = mlen; best_off = i; last_pos = cur + 1; best_mlen = mlen;
best_off = i;
last_pos = cur + 1;
goto _storeSequence; goto _storeSequence;
} }
best_off = i - (opt[cur].mlen != 1); best_off = i - (opt[cur].mlen != 1);
if (mlen > best_mlen) best_mlen = mlen; if (mlen > best_mlen)
best_mlen = mlen;
do { do {
if (opt[cur].mlen == 1) { if (opt[cur].mlen == 1) {
litlen = opt[cur].litlen; litlen = opt[cur].litlen;
if (cur > litlen) { if (cur > litlen) {
price = opt[cur - litlen].price + ZSTD_getPrice(seqStorePtr, litlen, inr-litlen, best_off, mlen - MINMATCH, ultra); price = opt[cur - litlen].price + ZSTD_getPrice(seqStorePtr, litlen, inr - litlen,
best_off, mlen - MINMATCH, ultra);
} else } else
price = ZSTD_getPrice(seqStorePtr, litlen, anchor, best_off, mlen - MINMATCH, ultra); price = ZSTD_getPrice(seqStorePtr, litlen, anchor, best_off, mlen - MINMATCH, ultra);
} else { } else {
@@ -830,7 +898,9 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx,
SET_PRICE(cur + mlen, mlen, i, litlen, price); SET_PRICE(cur + mlen, mlen, i, litlen, price);
mlen--; mlen--;
} while (mlen >= minMatch); } while (mlen >= minMatch);
} } } }
}
}
match_num = ZSTD_BtGetAllMatches_selectMLS_extDict(ctx, inr, iend, maxSearches, mls, matches, minMatch); match_num = ZSTD_BtGetAllMatches_selectMLS_extDict(ctx, inr, iend, maxSearches, mls, matches, minMatch);
@@ -850,7 +920,8 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx,
if (opt[cur].mlen == 1) { if (opt[cur].mlen == 1) {
litlen = opt[cur].litlen; litlen = opt[cur].litlen;
if (cur > litlen) if (cur > litlen)
price = opt[cur - litlen].price + ZSTD_getPrice(seqStorePtr, litlen, ip+cur-litlen, matches[u].off-1, mlen - MINMATCH, ultra); price = opt[cur - litlen].price + ZSTD_getPrice(seqStorePtr, litlen, ip + cur - litlen,
matches[u].off - 1, mlen - MINMATCH, ultra);
else else
price = ZSTD_getPrice(seqStorePtr, litlen, anchor, matches[u].off - 1, mlen - MINMATCH, ultra); price = ZSTD_getPrice(seqStorePtr, litlen, anchor, matches[u].off - 1, mlen - MINMATCH, ultra);
} else { } else {
@@ -862,7 +933,9 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx,
SET_PRICE(cur + mlen, mlen, matches[u].off, litlen, price); SET_PRICE(cur + mlen, mlen, matches[u].off, litlen, price);
mlen++; mlen++;
} } } /* for (cur = 1; cur <= last_pos; cur++) */ }
}
} /* for (cur = 1; cur <= last_pos; cur++) */
best_mlen = opt[last_pos].mlen; best_mlen = opt[last_pos].mlen;
best_off = opt[last_pos].off; best_off = opt[last_pos].off;
@@ -879,7 +952,8 @@ _storeSequence: /* cur, last_pos, best_mlen, best_off have to be set */
opt[cur].off = best_off; opt[cur].off = best_off;
best_mlen = mlen; best_mlen = mlen;
best_off = offset; best_off = offset;
if (mlen > cur) break; if (mlen > cur)
break;
cur -= mlen; cur -= mlen;
} }
@@ -889,7 +963,11 @@ _storeSequence: /* cur, last_pos, best_mlen, best_off have to be set */
for (cur = 0; cur < last_pos;) { for (cur = 0; cur < last_pos;) {
mlen = opt[cur].mlen; mlen = opt[cur].mlen;
if (mlen == 1) { ip++; cur++; continue; } if (mlen == 1) {
ip++;
cur++;
continue;
}
offset = opt[cur].off; offset = opt[cur].off;
cur += mlen; cur += mlen;
litLength = (U32)(ip - anchor); litLength = (U32)(ip - anchor);
@@ -902,24 +980,32 @@ _storeSequence: /* cur, last_pos, best_mlen, best_off have to be set */
} else { } else {
if (offset != 0) { if (offset != 0) {
best_off = (offset == ZSTD_REP_MOVE_OPT) ? (rep[0] - 1) : (rep[offset]); best_off = (offset == ZSTD_REP_MOVE_OPT) ? (rep[0] - 1) : (rep[offset]);
if (offset != 1) rep[2] = rep[1]; if (offset != 1)
rep[2] = rep[1];
rep[1] = rep[0]; rep[1] = rep[0];
rep[0] = best_off; rep[0] = best_off;
} }
if (litLength==0) offset--; if (litLength == 0)
offset--;
} }
ZSTD_updatePrice(seqStorePtr, litLength, anchor, offset, mlen - MINMATCH); ZSTD_updatePrice(seqStorePtr, litLength, anchor, offset, mlen - MINMATCH);
ZSTD_storeSeq(seqStorePtr, litLength, anchor, offset, mlen - MINMATCH); ZSTD_storeSeq(seqStorePtr, litLength, anchor, offset, mlen - MINMATCH);
anchor = ip = ip + mlen; anchor = ip = ip + mlen;
} } /* for (cur=0; cur < last_pos; ) */ }
} /* for (cur=0; cur < last_pos; ) */
/* Save reps for next block */ /* Save reps for next block */
{ int i; for (i=0; i<ZSTD_REP_NUM; i++) ctx->repToConfirm[i] = rep[i]; } {
int i;
for (i = 0; i < ZSTD_REP_NUM; i++)
ctx->repToConfirm[i] = rep[i];
}
/* Last Literals */ /* Last Literals */
{ size_t lastLLSize = iend - anchor; {
size_t lastLLSize = iend - anchor;
memcpy(seqStorePtr->lit, anchor, lastLLSize); memcpy(seqStorePtr->lit, anchor, lastLLSize);
seqStorePtr->lit += lastLLSize; seqStorePtr->lit += lastLLSize;
} }