ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[linux-kernel] clang-format lib/zstd/

Browse Source
This commit is contained in:
Nick Terrell
2017-05-17 14:40:46 -07:00
parent 588579f3a1
commit 67101c6c77
15 changed files with 5241 additions and 4655 deletions
Download Patch File Download Diff File Expand all files Collapse all files
contrib/linux-kernel/lib/zstd/bitstream.h
+92 -100
View File
@@ -50,9 +50,8 @@
/*-**************************************** /*-****************************************
* Dependencies * Dependencies
******************************************/ ******************************************/
#include "mem.h" /* unaligned access routines */
#include "error_private.h" /* error codes and messages */ #include "error_private.h" /* error codes and messages */
#include "mem.h" /* unaligned access routines */
/*========================================= /*=========================================
* Target specific * Target specific
@@ -68,19 +67,18 @@
* A critical property of these streams is that they encode and decode in **reverse** direction. * A critical property of these streams is that they encode and decode in **reverse** direction.
* So the first bit sequence you add will be the last to be read, like a LIFO stack. * So the first bit sequence you add will be the last to be read, like a LIFO stack.
*/ */
typedef struct typedef struct {
{
size_t bitContainer; size_t bitContainer;
int bitPos; int bitPos;
char* startPtr; char *startPtr;
char* ptr; char *ptr;
char* endPtr; char *endPtr;
} BIT_CStream_t; } BIT_CStream_t;
ZSTD_STATIC size_t BIT_initCStream(BIT_CStream_t* bitC, void* dstBuffer, size_t dstCapacity); ZSTD_STATIC size_t BIT_initCStream(BIT_CStream_t *bitC, void *dstBuffer, size_t dstCapacity);
ZSTD_STATIC void BIT_addBits(BIT_CStream_t* bitC, size_t value, unsigned nbBits); ZSTD_STATIC void BIT_addBits(BIT_CStream_t *bitC, size_t value, unsigned nbBits);
ZSTD_STATIC void BIT_flushBits(BIT_CStream_t* bitC); ZSTD_STATIC void BIT_flushBits(BIT_CStream_t *bitC);
ZSTD_STATIC size_t BIT_closeCStream(BIT_CStream_t* bitC); ZSTD_STATIC size_t BIT_closeCStream(BIT_CStream_t *bitC);
/* Start with initCStream, providing the size of buffer to write into. /* Start with initCStream, providing the size of buffer to write into.
* bitStream will never write outside of this buffer. * bitStream will never write outside of this buffer.
@@ -99,29 +97,28 @@ ZSTD_STATIC size_t BIT_closeCStream(BIT_CStream_t* bitC);
* If data couldn't fit into `dstBuffer`, it will return a 0 ( == not storable) * If data couldn't fit into `dstBuffer`, it will return a 0 ( == not storable)
*/ */
/*-******************************************** /*-********************************************
* bitStream decoding API (read backward) * bitStream decoding API (read backward)
**********************************************/ **********************************************/
typedef struct typedef struct {
{
size_t bitContainer; size_t bitContainer;
unsigned bitsConsumed; unsigned bitsConsumed;
const char* ptr; const char *ptr;
const char* start; const char *start;
} BIT_DStream_t; } BIT_DStream_t;
typedef enum { BIT_DStream_unfinished = 0, typedef enum {
BIT_DStream_unfinished = 0,
BIT_DStream_endOfBuffer = 1, BIT_DStream_endOfBuffer = 1,
BIT_DStream_completed = 2, BIT_DStream_completed = 2,
BIT_DStream_overflow = 3 } BIT_DStream_status; /* result of BIT_reloadDStream() */ BIT_DStream_overflow = 3
} BIT_DStream_status; /* result of BIT_reloadDStream() */
/* 1,2,4,8 would be better for bitmap combinations, but slows down performance a bit ... :( */ /* 1,2,4,8 would be better for bitmap combinations, but slows down performance a bit ... :( */
ZSTD_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, size_t srcSize); ZSTD_STATIC size_t BIT_initDStream(BIT_DStream_t *bitD, const void *srcBuffer, size_t srcSize);
ZSTD_STATIC size_t BIT_readBits(BIT_DStream_t* bitD, unsigned nbBits); ZSTD_STATIC size_t BIT_readBits(BIT_DStream_t *bitD, unsigned nbBits);
ZSTD_STATIC BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD); ZSTD_STATIC BIT_DStream_status BIT_reloadDStream(BIT_DStream_t *bitD);
ZSTD_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* bitD); ZSTD_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t *bitD);
/* Start by invoking BIT_initDStream(). /* Start by invoking BIT_initDStream().
* A chunk of the bitStream is then stored into a local register. * A chunk of the bitStream is then stored into a local register.
@@ -133,32 +130,27 @@ ZSTD_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* bitD);
* Checking if DStream has reached its end can be performed with BIT_endOfDStream(). * Checking if DStream has reached its end can be performed with BIT_endOfDStream().
*/ */
/*-**************************************** /*-****************************************
* unsafe API * unsafe API
******************************************/ ******************************************/
ZSTD_STATIC void BIT_addBitsFast(BIT_CStream_t* bitC, size_t value, unsigned nbBits); ZSTD_STATIC void BIT_addBitsFast(BIT_CStream_t *bitC, size_t value, unsigned nbBits);
/* faster, but works only if value is "clean", meaning all high bits above nbBits are 0 */ /* faster, but works only if value is "clean", meaning all high bits above nbBits are 0 */
ZSTD_STATIC void BIT_flushBitsFast(BIT_CStream_t* bitC); ZSTD_STATIC void BIT_flushBitsFast(BIT_CStream_t *bitC);
/* unsafe version; does not check buffer overflow */ /* unsafe version; does not check buffer overflow */
ZSTD_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, unsigned nbBits); ZSTD_STATIC size_t BIT_readBitsFast(BIT_DStream_t *bitD, unsigned nbBits);
/* faster, but works only if nbBits >= 1 */ /* faster, but works only if nbBits >= 1 */
/*-************************************************************** /*-**************************************************************
* Internal functions * Internal functions
****************************************************************/ ****************************************************************/
ZSTD_STATIC unsigned BIT_highbit32 (register U32 val) ZSTD_STATIC unsigned BIT_highbit32(register U32 val) { return 31 - __builtin_clz(val); }
{
return 31 - __builtin_clz(val);
}
/*===== Local Constants =====*/ /*===== Local Constants =====*/
static const unsigned BIT_mask[] = { 0, 1, 3, 7, 0xF, 0x1F, 0x3F, 0x7F, 0xFF, 0x1FF, 0x3FF, 0x7FF, 0xFFF, 0x1FFF, 0x3FFF, 0x7FFF, 0xFFFF, 0x1FFFF, 0x3FFFF, 0x7FFFF, 0xFFFFF, 0x1FFFFF, 0x3FFFFF, 0x7FFFFF, 0xFFFFFF, 0x1FFFFFF, 0x3FFFFFF }; /* up to 26 bits */ static const unsigned BIT_mask[] = {0, 1, 3, 7, 0xF, 0x1F, 0x3F, 0x7F, 0xFF,
0x1FF, 0x3FF, 0x7FF, 0xFFF, 0x1FFF, 0x3FFF, 0x7FFF, 0xFFFF, 0x1FFFF,
0x3FFFF, 0x7FFFF, 0xFFFFF, 0x1FFFFF, 0x3FFFFF, 0x7FFFFF, 0xFFFFFF, 0x1FFFFFF, 0x3FFFFFF}; /* up to 26 bits */
/*-************************************************************** /*-**************************************************************
* bitStream encoding * bitStream encoding
@@ -167,21 +159,22 @@ static const unsigned BIT_mask[] = { 0, 1, 3, 7, 0xF, 0x1F, 0x3F, 0x7F, 0xFF, 0x
* `dstCapacity` must be > sizeof(void*) * `dstCapacity` must be > sizeof(void*)
* @return : 0 if success, * @return : 0 if success,
otherwise an error code (can be tested using ERR_isError() ) */ otherwise an error code (can be tested using ERR_isError() ) */
ZSTD_STATIC size_t BIT_initCStream(BIT_CStream_t* bitC, void* startPtr, size_t dstCapacity) ZSTD_STATIC size_t BIT_initCStream(BIT_CStream_t *bitC, void *startPtr, size_t dstCapacity)
{ {
bitC->bitContainer = 0; bitC->bitContainer = 0;
bitC->bitPos = 0; bitC->bitPos = 0;
bitC->startPtr = (char*)startPtr; bitC->startPtr = (char *)startPtr;
bitC->ptr = bitC->startPtr; bitC->ptr = bitC->startPtr;
bitC->endPtr = bitC->startPtr + dstCapacity - sizeof(bitC->ptr); bitC->endPtr = bitC->startPtr + dstCapacity - sizeof(bitC->ptr);
if (dstCapacity <= sizeof(bitC->ptr)) return ERROR(dstSize_tooSmall); if (dstCapacity <= sizeof(bitC->ptr))
return ERROR(dstSize_tooSmall);
return 0; return 0;
} }
/*! BIT_addBits() : /*! BIT_addBits() :
can add up to 26 bits into `bitC`. can add up to 26 bits into `bitC`.
Does not check for register overflow ! */ Does not check for register overflow ! */
ZSTD_STATIC void BIT_addBits(BIT_CStream_t* bitC, size_t value, unsigned nbBits) ZSTD_STATIC void BIT_addBits(BIT_CStream_t *bitC, size_t value, unsigned nbBits)
{ {
bitC->bitContainer |= (value & BIT_mask[nbBits]) << bitC->bitPos; bitC->bitContainer |= (value & BIT_mask[nbBits]) << bitC->bitPos;
bitC->bitPos += nbBits; bitC->bitPos += nbBits;
@@ -189,7 +182,7 @@ ZSTD_STATIC void BIT_addBits(BIT_CStream_t* bitC, size_t value, unsigned nbBits)
/*! BIT_addBitsFast() : /*! BIT_addBitsFast() :
* works only if `value` is _clean_, meaning all high bits above nbBits are 0 */ * works only if `value` is _clean_, meaning all high bits above nbBits are 0 */
ZSTD_STATIC void BIT_addBitsFast(BIT_CStream_t* bitC, size_t value, unsigned nbBits) ZSTD_STATIC void BIT_addBitsFast(BIT_CStream_t *bitC, size_t value, unsigned nbBits)
{ {
bitC->bitContainer |= value << bitC->bitPos; bitC->bitContainer |= value << bitC->bitPos;
bitC->bitPos += nbBits; bitC->bitPos += nbBits;
@@ -197,42 +190,43 @@ ZSTD_STATIC void BIT_addBitsFast(BIT_CStream_t* bitC, size_t value, unsigned nbB
/*! BIT_flushBitsFast() : /*! BIT_flushBitsFast() :
* unsafe version; does not check buffer overflow */ * unsafe version; does not check buffer overflow */
ZSTD_STATIC void BIT_flushBitsFast(BIT_CStream_t* bitC) ZSTD_STATIC void BIT_flushBitsFast(BIT_CStream_t *bitC)
{ {
size_t const nbBytes = bitC->bitPos >> 3; size_t const nbBytes = bitC->bitPos >> 3;
ZSTD_writeLEST(bitC->ptr, bitC->bitContainer); ZSTD_writeLEST(bitC->ptr, bitC->bitContainer);
bitC->ptr += nbBytes; bitC->ptr += nbBytes;
bitC->bitPos &= 7; bitC->bitPos &= 7;
bitC->bitContainer >>= nbBytes*8; /* if bitPos >= sizeof(bitContainer)*8 --> undefined behavior */ bitC->bitContainer >>= nbBytes * 8; /* if bitPos >= sizeof(bitContainer)*8 --> undefined behavior */
} }
/*! BIT_flushBits() : /*! BIT_flushBits() :
* safe version; check for buffer overflow, and prevents it. * safe version; check for buffer overflow, and prevents it.
* note : does not signal buffer overflow. This will be revealed later on using BIT_closeCStream() */ * note : does not signal buffer overflow. This will be revealed later on using BIT_closeCStream() */
ZSTD_STATIC void BIT_flushBits(BIT_CStream_t* bitC) ZSTD_STATIC void BIT_flushBits(BIT_CStream_t *bitC)
{ {
size_t const nbBytes = bitC->bitPos >> 3; size_t const nbBytes = bitC->bitPos >> 3;
ZSTD_writeLEST(bitC->ptr, bitC->bitContainer); ZSTD_writeLEST(bitC->ptr, bitC->bitContainer);
bitC->ptr += nbBytes; bitC->ptr += nbBytes;
if (bitC->ptr > bitC->endPtr) bitC->ptr = bitC->endPtr; if (bitC->ptr > bitC->endPtr)
bitC->ptr = bitC->endPtr;
bitC->bitPos &= 7; bitC->bitPos &= 7;
bitC->bitContainer >>= nbBytes*8; /* if bitPos >= sizeof(bitContainer)*8 --> undefined behavior */ bitC->bitContainer >>= nbBytes * 8; /* if bitPos >= sizeof(bitContainer)*8 --> undefined behavior */
} }
/*! BIT_closeCStream() : /*! BIT_closeCStream() :
* @return : size of CStream, in bytes, * @return : size of CStream, in bytes,
or 0 if it could not fit into dstBuffer */ or 0 if it could not fit into dstBuffer */
ZSTD_STATIC size_t BIT_closeCStream(BIT_CStream_t* bitC) ZSTD_STATIC size_t BIT_closeCStream(BIT_CStream_t *bitC)
{ {
BIT_addBitsFast(bitC, 1, 1); /* endMark */ BIT_addBitsFast(bitC, 1, 1); /* endMark */
BIT_flushBits(bitC); BIT_flushBits(bitC);
if (bitC->ptr >= bitC->endPtr) return 0; /* doesn't fit within authorized budget : cancel */ if (bitC->ptr >= bitC->endPtr)
return 0; /* doesn't fit within authorized budget : cancel */
return (bitC->ptr - bitC->startPtr) + (bitC->bitPos > 0); return (bitC->ptr - bitC->startPtr) + (bitC->bitPos > 0);
} }
/*-******************************************************** /*-********************************************************
* bitStream decoding * bitStream decoding
**********************************************************/ **********************************************************/
@@ -242,54 +236,53 @@ ZSTD_STATIC size_t BIT_closeCStream(BIT_CStream_t* bitC)
* `srcSize` must be the *exact* size of the bitStream, in bytes. * `srcSize` must be the *exact* size of the bitStream, in bytes.
* @return : size of stream (== srcSize) or an errorCode if a problem is detected * @return : size of stream (== srcSize) or an errorCode if a problem is detected
*/ */
ZSTD_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, size_t srcSize) ZSTD_STATIC size_t BIT_initDStream(BIT_DStream_t *bitD, const void *srcBuffer, size_t srcSize)
{ {
if (srcSize < 1) { memset(bitD, 0, sizeof(*bitD)); return ERROR(srcSize_wrong); } if (srcSize < 1) {
memset(bitD, 0, sizeof(*bitD));
return ERROR(srcSize_wrong);
}
if (srcSize >= sizeof(bitD->bitContainer)) { /* normal case */ if (srcSize >= sizeof(bitD->bitContainer)) { /* normal case */
bitD->start = (const char*)srcBuffer; bitD->start = (const char *)srcBuffer;
bitD->ptr = (const char*)srcBuffer + srcSize - sizeof(bitD->bitContainer); bitD->ptr = (const char *)srcBuffer + srcSize - sizeof(bitD->bitContainer);
bitD->bitContainer = ZSTD_readLEST(bitD->ptr); bitD->bitContainer = ZSTD_readLEST(bitD->ptr);
{ BYTE const lastByte = ((const BYTE*)srcBuffer)[srcSize-1];
bitD->bitsConsumed = lastByte ? 8 - BIT_highbit32(lastByte) : 0; /* ensures bitsConsumed is always set */
if (lastByte == 0) return ERROR(GENERIC); /* endMark not present */ }
} else {
bitD->start = (const char*)srcBuffer;
bitD->ptr = bitD->start;
bitD->bitContainer = *(const BYTE*)(bitD->start);
switch(srcSize)
{ {
case 7: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[6]) << (sizeof(bitD->bitContainer)*8 - 16); BYTE const lastByte = ((const BYTE *)srcBuffer)[srcSize - 1];
case 6: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[5]) << (sizeof(bitD->bitContainer)*8 - 24); bitD->bitsConsumed = lastByte ? 8 - BIT_highbit32(lastByte) : 0; /* ensures bitsConsumed is always set */
case 5: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[4]) << (sizeof(bitD->bitContainer)*8 - 32); if (lastByte == 0)
case 4: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[3]) << 24; return ERROR(GENERIC); /* endMark not present */
case 3: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[2]) << 16; }
case 2: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[1]) << 8; } else {
bitD->start = (const char *)srcBuffer;
bitD->ptr = bitD->start;
bitD->bitContainer = *(const BYTE *)(bitD->start);
switch (srcSize) {
case 7: bitD->bitContainer += (size_t)(((const BYTE *)(srcBuffer))[6]) << (sizeof(bitD->bitContainer) * 8 - 16);
case 6: bitD->bitContainer += (size_t)(((const BYTE *)(srcBuffer))[5]) << (sizeof(bitD->bitContainer) * 8 - 24);
case 5: bitD->bitContainer += (size_t)(((const BYTE *)(srcBuffer))[4]) << (sizeof(bitD->bitContainer) * 8 - 32);
case 4: bitD->bitContainer += (size_t)(((const BYTE *)(srcBuffer))[3]) << 24;
case 3: bitD->bitContainer += (size_t)(((const BYTE *)(srcBuffer))[2]) << 16;
case 2: bitD->bitContainer += (size_t)(((const BYTE *)(srcBuffer))[1]) << 8;
default:; default:;
} }
{ BYTE const lastByte = ((const BYTE*)srcBuffer)[srcSize-1]; {
BYTE const lastByte = ((const BYTE *)srcBuffer)[srcSize - 1];
bitD->bitsConsumed = lastByte ? 8 - BIT_highbit32(lastByte) : 0; bitD->bitsConsumed = lastByte ? 8 - BIT_highbit32(lastByte) : 0;
if (lastByte == 0) return ERROR(GENERIC); /* endMark not present */ } if (lastByte == 0)
bitD->bitsConsumed += (U32)(sizeof(bitD->bitContainer) - srcSize)*8; return ERROR(GENERIC); /* endMark not present */
}
bitD->bitsConsumed += (U32)(sizeof(bitD->bitContainer) - srcSize) * 8;
} }
return srcSize; return srcSize;
} }
ZSTD_STATIC size_t BIT_getUpperBits(size_t bitContainer, U32 const start) ZSTD_STATIC size_t BIT_getUpperBits(size_t bitContainer, U32 const start) { return bitContainer >> start; }
{
return bitContainer >> start;
}
ZSTD_STATIC size_t BIT_getMiddleBits(size_t bitContainer, U32 const start, U32 const nbBits) ZSTD_STATIC size_t BIT_getMiddleBits(size_t bitContainer, U32 const start, U32 const nbBits) { return (bitContainer >> start) & BIT_mask[nbBits]; }
{
return (bitContainer >> start) & BIT_mask[nbBits];
}
ZSTD_STATIC size_t BIT_getLowerBits(size_t bitContainer, U32 const nbBits) ZSTD_STATIC size_t BIT_getLowerBits(size_t bitContainer, U32 const nbBits) { return bitContainer & BIT_mask[nbBits]; }
{
return bitContainer & BIT_mask[nbBits];
}
/*! BIT_lookBits() : /*! BIT_lookBits() :
* Provides next n bits from local register. * Provides next n bits from local register.
@@ -298,31 +291,28 @@ ZSTD_STATIC size_t BIT_getLowerBits(size_t bitContainer, U32 const nbBits)
* On 64-bits, maxNbBits==56. * On 64-bits, maxNbBits==56.
* @return : value extracted * @return : value extracted
*/ */
ZSTD_STATIC size_t BIT_lookBits(const BIT_DStream_t* bitD, U32 nbBits) ZSTD_STATIC size_t BIT_lookBits(const BIT_DStream_t *bitD, U32 nbBits)
{ {
U32 const bitMask = sizeof(bitD->bitContainer)*8 - 1; U32 const bitMask = sizeof(bitD->bitContainer) * 8 - 1;
return ((bitD->bitContainer << (bitD->bitsConsumed & bitMask)) >> 1) >> ((bitMask-nbBits) & bitMask); return ((bitD->bitContainer << (bitD->bitsConsumed & bitMask)) >> 1) >> ((bitMask - nbBits) & bitMask);
} }
/*! BIT_lookBitsFast() : /*! BIT_lookBitsFast() :
* unsafe version; only works only if nbBits >= 1 */ * unsafe version; only works only if nbBits >= 1 */
ZSTD_STATIC size_t BIT_lookBitsFast(const BIT_DStream_t* bitD, U32 nbBits) ZSTD_STATIC size_t BIT_lookBitsFast(const BIT_DStream_t *bitD, U32 nbBits)
{ {
U32 const bitMask = sizeof(bitD->bitContainer)*8 - 1; U32 const bitMask = sizeof(bitD->bitContainer) * 8 - 1;
return (bitD->bitContainer << (bitD->bitsConsumed & bitMask)) >> (((bitMask+1)-nbBits) & bitMask); return (bitD->bitContainer << (bitD->bitsConsumed & bitMask)) >> (((bitMask + 1) - nbBits) & bitMask);
} }
ZSTD_STATIC void BIT_skipBits(BIT_DStream_t* bitD, U32 nbBits) ZSTD_STATIC void BIT_skipBits(BIT_DStream_t *bitD, U32 nbBits) { bitD->bitsConsumed += nbBits; }
{
bitD->bitsConsumed += nbBits;
}
/*! BIT_readBits() : /*! BIT_readBits() :
* Read (consume) next n bits from local register and update. * Read (consume) next n bits from local register and update.
* Pay attention to not read more than nbBits contained into local register. * Pay attention to not read more than nbBits contained into local register.
* @return : extracted value. * @return : extracted value.
*/ */
ZSTD_STATIC size_t BIT_readBits(BIT_DStream_t* bitD, U32 nbBits) ZSTD_STATIC size_t BIT_readBits(BIT_DStream_t *bitD, U32 nbBits)
{ {
size_t const value = BIT_lookBits(bitD, nbBits); size_t const value = BIT_lookBits(bitD, nbBits);
BIT_skipBits(bitD, nbBits); BIT_skipBits(bitD, nbBits);
@@ -331,7 +321,7 @@ ZSTD_STATIC size_t BIT_readBits(BIT_DStream_t* bitD, U32 nbBits)
/*! BIT_readBitsFast() : /*! BIT_readBitsFast() :
* unsafe version; only works only if nbBits >= 1 */ * unsafe version; only works only if nbBits >= 1 */
ZSTD_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, U32 nbBits) ZSTD_STATIC size_t BIT_readBitsFast(BIT_DStream_t *bitD, U32 nbBits)
{ {
size_t const value = BIT_lookBitsFast(bitD, nbBits); size_t const value = BIT_lookBitsFast(bitD, nbBits);
BIT_skipBits(bitD, nbBits); BIT_skipBits(bitD, nbBits);
@@ -343,9 +333,9 @@ ZSTD_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, U32 nbBits)
* This function is safe, it guarantees it will not read beyond src buffer. * This function is safe, it guarantees it will not read beyond src buffer.
* @return : status of `BIT_DStream_t` internal register. * @return : status of `BIT_DStream_t` internal register.
if status == BIT_DStream_unfinished, internal register is filled with >= (sizeof(bitD->bitContainer)*8 - 7) bits */ if status == BIT_DStream_unfinished, internal register is filled with >= (sizeof(bitD->bitContainer)*8 - 7) bits */
ZSTD_STATIC BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD) ZSTD_STATIC BIT_DStream_status BIT_reloadDStream(BIT_DStream_t *bitD)
{ {
if (bitD->bitsConsumed > (sizeof(bitD->bitContainer)*8)) /* should not happen => corruption detected */ if (bitD->bitsConsumed > (sizeof(bitD->bitContainer) * 8)) /* should not happen => corruption detected */
return BIT_DStream_overflow; return BIT_DStream_overflow;
if (bitD->ptr >= bitD->start + sizeof(bitD->bitContainer)) { if (bitD->ptr >= bitD->start + sizeof(bitD->bitContainer)) {
@@ -355,17 +345,19 @@ ZSTD_STATIC BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD)
return BIT_DStream_unfinished; return BIT_DStream_unfinished;
} }
if (bitD->ptr == bitD->start) { if (bitD->ptr == bitD->start) {
if (bitD->bitsConsumed < sizeof(bitD->bitContainer)*8) return BIT_DStream_endOfBuffer; if (bitD->bitsConsumed < sizeof(bitD->bitContainer) * 8)
return BIT_DStream_endOfBuffer;
return BIT_DStream_completed; return BIT_DStream_completed;
} }
{ U32 nbBytes = bitD->bitsConsumed >> 3; {
U32 nbBytes = bitD->bitsConsumed >> 3;
BIT_DStream_status result = BIT_DStream_unfinished; BIT_DStream_status result = BIT_DStream_unfinished;
if (bitD->ptr - nbBytes < bitD->start) { if (bitD->ptr - nbBytes < bitD->start) {
nbBytes = (U32)(bitD->ptr - bitD->start); /* ptr > start */ nbBytes = (U32)(bitD->ptr - bitD->start); /* ptr > start */
result = BIT_DStream_endOfBuffer; result = BIT_DStream_endOfBuffer;
} }
bitD->ptr -= nbBytes; bitD->ptr -= nbBytes;
bitD->bitsConsumed -= nbBytes*8; bitD->bitsConsumed -= nbBytes * 8;
bitD->bitContainer = ZSTD_readLEST(bitD->ptr); /* reminder : srcSize > sizeof(bitD) */ bitD->bitContainer = ZSTD_readLEST(bitD->ptr); /* reminder : srcSize > sizeof(bitD) */
return result; return result;
} }
@@ -374,9 +366,9 @@ ZSTD_STATIC BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD)
/*! BIT_endOfDStream() : /*! BIT_endOfDStream() :
* @return Tells if DStream has exactly reached its end (all bits consumed). * @return Tells if DStream has exactly reached its end (all bits consumed).
*/ */
ZSTD_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* DStream) ZSTD_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t *DStream)
{ {
return ((DStream->ptr == DStream->start) && (DStream->bitsConsumed == sizeof(DStream->bitContainer)*8)); return ((DStream->ptr == DStream->start) && (DStream->bitsConsumed == sizeof(DStream->bitContainer) * 8));
} }
#endif /* BITSTREAM_H_MODULE */ #endif /* BITSTREAM_H_MODULE */
contrib/linux-kernel/lib/zstd/compress.c
+1517 -1356
View File
@@ -14,51 +14,50 @@
* ("BSD"). * ("BSD").
*/ */
/*-************************************* /*-*************************************
* Dependencies * Dependencies
***************************************/ ***************************************/
#include "fse.h"
#include "huf.h"
#include "mem.h"
#include "zstd_internal.h" /* includes zstd.h */
#include <linux/kernel.h> #include <linux/kernel.h>
#include <linux/module.h> #include <linux/module.h>
#include <linux/string.h> /* memset */ #include <linux/string.h> /* memset */
#include "mem.h"
#include "fse.h"
#include "huf.h"
#include "zstd_internal.h" /* includes zstd.h */
/*-************************************* /*-*************************************
* Constants * Constants
***************************************/ ***************************************/
static const U32 g_searchStrength = 8; /* control skip over incompressible data */ static const U32 g_searchStrength = 8; /* control skip over incompressible data */
#define HASH_READ_SIZE 8 #define HASH_READ_SIZE 8
typedef enum { ZSTDcs_created=0, ZSTDcs_init, ZSTDcs_ongoing, ZSTDcs_ending } ZSTD_compressionStage_e; typedef enum { ZSTDcs_created = 0, ZSTDcs_init, ZSTDcs_ongoing, ZSTDcs_ending } ZSTD_compressionStage_e;
/*-************************************* /*-*************************************
* Helper functions * Helper functions
***************************************/ ***************************************/
#define ZSTD_STATIC_ASSERT(c) { enum { ZSTD_static_assert = 1/(int)(!!(c)) }; } #define ZSTD_STATIC_ASSERT(c) \
{ \
enum { ZSTD_static_assert = 1 / (int)(!!(c)) }; \
}
size_t ZSTD_compressBound(size_t srcSize) { return FSE_compressBound(srcSize) + 12; } size_t ZSTD_compressBound(size_t srcSize) { return FSE_compressBound(srcSize) + 12; }
/*-************************************* /*-*************************************
* Sequence storage * Sequence storage
***************************************/ ***************************************/
static void ZSTD_resetSeqStore(seqStore_t* ssPtr) static void ZSTD_resetSeqStore(seqStore_t *ssPtr)
{ {
ssPtr->lit = ssPtr->litStart; ssPtr->lit = ssPtr->litStart;
ssPtr->sequences = ssPtr->sequencesStart; ssPtr->sequences = ssPtr->sequencesStart;
ssPtr->longLengthID = 0; ssPtr->longLengthID = 0;
} }
/*-************************************* /*-*************************************
* Context memory management * Context memory management
***************************************/ ***************************************/
struct ZSTD_CCtx_s { struct ZSTD_CCtx_s {
const BYTE* nextSrc; /* next block here to continue on curr prefix */ const BYTE *nextSrc; /* next block here to continue on curr prefix */
const BYTE* base; /* All regular indexes relative to this position */ const BYTE *base; /* All regular indexes relative to this position */
const BYTE* dictBase; /* extDict indexes relative to this position */ const BYTE *dictBase; /* extDict indexes relative to this position */
U32 dictLimit; /* below that point, need extDict */ U32 dictLimit; /* below that point, need extDict */
U32 lowLimit; /* below that point, no more data */ U32 lowLimit; /* below that point, no more data */
U32 nextToUpdate; /* index from which to continue dictionary update */ U32 nextToUpdate; /* index from which to continue dictionary update */
@@ -72,7 +71,7 @@ struct ZSTD_CCtx_s {
U32 repToConfirm[ZSTD_REP_NUM]; U32 repToConfirm[ZSTD_REP_NUM];
U32 dictID; U32 dictID;
ZSTD_parameters params; ZSTD_parameters params;
void* workSpace; void *workSpace;
size_t workSpaceSize; size_t workSpaceSize;
size_t blockSize; size_t blockSize;
U64 frameContentSize; U64 frameContentSize;
@@ -80,91 +79,94 @@ struct ZSTD_CCtx_s {
ZSTD_customMem customMem; ZSTD_customMem customMem;
seqStore_t seqStore; /* sequences storage ptrs */ seqStore_t seqStore; /* sequences storage ptrs */
U32* hashTable; U32 *hashTable;
U32* hashTable3; U32 *hashTable3;
U32* chainTable; U32 *chainTable;
HUF_CElt* hufTable; HUF_CElt *hufTable;
U32 flagStaticTables; U32 flagStaticTables;
HUF_repeat flagStaticHufTable; HUF_repeat flagStaticHufTable;
FSE_CTable offcodeCTable [FSE_CTABLE_SIZE_U32(OffFSELog, MaxOff)]; FSE_CTable offcodeCTable[FSE_CTABLE_SIZE_U32(OffFSELog, MaxOff)];
FSE_CTable matchlengthCTable[FSE_CTABLE_SIZE_U32(MLFSELog, MaxML)]; FSE_CTable matchlengthCTable[FSE_CTABLE_SIZE_U32(MLFSELog, MaxML)];
FSE_CTable litlengthCTable [FSE_CTABLE_SIZE_U32(LLFSELog, MaxLL)]; FSE_CTable litlengthCTable[FSE_CTABLE_SIZE_U32(LLFSELog, MaxLL)];
unsigned tmpCounters[HUF_WORKSPACE_SIZE_U32]; unsigned tmpCounters[HUF_WORKSPACE_SIZE_U32];
}; };
size_t ZSTD_CCtxWorkspaceBound(ZSTD_compressionParameters cParams) { size_t ZSTD_CCtxWorkspaceBound(ZSTD_compressionParameters cParams)
{
size_t const blockSize = MIN(ZSTD_BLOCKSIZE_ABSOLUTEMAX, (size_t)1 << cParams.windowLog); size_t const blockSize = MIN(ZSTD_BLOCKSIZE_ABSOLUTEMAX, (size_t)1 << cParams.windowLog);
U32 const divider = (cParams.searchLength==3) ? 3 : 4; U32 const divider = (cParams.searchLength == 3) ? 3 : 4;
size_t const maxNbSeq = blockSize / divider; size_t const maxNbSeq = blockSize / divider;
size_t const tokenSpace = blockSize + 11*maxNbSeq; size_t const tokenSpace = blockSize + 11 * maxNbSeq;
size_t const chainSize = (cParams.strategy == ZSTD_fast) ? 0 : (1 << cParams.chainLog); size_t const chainSize = (cParams.strategy == ZSTD_fast) ? 0 : (1 << cParams.chainLog);
size_t const hSize = ((size_t)1) << cParams.hashLog; size_t const hSize = ((size_t)1) << cParams.hashLog;
U32 const hashLog3 = (cParams.searchLength>3) ? 0 : MIN(ZSTD_HASHLOG3_MAX, cParams.windowLog); U32 const hashLog3 = (cParams.searchLength > 3) ? 0 : MIN(ZSTD_HASHLOG3_MAX, cParams.windowLog);
size_t const h3Size = ((size_t)1) << hashLog3; size_t const h3Size = ((size_t)1) << hashLog3;
size_t const tableSpace = (chainSize + hSize + h3Size) * sizeof(U32); size_t const tableSpace = (chainSize + hSize + h3Size) * sizeof(U32);
size_t const optSpace = ((MaxML+1) + (MaxLL+1) + (MaxOff+1) + (1<<Litbits))*sizeof(U32) + (ZSTD_OPT_NUM+1)*(sizeof(ZSTD_match_t) + sizeof(ZSTD_optimal_t)); size_t const optSpace =
size_t const workspaceSize = tableSpace + (256*sizeof(U32)) /* huffTable */ + tokenSpace + (((cParams.strategy == ZSTD_btopt) || (cParams.strategy == ZSTD_btopt2)) ? optSpace : 0); ((MaxML + 1) + (MaxLL + 1) + (MaxOff + 1) + (1 << Litbits)) * sizeof(U32) + (ZSTD_OPT_NUM + 1) * (sizeof(ZSTD_match_t) + sizeof(ZSTD_optimal_t));
size_t const workspaceSize = tableSpace + (256 * sizeof(U32)) /* huffTable */ + tokenSpace +
(((cParams.strategy == ZSTD_btopt) || (cParams.strategy == ZSTD_btopt2)) ? optSpace : 0);
return ZSTD_ALIGN(sizeof(ZSTD_stack)) + ZSTD_ALIGN(sizeof(ZSTD_CCtx)) + ZSTD_ALIGN(workspaceSize); return ZSTD_ALIGN(sizeof(ZSTD_stack)) + ZSTD_ALIGN(sizeof(ZSTD_CCtx)) + ZSTD_ALIGN(workspaceSize);
} }
static ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem) static ZSTD_CCtx *ZSTD_createCCtx_advanced(ZSTD_customMem customMem)
{ {
ZSTD_CCtx* cctx; ZSTD_CCtx *cctx;
if (!customMem.customAlloc || !customMem.customFree) return NULL; if (!customMem.customAlloc || !customMem.customFree)
cctx = (ZSTD_CCtx*) ZSTD_malloc(sizeof(ZSTD_CCtx), customMem); return NULL;
if (!cctx) return NULL; cctx = (ZSTD_CCtx *)ZSTD_malloc(sizeof(ZSTD_CCtx), customMem);
if (!cctx)
return NULL;
memset(cctx, 0, sizeof(ZSTD_CCtx)); memset(cctx, 0, sizeof(ZSTD_CCtx));
cctx->customMem = customMem; cctx->customMem = customMem;
return cctx; return cctx;
} }
ZSTD_CCtx* ZSTD_initCCtx(void* workspace, size_t workspaceSize) ZSTD_CCtx *ZSTD_initCCtx(void *workspace, size_t workspaceSize)
{ {
ZSTD_customMem const stackMem = ZSTD_initStack(workspace, workspaceSize); ZSTD_customMem const stackMem = ZSTD_initStack(workspace, workspaceSize);
ZSTD_CCtx* cctx = ZSTD_createCCtx_advanced(stackMem); ZSTD_CCtx *cctx = ZSTD_createCCtx_advanced(stackMem);
if (cctx) { if (cctx) {
cctx->workSpace = ZSTD_stackAllocAll(cctx->customMem.opaque, &cctx->workSpaceSize); cctx->workSpace = ZSTD_stackAllocAll(cctx->customMem.opaque, &cctx->workSpaceSize);
} }
return cctx; return cctx;
} }
size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx) size_t ZSTD_freeCCtx(ZSTD_CCtx *cctx)
{ {
if (cctx==NULL) return 0; /* support free on NULL */ if (cctx == NULL)
return 0; /* support free on NULL */
ZSTD_free(cctx->workSpace, cctx->customMem); ZSTD_free(cctx->workSpace, cctx->customMem);
ZSTD_free(cctx, cctx->customMem); ZSTD_free(cctx, cctx->customMem);
return 0; /* reserved as a potential error code in the future */ return 0; /* reserved as a potential error code in the future */
} }
const seqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx) /* hidden interface */ const seqStore_t *ZSTD_getSeqStore(const ZSTD_CCtx *ctx) /* hidden interface */ { return &(ctx->seqStore); }
{
return &(ctx->seqStore);
}
static ZSTD_parameters ZSTD_getParamsFromCCtx(const ZSTD_CCtx* cctx)
{
return cctx->params;
}
static ZSTD_parameters ZSTD_getParamsFromCCtx(const ZSTD_CCtx *cctx) { return cctx->params; }
/** ZSTD_checkParams() : /** ZSTD_checkParams() :
ensure param values remain within authorized range. ensure param values remain within authorized range.
@return : 0, or an error code if one value is beyond authorized range */ @return : 0, or an error code if one value is beyond authorized range */
size_t ZSTD_checkCParams(ZSTD_compressionParameters cParams) size_t ZSTD_checkCParams(ZSTD_compressionParameters cParams)
{ {
# define CLAMPCHECK(val,min,max) { if ((val<min) | (val>max)) return ERROR(compressionParameter_unsupported); } #define CLAMPCHECK(val, min, max) \
{ \
if ((val < min) | (val > max)) \
return ERROR(compressionParameter_unsupported); \
}
CLAMPCHECK(cParams.windowLog, ZSTD_WINDOWLOG_MIN, ZSTD_WINDOWLOG_MAX); CLAMPCHECK(cParams.windowLog, ZSTD_WINDOWLOG_MIN, ZSTD_WINDOWLOG_MAX);
CLAMPCHECK(cParams.chainLog, ZSTD_CHAINLOG_MIN, ZSTD_CHAINLOG_MAX); CLAMPCHECK(cParams.chainLog, ZSTD_CHAINLOG_MIN, ZSTD_CHAINLOG_MAX);
CLAMPCHECK(cParams.hashLog, ZSTD_HASHLOG_MIN, ZSTD_HASHLOG_MAX); CLAMPCHECK(cParams.hashLog, ZSTD_HASHLOG_MIN, ZSTD_HASHLOG_MAX);
CLAMPCHECK(cParams.searchLog, ZSTD_SEARCHLOG_MIN, ZSTD_SEARCHLOG_MAX); CLAMPCHECK(cParams.searchLog, ZSTD_SEARCHLOG_MIN, ZSTD_SEARCHLOG_MAX);
CLAMPCHECK(cParams.searchLength, ZSTD_SEARCHLENGTH_MIN, ZSTD_SEARCHLENGTH_MAX); CLAMPCHECK(cParams.searchLength, ZSTD_SEARCHLENGTH_MIN, ZSTD_SEARCHLENGTH_MAX);
CLAMPCHECK(cParams.targetLength, ZSTD_TARGETLENGTH_MIN, ZSTD_TARGETLENGTH_MAX); CLAMPCHECK(cParams.targetLength, ZSTD_TARGETLENGTH_MIN, ZSTD_TARGETLENGTH_MAX);
if ((U32)(cParams.strategy) > (U32)ZSTD_btopt2) return ERROR(compressionParameter_unsupported); if ((U32)(cParams.strategy) > (U32)ZSTD_btopt2)
return ERROR(compressionParameter_unsupported);
return 0; return 0;
} }
/** ZSTD_cycleLog() : /** ZSTD_cycleLog() :
* condition for correct operation : hashLog > 1 */ * condition for correct operation : hashLog > 1 */
static U32 ZSTD_cycleLog(U32 hashLog, ZSTD_strategy strat) static U32 ZSTD_cycleLog(U32 hashLog, ZSTD_strategy strat)
@@ -181,48 +183,57 @@ static U32 ZSTD_cycleLog(U32 hashLog, ZSTD_strategy strat)
Note : cPar is considered validated at this stage. Use ZSTD_checkParams() to ensure that. */ Note : cPar is considered validated at this stage. Use ZSTD_checkParams() to ensure that. */
ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, unsigned long long srcSize, size_t dictSize) ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, unsigned long long srcSize, size_t dictSize)
{ {
if (srcSize+dictSize == 0) return cPar; /* no size information available : no adjustment */ if (srcSize + dictSize == 0)
return cPar; /* no size information available : no adjustment */
/* resize params, to use less memory when necessary */ /* resize params, to use less memory when necessary */
{ U32 const minSrcSize = (srcSize==0) ? 500 : 0; {
U32 const minSrcSize = (srcSize == 0) ? 500 : 0;
U64 const rSize = srcSize + dictSize + minSrcSize; U64 const rSize = srcSize + dictSize + minSrcSize;
if (rSize < ((U64)1<<ZSTD_WINDOWLOG_MAX)) { if (rSize < ((U64)1 << ZSTD_WINDOWLOG_MAX)) {
U32 const srcLog = MAX(ZSTD_HASHLOG_MIN, ZSTD_highbit32((U32)(rSize)-1) + 1); U32 const srcLog = MAX(ZSTD_HASHLOG_MIN, ZSTD_highbit32((U32)(rSize)-1) + 1);
if (cPar.windowLog > srcLog) cPar.windowLog = srcLog; if (cPar.windowLog > srcLog)
} } cPar.windowLog = srcLog;
if (cPar.hashLog > cPar.windowLog) cPar.hashLog = cPar.windowLog; }
{ U32 const cycleLog = ZSTD_cycleLog(cPar.chainLog, cPar.strategy); }
if (cycleLog > cPar.windowLog) cPar.chainLog -= (cycleLog - cPar.windowLog); if (cPar.hashLog > cPar.windowLog)
cPar.hashLog = cPar.windowLog;
{
U32 const cycleLog = ZSTD_cycleLog(cPar.chainLog, cPar.strategy);
if (cycleLog > cPar.windowLog)
cPar.chainLog -= (cycleLog - cPar.windowLog);
} }
if (cPar.windowLog < ZSTD_WINDOWLOG_ABSOLUTEMIN) cPar.windowLog = ZSTD_WINDOWLOG_ABSOLUTEMIN; /* required for frame header */ if (cPar.windowLog < ZSTD_WINDOWLOG_ABSOLUTEMIN)
cPar.windowLog = ZSTD_WINDOWLOG_ABSOLUTEMIN; /* required for frame header */
return cPar; return cPar;
} }
static U32 ZSTD_equivalentParams(ZSTD_parameters param1, ZSTD_parameters param2) static U32 ZSTD_equivalentParams(ZSTD_parameters param1, ZSTD_parameters param2)
{ {
return (param1.cParams.hashLog == param2.cParams.hashLog) return (param1.cParams.hashLog == param2.cParams.hashLog) & (param1.cParams.chainLog == param2.cParams.chainLog) &
& (param1.cParams.chainLog == param2.cParams.chainLog) (param1.cParams.strategy == param2.cParams.strategy) & ((param1.cParams.searchLength == 3) == (param2.cParams.searchLength == 3));
& (param1.cParams.strategy == param2.cParams.strategy)
& ((param1.cParams.searchLength==3) == (param2.cParams.searchLength==3));
} }
/*! ZSTD_continueCCtx() : /*! ZSTD_continueCCtx() :
reuse CCtx without reset (note : requires no dictionary) */ reuse CCtx without reset (note : requires no dictionary) */
static size_t ZSTD_continueCCtx(ZSTD_CCtx* cctx, ZSTD_parameters params, U64 frameContentSize) static size_t ZSTD_continueCCtx(ZSTD_CCtx *cctx, ZSTD_parameters params, U64 frameContentSize)
{ {
U32 const end = (U32)(cctx->nextSrc - cctx->base); U32 const end = (U32)(cctx->nextSrc - cctx->base);
cctx->params = params; cctx->params = params;
cctx->frameContentSize = frameContentSize; cctx->frameContentSize = frameContentSize;
cctx->lowLimit = end; cctx->lowLimit = end;
cctx->dictLimit = end; cctx->dictLimit = end;
cctx->nextToUpdate = end+1; cctx->nextToUpdate = end + 1;
cctx->stage = ZSTDcs_init; cctx->stage = ZSTDcs_init;
cctx->dictID = 0; cctx->dictID = 0;
cctx->loadedDictEnd = 0; cctx->loadedDictEnd = 0;
{ int i; for (i=0; i<ZSTD_REP_NUM; i++) cctx->rep[i] = repStartValue[i]; } {
int i;
for (i = 0; i < ZSTD_REP_NUM; i++)
cctx->rep[i] = repStartValue[i];
}
cctx->seqStore.litLengthSum = 0; /* force reset of btopt stats */ cctx->seqStore.litLengthSum = 0; /* force reset of btopt stats */
xxh64_reset(&cctx->xxhState, 0); xxh64_reset(&cctx->xxhState, 0);
return 0; return 0;
@@ -232,9 +243,7 @@ typedef enum { ZSTDcrp_continue, ZSTDcrp_noMemset, ZSTDcrp_fullReset } ZSTD_comp
/*! ZSTD_resetCCtx_advanced() : /*! ZSTD_resetCCtx_advanced() :
note : `params` must be validated */ note : `params` must be validated */
static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc, static size_t ZSTD_resetCCtx_advanced(ZSTD_CCtx *zc, ZSTD_parameters params, U64 frameContentSize, ZSTD_compResetPolicy_e const crp)
ZSTD_parameters params, U64 frameContentSize,
ZSTD_compResetPolicy_e const crp)
{ {
if (crp == ZSTDcrp_continue) if (crp == ZSTDcrp_continue)
if (ZSTD_equivalentParams(params, zc->params)) { if (ZSTD_equivalentParams(params, zc->params)) {
@@ -243,40 +252,45 @@ static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc,
return ZSTD_continueCCtx(zc, params, frameContentSize); return ZSTD_continueCCtx(zc, params, frameContentSize);
} }
{ size_t const blockSize = MIN(ZSTD_BLOCKSIZE_ABSOLUTEMAX, (size_t)1 << params.cParams.windowLog); {
U32 const divider = (params.cParams.searchLength==3) ? 3 : 4; size_t const blockSize = MIN(ZSTD_BLOCKSIZE_ABSOLUTEMAX, (size_t)1 << params.cParams.windowLog);
U32 const divider = (params.cParams.searchLength == 3) ? 3 : 4;
size_t const maxNbSeq = blockSize / divider; size_t const maxNbSeq = blockSize / divider;
size_t const tokenSpace = blockSize + 11*maxNbSeq; size_t const tokenSpace = blockSize + 11 * maxNbSeq;
size_t const chainSize = (params.cParams.strategy == ZSTD_fast) ? 0 : (1 << params.cParams.chainLog); size_t const chainSize = (params.cParams.strategy == ZSTD_fast) ? 0 : (1 << params.cParams.chainLog);
size_t const hSize = ((size_t)1) << params.cParams.hashLog; size_t const hSize = ((size_t)1) << params.cParams.hashLog;
U32 const hashLog3 = (params.cParams.searchLength>3) ? 0 : MIN(ZSTD_HASHLOG3_MAX, params.cParams.windowLog); U32 const hashLog3 = (params.cParams.searchLength > 3) ? 0 : MIN(ZSTD_HASHLOG3_MAX, params.cParams.windowLog);
size_t const h3Size = ((size_t)1) << hashLog3; size_t const h3Size = ((size_t)1) << hashLog3;
size_t const tableSpace = (chainSize + hSize + h3Size) * sizeof(U32); size_t const tableSpace = (chainSize + hSize + h3Size) * sizeof(U32);
void* ptr; void *ptr;
/* Check if workSpace is large enough, alloc a new one if needed */ /* Check if workSpace is large enough, alloc a new one if needed */
{ size_t const optSpace = ((MaxML+1) + (MaxLL+1) + (MaxOff+1) + (1<<Litbits))*sizeof(U32) {
+ (ZSTD_OPT_NUM+1)*(sizeof(ZSTD_match_t) + sizeof(ZSTD_optimal_t)); size_t const optSpace = ((MaxML + 1) + (MaxLL + 1) + (MaxOff + 1) + (1 << Litbits)) * sizeof(U32) +
size_t const neededSpace = tableSpace + (256*sizeof(U32)) /* huffTable */ + tokenSpace (ZSTD_OPT_NUM + 1) * (sizeof(ZSTD_match_t) + sizeof(ZSTD_optimal_t));
+ (((params.cParams.strategy == ZSTD_btopt) || (params.cParams.strategy == ZSTD_btopt2)) ? optSpace : 0); size_t const neededSpace = tableSpace + (256 * sizeof(U32)) /* huffTable */ + tokenSpace +
(((params.cParams.strategy == ZSTD_btopt) || (params.cParams.strategy == ZSTD_btopt2)) ? optSpace : 0);
if (zc->workSpaceSize < neededSpace) { if (zc->workSpaceSize < neededSpace) {
ZSTD_free(zc->workSpace, zc->customMem); ZSTD_free(zc->workSpace, zc->customMem);
zc->workSpace = ZSTD_malloc(neededSpace, zc->customMem); zc->workSpace = ZSTD_malloc(neededSpace, zc->customMem);
if (zc->workSpace == NULL) return ERROR(memory_allocation); if (zc->workSpace == NULL)
return ERROR(memory_allocation);
zc->workSpaceSize = neededSpace; zc->workSpaceSize = neededSpace;
} } }
}
if (crp!=ZSTDcrp_noMemset) memset(zc->workSpace, 0, tableSpace); /* reset tables only */ if (crp != ZSTDcrp_noMemset)
memset(zc->workSpace, 0, tableSpace); /* reset tables only */
xxh64_reset(&zc->xxhState, 0); xxh64_reset(&zc->xxhState, 0);
zc->hashLog3 = hashLog3; zc->hashLog3 = hashLog3;
zc->hashTable = (U32*)(zc->workSpace); zc->hashTable = (U32 *)(zc->workSpace);
zc->chainTable = zc->hashTable + hSize; zc->chainTable = zc->hashTable + hSize;
zc->hashTable3 = zc->chainTable + chainSize; zc->hashTable3 = zc->chainTable + chainSize;
ptr = zc->hashTable3 + h3Size; ptr = zc->hashTable3 + h3Size;
zc->hufTable = (HUF_CElt*)ptr; zc->hufTable = (HUF_CElt *)ptr;
zc->flagStaticTables = 0; zc->flagStaticTables = 0;
zc->flagStaticHufTable = HUF_repeat_none; zc->flagStaticHufTable = HUF_repeat_none;
ptr = ((U32*)ptr) + 256; /* note : HUF_CElt* is incomplete type, size is simulated using U32 */ ptr = ((U32 *)ptr) + 256; /* note : HUF_CElt* is incomplete type, size is simulated using U32 */
zc->nextToUpdate = 1; zc->nextToUpdate = 1;
zc->nextSrc = NULL; zc->nextSrc = NULL;
@@ -287,23 +301,27 @@ static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc,
zc->params = params; zc->params = params;
zc->blockSize = blockSize; zc->blockSize = blockSize;
zc->frameContentSize = frameContentSize; zc->frameContentSize = frameContentSize;
{ int i; for (i=0; i<ZSTD_REP_NUM; i++) zc->rep[i] = repStartValue[i]; } {
int i;
for (i = 0; i < ZSTD_REP_NUM; i++)
zc->rep[i] = repStartValue[i];
}
if ((params.cParams.strategy == ZSTD_btopt) || (params.cParams.strategy == ZSTD_btopt2)) { if ((params.cParams.strategy == ZSTD_btopt) || (params.cParams.strategy == ZSTD_btopt2)) {
zc->seqStore.litFreq = (U32*)ptr; zc->seqStore.litFreq = (U32 *)ptr;
zc->seqStore.litLengthFreq = zc->seqStore.litFreq + (1<<Litbits); zc->seqStore.litLengthFreq = zc->seqStore.litFreq + (1 << Litbits);
zc->seqStore.matchLengthFreq = zc->seqStore.litLengthFreq + (MaxLL+1); zc->seqStore.matchLengthFreq = zc->seqStore.litLengthFreq + (MaxLL + 1);
zc->seqStore.offCodeFreq = zc->seqStore.matchLengthFreq + (MaxML+1); zc->seqStore.offCodeFreq = zc->seqStore.matchLengthFreq + (MaxML + 1);
ptr = zc->seqStore.offCodeFreq + (MaxOff+1); ptr = zc->seqStore.offCodeFreq + (MaxOff + 1);
zc->seqStore.matchTable = (ZSTD_match_t*)ptr; zc->seqStore.matchTable = (ZSTD_match_t *)ptr;
ptr = zc->seqStore.matchTable + ZSTD_OPT_NUM+1; ptr = zc->seqStore.matchTable + ZSTD_OPT_NUM + 1;
zc->seqStore.priceTable = (ZSTD_optimal_t*)ptr; zc->seqStore.priceTable = (ZSTD_optimal_t *)ptr;
ptr = zc->seqStore.priceTable + ZSTD_OPT_NUM+1; ptr = zc->seqStore.priceTable + ZSTD_OPT_NUM + 1;
zc->seqStore.litLengthSum = 0; zc->seqStore.litLengthSum = 0;
} }
zc->seqStore.sequencesStart = (seqDef*)ptr; zc->seqStore.sequencesStart = (seqDef *)ptr;
ptr = zc->seqStore.sequencesStart + maxNbSeq; ptr = zc->seqStore.sequencesStart + maxNbSeq;
zc->seqStore.llCode = (BYTE*) ptr; zc->seqStore.llCode = (BYTE *)ptr;
zc->seqStore.mlCode = zc->seqStore.llCode + maxNbSeq; zc->seqStore.mlCode = zc->seqStore.llCode + maxNbSeq;
zc->seqStore.ofCode = zc->seqStore.mlCode + maxNbSeq; zc->seqStore.ofCode = zc->seqStore.mlCode + maxNbSeq;
zc->seqStore.litStart = zc->seqStore.ofCode + maxNbSeq; zc->seqStore.litStart = zc->seqStore.ofCode + maxNbSeq;
@@ -320,28 +338,32 @@ static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc,
* ensures next compression will not use repcodes from previous block. * ensures next compression will not use repcodes from previous block.
* Note : only works with regular variant; * Note : only works with regular variant;
* do not use with extDict variant ! */ * do not use with extDict variant ! */
void ZSTD_invalidateRepCodes(ZSTD_CCtx* cctx) { void ZSTD_invalidateRepCodes(ZSTD_CCtx *cctx)
{
int i; int i;
for (i=0; i<ZSTD_REP_NUM; i++) cctx->rep[i] = 0; for (i = 0; i < ZSTD_REP_NUM; i++)
cctx->rep[i] = 0;
} }
/*! ZSTD_copyCCtx() : /*! ZSTD_copyCCtx() :
* Duplicate an existing context `srcCCtx` into another one `dstCCtx`. * Duplicate an existing context `srcCCtx` into another one `dstCCtx`.
* Only works during stage ZSTDcs_init (i.e. after creation, but before first call to ZSTD_compressContinue()). * Only works during stage ZSTDcs_init (i.e. after creation, but before first call to ZSTD_compressContinue()).
* @return : 0, or an error code */ * @return : 0, or an error code */
size_t ZSTD_copyCCtx(ZSTD_CCtx* dstCCtx, const ZSTD_CCtx* srcCCtx, unsigned long long pledgedSrcSize) size_t ZSTD_copyCCtx(ZSTD_CCtx *dstCCtx, const ZSTD_CCtx *srcCCtx, unsigned long long pledgedSrcSize)
{ {
if (srcCCtx->stage!=ZSTDcs_init) return ERROR(stage_wrong); if (srcCCtx->stage != ZSTDcs_init)
return ERROR(stage_wrong);
memcpy(&dstCCtx->customMem, &srcCCtx->customMem, sizeof(ZSTD_customMem)); memcpy(&dstCCtx->customMem, &srcCCtx->customMem, sizeof(ZSTD_customMem));
{ ZSTD_parameters params = srcCCtx->params; {
ZSTD_parameters params = srcCCtx->params;
params.fParams.contentSizeFlag = (pledgedSrcSize > 0); params.fParams.contentSizeFlag = (pledgedSrcSize > 0);
ZSTD_resetCCtx_advanced(dstCCtx, params, pledgedSrcSize, ZSTDcrp_noMemset); ZSTD_resetCCtx_advanced(dstCCtx, params, pledgedSrcSize, ZSTDcrp_noMemset);
} }
/* copy tables */ /* copy tables */
{ size_t const chainSize = (srcCCtx->params.cParams.strategy == ZSTD_fast) ? 0 : (1 << srcCCtx->params.cParams.chainLog); {
size_t const chainSize = (srcCCtx->params.cParams.strategy == ZSTD_fast) ? 0 : (1 << srcCCtx->params.cParams.chainLog);
size_t const hSize = ((size_t)1) << srcCCtx->params.cParams.hashLog; size_t const hSize = ((size_t)1) << srcCCtx->params.cParams.hashLog;
size_t const h3Size = (size_t)1 << srcCCtx->hashLog3; size_t const h3Size = (size_t)1 << srcCCtx->hashLog3;
size_t const tableSpace = (chainSize + hSize + h3Size) * sizeof(U32); size_t const tableSpace = (chainSize + hSize + h3Size) * sizeof(U32);
@@ -350,13 +372,13 @@ size_t ZSTD_copyCCtx(ZSTD_CCtx* dstCCtx, const ZSTD_CCtx* srcCCtx, unsigned long
/* copy dictionary offsets */ /* copy dictionary offsets */
dstCCtx->nextToUpdate = srcCCtx->nextToUpdate; dstCCtx->nextToUpdate = srcCCtx->nextToUpdate;
dstCCtx->nextToUpdate3= srcCCtx->nextToUpdate3; dstCCtx->nextToUpdate3 = srcCCtx->nextToUpdate3;
dstCCtx->nextSrc = srcCCtx->nextSrc; dstCCtx->nextSrc = srcCCtx->nextSrc;
dstCCtx->base = srcCCtx->base; dstCCtx->base = srcCCtx->base;
dstCCtx->dictBase = srcCCtx->dictBase; dstCCtx->dictBase = srcCCtx->dictBase;
dstCCtx->dictLimit = srcCCtx->dictLimit; dstCCtx->dictLimit = srcCCtx->dictLimit;
dstCCtx->lowLimit = srcCCtx->lowLimit; dstCCtx->lowLimit = srcCCtx->lowLimit;
dstCCtx->loadedDictEnd= srcCCtx->loadedDictEnd; dstCCtx->loadedDictEnd = srcCCtx->loadedDictEnd;
dstCCtx->dictID = srcCCtx->dictID; dstCCtx->dictID = srcCCtx->dictID;
/* copy entropy tables */ /* copy entropy tables */
@@ -368,247 +390,245 @@ size_t ZSTD_copyCCtx(ZSTD_CCtx* dstCCtx, const ZSTD_CCtx* srcCCtx, unsigned long
memcpy(dstCCtx->offcodeCTable, srcCCtx->offcodeCTable, sizeof(dstCCtx->offcodeCTable)); memcpy(dstCCtx->offcodeCTable, srcCCtx->offcodeCTable, sizeof(dstCCtx->offcodeCTable));
} }
if (srcCCtx->flagStaticHufTable) { if (srcCCtx->flagStaticHufTable) {
memcpy(dstCCtx->hufTable, srcCCtx->hufTable, 256*4); memcpy(dstCCtx->hufTable, srcCCtx->hufTable, 256 * 4);
} }
return 0; return 0;
} }
/*! ZSTD_reduceTable() : /*! ZSTD_reduceTable() :
* reduce table indexes by `reducerValue` */ * reduce table indexes by `reducerValue` */
static void ZSTD_reduceTable (U32* const table, U32 const size, U32 const reducerValue) static void ZSTD_reduceTable(U32 *const table, U32 const size, U32 const reducerValue)
{ {
U32 u; U32 u;
for (u=0 ; u < size ; u++) { for (u = 0; u < size; u++) {
if (table[u] < reducerValue) table[u] = 0; if (table[u] < reducerValue)
else table[u] -= reducerValue; table[u] = 0;
else
table[u] -= reducerValue;
} }
} }
/*! ZSTD_reduceIndex() : /*! ZSTD_reduceIndex() :
* rescale all indexes to avoid future overflow (indexes are U32) */ * rescale all indexes to avoid future overflow (indexes are U32) */
static void ZSTD_reduceIndex (ZSTD_CCtx* zc, const U32 reducerValue) static void ZSTD_reduceIndex(ZSTD_CCtx *zc, const U32 reducerValue)
{ {
{ U32 const hSize = 1 << zc->params.cParams.hashLog; {
ZSTD_reduceTable(zc->hashTable, hSize, reducerValue); } U32 const hSize = 1 << zc->params.cParams.hashLog;
ZSTD_reduceTable(zc->hashTable, hSize, reducerValue);
}
{ U32 const chainSize = (zc->params.cParams.strategy == ZSTD_fast) ? 0 : (1 << zc->params.cParams.chainLog); {
ZSTD_reduceTable(zc->chainTable, chainSize, reducerValue); } U32 const chainSize = (zc->params.cParams.strategy == ZSTD_fast) ? 0 : (1 << zc->params.cParams.chainLog);
ZSTD_reduceTable(zc->chainTable, chainSize, reducerValue);
}
{ U32 const h3Size = (zc->hashLog3) ? 1 << zc->hashLog3 : 0; {
ZSTD_reduceTable(zc->hashTable3, h3Size, reducerValue); } U32 const h3Size = (zc->hashLog3) ? 1 << zc->hashLog3 : 0;
ZSTD_reduceTable(zc->hashTable3, h3Size, reducerValue);
}
} }
/*-******************************************************* /*-*******************************************************
* Block entropic compression * Block entropic compression
*********************************************************/ *********************************************************/
/* See doc/zstd_compression_format.md for detailed format description */ /* See doc/zstd_compression_format.md for detailed format description */
size_t ZSTD_noCompressBlock (void* dst, size_t dstCapacity, const void* src, size_t srcSize) size_t ZSTD_noCompressBlock(void *dst, size_t dstCapacity, const void *src, size_t srcSize)
{ {
if (srcSize + ZSTD_blockHeaderSize > dstCapacity) return ERROR(dstSize_tooSmall); if (srcSize + ZSTD_blockHeaderSize > dstCapacity)
memcpy((BYTE*)dst + ZSTD_blockHeaderSize, src, srcSize); return ERROR(dstSize_tooSmall);
memcpy((BYTE *)dst + ZSTD_blockHeaderSize, src, srcSize);
ZSTD_writeLE24(dst, (U32)(srcSize << 2) + (U32)bt_raw); ZSTD_writeLE24(dst, (U32)(srcSize << 2) + (U32)bt_raw);
return ZSTD_blockHeaderSize+srcSize; return ZSTD_blockHeaderSize + srcSize;
} }
static size_t ZSTD_noCompressLiterals(void *dst, size_t dstCapacity, const void *src, size_t srcSize)
static size_t ZSTD_noCompressLiterals (void* dst, size_t dstCapacity, const void* src, size_t srcSize)
{ {
BYTE* const ostart = (BYTE* const)dst; BYTE *const ostart = (BYTE * const)dst;
U32 const flSize = 1 + (srcSize>31) + (srcSize>4095); U32 const flSize = 1 + (srcSize > 31) + (srcSize > 4095);
if (srcSize + flSize > dstCapacity) return ERROR(dstSize_tooSmall); if (srcSize + flSize > dstCapacity)
return ERROR(dstSize_tooSmall);
switch(flSize) switch (flSize) {
{ case 1: /* 2 - 1 - 5 */ ostart[0] = (BYTE)((U32)set_basic + (srcSize << 3)); break;
case 1: /* 2 - 1 - 5 */ case 2: /* 2 - 2 - 12 */ ZSTD_writeLE16(ostart, (U16)((U32)set_basic + (1 << 2) + (srcSize << 4))); break;
ostart[0] = (BYTE)((U32)set_basic + (srcSize<<3));
break;
case 2: /* 2 - 2 - 12 */
ZSTD_writeLE16(ostart, (U16)((U32)set_basic + (1<<2) + (srcSize<<4)));
break;
default: /*note : should not be necessary : flSize is within {1,2,3} */ default: /*note : should not be necessary : flSize is within {1,2,3} */
case 3: /* 2 - 2 - 20 */ case 3: /* 2 - 2 - 20 */ ZSTD_writeLE32(ostart, (U32)((U32)set_basic + (3 << 2) + (srcSize << 4))); break;
ZSTD_writeLE32(ostart, (U32)((U32)set_basic + (3<<2) + (srcSize<<4)));
break;
} }
memcpy(ostart + flSize, src, srcSize); memcpy(ostart + flSize, src, srcSize);
return srcSize + flSize; return srcSize + flSize;
} }
static size_t ZSTD_compressRleLiteralsBlock (void* dst, size_t dstCapacity, const void* src, size_t srcSize) static size_t ZSTD_compressRleLiteralsBlock(void *dst, size_t dstCapacity, const void *src, size_t srcSize)
{ {
BYTE* const ostart = (BYTE* const)dst; BYTE *const ostart = (BYTE * const)dst;
U32 const flSize = 1 + (srcSize>31) + (srcSize>4095); U32 const flSize = 1 + (srcSize > 31) + (srcSize > 4095);
(void)dstCapacity; /* dstCapacity already guaranteed to be >=4, hence large enough */ (void)dstCapacity; /* dstCapacity already guaranteed to be >=4, hence large enough */
switch(flSize) switch (flSize) {
{ case 1: /* 2 - 1 - 5 */ ostart[0] = (BYTE)((U32)set_rle + (srcSize << 3)); break;
case 1: /* 2 - 1 - 5 */ case 2: /* 2 - 2 - 12 */ ZSTD_writeLE16(ostart, (U16)((U32)set_rle + (1 << 2) + (srcSize << 4))); break;
ostart[0] = (BYTE)((U32)set_rle + (srcSize<<3));
break;
case 2: /* 2 - 2 - 12 */
ZSTD_writeLE16(ostart, (U16)((U32)set_rle + (1<<2) + (srcSize<<4)));
break;
default: /*note : should not be necessary : flSize is necessarily within {1,2,3} */ default: /*note : should not be necessary : flSize is necessarily within {1,2,3} */
case 3: /* 2 - 2 - 20 */ case 3: /* 2 - 2 - 20 */ ZSTD_writeLE32(ostart, (U32)((U32)set_rle + (3 << 2) + (srcSize << 4))); break;
ZSTD_writeLE32(ostart, (U32)((U32)set_rle + (3<<2) + (srcSize<<4)));
break;
} }
ostart[flSize] = *(const BYTE*)src; ostart[flSize] = *(const BYTE *)src;
return flSize+1; return flSize + 1;
} }
static size_t ZSTD_minGain(size_t srcSize) { return (srcSize >> 6) + 2; } static size_t ZSTD_minGain(size_t srcSize) { return (srcSize >> 6) + 2; }
static size_t ZSTD_compressLiterals (ZSTD_CCtx* zc, static size_t ZSTD_compressLiterals(ZSTD_CCtx *zc, void *dst, size_t dstCapacity, const void *src, size_t srcSize)
void* dst, size_t dstCapacity,
const void* src, size_t srcSize)
{ {
size_t const minGain = ZSTD_minGain(srcSize); size_t const minGain = ZSTD_minGain(srcSize);
size_t const lhSize = 3 + (srcSize >= 1 KB) + (srcSize >= 16 KB); size_t const lhSize = 3 + (srcSize >= 1 KB) + (srcSize >= 16 KB);
BYTE* const ostart = (BYTE*)dst; BYTE *const ostart = (BYTE *)dst;
U32 singleStream = srcSize < 256; U32 singleStream = srcSize < 256;
symbolEncodingType_e hType = set_compressed; symbolEncodingType_e hType = set_compressed;
size_t cLitSize; size_t cLitSize;
/* small ? don't even attempt compression (speed opt) */
/* small ? don't even attempt compression (speed opt) */ #define LITERAL_NOENTROPY 63
# define LITERAL_NOENTROPY 63 {
{ size_t const minLitSize = zc->flagStaticHufTable == HUF_repeat_valid ? 6 : LITERAL_NOENTROPY; size_t const minLitSize = zc->flagStaticHufTable == HUF_repeat_valid ? 6 : LITERAL_NOENTROPY;
if (srcSize <= minLitSize) return ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize); if (srcSize <= minLitSize)
return ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize);
} }
if (dstCapacity < lhSize+1) return ERROR(dstSize_tooSmall); /* not enough space for compression */ if (dstCapacity < lhSize + 1)
{ HUF_repeat repeat = zc->flagStaticHufTable; return ERROR(dstSize_tooSmall); /* not enough space for compression */
{
HUF_repeat repeat = zc->flagStaticHufTable;
int const preferRepeat = zc->params.cParams.strategy < ZSTD_lazy ? srcSize <= 1024 : 0; int const preferRepeat = zc->params.cParams.strategy < ZSTD_lazy ? srcSize <= 1024 : 0;
if (repeat == HUF_repeat_valid && lhSize == 3) singleStream = 1; if (repeat == HUF_repeat_valid && lhSize == 3)
cLitSize = singleStream ? HUF_compress1X_repeat(ostart+lhSize, dstCapacity-lhSize, src, srcSize, 255, 11, zc->tmpCounters, sizeof(zc->tmpCounters), zc->hufTable, &repeat, preferRepeat) singleStream = 1;
: HUF_compress4X_repeat(ostart+lhSize, dstCapacity-lhSize, src, srcSize, 255, 11, zc->tmpCounters, sizeof(zc->tmpCounters), zc->hufTable, &repeat, preferRepeat); cLitSize = singleStream ? HUF_compress1X_repeat(ostart + lhSize, dstCapacity - lhSize, src, srcSize, 255, 11, zc->tmpCounters,
if (repeat != HUF_repeat_none) { hType = set_repeat; } /* reused the existing table */ sizeof(zc->tmpCounters), zc->hufTable, &repeat, preferRepeat)
else { zc->flagStaticHufTable = HUF_repeat_check; } /* now have a table to reuse */ : HUF_compress4X_repeat(ostart + lhSize, dstCapacity - lhSize, src, srcSize, 255, 11, zc->tmpCounters,
sizeof(zc->tmpCounters), zc->hufTable, &repeat, preferRepeat);
if (repeat != HUF_repeat_none) {
hType = set_repeat;
} /* reused the existing table */
else {
zc->flagStaticHufTable = HUF_repeat_check;
} /* now have a table to reuse */
} }
if ((cLitSize==0) | (cLitSize >= srcSize - minGain)) { if ((cLitSize == 0) | (cLitSize >= srcSize - minGain)) {
zc->flagStaticHufTable = HUF_repeat_none; zc->flagStaticHufTable = HUF_repeat_none;
return ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize); return ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize);
} }
if (cLitSize==1) { if (cLitSize == 1) {
zc->flagStaticHufTable = HUF_repeat_none; zc->flagStaticHufTable = HUF_repeat_none;
return ZSTD_compressRleLiteralsBlock(dst, dstCapacity, src, srcSize); return ZSTD_compressRleLiteralsBlock(dst, dstCapacity, src, srcSize);
} }
/* Build header */ /* Build header */
switch(lhSize) switch (lhSize) {
{
case 3: /* 2 - 2 - 10 - 10 */ case 3: /* 2 - 2 - 10 - 10 */
{ U32 const lhc = hType + ((!singleStream) << 2) + ((U32)srcSize<<4) + ((U32)cLitSize<<14); {
U32 const lhc = hType + ((!singleStream) << 2) + ((U32)srcSize << 4) + ((U32)cLitSize << 14);
ZSTD_writeLE24(ostart, lhc); ZSTD_writeLE24(ostart, lhc);
break; break;
} }
case 4: /* 2 - 2 - 14 - 14 */ case 4: /* 2 - 2 - 14 - 14 */
{ U32 const lhc = hType + (2 << 2) + ((U32)srcSize<<4) + ((U32)cLitSize<<18); {
U32 const lhc = hType + (2 << 2) + ((U32)srcSize << 4) + ((U32)cLitSize << 18);
ZSTD_writeLE32(ostart, lhc); ZSTD_writeLE32(ostart, lhc);
break; break;
} }
default: /* should not be necessary, lhSize is only {3,4,5} */ default: /* should not be necessary, lhSize is only {3,4,5} */
case 5: /* 2 - 2 - 18 - 18 */ case 5: /* 2 - 2 - 18 - 18 */
{ U32 const lhc = hType + (3 << 2) + ((U32)srcSize<<4) + ((U32)cLitSize<<22); {
U32 const lhc = hType + (3 << 2) + ((U32)srcSize << 4) + ((U32)cLitSize << 22);
ZSTD_writeLE32(ostart, lhc); ZSTD_writeLE32(ostart, lhc);
ostart[4] = (BYTE)(cLitSize >> 10); ostart[4] = (BYTE)(cLitSize >> 10);
break; break;
} }
} }
return lhSize+cLitSize; return lhSize + cLitSize;
} }
static const BYTE LL_Code[64] = { 0, 1, 2, 3, 4, 5, 6, 7, static const BYTE LL_Code[64] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 16, 17, 17, 18, 18,
8, 9, 10, 11, 12, 13, 14, 15, 19, 19, 20, 20, 20, 20, 21, 21, 21, 21, 22, 22, 22, 22, 22, 22, 22, 22, 23, 23, 23, 23,
16, 16, 17, 17, 18, 18, 19, 19, 23, 23, 23, 23, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24};
20, 20, 20, 20, 21, 21, 21, 21,
22, 22, 22, 22, 22, 22, 22, 22,
23, 23, 23, 23, 23, 23, 23, 23,
24, 24, 24, 24, 24, 24, 24, 24,
24, 24, 24, 24, 24, 24, 24, 24 };
static const BYTE ML_Code[128] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, static const BYTE ML_Code[128] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25,
16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 26, 27, 28, 29, 30, 31, 32, 32, 33, 33, 34, 34, 35, 35, 36, 36, 36, 36, 37, 37, 37, 37, 38, 38, 38, 38,
32, 32, 33, 33, 34, 34, 35, 35, 36, 36, 36, 36, 37, 37, 37, 37, 38, 38, 38, 38, 39, 39, 39, 39, 39, 39, 39, 39, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40,
38, 38, 38, 38, 38, 38, 38, 38, 39, 39, 39, 39, 39, 39, 39, 39, 40, 40, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 42, 42, 42, 42, 42, 42, 42, 42,
40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42};
41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41,
42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42,
42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42 };
void ZSTD_seqToCodes(const seqStore_t *seqStorePtr)
void ZSTD_seqToCodes(const seqStore_t* seqStorePtr)
{ {
BYTE const LL_deltaCode = 19; BYTE const LL_deltaCode = 19;
BYTE const ML_deltaCode = 36; BYTE const ML_deltaCode = 36;
const seqDef* const sequences = seqStorePtr->sequencesStart; const seqDef *const sequences = seqStorePtr->sequencesStart;
BYTE* const llCodeTable = seqStorePtr->llCode; BYTE *const llCodeTable = seqStorePtr->llCode;
BYTE* const ofCodeTable = seqStorePtr->ofCode; BYTE *const ofCodeTable = seqStorePtr->ofCode;
BYTE* const mlCodeTable = seqStorePtr->mlCode; BYTE *const mlCodeTable = seqStorePtr->mlCode;
U32 const nbSeq = (U32)(seqStorePtr->sequences - seqStorePtr->sequencesStart); U32 const nbSeq = (U32)(seqStorePtr->sequences - seqStorePtr->sequencesStart);
U32 u; U32 u;
for (u=0; u<nbSeq; u++) { for (u = 0; u < nbSeq; u++) {
U32 const llv = sequences[u].litLength; U32 const llv = sequences[u].litLength;
U32 const mlv = sequences[u].matchLength; U32 const mlv = sequences[u].matchLength;
llCodeTable[u] = (llv> 63) ? (BYTE)ZSTD_highbit32(llv) + LL_deltaCode : LL_Code[llv]; llCodeTable[u] = (llv > 63) ? (BYTE)ZSTD_highbit32(llv) + LL_deltaCode : LL_Code[llv];
ofCodeTable[u] = (BYTE)ZSTD_highbit32(sequences[u].offset); ofCodeTable[u] = (BYTE)ZSTD_highbit32(sequences[u].offset);
mlCodeTable[u] = (mlv>127) ? (BYTE)ZSTD_highbit32(mlv) + ML_deltaCode : ML_Code[mlv]; mlCodeTable[u] = (mlv > 127) ? (BYTE)ZSTD_highbit32(mlv) + ML_deltaCode : ML_Code[mlv];
} }
if (seqStorePtr->longLengthID==1) if (seqStorePtr->longLengthID == 1)
llCodeTable[seqStorePtr->longLengthPos] = MaxLL; llCodeTable[seqStorePtr->longLengthPos] = MaxLL;
if (seqStorePtr->longLengthID==2) if (seqStorePtr->longLengthID == 2)
mlCodeTable[seqStorePtr->longLengthPos] = MaxML; mlCodeTable[seqStorePtr->longLengthPos] = MaxML;
} }
ZSTD_STATIC size_t ZSTD_compressSequences (ZSTD_CCtx* zc, ZSTD_STATIC size_t ZSTD_compressSequences(ZSTD_CCtx *zc, void *dst, size_t dstCapacity, size_t srcSize)
void* dst, size_t dstCapacity,
size_t srcSize)
{ {
const int longOffsets = zc->params.cParams.windowLog > STREAM_ACCUMULATOR_MIN; const int longOffsets = zc->params.cParams.windowLog > STREAM_ACCUMULATOR_MIN;
const seqStore_t* seqStorePtr = &(zc->seqStore); const seqStore_t *seqStorePtr = &(zc->seqStore);
U32 count[MaxSeq+1]; U32 count[MaxSeq + 1];
S16 norm[MaxSeq+1]; S16 norm[MaxSeq + 1];
FSE_CTable* CTable_LitLength = zc->litlengthCTable; FSE_CTable *CTable_LitLength = zc->litlengthCTable;
FSE_CTable* CTable_OffsetBits = zc->offcodeCTable; FSE_CTable *CTable_OffsetBits = zc->offcodeCTable;
FSE_CTable* CTable_MatchLength = zc->matchlengthCTable; FSE_CTable *CTable_MatchLength = zc->matchlengthCTable;
U32 LLtype, Offtype, MLtype; /* compressed, raw or rle */ U32 LLtype, Offtype, MLtype; /* compressed, raw or rle */
const seqDef* const sequences = seqStorePtr->sequencesStart; const seqDef *const sequences = seqStorePtr->sequencesStart;
const BYTE* const ofCodeTable = seqStorePtr->ofCode; const BYTE *const ofCodeTable = seqStorePtr->ofCode;
const BYTE* const llCodeTable = seqStorePtr->llCode; const BYTE *const llCodeTable = seqStorePtr->llCode;
const BYTE* const mlCodeTable = seqStorePtr->mlCode; const BYTE *const mlCodeTable = seqStorePtr->mlCode;
BYTE* const ostart = (BYTE*)dst; BYTE *const ostart = (BYTE *)dst;
BYTE* const oend = ostart + dstCapacity; BYTE *const oend = ostart + dstCapacity;
BYTE* op = ostart; BYTE *op = ostart;
size_t const nbSeq = seqStorePtr->sequences - seqStorePtr->sequencesStart; size_t const nbSeq = seqStorePtr->sequences - seqStorePtr->sequencesStart;
BYTE* seqHead; BYTE *seqHead;
BYTE scratchBuffer[1<<MAX(MLFSELog,LLFSELog)]; BYTE scratchBuffer[1 << MAX(MLFSELog, LLFSELog)];
/* Compress literals */ /* Compress literals */
{ const BYTE* const literals = seqStorePtr->litStart; {
const BYTE *const literals = seqStorePtr->litStart;
size_t const litSize = seqStorePtr->lit - literals; size_t const litSize = seqStorePtr->lit - literals;
size_t const cSize = ZSTD_compressLiterals(zc, op, dstCapacity, literals, litSize); size_t const cSize = ZSTD_compressLiterals(zc, op, dstCapacity, literals, litSize);
if (ZSTD_isError(cSize)) return cSize; if (ZSTD_isError(cSize))
return cSize;
op += cSize; op += cSize;
} }
/* Sequences Header */ /* Sequences Header */
if ((oend-op) < 3 /*max nbSeq Size*/ + 1 /*seqHead */) return ERROR(dstSize_tooSmall); if ((oend - op) < 3 /*max nbSeq Size*/ + 1 /*seqHead */)
if (nbSeq < 0x7F) *op++ = (BYTE)nbSeq; return ERROR(dstSize_tooSmall);
else if (nbSeq < LONGNBSEQ) op[0] = (BYTE)((nbSeq>>8) + 0x80), op[1] = (BYTE)nbSeq, op+=2; if (nbSeq < 0x7F)
else op[0]=0xFF, ZSTD_writeLE16(op+1, (U16)(nbSeq - LONGNBSEQ)), op+=3; *op++ = (BYTE)nbSeq;
if (nbSeq==0) goto _check_compressibility; else if (nbSeq < LONGNBSEQ)
op[0] = (BYTE)((nbSeq >> 8) + 0x80), op[1] = (BYTE)nbSeq, op += 2;
else
op[0] = 0xFF, ZSTD_writeLE16(op + 1, (U16)(nbSeq - LONGNBSEQ)), op += 3;
if (nbSeq == 0)
goto _check_compressibility;
/* seqHead : flags for FSE encoding type */ /* seqHead : flags for FSE encoding type */
seqHead = op++; seqHead = op++;
@@ -620,7 +640,8 @@ ZSTD_STATIC size_t ZSTD_compressSequences (ZSTD_CCtx* zc,
ZSTD_seqToCodes(seqStorePtr); ZSTD_seqToCodes(seqStorePtr);
/* CTable for Literal Lengths */ /* CTable for Literal Lengths */
{ U32 max = MaxLL; {
U32 max = MaxLL;
size_t const mostFrequent = FSE_countFast_wksp(count, &max, llCodeTable, nbSeq, zc->tmpCounters); size_t const mostFrequent = FSE_countFast_wksp(count, &max, llCodeTable, nbSeq, zc->tmpCounters);
if ((mostFrequent == nbSeq) && (nbSeq > 2)) { if ((mostFrequent == nbSeq) && (nbSeq > 2)) {
*op++ = llCodeTable[0]; *op++ = llCodeTable[0];
@@ -628,23 +649,31 @@ ZSTD_STATIC size_t ZSTD_compressSequences (ZSTD_CCtx* zc,
LLtype = set_rle; LLtype = set_rle;
} else if ((zc->flagStaticTables) && (nbSeq < MAX_SEQ_FOR_STATIC_FSE)) { } else if ((zc->flagStaticTables) && (nbSeq < MAX_SEQ_FOR_STATIC_FSE)) {
LLtype = set_repeat; LLtype = set_repeat;
} else if ((nbSeq < MIN_SEQ_FOR_DYNAMIC_FSE) || (mostFrequent < (nbSeq >> (LL_defaultNormLog-1)))) { } else if ((nbSeq < MIN_SEQ_FOR_DYNAMIC_FSE) || (mostFrequent < (nbSeq >> (LL_defaultNormLog - 1)))) {
FSE_buildCTable_wksp(CTable_LitLength, LL_defaultNorm, MaxLL, LL_defaultNormLog, scratchBuffer, sizeof(scratchBuffer)); FSE_buildCTable_wksp(CTable_LitLength, LL_defaultNorm, MaxLL, LL_defaultNormLog, scratchBuffer, sizeof(scratchBuffer));
LLtype = set_basic; LLtype = set_basic;
} else { } else {
size_t nbSeq_1 = nbSeq; size_t nbSeq_1 = nbSeq;
const U32 tableLog = FSE_optimalTableLog(LLFSELog, nbSeq, max); const U32 tableLog = FSE_optimalTableLog(LLFSELog, nbSeq, max);
if (count[llCodeTable[nbSeq-1]]>1) { count[llCodeTable[nbSeq-1]]--; nbSeq_1--; } if (count[llCodeTable[nbSeq - 1]] > 1) {
count[llCodeTable[nbSeq - 1]]--;
nbSeq_1--;
}
FSE_normalizeCount(norm, tableLog, count, nbSeq_1, max); FSE_normalizeCount(norm, tableLog, count, nbSeq_1, max);
{ size_t const NCountSize = FSE_writeNCount(op, oend-op, norm, max, tableLog); /* overflow protected */ {
if (FSE_isError(NCountSize)) return NCountSize; size_t const NCountSize = FSE_writeNCount(op, oend - op, norm, max, tableLog); /* overflow protected */
op += NCountSize; } if (FSE_isError(NCountSize))
return NCountSize;
op += NCountSize;
}
FSE_buildCTable_wksp(CTable_LitLength, norm, max, tableLog, scratchBuffer, sizeof(scratchBuffer)); FSE_buildCTable_wksp(CTable_LitLength, norm, max, tableLog, scratchBuffer, sizeof(scratchBuffer));
LLtype = set_compressed; LLtype = set_compressed;
} } }
}
/* CTable for Offsets */ /* CTable for Offsets */
{ U32 max = MaxOff; {
U32 max = MaxOff;
size_t const mostFrequent = FSE_countFast_wksp(count, &max, ofCodeTable, nbSeq, zc->tmpCounters); size_t const mostFrequent = FSE_countFast_wksp(count, &max, ofCodeTable, nbSeq, zc->tmpCounters);
if ((mostFrequent == nbSeq) && (nbSeq > 2)) { if ((mostFrequent == nbSeq) && (nbSeq > 2)) {
*op++ = ofCodeTable[0]; *op++ = ofCodeTable[0];
@@ -652,23 +681,31 @@ ZSTD_STATIC size_t ZSTD_compressSequences (ZSTD_CCtx* zc,
Offtype = set_rle; Offtype = set_rle;
} else if ((zc->flagStaticTables) && (nbSeq < MAX_SEQ_FOR_STATIC_FSE)) { } else if ((zc->flagStaticTables) && (nbSeq < MAX_SEQ_FOR_STATIC_FSE)) {
Offtype = set_repeat; Offtype = set_repeat;
} else if ((nbSeq < MIN_SEQ_FOR_DYNAMIC_FSE) || (mostFrequent < (nbSeq >> (OF_defaultNormLog-1)))) { } else if ((nbSeq < MIN_SEQ_FOR_DYNAMIC_FSE) || (mostFrequent < (nbSeq >> (OF_defaultNormLog - 1)))) {
FSE_buildCTable_wksp(CTable_OffsetBits, OF_defaultNorm, MaxOff, OF_defaultNormLog, scratchBuffer, sizeof(scratchBuffer)); FSE_buildCTable_wksp(CTable_OffsetBits, OF_defaultNorm, MaxOff, OF_defaultNormLog, scratchBuffer, sizeof(scratchBuffer));
Offtype = set_basic; Offtype = set_basic;
} else { } else {
size_t nbSeq_1 = nbSeq; size_t nbSeq_1 = nbSeq;
const U32 tableLog = FSE_optimalTableLog(OffFSELog, nbSeq, max); const U32 tableLog = FSE_optimalTableLog(OffFSELog, nbSeq, max);
if (count[ofCodeTable[nbSeq-1]]>1) { count[ofCodeTable[nbSeq-1]]--; nbSeq_1--; } if (count[ofCodeTable[nbSeq - 1]] > 1) {
count[ofCodeTable[nbSeq - 1]]--;
nbSeq_1--;
}
FSE_normalizeCount(norm, tableLog, count, nbSeq_1, max); FSE_normalizeCount(norm, tableLog, count, nbSeq_1, max);
{ size_t const NCountSize = FSE_writeNCount(op, oend-op, norm, max, tableLog); /* overflow protected */ {
if (FSE_isError(NCountSize)) return NCountSize; size_t const NCountSize = FSE_writeNCount(op, oend - op, norm, max, tableLog); /* overflow protected */
op += NCountSize; } if (FSE_isError(NCountSize))
return NCountSize;
op += NCountSize;
}
FSE_buildCTable_wksp(CTable_OffsetBits, norm, max, tableLog, scratchBuffer, sizeof(scratchBuffer)); FSE_buildCTable_wksp(CTable_OffsetBits, norm, max, tableLog, scratchBuffer, sizeof(scratchBuffer));
Offtype = set_compressed; Offtype = set_compressed;
} } }
}
/* CTable for MatchLengths */ /* CTable for MatchLengths */
{ U32 max = MaxML; {
U32 max = MaxML;
size_t const mostFrequent = FSE_countFast_wksp(count, &max, mlCodeTable, nbSeq, zc->tmpCounters); size_t const mostFrequent = FSE_countFast_wksp(count, &max, mlCodeTable, nbSeq, zc->tmpCounters);
if ((mostFrequent == nbSeq) && (nbSeq > 2)) { if ((mostFrequent == nbSeq) && (nbSeq > 2)) {
*op++ = *mlCodeTable; *op++ = *mlCodeTable;
@@ -676,56 +713,66 @@ ZSTD_STATIC size_t ZSTD_compressSequences (ZSTD_CCtx* zc,
MLtype = set_rle; MLtype = set_rle;
} else if ((zc->flagStaticTables) && (nbSeq < MAX_SEQ_FOR_STATIC_FSE)) { } else if ((zc->flagStaticTables) && (nbSeq < MAX_SEQ_FOR_STATIC_FSE)) {
MLtype = set_repeat; MLtype = set_repeat;
} else if ((nbSeq < MIN_SEQ_FOR_DYNAMIC_FSE) || (mostFrequent < (nbSeq >> (ML_defaultNormLog-1)))) { } else if ((nbSeq < MIN_SEQ_FOR_DYNAMIC_FSE) || (mostFrequent < (nbSeq >> (ML_defaultNormLog - 1)))) {
FSE_buildCTable_wksp(CTable_MatchLength, ML_defaultNorm, MaxML, ML_defaultNormLog, scratchBuffer, sizeof(scratchBuffer)); FSE_buildCTable_wksp(CTable_MatchLength, ML_defaultNorm, MaxML, ML_defaultNormLog, scratchBuffer, sizeof(scratchBuffer));
MLtype = set_basic; MLtype = set_basic;
} else { } else {
size_t nbSeq_1 = nbSeq; size_t nbSeq_1 = nbSeq;
const U32 tableLog = FSE_optimalTableLog(MLFSELog, nbSeq, max); const U32 tableLog = FSE_optimalTableLog(MLFSELog, nbSeq, max);
if (count[mlCodeTable[nbSeq-1]]>1) { count[mlCodeTable[nbSeq-1]]--; nbSeq_1--; } if (count[mlCodeTable[nbSeq - 1]] > 1) {
count[mlCodeTable[nbSeq - 1]]--;
nbSeq_1--;
}
FSE_normalizeCount(norm, tableLog, count, nbSeq_1, max); FSE_normalizeCount(norm, tableLog, count, nbSeq_1, max);
{ size_t const NCountSize = FSE_writeNCount(op, oend-op, norm, max, tableLog); /* overflow protected */ {
if (FSE_isError(NCountSize)) return NCountSize; size_t const NCountSize = FSE_writeNCount(op, oend - op, norm, max, tableLog); /* overflow protected */
op += NCountSize; } if (FSE_isError(NCountSize))
return NCountSize;
op += NCountSize;
}
FSE_buildCTable_wksp(CTable_MatchLength, norm, max, tableLog, scratchBuffer, sizeof(scratchBuffer)); FSE_buildCTable_wksp(CTable_MatchLength, norm, max, tableLog, scratchBuffer, sizeof(scratchBuffer));
MLtype = set_compressed; MLtype = set_compressed;
} } }
}
*seqHead = (BYTE)((LLtype<<6) + (Offtype<<4) + (MLtype<<2)); *seqHead = (BYTE)((LLtype << 6) + (Offtype << 4) + (MLtype << 2));
zc->flagStaticTables = 0; zc->flagStaticTables = 0;
/* Encoding Sequences */ /* Encoding Sequences */
{ BIT_CStream_t blockStream; {
BIT_CStream_t blockStream;
FSE_CState_t stateMatchLength; FSE_CState_t stateMatchLength;
FSE_CState_t stateOffsetBits; FSE_CState_t stateOffsetBits;
FSE_CState_t stateLitLength; FSE_CState_t stateLitLength;
CHECK_E(BIT_initCStream(&blockStream, op, oend-op), dstSize_tooSmall); /* not enough space remaining */ CHECK_E(BIT_initCStream(&blockStream, op, oend - op), dstSize_tooSmall); /* not enough space remaining */
/* first symbols */ /* first symbols */
FSE_initCState2(&stateMatchLength, CTable_MatchLength, mlCodeTable[nbSeq-1]); FSE_initCState2(&stateMatchLength, CTable_MatchLength, mlCodeTable[nbSeq - 1]);
FSE_initCState2(&stateOffsetBits, CTable_OffsetBits, ofCodeTable[nbSeq-1]); FSE_initCState2(&stateOffsetBits, CTable_OffsetBits, ofCodeTable[nbSeq - 1]);
FSE_initCState2(&stateLitLength, CTable_LitLength, llCodeTable[nbSeq-1]); FSE_initCState2(&stateLitLength, CTable_LitLength, llCodeTable[nbSeq - 1]);
BIT_addBits(&blockStream, sequences[nbSeq-1].litLength, LL_bits[llCodeTable[nbSeq-1]]); BIT_addBits(&blockStream, sequences[nbSeq - 1].litLength, LL_bits[llCodeTable[nbSeq - 1]]);
if (ZSTD_32bits()) BIT_flushBits(&blockStream); if (ZSTD_32bits())
BIT_addBits(&blockStream, sequences[nbSeq-1].matchLength, ML_bits[mlCodeTable[nbSeq-1]]); BIT_flushBits(&blockStream);
if (ZSTD_32bits()) BIT_flushBits(&blockStream); BIT_addBits(&blockStream, sequences[nbSeq - 1].matchLength, ML_bits[mlCodeTable[nbSeq - 1]]);
if (ZSTD_32bits())
BIT_flushBits(&blockStream);
if (longOffsets) { if (longOffsets) {
U32 const ofBits = ofCodeTable[nbSeq-1]; U32 const ofBits = ofCodeTable[nbSeq - 1];
int const extraBits = ofBits - MIN(ofBits, STREAM_ACCUMULATOR_MIN-1); int const extraBits = ofBits - MIN(ofBits, STREAM_ACCUMULATOR_MIN - 1);
if (extraBits) { if (extraBits) {
BIT_addBits(&blockStream, sequences[nbSeq-1].offset, extraBits); BIT_addBits(&blockStream, sequences[nbSeq - 1].offset, extraBits);
BIT_flushBits(&blockStream); BIT_flushBits(&blockStream);
} }
BIT_addBits(&blockStream, sequences[nbSeq-1].offset >> extraBits, BIT_addBits(&blockStream, sequences[nbSeq - 1].offset >> extraBits, ofBits - extraBits);
ofBits - extraBits);
} else { } else {
BIT_addBits(&blockStream, sequences[nbSeq-1].offset, ofCodeTable[nbSeq-1]); BIT_addBits(&blockStream, sequences[nbSeq - 1].offset, ofCodeTable[nbSeq - 1]);
} }
BIT_flushBits(&blockStream); BIT_flushBits(&blockStream);
{ size_t n; {
for (n=nbSeq-2 ; n<nbSeq ; n--) { /* intentional underflow */ size_t n;
for (n = nbSeq - 2; n < nbSeq; n--) { /* intentional underflow */
BYTE const llCode = llCodeTable[n]; BYTE const llCode = llCodeTable[n];
BYTE const ofCode = ofCodeTable[n]; BYTE const ofCode = ofCodeTable[n];
BYTE const mlCode = mlCodeTable[n]; BYTE const mlCode = mlCodeTable[n];
@@ -735,48 +782,59 @@ ZSTD_STATIC size_t ZSTD_compressSequences (ZSTD_CCtx* zc,
/* (7)*/ /* (7)*/ /* (7)*/ /* (7)*/
FSE_encodeSymbol(&blockStream, &stateOffsetBits, ofCode); /* 15 */ /* 15 */ FSE_encodeSymbol(&blockStream, &stateOffsetBits, ofCode); /* 15 */ /* 15 */
FSE_encodeSymbol(&blockStream, &stateMatchLength, mlCode); /* 24 */ /* 24 */ FSE_encodeSymbol(&blockStream, &stateMatchLength, mlCode); /* 24 */ /* 24 */
if (ZSTD_32bits()) BIT_flushBits(&blockStream); /* (7)*/ if (ZSTD_32bits())
BIT_flushBits(&blockStream); /* (7)*/
FSE_encodeSymbol(&blockStream, &stateLitLength, llCode); /* 16 */ /* 33 */ FSE_encodeSymbol(&blockStream, &stateLitLength, llCode); /* 16 */ /* 33 */
if (ZSTD_32bits() || (ofBits+mlBits+llBits >= 64-7-(LLFSELog+MLFSELog+OffFSELog))) if (ZSTD_32bits() || (ofBits + mlBits + llBits >= 64 - 7 - (LLFSELog + MLFSELog + OffFSELog)))
BIT_flushBits(&blockStream); /* (7)*/ BIT_flushBits(&blockStream); /* (7)*/
BIT_addBits(&blockStream, sequences[n].litLength, llBits); BIT_addBits(&blockStream, sequences[n].litLength, llBits);
if (ZSTD_32bits() && ((llBits+mlBits)>24)) BIT_flushBits(&blockStream); if (ZSTD_32bits() && ((llBits + mlBits) > 24))
BIT_flushBits(&blockStream);
BIT_addBits(&blockStream, sequences[n].matchLength, mlBits); BIT_addBits(&blockStream, sequences[n].matchLength, mlBits);
if (ZSTD_32bits()) BIT_flushBits(&blockStream); /* (7)*/ if (ZSTD_32bits())
BIT_flushBits(&blockStream); /* (7)*/
if (longOffsets) { if (longOffsets) {
int const extraBits = ofBits - MIN(ofBits, STREAM_ACCUMULATOR_MIN-1); int const extraBits = ofBits - MIN(ofBits, STREAM_ACCUMULATOR_MIN - 1);
if (extraBits) { if (extraBits) {
BIT_addBits(&blockStream, sequences[n].offset, extraBits); BIT_addBits(&blockStream, sequences[n].offset, extraBits);
BIT_flushBits(&blockStream); /* (7)*/ BIT_flushBits(&blockStream); /* (7)*/
} }
BIT_addBits(&blockStream, sequences[n].offset >> extraBits, BIT_addBits(&blockStream, sequences[n].offset >> extraBits, ofBits - extraBits); /* 31 */
ofBits - extraBits); /* 31 */
} else { } else {
BIT_addBits(&blockStream, sequences[n].offset, ofBits); /* 31 */ BIT_addBits(&blockStream, sequences[n].offset, ofBits); /* 31 */
} }
BIT_flushBits(&blockStream); /* (7)*/ BIT_flushBits(&blockStream); /* (7)*/
} } }
}
FSE_flushCState(&blockStream, &stateMatchLength); FSE_flushCState(&blockStream, &stateMatchLength);
FSE_flushCState(&blockStream, &stateOffsetBits); FSE_flushCState(&blockStream, &stateOffsetBits);
FSE_flushCState(&blockStream, &stateLitLength); FSE_flushCState(&blockStream, &stateLitLength);
{ size_t const streamSize = BIT_closeCStream(&blockStream); {
if (streamSize==0) return ERROR(dstSize_tooSmall); /* not enough space */ size_t const streamSize = BIT_closeCStream(&blockStream);
if (streamSize == 0)
return ERROR(dstSize_tooSmall); /* not enough space */
op += streamSize; op += streamSize;
} } }
}
/* check compressibility */ /* check compressibility */
_check_compressibility: _check_compressibility : {
{ size_t const minGain = ZSTD_minGain(srcSize); size_t const minGain = ZSTD_minGain(srcSize);
size_t const maxCSize = srcSize - minGain; size_t const maxCSize = srcSize - minGain;
if ((size_t)(op-ostart) >= maxCSize) { if ((size_t)(op - ostart) >= maxCSize) {
zc->flagStaticHufTable = HUF_repeat_none; zc->flagStaticHufTable = HUF_repeat_none;
return 0; return 0;
} } }
}
/* confirm repcodes */ /* confirm repcodes */
{ int i; for (i=0; i<ZSTD_REP_NUM; i++) zc->rep[i] = zc->repToConfirm[i]; } {
int i;
for (i = 0; i < ZSTD_REP_NUM; i++)
zc->rep[i] = zc->repToConfirm[i];
}
return op - ostart; return op - ostart;
} }
@@ -786,31 +844,36 @@ _check_compressibility:
`offsetCode` : distance to match, or 0 == repCode. `offsetCode` : distance to match, or 0 == repCode.
`matchCode` : matchLength - MINMATCH `matchCode` : matchLength - MINMATCH
*/ */
ZSTD_STATIC void ZSTD_storeSeq(seqStore_t* seqStorePtr, size_t litLength, const void* literals, U32 offsetCode, size_t matchCode) ZSTD_STATIC void ZSTD_storeSeq(seqStore_t *seqStorePtr, size_t litLength, const void *literals, U32 offsetCode, size_t matchCode)
{ {
/* copy Literals */ /* copy Literals */
ZSTD_wildcopy(seqStorePtr->lit, literals, litLength); ZSTD_wildcopy(seqStorePtr->lit, literals, litLength);
seqStorePtr->lit += litLength; seqStorePtr->lit += litLength;
/* literal Length */ /* literal Length */
if (litLength>0xFFFF) { seqStorePtr->longLengthID = 1; seqStorePtr->longLengthPos = (U32)(seqStorePtr->sequences - seqStorePtr->sequencesStart); } if (litLength > 0xFFFF) {
seqStorePtr->longLengthID = 1;
seqStorePtr->longLengthPos = (U32)(seqStorePtr->sequences - seqStorePtr->sequencesStart);
}
seqStorePtr->sequences[0].litLength = (U16)litLength; seqStorePtr->sequences[0].litLength = (U16)litLength;
/* match offset */ /* match offset */
seqStorePtr->sequences[0].offset = offsetCode + 1; seqStorePtr->sequences[0].offset = offsetCode + 1;
/* match Length */ /* match Length */
if (matchCode>0xFFFF) { seqStorePtr->longLengthID = 2; seqStorePtr->longLengthPos = (U32)(seqStorePtr->sequences - seqStorePtr->sequencesStart); } if (matchCode > 0xFFFF) {
seqStorePtr->longLengthID = 2;
seqStorePtr->longLengthPos = (U32)(seqStorePtr->sequences - seqStorePtr->sequencesStart);
}
seqStorePtr->sequences[0].matchLength = (U16)matchCode; seqStorePtr->sequences[0].matchLength = (U16)matchCode;
seqStorePtr->sequences++; seqStorePtr->sequences++;
} }
/*-************************************* /*-*************************************
* Match length counter * Match length counter
***************************************/ ***************************************/
static unsigned ZSTD_NbCommonBytes (register size_t val) static unsigned ZSTD_NbCommonBytes(register size_t val)
{ {
if (ZSTD_isLittleEndian()) { if (ZSTD_isLittleEndian()) {
if (ZSTD_64bits()) { if (ZSTD_64bits()) {
@@ -823,24 +886,36 @@ static unsigned ZSTD_NbCommonBytes (register size_t val)
return (__builtin_clzll(val) >> 3); return (__builtin_clzll(val) >> 3);
} else { /* 32 bits */ } else { /* 32 bits */
return (__builtin_clz((U32)val) >> 3); return (__builtin_clz((U32)val) >> 3);
} } }
}
} }
static size_t ZSTD_count(const BYTE *pIn, const BYTE *pMatch, const BYTE *const pInLimit)
static size_t ZSTD_count(const BYTE* pIn, const BYTE* pMatch, const BYTE* const pInLimit)
{ {
const BYTE* const pStart = pIn; const BYTE *const pStart = pIn;
const BYTE* const pInLoopLimit = pInLimit - (sizeof(size_t)-1); const BYTE *const pInLoopLimit = pInLimit - (sizeof(size_t) - 1);
while (pIn < pInLoopLimit) { while (pIn < pInLoopLimit) {
size_t const diff = ZSTD_readST(pMatch) ^ ZSTD_readST(pIn); size_t const diff = ZSTD_readST(pMatch) ^ ZSTD_readST(pIn);
if (!diff) { pIn+=sizeof(size_t); pMatch+=sizeof(size_t); continue; } if (!diff) {
pIn += sizeof(size_t);
pMatch += sizeof(size_t);
continue;
}
pIn += ZSTD_NbCommonBytes(diff); pIn += ZSTD_NbCommonBytes(diff);
return (size_t)(pIn - pStart); return (size_t)(pIn - pStart);
} }
if (ZSTD_64bits()) if ((pIn<(pInLimit-3)) && (ZSTD_read32(pMatch) == ZSTD_read32(pIn))) { pIn+=4; pMatch+=4; } if (ZSTD_64bits())
if ((pIn<(pInLimit-1)) && (ZSTD_read16(pMatch) == ZSTD_read16(pIn))) { pIn+=2; pMatch+=2; } if ((pIn < (pInLimit - 3)) && (ZSTD_read32(pMatch) == ZSTD_read32(pIn))) {
if ((pIn<pInLimit) && (*pMatch == *pIn)) pIn++; pIn += 4;
pMatch += 4;
}
if ((pIn < (pInLimit - 1)) && (ZSTD_read16(pMatch) == ZSTD_read16(pIn))) {
pIn += 2;
pMatch += 2;
}
if ((pIn < pInLimit) && (*pMatch == *pIn))
pIn++;
return (size_t)(pIn - pStart); return (size_t)(pIn - pStart);
} }
@@ -848,47 +923,46 @@ static size_t ZSTD_count(const BYTE* pIn, const BYTE* pMatch, const BYTE* const
* can count match length with `ip` & `match` in 2 different segments. * can count match length with `ip` & `match` in 2 different segments.
* convention : on reaching mEnd, match count continue starting from iStart * convention : on reaching mEnd, match count continue starting from iStart
*/ */
static size_t ZSTD_count_2segments(const BYTE* ip, const BYTE* match, const BYTE* iEnd, const BYTE* mEnd, const BYTE* iStart) static size_t ZSTD_count_2segments(const BYTE *ip, const BYTE *match, const BYTE *iEnd, const BYTE *mEnd, const BYTE *iStart)
{ {
const BYTE* const vEnd = MIN( ip + (mEnd - match), iEnd); const BYTE *const vEnd = MIN(ip + (mEnd - match), iEnd);
size_t const matchLength = ZSTD_count(ip, match, vEnd); size_t const matchLength = ZSTD_count(ip, match, vEnd);
if (match + matchLength != mEnd) return matchLength; if (match + matchLength != mEnd)
return matchLength + ZSTD_count(ip+matchLength, iStart, iEnd); return matchLength;
return matchLength + ZSTD_count(ip + matchLength, iStart, iEnd);
} }
/*-************************************* /*-*************************************
* Hashes * Hashes
***************************************/ ***************************************/
static const U32 prime3bytes = 506832829U; static const U32 prime3bytes = 506832829U;
static U32 ZSTD_hash3(U32 u, U32 h) { return ((u << (32-24)) * prime3bytes) >> (32-h) ; } static U32 ZSTD_hash3(U32 u, U32 h) { return ((u << (32 - 24)) * prime3bytes) >> (32 - h); }
ZSTD_STATIC size_t ZSTD_hash3Ptr(const void* ptr, U32 h) { return ZSTD_hash3(ZSTD_readLE32(ptr), h); } /* only in zstd_opt.h */ ZSTD_STATIC size_t ZSTD_hash3Ptr(const void *ptr, U32 h) { return ZSTD_hash3(ZSTD_readLE32(ptr), h); } /* only in zstd_opt.h */
static const U32 prime4bytes = 2654435761U; static const U32 prime4bytes = 2654435761U;
static U32 ZSTD_hash4(U32 u, U32 h) { return (u * prime4bytes) >> (32-h) ; } static U32 ZSTD_hash4(U32 u, U32 h) { return (u * prime4bytes) >> (32 - h); }
static size_t ZSTD_hash4Ptr(const void* ptr, U32 h) { return ZSTD_hash4(ZSTD_read32(ptr), h); } static size_t ZSTD_hash4Ptr(const void *ptr, U32 h) { return ZSTD_hash4(ZSTD_read32(ptr), h); }
static const U64 prime5bytes = 889523592379ULL; static const U64 prime5bytes = 889523592379ULL;
static size_t ZSTD_hash5(U64 u, U32 h) { return (size_t)(((u << (64-40)) * prime5bytes) >> (64-h)) ; } static size_t ZSTD_hash5(U64 u, U32 h) { return (size_t)(((u << (64 - 40)) * prime5bytes) >> (64 - h)); }
static size_t ZSTD_hash5Ptr(const void* p, U32 h) { return ZSTD_hash5(ZSTD_readLE64(p), h); } static size_t ZSTD_hash5Ptr(const void *p, U32 h) { return ZSTD_hash5(ZSTD_readLE64(p), h); }
static const U64 prime6bytes = 227718039650203ULL; static const U64 prime6bytes = 227718039650203ULL;
static size_t ZSTD_hash6(U64 u, U32 h) { return (size_t)(((u << (64-48)) * prime6bytes) >> (64-h)) ; } static size_t ZSTD_hash6(U64 u, U32 h) { return (size_t)(((u << (64 - 48)) * prime6bytes) >> (64 - h)); }
static size_t ZSTD_hash6Ptr(const void* p, U32 h) { return ZSTD_hash6(ZSTD_readLE64(p), h); } static size_t ZSTD_hash6Ptr(const void *p, U32 h) { return ZSTD_hash6(ZSTD_readLE64(p), h); }
static const U64 prime7bytes = 58295818150454627ULL; static const U64 prime7bytes = 58295818150454627ULL;
static size_t ZSTD_hash7(U64 u, U32 h) { return (size_t)(((u << (64-56)) * prime7bytes) >> (64-h)) ; } static size_t ZSTD_hash7(U64 u, U32 h) { return (size_t)(((u << (64 - 56)) * prime7bytes) >> (64 - h)); }
static size_t ZSTD_hash7Ptr(const void* p, U32 h) { return ZSTD_hash7(ZSTD_readLE64(p), h); } static size_t ZSTD_hash7Ptr(const void *p, U32 h) { return ZSTD_hash7(ZSTD_readLE64(p), h); }
static const U64 prime8bytes = 0xCF1BBCDCB7A56463ULL; static const U64 prime8bytes = 0xCF1BBCDCB7A56463ULL;
static size_t ZSTD_hash8(U64 u, U32 h) { return (size_t)(((u) * prime8bytes) >> (64-h)) ; } static size_t ZSTD_hash8(U64 u, U32 h) { return (size_t)(((u)*prime8bytes) >> (64 - h)); }
static size_t ZSTD_hash8Ptr(const void* p, U32 h) { return ZSTD_hash8(ZSTD_readLE64(p), h); } static size_t ZSTD_hash8Ptr(const void *p, U32 h) { return ZSTD_hash8(ZSTD_readLE64(p), h); }
static size_t ZSTD_hashPtr(const void* p, U32 hBits, U32 mls) static size_t ZSTD_hashPtr(const void *p, U32 hBits, U32 mls)
{ {
switch(mls) switch (mls) {
{ // case 3: return ZSTD_hash3Ptr(p, hBits);
//case 3: return ZSTD_hash3Ptr(p, hBits);
default: default:
case 4: return ZSTD_hash4Ptr(p, hBits); case 4: return ZSTD_hash4Ptr(p, hBits);
case 5: return ZSTD_hash5Ptr(p, hBits); case 5: return ZSTD_hash5Ptr(p, hBits);
@@ -898,78 +972,81 @@ static size_t ZSTD_hashPtr(const void* p, U32 hBits, U32 mls)
} }
} }
/*-************************************* /*-*************************************
* Fast Scan * Fast Scan
***************************************/ ***************************************/
static void ZSTD_fillHashTable (ZSTD_CCtx* zc, const void* end, const U32 mls) static void ZSTD_fillHashTable(ZSTD_CCtx *zc, const void *end, const U32 mls)
{ {
U32* const hashTable = zc->hashTable; U32 *const hashTable = zc->hashTable;
U32 const hBits = zc->params.cParams.hashLog; U32 const hBits = zc->params.cParams.hashLog;
const BYTE* const base = zc->base; const BYTE *const base = zc->base;
const BYTE* ip = base + zc->nextToUpdate; const BYTE *ip = base + zc->nextToUpdate;
const BYTE* const iend = ((const BYTE*)end) - HASH_READ_SIZE; const BYTE *const iend = ((const BYTE *)end) - HASH_READ_SIZE;
const size_t fastHashFillStep = 3; const size_t fastHashFillStep = 3;
while(ip <= iend) { while (ip <= iend) {
hashTable[ZSTD_hashPtr(ip, hBits, mls)] = (U32)(ip - base); hashTable[ZSTD_hashPtr(ip, hBits, mls)] = (U32)(ip - base);
ip += fastHashFillStep; ip += fastHashFillStep;
} }
} }
FORCE_INLINE FORCE_INLINE
void ZSTD_compressBlock_fast_generic(ZSTD_CCtx* cctx, void ZSTD_compressBlock_fast_generic(ZSTD_CCtx *cctx, const void *src, size_t srcSize, const U32 mls)
const void* src, size_t srcSize,
const U32 mls)
{ {
U32* const hashTable = cctx->hashTable; U32 *const hashTable = cctx->hashTable;
U32 const hBits = cctx->params.cParams.hashLog; U32 const hBits = cctx->params.cParams.hashLog;
seqStore_t* seqStorePtr = &(cctx->seqStore); seqStore_t *seqStorePtr = &(cctx->seqStore);
const BYTE* const base = cctx->base; const BYTE *const base = cctx->base;
const BYTE* const istart = (const BYTE*)src; const BYTE *const istart = (const BYTE *)src;
const BYTE* ip = istart; const BYTE *ip = istart;
const BYTE* anchor = istart; const BYTE *anchor = istart;
const U32 lowestIndex = cctx->dictLimit; const U32 lowestIndex = cctx->dictLimit;
const BYTE* const lowest = base + lowestIndex; const BYTE *const lowest = base + lowestIndex;
const BYTE* const iend = istart + srcSize; const BYTE *const iend = istart + srcSize;
const BYTE* const ilimit = iend - HASH_READ_SIZE; const BYTE *const ilimit = iend - HASH_READ_SIZE;
U32 offset_1=cctx->rep[0], offset_2=cctx->rep[1]; U32 offset_1 = cctx->rep[0], offset_2 = cctx->rep[1];
U32 offsetSaved = 0; U32 offsetSaved = 0;
/* init */ /* init */
ip += (ip==lowest); ip += (ip == lowest);
{ U32 const maxRep = (U32)(ip-lowest); {
if (offset_2 > maxRep) offsetSaved = offset_2, offset_2 = 0; U32 const maxRep = (U32)(ip - lowest);
if (offset_1 > maxRep) offsetSaved = offset_1, offset_1 = 0; if (offset_2 > maxRep)
offsetSaved = offset_2, offset_2 = 0;
if (offset_1 > maxRep)
offsetSaved = offset_1, offset_1 = 0;
} }
/* Main Search Loop */ /* Main Search Loop */
while (ip < ilimit) { /* < instead of <=, because repcode check at (ip+1) */ while (ip < ilimit) { /* < instead of <=, because repcode check at (ip+1) */
size_t mLength; size_t mLength;
size_t const h = ZSTD_hashPtr(ip, hBits, mls); size_t const h = ZSTD_hashPtr(ip, hBits, mls);
U32 const curr = (U32)(ip-base); U32 const curr = (U32)(ip - base);
U32 const matchIndex = hashTable[h]; U32 const matchIndex = hashTable[h];
const BYTE* match = base + matchIndex; const BYTE *match = base + matchIndex;
hashTable[h] = curr; /* update hash table */ hashTable[h] = curr; /* update hash table */
if ((offset_1 > 0) & (ZSTD_read32(ip+1-offset_1) == ZSTD_read32(ip+1))) { if ((offset_1 > 0) & (ZSTD_read32(ip + 1 - offset_1) == ZSTD_read32(ip + 1))) {
mLength = ZSTD_count(ip+1+4, ip+1+4-offset_1, iend) + 4; mLength = ZSTD_count(ip + 1 + 4, ip + 1 + 4 - offset_1, iend) + 4;
ip++; ip++;
ZSTD_storeSeq(seqStorePtr, ip-anchor, anchor, 0, mLength-MINMATCH); ZSTD_storeSeq(seqStorePtr, ip - anchor, anchor, 0, mLength - MINMATCH);
} else { } else {
U32 offset; U32 offset;
if ( (matchIndex <= lowestIndex) || (ZSTD_read32(match) != ZSTD_read32(ip)) ) { if ((matchIndex <= lowestIndex) || (ZSTD_read32(match) != ZSTD_read32(ip))) {
ip += ((ip-anchor) >> g_searchStrength) + 1; ip += ((ip - anchor) >> g_searchStrength) + 1;
continue; continue;
} }
mLength = ZSTD_count(ip+4, match+4, iend) + 4; mLength = ZSTD_count(ip + 4, match + 4, iend) + 4;
offset = (U32)(ip-match); offset = (U32)(ip - match);
while (((ip>anchor) & (match>lowest)) && (ip[-1] == match[-1])) { ip--; match--; mLength++; } /* catch up */ while (((ip > anchor) & (match > lowest)) && (ip[-1] == match[-1])) {
ip--;
match--;
mLength++;
} /* catch up */
offset_2 = offset_1; offset_2 = offset_1;
offset_1 = offset; offset_1 = offset;
ZSTD_storeSeq(seqStorePtr, ip-anchor, anchor, offset + ZSTD_REP_MOVE, mLength-MINMATCH); ZSTD_storeSeq(seqStorePtr, ip - anchor, anchor, offset + ZSTD_REP_MOVE, mLength - MINMATCH);
} }
/* match found */ /* match found */
@@ -978,109 +1055,109 @@ void ZSTD_compressBlock_fast_generic(ZSTD_CCtx* cctx,
if (ip <= ilimit) { if (ip <= ilimit) {
/* Fill Table */ /* Fill Table */
hashTable[ZSTD_hashPtr(base+curr+2, hBits, mls)] = curr+2; /* here because curr+2 could be > iend-8 */ hashTable[ZSTD_hashPtr(base + curr + 2, hBits, mls)] = curr + 2; /* here because curr+2 could be > iend-8 */
hashTable[ZSTD_hashPtr(ip-2, hBits, mls)] = (U32)(ip-2-base); hashTable[ZSTD_hashPtr(ip - 2, hBits, mls)] = (U32)(ip - 2 - base);
/* check immediate repcode */ /* check immediate repcode */
while ( (ip <= ilimit) while ((ip <= ilimit) && ((offset_2 > 0) & (ZSTD_read32(ip) == ZSTD_read32(ip - offset_2)))) {
&& ( (offset_2>0)
& (ZSTD_read32(ip) == ZSTD_read32(ip - offset_2)) )) {
/* store sequence */ /* store sequence */
size_t const rLength = ZSTD_count(ip+4, ip+4-offset_2, iend) + 4; size_t const rLength = ZSTD_count(ip + 4, ip + 4 - offset_2, iend) + 4;
{ U32 const tmpOff = offset_2; offset_2 = offset_1; offset_1 = tmpOff; } /* swap offset_2 <=> offset_1 */ {
hashTable[ZSTD_hashPtr(ip, hBits, mls)] = (U32)(ip-base); U32 const tmpOff = offset_2;
ZSTD_storeSeq(seqStorePtr, 0, anchor, 0, rLength-MINMATCH); offset_2 = offset_1;
offset_1 = tmpOff;
} /* swap offset_2 <=> offset_1 */
hashTable[ZSTD_hashPtr(ip, hBits, mls)] = (U32)(ip - base);
ZSTD_storeSeq(seqStorePtr, 0, anchor, 0, rLength - MINMATCH);
ip += rLength; ip += rLength;
anchor = ip; anchor = ip;
continue; /* faster when present ... (?) */ continue; /* faster when present ... (?) */
} } } }
}
}
/* save reps for next block */ /* save reps for next block */
cctx->repToConfirm[0] = offset_1 ? offset_1 : offsetSaved; cctx->repToConfirm[0] = offset_1 ? offset_1 : offsetSaved;
cctx->repToConfirm[1] = offset_2 ? offset_2 : offsetSaved; cctx->repToConfirm[1] = offset_2 ? offset_2 : offsetSaved;
/* Last Literals */ /* Last Literals */
{ size_t const lastLLSize = iend - anchor; {
size_t const lastLLSize = iend - anchor;
memcpy(seqStorePtr->lit, anchor, lastLLSize); memcpy(seqStorePtr->lit, anchor, lastLLSize);
seqStorePtr->lit += lastLLSize; seqStorePtr->lit += lastLLSize;
} }
} }
static void ZSTD_compressBlock_fast(ZSTD_CCtx *ctx, const void *src, size_t srcSize)
static void ZSTD_compressBlock_fast(ZSTD_CCtx* ctx,
const void* src, size_t srcSize)
{ {
const U32 mls = ctx->params.cParams.searchLength; const U32 mls = ctx->params.cParams.searchLength;
switch(mls) switch (mls) {
{
default: /* includes case 3 */ default: /* includes case 3 */
case 4 : case 4: ZSTD_compressBlock_fast_generic(ctx, src, srcSize, 4); return;
ZSTD_compressBlock_fast_generic(ctx, src, srcSize, 4); return; case 5: ZSTD_compressBlock_fast_generic(ctx, src, srcSize, 5); return;
case 5 : case 6: ZSTD_compressBlock_fast_generic(ctx, src, srcSize, 6); return;
ZSTD_compressBlock_fast_generic(ctx, src, srcSize, 5); return; case 7: ZSTD_compressBlock_fast_generic(ctx, src, srcSize, 7); return;
case 6 :
ZSTD_compressBlock_fast_generic(ctx, src, srcSize, 6); return;
case 7 :
ZSTD_compressBlock_fast_generic(ctx, src, srcSize, 7); return;
} }
} }
static void ZSTD_compressBlock_fast_extDict_generic(ZSTD_CCtx *ctx, const void *src, size_t srcSize, const U32 mls)
static void ZSTD_compressBlock_fast_extDict_generic(ZSTD_CCtx* ctx,
const void* src, size_t srcSize,
const U32 mls)
{ {
U32* hashTable = ctx->hashTable; U32 *hashTable = ctx->hashTable;
const U32 hBits = ctx->params.cParams.hashLog; const U32 hBits = ctx->params.cParams.hashLog;
seqStore_t* seqStorePtr = &(ctx->seqStore); seqStore_t *seqStorePtr = &(ctx->seqStore);
const BYTE* const base = ctx->base; const BYTE *const base = ctx->base;
const BYTE* const dictBase = ctx->dictBase; const BYTE *const dictBase = ctx->dictBase;
const BYTE* const istart = (const BYTE*)src; const BYTE *const istart = (const BYTE *)src;
const BYTE* ip = istart; const BYTE *ip = istart;
const BYTE* anchor = istart; const BYTE *anchor = istart;
const U32 lowestIndex = ctx->lowLimit; const U32 lowestIndex = ctx->lowLimit;
const BYTE* const dictStart = dictBase + lowestIndex; const BYTE *const dictStart = dictBase + lowestIndex;
const U32 dictLimit = ctx->dictLimit; const U32 dictLimit = ctx->dictLimit;
const BYTE* const lowPrefixPtr = base + dictLimit; const BYTE *const lowPrefixPtr = base + dictLimit;
const BYTE* const dictEnd = dictBase + dictLimit; const BYTE *const dictEnd = dictBase + dictLimit;
const BYTE* const iend = istart + srcSize; const BYTE *const iend = istart + srcSize;
const BYTE* const ilimit = iend - 8; const BYTE *const ilimit = iend - 8;
U32 offset_1=ctx->rep[0], offset_2=ctx->rep[1]; U32 offset_1 = ctx->rep[0], offset_2 = ctx->rep[1];
/* Search Loop */ /* Search Loop */
while (ip < ilimit) { /* < instead of <=, because (ip+1) */ while (ip < ilimit) { /* < instead of <=, because (ip+1) */
const size_t h = ZSTD_hashPtr(ip, hBits, mls); const size_t h = ZSTD_hashPtr(ip, hBits, mls);
const U32 matchIndex = hashTable[h]; const U32 matchIndex = hashTable[h];
const BYTE* matchBase = matchIndex < dictLimit ? dictBase : base; const BYTE *matchBase = matchIndex < dictLimit ? dictBase : base;
const BYTE* match = matchBase + matchIndex; const BYTE *match = matchBase + matchIndex;
const U32 curr = (U32)(ip-base); const U32 curr = (U32)(ip - base);
const U32 repIndex = curr + 1 - offset_1; /* offset_1 expected <= curr +1 */ const U32 repIndex = curr + 1 - offset_1; /* offset_1 expected <= curr +1 */
const BYTE* repBase = repIndex < dictLimit ? dictBase : base; const BYTE *repBase = repIndex < dictLimit ? dictBase : base;
const BYTE* repMatch = repBase + repIndex; const BYTE *repMatch = repBase + repIndex;
size_t mLength; size_t mLength;
hashTable[h] = curr; /* update hash table */ hashTable[h] = curr; /* update hash table */
if ( (((U32)((dictLimit-1) - repIndex) >= 3) /* intentional underflow */ & (repIndex > lowestIndex)) if ((((U32)((dictLimit - 1) - repIndex) >= 3) /* intentional underflow */ & (repIndex > lowestIndex)) &&
&& (ZSTD_read32(repMatch) == ZSTD_read32(ip+1)) ) { (ZSTD_read32(repMatch) == ZSTD_read32(ip + 1))) {
const BYTE* repMatchEnd = repIndex < dictLimit ? dictEnd : iend; const BYTE *repMatchEnd = repIndex < dictLimit ? dictEnd : iend;
mLength = ZSTD_count_2segments(ip+1+EQUAL_READ32, repMatch+EQUAL_READ32, iend, repMatchEnd, lowPrefixPtr) + EQUAL_READ32; mLength = ZSTD_count_2segments(ip + 1 + EQUAL_READ32, repMatch + EQUAL_READ32, iend, repMatchEnd, lowPrefixPtr) + EQUAL_READ32;
ip++; ip++;
ZSTD_storeSeq(seqStorePtr, ip-anchor, anchor, 0, mLength-MINMATCH); ZSTD_storeSeq(seqStorePtr, ip - anchor, anchor, 0, mLength - MINMATCH);
} else { } else {
if ( (matchIndex < lowestIndex) || if ((matchIndex < lowestIndex) || (ZSTD_read32(match) != ZSTD_read32(ip))) {
(ZSTD_read32(match) != ZSTD_read32(ip)) ) { ip += ((ip - anchor) >> g_searchStrength) + 1;
ip += ((ip-anchor) >> g_searchStrength) + 1;
continue; continue;
} }
{ const BYTE* matchEnd = matchIndex < dictLimit ? dictEnd : iend; {
const BYTE* lowMatchPtr = matchIndex < dictLimit ? dictStart : lowPrefixPtr; const BYTE *matchEnd = matchIndex < dictLimit ? dictEnd : iend;
const BYTE *lowMatchPtr = matchIndex < dictLimit ? dictStart : lowPrefixPtr;
U32 offset; U32 offset;
mLength = ZSTD_count_2segments(ip+EQUAL_READ32, match+EQUAL_READ32, iend, matchEnd, lowPrefixPtr) + EQUAL_READ32; mLength = ZSTD_count_2segments(ip + EQUAL_READ32, match + EQUAL_READ32, iend, matchEnd, lowPrefixPtr) + EQUAL_READ32;
while (((ip>anchor) & (match>lowMatchPtr)) && (ip[-1] == match[-1])) { ip--; match--; mLength++; } /* catch up */ while (((ip > anchor) & (match > lowMatchPtr)) && (ip[-1] == match[-1])) {
ip--;
match--;
mLength++;
} /* catch up */
offset = curr - matchIndex; offset = curr - matchIndex;
offset_2 = offset_1; offset_2 = offset_1;
offset_1 = offset; offset_1 = offset;
ZSTD_storeSeq(seqStorePtr, ip-anchor, anchor, offset + ZSTD_REP_MOVE, mLength-MINMATCH); ZSTD_storeSeq(seqStorePtr, ip - anchor, anchor, offset + ZSTD_REP_MOVE, mLength - MINMATCH);
} } }
}
/* found a match : store it */ /* found a match : store it */
ip += mLength; ip += mLength;
@@ -1088,105 +1165,104 @@ static void ZSTD_compressBlock_fast_extDict_generic(ZSTD_CCtx* ctx,
if (ip <= ilimit) { if (ip <= ilimit) {
/* Fill Table */ /* Fill Table */
hashTable[ZSTD_hashPtr(base+curr+2, hBits, mls)] = curr+2; hashTable[ZSTD_hashPtr(base + curr + 2, hBits, mls)] = curr + 2;
hashTable[ZSTD_hashPtr(ip-2, hBits, mls)] = (U32)(ip-2-base); hashTable[ZSTD_hashPtr(ip - 2, hBits, mls)] = (U32)(ip - 2 - base);
/* check immediate repcode */ /* check immediate repcode */
while (ip <= ilimit) { while (ip <= ilimit) {
U32 const curr2 = (U32)(ip-base); U32 const curr2 = (U32)(ip - base);
U32 const repIndex2 = curr2 - offset_2; U32 const repIndex2 = curr2 - offset_2;
const BYTE* repMatch2 = repIndex2 < dictLimit ? dictBase + repIndex2 : base + repIndex2; const BYTE *repMatch2 = repIndex2 < dictLimit ? dictBase + repIndex2 : base + repIndex2;
if ( (((U32)((dictLimit-1) - repIndex2) >= 3) & (repIndex2 > lowestIndex)) /* intentional overflow */ if ((((U32)((dictLimit - 1) - repIndex2) >= 3) & (repIndex2 > lowestIndex)) /* intentional overflow */
&& (ZSTD_read32(repMatch2) == ZSTD_read32(ip)) ) { && (ZSTD_read32(repMatch2) == ZSTD_read32(ip))) {
const BYTE* const repEnd2 = repIndex2 < dictLimit ? dictEnd : iend; const BYTE *const repEnd2 = repIndex2 < dictLimit ? dictEnd : iend;
size_t repLength2 = ZSTD_count_2segments(ip+EQUAL_READ32, repMatch2+EQUAL_READ32, iend, repEnd2, lowPrefixPtr) + EQUAL_READ32; size_t repLength2 =
U32 tmpOffset = offset_2; offset_2 = offset_1; offset_1 = tmpOffset; /* swap offset_2 <=> offset_1 */ ZSTD_count_2segments(ip + EQUAL_READ32, repMatch2 + EQUAL_READ32, iend, repEnd2, lowPrefixPtr) + EQUAL_READ32;
ZSTD_storeSeq(seqStorePtr, 0, anchor, 0, repLength2-MINMATCH); U32 tmpOffset = offset_2;
offset_2 = offset_1;
offset_1 = tmpOffset; /* swap offset_2 <=> offset_1 */
ZSTD_storeSeq(seqStorePtr, 0, anchor, 0, repLength2 - MINMATCH);
hashTable[ZSTD_hashPtr(ip, hBits, mls)] = curr2; hashTable[ZSTD_hashPtr(ip, hBits, mls)] = curr2;
ip += repLength2; ip += repLength2;
anchor = ip; anchor = ip;
continue; continue;
} }
break; break;
} } } }
}
}
/* save reps for next block */ /* save reps for next block */
ctx->repToConfirm[0] = offset_1; ctx->repToConfirm[1] = offset_2; ctx->repToConfirm[0] = offset_1;
ctx->repToConfirm[1] = offset_2;
/* Last Literals */ /* Last Literals */
{ size_t const lastLLSize = iend - anchor; {
size_t const lastLLSize = iend - anchor;
memcpy(seqStorePtr->lit, anchor, lastLLSize); memcpy(seqStorePtr->lit, anchor, lastLLSize);
seqStorePtr->lit += lastLLSize; seqStorePtr->lit += lastLLSize;
} }
} }
static void ZSTD_compressBlock_fast_extDict(ZSTD_CCtx *ctx, const void *src, size_t srcSize)
static void ZSTD_compressBlock_fast_extDict(ZSTD_CCtx* ctx,
const void* src, size_t srcSize)
{ {
U32 const mls = ctx->params.cParams.searchLength; U32 const mls = ctx->params.cParams.searchLength;
switch(mls) switch (mls) {
{
default: /* includes case 3 */ default: /* includes case 3 */
case 4 : case 4: ZSTD_compressBlock_fast_extDict_generic(ctx, src, srcSize, 4); return;
ZSTD_compressBlock_fast_extDict_generic(ctx, src, srcSize, 4); return; case 5: ZSTD_compressBlock_fast_extDict_generic(ctx, src, srcSize, 5); return;
case 5 : case 6: ZSTD_compressBlock_fast_extDict_generic(ctx, src, srcSize, 6); return;
ZSTD_compressBlock_fast_extDict_generic(ctx, src, srcSize, 5); return; case 7: ZSTD_compressBlock_fast_extDict_generic(ctx, src, srcSize, 7); return;
case 6 :
ZSTD_compressBlock_fast_extDict_generic(ctx, src, srcSize, 6); return;
case 7 :
ZSTD_compressBlock_fast_extDict_generic(ctx, src, srcSize, 7); return;
} }
} }
/*-************************************* /*-*************************************
* Double Fast * Double Fast
***************************************/ ***************************************/
static void ZSTD_fillDoubleHashTable (ZSTD_CCtx* cctx, const void* end, const U32 mls) static void ZSTD_fillDoubleHashTable(ZSTD_CCtx *cctx, const void *end, const U32 mls)
{ {
U32* const hashLarge = cctx->hashTable; U32 *const hashLarge = cctx->hashTable;
U32 const hBitsL = cctx->params.cParams.hashLog; U32 const hBitsL = cctx->params.cParams.hashLog;
U32* const hashSmall = cctx->chainTable; U32 *const hashSmall = cctx->chainTable;
U32 const hBitsS = cctx->params.cParams.chainLog; U32 const hBitsS = cctx->params.cParams.chainLog;
const BYTE* const base = cctx->base; const BYTE *const base = cctx->base;
const BYTE* ip = base + cctx->nextToUpdate; const BYTE *ip = base + cctx->nextToUpdate;
const BYTE* const iend = ((const BYTE*)end) - HASH_READ_SIZE; const BYTE *const iend = ((const BYTE *)end) - HASH_READ_SIZE;
const size_t fastHashFillStep = 3; const size_t fastHashFillStep = 3;
while(ip <= iend) { while (ip <= iend) {
hashSmall[ZSTD_hashPtr(ip, hBitsS, mls)] = (U32)(ip - base); hashSmall[ZSTD_hashPtr(ip, hBitsS, mls)] = (U32)(ip - base);
hashLarge[ZSTD_hashPtr(ip, hBitsL, 8)] = (U32)(ip - base); hashLarge[ZSTD_hashPtr(ip, hBitsL, 8)] = (U32)(ip - base);
ip += fastHashFillStep; ip += fastHashFillStep;
} }
} }
FORCE_INLINE FORCE_INLINE
void ZSTD_compressBlock_doubleFast_generic(ZSTD_CCtx* cctx, void ZSTD_compressBlock_doubleFast_generic(ZSTD_CCtx *cctx, const void *src, size_t srcSize, const U32 mls)
const void* src, size_t srcSize,
const U32 mls)
{ {
U32* const hashLong = cctx->hashTable; U32 *const hashLong = cctx->hashTable;
const U32 hBitsL = cctx->params.cParams.hashLog; const U32 hBitsL = cctx->params.cParams.hashLog;
U32* const hashSmall = cctx->chainTable; U32 *const hashSmall = cctx->chainTable;
const U32 hBitsS = cctx->params.cParams.chainLog; const U32 hBitsS = cctx->params.cParams.chainLog;
seqStore_t* seqStorePtr = &(cctx->seqStore); seqStore_t *seqStorePtr = &(cctx->seqStore);
const BYTE* const base = cctx->base; const BYTE *const base = cctx->base;
const BYTE* const istart = (const BYTE*)src; const BYTE *const istart = (const BYTE *)src;
const BYTE* ip = istart; const BYTE *ip = istart;
const BYTE* anchor = istart; const BYTE *anchor = istart;
const U32 lowestIndex = cctx->dictLimit; const U32 lowestIndex = cctx->dictLimit;
const BYTE* const lowest = base + lowestIndex; const BYTE *const lowest = base + lowestIndex;
const BYTE* const iend = istart + srcSize; const BYTE *const iend = istart + srcSize;
const BYTE* const ilimit = iend - HASH_READ_SIZE; const BYTE *const ilimit = iend - HASH_READ_SIZE;
U32 offset_1=cctx->rep[0], offset_2=cctx->rep[1]; U32 offset_1 = cctx->rep[0], offset_2 = cctx->rep[1];
U32 offsetSaved = 0; U32 offsetSaved = 0;
/* init */ /* init */
ip += (ip==lowest); ip += (ip == lowest);
{ U32 const maxRep = (U32)(ip-lowest); {
if (offset_2 > maxRep) offsetSaved = offset_2, offset_2 = 0; U32 const maxRep = (U32)(ip - lowest);
if (offset_1 > maxRep) offsetSaved = offset_1, offset_1 = 0; if (offset_2 > maxRep)
offsetSaved = offset_2, offset_2 = 0;
if (offset_1 > maxRep)
offsetSaved = offset_1, offset_1 = 0;
} }
/* Main Search Loop */ /* Main Search Loop */
@@ -1194,47 +1270,59 @@ void ZSTD_compressBlock_doubleFast_generic(ZSTD_CCtx* cctx,
size_t mLength; size_t mLength;
size_t const h2 = ZSTD_hashPtr(ip, hBitsL, 8); size_t const h2 = ZSTD_hashPtr(ip, hBitsL, 8);
size_t const h = ZSTD_hashPtr(ip, hBitsS, mls); size_t const h = ZSTD_hashPtr(ip, hBitsS, mls);
U32 const curr = (U32)(ip-base); U32 const curr = (U32)(ip - base);
U32 const matchIndexL = hashLong[h2]; U32 const matchIndexL = hashLong[h2];
U32 const matchIndexS = hashSmall[h]; U32 const matchIndexS = hashSmall[h];
const BYTE* matchLong = base + matchIndexL; const BYTE *matchLong = base + matchIndexL;
const BYTE* match = base + matchIndexS; const BYTE *match = base + matchIndexS;
hashLong[h2] = hashSmall[h] = curr; /* update hash tables */ hashLong[h2] = hashSmall[h] = curr; /* update hash tables */
if ((offset_1 > 0) & (ZSTD_read32(ip+1-offset_1) == ZSTD_read32(ip+1))) { /* note : by construction, offset_1 <= curr */ if ((offset_1 > 0) & (ZSTD_read32(ip + 1 - offset_1) == ZSTD_read32(ip + 1))) { /* note : by construction, offset_1 <= curr */
mLength = ZSTD_count(ip+1+4, ip+1+4-offset_1, iend) + 4; mLength = ZSTD_count(ip + 1 + 4, ip + 1 + 4 - offset_1, iend) + 4;
ip++; ip++;
ZSTD_storeSeq(seqStorePtr, ip-anchor, anchor, 0, mLength-MINMATCH); ZSTD_storeSeq(seqStorePtr, ip - anchor, anchor, 0, mLength - MINMATCH);
} else { } else {
U32 offset; U32 offset;
if ( (matchIndexL > lowestIndex) && (ZSTD_read64(matchLong) == ZSTD_read64(ip)) ) { if ((matchIndexL > lowestIndex) && (ZSTD_read64(matchLong) == ZSTD_read64(ip))) {
mLength = ZSTD_count(ip+8, matchLong+8, iend) + 8; mLength = ZSTD_count(ip + 8, matchLong + 8, iend) + 8;
offset = (U32)(ip-matchLong); offset = (U32)(ip - matchLong);
while (((ip>anchor) & (matchLong>lowest)) && (ip[-1] == matchLong[-1])) { ip--; matchLong--; mLength++; } /* catch up */ while (((ip > anchor) & (matchLong > lowest)) && (ip[-1] == matchLong[-1])) {
} else if ( (matchIndexS > lowestIndex) && (ZSTD_read32(match) == ZSTD_read32(ip)) ) { ip--;
size_t const h3 = ZSTD_hashPtr(ip+1, hBitsL, 8); matchLong--;
mLength++;
} /* catch up */
} else if ((matchIndexS > lowestIndex) && (ZSTD_read32(match) == ZSTD_read32(ip))) {
size_t const h3 = ZSTD_hashPtr(ip + 1, hBitsL, 8);
U32 const matchIndex3 = hashLong[h3]; U32 const matchIndex3 = hashLong[h3];
const BYTE* match3 = base + matchIndex3; const BYTE *match3 = base + matchIndex3;
hashLong[h3] = curr + 1; hashLong[h3] = curr + 1;
if ( (matchIndex3 > lowestIndex) && (ZSTD_read64(match3) == ZSTD_read64(ip+1)) ) { if ((matchIndex3 > lowestIndex) && (ZSTD_read64(match3) == ZSTD_read64(ip + 1))) {
mLength = ZSTD_count(ip+9, match3+8, iend) + 8; mLength = ZSTD_count(ip + 9, match3 + 8, iend) + 8;
ip++; ip++;
offset = (U32)(ip-match3); offset = (U32)(ip - match3);
while (((ip>anchor) & (match3>lowest)) && (ip[-1] == match3[-1])) { ip--; match3--; mLength++; } /* catch up */ while (((ip > anchor) & (match3 > lowest)) && (ip[-1] == match3[-1])) {
ip--;
match3--;
mLength++;
} /* catch up */
} else { } else {
mLength = ZSTD_count(ip+4, match+4, iend) + 4; mLength = ZSTD_count(ip + 4, match + 4, iend) + 4;
offset = (U32)(ip-match); offset = (U32)(ip - match);
while (((ip>anchor) & (match>lowest)) && (ip[-1] == match[-1])) { ip--; match--; mLength++; } /* catch up */ while (((ip > anchor) & (match > lowest)) && (ip[-1] == match[-1])) {
ip--;
match--;
mLength++;
} /* catch up */
} }
} else { } else {
ip += ((ip-anchor) >> g_searchStrength) + 1; ip += ((ip - anchor) >> g_searchStrength) + 1;
continue; continue;
} }
offset_2 = offset_1; offset_2 = offset_1;
offset_1 = offset; offset_1 = offset;
ZSTD_storeSeq(seqStorePtr, ip-anchor, anchor, offset + ZSTD_REP_MOVE, mLength-MINMATCH); ZSTD_storeSeq(seqStorePtr, ip - anchor, anchor, offset + ZSTD_REP_MOVE, mLength - MINMATCH);
} }
/* match found */ /* match found */
@@ -1243,145 +1331,153 @@ void ZSTD_compressBlock_doubleFast_generic(ZSTD_CCtx* cctx,
if (ip <= ilimit) { if (ip <= ilimit) {
/* Fill Table */ /* Fill Table */
hashLong[ZSTD_hashPtr(base+curr+2, hBitsL, 8)] = hashLong[ZSTD_hashPtr(base + curr + 2, hBitsL, 8)] = hashSmall[ZSTD_hashPtr(base + curr + 2, hBitsS, mls)] =
hashSmall[ZSTD_hashPtr(base+curr+2, hBitsS, mls)] = curr+2; /* here because curr+2 could be > iend-8 */ curr + 2; /* here because curr+2 could be > iend-8 */
hashLong[ZSTD_hashPtr(ip-2, hBitsL, 8)] = hashLong[ZSTD_hashPtr(ip - 2, hBitsL, 8)] = hashSmall[ZSTD_hashPtr(ip - 2, hBitsS, mls)] = (U32)(ip - 2 - base);
hashSmall[ZSTD_hashPtr(ip-2, hBitsS, mls)] = (U32)(ip-2-base);
/* check immediate repcode */ /* check immediate repcode */
while ( (ip <= ilimit) while ((ip <= ilimit) && ((offset_2 > 0) & (ZSTD_read32(ip) == ZSTD_read32(ip - offset_2)))) {
&& ( (offset_2>0)
& (ZSTD_read32(ip) == ZSTD_read32(ip - offset_2)) )) {
/* store sequence */ /* store sequence */
size_t const rLength = ZSTD_count(ip+4, ip+4-offset_2, iend) + 4; size_t const rLength = ZSTD_count(ip + 4, ip + 4 - offset_2, iend) + 4;
{ U32 const tmpOff = offset_2; offset_2 = offset_1; offset_1 = tmpOff; } /* swap offset_2 <=> offset_1 */ {
hashSmall[ZSTD_hashPtr(ip, hBitsS, mls)] = (U32)(ip-base); U32 const tmpOff = offset_2;
hashLong[ZSTD_hashPtr(ip, hBitsL, 8)] = (U32)(ip-base); offset_2 = offset_1;
ZSTD_storeSeq(seqStorePtr, 0, anchor, 0, rLength-MINMATCH); offset_1 = tmpOff;
} /* swap offset_2 <=> offset_1 */
hashSmall[ZSTD_hashPtr(ip, hBitsS, mls)] = (U32)(ip - base);
hashLong[ZSTD_hashPtr(ip, hBitsL, 8)] = (U32)(ip - base);
ZSTD_storeSeq(seqStorePtr, 0, anchor, 0, rLength - MINMATCH);
ip += rLength; ip += rLength;
anchor = ip; anchor = ip;
continue; /* faster when present ... (?) */ continue; /* faster when present ... (?) */
} } } }
}
}
/* save reps for next block */ /* save reps for next block */
cctx->repToConfirm[0] = offset_1 ? offset_1 : offsetSaved; cctx->repToConfirm[0] = offset_1 ? offset_1 : offsetSaved;
cctx->repToConfirm[1] = offset_2 ? offset_2 : offsetSaved; cctx->repToConfirm[1] = offset_2 ? offset_2 : offsetSaved;
/* Last Literals */ /* Last Literals */
{ size_t const lastLLSize = iend - anchor; {
size_t const lastLLSize = iend - anchor;
memcpy(seqStorePtr->lit, anchor, lastLLSize); memcpy(seqStorePtr->lit, anchor, lastLLSize);
seqStorePtr->lit += lastLLSize; seqStorePtr->lit += lastLLSize;
} }
} }
static void ZSTD_compressBlock_doubleFast(ZSTD_CCtx *ctx, const void *src, size_t srcSize)
static void ZSTD_compressBlock_doubleFast(ZSTD_CCtx* ctx, const void* src, size_t srcSize)
{ {
const U32 mls = ctx->params.cParams.searchLength; const U32 mls = ctx->params.cParams.searchLength;
switch(mls) switch (mls) {
{
default: /* includes case 3 */ default: /* includes case 3 */
case 4 : case 4: ZSTD_compressBlock_doubleFast_generic(ctx, src, srcSize, 4); return;
ZSTD_compressBlock_doubleFast_generic(ctx, src, srcSize, 4); return; case 5: ZSTD_compressBlock_doubleFast_generic(ctx, src, srcSize, 5); return;
case 5 : case 6: ZSTD_compressBlock_doubleFast_generic(ctx, src, srcSize, 6); return;
ZSTD_compressBlock_doubleFast_generic(ctx, src, srcSize, 5); return; case 7: ZSTD_compressBlock_doubleFast_generic(ctx, src, srcSize, 7); return;
case 6 :
ZSTD_compressBlock_doubleFast_generic(ctx, src, srcSize, 6); return;
case 7 :
ZSTD_compressBlock_doubleFast_generic(ctx, src, srcSize, 7); return;
} }
} }
static void ZSTD_compressBlock_doubleFast_extDict_generic(ZSTD_CCtx *ctx, const void *src, size_t srcSize, const U32 mls)
static void ZSTD_compressBlock_doubleFast_extDict_generic(ZSTD_CCtx* ctx,
const void* src, size_t srcSize,
const U32 mls)
{ {
U32* const hashLong = ctx->hashTable; U32 *const hashLong = ctx->hashTable;
U32 const hBitsL = ctx->params.cParams.hashLog; U32 const hBitsL = ctx->params.cParams.hashLog;
U32* const hashSmall = ctx->chainTable; U32 *const hashSmall = ctx->chainTable;
U32 const hBitsS = ctx->params.cParams.chainLog; U32 const hBitsS = ctx->params.cParams.chainLog;
seqStore_t* seqStorePtr = &(ctx->seqStore); seqStore_t *seqStorePtr = &(ctx->seqStore);
const BYTE* const base = ctx->base; const BYTE *const base = ctx->base;
const BYTE* const dictBase = ctx->dictBase; const BYTE *const dictBase = ctx->dictBase;
const BYTE* const istart = (const BYTE*)src; const BYTE *const istart = (const BYTE *)src;
const BYTE* ip = istart; const BYTE *ip = istart;
const BYTE* anchor = istart; const BYTE *anchor = istart;
const U32 lowestIndex = ctx->lowLimit; const U32 lowestIndex = ctx->lowLimit;
const BYTE* const dictStart = dictBase + lowestIndex; const BYTE *const dictStart = dictBase + lowestIndex;
const U32 dictLimit = ctx->dictLimit; const U32 dictLimit = ctx->dictLimit;
const BYTE* const lowPrefixPtr = base + dictLimit; const BYTE *const lowPrefixPtr = base + dictLimit;
const BYTE* const dictEnd = dictBase + dictLimit; const BYTE *const dictEnd = dictBase + dictLimit;
const BYTE* const iend = istart + srcSize; const BYTE *const iend = istart + srcSize;
const BYTE* const ilimit = iend - 8; const BYTE *const ilimit = iend - 8;
U32 offset_1=ctx->rep[0], offset_2=ctx->rep[1]; U32 offset_1 = ctx->rep[0], offset_2 = ctx->rep[1];
/* Search Loop */ /* Search Loop */
while (ip < ilimit) { /* < instead of <=, because (ip+1) */ while (ip < ilimit) { /* < instead of <=, because (ip+1) */
const size_t hSmall = ZSTD_hashPtr(ip, hBitsS, mls); const size_t hSmall = ZSTD_hashPtr(ip, hBitsS, mls);
const U32 matchIndex = hashSmall[hSmall]; const U32 matchIndex = hashSmall[hSmall];
const BYTE* matchBase = matchIndex < dictLimit ? dictBase : base; const BYTE *matchBase = matchIndex < dictLimit ? dictBase : base;
const BYTE* match = matchBase + matchIndex; const BYTE *match = matchBase + matchIndex;
const size_t hLong = ZSTD_hashPtr(ip, hBitsL, 8); const size_t hLong = ZSTD_hashPtr(ip, hBitsL, 8);
const U32 matchLongIndex = hashLong[hLong]; const U32 matchLongIndex = hashLong[hLong];
const BYTE* matchLongBase = matchLongIndex < dictLimit ? dictBase : base; const BYTE *matchLongBase = matchLongIndex < dictLimit ? dictBase : base;
const BYTE* matchLong = matchLongBase + matchLongIndex; const BYTE *matchLong = matchLongBase + matchLongIndex;
const U32 curr = (U32)(ip-base); const U32 curr = (U32)(ip - base);
const U32 repIndex = curr + 1 - offset_1; /* offset_1 expected <= curr +1 */ const U32 repIndex = curr + 1 - offset_1; /* offset_1 expected <= curr +1 */
const BYTE* repBase = repIndex < dictLimit ? dictBase : base; const BYTE *repBase = repIndex < dictLimit ? dictBase : base;
const BYTE* repMatch = repBase + repIndex; const BYTE *repMatch = repBase + repIndex;
size_t mLength; size_t mLength;
hashSmall[hSmall] = hashLong[hLong] = curr; /* update hash table */ hashSmall[hSmall] = hashLong[hLong] = curr; /* update hash table */
if ( (((U32)((dictLimit-1) - repIndex) >= 3) /* intentional underflow */ & (repIndex > lowestIndex)) if ((((U32)((dictLimit - 1) - repIndex) >= 3) /* intentional underflow */ & (repIndex > lowestIndex)) &&
&& (ZSTD_read32(repMatch) == ZSTD_read32(ip+1)) ) { (ZSTD_read32(repMatch) == ZSTD_read32(ip + 1))) {
const BYTE* repMatchEnd = repIndex < dictLimit ? dictEnd : iend; const BYTE *repMatchEnd = repIndex < dictLimit ? dictEnd : iend;
mLength = ZSTD_count_2segments(ip+1+4, repMatch+4, iend, repMatchEnd, lowPrefixPtr) + 4; mLength = ZSTD_count_2segments(ip + 1 + 4, repMatch + 4, iend, repMatchEnd, lowPrefixPtr) + 4;
ip++; ip++;
ZSTD_storeSeq(seqStorePtr, ip-anchor, anchor, 0, mLength-MINMATCH); ZSTD_storeSeq(seqStorePtr, ip - anchor, anchor, 0, mLength - MINMATCH);
} else { } else {
if ((matchLongIndex > lowestIndex) && (ZSTD_read64(matchLong) == ZSTD_read64(ip))) { if ((matchLongIndex > lowestIndex) && (ZSTD_read64(matchLong) == ZSTD_read64(ip))) {
const BYTE* matchEnd = matchLongIndex < dictLimit ? dictEnd : iend; const BYTE *matchEnd = matchLongIndex < dictLimit ? dictEnd : iend;
const BYTE* lowMatchPtr = matchLongIndex < dictLimit ? dictStart : lowPrefixPtr; const BYTE *lowMatchPtr = matchLongIndex < dictLimit ? dictStart : lowPrefixPtr;
U32 offset; U32 offset;
mLength = ZSTD_count_2segments(ip+8, matchLong+8, iend, matchEnd, lowPrefixPtr) + 8; mLength = ZSTD_count_2segments(ip + 8, matchLong + 8, iend, matchEnd, lowPrefixPtr) + 8;
offset = curr - matchLongIndex; offset = curr - matchLongIndex;
while (((ip>anchor) & (matchLong>lowMatchPtr)) && (ip[-1] == matchLong[-1])) { ip--; matchLong--; mLength++; } /* catch up */ while (((ip > anchor) & (matchLong > lowMatchPtr)) && (ip[-1] == matchLong[-1])) {
ip--;
matchLong--;
mLength++;
} /* catch up */
offset_2 = offset_1; offset_2 = offset_1;
offset_1 = offset; offset_1 = offset;
ZSTD_storeSeq(seqStorePtr, ip-anchor, anchor, offset + ZSTD_REP_MOVE, mLength-MINMATCH); ZSTD_storeSeq(seqStorePtr, ip - anchor, anchor, offset + ZSTD_REP_MOVE, mLength - MINMATCH);
} else if ((matchIndex > lowestIndex) && (ZSTD_read32(match) == ZSTD_read32(ip))) { } else if ((matchIndex > lowestIndex) && (ZSTD_read32(match) == ZSTD_read32(ip))) {
size_t const h3 = ZSTD_hashPtr(ip+1, hBitsL, 8); size_t const h3 = ZSTD_hashPtr(ip + 1, hBitsL, 8);
U32 const matchIndex3 = hashLong[h3]; U32 const matchIndex3 = hashLong[h3];
const BYTE* const match3Base = matchIndex3 < dictLimit ? dictBase : base; const BYTE *const match3Base = matchIndex3 < dictLimit ? dictBase : base;
const BYTE* match3 = match3Base + matchIndex3; const BYTE *match3 = match3Base + matchIndex3;
U32 offset; U32 offset;
hashLong[h3] = curr + 1; hashLong[h3] = curr + 1;
if ( (matchIndex3 > lowestIndex) && (ZSTD_read64(match3) == ZSTD_read64(ip+1)) ) { if ((matchIndex3 > lowestIndex) && (ZSTD_read64(match3) == ZSTD_read64(ip + 1))) {
const BYTE* matchEnd = matchIndex3 < dictLimit ? dictEnd : iend; const BYTE *matchEnd = matchIndex3 < dictLimit ? dictEnd : iend;
const BYTE* lowMatchPtr = matchIndex3 < dictLimit ? dictStart : lowPrefixPtr; const BYTE *lowMatchPtr = matchIndex3 < dictLimit ? dictStart : lowPrefixPtr;
mLength = ZSTD_count_2segments(ip+9, match3+8, iend, matchEnd, lowPrefixPtr) + 8; mLength = ZSTD_count_2segments(ip + 9, match3 + 8, iend, matchEnd, lowPrefixPtr) + 8;
ip++; ip++;
offset = curr+1 - matchIndex3; offset = curr + 1 - matchIndex3;
while (((ip>anchor) & (match3>lowMatchPtr)) && (ip[-1] == match3[-1])) { ip--; match3--; mLength++; } /* catch up */ while (((ip > anchor) & (match3 > lowMatchPtr)) && (ip[-1] == match3[-1])) {
ip--;
match3--;
mLength++;
} /* catch up */
} else { } else {
const BYTE* matchEnd = matchIndex < dictLimit ? dictEnd : iend; const BYTE *matchEnd = matchIndex < dictLimit ? dictEnd : iend;
const BYTE* lowMatchPtr = matchIndex < dictLimit ? dictStart : lowPrefixPtr; const BYTE *lowMatchPtr = matchIndex < dictLimit ? dictStart : lowPrefixPtr;
mLength = ZSTD_count_2segments(ip+4, match+4, iend, matchEnd, lowPrefixPtr) + 4; mLength = ZSTD_count_2segments(ip + 4, match + 4, iend, matchEnd, lowPrefixPtr) + 4;
offset = curr - matchIndex; offset = curr - matchIndex;
while (((ip>anchor) & (match>lowMatchPtr)) && (ip[-1] == match[-1])) { ip--; match--; mLength++; } /* catch up */ while (((ip > anchor) & (match > lowMatchPtr)) && (ip[-1] == match[-1])) {
ip--;
match--;
mLength++;
} /* catch up */
} }
offset_2 = offset_1; offset_2 = offset_1;
offset_1 = offset; offset_1 = offset;
ZSTD_storeSeq(seqStorePtr, ip-anchor, anchor, offset + ZSTD_REP_MOVE, mLength-MINMATCH); ZSTD_storeSeq(seqStorePtr, ip - anchor, anchor, offset + ZSTD_REP_MOVE, mLength - MINMATCH);
} else { } else {
ip += ((ip-anchor) >> g_searchStrength) + 1; ip += ((ip - anchor) >> g_searchStrength) + 1;
continue; continue;
} } }
}
/* found a match : store it */ /* found a match : store it */
ip += mLength; ip += mLength;
@@ -1389,21 +1485,24 @@ static void ZSTD_compressBlock_doubleFast_extDict_generic(ZSTD_CCtx* ctx,
if (ip <= ilimit) { if (ip <= ilimit) {
/* Fill Table */ /* Fill Table */
hashSmall[ZSTD_hashPtr(base+curr+2, hBitsS, mls)] = curr+2; hashSmall[ZSTD_hashPtr(base + curr + 2, hBitsS, mls)] = curr + 2;
hashLong[ZSTD_hashPtr(base+curr+2, hBitsL, 8)] = curr+2; hashLong[ZSTD_hashPtr(base + curr + 2, hBitsL, 8)] = curr + 2;
hashSmall[ZSTD_hashPtr(ip-2, hBitsS, mls)] = (U32)(ip-2-base); hashSmall[ZSTD_hashPtr(ip - 2, hBitsS, mls)] = (U32)(ip - 2 - base);
hashLong[ZSTD_hashPtr(ip-2, hBitsL, 8)] = (U32)(ip-2-base); hashLong[ZSTD_hashPtr(ip - 2, hBitsL, 8)] = (U32)(ip - 2 - base);
/* check immediate repcode */ /* check immediate repcode */
while (ip <= ilimit) { while (ip <= ilimit) {
U32 const curr2 = (U32)(ip-base); U32 const curr2 = (U32)(ip - base);
U32 const repIndex2 = curr2 - offset_2; U32 const repIndex2 = curr2 - offset_2;
const BYTE* repMatch2 = repIndex2 < dictLimit ? dictBase + repIndex2 : base + repIndex2; const BYTE *repMatch2 = repIndex2 < dictLimit ? dictBase + repIndex2 : base + repIndex2;
if ( (((U32)((dictLimit-1) - repIndex2) >= 3) & (repIndex2 > lowestIndex)) /* intentional overflow */ if ((((U32)((dictLimit - 1) - repIndex2) >= 3) & (repIndex2 > lowestIndex)) /* intentional overflow */
&& (ZSTD_read32(repMatch2) == ZSTD_read32(ip)) ) { && (ZSTD_read32(repMatch2) == ZSTD_read32(ip))) {
const BYTE* const repEnd2 = repIndex2 < dictLimit ? dictEnd : iend; const BYTE *const repEnd2 = repIndex2 < dictLimit ? dictEnd : iend;
size_t const repLength2 = ZSTD_count_2segments(ip+EQUAL_READ32, repMatch2+EQUAL_READ32, iend, repEnd2, lowPrefixPtr) + EQUAL_READ32; size_t const repLength2 =
U32 tmpOffset = offset_2; offset_2 = offset_1; offset_1 = tmpOffset; /* swap offset_2 <=> offset_1 */ ZSTD_count_2segments(ip + EQUAL_READ32, repMatch2 + EQUAL_READ32, iend, repEnd2, lowPrefixPtr) + EQUAL_READ32;
ZSTD_storeSeq(seqStorePtr, 0, anchor, 0, repLength2-MINMATCH); U32 tmpOffset = offset_2;
offset_2 = offset_1;
offset_1 = tmpOffset; /* swap offset_2 <=> offset_1 */
ZSTD_storeSeq(seqStorePtr, 0, anchor, 0, repLength2 - MINMATCH);
hashSmall[ZSTD_hashPtr(ip, hBitsS, mls)] = curr2; hashSmall[ZSTD_hashPtr(ip, hBitsS, mls)] = curr2;
hashLong[ZSTD_hashPtr(ip, hBitsL, 8)] = curr2; hashLong[ZSTD_hashPtr(ip, hBitsL, 8)] = curr2;
ip += repLength2; ip += repLength2;
@@ -1411,84 +1510,79 @@ static void ZSTD_compressBlock_doubleFast_extDict_generic(ZSTD_CCtx* ctx,
continue; continue;
} }
break; break;
} } } }
}
}
/* save reps for next block */ /* save reps for next block */
ctx->repToConfirm[0] = offset_1; ctx->repToConfirm[1] = offset_2; ctx->repToConfirm[0] = offset_1;
ctx->repToConfirm[1] = offset_2;
/* Last Literals */ /* Last Literals */
{ size_t const lastLLSize = iend - anchor; {
size_t const lastLLSize = iend - anchor;
memcpy(seqStorePtr->lit, anchor, lastLLSize); memcpy(seqStorePtr->lit, anchor, lastLLSize);
seqStorePtr->lit += lastLLSize; seqStorePtr->lit += lastLLSize;
} }
} }
static void ZSTD_compressBlock_doubleFast_extDict(ZSTD_CCtx *ctx, const void *src, size_t srcSize)
static void ZSTD_compressBlock_doubleFast_extDict(ZSTD_CCtx* ctx,
const void* src, size_t srcSize)
{ {
U32 const mls = ctx->params.cParams.searchLength; U32 const mls = ctx->params.cParams.searchLength;
switch(mls) switch (mls) {
{
default: /* includes case 3 */ default: /* includes case 3 */
case 4 : case 4: ZSTD_compressBlock_doubleFast_extDict_generic(ctx, src, srcSize, 4); return;
ZSTD_compressBlock_doubleFast_extDict_generic(ctx, src, srcSize, 4); return; case 5: ZSTD_compressBlock_doubleFast_extDict_generic(ctx, src, srcSize, 5); return;
case 5 : case 6: ZSTD_compressBlock_doubleFast_extDict_generic(ctx, src, srcSize, 6); return;
ZSTD_compressBlock_doubleFast_extDict_generic(ctx, src, srcSize, 5); return; case 7: ZSTD_compressBlock_doubleFast_extDict_generic(ctx, src, srcSize, 7); return;
case 6 :
ZSTD_compressBlock_doubleFast_extDict_generic(ctx, src, srcSize, 6); return;
case 7 :
ZSTD_compressBlock_doubleFast_extDict_generic(ctx, src, srcSize, 7); return;
} }
} }
/*-************************************* /*-*************************************
* Binary Tree search * Binary Tree search
***************************************/ ***************************************/
/** ZSTD_insertBt1() : add one or multiple positions to tree. /** ZSTD_insertBt1() : add one or multiple positions to tree.
* ip : assumed <= iend-8 . * ip : assumed <= iend-8 .
* @return : nb of positions added */ * @return : nb of positions added */
static U32 ZSTD_insertBt1(ZSTD_CCtx* zc, const BYTE* const ip, const U32 mls, const BYTE* const iend, U32 nbCompares, static U32 ZSTD_insertBt1(ZSTD_CCtx *zc, const BYTE *const ip, const U32 mls, const BYTE *const iend, U32 nbCompares, U32 extDict)
U32 extDict)
{ {
U32* const hashTable = zc->hashTable; U32 *const hashTable = zc->hashTable;
U32 const hashLog = zc->params.cParams.hashLog; U32 const hashLog = zc->params.cParams.hashLog;
size_t const h = ZSTD_hashPtr(ip, hashLog, mls); size_t const h = ZSTD_hashPtr(ip, hashLog, mls);
U32* const bt = zc->chainTable; U32 *const bt = zc->chainTable;
U32 const btLog = zc->params.cParams.chainLog - 1; U32 const btLog = zc->params.cParams.chainLog - 1;
U32 const btMask = (1 << btLog) - 1; U32 const btMask = (1 << btLog) - 1;
U32 matchIndex = hashTable[h]; U32 matchIndex = hashTable[h];
size_t commonLengthSmaller=0, commonLengthLarger=0; size_t commonLengthSmaller = 0, commonLengthLarger = 0;
const BYTE* const base = zc->base; const BYTE *const base = zc->base;
const BYTE* const dictBase = zc->dictBase; const BYTE *const dictBase = zc->dictBase;
const U32 dictLimit = zc->dictLimit; const U32 dictLimit = zc->dictLimit;
const BYTE* const dictEnd = dictBase + dictLimit; const BYTE *const dictEnd = dictBase + dictLimit;
const BYTE* const prefixStart = base + dictLimit; const BYTE *const prefixStart = base + dictLimit;
const BYTE* match; const BYTE *match;
const U32 curr = (U32)(ip-base); const U32 curr = (U32)(ip - base);
const U32 btLow = btMask >= curr ? 0 : curr - btMask; const U32 btLow = btMask >= curr ? 0 : curr - btMask;
U32* smallerPtr = bt + 2*(curr&btMask); U32 *smallerPtr = bt + 2 * (curr & btMask);
U32* largerPtr = smallerPtr + 1; U32 *largerPtr = smallerPtr + 1;
U32 dummy32; /* to be nullified at the end */ U32 dummy32; /* to be nullified at the end */
U32 const windowLow = zc->lowLimit; U32 const windowLow = zc->lowLimit;
U32 matchEndIdx = curr+8; U32 matchEndIdx = curr + 8;
size_t bestLength = 8; size_t bestLength = 8;
hashTable[h] = curr; /* Update Hash Table */ hashTable[h] = curr; /* Update Hash Table */
while (nbCompares-- && (matchIndex > windowLow)) { while (nbCompares-- && (matchIndex > windowLow)) {
U32* const nextPtr = bt + 2*(matchIndex & btMask); U32 *const nextPtr = bt + 2 * (matchIndex & btMask);
size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */ size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */
if ((!extDict) || (matchIndex+matchLength >= dictLimit)) { if ((!extDict) || (matchIndex + matchLength >= dictLimit)) {
match = base + matchIndex; match = base + matchIndex;
if (match[matchLength] == ip[matchLength]) if (match[matchLength] == ip[matchLength])
matchLength += ZSTD_count(ip+matchLength+1, match+matchLength+1, iend) +1; matchLength += ZSTD_count(ip + matchLength + 1, match + matchLength + 1, iend) + 1;
} else { } else {
match = dictBase + matchIndex; match = dictBase + matchIndex;
matchLength += ZSTD_count_2segments(ip+matchLength, match+matchLength, iend, dictEnd, prefixStart); matchLength += ZSTD_count_2segments(ip + matchLength, match + matchLength, iend, dictEnd, prefixStart);
if (matchIndex+matchLength >= dictLimit) if (matchIndex + matchLength >= dictLimit)
match = base + matchIndex; /* to prepare for next usage of match[matchLength] */ match = base + matchIndex; /* to prepare for next usage of match[matchLength] */
} }
@@ -1498,85 +1592,89 @@ static U32 ZSTD_insertBt1(ZSTD_CCtx* zc, const BYTE* const ip, const U32 mls, co
matchEndIdx = matchIndex + (U32)matchLength; matchEndIdx = matchIndex + (U32)matchLength;
} }
if (ip+matchLength == iend) /* equal : no way to know if inf or sup */ if (ip + matchLength == iend) /* equal : no way to know if inf or sup */
break; /* drop , to guarantee consistency ; miss a bit of compression, but other solutions can corrupt the tree */ break; /* drop , to guarantee consistency ; miss a bit of compression, but other solutions can corrupt the tree */
if (match[matchLength] < ip[matchLength]) { /* necessarily within correct buffer */ if (match[matchLength] < ip[matchLength]) { /* necessarily within correct buffer */
/* match is smaller than curr */ /* match is smaller than curr */
*smallerPtr = matchIndex; /* update smaller idx */ *smallerPtr = matchIndex; /* update smaller idx */
commonLengthSmaller = matchLength; /* all smaller will now have at least this guaranteed common length */ commonLengthSmaller = matchLength; /* all smaller will now have at least this guaranteed common length */
if (matchIndex <= btLow) { smallerPtr=&dummy32; break; } /* beyond tree size, stop the search */ if (matchIndex <= btLow) {
smallerPtr = nextPtr+1; /* new "smaller" => larger of match */ smallerPtr = &dummy32;
break;
} /* beyond tree size, stop the search */
smallerPtr = nextPtr + 1; /* new "smaller" => larger of match */
matchIndex = nextPtr[1]; /* new matchIndex larger than previous (closer to curr) */ matchIndex = nextPtr[1]; /* new matchIndex larger than previous (closer to curr) */
} else { } else {
/* match is larger than curr */ /* match is larger than curr */
*largerPtr = matchIndex; *largerPtr = matchIndex;
commonLengthLarger = matchLength; commonLengthLarger = matchLength;
if (matchIndex <= btLow) { largerPtr=&dummy32; break; } /* beyond tree size, stop the search */ if (matchIndex <= btLow) {
largerPtr = &dummy32;
break;
} /* beyond tree size, stop the search */
largerPtr = nextPtr; largerPtr = nextPtr;
matchIndex = nextPtr[0]; matchIndex = nextPtr[0];
} } }
}
*smallerPtr = *largerPtr = 0; *smallerPtr = *largerPtr = 0;
if (bestLength > 384) return MIN(192, (U32)(bestLength - 384)); /* speed optimization */ if (bestLength > 384)
if (matchEndIdx > curr + 8) return matchEndIdx - curr - 8; return MIN(192, (U32)(bestLength - 384)); /* speed optimization */
if (matchEndIdx > curr + 8)
return matchEndIdx - curr - 8;
return 1; return 1;
} }
static size_t ZSTD_insertBtAndFindBestMatch(ZSTD_CCtx *zc, const BYTE *const ip, const BYTE *const iend, size_t *offsetPtr, U32 nbCompares, const U32 mls,
static size_t ZSTD_insertBtAndFindBestMatch (
ZSTD_CCtx* zc,
const BYTE* const ip, const BYTE* const iend,
size_t* offsetPtr,
U32 nbCompares, const U32 mls,
U32 extDict) U32 extDict)
{ {
U32* const hashTable = zc->hashTable; U32 *const hashTable = zc->hashTable;
U32 const hashLog = zc->params.cParams.hashLog; U32 const hashLog = zc->params.cParams.hashLog;
size_t const h = ZSTD_hashPtr(ip, hashLog, mls); size_t const h = ZSTD_hashPtr(ip, hashLog, mls);
U32* const bt = zc->chainTable; U32 *const bt = zc->chainTable;
U32 const btLog = zc->params.cParams.chainLog - 1; U32 const btLog = zc->params.cParams.chainLog - 1;
U32 const btMask = (1 << btLog) - 1; U32 const btMask = (1 << btLog) - 1;
U32 matchIndex = hashTable[h]; U32 matchIndex = hashTable[h];
size_t commonLengthSmaller=0, commonLengthLarger=0; size_t commonLengthSmaller = 0, commonLengthLarger = 0;
const BYTE* const base = zc->base; const BYTE *const base = zc->base;
const BYTE* const dictBase = zc->dictBase; const BYTE *const dictBase = zc->dictBase;
const U32 dictLimit = zc->dictLimit; const U32 dictLimit = zc->dictLimit;
const BYTE* const dictEnd = dictBase + dictLimit; const BYTE *const dictEnd = dictBase + dictLimit;
const BYTE* const prefixStart = base + dictLimit; const BYTE *const prefixStart = base + dictLimit;
const U32 curr = (U32)(ip-base); const U32 curr = (U32)(ip - base);
const U32 btLow = btMask >= curr ? 0 : curr - btMask; const U32 btLow = btMask >= curr ? 0 : curr - btMask;
const U32 windowLow = zc->lowLimit; const U32 windowLow = zc->lowLimit;
U32* smallerPtr = bt + 2*(curr&btMask); U32 *smallerPtr = bt + 2 * (curr & btMask);
U32* largerPtr = bt + 2*(curr&btMask) + 1; U32 *largerPtr = bt + 2 * (curr & btMask) + 1;
U32 matchEndIdx = curr+8; U32 matchEndIdx = curr + 8;
U32 dummy32; /* to be nullified at the end */ U32 dummy32; /* to be nullified at the end */
size_t bestLength = 0; size_t bestLength = 0;
hashTable[h] = curr; /* Update Hash Table */ hashTable[h] = curr; /* Update Hash Table */
while (nbCompares-- && (matchIndex > windowLow)) { while (nbCompares-- && (matchIndex > windowLow)) {
U32* const nextPtr = bt + 2*(matchIndex & btMask); U32 *const nextPtr = bt + 2 * (matchIndex & btMask);
size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */ size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */
const BYTE* match; const BYTE *match;
if ((!extDict) || (matchIndex+matchLength >= dictLimit)) { if ((!extDict) || (matchIndex + matchLength >= dictLimit)) {
match = base + matchIndex; match = base + matchIndex;
if (match[matchLength] == ip[matchLength]) if (match[matchLength] == ip[matchLength])
matchLength += ZSTD_count(ip+matchLength+1, match+matchLength+1, iend) +1; matchLength += ZSTD_count(ip + matchLength + 1, match + matchLength + 1, iend) + 1;
} else { } else {
match = dictBase + matchIndex; match = dictBase + matchIndex;
matchLength += ZSTD_count_2segments(ip+matchLength, match+matchLength, iend, dictEnd, prefixStart); matchLength += ZSTD_count_2segments(ip + matchLength, match + matchLength, iend, dictEnd, prefixStart);
if (matchIndex+matchLength >= dictLimit) if (matchIndex + matchLength >= dictLimit)
match = base + matchIndex; /* to prepare for next usage of match[matchLength] */ match = base + matchIndex; /* to prepare for next usage of match[matchLength] */
} }
if (matchLength > bestLength) { if (matchLength > bestLength) {
if (matchLength > matchEndIdx - matchIndex) if (matchLength > matchEndIdx - matchIndex)
matchEndIdx = matchIndex + (U32)matchLength; matchEndIdx = matchIndex + (U32)matchLength;
if ( (4*(int)(matchLength-bestLength)) > (int)(ZSTD_highbit32(curr-matchIndex+1) - ZSTD_highbit32((U32)offsetPtr[0]+1)) ) if ((4 * (int)(matchLength - bestLength)) > (int)(ZSTD_highbit32(curr - matchIndex + 1) - ZSTD_highbit32((U32)offsetPtr[0] + 1)))
bestLength = matchLength, *offsetPtr = ZSTD_REP_MOVE + curr - matchIndex; bestLength = matchLength, *offsetPtr = ZSTD_REP_MOVE + curr - matchIndex;
if (ip+matchLength == iend) /* equal : no way to know if inf or sup */ if (ip + matchLength == iend) /* equal : no way to know if inf or sup */
break; /* drop, to guarantee consistency (miss a little bit of compression) */ break; /* drop, to guarantee consistency (miss a little bit of compression) */
} }
@@ -1584,126 +1682,115 @@ static size_t ZSTD_insertBtAndFindBestMatch (
/* match is smaller than curr */ /* match is smaller than curr */
*smallerPtr = matchIndex; /* update smaller idx */ *smallerPtr = matchIndex; /* update smaller idx */
commonLengthSmaller = matchLength; /* all smaller will now have at least this guaranteed common length */ commonLengthSmaller = matchLength; /* all smaller will now have at least this guaranteed common length */
if (matchIndex <= btLow) { smallerPtr=&dummy32; break; } /* beyond tree size, stop the search */ if (matchIndex <= btLow) {
smallerPtr = nextPtr+1; /* new "smaller" => larger of match */ smallerPtr = &dummy32;
break;
} /* beyond tree size, stop the search */
smallerPtr = nextPtr + 1; /* new "smaller" => larger of match */
matchIndex = nextPtr[1]; /* new matchIndex larger than previous (closer to curr) */ matchIndex = nextPtr[1]; /* new matchIndex larger than previous (closer to curr) */
} else { } else {
/* match is larger than curr */ /* match is larger than curr */
*largerPtr = matchIndex; *largerPtr = matchIndex;
commonLengthLarger = matchLength; commonLengthLarger = matchLength;
if (matchIndex <= btLow) { largerPtr=&dummy32; break; } /* beyond tree size, stop the search */ if (matchIndex <= btLow) {
largerPtr = &dummy32;
break;
} /* beyond tree size, stop the search */
largerPtr = nextPtr; largerPtr = nextPtr;
matchIndex = nextPtr[0]; matchIndex = nextPtr[0];
} } }
}
*smallerPtr = *largerPtr = 0; *smallerPtr = *largerPtr = 0;
zc->nextToUpdate = (matchEndIdx > curr + 8) ? matchEndIdx - 8 : curr+1; zc->nextToUpdate = (matchEndIdx > curr + 8) ? matchEndIdx - 8 : curr + 1;
return bestLength; return bestLength;
} }
static void ZSTD_updateTree(ZSTD_CCtx *zc, const BYTE *const ip, const BYTE *const iend, const U32 nbCompares, const U32 mls)
static void ZSTD_updateTree(ZSTD_CCtx* zc, const BYTE* const ip, const BYTE* const iend, const U32 nbCompares, const U32 mls)
{ {
const BYTE* const base = zc->base; const BYTE *const base = zc->base;
const U32 target = (U32)(ip - base); const U32 target = (U32)(ip - base);
U32 idx = zc->nextToUpdate; U32 idx = zc->nextToUpdate;
while(idx < target) while (idx < target)
idx += ZSTD_insertBt1(zc, base+idx, mls, iend, nbCompares, 0); idx += ZSTD_insertBt1(zc, base + idx, mls, iend, nbCompares, 0);
} }
/** ZSTD_BtFindBestMatch() : Tree updater, providing best match */ /** ZSTD_BtFindBestMatch() : Tree updater, providing best match */
static size_t ZSTD_BtFindBestMatch ( static size_t ZSTD_BtFindBestMatch(ZSTD_CCtx *zc, const BYTE *const ip, const BYTE *const iLimit, size_t *offsetPtr, const U32 maxNbAttempts, const U32 mls)
ZSTD_CCtx* zc,
const BYTE* const ip, const BYTE* const iLimit,
size_t* offsetPtr,
const U32 maxNbAttempts, const U32 mls)
{ {
if (ip < zc->base + zc->nextToUpdate) return 0; /* skipped area */ if (ip < zc->base + zc->nextToUpdate)
return 0; /* skipped area */
ZSTD_updateTree(zc, ip, iLimit, maxNbAttempts, mls); ZSTD_updateTree(zc, ip, iLimit, maxNbAttempts, mls);
return ZSTD_insertBtAndFindBestMatch(zc, ip, iLimit, offsetPtr, maxNbAttempts, mls, 0); return ZSTD_insertBtAndFindBestMatch(zc, ip, iLimit, offsetPtr, maxNbAttempts, mls, 0);
} }
static size_t ZSTD_BtFindBestMatch_selectMLS(ZSTD_CCtx *zc, /* Index table will be updated */
static size_t ZSTD_BtFindBestMatch_selectMLS ( const BYTE *ip, const BYTE *const iLimit, size_t *offsetPtr, const U32 maxNbAttempts, const U32 matchLengthSearch)
ZSTD_CCtx* zc, /* Index table will be updated */
const BYTE* ip, const BYTE* const iLimit,
size_t* offsetPtr,
const U32 maxNbAttempts, const U32 matchLengthSearch)
{ {
switch(matchLengthSearch) switch (matchLengthSearch) {
{ default: /* includes case 3 */
default : /* includes case 3 */ case 4: return ZSTD_BtFindBestMatch(zc, ip, iLimit, offsetPtr, maxNbAttempts, 4);
case 4 : return ZSTD_BtFindBestMatch(zc, ip, iLimit, offsetPtr, maxNbAttempts, 4); case 5: return ZSTD_BtFindBestMatch(zc, ip, iLimit, offsetPtr, maxNbAttempts, 5);
case 5 : return ZSTD_BtFindBestMatch(zc, ip, iLimit, offsetPtr, maxNbAttempts, 5); case 7:
case 7 : case 6: return ZSTD_BtFindBestMatch(zc, ip, iLimit, offsetPtr, maxNbAttempts, 6);
case 6 : return ZSTD_BtFindBestMatch(zc, ip, iLimit, offsetPtr, maxNbAttempts, 6);
} }
} }
static void ZSTD_updateTree_extDict(ZSTD_CCtx *zc, const BYTE *const ip, const BYTE *const iend, const U32 nbCompares, const U32 mls)
static void ZSTD_updateTree_extDict(ZSTD_CCtx* zc, const BYTE* const ip, const BYTE* const iend, const U32 nbCompares, const U32 mls)
{ {
const BYTE* const base = zc->base; const BYTE *const base = zc->base;
const U32 target = (U32)(ip - base); const U32 target = (U32)(ip - base);
U32 idx = zc->nextToUpdate; U32 idx = zc->nextToUpdate;
while (idx < target) idx += ZSTD_insertBt1(zc, base+idx, mls, iend, nbCompares, 1); while (idx < target)
idx += ZSTD_insertBt1(zc, base + idx, mls, iend, nbCompares, 1);
} }
/** Tree updater, providing best match */ /** Tree updater, providing best match */
static size_t ZSTD_BtFindBestMatch_extDict ( static size_t ZSTD_BtFindBestMatch_extDict(ZSTD_CCtx *zc, const BYTE *const ip, const BYTE *const iLimit, size_t *offsetPtr, const U32 maxNbAttempts,
ZSTD_CCtx* zc, const U32 mls)
const BYTE* const ip, const BYTE* const iLimit,
size_t* offsetPtr,
const U32 maxNbAttempts, const U32 mls)
{ {
if (ip < zc->base + zc->nextToUpdate) return 0; /* skipped area */ if (ip < zc->base + zc->nextToUpdate)
return 0; /* skipped area */
ZSTD_updateTree_extDict(zc, ip, iLimit, maxNbAttempts, mls); ZSTD_updateTree_extDict(zc, ip, iLimit, maxNbAttempts, mls);
return ZSTD_insertBtAndFindBestMatch(zc, ip, iLimit, offsetPtr, maxNbAttempts, mls, 1); return ZSTD_insertBtAndFindBestMatch(zc, ip, iLimit, offsetPtr, maxNbAttempts, mls, 1);
} }
static size_t ZSTD_BtFindBestMatch_selectMLS_extDict(ZSTD_CCtx *zc, /* Index table will be updated */
static size_t ZSTD_BtFindBestMatch_selectMLS_extDict ( const BYTE *ip, const BYTE *const iLimit, size_t *offsetPtr, const U32 maxNbAttempts,
ZSTD_CCtx* zc, /* Index table will be updated */ const U32 matchLengthSearch)
const BYTE* ip, const BYTE* const iLimit,
size_t* offsetPtr,
const U32 maxNbAttempts, const U32 matchLengthSearch)
{ {
switch(matchLengthSearch) switch (matchLengthSearch) {
{ default: /* includes case 3 */
default : /* includes case 3 */ case 4: return ZSTD_BtFindBestMatch_extDict(zc, ip, iLimit, offsetPtr, maxNbAttempts, 4);
case 4 : return ZSTD_BtFindBestMatch_extDict(zc, ip, iLimit, offsetPtr, maxNbAttempts, 4); case 5: return ZSTD_BtFindBestMatch_extDict(zc, ip, iLimit, offsetPtr, maxNbAttempts, 5);
case 5 : return ZSTD_BtFindBestMatch_extDict(zc, ip, iLimit, offsetPtr, maxNbAttempts, 5); case 7:
case 7 : case 6: return ZSTD_BtFindBestMatch_extDict(zc, ip, iLimit, offsetPtr, maxNbAttempts, 6);
case 6 : return ZSTD_BtFindBestMatch_extDict(zc, ip, iLimit, offsetPtr, maxNbAttempts, 6);
} }
} }
/* ********************************* /* *********************************
* Hash Chain * Hash Chain
***********************************/ ***********************************/
#define NEXT_IN_CHAIN(d, mask) chainTable[(d) & mask] #define NEXT_IN_CHAIN(d, mask) chainTable[(d)&mask]
/* Update chains up to ip (excluded) /* Update chains up to ip (excluded)
Assumption : always within prefix (i.e. not within extDict) */ Assumption : always within prefix (i.e. not within extDict) */
FORCE_INLINE FORCE_INLINE
U32 ZSTD_insertAndFindFirstIndex (ZSTD_CCtx* zc, const BYTE* ip, U32 mls) U32 ZSTD_insertAndFindFirstIndex(ZSTD_CCtx *zc, const BYTE *ip, U32 mls)
{ {
U32* const hashTable = zc->hashTable; U32 *const hashTable = zc->hashTable;
const U32 hashLog = zc->params.cParams.hashLog; const U32 hashLog = zc->params.cParams.hashLog;
U32* const chainTable = zc->chainTable; U32 *const chainTable = zc->chainTable;
const U32 chainMask = (1 << zc->params.cParams.chainLog) - 1; const U32 chainMask = (1 << zc->params.cParams.chainLog) - 1;
const BYTE* const base = zc->base; const BYTE *const base = zc->base;
const U32 target = (U32)(ip - base); const U32 target = (U32)(ip - base);
U32 idx = zc->nextToUpdate; U32 idx = zc->nextToUpdate;
while(idx < target) { /* catch up */ while (idx < target) { /* catch up */
size_t const h = ZSTD_hashPtr(base+idx, hashLog, mls); size_t const h = ZSTD_hashPtr(base + idx, hashLog, mls);
NEXT_IN_CHAIN(idx, chainMask) = hashTable[h]; NEXT_IN_CHAIN(idx, chainMask) = hashTable[h];
hashTable[h] = idx; hashTable[h] = idx;
idx++; idx++;
@@ -1713,35 +1800,31 @@ U32 ZSTD_insertAndFindFirstIndex (ZSTD_CCtx* zc, const BYTE* ip, U32 mls)
return hashTable[ZSTD_hashPtr(ip, hashLog, mls)]; return hashTable[ZSTD_hashPtr(ip, hashLog, mls)];
} }
FORCE_INLINE /* inlining is important to hardwire a hot branch (template emulation) */ FORCE_INLINE /* inlining is important to hardwire a hot branch (template emulation) */
size_t ZSTD_HcFindBestMatch_generic ( size_t
ZSTD_CCtx* zc, /* Index table will be updated */ ZSTD_HcFindBestMatch_generic(ZSTD_CCtx *zc, /* Index table will be updated */
const BYTE* const ip, const BYTE* const iLimit, const BYTE *const ip, const BYTE *const iLimit, size_t *offsetPtr, const U32 maxNbAttempts, const U32 mls, const U32 extDict)
size_t* offsetPtr,
const U32 maxNbAttempts, const U32 mls, const U32 extDict)
{ {
U32* const chainTable = zc->chainTable; U32 *const chainTable = zc->chainTable;
const U32 chainSize = (1 << zc->params.cParams.chainLog); const U32 chainSize = (1 << zc->params.cParams.chainLog);
const U32 chainMask = chainSize-1; const U32 chainMask = chainSize - 1;
const BYTE* const base = zc->base; const BYTE *const base = zc->base;
const BYTE* const dictBase = zc->dictBase; const BYTE *const dictBase = zc->dictBase;
const U32 dictLimit = zc->dictLimit; const U32 dictLimit = zc->dictLimit;
const BYTE* const prefixStart = base + dictLimit; const BYTE *const prefixStart = base + dictLimit;
const BYTE* const dictEnd = dictBase + dictLimit; const BYTE *const dictEnd = dictBase + dictLimit;
const U32 lowLimit = zc->lowLimit; const U32 lowLimit = zc->lowLimit;
const U32 curr = (U32)(ip-base); const U32 curr = (U32)(ip - base);
const U32 minChain = curr > chainSize ? curr - chainSize : 0; const U32 minChain = curr > chainSize ? curr - chainSize : 0;
int nbAttempts=maxNbAttempts; int nbAttempts = maxNbAttempts;
size_t ml=EQUAL_READ32-1; size_t ml = EQUAL_READ32 - 1;
/* HC4 match finder */ /* HC4 match finder */
U32 matchIndex = ZSTD_insertAndFindFirstIndex (zc, ip, mls); U32 matchIndex = ZSTD_insertAndFindFirstIndex(zc, ip, mls);
for ( ; (matchIndex>lowLimit) & (nbAttempts>0) ; nbAttempts--) { for (; (matchIndex > lowLimit) & (nbAttempts > 0); nbAttempts--) {
const BYTE* match; const BYTE *match;
size_t currMl=0; size_t currMl = 0;
if ((!extDict) || matchIndex >= dictLimit) { if ((!extDict) || matchIndex >= dictLimit) {
match = base + matchIndex; match = base + matchIndex;
if (match[ml] == ip[ml]) /* potentially better */ if (match[ml] == ip[ml]) /* potentially better */
@@ -1749,238 +1832,228 @@ size_t ZSTD_HcFindBestMatch_generic (
} else { } else {
match = dictBase + matchIndex; match = dictBase + matchIndex;
if (ZSTD_read32(match) == ZSTD_read32(ip)) /* assumption : matchIndex <= dictLimit-4 (by table construction) */ if (ZSTD_read32(match) == ZSTD_read32(ip)) /* assumption : matchIndex <= dictLimit-4 (by table construction) */
currMl = ZSTD_count_2segments(ip+EQUAL_READ32, match+EQUAL_READ32, iLimit, dictEnd, prefixStart) + EQUAL_READ32; currMl = ZSTD_count_2segments(ip + EQUAL_READ32, match + EQUAL_READ32, iLimit, dictEnd, prefixStart) + EQUAL_READ32;
} }
/* save best solution */ /* save best solution */
if (currMl > ml) { ml = currMl; *offsetPtr = curr - matchIndex + ZSTD_REP_MOVE; if (ip+currMl == iLimit) break; /* best possible, and avoid read overflow*/ } if (currMl > ml) {
ml = currMl;
*offsetPtr = curr - matchIndex + ZSTD_REP_MOVE;
if (ip + currMl == iLimit)
break; /* best possible, and avoid read overflow*/
}
if (matchIndex <= minChain) break; if (matchIndex <= minChain)
break;
matchIndex = NEXT_IN_CHAIN(matchIndex, chainMask); matchIndex = NEXT_IN_CHAIN(matchIndex, chainMask);
} }
return ml; return ml;
} }
FORCE_INLINE size_t ZSTD_HcFindBestMatch_selectMLS(ZSTD_CCtx *zc, const BYTE *ip, const BYTE *const iLimit, size_t *offsetPtr, const U32 maxNbAttempts,
FORCE_INLINE size_t ZSTD_HcFindBestMatch_selectMLS ( const U32 matchLengthSearch)
ZSTD_CCtx* zc,
const BYTE* ip, const BYTE* const iLimit,
size_t* offsetPtr,
const U32 maxNbAttempts, const U32 matchLengthSearch)
{ {
switch(matchLengthSearch) switch (matchLengthSearch) {
{ default: /* includes case 3 */
default : /* includes case 3 */ case 4: return ZSTD_HcFindBestMatch_generic(zc, ip, iLimit, offsetPtr, maxNbAttempts, 4, 0);
case 4 : return ZSTD_HcFindBestMatch_generic(zc, ip, iLimit, offsetPtr, maxNbAttempts, 4, 0); case 5: return ZSTD_HcFindBestMatch_generic(zc, ip, iLimit, offsetPtr, maxNbAttempts, 5, 0);
case 5 : return ZSTD_HcFindBestMatch_generic(zc, ip, iLimit, offsetPtr, maxNbAttempts, 5, 0); case 7:
case 7 : case 6: return ZSTD_HcFindBestMatch_generic(zc, ip, iLimit, offsetPtr, maxNbAttempts, 6, 0);
case 6 : return ZSTD_HcFindBestMatch_generic(zc, ip, iLimit, offsetPtr, maxNbAttempts, 6, 0);
} }
} }
FORCE_INLINE size_t ZSTD_HcFindBestMatch_extDict_selectMLS(ZSTD_CCtx *zc, const BYTE *ip, const BYTE *const iLimit, size_t *offsetPtr, const U32 maxNbAttempts,
FORCE_INLINE size_t ZSTD_HcFindBestMatch_extDict_selectMLS ( const U32 matchLengthSearch)
ZSTD_CCtx* zc,
const BYTE* ip, const BYTE* const iLimit,
size_t* offsetPtr,
const U32 maxNbAttempts, const U32 matchLengthSearch)
{ {
switch(matchLengthSearch) switch (matchLengthSearch) {
{ default: /* includes case 3 */
default : /* includes case 3 */ case 4: return ZSTD_HcFindBestMatch_generic(zc, ip, iLimit, offsetPtr, maxNbAttempts, 4, 1);
case 4 : return ZSTD_HcFindBestMatch_generic(zc, ip, iLimit, offsetPtr, maxNbAttempts, 4, 1); case 5: return ZSTD_HcFindBestMatch_generic(zc, ip, iLimit, offsetPtr, maxNbAttempts, 5, 1);
case 5 : return ZSTD_HcFindBestMatch_generic(zc, ip, iLimit, offsetPtr, maxNbAttempts, 5, 1); case 7:
case 7 : case 6: return ZSTD_HcFindBestMatch_generic(zc, ip, iLimit, offsetPtr, maxNbAttempts, 6, 1);
case 6 : return ZSTD_HcFindBestMatch_generic(zc, ip, iLimit, offsetPtr, maxNbAttempts, 6, 1);
} }
} }
/* ******************************* /* *******************************
* Common parser - lazy strategy * Common parser - lazy strategy
*********************************/ *********************************/
FORCE_INLINE FORCE_INLINE
void ZSTD_compressBlock_lazy_generic(ZSTD_CCtx* ctx, void ZSTD_compressBlock_lazy_generic(ZSTD_CCtx *ctx, const void *src, size_t srcSize, const U32 searchMethod, const U32 depth)
const void* src, size_t srcSize,
const U32 searchMethod, const U32 depth)
{ {
seqStore_t* seqStorePtr = &(ctx->seqStore); seqStore_t *seqStorePtr = &(ctx->seqStore);
const BYTE* const istart = (const BYTE*)src; const BYTE *const istart = (const BYTE *)src;
const BYTE* ip = istart; const BYTE *ip = istart;
const BYTE* anchor = istart; const BYTE *anchor = istart;
const BYTE* const iend = istart + srcSize; const BYTE *const iend = istart + srcSize;
const BYTE* const ilimit = iend - 8; const BYTE *const ilimit = iend - 8;
const BYTE* const base = ctx->base + ctx->dictLimit; const BYTE *const base = ctx->base + ctx->dictLimit;
U32 const maxSearches = 1 << ctx->params.cParams.searchLog; U32 const maxSearches = 1 << ctx->params.cParams.searchLog;
U32 const mls = ctx->params.cParams.searchLength; U32 const mls = ctx->params.cParams.searchLength;
typedef size_t (*searchMax_f)(ZSTD_CCtx* zc, const BYTE* ip, const BYTE* iLimit, typedef size_t (*searchMax_f)(ZSTD_CCtx * zc, const BYTE *ip, const BYTE *iLimit, size_t *offsetPtr, U32 maxNbAttempts, U32 matchLengthSearch);
size_t* offsetPtr,
U32 maxNbAttempts, U32 matchLengthSearch);
searchMax_f const searchMax = searchMethod ? ZSTD_BtFindBestMatch_selectMLS : ZSTD_HcFindBestMatch_selectMLS; searchMax_f const searchMax = searchMethod ? ZSTD_BtFindBestMatch_selectMLS : ZSTD_HcFindBestMatch_selectMLS;
U32 offset_1 = ctx->rep[0], offset_2 = ctx->rep[1], savedOffset=0; U32 offset_1 = ctx->rep[0], offset_2 = ctx->rep[1], savedOffset = 0;
/* init */ /* init */
ip += (ip==base); ip += (ip == base);
ctx->nextToUpdate3 = ctx->nextToUpdate; ctx->nextToUpdate3 = ctx->nextToUpdate;
{ U32 const maxRep = (U32)(ip-base); {
if (offset_2 > maxRep) savedOffset = offset_2, offset_2 = 0; U32 const maxRep = (U32)(ip - base);
if (offset_1 > maxRep) savedOffset = offset_1, offset_1 = 0; if (offset_2 > maxRep)
savedOffset = offset_2, offset_2 = 0;
if (offset_1 > maxRep)
savedOffset = offset_1, offset_1 = 0;
} }
/* Match Loop */ /* Match Loop */
while (ip < ilimit) { while (ip < ilimit) {
size_t matchLength=0; size_t matchLength = 0;
size_t offset=0; size_t offset = 0;
const BYTE* start=ip+1; const BYTE *start = ip + 1;
/* check repCode */ /* check repCode */
if ((offset_1>0) & (ZSTD_read32(ip+1) == ZSTD_read32(ip+1 - offset_1))) { if ((offset_1 > 0) & (ZSTD_read32(ip + 1) == ZSTD_read32(ip + 1 - offset_1))) {
/* repcode : we take it */ /* repcode : we take it */
matchLength = ZSTD_count(ip+1+EQUAL_READ32, ip+1+EQUAL_READ32-offset_1, iend) + EQUAL_READ32; matchLength = ZSTD_count(ip + 1 + EQUAL_READ32, ip + 1 + EQUAL_READ32 - offset_1, iend) + EQUAL_READ32;
if (depth==0) goto _storeSequence; if (depth == 0)
goto _storeSequence;
} }
/* first search (depth 0) */ /* first search (depth 0) */
{ size_t offsetFound = 99999999; {
size_t offsetFound = 99999999;
size_t const ml2 = searchMax(ctx, ip, iend, &offsetFound, maxSearches, mls); size_t const ml2 = searchMax(ctx, ip, iend, &offsetFound, maxSearches, mls);
if (ml2 > matchLength) if (ml2 > matchLength)
matchLength = ml2, start = ip, offset=offsetFound; matchLength = ml2, start = ip, offset = offsetFound;
} }
if (matchLength < EQUAL_READ32) { if (matchLength < EQUAL_READ32) {
ip += ((ip-anchor) >> g_searchStrength) + 1; /* jump faster over incompressible sections */ ip += ((ip - anchor) >> g_searchStrength) + 1; /* jump faster over incompressible sections */
continue; continue;
} }
/* let's try to find a better solution */ /* let's try to find a better solution */
if (depth>=1) if (depth >= 1)
while (ip<ilimit) { while (ip < ilimit) {
ip ++; ip++;
if ((offset) && ((offset_1>0) & (ZSTD_read32(ip) == ZSTD_read32(ip - offset_1)))) { if ((offset) && ((offset_1 > 0) & (ZSTD_read32(ip) == ZSTD_read32(ip - offset_1)))) {
size_t const mlRep = ZSTD_count(ip+EQUAL_READ32, ip+EQUAL_READ32-offset_1, iend) + EQUAL_READ32; size_t const mlRep = ZSTD_count(ip + EQUAL_READ32, ip + EQUAL_READ32 - offset_1, iend) + EQUAL_READ32;
int const gain2 = (int)(mlRep * 3); int const gain2 = (int)(mlRep * 3);
int const gain1 = (int)(matchLength*3 - ZSTD_highbit32((U32)offset+1) + 1); int const gain1 = (int)(matchLength * 3 - ZSTD_highbit32((U32)offset + 1) + 1);
if ((mlRep >= EQUAL_READ32) && (gain2 > gain1)) if ((mlRep >= EQUAL_READ32) && (gain2 > gain1))
matchLength = mlRep, offset = 0, start = ip; matchLength = mlRep, offset = 0, start = ip;
} }
{ size_t offset2=99999999; {
size_t offset2 = 99999999;
size_t const ml2 = searchMax(ctx, ip, iend, &offset2, maxSearches, mls); size_t const ml2 = searchMax(ctx, ip, iend, &offset2, maxSearches, mls);
int const gain2 = (int)(ml2*4 - ZSTD_highbit32((U32)offset2+1)); /* raw approx */ int const gain2 = (int)(ml2 * 4 - ZSTD_highbit32((U32)offset2 + 1)); /* raw approx */
int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offset+1) + 4); int const gain1 = (int)(matchLength * 4 - ZSTD_highbit32((U32)offset + 1) + 4);
if ((ml2 >= EQUAL_READ32) && (gain2 > gain1)) { if ((ml2 >= EQUAL_READ32) && (gain2 > gain1)) {
matchLength = ml2, offset = offset2, start = ip; matchLength = ml2, offset = offset2, start = ip;
continue; /* search a better one */ continue; /* search a better one */
} } }
}
/* let's find an even better one */ /* let's find an even better one */
if ((depth==2) && (ip<ilimit)) { if ((depth == 2) && (ip < ilimit)) {
ip ++; ip++;
if ((offset) && ((offset_1>0) & (ZSTD_read32(ip) == ZSTD_read32(ip - offset_1)))) { if ((offset) && ((offset_1 > 0) & (ZSTD_read32(ip) == ZSTD_read32(ip - offset_1)))) {
size_t const ml2 = ZSTD_count(ip+EQUAL_READ32, ip+EQUAL_READ32-offset_1, iend) + EQUAL_READ32; size_t const ml2 = ZSTD_count(ip + EQUAL_READ32, ip + EQUAL_READ32 - offset_1, iend) + EQUAL_READ32;
int const gain2 = (int)(ml2 * 4); int const gain2 = (int)(ml2 * 4);
int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offset+1) + 1); int const gain1 = (int)(matchLength * 4 - ZSTD_highbit32((U32)offset + 1) + 1);
if ((ml2 >= EQUAL_READ32) && (gain2 > gain1)) if ((ml2 >= EQUAL_READ32) && (gain2 > gain1))
matchLength = ml2, offset = 0, start = ip; matchLength = ml2, offset = 0, start = ip;
} }
{ size_t offset2=99999999; {
size_t offset2 = 99999999;
size_t const ml2 = searchMax(ctx, ip, iend, &offset2, maxSearches, mls); size_t const ml2 = searchMax(ctx, ip, iend, &offset2, maxSearches, mls);
int const gain2 = (int)(ml2*4 - ZSTD_highbit32((U32)offset2+1)); /* raw approx */ int const gain2 = (int)(ml2 * 4 - ZSTD_highbit32((U32)offset2 + 1)); /* raw approx */
int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offset+1) + 7); int const gain1 = (int)(matchLength * 4 - ZSTD_highbit32((U32)offset + 1) + 7);
if ((ml2 >= EQUAL_READ32) && (gain2 > gain1)) { if ((ml2 >= EQUAL_READ32) && (gain2 > gain1)) {
matchLength = ml2, offset = offset2, start = ip; matchLength = ml2, offset = offset2, start = ip;
continue; continue;
} } } }
}
}
break; /* nothing found : store previous solution */ break; /* nothing found : store previous solution */
} }
/* catch up */ /* catch up */
if (offset) { if (offset) {
while ((start>anchor) && (start>base+offset-ZSTD_REP_MOVE) && (start[-1] == start[-1-offset+ZSTD_REP_MOVE])) /* only search for offset within prefix */ while ((start > anchor) && (start > base + offset - ZSTD_REP_MOVE) &&
{ start--; matchLength++; } (start[-1] == start[-1 - offset + ZSTD_REP_MOVE])) /* only search for offset within prefix */
offset_2 = offset_1; offset_1 = (U32)(offset - ZSTD_REP_MOVE); {
start--;
matchLength++;
}
offset_2 = offset_1;
offset_1 = (U32)(offset - ZSTD_REP_MOVE);
} }
/* store sequence */ /* store sequence */
_storeSequence: _storeSequence : {
{ size_t const litLength = start - anchor; size_t const litLength = start - anchor;
ZSTD_storeSeq(seqStorePtr, litLength, anchor, (U32)offset, matchLength-MINMATCH); ZSTD_storeSeq(seqStorePtr, litLength, anchor, (U32)offset, matchLength - MINMATCH);
anchor = ip = start + matchLength; anchor = ip = start + matchLength;
} }
/* check immediate repcode */ /* check immediate repcode */
while ( (ip <= ilimit) while ((ip <= ilimit) && ((offset_2 > 0) & (ZSTD_read32(ip) == ZSTD_read32(ip - offset_2)))) {
&& ((offset_2>0)
& (ZSTD_read32(ip) == ZSTD_read32(ip - offset_2)) )) {
/* store sequence */ /* store sequence */
matchLength = ZSTD_count(ip+EQUAL_READ32, ip+EQUAL_READ32-offset_2, iend) + EQUAL_READ32; matchLength = ZSTD_count(ip + EQUAL_READ32, ip + EQUAL_READ32 - offset_2, iend) + EQUAL_READ32;
offset = offset_2; offset_2 = offset_1; offset_1 = (U32)offset; /* swap repcodes */ offset = offset_2;
ZSTD_storeSeq(seqStorePtr, 0, anchor, 0, matchLength-MINMATCH); offset_2 = offset_1;
offset_1 = (U32)offset; /* swap repcodes */
ZSTD_storeSeq(seqStorePtr, 0, anchor, 0, matchLength - MINMATCH);
ip += matchLength; ip += matchLength;
anchor = ip; anchor = ip;
continue; /* faster when present ... (?) */ continue; /* faster when present ... (?) */
} } }
}
/* Save reps for next block */ /* Save reps for next block */
ctx->repToConfirm[0] = offset_1 ? offset_1 : savedOffset; ctx->repToConfirm[0] = offset_1 ? offset_1 : savedOffset;
ctx->repToConfirm[1] = offset_2 ? offset_2 : savedOffset; ctx->repToConfirm[1] = offset_2 ? offset_2 : savedOffset;
/* Last Literals */ /* Last Literals */
{ size_t const lastLLSize = iend - anchor; {
size_t const lastLLSize = iend - anchor;
memcpy(seqStorePtr->lit, anchor, lastLLSize); memcpy(seqStorePtr->lit, anchor, lastLLSize);
seqStorePtr->lit += lastLLSize; seqStorePtr->lit += lastLLSize;
} }
} }
static void ZSTD_compressBlock_btlazy2(ZSTD_CCtx *ctx, const void *src, size_t srcSize) { ZSTD_compressBlock_lazy_generic(ctx, src, srcSize, 1, 2); }
static void ZSTD_compressBlock_btlazy2(ZSTD_CCtx* ctx, const void* src, size_t srcSize) static void ZSTD_compressBlock_lazy2(ZSTD_CCtx *ctx, const void *src, size_t srcSize) { ZSTD_compressBlock_lazy_generic(ctx, src, srcSize, 0, 2); }
{
ZSTD_compressBlock_lazy_generic(ctx, src, srcSize, 1, 2);
}
static void ZSTD_compressBlock_lazy2(ZSTD_CCtx* ctx, const void* src, size_t srcSize) static void ZSTD_compressBlock_lazy(ZSTD_CCtx *ctx, const void *src, size_t srcSize) { ZSTD_compressBlock_lazy_generic(ctx, src, srcSize, 0, 1); }
{
ZSTD_compressBlock_lazy_generic(ctx, src, srcSize, 0, 2);
}
static void ZSTD_compressBlock_lazy(ZSTD_CCtx* ctx, const void* src, size_t srcSize)
{
ZSTD_compressBlock_lazy_generic(ctx, src, srcSize, 0, 1);
}
static void ZSTD_compressBlock_greedy(ZSTD_CCtx* ctx, const void* src, size_t srcSize)
{
ZSTD_compressBlock_lazy_generic(ctx, src, srcSize, 0, 0);
}
static void ZSTD_compressBlock_greedy(ZSTD_CCtx *ctx, const void *src, size_t srcSize) { ZSTD_compressBlock_lazy_generic(ctx, src, srcSize, 0, 0); }
FORCE_INLINE FORCE_INLINE
void ZSTD_compressBlock_lazy_extDict_generic(ZSTD_CCtx* ctx, void ZSTD_compressBlock_lazy_extDict_generic(ZSTD_CCtx *ctx, const void *src, size_t srcSize, const U32 searchMethod, const U32 depth)
const void* src, size_t srcSize,
const U32 searchMethod, const U32 depth)
{ {
seqStore_t* seqStorePtr = &(ctx->seqStore); seqStore_t *seqStorePtr = &(ctx->seqStore);
const BYTE* const istart = (const BYTE*)src; const BYTE *const istart = (const BYTE *)src;
const BYTE* ip = istart; const BYTE *ip = istart;
const BYTE* anchor = istart; const BYTE *anchor = istart;
const BYTE* const iend = istart + srcSize; const BYTE *const iend = istart + srcSize;
const BYTE* const ilimit = iend - 8; const BYTE *const ilimit = iend - 8;
const BYTE* const base = ctx->base; const BYTE *const base = ctx->base;
const U32 dictLimit = ctx->dictLimit; const U32 dictLimit = ctx->dictLimit;
const U32 lowestIndex = ctx->lowLimit; const U32 lowestIndex = ctx->lowLimit;
const BYTE* const prefixStart = base + dictLimit; const BYTE *const prefixStart = base + dictLimit;
const BYTE* const dictBase = ctx->dictBase; const BYTE *const dictBase = ctx->dictBase;
const BYTE* const dictEnd = dictBase + dictLimit; const BYTE *const dictEnd = dictBase + dictLimit;
const BYTE* const dictStart = dictBase + ctx->lowLimit; const BYTE *const dictStart = dictBase + ctx->lowLimit;
const U32 maxSearches = 1 << ctx->params.cParams.searchLog; const U32 maxSearches = 1 << ctx->params.cParams.searchLog;
const U32 mls = ctx->params.cParams.searchLength; const U32 mls = ctx->params.cParams.searchLength;
typedef size_t (*searchMax_f)(ZSTD_CCtx* zc, const BYTE* ip, const BYTE* iLimit, typedef size_t (*searchMax_f)(ZSTD_CCtx * zc, const BYTE *ip, const BYTE *iLimit, size_t *offsetPtr, U32 maxNbAttempts, U32 matchLengthSearch);
size_t* offsetPtr,
U32 maxNbAttempts, U32 matchLengthSearch);
searchMax_f searchMax = searchMethod ? ZSTD_BtFindBestMatch_selectMLS_extDict : ZSTD_HcFindBestMatch_extDict_selectMLS; searchMax_f searchMax = searchMethod ? ZSTD_BtFindBestMatch_selectMLS_extDict : ZSTD_HcFindBestMatch_extDict_selectMLS;
U32 offset_1 = ctx->rep[0], offset_2 = ctx->rep[1]; U32 offset_1 = ctx->rep[0], offset_2 = ctx->rep[1];
@@ -1991,229 +2064,259 @@ void ZSTD_compressBlock_lazy_extDict_generic(ZSTD_CCtx* ctx,
/* Match Loop */ /* Match Loop */
while (ip < ilimit) { while (ip < ilimit) {
size_t matchLength=0; size_t matchLength = 0;
size_t offset=0; size_t offset = 0;
const BYTE* start=ip+1; const BYTE *start = ip + 1;
U32 curr = (U32)(ip-base); U32 curr = (U32)(ip - base);
/* check repCode */ /* check repCode */
{ const U32 repIndex = (U32)(curr+1 - offset_1); {
const BYTE* const repBase = repIndex < dictLimit ? dictBase : base; const U32 repIndex = (U32)(curr + 1 - offset_1);
const BYTE* const repMatch = repBase + repIndex; const BYTE *const repBase = repIndex < dictLimit ? dictBase : base;
if (((U32)((dictLimit-1) - repIndex) >= 3) & (repIndex > lowestIndex)) /* intentional overflow */ const BYTE *const repMatch = repBase + repIndex;
if (ZSTD_read32(ip+1) == ZSTD_read32(repMatch)) { if (((U32)((dictLimit - 1) - repIndex) >= 3) & (repIndex > lowestIndex)) /* intentional overflow */
if (ZSTD_read32(ip + 1) == ZSTD_read32(repMatch)) {
/* repcode detected we should take it */ /* repcode detected we should take it */
const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend; const BYTE *const repEnd = repIndex < dictLimit ? dictEnd : iend;
matchLength = ZSTD_count_2segments(ip+1+EQUAL_READ32, repMatch+EQUAL_READ32, iend, repEnd, prefixStart) + EQUAL_READ32; matchLength =
if (depth==0) goto _storeSequence; ZSTD_count_2segments(ip + 1 + EQUAL_READ32, repMatch + EQUAL_READ32, iend, repEnd, prefixStart) + EQUAL_READ32;
} } if (depth == 0)
goto _storeSequence;
}
}
/* first search (depth 0) */ /* first search (depth 0) */
{ size_t offsetFound = 99999999; {
size_t offsetFound = 99999999;
size_t const ml2 = searchMax(ctx, ip, iend, &offsetFound, maxSearches, mls); size_t const ml2 = searchMax(ctx, ip, iend, &offsetFound, maxSearches, mls);
if (ml2 > matchLength) if (ml2 > matchLength)
matchLength = ml2, start = ip, offset=offsetFound; matchLength = ml2, start = ip, offset = offsetFound;
} }
if (matchLength < EQUAL_READ32) { if (matchLength < EQUAL_READ32) {
ip += ((ip-anchor) >> g_searchStrength) + 1; /* jump faster over incompressible sections */ ip += ((ip - anchor) >> g_searchStrength) + 1; /* jump faster over incompressible sections */
continue; continue;
} }
/* let's try to find a better solution */ /* let's try to find a better solution */
if (depth>=1) if (depth >= 1)
while (ip<ilimit) { while (ip < ilimit) {
ip ++; ip++;
curr++; curr++;
/* check repCode */ /* check repCode */
if (offset) { if (offset) {
const U32 repIndex = (U32)(curr - offset_1); const U32 repIndex = (U32)(curr - offset_1);
const BYTE* const repBase = repIndex < dictLimit ? dictBase : base; const BYTE *const repBase = repIndex < dictLimit ? dictBase : base;
const BYTE* const repMatch = repBase + repIndex; const BYTE *const repMatch = repBase + repIndex;
if (((U32)((dictLimit-1) - repIndex) >= 3) & (repIndex > lowestIndex)) /* intentional overflow */ if (((U32)((dictLimit - 1) - repIndex) >= 3) & (repIndex > lowestIndex)) /* intentional overflow */
if (ZSTD_read32(ip) == ZSTD_read32(repMatch)) { if (ZSTD_read32(ip) == ZSTD_read32(repMatch)) {
/* repcode detected */ /* repcode detected */
const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend; const BYTE *const repEnd = repIndex < dictLimit ? dictEnd : iend;
size_t const repLength = ZSTD_count_2segments(ip+EQUAL_READ32, repMatch+EQUAL_READ32, iend, repEnd, prefixStart) + EQUAL_READ32; size_t const repLength =
ZSTD_count_2segments(ip + EQUAL_READ32, repMatch + EQUAL_READ32, iend, repEnd, prefixStart) +
EQUAL_READ32;
int const gain2 = (int)(repLength * 3); int const gain2 = (int)(repLength * 3);
int const gain1 = (int)(matchLength*3 - ZSTD_highbit32((U32)offset+1) + 1); int const gain1 = (int)(matchLength * 3 - ZSTD_highbit32((U32)offset + 1) + 1);
if ((repLength >= EQUAL_READ32) && (gain2 > gain1)) if ((repLength >= EQUAL_READ32) && (gain2 > gain1))
matchLength = repLength, offset = 0, start = ip; matchLength = repLength, offset = 0, start = ip;
} } }
}
/* search match, depth 1 */ /* search match, depth 1 */
{ size_t offset2=99999999; {
size_t offset2 = 99999999;
size_t const ml2 = searchMax(ctx, ip, iend, &offset2, maxSearches, mls); size_t const ml2 = searchMax(ctx, ip, iend, &offset2, maxSearches, mls);
int const gain2 = (int)(ml2*4 - ZSTD_highbit32((U32)offset2+1)); /* raw approx */ int const gain2 = (int)(ml2 * 4 - ZSTD_highbit32((U32)offset2 + 1)); /* raw approx */
int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offset+1) + 4); int const gain1 = (int)(matchLength * 4 - ZSTD_highbit32((U32)offset + 1) + 4);
if ((ml2 >= EQUAL_READ32) && (gain2 > gain1)) { if ((ml2 >= EQUAL_READ32) && (gain2 > gain1)) {
matchLength = ml2, offset = offset2, start = ip; matchLength = ml2, offset = offset2, start = ip;
continue; /* search a better one */ continue; /* search a better one */
} } }
}
/* let's find an even better one */ /* let's find an even better one */
if ((depth==2) && (ip<ilimit)) { if ((depth == 2) && (ip < ilimit)) {
ip ++; ip++;
curr++; curr++;
/* check repCode */ /* check repCode */
if (offset) { if (offset) {
const U32 repIndex = (U32)(curr - offset_1); const U32 repIndex = (U32)(curr - offset_1);
const BYTE* const repBase = repIndex < dictLimit ? dictBase : base; const BYTE *const repBase = repIndex < dictLimit ? dictBase : base;
const BYTE* const repMatch = repBase + repIndex; const BYTE *const repMatch = repBase + repIndex;
if (((U32)((dictLimit-1) - repIndex) >= 3) & (repIndex > lowestIndex)) /* intentional overflow */ if (((U32)((dictLimit - 1) - repIndex) >= 3) & (repIndex > lowestIndex)) /* intentional overflow */
if (ZSTD_read32(ip) == ZSTD_read32(repMatch)) { if (ZSTD_read32(ip) == ZSTD_read32(repMatch)) {
/* repcode detected */ /* repcode detected */
const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend; const BYTE *const repEnd = repIndex < dictLimit ? dictEnd : iend;
size_t repLength = ZSTD_count_2segments(ip+EQUAL_READ32, repMatch+EQUAL_READ32, iend, repEnd, prefixStart) + EQUAL_READ32; size_t repLength = ZSTD_count_2segments(ip + EQUAL_READ32, repMatch + EQUAL_READ32, iend,
repEnd, prefixStart) +
EQUAL_READ32;
int gain2 = (int)(repLength * 4); int gain2 = (int)(repLength * 4);
int gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offset+1) + 1); int gain1 = (int)(matchLength * 4 - ZSTD_highbit32((U32)offset + 1) + 1);
if ((repLength >= EQUAL_READ32) && (gain2 > gain1)) if ((repLength >= EQUAL_READ32) && (gain2 > gain1))
matchLength = repLength, offset = 0, start = ip; matchLength = repLength, offset = 0, start = ip;
} } }
}
/* search match, depth 2 */ /* search match, depth 2 */
{ size_t offset2=99999999; {
size_t offset2 = 99999999;
size_t const ml2 = searchMax(ctx, ip, iend, &offset2, maxSearches, mls); size_t const ml2 = searchMax(ctx, ip, iend, &offset2, maxSearches, mls);
int const gain2 = (int)(ml2*4 - ZSTD_highbit32((U32)offset2+1)); /* raw approx */ int const gain2 = (int)(ml2 * 4 - ZSTD_highbit32((U32)offset2 + 1)); /* raw approx */
int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offset+1) + 7); int const gain1 = (int)(matchLength * 4 - ZSTD_highbit32((U32)offset + 1) + 7);
if ((ml2 >= EQUAL_READ32) && (gain2 > gain1)) { if ((ml2 >= EQUAL_READ32) && (gain2 > gain1)) {
matchLength = ml2, offset = offset2, start = ip; matchLength = ml2, offset = offset2, start = ip;
continue; continue;
} } } }
}
}
break; /* nothing found : store previous solution */ break; /* nothing found : store previous solution */
} }
/* catch up */ /* catch up */
if (offset) { if (offset) {
U32 const matchIndex = (U32)((start-base) - (offset - ZSTD_REP_MOVE)); U32 const matchIndex = (U32)((start - base) - (offset - ZSTD_REP_MOVE));
const BYTE* match = (matchIndex < dictLimit) ? dictBase + matchIndex : base + matchIndex; const BYTE *match = (matchIndex < dictLimit) ? dictBase + matchIndex : base + matchIndex;
const BYTE* const mStart = (matchIndex < dictLimit) ? dictStart : prefixStart; const BYTE *const mStart = (matchIndex < dictLimit) ? dictStart : prefixStart;
while ((start>anchor) && (match>mStart) && (start[-1] == match[-1])) { start--; match--; matchLength++; } /* catch up */ while ((start > anchor) && (match > mStart) && (start[-1] == match[-1])) {
offset_2 = offset_1; offset_1 = (U32)(offset - ZSTD_REP_MOVE); start--;
match--;
matchLength++;
} /* catch up */
offset_2 = offset_1;
offset_1 = (U32)(offset - ZSTD_REP_MOVE);
} }
/* store sequence */ /* store sequence */
_storeSequence: _storeSequence : {
{ size_t const litLength = start - anchor; size_t const litLength = start - anchor;
ZSTD_storeSeq(seqStorePtr, litLength, anchor, (U32)offset, matchLength-MINMATCH); ZSTD_storeSeq(seqStorePtr, litLength, anchor, (U32)offset, matchLength - MINMATCH);
anchor = ip = start + matchLength; anchor = ip = start + matchLength;
} }
/* check immediate repcode */ /* check immediate repcode */
while (ip <= ilimit) { while (ip <= ilimit) {
const U32 repIndex = (U32)((ip-base) - offset_2); const U32 repIndex = (U32)((ip - base) - offset_2);
const BYTE* const repBase = repIndex < dictLimit ? dictBase : base; const BYTE *const repBase = repIndex < dictLimit ? dictBase : base;
const BYTE* const repMatch = repBase + repIndex; const BYTE *const repMatch = repBase + repIndex;
if (((U32)((dictLimit-1) - repIndex) >= 3) & (repIndex > lowestIndex)) /* intentional overflow */ if (((U32)((dictLimit - 1) - repIndex) >= 3) & (repIndex > lowestIndex)) /* intentional overflow */
if (ZSTD_read32(ip) == ZSTD_read32(repMatch)) { if (ZSTD_read32(ip) == ZSTD_read32(repMatch)) {
/* repcode detected we should take it */ /* repcode detected we should take it */
const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend; const BYTE *const repEnd = repIndex < dictLimit ? dictEnd : iend;
matchLength = ZSTD_count_2segments(ip+EQUAL_READ32, repMatch+EQUAL_READ32, iend, repEnd, prefixStart) + EQUAL_READ32; matchLength =
offset = offset_2; offset_2 = offset_1; offset_1 = (U32)offset; /* swap offset history */ ZSTD_count_2segments(ip + EQUAL_READ32, repMatch + EQUAL_READ32, iend, repEnd, prefixStart) + EQUAL_READ32;
ZSTD_storeSeq(seqStorePtr, 0, anchor, 0, matchLength-MINMATCH); offset = offset_2;
offset_2 = offset_1;
offset_1 = (U32)offset; /* swap offset history */
ZSTD_storeSeq(seqStorePtr, 0, anchor, 0, matchLength - MINMATCH);
ip += matchLength; ip += matchLength;
anchor = ip; anchor = ip;
continue; /* faster when present ... (?) */ continue; /* faster when present ... (?) */
} }
break; break;
} } }
}
/* Save reps for next block */ /* Save reps for next block */
ctx->repToConfirm[0] = offset_1; ctx->repToConfirm[1] = offset_2; ctx->repToConfirm[0] = offset_1;
ctx->repToConfirm[1] = offset_2;
/* Last Literals */ /* Last Literals */
{ size_t const lastLLSize = iend - anchor; {
size_t const lastLLSize = iend - anchor;
memcpy(seqStorePtr->lit, anchor, lastLLSize); memcpy(seqStorePtr->lit, anchor, lastLLSize);
seqStorePtr->lit += lastLLSize; seqStorePtr->lit += lastLLSize;
} }
} }
void ZSTD_compressBlock_greedy_extDict(ZSTD_CCtx *ctx, const void *src, size_t srcSize) { ZSTD_compressBlock_lazy_extDict_generic(ctx, src, srcSize, 0, 0); }
void ZSTD_compressBlock_greedy_extDict(ZSTD_CCtx* ctx, const void* src, size_t srcSize) static void ZSTD_compressBlock_lazy_extDict(ZSTD_CCtx *ctx, const void *src, size_t srcSize)
{
ZSTD_compressBlock_lazy_extDict_generic(ctx, src, srcSize, 0, 0);
}
static void ZSTD_compressBlock_lazy_extDict(ZSTD_CCtx* ctx, const void* src, size_t srcSize)
{ {
ZSTD_compressBlock_lazy_extDict_generic(ctx, src, srcSize, 0, 1); ZSTD_compressBlock_lazy_extDict_generic(ctx, src, srcSize, 0, 1);
} }
static void ZSTD_compressBlock_lazy2_extDict(ZSTD_CCtx* ctx, const void* src, size_t srcSize) static void ZSTD_compressBlock_lazy2_extDict(ZSTD_CCtx *ctx, const void *src, size_t srcSize)
{ {
ZSTD_compressBlock_lazy_extDict_generic(ctx, src, srcSize, 0, 2); ZSTD_compressBlock_lazy_extDict_generic(ctx, src, srcSize, 0, 2);
} }
static void ZSTD_compressBlock_btlazy2_extDict(ZSTD_CCtx* ctx, const void* src, size_t srcSize) static void ZSTD_compressBlock_btlazy2_extDict(ZSTD_CCtx *ctx, const void *src, size_t srcSize)
{ {
ZSTD_compressBlock_lazy_extDict_generic(ctx, src, srcSize, 1, 2); ZSTD_compressBlock_lazy_extDict_generic(ctx, src, srcSize, 1, 2);
} }
/* The optimal parser */ /* The optimal parser */
#include "zstd_opt.h" #include "zstd_opt.h"
static void ZSTD_compressBlock_btopt(ZSTD_CCtx* ctx, const void* src, size_t srcSize) static void ZSTD_compressBlock_btopt(ZSTD_CCtx *ctx, const void *src, size_t srcSize)
{ {
#ifdef ZSTD_OPT_H_91842398743 #ifdef ZSTD_OPT_H_91842398743
ZSTD_compressBlock_opt_generic(ctx, src, srcSize, 0); ZSTD_compressBlock_opt_generic(ctx, src, srcSize, 0);
#else #else
(void)ctx; (void)src; (void)srcSize; (void)ctx;
(void)src;
(void)srcSize;
return; return;
#endif #endif
} }
static void ZSTD_compressBlock_btopt2(ZSTD_CCtx* ctx, const void* src, size_t srcSize) static void ZSTD_compressBlock_btopt2(ZSTD_CCtx *ctx, const void *src, size_t srcSize)
{ {
#ifdef ZSTD_OPT_H_91842398743 #ifdef ZSTD_OPT_H_91842398743
ZSTD_compressBlock_opt_generic(ctx, src, srcSize, 1); ZSTD_compressBlock_opt_generic(ctx, src, srcSize, 1);
#else #else
(void)ctx; (void)src; (void)srcSize; (void)ctx;
(void)src;
(void)srcSize;
return; return;
#endif #endif
} }
static void ZSTD_compressBlock_btopt_extDict(ZSTD_CCtx* ctx, const void* src, size_t srcSize) static void ZSTD_compressBlock_btopt_extDict(ZSTD_CCtx *ctx, const void *src, size_t srcSize)
{ {
#ifdef ZSTD_OPT_H_91842398743 #ifdef ZSTD_OPT_H_91842398743
ZSTD_compressBlock_opt_extDict_generic(ctx, src, srcSize, 0); ZSTD_compressBlock_opt_extDict_generic(ctx, src, srcSize, 0);
#else #else
(void)ctx; (void)src; (void)srcSize; (void)ctx;
(void)src;
(void)srcSize;
return; return;
#endif #endif
} }
static void ZSTD_compressBlock_btopt2_extDict(ZSTD_CCtx* ctx, const void* src, size_t srcSize) static void ZSTD_compressBlock_btopt2_extDict(ZSTD_CCtx *ctx, const void *src, size_t srcSize)
{ {
#ifdef ZSTD_OPT_H_91842398743 #ifdef ZSTD_OPT_H_91842398743
ZSTD_compressBlock_opt_extDict_generic(ctx, src, srcSize, 1); ZSTD_compressBlock_opt_extDict_generic(ctx, src, srcSize, 1);
#else #else
(void)ctx; (void)src; (void)srcSize; (void)ctx;
(void)src;
(void)srcSize;
return; return;
#endif #endif
} }
typedef void (*ZSTD_blockCompressor)(ZSTD_CCtx *ctx, const void *src, size_t srcSize);
typedef void (*ZSTD_blockCompressor) (ZSTD_CCtx* ctx, const void* src, size_t srcSize);
static ZSTD_blockCompressor ZSTD_selectBlockCompressor(ZSTD_strategy strat, int extDict) static ZSTD_blockCompressor ZSTD_selectBlockCompressor(ZSTD_strategy strat, int extDict)
{ {
static const ZSTD_blockCompressor blockCompressor[2][8] = { static const ZSTD_blockCompressor blockCompressor[2][8] = {
{ ZSTD_compressBlock_fast, ZSTD_compressBlock_doubleFast, ZSTD_compressBlock_greedy, ZSTD_compressBlock_lazy, ZSTD_compressBlock_lazy2, ZSTD_compressBlock_btlazy2, ZSTD_compressBlock_btopt, ZSTD_compressBlock_btopt2 }, {ZSTD_compressBlock_fast, ZSTD_compressBlock_doubleFast, ZSTD_compressBlock_greedy, ZSTD_compressBlock_lazy, ZSTD_compressBlock_lazy2,
{ ZSTD_compressBlock_fast_extDict, ZSTD_compressBlock_doubleFast_extDict, ZSTD_compressBlock_greedy_extDict, ZSTD_compressBlock_lazy_extDict,ZSTD_compressBlock_lazy2_extDict, ZSTD_compressBlock_btlazy2_extDict, ZSTD_compressBlock_btopt_extDict, ZSTD_compressBlock_btopt2_extDict } ZSTD_compressBlock_btlazy2, ZSTD_compressBlock_btopt, ZSTD_compressBlock_btopt2},
}; {ZSTD_compressBlock_fast_extDict, ZSTD_compressBlock_doubleFast_extDict, ZSTD_compressBlock_greedy_extDict, ZSTD_compressBlock_lazy_extDict,
ZSTD_compressBlock_lazy2_extDict, ZSTD_compressBlock_btlazy2_extDict, ZSTD_compressBlock_btopt_extDict, ZSTD_compressBlock_btopt2_extDict}};
return blockCompressor[extDict][(U32)strat]; return blockCompressor[extDict][(U32)strat];
} }
static size_t ZSTD_compressBlock_internal(ZSTD_CCtx *zc, void *dst, size_t dstCapacity, const void *src, size_t srcSize)
static size_t ZSTD_compressBlock_internal(ZSTD_CCtx* zc, void* dst, size_t dstCapacity, const void* src, size_t srcSize)
{ {
ZSTD_blockCompressor const blockCompressor = ZSTD_selectBlockCompressor(zc->params.cParams.strategy, zc->lowLimit < zc->dictLimit); ZSTD_blockCompressor const blockCompressor = ZSTD_selectBlockCompressor(zc->params.cParams.strategy, zc->lowLimit < zc->dictLimit);
const BYTE* const base = zc->base; const BYTE *const base = zc->base;
const BYTE* const istart = (const BYTE*)src; const BYTE *const istart = (const BYTE *)src;
const U32 curr = (U32)(istart-base); const U32 curr = (U32)(istart - base);
if (srcSize < MIN_CBLOCK_SIZE+ZSTD_blockHeaderSize+1) return 0; /* don't even attempt compression below a certain srcSize */ if (srcSize < MIN_CBLOCK_SIZE + ZSTD_blockHeaderSize + 1)
return 0; /* don't even attempt compression below a certain srcSize */
ZSTD_resetSeqStore(&(zc->seqStore)); ZSTD_resetSeqStore(&(zc->seqStore));
if (curr > zc->nextToUpdate + 384) if (curr > zc->nextToUpdate + 384)
zc->nextToUpdate = curr - MIN(192, (U32)(curr - zc->nextToUpdate - 384)); /* update tree not updated after finding very long rep matches */ zc->nextToUpdate = curr - MIN(192, (U32)(curr - zc->nextToUpdate - 384)); /* update tree not updated after finding very long rep matches */
@@ -2221,7 +2324,6 @@ static size_t ZSTD_compressBlock_internal(ZSTD_CCtx* zc, void* dst, size_t dstCa
return ZSTD_compressSequences(zc, dst, dstCapacity, srcSize); return ZSTD_compressSequences(zc, dst, dstCapacity, srcSize);
} }
/*! ZSTD_compress_generic() : /*! ZSTD_compress_generic() :
* Compress a chunk of data into one or multiple blocks. * Compress a chunk of data into one or multiple blocks.
* All blocks will be terminated, all input will be consumed. * All blocks will be terminated, all input will be consumed.
@@ -2229,16 +2331,13 @@ static size_t ZSTD_compressBlock_internal(ZSTD_CCtx* zc, void* dst, size_t dstCa
* Frame is supposed already started (header already produced) * Frame is supposed already started (header already produced)
* @return : compressed size, or an error code * @return : compressed size, or an error code
*/ */
static size_t ZSTD_compress_generic (ZSTD_CCtx* cctx, static size_t ZSTD_compress_generic(ZSTD_CCtx *cctx, void *dst, size_t dstCapacity, const void *src, size_t srcSize, U32 lastFrameChunk)
void* dst, size_t dstCapacity,
const void* src, size_t srcSize,
U32 lastFrameChunk)
{ {
size_t blockSize = cctx->blockSize; size_t blockSize = cctx->blockSize;
size_t remaining = srcSize; size_t remaining = srcSize;
const BYTE* ip = (const BYTE*)src; const BYTE *ip = (const BYTE *)src;
BYTE* const ostart = (BYTE*)dst; BYTE *const ostart = (BYTE *)dst;
BYTE* op = ostart; BYTE *op = ostart;
U32 const maxDist = 1 << cctx->params.cParams.windowLog; U32 const maxDist = 1 << cctx->params.cParams.windowLog;
if (cctx->params.fParams.checksumFlag && srcSize) if (cctx->params.fParams.checksumFlag && srcSize)
@@ -2248,11 +2347,13 @@ static size_t ZSTD_compress_generic (ZSTD_CCtx* cctx,
U32 const lastBlock = lastFrameChunk & (blockSize >= remaining); U32 const lastBlock = lastFrameChunk & (blockSize >= remaining);
size_t cSize; size_t cSize;
if (dstCapacity < ZSTD_blockHeaderSize + MIN_CBLOCK_SIZE) return ERROR(dstSize_tooSmall); /* not enough space to store compressed block */ if (dstCapacity < ZSTD_blockHeaderSize + MIN_CBLOCK_SIZE)
if (remaining < blockSize) blockSize = remaining; return ERROR(dstSize_tooSmall); /* not enough space to store compressed block */
if (remaining < blockSize)
blockSize = remaining;
/* preemptive overflow correction */ /* preemptive overflow correction */
if (cctx->lowLimit > (3U<<29)) { if (cctx->lowLimit > (3U << 29)) {
U32 const cycleMask = (1 << ZSTD_cycleLog(cctx->params.cParams.hashLog, cctx->params.cParams.strategy)) - 1; U32 const cycleMask = (1 << ZSTD_cycleLog(cctx->params.cParams.hashLog, cctx->params.cParams.strategy)) - 1;
U32 const curr = (U32)(ip - cctx->base); U32 const curr = (U32)(ip - cctx->base);
U32 const newCurr = (curr & cycleMask) + (1 << cctx->params.cParams.windowLog); U32 const newCurr = (curr & cycleMask) + (1 << cctx->params.cParams.windowLog);
@@ -2263,28 +2364,34 @@ static size_t ZSTD_compress_generic (ZSTD_CCtx* cctx,
cctx->dictBase += correction; cctx->dictBase += correction;
cctx->lowLimit -= correction; cctx->lowLimit -= correction;
cctx->dictLimit -= correction; cctx->dictLimit -= correction;
if (cctx->nextToUpdate < correction) cctx->nextToUpdate = 0; if (cctx->nextToUpdate < correction)
else cctx->nextToUpdate -= correction; cctx->nextToUpdate = 0;
else
cctx->nextToUpdate -= correction;
} }
if ((U32)(ip+blockSize - cctx->base) > cctx->loadedDictEnd + maxDist) { if ((U32)(ip + blockSize - cctx->base) > cctx->loadedDictEnd + maxDist) {
/* enforce maxDist */ /* enforce maxDist */
U32 const newLowLimit = (U32)(ip+blockSize - cctx->base) - maxDist; U32 const newLowLimit = (U32)(ip + blockSize - cctx->base) - maxDist;
if (cctx->lowLimit < newLowLimit) cctx->lowLimit = newLowLimit; if (cctx->lowLimit < newLowLimit)
if (cctx->dictLimit < cctx->lowLimit) cctx->dictLimit = cctx->lowLimit; cctx->lowLimit = newLowLimit;
if (cctx->dictLimit < cctx->lowLimit)
cctx->dictLimit = cctx->lowLimit;
} }
cSize = ZSTD_compressBlock_internal(cctx, op+ZSTD_blockHeaderSize, dstCapacity-ZSTD_blockHeaderSize, ip, blockSize); cSize = ZSTD_compressBlock_internal(cctx, op + ZSTD_blockHeaderSize, dstCapacity - ZSTD_blockHeaderSize, ip, blockSize);
if (ZSTD_isError(cSize)) return cSize; if (ZSTD_isError(cSize))
return cSize;
if (cSize == 0) { /* block is not compressible */ if (cSize == 0) { /* block is not compressible */
U32 const cBlockHeader24 = lastBlock + (((U32)bt_raw)<<1) + (U32)(blockSize << 3); U32 const cBlockHeader24 = lastBlock + (((U32)bt_raw) << 1) + (U32)(blockSize << 3);
if (blockSize + ZSTD_blockHeaderSize > dstCapacity) return ERROR(dstSize_tooSmall); if (blockSize + ZSTD_blockHeaderSize > dstCapacity)
return ERROR(dstSize_tooSmall);
ZSTD_writeLE32(op, cBlockHeader24); /* no pb, 4th byte will be overwritten */ ZSTD_writeLE32(op, cBlockHeader24); /* no pb, 4th byte will be overwritten */
memcpy(op + ZSTD_blockHeaderSize, ip, blockSize); memcpy(op + ZSTD_blockHeaderSize, ip, blockSize);
cSize = ZSTD_blockHeaderSize+blockSize; cSize = ZSTD_blockHeaderSize + blockSize;
} else { } else {
U32 const cBlockHeader24 = lastBlock + (((U32)bt_compressed)<<1) + (U32)(cSize << 3); U32 const cBlockHeader24 = lastBlock + (((U32)bt_compressed) << 1) + (U32)(cSize << 3);
ZSTD_writeLE24(op, cBlockHeader24); ZSTD_writeLE24(op, cBlockHeader24);
cSize += ZSTD_blockHeaderSize; cSize += ZSTD_blockHeaderSize;
} }
@@ -2295,65 +2402,85 @@ static size_t ZSTD_compress_generic (ZSTD_CCtx* cctx,
op += cSize; op += cSize;
} }
if (lastFrameChunk && (op>ostart)) cctx->stage = ZSTDcs_ending; if (lastFrameChunk && (op > ostart))
return op-ostart; cctx->stage = ZSTDcs_ending;
return op - ostart;
} }
static size_t ZSTD_writeFrameHeader(void *dst, size_t dstCapacity, ZSTD_parameters params, U64 pledgedSrcSize, U32 dictID)
static size_t ZSTD_writeFrameHeader(void* dst, size_t dstCapacity, {
ZSTD_parameters params, U64 pledgedSrcSize, U32 dictID) BYTE *const op = (BYTE *)dst;
{ BYTE* const op = (BYTE*)dst; U32 const dictIDSizeCode = (dictID > 0) + (dictID >= 256) + (dictID >= 65536); /* 0-3 */
U32 const dictIDSizeCode = (dictID>0) + (dictID>=256) + (dictID>=65536); /* 0-3 */ U32 const checksumFlag = params.fParams.checksumFlag > 0;
U32 const checksumFlag = params.fParams.checksumFlag>0;
U32 const windowSize = 1U << params.cParams.windowLog; U32 const windowSize = 1U << params.cParams.windowLog;
U32 const singleSegment = params.fParams.contentSizeFlag && (windowSize >= pledgedSrcSize); U32 const singleSegment = params.fParams.contentSizeFlag && (windowSize >= pledgedSrcSize);
BYTE const windowLogByte = (BYTE)((params.cParams.windowLog - ZSTD_WINDOWLOG_ABSOLUTEMIN) << 3); BYTE const windowLogByte = (BYTE)((params.cParams.windowLog - ZSTD_WINDOWLOG_ABSOLUTEMIN) << 3);
U32 const fcsCode = params.fParams.contentSizeFlag ? U32 const fcsCode = params.fParams.contentSizeFlag ? (pledgedSrcSize >= 256) + (pledgedSrcSize >= 65536 + 256) + (pledgedSrcSize >= 0xFFFFFFFFU)
(pledgedSrcSize>=256) + (pledgedSrcSize>=65536+256) + (pledgedSrcSize>=0xFFFFFFFFU) : /* 0-3 */ : /* 0-3 */
0; 0;
BYTE const frameHeaderDecriptionByte = (BYTE)(dictIDSizeCode + (checksumFlag<<2) + (singleSegment<<5) + (fcsCode<<6) ); BYTE const frameHeaderDecriptionByte = (BYTE)(dictIDSizeCode + (checksumFlag << 2) + (singleSegment << 5) + (fcsCode << 6));
size_t pos; size_t pos;
if (dstCapacity < ZSTD_frameHeaderSize_max) return ERROR(dstSize_tooSmall); if (dstCapacity < ZSTD_frameHeaderSize_max)
return ERROR(dstSize_tooSmall);
ZSTD_writeLE32(dst, ZSTD_MAGICNUMBER); ZSTD_writeLE32(dst, ZSTD_MAGICNUMBER);
op[4] = frameHeaderDecriptionByte; pos=5; op[4] = frameHeaderDecriptionByte;
if (!singleSegment) op[pos++] = windowLogByte; pos = 5;
switch(dictIDSizeCode) if (!singleSegment)
{ op[pos++] = windowLogByte;
switch (dictIDSizeCode) {
default: /* impossible */ default: /* impossible */
case 0 : break; case 0: break;
case 1 : op[pos] = (BYTE)(dictID); pos++; break; case 1:
case 2 : ZSTD_writeLE16(op+pos, (U16)dictID); pos+=2; break; op[pos] = (BYTE)(dictID);
case 3 : ZSTD_writeLE32(op+pos, dictID); pos+=4; break; pos++;
break;
case 2:
ZSTD_writeLE16(op + pos, (U16)dictID);
pos += 2;
break;
case 3:
ZSTD_writeLE32(op + pos, dictID);
pos += 4;
break;
} }
switch(fcsCode) switch (fcsCode) {
{
default: /* impossible */ default: /* impossible */
case 0 : if (singleSegment) op[pos++] = (BYTE)(pledgedSrcSize); break; case 0:
case 1 : ZSTD_writeLE16(op+pos, (U16)(pledgedSrcSize-256)); pos+=2; break; if (singleSegment)
case 2 : ZSTD_writeLE32(op+pos, (U32)(pledgedSrcSize)); pos+=4; break; op[pos++] = (BYTE)(pledgedSrcSize);
case 3 : ZSTD_writeLE64(op+pos, (U64)(pledgedSrcSize)); pos+=8; break; break;
case 1:
ZSTD_writeLE16(op + pos, (U16)(pledgedSrcSize - 256));
pos += 2;
break;
case 2:
ZSTD_writeLE32(op + pos, (U32)(pledgedSrcSize));
pos += 4;
break;
case 3:
ZSTD_writeLE64(op + pos, (U64)(pledgedSrcSize));
pos += 8;
break;
} }
return pos; return pos;
} }
static size_t ZSTD_compressContinue_internal(ZSTD_CCtx *cctx, void *dst, size_t dstCapacity, const void *src, size_t srcSize, U32 frame, U32 lastFrameChunk)
static size_t ZSTD_compressContinue_internal (ZSTD_CCtx* cctx,
void* dst, size_t dstCapacity,
const void* src, size_t srcSize,
U32 frame, U32 lastFrameChunk)
{ {
const BYTE* const ip = (const BYTE*) src; const BYTE *const ip = (const BYTE *)src;
size_t fhSize = 0; size_t fhSize = 0;
if (cctx->stage==ZSTDcs_created) return ERROR(stage_wrong); /* missing init (ZSTD_compressBegin) */ if (cctx->stage == ZSTDcs_created)
return ERROR(stage_wrong); /* missing init (ZSTD_compressBegin) */
if (frame && (cctx->stage==ZSTDcs_init)) { if (frame && (cctx->stage == ZSTDcs_init)) {
fhSize = ZSTD_writeFrameHeader(dst, dstCapacity, cctx->params, cctx->frameContentSize, cctx->dictID); fhSize = ZSTD_writeFrameHeader(dst, dstCapacity, cctx->params, cctx->frameContentSize, cctx->dictID);
if (ZSTD_isError(fhSize)) return fhSize; if (ZSTD_isError(fhSize))
return fhSize;
dstCapacity -= fhSize; dstCapacity -= fhSize;
dst = (char*)dst + fhSize; dst = (char *)dst + fhSize;
cctx->stage = ZSTDcs_ongoing; cctx->stage = ZSTDcs_ongoing;
} }
@@ -2366,11 +2493,12 @@ static size_t ZSTD_compressContinue_internal (ZSTD_CCtx* cctx,
cctx->dictBase = cctx->base; cctx->dictBase = cctx->base;
cctx->base -= delta; cctx->base -= delta;
cctx->nextToUpdate = cctx->dictLimit; cctx->nextToUpdate = cctx->dictLimit;
if (cctx->dictLimit - cctx->lowLimit < HASH_READ_SIZE) cctx->lowLimit = cctx->dictLimit; /* too small extDict */ if (cctx->dictLimit - cctx->lowLimit < HASH_READ_SIZE)
cctx->lowLimit = cctx->dictLimit; /* too small extDict */
} }
/* if input and dictionary overlap : reduce dictionary (area presumed modified by input) */ /* if input and dictionary overlap : reduce dictionary (area presumed modified by input) */
if ((ip+srcSize > cctx->dictBase + cctx->lowLimit) & (ip < cctx->dictBase + cctx->dictLimit)) { if ((ip + srcSize > cctx->dictBase + cctx->lowLimit) & (ip < cctx->dictBase + cctx->dictLimit)) {
ptrdiff_t const highInputIdx = (ip + srcSize) - cctx->dictBase; ptrdiff_t const highInputIdx = (ip + srcSize) - cctx->dictBase;
U32 const lowLimitMax = (highInputIdx > (ptrdiff_t)cctx->dictLimit) ? cctx->dictLimit : (U32)highInputIdx; U32 const lowLimitMax = (highInputIdx > (ptrdiff_t)cctx->dictLimit) ? cctx->dictLimit : (U32)highInputIdx;
cctx->lowLimit = lowLimitMax; cctx->lowLimit = lowLimitMax;
@@ -2379,43 +2507,37 @@ static size_t ZSTD_compressContinue_internal (ZSTD_CCtx* cctx,
cctx->nextSrc = ip + srcSize; cctx->nextSrc = ip + srcSize;
if (srcSize) { if (srcSize) {
size_t const cSize = frame ? size_t const cSize = frame ? ZSTD_compress_generic(cctx, dst, dstCapacity, src, srcSize, lastFrameChunk)
ZSTD_compress_generic (cctx, dst, dstCapacity, src, srcSize, lastFrameChunk) : : ZSTD_compressBlock_internal(cctx, dst, dstCapacity, src, srcSize);
ZSTD_compressBlock_internal (cctx, dst, dstCapacity, src, srcSize); if (ZSTD_isError(cSize))
if (ZSTD_isError(cSize)) return cSize; return cSize;
return cSize + fhSize; return cSize + fhSize;
} else } else
return fhSize; return fhSize;
} }
size_t ZSTD_compressContinue(ZSTD_CCtx *cctx, void *dst, size_t dstCapacity, const void *src, size_t srcSize)
size_t ZSTD_compressContinue (ZSTD_CCtx* cctx,
void* dst, size_t dstCapacity,
const void* src, size_t srcSize)
{ {
return ZSTD_compressContinue_internal(cctx, dst, dstCapacity, src, srcSize, 1, 0); return ZSTD_compressContinue_internal(cctx, dst, dstCapacity, src, srcSize, 1, 0);
} }
size_t ZSTD_getBlockSizeMax(ZSTD_CCtx *cctx) { return MIN(ZSTD_BLOCKSIZE_ABSOLUTEMAX, 1 << cctx->params.cParams.windowLog); }
size_t ZSTD_getBlockSizeMax(ZSTD_CCtx* cctx) size_t ZSTD_compressBlock(ZSTD_CCtx *cctx, void *dst, size_t dstCapacity, const void *src, size_t srcSize)
{
return MIN (ZSTD_BLOCKSIZE_ABSOLUTEMAX, 1 << cctx->params.cParams.windowLog);
}
size_t ZSTD_compressBlock(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize)
{ {
size_t const blockSizeMax = ZSTD_getBlockSizeMax(cctx); size_t const blockSizeMax = ZSTD_getBlockSizeMax(cctx);
if (srcSize > blockSizeMax) return ERROR(srcSize_wrong); if (srcSize > blockSizeMax)
return ERROR(srcSize_wrong);
return ZSTD_compressContinue_internal(cctx, dst, dstCapacity, src, srcSize, 0, 0); return ZSTD_compressContinue_internal(cctx, dst, dstCapacity, src, srcSize, 0, 0);
} }
/*! ZSTD_loadDictionaryContent() : /*! ZSTD_loadDictionaryContent() :
* @return : 0, or an error code * @return : 0, or an error code
*/ */
static size_t ZSTD_loadDictionaryContent(ZSTD_CCtx* zc, const void* src, size_t srcSize) static size_t ZSTD_loadDictionaryContent(ZSTD_CCtx *zc, const void *src, size_t srcSize)
{ {
const BYTE* const ip = (const BYTE*) src; const BYTE *const ip = (const BYTE *)src;
const BYTE* const iend = ip + srcSize; const BYTE *const iend = ip + srcSize;
/* input becomes curr prefix */ /* input becomes curr prefix */
zc->lowLimit = zc->dictLimit; zc->lowLimit = zc->dictLimit;
@@ -2426,30 +2548,26 @@ static size_t ZSTD_loadDictionaryContent(ZSTD_CCtx* zc, const void* src, size_t
zc->loadedDictEnd = zc->forceWindow ? 0 : (U32)(iend - zc->base); zc->loadedDictEnd = zc->forceWindow ? 0 : (U32)(iend - zc->base);
zc->nextSrc = iend; zc->nextSrc = iend;
if (srcSize <= HASH_READ_SIZE) return 0; if (srcSize <= HASH_READ_SIZE)
return 0;
switch(zc->params.cParams.strategy) switch (zc->params.cParams.strategy) {
{ case ZSTD_fast: ZSTD_fillHashTable(zc, iend, zc->params.cParams.searchLength); break;
case ZSTD_fast:
ZSTD_fillHashTable (zc, iend, zc->params.cParams.searchLength);
break;
case ZSTD_dfast: case ZSTD_dfast: ZSTD_fillDoubleHashTable(zc, iend, zc->params.cParams.searchLength); break;
ZSTD_fillDoubleHashTable (zc, iend, zc->params.cParams.searchLength);
break;
case ZSTD_greedy: case ZSTD_greedy:
case ZSTD_lazy: case ZSTD_lazy:
case ZSTD_lazy2: case ZSTD_lazy2:
if (srcSize >= HASH_READ_SIZE) if (srcSize >= HASH_READ_SIZE)
ZSTD_insertAndFindFirstIndex(zc, iend-HASH_READ_SIZE, zc->params.cParams.searchLength); ZSTD_insertAndFindFirstIndex(zc, iend - HASH_READ_SIZE, zc->params.cParams.searchLength);
break; break;
case ZSTD_btlazy2: case ZSTD_btlazy2:
case ZSTD_btopt: case ZSTD_btopt:
case ZSTD_btopt2: case ZSTD_btopt2:
if (srcSize >= HASH_READ_SIZE) if (srcSize >= HASH_READ_SIZE)
ZSTD_updateTree(zc, iend-HASH_READ_SIZE, iend, 1 << zc->params.cParams.searchLog, zc->params.cParams.searchLength); ZSTD_updateTree(zc, iend - HASH_READ_SIZE, iend, 1 << zc->params.cParams.searchLog, zc->params.cParams.searchLength);
break; break;
default: default:
@@ -2460,21 +2578,22 @@ static size_t ZSTD_loadDictionaryContent(ZSTD_CCtx* zc, const void* src, size_t
return 0; return 0;
} }
/* Dictionaries that assign zero probability to symbols that show up causes problems /* Dictionaries that assign zero probability to symbols that show up causes problems
when FSE encoding. Refuse dictionaries that assign zero probability to symbols when FSE encoding. Refuse dictionaries that assign zero probability to symbols
that we may encounter during compression. that we may encounter during compression.
NOTE: This behavior is not standard and could be improved in the future. */ NOTE: This behavior is not standard and could be improved in the future. */
static size_t ZSTD_checkDictNCount(short* normalizedCounter, unsigned dictMaxSymbolValue, unsigned maxSymbolValue) { static size_t ZSTD_checkDictNCount(short *normalizedCounter, unsigned dictMaxSymbolValue, unsigned maxSymbolValue)
{
U32 s; U32 s;
if (dictMaxSymbolValue < maxSymbolValue) return ERROR(dictionary_corrupted); if (dictMaxSymbolValue < maxSymbolValue)
return ERROR(dictionary_corrupted);
for (s = 0; s <= maxSymbolValue; ++s) { for (s = 0; s <= maxSymbolValue; ++s) {
if (normalizedCounter[s] == 0) return ERROR(dictionary_corrupted); if (normalizedCounter[s] == 0)
return ERROR(dictionary_corrupted);
} }
return 0; return 0;
} }
/* Dictionary format : /* Dictionary format :
* See : * See :
* https://github.com/facebook/zstd/blob/master/doc/zstd_compression_format.md#dictionary-format * https://github.com/facebook/zstd/blob/master/doc/zstd_compression_format.md#dictionary-format
@@ -2484,74 +2603,95 @@ static size_t ZSTD_checkDictNCount(short* normalizedCounter, unsigned dictMaxSym
* assumptions : magic number supposed already checked * assumptions : magic number supposed already checked
* dictSize supposed > 8 * dictSize supposed > 8
*/ */
static size_t ZSTD_loadZstdDictionary(ZSTD_CCtx* cctx, const void* dict, size_t dictSize) static size_t ZSTD_loadZstdDictionary(ZSTD_CCtx *cctx, const void *dict, size_t dictSize)
{ {
const BYTE* dictPtr = (const BYTE*)dict; const BYTE *dictPtr = (const BYTE *)dict;
const BYTE* const dictEnd = dictPtr + dictSize; const BYTE *const dictEnd = dictPtr + dictSize;
short offcodeNCount[MaxOff+1]; short offcodeNCount[MaxOff + 1];
unsigned offcodeMaxValue = MaxOff; unsigned offcodeMaxValue = MaxOff;
BYTE scratchBuffer[1<<MAX(MLFSELog,LLFSELog)]; BYTE scratchBuffer[1 << MAX(MLFSELog, LLFSELog)];
dictPtr += 4; /* skip magic number */ dictPtr += 4; /* skip magic number */
cctx->dictID = cctx->params.fParams.noDictIDFlag ? 0 : ZSTD_readLE32(dictPtr); cctx->dictID = cctx->params.fParams.noDictIDFlag ? 0 : ZSTD_readLE32(dictPtr);
dictPtr += 4; dictPtr += 4;
{ size_t const hufHeaderSize = HUF_readCTable(cctx->hufTable, 255, dictPtr, dictEnd-dictPtr); {
if (HUF_isError(hufHeaderSize)) return ERROR(dictionary_corrupted); size_t const hufHeaderSize = HUF_readCTable(cctx->hufTable, 255, dictPtr, dictEnd - dictPtr);
if (HUF_isError(hufHeaderSize))
return ERROR(dictionary_corrupted);
dictPtr += hufHeaderSize; dictPtr += hufHeaderSize;
} }
{ unsigned offcodeLog; {
size_t const offcodeHeaderSize = FSE_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dictPtr, dictEnd-dictPtr); unsigned offcodeLog;
if (FSE_isError(offcodeHeaderSize)) return ERROR(dictionary_corrupted); size_t const offcodeHeaderSize = FSE_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dictPtr, dictEnd - dictPtr);
if (offcodeLog > OffFSELog) return ERROR(dictionary_corrupted); if (FSE_isError(offcodeHeaderSize))
return ERROR(dictionary_corrupted);
if (offcodeLog > OffFSELog)
return ERROR(dictionary_corrupted);
/* Defer checking offcodeMaxValue because we need to know the size of the dictionary content */ /* Defer checking offcodeMaxValue because we need to know the size of the dictionary content */
CHECK_E (FSE_buildCTable_wksp(cctx->offcodeCTable, offcodeNCount, offcodeMaxValue, offcodeLog, scratchBuffer, sizeof(scratchBuffer)), dictionary_corrupted); CHECK_E(FSE_buildCTable_wksp(cctx->offcodeCTable, offcodeNCount, offcodeMaxValue, offcodeLog, scratchBuffer, sizeof(scratchBuffer)),
dictionary_corrupted);
dictPtr += offcodeHeaderSize; dictPtr += offcodeHeaderSize;
} }
{ short matchlengthNCount[MaxML+1]; {
short matchlengthNCount[MaxML + 1];
unsigned matchlengthMaxValue = MaxML, matchlengthLog; unsigned matchlengthMaxValue = MaxML, matchlengthLog;
size_t const matchlengthHeaderSize = FSE_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dictPtr, dictEnd-dictPtr); size_t const matchlengthHeaderSize = FSE_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dictPtr, dictEnd - dictPtr);
if (FSE_isError(matchlengthHeaderSize)) return ERROR(dictionary_corrupted); if (FSE_isError(matchlengthHeaderSize))
if (matchlengthLog > MLFSELog) return ERROR(dictionary_corrupted); return ERROR(dictionary_corrupted);
if (matchlengthLog > MLFSELog)
return ERROR(dictionary_corrupted);
/* Every match length code must have non-zero probability */ /* Every match length code must have non-zero probability */
CHECK_F (ZSTD_checkDictNCount(matchlengthNCount, matchlengthMaxValue, MaxML)); CHECK_F(ZSTD_checkDictNCount(matchlengthNCount, matchlengthMaxValue, MaxML));
CHECK_E (FSE_buildCTable_wksp(cctx->matchlengthCTable, matchlengthNCount, matchlengthMaxValue, matchlengthLog, scratchBuffer, sizeof(scratchBuffer)), dictionary_corrupted); CHECK_E(
FSE_buildCTable_wksp(cctx->matchlengthCTable, matchlengthNCount, matchlengthMaxValue, matchlengthLog, scratchBuffer, sizeof(scratchBuffer)),
dictionary_corrupted);
dictPtr += matchlengthHeaderSize; dictPtr += matchlengthHeaderSize;
} }
{ short litlengthNCount[MaxLL+1]; {
short litlengthNCount[MaxLL + 1];
unsigned litlengthMaxValue = MaxLL, litlengthLog; unsigned litlengthMaxValue = MaxLL, litlengthLog;
size_t const litlengthHeaderSize = FSE_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dictPtr, dictEnd-dictPtr); size_t const litlengthHeaderSize = FSE_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dictPtr, dictEnd - dictPtr);
if (FSE_isError(litlengthHeaderSize)) return ERROR(dictionary_corrupted); if (FSE_isError(litlengthHeaderSize))
if (litlengthLog > LLFSELog) return ERROR(dictionary_corrupted); return ERROR(dictionary_corrupted);
if (litlengthLog > LLFSELog)
return ERROR(dictionary_corrupted);
/* Every literal length code must have non-zero probability */ /* Every literal length code must have non-zero probability */
CHECK_F (ZSTD_checkDictNCount(litlengthNCount, litlengthMaxValue, MaxLL)); CHECK_F(ZSTD_checkDictNCount(litlengthNCount, litlengthMaxValue, MaxLL));
CHECK_E(FSE_buildCTable_wksp(cctx->litlengthCTable, litlengthNCount, litlengthMaxValue, litlengthLog, scratchBuffer, sizeof(scratchBuffer)), dictionary_corrupted); CHECK_E(FSE_buildCTable_wksp(cctx->litlengthCTable, litlengthNCount, litlengthMaxValue, litlengthLog, scratchBuffer, sizeof(scratchBuffer)),
dictionary_corrupted);
dictPtr += litlengthHeaderSize; dictPtr += litlengthHeaderSize;
} }
if (dictPtr+12 > dictEnd) return ERROR(dictionary_corrupted); if (dictPtr + 12 > dictEnd)
cctx->rep[0] = ZSTD_readLE32(dictPtr+0); return ERROR(dictionary_corrupted);
cctx->rep[1] = ZSTD_readLE32(dictPtr+4); cctx->rep[0] = ZSTD_readLE32(dictPtr + 0);
cctx->rep[2] = ZSTD_readLE32(dictPtr+8); cctx->rep[1] = ZSTD_readLE32(dictPtr + 4);
cctx->rep[2] = ZSTD_readLE32(dictPtr + 8);
dictPtr += 12; dictPtr += 12;
{ size_t const dictContentSize = (size_t)(dictEnd - dictPtr); {
size_t const dictContentSize = (size_t)(dictEnd - dictPtr);
U32 offcodeMax = MaxOff; U32 offcodeMax = MaxOff;
if (dictContentSize <= ((U32)-1) - 128 KB) { if (dictContentSize <= ((U32)-1) - 128 KB) {
U32 const maxOffset = (U32)dictContentSize + 128 KB; /* The maximum offset that must be supported */ U32 const maxOffset = (U32)dictContentSize + 128 KB; /* The maximum offset that must be supported */
offcodeMax = ZSTD_highbit32(maxOffset); /* Calculate minimum offset code required to represent maxOffset */ offcodeMax = ZSTD_highbit32(maxOffset); /* Calculate minimum offset code required to represent maxOffset */
} }
/* All offset values <= dictContentSize + 128 KB must be representable */ /* All offset values <= dictContentSize + 128 KB must be representable */
CHECK_F (ZSTD_checkDictNCount(offcodeNCount, offcodeMaxValue, MIN(offcodeMax, MaxOff))); CHECK_F(ZSTD_checkDictNCount(offcodeNCount, offcodeMaxValue, MIN(offcodeMax, MaxOff)));
/* All repCodes must be <= dictContentSize and != 0*/ /* All repCodes must be <= dictContentSize and != 0*/
{ U32 u; {
for (u=0; u<3; u++) { U32 u;
if (cctx->rep[u] == 0) return ERROR(dictionary_corrupted); for (u = 0; u < 3; u++) {
if (cctx->rep[u] > dictContentSize) return ERROR(dictionary_corrupted); if (cctx->rep[u] == 0)
} } return ERROR(dictionary_corrupted);
if (cctx->rep[u] > dictContentSize)
return ERROR(dictionary_corrupted);
}
}
cctx->flagStaticTables = 1; cctx->flagStaticTables = 1;
cctx->flagStaticHufTable = HUF_repeat_valid; cctx->flagStaticHufTable = HUF_repeat_valid;
@@ -2561,9 +2701,10 @@ static size_t ZSTD_loadZstdDictionary(ZSTD_CCtx* cctx, const void* dict, size_t
/** ZSTD_compress_insertDictionary() : /** ZSTD_compress_insertDictionary() :
* @return : 0, or an error code */ * @return : 0, or an error code */
static size_t ZSTD_compress_insertDictionary(ZSTD_CCtx* cctx, const void* dict, size_t dictSize) static size_t ZSTD_compress_insertDictionary(ZSTD_CCtx *cctx, const void *dict, size_t dictSize)
{ {
if ((dict==NULL) || (dictSize<=8)) return 0; if ((dict == NULL) || (dictSize <= 8))
return 0;
/* dict as pure content */ /* dict as pure content */
if ((ZSTD_readLE32(dict) != ZSTD_DICT_MAGIC) || (cctx->forceRawDict)) if ((ZSTD_readLE32(dict) != ZSTD_DICT_MAGIC) || (cctx->forceRawDict))
@@ -2575,56 +2716,47 @@ static size_t ZSTD_compress_insertDictionary(ZSTD_CCtx* cctx, const void* dict,
/*! ZSTD_compressBegin_internal() : /*! ZSTD_compressBegin_internal() :
* @return : 0, or an error code */ * @return : 0, or an error code */
static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* cctx, static size_t ZSTD_compressBegin_internal(ZSTD_CCtx *cctx, const void *dict, size_t dictSize, ZSTD_parameters params, U64 pledgedSrcSize)
const void* dict, size_t dictSize,
ZSTD_parameters params, U64 pledgedSrcSize)
{ {
ZSTD_compResetPolicy_e const crp = dictSize ? ZSTDcrp_fullReset : ZSTDcrp_continue; ZSTD_compResetPolicy_e const crp = dictSize ? ZSTDcrp_fullReset : ZSTDcrp_continue;
CHECK_F(ZSTD_resetCCtx_advanced(cctx, params, pledgedSrcSize, crp)); CHECK_F(ZSTD_resetCCtx_advanced(cctx, params, pledgedSrcSize, crp));
return ZSTD_compress_insertDictionary(cctx, dict, dictSize); return ZSTD_compress_insertDictionary(cctx, dict, dictSize);
} }
/*! ZSTD_compressBegin_advanced() : /*! ZSTD_compressBegin_advanced() :
* @return : 0, or an error code */ * @return : 0, or an error code */
size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, size_t ZSTD_compressBegin_advanced(ZSTD_CCtx *cctx, const void *dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize)
const void* dict, size_t dictSize,
ZSTD_parameters params, unsigned long long pledgedSrcSize)
{ {
/* compression parameters verification and optimization */ /* compression parameters verification and optimization */
CHECK_F(ZSTD_checkCParams(params.cParams)); CHECK_F(ZSTD_checkCParams(params.cParams));
return ZSTD_compressBegin_internal(cctx, dict, dictSize, params, pledgedSrcSize); return ZSTD_compressBegin_internal(cctx, dict, dictSize, params, pledgedSrcSize);
} }
size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx *cctx, const void *dict, size_t dictSize, int compressionLevel)
size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel)
{ {
ZSTD_parameters const params = ZSTD_getParams(compressionLevel, 0, dictSize); ZSTD_parameters const params = ZSTD_getParams(compressionLevel, 0, dictSize);
return ZSTD_compressBegin_internal(cctx, dict, dictSize, params, 0); return ZSTD_compressBegin_internal(cctx, dict, dictSize, params, 0);
} }
size_t ZSTD_compressBegin(ZSTD_CCtx *cctx, int compressionLevel) { return ZSTD_compressBegin_usingDict(cctx, NULL, 0, compressionLevel); }
size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel)
{
return ZSTD_compressBegin_usingDict(cctx, NULL, 0, compressionLevel);
}
/*! ZSTD_writeEpilogue() : /*! ZSTD_writeEpilogue() :
* Ends a frame. * Ends a frame.
* @return : nb of bytes written into dst (or an error code) */ * @return : nb of bytes written into dst (or an error code) */
static size_t ZSTD_writeEpilogue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity) static size_t ZSTD_writeEpilogue(ZSTD_CCtx *cctx, void *dst, size_t dstCapacity)
{ {
BYTE* const ostart = (BYTE*)dst; BYTE *const ostart = (BYTE *)dst;
BYTE* op = ostart; BYTE *op = ostart;
size_t fhSize = 0; size_t fhSize = 0;
if (cctx->stage == ZSTDcs_created) return ERROR(stage_wrong); /* init missing */ if (cctx->stage == ZSTDcs_created)
return ERROR(stage_wrong); /* init missing */
/* special case : empty frame */ /* special case : empty frame */
if (cctx->stage == ZSTDcs_init) { if (cctx->stage == ZSTDcs_init) {
fhSize = ZSTD_writeFrameHeader(dst, dstCapacity, cctx->params, 0, 0); fhSize = ZSTD_writeFrameHeader(dst, dstCapacity, cctx->params, 0, 0);
if (ZSTD_isError(fhSize)) return fhSize; if (ZSTD_isError(fhSize))
return fhSize;
dstCapacity -= fhSize; dstCapacity -= fhSize;
op += fhSize; op += fhSize;
cctx->stage = ZSTDcs_ongoing; cctx->stage = ZSTDcs_ongoing;
@@ -2632,81 +2764,75 @@ static size_t ZSTD_writeEpilogue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity)
if (cctx->stage != ZSTDcs_ending) { if (cctx->stage != ZSTDcs_ending) {
/* write one last empty block, make it the "last" block */ /* write one last empty block, make it the "last" block */
U32 const cBlockHeader24 = 1 /* last block */ + (((U32)bt_raw)<<1) + 0; U32 const cBlockHeader24 = 1 /* last block */ + (((U32)bt_raw) << 1) + 0;
if (dstCapacity<4) return ERROR(dstSize_tooSmall); if (dstCapacity < 4)
return ERROR(dstSize_tooSmall);
ZSTD_writeLE32(op, cBlockHeader24); ZSTD_writeLE32(op, cBlockHeader24);
op += ZSTD_blockHeaderSize; op += ZSTD_blockHeaderSize;
dstCapacity -= ZSTD_blockHeaderSize; dstCapacity -= ZSTD_blockHeaderSize;
} }
if (cctx->params.fParams.checksumFlag) { if (cctx->params.fParams.checksumFlag) {
U32 const checksum = (U32) xxh64_digest(&cctx->xxhState); U32 const checksum = (U32)xxh64_digest(&cctx->xxhState);
if (dstCapacity<4) return ERROR(dstSize_tooSmall); if (dstCapacity < 4)
return ERROR(dstSize_tooSmall);
ZSTD_writeLE32(op, checksum); ZSTD_writeLE32(op, checksum);
op += 4; op += 4;
} }
cctx->stage = ZSTDcs_created; /* return to "created but no init" status */ cctx->stage = ZSTDcs_created; /* return to "created but no init" status */
return op-ostart; return op - ostart;
} }
size_t ZSTD_compressEnd(ZSTD_CCtx *cctx, void *dst, size_t dstCapacity, const void *src, size_t srcSize)
size_t ZSTD_compressEnd (ZSTD_CCtx* cctx,
void* dst, size_t dstCapacity,
const void* src, size_t srcSize)
{ {
size_t endResult; size_t endResult;
size_t const cSize = ZSTD_compressContinue_internal(cctx, dst, dstCapacity, src, srcSize, 1, 1); size_t const cSize = ZSTD_compressContinue_internal(cctx, dst, dstCapacity, src, srcSize, 1, 1);
if (ZSTD_isError(cSize)) return cSize; if (ZSTD_isError(cSize))
endResult = ZSTD_writeEpilogue(cctx, (char*)dst + cSize, dstCapacity-cSize); return cSize;
if (ZSTD_isError(endResult)) return endResult; endResult = ZSTD_writeEpilogue(cctx, (char *)dst + cSize, dstCapacity - cSize);
if (ZSTD_isError(endResult))
return endResult;
return cSize + endResult; return cSize + endResult;
} }
static size_t ZSTD_compress_internal(ZSTD_CCtx *cctx, void *dst, size_t dstCapacity, const void *src, size_t srcSize, const void *dict, size_t dictSize,
static size_t ZSTD_compress_internal (ZSTD_CCtx* cctx,
void* dst, size_t dstCapacity,
const void* src, size_t srcSize,
const void* dict,size_t dictSize,
ZSTD_parameters params) ZSTD_parameters params)
{ {
CHECK_F(ZSTD_compressBegin_internal(cctx, dict, dictSize, params, srcSize)); CHECK_F(ZSTD_compressBegin_internal(cctx, dict, dictSize, params, srcSize));
return ZSTD_compressEnd(cctx, dst, dstCapacity, src, srcSize); return ZSTD_compressEnd(cctx, dst, dstCapacity, src, srcSize);
} }
size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, const void* dict, size_t dictSize, ZSTD_parameters params) size_t ZSTD_compress_usingDict(ZSTD_CCtx *ctx, void *dst, size_t dstCapacity, const void *src, size_t srcSize, const void *dict, size_t dictSize,
ZSTD_parameters params)
{ {
return ZSTD_compress_internal(ctx, dst, dstCapacity, src, srcSize, dict, dictSize, params); return ZSTD_compress_internal(ctx, dst, dstCapacity, src, srcSize, dict, dictSize, params);
} }
size_t ZSTD_compressCCtx(ZSTD_CCtx *ctx, void *dst, size_t dstCapacity, const void *src, size_t srcSize, ZSTD_parameters params)
size_t ZSTD_compressCCtx(ZSTD_CCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, ZSTD_parameters params)
{ {
return ZSTD_compress_internal(ctx, dst, dstCapacity, src, srcSize, NULL, 0, params); return ZSTD_compress_internal(ctx, dst, dstCapacity, src, srcSize, NULL, 0, params);
} }
/* ===== Dictionary API ===== */ /* ===== Dictionary API ===== */
struct ZSTD_CDict_s { struct ZSTD_CDict_s {
void* dictBuffer; void *dictBuffer;
const void* dictContent; const void *dictContent;
size_t dictContentSize; size_t dictContentSize;
ZSTD_CCtx* refContext; ZSTD_CCtx *refContext;
}; /* typedef'd tp ZSTD_CDict within "zstd.h" */ }; /* typedef'd tp ZSTD_CDict within "zstd.h" */
size_t ZSTD_CDictWorkspaceBound(ZSTD_compressionParameters cParams) size_t ZSTD_CDictWorkspaceBound(ZSTD_compressionParameters cParams) { return ZSTD_CCtxWorkspaceBound(cParams) + ZSTD_ALIGN(sizeof(ZSTD_CDict)); }
{
return ZSTD_CCtxWorkspaceBound(cParams) + ZSTD_ALIGN(sizeof(ZSTD_CDict));
}
static ZSTD_CDict* ZSTD_createCDict_advanced(const void* dictBuffer, size_t dictSize, unsigned byReference, static ZSTD_CDict *ZSTD_createCDict_advanced(const void *dictBuffer, size_t dictSize, unsigned byReference, ZSTD_parameters params, ZSTD_customMem customMem)
ZSTD_parameters params, ZSTD_customMem customMem)
{ {
if (!customMem.customAlloc || !customMem.customFree) return NULL; if (!customMem.customAlloc || !customMem.customFree)
return NULL;
{ ZSTD_CDict* const cdict = (ZSTD_CDict*) ZSTD_malloc(sizeof(ZSTD_CDict), customMem); {
ZSTD_CCtx* const cctx = ZSTD_createCCtx_advanced(customMem); ZSTD_CDict *const cdict = (ZSTD_CDict *)ZSTD_malloc(sizeof(ZSTD_CDict), customMem);
ZSTD_CCtx *const cctx = ZSTD_createCCtx_advanced(customMem);
if (!cdict || !cctx) { if (!cdict || !cctx) {
ZSTD_free(cdict, customMem); ZSTD_free(cdict, customMem);
@@ -2718,20 +2844,26 @@ static ZSTD_CDict* ZSTD_createCDict_advanced(const void* dictBuffer, size_t dict
cdict->dictBuffer = NULL; cdict->dictBuffer = NULL;
cdict->dictContent = dictBuffer; cdict->dictContent = dictBuffer;
} else { } else {
void* const internalBuffer = ZSTD_malloc(dictSize, customMem); void *const internalBuffer = ZSTD_malloc(dictSize, customMem);
if (!internalBuffer) { ZSTD_free(cctx, customMem); ZSTD_free(cdict, customMem); return NULL; } if (!internalBuffer) {
ZSTD_free(cctx, customMem);
ZSTD_free(cdict, customMem);
return NULL;
}
memcpy(internalBuffer, dictBuffer, dictSize); memcpy(internalBuffer, dictBuffer, dictSize);
cdict->dictBuffer = internalBuffer; cdict->dictBuffer = internalBuffer;
cdict->dictContent = internalBuffer; cdict->dictContent = internalBuffer;
} }
{ size_t const errorCode = ZSTD_compressBegin_advanced(cctx, cdict->dictContent, dictSize, params, 0); {
size_t const errorCode = ZSTD_compressBegin_advanced(cctx, cdict->dictContent, dictSize, params, 0);
if (ZSTD_isError(errorCode)) { if (ZSTD_isError(errorCode)) {
ZSTD_free(cdict->dictBuffer, customMem); ZSTD_free(cdict->dictBuffer, customMem);
ZSTD_free(cdict, customMem); ZSTD_free(cdict, customMem);
ZSTD_freeCCtx(cctx); ZSTD_freeCCtx(cctx);
return NULL; return NULL;
} } }
}
cdict->refContext = cctx; cdict->refContext = cctx;
cdict->dictContentSize = dictSize; cdict->dictContentSize = dictSize;
@@ -2739,16 +2871,18 @@ static ZSTD_CDict* ZSTD_createCDict_advanced(const void* dictBuffer, size_t dict
} }
} }
ZSTD_CDict* ZSTD_initCDict(const void* dict, size_t dictSize, ZSTD_parameters params, void* workspace, size_t workspaceSize) ZSTD_CDict *ZSTD_initCDict(const void *dict, size_t dictSize, ZSTD_parameters params, void *workspace, size_t workspaceSize)
{ {
ZSTD_customMem const stackMem = ZSTD_initStack(workspace, workspaceSize); ZSTD_customMem const stackMem = ZSTD_initStack(workspace, workspaceSize);
return ZSTD_createCDict_advanced(dict, dictSize, 1, params, stackMem); return ZSTD_createCDict_advanced(dict, dictSize, 1, params, stackMem);
} }
size_t ZSTD_freeCDict(ZSTD_CDict* cdict) size_t ZSTD_freeCDict(ZSTD_CDict *cdict)
{ {
if (cdict==NULL) return 0; /* support free on NULL */ if (cdict == NULL)
{ ZSTD_customMem const cMem = cdict->refContext->customMem; return 0; /* support free on NULL */
{
ZSTD_customMem const cMem = cdict->refContext->customMem;
ZSTD_freeCCtx(cdict->refContext); ZSTD_freeCCtx(cdict->refContext);
ZSTD_free(cdict->dictBuffer, cMem); ZSTD_free(cdict->dictBuffer, cMem);
ZSTD_free(cdict, cMem); ZSTD_free(cdict, cMem);
@@ -2756,13 +2890,12 @@ size_t ZSTD_freeCDict(ZSTD_CDict* cdict)
} }
} }
static ZSTD_parameters ZSTD_getParamsFromCDict(const ZSTD_CDict* cdict) { static ZSTD_parameters ZSTD_getParamsFromCDict(const ZSTD_CDict *cdict) { return ZSTD_getParamsFromCCtx(cdict->refContext); }
return ZSTD_getParamsFromCCtx(cdict->refContext);
}
size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict, unsigned long long pledgedSrcSize) size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx *cctx, const ZSTD_CDict *cdict, unsigned long long pledgedSrcSize)
{ {
if (cdict->dictContentSize) CHECK_F(ZSTD_copyCCtx(cctx, cdict->refContext, pledgedSrcSize)) if (cdict->dictContentSize)
CHECK_F(ZSTD_copyCCtx(cctx, cdict->refContext, pledgedSrcSize))
else { else {
ZSTD_parameters params = cdict->refContext->params; ZSTD_parameters params = cdict->refContext->params;
params.fParams.contentSizeFlag = (pledgedSrcSize > 0); params.fParams.contentSizeFlag = (pledgedSrcSize > 0);
@@ -2775,14 +2908,11 @@ size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict, u
* Compression using a digested Dictionary. * Compression using a digested Dictionary.
* Faster startup than ZSTD_compress_usingDict(), recommended when same dictionary is used multiple times. * Faster startup than ZSTD_compress_usingDict(), recommended when same dictionary is used multiple times.
* Note that compression level is decided during dictionary creation */ * Note that compression level is decided during dictionary creation */
size_t ZSTD_compress_usingCDict(ZSTD_CCtx* cctx, size_t ZSTD_compress_usingCDict(ZSTD_CCtx *cctx, void *dst, size_t dstCapacity, const void *src, size_t srcSize, const ZSTD_CDict *cdict)
void* dst, size_t dstCapacity,
const void* src, size_t srcSize,
const ZSTD_CDict* cdict)
{ {
CHECK_F(ZSTD_compressBegin_usingCDict(cctx, cdict, srcSize)); CHECK_F(ZSTD_compressBegin_usingCDict(cctx, cdict, srcSize));
if (cdict->refContext->params.fParams.contentSizeFlag==1) { if (cdict->refContext->params.fParams.contentSizeFlag == 1) {
cctx->params.fParams.contentSizeFlag = 1; cctx->params.fParams.contentSizeFlag = 1;
cctx->frameContentSize = srcSize; cctx->frameContentSize = srcSize;
} else { } else {
@@ -2792,8 +2922,6 @@ size_t ZSTD_compress_usingCDict(ZSTD_CCtx* cctx,
return ZSTD_compressEnd(cctx, dst, dstCapacity, src, srcSize); return ZSTD_compressEnd(cctx, dst, dstCapacity, src, srcSize);
} }
/* ****************************************************************** /* ******************************************************************
* Streaming * Streaming
********************************************************************/ ********************************************************************/
@@ -2801,16 +2929,16 @@ size_t ZSTD_compress_usingCDict(ZSTD_CCtx* cctx,
typedef enum { zcss_init, zcss_load, zcss_flush, zcss_final } ZSTD_cStreamStage; typedef enum { zcss_init, zcss_load, zcss_flush, zcss_final } ZSTD_cStreamStage;
struct ZSTD_CStream_s { struct ZSTD_CStream_s {
ZSTD_CCtx* cctx; ZSTD_CCtx *cctx;
ZSTD_CDict* cdictLocal; ZSTD_CDict *cdictLocal;
const ZSTD_CDict* cdict; const ZSTD_CDict *cdict;
char* inBuff; char *inBuff;
size_t inBuffSize; size_t inBuffSize;
size_t inToCompress; size_t inToCompress;
size_t inBuffPos; size_t inBuffPos;
size_t inBuffTarget; size_t inBuffTarget;
size_t blockSize; size_t blockSize;
char* outBuff; char *outBuff;
size_t outBuffSize; size_t outBuffSize;
size_t outBuffContentSize; size_t outBuffContentSize;
size_t outBuffFlushedSize; size_t outBuffFlushedSize;
@@ -2832,25 +2960,32 @@ size_t ZSTD_CStreamWorkspaceBound(ZSTD_compressionParameters cParams)
return ZSTD_CCtxWorkspaceBound(cParams) + ZSTD_ALIGN(sizeof(ZSTD_CStream)) + ZSTD_ALIGN(inBuffSize) + ZSTD_ALIGN(outBuffSize); return ZSTD_CCtxWorkspaceBound(cParams) + ZSTD_ALIGN(sizeof(ZSTD_CStream)) + ZSTD_ALIGN(inBuffSize) + ZSTD_ALIGN(outBuffSize);
} }
ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem) ZSTD_CStream *ZSTD_createCStream_advanced(ZSTD_customMem customMem)
{ {
ZSTD_CStream* zcs; ZSTD_CStream *zcs;
if (!customMem.customAlloc || !customMem.customFree) return NULL; if (!customMem.customAlloc || !customMem.customFree)
return NULL;
zcs = (ZSTD_CStream*)ZSTD_malloc(sizeof(ZSTD_CStream), customMem); zcs = (ZSTD_CStream *)ZSTD_malloc(sizeof(ZSTD_CStream), customMem);
if (zcs==NULL) return NULL; if (zcs == NULL)
return NULL;
memset(zcs, 0, sizeof(ZSTD_CStream)); memset(zcs, 0, sizeof(ZSTD_CStream));
memcpy(&zcs->customMem, &customMem, sizeof(ZSTD_customMem)); memcpy(&zcs->customMem, &customMem, sizeof(ZSTD_customMem));
zcs->cctx = ZSTD_createCCtx_advanced(customMem); zcs->cctx = ZSTD_createCCtx_advanced(customMem);
if (zcs->cctx == NULL) { ZSTD_freeCStream(zcs); return NULL; } if (zcs->cctx == NULL) {
ZSTD_freeCStream(zcs);
return NULL;
}
return zcs; return zcs;
} }
size_t ZSTD_freeCStream(ZSTD_CStream* zcs) size_t ZSTD_freeCStream(ZSTD_CStream *zcs)
{ {
if (zcs==NULL) return 0; /* support free on NULL */ if (zcs == NULL)
{ ZSTD_customMem const cMem = zcs->customMem; return 0; /* support free on NULL */
{
ZSTD_customMem const cMem = zcs->customMem;
ZSTD_freeCCtx(zcs->cctx); ZSTD_freeCCtx(zcs->cctx);
zcs->cctx = NULL; zcs->cctx = NULL;
ZSTD_freeCDict(zcs->cdictLocal); ZSTD_freeCDict(zcs->cdictLocal);
@@ -2864,18 +2999,20 @@ size_t ZSTD_freeCStream(ZSTD_CStream* zcs)
} }
} }
/*====== Initialization ======*/ /*====== Initialization ======*/
size_t ZSTD_CStreamInSize(void) { return ZSTD_BLOCKSIZE_ABSOLUTEMAX; } size_t ZSTD_CStreamInSize(void) { return ZSTD_BLOCKSIZE_ABSOLUTEMAX; }
size_t ZSTD_CStreamOutSize(void) { return ZSTD_compressBound(ZSTD_BLOCKSIZE_ABSOLUTEMAX) + ZSTD_blockHeaderSize + 4 /* 32-bits hash */ ; } size_t ZSTD_CStreamOutSize(void) { return ZSTD_compressBound(ZSTD_BLOCKSIZE_ABSOLUTEMAX) + ZSTD_blockHeaderSize + 4 /* 32-bits hash */; }
static size_t ZSTD_resetCStream_internal(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize) static size_t ZSTD_resetCStream_internal(ZSTD_CStream *zcs, unsigned long long pledgedSrcSize)
{ {
if (zcs->inBuffSize==0) return ERROR(stage_wrong); /* zcs has not been init at least once => can't reset */ if (zcs->inBuffSize == 0)
return ERROR(stage_wrong); /* zcs has not been init at least once => can't reset */
if (zcs->cdict) CHECK_F(ZSTD_compressBegin_usingCDict(zcs->cctx, zcs->cdict, pledgedSrcSize)) if (zcs->cdict)
else CHECK_F(ZSTD_compressBegin_advanced(zcs->cctx, NULL, 0, zcs->params, pledgedSrcSize)); CHECK_F(ZSTD_compressBegin_usingCDict(zcs->cctx, zcs->cdict, pledgedSrcSize))
else
CHECK_F(ZSTD_compressBegin_advanced(zcs->cctx, NULL, 0, zcs->params, pledgedSrcSize));
zcs->inToCompress = 0; zcs->inToCompress = 0;
zcs->inBuffPos = 0; zcs->inBuffPos = 0;
@@ -2888,7 +3025,7 @@ static size_t ZSTD_resetCStream_internal(ZSTD_CStream* zcs, unsigned long long p
return 0; /* ready to go */ return 0; /* ready to go */
} }
size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize) size_t ZSTD_resetCStream(ZSTD_CStream *zcs, unsigned long long pledgedSrcSize)
{ {
zcs->params.fParams.contentSizeFlag = (pledgedSrcSize > 0); zcs->params.fParams.contentSizeFlag = (pledgedSrcSize > 0);
@@ -2896,33 +3033,36 @@ size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize)
return ZSTD_resetCStream_internal(zcs, pledgedSrcSize); return ZSTD_resetCStream_internal(zcs, pledgedSrcSize);
} }
static size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, static size_t ZSTD_initCStream_advanced(ZSTD_CStream *zcs, const void *dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize)
const void* dict, size_t dictSize,
ZSTD_parameters params, unsigned long long pledgedSrcSize)
{ {
/* allocate buffers */ /* allocate buffers */
{ size_t const neededInBuffSize = (size_t)1 << params.cParams.windowLog; {
size_t const neededInBuffSize = (size_t)1 << params.cParams.windowLog;
if (zcs->inBuffSize < neededInBuffSize) { if (zcs->inBuffSize < neededInBuffSize) {
zcs->inBuffSize = neededInBuffSize; zcs->inBuffSize = neededInBuffSize;
ZSTD_free(zcs->inBuff, zcs->customMem); ZSTD_free(zcs->inBuff, zcs->customMem);
zcs->inBuff = (char*) ZSTD_malloc(neededInBuffSize, zcs->customMem); zcs->inBuff = (char *)ZSTD_malloc(neededInBuffSize, zcs->customMem);
if (zcs->inBuff == NULL) return ERROR(memory_allocation); if (zcs->inBuff == NULL)
return ERROR(memory_allocation);
} }
zcs->blockSize = MIN(ZSTD_BLOCKSIZE_ABSOLUTEMAX, neededInBuffSize); zcs->blockSize = MIN(ZSTD_BLOCKSIZE_ABSOLUTEMAX, neededInBuffSize);
} }
if (zcs->outBuffSize < ZSTD_compressBound(zcs->blockSize)+1) { if (zcs->outBuffSize < ZSTD_compressBound(zcs->blockSize) + 1) {
zcs->outBuffSize = ZSTD_compressBound(zcs->blockSize)+1; zcs->outBuffSize = ZSTD_compressBound(zcs->blockSize) + 1;
ZSTD_free(zcs->outBuff, zcs->customMem); ZSTD_free(zcs->outBuff, zcs->customMem);
zcs->outBuff = (char*) ZSTD_malloc(zcs->outBuffSize, zcs->customMem); zcs->outBuff = (char *)ZSTD_malloc(zcs->outBuffSize, zcs->customMem);
if (zcs->outBuff == NULL) return ERROR(memory_allocation); if (zcs->outBuff == NULL)
return ERROR(memory_allocation);
} }
if (dict && dictSize >= 8) { if (dict && dictSize >= 8) {
ZSTD_freeCDict(zcs->cdictLocal); ZSTD_freeCDict(zcs->cdictLocal);
zcs->cdictLocal = ZSTD_createCDict_advanced(dict, dictSize, 0, params, zcs->customMem); zcs->cdictLocal = ZSTD_createCDict_advanced(dict, dictSize, 0, params, zcs->customMem);
if (zcs->cdictLocal == NULL) return ERROR(memory_allocation); if (zcs->cdictLocal == NULL)
return ERROR(memory_allocation);
zcs->cdict = zcs->cdictLocal; zcs->cdict = zcs->cdictLocal;
} else zcs->cdict = NULL; } else
zcs->cdict = NULL;
zcs->checksum = params.fParams.checksumFlag > 0; zcs->checksum = params.fParams.checksumFlag > 0;
zcs->params = params; zcs->params = params;
@@ -2930,21 +3070,23 @@ static size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs,
return ZSTD_resetCStream_internal(zcs, pledgedSrcSize); return ZSTD_resetCStream_internal(zcs, pledgedSrcSize);
} }
ZSTD_CStream* ZSTD_initCStream(ZSTD_parameters params, unsigned long long pledgedSrcSize, void* workspace, size_t workspaceSize) ZSTD_CStream *ZSTD_initCStream(ZSTD_parameters params, unsigned long long pledgedSrcSize, void *workspace, size_t workspaceSize)
{ {
ZSTD_customMem const stackMem = ZSTD_initStack(workspace, workspaceSize); ZSTD_customMem const stackMem = ZSTD_initStack(workspace, workspaceSize);
ZSTD_CStream* const zcs = ZSTD_createCStream_advanced(stackMem); ZSTD_CStream *const zcs = ZSTD_createCStream_advanced(stackMem);
if (zcs) { if (zcs) {
size_t const code = ZSTD_initCStream_advanced(zcs, NULL, 0, params, pledgedSrcSize); size_t const code = ZSTD_initCStream_advanced(zcs, NULL, 0, params, pledgedSrcSize);
if (ZSTD_isError(code)) { return NULL; } if (ZSTD_isError(code)) {
return NULL;
}
} }
return zcs; return zcs;
} }
ZSTD_CStream* ZSTD_initCStream_usingCDict(const ZSTD_CDict* cdict, unsigned long long pledgedSrcSize, void* workspace, size_t workspaceSize) ZSTD_CStream *ZSTD_initCStream_usingCDict(const ZSTD_CDict *cdict, unsigned long long pledgedSrcSize, void *workspace, size_t workspaceSize)
{ {
ZSTD_parameters const params = ZSTD_getParamsFromCDict(cdict); ZSTD_parameters const params = ZSTD_getParamsFromCDict(cdict);
ZSTD_CStream* const zcs = ZSTD_initCStream(params, pledgedSrcSize, workspace, workspaceSize); ZSTD_CStream *const zcs = ZSTD_initCStream(params, pledgedSrcSize, workspace, workspaceSize);
if (zcs) { if (zcs) {
zcs->cdict = cdict; zcs->cdict = cdict;
if (ZSTD_isError(ZSTD_resetCStream_internal(zcs, pledgedSrcSize))) { if (ZSTD_isError(ZSTD_resetCStream_internal(zcs, pledgedSrcSize))) {
@@ -2958,71 +3100,79 @@ ZSTD_CStream* ZSTD_initCStream_usingCDict(const ZSTD_CDict* cdict, unsigned long
typedef enum { zsf_gather, zsf_flush, zsf_end } ZSTD_flush_e; typedef enum { zsf_gather, zsf_flush, zsf_end } ZSTD_flush_e;
ZSTD_STATIC size_t ZSTD_limitCopy(void* dst, size_t dstCapacity, const void* src, size_t srcSize) ZSTD_STATIC size_t ZSTD_limitCopy(void *dst, size_t dstCapacity, const void *src, size_t srcSize)
{ {
size_t const length = MIN(dstCapacity, srcSize); size_t const length = MIN(dstCapacity, srcSize);
memcpy(dst, src, length); memcpy(dst, src, length);
return length; return length;
} }
static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs, static size_t ZSTD_compressStream_generic(ZSTD_CStream *zcs, void *dst, size_t *dstCapacityPtr, const void *src, size_t *srcSizePtr, ZSTD_flush_e const flush)
void* dst, size_t* dstCapacityPtr,
const void* src, size_t* srcSizePtr,
ZSTD_flush_e const flush)
{ {
U32 someMoreWork = 1; U32 someMoreWork = 1;
const char* const istart = (const char*)src; const char *const istart = (const char *)src;
const char* const iend = istart + *srcSizePtr; const char *const iend = istart + *srcSizePtr;
const char* ip = istart; const char *ip = istart;
char* const ostart = (char*)dst; char *const ostart = (char *)dst;
char* const oend = ostart + *dstCapacityPtr; char *const oend = ostart + *dstCapacityPtr;
char* op = ostart; char *op = ostart;
while (someMoreWork) { while (someMoreWork) {
switch(zcs->stage) switch (zcs->stage) {
{ case zcss_init:
case zcss_init: return ERROR(init_missing); /* call ZBUFF_compressInit() first ! */ return ERROR(init_missing); /* call ZBUFF_compressInit() first ! */
case zcss_load: case zcss_load:
/* complete inBuffer */ /* complete inBuffer */
{ size_t const toLoad = zcs->inBuffTarget - zcs->inBuffPos; {
size_t const loaded = ZSTD_limitCopy(zcs->inBuff + zcs->inBuffPos, toLoad, ip, iend-ip); size_t const toLoad = zcs->inBuffTarget - zcs->inBuffPos;
size_t const loaded = ZSTD_limitCopy(zcs->inBuff + zcs->inBuffPos, toLoad, ip, iend - ip);
zcs->inBuffPos += loaded; zcs->inBuffPos += loaded;
ip += loaded; ip += loaded;
if ( (zcs->inBuffPos==zcs->inToCompress) || (!flush && (toLoad != loaded)) ) { if ((zcs->inBuffPos == zcs->inToCompress) || (!flush && (toLoad != loaded))) {
someMoreWork = 0; break; /* not enough input to get a full block : stop there, wait for more */ someMoreWork = 0;
} } break; /* not enough input to get a full block : stop there, wait for more */
}
}
/* compress curr block (note : this stage cannot be stopped in the middle) */ /* compress curr block (note : this stage cannot be stopped in the middle) */
{ void* cDst; {
void *cDst;
size_t cSize; size_t cSize;
size_t const iSize = zcs->inBuffPos - zcs->inToCompress; size_t const iSize = zcs->inBuffPos - zcs->inToCompress;
size_t oSize = oend-op; size_t oSize = oend - op;
if (oSize >= ZSTD_compressBound(iSize)) if (oSize >= ZSTD_compressBound(iSize))
cDst = op; /* compress directly into output buffer (avoid flush stage) */ cDst = op; /* compress directly into output buffer (avoid flush stage) */
else else
cDst = zcs->outBuff, oSize = zcs->outBuffSize; cDst = zcs->outBuff, oSize = zcs->outBuffSize;
cSize = (flush == zsf_end) ? cSize = (flush == zsf_end) ? ZSTD_compressEnd(zcs->cctx, cDst, oSize, zcs->inBuff + zcs->inToCompress, iSize)
ZSTD_compressEnd(zcs->cctx, cDst, oSize, zcs->inBuff + zcs->inToCompress, iSize) : : ZSTD_compressContinue(zcs->cctx, cDst, oSize, zcs->inBuff + zcs->inToCompress, iSize);
ZSTD_compressContinue(zcs->cctx, cDst, oSize, zcs->inBuff + zcs->inToCompress, iSize); if (ZSTD_isError(cSize))
if (ZSTD_isError(cSize)) return cSize; return cSize;
if (flush == zsf_end) zcs->frameEnded = 1; if (flush == zsf_end)
zcs->frameEnded = 1;
/* prepare next block */ /* prepare next block */
zcs->inBuffTarget = zcs->inBuffPos + zcs->blockSize; zcs->inBuffTarget = zcs->inBuffPos + zcs->blockSize;
if (zcs->inBuffTarget > zcs->inBuffSize) if (zcs->inBuffTarget > zcs->inBuffSize)
zcs->inBuffPos = 0, zcs->inBuffTarget = zcs->blockSize; /* note : inBuffSize >= blockSize */ zcs->inBuffPos = 0, zcs->inBuffTarget = zcs->blockSize; /* note : inBuffSize >= blockSize */
zcs->inToCompress = zcs->inBuffPos; zcs->inToCompress = zcs->inBuffPos;
if (cDst == op) { op += cSize; break; } /* no need to flush */ if (cDst == op) {
op += cSize;
break;
} /* no need to flush */
zcs->outBuffContentSize = cSize; zcs->outBuffContentSize = cSize;
zcs->outBuffFlushedSize = 0; zcs->outBuffFlushedSize = 0;
zcs->stage = zcss_flush; /* pass-through to flush stage */ zcs->stage = zcss_flush; /* pass-through to flush stage */
} }
case zcss_flush: case zcss_flush: {
{ size_t const toFlush = zcs->outBuffContentSize - zcs->outBuffFlushedSize; size_t const toFlush = zcs->outBuffContentSize - zcs->outBuffFlushedSize;
size_t const flushed = ZSTD_limitCopy(op, oend-op, zcs->outBuff + zcs->outBuffFlushedSize, toFlush); size_t const flushed = ZSTD_limitCopy(op, oend - op, zcs->outBuff + zcs->outBuffFlushedSize, toFlush);
op += flushed; op += flushed;
zcs->outBuffFlushedSize += flushed; zcs->outBuffFlushedSize += flushed;
if (toFlush!=flushed) { someMoreWork = 0; break; } /* dst too small to store flushed data : stop there */ if (toFlush != flushed) {
someMoreWork = 0;
break;
} /* dst too small to store flushed data : stop there */
zcs->outBuffContentSize = zcs->outBuffFlushedSize = 0; zcs->outBuffContentSize = zcs->outBuffFlushedSize = 0;
zcs->stage = zcss_load; zcs->stage = zcss_load;
break; break;
@@ -3040,49 +3190,49 @@ static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs,
*srcSizePtr = ip - istart; *srcSizePtr = ip - istart;
*dstCapacityPtr = op - ostart; *dstCapacityPtr = op - ostart;
zcs->inputProcessed += *srcSizePtr; zcs->inputProcessed += *srcSizePtr;
if (zcs->frameEnded) return 0; if (zcs->frameEnded)
{ size_t hintInSize = zcs->inBuffTarget - zcs->inBuffPos; return 0;
if (hintInSize==0) hintInSize = zcs->blockSize; {
size_t hintInSize = zcs->inBuffTarget - zcs->inBuffPos;
if (hintInSize == 0)
hintInSize = zcs->blockSize;
return hintInSize; return hintInSize;
} }
} }
size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuffer* input) size_t ZSTD_compressStream(ZSTD_CStream *zcs, ZSTD_outBuffer *output, ZSTD_inBuffer *input)
{ {
size_t sizeRead = input->size - input->pos; size_t sizeRead = input->size - input->pos;
size_t sizeWritten = output->size - output->pos; size_t sizeWritten = output->size - output->pos;
size_t const result = ZSTD_compressStream_generic(zcs, size_t const result =
(char*)(output->dst) + output->pos, &sizeWritten, ZSTD_compressStream_generic(zcs, (char *)(output->dst) + output->pos, &sizeWritten, (const char *)(input->src) + input->pos, &sizeRead, zsf_gather);
(const char*)(input->src) + input->pos, &sizeRead, zsf_gather);
input->pos += sizeRead; input->pos += sizeRead;
output->pos += sizeWritten; output->pos += sizeWritten;
return result; return result;
} }
/*====== Finalize ======*/ /*====== Finalize ======*/
/*! ZSTD_flushStream() : /*! ZSTD_flushStream() :
* @return : amount of data remaining to flush */ * @return : amount of data remaining to flush */
size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output) size_t ZSTD_flushStream(ZSTD_CStream *zcs, ZSTD_outBuffer *output)
{ {
size_t srcSize = 0; size_t srcSize = 0;
size_t sizeWritten = output->size - output->pos; size_t sizeWritten = output->size - output->pos;
size_t const result = ZSTD_compressStream_generic(zcs, size_t const result = ZSTD_compressStream_generic(zcs, (char *)(output->dst) + output->pos, &sizeWritten, &srcSize,
(char*)(output->dst) + output->pos, &sizeWritten, &srcSize, /* use a valid src address instead of NULL */
&srcSize, &srcSize, /* use a valid src address instead of NULL */
zsf_flush); zsf_flush);
output->pos += sizeWritten; output->pos += sizeWritten;
if (ZSTD_isError(result)) return result; if (ZSTD_isError(result))
return result;
return zcs->outBuffContentSize - zcs->outBuffFlushedSize; /* remaining to flush */ return zcs->outBuffContentSize - zcs->outBuffFlushedSize; /* remaining to flush */
} }
size_t ZSTD_endStream(ZSTD_CStream *zcs, ZSTD_outBuffer *output)
size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output)
{ {
BYTE* const ostart = (BYTE*)(output->dst) + output->pos; BYTE *const ostart = (BYTE *)(output->dst) + output->pos;
BYTE* const oend = (BYTE*)(output->dst) + output->size; BYTE *const oend = (BYTE *)(output->dst) + output->size;
BYTE* op = ostart; BYTE *op = ostart;
if ((zcs->pledgedSrcSize) && (zcs->inputProcessed != zcs->pledgedSrcSize)) if ((zcs->pledgedSrcSize) && (zcs->inputProcessed != zcs->pledgedSrcSize))
return ERROR(srcSize_wrong); /* pledgedSrcSize not respected */ return ERROR(srcSize_wrong); /* pledgedSrcSize not respected */
@@ -3091,7 +3241,8 @@ size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output)
/* flush whatever remains */ /* flush whatever remains */
size_t srcSize = 0; size_t srcSize = 0;
size_t sizeWritten = output->size - output->pos; size_t sizeWritten = output->size - output->pos;
size_t const notEnded = ZSTD_compressStream_generic(zcs, ostart, &sizeWritten, &srcSize, &srcSize, zsf_end); /* use a valid src address instead of NULL */ size_t const notEnded =
ZSTD_compressStream_generic(zcs, ostart, &sizeWritten, &srcSize, &srcSize, zsf_end); /* use a valid src address instead of NULL */
size_t const remainingToFlush = zcs->outBuffContentSize - zcs->outBuffFlushedSize; size_t const remainingToFlush = zcs->outBuffContentSize - zcs->outBuffFlushedSize;
op += sizeWritten; op += sizeWritten;
if (remainingToFlush) { if (remainingToFlush) {
@@ -3100,134 +3251,138 @@ size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output)
} }
/* create epilogue */ /* create epilogue */
zcs->stage = zcss_final; zcs->stage = zcss_final;
zcs->outBuffContentSize = !notEnded ? 0 : zcs->outBuffContentSize = !notEnded ? 0 : ZSTD_compressEnd(zcs->cctx, zcs->outBuff, zcs->outBuffSize, NULL,
ZSTD_compressEnd(zcs->cctx, zcs->outBuff, zcs->outBuffSize, NULL, 0); /* write epilogue, including final empty block, into outBuff */ 0); /* write epilogue, including final empty block, into outBuff */
} }
/* flush epilogue */ /* flush epilogue */
{ size_t const toFlush = zcs->outBuffContentSize - zcs->outBuffFlushedSize; {
size_t const flushed = ZSTD_limitCopy(op, oend-op, zcs->outBuff + zcs->outBuffFlushedSize, toFlush); size_t const toFlush = zcs->outBuffContentSize - zcs->outBuffFlushedSize;
size_t const flushed = ZSTD_limitCopy(op, oend - op, zcs->outBuff + zcs->outBuffFlushedSize, toFlush);
op += flushed; op += flushed;
zcs->outBuffFlushedSize += flushed; zcs->outBuffFlushedSize += flushed;
output->pos += op-ostart; output->pos += op - ostart;
if (toFlush==flushed) zcs->stage = zcss_init; /* end reached */ if (toFlush == flushed)
zcs->stage = zcss_init; /* end reached */
return toFlush - flushed; return toFlush - flushed;
} }
} }
/*-===== Pre-defined compression levels =====-*/ /*-===== Pre-defined compression levels =====-*/
#define ZSTD_DEFAULT_CLEVEL 1 #define ZSTD_DEFAULT_CLEVEL 1
#define ZSTD_MAX_CLEVEL 22 #define ZSTD_MAX_CLEVEL 22
int ZSTD_maxCLevel(void) { return ZSTD_MAX_CLEVEL; } int ZSTD_maxCLevel(void) { return ZSTD_MAX_CLEVEL; }
static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEVEL+1] = { static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEVEL + 1] = {
{ /* "default" */ {
/* "default" */
/* W, C, H, S, L, TL, strat */ /* W, C, H, S, L, TL, strat */
{ 18, 12, 12, 1, 7, 16, ZSTD_fast }, /* level 0 - never used */ {18, 12, 12, 1, 7, 16, ZSTD_fast}, /* level 0 - never used */
{ 19, 13, 14, 1, 7, 16, ZSTD_fast }, /* level 1 */ {19, 13, 14, 1, 7, 16, ZSTD_fast}, /* level 1 */
{ 19, 15, 16, 1, 6, 16, ZSTD_fast }, /* level 2 */ {19, 15, 16, 1, 6, 16, ZSTD_fast}, /* level 2 */
{ 20, 16, 17, 1, 5, 16, ZSTD_dfast }, /* level 3.*/ {20, 16, 17, 1, 5, 16, ZSTD_dfast}, /* level 3.*/
{ 20, 18, 18, 1, 5, 16, ZSTD_dfast }, /* level 4.*/ {20, 18, 18, 1, 5, 16, ZSTD_dfast}, /* level 4.*/
{ 20, 15, 18, 3, 5, 16, ZSTD_greedy }, /* level 5 */ {20, 15, 18, 3, 5, 16, ZSTD_greedy}, /* level 5 */
{ 21, 16, 19, 2, 5, 16, ZSTD_lazy }, /* level 6 */ {21, 16, 19, 2, 5, 16, ZSTD_lazy}, /* level 6 */
{ 21, 17, 20, 3, 5, 16, ZSTD_lazy }, /* level 7 */ {21, 17, 20, 3, 5, 16, ZSTD_lazy}, /* level 7 */
{ 21, 18, 20, 3, 5, 16, ZSTD_lazy2 }, /* level 8 */ {21, 18, 20, 3, 5, 16, ZSTD_lazy2}, /* level 8 */
{ 21, 20, 20, 3, 5, 16, ZSTD_lazy2 }, /* level 9 */ {21, 20, 20, 3, 5, 16, ZSTD_lazy2}, /* level 9 */
{ 21, 19, 21, 4, 5, 16, ZSTD_lazy2 }, /* level 10 */ {21, 19, 21, 4, 5, 16, ZSTD_lazy2}, /* level 10 */
{ 22, 20, 22, 4, 5, 16, ZSTD_lazy2 }, /* level 11 */ {22, 20, 22, 4, 5, 16, ZSTD_lazy2}, /* level 11 */
{ 22, 20, 22, 5, 5, 16, ZSTD_lazy2 }, /* level 12 */ {22, 20, 22, 5, 5, 16, ZSTD_lazy2}, /* level 12 */
{ 22, 21, 22, 5, 5, 16, ZSTD_lazy2 }, /* level 13 */ {22, 21, 22, 5, 5, 16, ZSTD_lazy2}, /* level 13 */
{ 22, 21, 22, 6, 5, 16, ZSTD_lazy2 }, /* level 14 */ {22, 21, 22, 6, 5, 16, ZSTD_lazy2}, /* level 14 */
{ 22, 21, 21, 5, 5, 16, ZSTD_btlazy2 }, /* level 15 */ {22, 21, 21, 5, 5, 16, ZSTD_btlazy2}, /* level 15 */
{ 23, 22, 22, 5, 5, 16, ZSTD_btlazy2 }, /* level 16 */ {23, 22, 22, 5, 5, 16, ZSTD_btlazy2}, /* level 16 */
{ 23, 21, 22, 4, 5, 24, ZSTD_btopt }, /* level 17 */ {23, 21, 22, 4, 5, 24, ZSTD_btopt}, /* level 17 */
{ 23, 23, 22, 6, 5, 32, ZSTD_btopt }, /* level 18 */ {23, 23, 22, 6, 5, 32, ZSTD_btopt}, /* level 18 */
{ 23, 23, 22, 6, 3, 48, ZSTD_btopt }, /* level 19 */ {23, 23, 22, 6, 3, 48, ZSTD_btopt}, /* level 19 */
{ 25, 25, 23, 7, 3, 64, ZSTD_btopt2 }, /* level 20 */ {25, 25, 23, 7, 3, 64, ZSTD_btopt2}, /* level 20 */
{ 26, 26, 23, 7, 3,256, ZSTD_btopt2 }, /* level 21 */ {26, 26, 23, 7, 3, 256, ZSTD_btopt2}, /* level 21 */
{ 27, 27, 25, 9, 3,512, ZSTD_btopt2 }, /* level 22 */ {27, 27, 25, 9, 3, 512, ZSTD_btopt2}, /* level 22 */
}, },
{ /* for srcSize <= 256 KB */ {
/* for srcSize <= 256 KB */
/* W, C, H, S, L, T, strat */ /* W, C, H, S, L, T, strat */
{ 0, 0, 0, 0, 0, 0, ZSTD_fast }, /* level 0 - not used */ {0, 0, 0, 0, 0, 0, ZSTD_fast}, /* level 0 - not used */
{ 18, 13, 14, 1, 6, 8, ZSTD_fast }, /* level 1 */ {18, 13, 14, 1, 6, 8, ZSTD_fast}, /* level 1 */
{ 18, 14, 13, 1, 5, 8, ZSTD_dfast }, /* level 2 */ {18, 14, 13, 1, 5, 8, ZSTD_dfast}, /* level 2 */
{ 18, 16, 15, 1, 5, 8, ZSTD_dfast }, /* level 3 */ {18, 16, 15, 1, 5, 8, ZSTD_dfast}, /* level 3 */
{ 18, 15, 17, 1, 5, 8, ZSTD_greedy }, /* level 4.*/ {18, 15, 17, 1, 5, 8, ZSTD_greedy}, /* level 4.*/
{ 18, 16, 17, 4, 5, 8, ZSTD_greedy }, /* level 5.*/ {18, 16, 17, 4, 5, 8, ZSTD_greedy}, /* level 5.*/
{ 18, 16, 17, 3, 5, 8, ZSTD_lazy }, /* level 6.*/ {18, 16, 17, 3, 5, 8, ZSTD_lazy}, /* level 6.*/
{ 18, 17, 17, 4, 4, 8, ZSTD_lazy }, /* level 7 */ {18, 17, 17, 4, 4, 8, ZSTD_lazy}, /* level 7 */
{ 18, 17, 17, 4, 4, 8, ZSTD_lazy2 }, /* level 8 */ {18, 17, 17, 4, 4, 8, ZSTD_lazy2}, /* level 8 */
{ 18, 17, 17, 5, 4, 8, ZSTD_lazy2 }, /* level 9 */ {18, 17, 17, 5, 4, 8, ZSTD_lazy2}, /* level 9 */
{ 18, 17, 17, 6, 4, 8, ZSTD_lazy2 }, /* level 10 */ {18, 17, 17, 6, 4, 8, ZSTD_lazy2}, /* level 10 */
{ 18, 18, 17, 6, 4, 8, ZSTD_lazy2 }, /* level 11.*/ {18, 18, 17, 6, 4, 8, ZSTD_lazy2}, /* level 11.*/
{ 18, 18, 17, 7, 4, 8, ZSTD_lazy2 }, /* level 12.*/ {18, 18, 17, 7, 4, 8, ZSTD_lazy2}, /* level 12.*/
{ 18, 19, 17, 6, 4, 8, ZSTD_btlazy2 }, /* level 13 */ {18, 19, 17, 6, 4, 8, ZSTD_btlazy2}, /* level 13 */
{ 18, 18, 18, 4, 4, 16, ZSTD_btopt }, /* level 14.*/ {18, 18, 18, 4, 4, 16, ZSTD_btopt}, /* level 14.*/
{ 18, 18, 18, 4, 3, 16, ZSTD_btopt }, /* level 15.*/ {18, 18, 18, 4, 3, 16, ZSTD_btopt}, /* level 15.*/
{ 18, 19, 18, 6, 3, 32, ZSTD_btopt }, /* level 16.*/ {18, 19, 18, 6, 3, 32, ZSTD_btopt}, /* level 16.*/
{ 18, 19, 18, 8, 3, 64, ZSTD_btopt }, /* level 17.*/ {18, 19, 18, 8, 3, 64, ZSTD_btopt}, /* level 17.*/
{ 18, 19, 18, 9, 3,128, ZSTD_btopt }, /* level 18.*/ {18, 19, 18, 9, 3, 128, ZSTD_btopt}, /* level 18.*/
{ 18, 19, 18, 10, 3,256, ZSTD_btopt }, /* level 19.*/ {18, 19, 18, 10, 3, 256, ZSTD_btopt}, /* level 19.*/
{ 18, 19, 18, 11, 3,512, ZSTD_btopt2 }, /* level 20.*/ {18, 19, 18, 11, 3, 512, ZSTD_btopt2}, /* level 20.*/
{ 18, 19, 18, 12, 3,512, ZSTD_btopt2 }, /* level 21.*/ {18, 19, 18, 12, 3, 512, ZSTD_btopt2}, /* level 21.*/
{ 18, 19, 18, 13, 3,512, ZSTD_btopt2 }, /* level 22.*/ {18, 19, 18, 13, 3, 512, ZSTD_btopt2}, /* level 22.*/
}, },
{ /* for srcSize <= 128 KB */ {
/* for srcSize <= 128 KB */
/* W, C, H, S, L, T, strat */ /* W, C, H, S, L, T, strat */
{ 17, 12, 12, 1, 7, 8, ZSTD_fast }, /* level 0 - not used */ {17, 12, 12, 1, 7, 8, ZSTD_fast}, /* level 0 - not used */
{ 17, 12, 13, 1, 6, 8, ZSTD_fast }, /* level 1 */ {17, 12, 13, 1, 6, 8, ZSTD_fast}, /* level 1 */
{ 17, 13, 16, 1, 5, 8, ZSTD_fast }, /* level 2 */ {17, 13, 16, 1, 5, 8, ZSTD_fast}, /* level 2 */
{ 17, 16, 16, 2, 5, 8, ZSTD_dfast }, /* level 3 */ {17, 16, 16, 2, 5, 8, ZSTD_dfast}, /* level 3 */
{ 17, 13, 15, 3, 4, 8, ZSTD_greedy }, /* level 4 */ {17, 13, 15, 3, 4, 8, ZSTD_greedy}, /* level 4 */
{ 17, 15, 17, 4, 4, 8, ZSTD_greedy }, /* level 5 */ {17, 15, 17, 4, 4, 8, ZSTD_greedy}, /* level 5 */
{ 17, 16, 17, 3, 4, 8, ZSTD_lazy }, /* level 6 */ {17, 16, 17, 3, 4, 8, ZSTD_lazy}, /* level 6 */
{ 17, 15, 17, 4, 4, 8, ZSTD_lazy2 }, /* level 7 */ {17, 15, 17, 4, 4, 8, ZSTD_lazy2}, /* level 7 */
{ 17, 17, 17, 4, 4, 8, ZSTD_lazy2 }, /* level 8 */ {17, 17, 17, 4, 4, 8, ZSTD_lazy2}, /* level 8 */
{ 17, 17, 17, 5, 4, 8, ZSTD_lazy2 }, /* level 9 */ {17, 17, 17, 5, 4, 8, ZSTD_lazy2}, /* level 9 */
{ 17, 17, 17, 6, 4, 8, ZSTD_lazy2 }, /* level 10 */ {17, 17, 17, 6, 4, 8, ZSTD_lazy2}, /* level 10 */
{ 17, 17, 17, 7, 4, 8, ZSTD_lazy2 }, /* level 11 */ {17, 17, 17, 7, 4, 8, ZSTD_lazy2}, /* level 11 */
{ 17, 17, 17, 8, 4, 8, ZSTD_lazy2 }, /* level 12 */ {17, 17, 17, 8, 4, 8, ZSTD_lazy2}, /* level 12 */
{ 17, 18, 17, 6, 4, 8, ZSTD_btlazy2 }, /* level 13.*/ {17, 18, 17, 6, 4, 8, ZSTD_btlazy2}, /* level 13.*/
{ 17, 17, 17, 7, 3, 8, ZSTD_btopt }, /* level 14.*/ {17, 17, 17, 7, 3, 8, ZSTD_btopt}, /* level 14.*/
{ 17, 17, 17, 7, 3, 16, ZSTD_btopt }, /* level 15.*/ {17, 17, 17, 7, 3, 16, ZSTD_btopt}, /* level 15.*/
{ 17, 18, 17, 7, 3, 32, ZSTD_btopt }, /* level 16.*/ {17, 18, 17, 7, 3, 32, ZSTD_btopt}, /* level 16.*/
{ 17, 18, 17, 7, 3, 64, ZSTD_btopt }, /* level 17.*/ {17, 18, 17, 7, 3, 64, ZSTD_btopt}, /* level 17.*/
{ 17, 18, 17, 7, 3,256, ZSTD_btopt }, /* level 18.*/ {17, 18, 17, 7, 3, 256, ZSTD_btopt}, /* level 18.*/
{ 17, 18, 17, 8, 3,256, ZSTD_btopt }, /* level 19.*/ {17, 18, 17, 8, 3, 256, ZSTD_btopt}, /* level 19.*/
{ 17, 18, 17, 9, 3,256, ZSTD_btopt2 }, /* level 20.*/ {17, 18, 17, 9, 3, 256, ZSTD_btopt2}, /* level 20.*/
{ 17, 18, 17, 10, 3,256, ZSTD_btopt2 }, /* level 21.*/ {17, 18, 17, 10, 3, 256, ZSTD_btopt2}, /* level 21.*/
{ 17, 18, 17, 11, 3,512, ZSTD_btopt2 }, /* level 22.*/ {17, 18, 17, 11, 3, 512, ZSTD_btopt2}, /* level 22.*/
}, },
{ /* for srcSize <= 16 KB */ {
/* for srcSize <= 16 KB */
/* W, C, H, S, L, T, strat */ /* W, C, H, S, L, T, strat */
{ 14, 12, 12, 1, 7, 6, ZSTD_fast }, /* level 0 - not used */ {14, 12, 12, 1, 7, 6, ZSTD_fast}, /* level 0 - not used */
{ 14, 14, 14, 1, 6, 6, ZSTD_fast }, /* level 1 */ {14, 14, 14, 1, 6, 6, ZSTD_fast}, /* level 1 */
{ 14, 14, 14, 1, 4, 6, ZSTD_fast }, /* level 2 */ {14, 14, 14, 1, 4, 6, ZSTD_fast}, /* level 2 */
{ 14, 14, 14, 1, 4, 6, ZSTD_dfast }, /* level 3.*/ {14, 14, 14, 1, 4, 6, ZSTD_dfast}, /* level 3.*/
{ 14, 14, 14, 4, 4, 6, ZSTD_greedy }, /* level 4.*/ {14, 14, 14, 4, 4, 6, ZSTD_greedy}, /* level 4.*/
{ 14, 14, 14, 3, 4, 6, ZSTD_lazy }, /* level 5.*/ {14, 14, 14, 3, 4, 6, ZSTD_lazy}, /* level 5.*/
{ 14, 14, 14, 4, 4, 6, ZSTD_lazy2 }, /* level 6 */ {14, 14, 14, 4, 4, 6, ZSTD_lazy2}, /* level 6 */
{ 14, 14, 14, 5, 4, 6, ZSTD_lazy2 }, /* level 7 */ {14, 14, 14, 5, 4, 6, ZSTD_lazy2}, /* level 7 */
{ 14, 14, 14, 6, 4, 6, ZSTD_lazy2 }, /* level 8.*/ {14, 14, 14, 6, 4, 6, ZSTD_lazy2}, /* level 8.*/
{ 14, 15, 14, 6, 4, 6, ZSTD_btlazy2 }, /* level 9.*/ {14, 15, 14, 6, 4, 6, ZSTD_btlazy2}, /* level 9.*/
{ 14, 15, 14, 3, 3, 6, ZSTD_btopt }, /* level 10.*/ {14, 15, 14, 3, 3, 6, ZSTD_btopt}, /* level 10.*/
{ 14, 15, 14, 6, 3, 8, ZSTD_btopt }, /* level 11.*/ {14, 15, 14, 6, 3, 8, ZSTD_btopt}, /* level 11.*/
{ 14, 15, 14, 6, 3, 16, ZSTD_btopt }, /* level 12.*/ {14, 15, 14, 6, 3, 16, ZSTD_btopt}, /* level 12.*/
{ 14, 15, 14, 6, 3, 24, ZSTD_btopt }, /* level 13.*/ {14, 15, 14, 6, 3, 24, ZSTD_btopt}, /* level 13.*/
{ 14, 15, 15, 6, 3, 48, ZSTD_btopt }, /* level 14.*/ {14, 15, 15, 6, 3, 48, ZSTD_btopt}, /* level 14.*/
{ 14, 15, 15, 6, 3, 64, ZSTD_btopt }, /* level 15.*/ {14, 15, 15, 6, 3, 64, ZSTD_btopt}, /* level 15.*/
{ 14, 15, 15, 6, 3, 96, ZSTD_btopt }, /* level 16.*/ {14, 15, 15, 6, 3, 96, ZSTD_btopt}, /* level 16.*/
{ 14, 15, 15, 6, 3,128, ZSTD_btopt }, /* level 17.*/ {14, 15, 15, 6, 3, 128, ZSTD_btopt}, /* level 17.*/
{ 14, 15, 15, 6, 3,256, ZSTD_btopt }, /* level 18.*/ {14, 15, 15, 6, 3, 256, ZSTD_btopt}, /* level 18.*/
{ 14, 15, 15, 7, 3,256, ZSTD_btopt }, /* level 19.*/ {14, 15, 15, 7, 3, 256, ZSTD_btopt}, /* level 19.*/
{ 14, 15, 15, 8, 3,256, ZSTD_btopt2 }, /* level 20.*/ {14, 15, 15, 8, 3, 256, ZSTD_btopt2}, /* level 20.*/
{ 14, 15, 15, 9, 3,256, ZSTD_btopt2 }, /* level 21.*/ {14, 15, 15, 9, 3, 256, ZSTD_btopt2}, /* level 21.*/
{ 14, 15, 15, 10, 3,256, ZSTD_btopt2 }, /* level 22.*/ {14, 15, 15, 10, 3, 256, ZSTD_btopt2}, /* level 22.*/
}, },
}; };
/*! ZSTD_getCParams() : /*! ZSTD_getCParams() :
@@ -3237,15 +3392,20 @@ ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, unsigned long l
{ {
ZSTD_compressionParameters cp; ZSTD_compressionParameters cp;
size_t const addedSize = srcSize ? 0 : 500; size_t const addedSize = srcSize ? 0 : 500;
U64 const rSize = srcSize+dictSize ? srcSize+dictSize+addedSize : (U64)-1; U64 const rSize = srcSize + dictSize ? srcSize + dictSize + addedSize : (U64)-1;
U32 const tableID = (rSize <= 256 KB) + (rSize <= 128 KB) + (rSize <= 16 KB); /* intentional underflow for srcSizeHint == 0 */ U32 const tableID = (rSize <= 256 KB) + (rSize <= 128 KB) + (rSize <= 16 KB); /* intentional underflow for srcSizeHint == 0 */
if (compressionLevel <= 0) compressionLevel = ZSTD_DEFAULT_CLEVEL; /* 0 == default; no negative compressionLevel yet */ if (compressionLevel <= 0)
if (compressionLevel > ZSTD_MAX_CLEVEL) compressionLevel = ZSTD_MAX_CLEVEL; compressionLevel = ZSTD_DEFAULT_CLEVEL; /* 0 == default; no negative compressionLevel yet */
if (compressionLevel > ZSTD_MAX_CLEVEL)
compressionLevel = ZSTD_MAX_CLEVEL;
cp = ZSTD_defaultCParameters[tableID][compressionLevel]; cp = ZSTD_defaultCParameters[tableID][compressionLevel];
if (ZSTD_32bits()) { /* auto-correction, for 32-bits mode */ if (ZSTD_32bits()) { /* auto-correction, for 32-bits mode */
if (cp.windowLog > ZSTD_WINDOWLOG_MAX) cp.windowLog = ZSTD_WINDOWLOG_MAX; if (cp.windowLog > ZSTD_WINDOWLOG_MAX)
if (cp.chainLog > ZSTD_CHAINLOG_MAX) cp.chainLog = ZSTD_CHAINLOG_MAX; cp.windowLog = ZSTD_WINDOWLOG_MAX;
if (cp.hashLog > ZSTD_HASHLOG_MAX) cp.hashLog = ZSTD_HASHLOG_MAX; if (cp.chainLog > ZSTD_CHAINLOG_MAX)
cp.chainLog = ZSTD_CHAINLOG_MAX;
if (cp.hashLog > ZSTD_HASHLOG_MAX)
cp.hashLog = ZSTD_HASHLOG_MAX;
} }
cp = ZSTD_adjustCParams(cp, srcSize, dictSize); cp = ZSTD_adjustCParams(cp, srcSize, dictSize);
return cp; return cp;
@@ -3254,7 +3414,8 @@ ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, unsigned long l
/*! ZSTD_getParams() : /*! ZSTD_getParams() :
* same as ZSTD_getCParams(), but @return a `ZSTD_parameters` object (instead of `ZSTD_compressionParameters`). * same as ZSTD_getCParams(), but @return a `ZSTD_parameters` object (instead of `ZSTD_compressionParameters`).
* All fields of `ZSTD_frameParameters` are set to default (0) */ * All fields of `ZSTD_frameParameters` are set to default (0) */
ZSTD_parameters ZSTD_getParams(int compressionLevel, unsigned long long srcSize, size_t dictSize) { ZSTD_parameters ZSTD_getParams(int compressionLevel, unsigned long long srcSize, size_t dictSize)
{
ZSTD_parameters params; ZSTD_parameters params;
ZSTD_compressionParameters const cParams = ZSTD_getCParams(compressionLevel, srcSize, dictSize); ZSTD_compressionParameters const cParams = ZSTD_getCParams(compressionLevel, srcSize, dictSize);
memset(&params, 0, sizeof(params)); memset(&params, 0, sizeof(params));
contrib/linux-kernel/lib/zstd/decompress.c
+1049 -897
View File
@@ -14,7 +14,6 @@
* ("BSD"). * ("BSD").
*/ */
/* *************************************************************** /* ***************************************************************
* Tuning parameters * Tuning parameters
*****************************************************************/ *****************************************************************/
@@ -24,20 +23,19 @@
* Frames requiring more memory will be rejected. * Frames requiring more memory will be rejected.
*/ */
#ifndef ZSTD_MAXWINDOWSIZE_DEFAULT #ifndef ZSTD_MAXWINDOWSIZE_DEFAULT
# define ZSTD_MAXWINDOWSIZE_DEFAULT ((1 << ZSTD_WINDOWLOG_MAX) + 1) /* defined within zstd.h */ #define ZSTD_MAXWINDOWSIZE_DEFAULT ((1 << ZSTD_WINDOWLOG_MAX) + 1) /* defined within zstd.h */
#endif #endif
/*-******************************************************* /*-*******************************************************
* Dependencies * Dependencies
*********************************************************/ *********************************************************/
#include "fse.h"
#include "huf.h"
#include "mem.h" /* low level memory routines */
#include "zstd_internal.h"
#include <linux/kernel.h> #include <linux/kernel.h>
#include <linux/module.h> #include <linux/module.h>
#include <linux/string.h> /* memcpy, memmove, memset */ #include <linux/string.h> /* memcpy, memmove, memset */
#include "mem.h" /* low level memory routines */
#include "fse.h"
#include "huf.h"
#include "zstd_internal.h"
#define ZSTD_PREFETCH(ptr) __builtin_prefetch(ptr, 0, 0) #define ZSTD_PREFETCH(ptr) __builtin_prefetch(ptr, 0, 0)
@@ -48,20 +46,24 @@
#define FSE_isError ERR_isError #define FSE_isError ERR_isError
#define HUF_isError ERR_isError #define HUF_isError ERR_isError
/*_******************************************************* /*_*******************************************************
* Memory operations * Memory operations
**********************************************************/ **********************************************************/
static void ZSTD_copy4(void* dst, const void* src) { memcpy(dst, src, 4); } static void ZSTD_copy4(void *dst, const void *src) { memcpy(dst, src, 4); }
/*-************************************************************* /*-*************************************************************
* Context management * Context management
***************************************************************/ ***************************************************************/
typedef enum { ZSTDds_getFrameHeaderSize, ZSTDds_decodeFrameHeader, typedef enum {
ZSTDds_decodeBlockHeader, ZSTDds_decompressBlock, ZSTDds_getFrameHeaderSize,
ZSTDds_decompressLastBlock, ZSTDds_checkChecksum, ZSTDds_decodeFrameHeader,
ZSTDds_decodeSkippableHeader, ZSTDds_skipFrame } ZSTD_dStage; ZSTDds_decodeBlockHeader,
ZSTDds_decompressBlock,
ZSTDds_decompressLastBlock,
ZSTDds_checkChecksum,
ZSTDds_decodeSkippableHeader,
ZSTDds_skipFrame
} ZSTD_dStage;
typedef struct { typedef struct {
FSE_DTable LLTable[FSE_DTABLE_SIZE_U32(LLFSELog)]; FSE_DTable LLTable[FSE_DTABLE_SIZE_U32(LLFSELog)];
@@ -71,17 +73,16 @@ typedef struct {
U32 rep[ZSTD_REP_NUM]; U32 rep[ZSTD_REP_NUM];
} ZSTD_entropyTables_t; } ZSTD_entropyTables_t;
struct ZSTD_DCtx_s struct ZSTD_DCtx_s {
{ const FSE_DTable *LLTptr;
const FSE_DTable* LLTptr; const FSE_DTable *MLTptr;
const FSE_DTable* MLTptr; const FSE_DTable *OFTptr;
const FSE_DTable* OFTptr; const HUF_DTable *HUFptr;
const HUF_DTable* HUFptr;
ZSTD_entropyTables_t entropy; ZSTD_entropyTables_t entropy;
const void* previousDstEnd; /* detect continuity */ const void *previousDstEnd; /* detect continuity */
const void* base; /* start of curr segment */ const void *base; /* start of curr segment */
const void* vBase; /* virtual start of previous segment if it was just before curr one */ const void *vBase; /* virtual start of previous segment if it was just before curr one */
const void* dictEnd; /* end of previous segment */ const void *dictEnd; /* end of previous segment */
size_t expected; size_t expected;
ZSTD_frameParams fParams; ZSTD_frameParams fParams;
blockType_e bType; /* used in ZSTD_decompressContinue(), to transfer blockType between header decoding and block decoding stages */ blockType_e bType; /* used in ZSTD_decompressContinue(), to transfer blockType between header decoding and block decoding stages */
@@ -91,7 +92,7 @@ struct ZSTD_DCtx_s
struct xxh64_state xxhState; struct xxh64_state xxhState;
size_t headerSize; size_t headerSize;
U32 dictID; U32 dictID;
const BYTE* litPtr; const BYTE *litPtr;
ZSTD_customMem customMem; ZSTD_customMem customMem;
size_t litSize; size_t litSize;
size_t rleSize; size_t rleSize;
@@ -99,12 +100,9 @@ struct ZSTD_DCtx_s
BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX]; BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX];
}; /* typedef'd to ZSTD_DCtx within "zstd.h" */ }; /* typedef'd to ZSTD_DCtx within "zstd.h" */
size_t ZSTD_DCtxWorkspaceBound(void) size_t ZSTD_DCtxWorkspaceBound(void) { return ZSTD_ALIGN(sizeof(ZSTD_stack)) + ZSTD_ALIGN(sizeof(ZSTD_DCtx)); }
{
return ZSTD_ALIGN(sizeof(ZSTD_stack)) + ZSTD_ALIGN(sizeof(ZSTD_DCtx));
}
size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx) size_t ZSTD_decompressBegin(ZSTD_DCtx *dctx)
{ {
dctx->expected = ZSTD_frameHeaderSize_prefix; dctx->expected = ZSTD_frameHeaderSize_prefix;
dctx->stage = ZSTDds_getFrameHeaderSize; dctx->stage = ZSTDds_getFrameHeaderSize;
@@ -124,40 +122,42 @@ size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx)
return 0; return 0;
} }
ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem) ZSTD_DCtx *ZSTD_createDCtx_advanced(ZSTD_customMem customMem)
{ {
ZSTD_DCtx* dctx; ZSTD_DCtx *dctx;
if (!customMem.customAlloc || !customMem.customFree) return NULL; if (!customMem.customAlloc || !customMem.customFree)
return NULL;
dctx = (ZSTD_DCtx*)ZSTD_malloc(sizeof(ZSTD_DCtx), customMem); dctx = (ZSTD_DCtx *)ZSTD_malloc(sizeof(ZSTD_DCtx), customMem);
if (!dctx) return NULL; if (!dctx)
return NULL;
memcpy(&dctx->customMem, &customMem, sizeof(customMem)); memcpy(&dctx->customMem, &customMem, sizeof(customMem));
ZSTD_decompressBegin(dctx); ZSTD_decompressBegin(dctx);
return dctx; return dctx;
} }
ZSTD_DCtx* ZSTD_initDCtx(void* workspace, size_t workspaceSize) ZSTD_DCtx *ZSTD_initDCtx(void *workspace, size_t workspaceSize)
{ {
ZSTD_customMem const stackMem = ZSTD_initStack(workspace, workspaceSize); ZSTD_customMem const stackMem = ZSTD_initStack(workspace, workspaceSize);
return ZSTD_createDCtx_advanced(stackMem); return ZSTD_createDCtx_advanced(stackMem);
} }
size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx) size_t ZSTD_freeDCtx(ZSTD_DCtx *dctx)
{ {
if (dctx==NULL) return 0; /* support free on NULL */ if (dctx == NULL)
return 0; /* support free on NULL */
ZSTD_free(dctx, dctx->customMem); ZSTD_free(dctx, dctx->customMem);
return 0; /* reserved as a potential error code in the future */ return 0; /* reserved as a potential error code in the future */
} }
void ZSTD_copyDCtx(ZSTD_DCtx* dstDCtx, const ZSTD_DCtx* srcDCtx) void ZSTD_copyDCtx(ZSTD_DCtx *dstDCtx, const ZSTD_DCtx *srcDCtx)
{ {
size_t const workSpaceSize = (ZSTD_BLOCKSIZE_ABSOLUTEMAX+WILDCOPY_OVERLENGTH) + ZSTD_frameHeaderSize_max; size_t const workSpaceSize = (ZSTD_BLOCKSIZE_ABSOLUTEMAX + WILDCOPY_OVERLENGTH) + ZSTD_frameHeaderSize_max;
memcpy(dstDCtx, srcDCtx, sizeof(ZSTD_DCtx) - workSpaceSize); /* no need to copy workspace */ memcpy(dstDCtx, srcDCtx, sizeof(ZSTD_DCtx) - workSpaceSize); /* no need to copy workspace */
} }
static void ZSTD_refDDict(ZSTD_DCtx* dstDCtx, const ZSTD_DDict* ddict); static void ZSTD_refDDict(ZSTD_DCtx *dstDCtx, const ZSTD_DDict *ddict);
/*-************************************************************* /*-*************************************************************
* Decompression section * Decompression section
@@ -168,46 +168,51 @@ static void ZSTD_refDDict(ZSTD_DCtx* dstDCtx, const ZSTD_DDict* ddict);
* Note : Frame Identifier is 4 bytes. If `size < 4`, @return will always be 0. * Note : Frame Identifier is 4 bytes. If `size < 4`, @return will always be 0.
* Note 2 : Legacy Frame Identifiers are considered valid only if Legacy Support is enabled. * Note 2 : Legacy Frame Identifiers are considered valid only if Legacy Support is enabled.
* Note 3 : Skippable Frame Identifiers are considered valid. */ * Note 3 : Skippable Frame Identifiers are considered valid. */
unsigned ZSTD_isFrame(const void* buffer, size_t size) unsigned ZSTD_isFrame(const void *buffer, size_t size)
{ {
if (size < 4) return 0; if (size < 4)
{ U32 const magic = ZSTD_readLE32(buffer); return 0;
if (magic == ZSTD_MAGICNUMBER) return 1; {
if ((magic & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) return 1; U32 const magic = ZSTD_readLE32(buffer);
if (magic == ZSTD_MAGICNUMBER)
return 1;
if ((magic & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START)
return 1;
} }
return 0; return 0;
} }
/** ZSTD_frameHeaderSize() : /** ZSTD_frameHeaderSize() :
* srcSize must be >= ZSTD_frameHeaderSize_prefix. * srcSize must be >= ZSTD_frameHeaderSize_prefix.
* @return : size of the Frame Header */ * @return : size of the Frame Header */
static size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize) static size_t ZSTD_frameHeaderSize(const void *src, size_t srcSize)
{ {
if (srcSize < ZSTD_frameHeaderSize_prefix) return ERROR(srcSize_wrong); if (srcSize < ZSTD_frameHeaderSize_prefix)
{ BYTE const fhd = ((const BYTE*)src)[4]; return ERROR(srcSize_wrong);
U32 const dictID= fhd & 3; {
BYTE const fhd = ((const BYTE *)src)[4];
U32 const dictID = fhd & 3;
U32 const singleSegment = (fhd >> 5) & 1; U32 const singleSegment = (fhd >> 5) & 1;
U32 const fcsId = fhd >> 6; U32 const fcsId = fhd >> 6;
return ZSTD_frameHeaderSize_prefix + !singleSegment + ZSTD_did_fieldSize[dictID] + ZSTD_fcs_fieldSize[fcsId] return ZSTD_frameHeaderSize_prefix + !singleSegment + ZSTD_did_fieldSize[dictID] + ZSTD_fcs_fieldSize[fcsId] + (singleSegment && !fcsId);
+ (singleSegment && !fcsId);
} }
} }
/** ZSTD_getFrameParams() : /** ZSTD_getFrameParams() :
* decode Frame Header, or require larger `srcSize`. * decode Frame Header, or require larger `srcSize`.
* @return : 0, `fparamsPtr` is correctly filled, * @return : 0, `fparamsPtr` is correctly filled,
* >0, `srcSize` is too small, result is expected `srcSize`, * >0, `srcSize` is too small, result is expected `srcSize`,
* or an error code, which can be tested using ZSTD_isError() */ * or an error code, which can be tested using ZSTD_isError() */
size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t srcSize) size_t ZSTD_getFrameParams(ZSTD_frameParams *fparamsPtr, const void *src, size_t srcSize)
{ {
const BYTE* ip = (const BYTE*)src; const BYTE *ip = (const BYTE *)src;
if (srcSize < ZSTD_frameHeaderSize_prefix) return ZSTD_frameHeaderSize_prefix; if (srcSize < ZSTD_frameHeaderSize_prefix)
return ZSTD_frameHeaderSize_prefix;
if (ZSTD_readLE32(src) != ZSTD_MAGICNUMBER) { if (ZSTD_readLE32(src) != ZSTD_MAGICNUMBER) {
if ((ZSTD_readLE32(src) & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) { if ((ZSTD_readLE32(src) & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) {
if (srcSize < ZSTD_skippableHeaderSize) return ZSTD_skippableHeaderSize; /* magic number + skippable frame length */ if (srcSize < ZSTD_skippableHeaderSize)
return ZSTD_skippableHeaderSize; /* magic number + skippable frame length */
memset(fparamsPtr, 0, sizeof(*fparamsPtr)); memset(fparamsPtr, 0, sizeof(*fparamsPtr));
fparamsPtr->frameContentSize = ZSTD_readLE32((const char *)src + 4); fparamsPtr->frameContentSize = ZSTD_readLE32((const char *)src + 4);
fparamsPtr->windowSize = 0; /* windowSize==0 means a frame is skippable */ fparamsPtr->windowSize = 0; /* windowSize==0 means a frame is skippable */
@@ -217,46 +222,64 @@ size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t
} }
/* ensure there is enough `srcSize` to fully read/decode frame header */ /* ensure there is enough `srcSize` to fully read/decode frame header */
{ size_t const fhsize = ZSTD_frameHeaderSize(src, srcSize); {
if (srcSize < fhsize) return fhsize; } size_t const fhsize = ZSTD_frameHeaderSize(src, srcSize);
if (srcSize < fhsize)
return fhsize;
}
{ BYTE const fhdByte = ip[4]; {
BYTE const fhdByte = ip[4];
size_t pos = 5; size_t pos = 5;
U32 const dictIDSizeCode = fhdByte&3; U32 const dictIDSizeCode = fhdByte & 3;
U32 const checksumFlag = (fhdByte>>2)&1; U32 const checksumFlag = (fhdByte >> 2) & 1;
U32 const singleSegment = (fhdByte>>5)&1; U32 const singleSegment = (fhdByte >> 5) & 1;
U32 const fcsID = fhdByte>>6; U32 const fcsID = fhdByte >> 6;
U32 const windowSizeMax = 1U << ZSTD_WINDOWLOG_MAX; U32 const windowSizeMax = 1U << ZSTD_WINDOWLOG_MAX;
U32 windowSize = 0; U32 windowSize = 0;
U32 dictID = 0; U32 dictID = 0;
U64 frameContentSize = 0; U64 frameContentSize = 0;
if ((fhdByte & 0x08) != 0) return ERROR(frameParameter_unsupported); /* reserved bits, which must be zero */ if ((fhdByte & 0x08) != 0)
return ERROR(frameParameter_unsupported); /* reserved bits, which must be zero */
if (!singleSegment) { if (!singleSegment) {
BYTE const wlByte = ip[pos++]; BYTE const wlByte = ip[pos++];
U32 const windowLog = (wlByte >> 3) + ZSTD_WINDOWLOG_ABSOLUTEMIN; U32 const windowLog = (wlByte >> 3) + ZSTD_WINDOWLOG_ABSOLUTEMIN;
if (windowLog > ZSTD_WINDOWLOG_MAX) return ERROR(frameParameter_windowTooLarge); /* avoids issue with 1 << windowLog */ if (windowLog > ZSTD_WINDOWLOG_MAX)
return ERROR(frameParameter_windowTooLarge); /* avoids issue with 1 << windowLog */
windowSize = (1U << windowLog); windowSize = (1U << windowLog);
windowSize += (windowSize >> 3) * (wlByte&7); windowSize += (windowSize >> 3) * (wlByte & 7);
} }
switch(dictIDSizeCode) switch (dictIDSizeCode) {
{
default: /* impossible */ default: /* impossible */
case 0 : break; case 0: break;
case 1 : dictID = ip[pos]; pos++; break; case 1:
case 2 : dictID = ZSTD_readLE16(ip+pos); pos+=2; break; dictID = ip[pos];
case 3 : dictID = ZSTD_readLE32(ip+pos); pos+=4; break; pos++;
break;
case 2:
dictID = ZSTD_readLE16(ip + pos);
pos += 2;
break;
case 3:
dictID = ZSTD_readLE32(ip + pos);
pos += 4;
break;
} }
switch(fcsID) switch (fcsID) {
{
default: /* impossible */ default: /* impossible */
case 0 : if (singleSegment) frameContentSize = ip[pos]; break; case 0:
case 1 : frameContentSize = ZSTD_readLE16(ip+pos)+256; break; if (singleSegment)
case 2 : frameContentSize = ZSTD_readLE32(ip+pos); break; frameContentSize = ip[pos];
case 3 : frameContentSize = ZSTD_readLE64(ip+pos); break; break;
case 1: frameContentSize = ZSTD_readLE16(ip + pos) + 256; break;
case 2: frameContentSize = ZSTD_readLE32(ip + pos); break;
case 3: frameContentSize = ZSTD_readLE64(ip + pos); break;
} }
if (!windowSize) windowSize = (U32)frameContentSize; if (!windowSize)
if (windowSize > windowSizeMax) return ERROR(frameParameter_windowTooLarge); windowSize = (U32)frameContentSize;
if (windowSize > windowSizeMax)
return ERROR(frameParameter_windowTooLarge);
fparamsPtr->frameContentSize = frameContentSize; fparamsPtr->frameContentSize = frameContentSize;
fparamsPtr->windowSize = windowSize; fparamsPtr->windowSize = windowSize;
fparamsPtr->dictID = dictID; fparamsPtr->dictID = dictID;
@@ -274,7 +297,8 @@ unsigned long long ZSTD_getFrameContentSize(const void *src, size_t srcSize)
{ {
{ {
ZSTD_frameParams fParams; ZSTD_frameParams fParams;
if (ZSTD_getFrameParams(&fParams, src, srcSize) != 0) return ZSTD_CONTENTSIZE_ERROR; if (ZSTD_getFrameParams(&fParams, src, srcSize) != 0)
return ZSTD_CONTENTSIZE_ERROR;
if (fParams.windowSize == 0) { if (fParams.windowSize == 0) {
/* Either skippable or empty frame, size == 0 either way */ /* Either skippable or empty frame, size == 0 either way */
return 0; return 0;
@@ -291,7 +315,7 @@ unsigned long long ZSTD_getFrameContentSize(const void *src, size_t srcSize)
* `srcSize` must be the exact length of some number of ZSTD compressed and/or * `srcSize` must be the exact length of some number of ZSTD compressed and/or
* skippable frames * skippable frames
* @return : decompressed size of the frames contained */ * @return : decompressed size of the frames contained */
unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize) unsigned long long ZSTD_findDecompressedSize(const void *src, size_t srcSize)
{ {
{ {
unsigned long long totalDstSize = 0; unsigned long long totalDstSize = 0;
@@ -302,8 +326,7 @@ unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize)
size_t skippableSize; size_t skippableSize;
if (srcSize < ZSTD_skippableHeaderSize) if (srcSize < ZSTD_skippableHeaderSize)
return ERROR(srcSize_wrong); return ERROR(srcSize_wrong);
skippableSize = ZSTD_readLE32((const BYTE *)src + 4) + skippableSize = ZSTD_readLE32((const BYTE *)src + 4) + ZSTD_skippableHeaderSize;
ZSTD_skippableHeaderSize;
if (srcSize < skippableSize) { if (srcSize < skippableSize) {
return ZSTD_CONTENTSIZE_ERROR; return ZSTD_CONTENTSIZE_ERROR;
} }
@@ -315,10 +338,12 @@ unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize)
{ {
unsigned long long const ret = ZSTD_getFrameContentSize(src, srcSize); unsigned long long const ret = ZSTD_getFrameContentSize(src, srcSize);
if (ret >= ZSTD_CONTENTSIZE_ERROR) return ret; if (ret >= ZSTD_CONTENTSIZE_ERROR)
return ret;
/* check for overflow */ /* check for overflow */
if (totalDstSize + ret < totalDstSize) return ZSTD_CONTENTSIZE_ERROR; if (totalDstSize + ret < totalDstSize)
return ZSTD_CONTENTSIZE_ERROR;
totalDstSize += ret; totalDstSize += ret;
} }
{ {
@@ -343,19 +368,21 @@ unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize)
/** ZSTD_decodeFrameHeader() : /** ZSTD_decodeFrameHeader() :
* `headerSize` must be the size provided by ZSTD_frameHeaderSize(). * `headerSize` must be the size provided by ZSTD_frameHeaderSize().
* @return : 0 if success, or an error code, which can be tested using ZSTD_isError() */ * @return : 0 if success, or an error code, which can be tested using ZSTD_isError() */
static size_t ZSTD_decodeFrameHeader(ZSTD_DCtx* dctx, const void* src, size_t headerSize) static size_t ZSTD_decodeFrameHeader(ZSTD_DCtx *dctx, const void *src, size_t headerSize)
{ {
size_t const result = ZSTD_getFrameParams(&(dctx->fParams), src, headerSize); size_t const result = ZSTD_getFrameParams(&(dctx->fParams), src, headerSize);
if (ZSTD_isError(result)) return result; /* invalid header */ if (ZSTD_isError(result))
if (result>0) return ERROR(srcSize_wrong); /* headerSize too small */ return result; /* invalid header */
if (dctx->fParams.dictID && (dctx->dictID != dctx->fParams.dictID)) return ERROR(dictionary_wrong); if (result > 0)
if (dctx->fParams.checksumFlag) xxh64_reset(&dctx->xxhState, 0); return ERROR(srcSize_wrong); /* headerSize too small */
if (dctx->fParams.dictID && (dctx->dictID != dctx->fParams.dictID))
return ERROR(dictionary_wrong);
if (dctx->fParams.checksumFlag)
xxh64_reset(&dctx->xxhState, 0);
return 0; return 0;
} }
typedef struct {
typedef struct
{
blockType_e blockType; blockType_e blockType;
U32 lastBlock; U32 lastBlock;
U32 origSize; U32 origSize;
@@ -363,61 +390,70 @@ typedef struct
/*! ZSTD_getcBlockSize() : /*! ZSTD_getcBlockSize() :
* Provides the size of compressed block from block header `src` */ * Provides the size of compressed block from block header `src` */
size_t ZSTD_getcBlockSize(const void* src, size_t srcSize, blockProperties_t* bpPtr) size_t ZSTD_getcBlockSize(const void *src, size_t srcSize, blockProperties_t *bpPtr)
{ {
if (srcSize < ZSTD_blockHeaderSize) return ERROR(srcSize_wrong); if (srcSize < ZSTD_blockHeaderSize)
{ U32 const cBlockHeader = ZSTD_readLE24(src); return ERROR(srcSize_wrong);
{
U32 const cBlockHeader = ZSTD_readLE24(src);
U32 const cSize = cBlockHeader >> 3; U32 const cSize = cBlockHeader >> 3;
bpPtr->lastBlock = cBlockHeader & 1; bpPtr->lastBlock = cBlockHeader & 1;
bpPtr->blockType = (blockType_e)((cBlockHeader >> 1) & 3); bpPtr->blockType = (blockType_e)((cBlockHeader >> 1) & 3);
bpPtr->origSize = cSize; /* only useful for RLE */ bpPtr->origSize = cSize; /* only useful for RLE */
if (bpPtr->blockType == bt_rle) return 1; if (bpPtr->blockType == bt_rle)
if (bpPtr->blockType == bt_reserved) return ERROR(corruption_detected); return 1;
if (bpPtr->blockType == bt_reserved)
return ERROR(corruption_detected);
return cSize; return cSize;
} }
} }
static size_t ZSTD_copyRawBlock(void *dst, size_t dstCapacity, const void *src, size_t srcSize)
static size_t ZSTD_copyRawBlock(void* dst, size_t dstCapacity, const void* src, size_t srcSize)
{ {
if (srcSize > dstCapacity) return ERROR(dstSize_tooSmall); if (srcSize > dstCapacity)
return ERROR(dstSize_tooSmall);
memcpy(dst, src, srcSize); memcpy(dst, src, srcSize);
return srcSize; return srcSize;
} }
static size_t ZSTD_setRleBlock(void *dst, size_t dstCapacity, const void *src, size_t srcSize, size_t regenSize)
static size_t ZSTD_setRleBlock(void* dst, size_t dstCapacity, const void* src, size_t srcSize, size_t regenSize)
{ {
if (srcSize != 1) return ERROR(srcSize_wrong); if (srcSize != 1)
if (regenSize > dstCapacity) return ERROR(dstSize_tooSmall); return ERROR(srcSize_wrong);
memset(dst, *(const BYTE*)src, regenSize); if (regenSize > dstCapacity)
return ERROR(dstSize_tooSmall);
memset(dst, *(const BYTE *)src, regenSize);
return regenSize; return regenSize;
} }
/*! ZSTD_decodeLiteralsBlock() : /*! ZSTD_decodeLiteralsBlock() :
@return : nb of bytes read from src (< srcSize ) */ @return : nb of bytes read from src (< srcSize ) */
size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx, size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx *dctx, const void *src, size_t srcSize) /* note : srcSize < BLOCKSIZE */
const void* src, size_t srcSize) /* note : srcSize < BLOCKSIZE */
{ {
if (srcSize < MIN_CBLOCK_SIZE) return ERROR(corruption_detected); if (srcSize < MIN_CBLOCK_SIZE)
return ERROR(corruption_detected);
{ const BYTE* const istart = (const BYTE*) src; {
const BYTE *const istart = (const BYTE *)src;
symbolEncodingType_e const litEncType = (symbolEncodingType_e)(istart[0] & 3); symbolEncodingType_e const litEncType = (symbolEncodingType_e)(istart[0] & 3);
switch(litEncType) switch (litEncType) {
{
case set_repeat: case set_repeat:
if (dctx->litEntropy==0) return ERROR(dictionary_corrupted); if (dctx->litEntropy == 0)
return ERROR(dictionary_corrupted);
/* fall-through */ /* fall-through */
case set_compressed: case set_compressed:
if (srcSize < 5) return ERROR(corruption_detected); /* srcSize >= MIN_CBLOCK_SIZE == 3; here we need up to 5 for case 3 */ if (srcSize < 5)
{ size_t lhSize, litSize, litCSize; return ERROR(corruption_detected); /* srcSize >= MIN_CBLOCK_SIZE == 3; here we need up to 5 for case 3 */
U32 singleStream=0; {
size_t lhSize, litSize, litCSize;
U32 singleStream = 0;
U32 const lhlCode = (istart[0] >> 2) & 3; U32 const lhlCode = (istart[0] >> 2) & 3;
U32 const lhc = ZSTD_readLE32(istart); U32 const lhc = ZSTD_readLE32(istart);
switch(lhlCode) switch (lhlCode) {
{ case 0:
case 0: case 1: default: /* note : default is impossible, since lhlCode into [0..3] */ case 1:
default: /* note : default is impossible, since lhlCode into [0..3] */
/* 2 - 2 - 10 - 10 */ /* 2 - 2 - 10 - 10 */
singleStream = !lhlCode; singleStream = !lhlCode;
lhSize = 3; lhSize = 3;
@@ -437,32 +473,36 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx,
litCSize = (lhc >> 22) + (istart[4] << 10); litCSize = (lhc >> 22) + (istart[4] << 10);
break; break;
} }
if (litSize > ZSTD_BLOCKSIZE_ABSOLUTEMAX) return ERROR(corruption_detected); if (litSize > ZSTD_BLOCKSIZE_ABSOLUTEMAX)
if (litCSize + lhSize > srcSize) return ERROR(corruption_detected); return ERROR(corruption_detected);
if (litCSize + lhSize > srcSize)
return ERROR(corruption_detected);
if (HUF_isError((litEncType==set_repeat) ? if (HUF_isError(
( singleStream ? (litEncType == set_repeat)
HUF_decompress1X_usingDTable(dctx->litBuffer, litSize, istart+lhSize, litCSize, dctx->HUFptr) : ? (singleStream ? HUF_decompress1X_usingDTable(dctx->litBuffer, litSize, istart + lhSize, litCSize, dctx->HUFptr)
HUF_decompress4X_usingDTable(dctx->litBuffer, litSize, istart+lhSize, litCSize, dctx->HUFptr) ) : : HUF_decompress4X_usingDTable(dctx->litBuffer, litSize, istart + lhSize, litCSize, dctx->HUFptr))
( singleStream ? : (singleStream
HUF_decompress1X2_DCtx(dctx->entropy.hufTable, dctx->litBuffer, litSize, istart+lhSize, litCSize) : ? HUF_decompress1X2_DCtx(dctx->entropy.hufTable, dctx->litBuffer, litSize, istart + lhSize, litCSize)
HUF_decompress4X_hufOnly (dctx->entropy.hufTable, dctx->litBuffer, litSize, istart+lhSize, litCSize)) )) : HUF_decompress4X_hufOnly(dctx->entropy.hufTable, dctx->litBuffer, litSize, istart + lhSize, litCSize))))
return ERROR(corruption_detected); return ERROR(corruption_detected);
dctx->litPtr = dctx->litBuffer; dctx->litPtr = dctx->litBuffer;
dctx->litSize = litSize; dctx->litSize = litSize;
dctx->litEntropy = 1; dctx->litEntropy = 1;
if (litEncType==set_compressed) dctx->HUFptr = dctx->entropy.hufTable; if (litEncType == set_compressed)
dctx->HUFptr = dctx->entropy.hufTable;
memset(dctx->litBuffer + dctx->litSize, 0, WILDCOPY_OVERLENGTH); memset(dctx->litBuffer + dctx->litSize, 0, WILDCOPY_OVERLENGTH);
return litCSize + lhSize; return litCSize + lhSize;
} }
case set_basic: case set_basic: {
{ size_t litSize, lhSize; size_t litSize, lhSize;
U32 const lhlCode = ((istart[0]) >> 2) & 3; U32 const lhlCode = ((istart[0]) >> 2) & 3;
switch(lhlCode) switch (lhlCode) {
{ case 0:
case 0: case 2: default: /* note : default is impossible, since lhlCode into [0..3] */ case 2:
default: /* note : default is impossible, since lhlCode into [0..3] */
lhSize = 1; lhSize = 1;
litSize = istart[0] >> 3; litSize = istart[0] >> 3;
break; break;
@@ -476,26 +516,28 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx,
break; break;
} }
if (lhSize+litSize+WILDCOPY_OVERLENGTH > srcSize) { /* risk reading beyond src buffer with wildcopy */ if (lhSize + litSize + WILDCOPY_OVERLENGTH > srcSize) { /* risk reading beyond src buffer with wildcopy */
if (litSize+lhSize > srcSize) return ERROR(corruption_detected); if (litSize + lhSize > srcSize)
memcpy(dctx->litBuffer, istart+lhSize, litSize); return ERROR(corruption_detected);
memcpy(dctx->litBuffer, istart + lhSize, litSize);
dctx->litPtr = dctx->litBuffer; dctx->litPtr = dctx->litBuffer;
dctx->litSize = litSize; dctx->litSize = litSize;
memset(dctx->litBuffer + dctx->litSize, 0, WILDCOPY_OVERLENGTH); memset(dctx->litBuffer + dctx->litSize, 0, WILDCOPY_OVERLENGTH);
return lhSize+litSize; return lhSize + litSize;
} }
/* direct reference into compressed stream */ /* direct reference into compressed stream */
dctx->litPtr = istart+lhSize; dctx->litPtr = istart + lhSize;
dctx->litSize = litSize; dctx->litSize = litSize;
return lhSize+litSize; return lhSize + litSize;
} }
case set_rle: case set_rle: {
{ U32 const lhlCode = ((istart[0]) >> 2) & 3; U32 const lhlCode = ((istart[0]) >> 2) & 3;
size_t litSize, lhSize; size_t litSize, lhSize;
switch(lhlCode) switch (lhlCode) {
{ case 0:
case 0: case 2: default: /* note : default is impossible, since lhlCode into [0..3] */ case 2:
default: /* note : default is impossible, since lhlCode into [0..3] */
lhSize = 1; lhSize = 1;
litSize = istart[0] >> 3; litSize = istart[0] >> 3;
break; break;
@@ -506,14 +548,16 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx,
case 3: case 3:
lhSize = 3; lhSize = 3;
litSize = ZSTD_readLE24(istart) >> 4; litSize = ZSTD_readLE24(istart) >> 4;
if (srcSize<4) return ERROR(corruption_detected); /* srcSize >= MIN_CBLOCK_SIZE == 3; here we need lhSize+1 = 4 */ if (srcSize < 4)
return ERROR(corruption_detected); /* srcSize >= MIN_CBLOCK_SIZE == 3; here we need lhSize+1 = 4 */
break; break;
} }
if (litSize > ZSTD_BLOCKSIZE_ABSOLUTEMAX) return ERROR(corruption_detected); if (litSize > ZSTD_BLOCKSIZE_ABSOLUTEMAX)
return ERROR(corruption_detected);
memset(dctx->litBuffer, istart[lhSize], litSize + WILDCOPY_OVERLENGTH); memset(dctx->litBuffer, istart[lhSize], litSize + WILDCOPY_OVERLENGTH);
dctx->litPtr = dctx->litBuffer; dctx->litPtr = dctx->litBuffer;
dctx->litSize = litSize; dctx->litSize = litSize;
return lhSize+1; return lhSize + 1;
} }
default: default:
return ERROR(corruption_detected); /* impossible */ return ERROR(corruption_detected); /* impossible */
@@ -521,283 +565,293 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx,
} }
} }
typedef union { typedef union {
FSE_decode_t realData; FSE_decode_t realData;
U32 alignedBy4; U32 alignedBy4;
} FSE_decode_t4; } FSE_decode_t4;
static const FSE_decode_t4 LL_defaultDTable[(1<<LL_DEFAULTNORMLOG)+1] = { static const FSE_decode_t4 LL_defaultDTable[(1 << LL_DEFAULTNORMLOG) + 1] = {
{ { LL_DEFAULTNORMLOG, 1, 1 } }, /* header : tableLog, fastMode, fastMode */ {{LL_DEFAULTNORMLOG, 1, 1}}, /* header : tableLog, fastMode, fastMode */
{ { 0, 0, 4 } }, /* 0 : base, symbol, bits */ {{0, 0, 4}}, /* 0 : base, symbol, bits */
{ { 16, 0, 4 } }, {{16, 0, 4}},
{ { 32, 1, 5 } }, {{32, 1, 5}},
{ { 0, 3, 5 } }, {{0, 3, 5}},
{ { 0, 4, 5 } }, {{0, 4, 5}},
{ { 0, 6, 5 } }, {{0, 6, 5}},
{ { 0, 7, 5 } }, {{0, 7, 5}},
{ { 0, 9, 5 } }, {{0, 9, 5}},
{ { 0, 10, 5 } }, {{0, 10, 5}},
{ { 0, 12, 5 } }, {{0, 12, 5}},
{ { 0, 14, 6 } }, {{0, 14, 6}},
{ { 0, 16, 5 } }, {{0, 16, 5}},
{ { 0, 18, 5 } }, {{0, 18, 5}},
{ { 0, 19, 5 } }, {{0, 19, 5}},
{ { 0, 21, 5 } }, {{0, 21, 5}},
{ { 0, 22, 5 } }, {{0, 22, 5}},
{ { 0, 24, 5 } }, {{0, 24, 5}},
{ { 32, 25, 5 } }, {{32, 25, 5}},
{ { 0, 26, 5 } }, {{0, 26, 5}},
{ { 0, 27, 6 } }, {{0, 27, 6}},
{ { 0, 29, 6 } }, {{0, 29, 6}},
{ { 0, 31, 6 } }, {{0, 31, 6}},
{ { 32, 0, 4 } }, {{32, 0, 4}},
{ { 0, 1, 4 } }, {{0, 1, 4}},
{ { 0, 2, 5 } }, {{0, 2, 5}},
{ { 32, 4, 5 } }, {{32, 4, 5}},
{ { 0, 5, 5 } }, {{0, 5, 5}},
{ { 32, 7, 5 } }, {{32, 7, 5}},
{ { 0, 8, 5 } }, {{0, 8, 5}},
{ { 32, 10, 5 } }, {{32, 10, 5}},
{ { 0, 11, 5 } }, {{0, 11, 5}},
{ { 0, 13, 6 } }, {{0, 13, 6}},
{ { 32, 16, 5 } }, {{32, 16, 5}},
{ { 0, 17, 5 } }, {{0, 17, 5}},
{ { 32, 19, 5 } }, {{32, 19, 5}},
{ { 0, 20, 5 } }, {{0, 20, 5}},
{ { 32, 22, 5 } }, {{32, 22, 5}},
{ { 0, 23, 5 } }, {{0, 23, 5}},
{ { 0, 25, 4 } }, {{0, 25, 4}},
{ { 16, 25, 4 } }, {{16, 25, 4}},
{ { 32, 26, 5 } }, {{32, 26, 5}},
{ { 0, 28, 6 } }, {{0, 28, 6}},
{ { 0, 30, 6 } }, {{0, 30, 6}},
{ { 48, 0, 4 } }, {{48, 0, 4}},
{ { 16, 1, 4 } }, {{16, 1, 4}},
{ { 32, 2, 5 } }, {{32, 2, 5}},
{ { 32, 3, 5 } }, {{32, 3, 5}},
{ { 32, 5, 5 } }, {{32, 5, 5}},
{ { 32, 6, 5 } }, {{32, 6, 5}},
{ { 32, 8, 5 } }, {{32, 8, 5}},
{ { 32, 9, 5 } }, {{32, 9, 5}},
{ { 32, 11, 5 } }, {{32, 11, 5}},
{ { 32, 12, 5 } }, {{32, 12, 5}},
{ { 0, 15, 6 } }, {{0, 15, 6}},
{ { 32, 17, 5 } }, {{32, 17, 5}},
{ { 32, 18, 5 } }, {{32, 18, 5}},
{ { 32, 20, 5 } }, {{32, 20, 5}},
{ { 32, 21, 5 } }, {{32, 21, 5}},
{ { 32, 23, 5 } }, {{32, 23, 5}},
{ { 32, 24, 5 } }, {{32, 24, 5}},
{ { 0, 35, 6 } }, {{0, 35, 6}},
{ { 0, 34, 6 } }, {{0, 34, 6}},
{ { 0, 33, 6 } }, {{0, 33, 6}},
{ { 0, 32, 6 } }, {{0, 32, 6}},
}; /* LL_defaultDTable */ }; /* LL_defaultDTable */
static const FSE_decode_t4 ML_defaultDTable[(1<<ML_DEFAULTNORMLOG)+1] = { static const FSE_decode_t4 ML_defaultDTable[(1 << ML_DEFAULTNORMLOG) + 1] = {
{ { ML_DEFAULTNORMLOG, 1, 1 } }, /* header : tableLog, fastMode, fastMode */ {{ML_DEFAULTNORMLOG, 1, 1}}, /* header : tableLog, fastMode, fastMode */
{ { 0, 0, 6 } }, /* 0 : base, symbol, bits */ {{0, 0, 6}}, /* 0 : base, symbol, bits */
{ { 0, 1, 4 } }, {{0, 1, 4}},
{ { 32, 2, 5 } }, {{32, 2, 5}},
{ { 0, 3, 5 } }, {{0, 3, 5}},
{ { 0, 5, 5 } }, {{0, 5, 5}},
{ { 0, 6, 5 } }, {{0, 6, 5}},
{ { 0, 8, 5 } }, {{0, 8, 5}},
{ { 0, 10, 6 } }, {{0, 10, 6}},
{ { 0, 13, 6 } }, {{0, 13, 6}},
{ { 0, 16, 6 } }, {{0, 16, 6}},
{ { 0, 19, 6 } }, {{0, 19, 6}},
{ { 0, 22, 6 } }, {{0, 22, 6}},
{ { 0, 25, 6 } }, {{0, 25, 6}},
{ { 0, 28, 6 } }, {{0, 28, 6}},
{ { 0, 31, 6 } }, {{0, 31, 6}},
{ { 0, 33, 6 } }, {{0, 33, 6}},
{ { 0, 35, 6 } }, {{0, 35, 6}},
{ { 0, 37, 6 } }, {{0, 37, 6}},
{ { 0, 39, 6 } }, {{0, 39, 6}},
{ { 0, 41, 6 } }, {{0, 41, 6}},
{ { 0, 43, 6 } }, {{0, 43, 6}},
{ { 0, 45, 6 } }, {{0, 45, 6}},
{ { 16, 1, 4 } }, {{16, 1, 4}},
{ { 0, 2, 4 } }, {{0, 2, 4}},
{ { 32, 3, 5 } }, {{32, 3, 5}},
{ { 0, 4, 5 } }, {{0, 4, 5}},
{ { 32, 6, 5 } }, {{32, 6, 5}},
{ { 0, 7, 5 } }, {{0, 7, 5}},
{ { 0, 9, 6 } }, {{0, 9, 6}},
{ { 0, 12, 6 } }, {{0, 12, 6}},
{ { 0, 15, 6 } }, {{0, 15, 6}},
{ { 0, 18, 6 } }, {{0, 18, 6}},
{ { 0, 21, 6 } }, {{0, 21, 6}},
{ { 0, 24, 6 } }, {{0, 24, 6}},
{ { 0, 27, 6 } }, {{0, 27, 6}},
{ { 0, 30, 6 } }, {{0, 30, 6}},
{ { 0, 32, 6 } }, {{0, 32, 6}},
{ { 0, 34, 6 } }, {{0, 34, 6}},
{ { 0, 36, 6 } }, {{0, 36, 6}},
{ { 0, 38, 6 } }, {{0, 38, 6}},
{ { 0, 40, 6 } }, {{0, 40, 6}},
{ { 0, 42, 6 } }, {{0, 42, 6}},
{ { 0, 44, 6 } }, {{0, 44, 6}},
{ { 32, 1, 4 } }, {{32, 1, 4}},
{ { 48, 1, 4 } }, {{48, 1, 4}},
{ { 16, 2, 4 } }, {{16, 2, 4}},
{ { 32, 4, 5 } }, {{32, 4, 5}},
{ { 32, 5, 5 } }, {{32, 5, 5}},
{ { 32, 7, 5 } }, {{32, 7, 5}},
{ { 32, 8, 5 } }, {{32, 8, 5}},
{ { 0, 11, 6 } }, {{0, 11, 6}},
{ { 0, 14, 6 } }, {{0, 14, 6}},
{ { 0, 17, 6 } }, {{0, 17, 6}},
{ { 0, 20, 6 } }, {{0, 20, 6}},
{ { 0, 23, 6 } }, {{0, 23, 6}},
{ { 0, 26, 6 } }, {{0, 26, 6}},
{ { 0, 29, 6 } }, {{0, 29, 6}},
{ { 0, 52, 6 } }, {{0, 52, 6}},
{ { 0, 51, 6 } }, {{0, 51, 6}},
{ { 0, 50, 6 } }, {{0, 50, 6}},
{ { 0, 49, 6 } }, {{0, 49, 6}},
{ { 0, 48, 6 } }, {{0, 48, 6}},
{ { 0, 47, 6 } }, {{0, 47, 6}},
{ { 0, 46, 6 } }, {{0, 46, 6}},
}; /* ML_defaultDTable */ }; /* ML_defaultDTable */
static const FSE_decode_t4 OF_defaultDTable[(1<<OF_DEFAULTNORMLOG)+1] = { static const FSE_decode_t4 OF_defaultDTable[(1 << OF_DEFAULTNORMLOG) + 1] = {
{ { OF_DEFAULTNORMLOG, 1, 1 } }, /* header : tableLog, fastMode, fastMode */ {{OF_DEFAULTNORMLOG, 1, 1}}, /* header : tableLog, fastMode, fastMode */
{ { 0, 0, 5 } }, /* 0 : base, symbol, bits */ {{0, 0, 5}}, /* 0 : base, symbol, bits */
{ { 0, 6, 4 } }, {{0, 6, 4}},
{ { 0, 9, 5 } }, {{0, 9, 5}},
{ { 0, 15, 5 } }, {{0, 15, 5}},
{ { 0, 21, 5 } }, {{0, 21, 5}},
{ { 0, 3, 5 } }, {{0, 3, 5}},
{ { 0, 7, 4 } }, {{0, 7, 4}},
{ { 0, 12, 5 } }, {{0, 12, 5}},
{ { 0, 18, 5 } }, {{0, 18, 5}},
{ { 0, 23, 5 } }, {{0, 23, 5}},
{ { 0, 5, 5 } }, {{0, 5, 5}},
{ { 0, 8, 4 } }, {{0, 8, 4}},
{ { 0, 14, 5 } }, {{0, 14, 5}},
{ { 0, 20, 5 } }, {{0, 20, 5}},
{ { 0, 2, 5 } }, {{0, 2, 5}},
{ { 16, 7, 4 } }, {{16, 7, 4}},
{ { 0, 11, 5 } }, {{0, 11, 5}},
{ { 0, 17, 5 } }, {{0, 17, 5}},
{ { 0, 22, 5 } }, {{0, 22, 5}},
{ { 0, 4, 5 } }, {{0, 4, 5}},
{ { 16, 8, 4 } }, {{16, 8, 4}},
{ { 0, 13, 5 } }, {{0, 13, 5}},
{ { 0, 19, 5 } }, {{0, 19, 5}},
{ { 0, 1, 5 } }, {{0, 1, 5}},
{ { 16, 6, 4 } }, {{16, 6, 4}},
{ { 0, 10, 5 } }, {{0, 10, 5}},
{ { 0, 16, 5 } }, {{0, 16, 5}},
{ { 0, 28, 5 } }, {{0, 28, 5}},
{ { 0, 27, 5 } }, {{0, 27, 5}},
{ { 0, 26, 5 } }, {{0, 26, 5}},
{ { 0, 25, 5 } }, {{0, 25, 5}},
{ { 0, 24, 5 } }, {{0, 24, 5}},
}; /* OF_defaultDTable */ }; /* OF_defaultDTable */
/*! ZSTD_buildSeqTable() : /*! ZSTD_buildSeqTable() :
@return : nb bytes read from src, @return : nb bytes read from src,
or an error code if it fails, testable with ZSTD_isError() or an error code if it fails, testable with ZSTD_isError()
*/ */
static size_t ZSTD_buildSeqTable(FSE_DTable* DTableSpace, const FSE_DTable** DTablePtr, static size_t ZSTD_buildSeqTable(FSE_DTable *DTableSpace, const FSE_DTable **DTablePtr, symbolEncodingType_e type, U32 max, U32 maxLog, const void *src,
symbolEncodingType_e type, U32 max, U32 maxLog, size_t srcSize, const FSE_decode_t4 *defaultTable, U32 flagRepeatTable)
const void* src, size_t srcSize,
const FSE_decode_t4* defaultTable, U32 flagRepeatTable)
{ {
const void* const tmpPtr = defaultTable; /* bypass strict aliasing */ const void *const tmpPtr = defaultTable; /* bypass strict aliasing */
switch(type) switch (type) {
{ case set_rle:
case set_rle : if (!srcSize)
if (!srcSize) return ERROR(srcSize_wrong); return ERROR(srcSize_wrong);
if ( (*(const BYTE*)src) > max) return ERROR(corruption_detected); if ((*(const BYTE *)src) > max)
FSE_buildDTable_rle(DTableSpace, *(const BYTE*)src); return ERROR(corruption_detected);
FSE_buildDTable_rle(DTableSpace, *(const BYTE *)src);
*DTablePtr = DTableSpace; *DTablePtr = DTableSpace;
return 1; return 1;
case set_basic : case set_basic: *DTablePtr = (const FSE_DTable *)tmpPtr; return 0;
*DTablePtr = (const FSE_DTable*)tmpPtr;
return 0;
case set_repeat: case set_repeat:
if (!flagRepeatTable) return ERROR(corruption_detected); if (!flagRepeatTable)
return ERROR(corruption_detected);
return 0; return 0;
default : /* impossible */ default: /* impossible */
case set_compressed : case set_compressed: {
{ U32 tableLog; U32 tableLog;
S16 norm[MaxSeq+1]; S16 norm[MaxSeq + 1];
size_t const headerSize = FSE_readNCount(norm, &max, &tableLog, src, srcSize); size_t const headerSize = FSE_readNCount(norm, &max, &tableLog, src, srcSize);
if (FSE_isError(headerSize)) return ERROR(corruption_detected); if (FSE_isError(headerSize))
if (tableLog > maxLog) return ERROR(corruption_detected); return ERROR(corruption_detected);
if (tableLog > maxLog)
return ERROR(corruption_detected);
FSE_buildDTable(DTableSpace, norm, max, tableLog); FSE_buildDTable(DTableSpace, norm, max, tableLog);
*DTablePtr = DTableSpace; *DTablePtr = DTableSpace;
return headerSize; return headerSize;
} } }
}
} }
size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeqPtr, size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx *dctx, int *nbSeqPtr, const void *src, size_t srcSize)
const void* src, size_t srcSize)
{ {
const BYTE* const istart = (const BYTE* const)src; const BYTE *const istart = (const BYTE *const)src;
const BYTE* const iend = istart + srcSize; const BYTE *const iend = istart + srcSize;
const BYTE* ip = istart; const BYTE *ip = istart;
/* check */ /* check */
if (srcSize < MIN_SEQUENCES_SIZE) return ERROR(srcSize_wrong); if (srcSize < MIN_SEQUENCES_SIZE)
return ERROR(srcSize_wrong);
/* SeqHead */ /* SeqHead */
{ int nbSeq = *ip++; {
if (!nbSeq) { *nbSeqPtr=0; return 1; } int nbSeq = *ip++;
if (!nbSeq) {
*nbSeqPtr = 0;
return 1;
}
if (nbSeq > 0x7F) { if (nbSeq > 0x7F) {
if (nbSeq == 0xFF) { if (nbSeq == 0xFF) {
if (ip+2 > iend) return ERROR(srcSize_wrong); if (ip + 2 > iend)
nbSeq = ZSTD_readLE16(ip) + LONGNBSEQ, ip+=2; return ERROR(srcSize_wrong);
nbSeq = ZSTD_readLE16(ip) + LONGNBSEQ, ip += 2;
} else { } else {
if (ip >= iend) return ERROR(srcSize_wrong); if (ip >= iend)
nbSeq = ((nbSeq-0x80)<<8) + *ip++; return ERROR(srcSize_wrong);
nbSeq = ((nbSeq - 0x80) << 8) + *ip++;
} }
} }
*nbSeqPtr = nbSeq; *nbSeqPtr = nbSeq;
} }
/* FSE table descriptors */ /* FSE table descriptors */
if (ip+4 > iend) return ERROR(srcSize_wrong); /* minimum possible size */ if (ip + 4 > iend)
{ symbolEncodingType_e const LLtype = (symbolEncodingType_e)(*ip >> 6); return ERROR(srcSize_wrong); /* minimum possible size */
{
symbolEncodingType_e const LLtype = (symbolEncodingType_e)(*ip >> 6);
symbolEncodingType_e const OFtype = (symbolEncodingType_e)((*ip >> 4) & 3); symbolEncodingType_e const OFtype = (symbolEncodingType_e)((*ip >> 4) & 3);
symbolEncodingType_e const MLtype = (symbolEncodingType_e)((*ip >> 2) & 3); symbolEncodingType_e const MLtype = (symbolEncodingType_e)((*ip >> 2) & 3);
ip++; ip++;
/* Build DTables */ /* Build DTables */
{ size_t const llhSize = ZSTD_buildSeqTable(dctx->entropy.LLTable, &dctx->LLTptr, {
LLtype, MaxLL, LLFSELog, size_t const llhSize = ZSTD_buildSeqTable(dctx->entropy.LLTable, &dctx->LLTptr, LLtype, MaxLL, LLFSELog, ip, iend - ip,
ip, iend-ip, LL_defaultDTable, dctx->fseEntropy); LL_defaultDTable, dctx->fseEntropy);
if (ZSTD_isError(llhSize)) return ERROR(corruption_detected); if (ZSTD_isError(llhSize))
return ERROR(corruption_detected);
ip += llhSize; ip += llhSize;
} }
{ size_t const ofhSize = ZSTD_buildSeqTable(dctx->entropy.OFTable, &dctx->OFTptr, {
OFtype, MaxOff, OffFSELog, size_t const ofhSize = ZSTD_buildSeqTable(dctx->entropy.OFTable, &dctx->OFTptr, OFtype, MaxOff, OffFSELog, ip, iend - ip,
ip, iend-ip, OF_defaultDTable, dctx->fseEntropy); OF_defaultDTable, dctx->fseEntropy);
if (ZSTD_isError(ofhSize)) return ERROR(corruption_detected); if (ZSTD_isError(ofhSize))
return ERROR(corruption_detected);
ip += ofhSize; ip += ofhSize;
} }
{ size_t const mlhSize = ZSTD_buildSeqTable(dctx->entropy.MLTable, &dctx->MLTptr, {
MLtype, MaxML, MLFSELog, size_t const mlhSize = ZSTD_buildSeqTable(dctx->entropy.MLTable, &dctx->MLTptr, MLtype, MaxML, MLFSELog, ip, iend - ip,
ip, iend-ip, ML_defaultDTable, dctx->fseEntropy); ML_defaultDTable, dctx->fseEntropy);
if (ZSTD_isError(mlhSize)) return ERROR(corruption_detected); if (ZSTD_isError(mlhSize))
return ERROR(corruption_detected);
ip += mlhSize; ip += mlhSize;
} }
} }
return ip-istart; return ip - istart;
} }
typedef struct { typedef struct {
size_t litLength; size_t litLength;
size_t matchLength; size_t matchLength;
size_t offset; size_t offset;
const BYTE* match; const BYTE *match;
} seq_t; } seq_t;
typedef struct { typedef struct {
@@ -806,29 +860,29 @@ typedef struct {
FSE_DState_t stateOffb; FSE_DState_t stateOffb;
FSE_DState_t stateML; FSE_DState_t stateML;
size_t prevOffset[ZSTD_REP_NUM]; size_t prevOffset[ZSTD_REP_NUM];
const BYTE* base; const BYTE *base;
size_t pos; size_t pos;
uPtrDiff gotoDict; uPtrDiff gotoDict;
} seqState_t; } seqState_t;
FORCE_NOINLINE FORCE_NOINLINE
size_t ZSTD_execSequenceLast7(BYTE* op, size_t ZSTD_execSequenceLast7(BYTE *op, BYTE *const oend, seq_t sequence, const BYTE **litPtr, const BYTE *const litLimit, const BYTE *const base,
BYTE* const oend, seq_t sequence, const BYTE *const vBase, const BYTE *const dictEnd)
const BYTE** litPtr, const BYTE* const litLimit,
const BYTE* const base, const BYTE* const vBase, const BYTE* const dictEnd)
{ {
BYTE* const oLitEnd = op + sequence.litLength; BYTE *const oLitEnd = op + sequence.litLength;
size_t const sequenceLength = sequence.litLength + sequence.matchLength; size_t const sequenceLength = sequence.litLength + sequence.matchLength;
BYTE* const oMatchEnd = op + sequenceLength; /* risk : address space overflow (32-bits) */ BYTE *const oMatchEnd = op + sequenceLength; /* risk : address space overflow (32-bits) */
BYTE* const oend_w = oend - WILDCOPY_OVERLENGTH; BYTE *const oend_w = oend - WILDCOPY_OVERLENGTH;
const BYTE* const iLitEnd = *litPtr + sequence.litLength; const BYTE *const iLitEnd = *litPtr + sequence.litLength;
const BYTE* match = oLitEnd - sequence.offset; const BYTE *match = oLitEnd - sequence.offset;
/* check */ /* check */
if (oMatchEnd>oend) return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of WILDCOPY_OVERLENGTH from oend */ if (oMatchEnd > oend)
if (iLitEnd > litLimit) return ERROR(corruption_detected); /* over-read beyond lit buffer */ return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of WILDCOPY_OVERLENGTH from oend */
if (oLitEnd <= oend_w) return ERROR(GENERIC); /* Precondition */ if (iLitEnd > litLimit)
return ERROR(corruption_detected); /* over-read beyond lit buffer */
if (oLitEnd <= oend_w)
return ERROR(GENERIC); /* Precondition */
/* copy literals */ /* copy literals */
if (op < oend_w) { if (op < oend_w) {
@@ -836,32 +890,34 @@ size_t ZSTD_execSequenceLast7(BYTE* op,
*litPtr += oend_w - op; *litPtr += oend_w - op;
op = oend_w; op = oend_w;
} }
while (op < oLitEnd) *op++ = *(*litPtr)++; while (op < oLitEnd)
*op++ = *(*litPtr)++;
/* copy Match */ /* copy Match */
if (sequence.offset > (size_t)(oLitEnd - base)) { if (sequence.offset > (size_t)(oLitEnd - base)) {
/* offset beyond prefix */ /* offset beyond prefix */
if (sequence.offset > (size_t)(oLitEnd - vBase)) return ERROR(corruption_detected); if (sequence.offset > (size_t)(oLitEnd - vBase))
match = dictEnd - (base-match); return ERROR(corruption_detected);
match = dictEnd - (base - match);
if (match + sequence.matchLength <= dictEnd) { if (match + sequence.matchLength <= dictEnd) {
memmove(oLitEnd, match, sequence.matchLength); memmove(oLitEnd, match, sequence.matchLength);
return sequenceLength; return sequenceLength;
} }
/* span extDict & currPrefixSegment */ /* span extDict & currPrefixSegment */
{ size_t const length1 = dictEnd - match; {
size_t const length1 = dictEnd - match;
memmove(oLitEnd, match, length1); memmove(oLitEnd, match, length1);
op = oLitEnd + length1; op = oLitEnd + length1;
sequence.matchLength -= length1; sequence.matchLength -= length1;
match = base; match = base;
} } }
while (op < oMatchEnd) *op++ = *match++; }
while (op < oMatchEnd)
*op++ = *match++;
return sequenceLength; return sequenceLength;
} }
static seq_t ZSTD_decodeSequence(seqState_t *seqState)
static seq_t ZSTD_decodeSequence(seqState_t* seqState)
{ {
seq_t seq; seq_t seq;
@@ -872,40 +928,37 @@ static seq_t ZSTD_decodeSequence(seqState_t* seqState)
U32 const llBits = LL_bits[llCode]; U32 const llBits = LL_bits[llCode];
U32 const mlBits = ML_bits[mlCode]; U32 const mlBits = ML_bits[mlCode];
U32 const ofBits = ofCode; U32 const ofBits = ofCode;
U32 const totalBits = llBits+mlBits+ofBits; U32 const totalBits = llBits + mlBits + ofBits;
static const U32 LL_base[MaxLL+1] = { static const U32 LL_base[MaxLL + 1] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 18,
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 20, 22, 24, 28, 32, 40, 48, 64, 0x80, 0x100, 0x200, 0x400, 0x800, 0x1000, 0x2000, 0x4000, 0x8000, 0x10000};
16, 18, 20, 22, 24, 28, 32, 40, 48, 64, 0x80, 0x100, 0x200, 0x400, 0x800, 0x1000,
0x2000, 0x4000, 0x8000, 0x10000 };
static const U32 ML_base[MaxML+1] = { static const U32 ML_base[MaxML + 1] = {3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20,
3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 37, 39, 41,
19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 43, 47, 51, 59, 67, 83, 99, 0x83, 0x103, 0x203, 0x403, 0x803, 0x1003, 0x2003, 0x4003, 0x8003, 0x10003};
35, 37, 39, 41, 43, 47, 51, 59, 67, 83, 99, 0x83, 0x103, 0x203, 0x403, 0x803,
0x1003, 0x2003, 0x4003, 0x8003, 0x10003 };
static const U32 OF_base[MaxOff+1] = { static const U32 OF_base[MaxOff + 1] = {0, 1, 1, 5, 0xD, 0x1D, 0x3D, 0x7D, 0xFD, 0x1FD,
0, 1, 1, 5, 0xD, 0x1D, 0x3D, 0x7D, 0x3FD, 0x7FD, 0xFFD, 0x1FFD, 0x3FFD, 0x7FFD, 0xFFFD, 0x1FFFD, 0x3FFFD, 0x7FFFD,
0xFD, 0x1FD, 0x3FD, 0x7FD, 0xFFD, 0x1FFD, 0x3FFD, 0x7FFD, 0xFFFFD, 0x1FFFFD, 0x3FFFFD, 0x7FFFFD, 0xFFFFFD, 0x1FFFFFD, 0x3FFFFFD, 0x7FFFFFD, 0xFFFFFFD};
0xFFFD, 0x1FFFD, 0x3FFFD, 0x7FFFD, 0xFFFFD, 0x1FFFFD, 0x3FFFFD, 0x7FFFFD,
0xFFFFFD, 0x1FFFFFD, 0x3FFFFFD, 0x7FFFFFD, 0xFFFFFFD };
/* sequence */ /* sequence */
{ size_t offset; {
size_t offset;
if (!ofCode) if (!ofCode)
offset = 0; offset = 0;
else { else {
offset = OF_base[ofCode] + BIT_readBitsFast(&seqState->DStream, ofBits); /* <= (ZSTD_WINDOWLOG_MAX-1) bits */ offset = OF_base[ofCode] + BIT_readBitsFast(&seqState->DStream, ofBits); /* <= (ZSTD_WINDOWLOG_MAX-1) bits */
if (ZSTD_32bits()) BIT_reloadDStream(&seqState->DStream); if (ZSTD_32bits())
BIT_reloadDStream(&seqState->DStream);
} }
if (ofCode <= 1) { if (ofCode <= 1) {
offset += (llCode==0); offset += (llCode == 0);
if (offset) { if (offset) {
size_t temp = (offset==3) ? seqState->prevOffset[0] - 1 : seqState->prevOffset[offset]; size_t temp = (offset == 3) ? seqState->prevOffset[0] - 1 : seqState->prevOffset[offset];
temp += !temp; /* 0 is not valid; input is corrupted; force offset to 1 */ temp += !temp; /* 0 is not valid; input is corrupted; force offset to 1 */
if (offset != 1) seqState->prevOffset[2] = seqState->prevOffset[1]; if (offset != 1)
seqState->prevOffset[2] = seqState->prevOffset[1];
seqState->prevOffset[1] = seqState->prevOffset[0]; seqState->prevOffset[1] = seqState->prevOffset[0];
seqState->prevOffset[0] = offset = temp; seqState->prevOffset[0] = offset = temp;
} else { } else {
@@ -919,123 +972,130 @@ static seq_t ZSTD_decodeSequence(seqState_t* seqState)
seq.offset = offset; seq.offset = offset;
} }
seq.matchLength = ML_base[mlCode] + ((mlCode>31) ? BIT_readBitsFast(&seqState->DStream, mlBits) : 0); /* <= 16 bits */ seq.matchLength = ML_base[mlCode] + ((mlCode > 31) ? BIT_readBitsFast(&seqState->DStream, mlBits) : 0); /* <= 16 bits */
if (ZSTD_32bits() && (mlBits+llBits>24)) BIT_reloadDStream(&seqState->DStream); if (ZSTD_32bits() && (mlBits + llBits > 24))
BIT_reloadDStream(&seqState->DStream);
seq.litLength = LL_base[llCode] + ((llCode>15) ? BIT_readBitsFast(&seqState->DStream, llBits) : 0); /* <= 16 bits */ seq.litLength = LL_base[llCode] + ((llCode > 15) ? BIT_readBitsFast(&seqState->DStream, llBits) : 0); /* <= 16 bits */
if (ZSTD_32bits() || if (ZSTD_32bits() || (totalBits > 64 - 7 - (LLFSELog + MLFSELog + OffFSELog)))
(totalBits > 64 - 7 - (LLFSELog+MLFSELog+OffFSELog)) ) BIT_reloadDStream(&seqState->DStream); BIT_reloadDStream(&seqState->DStream);
/* ANS state update */ /* ANS state update */
FSE_updateState(&seqState->stateLL, &seqState->DStream); /* <= 9 bits */ FSE_updateState(&seqState->stateLL, &seqState->DStream); /* <= 9 bits */
FSE_updateState(&seqState->stateML, &seqState->DStream); /* <= 9 bits */ FSE_updateState(&seqState->stateML, &seqState->DStream); /* <= 9 bits */
if (ZSTD_32bits()) BIT_reloadDStream(&seqState->DStream); /* <= 18 bits */ if (ZSTD_32bits())
BIT_reloadDStream(&seqState->DStream); /* <= 18 bits */
FSE_updateState(&seqState->stateOffb, &seqState->DStream); /* <= 8 bits */ FSE_updateState(&seqState->stateOffb, &seqState->DStream); /* <= 8 bits */
return seq; return seq;
} }
FORCE_INLINE FORCE_INLINE
size_t ZSTD_execSequence(BYTE* op, size_t ZSTD_execSequence(BYTE *op, BYTE *const oend, seq_t sequence, const BYTE **litPtr, const BYTE *const litLimit, const BYTE *const base,
BYTE* const oend, seq_t sequence, const BYTE *const vBase, const BYTE *const dictEnd)
const BYTE** litPtr, const BYTE* const litLimit,
const BYTE* const base, const BYTE* const vBase, const BYTE* const dictEnd)
{ {
BYTE* const oLitEnd = op + sequence.litLength; BYTE *const oLitEnd = op + sequence.litLength;
size_t const sequenceLength = sequence.litLength + sequence.matchLength; size_t const sequenceLength = sequence.litLength + sequence.matchLength;
BYTE* const oMatchEnd = op + sequenceLength; /* risk : address space overflow (32-bits) */ BYTE *const oMatchEnd = op + sequenceLength; /* risk : address space overflow (32-bits) */
BYTE* const oend_w = oend - WILDCOPY_OVERLENGTH; BYTE *const oend_w = oend - WILDCOPY_OVERLENGTH;
const BYTE* const iLitEnd = *litPtr + sequence.litLength; const BYTE *const iLitEnd = *litPtr + sequence.litLength;
const BYTE* match = oLitEnd - sequence.offset; const BYTE *match = oLitEnd - sequence.offset;
/* check */ /* check */
if (oMatchEnd>oend) return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of WILDCOPY_OVERLENGTH from oend */ if (oMatchEnd > oend)
if (iLitEnd > litLimit) return ERROR(corruption_detected); /* over-read beyond lit buffer */ return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of WILDCOPY_OVERLENGTH from oend */
if (oLitEnd>oend_w) return ZSTD_execSequenceLast7(op, oend, sequence, litPtr, litLimit, base, vBase, dictEnd); if (iLitEnd > litLimit)
return ERROR(corruption_detected); /* over-read beyond lit buffer */
if (oLitEnd > oend_w)
return ZSTD_execSequenceLast7(op, oend, sequence, litPtr, litLimit, base, vBase, dictEnd);
/* copy Literals */ /* copy Literals */
ZSTD_copy8(op, *litPtr); ZSTD_copy8(op, *litPtr);
if (sequence.litLength > 8) if (sequence.litLength > 8)
ZSTD_wildcopy(op+8, (*litPtr)+8, sequence.litLength - 8); /* note : since oLitEnd <= oend-WILDCOPY_OVERLENGTH, no risk of overwrite beyond oend */ ZSTD_wildcopy(op + 8, (*litPtr) + 8,
sequence.litLength - 8); /* note : since oLitEnd <= oend-WILDCOPY_OVERLENGTH, no risk of overwrite beyond oend */
op = oLitEnd; op = oLitEnd;
*litPtr = iLitEnd; /* update for next sequence */ *litPtr = iLitEnd; /* update for next sequence */
/* copy Match */ /* copy Match */
if (sequence.offset > (size_t)(oLitEnd - base)) { if (sequence.offset > (size_t)(oLitEnd - base)) {
/* offset beyond prefix */ /* offset beyond prefix */
if (sequence.offset > (size_t)(oLitEnd - vBase)) return ERROR(corruption_detected); if (sequence.offset > (size_t)(oLitEnd - vBase))
return ERROR(corruption_detected);
match = dictEnd + (match - base); match = dictEnd + (match - base);
if (match + sequence.matchLength <= dictEnd) { if (match + sequence.matchLength <= dictEnd) {
memmove(oLitEnd, match, sequence.matchLength); memmove(oLitEnd, match, sequence.matchLength);
return sequenceLength; return sequenceLength;
} }
/* span extDict & currPrefixSegment */ /* span extDict & currPrefixSegment */
{ size_t const length1 = dictEnd - match; {
size_t const length1 = dictEnd - match;
memmove(oLitEnd, match, length1); memmove(oLitEnd, match, length1);
op = oLitEnd + length1; op = oLitEnd + length1;
sequence.matchLength -= length1; sequence.matchLength -= length1;
match = base; match = base;
if (op > oend_w || sequence.matchLength < MINMATCH) { if (op > oend_w || sequence.matchLength < MINMATCH) {
U32 i; U32 i;
for (i = 0; i < sequence.matchLength; ++i) op[i] = match[i]; for (i = 0; i < sequence.matchLength; ++i)
op[i] = match[i];
return sequenceLength; return sequenceLength;
} }
} } }
}
/* Requirement: op <= oend_w && sequence.matchLength >= MINMATCH */ /* Requirement: op <= oend_w && sequence.matchLength >= MINMATCH */
/* match within prefix */ /* match within prefix */
if (sequence.offset < 8) { if (sequence.offset < 8) {
/* close range match, overlap */ /* close range match, overlap */
static const U32 dec32table[] = { 0, 1, 2, 1, 4, 4, 4, 4 }; /* added */ static const U32 dec32table[] = {0, 1, 2, 1, 4, 4, 4, 4}; /* added */
static const int dec64table[] = { 8, 8, 8, 7, 8, 9,10,11 }; /* subtracted */ static const int dec64table[] = {8, 8, 8, 7, 8, 9, 10, 11}; /* subtracted */
int const sub2 = dec64table[sequence.offset]; int const sub2 = dec64table[sequence.offset];
op[0] = match[0]; op[0] = match[0];
op[1] = match[1]; op[1] = match[1];
op[2] = match[2]; op[2] = match[2];
op[3] = match[3]; op[3] = match[3];
match += dec32table[sequence.offset]; match += dec32table[sequence.offset];
ZSTD_copy4(op+4, match); ZSTD_copy4(op + 4, match);
match -= sub2; match -= sub2;
} else { } else {
ZSTD_copy8(op, match); ZSTD_copy8(op, match);
} }
op += 8; match += 8; op += 8;
match += 8;
if (oMatchEnd > oend-(16-MINMATCH)) { if (oMatchEnd > oend - (16 - MINMATCH)) {
if (op < oend_w) { if (op < oend_w) {
ZSTD_wildcopy(op, match, oend_w - op); ZSTD_wildcopy(op, match, oend_w - op);
match += oend_w - op; match += oend_w - op;
op = oend_w; op = oend_w;
} }
while (op < oMatchEnd) *op++ = *match++; while (op < oMatchEnd)
*op++ = *match++;
} else { } else {
ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength-8); /* works even if matchLength < 8 */ ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength - 8); /* works even if matchLength < 8 */
} }
return sequenceLength; return sequenceLength;
} }
static size_t ZSTD_decompressSequences(ZSTD_DCtx *dctx, void *dst, size_t maxDstSize, const void *seqStart, size_t seqSize)
static size_t ZSTD_decompressSequences(
ZSTD_DCtx* dctx,
void* dst, size_t maxDstSize,
const void* seqStart, size_t seqSize)
{ {
const BYTE* ip = (const BYTE*)seqStart; const BYTE *ip = (const BYTE *)seqStart;
const BYTE* const iend = ip + seqSize; const BYTE *const iend = ip + seqSize;
BYTE* const ostart = (BYTE* const)dst; BYTE *const ostart = (BYTE * const)dst;
BYTE* const oend = ostart + maxDstSize; BYTE *const oend = ostart + maxDstSize;
BYTE* op = ostart; BYTE *op = ostart;
const BYTE* litPtr = dctx->litPtr; const BYTE *litPtr = dctx->litPtr;
const BYTE* const litEnd = litPtr + dctx->litSize; const BYTE *const litEnd = litPtr + dctx->litSize;
const BYTE* const base = (const BYTE*) (dctx->base); const BYTE *const base = (const BYTE *)(dctx->base);
const BYTE* const vBase = (const BYTE*) (dctx->vBase); const BYTE *const vBase = (const BYTE *)(dctx->vBase);
const BYTE* const dictEnd = (const BYTE*) (dctx->dictEnd); const BYTE *const dictEnd = (const BYTE *)(dctx->dictEnd);
int nbSeq; int nbSeq;
/* Build Decoding Tables */ /* Build Decoding Tables */
{ size_t const seqHSize = ZSTD_decodeSeqHeaders(dctx, &nbSeq, ip, seqSize); {
if (ZSTD_isError(seqHSize)) return seqHSize; size_t const seqHSize = ZSTD_decodeSeqHeaders(dctx, &nbSeq, ip, seqSize);
if (ZSTD_isError(seqHSize))
return seqHSize;
ip += seqHSize; ip += seqHSize;
} }
@@ -1043,38 +1103,51 @@ static size_t ZSTD_decompressSequences(
if (nbSeq) { if (nbSeq) {
seqState_t seqState; seqState_t seqState;
dctx->fseEntropy = 1; dctx->fseEntropy = 1;
{ U32 i; for (i=0; i<ZSTD_REP_NUM; i++) seqState.prevOffset[i] = dctx->entropy.rep[i]; } {
CHECK_E(BIT_initDStream(&seqState.DStream, ip, iend-ip), corruption_detected); U32 i;
for (i = 0; i < ZSTD_REP_NUM; i++)
seqState.prevOffset[i] = dctx->entropy.rep[i];
}
CHECK_E(BIT_initDStream(&seqState.DStream, ip, iend - ip), corruption_detected);
FSE_initDState(&seqState.stateLL, &seqState.DStream, dctx->LLTptr); FSE_initDState(&seqState.stateLL, &seqState.DStream, dctx->LLTptr);
FSE_initDState(&seqState.stateOffb, &seqState.DStream, dctx->OFTptr); FSE_initDState(&seqState.stateOffb, &seqState.DStream, dctx->OFTptr);
FSE_initDState(&seqState.stateML, &seqState.DStream, dctx->MLTptr); FSE_initDState(&seqState.stateML, &seqState.DStream, dctx->MLTptr);
for ( ; (BIT_reloadDStream(&(seqState.DStream)) <= BIT_DStream_completed) && nbSeq ; ) { for (; (BIT_reloadDStream(&(seqState.DStream)) <= BIT_DStream_completed) && nbSeq;) {
nbSeq--; nbSeq--;
{ seq_t const sequence = ZSTD_decodeSequence(&seqState); {
seq_t const sequence = ZSTD_decodeSequence(&seqState);
size_t const oneSeqSize = ZSTD_execSequence(op, oend, sequence, &litPtr, litEnd, base, vBase, dictEnd); size_t const oneSeqSize = ZSTD_execSequence(op, oend, sequence, &litPtr, litEnd, base, vBase, dictEnd);
if (ZSTD_isError(oneSeqSize)) return oneSeqSize; if (ZSTD_isError(oneSeqSize))
return oneSeqSize;
op += oneSeqSize; op += oneSeqSize;
} } }
}
/* check if reached exact end */ /* check if reached exact end */
if (nbSeq) return ERROR(corruption_detected); if (nbSeq)
return ERROR(corruption_detected);
/* save reps for next block */ /* save reps for next block */
{ U32 i; for (i=0; i<ZSTD_REP_NUM; i++) dctx->entropy.rep[i] = (U32)(seqState.prevOffset[i]); } {
U32 i;
for (i = 0; i < ZSTD_REP_NUM; i++)
dctx->entropy.rep[i] = (U32)(seqState.prevOffset[i]);
}
} }
/* last literal segment */ /* last literal segment */
{ size_t const lastLLSize = litEnd - litPtr; {
if (lastLLSize > (size_t)(oend-op)) return ERROR(dstSize_tooSmall); size_t const lastLLSize = litEnd - litPtr;
if (lastLLSize > (size_t)(oend - op))
return ERROR(dstSize_tooSmall);
memcpy(op, litPtr, lastLLSize); memcpy(op, litPtr, lastLLSize);
op += lastLLSize; op += lastLLSize;
} }
return op-ostart; return op - ostart;
} }
FORCE_INLINE seq_t ZSTD_decodeSequenceLong_generic(seqState_t *seqState, int const longOffsets)
FORCE_INLINE seq_t ZSTD_decodeSequenceLong_generic(seqState_t* seqState, int const longOffsets)
{ {
seq_t seq; seq_t seq;
@@ -1085,47 +1158,46 @@ FORCE_INLINE seq_t ZSTD_decodeSequenceLong_generic(seqState_t* seqState, int con
U32 const llBits = LL_bits[llCode]; U32 const llBits = LL_bits[llCode];
U32 const mlBits = ML_bits[mlCode]; U32 const mlBits = ML_bits[mlCode];
U32 const ofBits = ofCode; U32 const ofBits = ofCode;
U32 const totalBits = llBits+mlBits+ofBits; U32 const totalBits = llBits + mlBits + ofBits;
static const U32 LL_base[MaxLL+1] = { static const U32 LL_base[MaxLL + 1] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 18,
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 20, 22, 24, 28, 32, 40, 48, 64, 0x80, 0x100, 0x200, 0x400, 0x800, 0x1000, 0x2000, 0x4000, 0x8000, 0x10000};
16, 18, 20, 22, 24, 28, 32, 40, 48, 64, 0x80, 0x100, 0x200, 0x400, 0x800, 0x1000,
0x2000, 0x4000, 0x8000, 0x10000 };
static const U32 ML_base[MaxML+1] = { static const U32 ML_base[MaxML + 1] = {3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20,
3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 37, 39, 41,
19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 43, 47, 51, 59, 67, 83, 99, 0x83, 0x103, 0x203, 0x403, 0x803, 0x1003, 0x2003, 0x4003, 0x8003, 0x10003};
35, 37, 39, 41, 43, 47, 51, 59, 67, 83, 99, 0x83, 0x103, 0x203, 0x403, 0x803,
0x1003, 0x2003, 0x4003, 0x8003, 0x10003 };
static const U32 OF_base[MaxOff+1] = { static const U32 OF_base[MaxOff + 1] = {0, 1, 1, 5, 0xD, 0x1D, 0x3D, 0x7D, 0xFD, 0x1FD,
0, 1, 1, 5, 0xD, 0x1D, 0x3D, 0x7D, 0x3FD, 0x7FD, 0xFFD, 0x1FFD, 0x3FFD, 0x7FFD, 0xFFFD, 0x1FFFD, 0x3FFFD, 0x7FFFD,
0xFD, 0x1FD, 0x3FD, 0x7FD, 0xFFD, 0x1FFD, 0x3FFD, 0x7FFD, 0xFFFFD, 0x1FFFFD, 0x3FFFFD, 0x7FFFFD, 0xFFFFFD, 0x1FFFFFD, 0x3FFFFFD, 0x7FFFFFD, 0xFFFFFFD};
0xFFFD, 0x1FFFD, 0x3FFFD, 0x7FFFD, 0xFFFFD, 0x1FFFFD, 0x3FFFFD, 0x7FFFFD,
0xFFFFFD, 0x1FFFFFD, 0x3FFFFFD, 0x7FFFFFD, 0xFFFFFFD };
/* sequence */ /* sequence */
{ size_t offset; {
size_t offset;
if (!ofCode) if (!ofCode)
offset = 0; offset = 0;
else { else {
if (longOffsets) { if (longOffsets) {
int const extraBits = ofBits - MIN(ofBits, STREAM_ACCUMULATOR_MIN); int const extraBits = ofBits - MIN(ofBits, STREAM_ACCUMULATOR_MIN);
offset = OF_base[ofCode] + (BIT_readBitsFast(&seqState->DStream, ofBits - extraBits) << extraBits); offset = OF_base[ofCode] + (BIT_readBitsFast(&seqState->DStream, ofBits - extraBits) << extraBits);
if (ZSTD_32bits() || extraBits) BIT_reloadDStream(&seqState->DStream); if (ZSTD_32bits() || extraBits)
if (extraBits) offset += BIT_readBitsFast(&seqState->DStream, extraBits); BIT_reloadDStream(&seqState->DStream);
if (extraBits)
offset += BIT_readBitsFast(&seqState->DStream, extraBits);
} else { } else {
offset = OF_base[ofCode] + BIT_readBitsFast(&seqState->DStream, ofBits); /* <= (ZSTD_WINDOWLOG_MAX-1) bits */ offset = OF_base[ofCode] + BIT_readBitsFast(&seqState->DStream, ofBits); /* <= (ZSTD_WINDOWLOG_MAX-1) bits */
if (ZSTD_32bits()) BIT_reloadDStream(&seqState->DStream); if (ZSTD_32bits())
BIT_reloadDStream(&seqState->DStream);
} }
} }
if (ofCode <= 1) { if (ofCode <= 1) {
offset += (llCode==0); offset += (llCode == 0);
if (offset) { if (offset) {
size_t temp = (offset==3) ? seqState->prevOffset[0] - 1 : seqState->prevOffset[offset]; size_t temp = (offset == 3) ? seqState->prevOffset[0] - 1 : seqState->prevOffset[offset];
temp += !temp; /* 0 is not valid; input is corrupted; force offset to 1 */ temp += !temp; /* 0 is not valid; input is corrupted; force offset to 1 */
if (offset != 1) seqState->prevOffset[2] = seqState->prevOffset[1]; if (offset != 1)
seqState->prevOffset[2] = seqState->prevOffset[1];
seqState->prevOffset[1] = seqState->prevOffset[0]; seqState->prevOffset[1] = seqState->prevOffset[0];
seqState->prevOffset[0] = offset = temp; seqState->prevOffset[0] = offset = temp;
} else { } else {
@@ -1139,29 +1211,34 @@ FORCE_INLINE seq_t ZSTD_decodeSequenceLong_generic(seqState_t* seqState, int con
seq.offset = offset; seq.offset = offset;
} }
seq.matchLength = ML_base[mlCode] + ((mlCode>31) ? BIT_readBitsFast(&seqState->DStream, mlBits) : 0); /* <= 16 bits */ seq.matchLength = ML_base[mlCode] + ((mlCode > 31) ? BIT_readBitsFast(&seqState->DStream, mlBits) : 0); /* <= 16 bits */
if (ZSTD_32bits() && (mlBits+llBits>24)) BIT_reloadDStream(&seqState->DStream); if (ZSTD_32bits() && (mlBits + llBits > 24))
BIT_reloadDStream(&seqState->DStream);
seq.litLength = LL_base[llCode] + ((llCode>15) ? BIT_readBitsFast(&seqState->DStream, llBits) : 0); /* <= 16 bits */ seq.litLength = LL_base[llCode] + ((llCode > 15) ? BIT_readBitsFast(&seqState->DStream, llBits) : 0); /* <= 16 bits */
if (ZSTD_32bits() || if (ZSTD_32bits() || (totalBits > 64 - 7 - (LLFSELog + MLFSELog + OffFSELog)))
(totalBits > 64 - 7 - (LLFSELog+MLFSELog+OffFSELog)) ) BIT_reloadDStream(&seqState->DStream); BIT_reloadDStream(&seqState->DStream);
{ size_t const pos = seqState->pos + seq.litLength; {
size_t const pos = seqState->pos + seq.litLength;
seq.match = seqState->base + pos - seq.offset; /* single memory segment */ seq.match = seqState->base + pos - seq.offset; /* single memory segment */
if (seq.offset > pos) seq.match += seqState->gotoDict; /* separate memory segment */ if (seq.offset > pos)
seq.match += seqState->gotoDict; /* separate memory segment */
seqState->pos = pos + seq.matchLength; seqState->pos = pos + seq.matchLength;
} }
/* ANS state update */ /* ANS state update */
FSE_updateState(&seqState->stateLL, &seqState->DStream); /* <= 9 bits */ FSE_updateState(&seqState->stateLL, &seqState->DStream); /* <= 9 bits */
FSE_updateState(&seqState->stateML, &seqState->DStream); /* <= 9 bits */ FSE_updateState(&seqState->stateML, &seqState->DStream); /* <= 9 bits */
if (ZSTD_32bits()) BIT_reloadDStream(&seqState->DStream); /* <= 18 bits */ if (ZSTD_32bits())
BIT_reloadDStream(&seqState->DStream); /* <= 18 bits */
FSE_updateState(&seqState->stateOffb, &seqState->DStream); /* <= 8 bits */ FSE_updateState(&seqState->stateOffb, &seqState->DStream); /* <= 8 bits */
return seq; return seq;
} }
static seq_t ZSTD_decodeSequenceLong(seqState_t* seqState, unsigned const windowSize) { static seq_t ZSTD_decodeSequenceLong(seqState_t *seqState, unsigned const windowSize)
{
if (ZSTD_highbit32(windowSize) > STREAM_ACCUMULATOR_MIN) { if (ZSTD_highbit32(windowSize) > STREAM_ACCUMULATOR_MIN) {
return ZSTD_decodeSequenceLong_generic(seqState, 1); return ZSTD_decodeSequenceLong_generic(seqState, 1);
} else { } else {
@@ -1170,177 +1247,199 @@ static seq_t ZSTD_decodeSequenceLong(seqState_t* seqState, unsigned const window
} }
FORCE_INLINE FORCE_INLINE
size_t ZSTD_execSequenceLong(BYTE* op, size_t ZSTD_execSequenceLong(BYTE *op, BYTE *const oend, seq_t sequence, const BYTE **litPtr, const BYTE *const litLimit, const BYTE *const base,
BYTE* const oend, seq_t sequence, const BYTE *const vBase, const BYTE *const dictEnd)
const BYTE** litPtr, const BYTE* const litLimit,
const BYTE* const base, const BYTE* const vBase, const BYTE* const dictEnd)
{ {
BYTE* const oLitEnd = op + sequence.litLength; BYTE *const oLitEnd = op + sequence.litLength;
size_t const sequenceLength = sequence.litLength + sequence.matchLength; size_t const sequenceLength = sequence.litLength + sequence.matchLength;
BYTE* const oMatchEnd = op + sequenceLength; /* risk : address space overflow (32-bits) */ BYTE *const oMatchEnd = op + sequenceLength; /* risk : address space overflow (32-bits) */
BYTE* const oend_w = oend - WILDCOPY_OVERLENGTH; BYTE *const oend_w = oend - WILDCOPY_OVERLENGTH;
const BYTE* const iLitEnd = *litPtr + sequence.litLength; const BYTE *const iLitEnd = *litPtr + sequence.litLength;
const BYTE* match = sequence.match; const BYTE *match = sequence.match;
/* check */ /* check */
if (oMatchEnd>oend) return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of WILDCOPY_OVERLENGTH from oend */ if (oMatchEnd > oend)
if (iLitEnd > litLimit) return ERROR(corruption_detected); /* over-read beyond lit buffer */ return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of WILDCOPY_OVERLENGTH from oend */
if (oLitEnd>oend_w) return ZSTD_execSequenceLast7(op, oend, sequence, litPtr, litLimit, base, vBase, dictEnd); if (iLitEnd > litLimit)
return ERROR(corruption_detected); /* over-read beyond lit buffer */
if (oLitEnd > oend_w)
return ZSTD_execSequenceLast7(op, oend, sequence, litPtr, litLimit, base, vBase, dictEnd);
/* copy Literals */ /* copy Literals */
ZSTD_copy8(op, *litPtr); ZSTD_copy8(op, *litPtr);
if (sequence.litLength > 8) if (sequence.litLength > 8)
ZSTD_wildcopy(op+8, (*litPtr)+8, sequence.litLength - 8); /* note : since oLitEnd <= oend-WILDCOPY_OVERLENGTH, no risk of overwrite beyond oend */ ZSTD_wildcopy(op + 8, (*litPtr) + 8,
sequence.litLength - 8); /* note : since oLitEnd <= oend-WILDCOPY_OVERLENGTH, no risk of overwrite beyond oend */
op = oLitEnd; op = oLitEnd;
*litPtr = iLitEnd; /* update for next sequence */ *litPtr = iLitEnd; /* update for next sequence */
/* copy Match */ /* copy Match */
if (sequence.offset > (size_t)(oLitEnd - base)) { if (sequence.offset > (size_t)(oLitEnd - base)) {
/* offset beyond prefix */ /* offset beyond prefix */
if (sequence.offset > (size_t)(oLitEnd - vBase)) return ERROR(corruption_detected); if (sequence.offset > (size_t)(oLitEnd - vBase))
return ERROR(corruption_detected);
if (match + sequence.matchLength <= dictEnd) { if (match + sequence.matchLength <= dictEnd) {
memmove(oLitEnd, match, sequence.matchLength); memmove(oLitEnd, match, sequence.matchLength);
return sequenceLength; return sequenceLength;
} }
/* span extDict & currPrefixSegment */ /* span extDict & currPrefixSegment */
{ size_t const length1 = dictEnd - match; {
size_t const length1 = dictEnd - match;
memmove(oLitEnd, match, length1); memmove(oLitEnd, match, length1);
op = oLitEnd + length1; op = oLitEnd + length1;
sequence.matchLength -= length1; sequence.matchLength -= length1;
match = base; match = base;
if (op > oend_w || sequence.matchLength < MINMATCH) { if (op > oend_w || sequence.matchLength < MINMATCH) {
U32 i; U32 i;
for (i = 0; i < sequence.matchLength; ++i) op[i] = match[i]; for (i = 0; i < sequence.matchLength; ++i)
op[i] = match[i];
return sequenceLength; return sequenceLength;
} }
} } }
}
/* Requirement: op <= oend_w && sequence.matchLength >= MINMATCH */ /* Requirement: op <= oend_w && sequence.matchLength >= MINMATCH */
/* match within prefix */ /* match within prefix */
if (sequence.offset < 8) { if (sequence.offset < 8) {
/* close range match, overlap */ /* close range match, overlap */
static const U32 dec32table[] = { 0, 1, 2, 1, 4, 4, 4, 4 }; /* added */ static const U32 dec32table[] = {0, 1, 2, 1, 4, 4, 4, 4}; /* added */
static const int dec64table[] = { 8, 8, 8, 7, 8, 9,10,11 }; /* subtracted */ static const int dec64table[] = {8, 8, 8, 7, 8, 9, 10, 11}; /* subtracted */
int const sub2 = dec64table[sequence.offset]; int const sub2 = dec64table[sequence.offset];
op[0] = match[0]; op[0] = match[0];
op[1] = match[1]; op[1] = match[1];
op[2] = match[2]; op[2] = match[2];
op[3] = match[3]; op[3] = match[3];
match += dec32table[sequence.offset]; match += dec32table[sequence.offset];
ZSTD_copy4(op+4, match); ZSTD_copy4(op + 4, match);
match -= sub2; match -= sub2;
} else { } else {
ZSTD_copy8(op, match); ZSTD_copy8(op, match);
} }
op += 8; match += 8; op += 8;
match += 8;
if (oMatchEnd > oend-(16-MINMATCH)) { if (oMatchEnd > oend - (16 - MINMATCH)) {
if (op < oend_w) { if (op < oend_w) {
ZSTD_wildcopy(op, match, oend_w - op); ZSTD_wildcopy(op, match, oend_w - op);
match += oend_w - op; match += oend_w - op;
op = oend_w; op = oend_w;
} }
while (op < oMatchEnd) *op++ = *match++; while (op < oMatchEnd)
*op++ = *match++;
} else { } else {
ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength-8); /* works even if matchLength < 8 */ ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength - 8); /* works even if matchLength < 8 */
} }
return sequenceLength; return sequenceLength;
} }
static size_t ZSTD_decompressSequencesLong( static size_t ZSTD_decompressSequencesLong(ZSTD_DCtx *dctx, void *dst, size_t maxDstSize, const void *seqStart, size_t seqSize)
ZSTD_DCtx* dctx,
void* dst, size_t maxDstSize,
const void* seqStart, size_t seqSize)
{ {
const BYTE* ip = (const BYTE*)seqStart; const BYTE *ip = (const BYTE *)seqStart;
const BYTE* const iend = ip + seqSize; const BYTE *const iend = ip + seqSize;
BYTE* const ostart = (BYTE* const)dst; BYTE *const ostart = (BYTE * const)dst;
BYTE* const oend = ostart + maxDstSize; BYTE *const oend = ostart + maxDstSize;
BYTE* op = ostart; BYTE *op = ostart;
const BYTE* litPtr = dctx->litPtr; const BYTE *litPtr = dctx->litPtr;
const BYTE* const litEnd = litPtr + dctx->litSize; const BYTE *const litEnd = litPtr + dctx->litSize;
const BYTE* const base = (const BYTE*) (dctx->base); const BYTE *const base = (const BYTE *)(dctx->base);
const BYTE* const vBase = (const BYTE*) (dctx->vBase); const BYTE *const vBase = (const BYTE *)(dctx->vBase);
const BYTE* const dictEnd = (const BYTE*) (dctx->dictEnd); const BYTE *const dictEnd = (const BYTE *)(dctx->dictEnd);
unsigned const windowSize = dctx->fParams.windowSize; unsigned const windowSize = dctx->fParams.windowSize;
int nbSeq; int nbSeq;
/* Build Decoding Tables */ /* Build Decoding Tables */
{ size_t const seqHSize = ZSTD_decodeSeqHeaders(dctx, &nbSeq, ip, seqSize); {
if (ZSTD_isError(seqHSize)) return seqHSize; size_t const seqHSize = ZSTD_decodeSeqHeaders(dctx, &nbSeq, ip, seqSize);
if (ZSTD_isError(seqHSize))
return seqHSize;
ip += seqHSize; ip += seqHSize;
} }
/* Regen sequences */ /* Regen sequences */
if (nbSeq) { if (nbSeq) {
#define STORED_SEQS 4 #define STORED_SEQS 4
#define STOSEQ_MASK (STORED_SEQS-1) #define STOSEQ_MASK (STORED_SEQS - 1)
#define ADVANCED_SEQS 4 #define ADVANCED_SEQS 4
seq_t sequences[STORED_SEQS]; seq_t sequences[STORED_SEQS];
int const seqAdvance = MIN(nbSeq, ADVANCED_SEQS); int const seqAdvance = MIN(nbSeq, ADVANCED_SEQS);
seqState_t seqState; seqState_t seqState;
int seqNb; int seqNb;
dctx->fseEntropy = 1; dctx->fseEntropy = 1;
{ U32 i; for (i=0; i<ZSTD_REP_NUM; i++) seqState.prevOffset[i] = dctx->entropy.rep[i]; } {
U32 i;
for (i = 0; i < ZSTD_REP_NUM; i++)
seqState.prevOffset[i] = dctx->entropy.rep[i];
}
seqState.base = base; seqState.base = base;
seqState.pos = (size_t)(op-base); seqState.pos = (size_t)(op - base);
seqState.gotoDict = (uPtrDiff)dictEnd - (uPtrDiff)base; /* cast to avoid undefined behaviour */ seqState.gotoDict = (uPtrDiff)dictEnd - (uPtrDiff)base; /* cast to avoid undefined behaviour */
CHECK_E(BIT_initDStream(&seqState.DStream, ip, iend-ip), corruption_detected); CHECK_E(BIT_initDStream(&seqState.DStream, ip, iend - ip), corruption_detected);
FSE_initDState(&seqState.stateLL, &seqState.DStream, dctx->LLTptr); FSE_initDState(&seqState.stateLL, &seqState.DStream, dctx->LLTptr);
FSE_initDState(&seqState.stateOffb, &seqState.DStream, dctx->OFTptr); FSE_initDState(&seqState.stateOffb, &seqState.DStream, dctx->OFTptr);
FSE_initDState(&seqState.stateML, &seqState.DStream, dctx->MLTptr); FSE_initDState(&seqState.stateML, &seqState.DStream, dctx->MLTptr);
/* prepare in advance */ /* prepare in advance */
for (seqNb=0; (BIT_reloadDStream(&seqState.DStream) <= BIT_DStream_completed) && seqNb<seqAdvance; seqNb++) { for (seqNb = 0; (BIT_reloadDStream(&seqState.DStream) <= BIT_DStream_completed) && seqNb < seqAdvance; seqNb++) {
sequences[seqNb] = ZSTD_decodeSequenceLong(&seqState, windowSize); sequences[seqNb] = ZSTD_decodeSequenceLong(&seqState, windowSize);
} }
if (seqNb<seqAdvance) return ERROR(corruption_detected); if (seqNb < seqAdvance)
return ERROR(corruption_detected);
/* decode and decompress */ /* decode and decompress */
for ( ; (BIT_reloadDStream(&(seqState.DStream)) <= BIT_DStream_completed) && seqNb<nbSeq ; seqNb++) { for (; (BIT_reloadDStream(&(seqState.DStream)) <= BIT_DStream_completed) && seqNb < nbSeq; seqNb++) {
seq_t const sequence = ZSTD_decodeSequenceLong(&seqState, windowSize); seq_t const sequence = ZSTD_decodeSequenceLong(&seqState, windowSize);
size_t const oneSeqSize = ZSTD_execSequenceLong(op, oend, sequences[(seqNb-ADVANCED_SEQS) & STOSEQ_MASK], &litPtr, litEnd, base, vBase, dictEnd); size_t const oneSeqSize =
if (ZSTD_isError(oneSeqSize)) return oneSeqSize; ZSTD_execSequenceLong(op, oend, sequences[(seqNb - ADVANCED_SEQS) & STOSEQ_MASK], &litPtr, litEnd, base, vBase, dictEnd);
if (ZSTD_isError(oneSeqSize))
return oneSeqSize;
ZSTD_PREFETCH(sequence.match); ZSTD_PREFETCH(sequence.match);
sequences[seqNb&STOSEQ_MASK] = sequence; sequences[seqNb & STOSEQ_MASK] = sequence;
op += oneSeqSize; op += oneSeqSize;
} }
if (seqNb<nbSeq) return ERROR(corruption_detected); if (seqNb < nbSeq)
return ERROR(corruption_detected);
/* finish queue */ /* finish queue */
seqNb -= seqAdvance; seqNb -= seqAdvance;
for ( ; seqNb<nbSeq ; seqNb++) { for (; seqNb < nbSeq; seqNb++) {
size_t const oneSeqSize = ZSTD_execSequenceLong(op, oend, sequences[seqNb&STOSEQ_MASK], &litPtr, litEnd, base, vBase, dictEnd); size_t const oneSeqSize = ZSTD_execSequenceLong(op, oend, sequences[seqNb & STOSEQ_MASK], &litPtr, litEnd, base, vBase, dictEnd);
if (ZSTD_isError(oneSeqSize)) return oneSeqSize; if (ZSTD_isError(oneSeqSize))
return oneSeqSize;
op += oneSeqSize; op += oneSeqSize;
} }
/* save reps for next block */ /* save reps for next block */
{ U32 i; for (i=0; i<ZSTD_REP_NUM; i++) dctx->entropy.rep[i] = (U32)(seqState.prevOffset[i]); } {
U32 i;
for (i = 0; i < ZSTD_REP_NUM; i++)
dctx->entropy.rep[i] = (U32)(seqState.prevOffset[i]);
}
} }
/* last literal segment */ /* last literal segment */
{ size_t const lastLLSize = litEnd - litPtr; {
if (lastLLSize > (size_t)(oend-op)) return ERROR(dstSize_tooSmall); size_t const lastLLSize = litEnd - litPtr;
if (lastLLSize > (size_t)(oend - op))
return ERROR(dstSize_tooSmall);
memcpy(op, litPtr, lastLLSize); memcpy(op, litPtr, lastLLSize);
op += lastLLSize; op += lastLLSize;
} }
return op-ostart; return op - ostart;
} }
static size_t ZSTD_decompressBlock_internal(ZSTD_DCtx *dctx, void *dst, size_t dstCapacity, const void *src, size_t srcSize)
static size_t ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx,
void* dst, size_t dstCapacity,
const void* src, size_t srcSize)
{ /* blockType == blockCompressed */ { /* blockType == blockCompressed */
const BYTE* ip = (const BYTE*)src; const BYTE *ip = (const BYTE *)src;
if (srcSize >= ZSTD_BLOCKSIZE_ABSOLUTEMAX) return ERROR(srcSize_wrong); if (srcSize >= ZSTD_BLOCKSIZE_ABSOLUTEMAX)
return ERROR(srcSize_wrong);
/* Decode literals section */ /* Decode literals section */
{ size_t const litCSize = ZSTD_decodeLiteralsBlock(dctx, src, srcSize); {
if (ZSTD_isError(litCSize)) return litCSize; size_t const litCSize = ZSTD_decodeLiteralsBlock(dctx, src, srcSize);
if (ZSTD_isError(litCSize))
return litCSize;
ip += litCSize; ip += litCSize;
srcSize -= litCSize; srcSize -= litCSize;
} }
@@ -1348,47 +1447,43 @@ static size_t ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx,
/* likely because of register pressure */ /* likely because of register pressure */
/* if that's the correct cause, then 32-bits ARM should be affected differently */ /* if that's the correct cause, then 32-bits ARM should be affected differently */
/* it would be good to test this on ARM real hardware, to see if prefetch version improves speed */ /* it would be good to test this on ARM real hardware, to see if prefetch version improves speed */
if (dctx->fParams.windowSize > (1<<23)) if (dctx->fParams.windowSize > (1 << 23))
return ZSTD_decompressSequencesLong(dctx, dst, dstCapacity, ip, srcSize); return ZSTD_decompressSequencesLong(dctx, dst, dstCapacity, ip, srcSize);
return ZSTD_decompressSequences(dctx, dst, dstCapacity, ip, srcSize); return ZSTD_decompressSequences(dctx, dst, dstCapacity, ip, srcSize);
} }
static void ZSTD_checkContinuity(ZSTD_DCtx *dctx, const void *dst)
static void ZSTD_checkContinuity(ZSTD_DCtx* dctx, const void* dst)
{ {
if (dst != dctx->previousDstEnd) { /* not contiguous */ if (dst != dctx->previousDstEnd) { /* not contiguous */
dctx->dictEnd = dctx->previousDstEnd; dctx->dictEnd = dctx->previousDstEnd;
dctx->vBase = (const char*)dst - ((const char*)(dctx->previousDstEnd) - (const char*)(dctx->base)); dctx->vBase = (const char *)dst - ((const char *)(dctx->previousDstEnd) - (const char *)(dctx->base));
dctx->base = dst; dctx->base = dst;
dctx->previousDstEnd = dst; dctx->previousDstEnd = dst;
} }
} }
size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, size_t ZSTD_decompressBlock(ZSTD_DCtx *dctx, void *dst, size_t dstCapacity, const void *src, size_t srcSize)
void* dst, size_t dstCapacity,
const void* src, size_t srcSize)
{ {
size_t dSize; size_t dSize;
ZSTD_checkContinuity(dctx, dst); ZSTD_checkContinuity(dctx, dst);
dSize = ZSTD_decompressBlock_internal(dctx, dst, dstCapacity, src, srcSize); dSize = ZSTD_decompressBlock_internal(dctx, dst, dstCapacity, src, srcSize);
dctx->previousDstEnd = (char*)dst + dSize; dctx->previousDstEnd = (char *)dst + dSize;
return dSize; return dSize;
} }
/** ZSTD_insertBlock() : /** ZSTD_insertBlock() :
insert `src` block into `dctx` history. Useful to track uncompressed blocks. */ insert `src` block into `dctx` history. Useful to track uncompressed blocks. */
size_t ZSTD_insertBlock(ZSTD_DCtx* dctx, const void* blockStart, size_t blockSize) size_t ZSTD_insertBlock(ZSTD_DCtx *dctx, const void *blockStart, size_t blockSize)
{ {
ZSTD_checkContinuity(dctx, blockStart); ZSTD_checkContinuity(dctx, blockStart);
dctx->previousDstEnd = (const char*)blockStart + blockSize; dctx->previousDstEnd = (const char *)blockStart + blockSize;
return blockSize; return blockSize;
} }
size_t ZSTD_generateNxBytes(void *dst, size_t dstCapacity, BYTE byte, size_t length)
size_t ZSTD_generateNxBytes(void* dst, size_t dstCapacity, BYTE byte, size_t length)
{ {
if (length > dstCapacity) return ERROR(dstSize_tooSmall); if (length > dstCapacity)
return ERROR(dstSize_tooSmall);
memset(dst, byte, length); memset(dst, byte, length);
return length; return length;
} }
@@ -1400,22 +1495,25 @@ size_t ZSTD_generateNxBytes(void* dst, size_t dstCapacity, BYTE byte, size_t len
* @return : the compressed size of the frame starting at `src` */ * @return : the compressed size of the frame starting at `src` */
size_t ZSTD_findFrameCompressedSize(const void *src, size_t srcSize) size_t ZSTD_findFrameCompressedSize(const void *src, size_t srcSize)
{ {
if (srcSize >= ZSTD_skippableHeaderSize && if (srcSize >= ZSTD_skippableHeaderSize && (ZSTD_readLE32(src) & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) {
(ZSTD_readLE32(src) & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) { return ZSTD_skippableHeaderSize + ZSTD_readLE32((const BYTE *)src + 4);
return ZSTD_skippableHeaderSize + ZSTD_readLE32((const BYTE*)src + 4);
} else { } else {
const BYTE* ip = (const BYTE*)src; const BYTE *ip = (const BYTE *)src;
const BYTE* const ipstart = ip; const BYTE *const ipstart = ip;
size_t remainingSize = srcSize; size_t remainingSize = srcSize;
ZSTD_frameParams fParams; ZSTD_frameParams fParams;
size_t const headerSize = ZSTD_frameHeaderSize(ip, remainingSize); size_t const headerSize = ZSTD_frameHeaderSize(ip, remainingSize);
if (ZSTD_isError(headerSize)) return headerSize; if (ZSTD_isError(headerSize))
return headerSize;
/* Frame Header */ /* Frame Header */
{ size_t const ret = ZSTD_getFrameParams(&fParams, ip, remainingSize); {
if (ZSTD_isError(ret)) return ret; size_t const ret = ZSTD_getFrameParams(&fParams, ip, remainingSize);
if (ret > 0) return ERROR(srcSize_wrong); if (ZSTD_isError(ret))
return ret;
if (ret > 0)
return ERROR(srcSize_wrong);
} }
ip += headerSize; ip += headerSize;
@@ -1425,18 +1523,22 @@ size_t ZSTD_findFrameCompressedSize(const void *src, size_t srcSize)
while (1) { while (1) {
blockProperties_t blockProperties; blockProperties_t blockProperties;
size_t const cBlockSize = ZSTD_getcBlockSize(ip, remainingSize, &blockProperties); size_t const cBlockSize = ZSTD_getcBlockSize(ip, remainingSize, &blockProperties);
if (ZSTD_isError(cBlockSize)) return cBlockSize; if (ZSTD_isError(cBlockSize))
return cBlockSize;
if (ZSTD_blockHeaderSize + cBlockSize > remainingSize) return ERROR(srcSize_wrong); if (ZSTD_blockHeaderSize + cBlockSize > remainingSize)
return ERROR(srcSize_wrong);
ip += ZSTD_blockHeaderSize + cBlockSize; ip += ZSTD_blockHeaderSize + cBlockSize;
remainingSize -= ZSTD_blockHeaderSize + cBlockSize; remainingSize -= ZSTD_blockHeaderSize + cBlockSize;
if (blockProperties.lastBlock) break; if (blockProperties.lastBlock)
break;
} }
if (fParams.checksumFlag) { /* Frame content checksum */ if (fParams.checksumFlag) { /* Frame content checksum */
if (remainingSize < 4) return ERROR(srcSize_wrong); if (remainingSize < 4)
return ERROR(srcSize_wrong);
ip += 4; ip += 4;
remainingSize -= 4; remainingSize -= 4;
} }
@@ -1447,25 +1549,28 @@ size_t ZSTD_findFrameCompressedSize(const void *src, size_t srcSize)
/*! ZSTD_decompressFrame() : /*! ZSTD_decompressFrame() :
* @dctx must be properly initialized */ * @dctx must be properly initialized */
static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx, static size_t ZSTD_decompressFrame(ZSTD_DCtx *dctx, void *dst, size_t dstCapacity, const void **srcPtr, size_t *srcSizePtr)
void* dst, size_t dstCapacity,
const void** srcPtr, size_t *srcSizePtr)
{ {
const BYTE* ip = (const BYTE*)(*srcPtr); const BYTE *ip = (const BYTE *)(*srcPtr);
BYTE* const ostart = (BYTE* const)dst; BYTE *const ostart = (BYTE * const)dst;
BYTE* const oend = ostart + dstCapacity; BYTE *const oend = ostart + dstCapacity;
BYTE* op = ostart; BYTE *op = ostart;
size_t remainingSize = *srcSizePtr; size_t remainingSize = *srcSizePtr;
/* check */ /* check */
if (remainingSize < ZSTD_frameHeaderSize_min+ZSTD_blockHeaderSize) return ERROR(srcSize_wrong); if (remainingSize < ZSTD_frameHeaderSize_min + ZSTD_blockHeaderSize)
return ERROR(srcSize_wrong);
/* Frame Header */ /* Frame Header */
{ size_t const frameHeaderSize = ZSTD_frameHeaderSize(ip, ZSTD_frameHeaderSize_prefix); {
if (ZSTD_isError(frameHeaderSize)) return frameHeaderSize; size_t const frameHeaderSize = ZSTD_frameHeaderSize(ip, ZSTD_frameHeaderSize_prefix);
if (remainingSize < frameHeaderSize+ZSTD_blockHeaderSize) return ERROR(srcSize_wrong); if (ZSTD_isError(frameHeaderSize))
return frameHeaderSize;
if (remainingSize < frameHeaderSize + ZSTD_blockHeaderSize)
return ERROR(srcSize_wrong);
CHECK_F(ZSTD_decodeFrameHeader(dctx, ip, frameHeaderSize)); CHECK_F(ZSTD_decodeFrameHeader(dctx, ip, frameHeaderSize));
ip += frameHeaderSize; remainingSize -= frameHeaderSize; ip += frameHeaderSize;
remainingSize -= frameHeaderSize;
} }
/* Loop on each block */ /* Loop on each block */
@@ -1473,42 +1578,41 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx,
size_t decodedSize; size_t decodedSize;
blockProperties_t blockProperties; blockProperties_t blockProperties;
size_t const cBlockSize = ZSTD_getcBlockSize(ip, remainingSize, &blockProperties); size_t const cBlockSize = ZSTD_getcBlockSize(ip, remainingSize, &blockProperties);
if (ZSTD_isError(cBlockSize)) return cBlockSize; if (ZSTD_isError(cBlockSize))
return cBlockSize;
ip += ZSTD_blockHeaderSize; ip += ZSTD_blockHeaderSize;
remainingSize -= ZSTD_blockHeaderSize; remainingSize -= ZSTD_blockHeaderSize;
if (cBlockSize > remainingSize) return ERROR(srcSize_wrong); if (cBlockSize > remainingSize)
return ERROR(srcSize_wrong);
switch(blockProperties.blockType) switch (blockProperties.blockType) {
{ case bt_compressed: decodedSize = ZSTD_decompressBlock_internal(dctx, op, oend - op, ip, cBlockSize); break;
case bt_compressed: case bt_raw: decodedSize = ZSTD_copyRawBlock(op, oend - op, ip, cBlockSize); break;
decodedSize = ZSTD_decompressBlock_internal(dctx, op, oend-op, ip, cBlockSize); case bt_rle: decodedSize = ZSTD_generateNxBytes(op, oend - op, *ip, blockProperties.origSize); break;
break; case bt_reserved:
case bt_raw : default: return ERROR(corruption_detected);
decodedSize = ZSTD_copyRawBlock(op, oend-op, ip, cBlockSize);
break;
case bt_rle :
decodedSize = ZSTD_generateNxBytes(op, oend-op, *ip, blockProperties.origSize);
break;
case bt_reserved :
default:
return ERROR(corruption_detected);
} }
if (ZSTD_isError(decodedSize)) return decodedSize; if (ZSTD_isError(decodedSize))
if (dctx->fParams.checksumFlag) xxh64_update(&dctx->xxhState, op, decodedSize); return decodedSize;
if (dctx->fParams.checksumFlag)
xxh64_update(&dctx->xxhState, op, decodedSize);
op += decodedSize; op += decodedSize;
ip += cBlockSize; ip += cBlockSize;
remainingSize -= cBlockSize; remainingSize -= cBlockSize;
if (blockProperties.lastBlock) break; if (blockProperties.lastBlock)
break;
} }
if (dctx->fParams.checksumFlag) { /* Frame content checksum verification */ if (dctx->fParams.checksumFlag) { /* Frame content checksum verification */
U32 const checkCalc = (U32)xxh64_digest(&dctx->xxhState); U32 const checkCalc = (U32)xxh64_digest(&dctx->xxhState);
U32 checkRead; U32 checkRead;
if (remainingSize<4) return ERROR(checksum_wrong); if (remainingSize < 4)
return ERROR(checksum_wrong);
checkRead = ZSTD_readLE32(ip); checkRead = ZSTD_readLE32(ip);
if (checkRead != checkCalc) return ERROR(checksum_wrong); if (checkRead != checkCalc)
return ERROR(checksum_wrong);
ip += 4; ip += 4;
remainingSize -= 4; remainingSize -= 4;
} }
@@ -1516,19 +1620,16 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx,
/* Allow caller to get size read */ /* Allow caller to get size read */
*srcPtr = ip; *srcPtr = ip;
*srcSizePtr = remainingSize; *srcSizePtr = remainingSize;
return op-ostart; return op - ostart;
} }
static const void* ZSTD_DDictDictContent(const ZSTD_DDict* ddict); static const void *ZSTD_DDictDictContent(const ZSTD_DDict *ddict);
static size_t ZSTD_DDictDictSize(const ZSTD_DDict* ddict); static size_t ZSTD_DDictDictSize(const ZSTD_DDict *ddict);
static size_t ZSTD_decompressMultiFrame(ZSTD_DCtx* dctx, static size_t ZSTD_decompressMultiFrame(ZSTD_DCtx *dctx, void *dst, size_t dstCapacity, const void *src, size_t srcSize, const void *dict, size_t dictSize,
void* dst, size_t dstCapacity, const ZSTD_DDict *ddict)
const void* src, size_t srcSize,
const void *dict, size_t dictSize,
const ZSTD_DDict* ddict)
{ {
void* const dststart = dst; void *const dststart = dst;
if (ddict) { if (ddict) {
if (dict) { if (dict) {
@@ -1549,8 +1650,7 @@ static size_t ZSTD_decompressMultiFrame(ZSTD_DCtx* dctx,
size_t skippableSize; size_t skippableSize;
if (srcSize < ZSTD_skippableHeaderSize) if (srcSize < ZSTD_skippableHeaderSize)
return ERROR(srcSize_wrong); return ERROR(srcSize_wrong);
skippableSize = ZSTD_readLE32((const BYTE *)src + 4) + skippableSize = ZSTD_readLE32((const BYTE *)src + 4) + ZSTD_skippableHeaderSize;
ZSTD_skippableHeaderSize;
if (srcSize < skippableSize) { if (srcSize < skippableSize) {
return ERROR(srcSize_wrong); return ERROR(srcSize_wrong);
} }
@@ -1573,75 +1673,71 @@ static size_t ZSTD_decompressMultiFrame(ZSTD_DCtx* dctx,
} }
ZSTD_checkContinuity(dctx, dst); ZSTD_checkContinuity(dctx, dst);
{ const size_t res = ZSTD_decompressFrame(dctx, dst, dstCapacity, {
&src, &srcSize); const size_t res = ZSTD_decompressFrame(dctx, dst, dstCapacity, &src, &srcSize);
if (ZSTD_isError(res)) return res; if (ZSTD_isError(res))
return res;
/* don't need to bounds check this, ZSTD_decompressFrame will have /* don't need to bounds check this, ZSTD_decompressFrame will have
* already */ * already */
dst = (BYTE*)dst + res; dst = (BYTE *)dst + res;
dstCapacity -= res; dstCapacity -= res;
} }
} }
if (srcSize) return ERROR(srcSize_wrong); /* input not entirely consumed */ if (srcSize)
return ERROR(srcSize_wrong); /* input not entirely consumed */
return (BYTE*)dst - (BYTE*)dststart; return (BYTE *)dst - (BYTE *)dststart;
} }
size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, const void *dict, size_t dictSize) size_t ZSTD_decompress_usingDict(ZSTD_DCtx *dctx, void *dst, size_t dstCapacity, const void *src, size_t srcSize, const void *dict, size_t dictSize)
{ {
return ZSTD_decompressMultiFrame(dctx, dst, dstCapacity, src, srcSize, dict, dictSize, NULL); return ZSTD_decompressMultiFrame(dctx, dst, dstCapacity, src, srcSize, dict, dictSize, NULL);
} }
size_t ZSTD_decompressDCtx(ZSTD_DCtx *dctx, void *dst, size_t dstCapacity, const void *src, size_t srcSize)
size_t ZSTD_decompressDCtx(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize)
{ {
return ZSTD_decompress_usingDict(dctx, dst, dstCapacity, src, srcSize, NULL, 0); return ZSTD_decompress_usingDict(dctx, dst, dstCapacity, src, srcSize, NULL, 0);
} }
/*-************************************** /*-**************************************
* Advanced Streaming Decompression API * Advanced Streaming Decompression API
* Bufferless and synchronous * Bufferless and synchronous
****************************************/ ****************************************/
size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx) { return dctx->expected; } size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx *dctx) { return dctx->expected; }
ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx) { ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx *dctx)
switch(dctx->stage) {
{ switch (dctx->stage) {
default: /* should not happen */ default: /* should not happen */
case ZSTDds_getFrameHeaderSize: case ZSTDds_getFrameHeaderSize:
case ZSTDds_decodeFrameHeader: case ZSTDds_decodeFrameHeader: return ZSTDnit_frameHeader;
return ZSTDnit_frameHeader; case ZSTDds_decodeBlockHeader: return ZSTDnit_blockHeader;
case ZSTDds_decodeBlockHeader: case ZSTDds_decompressBlock: return ZSTDnit_block;
return ZSTDnit_blockHeader; case ZSTDds_decompressLastBlock: return ZSTDnit_lastBlock;
case ZSTDds_decompressBlock: case ZSTDds_checkChecksum: return ZSTDnit_checksum;
return ZSTDnit_block;
case ZSTDds_decompressLastBlock:
return ZSTDnit_lastBlock;
case ZSTDds_checkChecksum:
return ZSTDnit_checksum;
case ZSTDds_decodeSkippableHeader: case ZSTDds_decodeSkippableHeader:
case ZSTDds_skipFrame: case ZSTDds_skipFrame: return ZSTDnit_skippableFrame;
return ZSTDnit_skippableFrame;
} }
} }
int ZSTD_isSkipFrame(ZSTD_DCtx* dctx) { return dctx->stage == ZSTDds_skipFrame; } /* for zbuff */ int ZSTD_isSkipFrame(ZSTD_DCtx *dctx) { return dctx->stage == ZSTDds_skipFrame; } /* for zbuff */
/** ZSTD_decompressContinue() : /** ZSTD_decompressContinue() :
* @return : nb of bytes generated into `dst` (necessarily <= `dstCapacity) * @return : nb of bytes generated into `dst` (necessarily <= `dstCapacity)
* or an error code, which can be tested using ZSTD_isError() */ * or an error code, which can be tested using ZSTD_isError() */
size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize) size_t ZSTD_decompressContinue(ZSTD_DCtx *dctx, void *dst, size_t dstCapacity, const void *src, size_t srcSize)
{ {
/* Sanity check */ /* Sanity check */
if (srcSize != dctx->expected) return ERROR(srcSize_wrong); if (srcSize != dctx->expected)
if (dstCapacity) ZSTD_checkContinuity(dctx, dst); return ERROR(srcSize_wrong);
if (dstCapacity)
ZSTD_checkContinuity(dctx, dst);
switch (dctx->stage) switch (dctx->stage) {
{ case ZSTDds_getFrameHeaderSize:
case ZSTDds_getFrameHeaderSize : if (srcSize != ZSTD_frameHeaderSize_prefix)
if (srcSize != ZSTD_frameHeaderSize_prefix) return ERROR(srcSize_wrong); /* impossible */ return ERROR(srcSize_wrong); /* impossible */
if ((ZSTD_readLE32(src) & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) { /* skippable frame */ if ((ZSTD_readLE32(src) & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) { /* skippable frame */
memcpy(dctx->headerBuffer, src, ZSTD_frameHeaderSize_prefix); memcpy(dctx->headerBuffer, src, ZSTD_frameHeaderSize_prefix);
dctx->expected = ZSTD_skippableHeaderSize - ZSTD_frameHeaderSize_prefix; /* magic number + skippable frame length */ dctx->expected = ZSTD_skippableHeaderSize - ZSTD_frameHeaderSize_prefix; /* magic number + skippable frame length */
@@ -1649,7 +1745,8 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c
return 0; return 0;
} }
dctx->headerSize = ZSTD_frameHeaderSize(src, ZSTD_frameHeaderSize_prefix); dctx->headerSize = ZSTD_frameHeaderSize(src, ZSTD_frameHeaderSize_prefix);
if (ZSTD_isError(dctx->headerSize)) return dctx->headerSize; if (ZSTD_isError(dctx->headerSize))
return dctx->headerSize;
memcpy(dctx->headerBuffer, src, ZSTD_frameHeaderSize_prefix); memcpy(dctx->headerBuffer, src, ZSTD_frameHeaderSize_prefix);
if (dctx->headerSize > ZSTD_frameHeaderSize_prefix) { if (dctx->headerSize > ZSTD_frameHeaderSize_prefix) {
dctx->expected = dctx->headerSize - ZSTD_frameHeaderSize_prefix; dctx->expected = dctx->headerSize - ZSTD_frameHeaderSize_prefix;
@@ -1665,10 +1762,11 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c
dctx->stage = ZSTDds_decodeBlockHeader; dctx->stage = ZSTDds_decodeBlockHeader;
return 0; return 0;
case ZSTDds_decodeBlockHeader: case ZSTDds_decodeBlockHeader: {
{ blockProperties_t bp; blockProperties_t bp;
size_t const cBlockSize = ZSTD_getcBlockSize(src, ZSTD_blockHeaderSize, &bp); size_t const cBlockSize = ZSTD_getcBlockSize(src, ZSTD_blockHeaderSize, &bp);
if (ZSTD_isError(cBlockSize)) return cBlockSize; if (ZSTD_isError(cBlockSize))
return cBlockSize;
dctx->expected = cBlockSize; dctx->expected = cBlockSize;
dctx->bType = bp.blockType; dctx->bType = bp.blockType;
dctx->rleSize = bp.origSize; dctx->rleSize = bp.origSize;
@@ -1692,25 +1790,19 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c
return 0; return 0;
} }
case ZSTDds_decompressLastBlock: case ZSTDds_decompressLastBlock:
case ZSTDds_decompressBlock: case ZSTDds_decompressBlock: {
{ size_t rSize; size_t rSize;
switch(dctx->bType) switch (dctx->bType) {
{ case bt_compressed: rSize = ZSTD_decompressBlock_internal(dctx, dst, dstCapacity, src, srcSize); break;
case bt_compressed: case bt_raw: rSize = ZSTD_copyRawBlock(dst, dstCapacity, src, srcSize); break;
rSize = ZSTD_decompressBlock_internal(dctx, dst, dstCapacity, src, srcSize); case bt_rle: rSize = ZSTD_setRleBlock(dst, dstCapacity, src, srcSize, dctx->rleSize); break;
break; case bt_reserved: /* should never happen */
case bt_raw : default: return ERROR(corruption_detected);
rSize = ZSTD_copyRawBlock(dst, dstCapacity, src, srcSize);
break;
case bt_rle :
rSize = ZSTD_setRleBlock(dst, dstCapacity, src, srcSize, dctx->rleSize);
break;
case bt_reserved : /* should never happen */
default:
return ERROR(corruption_detected);
} }
if (ZSTD_isError(rSize)) return rSize; if (ZSTD_isError(rSize))
if (dctx->fParams.checksumFlag) xxh64_update(&dctx->xxhState, dst, rSize); return rSize;
if (dctx->fParams.checksumFlag)
xxh64_update(&dctx->xxhState, dst, rSize);
if (dctx->stage == ZSTDds_decompressLastBlock) { /* end of frame */ if (dctx->stage == ZSTDds_decompressLastBlock) { /* end of frame */
if (dctx->fParams.checksumFlag) { /* another round for frame checksum */ if (dctx->fParams.checksumFlag) { /* another round for frame checksum */
@@ -1723,26 +1815,27 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c
} else { } else {
dctx->stage = ZSTDds_decodeBlockHeader; dctx->stage = ZSTDds_decodeBlockHeader;
dctx->expected = ZSTD_blockHeaderSize; dctx->expected = ZSTD_blockHeaderSize;
dctx->previousDstEnd = (char*)dst + rSize; dctx->previousDstEnd = (char *)dst + rSize;
} }
return rSize; return rSize;
} }
case ZSTDds_checkChecksum: case ZSTDds_checkChecksum: {
{ U32 const h32 = (U32)xxh64_digest(&dctx->xxhState); U32 const h32 = (U32)xxh64_digest(&dctx->xxhState);
U32 const check32 = ZSTD_readLE32(src); /* srcSize == 4, guaranteed by dctx->expected */ U32 const check32 = ZSTD_readLE32(src); /* srcSize == 4, guaranteed by dctx->expected */
if (check32 != h32) return ERROR(checksum_wrong); if (check32 != h32)
return ERROR(checksum_wrong);
dctx->expected = 0; dctx->expected = 0;
dctx->stage = ZSTDds_getFrameHeaderSize; dctx->stage = ZSTDds_getFrameHeaderSize;
return 0; return 0;
} }
case ZSTDds_decodeSkippableHeader: case ZSTDds_decodeSkippableHeader: {
{ memcpy(dctx->headerBuffer + ZSTD_frameHeaderSize_prefix, src, dctx->expected); memcpy(dctx->headerBuffer + ZSTD_frameHeaderSize_prefix, src, dctx->expected);
dctx->expected = ZSTD_readLE32(dctx->headerBuffer + 4); dctx->expected = ZSTD_readLE32(dctx->headerBuffer + 4);
dctx->stage = ZSTDds_skipFrame; dctx->stage = ZSTDds_skipFrame;
return 0; return 0;
} }
case ZSTDds_skipFrame: case ZSTDds_skipFrame: {
{ dctx->expected = 0; dctx->expected = 0;
dctx->stage = ZSTDds_getFrameHeaderSize; dctx->stage = ZSTDds_getFrameHeaderSize;
return 0; return 0;
} }
@@ -1751,85 +1844,105 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c
} }
} }
static size_t ZSTD_refDictContent(ZSTD_DCtx *dctx, const void *dict, size_t dictSize)
static size_t ZSTD_refDictContent(ZSTD_DCtx* dctx, const void* dict, size_t dictSize)
{ {
dctx->dictEnd = dctx->previousDstEnd; dctx->dictEnd = dctx->previousDstEnd;
dctx->vBase = (const char*)dict - ((const char*)(dctx->previousDstEnd) - (const char*)(dctx->base)); dctx->vBase = (const char *)dict - ((const char *)(dctx->previousDstEnd) - (const char *)(dctx->base));
dctx->base = dict; dctx->base = dict;
dctx->previousDstEnd = (const char*)dict + dictSize; dctx->previousDstEnd = (const char *)dict + dictSize;
return 0; return 0;
} }
/* ZSTD_loadEntropy() : /* ZSTD_loadEntropy() :
* dict : must point at beginning of a valid zstd dictionary * dict : must point at beginning of a valid zstd dictionary
* @return : size of entropy tables read */ * @return : size of entropy tables read */
static size_t ZSTD_loadEntropy(ZSTD_entropyTables_t* entropy, const void* const dict, size_t const dictSize) static size_t ZSTD_loadEntropy(ZSTD_entropyTables_t *entropy, const void *const dict, size_t const dictSize)
{ {
const BYTE* dictPtr = (const BYTE*)dict; const BYTE *dictPtr = (const BYTE *)dict;
const BYTE* const dictEnd = dictPtr + dictSize; const BYTE *const dictEnd = dictPtr + dictSize;
if (dictSize <= 8) return ERROR(dictionary_corrupted); if (dictSize <= 8)
return ERROR(dictionary_corrupted);
dictPtr += 8; /* skip header = magic + dictID */ dictPtr += 8; /* skip header = magic + dictID */
{
{ size_t const hSize = HUF_readDTableX4(entropy->hufTable, dictPtr, dictEnd-dictPtr); size_t const hSize = HUF_readDTableX4(entropy->hufTable, dictPtr, dictEnd - dictPtr);
if (HUF_isError(hSize)) return ERROR(dictionary_corrupted); if (HUF_isError(hSize))
return ERROR(dictionary_corrupted);
dictPtr += hSize; dictPtr += hSize;
} }
{ short offcodeNCount[MaxOff+1]; {
short offcodeNCount[MaxOff + 1];
U32 offcodeMaxValue = MaxOff, offcodeLog; U32 offcodeMaxValue = MaxOff, offcodeLog;
size_t const offcodeHeaderSize = FSE_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dictPtr, dictEnd-dictPtr); size_t const offcodeHeaderSize = FSE_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dictPtr, dictEnd - dictPtr);
if (FSE_isError(offcodeHeaderSize)) return ERROR(dictionary_corrupted); if (FSE_isError(offcodeHeaderSize))
if (offcodeLog > OffFSELog) return ERROR(dictionary_corrupted); return ERROR(dictionary_corrupted);
if (offcodeLog > OffFSELog)
return ERROR(dictionary_corrupted);
CHECK_E(FSE_buildDTable(entropy->OFTable, offcodeNCount, offcodeMaxValue, offcodeLog), dictionary_corrupted); CHECK_E(FSE_buildDTable(entropy->OFTable, offcodeNCount, offcodeMaxValue, offcodeLog), dictionary_corrupted);
dictPtr += offcodeHeaderSize; dictPtr += offcodeHeaderSize;
} }
{ short matchlengthNCount[MaxML+1]; {
short matchlengthNCount[MaxML + 1];
unsigned matchlengthMaxValue = MaxML, matchlengthLog; unsigned matchlengthMaxValue = MaxML, matchlengthLog;
size_t const matchlengthHeaderSize = FSE_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dictPtr, dictEnd-dictPtr); size_t const matchlengthHeaderSize = FSE_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dictPtr, dictEnd - dictPtr);
if (FSE_isError(matchlengthHeaderSize)) return ERROR(dictionary_corrupted); if (FSE_isError(matchlengthHeaderSize))
if (matchlengthLog > MLFSELog) return ERROR(dictionary_corrupted); return ERROR(dictionary_corrupted);
if (matchlengthLog > MLFSELog)
return ERROR(dictionary_corrupted);
CHECK_E(FSE_buildDTable(entropy->MLTable, matchlengthNCount, matchlengthMaxValue, matchlengthLog), dictionary_corrupted); CHECK_E(FSE_buildDTable(entropy->MLTable, matchlengthNCount, matchlengthMaxValue, matchlengthLog), dictionary_corrupted);
dictPtr += matchlengthHeaderSize; dictPtr += matchlengthHeaderSize;
} }
{ short litlengthNCount[MaxLL+1]; {
short litlengthNCount[MaxLL + 1];
unsigned litlengthMaxValue = MaxLL, litlengthLog; unsigned litlengthMaxValue = MaxLL, litlengthLog;
size_t const litlengthHeaderSize = FSE_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dictPtr, dictEnd-dictPtr); size_t const litlengthHeaderSize = FSE_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dictPtr, dictEnd - dictPtr);
if (FSE_isError(litlengthHeaderSize)) return ERROR(dictionary_corrupted); if (FSE_isError(litlengthHeaderSize))
if (litlengthLog > LLFSELog) return ERROR(dictionary_corrupted); return ERROR(dictionary_corrupted);
if (litlengthLog > LLFSELog)
return ERROR(dictionary_corrupted);
CHECK_E(FSE_buildDTable(entropy->LLTable, litlengthNCount, litlengthMaxValue, litlengthLog), dictionary_corrupted); CHECK_E(FSE_buildDTable(entropy->LLTable, litlengthNCount, litlengthMaxValue, litlengthLog), dictionary_corrupted);
dictPtr += litlengthHeaderSize; dictPtr += litlengthHeaderSize;
} }
if (dictPtr+12 > dictEnd) return ERROR(dictionary_corrupted); if (dictPtr + 12 > dictEnd)
{ int i; return ERROR(dictionary_corrupted);
size_t const dictContentSize = (size_t)(dictEnd - (dictPtr+12)); {
for (i=0; i<3; i++) { int i;
U32 const rep = ZSTD_readLE32(dictPtr); dictPtr += 4; size_t const dictContentSize = (size_t)(dictEnd - (dictPtr + 12));
if (rep==0 || rep >= dictContentSize) return ERROR(dictionary_corrupted); for (i = 0; i < 3; i++) {
U32 const rep = ZSTD_readLE32(dictPtr);
dictPtr += 4;
if (rep == 0 || rep >= dictContentSize)
return ERROR(dictionary_corrupted);
entropy->rep[i] = rep; entropy->rep[i] = rep;
} } }
}
return dictPtr - (const BYTE*)dict; return dictPtr - (const BYTE *)dict;
} }
static size_t ZSTD_decompress_insertDictionary(ZSTD_DCtx* dctx, const void* dict, size_t dictSize) static size_t ZSTD_decompress_insertDictionary(ZSTD_DCtx *dctx, const void *dict, size_t dictSize)
{ {
if (dictSize < 8) return ZSTD_refDictContent(dctx, dict, dictSize); if (dictSize < 8)
{ U32 const magic = ZSTD_readLE32(dict); return ZSTD_refDictContent(dctx, dict, dictSize);
{
U32 const magic = ZSTD_readLE32(dict);
if (magic != ZSTD_DICT_MAGIC) { if (magic != ZSTD_DICT_MAGIC) {
return ZSTD_refDictContent(dctx, dict, dictSize); /* pure content mode */ return ZSTD_refDictContent(dctx, dict, dictSize); /* pure content mode */
} } }
dctx->dictID = ZSTD_readLE32((const char*)dict + 4); }
dctx->dictID = ZSTD_readLE32((const char *)dict + 4);
/* load entropy tables */ /* load entropy tables */
{ size_t const eSize = ZSTD_loadEntropy(&dctx->entropy, dict, dictSize); {
if (ZSTD_isError(eSize)) return ERROR(dictionary_corrupted); size_t const eSize = ZSTD_loadEntropy(&dctx->entropy, dict, dictSize);
dict = (const char*)dict + eSize; if (ZSTD_isError(eSize))
return ERROR(dictionary_corrupted);
dict = (const char *)dict + eSize;
dictSize -= eSize; dictSize -= eSize;
} }
dctx->litEntropy = dctx->fseEntropy = 1; dctx->litEntropy = dctx->fseEntropy = 1;
@@ -1838,19 +1951,19 @@ static size_t ZSTD_decompress_insertDictionary(ZSTD_DCtx* dctx, const void* dict
return ZSTD_refDictContent(dctx, dict, dictSize); return ZSTD_refDictContent(dctx, dict, dictSize);
} }
size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t dictSize) size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx *dctx, const void *dict, size_t dictSize)
{ {
CHECK_F(ZSTD_decompressBegin(dctx)); CHECK_F(ZSTD_decompressBegin(dctx));
if (dict && dictSize) CHECK_E(ZSTD_decompress_insertDictionary(dctx, dict, dictSize), dictionary_corrupted); if (dict && dictSize)
CHECK_E(ZSTD_decompress_insertDictionary(dctx, dict, dictSize), dictionary_corrupted);
return 0; return 0;
} }
/* ====== ZSTD_DDict ====== */ /* ====== ZSTD_DDict ====== */
struct ZSTD_DDict_s { struct ZSTD_DDict_s {
void* dictBuffer; void *dictBuffer;
const void* dictContent; const void *dictContent;
size_t dictSize; size_t dictSize;
ZSTD_entropyTables_t entropy; ZSTD_entropyTables_t entropy;
U32 dictID; U32 dictID;
@@ -1858,29 +1971,20 @@ struct ZSTD_DDict_s {
ZSTD_customMem cMem; ZSTD_customMem cMem;
}; /* typedef'd to ZSTD_DDict within "zstd.h" */ }; /* typedef'd to ZSTD_DDict within "zstd.h" */
size_t ZSTD_DDictWorkspaceBound(void) size_t ZSTD_DDictWorkspaceBound(void) { return ZSTD_ALIGN(sizeof(ZSTD_stack)) + ZSTD_ALIGN(sizeof(ZSTD_DDict)); }
{
return ZSTD_ALIGN(sizeof(ZSTD_stack)) + ZSTD_ALIGN(sizeof(ZSTD_DDict));
}
static const void* ZSTD_DDictDictContent(const ZSTD_DDict* ddict) static const void *ZSTD_DDictDictContent(const ZSTD_DDict *ddict) { return ddict->dictContent; }
{
return ddict->dictContent;
}
static size_t ZSTD_DDictDictSize(const ZSTD_DDict* ddict) static size_t ZSTD_DDictDictSize(const ZSTD_DDict *ddict) { return ddict->dictSize; }
{
return ddict->dictSize;
}
static void ZSTD_refDDict(ZSTD_DCtx* dstDCtx, const ZSTD_DDict* ddict) static void ZSTD_refDDict(ZSTD_DCtx *dstDCtx, const ZSTD_DDict *ddict)
{ {
ZSTD_decompressBegin(dstDCtx); /* init */ ZSTD_decompressBegin(dstDCtx); /* init */
if (ddict) { /* support refDDict on NULL */ if (ddict) { /* support refDDict on NULL */
dstDCtx->dictID = ddict->dictID; dstDCtx->dictID = ddict->dictID;
dstDCtx->base = ddict->dictContent; dstDCtx->base = ddict->dictContent;
dstDCtx->vBase = ddict->dictContent; dstDCtx->vBase = ddict->dictContent;
dstDCtx->dictEnd = (const BYTE*)ddict->dictContent + ddict->dictSize; dstDCtx->dictEnd = (const BYTE *)ddict->dictContent + ddict->dictSize;
dstDCtx->previousDstEnd = dstDCtx->dictEnd; dstDCtx->previousDstEnd = dstDCtx->dictEnd;
if (ddict->entropyPresent) { if (ddict->entropyPresent) {
dstDCtx->litEntropy = 1; dstDCtx->litEntropy = 1;
@@ -1899,37 +2003,45 @@ static void ZSTD_refDDict(ZSTD_DCtx* dstDCtx, const ZSTD_DDict* ddict)
} }
} }
static size_t ZSTD_loadEntropy_inDDict(ZSTD_DDict* ddict) static size_t ZSTD_loadEntropy_inDDict(ZSTD_DDict *ddict)
{ {
ddict->dictID = 0; ddict->dictID = 0;
ddict->entropyPresent = 0; ddict->entropyPresent = 0;
if (ddict->dictSize < 8) return 0; if (ddict->dictSize < 8)
{ U32 const magic = ZSTD_readLE32(ddict->dictContent); return 0;
if (magic != ZSTD_DICT_MAGIC) return 0; /* pure content mode */ {
U32 const magic = ZSTD_readLE32(ddict->dictContent);
if (magic != ZSTD_DICT_MAGIC)
return 0; /* pure content mode */
} }
ddict->dictID = ZSTD_readLE32((const char*)ddict->dictContent + 4); ddict->dictID = ZSTD_readLE32((const char *)ddict->dictContent + 4);
/* load entropy tables */ /* load entropy tables */
CHECK_E( ZSTD_loadEntropy(&ddict->entropy, ddict->dictContent, ddict->dictSize), dictionary_corrupted ); CHECK_E(ZSTD_loadEntropy(&ddict->entropy, ddict->dictContent, ddict->dictSize), dictionary_corrupted);
ddict->entropyPresent = 1; ddict->entropyPresent = 1;
return 0; return 0;
} }
static ZSTD_DDict *ZSTD_createDDict_advanced(const void *dict, size_t dictSize, unsigned byReference, ZSTD_customMem customMem)
static ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictSize, unsigned byReference, ZSTD_customMem customMem)
{ {
if (!customMem.customAlloc || !customMem.customFree) return NULL; if (!customMem.customAlloc || !customMem.customFree)
return NULL;
{ ZSTD_DDict* const ddict = (ZSTD_DDict*) ZSTD_malloc(sizeof(ZSTD_DDict), customMem); {
if (!ddict) return NULL; ZSTD_DDict *const ddict = (ZSTD_DDict *)ZSTD_malloc(sizeof(ZSTD_DDict), customMem);
if (!ddict)
return NULL;
ddict->cMem = customMem; ddict->cMem = customMem;
if ((byReference) || (!dict) || (!dictSize)) { if ((byReference) || (!dict) || (!dictSize)) {
ddict->dictBuffer = NULL; ddict->dictBuffer = NULL;
ddict->dictContent = dict; ddict->dictContent = dict;
} else { } else {
void* const internalBuffer = ZSTD_malloc(dictSize, customMem); void *const internalBuffer = ZSTD_malloc(dictSize, customMem);
if (!internalBuffer) { ZSTD_freeDDict(ddict); return NULL; } if (!internalBuffer) {
ZSTD_freeDDict(ddict);
return NULL;
}
memcpy(internalBuffer, dict, dictSize); memcpy(internalBuffer, dict, dictSize);
ddict->dictBuffer = internalBuffer; ddict->dictBuffer = internalBuffer;
ddict->dictContent = internalBuffer; ddict->dictContent = internalBuffer;
@@ -1937,11 +2049,13 @@ static ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictSize,
ddict->dictSize = dictSize; ddict->dictSize = dictSize;
ddict->entropy.hufTable[0] = (HUF_DTable)((HufLog)*0x1000001); /* cover both little and big endian */ ddict->entropy.hufTable[0] = (HUF_DTable)((HufLog)*0x1000001); /* cover both little and big endian */
/* parse dictionary content */ /* parse dictionary content */
{ size_t const errorCode = ZSTD_loadEntropy_inDDict(ddict); {
size_t const errorCode = ZSTD_loadEntropy_inDDict(ddict);
if (ZSTD_isError(errorCode)) { if (ZSTD_isError(errorCode)) {
ZSTD_freeDDict(ddict); ZSTD_freeDDict(ddict);
return NULL; return NULL;
} } }
}
return ddict; return ddict;
} }
@@ -1951,17 +2065,18 @@ static ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictSize,
* Create a digested dictionary, to start decompression without startup delay. * Create a digested dictionary, to start decompression without startup delay.
* `dict` content is copied inside DDict. * `dict` content is copied inside DDict.
* Consequently, `dict` can be released after `ZSTD_DDict` creation */ * Consequently, `dict` can be released after `ZSTD_DDict` creation */
ZSTD_DDict* ZSTD_initDDict(const void* dict, size_t dictSize, void* workspace, size_t workspaceSize) ZSTD_DDict *ZSTD_initDDict(const void *dict, size_t dictSize, void *workspace, size_t workspaceSize)
{ {
ZSTD_customMem const stackMem = ZSTD_initStack(workspace, workspaceSize); ZSTD_customMem const stackMem = ZSTD_initStack(workspace, workspaceSize);
return ZSTD_createDDict_advanced(dict, dictSize, 1, stackMem); return ZSTD_createDDict_advanced(dict, dictSize, 1, stackMem);
} }
size_t ZSTD_freeDDict(ZSTD_DDict *ddict)
size_t ZSTD_freeDDict(ZSTD_DDict* ddict)
{ {
if (ddict==NULL) return 0; /* support free on NULL */ if (ddict == NULL)
{ ZSTD_customMem const cMem = ddict->cMem; return 0; /* support free on NULL */
{
ZSTD_customMem const cMem = ddict->cMem;
ZSTD_free(ddict->dictBuffer, cMem); ZSTD_free(ddict->dictBuffer, cMem);
ZSTD_free(ddict, cMem); ZSTD_free(ddict, cMem);
return 0; return 0;
@@ -1972,20 +2087,23 @@ size_t ZSTD_freeDDict(ZSTD_DDict* ddict)
* Provides the dictID stored within dictionary. * Provides the dictID stored within dictionary.
* if @return == 0, the dictionary is not conformant with Zstandard specification. * if @return == 0, the dictionary is not conformant with Zstandard specification.
* It can still be loaded, but as a content-only dictionary. */ * It can still be loaded, but as a content-only dictionary. */
unsigned ZSTD_getDictID_fromDict(const void* dict, size_t dictSize) unsigned ZSTD_getDictID_fromDict(const void *dict, size_t dictSize)
{ {
if (dictSize < 8) return 0; if (dictSize < 8)
if (ZSTD_readLE32(dict) != ZSTD_DICT_MAGIC) return 0; return 0;
return ZSTD_readLE32((const char*)dict + 4); if (ZSTD_readLE32(dict) != ZSTD_DICT_MAGIC)
return 0;
return ZSTD_readLE32((const char *)dict + 4);
} }
/*! ZSTD_getDictID_fromDDict() : /*! ZSTD_getDictID_fromDDict() :
* Provides the dictID of the dictionary loaded into `ddict`. * Provides the dictID of the dictionary loaded into `ddict`.
* If @return == 0, the dictionary is not conformant to Zstandard specification, or empty. * If @return == 0, the dictionary is not conformant to Zstandard specification, or empty.
* Non-conformant dictionaries can still be loaded, but as content-only dictionaries. */ * Non-conformant dictionaries can still be loaded, but as content-only dictionaries. */
unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict) unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict *ddict)
{ {
if (ddict==NULL) return 0; if (ddict == NULL)
return 0;
return ZSTD_getDictID_fromDict(ddict->dictContent, ddict->dictSize); return ZSTD_getDictID_fromDict(ddict->dictContent, ddict->dictSize);
} }
@@ -1999,49 +2117,42 @@ unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict)
* - `srcSize` is too small, and as a result, the frame header could not be decoded (only possible if `srcSize < ZSTD_FRAMEHEADERSIZE_MAX`). * - `srcSize` is too small, and as a result, the frame header could not be decoded (only possible if `srcSize < ZSTD_FRAMEHEADERSIZE_MAX`).
* - This is not a Zstandard frame. * - This is not a Zstandard frame.
* When identifying the exact failure cause, it's possible to used ZSTD_getFrameParams(), which will provide a more precise error code. */ * When identifying the exact failure cause, it's possible to used ZSTD_getFrameParams(), which will provide a more precise error code. */
unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize) unsigned ZSTD_getDictID_fromFrame(const void *src, size_t srcSize)
{ {
ZSTD_frameParams zfp = { 0 , 0 , 0 , 0 }; ZSTD_frameParams zfp = {0, 0, 0, 0};
size_t const hError = ZSTD_getFrameParams(&zfp, src, srcSize); size_t const hError = ZSTD_getFrameParams(&zfp, src, srcSize);
if (ZSTD_isError(hError)) return 0; if (ZSTD_isError(hError))
return 0;
return zfp.dictID; return zfp.dictID;
} }
/*! ZSTD_decompress_usingDDict() : /*! ZSTD_decompress_usingDDict() :
* Decompression using a pre-digested Dictionary * Decompression using a pre-digested Dictionary
* Use dictionary without significant overhead. */ * Use dictionary without significant overhead. */
size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx, size_t ZSTD_decompress_usingDDict(ZSTD_DCtx *dctx, void *dst, size_t dstCapacity, const void *src, size_t srcSize, const ZSTD_DDict *ddict)
void* dst, size_t dstCapacity,
const void* src, size_t srcSize,
const ZSTD_DDict* ddict)
{ {
/* pass content and size in case legacy frames are encountered */ /* pass content and size in case legacy frames are encountered */
return ZSTD_decompressMultiFrame(dctx, dst, dstCapacity, src, srcSize, return ZSTD_decompressMultiFrame(dctx, dst, dstCapacity, src, srcSize, NULL, 0, ddict);
NULL, 0,
ddict);
} }
/*===================================== /*=====================================
* Streaming decompression * Streaming decompression
*====================================*/ *====================================*/
typedef enum { zdss_init, zdss_loadHeader, typedef enum { zdss_init, zdss_loadHeader, zdss_read, zdss_load, zdss_flush } ZSTD_dStreamStage;
zdss_read, zdss_load, zdss_flush } ZSTD_dStreamStage;
/* *** Resource management *** */ /* *** Resource management *** */
struct ZSTD_DStream_s { struct ZSTD_DStream_s {
ZSTD_DCtx* dctx; ZSTD_DCtx *dctx;
ZSTD_DDict* ddictLocal; ZSTD_DDict *ddictLocal;
const ZSTD_DDict* ddict; const ZSTD_DDict *ddict;
ZSTD_frameParams fParams; ZSTD_frameParams fParams;
ZSTD_dStreamStage stage; ZSTD_dStreamStage stage;
char* inBuff; char *inBuff;
size_t inBuffSize; size_t inBuffSize;
size_t inPos; size_t inPos;
size_t maxWindowSize; size_t maxWindowSize;
char* outBuff; char *outBuff;
size_t outBuffSize; size_t outBuffSize;
size_t outStart; size_t outStart;
size_t outEnd; size_t outEnd;
@@ -2049,41 +2160,49 @@ struct ZSTD_DStream_s {
BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX]; /* tmp buffer to store frame header */ BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX]; /* tmp buffer to store frame header */
size_t lhSize; size_t lhSize;
ZSTD_customMem customMem; ZSTD_customMem customMem;
void* legacyContext; void *legacyContext;
U32 previousLegacyVersion; U32 previousLegacyVersion;
U32 legacyVersion; U32 legacyVersion;
U32 hostageByte; U32 hostageByte;
}; /* typedef'd to ZSTD_DStream within "zstd.h" */ }; /* typedef'd to ZSTD_DStream within "zstd.h" */
size_t ZSTD_DStreamWorkspaceBound(size_t maxWindowSize) { size_t ZSTD_DStreamWorkspaceBound(size_t maxWindowSize)
{
size_t const blockSize = MIN(maxWindowSize, ZSTD_BLOCKSIZE_ABSOLUTEMAX); size_t const blockSize = MIN(maxWindowSize, ZSTD_BLOCKSIZE_ABSOLUTEMAX);
size_t const inBuffSize = blockSize; size_t const inBuffSize = blockSize;
size_t const outBuffSize = maxWindowSize + blockSize + WILDCOPY_OVERLENGTH * 2; size_t const outBuffSize = maxWindowSize + blockSize + WILDCOPY_OVERLENGTH * 2;
return ZSTD_DCtxWorkspaceBound() + ZSTD_ALIGN(sizeof(ZSTD_DStream)) + ZSTD_ALIGN(inBuffSize) + ZSTD_ALIGN(outBuffSize); return ZSTD_DCtxWorkspaceBound() + ZSTD_ALIGN(sizeof(ZSTD_DStream)) + ZSTD_ALIGN(inBuffSize) + ZSTD_ALIGN(outBuffSize);
} }
static ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem) static ZSTD_DStream *ZSTD_createDStream_advanced(ZSTD_customMem customMem)
{ {
ZSTD_DStream* zds; ZSTD_DStream *zds;
if (!customMem.customAlloc || !customMem.customFree) return NULL; if (!customMem.customAlloc || !customMem.customFree)
return NULL;
zds = (ZSTD_DStream*) ZSTD_malloc(sizeof(ZSTD_DStream), customMem); zds = (ZSTD_DStream *)ZSTD_malloc(sizeof(ZSTD_DStream), customMem);
if (zds==NULL) return NULL; if (zds == NULL)
return NULL;
memset(zds, 0, sizeof(ZSTD_DStream)); memset(zds, 0, sizeof(ZSTD_DStream));
memcpy(&zds->customMem, &customMem, sizeof(ZSTD_customMem)); memcpy(&zds->customMem, &customMem, sizeof(ZSTD_customMem));
zds->dctx = ZSTD_createDCtx_advanced(customMem); zds->dctx = ZSTD_createDCtx_advanced(customMem);
if (zds->dctx == NULL) { ZSTD_freeDStream(zds); return NULL; } if (zds->dctx == NULL) {
ZSTD_freeDStream(zds);
return NULL;
}
zds->stage = zdss_init; zds->stage = zdss_init;
zds->maxWindowSize = ZSTD_MAXWINDOWSIZE_DEFAULT; zds->maxWindowSize = ZSTD_MAXWINDOWSIZE_DEFAULT;
return zds; return zds;
} }
ZSTD_DStream* ZSTD_initDStream(size_t maxWindowSize, void* workspace, size_t workspaceSize) ZSTD_DStream *ZSTD_initDStream(size_t maxWindowSize, void *workspace, size_t workspaceSize)
{ {
ZSTD_customMem const stackMem = ZSTD_initStack(workspace, workspaceSize); ZSTD_customMem const stackMem = ZSTD_initStack(workspace, workspaceSize);
ZSTD_DStream* zds = ZSTD_createDStream_advanced(stackMem); ZSTD_DStream *zds = ZSTD_createDStream_advanced(stackMem);
if (!zds) { return NULL; } if (!zds) {
return NULL;
}
zds->maxWindowSize = maxWindowSize; zds->maxWindowSize = maxWindowSize;
zds->stage = zdss_loadHeader; zds->stage = zdss_loadHeader;
@@ -2096,19 +2215,21 @@ ZSTD_DStream* ZSTD_initDStream(size_t maxWindowSize, void* workspace, size_t wor
return zds; return zds;
} }
ZSTD_DStream* ZSTD_initDStream_usingDDict(size_t maxWindowSize, const ZSTD_DDict* ddict, void* workspace, size_t workspaceSize) ZSTD_DStream *ZSTD_initDStream_usingDDict(size_t maxWindowSize, const ZSTD_DDict *ddict, void *workspace, size_t workspaceSize)
{ {
ZSTD_DStream* zds = ZSTD_initDStream(maxWindowSize, workspace, workspaceSize); ZSTD_DStream *zds = ZSTD_initDStream(maxWindowSize, workspace, workspaceSize);
if (zds) { if (zds) {
zds->ddict = ddict; zds->ddict = ddict;
} }
return zds; return zds;
} }
size_t ZSTD_freeDStream(ZSTD_DStream* zds) size_t ZSTD_freeDStream(ZSTD_DStream *zds)
{ {
if (zds==NULL) return 0; /* support free on null */ if (zds == NULL)
{ ZSTD_customMem const cMem = zds->customMem; return 0; /* support free on null */
{
ZSTD_customMem const cMem = zds->customMem;
ZSTD_freeDCtx(zds->dctx); ZSTD_freeDCtx(zds->dctx);
zds->dctx = NULL; zds->dctx = NULL;
ZSTD_freeDDict(zds->ddictLocal); ZSTD_freeDDict(zds->ddictLocal);
@@ -2122,13 +2243,12 @@ size_t ZSTD_freeDStream(ZSTD_DStream* zds)
} }
} }
/* *** Initialization *** */ /* *** Initialization *** */
size_t ZSTD_DStreamInSize(void) { return ZSTD_BLOCKSIZE_ABSOLUTEMAX + ZSTD_blockHeaderSize; } size_t ZSTD_DStreamInSize(void) { return ZSTD_BLOCKSIZE_ABSOLUTEMAX + ZSTD_blockHeaderSize; }
size_t ZSTD_DStreamOutSize(void) { return ZSTD_BLOCKSIZE_ABSOLUTEMAX; } size_t ZSTD_DStreamOutSize(void) { return ZSTD_BLOCKSIZE_ABSOLUTEMAX; }
size_t ZSTD_resetDStream(ZSTD_DStream* zds) size_t ZSTD_resetDStream(ZSTD_DStream *zds)
{ {
zds->stage = zdss_loadHeader; zds->stage = zdss_loadHeader;
zds->lhSize = zds->inPos = zds->outStart = zds->outEnd = 0; zds->lhSize = zds->inPos = zds->outStart = zds->outEnd = 0;
@@ -2139,142 +2259,167 @@ size_t ZSTD_resetDStream(ZSTD_DStream* zds)
/* ***** Decompression ***** */ /* ***** Decompression ***** */
ZSTD_STATIC size_t ZSTD_limitCopy(void* dst, size_t dstCapacity, const void* src, size_t srcSize) ZSTD_STATIC size_t ZSTD_limitCopy(void *dst, size_t dstCapacity, const void *src, size_t srcSize)
{ {
size_t const length = MIN(dstCapacity, srcSize); size_t const length = MIN(dstCapacity, srcSize);
memcpy(dst, src, length); memcpy(dst, src, length);
return length; return length;
} }
size_t ZSTD_decompressStream(ZSTD_DStream *zds, ZSTD_outBuffer *output, ZSTD_inBuffer *input)
size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inBuffer* input)
{ {
const char* const istart = (const char*)(input->src) + input->pos; const char *const istart = (const char *)(input->src) + input->pos;
const char* const iend = (const char*)(input->src) + input->size; const char *const iend = (const char *)(input->src) + input->size;
const char* ip = istart; const char *ip = istart;
char* const ostart = (char*)(output->dst) + output->pos; char *const ostart = (char *)(output->dst) + output->pos;
char* const oend = (char*)(output->dst) + output->size; char *const oend = (char *)(output->dst) + output->size;
char* op = ostart; char *op = ostart;
U32 someMoreWork = 1; U32 someMoreWork = 1;
while (someMoreWork) { while (someMoreWork) {
switch(zds->stage) switch (zds->stage) {
{ case zdss_init:
case zdss_init :
ZSTD_resetDStream(zds); /* transparent reset on starting decoding a new frame */ ZSTD_resetDStream(zds); /* transparent reset on starting decoding a new frame */
/* fall-through */ /* fall-through */
case zdss_loadHeader : case zdss_loadHeader: {
{ size_t const hSize = ZSTD_getFrameParams(&zds->fParams, zds->headerBuffer, zds->lhSize); size_t const hSize = ZSTD_getFrameParams(&zds->fParams, zds->headerBuffer, zds->lhSize);
if (ZSTD_isError(hSize)) if (ZSTD_isError(hSize))
return hSize; return hSize;
if (hSize != 0) { /* need more input */ if (hSize != 0) { /* need more input */
size_t const toLoad = hSize - zds->lhSize; /* if hSize!=0, hSize > zds->lhSize */ size_t const toLoad = hSize - zds->lhSize; /* if hSize!=0, hSize > zds->lhSize */
if (toLoad > (size_t)(iend-ip)) { /* not enough input to load full header */ if (toLoad > (size_t)(iend - ip)) { /* not enough input to load full header */
memcpy(zds->headerBuffer + zds->lhSize, ip, iend-ip); memcpy(zds->headerBuffer + zds->lhSize, ip, iend - ip);
zds->lhSize += iend-ip; zds->lhSize += iend - ip;
input->pos = input->size; input->pos = input->size;
return (MAX(ZSTD_frameHeaderSize_min, hSize) - zds->lhSize) + ZSTD_blockHeaderSize; /* remaining header bytes + next block header */ return (MAX(ZSTD_frameHeaderSize_min, hSize) - zds->lhSize) +
ZSTD_blockHeaderSize; /* remaining header bytes + next block header */
} }
memcpy(zds->headerBuffer + zds->lhSize, ip, toLoad); zds->lhSize = hSize; ip += toLoad; memcpy(zds->headerBuffer + zds->lhSize, ip, toLoad);
zds->lhSize = hSize;
ip += toLoad;
break; break;
} } }
}
/* check for single-pass mode opportunity */ /* check for single-pass mode opportunity */
if (zds->fParams.frameContentSize && zds->fParams.windowSize /* skippable frame if == 0 */ if (zds->fParams.frameContentSize && zds->fParams.windowSize /* skippable frame if == 0 */
&& (U64)(size_t)(oend-op) >= zds->fParams.frameContentSize) { && (U64)(size_t)(oend - op) >= zds->fParams.frameContentSize) {
size_t const cSize = ZSTD_findFrameCompressedSize(istart, iend-istart); size_t const cSize = ZSTD_findFrameCompressedSize(istart, iend - istart);
if (cSize <= (size_t)(iend-istart)) { if (cSize <= (size_t)(iend - istart)) {
size_t const decompressedSize = ZSTD_decompress_usingDDict(zds->dctx, op, oend-op, istart, cSize, zds->ddict); size_t const decompressedSize = ZSTD_decompress_usingDDict(zds->dctx, op, oend - op, istart, cSize, zds->ddict);
if (ZSTD_isError(decompressedSize)) return decompressedSize; if (ZSTD_isError(decompressedSize))
return decompressedSize;
ip = istart + cSize; ip = istart + cSize;
op += decompressedSize; op += decompressedSize;
zds->dctx->expected = 0; zds->dctx->expected = 0;
zds->stage = zdss_init; zds->stage = zdss_init;
someMoreWork = 0; someMoreWork = 0;
break; break;
} } }
}
/* Consume header */ /* Consume header */
ZSTD_refDDict(zds->dctx, zds->ddict); ZSTD_refDDict(zds->dctx, zds->ddict);
{ size_t const h1Size = ZSTD_nextSrcSizeToDecompress(zds->dctx); /* == ZSTD_frameHeaderSize_prefix */ {
size_t const h1Size = ZSTD_nextSrcSizeToDecompress(zds->dctx); /* == ZSTD_frameHeaderSize_prefix */
CHECK_F(ZSTD_decompressContinue(zds->dctx, NULL, 0, zds->headerBuffer, h1Size)); CHECK_F(ZSTD_decompressContinue(zds->dctx, NULL, 0, zds->headerBuffer, h1Size));
{ size_t const h2Size = ZSTD_nextSrcSizeToDecompress(zds->dctx); {
CHECK_F(ZSTD_decompressContinue(zds->dctx, NULL, 0, zds->headerBuffer+h1Size, h2Size)); size_t const h2Size = ZSTD_nextSrcSizeToDecompress(zds->dctx);
} } CHECK_F(ZSTD_decompressContinue(zds->dctx, NULL, 0, zds->headerBuffer + h1Size, h2Size));
}
}
zds->fParams.windowSize = MAX(zds->fParams.windowSize, 1U << ZSTD_WINDOWLOG_ABSOLUTEMIN); zds->fParams.windowSize = MAX(zds->fParams.windowSize, 1U << ZSTD_WINDOWLOG_ABSOLUTEMIN);
if (zds->fParams.windowSize > zds->maxWindowSize) return ERROR(frameParameter_windowTooLarge); if (zds->fParams.windowSize > zds->maxWindowSize)
return ERROR(frameParameter_windowTooLarge);
/* Adapt buffer sizes to frame header instructions */ /* Adapt buffer sizes to frame header instructions */
{ size_t const blockSize = MIN(zds->fParams.windowSize, ZSTD_BLOCKSIZE_ABSOLUTEMAX); {
size_t const blockSize = MIN(zds->fParams.windowSize, ZSTD_BLOCKSIZE_ABSOLUTEMAX);
size_t const neededOutSize = zds->fParams.windowSize + blockSize + WILDCOPY_OVERLENGTH * 2; size_t const neededOutSize = zds->fParams.windowSize + blockSize + WILDCOPY_OVERLENGTH * 2;
zds->blockSize = blockSize; zds->blockSize = blockSize;
if (zds->inBuffSize < blockSize) { if (zds->inBuffSize < blockSize) {
ZSTD_free(zds->inBuff, zds->customMem); ZSTD_free(zds->inBuff, zds->customMem);
zds->inBuffSize = blockSize; zds->inBuffSize = blockSize;
zds->inBuff = (char*)ZSTD_malloc(blockSize, zds->customMem); zds->inBuff = (char *)ZSTD_malloc(blockSize, zds->customMem);
if (zds->inBuff == NULL) return ERROR(memory_allocation); if (zds->inBuff == NULL)
return ERROR(memory_allocation);
} }
if (zds->outBuffSize < neededOutSize) { if (zds->outBuffSize < neededOutSize) {
ZSTD_free(zds->outBuff, zds->customMem); ZSTD_free(zds->outBuff, zds->customMem);
zds->outBuffSize = neededOutSize; zds->outBuffSize = neededOutSize;
zds->outBuff = (char*)ZSTD_malloc(neededOutSize, zds->customMem); zds->outBuff = (char *)ZSTD_malloc(neededOutSize, zds->customMem);
if (zds->outBuff == NULL) return ERROR(memory_allocation); if (zds->outBuff == NULL)
} } return ERROR(memory_allocation);
}
}
zds->stage = zdss_read; zds->stage = zdss_read;
/* pass-through */ /* pass-through */
case zdss_read: case zdss_read: {
{ size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zds->dctx); size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zds->dctx);
if (neededInSize==0) { /* end of frame */ if (neededInSize == 0) { /* end of frame */
zds->stage = zdss_init; zds->stage = zdss_init;
someMoreWork = 0; someMoreWork = 0;
break; break;
} }
if ((size_t)(iend-ip) >= neededInSize) { /* decode directly from src */ if ((size_t)(iend - ip) >= neededInSize) { /* decode directly from src */
const int isSkipFrame = ZSTD_isSkipFrame(zds->dctx); const int isSkipFrame = ZSTD_isSkipFrame(zds->dctx);
size_t const decodedSize = ZSTD_decompressContinue(zds->dctx, size_t const decodedSize = ZSTD_decompressContinue(zds->dctx, zds->outBuff + zds->outStart,
zds->outBuff + zds->outStart, (isSkipFrame ? 0 : zds->outBuffSize - zds->outStart), (isSkipFrame ? 0 : zds->outBuffSize - zds->outStart), ip, neededInSize);
ip, neededInSize); if (ZSTD_isError(decodedSize))
if (ZSTD_isError(decodedSize)) return decodedSize; return decodedSize;
ip += neededInSize; ip += neededInSize;
if (!decodedSize && !isSkipFrame) break; /* this was just a header */ if (!decodedSize && !isSkipFrame)
break; /* this was just a header */
zds->outEnd = zds->outStart + decodedSize; zds->outEnd = zds->outStart + decodedSize;
zds->stage = zdss_flush; zds->stage = zdss_flush;
break; break;
} }
if (ip==iend) { someMoreWork = 0; break; } /* no more input */ if (ip == iend) {
someMoreWork = 0;
break;
} /* no more input */
zds->stage = zdss_load; zds->stage = zdss_load;
/* pass-through */ /* pass-through */
} }
case zdss_load: case zdss_load: {
{ size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zds->dctx); size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zds->dctx);
size_t const toLoad = neededInSize - zds->inPos; /* should always be <= remaining space within inBuff */ size_t const toLoad = neededInSize - zds->inPos; /* should always be <= remaining space within inBuff */
size_t loadedSize; size_t loadedSize;
if (toLoad > zds->inBuffSize - zds->inPos) return ERROR(corruption_detected); /* should never happen */ if (toLoad > zds->inBuffSize - zds->inPos)
loadedSize = ZSTD_limitCopy(zds->inBuff + zds->inPos, toLoad, ip, iend-ip); return ERROR(corruption_detected); /* should never happen */
loadedSize = ZSTD_limitCopy(zds->inBuff + zds->inPos, toLoad, ip, iend - ip);
ip += loadedSize; ip += loadedSize;
zds->inPos += loadedSize; zds->inPos += loadedSize;
if (loadedSize < toLoad) { someMoreWork = 0; break; } /* not enough input, wait for more */ if (loadedSize < toLoad) {
someMoreWork = 0;
break;
} /* not enough input, wait for more */
/* decode loaded input */ /* decode loaded input */
{ const int isSkipFrame = ZSTD_isSkipFrame(zds->dctx); {
size_t const decodedSize = ZSTD_decompressContinue(zds->dctx, const int isSkipFrame = ZSTD_isSkipFrame(zds->dctx);
zds->outBuff + zds->outStart, zds->outBuffSize - zds->outStart, size_t const decodedSize = ZSTD_decompressContinue(zds->dctx, zds->outBuff + zds->outStart, zds->outBuffSize - zds->outStart,
zds->inBuff, neededInSize); zds->inBuff, neededInSize);
if (ZSTD_isError(decodedSize)) return decodedSize; if (ZSTD_isError(decodedSize))
return decodedSize;
zds->inPos = 0; /* input is consumed */ zds->inPos = 0; /* input is consumed */
if (!decodedSize && !isSkipFrame) { zds->stage = zdss_read; break; } /* this was just a header */ if (!decodedSize && !isSkipFrame) {
zds->stage = zdss_read;
break;
} /* this was just a header */
zds->outEnd = zds->outStart + decodedSize; zds->outEnd = zds->outStart + decodedSize;
zds->stage = zdss_flush; zds->stage = zdss_flush;
/* pass-through */ /* pass-through */
} } }
}
case zdss_flush: case zdss_flush: {
{ size_t const toFlushSize = zds->outEnd - zds->outStart; size_t const toFlushSize = zds->outEnd - zds->outStart;
size_t const flushedSize = ZSTD_limitCopy(op, oend-op, zds->outBuff + zds->outStart, toFlushSize); size_t const flushedSize = ZSTD_limitCopy(op, oend - op, zds->outBuff + zds->outStart, toFlushSize);
op += flushedSize; op += flushedSize;
zds->outStart += flushedSize; zds->outStart += flushedSize;
if (flushedSize == toFlushSize) { /* flush completed */ if (flushedSize == toFlushSize) { /* flush completed */
@@ -2287,29 +2432,36 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
someMoreWork = 0; someMoreWork = 0;
break; break;
} }
default: return ERROR(GENERIC); /* impossible */ default:
} } return ERROR(GENERIC); /* impossible */
}
}
/* result */ /* result */
input->pos += (size_t)(ip-istart); input->pos += (size_t)(ip - istart);
output->pos += (size_t)(op-ostart); output->pos += (size_t)(op - ostart);
{ size_t nextSrcSizeHint = ZSTD_nextSrcSizeToDecompress(zds->dctx); {
size_t nextSrcSizeHint = ZSTD_nextSrcSizeToDecompress(zds->dctx);
if (!nextSrcSizeHint) { /* frame fully decoded */ if (!nextSrcSizeHint) { /* frame fully decoded */
if (zds->outEnd == zds->outStart) { /* output fully flushed */ if (zds->outEnd == zds->outStart) { /* output fully flushed */
if (zds->hostageByte) { if (zds->hostageByte) {
if (input->pos >= input->size) { zds->stage = zdss_read; return 1; } /* can't release hostage (not present) */ if (input->pos >= input->size) {
zds->stage = zdss_read;
return 1;
} /* can't release hostage (not present) */
input->pos++; /* release hostage */ input->pos++; /* release hostage */
} }
return 0; return 0;
} }
if (!zds->hostageByte) { /* output not fully flushed; keep last byte as hostage; will be released when all output is flushed */ if (!zds->hostageByte) { /* output not fully flushed; keep last byte as hostage; will be released when all output is flushed */
input->pos--; /* note : pos > 0, otherwise, impossible to finish reading last block */ input->pos--; /* note : pos > 0, otherwise, impossible to finish reading last block */
zds->hostageByte=1; zds->hostageByte = 1;
} }
return 1; return 1;
} }
nextSrcSizeHint += ZSTD_blockHeaderSize * (ZSTD_nextInputType(zds->dctx) == ZSTDnit_block); /* preload header of next block */ nextSrcSizeHint += ZSTD_blockHeaderSize * (ZSTD_nextInputType(zds->dctx) == ZSTDnit_block); /* preload header of next block */
if (zds->inPos > nextSrcSizeHint) return ERROR(GENERIC); /* should never happen */ if (zds->inPos > nextSrcSizeHint)
return ERROR(GENERIC); /* should never happen */
nextSrcSizeHint -= zds->inPos; /* already loaded*/ nextSrcSizeHint -= zds->inPos; /* already loaded*/
return nextSrcSizeHint; return nextSrcSizeHint;
} }
contrib/linux-kernel/lib/zstd/entropy_common.c
+88 -66
View File
@@ -40,31 +40,27 @@
/* ************************************* /* *************************************
* Dependencies * Dependencies
***************************************/ ***************************************/
#include "mem.h"
#include "error_private.h" /* ERR_*, ERROR */ #include "error_private.h" /* ERR_*, ERROR */
#include "fse.h" #include "fse.h"
#include "huf.h" #include "huf.h"
#include "mem.h"
/*=== Version ===*/ /*=== Version ===*/
unsigned FSE_versionNumber(void) { return FSE_VERSION_NUMBER; } unsigned FSE_versionNumber(void) { return FSE_VERSION_NUMBER; }
/*=== Error Management ===*/ /*=== Error Management ===*/
unsigned FSE_isError(size_t code) { return ERR_isError(code); } unsigned FSE_isError(size_t code) { return ERR_isError(code); }
unsigned HUF_isError(size_t code) { return ERR_isError(code); } unsigned HUF_isError(size_t code) { return ERR_isError(code); }
/*-************************************************************** /*-**************************************************************
* FSE NCount encoding-decoding * FSE NCount encoding-decoding
****************************************************************/ ****************************************************************/
size_t FSE_readNCount (short* normalizedCounter, unsigned* maxSVPtr, unsigned* tableLogPtr, size_t FSE_readNCount(short *normalizedCounter, unsigned *maxSVPtr, unsigned *tableLogPtr, const void *headerBuffer, size_t hbSize)
const void* headerBuffer, size_t hbSize)
{ {
const BYTE* const istart = (const BYTE*) headerBuffer; const BYTE *const istart = (const BYTE *)headerBuffer;
const BYTE* const iend = istart + hbSize; const BYTE *const iend = istart + hbSize;
const BYTE* ip = istart; const BYTE *ip = istart;
int nbBits; int nbBits;
int remaining; int remaining;
int threshold; int threshold;
@@ -73,29 +69,32 @@ size_t FSE_readNCount (short* normalizedCounter, unsigned* maxSVPtr, unsigned* t
unsigned charnum = 0; unsigned charnum = 0;
int previous0 = 0; int previous0 = 0;
if (hbSize < 4) return ERROR(srcSize_wrong); if (hbSize < 4)
return ERROR(srcSize_wrong);
bitStream = ZSTD_readLE32(ip); bitStream = ZSTD_readLE32(ip);
nbBits = (bitStream & 0xF) + FSE_MIN_TABLELOG; /* extract tableLog */ nbBits = (bitStream & 0xF) + FSE_MIN_TABLELOG; /* extract tableLog */
if (nbBits > FSE_TABLELOG_ABSOLUTE_MAX) return ERROR(tableLog_tooLarge); if (nbBits > FSE_TABLELOG_ABSOLUTE_MAX)
return ERROR(tableLog_tooLarge);
bitStream >>= 4; bitStream >>= 4;
bitCount = 4; bitCount = 4;
*tableLogPtr = nbBits; *tableLogPtr = nbBits;
remaining = (1<<nbBits)+1; remaining = (1 << nbBits) + 1;
threshold = 1<<nbBits; threshold = 1 << nbBits;
nbBits++; nbBits++;
while ((remaining>1) & (charnum<=*maxSVPtr)) { while ((remaining > 1) & (charnum <= *maxSVPtr)) {
if (previous0) { if (previous0) {
unsigned n0 = charnum; unsigned n0 = charnum;
while ((bitStream & 0xFFFF) == 0xFFFF) { while ((bitStream & 0xFFFF) == 0xFFFF) {
n0 += 24; n0 += 24;
if (ip < iend-5) { if (ip < iend - 5) {
ip += 2; ip += 2;
bitStream = ZSTD_readLE32(ip) >> bitCount; bitStream = ZSTD_readLE32(ip) >> bitCount;
} else { } else {
bitStream >>= 16; bitStream >>= 16;
bitCount += 16; bitCount += 16;
} } }
}
while ((bitStream & 3) == 3) { while ((bitStream & 3) == 3) {
n0 += 3; n0 += 3;
bitStream >>= 2; bitStream >>= 2;
@@ -103,24 +102,29 @@ size_t FSE_readNCount (short* normalizedCounter, unsigned* maxSVPtr, unsigned* t
} }
n0 += bitStream & 3; n0 += bitStream & 3;
bitCount += 2; bitCount += 2;
if (n0 > *maxSVPtr) return ERROR(maxSymbolValue_tooSmall); if (n0 > *maxSVPtr)
while (charnum < n0) normalizedCounter[charnum++] = 0; return ERROR(maxSymbolValue_tooSmall);
if ((ip <= iend-7) || (ip + (bitCount>>3) <= iend-4)) { while (charnum < n0)
ip += bitCount>>3; normalizedCounter[charnum++] = 0;
if ((ip <= iend - 7) || (ip + (bitCount >> 3) <= iend - 4)) {
ip += bitCount >> 3;
bitCount &= 7; bitCount &= 7;
bitStream = ZSTD_readLE32(ip) >> bitCount; bitStream = ZSTD_readLE32(ip) >> bitCount;
} else { } else {
bitStream >>= 2; bitStream >>= 2;
} } }
{ int const max = (2*threshold-1) - remaining; }
{
int const max = (2 * threshold - 1) - remaining;
int count; int count;
if ((bitStream & (threshold-1)) < (U32)max) { if ((bitStream & (threshold - 1)) < (U32)max) {
count = bitStream & (threshold-1); count = bitStream & (threshold - 1);
bitCount += nbBits-1; bitCount += nbBits - 1;
} else { } else {
count = bitStream & (2*threshold-1); count = bitStream & (2 * threshold - 1);
if (count >= threshold) count -= max; if (count >= threshold)
count -= max;
bitCount += nbBits; bitCount += nbBits;
} }
@@ -133,24 +137,26 @@ size_t FSE_readNCount (short* normalizedCounter, unsigned* maxSVPtr, unsigned* t
threshold >>= 1; threshold >>= 1;
} }
if ((ip <= iend-7) || (ip + (bitCount>>3) <= iend-4)) { if ((ip <= iend - 7) || (ip + (bitCount >> 3) <= iend - 4)) {
ip += bitCount>>3; ip += bitCount >> 3;
bitCount &= 7; bitCount &= 7;
} else { } else {
bitCount -= (int)(8 * (iend - 4 - ip)); bitCount -= (int)(8 * (iend - 4 - ip));
ip = iend - 4; ip = iend - 4;
} }
bitStream = ZSTD_readLE32(ip) >> (bitCount & 31); bitStream = ZSTD_readLE32(ip) >> (bitCount & 31);
} } /* while ((remaining>1) & (charnum<=*maxSVPtr)) */ }
if (remaining != 1) return ERROR(corruption_detected); } /* while ((remaining>1) & (charnum<=*maxSVPtr)) */
if (bitCount > 32) return ERROR(corruption_detected); if (remaining != 1)
*maxSVPtr = charnum-1; return ERROR(corruption_detected);
if (bitCount > 32)
return ERROR(corruption_detected);
*maxSVPtr = charnum - 1;
ip += (bitCount+7)>>3; ip += (bitCount + 7) >> 3;
return ip-istart; return ip - istart;
} }
/*! HUF_readStats() : /*! HUF_readStats() :
Read compact Huffman tree, saved by HUF_writeCTable(). Read compact Huffman tree, saved by HUF_writeCTable().
`huffWeight` is destination buffer. `huffWeight` is destination buffer.
@@ -158,65 +164,81 @@ size_t FSE_readNCount (short* normalizedCounter, unsigned* maxSVPtr, unsigned* t
@return : size read from `src` , or an error Code . @return : size read from `src` , or an error Code .
Note : Needed by HUF_readCTable() and HUF_readDTableX?() . Note : Needed by HUF_readCTable() and HUF_readDTableX?() .
*/ */
size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats, size_t HUF_readStats(BYTE *huffWeight, size_t hwSize, U32 *rankStats, U32 *nbSymbolsPtr, U32 *tableLogPtr, const void *src, size_t srcSize)
U32* nbSymbolsPtr, U32* tableLogPtr,
const void* src, size_t srcSize)
{ {
U32 weightTotal; U32 weightTotal;
const BYTE* ip = (const BYTE*) src; const BYTE *ip = (const BYTE *)src;
size_t iSize; size_t iSize;
size_t oSize; size_t oSize;
if (!srcSize) return ERROR(srcSize_wrong); if (!srcSize)
return ERROR(srcSize_wrong);
iSize = ip[0]; iSize = ip[0];
/* memset(huffWeight, 0, hwSize); *//* is not necessary, even though some analyzer complain ... */ /* memset(huffWeight, 0, hwSize); */ /* is not necessary, even though some analyzer complain ... */
if (iSize >= 128) { /* special header */ if (iSize >= 128) { /* special header */
oSize = iSize - 127; oSize = iSize - 127;
iSize = ((oSize+1)/2); iSize = ((oSize + 1) / 2);
if (iSize+1 > srcSize) return ERROR(srcSize_wrong); if (iSize + 1 > srcSize)
if (oSize >= hwSize) return ERROR(corruption_detected); return ERROR(srcSize_wrong);
if (oSize >= hwSize)
return ERROR(corruption_detected);
ip += 1; ip += 1;
{ U32 n; {
for (n=0; n<oSize; n+=2) { U32 n;
huffWeight[n] = ip[n/2] >> 4; for (n = 0; n < oSize; n += 2) {
huffWeight[n+1] = ip[n/2] & 15; huffWeight[n] = ip[n / 2] >> 4;
} } } huffWeight[n + 1] = ip[n / 2] & 15;
else { /* header compressed with FSE (normal case) */ }
}
} else { /* header compressed with FSE (normal case) */
FSE_DTable fseWorkspace[FSE_DTABLE_SIZE_U32(6)]; /* 6 is max possible tableLog for HUF header (maybe even 5, to be tested) */ FSE_DTable fseWorkspace[FSE_DTABLE_SIZE_U32(6)]; /* 6 is max possible tableLog for HUF header (maybe even 5, to be tested) */
if (iSize+1 > srcSize) return ERROR(srcSize_wrong); if (iSize + 1 > srcSize)
oSize = FSE_decompress_wksp(huffWeight, hwSize-1, ip+1, iSize, fseWorkspace, 6); /* max (hwSize-1) values decoded, as last one is implied */ return ERROR(srcSize_wrong);
if (FSE_isError(oSize)) return oSize; oSize = FSE_decompress_wksp(huffWeight, hwSize - 1, ip + 1, iSize, fseWorkspace, 6); /* max (hwSize-1) values decoded, as last one is implied */
if (FSE_isError(oSize))
return oSize;
} }
/* collect weight stats */ /* collect weight stats */
memset(rankStats, 0, (HUF_TABLELOG_MAX + 1) * sizeof(U32)); memset(rankStats, 0, (HUF_TABLELOG_MAX + 1) * sizeof(U32));
weightTotal = 0; weightTotal = 0;
{ U32 n; for (n=0; n<oSize; n++) { {
if (huffWeight[n] >= HUF_TABLELOG_MAX) return ERROR(corruption_detected); U32 n;
for (n = 0; n < oSize; n++) {
if (huffWeight[n] >= HUF_TABLELOG_MAX)
return ERROR(corruption_detected);
rankStats[huffWeight[n]]++; rankStats[huffWeight[n]]++;
weightTotal += (1 << huffWeight[n]) >> 1; weightTotal += (1 << huffWeight[n]) >> 1;
} } }
if (weightTotal == 0) return ERROR(corruption_detected); }
if (weightTotal == 0)
return ERROR(corruption_detected);
/* get last non-null symbol weight (implied, total must be 2^n) */ /* get last non-null symbol weight (implied, total must be 2^n) */
{ U32 const tableLog = BIT_highbit32(weightTotal) + 1; {
if (tableLog > HUF_TABLELOG_MAX) return ERROR(corruption_detected); U32 const tableLog = BIT_highbit32(weightTotal) + 1;
if (tableLog > HUF_TABLELOG_MAX)
return ERROR(corruption_detected);
*tableLogPtr = tableLog; *tableLogPtr = tableLog;
/* determine last weight */ /* determine last weight */
{ U32 const total = 1 << tableLog; {
U32 const total = 1 << tableLog;
U32 const rest = total - weightTotal; U32 const rest = total - weightTotal;
U32 const verif = 1 << BIT_highbit32(rest); U32 const verif = 1 << BIT_highbit32(rest);
U32 const lastWeight = BIT_highbit32(rest) + 1; U32 const lastWeight = BIT_highbit32(rest) + 1;
if (verif != rest) return ERROR(corruption_detected); /* last value must be a clean power of 2 */ if (verif != rest)
return ERROR(corruption_detected); /* last value must be a clean power of 2 */
huffWeight[oSize] = (BYTE)lastWeight; huffWeight[oSize] = (BYTE)lastWeight;
rankStats[lastWeight]++; rankStats[lastWeight]++;
} } }
}
/* check tree construction validity */ /* check tree construction validity */
if ((rankStats[1] < 2) || (rankStats[1] & 1)) return ERROR(corruption_detected); /* by construction : at least 2 elts of rank 1, must be even */ if ((rankStats[1] < 2) || (rankStats[1] & 1))
return ERROR(corruption_detected); /* by construction : at least 2 elts of rank 1, must be even */
/* results */ /* results */
*nbSymbolsPtr = (U32)(oSize+1); *nbSymbolsPtr = (U32)(oSize + 1);
return iSize+1; return iSize + 1;
} }
contrib/linux-kernel/lib/zstd/error_private.h
+6 -4
View File
@@ -25,20 +25,17 @@
#include <linux/types.h> /* size_t */ #include <linux/types.h> /* size_t */
#include <linux/zstd.h> /* enum list */ #include <linux/zstd.h> /* enum list */
/* **************************************** /* ****************************************
* Compiler-specific * Compiler-specific
******************************************/ ******************************************/
#define ERR_STATIC static __attribute__((unused)) #define ERR_STATIC static __attribute__((unused))
/*-**************************************** /*-****************************************
* Customization (error_public.h) * Customization (error_public.h)
******************************************/ ******************************************/
typedef ZSTD_ErrorCode ERR_enum; typedef ZSTD_ErrorCode ERR_enum;
#define PREFIX(name) ZSTD_error_##name #define PREFIX(name) ZSTD_error_##name
/*-**************************************** /*-****************************************
* Error codes handling * Error codes handling
******************************************/ ******************************************/
@@ -46,6 +43,11 @@ typedef ZSTD_ErrorCode ERR_enum;
ERR_STATIC unsigned ERR_isError(size_t code) { return (code > ERROR(maxCode)); } ERR_STATIC unsigned ERR_isError(size_t code) { return (code > ERROR(maxCode)); }
ERR_STATIC ERR_enum ERR_getErrorCode(size_t code) { if (!ERR_isError(code)) return (ERR_enum)0; return (ERR_enum) (0-code); } ERR_STATIC ERR_enum ERR_getErrorCode(size_t code)
{
if (!ERR_isError(code))
return (ERR_enum)0;
return (ERR_enum)(0 - code);
}
#endif /* ERROR_H_MODULE */ #endif /* ERROR_H_MODULE */
contrib/linux-kernel/lib/zstd/fse.h
+71 -95
View File
@@ -40,13 +40,11 @@
#ifndef FSE_H #ifndef FSE_H
#define FSE_H #define FSE_H
/*-***************************************** /*-*****************************************
* Dependencies * Dependencies
******************************************/ ******************************************/
#include <linux/types.h> /* size_t, ptrdiff_t */ #include <linux/types.h> /* size_t, ptrdiff_t */
/*-***************************************** /*-*****************************************
* FSE_PUBLIC_API : control library symbols visibility * FSE_PUBLIC_API : control library symbols visibility
******************************************/ ******************************************/
@@ -62,7 +60,7 @@
#define FSE_EXPAND_AND_QUOTE(str) FSE_QUOTE(str) #define FSE_EXPAND_AND_QUOTE(str) FSE_QUOTE(str)
#define FSE_VERSION_STRING FSE_EXPAND_AND_QUOTE(FSE_LIB_VERSION) #define FSE_VERSION_STRING FSE_EXPAND_AND_QUOTE(FSE_LIB_VERSION)
#define FSE_VERSION_NUMBER (FSE_VERSION_MAJOR *100*100 + FSE_VERSION_MINOR *100 + FSE_VERSION_RELEASE) #define FSE_VERSION_NUMBER (FSE_VERSION_MAJOR * 100 * 100 + FSE_VERSION_MINOR * 100 + FSE_VERSION_RELEASE)
FSE_PUBLIC_API unsigned FSE_versionNumber(void); /**< library version number; to be used when checking dll version */ FSE_PUBLIC_API unsigned FSE_versionNumber(void); /**< library version number; to be used when checking dll version */
/*-***************************************** /*-*****************************************
@@ -73,7 +71,6 @@ FSE_PUBLIC_API size_t FSE_compressBound(size_t size); /* maximum compresse
/* Error Management */ /* Error Management */
FSE_PUBLIC_API unsigned FSE_isError(size_t code); /* tells if a return value is an error code */ FSE_PUBLIC_API unsigned FSE_isError(size_t code); /* tells if a return value is an error code */
/*-***************************************** /*-*****************************************
* FSE detailed API * FSE detailed API
******************************************/ ******************************************/
@@ -107,7 +104,7 @@ FSE_PUBLIC_API unsigned FSE_optimalTableLog(unsigned maxTableLog, size_t srcSize
'normalizedCounter' is a table of short, of minimum size (maxSymbolValue+1). 'normalizedCounter' is a table of short, of minimum size (maxSymbolValue+1).
@return : tableLog, @return : tableLog,
or an errorCode, which can be tested using FSE_isError() */ or an errorCode, which can be tested using FSE_isError() */
FSE_PUBLIC_API size_t FSE_normalizeCount(short* normalizedCounter, unsigned tableLog, const unsigned* count, size_t srcSize, unsigned maxSymbolValue); FSE_PUBLIC_API size_t FSE_normalizeCount(short *normalizedCounter, unsigned tableLog, const unsigned *count, size_t srcSize, unsigned maxSymbolValue);
/*! FSE_NCountWriteBound(): /*! FSE_NCountWriteBound():
Provides the maximum possible size of an FSE normalized table, given 'maxSymbolValue' and 'tableLog'. Provides the maximum possible size of an FSE normalized table, given 'maxSymbolValue' and 'tableLog'.
@@ -118,8 +115,7 @@ FSE_PUBLIC_API size_t FSE_NCountWriteBound(unsigned maxSymbolValue, unsigned tab
Compactly save 'normalizedCounter' into 'buffer'. Compactly save 'normalizedCounter' into 'buffer'.
@return : size of the compressed table, @return : size of the compressed table,
or an errorCode, which can be tested using FSE_isError(). */ or an errorCode, which can be tested using FSE_isError(). */
FSE_PUBLIC_API size_t FSE_writeNCount (void* buffer, size_t bufferSize, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog); FSE_PUBLIC_API size_t FSE_writeNCount(void *buffer, size_t bufferSize, const short *normalizedCounter, unsigned maxSymbolValue, unsigned tableLog);
/*! Constructor and Destructor of FSE_CTable. /*! Constructor and Destructor of FSE_CTable.
Note that FSE_CTable size depends on 'tableLog' and 'maxSymbolValue' */ Note that FSE_CTable size depends on 'tableLog' and 'maxSymbolValue' */
@@ -130,7 +126,7 @@ typedef unsigned FSE_CTable; /* don't allocate that. It's only meant to be mor
@return : size of compressed data (<= `dstCapacity`), @return : size of compressed data (<= `dstCapacity`),
or 0 if compressed data could not fit into `dst`, or 0 if compressed data could not fit into `dst`,
or an errorCode, which can be tested using FSE_isError() */ or an errorCode, which can be tested using FSE_isError() */
FSE_PUBLIC_API size_t FSE_compress_usingCTable (void* dst, size_t dstCapacity, const void* src, size_t srcSize, const FSE_CTable* ct); FSE_PUBLIC_API size_t FSE_compress_usingCTable(void *dst, size_t dstCapacity, const void *src, size_t srcSize, const FSE_CTable *ct);
/*! /*!
Tutorial : Tutorial :
@@ -175,7 +171,6 @@ If it returns '0', compressed data could not fit into 'dst'.
If there is an error, the function will return an ErrorCode (which can be tested using FSE_isError()). If there is an error, the function will return an ErrorCode (which can be tested using FSE_isError()).
*/ */
/* *** DECOMPRESSION *** */ /* *** DECOMPRESSION *** */
/*! FSE_readNCount(): /*! FSE_readNCount():
@@ -183,7 +178,7 @@ If there is an error, the function will return an ErrorCode (which can be tested
@return : size read from 'rBuffer', @return : size read from 'rBuffer',
or an errorCode, which can be tested using FSE_isError(). or an errorCode, which can be tested using FSE_isError().
maxSymbolValuePtr[0] and tableLogPtr[0] will also be updated with their respective values */ maxSymbolValuePtr[0] and tableLogPtr[0] will also be updated with their respective values */
FSE_PUBLIC_API size_t FSE_readNCount (short* normalizedCounter, unsigned* maxSymbolValuePtr, unsigned* tableLogPtr, const void* rBuffer, size_t rBuffSize); FSE_PUBLIC_API size_t FSE_readNCount(short *normalizedCounter, unsigned *maxSymbolValuePtr, unsigned *tableLogPtr, const void *rBuffer, size_t rBuffSize);
/*! Constructor and Destructor of FSE_DTable. /*! Constructor and Destructor of FSE_DTable.
Note that its size depends on 'tableLog' */ Note that its size depends on 'tableLog' */
@@ -192,14 +187,14 @@ typedef unsigned FSE_DTable; /* don't allocate that. It's just a way to be mor
/*! FSE_buildDTable(): /*! FSE_buildDTable():
Builds 'dt', which must be already allocated, using FSE_createDTable(). Builds 'dt', which must be already allocated, using FSE_createDTable().
return : 0, or an errorCode, which can be tested using FSE_isError() */ return : 0, or an errorCode, which can be tested using FSE_isError() */
FSE_PUBLIC_API size_t FSE_buildDTable (FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog); FSE_PUBLIC_API size_t FSE_buildDTable(FSE_DTable *dt, const short *normalizedCounter, unsigned maxSymbolValue, unsigned tableLog);
/*! FSE_decompress_usingDTable(): /*! FSE_decompress_usingDTable():
Decompress compressed source `cSrc` of size `cSrcSize` using `dt` Decompress compressed source `cSrc` of size `cSrcSize` using `dt`
into `dst` which must be already allocated. into `dst` which must be already allocated.
@return : size of regenerated data (necessarily <= `dstCapacity`), @return : size of regenerated data (necessarily <= `dstCapacity`),
or an errorCode, which can be tested using FSE_isError() */ or an errorCode, which can be tested using FSE_isError() */
FSE_PUBLIC_API size_t FSE_decompress_usingDTable(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, const FSE_DTable* dt); FSE_PUBLIC_API size_t FSE_decompress_usingDTable(void *dst, size_t dstCapacity, const void *cSrc, size_t cSrcSize, const FSE_DTable *dt);
/*! /*!
Tutorial : Tutorial :
@@ -229,23 +224,20 @@ FSE_decompress_usingDTable() result will tell how many bytes were regenerated (<
If there is an error, the function will return an error code, which can be tested using FSE_isError(). (ex: dst buffer too small) If there is an error, the function will return an error code, which can be tested using FSE_isError(). (ex: dst buffer too small)
*/ */
/* *** Dependency *** */ /* *** Dependency *** */
#include "bitstream.h" #include "bitstream.h"
/* ***************************************** /* *****************************************
* Static allocation * Static allocation
*******************************************/ *******************************************/
/* FSE buffer bounds */ /* FSE buffer bounds */
#define FSE_NCOUNTBOUND 512 #define FSE_NCOUNTBOUND 512
#define FSE_BLOCKBOUND(size) (size + (size>>7)) #define FSE_BLOCKBOUND(size) (size + (size >> 7))
#define FSE_COMPRESSBOUND(size) (FSE_NCOUNTBOUND + FSE_BLOCKBOUND(size)) /* Macro version, useful for static allocation */ #define FSE_COMPRESSBOUND(size) (FSE_NCOUNTBOUND + FSE_BLOCKBOUND(size)) /* Macro version, useful for static allocation */
/* It is possible to statically allocate FSE CTable/DTable as a table of FSE_CTable/FSE_DTable using below macros */ /* It is possible to statically allocate FSE CTable/DTable as a table of FSE_CTable/FSE_DTable using below macros */
#define FSE_CTABLE_SIZE_U32(maxTableLog, maxSymbolValue) (1 + (1<<(maxTableLog-1)) + ((maxSymbolValue+1)*2)) #define FSE_CTABLE_SIZE_U32(maxTableLog, maxSymbolValue) (1 + (1 << (maxTableLog - 1)) + ((maxSymbolValue + 1) * 2))
#define FSE_DTABLE_SIZE_U32(maxTableLog) (1 + (1<<maxTableLog)) #define FSE_DTABLE_SIZE_U32(maxTableLog) (1 + (1 << maxTableLog))
/* ***************************************** /* *****************************************
* FSE advanced API * FSE advanced API
@@ -254,22 +246,19 @@ If there is an error, the function will return an error code, which can be teste
* Same as FSE_count(), but using an externally provided scratch buffer. * Same as FSE_count(), but using an externally provided scratch buffer.
* `workSpace` size must be table of >= `1024` unsigned * `workSpace` size must be table of >= `1024` unsigned
*/ */
size_t FSE_count_wksp(unsigned* count, unsigned* maxSymbolValuePtr, size_t FSE_count_wksp(unsigned *count, unsigned *maxSymbolValuePtr, const void *source, size_t sourceSize, unsigned *workSpace);
const void* source, size_t sourceSize, unsigned* workSpace);
/* FSE_countFast_wksp() : /* FSE_countFast_wksp() :
* Same as FSE_countFast(), but using an externally provided scratch buffer. * Same as FSE_countFast(), but using an externally provided scratch buffer.
* `workSpace` must be a table of minimum `1024` unsigned * `workSpace` must be a table of minimum `1024` unsigned
*/ */
size_t FSE_countFast_wksp(unsigned* count, unsigned* maxSymbolValuePtr, const void* src, size_t srcSize, unsigned* workSpace); size_t FSE_countFast_wksp(unsigned *count, unsigned *maxSymbolValuePtr, const void *src, size_t srcSize, unsigned *workSpace);
/*! FSE_count_simple /*! FSE_count_simple
* Same as FSE_countFast(), but does not use any additional memory (not even on stack). * Same as FSE_countFast(), but does not use any additional memory (not even on stack).
* This function is unsafe, and will segfault if any value within `src` is `> *maxSymbolValuePtr` (presuming it's also the size of `count`). * This function is unsafe, and will segfault if any value within `src` is `> *maxSymbolValuePtr` (presuming it's also the size of `count`).
*/ */
size_t FSE_count_simple(unsigned* count, unsigned* maxSymbolValuePtr, const void* src, size_t srcSize); size_t FSE_count_simple(unsigned *count, unsigned *maxSymbolValuePtr, const void *src, size_t srcSize);
unsigned FSE_optimalTableLog_internal(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue, unsigned minus); unsigned FSE_optimalTableLog_internal(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue, unsigned minus);
/**< same as FSE_optimalTableLog(), which used `minus==2` */ /**< same as FSE_optimalTableLog(), which used `minus==2` */
@@ -278,31 +267,32 @@ unsigned FSE_optimalTableLog_internal(unsigned maxTableLog, size_t srcSize, unsi
* Same as FSE_compress2(), but using an externally allocated scratch buffer (`workSpace`). * Same as FSE_compress2(), but using an externally allocated scratch buffer (`workSpace`).
* FSE_WKSP_SIZE_U32() provides the minimum size required for `workSpace` as a table of FSE_CTable. * FSE_WKSP_SIZE_U32() provides the minimum size required for `workSpace` as a table of FSE_CTable.
*/ */
#define FSE_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) ( FSE_CTABLE_SIZE_U32(maxTableLog, maxSymbolValue) + ((maxTableLog > 12) ? (1 << (maxTableLog - 2)) : 1024) ) #define FSE_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) \
size_t FSE_compress_wksp (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize); (FSE_CTABLE_SIZE_U32(maxTableLog, maxSymbolValue) + ((maxTableLog > 12) ? (1 << (maxTableLog - 2)) : 1024))
size_t FSE_compress_wksp(void *dst, size_t dstSize, const void *src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void *workSpace,
size_t wkspSize);
size_t FSE_buildCTable_raw (FSE_CTable* ct, unsigned nbBits); size_t FSE_buildCTable_raw(FSE_CTable *ct, unsigned nbBits);
/**< build a fake FSE_CTable, designed for a flat distribution, where each symbol uses nbBits */ /**< build a fake FSE_CTable, designed for a flat distribution, where each symbol uses nbBits */
size_t FSE_buildCTable_rle (FSE_CTable* ct, unsigned char symbolValue); size_t FSE_buildCTable_rle(FSE_CTable *ct, unsigned char symbolValue);
/**< build a fake FSE_CTable, designed to compress always the same symbolValue */ /**< build a fake FSE_CTable, designed to compress always the same symbolValue */
/* FSE_buildCTable_wksp() : /* FSE_buildCTable_wksp() :
* Same as FSE_buildCTable(), but using an externally allocated scratch buffer (`workSpace`). * Same as FSE_buildCTable(), but using an externally allocated scratch buffer (`workSpace`).
* `wkspSize` must be >= `(1<<tableLog)`. * `wkspSize` must be >= `(1<<tableLog)`.
*/ */
size_t FSE_buildCTable_wksp(FSE_CTable* ct, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize); size_t FSE_buildCTable_wksp(FSE_CTable *ct, const short *normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, void *workSpace, size_t wkspSize);
size_t FSE_buildDTable_raw (FSE_DTable* dt, unsigned nbBits); size_t FSE_buildDTable_raw(FSE_DTable *dt, unsigned nbBits);
/**< build a fake FSE_DTable, designed to read a flat distribution where each symbol uses nbBits */ /**< build a fake FSE_DTable, designed to read a flat distribution where each symbol uses nbBits */
size_t FSE_buildDTable_rle (FSE_DTable* dt, unsigned char symbolValue); size_t FSE_buildDTable_rle(FSE_DTable *dt, unsigned char symbolValue);
/**< build a fake FSE_DTable, designed to always generate the same symbolValue */ /**< build a fake FSE_DTable, designed to always generate the same symbolValue */
size_t FSE_decompress_wksp(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, FSE_DTable* workSpace, unsigned maxLog); size_t FSE_decompress_wksp(void *dst, size_t dstCapacity, const void *cSrc, size_t cSrcSize, FSE_DTable *workSpace, unsigned maxLog);
/**< same as FSE_decompress(), using an externally allocated `workSpace` produced with `FSE_DTABLE_SIZE_U32(maxLog)` */ /**< same as FSE_decompress(), using an externally allocated `workSpace` produced with `FSE_DTABLE_SIZE_U32(maxLog)` */
/* ***************************************** /* *****************************************
* FSE symbol compression API * FSE symbol compression API
*******************************************/ *******************************************/
@@ -312,16 +302,16 @@ size_t FSE_decompress_wksp(void* dst, size_t dstCapacity, const void* cSrc, size
*/ */
typedef struct { typedef struct {
ptrdiff_t value; ptrdiff_t value;
const void* stateTable; const void *stateTable;
const void* symbolTT; const void *symbolTT;
unsigned stateLog; unsigned stateLog;
} FSE_CState_t; } FSE_CState_t;
static void FSE_initCState(FSE_CState_t* CStatePtr, const FSE_CTable* ct); static void FSE_initCState(FSE_CState_t *CStatePtr, const FSE_CTable *ct);
static void FSE_encodeSymbol(BIT_CStream_t* bitC, FSE_CState_t* CStatePtr, unsigned symbol); static void FSE_encodeSymbol(BIT_CStream_t *bitC, FSE_CState_t *CStatePtr, unsigned symbol);
static void FSE_flushCState(BIT_CStream_t* bitC, const FSE_CState_t* CStatePtr); static void FSE_flushCState(BIT_CStream_t *bitC, const FSE_CState_t *CStatePtr);
/**< /**<
These functions are inner components of FSE_compress_usingCTable(). These functions are inner components of FSE_compress_usingCTable().
@@ -366,21 +356,19 @@ If there is an error, it returns an errorCode (which can be tested using FSE_isE
size_t size = BIT_closeCStream(&bitStream); size_t size = BIT_closeCStream(&bitStream);
*/ */
/* ***************************************** /* *****************************************
* FSE symbol decompression API * FSE symbol decompression API
*******************************************/ *******************************************/
typedef struct { typedef struct {
size_t state; size_t state;
const void* table; /* precise table may vary, depending on U16 */ const void *table; /* precise table may vary, depending on U16 */
} FSE_DState_t; } FSE_DState_t;
static void FSE_initDState(FSE_DState_t *DStatePtr, BIT_DStream_t *bitD, const FSE_DTable *dt);
static void FSE_initDState(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD, const FSE_DTable* dt); static unsigned char FSE_decodeSymbol(FSE_DState_t *DStatePtr, BIT_DStream_t *bitD);
static unsigned char FSE_decodeSymbol(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD); static unsigned FSE_endOfDState(const FSE_DState_t *DStatePtr);
static unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr);
/**< /**<
Let's now decompose FSE_decompress_usingDTable() into its unitary components. Let's now decompose FSE_decompress_usingDTable() into its unitary components.
@@ -431,14 +419,12 @@ Check also the states. There might be some symbols left there, if some high prob
FSE_endOfDState(&DState); FSE_endOfDState(&DState);
*/ */
/* ***************************************** /* *****************************************
* FSE unsafe API * FSE unsafe API
*******************************************/ *******************************************/
static unsigned char FSE_decodeSymbolFast(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD); static unsigned char FSE_decodeSymbolFast(FSE_DState_t *DStatePtr, BIT_DStream_t *bitD);
/* faster, but works only if nbBits is always >= 1 (otherwise, result will be corrupted) */ /* faster, but works only if nbBits is always >= 1 (otherwise, result will be corrupted) */
/* ***************************************** /* *****************************************
* Implementation of inlined functions * Implementation of inlined functions
*******************************************/ *******************************************/
@@ -447,48 +433,47 @@ typedef struct {
U32 deltaNbBits; U32 deltaNbBits;
} FSE_symbolCompressionTransform; /* total 8 bytes */ } FSE_symbolCompressionTransform; /* total 8 bytes */
ZSTD_STATIC void FSE_initCState(FSE_CState_t* statePtr, const FSE_CTable* ct) ZSTD_STATIC void FSE_initCState(FSE_CState_t *statePtr, const FSE_CTable *ct)
{ {
const void* ptr = ct; const void *ptr = ct;
const U16* u16ptr = (const U16*) ptr; const U16 *u16ptr = (const U16 *)ptr;
const U32 tableLog = ZSTD_read16(ptr); const U32 tableLog = ZSTD_read16(ptr);
statePtr->value = (ptrdiff_t)1<<tableLog; statePtr->value = (ptrdiff_t)1 << tableLog;
statePtr->stateTable = u16ptr+2; statePtr->stateTable = u16ptr + 2;
statePtr->symbolTT = ((const U32*)ct + 1 + (tableLog ? (1<<(tableLog-1)) : 1)); statePtr->symbolTT = ((const U32 *)ct + 1 + (tableLog ? (1 << (tableLog - 1)) : 1));
statePtr->stateLog = tableLog; statePtr->stateLog = tableLog;
} }
/*! FSE_initCState2() : /*! FSE_initCState2() :
* Same as FSE_initCState(), but the first symbol to include (which will be the last to be read) * Same as FSE_initCState(), but the first symbol to include (which will be the last to be read)
* uses the smallest state value possible, saving the cost of this symbol */ * uses the smallest state value possible, saving the cost of this symbol */
ZSTD_STATIC void FSE_initCState2(FSE_CState_t* statePtr, const FSE_CTable* ct, U32 symbol) ZSTD_STATIC void FSE_initCState2(FSE_CState_t *statePtr, const FSE_CTable *ct, U32 symbol)
{ {
FSE_initCState(statePtr, ct); FSE_initCState(statePtr, ct);
{ const FSE_symbolCompressionTransform symbolTT = ((const FSE_symbolCompressionTransform*)(statePtr->symbolTT))[symbol]; {
const U16* stateTable = (const U16*)(statePtr->stateTable); const FSE_symbolCompressionTransform symbolTT = ((const FSE_symbolCompressionTransform *)(statePtr->symbolTT))[symbol];
U32 nbBitsOut = (U32)((symbolTT.deltaNbBits + (1<<15)) >> 16); const U16 *stateTable = (const U16 *)(statePtr->stateTable);
U32 nbBitsOut = (U32)((symbolTT.deltaNbBits + (1 << 15)) >> 16);
statePtr->value = (nbBitsOut << 16) - symbolTT.deltaNbBits; statePtr->value = (nbBitsOut << 16) - symbolTT.deltaNbBits;
statePtr->value = stateTable[(statePtr->value >> nbBitsOut) + symbolTT.deltaFindState]; statePtr->value = stateTable[(statePtr->value >> nbBitsOut) + symbolTT.deltaFindState];
} }
} }
ZSTD_STATIC void FSE_encodeSymbol(BIT_CStream_t* bitC, FSE_CState_t* statePtr, U32 symbol) ZSTD_STATIC void FSE_encodeSymbol(BIT_CStream_t *bitC, FSE_CState_t *statePtr, U32 symbol)
{ {
const FSE_symbolCompressionTransform symbolTT = ((const FSE_symbolCompressionTransform*)(statePtr->symbolTT))[symbol]; const FSE_symbolCompressionTransform symbolTT = ((const FSE_symbolCompressionTransform *)(statePtr->symbolTT))[symbol];
const U16* const stateTable = (const U16*)(statePtr->stateTable); const U16 *const stateTable = (const U16 *)(statePtr->stateTable);
U32 nbBitsOut = (U32)((statePtr->value + symbolTT.deltaNbBits) >> 16); U32 nbBitsOut = (U32)((statePtr->value + symbolTT.deltaNbBits) >> 16);
BIT_addBits(bitC, statePtr->value, nbBitsOut); BIT_addBits(bitC, statePtr->value, nbBitsOut);
statePtr->value = stateTable[ (statePtr->value >> nbBitsOut) + symbolTT.deltaFindState]; statePtr->value = stateTable[(statePtr->value >> nbBitsOut) + symbolTT.deltaFindState];
} }
ZSTD_STATIC void FSE_flushCState(BIT_CStream_t* bitC, const FSE_CState_t* statePtr) ZSTD_STATIC void FSE_flushCState(BIT_CStream_t *bitC, const FSE_CState_t *statePtr)
{ {
BIT_addBits(bitC, statePtr->value, statePtr->stateLog); BIT_addBits(bitC, statePtr->value, statePtr->stateLog);
BIT_flushBits(bitC); BIT_flushBits(bitC);
} }
/* ====== Decompression ====== */ /* ====== Decompression ====== */
typedef struct { typedef struct {
@@ -496,39 +481,38 @@ typedef struct {
U16 fastMode; U16 fastMode;
} FSE_DTableHeader; /* sizeof U32 */ } FSE_DTableHeader; /* sizeof U32 */
typedef struct typedef struct {
{
unsigned short newState; unsigned short newState;
unsigned char symbol; unsigned char symbol;
unsigned char nbBits; unsigned char nbBits;
} FSE_decode_t; /* size == U32 */ } FSE_decode_t; /* size == U32 */
ZSTD_STATIC void FSE_initDState(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD, const FSE_DTable* dt) ZSTD_STATIC void FSE_initDState(FSE_DState_t *DStatePtr, BIT_DStream_t *bitD, const FSE_DTable *dt)
{ {
const void* ptr = dt; const void *ptr = dt;
const FSE_DTableHeader* const DTableH = (const FSE_DTableHeader*)ptr; const FSE_DTableHeader *const DTableH = (const FSE_DTableHeader *)ptr;
DStatePtr->state = BIT_readBits(bitD, DTableH->tableLog); DStatePtr->state = BIT_readBits(bitD, DTableH->tableLog);
BIT_reloadDStream(bitD); BIT_reloadDStream(bitD);
DStatePtr->table = dt + 1; DStatePtr->table = dt + 1;
} }
ZSTD_STATIC BYTE FSE_peekSymbol(const FSE_DState_t* DStatePtr) ZSTD_STATIC BYTE FSE_peekSymbol(const FSE_DState_t *DStatePtr)
{ {
FSE_decode_t const DInfo = ((const FSE_decode_t*)(DStatePtr->table))[DStatePtr->state]; FSE_decode_t const DInfo = ((const FSE_decode_t *)(DStatePtr->table))[DStatePtr->state];
return DInfo.symbol; return DInfo.symbol;
} }
ZSTD_STATIC void FSE_updateState(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD) ZSTD_STATIC void FSE_updateState(FSE_DState_t *DStatePtr, BIT_DStream_t *bitD)
{ {
FSE_decode_t const DInfo = ((const FSE_decode_t*)(DStatePtr->table))[DStatePtr->state]; FSE_decode_t const DInfo = ((const FSE_decode_t *)(DStatePtr->table))[DStatePtr->state];
U32 const nbBits = DInfo.nbBits; U32 const nbBits = DInfo.nbBits;
size_t const lowBits = BIT_readBits(bitD, nbBits); size_t const lowBits = BIT_readBits(bitD, nbBits);
DStatePtr->state = DInfo.newState + lowBits; DStatePtr->state = DInfo.newState + lowBits;
} }
ZSTD_STATIC BYTE FSE_decodeSymbol(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD) ZSTD_STATIC BYTE FSE_decodeSymbol(FSE_DState_t *DStatePtr, BIT_DStream_t *bitD)
{ {
FSE_decode_t const DInfo = ((const FSE_decode_t*)(DStatePtr->table))[DStatePtr->state]; FSE_decode_t const DInfo = ((const FSE_decode_t *)(DStatePtr->table))[DStatePtr->state];
U32 const nbBits = DInfo.nbBits; U32 const nbBits = DInfo.nbBits;
BYTE const symbol = DInfo.symbol; BYTE const symbol = DInfo.symbol;
size_t const lowBits = BIT_readBits(bitD, nbBits); size_t const lowBits = BIT_readBits(bitD, nbBits);
@@ -539,9 +523,9 @@ ZSTD_STATIC BYTE FSE_decodeSymbol(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD)
/*! FSE_decodeSymbolFast() : /*! FSE_decodeSymbolFast() :
unsafe, only works if no symbol has a probability > 50% */ unsafe, only works if no symbol has a probability > 50% */
ZSTD_STATIC BYTE FSE_decodeSymbolFast(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD) ZSTD_STATIC BYTE FSE_decodeSymbolFast(FSE_DState_t *DStatePtr, BIT_DStream_t *bitD)
{ {
FSE_decode_t const DInfo = ((const FSE_decode_t*)(DStatePtr->table))[DStatePtr->state]; FSE_decode_t const DInfo = ((const FSE_decode_t *)(DStatePtr->table))[DStatePtr->state];
U32 const nbBits = DInfo.nbBits; U32 const nbBits = DInfo.nbBits;
BYTE const symbol = DInfo.symbol; BYTE const symbol = DInfo.symbol;
size_t const lowBits = BIT_readBitsFast(bitD, nbBits); size_t const lowBits = BIT_readBitsFast(bitD, nbBits);
@@ -550,12 +534,7 @@ ZSTD_STATIC BYTE FSE_decodeSymbolFast(FSE_DState_t* DStatePtr, BIT_DStream_t* bi
return symbol; return symbol;
} }
ZSTD_STATIC unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr) ZSTD_STATIC unsigned FSE_endOfDState(const FSE_DState_t *DStatePtr) { return DStatePtr->state == 0; }
{
return DStatePtr->state == 0;
}
#ifndef FSE_COMMONDEFS_ONLY #ifndef FSE_COMMONDEFS_ONLY
@@ -568,17 +547,17 @@ ZSTD_STATIC unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr)
* Reduced memory usage can improve speed, due to cache effect * Reduced memory usage can improve speed, due to cache effect
* Recommended max value is 14, for 16KB, which nicely fits into Intel x86 L1 cache */ * Recommended max value is 14, for 16KB, which nicely fits into Intel x86 L1 cache */
#ifndef FSE_MAX_MEMORY_USAGE #ifndef FSE_MAX_MEMORY_USAGE
# define FSE_MAX_MEMORY_USAGE 14 #define FSE_MAX_MEMORY_USAGE 14
#endif #endif
#ifndef FSE_DEFAULT_MEMORY_USAGE #ifndef FSE_DEFAULT_MEMORY_USAGE
# define FSE_DEFAULT_MEMORY_USAGE 13 #define FSE_DEFAULT_MEMORY_USAGE 13
#endif #endif
/*!FSE_MAX_SYMBOL_VALUE : /*!FSE_MAX_SYMBOL_VALUE :
* Maximum symbol value authorized. * Maximum symbol value authorized.
* Required for proper stack allocation */ * Required for proper stack allocation */
#ifndef FSE_MAX_SYMBOL_VALUE #ifndef FSE_MAX_SYMBOL_VALUE
# define FSE_MAX_SYMBOL_VALUE 255 #define FSE_MAX_SYMBOL_VALUE 255
#endif #endif
/* ************************************************************** /* **************************************************************
@@ -588,25 +567,22 @@ ZSTD_STATIC unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr)
#define FSE_FUNCTION_EXTENSION #define FSE_FUNCTION_EXTENSION
#define FSE_DECODE_TYPE FSE_decode_t #define FSE_DECODE_TYPE FSE_decode_t
#endif /* !FSE_COMMONDEFS_ONLY */ #endif /* !FSE_COMMONDEFS_ONLY */
/* *************************************************************** /* ***************************************************************
* Constants * Constants
*****************************************************************/ *****************************************************************/
#define FSE_MAX_TABLELOG (FSE_MAX_MEMORY_USAGE-2) #define FSE_MAX_TABLELOG (FSE_MAX_MEMORY_USAGE - 2)
#define FSE_MAX_TABLESIZE (1U<<FSE_MAX_TABLELOG) #define FSE_MAX_TABLESIZE (1U << FSE_MAX_TABLELOG)
#define FSE_MAXTABLESIZE_MASK (FSE_MAX_TABLESIZE-1) #define FSE_MAXTABLESIZE_MASK (FSE_MAX_TABLESIZE - 1)
#define FSE_DEFAULT_TABLELOG (FSE_DEFAULT_MEMORY_USAGE-2) #define FSE_DEFAULT_TABLELOG (FSE_DEFAULT_MEMORY_USAGE - 2)
#define FSE_MIN_TABLELOG 5 #define FSE_MIN_TABLELOG 5
#define FSE_TABLELOG_ABSOLUTE_MAX 15 #define FSE_TABLELOG_ABSOLUTE_MAX 15
#if FSE_MAX_TABLELOG > FSE_TABLELOG_ABSOLUTE_MAX #if FSE_MAX_TABLELOG > FSE_TABLELOG_ABSOLUTE_MAX
# error "FSE_MAX_TABLELOG > FSE_TABLELOG_ABSOLUTE_MAX is not supported" #error "FSE_MAX_TABLELOG > FSE_TABLELOG_ABSOLUTE_MAX is not supported"
#endif #endif
#define FSE_TABLESTEP(tableSize) ((tableSize>>1) + (tableSize>>3) + 3) #define FSE_TABLESTEP(tableSize) ((tableSize >> 1) + (tableSize >> 3) + 3)
#endif /* FSE_H */ #endif /* FSE_H */
contrib/linux-kernel/lib/zstd/fse_compress.c
+357 -275
View File
@@ -42,21 +42,21 @@
****************************************************************/ ****************************************************************/
#define FORCE_INLINE static __always_inline #define FORCE_INLINE static __always_inline
/* ************************************************************** /* **************************************************************
* Includes * Includes
****************************************************************/ ****************************************************************/
#include <linux/compiler.h>
#include <linux/string.h> /* memcpy, memset */
#include "bitstream.h" #include "bitstream.h"
#include "fse.h" #include "fse.h"
#include <linux/compiler.h>
#include <linux/string.h> /* memcpy, memset */
/* ************************************************************** /* **************************************************************
* Error Management * Error Management
****************************************************************/ ****************************************************************/
#define FSE_STATIC_ASSERT(c) { enum { FSE_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */ #define FSE_STATIC_ASSERT(c) \
{ \
enum { FSE_static_assert = 1 / (int)(!!(c)) }; \
} /* use only *after* variable declarations */
/* ************************************************************** /* **************************************************************
* Templates * Templates
@@ -69,17 +69,16 @@
/* safety checks */ /* safety checks */
#ifndef FSE_FUNCTION_EXTENSION #ifndef FSE_FUNCTION_EXTENSION
# error "FSE_FUNCTION_EXTENSION must be defined" #error "FSE_FUNCTION_EXTENSION must be defined"
#endif #endif
#ifndef FSE_FUNCTION_TYPE #ifndef FSE_FUNCTION_TYPE
# error "FSE_FUNCTION_TYPE must be defined" #error "FSE_FUNCTION_TYPE must be defined"
#endif #endif
/* Function names */ /* Function names */
#define FSE_CAT(X,Y) X##Y #define FSE_CAT(X, Y) X##Y
#define FSE_FUNCTION_NAME(X,Y) FSE_CAT(X,Y) #define FSE_FUNCTION_NAME(X, Y) FSE_CAT(X, Y)
#define FSE_TYPE_NAME(X,Y) FSE_CAT(X,Y) #define FSE_TYPE_NAME(X, Y) FSE_CAT(X, Y)
/* Function templates */ /* Function templates */
@@ -88,88 +87,99 @@
* wkspSize should be sized to handle worst case situation, which is `1<<max_tableLog * sizeof(FSE_FUNCTION_TYPE)` * wkspSize should be sized to handle worst case situation, which is `1<<max_tableLog * sizeof(FSE_FUNCTION_TYPE)`
* workSpace must also be properly aligned with FSE_FUNCTION_TYPE requirements * workSpace must also be properly aligned with FSE_FUNCTION_TYPE requirements
*/ */
size_t FSE_buildCTable_wksp(FSE_CTable* ct, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize) size_t FSE_buildCTable_wksp(FSE_CTable *ct, const short *normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, void *workSpace, size_t wkspSize)
{ {
U32 const tableSize = 1 << tableLog; U32 const tableSize = 1 << tableLog;
U32 const tableMask = tableSize - 1; U32 const tableMask = tableSize - 1;
void* const ptr = ct; void *const ptr = ct;
U16* const tableU16 = ( (U16*) ptr) + 2; U16 *const tableU16 = ((U16 *)ptr) + 2;
void* const FSCT = ((U32*)ptr) + 1 /* header */ + (tableLog ? tableSize>>1 : 1) ; void *const FSCT = ((U32 *)ptr) + 1 /* header */ + (tableLog ? tableSize >> 1 : 1);
FSE_symbolCompressionTransform* const symbolTT = (FSE_symbolCompressionTransform*) (FSCT); FSE_symbolCompressionTransform *const symbolTT = (FSE_symbolCompressionTransform *)(FSCT);
U32 const step = FSE_TABLESTEP(tableSize); U32 const step = FSE_TABLESTEP(tableSize);
U32 cumul[FSE_MAX_SYMBOL_VALUE+2]; U32 cumul[FSE_MAX_SYMBOL_VALUE + 2];
FSE_FUNCTION_TYPE* const tableSymbol = (FSE_FUNCTION_TYPE*)workSpace; FSE_FUNCTION_TYPE *const tableSymbol = (FSE_FUNCTION_TYPE *)workSpace;
U32 highThreshold = tableSize-1; U32 highThreshold = tableSize - 1;
/* CTable header */ /* CTable header */
if (((size_t)1 << tableLog) * sizeof(FSE_FUNCTION_TYPE) > wkspSize) return ERROR(tableLog_tooLarge); if (((size_t)1 << tableLog) * sizeof(FSE_FUNCTION_TYPE) > wkspSize)
tableU16[-2] = (U16) tableLog; return ERROR(tableLog_tooLarge);
tableU16[-1] = (U16) maxSymbolValue; tableU16[-2] = (U16)tableLog;
tableU16[-1] = (U16)maxSymbolValue;
/* For explanations on how to distribute symbol values over the table : /* For explanations on how to distribute symbol values over the table :
* http://fastcompression.blogspot.fr/2014/02/fse-distributing-symbol-values.html */ * http://fastcompression.blogspot.fr/2014/02/fse-distributing-symbol-values.html */
/* symbol start positions */ /* symbol start positions */
{ U32 u; {
U32 u;
cumul[0] = 0; cumul[0] = 0;
for (u=1; u<=maxSymbolValue+1; u++) { for (u = 1; u <= maxSymbolValue + 1; u++) {
if (normalizedCounter[u-1]==-1) { /* Low proba symbol */ if (normalizedCounter[u - 1] == -1) { /* Low proba symbol */
cumul[u] = cumul[u-1] + 1; cumul[u] = cumul[u - 1] + 1;
tableSymbol[highThreshold--] = (FSE_FUNCTION_TYPE)(u-1); tableSymbol[highThreshold--] = (FSE_FUNCTION_TYPE)(u - 1);
} else { } else {
cumul[u] = cumul[u-1] + normalizedCounter[u-1]; cumul[u] = cumul[u - 1] + normalizedCounter[u - 1];
} } }
cumul[maxSymbolValue+1] = tableSize+1; }
cumul[maxSymbolValue + 1] = tableSize + 1;
} }
/* Spread symbols */ /* Spread symbols */
{ U32 position = 0; {
U32 position = 0;
U32 symbol; U32 symbol;
for (symbol=0; symbol<=maxSymbolValue; symbol++) { for (symbol = 0; symbol <= maxSymbolValue; symbol++) {
int nbOccurences; int nbOccurences;
for (nbOccurences=0; nbOccurences<normalizedCounter[symbol]; nbOccurences++) { for (nbOccurences = 0; nbOccurences < normalizedCounter[symbol]; nbOccurences++) {
tableSymbol[position] = (FSE_FUNCTION_TYPE)symbol; tableSymbol[position] = (FSE_FUNCTION_TYPE)symbol;
position = (position + step) & tableMask; position = (position + step) & tableMask;
while (position > highThreshold) position = (position + step) & tableMask; /* Low proba area */ while (position > highThreshold)
} } position = (position + step) & tableMask; /* Low proba area */
}
}
if (position!=0) return ERROR(GENERIC); /* Must have gone through all positions */ if (position != 0)
return ERROR(GENERIC); /* Must have gone through all positions */
} }
/* Build table */ /* Build table */
{ U32 u; for (u=0; u<tableSize; u++) { {
U32 u;
for (u = 0; u < tableSize; u++) {
FSE_FUNCTION_TYPE s = tableSymbol[u]; /* note : static analyzer may not understand tableSymbol is properly initialized */ FSE_FUNCTION_TYPE s = tableSymbol[u]; /* note : static analyzer may not understand tableSymbol is properly initialized */
tableU16[cumul[s]++] = (U16) (tableSize+u); /* TableU16 : sorted by symbol order; gives next state value */ tableU16[cumul[s]++] = (U16)(tableSize + u); /* TableU16 : sorted by symbol order; gives next state value */
} } }
}
/* Build Symbol Transformation Table */ /* Build Symbol Transformation Table */
{ unsigned total = 0;
unsigned s;
for (s=0; s<=maxSymbolValue; s++) {
switch (normalizedCounter[s])
{ {
unsigned total = 0;
unsigned s;
for (s = 0; s <= maxSymbolValue; s++) {
switch (normalizedCounter[s]) {
case 0: break; case 0: break;
case -1: case -1:
case 1: case 1:
symbolTT[s].deltaNbBits = (tableLog << 16) - (1<<tableLog); symbolTT[s].deltaNbBits = (tableLog << 16) - (1 << tableLog);
symbolTT[s].deltaFindState = total - 1; symbolTT[s].deltaFindState = total - 1;
total ++; total++;
break; break;
default : default: {
{ U32 const maxBitsOut = tableLog - BIT_highbit32(normalizedCounter[s] - 1);
U32 const maxBitsOut = tableLog - BIT_highbit32 (normalizedCounter[s]-1);
U32 const minStatePlus = normalizedCounter[s] << maxBitsOut; U32 const minStatePlus = normalizedCounter[s] << maxBitsOut;
symbolTT[s].deltaNbBits = (maxBitsOut << 16) - minStatePlus; symbolTT[s].deltaNbBits = (maxBitsOut << 16) - minStatePlus;
symbolTT[s].deltaFindState = total - normalizedCounter[s]; symbolTT[s].deltaFindState = total - normalizedCounter[s];
total += normalizedCounter[s]; total += normalizedCounter[s];
} } } } }
}
}
}
return 0; return 0;
} }
#ifndef FSE_COMMONDEFS_ONLY #ifndef FSE_COMMONDEFS_ONLY
/*-************************************************************** /*-**************************************************************
@@ -177,17 +187,16 @@ size_t FSE_buildCTable_wksp(FSE_CTable* ct, const short* normalizedCounter, unsi
****************************************************************/ ****************************************************************/
size_t FSE_NCountWriteBound(unsigned maxSymbolValue, unsigned tableLog) size_t FSE_NCountWriteBound(unsigned maxSymbolValue, unsigned tableLog)
{ {
size_t const maxHeaderSize = (((maxSymbolValue+1) * tableLog) >> 3) + 3; size_t const maxHeaderSize = (((maxSymbolValue + 1) * tableLog) >> 3) + 3;
return maxSymbolValue ? maxHeaderSize : FSE_NCOUNTBOUND; /* maxSymbolValue==0 ? use default */ return maxSymbolValue ? maxHeaderSize : FSE_NCOUNTBOUND; /* maxSymbolValue==0 ? use default */
} }
static size_t FSE_writeNCount_generic (void* header, size_t headerBufferSize, static size_t FSE_writeNCount_generic(void *header, size_t headerBufferSize, const short *normalizedCounter, unsigned maxSymbolValue, unsigned tableLog,
const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog,
unsigned writeIsSafe) unsigned writeIsSafe)
{ {
BYTE* const ostart = (BYTE*) header; BYTE *const ostart = (BYTE *)header;
BYTE* out = ostart; BYTE *out = ostart;
BYTE* const oend = ostart + headerBufferSize; BYTE *const oend = ostart + headerBufferSize;
int nbBits; int nbBits;
const int tableSize = 1 << tableLog; const int tableSize = 1 << tableLog;
int remaining; int remaining;
@@ -200,79 +209,92 @@ static size_t FSE_writeNCount_generic (void* header, size_t headerBufferSize,
bitStream = 0; bitStream = 0;
bitCount = 0; bitCount = 0;
/* Table Size */ /* Table Size */
bitStream += (tableLog-FSE_MIN_TABLELOG) << bitCount; bitStream += (tableLog - FSE_MIN_TABLELOG) << bitCount;
bitCount += 4; bitCount += 4;
/* Init */ /* Init */
remaining = tableSize+1; /* +1 for extra accuracy */ remaining = tableSize + 1; /* +1 for extra accuracy */
threshold = tableSize; threshold = tableSize;
nbBits = tableLog+1; nbBits = tableLog + 1;
while (remaining>1) { /* stops at 1 */ while (remaining > 1) { /* stops at 1 */
if (previous0) { if (previous0) {
unsigned start = charnum; unsigned start = charnum;
while (!normalizedCounter[charnum]) charnum++; while (!normalizedCounter[charnum])
while (charnum >= start+24) { charnum++;
start+=24; while (charnum >= start + 24) {
start += 24;
bitStream += 0xFFFFU << bitCount; bitStream += 0xFFFFU << bitCount;
if ((!writeIsSafe) && (out > oend-2)) return ERROR(dstSize_tooSmall); /* Buffer overflow */ if ((!writeIsSafe) && (out > oend - 2))
out[0] = (BYTE) bitStream; return ERROR(dstSize_tooSmall); /* Buffer overflow */
out[1] = (BYTE)(bitStream>>8); out[0] = (BYTE)bitStream;
out+=2; out[1] = (BYTE)(bitStream >> 8);
bitStream>>=16; out += 2;
bitStream >>= 16;
} }
while (charnum >= start+3) { while (charnum >= start + 3) {
start+=3; start += 3;
bitStream += 3 << bitCount; bitStream += 3 << bitCount;
bitCount += 2; bitCount += 2;
} }
bitStream += (charnum-start) << bitCount; bitStream += (charnum - start) << bitCount;
bitCount += 2; bitCount += 2;
if (bitCount>16) { if (bitCount > 16) {
if ((!writeIsSafe) && (out > oend - 2)) return ERROR(dstSize_tooSmall); /* Buffer overflow */ if ((!writeIsSafe) && (out > oend - 2))
return ERROR(dstSize_tooSmall); /* Buffer overflow */
out[0] = (BYTE)bitStream; out[0] = (BYTE)bitStream;
out[1] = (BYTE)(bitStream>>8); out[1] = (BYTE)(bitStream >> 8);
out += 2; out += 2;
bitStream >>= 16; bitStream >>= 16;
bitCount -= 16; bitCount -= 16;
} } }
{ int count = normalizedCounter[charnum++]; }
int const max = (2*threshold-1)-remaining; {
int count = normalizedCounter[charnum++];
int const max = (2 * threshold - 1) - remaining;
remaining -= count < 0 ? -count : count; remaining -= count < 0 ? -count : count;
count++; /* +1 for extra accuracy */ count++; /* +1 for extra accuracy */
if (count>=threshold) count += max; /* [0..max[ [max..threshold[ (...) [threshold+max 2*threshold[ */ if (count >= threshold)
count += max; /* [0..max[ [max..threshold[ (...) [threshold+max 2*threshold[ */
bitStream += count << bitCount; bitStream += count << bitCount;
bitCount += nbBits; bitCount += nbBits;
bitCount -= (count<max); bitCount -= (count < max);
previous0 = (count==1); previous0 = (count == 1);
if (remaining<1) return ERROR(GENERIC); if (remaining < 1)
while (remaining<threshold) nbBits--, threshold>>=1; return ERROR(GENERIC);
while (remaining < threshold)
nbBits--, threshold >>= 1;
} }
if (bitCount>16) { if (bitCount > 16) {
if ((!writeIsSafe) && (out > oend - 2)) return ERROR(dstSize_tooSmall); /* Buffer overflow */ if ((!writeIsSafe) && (out > oend - 2))
return ERROR(dstSize_tooSmall); /* Buffer overflow */
out[0] = (BYTE)bitStream; out[0] = (BYTE)bitStream;
out[1] = (BYTE)(bitStream>>8); out[1] = (BYTE)(bitStream >> 8);
out += 2; out += 2;
bitStream >>= 16; bitStream >>= 16;
bitCount -= 16; bitCount -= 16;
} } }
}
/* flush remaining bitStream */ /* flush remaining bitStream */
if ((!writeIsSafe) && (out > oend - 2)) return ERROR(dstSize_tooSmall); /* Buffer overflow */ if ((!writeIsSafe) && (out > oend - 2))
return ERROR(dstSize_tooSmall); /* Buffer overflow */
out[0] = (BYTE)bitStream; out[0] = (BYTE)bitStream;
out[1] = (BYTE)(bitStream>>8); out[1] = (BYTE)(bitStream >> 8);
out+= (bitCount+7) /8; out += (bitCount + 7) / 8;
if (charnum > maxSymbolValue + 1) return ERROR(GENERIC); if (charnum > maxSymbolValue + 1)
return ERROR(GENERIC);
return (out-ostart); return (out - ostart);
} }
size_t FSE_writeNCount(void *buffer, size_t bufferSize, const short *normalizedCounter, unsigned maxSymbolValue, unsigned tableLog)
size_t FSE_writeNCount (void* buffer, size_t bufferSize, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog)
{ {
if (tableLog > FSE_MAX_TABLELOG) return ERROR(tableLog_tooLarge); /* Unsupported */ if (tableLog > FSE_MAX_TABLELOG)
if (tableLog < FSE_MIN_TABLELOG) return ERROR(GENERIC); /* Unsupported */ return ERROR(tableLog_tooLarge); /* Unsupported */
if (tableLog < FSE_MIN_TABLELOG)
return ERROR(GENERIC); /* Unsupported */
if (bufferSize < FSE_NCountWriteBound(maxSymbolValue, tableLog)) if (bufferSize < FSE_NCountWriteBound(maxSymbolValue, tableLog))
return FSE_writeNCount_generic(buffer, bufferSize, normalizedCounter, maxSymbolValue, tableLog, 0); return FSE_writeNCount_generic(buffer, bufferSize, normalizedCounter, maxSymbolValue, tableLog, 0);
@@ -280,8 +302,6 @@ size_t FSE_writeNCount (void* buffer, size_t bufferSize, const short* normalized
return FSE_writeNCount_generic(buffer, bufferSize, normalizedCounter, maxSymbolValue, tableLog, 1); return FSE_writeNCount_generic(buffer, bufferSize, normalizedCounter, maxSymbolValue, tableLog, 1);
} }
/*-************************************************************** /*-**************************************************************
* Counting histogram * Counting histogram
****************************************************************/ ****************************************************************/
@@ -292,46 +312,52 @@ size_t FSE_writeNCount (void* buffer, size_t bufferSize, const short* normalized
For this reason, prefer using a table `count` with 256 elements. For this reason, prefer using a table `count` with 256 elements.
@return : count of most numerous element @return : count of most numerous element
*/ */
size_t FSE_count_simple(unsigned* count, unsigned* maxSymbolValuePtr, size_t FSE_count_simple(unsigned *count, unsigned *maxSymbolValuePtr, const void *src, size_t srcSize)
const void* src, size_t srcSize)
{ {
const BYTE* ip = (const BYTE*)src; const BYTE *ip = (const BYTE *)src;
const BYTE* const end = ip + srcSize; const BYTE *const end = ip + srcSize;
unsigned maxSymbolValue = *maxSymbolValuePtr; unsigned maxSymbolValue = *maxSymbolValuePtr;
unsigned max=0; unsigned max = 0;
memset(count, 0, (maxSymbolValue+1)*sizeof(*count)); memset(count, 0, (maxSymbolValue + 1) * sizeof(*count));
if (srcSize==0) { *maxSymbolValuePtr = 0; return 0; } if (srcSize == 0) {
*maxSymbolValuePtr = 0;
return 0;
}
while (ip<end) count[*ip++]++; while (ip < end)
count[*ip++]++;
while (!count[maxSymbolValue]) maxSymbolValue--; while (!count[maxSymbolValue])
maxSymbolValue--;
*maxSymbolValuePtr = maxSymbolValue; *maxSymbolValuePtr = maxSymbolValue;
{ U32 s; for (s=0; s<=maxSymbolValue; s++) if (count[s] > max) max = count[s]; } {
U32 s;
for (s = 0; s <= maxSymbolValue; s++)
if (count[s] > max)
max = count[s];
}
return (size_t)max; return (size_t)max;
} }
/* FSE_count_parallel_wksp() : /* FSE_count_parallel_wksp() :
* Same as FSE_count_parallel(), but using an externally provided scratch buffer. * Same as FSE_count_parallel(), but using an externally provided scratch buffer.
* `workSpace` size must be a minimum of `1024 * sizeof(unsigned)`` */ * `workSpace` size must be a minimum of `1024 * sizeof(unsigned)`` */
static size_t FSE_count_parallel_wksp( static size_t FSE_count_parallel_wksp(unsigned *count, unsigned *maxSymbolValuePtr, const void *source, size_t sourceSize, unsigned checkMax,
unsigned* count, unsigned* maxSymbolValuePtr, unsigned *const workSpace)
const void* source, size_t sourceSize,
unsigned checkMax, unsigned* const workSpace)
{ {
const BYTE* ip = (const BYTE*)source; const BYTE *ip = (const BYTE *)source;
const BYTE* const iend = ip+sourceSize; const BYTE *const iend = ip + sourceSize;
unsigned maxSymbolValue = *maxSymbolValuePtr; unsigned maxSymbolValue = *maxSymbolValuePtr;
unsigned max=0; unsigned max = 0;
U32* const Counting1 = workSpace; U32 *const Counting1 = workSpace;
U32* const Counting2 = Counting1 + 256; U32 *const Counting2 = Counting1 + 256;
U32* const Counting3 = Counting2 + 256; U32 *const Counting3 = Counting2 + 256;
U32* const Counting4 = Counting3 + 256; U32 *const Counting4 = Counting3 + 256;
memset(Counting1, 0, 4*256*sizeof(unsigned)); memset(Counting1, 0, 4 * 256 * sizeof(unsigned));
/* safety checks */ /* safety checks */
if (!sourceSize) { if (!sourceSize) {
@@ -339,50 +365,70 @@ static size_t FSE_count_parallel_wksp(
*maxSymbolValuePtr = 0; *maxSymbolValuePtr = 0;
return 0; return 0;
} }
if (!maxSymbolValue) maxSymbolValue = 255; /* 0 == default */ if (!maxSymbolValue)
maxSymbolValue = 255; /* 0 == default */
/* by stripes of 16 bytes */ /* by stripes of 16 bytes */
{ U32 cached = ZSTD_read32(ip); ip += 4; {
while (ip < iend-15) { U32 cached = ZSTD_read32(ip);
U32 c = cached; cached = ZSTD_read32(ip); ip += 4; ip += 4;
Counting1[(BYTE) c ]++; while (ip < iend - 15) {
Counting2[(BYTE)(c>>8) ]++; U32 c = cached;
Counting3[(BYTE)(c>>16)]++; cached = ZSTD_read32(ip);
Counting4[ c>>24 ]++; ip += 4;
c = cached; cached = ZSTD_read32(ip); ip += 4; Counting1[(BYTE)c]++;
Counting1[(BYTE) c ]++; Counting2[(BYTE)(c >> 8)]++;
Counting2[(BYTE)(c>>8) ]++; Counting3[(BYTE)(c >> 16)]++;
Counting3[(BYTE)(c>>16)]++; Counting4[c >> 24]++;
Counting4[ c>>24 ]++; c = cached;
c = cached; cached = ZSTD_read32(ip); ip += 4; cached = ZSTD_read32(ip);
Counting1[(BYTE) c ]++; ip += 4;
Counting2[(BYTE)(c>>8) ]++; Counting1[(BYTE)c]++;
Counting3[(BYTE)(c>>16)]++; Counting2[(BYTE)(c >> 8)]++;
Counting4[ c>>24 ]++; Counting3[(BYTE)(c >> 16)]++;
c = cached; cached = ZSTD_read32(ip); ip += 4; Counting4[c >> 24]++;
Counting1[(BYTE) c ]++; c = cached;
Counting2[(BYTE)(c>>8) ]++; cached = ZSTD_read32(ip);
Counting3[(BYTE)(c>>16)]++; ip += 4;
Counting4[ c>>24 ]++; Counting1[(BYTE)c]++;
Counting2[(BYTE)(c >> 8)]++;
Counting3[(BYTE)(c >> 16)]++;
Counting4[c >> 24]++;
c = cached;
cached = ZSTD_read32(ip);
ip += 4;
Counting1[(BYTE)c]++;
Counting2[(BYTE)(c >> 8)]++;
Counting3[(BYTE)(c >> 16)]++;
Counting4[c >> 24]++;
} }
ip-=4; ip -= 4;
} }
/* finish last symbols */ /* finish last symbols */
while (ip<iend) Counting1[*ip++]++; while (ip < iend)
Counting1[*ip++]++;
if (checkMax) { /* verify stats will fit into destination table */ if (checkMax) { /* verify stats will fit into destination table */
U32 s; for (s=255; s>maxSymbolValue; s--) { U32 s;
for (s = 255; s > maxSymbolValue; s--) {
Counting1[s] += Counting2[s] + Counting3[s] + Counting4[s]; Counting1[s] += Counting2[s] + Counting3[s] + Counting4[s];
if (Counting1[s]) return ERROR(maxSymbolValue_tooSmall); if (Counting1[s])
} } return ERROR(maxSymbolValue_tooSmall);
}
}
{ U32 s; for (s=0; s<=maxSymbolValue; s++) { {
U32 s;
for (s = 0; s <= maxSymbolValue; s++) {
count[s] = Counting1[s] + Counting2[s] + Counting3[s] + Counting4[s]; count[s] = Counting1[s] + Counting2[s] + Counting3[s] + Counting4[s];
if (count[s] > max) max = count[s]; if (count[s] > max)
} } max = count[s];
}
}
while (!count[maxSymbolValue]) maxSymbolValue--; while (!count[maxSymbolValue])
maxSymbolValue--;
*maxSymbolValuePtr = maxSymbolValue; *maxSymbolValuePtr = maxSymbolValue;
return (size_t)max; return (size_t)max;
} }
@@ -390,18 +436,17 @@ static size_t FSE_count_parallel_wksp(
/* FSE_countFast_wksp() : /* FSE_countFast_wksp() :
* Same as FSE_countFast(), but using an externally provided scratch buffer. * Same as FSE_countFast(), but using an externally provided scratch buffer.
* `workSpace` size must be table of >= `1024` unsigned */ * `workSpace` size must be table of >= `1024` unsigned */
size_t FSE_countFast_wksp(unsigned* count, unsigned* maxSymbolValuePtr, size_t FSE_countFast_wksp(unsigned *count, unsigned *maxSymbolValuePtr, const void *source, size_t sourceSize, unsigned *workSpace)
const void* source, size_t sourceSize, unsigned* workSpace)
{ {
if (sourceSize < 1500) return FSE_count_simple(count, maxSymbolValuePtr, source, sourceSize); if (sourceSize < 1500)
return FSE_count_simple(count, maxSymbolValuePtr, source, sourceSize);
return FSE_count_parallel_wksp(count, maxSymbolValuePtr, source, sourceSize, 0, workSpace); return FSE_count_parallel_wksp(count, maxSymbolValuePtr, source, sourceSize, 0, workSpace);
} }
/* FSE_count_wksp() : /* FSE_count_wksp() :
* Same as FSE_count(), but using an externally provided scratch buffer. * Same as FSE_count(), but using an externally provided scratch buffer.
* `workSpace` size must be table of >= `1024` unsigned */ * `workSpace` size must be table of >= `1024` unsigned */
size_t FSE_count_wksp(unsigned* count, unsigned* maxSymbolValuePtr, size_t FSE_count_wksp(unsigned *count, unsigned *maxSymbolValuePtr, const void *source, size_t sourceSize, unsigned *workSpace)
const void* source, size_t sourceSize, unsigned* workSpace)
{ {
if (*maxSymbolValuePtr < 255) if (*maxSymbolValuePtr < 255)
return FSE_count_parallel_wksp(count, maxSymbolValuePtr, source, sourceSize, 1, workSpace); return FSE_count_parallel_wksp(count, maxSymbolValuePtr, source, sourceSize, 1, workSpace);
@@ -409,7 +454,6 @@ size_t FSE_count_wksp(unsigned* count, unsigned* maxSymbolValuePtr,
return FSE_countFast_wksp(count, maxSymbolValuePtr, source, sourceSize, workSpace); return FSE_countFast_wksp(count, maxSymbolValuePtr, source, sourceSize, workSpace);
} }
/*-************************************************************** /*-**************************************************************
* FSE Compression Code * FSE Compression Code
****************************************************************/ ****************************************************************/
@@ -421,10 +465,11 @@ size_t FSE_count_wksp(unsigned* count, unsigned* maxSymbolValuePtr,
`FSE_symbolCompressionTransform symbolTT[maxSymbolValue+1];` // This size is variable `FSE_symbolCompressionTransform symbolTT[maxSymbolValue+1];` // This size is variable
Allocation is manual (C standard does not support variable-size structures). Allocation is manual (C standard does not support variable-size structures).
*/ */
size_t FSE_sizeof_CTable (unsigned maxSymbolValue, unsigned tableLog) size_t FSE_sizeof_CTable(unsigned maxSymbolValue, unsigned tableLog)
{ {
if (tableLog > FSE_MAX_TABLELOG) return ERROR(tableLog_tooLarge); if (tableLog > FSE_MAX_TABLELOG)
return FSE_CTABLE_SIZE_U32 (tableLog, maxSymbolValue) * sizeof(U32); return ERROR(tableLog_tooLarge);
return FSE_CTABLE_SIZE_U32(tableLog, maxSymbolValue) * sizeof(U32);
} }
/* provides the minimum logSize to safely represent a distribution */ /* provides the minimum logSize to safely represent a distribution */
@@ -441,11 +486,16 @@ unsigned FSE_optimalTableLog_internal(unsigned maxTableLog, size_t srcSize, unsi
U32 maxBitsSrc = BIT_highbit32((U32)(srcSize - 1)) - minus; U32 maxBitsSrc = BIT_highbit32((U32)(srcSize - 1)) - minus;
U32 tableLog = maxTableLog; U32 tableLog = maxTableLog;
U32 minBits = FSE_minTableLog(srcSize, maxSymbolValue); U32 minBits = FSE_minTableLog(srcSize, maxSymbolValue);
if (tableLog==0) tableLog = FSE_DEFAULT_TABLELOG; if (tableLog == 0)
if (maxBitsSrc < tableLog) tableLog = maxBitsSrc; /* Accuracy can be reduced */ tableLog = FSE_DEFAULT_TABLELOG;
if (minBits > tableLog) tableLog = minBits; /* Need a minimum to safely represent all symbol values */ if (maxBitsSrc < tableLog)
if (tableLog < FSE_MIN_TABLELOG) tableLog = FSE_MIN_TABLELOG; tableLog = maxBitsSrc; /* Accuracy can be reduced */
if (tableLog > FSE_MAX_TABLELOG) tableLog = FSE_MAX_TABLELOG; if (minBits > tableLog)
tableLog = minBits; /* Need a minimum to safely represent all symbol values */
if (tableLog < FSE_MIN_TABLELOG)
tableLog = FSE_MIN_TABLELOG;
if (tableLog > FSE_MAX_TABLELOG)
tableLog = FSE_MAX_TABLELOG;
return tableLog; return tableLog;
} }
@@ -454,11 +504,10 @@ unsigned FSE_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxS
return FSE_optimalTableLog_internal(maxTableLog, srcSize, maxSymbolValue, 2); return FSE_optimalTableLog_internal(maxTableLog, srcSize, maxSymbolValue, 2);
} }
/* Secondary normalization method. /* Secondary normalization method.
To be used when primary method fails. */ To be used when primary method fails. */
static size_t FSE_normalizeM2(short* norm, U32 tableLog, const unsigned* count, size_t total, U32 maxSymbolValue) static size_t FSE_normalizeM2(short *norm, U32 tableLog, const unsigned *count, size_t total, U32 maxSymbolValue)
{ {
short const NOT_YET_ASSIGNED = -2; short const NOT_YET_ASSIGNED = -2;
U32 s; U32 s;
@@ -469,9 +518,9 @@ static size_t FSE_normalizeM2(short* norm, U32 tableLog, const unsigned* count,
U32 const lowThreshold = (U32)(total >> tableLog); U32 const lowThreshold = (U32)(total >> tableLog);
U32 lowOne = (U32)((total * 3) >> (tableLog + 1)); U32 lowOne = (U32)((total * 3) >> (tableLog + 1));
for (s=0; s<=maxSymbolValue; s++) { for (s = 0; s <= maxSymbolValue; s++) {
if (count[s] == 0) { if (count[s] == 0) {
norm[s]=0; norm[s] = 0;
continue; continue;
} }
if (count[s] <= lowThreshold) { if (count[s] <= lowThreshold) {
@@ -487,47 +536,51 @@ static size_t FSE_normalizeM2(short* norm, U32 tableLog, const unsigned* count,
continue; continue;
} }
norm[s]=NOT_YET_ASSIGNED; norm[s] = NOT_YET_ASSIGNED;
} }
ToDistribute = (1 << tableLog) - distributed; ToDistribute = (1 << tableLog) - distributed;
if ((total / ToDistribute) > lowOne) { if ((total / ToDistribute) > lowOne) {
/* risk of rounding to zero */ /* risk of rounding to zero */
lowOne = (U32)((total * 3) / (ToDistribute * 2)); lowOne = (U32)((total * 3) / (ToDistribute * 2));
for (s=0; s<=maxSymbolValue; s++) { for (s = 0; s <= maxSymbolValue; s++) {
if ((norm[s] == NOT_YET_ASSIGNED) && (count[s] <= lowOne)) { if ((norm[s] == NOT_YET_ASSIGNED) && (count[s] <= lowOne)) {
norm[s] = 1; norm[s] = 1;
distributed++; distributed++;
total -= count[s]; total -= count[s];
continue; continue;
} } }
}
ToDistribute = (1 << tableLog) - distributed; ToDistribute = (1 << tableLog) - distributed;
} }
if (distributed == maxSymbolValue+1) { if (distributed == maxSymbolValue + 1) {
/* all values are pretty poor; /* all values are pretty poor;
probably incompressible data (should have already been detected); probably incompressible data (should have already been detected);
find max, then give all remaining points to max */ find max, then give all remaining points to max */
U32 maxV = 0, maxC = 0; U32 maxV = 0, maxC = 0;
for (s=0; s<=maxSymbolValue; s++) for (s = 0; s <= maxSymbolValue; s++)
if (count[s] > maxC) maxV=s, maxC=count[s]; if (count[s] > maxC)
maxV = s, maxC = count[s];
norm[maxV] += (short)ToDistribute; norm[maxV] += (short)ToDistribute;
return 0; return 0;
} }
if (total == 0) { if (total == 0) {
/* all of the symbols were low enough for the lowOne or lowThreshold */ /* all of the symbols were low enough for the lowOne or lowThreshold */
for (s=0; ToDistribute > 0; s = (s+1)%(maxSymbolValue+1)) for (s = 0; ToDistribute > 0; s = (s + 1) % (maxSymbolValue + 1))
if (norm[s] > 0) ToDistribute--, norm[s]++; if (norm[s] > 0)
ToDistribute--, norm[s]++;
return 0; return 0;
} }
{ U64 const vStepLog = 62 - tableLog; {
U64 const mid = (1ULL << (vStepLog-1)) - 1; U64 const vStepLog = 62 - tableLog;
U64 const rStep = ((((U64)1<<vStepLog) * ToDistribute) + mid) / total; /* scale on remaining */ U64 const mid = (1ULL << (vStepLog - 1)) - 1;
U64 const rStep = ((((U64)1 << vStepLog) * ToDistribute) + mid) / total; /* scale on remaining */
U64 tmpTotal = mid; U64 tmpTotal = mid;
for (s=0; s<=maxSymbolValue; s++) { for (s = 0; s <= maxSymbolValue; s++) {
if (norm[s]==NOT_YET_ASSIGNED) { if (norm[s] == NOT_YET_ASSIGNED) {
U64 const end = tmpTotal + (count[s] * rStep); U64 const end = tmpTotal + (count[s] * rStep);
U32 const sStart = (U32)(tmpTotal >> vStepLog); U32 const sStart = (U32)(tmpTotal >> vStepLog);
U32 const sEnd = (U32)(end >> vStepLog); U32 const sEnd = (U32)(end >> vStepLog);
@@ -536,104 +589,117 @@ static size_t FSE_normalizeM2(short* norm, U32 tableLog, const unsigned* count,
return ERROR(GENERIC); return ERROR(GENERIC);
norm[s] = (short)weight; norm[s] = (short)weight;
tmpTotal = end; tmpTotal = end;
} } } }
}
}
return 0; return 0;
} }
size_t FSE_normalizeCount(short *normalizedCounter, unsigned tableLog, const unsigned *count, size_t total, unsigned maxSymbolValue)
size_t FSE_normalizeCount (short* normalizedCounter, unsigned tableLog,
const unsigned* count, size_t total,
unsigned maxSymbolValue)
{ {
/* Sanity checks */ /* Sanity checks */
if (tableLog==0) tableLog = FSE_DEFAULT_TABLELOG; if (tableLog == 0)
if (tableLog < FSE_MIN_TABLELOG) return ERROR(GENERIC); /* Unsupported size */ tableLog = FSE_DEFAULT_TABLELOG;
if (tableLog > FSE_MAX_TABLELOG) return ERROR(tableLog_tooLarge); /* Unsupported size */ if (tableLog < FSE_MIN_TABLELOG)
if (tableLog < FSE_minTableLog(total, maxSymbolValue)) return ERROR(GENERIC); /* Too small tableLog, compression potentially impossible */ return ERROR(GENERIC); /* Unsupported size */
if (tableLog > FSE_MAX_TABLELOG)
return ERROR(tableLog_tooLarge); /* Unsupported size */
if (tableLog < FSE_minTableLog(total, maxSymbolValue))
return ERROR(GENERIC); /* Too small tableLog, compression potentially impossible */
{ U32 const rtbTable[] = { 0, 473195, 504333, 520860, 550000, 700000, 750000, 830000 }; {
U32 const rtbTable[] = {0, 473195, 504333, 520860, 550000, 700000, 750000, 830000};
U64 const scale = 62 - tableLog; U64 const scale = 62 - tableLog;
U64 const step = ((U64)1<<62) / total; /* <== here, one division ! */ U64 const step = ((U64)1 << 62) / total; /* <== here, one division ! */
U64 const vStep = 1ULL<<(scale-20); U64 const vStep = 1ULL << (scale - 20);
int stillToDistribute = 1<<tableLog; int stillToDistribute = 1 << tableLog;
unsigned s; unsigned s;
unsigned largest=0; unsigned largest = 0;
short largestP=0; short largestP = 0;
U32 lowThreshold = (U32)(total >> tableLog); U32 lowThreshold = (U32)(total >> tableLog);
for (s=0; s<=maxSymbolValue; s++) { for (s = 0; s <= maxSymbolValue; s++) {
if (count[s] == total) return 0; /* rle special case */ if (count[s] == total)
if (count[s] == 0) { normalizedCounter[s]=0; continue; } return 0; /* rle special case */
if (count[s] == 0) {
normalizedCounter[s] = 0;
continue;
}
if (count[s] <= lowThreshold) { if (count[s] <= lowThreshold) {
normalizedCounter[s] = -1; normalizedCounter[s] = -1;
stillToDistribute--; stillToDistribute--;
} else { } else {
short proba = (short)((count[s]*step) >> scale); short proba = (short)((count[s] * step) >> scale);
if (proba<8) { if (proba < 8) {
U64 restToBeat = vStep * rtbTable[proba]; U64 restToBeat = vStep * rtbTable[proba];
proba += (count[s]*step) - ((U64)proba<<scale) > restToBeat; proba += (count[s] * step) - ((U64)proba << scale) > restToBeat;
} }
if (proba > largestP) largestP=proba, largest=s; if (proba > largestP)
largestP = proba, largest = s;
normalizedCounter[s] = proba; normalizedCounter[s] = proba;
stillToDistribute -= proba; stillToDistribute -= proba;
} } }
}
if (-stillToDistribute >= (normalizedCounter[largest] >> 1)) { if (-stillToDistribute >= (normalizedCounter[largest] >> 1)) {
/* corner case, need another normalization method */ /* corner case, need another normalization method */
size_t const errorCode = FSE_normalizeM2(normalizedCounter, tableLog, count, total, maxSymbolValue); size_t const errorCode = FSE_normalizeM2(normalizedCounter, tableLog, count, total, maxSymbolValue);
if (FSE_isError(errorCode)) return errorCode; if (FSE_isError(errorCode))
} return errorCode;
else normalizedCounter[largest] += (short)stillToDistribute; } else
normalizedCounter[largest] += (short)stillToDistribute;
} }
return tableLog; return tableLog;
} }
/* fake FSE_CTable, for raw (uncompressed) input */ /* fake FSE_CTable, for raw (uncompressed) input */
size_t FSE_buildCTable_raw (FSE_CTable* ct, unsigned nbBits) size_t FSE_buildCTable_raw(FSE_CTable *ct, unsigned nbBits)
{ {
const unsigned tableSize = 1 << nbBits; const unsigned tableSize = 1 << nbBits;
const unsigned tableMask = tableSize - 1; const unsigned tableMask = tableSize - 1;
const unsigned maxSymbolValue = tableMask; const unsigned maxSymbolValue = tableMask;
void* const ptr = ct; void *const ptr = ct;
U16* const tableU16 = ( (U16*) ptr) + 2; U16 *const tableU16 = ((U16 *)ptr) + 2;
void* const FSCT = ((U32*)ptr) + 1 /* header */ + (tableSize>>1); /* assumption : tableLog >= 1 */ void *const FSCT = ((U32 *)ptr) + 1 /* header */ + (tableSize >> 1); /* assumption : tableLog >= 1 */
FSE_symbolCompressionTransform* const symbolTT = (FSE_symbolCompressionTransform*) (FSCT); FSE_symbolCompressionTransform *const symbolTT = (FSE_symbolCompressionTransform *)(FSCT);
unsigned s; unsigned s;
/* Sanity checks */ /* Sanity checks */
if (nbBits < 1) return ERROR(GENERIC); /* min size */ if (nbBits < 1)
return ERROR(GENERIC); /* min size */
/* header */ /* header */
tableU16[-2] = (U16) nbBits; tableU16[-2] = (U16)nbBits;
tableU16[-1] = (U16) maxSymbolValue; tableU16[-1] = (U16)maxSymbolValue;
/* Build table */ /* Build table */
for (s=0; s<tableSize; s++) for (s = 0; s < tableSize; s++)
tableU16[s] = (U16)(tableSize + s); tableU16[s] = (U16)(tableSize + s);
/* Build Symbol Transformation Table */ /* Build Symbol Transformation Table */
{ const U32 deltaNbBits = (nbBits << 16) - (1 << nbBits); {
for (s=0; s<=maxSymbolValue; s++) { const U32 deltaNbBits = (nbBits << 16) - (1 << nbBits);
for (s = 0; s <= maxSymbolValue; s++) {
symbolTT[s].deltaNbBits = deltaNbBits; symbolTT[s].deltaNbBits = deltaNbBits;
symbolTT[s].deltaFindState = s-1; symbolTT[s].deltaFindState = s - 1;
} } }
}
return 0; return 0;
} }
/* fake FSE_CTable, for rle input (always same symbol) */ /* fake FSE_CTable, for rle input (always same symbol) */
size_t FSE_buildCTable_rle (FSE_CTable* ct, BYTE symbolValue) size_t FSE_buildCTable_rle(FSE_CTable *ct, BYTE symbolValue)
{ {
void* ptr = ct; void *ptr = ct;
U16* tableU16 = ( (U16*) ptr) + 2; U16 *tableU16 = ((U16 *)ptr) + 2;
void* FSCTptr = (U32*)ptr + 2; void *FSCTptr = (U32 *)ptr + 2;
FSE_symbolCompressionTransform* symbolTT = (FSE_symbolCompressionTransform*) FSCTptr; FSE_symbolCompressionTransform *symbolTT = (FSE_symbolCompressionTransform *)FSCTptr;
/* header */ /* header */
tableU16[-2] = (U16) 0; tableU16[-2] = (U16)0;
tableU16[-1] = (U16) symbolValue; tableU16[-1] = (U16)symbolValue;
/* Build table */ /* Build table */
tableU16[0] = 0; tableU16[0] = 0;
@@ -646,22 +712,23 @@ size_t FSE_buildCTable_rle (FSE_CTable* ct, BYTE symbolValue)
return 0; return 0;
} }
static size_t FSE_compress_usingCTable_generic(void *dst, size_t dstSize, const void *src, size_t srcSize, const FSE_CTable *ct, const unsigned fast)
static size_t FSE_compress_usingCTable_generic (void* dst, size_t dstSize,
const void* src, size_t srcSize,
const FSE_CTable* ct, const unsigned fast)
{ {
const BYTE* const istart = (const BYTE*) src; const BYTE *const istart = (const BYTE *)src;
const BYTE* const iend = istart + srcSize; const BYTE *const iend = istart + srcSize;
const BYTE* ip=iend; const BYTE *ip = iend;
BIT_CStream_t bitC; BIT_CStream_t bitC;
FSE_CState_t CState1, CState2; FSE_CState_t CState1, CState2;
/* init */ /* init */
if (srcSize <= 2) return 0; if (srcSize <= 2)
{ size_t const initError = BIT_initCStream(&bitC, dst, dstSize); return 0;
if (FSE_isError(initError)) return 0; /* not enough space available to write a bitstream */ } {
size_t const initError = BIT_initCStream(&bitC, dst, dstSize);
if (FSE_isError(initError))
return 0; /* not enough space available to write a bitstream */
}
#define FSE_FLUSHBITS(s) (fast ? BIT_flushBitsFast(s) : BIT_flushBits(s)) #define FSE_FLUSHBITS(s) (fast ? BIT_flushBitsFast(s) : BIT_flushBits(s))
@@ -677,23 +744,23 @@ static size_t FSE_compress_usingCTable_generic (void* dst, size_t dstSize,
/* join to mod 4 */ /* join to mod 4 */
srcSize -= 2; srcSize -= 2;
if ((sizeof(bitC.bitContainer)*8 > FSE_MAX_TABLELOG*4+7 ) && (srcSize & 2)) { /* test bit 2 */ if ((sizeof(bitC.bitContainer) * 8 > FSE_MAX_TABLELOG * 4 + 7) && (srcSize & 2)) { /* test bit 2 */
FSE_encodeSymbol(&bitC, &CState2, *--ip); FSE_encodeSymbol(&bitC, &CState2, *--ip);
FSE_encodeSymbol(&bitC, &CState1, *--ip); FSE_encodeSymbol(&bitC, &CState1, *--ip);
FSE_FLUSHBITS(&bitC); FSE_FLUSHBITS(&bitC);
} }
/* 2 or 4 encoding per loop */ /* 2 or 4 encoding per loop */
while ( ip>istart ) { while (ip > istart) {
FSE_encodeSymbol(&bitC, &CState2, *--ip); FSE_encodeSymbol(&bitC, &CState2, *--ip);
if (sizeof(bitC.bitContainer)*8 < FSE_MAX_TABLELOG*2+7 ) /* this test must be static */ if (sizeof(bitC.bitContainer) * 8 < FSE_MAX_TABLELOG * 2 + 7) /* this test must be static */
FSE_FLUSHBITS(&bitC); FSE_FLUSHBITS(&bitC);
FSE_encodeSymbol(&bitC, &CState1, *--ip); FSE_encodeSymbol(&bitC, &CState1, *--ip);
if (sizeof(bitC.bitContainer)*8 > FSE_MAX_TABLELOG*4+7 ) { /* this test must be static */ if (sizeof(bitC.bitContainer) * 8 > FSE_MAX_TABLELOG * 4 + 7) { /* this test must be static */
FSE_encodeSymbol(&bitC, &CState2, *--ip); FSE_encodeSymbol(&bitC, &CState2, *--ip);
FSE_encodeSymbol(&bitC, &CState1, *--ip); FSE_encodeSymbol(&bitC, &CState1, *--ip);
} }
@@ -706,9 +773,7 @@ static size_t FSE_compress_usingCTable_generic (void* dst, size_t dstSize,
return BIT_closeCStream(&bitC); return BIT_closeCStream(&bitC);
} }
size_t FSE_compress_usingCTable (void* dst, size_t dstSize, size_t FSE_compress_usingCTable(void *dst, size_t dstSize, const void *src, size_t srcSize, const FSE_CTable *ct)
const void* src, size_t srcSize,
const FSE_CTable* ct)
{ {
unsigned const fast = (dstSize >= FSE_BLOCKBOUND(srcSize)); unsigned const fast = (dstSize >= FSE_BLOCKBOUND(srcSize));
@@ -718,62 +783,79 @@ size_t FSE_compress_usingCTable (void* dst, size_t dstSize,
return FSE_compress_usingCTable_generic(dst, dstSize, src, srcSize, ct, 0); return FSE_compress_usingCTable_generic(dst, dstSize, src, srcSize, ct, 0);
} }
size_t FSE_compressBound(size_t size) { return FSE_COMPRESSBOUND(size); } size_t FSE_compressBound(size_t size) { return FSE_COMPRESSBOUND(size); }
#define CHECK_V_F(e, f) size_t const e = f; if (ERR_isError(e)) return f #define CHECK_V_F(e, f) \
#define CHECK_F(f) { CHECK_V_F(_var_err__, f); } size_t const e = f; \
if (ERR_isError(e)) \
return f
#define CHECK_F(f) \
{ \
CHECK_V_F(_var_err__, f); \
}
/* FSE_compress_wksp() : /* FSE_compress_wksp() :
* Same as FSE_compress2(), but using an externally allocated scratch buffer (`workSpace`). * Same as FSE_compress2(), but using an externally allocated scratch buffer (`workSpace`).
* `wkspSize` size must be `(1<<tableLog)`. * `wkspSize` size must be `(1<<tableLog)`.
*/ */
size_t FSE_compress_wksp (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize) size_t FSE_compress_wksp(void *dst, size_t dstSize, const void *src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void *workSpace,
size_t wkspSize)
{ {
BYTE* const ostart = (BYTE*) dst; BYTE *const ostart = (BYTE *)dst;
BYTE* op = ostart; BYTE *op = ostart;
BYTE* const oend = ostart + dstSize; BYTE *const oend = ostart + dstSize;
U32 count[FSE_MAX_SYMBOL_VALUE+1]; U32 count[FSE_MAX_SYMBOL_VALUE + 1];
S16 norm[FSE_MAX_SYMBOL_VALUE+1]; S16 norm[FSE_MAX_SYMBOL_VALUE + 1];
FSE_CTable* CTable = (FSE_CTable*)workSpace; FSE_CTable *CTable = (FSE_CTable *)workSpace;
size_t const CTableSize = FSE_CTABLE_SIZE_U32(tableLog, maxSymbolValue); size_t const CTableSize = FSE_CTABLE_SIZE_U32(tableLog, maxSymbolValue);
void* scratchBuffer = (void*)(CTable + CTableSize); void *scratchBuffer = (void *)(CTable + CTableSize);
size_t const scratchBufferSize = wkspSize - (CTableSize * sizeof(FSE_CTable)); size_t const scratchBufferSize = wkspSize - (CTableSize * sizeof(FSE_CTable));
/* init conditions */ /* init conditions */
if (wkspSize < FSE_WKSP_SIZE_U32(tableLog, maxSymbolValue)) return ERROR(tableLog_tooLarge); if (wkspSize < FSE_WKSP_SIZE_U32(tableLog, maxSymbolValue))
if (srcSize <= 1) return 0; /* Not compressible */ return ERROR(tableLog_tooLarge);
if (!maxSymbolValue) maxSymbolValue = FSE_MAX_SYMBOL_VALUE; if (srcSize <= 1)
if (!tableLog) tableLog = FSE_DEFAULT_TABLELOG; return 0; /* Not compressible */
if (!maxSymbolValue)
maxSymbolValue = FSE_MAX_SYMBOL_VALUE;
if (!tableLog)
tableLog = FSE_DEFAULT_TABLELOG;
/* Scan input and build symbol stats */ /* Scan input and build symbol stats */
{ CHECK_V_F(maxCount, FSE_count_wksp(count, &maxSymbolValue, src, srcSize, (unsigned*)scratchBuffer) ); {
if (maxCount == srcSize) return 1; /* only a single symbol in src : rle */ CHECK_V_F(maxCount, FSE_count_wksp(count, &maxSymbolValue, src, srcSize, (unsigned *)scratchBuffer));
if (maxCount == 1) return 0; /* each symbol present maximum once => not compressible */ if (maxCount == srcSize)
if (maxCount < (srcSize >> 7)) return 0; /* Heuristic : not compressible enough */ return 1; /* only a single symbol in src : rle */
if (maxCount == 1)
return 0; /* each symbol present maximum once => not compressible */
if (maxCount < (srcSize >> 7))
return 0; /* Heuristic : not compressible enough */
} }
tableLog = FSE_optimalTableLog(tableLog, srcSize, maxSymbolValue); tableLog = FSE_optimalTableLog(tableLog, srcSize, maxSymbolValue);
CHECK_F( FSE_normalizeCount(norm, tableLog, count, srcSize, maxSymbolValue) ); CHECK_F(FSE_normalizeCount(norm, tableLog, count, srcSize, maxSymbolValue));
/* Write table description header */ /* Write table description header */
{ CHECK_V_F(nc_err, FSE_writeNCount(op, oend-op, norm, maxSymbolValue, tableLog) ); {
CHECK_V_F(nc_err, FSE_writeNCount(op, oend - op, norm, maxSymbolValue, tableLog));
op += nc_err; op += nc_err;
} }
/* Compress */ /* Compress */
CHECK_F( FSE_buildCTable_wksp(CTable, norm, maxSymbolValue, tableLog, scratchBuffer, scratchBufferSize) ); CHECK_F(FSE_buildCTable_wksp(CTable, norm, maxSymbolValue, tableLog, scratchBuffer, scratchBufferSize));
{ CHECK_V_F(cSize, FSE_compress_usingCTable(op, oend - op, src, srcSize, CTable) ); {
if (cSize == 0) return 0; /* not enough space for compressed data */ CHECK_V_F(cSize, FSE_compress_usingCTable(op, oend - op, src, srcSize, CTable));
if (cSize == 0)
return 0; /* not enough space for compressed data */
op += cSize; op += cSize;
} }
/* check compressibility */ /* check compressibility */
if ( (size_t)(op-ostart) >= srcSize-1 ) return 0; if ((size_t)(op - ostart) >= srcSize - 1)
return 0;
return op-ostart; return op - ostart;
} }
#endif /* FSE_COMMONDEFS_ONLY */ #endif /* FSE_COMMONDEFS_ONLY */
contrib/linux-kernel/lib/zstd/fse_decompress.c
+111 -91
View File
@@ -37,31 +37,35 @@
* - Source repository : https://github.com/Cyan4973/FiniteStateEntropy * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy
*/ */
/* ************************************************************** /* **************************************************************
* Compiler specifics * Compiler specifics
****************************************************************/ ****************************************************************/
#define FORCE_INLINE static __always_inline #define FORCE_INLINE static __always_inline
/* ************************************************************** /* **************************************************************
* Includes * Includes
****************************************************************/ ****************************************************************/
#include <linux/compiler.h>
#include <linux/string.h> /* memcpy, memset */
#include "bitstream.h" #include "bitstream.h"
#include "fse.h" #include "fse.h"
#include <linux/compiler.h>
#include <linux/string.h> /* memcpy, memset */
/* ************************************************************** /* **************************************************************
* Error Management * Error Management
****************************************************************/ ****************************************************************/
#define FSE_isError ERR_isError #define FSE_isError ERR_isError
#define FSE_STATIC_ASSERT(c) { enum { FSE_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */ #define FSE_STATIC_ASSERT(c) \
{ \
enum { FSE_static_assert = 1 / (int)(!!(c)) }; \
} /* use only *after* variable declarations */
/* check and forward error code */ /* check and forward error code */
#define CHECK_F(f) { size_t const e = f; if (FSE_isError(e)) return e; } #define CHECK_F(f) \
{ \
size_t const e = f; \
if (FSE_isError(e)) \
return e; \
}
/* ************************************************************** /* **************************************************************
* Templates * Templates
@@ -74,89 +78,100 @@
/* safety checks */ /* safety checks */
#ifndef FSE_FUNCTION_EXTENSION #ifndef FSE_FUNCTION_EXTENSION
# error "FSE_FUNCTION_EXTENSION must be defined" #error "FSE_FUNCTION_EXTENSION must be defined"
#endif #endif
#ifndef FSE_FUNCTION_TYPE #ifndef FSE_FUNCTION_TYPE
# error "FSE_FUNCTION_TYPE must be defined" #error "FSE_FUNCTION_TYPE must be defined"
#endif #endif
/* Function names */ /* Function names */
#define FSE_CAT(X,Y) X##Y #define FSE_CAT(X, Y) X##Y
#define FSE_FUNCTION_NAME(X,Y) FSE_CAT(X,Y) #define FSE_FUNCTION_NAME(X, Y) FSE_CAT(X, Y)
#define FSE_TYPE_NAME(X,Y) FSE_CAT(X,Y) #define FSE_TYPE_NAME(X, Y) FSE_CAT(X, Y)
/* Function templates */ /* Function templates */
size_t FSE_buildDTable(FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog) size_t FSE_buildDTable(FSE_DTable *dt, const short *normalizedCounter, unsigned maxSymbolValue, unsigned tableLog)
{ {
void* const tdPtr = dt+1; /* because *dt is unsigned, 32-bits aligned on 32-bits */ void *const tdPtr = dt + 1; /* because *dt is unsigned, 32-bits aligned on 32-bits */
FSE_DECODE_TYPE* const tableDecode = (FSE_DECODE_TYPE*) (tdPtr); FSE_DECODE_TYPE *const tableDecode = (FSE_DECODE_TYPE *)(tdPtr);
U16 symbolNext[FSE_MAX_SYMBOL_VALUE+1]; U16 symbolNext[FSE_MAX_SYMBOL_VALUE + 1];
U32 const maxSV1 = maxSymbolValue + 1; U32 const maxSV1 = maxSymbolValue + 1;
U32 const tableSize = 1 << tableLog; U32 const tableSize = 1 << tableLog;
U32 highThreshold = tableSize-1; U32 highThreshold = tableSize - 1;
/* Sanity Checks */ /* Sanity Checks */
if (maxSymbolValue > FSE_MAX_SYMBOL_VALUE) return ERROR(maxSymbolValue_tooLarge); if (maxSymbolValue > FSE_MAX_SYMBOL_VALUE)
if (tableLog > FSE_MAX_TABLELOG) return ERROR(tableLog_tooLarge); return ERROR(maxSymbolValue_tooLarge);
if (tableLog > FSE_MAX_TABLELOG)
return ERROR(tableLog_tooLarge);
/* Init, lay down lowprob symbols */ /* Init, lay down lowprob symbols */
{ FSE_DTableHeader DTableH; {
FSE_DTableHeader DTableH;
DTableH.tableLog = (U16)tableLog; DTableH.tableLog = (U16)tableLog;
DTableH.fastMode = 1; DTableH.fastMode = 1;
{ S16 const largeLimit= (S16)(1 << (tableLog-1)); {
S16 const largeLimit = (S16)(1 << (tableLog - 1));
U32 s; U32 s;
for (s=0; s<maxSV1; s++) { for (s = 0; s < maxSV1; s++) {
if (normalizedCounter[s]==-1) { if (normalizedCounter[s] == -1) {
tableDecode[highThreshold--].symbol = (FSE_FUNCTION_TYPE)s; tableDecode[highThreshold--].symbol = (FSE_FUNCTION_TYPE)s;
symbolNext[s] = 1; symbolNext[s] = 1;
} else { } else {
if (normalizedCounter[s] >= largeLimit) DTableH.fastMode=0; if (normalizedCounter[s] >= largeLimit)
DTableH.fastMode = 0;
symbolNext[s] = normalizedCounter[s]; symbolNext[s] = normalizedCounter[s];
} } } }
}
}
memcpy(dt, &DTableH, sizeof(DTableH)); memcpy(dt, &DTableH, sizeof(DTableH));
} }
/* Spread symbols */ /* Spread symbols */
{ U32 const tableMask = tableSize-1; {
U32 const tableMask = tableSize - 1;
U32 const step = FSE_TABLESTEP(tableSize); U32 const step = FSE_TABLESTEP(tableSize);
U32 s, position = 0; U32 s, position = 0;
for (s=0; s<maxSV1; s++) { for (s = 0; s < maxSV1; s++) {
int i; int i;
for (i=0; i<normalizedCounter[s]; i++) { for (i = 0; i < normalizedCounter[s]; i++) {
tableDecode[position].symbol = (FSE_FUNCTION_TYPE)s; tableDecode[position].symbol = (FSE_FUNCTION_TYPE)s;
position = (position + step) & tableMask; position = (position + step) & tableMask;
while (position > highThreshold) position = (position + step) & tableMask; /* lowprob area */ while (position > highThreshold)
} } position = (position + step) & tableMask; /* lowprob area */
if (position!=0) return ERROR(GENERIC); /* position must reach all cells once, otherwise normalizedCounter is incorrect */ }
}
if (position != 0)
return ERROR(GENERIC); /* position must reach all cells once, otherwise normalizedCounter is incorrect */
} }
/* Build Decoding table */ /* Build Decoding table */
{ U32 u; {
for (u=0; u<tableSize; u++) { U32 u;
for (u = 0; u < tableSize; u++) {
FSE_FUNCTION_TYPE const symbol = (FSE_FUNCTION_TYPE)(tableDecode[u].symbol); FSE_FUNCTION_TYPE const symbol = (FSE_FUNCTION_TYPE)(tableDecode[u].symbol);
U16 nextState = symbolNext[symbol]++; U16 nextState = symbolNext[symbol]++;
tableDecode[u].nbBits = (BYTE) (tableLog - BIT_highbit32 ((U32)nextState) ); tableDecode[u].nbBits = (BYTE)(tableLog - BIT_highbit32((U32)nextState));
tableDecode[u].newState = (U16) ( (nextState << tableDecode[u].nbBits) - tableSize); tableDecode[u].newState = (U16)((nextState << tableDecode[u].nbBits) - tableSize);
} } }
}
return 0; return 0;
} }
#ifndef FSE_COMMONDEFS_ONLY #ifndef FSE_COMMONDEFS_ONLY
/*-******************************************************* /*-*******************************************************
* Decompression (Byte symbols) * Decompression (Byte symbols)
*********************************************************/ *********************************************************/
size_t FSE_buildDTable_rle (FSE_DTable* dt, BYTE symbolValue) size_t FSE_buildDTable_rle(FSE_DTable *dt, BYTE symbolValue)
{ {
void* ptr = dt; void *ptr = dt;
FSE_DTableHeader* const DTableH = (FSE_DTableHeader*)ptr; FSE_DTableHeader *const DTableH = (FSE_DTableHeader *)ptr;
void* dPtr = dt + 1; void *dPtr = dt + 1;
FSE_decode_t* const cell = (FSE_decode_t*)dPtr; FSE_decode_t *const cell = (FSE_decode_t *)dPtr;
DTableH->tableLog = 0; DTableH->tableLog = 0;
DTableH->fastMode = 0; DTableH->fastMode = 0;
@@ -168,25 +183,25 @@ size_t FSE_buildDTable_rle (FSE_DTable* dt, BYTE symbolValue)
return 0; return 0;
} }
size_t FSE_buildDTable_raw(FSE_DTable *dt, unsigned nbBits)
size_t FSE_buildDTable_raw (FSE_DTable* dt, unsigned nbBits)
{ {
void* ptr = dt; void *ptr = dt;
FSE_DTableHeader* const DTableH = (FSE_DTableHeader*)ptr; FSE_DTableHeader *const DTableH = (FSE_DTableHeader *)ptr;
void* dPtr = dt + 1; void *dPtr = dt + 1;
FSE_decode_t* const dinfo = (FSE_decode_t*)dPtr; FSE_decode_t *const dinfo = (FSE_decode_t *)dPtr;
const unsigned tableSize = 1 << nbBits; const unsigned tableSize = 1 << nbBits;
const unsigned tableMask = tableSize - 1; const unsigned tableMask = tableSize - 1;
const unsigned maxSV1 = tableMask+1; const unsigned maxSV1 = tableMask + 1;
unsigned s; unsigned s;
/* Sanity checks */ /* Sanity checks */
if (nbBits < 1) return ERROR(GENERIC); /* min size */ if (nbBits < 1)
return ERROR(GENERIC); /* min size */
/* Build Decoding Table */ /* Build Decoding Table */
DTableH->tableLog = (U16)nbBits; DTableH->tableLog = (U16)nbBits;
DTableH->fastMode = 1; DTableH->fastMode = 1;
for (s=0; s<maxSV1; s++) { for (s = 0; s < maxSV1; s++) {
dinfo[s].newState = 0; dinfo[s].newState = 0;
dinfo[s].symbol = (BYTE)s; dinfo[s].symbol = (BYTE)s;
dinfo[s].nbBits = (BYTE)nbBits; dinfo[s].nbBits = (BYTE)nbBits;
@@ -195,15 +210,13 @@ size_t FSE_buildDTable_raw (FSE_DTable* dt, unsigned nbBits)
return 0; return 0;
} }
FORCE_INLINE size_t FSE_decompress_usingDTable_generic( FORCE_INLINE size_t FSE_decompress_usingDTable_generic(void *dst, size_t maxDstSize, const void *cSrc, size_t cSrcSize, const FSE_DTable *dt,
void* dst, size_t maxDstSize, const unsigned fast)
const void* cSrc, size_t cSrcSize,
const FSE_DTable* dt, const unsigned fast)
{ {
BYTE* const ostart = (BYTE*) dst; BYTE *const ostart = (BYTE *)dst;
BYTE* op = ostart; BYTE *op = ostart;
BYTE* const omax = op + maxDstSize; BYTE *const omax = op + maxDstSize;
BYTE* const olimit = omax-3; BYTE *const olimit = omax - 3;
BIT_DStream_t bitD; BIT_DStream_t bitD;
FSE_DState_t state1; FSE_DState_t state1;
@@ -218,20 +231,25 @@ FORCE_INLINE size_t FSE_decompress_usingDTable_generic(
#define FSE_GETSYMBOL(statePtr) fast ? FSE_decodeSymbolFast(statePtr, &bitD) : FSE_decodeSymbol(statePtr, &bitD) #define FSE_GETSYMBOL(statePtr) fast ? FSE_decodeSymbolFast(statePtr, &bitD) : FSE_decodeSymbol(statePtr, &bitD)
/* 4 symbols per loop */ /* 4 symbols per loop */
for ( ; (BIT_reloadDStream(&bitD)==BIT_DStream_unfinished) & (op<olimit) ; op+=4) { for (; (BIT_reloadDStream(&bitD) == BIT_DStream_unfinished) & (op < olimit); op += 4) {
op[0] = FSE_GETSYMBOL(&state1); op[0] = FSE_GETSYMBOL(&state1);
if (FSE_MAX_TABLELOG*2+7 > sizeof(bitD.bitContainer)*8) /* This test must be static */ if (FSE_MAX_TABLELOG * 2 + 7 > sizeof(bitD.bitContainer) * 8) /* This test must be static */
BIT_reloadDStream(&bitD); BIT_reloadDStream(&bitD);
op[1] = FSE_GETSYMBOL(&state2); op[1] = FSE_GETSYMBOL(&state2);
if (FSE_MAX_TABLELOG*4+7 > sizeof(bitD.bitContainer)*8) /* This test must be static */ if (FSE_MAX_TABLELOG * 4 + 7 > sizeof(bitD.bitContainer) * 8) /* This test must be static */
{ if (BIT_reloadDStream(&bitD) > BIT_DStream_unfinished) { op+=2; break; } } {
if (BIT_reloadDStream(&bitD) > BIT_DStream_unfinished) {
op += 2;
break;
}
}
op[2] = FSE_GETSYMBOL(&state1); op[2] = FSE_GETSYMBOL(&state1);
if (FSE_MAX_TABLELOG*2+7 > sizeof(bitD.bitContainer)*8) /* This test must be static */ if (FSE_MAX_TABLELOG * 2 + 7 > sizeof(bitD.bitContainer) * 8) /* This test must be static */
BIT_reloadDStream(&bitD); BIT_reloadDStream(&bitD);
op[3] = FSE_GETSYMBOL(&state2); op[3] = FSE_GETSYMBOL(&state2);
@@ -240,58 +258,60 @@ FORCE_INLINE size_t FSE_decompress_usingDTable_generic(
/* tail */ /* tail */
/* note : BIT_reloadDStream(&bitD) >= FSE_DStream_partiallyFilled; Ends at exactly BIT_DStream_completed */ /* note : BIT_reloadDStream(&bitD) >= FSE_DStream_partiallyFilled; Ends at exactly BIT_DStream_completed */
while (1) { while (1) {
if (op>(omax-2)) return ERROR(dstSize_tooSmall); if (op > (omax - 2))
return ERROR(dstSize_tooSmall);
*op++ = FSE_GETSYMBOL(&state1); *op++ = FSE_GETSYMBOL(&state1);
if (BIT_reloadDStream(&bitD)==BIT_DStream_overflow) { if (BIT_reloadDStream(&bitD) == BIT_DStream_overflow) {
*op++ = FSE_GETSYMBOL(&state2); *op++ = FSE_GETSYMBOL(&state2);
break; break;
} }
if (op>(omax-2)) return ERROR(dstSize_tooSmall); if (op > (omax - 2))
return ERROR(dstSize_tooSmall);
*op++ = FSE_GETSYMBOL(&state2); *op++ = FSE_GETSYMBOL(&state2);
if (BIT_reloadDStream(&bitD)==BIT_DStream_overflow) { if (BIT_reloadDStream(&bitD) == BIT_DStream_overflow) {
*op++ = FSE_GETSYMBOL(&state1); *op++ = FSE_GETSYMBOL(&state1);
break; break;
} } }
}
return op-ostart; return op - ostart;
} }
size_t FSE_decompress_usingDTable(void *dst, size_t originalSize, const void *cSrc, size_t cSrcSize, const FSE_DTable *dt)
size_t FSE_decompress_usingDTable(void* dst, size_t originalSize,
const void* cSrc, size_t cSrcSize,
const FSE_DTable* dt)
{ {
const void* ptr = dt; const void *ptr = dt;
const FSE_DTableHeader* DTableH = (const FSE_DTableHeader*)ptr; const FSE_DTableHeader *DTableH = (const FSE_DTableHeader *)ptr;
const U32 fastMode = DTableH->fastMode; const U32 fastMode = DTableH->fastMode;
/* select fast mode (static) */ /* select fast mode (static) */
if (fastMode) return FSE_decompress_usingDTable_generic(dst, originalSize, cSrc, cSrcSize, dt, 1); if (fastMode)
return FSE_decompress_usingDTable_generic(dst, originalSize, cSrc, cSrcSize, dt, 1);
return FSE_decompress_usingDTable_generic(dst, originalSize, cSrc, cSrcSize, dt, 0); return FSE_decompress_usingDTable_generic(dst, originalSize, cSrc, cSrcSize, dt, 0);
} }
size_t FSE_decompress_wksp(void *dst, size_t dstCapacity, const void *cSrc, size_t cSrcSize, FSE_DTable *workSpace, unsigned maxLog)
size_t FSE_decompress_wksp(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, FSE_DTable* workSpace, unsigned maxLog)
{ {
const BYTE* const istart = (const BYTE*)cSrc; const BYTE *const istart = (const BYTE *)cSrc;
const BYTE* ip = istart; const BYTE *ip = istart;
short counting[FSE_MAX_SYMBOL_VALUE+1]; short counting[FSE_MAX_SYMBOL_VALUE + 1];
unsigned tableLog; unsigned tableLog;
unsigned maxSymbolValue = FSE_MAX_SYMBOL_VALUE; unsigned maxSymbolValue = FSE_MAX_SYMBOL_VALUE;
/* normal FSE decoding mode */ /* normal FSE decoding mode */
size_t const NCountLength = FSE_readNCount (counting, &maxSymbolValue, &tableLog, istart, cSrcSize); size_t const NCountLength = FSE_readNCount(counting, &maxSymbolValue, &tableLog, istart, cSrcSize);
if (FSE_isError(NCountLength)) return NCountLength; if (FSE_isError(NCountLength))
//if (NCountLength >= cSrcSize) return ERROR(srcSize_wrong); /* too small input size; supposed to be already checked in NCountLength, only remaining case : NCountLength==cSrcSize */ return NCountLength;
if (tableLog > maxLog) return ERROR(tableLog_tooLarge); // if (NCountLength >= cSrcSize) return ERROR(srcSize_wrong); /* too small input size; supposed to be already checked in NCountLength, only remaining
// case : NCountLength==cSrcSize */
if (tableLog > maxLog)
return ERROR(tableLog_tooLarge);
ip += NCountLength; ip += NCountLength;
cSrcSize -= NCountLength; cSrcSize -= NCountLength;
CHECK_F( FSE_buildDTable (workSpace, counting, maxSymbolValue, tableLog) ); CHECK_F(FSE_buildDTable(workSpace, counting, maxSymbolValue, tableLog));
return FSE_decompress_usingDTable (dst, dstCapacity, ip, cSrcSize, workSpace); /* always return, even if it is an error code */ return FSE_decompress_usingDTable(dst, dstCapacity, ip, cSrcSize, workSpace); /* always return, even if it is an error code */
} }
#endif /* FSE_COMMONDEFS_ONLY */ #endif /* FSE_COMMONDEFS_ONLY */
contrib/linux-kernel/lib/zstd/huf.h
+45 -51
View File
@@ -40,11 +40,9 @@
#ifndef HUF_H_298734234 #ifndef HUF_H_298734234
#define HUF_H_298734234 #define HUF_H_298734234
/* *** Dependencies *** */ /* *** Dependencies *** */
#include <linux/types.h> /* size_t */ #include <linux/types.h> /* size_t */
/* *** Tool functions *** */ /* *** Tool functions *** */
#define HUF_BLOCKSIZE_MAX (128 * 1024) /**< maximum input size for a single block compressed with HUF_compress */ #define HUF_BLOCKSIZE_MAX (128 * 1024) /**< maximum input size for a single block compressed with HUF_compress */
size_t HUF_compressBound(size_t size); /**< maximum compressed size (worst case) */ size_t HUF_compressBound(size_t size); /**< maximum compressed size (worst case) */
@@ -52,19 +50,16 @@ size_t HUF_compressBound(size_t size); /**< maximum compressed size (worst
/* Error Management */ /* Error Management */
unsigned HUF_isError(size_t code); /**< tells if a return value is an error code */ unsigned HUF_isError(size_t code); /**< tells if a return value is an error code */
/* *** Advanced function *** */ /* *** Advanced function *** */
/** HUF_compress4X_wksp() : /** HUF_compress4X_wksp() :
* Same as HUF_compress2(), but uses externally allocated `workSpace`, which must be a table of >= 1024 unsigned */ * Same as HUF_compress2(), but uses externally allocated `workSpace`, which must be a table of >= 1024 unsigned */
size_t HUF_compress4X_wksp (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize); /**< `workSpace` must be a table of at least HUF_WORKSPACE_SIZE_U32 unsigned */ size_t HUF_compress4X_wksp(void *dst, size_t dstSize, const void *src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void *workSpace,
size_t wkspSize); /**< `workSpace` must be a table of at least HUF_WORKSPACE_SIZE_U32 unsigned */
/* *** Dependencies *** */ /* *** Dependencies *** */
#include "mem.h" /* U32 */ #include "mem.h" /* U32 */
/* *** Constants *** */ /* *** Constants *** */
#define HUF_TABLELOG_MAX 12 /* max configured tableLog (for static allocation); can be modified up to HUF_ABSOLUTEMAX_TABLELOG */ #define HUF_TABLELOG_MAX 12 /* max configured tableLog (for static allocation); can be modified up to HUF_ABSOLUTEMAX_TABLELOG */
#define HUF_TABLELOG_DEFAULT 11 /* tableLog by default, when not specified */ #define HUF_TABLELOG_DEFAULT 11 /* tableLog by default, when not specified */
@@ -72,45 +67,41 @@ size_t HUF_compress4X_wksp (void* dst, size_t dstSize, const void* src, size_t s
#define HUF_TABLELOG_ABSOLUTEMAX 15 /* absolute limit of HUF_MAX_TABLELOG. Beyond that value, code does not work */ #define HUF_TABLELOG_ABSOLUTEMAX 15 /* absolute limit of HUF_MAX_TABLELOG. Beyond that value, code does not work */
#if (HUF_TABLELOG_MAX > HUF_TABLELOG_ABSOLUTEMAX) #if (HUF_TABLELOG_MAX > HUF_TABLELOG_ABSOLUTEMAX)
# error "HUF_TABLELOG_MAX is too large !" #error "HUF_TABLELOG_MAX is too large !"
#endif #endif
/* **************************************** /* ****************************************
* Static allocation * Static allocation
******************************************/ ******************************************/
/* HUF buffer bounds */ /* HUF buffer bounds */
#define HUF_CTABLEBOUND 129 #define HUF_CTABLEBOUND 129
#define HUF_BLOCKBOUND(size) (size + (size>>8) + 8) /* only true if incompressible pre-filtered with fast heuristic */ #define HUF_BLOCKBOUND(size) (size + (size >> 8) + 8) /* only true if incompressible pre-filtered with fast heuristic */
#define HUF_COMPRESSBOUND(size) (HUF_CTABLEBOUND + HUF_BLOCKBOUND(size)) /* Macro version, useful for static allocation */ #define HUF_COMPRESSBOUND(size) (HUF_CTABLEBOUND + HUF_BLOCKBOUND(size)) /* Macro version, useful for static allocation */
/* static allocation of HUF's Compression Table */ /* static allocation of HUF's Compression Table */
#define HUF_CREATE_STATIC_CTABLE(name, maxSymbolValue) \ #define HUF_CREATE_STATIC_CTABLE(name, maxSymbolValue) \
U32 name##hb[maxSymbolValue+1]; \ U32 name##hb[maxSymbolValue + 1]; \
void* name##hv = &(name##hb); \ void *name##hv = &(name##hb); \
HUF_CElt* name = (HUF_CElt*)(name##hv) /* no final ; */ HUF_CElt *name = (HUF_CElt *)(name##hv) /* no final ; */
/* static allocation of HUF's DTable */ /* static allocation of HUF's DTable */
typedef U32 HUF_DTable; typedef U32 HUF_DTable;
#define HUF_DTABLE_SIZE(maxTableLog) (1 + (1<<(maxTableLog))) #define HUF_DTABLE_SIZE(maxTableLog) (1 + (1 << (maxTableLog)))
#define HUF_CREATE_STATIC_DTABLEX2(DTable, maxTableLog) \ #define HUF_CREATE_STATIC_DTABLEX2(DTable, maxTableLog) HUF_DTable DTable[HUF_DTABLE_SIZE((maxTableLog)-1)] = {((U32)((maxTableLog)-1) * 0x01000001)}
HUF_DTable DTable[HUF_DTABLE_SIZE((maxTableLog)-1)] = { ((U32)((maxTableLog)-1) * 0x01000001) } #define HUF_CREATE_STATIC_DTABLEX4(DTable, maxTableLog) HUF_DTable DTable[HUF_DTABLE_SIZE(maxTableLog)] = {((U32)(maxTableLog)*0x01000001)}
#define HUF_CREATE_STATIC_DTABLEX4(DTable, maxTableLog) \
HUF_DTable DTable[HUF_DTABLE_SIZE(maxTableLog)] = { ((U32)(maxTableLog) * 0x01000001) }
/* The workspace must have alignment at least 4 and be at least this large */ /* The workspace must have alignment at least 4 and be at least this large */
#define HUF_WORKSPACE_SIZE (6 << 10) #define HUF_WORKSPACE_SIZE (6 << 10)
#define HUF_WORKSPACE_SIZE_U32 (HUF_WORKSPACE_SIZE / sizeof(U32)) #define HUF_WORKSPACE_SIZE_U32 (HUF_WORKSPACE_SIZE / sizeof(U32))
/* **************************************** /* ****************************************
* Advanced decompression functions * Advanced decompression functions
******************************************/ ******************************************/
size_t HUF_decompress4X_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< decodes RLE and uncompressed */ size_t HUF_decompress4X_DCtx(HUF_DTable *dctx, void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize); /**< decodes RLE and uncompressed */
size_t HUF_decompress4X_hufOnly(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< considers RLE and uncompressed as errors */ size_t HUF_decompress4X_hufOnly(HUF_DTable *dctx, void *dst, size_t dstSize, const void *cSrc,
size_t HUF_decompress4X2_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< single-symbol decoder */ size_t cSrcSize); /**< considers RLE and uncompressed as errors */
size_t HUF_decompress4X4_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< double-symbols decoder */ size_t HUF_decompress4X2_DCtx(HUF_DTable *dctx, void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize); /**< single-symbol decoder */
size_t HUF_decompress4X4_DCtx(HUF_DTable *dctx, void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize); /**< double-symbols decoder */
/* **************************************** /* ****************************************
* HUF detailed API * HUF detailed API
@@ -130,40 +121,40 @@ or to save and regenerate 'CTable' using external methods.
/* FSE_count() : find it within "fse.h" */ /* FSE_count() : find it within "fse.h" */
unsigned HUF_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue); unsigned HUF_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue);
typedef struct HUF_CElt_s HUF_CElt; /* incomplete type */ typedef struct HUF_CElt_s HUF_CElt; /* incomplete type */
size_t HUF_writeCTable (void* dst, size_t maxDstSize, const HUF_CElt* CTable, unsigned maxSymbolValue, unsigned huffLog); size_t HUF_writeCTable(void *dst, size_t maxDstSize, const HUF_CElt *CTable, unsigned maxSymbolValue, unsigned huffLog);
size_t HUF_compress4X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable); size_t HUF_compress4X_usingCTable(void *dst, size_t dstSize, const void *src, size_t srcSize, const HUF_CElt *CTable);
typedef enum { typedef enum {
HUF_repeat_none, /**< Cannot use the previous table */ HUF_repeat_none, /**< Cannot use the previous table */
HUF_repeat_check, /**< Can use the previous table but it must be checked. Note : The previous table must have been constructed by HUF_compress{1, 4}X_repeat */ HUF_repeat_check, /**< Can use the previous table but it must be checked. Note : The previous table must have been constructed by HUF_compress{1,
4}X_repeat */
HUF_repeat_valid /**< Can use the previous table and it is asumed to be valid */ HUF_repeat_valid /**< Can use the previous table and it is asumed to be valid */
} HUF_repeat; } HUF_repeat;
/** HUF_compress4X_repeat() : /** HUF_compress4X_repeat() :
* Same as HUF_compress4X_wksp(), but considers using hufTable if *repeat != HUF_repeat_none. * Same as HUF_compress4X_wksp(), but considers using hufTable if *repeat != HUF_repeat_none.
* If it uses hufTable it does not modify hufTable or repeat. * If it uses hufTable it does not modify hufTable or repeat.
* If it doesn't, it sets *repeat = HUF_repeat_none, and it sets hufTable to the table used. * If it doesn't, it sets *repeat = HUF_repeat_none, and it sets hufTable to the table used.
* If preferRepeat then the old table will always be used if valid. */ * If preferRepeat then the old table will always be used if valid. */
size_t HUF_compress4X_repeat(void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize, HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat); /**< `workSpace` must be a table of at least HUF_WORKSPACE_SIZE_U32 unsigned */ size_t HUF_compress4X_repeat(void *dst, size_t dstSize, const void *src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void *workSpace,
size_t wkspSize, HUF_CElt *hufTable, HUF_repeat *repeat,
int preferRepeat); /**< `workSpace` must be a table of at least HUF_WORKSPACE_SIZE_U32 unsigned */
/** HUF_buildCTable_wksp() : /** HUF_buildCTable_wksp() :
* Same as HUF_buildCTable(), but using externally allocated scratch buffer. * Same as HUF_buildCTable(), but using externally allocated scratch buffer.
* `workSpace` must be aligned on 4-bytes boundaries, and be at least as large as a table of 1024 unsigned. * `workSpace` must be aligned on 4-bytes boundaries, and be at least as large as a table of 1024 unsigned.
*/ */
size_t HUF_buildCTable_wksp (HUF_CElt* tree, const U32* count, U32 maxSymbolValue, U32 maxNbBits, void* workSpace, size_t wkspSize); size_t HUF_buildCTable_wksp(HUF_CElt *tree, const U32 *count, U32 maxSymbolValue, U32 maxNbBits, void *workSpace, size_t wkspSize);
/*! HUF_readStats() : /*! HUF_readStats() :
Read compact Huffman tree, saved by HUF_writeCTable(). Read compact Huffman tree, saved by HUF_writeCTable().
`huffWeight` is destination buffer. `huffWeight` is destination buffer.
@return : size read from `src` , or an error Code . @return : size read from `src` , or an error Code .
Note : Needed by HUF_readCTable() and HUF_readDTableXn() . */ Note : Needed by HUF_readCTable() and HUF_readDTableXn() . */
size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats, size_t HUF_readStats(BYTE *huffWeight, size_t hwSize, U32 *rankStats, U32 *nbSymbolsPtr, U32 *tableLogPtr, const void *src, size_t srcSize);
U32* nbSymbolsPtr, U32* tableLogPtr,
const void* src, size_t srcSize);
/** HUF_readCTable() : /** HUF_readCTable() :
* Loading a CTable saved with HUF_writeCTable() */ * Loading a CTable saved with HUF_writeCTable() */
size_t HUF_readCTable (HUF_CElt* CTable, unsigned maxSymbolValue, const void* src, size_t srcSize); size_t HUF_readCTable(HUF_CElt *CTable, unsigned maxSymbolValue, const void *src, size_t srcSize);
/* /*
HUF_decompress() does the following: HUF_decompress() does the following:
@@ -177,33 +168,36 @@ HUF_decompress() does the following:
* based on a set of pre-determined metrics. * based on a set of pre-determined metrics.
* @return : 0==HUF_decompress4X2, 1==HUF_decompress4X4 . * @return : 0==HUF_decompress4X2, 1==HUF_decompress4X4 .
* Assumption : 0 < cSrcSize < dstSize <= 128 KB */ * Assumption : 0 < cSrcSize < dstSize <= 128 KB */
U32 HUF_selectDecoder (size_t dstSize, size_t cSrcSize); U32 HUF_selectDecoder(size_t dstSize, size_t cSrcSize);
size_t HUF_readDTableX2 (HUF_DTable* DTable, const void* src, size_t srcSize); size_t HUF_readDTableX2(HUF_DTable *DTable, const void *src, size_t srcSize);
size_t HUF_readDTableX4 (HUF_DTable* DTable, const void* src, size_t srcSize); size_t HUF_readDTableX4(HUF_DTable *DTable, const void *src, size_t srcSize);
size_t HUF_decompress4X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable);
size_t HUF_decompress4X2_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable);
size_t HUF_decompress4X4_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable);
size_t HUF_decompress4X_usingDTable(void *dst, size_t maxDstSize, const void *cSrc, size_t cSrcSize, const HUF_DTable *DTable);
size_t HUF_decompress4X2_usingDTable(void *dst, size_t maxDstSize, const void *cSrc, size_t cSrcSize, const HUF_DTable *DTable);
size_t HUF_decompress4X4_usingDTable(void *dst, size_t maxDstSize, const void *cSrc, size_t cSrcSize, const HUF_DTable *DTable);
/* single stream variants */ /* single stream variants */
size_t HUF_compress1X_wksp (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize); /**< `workSpace` must be a table of at least HUF_WORKSPACE_SIZE_U32 unsigned */ size_t HUF_compress1X_wksp(void *dst, size_t dstSize, const void *src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void *workSpace,
size_t HUF_compress1X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable); size_t wkspSize); /**< `workSpace` must be a table of at least HUF_WORKSPACE_SIZE_U32 unsigned */
size_t HUF_compress1X_usingCTable(void *dst, size_t dstSize, const void *src, size_t srcSize, const HUF_CElt *CTable);
/** HUF_compress1X_repeat() : /** HUF_compress1X_repeat() :
* Same as HUF_compress1X_wksp(), but considers using hufTable if *repeat != HUF_repeat_none. * Same as HUF_compress1X_wksp(), but considers using hufTable if *repeat != HUF_repeat_none.
* If it uses hufTable it does not modify hufTable or repeat. * If it uses hufTable it does not modify hufTable or repeat.
* If it doesn't, it sets *repeat = HUF_repeat_none, and it sets hufTable to the table used. * If it doesn't, it sets *repeat = HUF_repeat_none, and it sets hufTable to the table used.
* If preferRepeat then the old table will always be used if valid. */ * If preferRepeat then the old table will always be used if valid. */
size_t HUF_compress1X_repeat(void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize, HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat); /**< `workSpace` must be a table of at least HUF_WORKSPACE_SIZE_U32 unsigned */ size_t HUF_compress1X_repeat(void *dst, size_t dstSize, const void *src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void *workSpace,
size_t wkspSize, HUF_CElt *hufTable, HUF_repeat *repeat,
int preferRepeat); /**< `workSpace` must be a table of at least HUF_WORKSPACE_SIZE_U32 unsigned */
size_t HUF_decompress1X_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); size_t HUF_decompress1X_DCtx(HUF_DTable *dctx, void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize);
size_t HUF_decompress1X2_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< single-symbol decoder */ size_t HUF_decompress1X2_DCtx(HUF_DTable *dctx, void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize); /**< single-symbol decoder */
size_t HUF_decompress1X4_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< double-symbols decoder */ size_t HUF_decompress1X4_DCtx(HUF_DTable *dctx, void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize); /**< double-symbols decoder */
size_t HUF_decompress1X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable); /**< automatic selection of sing or double symbol decoder, based on DTable */ size_t HUF_decompress1X_usingDTable(void *dst, size_t maxDstSize, const void *cSrc, size_t cSrcSize,
size_t HUF_decompress1X2_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable); const HUF_DTable *DTable); /**< automatic selection of sing or double symbol decoder, based on DTable */
size_t HUF_decompress1X4_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable); size_t HUF_decompress1X2_usingDTable(void *dst, size_t maxDstSize, const void *cSrc, size_t cSrcSize, const HUF_DTable *DTable);
size_t HUF_decompress1X4_usingDTable(void *dst, size_t maxDstSize, const void *cSrc, size_t cSrcSize, const HUF_DTable *DTable);
#endif /* HUF_H_298734234 */ #endif /* HUF_H_298734234 */
contrib/linux-kernel/lib/zstd/huf_compress.c
+332 -249
View File
@@ -37,23 +37,29 @@
* - Source repository : https://github.com/Cyan4973/FiniteStateEntropy * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy
*/ */
/* ************************************************************** /* **************************************************************
* Includes * Includes
****************************************************************/ ****************************************************************/
#include <linux/string.h> /* memcpy, memset */
#include "bitstream.h" #include "bitstream.h"
#include "fse.h" /* header compression */ #include "fse.h" /* header compression */
#include "huf.h" #include "huf.h"
#include <linux/string.h> /* memcpy, memset */
/* ************************************************************** /* **************************************************************
* Error Management * Error Management
****************************************************************/ ****************************************************************/
#define HUF_STATIC_ASSERT(c) { enum { HUF_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */ #define HUF_STATIC_ASSERT(c) \
#define CHECK_V_F(e, f) size_t const e = f; if (ERR_isError(e)) return f { \
#define CHECK_F(f) { CHECK_V_F(_var_err__, f); } enum { HUF_static_assert = 1 / (int)(!!(c)) }; \
} /* use only *after* variable declarations */
#define CHECK_V_F(e, f) \
size_t const e = f; \
if (ERR_isError(e)) \
return f
#define CHECK_F(f) \
{ \
CHECK_V_F(_var_err__, f); \
}
/* ************************************************************** /* **************************************************************
* Utils * Utils
@@ -63,7 +69,6 @@ unsigned HUF_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxS
return FSE_optimalTableLog_internal(maxTableLog, srcSize, maxSymbolValue, 1); return FSE_optimalTableLog_internal(maxTableLog, srcSize, maxSymbolValue, 1);
} }
/* ******************************************************* /* *******************************************************
* HUF : Huffman block compression * HUF : Huffman block compression
*********************************************************/ *********************************************************/
@@ -73,49 +78,55 @@ unsigned HUF_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxS
* Note : all elements within weightTable are supposed to be <= HUF_TABLELOG_MAX. * Note : all elements within weightTable are supposed to be <= HUF_TABLELOG_MAX.
*/ */
#define MAX_FSE_TABLELOG_FOR_HUFF_HEADER 6 #define MAX_FSE_TABLELOG_FOR_HUFF_HEADER 6
size_t HUF_compressWeights (void* dst, size_t dstSize, const void* weightTable, size_t wtSize) size_t HUF_compressWeights(void *dst, size_t dstSize, const void *weightTable, size_t wtSize)
{ {
BYTE* const ostart = (BYTE*) dst; BYTE *const ostart = (BYTE *)dst;
BYTE* op = ostart; BYTE *op = ostart;
BYTE* const oend = ostart + dstSize; BYTE *const oend = ostart + dstSize;
U32 maxSymbolValue = HUF_TABLELOG_MAX; U32 maxSymbolValue = HUF_TABLELOG_MAX;
U32 tableLog = MAX_FSE_TABLELOG_FOR_HUFF_HEADER; U32 tableLog = MAX_FSE_TABLELOG_FOR_HUFF_HEADER;
FSE_CTable CTable[FSE_CTABLE_SIZE_U32(MAX_FSE_TABLELOG_FOR_HUFF_HEADER, HUF_TABLELOG_MAX)]; FSE_CTable CTable[FSE_CTABLE_SIZE_U32(MAX_FSE_TABLELOG_FOR_HUFF_HEADER, HUF_TABLELOG_MAX)];
BYTE scratchBuffer[1<<MAX_FSE_TABLELOG_FOR_HUFF_HEADER]; BYTE scratchBuffer[1 << MAX_FSE_TABLELOG_FOR_HUFF_HEADER];
U32 count[HUF_TABLELOG_MAX+1]; U32 count[HUF_TABLELOG_MAX + 1];
S16 norm[HUF_TABLELOG_MAX+1]; S16 norm[HUF_TABLELOG_MAX + 1];
/* init conditions */ /* init conditions */
if (wtSize <= 1) return 0; /* Not compressible */ if (wtSize <= 1)
return 0; /* Not compressible */
/* Scan input and build symbol stats */ /* Scan input and build symbol stats */
{ CHECK_V_F(maxCount, FSE_count_simple(count, &maxSymbolValue, weightTable, wtSize) ); {
if (maxCount == wtSize) return 1; /* only a single symbol in src : rle */ CHECK_V_F(maxCount, FSE_count_simple(count, &maxSymbolValue, weightTable, wtSize));
if (maxCount == 1) return 0; /* each symbol present maximum once => not compressible */ if (maxCount == wtSize)
return 1; /* only a single symbol in src : rle */
if (maxCount == 1)
return 0; /* each symbol present maximum once => not compressible */
} }
tableLog = FSE_optimalTableLog(tableLog, wtSize, maxSymbolValue); tableLog = FSE_optimalTableLog(tableLog, wtSize, maxSymbolValue);
CHECK_F( FSE_normalizeCount(norm, tableLog, count, wtSize, maxSymbolValue) ); CHECK_F(FSE_normalizeCount(norm, tableLog, count, wtSize, maxSymbolValue));
/* Write table description header */ /* Write table description header */
{ CHECK_V_F(hSize, FSE_writeNCount(op, oend-op, norm, maxSymbolValue, tableLog) ); {
CHECK_V_F(hSize, FSE_writeNCount(op, oend - op, norm, maxSymbolValue, tableLog));
op += hSize; op += hSize;
} }
/* Compress */ /* Compress */
CHECK_F( FSE_buildCTable_wksp(CTable, norm, maxSymbolValue, tableLog, scratchBuffer, sizeof(scratchBuffer)) ); CHECK_F(FSE_buildCTable_wksp(CTable, norm, maxSymbolValue, tableLog, scratchBuffer, sizeof(scratchBuffer)));
{ CHECK_V_F(cSize, FSE_compress_usingCTable(op, oend - op, weightTable, wtSize, CTable) ); {
if (cSize == 0) return 0; /* not enough space for compressed data */ CHECK_V_F(cSize, FSE_compress_usingCTable(op, oend - op, weightTable, wtSize, CTable));
if (cSize == 0)
return 0; /* not enough space for compressed data */
op += cSize; op += cSize;
} }
return op-ostart; return op - ostart;
} }
struct HUF_CElt_s { struct HUF_CElt_s {
U16 val; U16 val;
BYTE nbBits; BYTE nbBits;
@@ -124,43 +135,46 @@ struct HUF_CElt_s {
/*! HUF_writeCTable() : /*! HUF_writeCTable() :
`CTable` : Huffman tree to save, using huf representation. `CTable` : Huffman tree to save, using huf representation.
@return : size of saved CTable */ @return : size of saved CTable */
size_t HUF_writeCTable (void* dst, size_t maxDstSize, size_t HUF_writeCTable(void *dst, size_t maxDstSize, const HUF_CElt *CTable, U32 maxSymbolValue, U32 huffLog)
const HUF_CElt* CTable, U32 maxSymbolValue, U32 huffLog)
{ {
BYTE bitsToWeight[HUF_TABLELOG_MAX + 1]; /* precomputed conversion table */ BYTE bitsToWeight[HUF_TABLELOG_MAX + 1]; /* precomputed conversion table */
BYTE huffWeight[HUF_SYMBOLVALUE_MAX]; BYTE huffWeight[HUF_SYMBOLVALUE_MAX];
BYTE* op = (BYTE*)dst; BYTE *op = (BYTE *)dst;
U32 n; U32 n;
/* check conditions */ /* check conditions */
if (maxSymbolValue > HUF_SYMBOLVALUE_MAX) return ERROR(maxSymbolValue_tooLarge); if (maxSymbolValue > HUF_SYMBOLVALUE_MAX)
return ERROR(maxSymbolValue_tooLarge);
/* convert to weight */ /* convert to weight */
bitsToWeight[0] = 0; bitsToWeight[0] = 0;
for (n=1; n<huffLog+1; n++) for (n = 1; n < huffLog + 1; n++)
bitsToWeight[n] = (BYTE)(huffLog + 1 - n); bitsToWeight[n] = (BYTE)(huffLog + 1 - n);
for (n=0; n<maxSymbolValue; n++) for (n = 0; n < maxSymbolValue; n++)
huffWeight[n] = bitsToWeight[CTable[n].nbBits]; huffWeight[n] = bitsToWeight[CTable[n].nbBits];
/* attempt weights compression by FSE */ /* attempt weights compression by FSE */
{ CHECK_V_F(hSize, HUF_compressWeights(op+1, maxDstSize-1, huffWeight, maxSymbolValue) ); {
if ((hSize>1) & (hSize < maxSymbolValue/2)) { /* FSE compressed */ CHECK_V_F(hSize, HUF_compressWeights(op + 1, maxDstSize - 1, huffWeight, maxSymbolValue));
if ((hSize > 1) & (hSize < maxSymbolValue / 2)) { /* FSE compressed */
op[0] = (BYTE)hSize; op[0] = (BYTE)hSize;
return hSize+1; return hSize + 1;
} } }
}
/* write raw values as 4-bits (max : 15) */ /* write raw values as 4-bits (max : 15) */
if (maxSymbolValue > (256-128)) return ERROR(GENERIC); /* should not happen : likely means source cannot be compressed */ if (maxSymbolValue > (256 - 128))
if (((maxSymbolValue+1)/2) + 1 > maxDstSize) return ERROR(dstSize_tooSmall); /* not enough space within dst buffer */ return ERROR(GENERIC); /* should not happen : likely means source cannot be compressed */
op[0] = (BYTE)(128 /*special case*/ + (maxSymbolValue-1)); if (((maxSymbolValue + 1) / 2) + 1 > maxDstSize)
return ERROR(dstSize_tooSmall); /* not enough space within dst buffer */
op[0] = (BYTE)(128 /*special case*/ + (maxSymbolValue - 1));
huffWeight[maxSymbolValue] = 0; /* to be sure it doesn't cause msan issue in final combination */ huffWeight[maxSymbolValue] = 0; /* to be sure it doesn't cause msan issue in final combination */
for (n=0; n<maxSymbolValue; n+=2) for (n = 0; n < maxSymbolValue; n += 2)
op[(n/2)+1] = (BYTE)((huffWeight[n] << 4) + huffWeight[n+1]); op[(n / 2) + 1] = (BYTE)((huffWeight[n] << 4) + huffWeight[n + 1]);
return ((maxSymbolValue+1)/2) + 1; return ((maxSymbolValue + 1) / 2) + 1;
} }
size_t HUF_readCTable(HUF_CElt *CTable, U32 maxSymbolValue, const void *src, size_t srcSize)
size_t HUF_readCTable (HUF_CElt* CTable, U32 maxSymbolValue, const void* src, size_t srcSize)
{ {
BYTE huffWeight[HUF_SYMBOLVALUE_MAX + 1]; /* init not required, even though some static analyzer may complain */ BYTE huffWeight[HUF_SYMBOLVALUE_MAX + 1]; /* init not required, even though some static analyzer may complain */
U32 rankVal[HUF_TABLELOG_ABSOLUTEMAX + 1]; /* large enough for values from 0 to 16 */ U32 rankVal[HUF_TABLELOG_ABSOLUTEMAX + 1]; /* large enough for values from 0 to 16 */
@@ -168,46 +182,64 @@ size_t HUF_readCTable (HUF_CElt* CTable, U32 maxSymbolValue, const void* src, si
U32 nbSymbols = 0; U32 nbSymbols = 0;
/* get symbol weights */ /* get symbol weights */
CHECK_V_F(readSize, HUF_readStats(huffWeight, HUF_SYMBOLVALUE_MAX+1, rankVal, &nbSymbols, &tableLog, src, srcSize)); CHECK_V_F(readSize, HUF_readStats(huffWeight, HUF_SYMBOLVALUE_MAX + 1, rankVal, &nbSymbols, &tableLog, src, srcSize));
/* check result */ /* check result */
if (tableLog > HUF_TABLELOG_MAX) return ERROR(tableLog_tooLarge); if (tableLog > HUF_TABLELOG_MAX)
if (nbSymbols > maxSymbolValue+1) return ERROR(maxSymbolValue_tooSmall); return ERROR(tableLog_tooLarge);
if (nbSymbols > maxSymbolValue + 1)
return ERROR(maxSymbolValue_tooSmall);
/* Prepare base value per rank */ /* Prepare base value per rank */
{ U32 n, nextRankStart = 0; {
for (n=1; n<=tableLog; n++) { U32 n, nextRankStart = 0;
for (n = 1; n <= tableLog; n++) {
U32 curr = nextRankStart; U32 curr = nextRankStart;
nextRankStart += (rankVal[n] << (n-1)); nextRankStart += (rankVal[n] << (n - 1));
rankVal[n] = curr; rankVal[n] = curr;
} } }
}
/* fill nbBits */ /* fill nbBits */
{ U32 n; for (n=0; n<nbSymbols; n++) { {
U32 n;
for (n = 0; n < nbSymbols; n++) {
const U32 w = huffWeight[n]; const U32 w = huffWeight[n];
CTable[n].nbBits = (BYTE)(tableLog + 1 - w); CTable[n].nbBits = (BYTE)(tableLog + 1 - w);
} } }
}
/* fill val */ /* fill val */
{ U16 nbPerRank[HUF_TABLELOG_MAX+2] = {0}; /* support w=0=>n=tableLog+1 */ {
U16 valPerRank[HUF_TABLELOG_MAX+2] = {0}; U16 nbPerRank[HUF_TABLELOG_MAX + 2] = {0}; /* support w=0=>n=tableLog+1 */
{ U32 n; for (n=0; n<nbSymbols; n++) nbPerRank[CTable[n].nbBits]++; } U16 valPerRank[HUF_TABLELOG_MAX + 2] = {0};
{
U32 n;
for (n = 0; n < nbSymbols; n++)
nbPerRank[CTable[n].nbBits]++;
}
/* determine stating value per rank */ /* determine stating value per rank */
valPerRank[tableLog+1] = 0; /* for w==0 */ valPerRank[tableLog + 1] = 0; /* for w==0 */
{ U16 min = 0; {
U32 n; for (n=tableLog; n>0; n--) { /* start at n=tablelog <-> w=1 */ U16 min = 0;
U32 n;
for (n = tableLog; n > 0; n--) { /* start at n=tablelog <-> w=1 */
valPerRank[n] = min; /* get starting value within each rank */ valPerRank[n] = min; /* get starting value within each rank */
min += nbPerRank[n]; min += nbPerRank[n];
min >>= 1; min >>= 1;
} } }
}
/* assign value within rank, symbol order */ /* assign value within rank, symbol order */
{ U32 n; for (n=0; n<=maxSymbolValue; n++) CTable[n].val = valPerRank[CTable[n].nbBits]++; } {
U32 n;
for (n = 0; n <= maxSymbolValue; n++)
CTable[n].val = valPerRank[CTable[n].nbBits]++;
}
} }
return readSize; return readSize;
} }
typedef struct nodeElt_s { typedef struct nodeElt_s {
U32 count; U32 count;
U16 parent; U16 parent;
@@ -215,130 +247,153 @@ typedef struct nodeElt_s {
BYTE nbBits; BYTE nbBits;
} nodeElt; } nodeElt;
static U32 HUF_setMaxHeight(nodeElt* huffNode, U32 lastNonNull, U32 maxNbBits) static U32 HUF_setMaxHeight(nodeElt *huffNode, U32 lastNonNull, U32 maxNbBits)
{ {
const U32 largestBits = huffNode[lastNonNull].nbBits; const U32 largestBits = huffNode[lastNonNull].nbBits;
if (largestBits <= maxNbBits) return largestBits; /* early exit : no elt > maxNbBits */ if (largestBits <= maxNbBits)
return largestBits; /* early exit : no elt > maxNbBits */
/* there are several too large elements (at least >= 2) */ /* there are several too large elements (at least >= 2) */
{ int totalCost = 0; {
int totalCost = 0;
const U32 baseCost = 1 << (largestBits - maxNbBits); const U32 baseCost = 1 << (largestBits - maxNbBits);
U32 n = lastNonNull; U32 n = lastNonNull;
while (huffNode[n].nbBits > maxNbBits) { while (huffNode[n].nbBits > maxNbBits) {
totalCost += baseCost - (1 << (largestBits - huffNode[n].nbBits)); totalCost += baseCost - (1 << (largestBits - huffNode[n].nbBits));
huffNode[n].nbBits = (BYTE)maxNbBits; huffNode[n].nbBits = (BYTE)maxNbBits;
n --; n--;
} /* n stops at huffNode[n].nbBits <= maxNbBits */ } /* n stops at huffNode[n].nbBits <= maxNbBits */
while (huffNode[n].nbBits == maxNbBits) n--; /* n end at index of smallest symbol using < maxNbBits */ while (huffNode[n].nbBits == maxNbBits)
n--; /* n end at index of smallest symbol using < maxNbBits */
/* renorm totalCost */ /* renorm totalCost */
totalCost >>= (largestBits - maxNbBits); /* note : totalCost is necessarily a multiple of baseCost */ totalCost >>= (largestBits - maxNbBits); /* note : totalCost is necessarily a multiple of baseCost */
/* repay normalized cost */ /* repay normalized cost */
{ U32 const noSymbol = 0xF0F0F0F0; {
U32 rankLast[HUF_TABLELOG_MAX+2]; U32 const noSymbol = 0xF0F0F0F0;
U32 rankLast[HUF_TABLELOG_MAX + 2];
int pos; int pos;
/* Get pos of last (smallest) symbol per rank */ /* Get pos of last (smallest) symbol per rank */
memset(rankLast, 0xF0, sizeof(rankLast)); memset(rankLast, 0xF0, sizeof(rankLast));
{ U32 currNbBits = maxNbBits; {
for (pos=n ; pos >= 0; pos--) { U32 currNbBits = maxNbBits;
if (huffNode[pos].nbBits >= currNbBits) continue; for (pos = n; pos >= 0; pos--) {
if (huffNode[pos].nbBits >= currNbBits)
continue;
currNbBits = huffNode[pos].nbBits; /* < maxNbBits */ currNbBits = huffNode[pos].nbBits; /* < maxNbBits */
rankLast[maxNbBits-currNbBits] = pos; rankLast[maxNbBits - currNbBits] = pos;
} } }
}
while (totalCost > 0) { while (totalCost > 0) {
U32 nBitsToDecrease = BIT_highbit32(totalCost) + 1; U32 nBitsToDecrease = BIT_highbit32(totalCost) + 1;
for ( ; nBitsToDecrease > 1; nBitsToDecrease--) { for (; nBitsToDecrease > 1; nBitsToDecrease--) {
U32 highPos = rankLast[nBitsToDecrease]; U32 highPos = rankLast[nBitsToDecrease];
U32 lowPos = rankLast[nBitsToDecrease-1]; U32 lowPos = rankLast[nBitsToDecrease - 1];
if (highPos == noSymbol) continue; if (highPos == noSymbol)
if (lowPos == noSymbol) break; continue;
{ U32 const highTotal = huffNode[highPos].count; if (lowPos == noSymbol)
break;
{
U32 const highTotal = huffNode[highPos].count;
U32 const lowTotal = 2 * huffNode[lowPos].count; U32 const lowTotal = 2 * huffNode[lowPos].count;
if (highTotal <= lowTotal) break; if (highTotal <= lowTotal)
} } break;
}
}
/* only triggered when no more rank 1 symbol left => find closest one (note : there is necessarily at least one !) */ /* only triggered when no more rank 1 symbol left => find closest one (note : there is necessarily at least one !) */
while ((nBitsToDecrease<=HUF_TABLELOG_MAX) && (rankLast[nBitsToDecrease] == noSymbol)) /* HUF_MAX_TABLELOG test just to please gcc 5+; but it should not be necessary */ while (
nBitsToDecrease ++; (nBitsToDecrease <= HUF_TABLELOG_MAX) &&
totalCost -= 1 << (nBitsToDecrease-1); (rankLast[nBitsToDecrease] == noSymbol)) /* HUF_MAX_TABLELOG test just to please gcc 5+; but it should not be necessary */
if (rankLast[nBitsToDecrease-1] == noSymbol) nBitsToDecrease++;
rankLast[nBitsToDecrease-1] = rankLast[nBitsToDecrease]; /* this rank is no longer empty */ totalCost -= 1 << (nBitsToDecrease - 1);
huffNode[rankLast[nBitsToDecrease]].nbBits ++; if (rankLast[nBitsToDecrease - 1] == noSymbol)
rankLast[nBitsToDecrease - 1] = rankLast[nBitsToDecrease]; /* this rank is no longer empty */
huffNode[rankLast[nBitsToDecrease]].nbBits++;
if (rankLast[nBitsToDecrease] == 0) /* special case, reached largest symbol */ if (rankLast[nBitsToDecrease] == 0) /* special case, reached largest symbol */
rankLast[nBitsToDecrease] = noSymbol; rankLast[nBitsToDecrease] = noSymbol;
else { else {
rankLast[nBitsToDecrease]--; rankLast[nBitsToDecrease]--;
if (huffNode[rankLast[nBitsToDecrease]].nbBits != maxNbBits-nBitsToDecrease) if (huffNode[rankLast[nBitsToDecrease]].nbBits != maxNbBits - nBitsToDecrease)
rankLast[nBitsToDecrease] = noSymbol; /* this rank is now empty */ rankLast[nBitsToDecrease] = noSymbol; /* this rank is now empty */
} } /* while (totalCost > 0) */ }
} /* while (totalCost > 0) */
while (totalCost < 0) { /* Sometimes, cost correction overshoot */ while (totalCost < 0) { /* Sometimes, cost correction overshoot */
if (rankLast[1] == noSymbol) { /* special case : no rank 1 symbol (using maxNbBits-1); let's create one from largest rank 0 (using maxNbBits) */ if (rankLast[1] == noSymbol) { /* special case : no rank 1 symbol (using maxNbBits-1); let's create one from largest rank 0
while (huffNode[n].nbBits == maxNbBits) n--; (using maxNbBits) */
huffNode[n+1].nbBits--; while (huffNode[n].nbBits == maxNbBits)
rankLast[1] = n+1; n--;
huffNode[n + 1].nbBits--;
rankLast[1] = n + 1;
totalCost++; totalCost++;
continue; continue;
} }
huffNode[ rankLast[1] + 1 ].nbBits--; huffNode[rankLast[1] + 1].nbBits--;
rankLast[1]++; rankLast[1]++;
totalCost ++; totalCost++;
} } } /* there are several too large elements (at least >= 2) */ }
}
} /* there are several too large elements (at least >= 2) */
return maxNbBits; return maxNbBits;
} }
typedef struct { typedef struct {
U32 base; U32 base;
U32 curr; U32 curr;
} rankPos; } rankPos;
static void HUF_sort(nodeElt* huffNode, const U32* count, U32 maxSymbolValue) static void HUF_sort(nodeElt *huffNode, const U32 *count, U32 maxSymbolValue)
{ {
rankPos rank[32]; rankPos rank[32];
U32 n; U32 n;
memset(rank, 0, sizeof(rank)); memset(rank, 0, sizeof(rank));
for (n=0; n<=maxSymbolValue; n++) { for (n = 0; n <= maxSymbolValue; n++) {
U32 r = BIT_highbit32(count[n] + 1); U32 r = BIT_highbit32(count[n] + 1);
rank[r].base ++; rank[r].base++;
} }
for (n=30; n>0; n--) rank[n-1].base += rank[n].base; for (n = 30; n > 0; n--)
for (n=0; n<32; n++) rank[n].curr = rank[n].base; rank[n - 1].base += rank[n].base;
for (n=0; n<=maxSymbolValue; n++) { for (n = 0; n < 32; n++)
rank[n].curr = rank[n].base;
for (n = 0; n <= maxSymbolValue; n++) {
U32 const c = count[n]; U32 const c = count[n];
U32 const r = BIT_highbit32(c+1) + 1; U32 const r = BIT_highbit32(c + 1) + 1;
U32 pos = rank[r].curr++; U32 pos = rank[r].curr++;
while ((pos > rank[r].base) && (c > huffNode[pos-1].count)) huffNode[pos]=huffNode[pos-1], pos--; while ((pos > rank[r].base) && (c > huffNode[pos - 1].count))
huffNode[pos] = huffNode[pos - 1], pos--;
huffNode[pos].count = c; huffNode[pos].count = c;
huffNode[pos].byte = (BYTE)n; huffNode[pos].byte = (BYTE)n;
} }
} }
/** HUF_buildCTable_wksp() : /** HUF_buildCTable_wksp() :
* Same as HUF_buildCTable(), but using externally allocated scratch buffer. * Same as HUF_buildCTable(), but using externally allocated scratch buffer.
* `workSpace` must be aligned on 4-bytes boundaries, and be at least as large as a table of 1024 unsigned. * `workSpace` must be aligned on 4-bytes boundaries, and be at least as large as a table of 1024 unsigned.
*/ */
#define STARTNODE (HUF_SYMBOLVALUE_MAX+1) #define STARTNODE (HUF_SYMBOLVALUE_MAX + 1)
typedef nodeElt huffNodeTable[2*HUF_SYMBOLVALUE_MAX+1 +1]; typedef nodeElt huffNodeTable[2 * HUF_SYMBOLVALUE_MAX + 1 + 1];
size_t HUF_buildCTable_wksp (HUF_CElt* tree, const U32* count, U32 maxSymbolValue, U32 maxNbBits, void* workSpace, size_t wkspSize) size_t HUF_buildCTable_wksp(HUF_CElt *tree, const U32 *count, U32 maxSymbolValue, U32 maxNbBits, void *workSpace, size_t wkspSize)
{ {
nodeElt* const huffNode0 = (nodeElt*)workSpace; nodeElt *const huffNode0 = (nodeElt *)workSpace;
nodeElt* const huffNode = huffNode0+1; nodeElt *const huffNode = huffNode0 + 1;
U32 n, nonNullRank; U32 n, nonNullRank;
int lowS, lowN; int lowS, lowN;
U16 nodeNb = STARTNODE; U16 nodeNb = STARTNODE;
U32 nodeRoot; U32 nodeRoot;
/* safety checks */ /* safety checks */
if (wkspSize < sizeof(huffNodeTable)) return ERROR(GENERIC); /* workSpace is not large enough */ if (wkspSize < sizeof(huffNodeTable))
if (maxNbBits == 0) maxNbBits = HUF_TABLELOG_DEFAULT; return ERROR(GENERIC); /* workSpace is not large enough */
if (maxSymbolValue > HUF_SYMBOLVALUE_MAX) return ERROR(GENERIC); if (maxNbBits == 0)
maxNbBits = HUF_TABLELOG_DEFAULT;
if (maxSymbolValue > HUF_SYMBOLVALUE_MAX)
return ERROR(GENERIC);
memset(huffNode0, 0, sizeof(huffNodeTable)); memset(huffNode0, 0, sizeof(huffNodeTable));
/* sort, decreasing order */ /* sort, decreasing order */
@@ -346,13 +401,18 @@ size_t HUF_buildCTable_wksp (HUF_CElt* tree, const U32* count, U32 maxSymbolValu
/* init for parents */ /* init for parents */
nonNullRank = maxSymbolValue; nonNullRank = maxSymbolValue;
while(huffNode[nonNullRank].count == 0) nonNullRank--; while (huffNode[nonNullRank].count == 0)
lowS = nonNullRank; nodeRoot = nodeNb + lowS - 1; lowN = nodeNb; nonNullRank--;
huffNode[nodeNb].count = huffNode[lowS].count + huffNode[lowS-1].count; lowS = nonNullRank;
huffNode[lowS].parent = huffNode[lowS-1].parent = nodeNb; nodeRoot = nodeNb + lowS - 1;
nodeNb++; lowS-=2; lowN = nodeNb;
for (n=nodeNb; n<=nodeRoot; n++) huffNode[n].count = (U32)(1U<<30); huffNode[nodeNb].count = huffNode[lowS].count + huffNode[lowS - 1].count;
huffNode0[0].count = (U32)(1U<<31); /* fake entry, strong barrier */ huffNode[lowS].parent = huffNode[lowS - 1].parent = nodeNb;
nodeNb++;
lowS -= 2;
for (n = nodeNb; n <= nodeRoot; n++)
huffNode[n].count = (U32)(1U << 30);
huffNode0[0].count = (U32)(1U << 31); /* fake entry, strong barrier */
/* create parents */ /* create parents */
while (nodeNb <= nodeRoot) { while (nodeNb <= nodeRoot) {
@@ -365,37 +425,41 @@ size_t HUF_buildCTable_wksp (HUF_CElt* tree, const U32* count, U32 maxSymbolValu
/* distribute weights (unlimited tree height) */ /* distribute weights (unlimited tree height) */
huffNode[nodeRoot].nbBits = 0; huffNode[nodeRoot].nbBits = 0;
for (n=nodeRoot-1; n>=STARTNODE; n--) for (n = nodeRoot - 1; n >= STARTNODE; n--)
huffNode[n].nbBits = huffNode[ huffNode[n].parent ].nbBits + 1; huffNode[n].nbBits = huffNode[huffNode[n].parent].nbBits + 1;
for (n=0; n<=nonNullRank; n++) for (n = 0; n <= nonNullRank; n++)
huffNode[n].nbBits = huffNode[ huffNode[n].parent ].nbBits + 1; huffNode[n].nbBits = huffNode[huffNode[n].parent].nbBits + 1;
/* enforce maxTableLog */ /* enforce maxTableLog */
maxNbBits = HUF_setMaxHeight(huffNode, nonNullRank, maxNbBits); maxNbBits = HUF_setMaxHeight(huffNode, nonNullRank, maxNbBits);
/* fill result into tree (val, nbBits) */ /* fill result into tree (val, nbBits) */
{ U16 nbPerRank[HUF_TABLELOG_MAX+1] = {0}; {
U16 valPerRank[HUF_TABLELOG_MAX+1] = {0}; U16 nbPerRank[HUF_TABLELOG_MAX + 1] = {0};
if (maxNbBits > HUF_TABLELOG_MAX) return ERROR(GENERIC); /* check fit into table */ U16 valPerRank[HUF_TABLELOG_MAX + 1] = {0};
for (n=0; n<=nonNullRank; n++) if (maxNbBits > HUF_TABLELOG_MAX)
return ERROR(GENERIC); /* check fit into table */
for (n = 0; n <= nonNullRank; n++)
nbPerRank[huffNode[n].nbBits]++; nbPerRank[huffNode[n].nbBits]++;
/* determine stating value per rank */ /* determine stating value per rank */
{ U16 min = 0; {
for (n=maxNbBits; n>0; n--) { U16 min = 0;
for (n = maxNbBits; n > 0; n--) {
valPerRank[n] = min; /* get starting value within each rank */ valPerRank[n] = min; /* get starting value within each rank */
min += nbPerRank[n]; min += nbPerRank[n];
min >>= 1; min >>= 1;
} } }
for (n=0; n<=maxSymbolValue; n++) }
for (n = 0; n <= maxSymbolValue; n++)
tree[huffNode[n].byte].nbBits = huffNode[n].nbBits; /* push nbBits per symbol, symbol order */ tree[huffNode[n].byte].nbBits = huffNode[n].nbBits; /* push nbBits per symbol, symbol order */
for (n=0; n<=maxSymbolValue; n++) for (n = 0; n <= maxSymbolValue; n++)
tree[n].val = valPerRank[tree[n].nbBits]++; /* assign value within rank, symbol order */ tree[n].val = valPerRank[tree[n].nbBits]++; /* assign value within rank, symbol order */
} }
return maxNbBits; return maxNbBits;
} }
static size_t HUF_estimateCompressedSize(HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue) static size_t HUF_estimateCompressedSize(HUF_CElt *CTable, const unsigned *count, unsigned maxSymbolValue)
{ {
size_t nbBits = 0; size_t nbBits = 0;
int s; int s;
@@ -405,7 +469,8 @@ static size_t HUF_estimateCompressedSize(HUF_CElt* CTable, const unsigned* count
return nbBits >> 3; return nbBits >> 3;
} }
static int HUF_validateCTable(const HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue) { static int HUF_validateCTable(const HUF_CElt *CTable, const unsigned *count, unsigned maxSymbolValue)
{
int bad = 0; int bad = 0;
int s; int s;
for (s = 0; s <= (int)maxSymbolValue; ++s) { for (s = 0; s <= (int)maxSymbolValue; ++s) {
@@ -414,7 +479,7 @@ static int HUF_validateCTable(const HUF_CElt* CTable, const unsigned* count, uns
return !bad; return !bad;
} }
static void HUF_encodeSymbol(BIT_CStream_t* bitCPtr, U32 symbol, const HUF_CElt* CTable) static void HUF_encodeSymbol(BIT_CStream_t *bitCPtr, U32 symbol, const HUF_CElt *CTable)
{ {
BIT_addBitsFast(bitCPtr, CTable[symbol].val, CTable[symbol].nbBits); BIT_addBitsFast(bitCPtr, CTable[symbol].val, CTable[symbol].nbBits);
} }
@@ -424,146 +489,160 @@ size_t HUF_compressBound(size_t size) { return HUF_COMPRESSBOUND(size); }
#define HUF_FLUSHBITS(s) (fast ? BIT_flushBitsFast(s) : BIT_flushBits(s)) #define HUF_FLUSHBITS(s) (fast ? BIT_flushBitsFast(s) : BIT_flushBits(s))
#define HUF_FLUSHBITS_1(stream) \ #define HUF_FLUSHBITS_1(stream) \
if (sizeof((stream)->bitContainer)*8 < HUF_TABLELOG_MAX*2+7) HUF_FLUSHBITS(stream) if (sizeof((stream)->bitContainer) * 8 < HUF_TABLELOG_MAX * 2 + 7) \
HUF_FLUSHBITS(stream)
#define HUF_FLUSHBITS_2(stream) \ #define HUF_FLUSHBITS_2(stream) \
if (sizeof((stream)->bitContainer)*8 < HUF_TABLELOG_MAX*4+7) HUF_FLUSHBITS(stream) if (sizeof((stream)->bitContainer) * 8 < HUF_TABLELOG_MAX * 4 + 7) \
HUF_FLUSHBITS(stream)
size_t HUF_compress1X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable) size_t HUF_compress1X_usingCTable(void *dst, size_t dstSize, const void *src, size_t srcSize, const HUF_CElt *CTable)
{ {
const BYTE* ip = (const BYTE*) src; const BYTE *ip = (const BYTE *)src;
BYTE* const ostart = (BYTE*)dst; BYTE *const ostart = (BYTE *)dst;
BYTE* const oend = ostart + dstSize; BYTE *const oend = ostart + dstSize;
BYTE* op = ostart; BYTE *op = ostart;
size_t n; size_t n;
const unsigned fast = (dstSize >= HUF_BLOCKBOUND(srcSize)); const unsigned fast = (dstSize >= HUF_BLOCKBOUND(srcSize));
BIT_CStream_t bitC; BIT_CStream_t bitC;
/* init */ /* init */
if (dstSize < 8) return 0; /* not enough space to compress */ if (dstSize < 8)
{ size_t const initErr = BIT_initCStream(&bitC, op, oend-op); return 0; /* not enough space to compress */
if (HUF_isError(initErr)) return 0; }
n = srcSize & ~3; /* join to mod 4 */
switch (srcSize & 3)
{ {
case 3 : HUF_encodeSymbol(&bitC, ip[n+ 2], CTable); size_t const initErr = BIT_initCStream(&bitC, op, oend - op);
HUF_FLUSHBITS_2(&bitC); if (HUF_isError(initErr))
case 2 : HUF_encodeSymbol(&bitC, ip[n+ 1], CTable); return 0;
HUF_FLUSHBITS_1(&bitC);
case 1 : HUF_encodeSymbol(&bitC, ip[n+ 0], CTable);
HUF_FLUSHBITS(&bitC);
case 0 :
default: ;
} }
for (; n>0; n-=4) { /* note : n&3==0 at this stage */ n = srcSize & ~3; /* join to mod 4 */
HUF_encodeSymbol(&bitC, ip[n- 1], CTable); switch (srcSize & 3) {
case 3: HUF_encodeSymbol(&bitC, ip[n + 2], CTable); HUF_FLUSHBITS_2(&bitC);
case 2: HUF_encodeSymbol(&bitC, ip[n + 1], CTable); HUF_FLUSHBITS_1(&bitC);
case 1: HUF_encodeSymbol(&bitC, ip[n + 0], CTable); HUF_FLUSHBITS(&bitC);
case 0:
default:;
}
for (; n > 0; n -= 4) { /* note : n&3==0 at this stage */
HUF_encodeSymbol(&bitC, ip[n - 1], CTable);
HUF_FLUSHBITS_1(&bitC); HUF_FLUSHBITS_1(&bitC);
HUF_encodeSymbol(&bitC, ip[n- 2], CTable); HUF_encodeSymbol(&bitC, ip[n - 2], CTable);
HUF_FLUSHBITS_2(&bitC); HUF_FLUSHBITS_2(&bitC);
HUF_encodeSymbol(&bitC, ip[n- 3], CTable); HUF_encodeSymbol(&bitC, ip[n - 3], CTable);
HUF_FLUSHBITS_1(&bitC); HUF_FLUSHBITS_1(&bitC);
HUF_encodeSymbol(&bitC, ip[n- 4], CTable); HUF_encodeSymbol(&bitC, ip[n - 4], CTable);
HUF_FLUSHBITS(&bitC); HUF_FLUSHBITS(&bitC);
} }
return BIT_closeCStream(&bitC); return BIT_closeCStream(&bitC);
} }
size_t HUF_compress4X_usingCTable(void *dst, size_t dstSize, const void *src, size_t srcSize, const HUF_CElt *CTable)
size_t HUF_compress4X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable)
{ {
size_t const segmentSize = (srcSize+3)/4; /* first 3 segments */ size_t const segmentSize = (srcSize + 3) / 4; /* first 3 segments */
const BYTE* ip = (const BYTE*) src; const BYTE *ip = (const BYTE *)src;
const BYTE* const iend = ip + srcSize; const BYTE *const iend = ip + srcSize;
BYTE* const ostart = (BYTE*) dst; BYTE *const ostart = (BYTE *)dst;
BYTE* const oend = ostart + dstSize; BYTE *const oend = ostart + dstSize;
BYTE* op = ostart; BYTE *op = ostart;
if (dstSize < 6 + 1 + 1 + 1 + 8) return 0; /* minimum space to compress successfully */ if (dstSize < 6 + 1 + 1 + 1 + 8)
if (srcSize < 12) return 0; /* no saving possible : too small input */ return 0; /* minimum space to compress successfully */
if (srcSize < 12)
return 0; /* no saving possible : too small input */
op += 6; /* jumpTable */ op += 6; /* jumpTable */
{ CHECK_V_F(cSize, HUF_compress1X_usingCTable(op, oend-op, ip, segmentSize, CTable) ); {
if (cSize==0) return 0; CHECK_V_F(cSize, HUF_compress1X_usingCTable(op, oend - op, ip, segmentSize, CTable));
if (cSize == 0)
return 0;
ZSTD_writeLE16(ostart, (U16)cSize); ZSTD_writeLE16(ostart, (U16)cSize);
op += cSize; op += cSize;
} }
ip += segmentSize; ip += segmentSize;
{ CHECK_V_F(cSize, HUF_compress1X_usingCTable(op, oend-op, ip, segmentSize, CTable) ); {
if (cSize==0) return 0; CHECK_V_F(cSize, HUF_compress1X_usingCTable(op, oend - op, ip, segmentSize, CTable));
ZSTD_writeLE16(ostart+2, (U16)cSize); if (cSize == 0)
return 0;
ZSTD_writeLE16(ostart + 2, (U16)cSize);
op += cSize; op += cSize;
} }
ip += segmentSize; ip += segmentSize;
{ CHECK_V_F(cSize, HUF_compress1X_usingCTable(op, oend-op, ip, segmentSize, CTable) ); {
if (cSize==0) return 0; CHECK_V_F(cSize, HUF_compress1X_usingCTable(op, oend - op, ip, segmentSize, CTable));
ZSTD_writeLE16(ostart+4, (U16)cSize); if (cSize == 0)
return 0;
ZSTD_writeLE16(ostart + 4, (U16)cSize);
op += cSize; op += cSize;
} }
ip += segmentSize; ip += segmentSize;
{ CHECK_V_F(cSize, HUF_compress1X_usingCTable(op, oend-op, ip, iend-ip, CTable) ); {
if (cSize==0) return 0; CHECK_V_F(cSize, HUF_compress1X_usingCTable(op, oend - op, ip, iend - ip, CTable));
if (cSize == 0)
return 0;
op += cSize; op += cSize;
} }
return op-ostart; return op - ostart;
} }
static size_t HUF_compressCTable_internal(BYTE *const ostart, BYTE *op, BYTE *const oend, const void *src, size_t srcSize, unsigned singleStream,
static size_t HUF_compressCTable_internal( const HUF_CElt *CTable)
BYTE* const ostart, BYTE* op, BYTE* const oend,
const void* src, size_t srcSize,
unsigned singleStream, const HUF_CElt* CTable)
{ {
size_t const cSize = singleStream ? size_t const cSize =
HUF_compress1X_usingCTable(op, oend - op, src, srcSize, CTable) : singleStream ? HUF_compress1X_usingCTable(op, oend - op, src, srcSize, CTable) : HUF_compress4X_usingCTable(op, oend - op, src, srcSize, CTable);
HUF_compress4X_usingCTable(op, oend - op, src, srcSize, CTable); if (HUF_isError(cSize)) {
if (HUF_isError(cSize)) { return cSize; } return cSize;
if (cSize==0) { return 0; } /* uncompressible */ }
if (cSize == 0) {
return 0;
} /* uncompressible */
op += cSize; op += cSize;
/* check compressibility */ /* check compressibility */
if ((size_t)(op-ostart) >= srcSize-1) { return 0; } if ((size_t)(op - ostart) >= srcSize - 1) {
return op-ostart; return 0;
}
return op - ostart;
} }
/* `workSpace` must a table of at least 1024 unsigned */ /* `workSpace` must a table of at least 1024 unsigned */
static size_t HUF_compress_internal ( static size_t HUF_compress_internal(void *dst, size_t dstSize, const void *src, size_t srcSize, unsigned maxSymbolValue, unsigned huffLog,
void* dst, size_t dstSize, unsigned singleStream, void *workSpace, size_t wkspSize, HUF_CElt *oldHufTable, HUF_repeat *repeat, int preferRepeat)
const void* src, size_t srcSize,
unsigned maxSymbolValue, unsigned huffLog,
unsigned singleStream,
void* workSpace, size_t wkspSize,
HUF_CElt* oldHufTable, HUF_repeat* repeat, int preferRepeat)
{ {
BYTE* const ostart = (BYTE*)dst; BYTE *const ostart = (BYTE *)dst;
BYTE* const oend = ostart + dstSize; BYTE *const oend = ostart + dstSize;
BYTE* op = ostart; BYTE *op = ostart;
U32* count; U32 *count;
size_t const countSize = sizeof(U32) * (HUF_SYMBOLVALUE_MAX + 1); size_t const countSize = sizeof(U32) * (HUF_SYMBOLVALUE_MAX + 1);
HUF_CElt* CTable; HUF_CElt *CTable;
size_t const CTableSize = sizeof(HUF_CElt) * (HUF_SYMBOLVALUE_MAX + 1); size_t const CTableSize = sizeof(HUF_CElt) * (HUF_SYMBOLVALUE_MAX + 1);
/* checks & inits */ /* checks & inits */
if (wkspSize < sizeof(huffNodeTable) + countSize + CTableSize) return ERROR(GENERIC); if (wkspSize < sizeof(huffNodeTable) + countSize + CTableSize)
if (!srcSize) return 0; /* Uncompressed (note : 1 means rle, so first byte must be correct) */ return ERROR(GENERIC);
if (!dstSize) return 0; /* cannot fit within dst budget */ if (!srcSize)
if (srcSize > HUF_BLOCKSIZE_MAX) return ERROR(srcSize_wrong); /* curr block size limit */ return 0; /* Uncompressed (note : 1 means rle, so first byte must be correct) */
if (huffLog > HUF_TABLELOG_MAX) return ERROR(tableLog_tooLarge); if (!dstSize)
if (!maxSymbolValue) maxSymbolValue = HUF_SYMBOLVALUE_MAX; return 0; /* cannot fit within dst budget */
if (!huffLog) huffLog = HUF_TABLELOG_DEFAULT; if (srcSize > HUF_BLOCKSIZE_MAX)
return ERROR(srcSize_wrong); /* curr block size limit */
if (huffLog > HUF_TABLELOG_MAX)
return ERROR(tableLog_tooLarge);
if (!maxSymbolValue)
maxSymbolValue = HUF_SYMBOLVALUE_MAX;
if (!huffLog)
huffLog = HUF_TABLELOG_DEFAULT;
count = (U32*)workSpace; count = (U32 *)workSpace;
workSpace = (BYTE*)workSpace + countSize; workSpace = (BYTE *)workSpace + countSize;
wkspSize -= countSize; wkspSize -= countSize;
CTable = (HUF_CElt*)workSpace; CTable = (HUF_CElt *)workSpace;
workSpace = (BYTE*)workSpace + CTableSize; workSpace = (BYTE *)workSpace + CTableSize;
wkspSize -= CTableSize; wkspSize -= CTableSize;
/* Heuristic : If we don't need to check the validity of the old table use the old table for small inputs */ /* Heuristic : If we don't need to check the validity of the old table use the old table for small inputs */
@@ -572,9 +651,14 @@ static size_t HUF_compress_internal (
} }
/* Scan input and build symbol stats */ /* Scan input and build symbol stats */
{ CHECK_V_F(largest, FSE_count_wksp (count, &maxSymbolValue, (const BYTE*)src, srcSize, (U32*)workSpace) ); {
if (largest == srcSize) { *ostart = ((const BYTE*)src)[0]; return 1; } /* single symbol, rle */ CHECK_V_F(largest, FSE_count_wksp(count, &maxSymbolValue, (const BYTE *)src, srcSize, (U32 *)workSpace));
if (largest <= (srcSize >> 7)+1) return 0; /* Fast heuristic : not compressible enough */ if (largest == srcSize) {
*ostart = ((const BYTE *)src)[0];
return 1;
} /* single symbol, rle */
if (largest <= (srcSize >> 7) + 1)
return 0; /* Fast heuristic : not compressible enough */
} }
/* Check validity of previous table */ /* Check validity of previous table */
@@ -588,14 +672,16 @@ static size_t HUF_compress_internal (
/* Build Huffman Tree */ /* Build Huffman Tree */
huffLog = HUF_optimalTableLog(huffLog, srcSize, maxSymbolValue); huffLog = HUF_optimalTableLog(huffLog, srcSize, maxSymbolValue);
{ CHECK_V_F(maxBits, HUF_buildCTable_wksp (CTable, count, maxSymbolValue, huffLog, workSpace, wkspSize) ); {
CHECK_V_F(maxBits, HUF_buildCTable_wksp(CTable, count, maxSymbolValue, huffLog, workSpace, wkspSize));
huffLog = (U32)maxBits; huffLog = (U32)maxBits;
/* Zero the unused symbols so we can check it for validity */ /* Zero the unused symbols so we can check it for validity */
memset(CTable + maxSymbolValue + 1, 0, CTableSize - (maxSymbolValue + 1) * sizeof(HUF_CElt)); memset(CTable + maxSymbolValue + 1, 0, CTableSize - (maxSymbolValue + 1) * sizeof(HUF_CElt));
} }
/* Write table description header */ /* Write table description header */
{ CHECK_V_F(hSize, HUF_writeCTable (op, dstSize, CTable, maxSymbolValue, huffLog) ); {
CHECK_V_F(hSize, HUF_writeCTable(op, dstSize, CTable, maxSymbolValue, huffLog));
/* Check if using the previous table will be beneficial */ /* Check if using the previous table will be beneficial */
if (repeat && *repeat != HUF_repeat_none) { if (repeat && *repeat != HUF_repeat_none) {
size_t const oldSize = HUF_estimateCompressedSize(oldHufTable, count, maxSymbolValue); size_t const oldSize = HUF_estimateCompressedSize(oldHufTable, count, maxSymbolValue);
@@ -605,45 +691,42 @@ static size_t HUF_compress_internal (
} }
} }
/* Use the new table */ /* Use the new table */
if (hSize + 12ul >= srcSize) { return 0; } if (hSize + 12ul >= srcSize) {
return 0;
}
op += hSize; op += hSize;
if (repeat) { *repeat = HUF_repeat_none; } if (repeat) {
if (oldHufTable) { memcpy(oldHufTable, CTable, CTableSize); } /* Save the new table */ *repeat = HUF_repeat_none;
}
if (oldHufTable) {
memcpy(oldHufTable, CTable, CTableSize);
} /* Save the new table */
} }
return HUF_compressCTable_internal(ostart, op, oend, src, srcSize, singleStream, CTable); return HUF_compressCTable_internal(ostart, op, oend, src, srcSize, singleStream, CTable);
} }
size_t HUF_compress1X_wksp(void *dst, size_t dstSize, const void *src, size_t srcSize, unsigned maxSymbolValue, unsigned huffLog, void *workSpace,
size_t HUF_compress1X_wksp (void* dst, size_t dstSize, size_t wkspSize)
const void* src, size_t srcSize,
unsigned maxSymbolValue, unsigned huffLog,
void* workSpace, size_t wkspSize)
{ {
return HUF_compress_internal(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, 1 /* single stream */, workSpace, wkspSize, NULL, NULL, 0); return HUF_compress_internal(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, 1 /* single stream */, workSpace, wkspSize, NULL, NULL, 0);
} }
size_t HUF_compress1X_repeat (void* dst, size_t dstSize, size_t HUF_compress1X_repeat(void *dst, size_t dstSize, const void *src, size_t srcSize, unsigned maxSymbolValue, unsigned huffLog, void *workSpace,
const void* src, size_t srcSize, size_t wkspSize, HUF_CElt *hufTable, HUF_repeat *repeat, int preferRepeat)
unsigned maxSymbolValue, unsigned huffLog,
void* workSpace, size_t wkspSize,
HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat)
{ {
return HUF_compress_internal(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, 1 /* single stream */, workSpace, wkspSize, hufTable, repeat, preferRepeat); return HUF_compress_internal(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, 1 /* single stream */, workSpace, wkspSize, hufTable, repeat,
preferRepeat);
} }
size_t HUF_compress4X_wksp (void* dst, size_t dstSize, size_t HUF_compress4X_wksp(void *dst, size_t dstSize, const void *src, size_t srcSize, unsigned maxSymbolValue, unsigned huffLog, void *workSpace,
const void* src, size_t srcSize, size_t wkspSize)
unsigned maxSymbolValue, unsigned huffLog,
void* workSpace, size_t wkspSize)
{ {
return HUF_compress_internal(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, 0 /* 4 streams */, workSpace, wkspSize, NULL, NULL, 0); return HUF_compress_internal(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, 0 /* 4 streams */, workSpace, wkspSize, NULL, NULL, 0);
} }
size_t HUF_compress4X_repeat (void* dst, size_t dstSize, size_t HUF_compress4X_repeat(void *dst, size_t dstSize, const void *src, size_t srcSize, unsigned maxSymbolValue, unsigned huffLog, void *workSpace,
const void* src, size_t srcSize, size_t wkspSize, HUF_CElt *hufTable, HUF_repeat *repeat, int preferRepeat)
unsigned maxSymbolValue, unsigned huffLog,
void* workSpace, size_t wkspSize,
HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat)
{ {
return HUF_compress_internal(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, 0 /* 4 streams */, workSpace, wkspSize, hufTable, repeat, preferRepeat); return HUF_compress_internal(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, 0 /* 4 streams */, workSpace, wkspSize, hufTable, repeat,
preferRepeat);
} }
contrib/linux-kernel/lib/zstd/huf_decompress.c
+396 -315
View File
@@ -42,93 +42,107 @@
****************************************************************/ ****************************************************************/
#define FORCE_INLINE static __always_inline #define FORCE_INLINE static __always_inline
/* ************************************************************** /* **************************************************************
* Dependencies * Dependencies
****************************************************************/ ****************************************************************/
#include <linux/compiler.h>
#include <linux/string.h> /* memcpy, memset */
#include "bitstream.h" /* BIT_* */ #include "bitstream.h" /* BIT_* */
#include "fse.h" /* header compression */ #include "fse.h" /* header compression */
#include "huf.h" #include "huf.h"
#include <linux/compiler.h>
#include <linux/string.h> /* memcpy, memset */
/* ************************************************************** /* **************************************************************
* Error Management * Error Management
****************************************************************/ ****************************************************************/
#define HUF_STATIC_ASSERT(c) { enum { HUF_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */ #define HUF_STATIC_ASSERT(c) \
{ \
enum { HUF_static_assert = 1 / (int)(!!(c)) }; \
} /* use only *after* variable declarations */
/*-***************************/ /*-***************************/
/* generic DTableDesc */ /* generic DTableDesc */
/*-***************************/ /*-***************************/
typedef struct { BYTE maxTableLog; BYTE tableType; BYTE tableLog; BYTE reserved; } DTableDesc; typedef struct {
BYTE maxTableLog;
BYTE tableType;
BYTE tableLog;
BYTE reserved;
} DTableDesc;
static DTableDesc HUF_getDTableDesc(const HUF_DTable* table) static DTableDesc HUF_getDTableDesc(const HUF_DTable *table)
{ {
DTableDesc dtd; DTableDesc dtd;
memcpy(&dtd, table, sizeof(dtd)); memcpy(&dtd, table, sizeof(dtd));
return dtd; return dtd;
} }
/*-***************************/ /*-***************************/
/* single-symbol decoding */ /* single-symbol decoding */
/*-***************************/ /*-***************************/
typedef struct { BYTE byte; BYTE nbBits; } HUF_DEltX2; /* single-symbol decoding */ typedef struct {
BYTE byte;
BYTE nbBits;
} HUF_DEltX2; /* single-symbol decoding */
size_t HUF_readDTableX2 (HUF_DTable* DTable, const void* src, size_t srcSize) size_t HUF_readDTableX2(HUF_DTable *DTable, const void *src, size_t srcSize)
{ {
BYTE huffWeight[HUF_SYMBOLVALUE_MAX + 1]; BYTE huffWeight[HUF_SYMBOLVALUE_MAX + 1];
U32 rankVal[HUF_TABLELOG_ABSOLUTEMAX + 1]; /* large enough for values from 0 to 16 */ U32 rankVal[HUF_TABLELOG_ABSOLUTEMAX + 1]; /* large enough for values from 0 to 16 */
U32 tableLog = 0; U32 tableLog = 0;
U32 nbSymbols = 0; U32 nbSymbols = 0;
size_t iSize; size_t iSize;
void* const dtPtr = DTable + 1; void *const dtPtr = DTable + 1;
HUF_DEltX2* const dt = (HUF_DEltX2*)dtPtr; HUF_DEltX2 *const dt = (HUF_DEltX2 *)dtPtr;
HUF_STATIC_ASSERT(sizeof(DTableDesc) == sizeof(HUF_DTable)); HUF_STATIC_ASSERT(sizeof(DTableDesc) == sizeof(HUF_DTable));
/* memset(huffWeight, 0, sizeof(huffWeight)); */ /* is not necessary, even though some analyzer complain ... */ /* memset(huffWeight, 0, sizeof(huffWeight)); */ /* is not necessary, even though some analyzer complain ... */
iSize = HUF_readStats(huffWeight, HUF_SYMBOLVALUE_MAX + 1, rankVal, &nbSymbols, &tableLog, src, srcSize); iSize = HUF_readStats(huffWeight, HUF_SYMBOLVALUE_MAX + 1, rankVal, &nbSymbols, &tableLog, src, srcSize);
if (HUF_isError(iSize)) return iSize; if (HUF_isError(iSize))
return iSize;
/* Table header */ /* Table header */
{ DTableDesc dtd = HUF_getDTableDesc(DTable); {
if (tableLog > (U32)(dtd.maxTableLog+1)) return ERROR(tableLog_tooLarge); /* DTable too small, Huffman tree cannot fit in */ DTableDesc dtd = HUF_getDTableDesc(DTable);
if (tableLog > (U32)(dtd.maxTableLog + 1))
return ERROR(tableLog_tooLarge); /* DTable too small, Huffman tree cannot fit in */
dtd.tableType = 0; dtd.tableType = 0;
dtd.tableLog = (BYTE)tableLog; dtd.tableLog = (BYTE)tableLog;
memcpy(DTable, &dtd, sizeof(dtd)); memcpy(DTable, &dtd, sizeof(dtd));
} }
/* Calculate starting value for each rank */ /* Calculate starting value for each rank */
{ U32 n, nextRankStart = 0; {
for (n=1; n<tableLog+1; n++) { U32 n, nextRankStart = 0;
for (n = 1; n < tableLog + 1; n++) {
U32 const curr = nextRankStart; U32 const curr = nextRankStart;
nextRankStart += (rankVal[n] << (n-1)); nextRankStart += (rankVal[n] << (n - 1));
rankVal[n] = curr; rankVal[n] = curr;
} } }
}
/* fill DTable */ /* fill DTable */
{ U32 n; {
for (n=0; n<nbSymbols; n++) { U32 n;
for (n = 0; n < nbSymbols; n++) {
U32 const w = huffWeight[n]; U32 const w = huffWeight[n];
U32 const length = (1 << w) >> 1; U32 const length = (1 << w) >> 1;
U32 u; U32 u;
HUF_DEltX2 D; HUF_DEltX2 D;
D.byte = (BYTE)n; D.nbBits = (BYTE)(tableLog + 1 - w); D.byte = (BYTE)n;
D.nbBits = (BYTE)(tableLog + 1 - w);
for (u = rankVal[w]; u < rankVal[w] + length; u++) for (u = rankVal[w]; u < rankVal[w] + length; u++)
dt[u] = D; dt[u] = D;
rankVal[w] += length; rankVal[w] += length;
} } }
}
return iSize; return iSize;
} }
static BYTE HUF_decodeSymbolX2(BIT_DStream_t *Dstream, const HUF_DEltX2 *dt, const U32 dtLog)
static BYTE HUF_decodeSymbolX2(BIT_DStream_t* Dstream, const HUF_DEltX2* dt, const U32 dtLog)
{ {
size_t const val = BIT_lookBitsFast(Dstream, dtLog); /* note : dtLog >= 1 */ size_t const val = BIT_lookBitsFast(Dstream, dtLog); /* note : dtLog >= 1 */
BYTE const c = dt[val].byte; BYTE const c = dt[val].byte;
@@ -136,23 +150,22 @@ static BYTE HUF_decodeSymbolX2(BIT_DStream_t* Dstream, const HUF_DEltX2* dt, con
return c; return c;
} }
#define HUF_DECODE_SYMBOLX2_0(ptr, DStreamPtr) \ #define HUF_DECODE_SYMBOLX2_0(ptr, DStreamPtr) *ptr++ = HUF_decodeSymbolX2(DStreamPtr, dt, dtLog)
*ptr++ = HUF_decodeSymbolX2(DStreamPtr, dt, dtLog)
#define HUF_DECODE_SYMBOLX2_1(ptr, DStreamPtr) \ #define HUF_DECODE_SYMBOLX2_1(ptr, DStreamPtr) \
if (ZSTD_64bits() || (HUF_TABLELOG_MAX<=12)) \ if (ZSTD_64bits() || (HUF_TABLELOG_MAX <= 12)) \
HUF_DECODE_SYMBOLX2_0(ptr, DStreamPtr) HUF_DECODE_SYMBOLX2_0(ptr, DStreamPtr)
#define HUF_DECODE_SYMBOLX2_2(ptr, DStreamPtr) \ #define HUF_DECODE_SYMBOLX2_2(ptr, DStreamPtr) \
if (ZSTD_64bits()) \ if (ZSTD_64bits()) \
HUF_DECODE_SYMBOLX2_0(ptr, DStreamPtr) HUF_DECODE_SYMBOLX2_0(ptr, DStreamPtr)
FORCE_INLINE size_t HUF_decodeStreamX2(BYTE* p, BIT_DStream_t* const bitDPtr, BYTE* const pEnd, const HUF_DEltX2* const dt, const U32 dtLog) FORCE_INLINE size_t HUF_decodeStreamX2(BYTE *p, BIT_DStream_t *const bitDPtr, BYTE *const pEnd, const HUF_DEltX2 *const dt, const U32 dtLog)
{ {
BYTE* const pStart = p; BYTE *const pStart = p;
/* up to 4 symbols at a time */ /* up to 4 symbols at a time */
while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) && (p <= pEnd-4)) { while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) && (p <= pEnd - 4)) {
HUF_DECODE_SYMBOLX2_2(p, bitDPtr); HUF_DECODE_SYMBOLX2_2(p, bitDPtr);
HUF_DECODE_SYMBOLX2_1(p, bitDPtr); HUF_DECODE_SYMBOLX2_1(p, bitDPtr);
HUF_DECODE_SYMBOLX2_2(p, bitDPtr); HUF_DECODE_SYMBOLX2_2(p, bitDPtr);
@@ -167,69 +180,69 @@ FORCE_INLINE size_t HUF_decodeStreamX2(BYTE* p, BIT_DStream_t* const bitDPtr, BY
while (p < pEnd) while (p < pEnd)
HUF_DECODE_SYMBOLX2_0(p, bitDPtr); HUF_DECODE_SYMBOLX2_0(p, bitDPtr);
return pEnd-pStart; return pEnd - pStart;
} }
static size_t HUF_decompress1X2_usingDTable_internal( static size_t HUF_decompress1X2_usingDTable_internal(void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize, const HUF_DTable *DTable)
void* dst, size_t dstSize,
const void* cSrc, size_t cSrcSize,
const HUF_DTable* DTable)
{ {
BYTE* op = (BYTE*)dst; BYTE *op = (BYTE *)dst;
BYTE* const oend = op + dstSize; BYTE *const oend = op + dstSize;
const void* dtPtr = DTable + 1; const void *dtPtr = DTable + 1;
const HUF_DEltX2* const dt = (const HUF_DEltX2*)dtPtr; const HUF_DEltX2 *const dt = (const HUF_DEltX2 *)dtPtr;
BIT_DStream_t bitD; BIT_DStream_t bitD;
DTableDesc const dtd = HUF_getDTableDesc(DTable); DTableDesc const dtd = HUF_getDTableDesc(DTable);
U32 const dtLog = dtd.tableLog; U32 const dtLog = dtd.tableLog;
{ size_t const errorCode = BIT_initDStream(&bitD, cSrc, cSrcSize); {
if (HUF_isError(errorCode)) return errorCode; } size_t const errorCode = BIT_initDStream(&bitD, cSrc, cSrcSize);
if (HUF_isError(errorCode))
return errorCode;
}
HUF_decodeStreamX2(op, &bitD, oend, dt, dtLog); HUF_decodeStreamX2(op, &bitD, oend, dt, dtLog);
/* check */ /* check */
if (!BIT_endOfDStream(&bitD)) return ERROR(corruption_detected); if (!BIT_endOfDStream(&bitD))
return ERROR(corruption_detected);
return dstSize; return dstSize;
} }
size_t HUF_decompress1X2_usingDTable( size_t HUF_decompress1X2_usingDTable(void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize, const HUF_DTable *DTable)
void* dst, size_t dstSize,
const void* cSrc, size_t cSrcSize,
const HUF_DTable* DTable)
{ {
DTableDesc dtd = HUF_getDTableDesc(DTable); DTableDesc dtd = HUF_getDTableDesc(DTable);
if (dtd.tableType != 0) return ERROR(GENERIC); if (dtd.tableType != 0)
return ERROR(GENERIC);
return HUF_decompress1X2_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable); return HUF_decompress1X2_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable);
} }
size_t HUF_decompress1X2_DCtx (HUF_DTable* DCtx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) size_t HUF_decompress1X2_DCtx(HUF_DTable *DCtx, void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize)
{ {
const BYTE* ip = (const BYTE*) cSrc; const BYTE *ip = (const BYTE *)cSrc;
size_t const hSize = HUF_readDTableX2 (DCtx, cSrc, cSrcSize); size_t const hSize = HUF_readDTableX2(DCtx, cSrc, cSrcSize);
if (HUF_isError(hSize)) return hSize; if (HUF_isError(hSize))
if (hSize >= cSrcSize) return ERROR(srcSize_wrong); return hSize;
ip += hSize; cSrcSize -= hSize; if (hSize >= cSrcSize)
return ERROR(srcSize_wrong);
ip += hSize;
cSrcSize -= hSize;
return HUF_decompress1X2_usingDTable_internal (dst, dstSize, ip, cSrcSize, DCtx); return HUF_decompress1X2_usingDTable_internal(dst, dstSize, ip, cSrcSize, DCtx);
} }
static size_t HUF_decompress4X2_usingDTable_internal(void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize, const HUF_DTable *DTable)
static size_t HUF_decompress4X2_usingDTable_internal(
void* dst, size_t dstSize,
const void* cSrc, size_t cSrcSize,
const HUF_DTable* DTable)
{ {
/* Check */ /* Check */
if (cSrcSize < 10) return ERROR(corruption_detected); /* strict minimum : jump table + 1 byte per stream */ if (cSrcSize < 10)
return ERROR(corruption_detected); /* strict minimum : jump table + 1 byte per stream */
{ const BYTE* const istart = (const BYTE*) cSrc; {
BYTE* const ostart = (BYTE*) dst; const BYTE *const istart = (const BYTE *)cSrc;
BYTE* const oend = ostart + dstSize; BYTE *const ostart = (BYTE *)dst;
const void* const dtPtr = DTable + 1; BYTE *const oend = ostart + dstSize;
const HUF_DEltX2* const dt = (const HUF_DEltX2*)dtPtr; const void *const dtPtr = DTable + 1;
const HUF_DEltX2 *const dt = (const HUF_DEltX2 *)dtPtr;
/* Init */ /* Init */
BIT_DStream_t bitD1; BIT_DStream_t bitD1;
@@ -237,38 +250,51 @@ static size_t HUF_decompress4X2_usingDTable_internal(
BIT_DStream_t bitD3; BIT_DStream_t bitD3;
BIT_DStream_t bitD4; BIT_DStream_t bitD4;
size_t const length1 = ZSTD_readLE16(istart); size_t const length1 = ZSTD_readLE16(istart);
size_t const length2 = ZSTD_readLE16(istart+2); size_t const length2 = ZSTD_readLE16(istart + 2);
size_t const length3 = ZSTD_readLE16(istart+4); size_t const length3 = ZSTD_readLE16(istart + 4);
size_t const length4 = cSrcSize - (length1 + length2 + length3 + 6); size_t const length4 = cSrcSize - (length1 + length2 + length3 + 6);
const BYTE* const istart1 = istart + 6; /* jumpTable */ const BYTE *const istart1 = istart + 6; /* jumpTable */
const BYTE* const istart2 = istart1 + length1; const BYTE *const istart2 = istart1 + length1;
const BYTE* const istart3 = istart2 + length2; const BYTE *const istart3 = istart2 + length2;
const BYTE* const istart4 = istart3 + length3; const BYTE *const istart4 = istart3 + length3;
const size_t segmentSize = (dstSize+3) / 4; const size_t segmentSize = (dstSize + 3) / 4;
BYTE* const opStart2 = ostart + segmentSize; BYTE *const opStart2 = ostart + segmentSize;
BYTE* const opStart3 = opStart2 + segmentSize; BYTE *const opStart3 = opStart2 + segmentSize;
BYTE* const opStart4 = opStart3 + segmentSize; BYTE *const opStart4 = opStart3 + segmentSize;
BYTE* op1 = ostart; BYTE *op1 = ostart;
BYTE* op2 = opStart2; BYTE *op2 = opStart2;
BYTE* op3 = opStart3; BYTE *op3 = opStart3;
BYTE* op4 = opStart4; BYTE *op4 = opStart4;
U32 endSignal; U32 endSignal;
DTableDesc const dtd = HUF_getDTableDesc(DTable); DTableDesc const dtd = HUF_getDTableDesc(DTable);
U32 const dtLog = dtd.tableLog; U32 const dtLog = dtd.tableLog;
if (length4 > cSrcSize) return ERROR(corruption_detected); /* overflow */ if (length4 > cSrcSize)
{ size_t const errorCode = BIT_initDStream(&bitD1, istart1, length1); return ERROR(corruption_detected); /* overflow */
if (HUF_isError(errorCode)) return errorCode; } {
{ size_t const errorCode = BIT_initDStream(&bitD2, istart2, length2); size_t const errorCode = BIT_initDStream(&bitD1, istart1, length1);
if (HUF_isError(errorCode)) return errorCode; } if (HUF_isError(errorCode))
{ size_t const errorCode = BIT_initDStream(&bitD3, istart3, length3); return errorCode;
if (HUF_isError(errorCode)) return errorCode; } }
{ size_t const errorCode = BIT_initDStream(&bitD4, istart4, length4); {
if (HUF_isError(errorCode)) return errorCode; } size_t const errorCode = BIT_initDStream(&bitD2, istart2, length2);
if (HUF_isError(errorCode))
return errorCode;
}
{
size_t const errorCode = BIT_initDStream(&bitD3, istart3, length3);
if (HUF_isError(errorCode))
return errorCode;
}
{
size_t const errorCode = BIT_initDStream(&bitD4, istart4, length4);
if (HUF_isError(errorCode))
return errorCode;
}
/* 16-32 symbols per loop (4-8 symbols per stream) */ /* 16-32 symbols per loop (4-8 symbols per stream) */
endSignal = BIT_reloadDStream(&bitD1) | BIT_reloadDStream(&bitD2) | BIT_reloadDStream(&bitD3) | BIT_reloadDStream(&bitD4); endSignal = BIT_reloadDStream(&bitD1) | BIT_reloadDStream(&bitD2) | BIT_reloadDStream(&bitD3) | BIT_reloadDStream(&bitD4);
for ( ; (endSignal==BIT_DStream_unfinished) && (op4<(oend-7)) ; ) { for (; (endSignal == BIT_DStream_unfinished) && (op4 < (oend - 7));) {
HUF_DECODE_SYMBOLX2_2(op1, &bitD1); HUF_DECODE_SYMBOLX2_2(op1, &bitD1);
HUF_DECODE_SYMBOLX2_2(op2, &bitD2); HUF_DECODE_SYMBOLX2_2(op2, &bitD2);
HUF_DECODE_SYMBOLX2_2(op3, &bitD3); HUF_DECODE_SYMBOLX2_2(op3, &bitD3);
@@ -289,9 +315,12 @@ static size_t HUF_decompress4X2_usingDTable_internal(
} }
/* check corruption */ /* check corruption */
if (op1 > opStart2) return ERROR(corruption_detected); if (op1 > opStart2)
if (op2 > opStart3) return ERROR(corruption_detected); return ERROR(corruption_detected);
if (op3 > opStart4) return ERROR(corruption_detected); if (op2 > opStart3)
return ERROR(corruption_detected);
if (op3 > opStart4)
return ERROR(corruption_detected);
/* note : op4 supposed already verified within main loop */ /* note : op4 supposed already verified within main loop */
/* finish bitStreams one by one */ /* finish bitStreams one by one */
@@ -302,50 +331,55 @@ static size_t HUF_decompress4X2_usingDTable_internal(
/* check */ /* check */
endSignal = BIT_endOfDStream(&bitD1) & BIT_endOfDStream(&bitD2) & BIT_endOfDStream(&bitD3) & BIT_endOfDStream(&bitD4); endSignal = BIT_endOfDStream(&bitD1) & BIT_endOfDStream(&bitD2) & BIT_endOfDStream(&bitD3) & BIT_endOfDStream(&bitD4);
if (!endSignal) return ERROR(corruption_detected); if (!endSignal)
return ERROR(corruption_detected);
/* decoded size */ /* decoded size */
return dstSize; return dstSize;
} }
} }
size_t HUF_decompress4X2_usingDTable(void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize, const HUF_DTable *DTable)
size_t HUF_decompress4X2_usingDTable(
void* dst, size_t dstSize,
const void* cSrc, size_t cSrcSize,
const HUF_DTable* DTable)
{ {
DTableDesc dtd = HUF_getDTableDesc(DTable); DTableDesc dtd = HUF_getDTableDesc(DTable);
if (dtd.tableType != 0) return ERROR(GENERIC); if (dtd.tableType != 0)
return ERROR(GENERIC);
return HUF_decompress4X2_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable); return HUF_decompress4X2_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable);
} }
size_t HUF_decompress4X2_DCtx(HUF_DTable *dctx, void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize)
size_t HUF_decompress4X2_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
{ {
const BYTE* ip = (const BYTE*) cSrc; const BYTE *ip = (const BYTE *)cSrc;
size_t const hSize = HUF_readDTableX2 (dctx, cSrc, cSrcSize); size_t const hSize = HUF_readDTableX2(dctx, cSrc, cSrcSize);
if (HUF_isError(hSize)) return hSize; if (HUF_isError(hSize))
if (hSize >= cSrcSize) return ERROR(srcSize_wrong); return hSize;
ip += hSize; cSrcSize -= hSize; if (hSize >= cSrcSize)
return ERROR(srcSize_wrong);
ip += hSize;
cSrcSize -= hSize;
return HUF_decompress4X2_usingDTable_internal (dst, dstSize, ip, cSrcSize, dctx); return HUF_decompress4X2_usingDTable_internal(dst, dstSize, ip, cSrcSize, dctx);
} }
/* *************************/ /* *************************/
/* double-symbols decoding */ /* double-symbols decoding */
/* *************************/ /* *************************/
typedef struct { U16 sequence; BYTE nbBits; BYTE length; } HUF_DEltX4; /* double-symbols decoding */ typedef struct {
U16 sequence;
BYTE nbBits;
BYTE length;
} HUF_DEltX4; /* double-symbols decoding */
typedef struct { BYTE symbol; BYTE weight; } sortedSymbol_t; typedef struct {
BYTE symbol;
BYTE weight;
} sortedSymbol_t;
/* HUF_fillDTableX4Level2() : /* HUF_fillDTableX4Level2() :
* `rankValOrigin` must be a table of at least (HUF_TABLELOG_MAX + 1) U32 */ * `rankValOrigin` must be a table of at least (HUF_TABLELOG_MAX + 1) U32 */
static void HUF_fillDTableX4Level2(HUF_DEltX4* DTable, U32 sizeLog, const U32 consumed, static void HUF_fillDTableX4Level2(HUF_DEltX4 *DTable, U32 sizeLog, const U32 consumed, const U32 *rankValOrigin, const int minWeight,
const U32* rankValOrigin, const int minWeight, const sortedSymbol_t *sortedSymbols, const U32 sortedListSize, U32 nbBitsBaseline, U16 baseSeq)
const sortedSymbol_t* sortedSymbols, const U32 sortedListSize,
U32 nbBitsBaseline, U16 baseSeq)
{ {
HUF_DEltX4 DElt; HUF_DEltX4 DElt;
U32 rankVal[HUF_TABLELOG_MAX + 1]; U32 rankVal[HUF_TABLELOG_MAX + 1];
@@ -354,7 +388,7 @@ static void HUF_fillDTableX4Level2(HUF_DEltX4* DTable, U32 sizeLog, const U32 co
memcpy(rankVal, rankValOrigin, sizeof(rankVal)); memcpy(rankVal, rankValOrigin, sizeof(rankVal));
/* fill skipped values */ /* fill skipped values */
if (minWeight>1) { if (minWeight > 1) {
U32 i, skipSize = rankVal[minWeight]; U32 i, skipSize = rankVal[minWeight];
ZSTD_writeLE16(&(DElt.sequence), baseSeq); ZSTD_writeLE16(&(DElt.sequence), baseSeq);
DElt.nbBits = (BYTE)(consumed); DElt.nbBits = (BYTE)(consumed);
@@ -364,11 +398,13 @@ static void HUF_fillDTableX4Level2(HUF_DEltX4* DTable, U32 sizeLog, const U32 co
} }
/* fill DTable */ /* fill DTable */
{ U32 s; for (s=0; s<sortedListSize; s++) { /* note : sortedSymbols already skipped */ {
U32 s;
for (s = 0; s < sortedListSize; s++) { /* note : sortedSymbols already skipped */
const U32 symbol = sortedSymbols[s].symbol; const U32 symbol = sortedSymbols[s].symbol;
const U32 weight = sortedSymbols[s].weight; const U32 weight = sortedSymbols[s].weight;
const U32 nbBits = nbBitsBaseline - weight; const U32 nbBits = nbBitsBaseline - weight;
const U32 length = 1 << (sizeLog-nbBits); const U32 length = 1 << (sizeLog - nbBits);
const U32 start = rankVal[weight]; const U32 start = rankVal[weight];
U32 i = start; U32 i = start;
const U32 end = start + length; const U32 end = start + length;
@@ -376,18 +412,19 @@ static void HUF_fillDTableX4Level2(HUF_DEltX4* DTable, U32 sizeLog, const U32 co
ZSTD_writeLE16(&(DElt.sequence), (U16)(baseSeq + (symbol << 8))); ZSTD_writeLE16(&(DElt.sequence), (U16)(baseSeq + (symbol << 8)));
DElt.nbBits = (BYTE)(nbBits + consumed); DElt.nbBits = (BYTE)(nbBits + consumed);
DElt.length = 2; DElt.length = 2;
do { DTable[i++] = DElt; } while (i<end); /* since length >= 1 */ do {
DTable[i++] = DElt;
} while (i < end); /* since length >= 1 */
rankVal[weight] += length; rankVal[weight] += length;
} } }
}
} }
typedef U32 rankVal_t[HUF_TABLELOG_MAX][HUF_TABLELOG_MAX + 1]; typedef U32 rankVal_t[HUF_TABLELOG_MAX][HUF_TABLELOG_MAX + 1];
static void HUF_fillDTableX4(HUF_DEltX4* DTable, const U32 targetLog, static void HUF_fillDTableX4(HUF_DEltX4 *DTable, const U32 targetLog, const sortedSymbol_t *sortedList, const U32 sortedListSize, const U32 *rankStart,
const sortedSymbol_t* sortedList, const U32 sortedListSize, rankVal_t rankValOrigin, const U32 maxWeight, const U32 nbBitsBaseline)
const U32* rankStart, rankVal_t rankValOrigin, const U32 maxWeight,
const U32 nbBitsBaseline)
{ {
U32 rankVal[HUF_TABLELOG_MAX + 1]; U32 rankVal[HUF_TABLELOG_MAX + 1];
const int scaleLog = nbBitsBaseline - targetLog; /* note : targetLog >= srcLog, hence scaleLog <= 1 */ const int scaleLog = nbBitsBaseline - targetLog; /* note : targetLog >= srcLog, hence scaleLog <= 1 */
@@ -397,66 +434,73 @@ static void HUF_fillDTableX4(HUF_DEltX4* DTable, const U32 targetLog,
memcpy(rankVal, rankValOrigin, sizeof(rankVal)); memcpy(rankVal, rankValOrigin, sizeof(rankVal));
/* fill DTable */ /* fill DTable */
for (s=0; s<sortedListSize; s++) { for (s = 0; s < sortedListSize; s++) {
const U16 symbol = sortedList[s].symbol; const U16 symbol = sortedList[s].symbol;
const U32 weight = sortedList[s].weight; const U32 weight = sortedList[s].weight;
const U32 nbBits = nbBitsBaseline - weight; const U32 nbBits = nbBitsBaseline - weight;
const U32 start = rankVal[weight]; const U32 start = rankVal[weight];
const U32 length = 1 << (targetLog-nbBits); const U32 length = 1 << (targetLog - nbBits);
if (targetLog-nbBits >= minBits) { /* enough room for a second symbol */ if (targetLog - nbBits >= minBits) { /* enough room for a second symbol */
U32 sortedRank; U32 sortedRank;
int minWeight = nbBits + scaleLog; int minWeight = nbBits + scaleLog;
if (minWeight < 1) minWeight = 1; if (minWeight < 1)
minWeight = 1;
sortedRank = rankStart[minWeight]; sortedRank = rankStart[minWeight];
HUF_fillDTableX4Level2(DTable+start, targetLog-nbBits, nbBits, HUF_fillDTableX4Level2(DTable + start, targetLog - nbBits, nbBits, rankValOrigin[nbBits], minWeight, sortedList + sortedRank,
rankValOrigin[nbBits], minWeight, sortedListSize - sortedRank, nbBitsBaseline, symbol);
sortedList+sortedRank, sortedListSize-sortedRank,
nbBitsBaseline, symbol);
} else { } else {
HUF_DEltX4 DElt; HUF_DEltX4 DElt;
ZSTD_writeLE16(&(DElt.sequence), symbol); ZSTD_writeLE16(&(DElt.sequence), symbol);
DElt.nbBits = (BYTE)(nbBits); DElt.nbBits = (BYTE)(nbBits);
DElt.length = 1; DElt.length = 1;
{ U32 const end = start + length; {
U32 const end = start + length;
U32 u; U32 u;
for (u = start; u < end; u++) DTable[u] = DElt; for (u = start; u < end; u++)
} } DTable[u] = DElt;
}
}
rankVal[weight] += length; rankVal[weight] += length;
} }
} }
size_t HUF_readDTableX4 (HUF_DTable* DTable, const void* src, size_t srcSize) size_t HUF_readDTableX4(HUF_DTable *DTable, const void *src, size_t srcSize)
{ {
BYTE weightList[HUF_SYMBOLVALUE_MAX + 1]; BYTE weightList[HUF_SYMBOLVALUE_MAX + 1];
sortedSymbol_t sortedSymbol[HUF_SYMBOLVALUE_MAX + 1]; sortedSymbol_t sortedSymbol[HUF_SYMBOLVALUE_MAX + 1];
U32 rankStats[HUF_TABLELOG_MAX + 1] = { 0 }; U32 rankStats[HUF_TABLELOG_MAX + 1] = {0};
U32 rankStart0[HUF_TABLELOG_MAX + 2] = { 0 }; U32 rankStart0[HUF_TABLELOG_MAX + 2] = {0};
U32* const rankStart = rankStart0+1; U32 *const rankStart = rankStart0 + 1;
rankVal_t rankVal; rankVal_t rankVal;
U32 tableLog, maxW, sizeOfSort, nbSymbols; U32 tableLog, maxW, sizeOfSort, nbSymbols;
DTableDesc dtd = HUF_getDTableDesc(DTable); DTableDesc dtd = HUF_getDTableDesc(DTable);
U32 const maxTableLog = dtd.maxTableLog; U32 const maxTableLog = dtd.maxTableLog;
size_t iSize; size_t iSize;
void* dtPtr = DTable+1; /* force compiler to avoid strict-aliasing */ void *dtPtr = DTable + 1; /* force compiler to avoid strict-aliasing */
HUF_DEltX4* const dt = (HUF_DEltX4*)dtPtr; HUF_DEltX4 *const dt = (HUF_DEltX4 *)dtPtr;
HUF_STATIC_ASSERT(sizeof(HUF_DEltX4) == sizeof(HUF_DTable)); /* if compiler fails here, assertion is wrong */ HUF_STATIC_ASSERT(sizeof(HUF_DEltX4) == sizeof(HUF_DTable)); /* if compiler fails here, assertion is wrong */
if (maxTableLog > HUF_TABLELOG_MAX) return ERROR(tableLog_tooLarge); if (maxTableLog > HUF_TABLELOG_MAX)
return ERROR(tableLog_tooLarge);
/* memset(weightList, 0, sizeof(weightList)); */ /* is not necessary, even though some analyzer complain ... */ /* memset(weightList, 0, sizeof(weightList)); */ /* is not necessary, even though some analyzer complain ... */
iSize = HUF_readStats(weightList, HUF_SYMBOLVALUE_MAX + 1, rankStats, &nbSymbols, &tableLog, src, srcSize); iSize = HUF_readStats(weightList, HUF_SYMBOLVALUE_MAX + 1, rankStats, &nbSymbols, &tableLog, src, srcSize);
if (HUF_isError(iSize)) return iSize; if (HUF_isError(iSize))
return iSize;
/* check result */ /* check result */
if (tableLog > maxTableLog) return ERROR(tableLog_tooLarge); /* DTable can't fit code depth */ if (tableLog > maxTableLog)
return ERROR(tableLog_tooLarge); /* DTable can't fit code depth */
/* find maxWeight */ /* find maxWeight */
for (maxW = tableLog; rankStats[maxW]==0; maxW--) {} /* necessarily finds a solution before 0 */ for (maxW = tableLog; rankStats[maxW] == 0; maxW--) {
} /* necessarily finds a solution before 0 */
/* Get start index of each weight */ /* Get start index of each weight */
{ U32 w, nextRankStart = 0; {
for (w=1; w<maxW+1; w++) { U32 w, nextRankStart = 0;
for (w = 1; w < maxW + 1; w++) {
U32 curr = nextRankStart; U32 curr = nextRankStart;
nextRankStart += rankStats[w]; nextRankStart += rankStats[w];
rankStart[w] = curr; rankStart[w] = curr;
@@ -466,8 +510,9 @@ size_t HUF_readDTableX4 (HUF_DTable* DTable, const void* src, size_t srcSize)
} }
/* sort symbols by weight */ /* sort symbols by weight */
{ U32 s; {
for (s=0; s<nbSymbols; s++) { U32 s;
for (s = 0; s < nbSymbols; s++) {
U32 const w = weightList[s]; U32 const w = weightList[s];
U32 const r = rankStart[w]++; U32 const r = rankStart[w]++;
sortedSymbol[r].symbol = (BYTE)s; sortedSymbol[r].symbol = (BYTE)s;
@@ -477,28 +522,32 @@ size_t HUF_readDTableX4 (HUF_DTable* DTable, const void* src, size_t srcSize)
} }
/* Build rankVal */ /* Build rankVal */
{ U32* const rankVal0 = rankVal[0]; {
{ int const rescale = (maxTableLog-tableLog) - 1; /* tableLog <= maxTableLog */ U32 *const rankVal0 = rankVal[0];
{
int const rescale = (maxTableLog - tableLog) - 1; /* tableLog <= maxTableLog */
U32 nextRankVal = 0; U32 nextRankVal = 0;
U32 w; U32 w;
for (w=1; w<maxW+1; w++) { for (w = 1; w < maxW + 1; w++) {
U32 curr = nextRankVal; U32 curr = nextRankVal;
nextRankVal += rankStats[w] << (w+rescale); nextRankVal += rankStats[w] << (w + rescale);
rankVal0[w] = curr; rankVal0[w] = curr;
} } }
{ U32 const minBits = tableLog+1 - maxW; }
{
U32 const minBits = tableLog + 1 - maxW;
U32 consumed; U32 consumed;
for (consumed = minBits; consumed < maxTableLog - minBits + 1; consumed++) { for (consumed = minBits; consumed < maxTableLog - minBits + 1; consumed++) {
U32* const rankValPtr = rankVal[consumed]; U32 *const rankValPtr = rankVal[consumed];
U32 w; U32 w;
for (w = 1; w < maxW+1; w++) { for (w = 1; w < maxW + 1; w++) {
rankValPtr[w] = rankVal0[w] >> consumed; rankValPtr[w] = rankVal0[w] >> consumed;
} } } } }
}
}
}
HUF_fillDTableX4(dt, maxTableLog, HUF_fillDTableX4(dt, maxTableLog, sortedSymbol, sizeOfSort, rankStart0, rankVal, maxW, tableLog + 1);
sortedSymbol, sizeOfSort,
rankStart0, rankVal, maxW,
tableLog+1);
dtd.tableLog = (BYTE)maxTableLog; dtd.tableLog = (BYTE)maxTableLog;
dtd.tableType = 1; dtd.tableType = 1;
@@ -506,47 +555,48 @@ size_t HUF_readDTableX4 (HUF_DTable* DTable, const void* src, size_t srcSize)
return iSize; return iSize;
} }
static U32 HUF_decodeSymbolX4(void *op, BIT_DStream_t *DStream, const HUF_DEltX4 *dt, const U32 dtLog)
static U32 HUF_decodeSymbolX4(void* op, BIT_DStream_t* DStream, const HUF_DEltX4* dt, const U32 dtLog)
{ {
size_t const val = BIT_lookBitsFast(DStream, dtLog); /* note : dtLog >= 1 */ size_t const val = BIT_lookBitsFast(DStream, dtLog); /* note : dtLog >= 1 */
memcpy(op, dt+val, 2); memcpy(op, dt + val, 2);
BIT_skipBits(DStream, dt[val].nbBits); BIT_skipBits(DStream, dt[val].nbBits);
return dt[val].length; return dt[val].length;
} }
static U32 HUF_decodeLastSymbolX4(void* op, BIT_DStream_t* DStream, const HUF_DEltX4* dt, const U32 dtLog) static U32 HUF_decodeLastSymbolX4(void *op, BIT_DStream_t *DStream, const HUF_DEltX4 *dt, const U32 dtLog)
{ {
size_t const val = BIT_lookBitsFast(DStream, dtLog); /* note : dtLog >= 1 */ size_t const val = BIT_lookBitsFast(DStream, dtLog); /* note : dtLog >= 1 */
memcpy(op, dt+val, 1); memcpy(op, dt + val, 1);
if (dt[val].length==1) BIT_skipBits(DStream, dt[val].nbBits); if (dt[val].length == 1)
else {
if (DStream->bitsConsumed < (sizeof(DStream->bitContainer)*8)) {
BIT_skipBits(DStream, dt[val].nbBits); BIT_skipBits(DStream, dt[val].nbBits);
if (DStream->bitsConsumed > (sizeof(DStream->bitContainer)*8)) else {
DStream->bitsConsumed = (sizeof(DStream->bitContainer)*8); /* ugly hack; works only because it's the last symbol. Note : can't easily extract nbBits from just this symbol */ if (DStream->bitsConsumed < (sizeof(DStream->bitContainer) * 8)) {
} } BIT_skipBits(DStream, dt[val].nbBits);
if (DStream->bitsConsumed > (sizeof(DStream->bitContainer) * 8))
DStream->bitsConsumed =
(sizeof(DStream->bitContainer) *
8); /* ugly hack; works only because it's the last symbol. Note : can't easily extract nbBits from just this symbol */
}
}
return 1; return 1;
} }
#define HUF_DECODE_SYMBOLX4_0(ptr, DStreamPtr) ptr += HUF_decodeSymbolX4(ptr, DStreamPtr, dt, dtLog)
#define HUF_DECODE_SYMBOLX4_0(ptr, DStreamPtr) \
ptr += HUF_decodeSymbolX4(ptr, DStreamPtr, dt, dtLog)
#define HUF_DECODE_SYMBOLX4_1(ptr, DStreamPtr) \ #define HUF_DECODE_SYMBOLX4_1(ptr, DStreamPtr) \
if (ZSTD_64bits() || (HUF_TABLELOG_MAX<=12)) \ if (ZSTD_64bits() || (HUF_TABLELOG_MAX <= 12)) \
ptr += HUF_decodeSymbolX4(ptr, DStreamPtr, dt, dtLog) ptr += HUF_decodeSymbolX4(ptr, DStreamPtr, dt, dtLog)
#define HUF_DECODE_SYMBOLX4_2(ptr, DStreamPtr) \ #define HUF_DECODE_SYMBOLX4_2(ptr, DStreamPtr) \
if (ZSTD_64bits()) \ if (ZSTD_64bits()) \
ptr += HUF_decodeSymbolX4(ptr, DStreamPtr, dt, dtLog) ptr += HUF_decodeSymbolX4(ptr, DStreamPtr, dt, dtLog)
FORCE_INLINE size_t HUF_decodeStreamX4(BYTE* p, BIT_DStream_t* bitDPtr, BYTE* const pEnd, const HUF_DEltX4* const dt, const U32 dtLog) FORCE_INLINE size_t HUF_decodeStreamX4(BYTE *p, BIT_DStream_t *bitDPtr, BYTE *const pEnd, const HUF_DEltX4 *const dt, const U32 dtLog)
{ {
BYTE* const pStart = p; BYTE *const pStart = p;
/* up to 8 symbols at a time */ /* up to 8 symbols at a time */
while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) & (p < pEnd-(sizeof(bitDPtr->bitContainer)-1))) { while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) & (p < pEnd - (sizeof(bitDPtr->bitContainer) - 1))) {
HUF_DECODE_SYMBOLX4_2(p, bitDPtr); HUF_DECODE_SYMBOLX4_2(p, bitDPtr);
HUF_DECODE_SYMBOLX4_1(p, bitDPtr); HUF_DECODE_SYMBOLX4_1(p, bitDPtr);
HUF_DECODE_SYMBOLX4_2(p, bitDPtr); HUF_DECODE_SYMBOLX4_2(p, bitDPtr);
@@ -554,81 +604,81 @@ FORCE_INLINE size_t HUF_decodeStreamX4(BYTE* p, BIT_DStream_t* bitDPtr, BYTE* co
} }
/* closer to end : up to 2 symbols at a time */ /* closer to end : up to 2 symbols at a time */
while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) & (p <= pEnd-2)) while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) & (p <= pEnd - 2))
HUF_DECODE_SYMBOLX4_0(p, bitDPtr); HUF_DECODE_SYMBOLX4_0(p, bitDPtr);
while (p <= pEnd-2) while (p <= pEnd - 2)
HUF_DECODE_SYMBOLX4_0(p, bitDPtr); /* no need to reload : reached the end of DStream */ HUF_DECODE_SYMBOLX4_0(p, bitDPtr); /* no need to reload : reached the end of DStream */
if (p < pEnd) if (p < pEnd)
p += HUF_decodeLastSymbolX4(p, bitDPtr, dt, dtLog); p += HUF_decodeLastSymbolX4(p, bitDPtr, dt, dtLog);
return p-pStart; return p - pStart;
} }
static size_t HUF_decompress1X4_usingDTable_internal(void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize, const HUF_DTable *DTable)
static size_t HUF_decompress1X4_usingDTable_internal(
void* dst, size_t dstSize,
const void* cSrc, size_t cSrcSize,
const HUF_DTable* DTable)
{ {
BIT_DStream_t bitD; BIT_DStream_t bitD;
/* Init */ /* Init */
{ size_t const errorCode = BIT_initDStream(&bitD, cSrc, cSrcSize); {
if (HUF_isError(errorCode)) return errorCode; size_t const errorCode = BIT_initDStream(&bitD, cSrc, cSrcSize);
if (HUF_isError(errorCode))
return errorCode;
} }
/* decode */ /* decode */
{ BYTE* const ostart = (BYTE*) dst; {
BYTE* const oend = ostart + dstSize; BYTE *const ostart = (BYTE *)dst;
const void* const dtPtr = DTable+1; /* force compiler to not use strict-aliasing */ BYTE *const oend = ostart + dstSize;
const HUF_DEltX4* const dt = (const HUF_DEltX4*)dtPtr; const void *const dtPtr = DTable + 1; /* force compiler to not use strict-aliasing */
const HUF_DEltX4 *const dt = (const HUF_DEltX4 *)dtPtr;
DTableDesc const dtd = HUF_getDTableDesc(DTable); DTableDesc const dtd = HUF_getDTableDesc(DTable);
HUF_decodeStreamX4(ostart, &bitD, oend, dt, dtd.tableLog); HUF_decodeStreamX4(ostart, &bitD, oend, dt, dtd.tableLog);
} }
/* check */ /* check */
if (!BIT_endOfDStream(&bitD)) return ERROR(corruption_detected); if (!BIT_endOfDStream(&bitD))
return ERROR(corruption_detected);
/* decoded size */ /* decoded size */
return dstSize; return dstSize;
} }
size_t HUF_decompress1X4_usingDTable( size_t HUF_decompress1X4_usingDTable(void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize, const HUF_DTable *DTable)
void* dst, size_t dstSize,
const void* cSrc, size_t cSrcSize,
const HUF_DTable* DTable)
{ {
DTableDesc dtd = HUF_getDTableDesc(DTable); DTableDesc dtd = HUF_getDTableDesc(DTable);
if (dtd.tableType != 1) return ERROR(GENERIC); if (dtd.tableType != 1)
return ERROR(GENERIC);
return HUF_decompress1X4_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable); return HUF_decompress1X4_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable);
} }
size_t HUF_decompress1X4_DCtx (HUF_DTable* DCtx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) size_t HUF_decompress1X4_DCtx(HUF_DTable *DCtx, void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize)
{ {
const BYTE* ip = (const BYTE*) cSrc; const BYTE *ip = (const BYTE *)cSrc;
size_t const hSize = HUF_readDTableX4 (DCtx, cSrc, cSrcSize); size_t const hSize = HUF_readDTableX4(DCtx, cSrc, cSrcSize);
if (HUF_isError(hSize)) return hSize; if (HUF_isError(hSize))
if (hSize >= cSrcSize) return ERROR(srcSize_wrong); return hSize;
ip += hSize; cSrcSize -= hSize; if (hSize >= cSrcSize)
return ERROR(srcSize_wrong);
ip += hSize;
cSrcSize -= hSize;
return HUF_decompress1X4_usingDTable_internal (dst, dstSize, ip, cSrcSize, DCtx); return HUF_decompress1X4_usingDTable_internal(dst, dstSize, ip, cSrcSize, DCtx);
} }
static size_t HUF_decompress4X4_usingDTable_internal( static size_t HUF_decompress4X4_usingDTable_internal(void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize, const HUF_DTable *DTable)
void* dst, size_t dstSize,
const void* cSrc, size_t cSrcSize,
const HUF_DTable* DTable)
{ {
if (cSrcSize < 10) return ERROR(corruption_detected); /* strict minimum : jump table + 1 byte per stream */ if (cSrcSize < 10)
return ERROR(corruption_detected); /* strict minimum : jump table + 1 byte per stream */
{ const BYTE* const istart = (const BYTE*) cSrc; {
BYTE* const ostart = (BYTE*) dst; const BYTE *const istart = (const BYTE *)cSrc;
BYTE* const oend = ostart + dstSize; BYTE *const ostart = (BYTE *)dst;
const void* const dtPtr = DTable+1; BYTE *const oend = ostart + dstSize;
const HUF_DEltX4* const dt = (const HUF_DEltX4*)dtPtr; const void *const dtPtr = DTable + 1;
const HUF_DEltX4 *const dt = (const HUF_DEltX4 *)dtPtr;
/* Init */ /* Init */
BIT_DStream_t bitD1; BIT_DStream_t bitD1;
@@ -636,38 +686,51 @@ static size_t HUF_decompress4X4_usingDTable_internal(
BIT_DStream_t bitD3; BIT_DStream_t bitD3;
BIT_DStream_t bitD4; BIT_DStream_t bitD4;
size_t const length1 = ZSTD_readLE16(istart); size_t const length1 = ZSTD_readLE16(istart);
size_t const length2 = ZSTD_readLE16(istart+2); size_t const length2 = ZSTD_readLE16(istart + 2);
size_t const length3 = ZSTD_readLE16(istart+4); size_t const length3 = ZSTD_readLE16(istart + 4);
size_t const length4 = cSrcSize - (length1 + length2 + length3 + 6); size_t const length4 = cSrcSize - (length1 + length2 + length3 + 6);
const BYTE* const istart1 = istart + 6; /* jumpTable */ const BYTE *const istart1 = istart + 6; /* jumpTable */
const BYTE* const istart2 = istart1 + length1; const BYTE *const istart2 = istart1 + length1;
const BYTE* const istart3 = istart2 + length2; const BYTE *const istart3 = istart2 + length2;
const BYTE* const istart4 = istart3 + length3; const BYTE *const istart4 = istart3 + length3;
size_t const segmentSize = (dstSize+3) / 4; size_t const segmentSize = (dstSize + 3) / 4;
BYTE* const opStart2 = ostart + segmentSize; BYTE *const opStart2 = ostart + segmentSize;
BYTE* const opStart3 = opStart2 + segmentSize; BYTE *const opStart3 = opStart2 + segmentSize;
BYTE* const opStart4 = opStart3 + segmentSize; BYTE *const opStart4 = opStart3 + segmentSize;
BYTE* op1 = ostart; BYTE *op1 = ostart;
BYTE* op2 = opStart2; BYTE *op2 = opStart2;
BYTE* op3 = opStart3; BYTE *op3 = opStart3;
BYTE* op4 = opStart4; BYTE *op4 = opStart4;
U32 endSignal; U32 endSignal;
DTableDesc const dtd = HUF_getDTableDesc(DTable); DTableDesc const dtd = HUF_getDTableDesc(DTable);
U32 const dtLog = dtd.tableLog; U32 const dtLog = dtd.tableLog;
if (length4 > cSrcSize) return ERROR(corruption_detected); /* overflow */ if (length4 > cSrcSize)
{ size_t const errorCode = BIT_initDStream(&bitD1, istart1, length1); return ERROR(corruption_detected); /* overflow */
if (HUF_isError(errorCode)) return errorCode; } {
{ size_t const errorCode = BIT_initDStream(&bitD2, istart2, length2); size_t const errorCode = BIT_initDStream(&bitD1, istart1, length1);
if (HUF_isError(errorCode)) return errorCode; } if (HUF_isError(errorCode))
{ size_t const errorCode = BIT_initDStream(&bitD3, istart3, length3); return errorCode;
if (HUF_isError(errorCode)) return errorCode; } }
{ size_t const errorCode = BIT_initDStream(&bitD4, istart4, length4); {
if (HUF_isError(errorCode)) return errorCode; } size_t const errorCode = BIT_initDStream(&bitD2, istart2, length2);
if (HUF_isError(errorCode))
return errorCode;
}
{
size_t const errorCode = BIT_initDStream(&bitD3, istart3, length3);
if (HUF_isError(errorCode))
return errorCode;
}
{
size_t const errorCode = BIT_initDStream(&bitD4, istart4, length4);
if (HUF_isError(errorCode))
return errorCode;
}
/* 16-32 symbols per loop (4-8 symbols per stream) */ /* 16-32 symbols per loop (4-8 symbols per stream) */
endSignal = BIT_reloadDStream(&bitD1) | BIT_reloadDStream(&bitD2) | BIT_reloadDStream(&bitD3) | BIT_reloadDStream(&bitD4); endSignal = BIT_reloadDStream(&bitD1) | BIT_reloadDStream(&bitD2) | BIT_reloadDStream(&bitD3) | BIT_reloadDStream(&bitD4);
for ( ; (endSignal==BIT_DStream_unfinished) & (op4<(oend-(sizeof(bitD4.bitContainer)-1))) ; ) { for (; (endSignal == BIT_DStream_unfinished) & (op4 < (oend - (sizeof(bitD4.bitContainer) - 1)));) {
HUF_DECODE_SYMBOLX4_2(op1, &bitD1); HUF_DECODE_SYMBOLX4_2(op1, &bitD1);
HUF_DECODE_SYMBOLX4_2(op2, &bitD2); HUF_DECODE_SYMBOLX4_2(op2, &bitD2);
HUF_DECODE_SYMBOLX4_2(op3, &bitD3); HUF_DECODE_SYMBOLX4_2(op3, &bitD3);
@@ -689,9 +752,12 @@ static size_t HUF_decompress4X4_usingDTable_internal(
} }
/* check corruption */ /* check corruption */
if (op1 > opStart2) return ERROR(corruption_detected); if (op1 > opStart2)
if (op2 > opStart3) return ERROR(corruption_detected); return ERROR(corruption_detected);
if (op3 > opStart4) return ERROR(corruption_detected); if (op2 > opStart3)
return ERROR(corruption_detected);
if (op3 > opStart4)
return ERROR(corruption_detected);
/* note : op4 already verified within main loop */ /* note : op4 already verified within main loop */
/* finish bitStreams one by one */ /* finish bitStreams one by one */
@@ -701,82 +767,80 @@ static size_t HUF_decompress4X4_usingDTable_internal(
HUF_decodeStreamX4(op4, &bitD4, oend, dt, dtLog); HUF_decodeStreamX4(op4, &bitD4, oend, dt, dtLog);
/* check */ /* check */
{ U32 const endCheck = BIT_endOfDStream(&bitD1) & BIT_endOfDStream(&bitD2) & BIT_endOfDStream(&bitD3) & BIT_endOfDStream(&bitD4); {
if (!endCheck) return ERROR(corruption_detected); } U32 const endCheck = BIT_endOfDStream(&bitD1) & BIT_endOfDStream(&bitD2) & BIT_endOfDStream(&bitD3) & BIT_endOfDStream(&bitD4);
if (!endCheck)
return ERROR(corruption_detected);
}
/* decoded size */ /* decoded size */
return dstSize; return dstSize;
} }
} }
size_t HUF_decompress4X4_usingDTable(void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize, const HUF_DTable *DTable)
size_t HUF_decompress4X4_usingDTable(
void* dst, size_t dstSize,
const void* cSrc, size_t cSrcSize,
const HUF_DTable* DTable)
{ {
DTableDesc dtd = HUF_getDTableDesc(DTable); DTableDesc dtd = HUF_getDTableDesc(DTable);
if (dtd.tableType != 1) return ERROR(GENERIC); if (dtd.tableType != 1)
return ERROR(GENERIC);
return HUF_decompress4X4_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable); return HUF_decompress4X4_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable);
} }
size_t HUF_decompress4X4_DCtx(HUF_DTable *dctx, void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize)
size_t HUF_decompress4X4_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
{ {
const BYTE* ip = (const BYTE*) cSrc; const BYTE *ip = (const BYTE *)cSrc;
size_t hSize = HUF_readDTableX4 (dctx, cSrc, cSrcSize); size_t hSize = HUF_readDTableX4(dctx, cSrc, cSrcSize);
if (HUF_isError(hSize)) return hSize; if (HUF_isError(hSize))
if (hSize >= cSrcSize) return ERROR(srcSize_wrong); return hSize;
ip += hSize; cSrcSize -= hSize; if (hSize >= cSrcSize)
return ERROR(srcSize_wrong);
ip += hSize;
cSrcSize -= hSize;
return HUF_decompress4X4_usingDTable_internal(dst, dstSize, ip, cSrcSize, dctx); return HUF_decompress4X4_usingDTable_internal(dst, dstSize, ip, cSrcSize, dctx);
} }
/* ********************************/ /* ********************************/
/* Generic decompression selector */ /* Generic decompression selector */
/* ********************************/ /* ********************************/
size_t HUF_decompress1X_usingDTable(void* dst, size_t maxDstSize, size_t HUF_decompress1X_usingDTable(void *dst, size_t maxDstSize, const void *cSrc, size_t cSrcSize, const HUF_DTable *DTable)
const void* cSrc, size_t cSrcSize,
const HUF_DTable* DTable)
{ {
DTableDesc const dtd = HUF_getDTableDesc(DTable); DTableDesc const dtd = HUF_getDTableDesc(DTable);
return dtd.tableType ? HUF_decompress1X4_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable) : return dtd.tableType ? HUF_decompress1X4_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable)
HUF_decompress1X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable); : HUF_decompress1X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable);
} }
size_t HUF_decompress4X_usingDTable(void* dst, size_t maxDstSize, size_t HUF_decompress4X_usingDTable(void *dst, size_t maxDstSize, const void *cSrc, size_t cSrcSize, const HUF_DTable *DTable)
const void* cSrc, size_t cSrcSize,
const HUF_DTable* DTable)
{ {
DTableDesc const dtd = HUF_getDTableDesc(DTable); DTableDesc const dtd = HUF_getDTableDesc(DTable);
return dtd.tableType ? HUF_decompress4X4_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable) : return dtd.tableType ? HUF_decompress4X4_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable)
HUF_decompress4X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable); : HUF_decompress4X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable);
} }
typedef struct {
typedef struct { U32 tableTime; U32 decode256Time; } algo_time_t; U32 tableTime;
static const algo_time_t algoTime[16 /* Quantization */][3 /* single, double, quad */] = U32 decode256Time;
{ } algo_time_t;
static const algo_time_t algoTime[16 /* Quantization */][3 /* single, double, quad */] = {
/* single, double, quad */ /* single, double, quad */
{{0,0}, {1,1}, {2,2}}, /* Q==0 : impossible */ {{0, 0}, {1, 1}, {2, 2}}, /* Q==0 : impossible */
{{0,0}, {1,1}, {2,2}}, /* Q==1 : impossible */ {{0, 0}, {1, 1}, {2, 2}}, /* Q==1 : impossible */
{{ 38,130}, {1313, 74}, {2151, 38}}, /* Q == 2 : 12-18% */ {{38, 130}, {1313, 74}, {2151, 38}}, /* Q == 2 : 12-18% */
{{ 448,128}, {1353, 74}, {2238, 41}}, /* Q == 3 : 18-25% */ {{448, 128}, {1353, 74}, {2238, 41}}, /* Q == 3 : 18-25% */
{{ 556,128}, {1353, 74}, {2238, 47}}, /* Q == 4 : 25-32% */ {{556, 128}, {1353, 74}, {2238, 47}}, /* Q == 4 : 25-32% */
{{ 714,128}, {1418, 74}, {2436, 53}}, /* Q == 5 : 32-38% */ {{714, 128}, {1418, 74}, {2436, 53}}, /* Q == 5 : 32-38% */
{{ 883,128}, {1437, 74}, {2464, 61}}, /* Q == 6 : 38-44% */ {{883, 128}, {1437, 74}, {2464, 61}}, /* Q == 6 : 38-44% */
{{ 897,128}, {1515, 75}, {2622, 68}}, /* Q == 7 : 44-50% */ {{897, 128}, {1515, 75}, {2622, 68}}, /* Q == 7 : 44-50% */
{{ 926,128}, {1613, 75}, {2730, 75}}, /* Q == 8 : 50-56% */ {{926, 128}, {1613, 75}, {2730, 75}}, /* Q == 8 : 50-56% */
{{ 947,128}, {1729, 77}, {3359, 77}}, /* Q == 9 : 56-62% */ {{947, 128}, {1729, 77}, {3359, 77}}, /* Q == 9 : 56-62% */
{{1107,128}, {2083, 81}, {4006, 84}}, /* Q ==10 : 62-69% */ {{1107, 128}, {2083, 81}, {4006, 84}}, /* Q ==10 : 62-69% */
{{1177,128}, {2379, 87}, {4785, 88}}, /* Q ==11 : 69-75% */ {{1177, 128}, {2379, 87}, {4785, 88}}, /* Q ==11 : 69-75% */
{{1242,128}, {2415, 93}, {5155, 84}}, /* Q ==12 : 75-81% */ {{1242, 128}, {2415, 93}, {5155, 84}}, /* Q ==12 : 75-81% */
{{1349,128}, {2644,106}, {5260,106}}, /* Q ==13 : 81-87% */ {{1349, 128}, {2644, 106}, {5260, 106}}, /* Q ==13 : 81-87% */
{{1455,128}, {2422,124}, {4174,124}}, /* Q ==14 : 87-93% */ {{1455, 128}, {2422, 124}, {4174, 124}}, /* Q ==14 : 87-93% */
{{ 722,128}, {1891,145}, {1936,146}}, /* Q ==15 : 93-99% */ {{722, 128}, {1891, 145}, {1936, 146}}, /* Q ==15 : 93-99% */
}; };
/** HUF_selectDecoder() : /** HUF_selectDecoder() :
@@ -784,7 +848,7 @@ static const algo_time_t algoTime[16 /* Quantization */][3 /* single, double, qu
* based on a set of pre-determined metrics. * based on a set of pre-determined metrics.
* @return : 0==HUF_decompress4X2, 1==HUF_decompress4X4 . * @return : 0==HUF_decompress4X2, 1==HUF_decompress4X4 .
* Assumption : 0 < cSrcSize < dstSize <= 128 KB */ * Assumption : 0 < cSrcSize < dstSize <= 128 KB */
U32 HUF_selectDecoder (size_t dstSize, size_t cSrcSize) U32 HUF_selectDecoder(size_t dstSize, size_t cSrcSize)
{ {
/* decoder timing evaluation */ /* decoder timing evaluation */
U32 const Q = (U32)(cSrcSize * 16 / dstSize); /* Q < 16 since dstSize > cSrcSize */ U32 const Q = (U32)(cSrcSize * 16 / dstSize); /* Q < 16 since dstSize > cSrcSize */
@@ -796,45 +860,62 @@ U32 HUF_selectDecoder (size_t dstSize, size_t cSrcSize)
return DTime1 < DTime0; return DTime1 < DTime0;
} }
typedef size_t (*decompressionAlgo)(void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize);
typedef size_t (*decompressionAlgo)(void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); size_t HUF_decompress4X_DCtx(HUF_DTable *dctx, void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize)
size_t HUF_decompress4X_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
{ {
/* validation checks */ /* validation checks */
if (dstSize == 0) return ERROR(dstSize_tooSmall); if (dstSize == 0)
if (cSrcSize > dstSize) return ERROR(corruption_detected); /* invalid */ return ERROR(dstSize_tooSmall);
if (cSrcSize == dstSize) { memcpy(dst, cSrc, dstSize); return dstSize; } /* not compressed */ if (cSrcSize > dstSize)
if (cSrcSize == 1) { memset(dst, *(const BYTE*)cSrc, dstSize); return dstSize; } /* RLE */ return ERROR(corruption_detected); /* invalid */
if (cSrcSize == dstSize) {
memcpy(dst, cSrc, dstSize);
return dstSize;
} /* not compressed */
if (cSrcSize == 1) {
memset(dst, *(const BYTE *)cSrc, dstSize);
return dstSize;
} /* RLE */
{ U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize); {
return algoNb ? HUF_decompress4X4_DCtx(dctx, dst, dstSize, cSrc, cSrcSize) : U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize);
HUF_decompress4X2_DCtx(dctx, dst, dstSize, cSrc, cSrcSize) ; return algoNb ? HUF_decompress4X4_DCtx(dctx, dst, dstSize, cSrc, cSrcSize) : HUF_decompress4X2_DCtx(dctx, dst, dstSize, cSrc, cSrcSize);
} }
} }
size_t HUF_decompress4X_hufOnly (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) size_t HUF_decompress4X_hufOnly(HUF_DTable *dctx, void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize)
{ {
/* validation checks */ /* validation checks */
if (dstSize == 0) return ERROR(dstSize_tooSmall); if (dstSize == 0)
if ((cSrcSize >= dstSize) || (cSrcSize <= 1)) return ERROR(corruption_detected); /* invalid */ return ERROR(dstSize_tooSmall);
if ((cSrcSize >= dstSize) || (cSrcSize <= 1))
return ERROR(corruption_detected); /* invalid */
{ U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize); {
return algoNb ? HUF_decompress4X4_DCtx(dctx, dst, dstSize, cSrc, cSrcSize) : U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize);
HUF_decompress4X2_DCtx(dctx, dst, dstSize, cSrc, cSrcSize) ; return algoNb ? HUF_decompress4X4_DCtx(dctx, dst, dstSize, cSrc, cSrcSize) : HUF_decompress4X2_DCtx(dctx, dst, dstSize, cSrc, cSrcSize);
} }
} }
size_t HUF_decompress1X_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) size_t HUF_decompress1X_DCtx(HUF_DTable *dctx, void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize)
{ {
/* validation checks */ /* validation checks */
if (dstSize == 0) return ERROR(dstSize_tooSmall); if (dstSize == 0)
if (cSrcSize > dstSize) return ERROR(corruption_detected); /* invalid */ return ERROR(dstSize_tooSmall);
if (cSrcSize == dstSize) { memcpy(dst, cSrc, dstSize); return dstSize; } /* not compressed */ if (cSrcSize > dstSize)
if (cSrcSize == 1) { memset(dst, *(const BYTE*)cSrc, dstSize); return dstSize; } /* RLE */ return ERROR(corruption_detected); /* invalid */
if (cSrcSize == dstSize) {
memcpy(dst, cSrc, dstSize);
return dstSize;
} /* not compressed */
if (cSrcSize == 1) {
memset(dst, *(const BYTE *)cSrc, dstSize);
return dstSize;
} /* RLE */
{ U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize); {
return algoNb ? HUF_decompress1X4_DCtx(dctx, dst, dstSize, cSrc, cSrcSize) : U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize);
HUF_decompress1X2_DCtx(dctx, dst, dstSize, cSrc, cSrcSize) ; return algoNb ? HUF_decompress1X4_DCtx(dctx, dst, dstSize, cSrc, cSrcSize) : HUF_decompress1X2_DCtx(dctx, dst, dstSize, cSrc, cSrcSize);
} }
} }
contrib/linux-kernel/lib/zstd/mem.h
+38 -99
View File
@@ -21,16 +21,14 @@
* Dependencies * Dependencies
******************************************/ ******************************************/
#include <asm/unaligned.h> #include <asm/unaligned.h>
#include <linux/types.h> /* size_t, ptrdiff_t */
#include <linux/string.h> /* memcpy */ #include <linux/string.h> /* memcpy */
#include <linux/types.h> /* size_t, ptrdiff_t */
/*-**************************************** /*-****************************************
* Compiler specifics * Compiler specifics
******************************************/ ******************************************/
#define ZSTD_STATIC static __inline __attribute__((unused)) #define ZSTD_STATIC static __inline __attribute__((unused))
/*-************************************************************** /*-**************************************************************
* Basic Types * Basic Types
*****************************************************************/ *****************************************************************/
@@ -44,103 +42,57 @@ typedef int64_t S64;
typedef ptrdiff_t iPtrDiff; typedef ptrdiff_t iPtrDiff;
typedef uintptr_t uPtrDiff; typedef uintptr_t uPtrDiff;
/*-************************************************************** /*-**************************************************************
* Memory I/O * Memory I/O
*****************************************************************/ *****************************************************************/
ZSTD_STATIC unsigned ZSTD_32bits(void) { return sizeof(size_t)==4; } ZSTD_STATIC unsigned ZSTD_32bits(void) { return sizeof(size_t) == 4; }
ZSTD_STATIC unsigned ZSTD_64bits(void) { return sizeof(size_t)==8; } ZSTD_STATIC unsigned ZSTD_64bits(void) { return sizeof(size_t) == 8; }
#if defined(__LITTLE_ENDIAN) #if defined(__LITTLE_ENDIAN)
# define ZSTD_LITTLE_ENDIAN 1 #define ZSTD_LITTLE_ENDIAN 1
#else #else
# define ZSTD_LITTLE_ENDIAN 0 #define ZSTD_LITTLE_ENDIAN 0
#endif #endif
ZSTD_STATIC unsigned ZSTD_isLittleEndian(void) ZSTD_STATIC unsigned ZSTD_isLittleEndian(void) { return ZSTD_LITTLE_ENDIAN; }
{
return ZSTD_LITTLE_ENDIAN;
}
ZSTD_STATIC U16 ZSTD_read16(const void* memPtr) ZSTD_STATIC U16 ZSTD_read16(const void *memPtr) { return get_unaligned((const U16 *)memPtr); }
{
return get_unaligned((const U16*)memPtr);
}
ZSTD_STATIC U32 ZSTD_read32(const void* memPtr) ZSTD_STATIC U32 ZSTD_read32(const void *memPtr) { return get_unaligned((const U32 *)memPtr); }
{
return get_unaligned((const U32*)memPtr);
}
ZSTD_STATIC U64 ZSTD_read64(const void* memPtr) ZSTD_STATIC U64 ZSTD_read64(const void *memPtr) { return get_unaligned((const U64 *)memPtr); }
{
return get_unaligned((const U64*)memPtr);
}
ZSTD_STATIC size_t ZSTD_readST(const void* memPtr) ZSTD_STATIC size_t ZSTD_readST(const void *memPtr) { return get_unaligned((const size_t *)memPtr); }
{
return get_unaligned((const size_t*)memPtr);
}
ZSTD_STATIC void ZSTD_write16(void* memPtr, U16 value) ZSTD_STATIC void ZSTD_write16(void *memPtr, U16 value) { put_unaligned(value, (U16 *)memPtr); }
{
put_unaligned(value, (U16*)memPtr);
}
ZSTD_STATIC void ZSTD_write32(void* memPtr, U32 value) ZSTD_STATIC void ZSTD_write32(void *memPtr, U32 value) { put_unaligned(value, (U32 *)memPtr); }
{
put_unaligned(value, (U32*)memPtr);
}
ZSTD_STATIC void ZSTD_write64(void* memPtr, U64 value) ZSTD_STATIC void ZSTD_write64(void *memPtr, U64 value) { put_unaligned(value, (U64 *)memPtr); }
{
put_unaligned(value, (U64*)memPtr);
}
/*=== Little endian r/w ===*/ /*=== Little endian r/w ===*/
ZSTD_STATIC U16 ZSTD_readLE16(const void* memPtr) ZSTD_STATIC U16 ZSTD_readLE16(const void *memPtr) { return get_unaligned_le16(memPtr); }
{
return get_unaligned_le16(memPtr);
}
ZSTD_STATIC void ZSTD_writeLE16(void* memPtr, U16 val) ZSTD_STATIC void ZSTD_writeLE16(void *memPtr, U16 val) { put_unaligned_le16(val, memPtr); }
{
put_unaligned_le16(val, memPtr);
}
ZSTD_STATIC U32 ZSTD_readLE24(const void* memPtr) ZSTD_STATIC U32 ZSTD_readLE24(const void *memPtr) { return ZSTD_readLE16(memPtr) + (((const BYTE *)memPtr)[2] << 16); }
{
return ZSTD_readLE16(memPtr) + (((const BYTE*)memPtr)[2] << 16);
}
ZSTD_STATIC void ZSTD_writeLE24(void* memPtr, U32 val) ZSTD_STATIC void ZSTD_writeLE24(void *memPtr, U32 val)
{ {
ZSTD_writeLE16(memPtr, (U16)val); ZSTD_writeLE16(memPtr, (U16)val);
((BYTE*)memPtr)[2] = (BYTE)(val>>16); ((BYTE *)memPtr)[2] = (BYTE)(val >> 16);
} }
ZSTD_STATIC U32 ZSTD_readLE32(const void* memPtr) ZSTD_STATIC U32 ZSTD_readLE32(const void *memPtr) { return get_unaligned_le32(memPtr); }
{
return get_unaligned_le32(memPtr);
}
ZSTD_STATIC void ZSTD_writeLE32(void* memPtr, U32 val32) ZSTD_STATIC void ZSTD_writeLE32(void *memPtr, U32 val32) { put_unaligned_le32(val32, memPtr); }
{
put_unaligned_le32(val32, memPtr);
}
ZSTD_STATIC U64 ZSTD_readLE64(const void* memPtr) ZSTD_STATIC U64 ZSTD_readLE64(const void *memPtr) { return get_unaligned_le64(memPtr); }
{
return get_unaligned_le64(memPtr);
}
ZSTD_STATIC void ZSTD_writeLE64(void* memPtr, U64 val64) ZSTD_STATIC void ZSTD_writeLE64(void *memPtr, U64 val64) { put_unaligned_le64(val64, memPtr); }
{
put_unaligned_le64(val64, memPtr);
}
ZSTD_STATIC size_t ZSTD_readLEST(const void* memPtr) ZSTD_STATIC size_t ZSTD_readLEST(const void *memPtr)
{ {
if (ZSTD_32bits()) if (ZSTD_32bits())
return (size_t)ZSTD_readLE32(memPtr); return (size_t)ZSTD_readLE32(memPtr);
@@ -148,7 +100,7 @@ ZSTD_STATIC size_t ZSTD_readLEST(const void* memPtr)
return (size_t)ZSTD_readLE64(memPtr); return (size_t)ZSTD_readLE64(memPtr);
} }
ZSTD_STATIC void ZSTD_writeLEST(void* memPtr, size_t val) ZSTD_STATIC void ZSTD_writeLEST(void *memPtr, size_t val)
{ {
if (ZSTD_32bits()) if (ZSTD_32bits())
ZSTD_writeLE32(memPtr, (U32)val); ZSTD_writeLE32(memPtr, (U32)val);
@@ -158,27 +110,15 @@ ZSTD_STATIC void ZSTD_writeLEST(void* memPtr, size_t val)
/*=== Big endian r/w ===*/ /*=== Big endian r/w ===*/
ZSTD_STATIC U32 ZSTD_readBE32(const void* memPtr) ZSTD_STATIC U32 ZSTD_readBE32(const void *memPtr) { return get_unaligned_be32(memPtr); }
{
return get_unaligned_be32(memPtr);
}
ZSTD_STATIC void ZSTD_writeBE32(void* memPtr, U32 val32) ZSTD_STATIC void ZSTD_writeBE32(void *memPtr, U32 val32) { put_unaligned_be32(val32, memPtr); }
{
put_unaligned_be32(val32, memPtr);
}
ZSTD_STATIC U64 ZSTD_readBE64(const void* memPtr) ZSTD_STATIC U64 ZSTD_readBE64(const void *memPtr) { return get_unaligned_be64(memPtr); }
{
return get_unaligned_be64(memPtr);
}
ZSTD_STATIC void ZSTD_writeBE64(void* memPtr, U64 val64) ZSTD_STATIC void ZSTD_writeBE64(void *memPtr, U64 val64) { put_unaligned_be64(val64, memPtr); }
{
put_unaligned_be64(val64, memPtr);
}
ZSTD_STATIC size_t ZSTD_readBEST(const void* memPtr) ZSTD_STATIC size_t ZSTD_readBEST(const void *memPtr)
{ {
if (ZSTD_32bits()) if (ZSTD_32bits())
return (size_t)ZSTD_readBE32(memPtr); return (size_t)ZSTD_readBE32(memPtr);
@@ -186,7 +126,7 @@ ZSTD_STATIC size_t ZSTD_readBEST(const void* memPtr)
return (size_t)ZSTD_readBE64(memPtr); return (size_t)ZSTD_readBE64(memPtr);
} }
ZSTD_STATIC void ZSTD_writeBEST(void* memPtr, size_t val) ZSTD_STATIC void ZSTD_writeBEST(void *memPtr, size_t val)
{ {
if (ZSTD_32bits()) if (ZSTD_32bits())
ZSTD_writeBE32(memPtr, (U32)val); ZSTD_writeBE32(memPtr, (U32)val);
@@ -194,18 +134,17 @@ ZSTD_STATIC void ZSTD_writeBEST(void* memPtr, size_t val)
ZSTD_writeBE64(memPtr, (U64)val); ZSTD_writeBE64(memPtr, (U64)val);
} }
/* function safe only for comparisons */ /* function safe only for comparisons */
ZSTD_STATIC U32 ZSTD_readMINMATCH(const void* memPtr, U32 length) ZSTD_STATIC U32 ZSTD_readMINMATCH(const void *memPtr, U32 length)
{ {
switch (length) switch (length) {
{ default:
default : case 4: return ZSTD_read32(memPtr);
case 4 : return ZSTD_read32(memPtr); case 3:
case 3 : if (ZSTD_isLittleEndian()) if (ZSTD_isLittleEndian())
return ZSTD_read32(memPtr)<<8; return ZSTD_read32(memPtr) << 8;
else else
return ZSTD_read32(memPtr)>>8; return ZSTD_read32(memPtr) >> 8;
} }
} }
contrib/linux-kernel/lib/zstd/zstd_common.c
+20 -21
View File
@@ -14,8 +14,6 @@
* ("BSD"). * ("BSD").
*/ */
/*-************************************* /*-*************************************
* Dependencies * Dependencies
***************************************/ ***************************************/
@@ -23,20 +21,21 @@
#include "zstd_internal.h" /* declaration of ZSTD_isError, ZSTD_getErrorName, ZSTD_getErrorCode, ZSTD_getErrorString, ZSTD_versionNumber */ #include "zstd_internal.h" /* declaration of ZSTD_isError, ZSTD_getErrorName, ZSTD_getErrorCode, ZSTD_getErrorString, ZSTD_versionNumber */
#include <linux/kernel.h> #include <linux/kernel.h>
/*=************************************************************** /*=**************************************************************
* Custom allocator * Custom allocator
****************************************************************/ ****************************************************************/
#define stack_push(stack, size) ({ \ #define stack_push(stack, size) \
void* const ptr = ZSTD_PTR_ALIGN((stack)->ptr); \ ({ \
(stack)->ptr = (char*)ptr + (size); \ void *const ptr = ZSTD_PTR_ALIGN((stack)->ptr); \
(stack)->ptr = (char *)ptr + (size); \
(stack)->ptr <= (stack)->end ? ptr : NULL; \ (stack)->ptr <= (stack)->end ? ptr : NULL; \
}) })
ZSTD_customMem ZSTD_initStack(void* workspace, size_t workspaceSize) { ZSTD_customMem ZSTD_initStack(void *workspace, size_t workspaceSize)
ZSTD_customMem stackMem = { ZSTD_stackAlloc, ZSTD_stackFree, workspace }; {
ZSTD_stack* stack = (ZSTD_stack*) workspace; ZSTD_customMem stackMem = {ZSTD_stackAlloc, ZSTD_stackFree, workspace};
ZSTD_stack *stack = (ZSTD_stack *)workspace;
/* Verify preconditions */ /* Verify preconditions */
if (!workspace || workspaceSize < sizeof(ZSTD_stack) || workspace != ZSTD_PTR_ALIGN(workspace)) { if (!workspace || workspaceSize < sizeof(ZSTD_stack) || workspace != ZSTD_PTR_ALIGN(workspace)) {
ZSTD_customMem error = {NULL, NULL, NULL}; ZSTD_customMem error = {NULL, NULL, NULL};
@@ -44,33 +43,33 @@ ZSTD_customMem ZSTD_initStack(void* workspace, size_t workspaceSize) {
} }
/* Initialize the stack */ /* Initialize the stack */
stack->ptr = workspace; stack->ptr = workspace;
stack->end = (char*)workspace + workspaceSize; stack->end = (char *)workspace + workspaceSize;
stack_push(stack, sizeof(ZSTD_stack)); stack_push(stack, sizeof(ZSTD_stack));
return stackMem; return stackMem;
} }
void* ZSTD_stackAllocAll(void* opaque, size_t* size) { void *ZSTD_stackAllocAll(void *opaque, size_t *size)
ZSTD_stack* stack = (ZSTD_stack*)opaque; {
ZSTD_stack *stack = (ZSTD_stack *)opaque;
*size = stack->end - ZSTD_PTR_ALIGN(stack->ptr); *size = stack->end - ZSTD_PTR_ALIGN(stack->ptr);
return stack_push(stack, *size); return stack_push(stack, *size);
} }
void* ZSTD_stackAlloc(void* opaque, size_t size) { void *ZSTD_stackAlloc(void *opaque, size_t size)
ZSTD_stack* stack = (ZSTD_stack*)opaque; {
ZSTD_stack *stack = (ZSTD_stack *)opaque;
return stack_push(stack, size); return stack_push(stack, size);
} }
void ZSTD_stackFree(void* opaque, void* address) { void ZSTD_stackFree(void *opaque, void *address)
{
(void)opaque; (void)opaque;
(void)address; (void)address;
} }
void* ZSTD_malloc(size_t size, ZSTD_customMem customMem) void *ZSTD_malloc(size_t size, ZSTD_customMem customMem) { return customMem.customAlloc(customMem.opaque, size); }
{
return customMem.customAlloc(customMem.opaque, size);
}
void ZSTD_free(void* ptr, ZSTD_customMem customMem) void ZSTD_free(void *ptr, ZSTD_customMem customMem)
{ {
if (ptr!=NULL) if (ptr != NULL)
customMem.customFree(customMem.opaque, ptr); customMem.customFree(customMem.opaque, ptr);
} }
contrib/linux-kernel/lib/zstd/zstd_internal.h
+87 -90
View File
@@ -23,42 +23,49 @@
#define FORCE_INLINE static __always_inline #define FORCE_INLINE static __always_inline
#define FORCE_NOINLINE static noinline #define FORCE_NOINLINE static noinline
/*-************************************* /*-*************************************
* Dependencies * Dependencies
***************************************/ ***************************************/
#include "error_private.h"
#include "mem.h"
#include <linux/compiler.h> #include <linux/compiler.h>
#include <linux/kernel.h> #include <linux/kernel.h>
#include <linux/xxhash.h> #include <linux/xxhash.h>
#include <linux/zstd.h> #include <linux/zstd.h>
#include "mem.h"
#include "error_private.h"
/*-************************************* /*-*************************************
* shared macros * shared macros
***************************************/ ***************************************/
#define MIN(a,b) ((a)<(b) ? (a) : (b)) #define MIN(a, b) ((a) < (b) ? (a) : (b))
#define MAX(a,b) ((a)>(b) ? (a) : (b)) #define MAX(a, b) ((a) > (b) ? (a) : (b))
#define CHECK_F(f) { size_t const errcod = f; if (ERR_isError(errcod)) return errcod; } /* check and Forward error code */ #define CHECK_F(f) \
#define CHECK_E(f, e) { size_t const errcod = f; if (ERR_isError(errcod)) return ERROR(e); } /* check and send Error code */ { \
size_t const errcod = f; \
if (ERR_isError(errcod)) \
return errcod; \
} /* check and Forward error code */
#define CHECK_E(f, e) \
{ \
size_t const errcod = f; \
if (ERR_isError(errcod)) \
return ERROR(e); \
} /* check and send Error code */
/*-************************************* /*-*************************************
* Common constants * Common constants
***************************************/ ***************************************/
#define ZSTD_OPT_NUM (1<<12) #define ZSTD_OPT_NUM (1 << 12)
#define ZSTD_DICT_MAGIC 0xEC30A437 /* v0.7+ */ #define ZSTD_DICT_MAGIC 0xEC30A437 /* v0.7+ */
#define ZSTD_REP_NUM 3 /* number of repcodes */ #define ZSTD_REP_NUM 3 /* number of repcodes */
#define ZSTD_REP_CHECK (ZSTD_REP_NUM) /* number of repcodes to check by the optimal parser */ #define ZSTD_REP_CHECK (ZSTD_REP_NUM) /* number of repcodes to check by the optimal parser */
#define ZSTD_REP_MOVE (ZSTD_REP_NUM-1) #define ZSTD_REP_MOVE (ZSTD_REP_NUM - 1)
#define ZSTD_REP_MOVE_OPT (ZSTD_REP_NUM) #define ZSTD_REP_MOVE_OPT (ZSTD_REP_NUM)
static const U32 repStartValue[ZSTD_REP_NUM] = { 1, 4, 8 }; static const U32 repStartValue[ZSTD_REP_NUM] = {1, 4, 8};
#define KB *(1 <<10) #define KB *(1 << 10)
#define MB *(1 <<20) #define MB *(1 << 20)
#define GB *(1U<<30) #define GB *(1U << 30)
#define BIT7 128 #define BIT7 128
#define BIT6 64 #define BIT6 64
@@ -68,8 +75,8 @@ static const U32 repStartValue[ZSTD_REP_NUM] = { 1, 4, 8 };
#define BIT0 1 #define BIT0 1
#define ZSTD_WINDOWLOG_ABSOLUTEMIN 10 #define ZSTD_WINDOWLOG_ABSOLUTEMIN 10
static const size_t ZSTD_fcs_fieldSize[4] = { 0, 2, 4, 8 }; static const size_t ZSTD_fcs_fieldSize[4] = {0, 2, 4, 8};
static const size_t ZSTD_did_fieldSize[4] = { 0, 1, 2, 4 }; static const size_t ZSTD_did_fieldSize[4] = {0, 1, 2, 4};
#define ZSTD_BLOCKHEADERSIZE 3 /* C standard doesn't allow `static const` variable to be init using another `static const` variable */ #define ZSTD_BLOCKHEADERSIZE 3 /* C standard doesn't allow `static const` variable to be init using another `static const` variable */
static const size_t ZSTD_blockHeaderSize = ZSTD_BLOCKHEADERSIZE; static const size_t ZSTD_blockHeaderSize = ZSTD_BLOCKHEADERSIZE;
@@ -87,7 +94,7 @@ typedef enum { set_basic, set_rle, set_compressed, set_repeat } symbolEncodingTy
#define EQUAL_READ32 4 #define EQUAL_READ32 4
#define Litbits 8 #define Litbits 8
#define MaxLit ((1<<Litbits) - 1) #define MaxLit ((1 << Litbits) - 1)
#define MaxML 52 #define MaxML 52
#define MaxLL 35 #define MaxLL 35
#define MaxOff 28 #define MaxOff 28
@@ -96,62 +103,56 @@ typedef enum { set_basic, set_rle, set_compressed, set_repeat } symbolEncodingTy
#define LLFSELog 9 #define LLFSELog 9
#define OffFSELog 8 #define OffFSELog 8
static const U32 LL_bits[MaxLL+1] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, static const U32 LL_bits[MaxLL + 1] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 3, 3, 4, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16};
1, 1, 1, 1, 2, 2, 3, 3, 4, 6, 7, 8, 9,10,11,12, static const S16 LL_defaultNorm[MaxLL + 1] = {4, 3, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 2, 1, 1, 1, 1, 1, -1, -1, -1, -1};
13,14,15,16 };
static const S16 LL_defaultNorm[MaxLL+1] = { 4, 3, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1,
2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 2, 1, 1, 1, 1, 1,
-1,-1,-1,-1 };
#define LL_DEFAULTNORMLOG 6 /* for static allocation */ #define LL_DEFAULTNORMLOG 6 /* for static allocation */
static const U32 LL_defaultNormLog = LL_DEFAULTNORMLOG; static const U32 LL_defaultNormLog = LL_DEFAULTNORMLOG;
static const U32 ML_bits[MaxML+1] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, static const U32 ML_bits[MaxML + 1] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 3, 3, 4, 4, 5, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16};
1, 1, 1, 1, 2, 2, 3, 3, 4, 4, 5, 7, 8, 9,10,11, static const S16 ML_defaultNorm[MaxML + 1] = {1, 4, 3, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
12,13,14,15,16 }; 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, -1, -1, -1, -1, -1, -1, -1};
static const S16 ML_defaultNorm[MaxML+1] = { 1, 4, 3, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,-1,-1,
-1,-1,-1,-1,-1 };
#define ML_DEFAULTNORMLOG 6 /* for static allocation */ #define ML_DEFAULTNORMLOG 6 /* for static allocation */
static const U32 ML_defaultNormLog = ML_DEFAULTNORMLOG; static const U32 ML_defaultNormLog = ML_DEFAULTNORMLOG;
static const S16 OF_defaultNorm[MaxOff+1] = { 1, 1, 1, 1, 1, 1, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1, static const S16 OF_defaultNorm[MaxOff + 1] = {1, 1, 1, 1, 1, 1, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, -1, -1, -1, -1, -1};
1, 1, 1, 1, 1, 1, 1, 1,-1,-1,-1,-1,-1 };
#define OF_DEFAULTNORMLOG 5 /* for static allocation */ #define OF_DEFAULTNORMLOG 5 /* for static allocation */
static const U32 OF_defaultNormLog = OF_DEFAULTNORMLOG; static const U32 OF_defaultNormLog = OF_DEFAULTNORMLOG;
/*-******************************************* /*-*******************************************
* Shared functions to include for inlining * Shared functions to include for inlining
*********************************************/ *********************************************/
static void ZSTD_copy8(void* dst, const void* src) { memcpy(dst, src, 8); } static void ZSTD_copy8(void *dst, const void *src) { memcpy(dst, src, 8); }
#define COPY8(d,s) { ZSTD_copy8(d,s); d+=8; s+=8; } #define COPY8(d, s) \
{ \
ZSTD_copy8(d, s); \
d += 8; \
s += 8; \
}
/*! ZSTD_wildcopy() : /*! ZSTD_wildcopy() :
* custom version of memcpy(), can copy up to 7 bytes too many (8 bytes if length==0) */ * custom version of memcpy(), can copy up to 7 bytes too many (8 bytes if length==0) */
#define WILDCOPY_OVERLENGTH 8 #define WILDCOPY_OVERLENGTH 8
ZSTD_STATIC void ZSTD_wildcopy(void* dst, const void* src, ptrdiff_t length) ZSTD_STATIC void ZSTD_wildcopy(void *dst, const void *src, ptrdiff_t length)
{ {
const BYTE* ip = (const BYTE*)src; const BYTE *ip = (const BYTE *)src;
BYTE* op = (BYTE*)dst; BYTE *op = (BYTE *)dst;
BYTE* const oend = op + length; BYTE *const oend = op + length;
do do
COPY8(op, ip) COPY8(op, ip)
while (op < oend); while (op < oend);
} }
ZSTD_STATIC void ZSTD_wildcopy_e(void* dst, const void* src, void* dstEnd) /* should be faster for decoding, but strangely, not verified on all platform */ ZSTD_STATIC void ZSTD_wildcopy_e(void *dst, const void *src, void *dstEnd) /* should be faster for decoding, but strangely, not verified on all platform */
{ {
const BYTE* ip = (const BYTE*)src; const BYTE *ip = (const BYTE *)src;
BYTE* op = (BYTE*)dst; BYTE *op = (BYTE *)dst;
BYTE* const oend = (BYTE*)dstEnd; BYTE *const oend = (BYTE *)dstEnd;
do do
COPY8(op, ip) COPY8(op, ip)
while (op < oend); while (op < oend);
} }
/*-******************************************* /*-*******************************************
* Private interfaces * Private interfaces
*********************************************/ *********************************************/
@@ -170,31 +171,29 @@ typedef struct {
U32 rep[ZSTD_REP_NUM]; U32 rep[ZSTD_REP_NUM];
} ZSTD_optimal_t; } ZSTD_optimal_t;
typedef struct seqDef_s { typedef struct seqDef_s {
U32 offset; U32 offset;
U16 litLength; U16 litLength;
U16 matchLength; U16 matchLength;
} seqDef; } seqDef;
typedef struct { typedef struct {
seqDef* sequencesStart; seqDef *sequencesStart;
seqDef* sequences; seqDef *sequences;
BYTE* litStart; BYTE *litStart;
BYTE* lit; BYTE *lit;
BYTE* llCode; BYTE *llCode;
BYTE* mlCode; BYTE *mlCode;
BYTE* ofCode; BYTE *ofCode;
U32 longLengthID; /* 0 == no longLength; 1 == Lit.longLength; 2 == Match.longLength; */ U32 longLengthID; /* 0 == no longLength; 1 == Lit.longLength; 2 == Match.longLength; */
U32 longLengthPos; U32 longLengthPos;
/* opt */ /* opt */
ZSTD_optimal_t* priceTable; ZSTD_optimal_t *priceTable;
ZSTD_match_t* matchTable; ZSTD_match_t *matchTable;
U32* matchLengthFreq; U32 *matchLengthFreq;
U32* litLengthFreq; U32 *litLengthFreq;
U32* litFreq; U32 *litFreq;
U32* offCodeFreq; U32 *offCodeFreq;
U32 matchLengthSum; U32 matchLengthSum;
U32 matchSum; U32 matchSum;
U32 litLengthSum; U32 litLengthSum;
@@ -209,45 +208,44 @@ typedef struct {
U32 staticPrices; U32 staticPrices;
U32 cachedPrice; U32 cachedPrice;
U32 cachedLitLength; U32 cachedLitLength;
const BYTE* cachedLiterals; const BYTE *cachedLiterals;
} seqStore_t; } seqStore_t;
const seqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx); const seqStore_t *ZSTD_getSeqStore(const ZSTD_CCtx *ctx);
void ZSTD_seqToCodes(const seqStore_t* seqStorePtr); void ZSTD_seqToCodes(const seqStore_t *seqStorePtr);
int ZSTD_isSkipFrame(ZSTD_DCtx* dctx); int ZSTD_isSkipFrame(ZSTD_DCtx *dctx);
/*= Custom memory allocation functions */ /*= Custom memory allocation functions */
typedef void* (*ZSTD_allocFunction) (void* opaque, size_t size); typedef void *(*ZSTD_allocFunction)(void *opaque, size_t size);
typedef void (*ZSTD_freeFunction) (void* opaque, void* address); typedef void (*ZSTD_freeFunction)(void *opaque, void *address);
typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; void* opaque; } ZSTD_customMem; typedef struct {
ZSTD_allocFunction customAlloc;
ZSTD_freeFunction customFree;
void *opaque;
} ZSTD_customMem;
void* ZSTD_malloc(size_t size, ZSTD_customMem customMem); void *ZSTD_malloc(size_t size, ZSTD_customMem customMem);
void ZSTD_free(void* ptr, ZSTD_customMem customMem); void ZSTD_free(void *ptr, ZSTD_customMem customMem);
/*====== stack allocation ======*/ /*====== stack allocation ======*/
typedef struct { typedef struct {
void* ptr; void *ptr;
const void* end; const void *end;
} ZSTD_stack; } ZSTD_stack;
#define ZSTD_ALIGN(x) ALIGN(x, sizeof(size_t)) #define ZSTD_ALIGN(x) ALIGN(x, sizeof(size_t))
#define ZSTD_PTR_ALIGN(p) PTR_ALIGN(p, sizeof(size_t)) #define ZSTD_PTR_ALIGN(p) PTR_ALIGN(p, sizeof(size_t))
ZSTD_customMem ZSTD_initStack(void* workspace, size_t workspaceSize); ZSTD_customMem ZSTD_initStack(void *workspace, size_t workspaceSize);
void* ZSTD_stackAllocAll(void* opaque, size_t* size);
void* ZSTD_stackAlloc(void* opaque, size_t size);
void ZSTD_stackFree(void* opaque, void* address);
void *ZSTD_stackAllocAll(void *opaque, size_t *size);
void *ZSTD_stackAlloc(void *opaque, size_t size);
void ZSTD_stackFree(void *opaque, void *address);
/*====== common function ======*/ /*====== common function ======*/
ZSTD_STATIC U32 ZSTD_highbit32(U32 val) ZSTD_STATIC U32 ZSTD_highbit32(U32 val) { return 31 - __builtin_clz(val); }
{
return 31 - __builtin_clz(val);
}
/* hidden functions */ /* hidden functions */
@@ -255,14 +253,13 @@ ZSTD_STATIC U32 ZSTD_highbit32(U32 val)
* ensures next compression will not use repcodes from previous block. * ensures next compression will not use repcodes from previous block.
* Note : only works with regular variant; * Note : only works with regular variant;
* do not use with extDict variant ! */ * do not use with extDict variant ! */
void ZSTD_invalidateRepCodes(ZSTD_CCtx* cctx); void ZSTD_invalidateRepCodes(ZSTD_CCtx *cctx);
size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx);
size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx);
size_t ZSTD_freeCDict(ZSTD_CDict* cdict);
size_t ZSTD_freeDDict(ZSTD_DDict* cdict);
size_t ZSTD_freeCStream(ZSTD_CStream* zcs);
size_t ZSTD_freeDStream(ZSTD_DStream* zds);
size_t ZSTD_freeCCtx(ZSTD_CCtx *cctx);
size_t ZSTD_freeDCtx(ZSTD_DCtx *dctx);
size_t ZSTD_freeCDict(ZSTD_CDict *cdict);
size_t ZSTD_freeDDict(ZSTD_DDict *cdict);
size_t ZSTD_freeCStream(ZSTD_CStream *zcs);
size_t ZSTD_freeDStream(ZSTD_DStream *zds);
#endif /* ZSTD_CCOMMON_H_MODULE */ #endif /* ZSTD_CCOMMON_H_MODULE */
contrib/linux-kernel/lib/zstd/zstd_opt.h
+398 -312
View File
@@ -14,32 +14,28 @@
* ("BSD"). * ("BSD").
*/ */
/* Note : this file is intended to be included within zstd_compress.c */ /* Note : this file is intended to be included within zstd_compress.c */
#ifndef ZSTD_OPT_H_91842398743 #ifndef ZSTD_OPT_H_91842398743
#define ZSTD_OPT_H_91842398743 #define ZSTD_OPT_H_91842398743
#define ZSTD_LITFREQ_ADD 2 #define ZSTD_LITFREQ_ADD 2
#define ZSTD_FREQ_DIV 4 #define ZSTD_FREQ_DIV 4
#define ZSTD_MAX_PRICE (1<<30) #define ZSTD_MAX_PRICE (1 << 30)
/*-************************************* /*-*************************************
* Price functions for optimal parser * Price functions for optimal parser
***************************************/ ***************************************/
FORCE_INLINE void ZSTD_setLog2Prices(seqStore_t* ssPtr) FORCE_INLINE void ZSTD_setLog2Prices(seqStore_t *ssPtr)
{ {
ssPtr->log2matchLengthSum = ZSTD_highbit32(ssPtr->matchLengthSum+1); ssPtr->log2matchLengthSum = ZSTD_highbit32(ssPtr->matchLengthSum + 1);
ssPtr->log2litLengthSum = ZSTD_highbit32(ssPtr->litLengthSum+1); ssPtr->log2litLengthSum = ZSTD_highbit32(ssPtr->litLengthSum + 1);
ssPtr->log2litSum = ZSTD_highbit32(ssPtr->litSum+1); ssPtr->log2litSum = ZSTD_highbit32(ssPtr->litSum + 1);
ssPtr->log2offCodeSum = ZSTD_highbit32(ssPtr->offCodeSum+1); ssPtr->log2offCodeSum = ZSTD_highbit32(ssPtr->offCodeSum + 1);
ssPtr->factor = 1 + ((ssPtr->litSum>>5) / ssPtr->litLengthSum) + ((ssPtr->litSum<<1) / (ssPtr->litSum + ssPtr->matchSum)); ssPtr->factor = 1 + ((ssPtr->litSum >> 5) / ssPtr->litLengthSum) + ((ssPtr->litSum << 1) / (ssPtr->litSum + ssPtr->matchSum));
} }
ZSTD_STATIC void ZSTD_rescaleFreqs(seqStore_t *ssPtr, const BYTE *src, size_t srcSize)
ZSTD_STATIC void ZSTD_rescaleFreqs(seqStore_t* ssPtr, const BYTE* src, size_t srcSize)
{ {
unsigned u; unsigned u;
@@ -48,28 +44,29 @@ ZSTD_STATIC void ZSTD_rescaleFreqs(seqStore_t* ssPtr, const BYTE* src, size_t sr
ssPtr->staticPrices = 0; ssPtr->staticPrices = 0;
if (ssPtr->litLengthSum == 0) { if (ssPtr->litLengthSum == 0) {
if (srcSize <= 1024) ssPtr->staticPrices = 1; if (srcSize <= 1024)
ssPtr->staticPrices = 1;
for (u=0; u<=MaxLit; u++) for (u = 0; u <= MaxLit; u++)
ssPtr->litFreq[u] = 0; ssPtr->litFreq[u] = 0;
for (u=0; u<srcSize; u++) for (u = 0; u < srcSize; u++)
ssPtr->litFreq[src[u]]++; ssPtr->litFreq[src[u]]++;
ssPtr->litSum = 0; ssPtr->litSum = 0;
ssPtr->litLengthSum = MaxLL+1; ssPtr->litLengthSum = MaxLL + 1;
ssPtr->matchLengthSum = MaxML+1; ssPtr->matchLengthSum = MaxML + 1;
ssPtr->offCodeSum = (MaxOff+1); ssPtr->offCodeSum = (MaxOff + 1);
ssPtr->matchSum = (ZSTD_LITFREQ_ADD<<Litbits); ssPtr->matchSum = (ZSTD_LITFREQ_ADD << Litbits);
for (u=0; u<=MaxLit; u++) { for (u = 0; u <= MaxLit; u++) {
ssPtr->litFreq[u] = 1 + (ssPtr->litFreq[u]>>ZSTD_FREQ_DIV); ssPtr->litFreq[u] = 1 + (ssPtr->litFreq[u] >> ZSTD_FREQ_DIV);
ssPtr->litSum += ssPtr->litFreq[u]; ssPtr->litSum += ssPtr->litFreq[u];
} }
for (u=0; u<=MaxLL; u++) for (u = 0; u <= MaxLL; u++)
ssPtr->litLengthFreq[u] = 1; ssPtr->litLengthFreq[u] = 1;
for (u=0; u<=MaxML; u++) for (u = 0; u <= MaxML; u++)
ssPtr->matchLengthFreq[u] = 1; ssPtr->matchLengthFreq[u] = 1;
for (u=0; u<=MaxOff; u++) for (u = 0; u <= MaxOff; u++)
ssPtr->offCodeFreq[u] = 1; ssPtr->offCodeFreq[u] = 1;
} else { } else {
ssPtr->matchLengthSum = 0; ssPtr->matchLengthSum = 0;
@@ -78,22 +75,22 @@ ZSTD_STATIC void ZSTD_rescaleFreqs(seqStore_t* ssPtr, const BYTE* src, size_t sr
ssPtr->matchSum = 0; ssPtr->matchSum = 0;
ssPtr->litSum = 0; ssPtr->litSum = 0;
for (u=0; u<=MaxLit; u++) { for (u = 0; u <= MaxLit; u++) {
ssPtr->litFreq[u] = 1 + (ssPtr->litFreq[u]>>(ZSTD_FREQ_DIV+1)); ssPtr->litFreq[u] = 1 + (ssPtr->litFreq[u] >> (ZSTD_FREQ_DIV + 1));
ssPtr->litSum += ssPtr->litFreq[u]; ssPtr->litSum += ssPtr->litFreq[u];
} }
for (u=0; u<=MaxLL; u++) { for (u = 0; u <= MaxLL; u++) {
ssPtr->litLengthFreq[u] = 1 + (ssPtr->litLengthFreq[u]>>(ZSTD_FREQ_DIV+1)); ssPtr->litLengthFreq[u] = 1 + (ssPtr->litLengthFreq[u] >> (ZSTD_FREQ_DIV + 1));
ssPtr->litLengthSum += ssPtr->litLengthFreq[u]; ssPtr->litLengthSum += ssPtr->litLengthFreq[u];
} }
for (u=0; u<=MaxML; u++) { for (u = 0; u <= MaxML; u++) {
ssPtr->matchLengthFreq[u] = 1 + (ssPtr->matchLengthFreq[u]>>ZSTD_FREQ_DIV); ssPtr->matchLengthFreq[u] = 1 + (ssPtr->matchLengthFreq[u] >> ZSTD_FREQ_DIV);
ssPtr->matchLengthSum += ssPtr->matchLengthFreq[u]; ssPtr->matchLengthSum += ssPtr->matchLengthFreq[u];
ssPtr->matchSum += ssPtr->matchLengthFreq[u] * (u + 3); ssPtr->matchSum += ssPtr->matchLengthFreq[u] * (u + 3);
} }
ssPtr->matchSum *= ZSTD_LITFREQ_ADD; ssPtr->matchSum *= ZSTD_LITFREQ_ADD;
for (u=0; u<=MaxOff; u++) { for (u = 0; u <= MaxOff; u++) {
ssPtr->offCodeFreq[u] = 1 + (ssPtr->offCodeFreq[u]>>ZSTD_FREQ_DIV); ssPtr->offCodeFreq[u] = 1 + (ssPtr->offCodeFreq[u] >> ZSTD_FREQ_DIV);
ssPtr->offCodeSum += ssPtr->offCodeFreq[u]; ssPtr->offCodeSum += ssPtr->offCodeFreq[u];
} }
} }
@@ -101,30 +98,29 @@ ZSTD_STATIC void ZSTD_rescaleFreqs(seqStore_t* ssPtr, const BYTE* src, size_t sr
ZSTD_setLog2Prices(ssPtr); ZSTD_setLog2Prices(ssPtr);
} }
FORCE_INLINE U32 ZSTD_getLiteralPrice(seqStore_t *ssPtr, U32 litLength, const BYTE *literals)
FORCE_INLINE U32 ZSTD_getLiteralPrice(seqStore_t* ssPtr, U32 litLength, const BYTE* literals)
{ {
U32 price, u; U32 price, u;
if (ssPtr->staticPrices) if (ssPtr->staticPrices)
return ZSTD_highbit32((U32)litLength+1) + (litLength*6); return ZSTD_highbit32((U32)litLength + 1) + (litLength * 6);
if (litLength == 0) if (litLength == 0)
return ssPtr->log2litLengthSum - ZSTD_highbit32(ssPtr->litLengthFreq[0]+1); return ssPtr->log2litLengthSum - ZSTD_highbit32(ssPtr->litLengthFreq[0] + 1);
/* literals */ /* literals */
if (ssPtr->cachedLiterals == literals) { if (ssPtr->cachedLiterals == literals) {
U32 const additional = litLength - ssPtr->cachedLitLength; U32 const additional = litLength - ssPtr->cachedLitLength;
const BYTE* literals2 = ssPtr->cachedLiterals + ssPtr->cachedLitLength; const BYTE *literals2 = ssPtr->cachedLiterals + ssPtr->cachedLitLength;
price = ssPtr->cachedPrice + additional * ssPtr->log2litSum; price = ssPtr->cachedPrice + additional * ssPtr->log2litSum;
for (u=0; u < additional; u++) for (u = 0; u < additional; u++)
price -= ZSTD_highbit32(ssPtr->litFreq[literals2[u]]+1); price -= ZSTD_highbit32(ssPtr->litFreq[literals2[u]] + 1);
ssPtr->cachedPrice = price; ssPtr->cachedPrice = price;
ssPtr->cachedLitLength = litLength; ssPtr->cachedLitLength = litLength;
} else { } else {
price = litLength * ssPtr->log2litSum; price = litLength * ssPtr->log2litSum;
for (u=0; u < litLength; u++) for (u = 0; u < litLength; u++)
price -= ZSTD_highbit32(ssPtr->litFreq[literals[u]]+1); price -= ZSTD_highbit32(ssPtr->litFreq[literals[u]] + 1);
if (litLength >= 12) { if (litLength >= 12) {
ssPtr->cachedLiterals = literals; ssPtr->cachedLiterals = literals;
@@ -134,62 +130,66 @@ FORCE_INLINE U32 ZSTD_getLiteralPrice(seqStore_t* ssPtr, U32 litLength, const BY
} }
/* literal Length */ /* literal Length */
{ const BYTE LL_deltaCode = 19; {
const BYTE llCode = (litLength>63) ? (BYTE)ZSTD_highbit32(litLength) + LL_deltaCode : LL_Code[litLength]; const BYTE LL_deltaCode = 19;
price += LL_bits[llCode] + ssPtr->log2litLengthSum - ZSTD_highbit32(ssPtr->litLengthFreq[llCode]+1); const BYTE llCode = (litLength > 63) ? (BYTE)ZSTD_highbit32(litLength) + LL_deltaCode : LL_Code[litLength];
price += LL_bits[llCode] + ssPtr->log2litLengthSum - ZSTD_highbit32(ssPtr->litLengthFreq[llCode] + 1);
} }
return price; return price;
} }
FORCE_INLINE U32 ZSTD_getPrice(seqStore_t *seqStorePtr, U32 litLength, const BYTE *literals, U32 offset, U32 matchLength, const int ultra)
FORCE_INLINE U32 ZSTD_getPrice(seqStore_t* seqStorePtr, U32 litLength, const BYTE* literals, U32 offset, U32 matchLength, const int ultra)
{ {
/* offset */ /* offset */
U32 price; U32 price;
BYTE const offCode = (BYTE)ZSTD_highbit32(offset+1); BYTE const offCode = (BYTE)ZSTD_highbit32(offset + 1);
if (seqStorePtr->staticPrices) if (seqStorePtr->staticPrices)
return ZSTD_getLiteralPrice(seqStorePtr, litLength, literals) + ZSTD_highbit32((U32)matchLength+1) + 16 + offCode; return ZSTD_getLiteralPrice(seqStorePtr, litLength, literals) + ZSTD_highbit32((U32)matchLength + 1) + 16 + offCode;
price = offCode + seqStorePtr->log2offCodeSum - ZSTD_highbit32(seqStorePtr->offCodeFreq[offCode]+1); price = offCode + seqStorePtr->log2offCodeSum - ZSTD_highbit32(seqStorePtr->offCodeFreq[offCode] + 1);
if (!ultra && offCode >= 20) price += (offCode-19)*2; if (!ultra && offCode >= 20)
price += (offCode - 19) * 2;
/* match Length */ /* match Length */
{ const BYTE ML_deltaCode = 36; {
const BYTE mlCode = (matchLength>127) ? (BYTE)ZSTD_highbit32(matchLength) + ML_deltaCode : ML_Code[matchLength]; const BYTE ML_deltaCode = 36;
price += ML_bits[mlCode] + seqStorePtr->log2matchLengthSum - ZSTD_highbit32(seqStorePtr->matchLengthFreq[mlCode]+1); const BYTE mlCode = (matchLength > 127) ? (BYTE)ZSTD_highbit32(matchLength) + ML_deltaCode : ML_Code[matchLength];
price += ML_bits[mlCode] + seqStorePtr->log2matchLengthSum - ZSTD_highbit32(seqStorePtr->matchLengthFreq[mlCode] + 1);
} }
return price + ZSTD_getLiteralPrice(seqStorePtr, litLength, literals) + seqStorePtr->factor; return price + ZSTD_getLiteralPrice(seqStorePtr, litLength, literals) + seqStorePtr->factor;
} }
ZSTD_STATIC void ZSTD_updatePrice(seqStore_t *seqStorePtr, U32 litLength, const BYTE *literals, U32 offset, U32 matchLength)
ZSTD_STATIC void ZSTD_updatePrice(seqStore_t* seqStorePtr, U32 litLength, const BYTE* literals, U32 offset, U32 matchLength)
{ {
U32 u; U32 u;
/* literals */ /* literals */
seqStorePtr->litSum += litLength*ZSTD_LITFREQ_ADD; seqStorePtr->litSum += litLength * ZSTD_LITFREQ_ADD;
for (u=0; u < litLength; u++) for (u = 0; u < litLength; u++)
seqStorePtr->litFreq[literals[u]] += ZSTD_LITFREQ_ADD; seqStorePtr->litFreq[literals[u]] += ZSTD_LITFREQ_ADD;
/* literal Length */ /* literal Length */
{ const BYTE LL_deltaCode = 19; {
const BYTE llCode = (litLength>63) ? (BYTE)ZSTD_highbit32(litLength) + LL_deltaCode : LL_Code[litLength]; const BYTE LL_deltaCode = 19;
const BYTE llCode = (litLength > 63) ? (BYTE)ZSTD_highbit32(litLength) + LL_deltaCode : LL_Code[litLength];
seqStorePtr->litLengthFreq[llCode]++; seqStorePtr->litLengthFreq[llCode]++;
seqStorePtr->litLengthSum++; seqStorePtr->litLengthSum++;
} }
/* match offset */ /* match offset */
{ BYTE const offCode = (BYTE)ZSTD_highbit32(offset+1); {
BYTE const offCode = (BYTE)ZSTD_highbit32(offset + 1);
seqStorePtr->offCodeSum++; seqStorePtr->offCodeSum++;
seqStorePtr->offCodeFreq[offCode]++; seqStorePtr->offCodeFreq[offCode]++;
} }
/* match Length */ /* match Length */
{ const BYTE ML_deltaCode = 36; {
const BYTE mlCode = (matchLength>127) ? (BYTE)ZSTD_highbit32(matchLength) + ML_deltaCode : ML_Code[matchLength]; const BYTE ML_deltaCode = 36;
const BYTE mlCode = (matchLength > 127) ? (BYTE)ZSTD_highbit32(matchLength) + ML_deltaCode : ML_Code[matchLength];
seqStorePtr->matchLengthFreq[mlCode]++; seqStorePtr->matchLengthFreq[mlCode]++;
seqStorePtr->matchLengthSum++; seqStorePtr->matchLengthSum++;
} }
@@ -197,85 +197,83 @@ ZSTD_STATIC void ZSTD_updatePrice(seqStore_t* seqStorePtr, U32 litLength, const
ZSTD_setLog2Prices(seqStorePtr); ZSTD_setLog2Prices(seqStorePtr);
} }
#define SET_PRICE(pos, mlen_, offset_, litlen_, price_) \ #define SET_PRICE(pos, mlen_, offset_, litlen_, price_) \
{ \ { \
while (last_pos < pos) { opt[last_pos+1].price = ZSTD_MAX_PRICE; last_pos++; } \ while (last_pos < pos) { \
opt[last_pos + 1].price = ZSTD_MAX_PRICE; \
last_pos++; \
} \
opt[pos].mlen = mlen_; \ opt[pos].mlen = mlen_; \
opt[pos].off = offset_; \ opt[pos].off = offset_; \
opt[pos].litlen = litlen_; \ opt[pos].litlen = litlen_; \
opt[pos].price = price_; \ opt[pos].price = price_; \
} }
/* Update hashTable3 up to ip (excluded) /* Update hashTable3 up to ip (excluded)
Assumption : always within prefix (i.e. not within extDict) */ Assumption : always within prefix (i.e. not within extDict) */
FORCE_INLINE FORCE_INLINE
U32 ZSTD_insertAndFindFirstIndexHash3 (ZSTD_CCtx* zc, const BYTE* ip) U32 ZSTD_insertAndFindFirstIndexHash3(ZSTD_CCtx *zc, const BYTE *ip)
{ {
U32* const hashTable3 = zc->hashTable3; U32 *const hashTable3 = zc->hashTable3;
U32 const hashLog3 = zc->hashLog3; U32 const hashLog3 = zc->hashLog3;
const BYTE* const base = zc->base; const BYTE *const base = zc->base;
U32 idx = zc->nextToUpdate3; U32 idx = zc->nextToUpdate3;
const U32 target = zc->nextToUpdate3 = (U32)(ip - base); const U32 target = zc->nextToUpdate3 = (U32)(ip - base);
const size_t hash3 = ZSTD_hash3Ptr(ip, hashLog3); const size_t hash3 = ZSTD_hash3Ptr(ip, hashLog3);
while(idx < target) { while (idx < target) {
hashTable3[ZSTD_hash3Ptr(base+idx, hashLog3)] = idx; hashTable3[ZSTD_hash3Ptr(base + idx, hashLog3)] = idx;
idx++; idx++;
} }
return hashTable3[hash3]; return hashTable3[hash3];
} }
/*-************************************* /*-*************************************
* Binary Tree search * Binary Tree search
***************************************/ ***************************************/
static U32 ZSTD_insertBtAndGetAllMatches ( static U32 ZSTD_insertBtAndGetAllMatches(ZSTD_CCtx *zc, const BYTE *const ip, const BYTE *const iLimit, U32 nbCompares, const U32 mls, U32 extDict,
ZSTD_CCtx* zc, ZSTD_match_t *matches, const U32 minMatchLen)
const BYTE* const ip, const BYTE* const iLimit,
U32 nbCompares, const U32 mls,
U32 extDict, ZSTD_match_t* matches, const U32 minMatchLen)
{ {
const BYTE* const base = zc->base; const BYTE *const base = zc->base;
const U32 curr = (U32)(ip-base); const U32 curr = (U32)(ip - base);
const U32 hashLog = zc->params.cParams.hashLog; const U32 hashLog = zc->params.cParams.hashLog;
const size_t h = ZSTD_hashPtr(ip, hashLog, mls); const size_t h = ZSTD_hashPtr(ip, hashLog, mls);
U32* const hashTable = zc->hashTable; U32 *const hashTable = zc->hashTable;
U32 matchIndex = hashTable[h]; U32 matchIndex = hashTable[h];
U32* const bt = zc->chainTable; U32 *const bt = zc->chainTable;
const U32 btLog = zc->params.cParams.chainLog - 1; const U32 btLog = zc->params.cParams.chainLog - 1;
const U32 btMask= (1U << btLog) - 1; const U32 btMask = (1U << btLog) - 1;
size_t commonLengthSmaller=0, commonLengthLarger=0; size_t commonLengthSmaller = 0, commonLengthLarger = 0;
const BYTE* const dictBase = zc->dictBase; const BYTE *const dictBase = zc->dictBase;
const U32 dictLimit = zc->dictLimit; const U32 dictLimit = zc->dictLimit;
const BYTE* const dictEnd = dictBase + dictLimit; const BYTE *const dictEnd = dictBase + dictLimit;
const BYTE* const prefixStart = base + dictLimit; const BYTE *const prefixStart = base + dictLimit;
const U32 btLow = btMask >= curr ? 0 : curr - btMask; const U32 btLow = btMask >= curr ? 0 : curr - btMask;
const U32 windowLow = zc->lowLimit; const U32 windowLow = zc->lowLimit;
U32* smallerPtr = bt + 2*(curr&btMask); U32 *smallerPtr = bt + 2 * (curr & btMask);
U32* largerPtr = bt + 2*(curr&btMask) + 1; U32 *largerPtr = bt + 2 * (curr & btMask) + 1;
U32 matchEndIdx = curr+8; U32 matchEndIdx = curr + 8;
U32 dummy32; /* to be nullified at the end */ U32 dummy32; /* to be nullified at the end */
U32 mnum = 0; U32 mnum = 0;
const U32 minMatch = (mls == 3) ? 3 : 4; const U32 minMatch = (mls == 3) ? 3 : 4;
size_t bestLength = minMatchLen-1; size_t bestLength = minMatchLen - 1;
if (minMatch == 3) { /* HC3 match finder */ if (minMatch == 3) { /* HC3 match finder */
U32 const matchIndex3 = ZSTD_insertAndFindFirstIndexHash3 (zc, ip); U32 const matchIndex3 = ZSTD_insertAndFindFirstIndexHash3(zc, ip);
if (matchIndex3>windowLow && (curr - matchIndex3 < (1<<18))) { if (matchIndex3 > windowLow && (curr - matchIndex3 < (1 << 18))) {
const BYTE* match; const BYTE *match;
size_t currMl=0; size_t currMl = 0;
if ((!extDict) || matchIndex3 >= dictLimit) { if ((!extDict) || matchIndex3 >= dictLimit) {
match = base + matchIndex3; match = base + matchIndex3;
if (match[bestLength] == ip[bestLength]) currMl = ZSTD_count(ip, match, iLimit); if (match[bestLength] == ip[bestLength])
currMl = ZSTD_count(ip, match, iLimit);
} else { } else {
match = dictBase + matchIndex3; match = dictBase + matchIndex3;
if (ZSTD_readMINMATCH(match, MINMATCH) == ZSTD_readMINMATCH(ip, MINMATCH)) /* assumption : matchIndex3 <= dictLimit-4 (by table construction) */ if (ZSTD_readMINMATCH(match, MINMATCH) ==
currMl = ZSTD_count_2segments(ip+MINMATCH, match+MINMATCH, iLimit, dictEnd, prefixStart) + MINMATCH; ZSTD_readMINMATCH(ip, MINMATCH)) /* assumption : matchIndex3 <= dictLimit-4 (by table construction) */
currMl = ZSTD_count_2segments(ip + MINMATCH, match + MINMATCH, iLimit, dictEnd, prefixStart) + MINMATCH;
} }
/* save best solution */ /* save best solution */
@@ -284,8 +282,10 @@ static U32 ZSTD_insertBtAndGetAllMatches (
matches[mnum].off = ZSTD_REP_MOVE_OPT + curr - matchIndex3; matches[mnum].off = ZSTD_REP_MOVE_OPT + curr - matchIndex3;
matches[mnum].len = (U32)currMl; matches[mnum].len = (U32)currMl;
mnum++; mnum++;
if (currMl > ZSTD_OPT_NUM) goto update; if (currMl > ZSTD_OPT_NUM)
if (ip+currMl == iLimit) goto update; /* best possible, and avoid read overflow*/ goto update;
if (ip + currMl == iLimit)
goto update; /* best possible, and avoid read overflow*/
} }
} }
} }
@@ -293,30 +293,32 @@ static U32 ZSTD_insertBtAndGetAllMatches (
hashTable[h] = curr; /* Update Hash Table */ hashTable[h] = curr; /* Update Hash Table */
while (nbCompares-- && (matchIndex > windowLow)) { while (nbCompares-- && (matchIndex > windowLow)) {
U32* nextPtr = bt + 2*(matchIndex & btMask); U32 *nextPtr = bt + 2 * (matchIndex & btMask);
size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */ size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */
const BYTE* match; const BYTE *match;
if ((!extDict) || (matchIndex+matchLength >= dictLimit)) { if ((!extDict) || (matchIndex + matchLength >= dictLimit)) {
match = base + matchIndex; match = base + matchIndex;
if (match[matchLength] == ip[matchLength]) { if (match[matchLength] == ip[matchLength]) {
matchLength += ZSTD_count(ip+matchLength+1, match+matchLength+1, iLimit) +1; matchLength += ZSTD_count(ip + matchLength + 1, match + matchLength + 1, iLimit) + 1;
} }
} else { } else {
match = dictBase + matchIndex; match = dictBase + matchIndex;
matchLength += ZSTD_count_2segments(ip+matchLength, match+matchLength, iLimit, dictEnd, prefixStart); matchLength += ZSTD_count_2segments(ip + matchLength, match + matchLength, iLimit, dictEnd, prefixStart);
if (matchIndex+matchLength >= dictLimit) if (matchIndex + matchLength >= dictLimit)
match = base + matchIndex; /* to prepare for next usage of match[matchLength] */ match = base + matchIndex; /* to prepare for next usage of match[matchLength] */
} }
if (matchLength > bestLength) { if (matchLength > bestLength) {
if (matchLength > matchEndIdx - matchIndex) matchEndIdx = matchIndex + (U32)matchLength; if (matchLength > matchEndIdx - matchIndex)
matchEndIdx = matchIndex + (U32)matchLength;
bestLength = matchLength; bestLength = matchLength;
matches[mnum].off = ZSTD_REP_MOVE_OPT + curr - matchIndex; matches[mnum].off = ZSTD_REP_MOVE_OPT + curr - matchIndex;
matches[mnum].len = (U32)matchLength; matches[mnum].len = (U32)matchLength;
mnum++; mnum++;
if (matchLength > ZSTD_OPT_NUM) break; if (matchLength > ZSTD_OPT_NUM)
if (ip+matchLength == iLimit) /* equal : no way to know if inf or sup */ break;
if (ip + matchLength == iLimit) /* equal : no way to know if inf or sup */
break; /* drop, to guarantee consistency (miss a little bit of compression) */ break; /* drop, to guarantee consistency (miss a little bit of compression) */
} }
@@ -324,114 +326,114 @@ static U32 ZSTD_insertBtAndGetAllMatches (
/* match is smaller than curr */ /* match is smaller than curr */
*smallerPtr = matchIndex; /* update smaller idx */ *smallerPtr = matchIndex; /* update smaller idx */
commonLengthSmaller = matchLength; /* all smaller will now have at least this guaranteed common length */ commonLengthSmaller = matchLength; /* all smaller will now have at least this guaranteed common length */
if (matchIndex <= btLow) { smallerPtr=&dummy32; break; } /* beyond tree size, stop the search */ if (matchIndex <= btLow) {
smallerPtr = nextPtr+1; /* new "smaller" => larger of match */ smallerPtr = &dummy32;
break;
} /* beyond tree size, stop the search */
smallerPtr = nextPtr + 1; /* new "smaller" => larger of match */
matchIndex = nextPtr[1]; /* new matchIndex larger than previous (closer to curr) */ matchIndex = nextPtr[1]; /* new matchIndex larger than previous (closer to curr) */
} else { } else {
/* match is larger than curr */ /* match is larger than curr */
*largerPtr = matchIndex; *largerPtr = matchIndex;
commonLengthLarger = matchLength; commonLengthLarger = matchLength;
if (matchIndex <= btLow) { largerPtr=&dummy32; break; } /* beyond tree size, stop the search */ if (matchIndex <= btLow) {
largerPtr = &dummy32;
break;
} /* beyond tree size, stop the search */
largerPtr = nextPtr; largerPtr = nextPtr;
matchIndex = nextPtr[0]; matchIndex = nextPtr[0];
} } }
}
*smallerPtr = *largerPtr = 0; *smallerPtr = *largerPtr = 0;
update: update:
zc->nextToUpdate = (matchEndIdx > curr + 8) ? matchEndIdx - 8 : curr+1; zc->nextToUpdate = (matchEndIdx > curr + 8) ? matchEndIdx - 8 : curr + 1;
return mnum; return mnum;
} }
/** Tree updater, providing best match */ /** Tree updater, providing best match */
static U32 ZSTD_BtGetAllMatches ( static U32 ZSTD_BtGetAllMatches(ZSTD_CCtx *zc, const BYTE *const ip, const BYTE *const iLimit, const U32 maxNbAttempts, const U32 mls, ZSTD_match_t *matches,
ZSTD_CCtx* zc, const U32 minMatchLen)
const BYTE* const ip, const BYTE* const iLimit,
const U32 maxNbAttempts, const U32 mls, ZSTD_match_t* matches, const U32 minMatchLen)
{ {
if (ip < zc->base + zc->nextToUpdate) return 0; /* skipped area */ if (ip < zc->base + zc->nextToUpdate)
return 0; /* skipped area */
ZSTD_updateTree(zc, ip, iLimit, maxNbAttempts, mls); ZSTD_updateTree(zc, ip, iLimit, maxNbAttempts, mls);
return ZSTD_insertBtAndGetAllMatches(zc, ip, iLimit, maxNbAttempts, mls, 0, matches, minMatchLen); return ZSTD_insertBtAndGetAllMatches(zc, ip, iLimit, maxNbAttempts, mls, 0, matches, minMatchLen);
} }
static U32 ZSTD_BtGetAllMatches_selectMLS(ZSTD_CCtx *zc, /* Index table will be updated */
static U32 ZSTD_BtGetAllMatches_selectMLS ( const BYTE *ip, const BYTE *const iHighLimit, const U32 maxNbAttempts, const U32 matchLengthSearch,
ZSTD_CCtx* zc, /* Index table will be updated */ ZSTD_match_t *matches, const U32 minMatchLen)
const BYTE* ip, const BYTE* const iHighLimit,
const U32 maxNbAttempts, const U32 matchLengthSearch, ZSTD_match_t* matches, const U32 minMatchLen)
{ {
switch(matchLengthSearch) switch (matchLengthSearch) {
{ case 3: return ZSTD_BtGetAllMatches(zc, ip, iHighLimit, maxNbAttempts, 3, matches, minMatchLen);
case 3 : return ZSTD_BtGetAllMatches(zc, ip, iHighLimit, maxNbAttempts, 3, matches, minMatchLen); default:
default : case 4: return ZSTD_BtGetAllMatches(zc, ip, iHighLimit, maxNbAttempts, 4, matches, minMatchLen);
case 4 : return ZSTD_BtGetAllMatches(zc, ip, iHighLimit, maxNbAttempts, 4, matches, minMatchLen); case 5: return ZSTD_BtGetAllMatches(zc, ip, iHighLimit, maxNbAttempts, 5, matches, minMatchLen);
case 5 : return ZSTD_BtGetAllMatches(zc, ip, iHighLimit, maxNbAttempts, 5, matches, minMatchLen); case 7:
case 7 : case 6: return ZSTD_BtGetAllMatches(zc, ip, iHighLimit, maxNbAttempts, 6, matches, minMatchLen);
case 6 : return ZSTD_BtGetAllMatches(zc, ip, iHighLimit, maxNbAttempts, 6, matches, minMatchLen);
} }
} }
/** Tree updater, providing best match */ /** Tree updater, providing best match */
static U32 ZSTD_BtGetAllMatches_extDict ( static U32 ZSTD_BtGetAllMatches_extDict(ZSTD_CCtx *zc, const BYTE *const ip, const BYTE *const iLimit, const U32 maxNbAttempts, const U32 mls,
ZSTD_CCtx* zc, ZSTD_match_t *matches, const U32 minMatchLen)
const BYTE* const ip, const BYTE* const iLimit,
const U32 maxNbAttempts, const U32 mls, ZSTD_match_t* matches, const U32 minMatchLen)
{ {
if (ip < zc->base + zc->nextToUpdate) return 0; /* skipped area */ if (ip < zc->base + zc->nextToUpdate)
return 0; /* skipped area */
ZSTD_updateTree_extDict(zc, ip, iLimit, maxNbAttempts, mls); ZSTD_updateTree_extDict(zc, ip, iLimit, maxNbAttempts, mls);
return ZSTD_insertBtAndGetAllMatches(zc, ip, iLimit, maxNbAttempts, mls, 1, matches, minMatchLen); return ZSTD_insertBtAndGetAllMatches(zc, ip, iLimit, maxNbAttempts, mls, 1, matches, minMatchLen);
} }
static U32 ZSTD_BtGetAllMatches_selectMLS_extDict(ZSTD_CCtx *zc, /* Index table will be updated */
static U32 ZSTD_BtGetAllMatches_selectMLS_extDict ( const BYTE *ip, const BYTE *const iHighLimit, const U32 maxNbAttempts, const U32 matchLengthSearch,
ZSTD_CCtx* zc, /* Index table will be updated */ ZSTD_match_t *matches, const U32 minMatchLen)
const BYTE* ip, const BYTE* const iHighLimit,
const U32 maxNbAttempts, const U32 matchLengthSearch, ZSTD_match_t* matches, const U32 minMatchLen)
{ {
switch(matchLengthSearch) switch (matchLengthSearch) {
{ case 3: return ZSTD_BtGetAllMatches_extDict(zc, ip, iHighLimit, maxNbAttempts, 3, matches, minMatchLen);
case 3 : return ZSTD_BtGetAllMatches_extDict(zc, ip, iHighLimit, maxNbAttempts, 3, matches, minMatchLen); default:
default : case 4: return ZSTD_BtGetAllMatches_extDict(zc, ip, iHighLimit, maxNbAttempts, 4, matches, minMatchLen);
case 4 : return ZSTD_BtGetAllMatches_extDict(zc, ip, iHighLimit, maxNbAttempts, 4, matches, minMatchLen); case 5: return ZSTD_BtGetAllMatches_extDict(zc, ip, iHighLimit, maxNbAttempts, 5, matches, minMatchLen);
case 5 : return ZSTD_BtGetAllMatches_extDict(zc, ip, iHighLimit, maxNbAttempts, 5, matches, minMatchLen); case 7:
case 7 : case 6: return ZSTD_BtGetAllMatches_extDict(zc, ip, iHighLimit, maxNbAttempts, 6, matches, minMatchLen);
case 6 : return ZSTD_BtGetAllMatches_extDict(zc, ip, iHighLimit, maxNbAttempts, 6, matches, minMatchLen);
} }
} }
/*-******************************* /*-*******************************
* Optimal parser * Optimal parser
*********************************/ *********************************/
FORCE_INLINE FORCE_INLINE
void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx, void ZSTD_compressBlock_opt_generic(ZSTD_CCtx *ctx, const void *src, size_t srcSize, const int ultra)
const void* src, size_t srcSize, const int ultra)
{ {
seqStore_t* seqStorePtr = &(ctx->seqStore); seqStore_t *seqStorePtr = &(ctx->seqStore);
const BYTE* const istart = (const BYTE*)src; const BYTE *const istart = (const BYTE *)src;
const BYTE* ip = istart; const BYTE *ip = istart;
const BYTE* anchor = istart; const BYTE *anchor = istart;
const BYTE* const iend = istart + srcSize; const BYTE *const iend = istart + srcSize;
const BYTE* const ilimit = iend - 8; const BYTE *const ilimit = iend - 8;
const BYTE* const base = ctx->base; const BYTE *const base = ctx->base;
const BYTE* const prefixStart = base + ctx->dictLimit; const BYTE *const prefixStart = base + ctx->dictLimit;
const U32 maxSearches = 1U << ctx->params.cParams.searchLog; const U32 maxSearches = 1U << ctx->params.cParams.searchLog;
const U32 sufficient_len = ctx->params.cParams.targetLength; const U32 sufficient_len = ctx->params.cParams.targetLength;
const U32 mls = ctx->params.cParams.searchLength; const U32 mls = ctx->params.cParams.searchLength;
const U32 minMatch = (ctx->params.cParams.searchLength == 3) ? 3 : 4; const U32 minMatch = (ctx->params.cParams.searchLength == 3) ? 3 : 4;
ZSTD_optimal_t* opt = seqStorePtr->priceTable; ZSTD_optimal_t *opt = seqStorePtr->priceTable;
ZSTD_match_t* matches = seqStorePtr->matchTable; ZSTD_match_t *matches = seqStorePtr->matchTable;
const BYTE* inr; const BYTE *inr;
U32 offset, rep[ZSTD_REP_NUM]; U32 offset, rep[ZSTD_REP_NUM];
/* init */ /* init */
ctx->nextToUpdate3 = ctx->nextToUpdate; ctx->nextToUpdate3 = ctx->nextToUpdate;
ZSTD_rescaleFreqs(seqStorePtr, (const BYTE*)src, srcSize); ZSTD_rescaleFreqs(seqStorePtr, (const BYTE *)src, srcSize);
ip += (ip==prefixStart); ip += (ip == prefixStart);
{ U32 i; for (i=0; i<ZSTD_REP_NUM; i++) rep[i]=ctx->rep[i]; } {
U32 i;
for (i = 0; i < ZSTD_REP_NUM; i++)
rep[i] = ctx->rep[i];
}
/* Match Loop */ /* Match Loop */
while (ip < ilimit) { while (ip < ilimit) {
@@ -442,14 +444,18 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx,
litlen = (U32)(ip - anchor); litlen = (U32)(ip - anchor);
/* check repCode */ /* check repCode */
{ U32 i, last_i = ZSTD_REP_CHECK + (ip==anchor); {
for (i=(ip == anchor); i<last_i; i++) { U32 i, last_i = ZSTD_REP_CHECK + (ip == anchor);
const S32 repCur = (i==ZSTD_REP_MOVE_OPT) ? (rep[0] - 1) : rep[i]; for (i = (ip == anchor); i < last_i; i++) {
if ( (repCur > 0) && (repCur < (S32)(ip-prefixStart)) const S32 repCur = (i == ZSTD_REP_MOVE_OPT) ? (rep[0] - 1) : rep[i];
&& (ZSTD_readMINMATCH(ip, minMatch) == ZSTD_readMINMATCH(ip - repCur, minMatch))) { if ((repCur > 0) && (repCur < (S32)(ip - prefixStart)) &&
mlen = (U32)ZSTD_count(ip+minMatch, ip+minMatch-repCur, iend) + minMatch; (ZSTD_readMINMATCH(ip, minMatch) == ZSTD_readMINMATCH(ip - repCur, minMatch))) {
mlen = (U32)ZSTD_count(ip + minMatch, ip + minMatch - repCur, iend) + minMatch;
if (mlen > sufficient_len || mlen >= ZSTD_OPT_NUM) { if (mlen > sufficient_len || mlen >= ZSTD_OPT_NUM) {
best_mlen = mlen; best_off = i; cur = 0; last_pos = 1; best_mlen = mlen;
best_off = i;
cur = 0;
last_pos = 1;
goto _storeSequence; goto _storeSequence;
} }
best_off = i - (ip == anchor); best_off = i - (ip == anchor);
@@ -459,15 +465,20 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx,
SET_PRICE(mlen, mlen, i, litlen, price); /* note : macro modifies last_pos */ SET_PRICE(mlen, mlen, i, litlen, price); /* note : macro modifies last_pos */
mlen--; mlen--;
} while (mlen >= minMatch); } while (mlen >= minMatch);
} } } }
}
}
match_num = ZSTD_BtGetAllMatches_selectMLS(ctx, ip, iend, maxSearches, mls, matches, minMatch); match_num = ZSTD_BtGetAllMatches_selectMLS(ctx, ip, iend, maxSearches, mls, matches, minMatch);
if (!last_pos && !match_num) { ip++; continue; } if (!last_pos && !match_num) {
ip++;
continue;
}
if (match_num && (matches[match_num-1].len > sufficient_len || matches[match_num-1].len >= ZSTD_OPT_NUM)) { if (match_num && (matches[match_num - 1].len > sufficient_len || matches[match_num - 1].len >= ZSTD_OPT_NUM)) {
best_mlen = matches[match_num-1].len; best_mlen = matches[match_num - 1].len;
best_off = matches[match_num-1].off; best_off = matches[match_num - 1].off;
cur = 0; cur = 0;
last_pos = 1; last_pos = 1;
goto _storeSequence; goto _storeSequence;
@@ -476,19 +487,27 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx,
/* set prices using matches at position = 0 */ /* set prices using matches at position = 0 */
best_mlen = (last_pos) ? last_pos : minMatch; best_mlen = (last_pos) ? last_pos : minMatch;
for (u = 0; u < match_num; u++) { for (u = 0; u < match_num; u++) {
mlen = (u>0) ? matches[u-1].len+1 : best_mlen; mlen = (u > 0) ? matches[u - 1].len + 1 : best_mlen;
best_mlen = matches[u].len; best_mlen = matches[u].len;
while (mlen <= best_mlen) { while (mlen <= best_mlen) {
price = ZSTD_getPrice(seqStorePtr, litlen, anchor, matches[u].off-1, mlen - MINMATCH, ultra); price = ZSTD_getPrice(seqStorePtr, litlen, anchor, matches[u].off - 1, mlen - MINMATCH, ultra);
if (mlen > last_pos || price < opt[mlen].price) if (mlen > last_pos || price < opt[mlen].price)
SET_PRICE(mlen, mlen, matches[u].off, litlen, price); /* note : macro modifies last_pos */ SET_PRICE(mlen, mlen, matches[u].off, litlen, price); /* note : macro modifies last_pos */
mlen++; mlen++;
} } }
}
if (last_pos < minMatch) { ip++; continue; } if (last_pos < minMatch) {
ip++;
continue;
}
/* initialize opt[0] */ /* initialize opt[0] */
{ U32 i ; for (i=0; i<ZSTD_REP_NUM; i++) opt[0].rep[i] = rep[i]; } {
U32 i;
for (i = 0; i < ZSTD_REP_NUM; i++)
opt[0].rep[i] = rep[i];
}
opt[0].mlen = 1; opt[0].mlen = 1;
opt[0].litlen = litlen; opt[0].litlen = litlen;
@@ -496,57 +515,64 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx,
for (cur = 1; cur <= last_pos; cur++) { for (cur = 1; cur <= last_pos; cur++) {
inr = ip + cur; inr = ip + cur;
if (opt[cur-1].mlen == 1) { if (opt[cur - 1].mlen == 1) {
litlen = opt[cur-1].litlen + 1; litlen = opt[cur - 1].litlen + 1;
if (cur > litlen) { if (cur > litlen) {
price = opt[cur - litlen].price + ZSTD_getLiteralPrice(seqStorePtr, litlen, inr-litlen); price = opt[cur - litlen].price + ZSTD_getLiteralPrice(seqStorePtr, litlen, inr - litlen);
} else } else
price = ZSTD_getLiteralPrice(seqStorePtr, litlen, anchor); price = ZSTD_getLiteralPrice(seqStorePtr, litlen, anchor);
} else { } else {
litlen = 1; litlen = 1;
price = opt[cur - 1].price + ZSTD_getLiteralPrice(seqStorePtr, litlen, inr-1); price = opt[cur - 1].price + ZSTD_getLiteralPrice(seqStorePtr, litlen, inr - 1);
} }
if (cur > last_pos || price <= opt[cur].price) if (cur > last_pos || price <= opt[cur].price)
SET_PRICE(cur, 1, 0, litlen, price); SET_PRICE(cur, 1, 0, litlen, price);
if (cur == last_pos) break; if (cur == last_pos)
break;
if (inr > ilimit) /* last match must start at a minimum distance of 8 from oend */ if (inr > ilimit) /* last match must start at a minimum distance of 8 from oend */
continue; continue;
mlen = opt[cur].mlen; mlen = opt[cur].mlen;
if (opt[cur].off > ZSTD_REP_MOVE_OPT) { if (opt[cur].off > ZSTD_REP_MOVE_OPT) {
opt[cur].rep[2] = opt[cur-mlen].rep[1]; opt[cur].rep[2] = opt[cur - mlen].rep[1];
opt[cur].rep[1] = opt[cur-mlen].rep[0]; opt[cur].rep[1] = opt[cur - mlen].rep[0];
opt[cur].rep[0] = opt[cur].off - ZSTD_REP_MOVE_OPT; opt[cur].rep[0] = opt[cur].off - ZSTD_REP_MOVE_OPT;
} else { } else {
opt[cur].rep[2] = (opt[cur].off > 1) ? opt[cur-mlen].rep[1] : opt[cur-mlen].rep[2]; opt[cur].rep[2] = (opt[cur].off > 1) ? opt[cur - mlen].rep[1] : opt[cur - mlen].rep[2];
opt[cur].rep[1] = (opt[cur].off > 0) ? opt[cur-mlen].rep[0] : opt[cur-mlen].rep[1]; opt[cur].rep[1] = (opt[cur].off > 0) ? opt[cur - mlen].rep[0] : opt[cur - mlen].rep[1];
opt[cur].rep[0] = ((opt[cur].off==ZSTD_REP_MOVE_OPT) && (mlen != 1)) ? (opt[cur-mlen].rep[0] - 1) : (opt[cur-mlen].rep[opt[cur].off]); opt[cur].rep[0] =
((opt[cur].off == ZSTD_REP_MOVE_OPT) && (mlen != 1)) ? (opt[cur - mlen].rep[0] - 1) : (opt[cur - mlen].rep[opt[cur].off]);
} }
best_mlen = minMatch; best_mlen = minMatch;
{ U32 i, last_i = ZSTD_REP_CHECK + (mlen != 1); {
for (i=(opt[cur].mlen != 1); i<last_i; i++) { /* check rep */ U32 i, last_i = ZSTD_REP_CHECK + (mlen != 1);
const S32 repCur = (i==ZSTD_REP_MOVE_OPT) ? (opt[cur].rep[0] - 1) : opt[cur].rep[i]; for (i = (opt[cur].mlen != 1); i < last_i; i++) { /* check rep */
if ( (repCur > 0) && (repCur < (S32)(inr-prefixStart)) const S32 repCur = (i == ZSTD_REP_MOVE_OPT) ? (opt[cur].rep[0] - 1) : opt[cur].rep[i];
&& (ZSTD_readMINMATCH(inr, minMatch) == ZSTD_readMINMATCH(inr - repCur, minMatch))) { if ((repCur > 0) && (repCur < (S32)(inr - prefixStart)) &&
mlen = (U32)ZSTD_count(inr+minMatch, inr+minMatch - repCur, iend) + minMatch; (ZSTD_readMINMATCH(inr, minMatch) == ZSTD_readMINMATCH(inr - repCur, minMatch))) {
mlen = (U32)ZSTD_count(inr + minMatch, inr + minMatch - repCur, iend) + minMatch;
if (mlen > sufficient_len || cur + mlen >= ZSTD_OPT_NUM) { if (mlen > sufficient_len || cur + mlen >= ZSTD_OPT_NUM) {
best_mlen = mlen; best_off = i; last_pos = cur + 1; best_mlen = mlen;
best_off = i;
last_pos = cur + 1;
goto _storeSequence; goto _storeSequence;
} }
best_off = i - (opt[cur].mlen != 1); best_off = i - (opt[cur].mlen != 1);
if (mlen > best_mlen) best_mlen = mlen; if (mlen > best_mlen)
best_mlen = mlen;
do { do {
if (opt[cur].mlen == 1) { if (opt[cur].mlen == 1) {
litlen = opt[cur].litlen; litlen = opt[cur].litlen;
if (cur > litlen) { if (cur > litlen) {
price = opt[cur - litlen].price + ZSTD_getPrice(seqStorePtr, litlen, inr-litlen, best_off, mlen - MINMATCH, ultra); price = opt[cur - litlen].price + ZSTD_getPrice(seqStorePtr, litlen, inr - litlen,
best_off, mlen - MINMATCH, ultra);
} else } else
price = ZSTD_getPrice(seqStorePtr, litlen, anchor, best_off, mlen - MINMATCH, ultra); price = ZSTD_getPrice(seqStorePtr, litlen, anchor, best_off, mlen - MINMATCH, ultra);
} else { } else {
@@ -558,46 +584,51 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx,
SET_PRICE(cur + mlen, mlen, i, litlen, price); SET_PRICE(cur + mlen, mlen, i, litlen, price);
mlen--; mlen--;
} while (mlen >= minMatch); } while (mlen >= minMatch);
} } } }
}
}
match_num = ZSTD_BtGetAllMatches_selectMLS(ctx, inr, iend, maxSearches, mls, matches, best_mlen); match_num = ZSTD_BtGetAllMatches_selectMLS(ctx, inr, iend, maxSearches, mls, matches, best_mlen);
if (match_num > 0 && (matches[match_num-1].len > sufficient_len || cur + matches[match_num-1].len >= ZSTD_OPT_NUM)) { if (match_num > 0 && (matches[match_num - 1].len > sufficient_len || cur + matches[match_num - 1].len >= ZSTD_OPT_NUM)) {
best_mlen = matches[match_num-1].len; best_mlen = matches[match_num - 1].len;
best_off = matches[match_num-1].off; best_off = matches[match_num - 1].off;
last_pos = cur + 1; last_pos = cur + 1;
goto _storeSequence; goto _storeSequence;
} }
/* set prices using matches at position = cur */ /* set prices using matches at position = cur */
for (u = 0; u < match_num; u++) { for (u = 0; u < match_num; u++) {
mlen = (u>0) ? matches[u-1].len+1 : best_mlen; mlen = (u > 0) ? matches[u - 1].len + 1 : best_mlen;
best_mlen = matches[u].len; best_mlen = matches[u].len;
while (mlen <= best_mlen) { while (mlen <= best_mlen) {
if (opt[cur].mlen == 1) { if (opt[cur].mlen == 1) {
litlen = opt[cur].litlen; litlen = opt[cur].litlen;
if (cur > litlen) if (cur > litlen)
price = opt[cur - litlen].price + ZSTD_getPrice(seqStorePtr, litlen, ip+cur-litlen, matches[u].off-1, mlen - MINMATCH, ultra); price = opt[cur - litlen].price + ZSTD_getPrice(seqStorePtr, litlen, ip + cur - litlen,
matches[u].off - 1, mlen - MINMATCH, ultra);
else else
price = ZSTD_getPrice(seqStorePtr, litlen, anchor, matches[u].off-1, mlen - MINMATCH, ultra); price = ZSTD_getPrice(seqStorePtr, litlen, anchor, matches[u].off - 1, mlen - MINMATCH, ultra);
} else { } else {
litlen = 0; litlen = 0;
price = opt[cur].price + ZSTD_getPrice(seqStorePtr, 0, NULL, matches[u].off-1, mlen - MINMATCH, ultra); price = opt[cur].price + ZSTD_getPrice(seqStorePtr, 0, NULL, matches[u].off - 1, mlen - MINMATCH, ultra);
} }
if (cur + mlen > last_pos || (price < opt[cur + mlen].price)) if (cur + mlen > last_pos || (price < opt[cur + mlen].price))
SET_PRICE(cur + mlen, mlen, matches[u].off, litlen, price); SET_PRICE(cur + mlen, mlen, matches[u].off, litlen, price);
mlen++; mlen++;
} } } }
}
}
best_mlen = opt[last_pos].mlen; best_mlen = opt[last_pos].mlen;
best_off = opt[last_pos].off; best_off = opt[last_pos].off;
cur = last_pos - best_mlen; cur = last_pos - best_mlen;
/* store sequence */ /* store sequence */
_storeSequence: /* cur, last_pos, best_mlen, best_off have to be set */ _storeSequence: /* cur, last_pos, best_mlen, best_off have to be set */
opt[0].mlen = 1; opt[0].mlen = 1;
while (1) { while (1) {
@@ -607,7 +638,8 @@ _storeSequence: /* cur, last_pos, best_mlen, best_off have to be set */
opt[cur].off = best_off; opt[cur].off = best_off;
best_mlen = mlen; best_mlen = mlen;
best_off = offset; best_off = offset;
if (mlen > cur) break; if (mlen > cur)
break;
cur -= mlen; cur -= mlen;
} }
@@ -615,9 +647,13 @@ _storeSequence: /* cur, last_pos, best_mlen, best_off have to be set */
u += opt[u].mlen; u += opt[u].mlen;
} }
for (cur=0; cur < last_pos; ) { for (cur = 0; cur < last_pos;) {
mlen = opt[cur].mlen; mlen = opt[cur].mlen;
if (mlen == 1) { ip++; cur++; continue; } if (mlen == 1) {
ip++;
cur++;
continue;
}
offset = opt[cur].off; offset = opt[cur].off;
cur += mlen; cur += mlen;
litLength = (U32)(ip - anchor); litLength = (U32)(ip - anchor);
@@ -629,93 +665,107 @@ _storeSequence: /* cur, last_pos, best_mlen, best_off have to be set */
offset--; offset--;
} else { } else {
if (offset != 0) { if (offset != 0) {
best_off = (offset==ZSTD_REP_MOVE_OPT) ? (rep[0] - 1) : (rep[offset]); best_off = (offset == ZSTD_REP_MOVE_OPT) ? (rep[0] - 1) : (rep[offset]);
if (offset != 1) rep[2] = rep[1]; if (offset != 1)
rep[2] = rep[1];
rep[1] = rep[0]; rep[1] = rep[0];
rep[0] = best_off; rep[0] = best_off;
} }
if (litLength==0) offset--; if (litLength == 0)
offset--;
} }
ZSTD_updatePrice(seqStorePtr, litLength, anchor, offset, mlen-MINMATCH); ZSTD_updatePrice(seqStorePtr, litLength, anchor, offset, mlen - MINMATCH);
ZSTD_storeSeq(seqStorePtr, litLength, anchor, offset, mlen-MINMATCH); ZSTD_storeSeq(seqStorePtr, litLength, anchor, offset, mlen - MINMATCH);
anchor = ip = ip + mlen; anchor = ip = ip + mlen;
} } /* for (cur=0; cur < last_pos; ) */ }
} /* for (cur=0; cur < last_pos; ) */
/* Save reps for next block */ /* Save reps for next block */
{ int i; for (i=0; i<ZSTD_REP_NUM; i++) ctx->repToConfirm[i] = rep[i]; } {
int i;
for (i = 0; i < ZSTD_REP_NUM; i++)
ctx->repToConfirm[i] = rep[i];
}
/* Last Literals */ /* Last Literals */
{ size_t const lastLLSize = iend - anchor; {
size_t const lastLLSize = iend - anchor;
memcpy(seqStorePtr->lit, anchor, lastLLSize); memcpy(seqStorePtr->lit, anchor, lastLLSize);
seqStorePtr->lit += lastLLSize; seqStorePtr->lit += lastLLSize;
} }
} }
FORCE_INLINE FORCE_INLINE
void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx, void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx *ctx, const void *src, size_t srcSize, const int ultra)
const void* src, size_t srcSize, const int ultra)
{ {
seqStore_t* seqStorePtr = &(ctx->seqStore); seqStore_t *seqStorePtr = &(ctx->seqStore);
const BYTE* const istart = (const BYTE*)src; const BYTE *const istart = (const BYTE *)src;
const BYTE* ip = istart; const BYTE *ip = istart;
const BYTE* anchor = istart; const BYTE *anchor = istart;
const BYTE* const iend = istart + srcSize; const BYTE *const iend = istart + srcSize;
const BYTE* const ilimit = iend - 8; const BYTE *const ilimit = iend - 8;
const BYTE* const base = ctx->base; const BYTE *const base = ctx->base;
const U32 lowestIndex = ctx->lowLimit; const U32 lowestIndex = ctx->lowLimit;
const U32 dictLimit = ctx->dictLimit; const U32 dictLimit = ctx->dictLimit;
const BYTE* const prefixStart = base + dictLimit; const BYTE *const prefixStart = base + dictLimit;
const BYTE* const dictBase = ctx->dictBase; const BYTE *const dictBase = ctx->dictBase;
const BYTE* const dictEnd = dictBase + dictLimit; const BYTE *const dictEnd = dictBase + dictLimit;
const U32 maxSearches = 1U << ctx->params.cParams.searchLog; const U32 maxSearches = 1U << ctx->params.cParams.searchLog;
const U32 sufficient_len = ctx->params.cParams.targetLength; const U32 sufficient_len = ctx->params.cParams.targetLength;
const U32 mls = ctx->params.cParams.searchLength; const U32 mls = ctx->params.cParams.searchLength;
const U32 minMatch = (ctx->params.cParams.searchLength == 3) ? 3 : 4; const U32 minMatch = (ctx->params.cParams.searchLength == 3) ? 3 : 4;
ZSTD_optimal_t* opt = seqStorePtr->priceTable; ZSTD_optimal_t *opt = seqStorePtr->priceTable;
ZSTD_match_t* matches = seqStorePtr->matchTable; ZSTD_match_t *matches = seqStorePtr->matchTable;
const BYTE* inr; const BYTE *inr;
/* init */ /* init */
U32 offset, rep[ZSTD_REP_NUM]; U32 offset, rep[ZSTD_REP_NUM];
{ U32 i; for (i=0; i<ZSTD_REP_NUM; i++) rep[i]=ctx->rep[i]; } {
U32 i;
for (i = 0; i < ZSTD_REP_NUM; i++)
rep[i] = ctx->rep[i];
}
ctx->nextToUpdate3 = ctx->nextToUpdate; ctx->nextToUpdate3 = ctx->nextToUpdate;
ZSTD_rescaleFreqs(seqStorePtr, (const BYTE*)src, srcSize); ZSTD_rescaleFreqs(seqStorePtr, (const BYTE *)src, srcSize);
ip += (ip==prefixStart); ip += (ip == prefixStart);
/* Match Loop */ /* Match Loop */
while (ip < ilimit) { while (ip < ilimit) {
U32 cur, match_num, last_pos, litlen, price; U32 cur, match_num, last_pos, litlen, price;
U32 u, mlen, best_mlen, best_off, litLength; U32 u, mlen, best_mlen, best_off, litLength;
U32 curr = (U32)(ip-base); U32 curr = (U32)(ip - base);
memset(opt, 0, sizeof(ZSTD_optimal_t)); memset(opt, 0, sizeof(ZSTD_optimal_t));
last_pos = 0; last_pos = 0;
opt[0].litlen = (U32)(ip - anchor); opt[0].litlen = (U32)(ip - anchor);
/* check repCode */ /* check repCode */
{ U32 i, last_i = ZSTD_REP_CHECK + (ip==anchor); {
for (i = (ip==anchor); i<last_i; i++) { U32 i, last_i = ZSTD_REP_CHECK + (ip == anchor);
const S32 repCur = (i==ZSTD_REP_MOVE_OPT) ? (rep[0] - 1) : rep[i]; for (i = (ip == anchor); i < last_i; i++) {
const S32 repCur = (i == ZSTD_REP_MOVE_OPT) ? (rep[0] - 1) : rep[i];
const U32 repIndex = (U32)(curr - repCur); const U32 repIndex = (U32)(curr - repCur);
const BYTE* const repBase = repIndex < dictLimit ? dictBase : base; const BYTE *const repBase = repIndex < dictLimit ? dictBase : base;
const BYTE* const repMatch = repBase + repIndex; const BYTE *const repMatch = repBase + repIndex;
if ( (repCur > 0 && repCur <= (S32)curr) if ((repCur > 0 && repCur <= (S32)curr) &&
&& (((U32)((dictLimit-1) - repIndex) >= 3) & (repIndex>lowestIndex)) /* intentional overflow */ (((U32)((dictLimit - 1) - repIndex) >= 3) & (repIndex > lowestIndex)) /* intentional overflow */
&& (ZSTD_readMINMATCH(ip, minMatch) == ZSTD_readMINMATCH(repMatch, minMatch)) ) { && (ZSTD_readMINMATCH(ip, minMatch) == ZSTD_readMINMATCH(repMatch, minMatch))) {
/* repcode detected we should take it */ /* repcode detected we should take it */
const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend; const BYTE *const repEnd = repIndex < dictLimit ? dictEnd : iend;
mlen = (U32)ZSTD_count_2segments(ip+minMatch, repMatch+minMatch, iend, repEnd, prefixStart) + minMatch; mlen = (U32)ZSTD_count_2segments(ip + minMatch, repMatch + minMatch, iend, repEnd, prefixStart) + minMatch;
if (mlen > sufficient_len || mlen >= ZSTD_OPT_NUM) { if (mlen > sufficient_len || mlen >= ZSTD_OPT_NUM) {
best_mlen = mlen; best_off = i; cur = 0; last_pos = 1; best_mlen = mlen;
best_off = i;
cur = 0;
last_pos = 1;
goto _storeSequence; goto _storeSequence;
} }
best_off = i - (ip==anchor); best_off = i - (ip == anchor);
litlen = opt[0].litlen; litlen = opt[0].litlen;
do { do {
price = ZSTD_getPrice(seqStorePtr, litlen, anchor, best_off, mlen - MINMATCH, ultra); price = ZSTD_getPrice(seqStorePtr, litlen, anchor, best_off, mlen - MINMATCH, ultra);
@@ -723,18 +773,27 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx,
SET_PRICE(mlen, mlen, i, litlen, price); /* note : macro modifies last_pos */ SET_PRICE(mlen, mlen, i, litlen, price); /* note : macro modifies last_pos */
mlen--; mlen--;
} while (mlen >= minMatch); } while (mlen >= minMatch);
} } } }
}
}
match_num = ZSTD_BtGetAllMatches_selectMLS_extDict(ctx, ip, iend, maxSearches, mls, matches, minMatch); /* first search (depth 0) */ match_num = ZSTD_BtGetAllMatches_selectMLS_extDict(ctx, ip, iend, maxSearches, mls, matches, minMatch); /* first search (depth 0) */
if (!last_pos && !match_num) { ip++; continue; } if (!last_pos && !match_num) {
ip++;
continue;
}
{ U32 i; for (i=0; i<ZSTD_REP_NUM; i++) opt[0].rep[i] = rep[i]; } {
U32 i;
for (i = 0; i < ZSTD_REP_NUM; i++)
opt[0].rep[i] = rep[i];
}
opt[0].mlen = 1; opt[0].mlen = 1;
if (match_num && (matches[match_num-1].len > sufficient_len || matches[match_num-1].len >= ZSTD_OPT_NUM)) { if (match_num && (matches[match_num - 1].len > sufficient_len || matches[match_num - 1].len >= ZSTD_OPT_NUM)) {
best_mlen = matches[match_num-1].len; best_mlen = matches[match_num - 1].len;
best_off = matches[match_num-1].off; best_off = matches[match_num - 1].off;
cur = 0; cur = 0;
last_pos = 1; last_pos = 1;
goto _storeSequence; goto _storeSequence;
@@ -744,81 +803,90 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx,
/* set prices using matches at position = 0 */ /* set prices using matches at position = 0 */
for (u = 0; u < match_num; u++) { for (u = 0; u < match_num; u++) {
mlen = (u>0) ? matches[u-1].len+1 : best_mlen; mlen = (u > 0) ? matches[u - 1].len + 1 : best_mlen;
best_mlen = matches[u].len; best_mlen = matches[u].len;
litlen = opt[0].litlen; litlen = opt[0].litlen;
while (mlen <= best_mlen) { while (mlen <= best_mlen) {
price = ZSTD_getPrice(seqStorePtr, litlen, anchor, matches[u].off-1, mlen - MINMATCH, ultra); price = ZSTD_getPrice(seqStorePtr, litlen, anchor, matches[u].off - 1, mlen - MINMATCH, ultra);
if (mlen > last_pos || price < opt[mlen].price) if (mlen > last_pos || price < opt[mlen].price)
SET_PRICE(mlen, mlen, matches[u].off, litlen, price); SET_PRICE(mlen, mlen, matches[u].off, litlen, price);
mlen++; mlen++;
} } }
}
if (last_pos < minMatch) { if (last_pos < minMatch) {
ip++; continue; ip++;
continue;
} }
/* check further positions */ /* check further positions */
for (cur = 1; cur <= last_pos; cur++) { for (cur = 1; cur <= last_pos; cur++) {
inr = ip + cur; inr = ip + cur;
if (opt[cur-1].mlen == 1) { if (opt[cur - 1].mlen == 1) {
litlen = opt[cur-1].litlen + 1; litlen = opt[cur - 1].litlen + 1;
if (cur > litlen) { if (cur > litlen) {
price = opt[cur - litlen].price + ZSTD_getLiteralPrice(seqStorePtr, litlen, inr-litlen); price = opt[cur - litlen].price + ZSTD_getLiteralPrice(seqStorePtr, litlen, inr - litlen);
} else } else
price = ZSTD_getLiteralPrice(seqStorePtr, litlen, anchor); price = ZSTD_getLiteralPrice(seqStorePtr, litlen, anchor);
} else { } else {
litlen = 1; litlen = 1;
price = opt[cur - 1].price + ZSTD_getLiteralPrice(seqStorePtr, litlen, inr-1); price = opt[cur - 1].price + ZSTD_getLiteralPrice(seqStorePtr, litlen, inr - 1);
} }
if (cur > last_pos || price <= opt[cur].price) if (cur > last_pos || price <= opt[cur].price)
SET_PRICE(cur, 1, 0, litlen, price); SET_PRICE(cur, 1, 0, litlen, price);
if (cur == last_pos) break; if (cur == last_pos)
break;
if (inr > ilimit) /* last match must start at a minimum distance of 8 from oend */ if (inr > ilimit) /* last match must start at a minimum distance of 8 from oend */
continue; continue;
mlen = opt[cur].mlen; mlen = opt[cur].mlen;
if (opt[cur].off > ZSTD_REP_MOVE_OPT) { if (opt[cur].off > ZSTD_REP_MOVE_OPT) {
opt[cur].rep[2] = opt[cur-mlen].rep[1]; opt[cur].rep[2] = opt[cur - mlen].rep[1];
opt[cur].rep[1] = opt[cur-mlen].rep[0]; opt[cur].rep[1] = opt[cur - mlen].rep[0];
opt[cur].rep[0] = opt[cur].off - ZSTD_REP_MOVE_OPT; opt[cur].rep[0] = opt[cur].off - ZSTD_REP_MOVE_OPT;
} else { } else {
opt[cur].rep[2] = (opt[cur].off > 1) ? opt[cur-mlen].rep[1] : opt[cur-mlen].rep[2]; opt[cur].rep[2] = (opt[cur].off > 1) ? opt[cur - mlen].rep[1] : opt[cur - mlen].rep[2];
opt[cur].rep[1] = (opt[cur].off > 0) ? opt[cur-mlen].rep[0] : opt[cur-mlen].rep[1]; opt[cur].rep[1] = (opt[cur].off > 0) ? opt[cur - mlen].rep[0] : opt[cur - mlen].rep[1];
opt[cur].rep[0] = ((opt[cur].off==ZSTD_REP_MOVE_OPT) && (mlen != 1)) ? (opt[cur-mlen].rep[0] - 1) : (opt[cur-mlen].rep[opt[cur].off]); opt[cur].rep[0] =
((opt[cur].off == ZSTD_REP_MOVE_OPT) && (mlen != 1)) ? (opt[cur - mlen].rep[0] - 1) : (opt[cur - mlen].rep[opt[cur].off]);
} }
best_mlen = minMatch; best_mlen = minMatch;
{ U32 i, last_i = ZSTD_REP_CHECK + (mlen != 1); {
for (i = (mlen != 1); i<last_i; i++) { U32 i, last_i = ZSTD_REP_CHECK + (mlen != 1);
const S32 repCur = (i==ZSTD_REP_MOVE_OPT) ? (opt[cur].rep[0] - 1) : opt[cur].rep[i]; for (i = (mlen != 1); i < last_i; i++) {
const U32 repIndex = (U32)(curr+cur - repCur); const S32 repCur = (i == ZSTD_REP_MOVE_OPT) ? (opt[cur].rep[0] - 1) : opt[cur].rep[i];
const BYTE* const repBase = repIndex < dictLimit ? dictBase : base; const U32 repIndex = (U32)(curr + cur - repCur);
const BYTE* const repMatch = repBase + repIndex; const BYTE *const repBase = repIndex < dictLimit ? dictBase : base;
if ( (repCur > 0 && repCur <= (S32)(curr+cur)) const BYTE *const repMatch = repBase + repIndex;
&& (((U32)((dictLimit-1) - repIndex) >= 3) & (repIndex>lowestIndex)) /* intentional overflow */ if ((repCur > 0 && repCur <= (S32)(curr + cur)) &&
&& (ZSTD_readMINMATCH(inr, minMatch) == ZSTD_readMINMATCH(repMatch, minMatch)) ) { (((U32)((dictLimit - 1) - repIndex) >= 3) & (repIndex > lowestIndex)) /* intentional overflow */
&& (ZSTD_readMINMATCH(inr, minMatch) == ZSTD_readMINMATCH(repMatch, minMatch))) {
/* repcode detected */ /* repcode detected */
const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend; const BYTE *const repEnd = repIndex < dictLimit ? dictEnd : iend;
mlen = (U32)ZSTD_count_2segments(inr+minMatch, repMatch+minMatch, iend, repEnd, prefixStart) + minMatch; mlen = (U32)ZSTD_count_2segments(inr + minMatch, repMatch + minMatch, iend, repEnd, prefixStart) + minMatch;
if (mlen > sufficient_len || cur + mlen >= ZSTD_OPT_NUM) { if (mlen > sufficient_len || cur + mlen >= ZSTD_OPT_NUM) {
best_mlen = mlen; best_off = i; last_pos = cur + 1; best_mlen = mlen;
best_off = i;
last_pos = cur + 1;
goto _storeSequence; goto _storeSequence;
} }
best_off = i - (opt[cur].mlen != 1); best_off = i - (opt[cur].mlen != 1);
if (mlen > best_mlen) best_mlen = mlen; if (mlen > best_mlen)
best_mlen = mlen;
do { do {
if (opt[cur].mlen == 1) { if (opt[cur].mlen == 1) {
litlen = opt[cur].litlen; litlen = opt[cur].litlen;
if (cur > litlen) { if (cur > litlen) {
price = opt[cur - litlen].price + ZSTD_getPrice(seqStorePtr, litlen, inr-litlen, best_off, mlen - MINMATCH, ultra); price = opt[cur - litlen].price + ZSTD_getPrice(seqStorePtr, litlen, inr - litlen,
best_off, mlen - MINMATCH, ultra);
} else } else
price = ZSTD_getPrice(seqStorePtr, litlen, anchor, best_off, mlen - MINMATCH, ultra); price = ZSTD_getPrice(seqStorePtr, litlen, anchor, best_off, mlen - MINMATCH, ultra);
} else { } else {
@@ -830,46 +898,51 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx,
SET_PRICE(cur + mlen, mlen, i, litlen, price); SET_PRICE(cur + mlen, mlen, i, litlen, price);
mlen--; mlen--;
} while (mlen >= minMatch); } while (mlen >= minMatch);
} } } }
}
}
match_num = ZSTD_BtGetAllMatches_selectMLS_extDict(ctx, inr, iend, maxSearches, mls, matches, minMatch); match_num = ZSTD_BtGetAllMatches_selectMLS_extDict(ctx, inr, iend, maxSearches, mls, matches, minMatch);
if (match_num > 0 && (matches[match_num-1].len > sufficient_len || cur + matches[match_num-1].len >= ZSTD_OPT_NUM)) { if (match_num > 0 && (matches[match_num - 1].len > sufficient_len || cur + matches[match_num - 1].len >= ZSTD_OPT_NUM)) {
best_mlen = matches[match_num-1].len; best_mlen = matches[match_num - 1].len;
best_off = matches[match_num-1].off; best_off = matches[match_num - 1].off;
last_pos = cur + 1; last_pos = cur + 1;
goto _storeSequence; goto _storeSequence;
} }
/* set prices using matches at position = cur */ /* set prices using matches at position = cur */
for (u = 0; u < match_num; u++) { for (u = 0; u < match_num; u++) {
mlen = (u>0) ? matches[u-1].len+1 : best_mlen; mlen = (u > 0) ? matches[u - 1].len + 1 : best_mlen;
best_mlen = matches[u].len; best_mlen = matches[u].len;
while (mlen <= best_mlen) { while (mlen <= best_mlen) {
if (opt[cur].mlen == 1) { if (opt[cur].mlen == 1) {
litlen = opt[cur].litlen; litlen = opt[cur].litlen;
if (cur > litlen) if (cur > litlen)
price = opt[cur - litlen].price + ZSTD_getPrice(seqStorePtr, litlen, ip+cur-litlen, matches[u].off-1, mlen - MINMATCH, ultra); price = opt[cur - litlen].price + ZSTD_getPrice(seqStorePtr, litlen, ip + cur - litlen,
matches[u].off - 1, mlen - MINMATCH, ultra);
else else
price = ZSTD_getPrice(seqStorePtr, litlen, anchor, matches[u].off-1, mlen - MINMATCH, ultra); price = ZSTD_getPrice(seqStorePtr, litlen, anchor, matches[u].off - 1, mlen - MINMATCH, ultra);
} else { } else {
litlen = 0; litlen = 0;
price = opt[cur].price + ZSTD_getPrice(seqStorePtr, 0, NULL, matches[u].off-1, mlen - MINMATCH, ultra); price = opt[cur].price + ZSTD_getPrice(seqStorePtr, 0, NULL, matches[u].off - 1, mlen - MINMATCH, ultra);
} }
if (cur + mlen > last_pos || (price < opt[cur + mlen].price)) if (cur + mlen > last_pos || (price < opt[cur + mlen].price))
SET_PRICE(cur + mlen, mlen, matches[u].off, litlen, price); SET_PRICE(cur + mlen, mlen, matches[u].off, litlen, price);
mlen++; mlen++;
} } } /* for (cur = 1; cur <= last_pos; cur++) */ }
}
} /* for (cur = 1; cur <= last_pos; cur++) */
best_mlen = opt[last_pos].mlen; best_mlen = opt[last_pos].mlen;
best_off = opt[last_pos].off; best_off = opt[last_pos].off;
cur = last_pos - best_mlen; cur = last_pos - best_mlen;
/* store sequence */ /* store sequence */
_storeSequence: /* cur, last_pos, best_mlen, best_off have to be set */ _storeSequence: /* cur, last_pos, best_mlen, best_off have to be set */
opt[0].mlen = 1; opt[0].mlen = 1;
while (1) { while (1) {
@@ -879,17 +952,22 @@ _storeSequence: /* cur, last_pos, best_mlen, best_off have to be set */
opt[cur].off = best_off; opt[cur].off = best_off;
best_mlen = mlen; best_mlen = mlen;
best_off = offset; best_off = offset;
if (mlen > cur) break; if (mlen > cur)
break;
cur -= mlen; cur -= mlen;
} }
for (u = 0; u <= last_pos; ) { for (u = 0; u <= last_pos;) {
u += opt[u].mlen; u += opt[u].mlen;
} }
for (cur=0; cur < last_pos; ) { for (cur = 0; cur < last_pos;) {
mlen = opt[cur].mlen; mlen = opt[cur].mlen;
if (mlen == 1) { ip++; cur++; continue; } if (mlen == 1) {
ip++;
cur++;
continue;
}
offset = opt[cur].off; offset = opt[cur].off;
cur += mlen; cur += mlen;
litLength = (U32)(ip - anchor); litLength = (U32)(ip - anchor);
@@ -901,25 +979,33 @@ _storeSequence: /* cur, last_pos, best_mlen, best_off have to be set */
offset--; offset--;
} else { } else {
if (offset != 0) { if (offset != 0) {
best_off = (offset==ZSTD_REP_MOVE_OPT) ? (rep[0] - 1) : (rep[offset]); best_off = (offset == ZSTD_REP_MOVE_OPT) ? (rep[0] - 1) : (rep[offset]);
if (offset != 1) rep[2] = rep[1]; if (offset != 1)
rep[2] = rep[1];
rep[1] = rep[0]; rep[1] = rep[0];
rep[0] = best_off; rep[0] = best_off;
} }
if (litLength==0) offset--; if (litLength == 0)
offset--;
} }
ZSTD_updatePrice(seqStorePtr, litLength, anchor, offset, mlen-MINMATCH); ZSTD_updatePrice(seqStorePtr, litLength, anchor, offset, mlen - MINMATCH);
ZSTD_storeSeq(seqStorePtr, litLength, anchor, offset, mlen-MINMATCH); ZSTD_storeSeq(seqStorePtr, litLength, anchor, offset, mlen - MINMATCH);
anchor = ip = ip + mlen; anchor = ip = ip + mlen;
} } /* for (cur=0; cur < last_pos; ) */ }
} /* for (cur=0; cur < last_pos; ) */
/* Save reps for next block */ /* Save reps for next block */
{ int i; for (i=0; i<ZSTD_REP_NUM; i++) ctx->repToConfirm[i] = rep[i]; } {
int i;
for (i = 0; i < ZSTD_REP_NUM; i++)
ctx->repToConfirm[i] = rep[i];
}
/* Last Literals */ /* Last Literals */
{ size_t lastLLSize = iend - anchor; {
size_t lastLLSize = iend - anchor;
memcpy(seqStorePtr->lit, anchor, lastLLSize); memcpy(seqStorePtr->lit, anchor, lastLLSize);
seqStorePtr->lit += lastLLSize; seqStorePtr->lit += lastLLSize;
} }