ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

quick first zstdhc version

Browse Source
This commit is contained in:
Yann Collet
2015-10-22 15:31:46 +01:00
parent 353c5d26cf
commit f3eca25322
9 changed files with 866 additions and 267 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/zstd.c
+18 -197
View File
@@ -130,12 +130,6 @@ static const U32 g_searchStrength = 8;
#define WORKPLACESIZE (BLOCKSIZE*3)
#define MINMATCH 4
#define MLbits 7
#define LLbits 6
#define Offbits 5
#define MaxML ((1<<MLbits )-1)
#define MaxLL ((1<<LLbits )-1)
#define MaxOff 31
#define LitFSELog 11
#define MLFSELog 10
#define LLFSELog 10
@@ -153,49 +147,12 @@ static const size_t ZSTD_frameHeaderSize = 4;
/* *******************************************************
* Memory operations
**********************************************************/
static void ZSTD_copy4(void* dst, const void* src) { memcpy(dst, src, 4); }
static void ZSTD_copy8(void* dst, const void* src) { memcpy(dst, src, 8); }
#define COPY8(d,s) { ZSTD_copy8(d,s); d+=8; s+=8; }
/*! ZSTD_wildcopy : custom version of memcpy(), can copy up to 7-8 bytes too many */
static void ZSTD_wildcopy(void* dst, const void* src, size_t length)
{
const BYTE* ip = (const BYTE*)src;
BYTE* op = (BYTE*)dst;
BYTE* const oend = op + length;
do COPY8(op, ip) while (op < oend);
}
static void ZSTD_copy4(void* dst, const void* src) { memcpy(dst, src, 4); }
/* **************************************
* Local structures
****************************************/
typedef enum { bt_compressed, bt_raw, bt_rle, bt_end } blockType_t;
typedef struct
{
blockType_t blockType;
U32 origSize;
} blockProperties_t;
typedef struct {
void* buffer;
U32* offsetStart;
U32* offset;
BYTE* offCodeStart;
BYTE* offCode;
BYTE* litStart;
BYTE* lit;
BYTE* litLengthStart;
BYTE* litLength;
BYTE* matchLengthStart;
BYTE* matchLength;
BYTE* dumpsStart;
BYTE* dumps;
} seqStore_t;
void ZSTD_resetSeqStore(seqStore_t* ssPtr)
{
ssPtr->offset = ssPtr->offsetStart;
@@ -205,7 +162,6 @@ void ZSTD_resetSeqStore(seqStore_t* ssPtr)
ssPtr->dumps = ssPtr->dumpsStart;
}
struct ZSTD_CCtx_s
{
const BYTE* base;
@@ -231,7 +187,7 @@ void ZSTD_resetCCtx(ZSTD_CCtx* ctx)
ctx->seqStore.litLengthStart = ctx->seqStore.litStart + BLOCKSIZE;
ctx->seqStore.matchLengthStart = ctx->seqStore.litLengthStart + (BLOCKSIZE>>2);
ctx->seqStore.dumpsStart = ctx->seqStore.matchLengthStart + (BLOCKSIZE>>2);
memset(ctx->hashTable, 0, HASH_TABLESIZE*4);
memset(ctx->hashTable, 0, sizeof(ctx->hashTable));
}
ZSTD_CCtx* ZSTD_createCCtx(void)
@@ -289,100 +245,6 @@ static unsigned ZSTD_highbit(U32 val)
}
/* *************************************
* Function body to include
***************************************/
#include "mem.h"
static size_t ZSTD_read_ARCH(const void* p) { size_t r; memcpy(&r, p, sizeof(r)); return r; }
MEM_STATIC unsigned ZSTD_NbCommonBytes (register size_t val)
{
if (MEM_isLittleEndian())
{
if (MEM_64bits())
{
# if defined(_MSC_VER) && defined(_WIN64) && !defined(LZ4_FORCE_SW_BITCOUNT)
unsigned long r = 0;
_BitScanForward64( &r, (U64)val );
return (int)(r>>3);
# elif defined(__GNUC__) && (__GNUC__ >= 3) && !defined(LZ4_FORCE_SW_BITCOUNT)
return (__builtin_ctzll((U64)val) >> 3);
# else
static const int DeBruijnBytePos[64] = { 0, 0, 0, 0, 0, 1, 1, 2, 0, 3, 1, 3, 1, 4, 2, 7, 0, 2, 3, 6, 1, 5, 3, 5, 1, 3, 4, 4, 2, 5, 6, 7, 7, 0, 1, 2, 3, 3, 4, 6, 2, 6, 5, 5, 3, 4, 5, 6, 7, 1, 2, 4, 6, 4, 4, 5, 7, 2, 6, 5, 7, 6, 7, 7 };
return DeBruijnBytePos[((U64)((val & -(long long)val) * 0x0218A392CDABBD3FULL)) >> 58];
# endif
}
else /* 32 bits */
{
# if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT)
unsigned long r;
_BitScanForward( &r, (U32)val );
return (int)(r>>3);
# elif defined(__GNUC__) && (__GNUC__ >= 3) && !defined(LZ4_FORCE_SW_BITCOUNT)
return (__builtin_ctz((U32)val) >> 3);
# else
static const int DeBruijnBytePos[32] = { 0, 0, 3, 0, 3, 1, 3, 0, 3, 2, 2, 1, 3, 2, 0, 1, 3, 3, 1, 2, 2, 2, 2, 0, 3, 1, 2, 0, 1, 0, 1, 1 };
return DeBruijnBytePos[((U32)((val & -(S32)val) * 0x077CB531U)) >> 27];
# endif
}
}
else /* Big Endian CPU */
{
if (MEM_32bits())
{
# if defined(_MSC_VER) && defined(_WIN64) && !defined(LZ4_FORCE_SW_BITCOUNT)
unsigned long r = 0;
_BitScanReverse64( &r, val );
return (unsigned)(r>>3);
# elif defined(__GNUC__) && (__GNUC__ >= 3) && !defined(LZ4_FORCE_SW_BITCOUNT)
return (__builtin_clzll(val) >> 3);
# else
unsigned r;
const unsigned n32 = sizeof(size_t)*4; /* calculate this way due to compiler complaining in 32-bits mode */
if (!(val>>n32)) { r=4; } else { r=0; val>>=n32; }
if (!(val>>16)) { r+=2; val>>=8; } else { val>>=24; }
r += (!val);
return r;
# endif
}
else /* 32 bits */
{
# if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT)
unsigned long r = 0;
_BitScanReverse( &r, (unsigned long)val );
return (unsigned)(r>>3);
# elif defined(__GNUC__) && (__GNUC__ >= 3) && !defined(LZ4_FORCE_SW_BITCOUNT)
return (__builtin_clz((U32)val) >> 3);
# else
unsigned r;
if (!(val>>16)) { r=2; val>>=8; } else { r=0; val>>=24; }
r += (!val);
return r;
# endif
}
}
}
MEM_STATIC size_t ZSTD_count(const BYTE* pIn, const BYTE* pMatch, const BYTE* pInLimit)
{
const BYTE* const pStart = pIn;
while ((pIn<pInLimit-(sizeof(size_t)-1)))
{
size_t diff = ZSTD_read_ARCH(pMatch) ^ ZSTD_read_ARCH(pIn);
if (!diff) { pIn+=sizeof(size_t); pMatch+=sizeof(size_t); continue; }
pIn += ZSTD_NbCommonBytes(diff);
return (size_t)(pIn - pStart);
}
if (MEM_32bits()) if ((pIn<(pInLimit-3)) && (MEM_read32(pMatch) == MEM_read32(pIn))) { pIn+=4; pMatch+=4; }
if ((pIn<(pInLimit-1)) && (MEM_read16(pMatch) == MEM_read16(pIn))) { pIn+=2; pMatch+=2; }
if ((pIn<pInLimit) && (*pMatch == *pIn)) pIn++;
return (size_t)(pIn - pStart);
}
/* *******************************************************
* Compression
*********************************************************/
@@ -392,7 +254,7 @@ size_t ZSTD_compressBound(size_t srcSize) /* maximum compressed size */
}
static size_t ZSTD_noCompressBlock (void* dst, size_t maxDstSize, const void* src, size_t srcSize)
size_t ZSTD_noCompressBlock (void* dst, size_t maxDstSize, const void* src, size_t srcSize)
{
BYTE* const ostart = (BYTE* const)dst;
@@ -433,7 +295,7 @@ static size_t ZSTD_compressRleLiteralsBlock (void* dst, size_t maxDstSize, const
size_t ZSTD_minGain(size_t srcSize) { return (srcSize >> 6) + 1; }
static size_t ZSTD_compressLiterals (void* dst, size_t maxDstSize,
const void* src, size_t srcSize)
const void* src, size_t srcSize)
{
const size_t minGain = ZSTD_minGain(srcSize);
BYTE* const ostart = (BYTE*)dst;
@@ -461,9 +323,9 @@ static size_t ZSTD_compressLiterals (void* dst, size_t maxDstSize,
}
static size_t ZSTD_compressSequences(BYTE* dst, size_t maxDstSize,
const seqStore_t* seqStorePtr,
size_t srcSize)
size_t ZSTD_compressSequences(BYTE* dst, size_t maxDstSize,
const seqStore_t* seqStorePtr,
size_t srcSize)
{
U32 count[MaxSeq+1];
S16 norm[MaxSeq+1];
@@ -475,7 +337,6 @@ static size_t ZSTD_compressSequences(BYTE* dst, size_t maxDstSize,
U32 CTable_MatchLength[FSE_CTABLE_SIZE_U32(MLFSELog, MaxML )];
U32 LLtype, Offtype, MLtype; /* compressed, raw or rle */
const BYTE* const op_lit_start = seqStorePtr->litStart;
const BYTE* op_lit = seqStorePtr->lit;
const BYTE* const llTable = seqStorePtr->litLengthStart;
const BYTE* const llPtr = seqStorePtr->litLength;
const BYTE* const mlTable = seqStorePtr->matchLengthStart;
@@ -492,7 +353,7 @@ static size_t ZSTD_compressSequences(BYTE* dst, size_t maxDstSize,
/* Compress literals */
{
size_t cSize;
size_t litSize = op_lit - op_lit_start;
size_t litSize = seqStorePtr->lit - op_lit_start;
if (litSize <= LITERAL_NOENTROPY)
cSize = ZSTD_compressRawLiteralsBlock(op, maxDstSize, op_lit_start, litSize);
@@ -665,48 +526,6 @@ static size_t ZSTD_compressSequences(BYTE* dst, size_t maxDstSize,
}
static void ZSTD_storeSeq(seqStore_t* seqStorePtr, size_t litLength, const BYTE* literals, size_t offset, size_t matchLength)
{
BYTE* op_lit = seqStorePtr->lit;
BYTE* const l_end = op_lit + litLength;
/* copy Literals */
while (op_lit<l_end) COPY8(op_lit, literals);
seqStorePtr->lit += litLength;
/* literal Length */
if (litLength >= MaxLL)
{
*(seqStorePtr->litLength++) = MaxLL;
if (litLength<255 + MaxLL)
*(seqStorePtr->dumps++) = (BYTE)(litLength - MaxLL);
else
{
*(seqStorePtr->dumps++) = 255;
MEM_writeLE32(seqStorePtr->dumps, (U32)litLength); seqStorePtr->dumps += 3;
}
}
else *(seqStorePtr->litLength++) = (BYTE)litLength;
/* match offset */
*(seqStorePtr->offset++) = (U32)offset;
/* match Length */
if (matchLength >= MaxML)
{
*(seqStorePtr->matchLength++) = MaxML;
if (matchLength < 255+MaxML)
*(seqStorePtr->dumps++) = (BYTE)(matchLength - MaxML);
else
{
*(seqStorePtr->dumps++) = 255;
MEM_writeLE32(seqStorePtr->dumps, (U32)matchLength); seqStorePtr->dumps+=3;
}
}
else *(seqStorePtr->matchLength++) = (BYTE)matchLength;
}
//static const U32 hashMask = (1<<HASH_LOG)-1;
//static const U64 prime5bytes = 889523592379ULL;
//static const U64 prime6bytes = 227718039650203ULL;
@@ -751,9 +570,9 @@ static size_t ZSTD_compressBlock(ZSTD_CCtx* ctx, void* dst, size_t maxDstSize, c
const BYTE* ip = istart + 1;
const BYTE* anchor = istart;
const BYTE* const iend = istart + srcSize;
const BYTE* const ilimit = iend - 16;
const BYTE* const ilimit = iend - 8;
size_t prevOffset=0, offset=0;
size_t prevOffset=4, offset=4;
/* init */
@@ -1095,7 +914,6 @@ size_t ZSTD_decodeLiteralsBlock(void* ctx,
switch(*istart & 3)
{
default:
case 0:
{
size_t nbLiterals = BLOCKSIZE;
@@ -1124,6 +942,8 @@ size_t ZSTD_decodeLiteralsBlock(void* ctx,
dctx->litSize = litSize;
return 4;
}
default:
return ERROR(corruption_detected); /* forbidden nominal case */
}
}
@@ -1343,8 +1163,8 @@ static size_t ZSTD_execSequence(BYTE* op,
const BYTE* match = op - sequence.offset;
/* check */
if (sequence.offset > (size_t)op) return ERROR(corruption_detected); /* address space overflow test (this test seems kept by clang optimizer) */
//if (match > op) return ERROR(corruption_detected); /* address space overflow test (is clang optimizer removing this test ?) */
if (sequence.offset > (size_t)op) return ERROR(corruption_detected); /* address space overflow test (this test seems kept by clang optimizer) */
if (match < base) return ERROR(corruption_detected);
/* close range match, overlap */
@@ -1419,9 +1239,10 @@ static size_t ZSTD_decompressSequences(
seqState_t seqState;
memset(&sequence, 0, sizeof(sequence));
sequence.offset = 4;
seqState.dumps = dumps;
seqState.dumpsEnd = dumps + dumpsLength;
seqState.prevOffset = 1;
seqState.prevOffset = 4;
errorCode = BIT_initDStream(&(seqState.DStream), ip, iend-ip);
if (ERR_isError(errorCode)) return ERROR(corruption_detected);
FSE_initDState(&(seqState.stateLL), &(seqState.DStream), DTableLL);
@@ -1447,7 +1268,7 @@ static size_t ZSTD_decompressSequences(
size_t lastLLSize = litEnd - litPtr;
if (litPtr > litEnd) return ERROR(corruption_detected);
if (op+lastLLSize > oend) return ERROR(dstSize_tooSmall);
if (op != litPtr) memmove(op, litPtr, lastLLSize);
if (op != litPtr) memcpy(op, litPtr, lastLLSize);
op += lastLLSize;
}
}
@@ -1542,9 +1363,9 @@ size_t ZSTD_decompress(void* dst, size_t maxDstSize, const void* src, size_t src
}
/*******************************
/* ******************************
* Streaming Decompression API
*******************************/
********************************/
size_t ZSTD_resetDCtx(ZSTD_DCtx* dctx)
{