ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fixed 32-bits compatibility

Browse Source
This commit is contained in:
Yann Collet
2016-03-26 17:18:11 +01:00
parent a5b66e34c7
commit b9151406de
3 changed files with 37 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/zstd_compress.c
+20 -15
View File
@@ -744,38 +744,43 @@ size_t ZSTD_compressSequences(ZSTD_CCtx* zc,
FSE_initCState2(&stateOffsetBits, CTable_OffsetBits, ofCodeTable[nbSeq-1]); FSE_initCState2(&stateOffsetBits, CTable_OffsetBits, ofCodeTable[nbSeq-1]);
FSE_initCState2(&stateLitLength, CTable_LitLength, llCodeTable[nbSeq-1]); FSE_initCState2(&stateLitLength, CTable_LitLength, llCodeTable[nbSeq-1]);
BIT_addBits(&blockStream, llTable[nbSeq-1], LL_bits[llCodeTable[nbSeq-1]]); BIT_addBits(&blockStream, llTable[nbSeq-1], LL_bits[llCodeTable[nbSeq-1]]);
if (MEM_32bits()) BIT_flushBits(&blockStream);
BIT_addBits(&blockStream, mlTable[nbSeq-1], ML_bits[mlCodeTable[nbSeq-1]]); BIT_addBits(&blockStream, mlTable[nbSeq-1], ML_bits[mlCodeTable[nbSeq-1]]);
if (MEM_32bits()) BIT_flushBits(&blockStream);
BIT_addBits(&blockStream, offsetTable[nbSeq-1], ofCodeTable[nbSeq-1]); BIT_addBits(&blockStream, offsetTable[nbSeq-1], ofCodeTable[nbSeq-1]);
BIT_flushBits(&blockStream); BIT_flushBits(&blockStream);
{ size_t n; { size_t n;
for (n=nbSeq-2 ; n<nbSeq ; n--) { /* intentional underflow */ for (n=nbSeq-2 ; n<nbSeq ; n--) { /* intentional underflow */
const BYTE ofCode = ofCodeTable[n]; /* 32b*/ /* 64b*/ const BYTE ofCode = ofCodeTable[n];
const BYTE mlCode = mlCodeTable[n]; const BYTE mlCode = mlCodeTable[n];
const BYTE llCode = llCodeTable[n]; const BYTE llCode = llCodeTable[n];
const U32 llBits = LL_bits[llCode]; const U32 llBits = LL_bits[llCode];
const U32 mlBits = ML_bits[mlCode]; const U32 mlBits = ML_bits[mlCode];
const U32 ofBits = ofCode; const U32 ofBits = ofCode; /* 32b*/ /* 64b*/
/* (7)*/ /* (7)*/ /* (7)*/ /* (7)*/
FSE_encodeSymbol(&blockStream, &stateOffsetBits, ofCode); /* 25 */ /* 35 */ FSE_encodeSymbol(&blockStream, &stateOffsetBits, ofCode); /* 15 */ /* 15 */
FSE_encodeSymbol(&blockStream, &stateMatchLength, mlCode); /* 17 */ /* 17 */ FSE_encodeSymbol(&blockStream, &stateMatchLength, mlCode); /* 24 */ /* 24 */
FSE_encodeSymbol(&blockStream, &stateLitLength, llCode); /* 16 */ /* 26 */ if (MEM_32bits()) BIT_flushBits(&blockStream); /* (7)*/
if (ofBits + mlBits + llBits > 64 - 7 - (LLFSELog+MLFSELog+OffFSELog)) FSE_encodeSymbol(&blockStream, &stateLitLength, llCode); /* 16 */ /* 33 */
BIT_flushBits(&blockStream); if (MEM_32bits() || (ofBits+mlBits+llBits > 64-7-(LLFSELog+MLFSELog+OffFSELog)))
BIT_flushBits(&blockStream); /* (7)*/
BIT_addBits(&blockStream, llTable[n], llBits); BIT_addBits(&blockStream, llTable[n], llBits);
if (MEM_32bits() && ((llBits+mlBits)>24)) BIT_flushBits(&blockStream);
BIT_addBits(&blockStream, mlTable[n], mlBits); BIT_addBits(&blockStream, mlTable[n], mlBits);
BIT_addBits(&blockStream, offsetTable[n], ofBits); /* 31 */ /* 61 */ /* 24 bits max in 32-bits mode */ if (MEM_32bits()) BIT_flushBits(&blockStream); /* (7)*/
BIT_flushBits(&blockStream); /* 7 */ /* 7 */ BIT_addBits(&blockStream, offsetTable[n], ofBits); /* 31 */
BIT_flushBits(&blockStream); /* (7)*/
} } } }
FSE_flushCState(&blockStream, &stateMatchLength); FSE_flushCState(&blockStream, &stateMatchLength);
FSE_flushCState(&blockStream, &stateOffsetBits); FSE_flushCState(&blockStream, &stateOffsetBits);
FSE_flushCState(&blockStream, &stateLitLength); FSE_flushCState(&blockStream, &stateLitLength);
{ size_t const streamSize = BIT_closeCStream(&blockStream); { size_t const streamSize = BIT_closeCStream(&blockStream);
if (streamSize==0) return ERROR(dstSize_tooSmall); /* not enough space */ if (streamSize==0) return ERROR(dstSize_tooSmall); /* not enough space */
op += streamSize; } op += streamSize;
} } }
/* check compressibility */ /* check compressibility */
_check_compressibility: _check_compressibility:
@@ -798,8 +803,8 @@ MEM_STATIC void ZSTD_storeSeq(seqStore_t* seqStorePtr, size_t litLength, const B
static const BYTE* g_start = NULL; static const BYTE* g_start = NULL;
const U32 pos = (U32)(literals - g_start); const U32 pos = (U32)(literals - g_start);
if (g_start==NULL) g_start = literals; if (g_start==NULL) g_start = literals;
if ((pos > 15181500) && (pos < 15183150)) if ((pos > 200000000) && (pos < 200900000))
printf("Cpos %6u :%4u literals & match %3u bytes at distance %6u \n", printf("Cpos %6u :%5u literals & match %3u bytes at distance %6u \n",
pos, (U32)litLength, (U32)matchCode+MINMATCH, (U32)offsetCode); pos, (U32)litLength, (U32)matchCode+MINMATCH, (U32)offsetCode);
#endif #endif
#if ZSTD_OPT_DEBUG == 3 #if ZSTD_OPT_DEBUG == 3
lib/zstd_decompress.c
+14 -11
View File
@@ -637,8 +637,6 @@ static void ZSTD_decodeSequence(seq_t* seq, seqState_t* seqState, const U32 mls)
U32 const ofBits = ofCode; U32 const ofBits = ofCode;
U32 const totalBits = llBits+mlBits+ofBits; U32 const totalBits = llBits+mlBits+ofBits;
//size_t const allBits = BIT_readBits(&(seqState->DStream), totalBits);
static const U32 LL_base[MaxLL+1] = { static const U32 LL_base[MaxLL+1] = {
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
16, 18, 20, 22, 24, 28, 32, 40, 48, 64, 0x80, 0x100, 0x200, 0x400, 0x800, 0x1000, 16, 18, 20, 22, 24, 28, 32, 40, 48, 64, 0x80, 0x100, 0x200, 0x400, 0x800, 0x1000,
@@ -657,20 +655,25 @@ static void ZSTD_decodeSequence(seq_t* seq, seqState_t* seqState, const U32 mls)
0xFFFFFF, 0x1FFFFFF, 0x3FFFFFF, /*fake*/ 1, 1, 1, 1, 1 }; 0xFFFFFF, 0x1FFFFFF, 0x3FFFFFF, /*fake*/ 1, 1, 1, 1, 1 };
/* sequence */ /* sequence */
{ size_t const offset = ofCode ? OF_base[ofCode] + BIT_readBits(&(seqState->DStream), ofBits) : { size_t const offset = ofCode ? OF_base[ofCode] + BIT_readBits(&(seqState->DStream), ofBits) : /* <= 26 bits */
llCode ? seq->offset : seqState->prevOffset; llCode ? seq->offset : seqState->prevOffset;
if (MEM_32bits()) BIT_reloadDStream(&(seqState->DStream));
if (ofCode | !llCode) seqState->prevOffset = seq->offset; /* cmove */ if (ofCode | !llCode) seqState->prevOffset = seq->offset; /* cmove */
seq->offset = offset; seq->offset = offset;
} }
seq->matchLength = ML_base[mlCode] + mls + ((mlCode>31) ? BIT_readBits(&(seqState->DStream), mlBits) : 0);
seq->litLength = LL_base[llCode] + ((llCode>15) ? BIT_readBits(&(seqState->DStream), llBits) : 0);
if (totalBits > 64 - 7 - (LLFSELog+MLFSELog+OffFSELog)) BIT_reloadDStream(&(seqState->DStream)); seq->matchLength = ML_base[mlCode] + mls + ((mlCode>31) ? BIT_readBits(&(seqState->DStream), mlBits) : 0); /* <= 16 bits */
if (MEM_32bits() && (mlBits+llBits>24)) BIT_reloadDStream(&(seqState->DStream));
seq->litLength = LL_base[llCode] + ((llCode>15) ? BIT_readBits(&(seqState->DStream), llBits) : 0); /* <= 16 bits */
if (MEM_32bits() ||
(totalBits > 64 - 7 - (LLFSELog+MLFSELog+OffFSELog)) ) BIT_reloadDStream(&(seqState->DStream));
/* ANS state update */ /* ANS state update */
FSE_updateState(&(seqState->stateLL), &(seqState->DStream)); FSE_updateState(&(seqState->stateLL), &(seqState->DStream)); /* <= 9 bits */
FSE_updateState(&(seqState->stateML), &(seqState->DStream)); FSE_updateState(&(seqState->stateML), &(seqState->DStream)); /* <= 9 bits */
FSE_updateState(&(seqState->stateOffb), &(seqState->DStream)); if (MEM_32bits()) BIT_reloadDStream(&(seqState->DStream)); /* <= 18 bits */
FSE_updateState(&(seqState->stateOffb), &(seqState->DStream)); /* <= 8 bits */
} }
@@ -795,8 +798,8 @@ static size_t ZSTD_decompressSequences(
ZSTD_decodeSequence(&sequence, &seqState, mls); ZSTD_decodeSequence(&sequence, &seqState, mls);
#if 0 /* for debug */ #if 0 /* for debug */
{ U32 pos = (U32)(op-base); { U32 pos = (U32)(op-base);
if ((pos > 15181500) && (pos < 15183150)) if ((pos > 200802300) && (pos < 200802400))
printf("Dpos %6u : %3u literals & match %3u bytes at distance %6u \n", printf("Dpos %6u :%5u literals & match %3u bytes at distance %6u \n",
pos, (U32)sequence.litLength, (U32)sequence.matchLength, (U32)sequence.offset); pos, (U32)sequence.litLength, (U32)sequence.matchLength, (U32)sequence.offset);
} }
#endif #endif
programs/bench.c
+3 -2
View File
@@ -195,14 +195,13 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
size_t const blockSize = (g_blockSize ? g_blockSize : srcSize) + (!srcSize); /* avoid div by 0 */ size_t const blockSize = (g_blockSize ? g_blockSize : srcSize) + (!srcSize); /* avoid div by 0 */
U32 const maxNbBlocks = (U32) ((srcSize + (blockSize-1)) / blockSize) + nbFiles; U32 const maxNbBlocks = (U32) ((srcSize + (blockSize-1)) / blockSize) + nbFiles;
blockParam_t* const blockTable = (blockParam_t*) malloc(maxNbBlocks * sizeof(blockParam_t)); blockParam_t* const blockTable = (blockParam_t*) malloc(maxNbBlocks * sizeof(blockParam_t));
const size_t maxCompressedSize = ZSTD_compressBound(srcSize) + (maxNbBlocks * 1024); /* add some room for safety */ size_t const maxCompressedSize = ZSTD_compressBound(srcSize) + (maxNbBlocks * 1024); /* add some room for safety */
void* const compressedBuffer = malloc(maxCompressedSize); void* const compressedBuffer = malloc(maxCompressedSize);
void* const resultBuffer = malloc(srcSize); void* const resultBuffer = malloc(srcSize);
ZSTD_CCtx* refCtx = ZSTD_createCCtx(); ZSTD_CCtx* refCtx = ZSTD_createCCtx();
ZSTD_CCtx* ctx = ZSTD_createCCtx(); ZSTD_CCtx* ctx = ZSTD_createCCtx();
ZSTD_DCtx* refDCtx = ZSTD_createDCtx(); ZSTD_DCtx* refDCtx = ZSTD_createDCtx();
ZSTD_DCtx* dctx = ZSTD_createDCtx(); ZSTD_DCtx* dctx = ZSTD_createDCtx();
U64 const crcOrig = XXH64(srcBuffer, srcSize, 0);
U32 nbBlocks; U32 nbBlocks;
/* checks */ /* checks */
@@ -240,6 +239,7 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
/* Bench */ /* Bench */
{ double fastestC = 100000000., fastestD = 100000000.; { double fastestC = 100000000., fastestD = 100000000.;
U64 const crcOrig = XXH64(srcBuffer, srcSize, 0);
clock_t coolTime = clock(); clock_t coolTime = clock();
U32 testNb; U32 testNb;
@@ -306,6 +306,7 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
if (ZSTD_isError(regenSize)) { if (ZSTD_isError(regenSize)) {
DISPLAY("ZSTD_decompress_usingPreparedDCtx() failed on block %u : %s \n", DISPLAY("ZSTD_decompress_usingPreparedDCtx() failed on block %u : %s \n",
blockNb, ZSTD_getErrorName(regenSize)); blockNb, ZSTD_getErrorName(regenSize));
clockStart -= clockLoop+1; /* force immediate test end */
break; break;
} }
blockTable[blockNb].resSize = regenSize; blockTable[blockNb].resSize = regenSize;