ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'dev' into oss

Browse Source
This commit is contained in:
Bimba Shrestha
2019-11-22 17:29:33 -08:00
committed by GitHub
parent 707a12c419 d4ce04c3e7
commit 826b555463
16 changed files with 137 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/Makefile
+1 -1
View File
@@ -161,7 +161,7 @@ ifneq (,$(filter Windows%,$(OS)))
LIBZSTD = dll\libzstd.dll
$(LIBZSTD): $(ZSTD_FILES)
@echo compiling dynamic library $(LIBVER)
$(CC) $(FLAGS) -DZSTD_DLL_EXPORT=1 -Wl,--out-implib,dll\libzstd.lib -shared $^ -o $@
$(CC) $(FLAGS) -DZSTD_DLL_EXPORT=1 -Wl,--out-implib,dll\libzstd.dll.a -shared $^ -o $@
else
lib/compress/zstd_compress.c
+19 -17
View File
@@ -1336,10 +1336,7 @@ ZSTD_reset_matchState(ZSTD_matchState_t* ms,
DEBUGLOG(4, "reset indices : %u", forceResetIndex == ZSTDirp_reset);
if (forceResetIndex == ZSTDirp_reset) {
memset(&ms->window, 0, sizeof(ms->window));
ms->window.dictLimit = 1; /* start from 1, so that 1st position is valid */
ms->window.lowLimit = 1; /* it ensures first and later CCtx usages compress the same */
ms->window.nextSrc = ms->window.base + 1; /* see issue #1241 */
ZSTD_window_init(&ms->window);
ZSTD_cwksp_mark_tables_dirty(ws);
}
@@ -1432,7 +1429,7 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc,
size_t const matchStateSize = ZSTD_sizeof_matchState(&params.cParams, /* forCCtx */ 1);
size_t const maxNbLdmSeq = ZSTD_ldm_getMaxNbSeq(params.ldmParams, blockSize);
ZSTD_indexResetPolicy_e needsIndexReset = ZSTDirp_continue;
ZSTD_indexResetPolicy_e needsIndexReset = zc->initialized ? ZSTDirp_continue : ZSTDirp_reset;
if (ZSTD_indexTooCloseToMax(zc->blockState.matchState.window)) {
needsIndexReset = ZSTDirp_reset;
@@ -1557,11 +1554,12 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc,
zc->ldmSequences = (rawSeq*)ZSTD_cwksp_reserve_aligned(ws, maxNbLdmSeq * sizeof(rawSeq));
zc->maxNbLdmSequences = maxNbLdmSeq;
memset(&zc->ldmState.window, 0, sizeof(zc->ldmState.window));
ZSTD_window_init(&zc->ldmState.window);
ZSTD_window_clear(&zc->ldmState.window);
}
DEBUGLOG(3, "wksp: finished allocating, %zd bytes remain available", ZSTD_cwksp_available_space(ws));
zc->initialized = 1;
return 0;
}
@@ -2593,23 +2591,25 @@ static size_t ZSTD_compress_frameChunk (ZSTD_CCtx* cctx,
/* Ensure hash/chain table insertion resumes no sooner than lowlimit */
if (ms->nextToUpdate < ms->window.lowLimit) ms->nextToUpdate = ms->window.lowLimit;
{ int useTargetCBlockSize = ZSTD_useTargetCBlockSize(&cctx->appliedParams);
size_t cSize = 0;
{ size_t cSize;
int useTargetCBlockSize = ZSTD_useTargetCBlockSize(&cctx->appliedParams);
if (useTargetCBlockSize) {
cSize = ZSTD_compressBlock_targetCBlockSize(cctx, op, dstCapacity, ip, blockSize, lastBlock);
FORWARD_IF_ERROR(cSize);
} else {
cSize = ZSTD_compressBlock_internal(cctx,
op+ZSTD_blockHeaderSize, dstCapacity-ZSTD_blockHeaderSize,
ip, blockSize, 0 /* frame */);
ip, blockSize, 1 /* frame */);
FORWARD_IF_ERROR(cSize);
if (cSize == 0) { /* block is not compressible */
cSize = ZSTD_noCompressBlock(op, dstCapacity, ip, blockSize, lastBlock);
FORWARD_IF_ERROR(cSize);
} else {
U32 const cBlockHeader24 = lastBlock + (((U32)bt_compressed)<<1) + (U32)(cSize << 3);
MEM_writeLE24(op, cBlockHeader24);
U32 const cBlockHeader = cSize == 1 ?
lastBlock + (((U32)bt_rle)<<1) + (U32)(blockSize << 3) :
lastBlock + (((U32)bt_compressed)<<1) + (U32)(cSize << 3);
MEM_writeLE24(op, cBlockHeader);
cSize += ZSTD_blockHeaderSize;
}
}
@@ -3767,11 +3767,11 @@ static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs,
ZSTD_EndDirective const flushMode)
{
const char* const istart = (const char*)input->src;
const char* const iend = istart + input->size;
const char* ip = istart + input->pos;
const char* const iend = input->size != 0 ? istart + input->size : istart;
const char* ip = input->pos != 0 ? istart + input->pos : istart;
char* const ostart = (char*)output->dst;
char* const oend = ostart + output->size;
char* op = ostart + output->pos;
char* const oend = output->size != 0 ? ostart + output->size : ostart;
char* op = output->pos != 0 ? ostart + output->pos : ostart;
U32 someMoreWork = 1;
/* check expectations */
@@ -3810,7 +3810,8 @@ static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs,
zcs->inBuff + zcs->inBuffPos, toLoad,
ip, iend-ip);
zcs->inBuffPos += loaded;
ip += loaded;
if (loaded != 0)
ip += loaded;
if ( (flushMode == ZSTD_e_continue)
&& (zcs->inBuffPos < zcs->inBuffTarget) ) {
/* not enough input to fill full block : stop here */
@@ -3870,7 +3871,8 @@ static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs,
zcs->outBuff + zcs->outBuffFlushedSize, toFlush);
DEBUGLOG(5, "toFlush: %u into %u ==> flushed: %u",
(unsigned)toFlush, (unsigned)(oend-op), (unsigned)flushed);
op += flushed;
if (flushed)
op += flushed;
zcs->outBuffFlushedSize += flushed;
if (toFlush!=flushed) {
/* flush not fully completed, presumably because dst is too small */
lib/compress/zstd_compress_internal.h
+14
View File
@@ -249,6 +249,7 @@ struct ZSTD_CCtx_s {
size_t staticSize;
SeqCollector seqCollector;
int isFirstBlock;
int initialized;
seqStore_t seqStore; /* sequences storage ptrs */
ldmState_t ldmState; /* long distance matching state */
@@ -858,6 +859,15 @@ ZSTD_checkDictValidity(const ZSTD_window_t* window,
} } }
}
MEM_STATIC void ZSTD_window_init(ZSTD_window_t* window) {
memset(window, 0, sizeof(*window));
window->base = (BYTE const*)"";
window->dictBase = (BYTE const*)"";
window->dictLimit = 1; /* start from 1, so that 1st position is valid */
window->lowLimit = 1; /* it ensures first and later CCtx usages compress the same */
window->nextSrc = window->base + 1; /* see issue #1241 */
}
/**
* ZSTD_window_update():
* Updates the window by appending [src, src + srcSize) to the window.
@@ -871,6 +881,10 @@ MEM_STATIC U32 ZSTD_window_update(ZSTD_window_t* window,
BYTE const* const ip = (BYTE const*)src;
U32 contiguous = 1;
DEBUGLOG(5, "ZSTD_window_update");
if (srcSize == 0)
return contiguous;
assert(window->base != NULL);
assert(window->dictBase != NULL);
/* Check if blocks follow each other */
if (src != window->nextSrc) {
/* not contiguous */
lib/compress/zstd_double_fast.c
+2 -2
View File
@@ -96,7 +96,7 @@ size_t ZSTD_compressBlock_doubleFast_generic(
dictCParams->hashLog : hBitsL;
const U32 dictHBitsS = dictMode == ZSTD_dictMatchState ?
dictCParams->chainLog : hBitsS;
const U32 dictAndPrefixLength = (U32)(ip - prefixLowest + dictEnd - dictStart);
const U32 dictAndPrefixLength = (U32)((ip - prefixLowest) + (dictEnd - dictStart));
DEBUGLOG(5, "ZSTD_compressBlock_doubleFast_generic");
@@ -271,7 +271,7 @@ _match_stored:
U32 const repIndex2 = current2 - offset_2;
const BYTE* repMatch2 = dictMode == ZSTD_dictMatchState
&& repIndex2 < prefixLowestIndex ?
dictBase - dictIndexDelta + repIndex2 :
dictBase + repIndex2 - dictIndexDelta :
base + repIndex2;
if ( ((U32)((prefixLowestIndex-1) - (U32)repIndex2) >= 3 /* intentional overflow */)
&& (MEM_read32(repMatch2) == MEM_read32(ip)) ) {
lib/compress/zstd_lazy.c
+1 -1
View File
@@ -660,7 +660,7 @@ ZSTD_compressBlock_lazy_generic(
const U32 dictIndexDelta = dictMode == ZSTD_dictMatchState ?
prefixLowestIndex - (U32)(dictEnd - dictBase) :
0;
const U32 dictAndPrefixLength = (U32)(ip - prefixLowest + dictEnd - dictLowest);
const U32 dictAndPrefixLength = (U32)((ip - prefixLowest) + (dictEnd - dictLowest));
/* init */
ip += (dictAndPrefixLength == 0);
lib/compress/zstdmt_compress.c
+2 -2
View File
@@ -490,7 +490,7 @@ static int ZSTDMT_serialState_reset(serialState_t* serialState, ZSTDMT_seqPool*
/* Size the seq pool tables */
ZSTDMT_setNbSeq(seqPool, ZSTD_ldm_getMaxNbSeq(params.ldmParams, jobSize));
/* Reset the window */
ZSTD_window_clear(&serialState->ldmState.window);
ZSTD_window_init(&serialState->ldmState.window);
serialState->ldmWindow = serialState->ldmState.window;
/* Resize tables and output space if necessary. */
if (serialState->ldmState.hashTable == NULL || serialState->params.ldmParams.hashLog < hashLog) {
@@ -1786,7 +1786,7 @@ static int ZSTDMT_isOverlapped(buffer_t buffer, range_t range)
BYTE const* const bufferStart = (BYTE const*)buffer.start;
BYTE const* const bufferEnd = bufferStart + buffer.capacity;
BYTE const* const rangeStart = (BYTE const*)range.start;
BYTE const* const rangeEnd = rangeStart + range.size;
BYTE const* const rangeEnd = range.size != 0 ? rangeStart + range.size : rangeStart;
if (rangeStart == NULL || bufferStart == NULL)
return 0;
lib/decompress/huf_decompress.c
+22 -10
View File
@@ -181,17 +181,29 @@ size_t HUF_readDTableX1_wksp(HUF_DTable* DTable, const void* src, size_t srcSize
/* fill DTable */
{ U32 n;
for (n=0; n<nbSymbols; n++) {
U32 const w = huffWeight[n];
U32 const length = (1 << w) >> 1;
U32 u;
size_t const nEnd = nbSymbols;
for (n=0; n<nEnd; n++) {
size_t const w = huffWeight[n];
size_t const length = (1 << w) >> 1;
size_t const uStart = rankVal[w];
size_t const uEnd = uStart + length;
size_t u;
HUF_DEltX1 D;
D.byte = (BYTE)n; D.nbBits = (BYTE)(tableLog + 1 - w);
for (u = rankVal[w]; u < rankVal[w] + length; u++)
dt[u] = D;
rankVal[w] += length;
} }
D.byte = (BYTE)n;
D.nbBits = (BYTE)(tableLog + 1 - w);
rankVal[w] = (U32)uEnd;
if (length < 4) {
/* Use length in the loop bound so the compiler knows it is short. */
for (u = 0; u < length; ++u)
dt[uStart + u] = D;
} else {
/* Unroll the loop 4 times, we know it is a power of 2. */
for (u = uStart; u < uEnd; u += 4) {
dt[u + 0] = D;
dt[u + 1] = D;
dt[u + 2] = D;
dt[u + 3] = D;
} } } }
return iSize;
}
lib/decompress/zstd_decompress.c
+12 -7
View File
@@ -618,7 +618,7 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx,
{
const BYTE* ip = (const BYTE*)(*srcPtr);
BYTE* const ostart = (BYTE* const)dst;
BYTE* const oend = ostart + dstCapacity;
BYTE* const oend = dstCapacity != 0 ? ostart + dstCapacity : ostart;
BYTE* op = ostart;
size_t remainingSrcSize = *srcSizePtr;
@@ -669,7 +669,9 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx,
if (ZSTD_isError(decodedSize)) return decodedSize;
if (dctx->fParams.checksumFlag)
XXH64_update(&dctx->xxhState, op, decodedSize);
op += decodedSize;
if (decodedSize != 0)
op += decodedSize;
assert(ip != NULL);
ip += cBlockSize;
remainingSrcSize -= cBlockSize;
if (blockProperties.lastBlock) break;
@@ -776,7 +778,8 @@ static size_t ZSTD_decompressMultiFrame(ZSTD_DCtx* dctx,
"error.");
if (ZSTD_isError(res)) return res;
assert(res <= dstCapacity);
dst = (BYTE*)dst + res;
if (res != 0)
dst = (BYTE*)dst + res;
dstCapacity -= res;
}
moreThan1Frame = 1;
@@ -1486,11 +1489,13 @@ MEM_STATIC size_t ZSTD_limitCopy(void* dst, size_t dstCapacity, const void* src,
size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inBuffer* input)
{
const char* const istart = (const char*)(input->src) + input->pos;
const char* const iend = (const char*)(input->src) + input->size;
const char* const src = (const char*)input->src;
const char* const istart = input->pos != 0 ? src + input->pos : src;
const char* const iend = input->size != 0 ? src + input->size : src;
const char* ip = istart;
char* const ostart = (char*)(output->dst) + output->pos;
char* const oend = (char*)(output->dst) + output->size;
char* const dst = (char*)output->dst;
char* const ostart = output->pos != 0 ? dst + output->pos : dst;
char* const oend = output->size != 0 ? dst + output->size : dst;
char* op = ostart;
U32 someMoreWork = 1;
lib/zstd.h
+1 -1
View File
@@ -763,7 +763,7 @@ ZSTDLIB_API size_t ZSTD_freeDStream(ZSTD_DStream* zds);
/* This function is redundant with the advanced API and equivalent to:
*
* ZSTD_DCtx_reset(zds);
* ZSTD_DCtx_reset(zds, ZSTD_reset_session_only);
* ZSTD_DCtx_refDDict(zds, NULL);
*/
ZSTDLIB_API size_t ZSTD_initDStream(ZSTD_DStream* zds);