skips the need to create a temporary cdict for --patch-from
thus saving a bit of memory and a little bit of cpu time
This commit is contained in:
@@ -714,7 +714,7 @@ static void ZSTDMT_compressionJob(void* jobDescription)
|
|||||||
size_t const initError = ZSTD_compressBegin_advanced_internal(cctx, NULL, 0, ZSTD_dct_auto, ZSTD_dtlm_fast, job->cdict, &jobParams, job->fullFrameSize);
|
size_t const initError = ZSTD_compressBegin_advanced_internal(cctx, NULL, 0, ZSTD_dct_auto, ZSTD_dtlm_fast, job->cdict, &jobParams, job->fullFrameSize);
|
||||||
assert(job->firstJob); /* only allowed for first job */
|
assert(job->firstJob); /* only allowed for first job */
|
||||||
if (ZSTD_isError(initError)) JOB_ERROR(initError);
|
if (ZSTD_isError(initError)) JOB_ERROR(initError);
|
||||||
} else { /* srcStart points at reloaded section */
|
} else {
|
||||||
U64 const pledgedSrcSize = job->firstJob ? job->fullFrameSize : job->src.size;
|
U64 const pledgedSrcSize = job->firstJob ? job->fullFrameSize : job->src.size;
|
||||||
{ size_t const forceWindowError = ZSTD_CCtxParams_setParameter(&jobParams, ZSTD_c_forceMaxWindow, !job->firstJob);
|
{ size_t const forceWindowError = ZSTD_CCtxParams_setParameter(&jobParams, ZSTD_c_forceMaxWindow, !job->firstJob);
|
||||||
if (ZSTD_isError(forceWindowError)) JOB_ERROR(forceWindowError);
|
if (ZSTD_isError(forceWindowError)) JOB_ERROR(forceWindowError);
|
||||||
@@ -741,7 +741,7 @@ static void ZSTDMT_compressionJob(void* jobDescription)
|
|||||||
ZSTD_invalidateRepCodes(cctx);
|
ZSTD_invalidateRepCodes(cctx);
|
||||||
}
|
}
|
||||||
|
|
||||||
/* compress */
|
/* compress the entire job by smaller chunks, for better granularity */
|
||||||
{ size_t const chunkSize = 4*ZSTD_BLOCKSIZE_MAX;
|
{ size_t const chunkSize = 4*ZSTD_BLOCKSIZE_MAX;
|
||||||
int const nbChunks = (int)((job->src.size + (chunkSize-1)) / chunkSize);
|
int const nbChunks = (int)((job->src.size + (chunkSize-1)) / chunkSize);
|
||||||
const BYTE* ip = (const BYTE*) job->src.start;
|
const BYTE* ip = (const BYTE*) job->src.start;
|
||||||
@@ -1258,18 +1258,6 @@ size_t ZSTDMT_initCStream_internal(
|
|||||||
|
|
||||||
mtctx->params = params;
|
mtctx->params = params;
|
||||||
mtctx->frameContentSize = pledgedSrcSize;
|
mtctx->frameContentSize = pledgedSrcSize;
|
||||||
ZSTD_freeCDict(mtctx->cdictLocal);
|
|
||||||
if (dict) {
|
|
||||||
mtctx->cdictLocal = ZSTD_createCDict_advanced(dict, dictSize,
|
|
||||||
dictLoadMethod, dictContentType, /* note : a loadPrefix becomes an internal CDict */
|
|
||||||
params.cParams, mtctx->cMem);
|
|
||||||
mtctx->cdict = mtctx->cdictLocal;
|
|
||||||
if (mtctx->cdictLocal == NULL) return ERROR(memory_allocation);
|
|
||||||
} else {
|
|
||||||
mtctx->cdictLocal = NULL;
|
|
||||||
mtctx->cdict = cdict;
|
|
||||||
}
|
|
||||||
|
|
||||||
mtctx->targetPrefixSize = ZSTDMT_computeOverlapSize(¶ms);
|
mtctx->targetPrefixSize = ZSTDMT_computeOverlapSize(¶ms);
|
||||||
DEBUGLOG(4, "overlapLog=%i => %u KB", params.overlapLog, (U32)(mtctx->targetPrefixSize>>10));
|
DEBUGLOG(4, "overlapLog=%i => %u KB", params.overlapLog, (U32)(mtctx->targetPrefixSize>>10));
|
||||||
mtctx->targetSectionSize = params.jobSize;
|
mtctx->targetSectionSize = params.jobSize;
|
||||||
@@ -1331,9 +1319,31 @@ size_t ZSTDMT_initCStream_internal(
|
|||||||
mtctx->allJobsCompleted = 0;
|
mtctx->allJobsCompleted = 0;
|
||||||
mtctx->consumed = 0;
|
mtctx->consumed = 0;
|
||||||
mtctx->produced = 0;
|
mtctx->produced = 0;
|
||||||
|
|
||||||
|
/* update dictionary */
|
||||||
|
ZSTD_freeCDict(mtctx->cdictLocal);
|
||||||
|
mtctx->cdictLocal = NULL;
|
||||||
|
if (dict) {
|
||||||
|
if (dictContentType == ZSTD_dct_rawContent) {
|
||||||
|
mtctx->inBuff.prefix.start = (const BYTE*)dict;
|
||||||
|
mtctx->inBuff.prefix.size = dictSize;
|
||||||
|
} else {
|
||||||
|
/* note : a loadPrefix becomes an internal CDict */
|
||||||
|
mtctx->cdictLocal = ZSTD_createCDict_advanced(dict, dictSize,
|
||||||
|
dictLoadMethod, dictContentType,
|
||||||
|
params.cParams, mtctx->cMem);
|
||||||
|
mtctx->cdict = mtctx->cdictLocal;
|
||||||
|
if (mtctx->cdictLocal == NULL) return ERROR(memory_allocation);
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
mtctx->cdict = cdict;
|
||||||
|
}
|
||||||
|
|
||||||
if (ZSTDMT_serialState_reset(&mtctx->serial, mtctx->seqPool, params, mtctx->targetSectionSize,
|
if (ZSTDMT_serialState_reset(&mtctx->serial, mtctx->seqPool, params, mtctx->targetSectionSize,
|
||||||
dict, dictSize, dictContentType))
|
dict, dictSize, dictContentType))
|
||||||
return ERROR(memory_allocation);
|
return ERROR(memory_allocation);
|
||||||
|
|
||||||
|
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1543,6 +1553,11 @@ static Range ZSTDMT_getInputDataInUse(ZSTDMT_CCtx* mtctx)
|
|||||||
unsigned const lastJobID = mtctx->nextJobID;
|
unsigned const lastJobID = mtctx->nextJobID;
|
||||||
unsigned jobID;
|
unsigned jobID;
|
||||||
|
|
||||||
|
/* no need to check during first round */
|
||||||
|
size_t roundBuffCapacity = mtctx->roundBuff.capacity;
|
||||||
|
size_t nbJobs1stRoundMin = roundBuffCapacity / mtctx->targetSectionSize;
|
||||||
|
if (lastJobID < nbJobs1stRoundMin) return kNullRange;
|
||||||
|
|
||||||
for (jobID = firstJobID; jobID < lastJobID; ++jobID) {
|
for (jobID = firstJobID; jobID < lastJobID; ++jobID) {
|
||||||
unsigned const wJobID = jobID & mtctx->jobIDMask;
|
unsigned const wJobID = jobID & mtctx->jobIDMask;
|
||||||
size_t consumed;
|
size_t consumed;
|
||||||
@@ -1637,14 +1652,14 @@ static int ZSTDMT_tryGetInputRange(ZSTDMT_CCtx* mtctx)
|
|||||||
{
|
{
|
||||||
Range const inUse = ZSTDMT_getInputDataInUse(mtctx);
|
Range const inUse = ZSTDMT_getInputDataInUse(mtctx);
|
||||||
size_t const spaceLeft = mtctx->roundBuff.capacity - mtctx->roundBuff.pos;
|
size_t const spaceLeft = mtctx->roundBuff.capacity - mtctx->roundBuff.pos;
|
||||||
size_t const target = mtctx->targetSectionSize;
|
size_t const spaceNeeded = mtctx->targetSectionSize;
|
||||||
Buffer buffer;
|
Buffer buffer;
|
||||||
|
|
||||||
DEBUGLOG(5, "ZSTDMT_tryGetInputRange");
|
DEBUGLOG(5, "ZSTDMT_tryGetInputRange");
|
||||||
assert(mtctx->inBuff.buffer.start == NULL);
|
assert(mtctx->inBuff.buffer.start == NULL);
|
||||||
assert(mtctx->roundBuff.capacity >= target);
|
assert(mtctx->roundBuff.capacity >= spaceNeeded);
|
||||||
|
|
||||||
if (spaceLeft < target) {
|
if (spaceLeft < spaceNeeded) {
|
||||||
/* ZSTD_invalidateRepCodes() doesn't work for extDict variants.
|
/* ZSTD_invalidateRepCodes() doesn't work for extDict variants.
|
||||||
* Simply copy the prefix to the beginning in that case.
|
* Simply copy the prefix to the beginning in that case.
|
||||||
*/
|
*/
|
||||||
@@ -1663,7 +1678,7 @@ static int ZSTDMT_tryGetInputRange(ZSTDMT_CCtx* mtctx)
|
|||||||
mtctx->roundBuff.pos = prefixSize;
|
mtctx->roundBuff.pos = prefixSize;
|
||||||
}
|
}
|
||||||
buffer.start = mtctx->roundBuff.buffer + mtctx->roundBuff.pos;
|
buffer.start = mtctx->roundBuff.buffer + mtctx->roundBuff.pos;
|
||||||
buffer.capacity = target;
|
buffer.capacity = spaceNeeded;
|
||||||
|
|
||||||
if (ZSTDMT_isOverlapped(buffer, inUse)) {
|
if (ZSTDMT_isOverlapped(buffer, inUse)) {
|
||||||
DEBUGLOG(5, "Waiting for buffer...");
|
DEBUGLOG(5, "Waiting for buffer...");
|
||||||
|
|||||||
Reference in New Issue
Block a user