From 64bd68e44bf9bdef8b2f0e50f5b56c55b6fdc1d2 Mon Sep 17 00:00:00 2001 From: senhuang42 Date: Thu, 10 Sep 2020 13:42:26 -0400 Subject: [PATCH 01/68] Adjust ZSTD_createCDict_byReference() function, and check for cdict when using compressStream2 --- lib/compress/zstd_compress.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 61369687a..3283ce943 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -3469,9 +3469,12 @@ ZSTD_CDict* ZSTD_createCDict(const void* dict, size_t dictSize, int compressionL ZSTD_CDict* ZSTD_createCDict_byReference(const void* dict, size_t dictSize, int compressionLevel) { ZSTD_compressionParameters cParams = ZSTD_getCParams_internal(compressionLevel, ZSTD_CONTENTSIZE_UNKNOWN, dictSize); - return ZSTD_createCDict_advanced(dict, dictSize, + ZSTD_CDict* cdict = ZSTD_createCDict_advanced(dict, dictSize, ZSTD_dlm_byRef, ZSTD_dct_auto, cParams, ZSTD_defaultCMem); + if (cdict) + cdict->compressionLevel = compressionLevel == 0 ? ZSTD_CLEVEL_DEFAULT : compressionLevel; + return cdict; } size_t ZSTD_freeCDict(ZSTD_CDict* cdict) @@ -3989,6 +3992,8 @@ size_t ZSTD_compressStream2( ZSTD_CCtx* cctx, FORWARD_IF_ERROR( ZSTD_initLocalDict(cctx) , ""); /* Init the local dict if present. */ ZSTD_memset(&cctx->prefixDict, 0, sizeof(cctx->prefixDict)); /* single usage */ assert(prefixDict.dict==NULL || cctx->cdict==NULL); /* only one can be set */ + if (cctx->cdict) + cctx->requestedParams.compressionLevel = cctx->cdict->compressionLevel; /* let cdict take priority in terms of compression level */ DEBUGLOG(4, "ZSTD_compressStream2 : transparent init stage"); if (endOp == ZSTD_e_end) cctx->pledgedSrcSizePlusOne = input->size + 1; /* auto-fix pledgedSrcSize */ params.cParams = ZSTD_getCParamsFromCCtxParams( From 801513b5e75aaa2ed15d5139fed7a62c0cda3fdf Mon Sep 17 00:00:00 2001 From: senhuang42 Date: Fri, 11 Sep 2020 11:41:10 -0400 Subject: [PATCH 02/68] Modify params rather than cctx->requestedParams --- lib/compress/zstd_compress.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 3283ce943..e67e6b194 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -3993,11 +3993,10 @@ size_t ZSTD_compressStream2( ZSTD_CCtx* cctx, ZSTD_memset(&cctx->prefixDict, 0, sizeof(cctx->prefixDict)); /* single usage */ assert(prefixDict.dict==NULL || cctx->cdict==NULL); /* only one can be set */ if (cctx->cdict) - cctx->requestedParams.compressionLevel = cctx->cdict->compressionLevel; /* let cdict take priority in terms of compression level */ + params.compressionLevel = cctx->cdict->compressionLevel; /* let cdict take priority in terms of compression level */ DEBUGLOG(4, "ZSTD_compressStream2 : transparent init stage"); if (endOp == ZSTD_e_end) cctx->pledgedSrcSizePlusOne = input->size + 1; /* auto-fix pledgedSrcSize */ - params.cParams = ZSTD_getCParamsFromCCtxParams( - &cctx->requestedParams, cctx->pledgedSrcSizePlusOne-1, 0 /*dictSize*/); + params.cParams = ZSTD_getCParamsFromCCtxParams(¶ms, cctx->pledgedSrcSizePlusOne-1, 0 /*dictSize*/); #ifdef ZSTD_MULTITHREAD From 17b56f934eee6b905602982adf0f92ccb0d6aa89 Mon Sep 17 00:00:00 2001 From: senhuang42 Date: Fri, 11 Sep 2020 11:42:12 -0400 Subject: [PATCH 03/68] Coding style cleanup --- lib/compress/zstd_compress.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index e67e6b194..dc601217c 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -3458,22 +3458,22 @@ ZSTD_CDict* ZSTD_createCDict_advanced(const void* dictBuffer, size_t dictSize, ZSTD_CDict* ZSTD_createCDict(const void* dict, size_t dictSize, int compressionLevel) { ZSTD_compressionParameters cParams = ZSTD_getCParams_internal(compressionLevel, ZSTD_CONTENTSIZE_UNKNOWN, dictSize); - ZSTD_CDict* cdict = ZSTD_createCDict_advanced(dict, dictSize, + ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dict, dictSize, ZSTD_dlm_byCopy, ZSTD_dct_auto, cParams, ZSTD_defaultCMem); if (cdict) - cdict->compressionLevel = compressionLevel == 0 ? ZSTD_CLEVEL_DEFAULT : compressionLevel; + cdict->compressionLevel = (compressionLevel == 0) ? ZSTD_CLEVEL_DEFAULT : compressionLevel; return cdict; } ZSTD_CDict* ZSTD_createCDict_byReference(const void* dict, size_t dictSize, int compressionLevel) { ZSTD_compressionParameters cParams = ZSTD_getCParams_internal(compressionLevel, ZSTD_CONTENTSIZE_UNKNOWN, dictSize); - ZSTD_CDict* cdict = ZSTD_createCDict_advanced(dict, dictSize, + ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dict, dictSize, ZSTD_dlm_byRef, ZSTD_dct_auto, cParams, ZSTD_defaultCMem); if (cdict) - cdict->compressionLevel = compressionLevel == 0 ? ZSTD_CLEVEL_DEFAULT : compressionLevel; + cdict->compressionLevel = (compressionLevel == 0) ? ZSTD_CLEVEL_DEFAULT : compressionLevel; return cdict; } From 5618e000bd58bcce76d2a0af9570577be4062d8e Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Mon, 21 Sep 2020 07:19:20 -0700 Subject: [PATCH 04/68] update fullbench for compressStream2() makes it possible to measure scenarios such as #2314 --- tests/fullbench.c | 46 +++++++++++++++++++++++++++++++++++++--------- 1 file changed, 37 insertions(+), 9 deletions(-) diff --git a/tests/fullbench.c b/tests/fullbench.c index 2207239e8..37f0e242f 100644 --- a/tests/fullbench.c +++ b/tests/fullbench.c @@ -232,23 +232,40 @@ local_ZSTD_compressStream_freshCCtx(const void* src, size_t srcSize, r = local_ZSTD_compressStream(src, srcSize, dst, dstCapacity, payload); ZSTD_freeCCtx(cctx); - return r; } static size_t -local_ZSTD_compress_generic_end(const void* src, size_t srcSize, - void* dst, size_t dstCapacity, - void* payload) +local_ZSTD_compress2(const void* src, size_t srcSize, + void* dst, size_t dstCapacity, + void* payload) { (void)payload; return ZSTD_compress2(g_cstream, dst, dstCapacity, src, srcSize); } static size_t -local_ZSTD_compress_generic_continue(const void* src, size_t srcSize, - void* dst, size_t dstCapacity, - void* payload) +local_ZSTD_compressStream2_end(const void* src, size_t srcSize, + void* dst, size_t dstCapacity, + void* payload) +{ + ZSTD_outBuffer buffOut; + ZSTD_inBuffer buffIn; + (void)payload; + buffOut.dst = dst; + buffOut.size = dstCapacity; + buffOut.pos = 0; + buffIn.src = src; + buffIn.size = srcSize; + buffIn.pos = 0; + ZSTD_compressStream2(g_cstream, &buffOut, &buffIn, ZSTD_e_end); + return buffOut.pos; +} + +static size_t +local_ZSTD_compressStream2_continue(const void* src, size_t srcSize, + void* dst, size_t dstCapacity, + void* payload) { ZSTD_outBuffer buffOut; ZSTD_inBuffer buffIn; @@ -437,11 +454,17 @@ static int benchMem(unsigned benchNb, case 43: benchFunction = local_ZSTD_compressStream_freshCCtx; benchName = "compressStream_freshCCtx"; break; + case 50: + benchFunction = local_ZSTD_compress2; benchName = "compress2"; + break; case 51: - benchFunction = local_ZSTD_compress_generic_continue; benchName = "compress_generic, continue"; + benchFunction = local_ZSTD_compressStream2_end; benchName = "compressStream2, end"; break; case 52: - benchFunction = local_ZSTD_compress_generic_end; benchName = "compress_generic, end"; + benchFunction = local_ZSTD_compressStream2_end; benchName = "compressStream2, end & short"; + break; + case 53: + benchFunction = local_ZSTD_compressStream2_continue; benchName = "compressStream2, continue"; break; case 61: benchFunction = local_ZSTD_compress_generic_T2_continue; benchName = "compress_generic, -T2, continue"; @@ -573,6 +596,11 @@ static int benchMem(unsigned benchNb, payload = &cparams; break; + case 52 : + /* compressStream2, short dstCapacity */ + dstBuffSize--; + break; + /* test functions */ /* convention: test functions have ID > 100 */ From b684900a4ae7153f4a4131a99cbfc252f833aad8 Mon Sep 17 00:00:00 2001 From: Martin Liska Date: Fri, 25 Sep 2020 14:12:14 +0200 Subject: [PATCH 05/68] Allow external creation of POOLs that can be shared. --- examples/streaming_compression_thread_pool.c | 178 +++++++++++++++++++ lib/common/pool.c | 8 +- lib/compress/zstd_compress.c | 10 +- lib/compress/zstd_compress_internal.h | 1 + lib/compress/zstdmt_compress.c | 22 ++- lib/compress/zstdmt_compress.h | 3 +- lib/zstd.h | 17 +- 7 files changed, 228 insertions(+), 11 deletions(-) create mode 100644 examples/streaming_compression_thread_pool.c diff --git a/examples/streaming_compression_thread_pool.c b/examples/streaming_compression_thread_pool.c new file mode 100644 index 000000000..22c3b2efa --- /dev/null +++ b/examples/streaming_compression_thread_pool.c @@ -0,0 +1,178 @@ +/* + * Copyright (c) 2020, Martin Liska, SUSE, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + + +#include // printf +#include // free +#include // memset, strcat, strlen +#include // presumes zstd library is installed +#include "common.h" // Helper functions, CHECK(), and CHECK_ZSTD() +#include + +typedef struct compress_args +{ + const char *fname; + char *outName; + int cLevel; +#if defined(ZSTD_STATIC_LINKING_ONLY) + ZSTD_threadPool *pool; +#endif +} compress_args_t; + +static void *compressFile_orDie(void *data) +{ + compress_args_t *args = (compress_args_t *)data; + fprintf (stderr, "Starting compression of %s with level %d\n", args->fname, args->cLevel); + /* Open the input and output files. */ + FILE* const fin = fopen_orDie(args->fname, "rb"); + FILE* const fout = fopen_orDie(args->outName, "wb"); + /* Create the input and output buffers. + * They may be any size, but we recommend using these functions to size them. + * Performance will only suffer significantly for very tiny buffers. + */ + size_t const buffInSize = ZSTD_CStreamInSize(); + void* const buffIn = malloc_orDie(buffInSize); + size_t const buffOutSize = ZSTD_CStreamOutSize(); + void* const buffOut = malloc_orDie(buffOutSize); + + /* Create the context. */ + ZSTD_CCtx* const cctx = ZSTD_createCCtx(); + CHECK(cctx != NULL, "ZSTD_createCCtx() failed!"); + +#if defined(ZSTD_STATIC_LINKING_ONLY) + size_t r = ZSTD_CCtx_refThreadPool(cctx, args->pool); + CHECK(r == 0, "ZSTD_CCtx_refThreadPool failed!"); +#endif + + /* Set any parameters you want. + * Here we set the compression level, and enable the checksum. + */ + CHECK_ZSTD( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, args->cLevel) ); + CHECK_ZSTD( ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1) ); + ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 16); + + /* This loop read from the input file, compresses that entire chunk, + * and writes all output produced to the output file. + */ + size_t const toRead = buffInSize; + for (;;) { + size_t read = fread_orDie(buffIn, toRead, fin); + /* Select the flush mode. + * If the read may not be finished (read == toRead) we use + * ZSTD_e_continue. If this is the last chunk, we use ZSTD_e_end. + * Zstd optimizes the case where the first flush mode is ZSTD_e_end, + * since it knows it is compressing the entire source in one pass. + */ + int const lastChunk = (read < toRead); + ZSTD_EndDirective const mode = lastChunk ? ZSTD_e_end : ZSTD_e_continue; + /* Set the input buffer to what we just read. + * We compress until the input buffer is empty, each time flushing the + * output. + */ + ZSTD_inBuffer input = { buffIn, read, 0 }; + int finished; + do { + /* Compress into the output buffer and write all of the output to + * the file so we can reuse the buffer next iteration. + */ + ZSTD_outBuffer output = { buffOut, buffOutSize, 0 }; + size_t const remaining = ZSTD_compressStream2(cctx, &output , &input, mode); + CHECK_ZSTD(remaining); + fwrite_orDie(buffOut, output.pos, fout); + /* If we're on the last chunk we're finished when zstd returns 0, + * which means its consumed all the input AND finished the frame. + * Otherwise, we're finished when we've consumed all the input. + */ + finished = lastChunk ? (remaining == 0) : (input.pos == input.size); + } while (!finished); + CHECK(input.pos == input.size, + "Impossible: zstd only returns 0 when the input is completely consumed!"); + + if (lastChunk) { + break; + } + } + + fprintf (stderr, "Finishing compression of %s\n", args->outName); + + ZSTD_freeCCtx(cctx); + fclose_orDie(fout); + fclose_orDie(fin); + free(buffIn); + free(buffOut); + free(args->outName); + + return NULL; +} + + +static char* createOutFilename_orDie(const char* filename) +{ + size_t const inL = strlen(filename); + size_t const outL = inL + 5; + void* const outSpace = malloc_orDie(outL); + memset(outSpace, 0, outL); + strcat(outSpace, filename); + strcat(outSpace, ".zst"); + return (char*)outSpace; +} + +int main(int argc, const char** argv) +{ + const char* const exeName = argv[0]; + + if (argc<=3) { + printf("wrong arguments\n"); + printf("usage:\n"); + printf("%s POOL_SIZE LEVEL FILES\n", exeName); + return 1; + } + + int pool_size = atoi (argv[1]); + CHECK(pool_size != 0, "can't parse POOL_SIZE!"); + + int level = atoi (argv[2]); + CHECK(level != 0, "can't parse LEVEL!"); + + argc -= 3; + argv += 3; + +#if defined(ZSTD_STATIC_LINKING_ONLY) + ZSTD_threadPool *pool = ZSTD_createThreadPool (pool_size); + CHECK(pool != NULL, "ZSTD_createThreadPool() failed!"); + fprintf (stderr, "Using shared thread pool of size %d\n", pool_size); +#else + fprintf (stderr, "All threads use its own thread pool\n"); +#endif + + pthread_t *threads = malloc_orDie(argc * sizeof(pthread_t)); + compress_args_t *args = malloc_orDie(argc * sizeof(compress_args_t)); + + for (unsigned i = 0; i < argc; i++) + { + args[i].fname = argv[i]; + args[i].outName = createOutFilename_orDie(args[i].fname); + args[i].cLevel = level; +#if defined(ZSTD_STATIC_LINKING_ONLY) + args[i].pool = pool; +#endif + + pthread_create (&threads[i], NULL, compressFile_orDie, &args[i]); + } + + for (unsigned i = 0; i < argc; i++) + pthread_join (threads[i], NULL); + +#if defined(ZSTD_STATIC_LINKING_ONLY) + ZSTD_freeThreadPool (pool); +#endif + + return 0; +} diff --git a/lib/common/pool.c b/lib/common/pool.c index 5def2bd55..4c1b83376 100644 --- a/lib/common/pool.c +++ b/lib/common/pool.c @@ -105,6 +105,10 @@ static void* POOL_thread(void* opaque) { assert(0); /* Unreachable */ } +POOL_ctx* ZSTD_createThreadPool(size_t numThreads) { + return POOL_create (numThreads, 0); +} + POOL_ctx* POOL_create(size_t numThreads, size_t queueSize) { return POOL_create_advanced(numThreads, queueSize, ZSTD_defaultCMem); } @@ -184,7 +188,9 @@ void POOL_free(POOL_ctx *ctx) { ZSTD_customFree(ctx, ctx->customMem); } - +void ZSTD_freeThreadPool (ZSTD_threadPool* pool) { + POOL_free (pool); +} size_t POOL_sizeof(POOL_ctx *ctx) { if (ctx==NULL) return 0; /* supports sizeof NULL */ diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index fd47ccd3f..26bf1e12b 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -958,6 +958,14 @@ size_t ZSTD_CCtx_refCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict) return 0; } +size_t ZSTD_CCtx_refThreadPool(ZSTD_CCtx* cctx, ZSTD_threadPool* pool) +{ + RETURN_ERROR_IF(cctx->streamStage != zcss_init, stage_wrong, + "Can't ref a pool when ctx not in init stage."); + cctx->pool = pool; + return 0; +} + size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize) { return ZSTD_CCtx_refPrefix_advanced(cctx, prefix, prefixSize, ZSTD_dct_rawContent); @@ -4115,7 +4123,7 @@ size_t ZSTD_compressStream2( ZSTD_CCtx* cctx, if (cctx->mtctx == NULL) { DEBUGLOG(4, "ZSTD_compressStream2: creating new mtctx for nbWorkers=%u", params.nbWorkers); - cctx->mtctx = ZSTDMT_createCCtx_advanced((U32)params.nbWorkers, cctx->customMem); + cctx->mtctx = ZSTDMT_createCCtx_advanced((U32)params.nbWorkers, cctx->customMem, cctx->pool); RETURN_ERROR_IF(cctx->mtctx == NULL, memory_allocation, "NULL pointer!"); } /* mt compression */ diff --git a/lib/compress/zstd_compress_internal.h b/lib/compress/zstd_compress_internal.h index b161a208c..b00366da2 100644 --- a/lib/compress/zstd_compress_internal.h +++ b/lib/compress/zstd_compress_internal.h @@ -255,6 +255,7 @@ struct ZSTD_CCtx_s { unsigned long long producedCSize; XXH64_state_t xxhState; ZSTD_customMem customMem; + ZSTD_threadPool* pool; size_t staticSize; SeqCollector seqCollector; int isFirstBlock; diff --git a/lib/compress/zstdmt_compress.c b/lib/compress/zstdmt_compress.c index baf6ef4ca..b809d602e 100644 --- a/lib/compress/zstdmt_compress.c +++ b/lib/compress/zstdmt_compress.c @@ -831,6 +831,7 @@ struct ZSTDMT_CCtx_s { ZSTD_customMem cMem; ZSTD_CDict* cdictLocal; const ZSTD_CDict* cdict; + unsigned providedFactory: 1; }; static void ZSTDMT_freeJobsTable(ZSTDMT_jobDescription* jobTable, U32 nbJobs, ZSTD_customMem cMem) @@ -889,7 +890,7 @@ size_t ZSTDMT_CCtxParam_setNbWorkers(ZSTD_CCtx_params* params, unsigned nbWorker return ZSTD_CCtxParams_setParameter(params, ZSTD_c_nbWorkers, (int)nbWorkers); } -MEM_STATIC ZSTDMT_CCtx* ZSTDMT_createCCtx_advanced_internal(unsigned nbWorkers, ZSTD_customMem cMem) +MEM_STATIC ZSTDMT_CCtx* ZSTDMT_createCCtx_advanced_internal(unsigned nbWorkers, ZSTD_customMem cMem, ZSTD_threadPool* pool) { ZSTDMT_CCtx* mtctx; U32 nbJobs = nbWorkers + 2; @@ -907,7 +908,14 @@ MEM_STATIC ZSTDMT_CCtx* ZSTDMT_createCCtx_advanced_internal(unsigned nbWorkers, ZSTDMT_CCtxParam_setNbWorkers(&mtctx->params, nbWorkers); mtctx->cMem = cMem; mtctx->allJobsCompleted = 1; - mtctx->factory = POOL_create_advanced(nbWorkers, 0, cMem); + if (pool != NULL) { + mtctx->factory = pool; + mtctx->providedFactory = 1; + } + else { + mtctx->factory = POOL_create_advanced(nbWorkers, 0, cMem); + mtctx->providedFactory = 0; + } mtctx->jobs = ZSTDMT_createJobsTable(&nbJobs, cMem); assert(nbJobs > 0); assert((nbJobs & (nbJobs - 1)) == 0); /* ensure nbJobs is a power of 2 */ mtctx->jobIDMask = nbJobs - 1; @@ -924,20 +932,21 @@ MEM_STATIC ZSTDMT_CCtx* ZSTDMT_createCCtx_advanced_internal(unsigned nbWorkers, return mtctx; } -ZSTDMT_CCtx* ZSTDMT_createCCtx_advanced(unsigned nbWorkers, ZSTD_customMem cMem) +ZSTDMT_CCtx* ZSTDMT_createCCtx_advanced(unsigned nbWorkers, ZSTD_customMem cMem, ZSTD_threadPool* pool) { #ifdef ZSTD_MULTITHREAD - return ZSTDMT_createCCtx_advanced_internal(nbWorkers, cMem); + return ZSTDMT_createCCtx_advanced_internal(nbWorkers, cMem, pool); #else (void)nbWorkers; (void)cMem; + (void)pool; return NULL; #endif } ZSTDMT_CCtx* ZSTDMT_createCCtx(unsigned nbWorkers) { - return ZSTDMT_createCCtx_advanced(nbWorkers, ZSTD_defaultCMem); + return ZSTDMT_createCCtx_advanced(nbWorkers, ZSTD_defaultCMem, NULL); } @@ -983,7 +992,8 @@ static void ZSTDMT_waitForAllJobsCompleted(ZSTDMT_CCtx* mtctx) size_t ZSTDMT_freeCCtx(ZSTDMT_CCtx* mtctx) { if (mtctx==NULL) return 0; /* compatible with free on NULL */ - POOL_free(mtctx->factory); /* stop and free worker threads */ + if (!mtctx->providedFactory) + POOL_free(mtctx->factory); /* stop and free worker threads */ ZSTDMT_releaseAllJobResources(mtctx); /* release job resources into pools first */ ZSTDMT_freeJobsTable(mtctx->jobs, mtctx->jobIDMask+1, mtctx->cMem); ZSTDMT_freeBufferPool(mtctx->bufPool); diff --git a/lib/compress/zstdmt_compress.h b/lib/compress/zstdmt_compress.h index 623fd97df..8aad78c4f 100644 --- a/lib/compress/zstdmt_compress.h +++ b/lib/compress/zstdmt_compress.h @@ -60,7 +60,8 @@ typedef struct ZSTDMT_CCtx_s ZSTDMT_CCtx; ZSTDMT_API ZSTDMT_CCtx* ZSTDMT_createCCtx(unsigned nbWorkers); /* Requires ZSTD_MULTITHREAD to be defined during compilation, otherwise it will return NULL. */ ZSTDMT_API ZSTDMT_CCtx* ZSTDMT_createCCtx_advanced(unsigned nbWorkers, - ZSTD_customMem cMem); + ZSTD_customMem cMem, + ZSTD_threadPool *pool); ZSTDMT_API size_t ZSTDMT_freeCCtx(ZSTDMT_CCtx* mtctx); ZSTDMT_API size_t ZSTDMT_sizeof_CCtx(ZSTDMT_CCtx* mtctx); diff --git a/lib/zstd.h b/lib/zstd.h index 0a93ee6f6..c6043ed67 100644 --- a/lib/zstd.h +++ b/lib/zstd.h @@ -1409,6 +1409,21 @@ ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictS ZSTD_compressionParameters cParams, ZSTD_customMem customMem); +/* ! Thread pool : + * These prototypes make it possible to share a thread pool among multiple compression contexts. + * This can limit resources for applications with multiple threads where each one uses + * a threaded compression mode (via ZSTD_c_nbWorkers parameter). + * ZSTD_createThreadPool creates a new thread pool with a given number of threads. + * Note that the lifetime of such pool must exist while being used. + * ZSTD_CCtx_refThreadPool assigns a thread pool to a context (use NULL argument value + * to use an internal thread pool). + * ZSTD_freeThreadPool frees a thread pool. + */ +typedef struct POOL_ctx_s ZSTD_threadPool; +ZSTDLIB_API ZSTD_threadPool* ZSTD_createThreadPool(size_t numThreads); +ZSTDLIB_API void ZSTD_freeThreadPool (ZSTD_threadPool* pool); +ZSTDLIB_API size_t ZSTD_CCtx_refThreadPool(ZSTD_CCtx* cctx, ZSTD_threadPool* pool); + /** * This API is temporary and is expected to change or disappear in the future! */ @@ -1424,8 +1439,6 @@ ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictS ZSTD_dictContentType_e dictContentType, ZSTD_customMem customMem); - - /*************************************** * Advanced compression functions ***************************************/ From 1ebe360d0f3213cc4e4664b461b37a5eec04949b Mon Sep 17 00:00:00 2001 From: senhuang42 Date: Wed, 7 Oct 2020 13:42:34 -0400 Subject: [PATCH 06/68] Add new stdoutOutput field --- programs/fileio.c | 42 +++++++++++++++++++++++++++--------------- 1 file changed, 27 insertions(+), 15 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index 76e5cbf57..e20b9e28c 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -331,6 +331,7 @@ struct FIO_ctx_s { /* file i/o info */ int nbFilesTotal; int hasStdinInput; + int hasStdoutOutput; /* file i/o state */ int currFileIdx; @@ -388,6 +389,7 @@ FIO_ctx_t* FIO_createContext(void) ret->currFileIdx = 0; ret->hasStdinInput = 0; + ret->hasStdoutOutput = 0; ret->nbFilesTotal = 1; ret->nbFilesProcessed = 0; ret->totalBytesInput = 0; @@ -536,6 +538,10 @@ void FIO_setContentSize(FIO_prefs_t* const prefs, int value) /* FIO_ctx_t functions */ +void FIO_setHasStdoutOutput(FIO_ctx_t* const fCtx, int value) { + fCtx->hasStdoutOutput = value; +} + void FIO_setNbFilesTotal(FIO_ctx_t* const fCtx, int value) { fCtx->nbFilesTotal = value; @@ -841,6 +847,7 @@ static void FIO_adjustMemLimitForPatchFromMode(FIO_prefs_t* const prefs, * If -q is specified with --rm, zstd will abort pre-emptively * If neither flag is specified, zstd will prompt the user for confirmation to proceed. * If --rm is not specified, then zstd will print a warning to the user (which can be silenced with -q). + * However, if the output is stdout, we will always abort rather than displaying the warning prompt. */ static int FIO_removeMultiFilesWarning(FIO_ctx_t* const fCtx, const FIO_prefs_t* const prefs, const char* outFileName, int displayLevelCutoff) { @@ -859,7 +866,12 @@ static int FIO_removeMultiFilesWarning(FIO_ctx_t* const fCtx, const FIO_prefs_t* } DISPLAYLEVEL(2, "\nThe concatenated output CANNOT regenerate the original directory tree. ") if (prefs->removeSrcFile) { - error = g_display_prefs.displayLevel > displayLevelCutoff && UTIL_requireUserConfirmation("This is a destructive operation. Proceed? (y/n): ", "Aborting...", "yY", fCtx->hasStdinInput); + if (fCtx->hasStdoutOutput) { + DISPLAYLEVEL(1, "\nAborting. Use -f if you really want to delete the files and output to stdout"); + error = 1; + } else { + error = g_display_prefs.displayLevel > displayLevelCutoff && UTIL_requireUserConfirmation("This is a destructive operation. Proceed? (y/n): ", "Aborting...", "yY", fCtx->hasStdinInput); + } } } DISPLAY("\n"); @@ -1532,20 +1544,20 @@ FIO_compressFilename_internal(FIO_ctx_t* const fCtx, fCtx->totalBytesInput += (size_t)readsize; fCtx->totalBytesOutput += (size_t)compressedfilesize; DISPLAYLEVEL(2, "\r%79s\r", ""); - if (g_display_prefs.displayLevel >= 2) { - if (g_display_prefs.displayLevel >= 3 || fCtx->nbFilesTotal <= 1) { - if (readsize == 0) { - DISPLAYLEVEL(2,"%-20s : (%6llu => %6llu bytes, %s) \n", - srcFileName, - (unsigned long long)readsize, (unsigned long long) compressedfilesize, - dstFileName); - } else { - DISPLAYLEVEL(2,"%-20s :%6.2f%% (%6llu => %6llu bytes, %s) \n", - srcFileName, - (double)compressedfilesize / readsize * 100, - (unsigned long long)readsize, (unsigned long long) compressedfilesize, - dstFileName); - } + if (g_display_prefs.displayLevel >= 2 && + !fCtx->hasStdoutOutput && + (g_display_prefs.displayLevel >= 3 || fCtx->nbFilesTotal <= 1)) { + if (readsize == 0) { + DISPLAYLEVEL(2,"%-20s : (%6llu => %6llu bytes, %s) \n", + srcFileName, + (unsigned long long)readsize, (unsigned long long) compressedfilesize, + dstFileName); + } else { + DISPLAYLEVEL(2,"%-20s :%6.2f%% (%6llu => %6llu bytes, %s) \n", + srcFileName, + (double)compressedfilesize / readsize * 100, + (unsigned long long)readsize, (unsigned long long) compressedfilesize, + dstFileName); } } From dd3dac9ce056838fa90b1aacb5f7ba078b3af36e Mon Sep 17 00:00:00 2001 From: senhuang42 Date: Wed, 7 Oct 2020 13:43:27 -0400 Subject: [PATCH 07/68] Fixed logic for stdout output --- programs/fileio.c | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index e20b9e28c..e02c3f83a 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -2124,19 +2124,21 @@ FIO_decompressZstdFrame(FIO_ctx_t* const fCtx, dRess_t* ress, FILE* finput, /* Write block */ storedSkips = FIO_fwriteSparse(ress->dstFile, ress->dstBuffer, outBuff.pos, prefs, storedSkips); frameSize += outBuff.pos; - if (fCtx->nbFilesTotal > 1) { - size_t srcFileNameSize = strlen(srcFileName); - if (srcFileNameSize > 18) { - const char* truncatedSrcFileName = srcFileName + srcFileNameSize - 15; - DISPLAYUPDATE(2, "\rDecompress: %2u/%2u files. Current: ...%s : %u MB... ", - fCtx->currFileIdx+1, fCtx->nbFilesTotal, truncatedSrcFileName, (unsigned)((alreadyDecoded+frameSize)>>20) ); + if (!fCtx->hasStdoutOutput) { + if (fCtx->nbFilesTotal > 1) { + size_t srcFileNameSize = strlen(srcFileName); + if (srcFileNameSize > 18) { + const char* truncatedSrcFileName = srcFileName + srcFileNameSize - 15; + DISPLAYUPDATE(2, "\rDecompress: %2u/%2u files. Current: ...%s : %u MB... ", + fCtx->currFileIdx+1, fCtx->nbFilesTotal, truncatedSrcFileName, (unsigned)((alreadyDecoded+frameSize)>>20) ); + } else { + DISPLAYUPDATE(2, "\rDecompress: %2u/%2u files. Current: %s : %u MB... ", + fCtx->currFileIdx+1, fCtx->nbFilesTotal, srcFileName, (unsigned)((alreadyDecoded+frameSize)>>20) ); + } } else { - DISPLAYUPDATE(2, "\rDecompress: %2u/%2u files. Current: %s : %u MB... ", - fCtx->currFileIdx+1, fCtx->nbFilesTotal, srcFileName, (unsigned)((alreadyDecoded+frameSize)>>20) ); + DISPLAYUPDATE(2, "\r%-20.20s : %u MB... ", + srcFileName, (unsigned)((alreadyDecoded+frameSize)>>20) ); } - } else { - DISPLAYUPDATE(2, "\r%-20.20s : %u MB... ", - srcFileName, (unsigned)((alreadyDecoded+frameSize)>>20) ); } if (inBuff.pos > 0) { From 93cd9d8a6e3b4d2fc1f96d330e03fbd794a2c0d5 Mon Sep 17 00:00:00 2001 From: senhuang42 Date: Wed, 7 Oct 2020 13:44:25 -0400 Subject: [PATCH 08/68] Add hasStdoutOutput setter to fileio.h --- programs/fileio.h | 1 + 1 file changed, 1 insertion(+) diff --git a/programs/fileio.h b/programs/fileio.h index e290efcce..05e6d0681 100644 --- a/programs/fileio.h +++ b/programs/fileio.h @@ -107,6 +107,7 @@ void FIO_setContentSize(FIO_prefs_t* const prefs, int value); /* FIO_ctx_t functions */ void FIO_setNbFilesTotal(FIO_ctx_t* const fCtx, int value); +void FIO_setHasStdoutOutput(FIO_ctx_t* const fCtx, int value); void FIO_determineHasStdinInput(FIO_ctx_t* const fCtx, const FileNamesTable* const filenames); /*-************************************* From 7259b258d18dfd3de5ae43b4fb16bb314650ec1d Mon Sep 17 00:00:00 2001 From: senhuang42 Date: Wed, 7 Oct 2020 13:47:38 -0400 Subject: [PATCH 09/68] Add callsites to zstdcli.c and tests to playTests.sh --- programs/zstdcli.c | 6 +++++- tests/playTests.sh | 5 ++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/programs/zstdcli.c b/programs/zstdcli.c index b1893ca47..9b6f91533 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -697,6 +697,7 @@ int main(int const argCount, const char* argv[]) int argNb, followLinks = 0, forceStdout = 0, + hasStdout = 0, ldmFlag = 0, main_pause = 0, nbWorkers = 0, @@ -1282,9 +1283,12 @@ int main(int const argCount, const char* argv[]) } /* No status message in pipe mode (stdin - stdout) */ - if (!strcmp(filenames->fileNames[0], stdinmark) && outFileName && !strcmp(outFileName,stdoutmark) && (g_displayLevel==2)) g_displayLevel=1; + hasStdout = outFileName && !strcmp(outFileName,stdoutmark); + + if (hasStdout && (g_displayLevel==2)) g_displayLevel=1; /* IO Stream/File */ + FIO_setHasStdoutOutput(fCtx, hasStdout); FIO_setNbFilesTotal(fCtx, (int)filenames->tableSize); FIO_determineHasStdinInput(fCtx, filenames); FIO_setNotificationLevel(g_displayLevel); diff --git a/tests/playTests.sh b/tests/playTests.sh index 7f75d8506..fd602e7ae 100755 --- a/tests/playTests.sh +++ b/tests/playTests.sh @@ -396,9 +396,8 @@ cp tmp_rm3.zst tmp_rm4.zst echo 'Y' | zstd -d tmp_rm3.zst tmp_rm4.zst -o tmp_rm_out --rm test ! -f tmp_rm3.zst test ! -f tmp_rm4.zst -echo 'yes' | zstd tmp_rm_out tmp_rm3 -c --rm > $INTOVOID -test ! -f tmp_rm_out -test ! -f tmp_rm3 +echo 'yes' | zstd tmp_rm_out tmp_rm3 -c --rm && die "compressing multiple files to stdout with --rm should fail unless -f is specified" +echo 'yes' | zstd tmp_rm_out tmp_rm3 -c --rm -v && die "compressing multiple files to stdout with --rm should fail unless -f is specified" println gooder > tmpexists1 zstd tmpexists1 tmpexists -c --rm -f > $INTOVOID From e1ae398ad539038d2148bca87fc35b8837ff2f42 Mon Sep 17 00:00:00 2001 From: senhuang42 Date: Sat, 26 Sep 2020 11:07:52 -0400 Subject: [PATCH 10/68] Add rawSeqStore to match state --- lib/compress/zstd_compress_internal.h | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/lib/compress/zstd_compress_internal.h b/lib/compress/zstd_compress_internal.h index 728afade7..524cbb025 100644 --- a/lib/compress/zstd_compress_internal.h +++ b/lib/compress/zstd_compress_internal.h @@ -86,6 +86,19 @@ typedef struct { U32 len; } ZSTD_match_t; +typedef struct { + U32 offset; + U32 litLength; + U32 matchLength; +} rawSeq; + +typedef struct { + rawSeq* seq; /* The start of the sequences */ + size_t pos; /* The position where reading stopped. <= size. */ + size_t size; /* The number of sequences. <= capacity. */ + size_t capacity; /* The capacity starting from `seq` pointer */ +} rawSeqStore_t; + typedef struct { int price; U32 off; @@ -152,6 +165,7 @@ struct ZSTD_matchState_t { optState_t opt; /* optimal parser state */ const ZSTD_matchState_t* dictMatchState; ZSTD_compressionParameters cParams; + rawSeqStore_t ldmSeqStore; }; typedef struct { @@ -183,19 +197,6 @@ typedef struct { U32 windowLog; /* Window log for the LDM */ } ldmParams_t; -typedef struct { - U32 offset; - U32 litLength; - U32 matchLength; -} rawSeq; - -typedef struct { - rawSeq* seq; /* The start of the sequences */ - size_t pos; /* The position where reading stopped. <= size. */ - size_t size; /* The number of sequences. <= capacity. */ - size_t capacity; /* The capacity starting from `seq` pointer */ -} rawSeqStore_t; - typedef struct { int collectSequences; ZSTD_Sequence* seqStart; From 35d9f488f5d3dd8918cba5e44d946ce5ce17339d Mon Sep 17 00:00:00 2001 From: senhuang42 Date: Sat, 26 Sep 2020 11:13:39 -0400 Subject: [PATCH 11/68] Modify codepath to use opt parser exclusively if the compression level is high enough --- lib/compress/zstd_ldm.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/lib/compress/zstd_ldm.c b/lib/compress/zstd_ldm.c index dbfce3dce..c68693bd6 100644 --- a/lib/compress/zstd_ldm.c +++ b/lib/compress/zstd_ldm.c @@ -576,6 +576,13 @@ size_t ZSTD_ldm_blockCompress(rawSeqStore_t* rawSeqStore, /* Input positions */ BYTE const* ip = istart; + if (cParams->strategy >= ZSTD_btopt) { + size_t lastLLSize; + ms->ldmSeqStore = *rawSeqStore; /* copy current seqStore */ + lastLLSize = blockCompressor(ms, seqStore, rep, src, srcSize); + return lastLLSize; + } + DEBUGLOG(5, "ZSTD_ldm_blockCompress: srcSize=%zu", srcSize); assert(rawSeqStore->pos <= rawSeqStore->size); assert(rawSeqStore->size <= rawSeqStore->capacity); From aea61e3c91a4ea608fa2bbea4144f3ef1a2c6e43 Mon Sep 17 00:00:00 2001 From: senhuang42 Date: Sat, 26 Sep 2020 11:22:50 -0400 Subject: [PATCH 12/68] Add ldm helper function declarations into opt parser --- lib/compress/zstd_opt.c | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/lib/compress/zstd_opt.c b/lib/compress/zstd_opt.c index 5acc9e0b6..240b94cfa 100644 --- a/lib/compress/zstd_opt.c +++ b/lib/compress/zstd_opt.c @@ -764,6 +764,34 @@ FORCE_INLINE_TEMPLATE U32 ZSTD_BtGetAllMatches ( } } +/*-******************************* +* LDM util functions +*********************************/ + +static int ldm_splitSequence() { + +} + +/* Returns 1 if the rest of the block is just LDM literals */ +static int ldm_getNextMatch() { + int ret = ldm_splitSequence(); +} + +/* Adds an LDM if it's long enough */ +static void ldm_maybeAddLdm() { + +} + +/* Updates the pos field in rawSeqStore */ +static void ldm_maybeUpdateSeqStoreReadPos() { + +} + +/* Wrapper function to call ldm functions as needed */ +static void ldm_handleLdm() { + int noMoreLdms = getNextMatch(); +} + /*-******************************* * Optimal parser From db7076103295969c53e42c4af964dde2a0b61613 Mon Sep 17 00:00:00 2001 From: senhuang42 Date: Sat, 26 Sep 2020 11:24:29 -0400 Subject: [PATCH 13/68] Add callsites to appropriate locations in ..opt_generic() --- lib/compress/zstd_opt.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/compress/zstd_opt.c b/lib/compress/zstd_opt.c index 240b94cfa..3c3f22313 100644 --- a/lib/compress/zstd_opt.c +++ b/lib/compress/zstd_opt.c @@ -788,7 +788,7 @@ static void ldm_maybeUpdateSeqStoreReadPos() { } /* Wrapper function to call ldm functions as needed */ -static void ldm_handleLdm() { +static void ldm_handleLdm(int* nbMatches) { int noMoreLdms = getNextMatch(); } @@ -861,6 +861,7 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms, { U32 const litlen = (U32)(ip - anchor); U32 const ll0 = !litlen; U32 const nbMatches = ZSTD_BtGetAllMatches(matches, ms, &nextToUpdate3, ip, iend, dictMode, rep, ll0, minMatch); + ldm_handleLdm(&nbMatches); if (!nbMatches) { ip++; continue; } /* initialize opt[0] */ @@ -975,6 +976,8 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms, U32 const basePrice = previousPrice + ZSTD_litLengthPrice(0, optStatePtr, optLevel); U32 const nbMatches = ZSTD_BtGetAllMatches(matches, ms, &nextToUpdate3, inr, iend, dictMode, opt[cur].rep, ll0, minMatch); U32 matchNb; + + ldm_handleLdm(&nbMatches); if (!nbMatches) { DEBUGLOG(7, "rPos:%u : no match found", cur); continue; From 634ab7830da587ece201563be9c6836ef50ce6e4 Mon Sep 17 00:00:00 2001 From: senhuang42 Date: Sat, 26 Sep 2020 11:59:07 -0400 Subject: [PATCH 14/68] Flesh out required args for ldm_handleLdm() --- lib/compress/zstd_opt.c | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/lib/compress/zstd_opt.c b/lib/compress/zstd_opt.c index 3c3f22313..0680e831b 100644 --- a/lib/compress/zstd_opt.c +++ b/lib/compress/zstd_opt.c @@ -768,12 +768,14 @@ FORCE_INLINE_TEMPLATE U32 ZSTD_BtGetAllMatches ( * LDM util functions *********************************/ -static int ldm_splitSequence() { +// The only function that can update pos (i think, for now) +static int ldm_splitSequence(rawSeqStore_t* ldmSeqStore, U32 remainingBytes) { + rawSeq currSeq = ldmSeqStore->seq[ldmSeqStore->pos]; } /* Returns 1 if the rest of the block is just LDM literals */ -static int ldm_getNextMatch() { +static int ldm_getNextMatch(U32* matchStartPosInBlock, U32* matchEndPosInBlock, U32 remainingBytes) { int ret = ldm_splitSequence(); } @@ -788,8 +790,13 @@ static void ldm_maybeUpdateSeqStoreReadPos() { } /* Wrapper function to call ldm functions as needed */ -static void ldm_handleLdm(int* nbMatches) { - int noMoreLdms = getNextMatch(); +static void ldm_handleLdm(ZSTD_match_t* matches, int* nbMatches, + U32* matchStartPosInBlock, U32* matchEndPosInBlock, + U32 currPosInBlock, U32 remainingBytes) { + if (currPosInBlock >= matchEndPosInBlock) { + int noMoreLdms = ldm_getNextMatch(matchStartPosInBlock, matchEndPosInBlock, remainingBytes); + } + ldm_maybeAddLdm(matches, currPosInBlock, matchStartPosInBlock, matchEndPosInBlock); } @@ -846,6 +853,10 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms, ZSTD_match_t* const matches = optStatePtr->matchTable; ZSTD_optimal_t lastSequence; + U32 matchStartPosInBlock = 0; + U32 matchEndPosInBlock = 0; + ldm_getNextMatch(matchStartPosInBlock, matchEndPosInBlock, (U32)(iend - ip)); + /* init */ DEBUGLOG(5, "ZSTD_compressBlock_opt_generic: current=%u, prefix=%u, nextToUpdate=%u", (U32)(ip - base), ms->window.dictLimit, ms->nextToUpdate); @@ -976,7 +987,7 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms, U32 const basePrice = previousPrice + ZSTD_litLengthPrice(0, optStatePtr, optLevel); U32 const nbMatches = ZSTD_BtGetAllMatches(matches, ms, &nextToUpdate3, inr, iend, dictMode, opt[cur].rep, ll0, minMatch); U32 matchNb; - + ldm_handleLdm(&nbMatches); if (!nbMatches) { DEBUGLOG(7, "rPos:%u : no match found", cur); From 28c74bf591bf18800d292353b4e1cb0c4b378129 Mon Sep 17 00:00:00 2001 From: senhuang42 Date: Sat, 26 Sep 2020 12:09:19 -0400 Subject: [PATCH 15/68] Implement basic splitSequence and skipSequence functions --- lib/compress/zstd_opt.c | 56 +++++++++++++++++++++++++++++++++++++---- 1 file changed, 51 insertions(+), 5 deletions(-) diff --git a/lib/compress/zstd_opt.c b/lib/compress/zstd_opt.c index 0680e831b..6c8e8cc1d 100644 --- a/lib/compress/zstd_opt.c +++ b/lib/compress/zstd_opt.c @@ -768,15 +768,61 @@ FORCE_INLINE_TEMPLATE U32 ZSTD_BtGetAllMatches ( * LDM util functions *********************************/ -// The only function that can update pos (i think, for now) -static int ldm_splitSequence(rawSeqStore_t* ldmSeqStore, U32 remainingBytes) { - rawSeq currSeq = ldmSeqStore->seq[ldmSeqStore->pos]; +static void ldm_skipSequences(rawSeqStore_t* rawSeqStore, size_t srcSize, U32 const minMatch) { + while (srcSize > 0 && rawSeqStore->pos < rawSeqStore->size) { + rawSeq* seq = rawSeqStore->seq + rawSeqStore->pos; + if (srcSize <= seq->litLength) { + /* Skip past srcSize literals */ + seq->litLength -= (U32)srcSize; + return; + } + srcSize -= seq->litLength; + seq->litLength = 0; + if (srcSize < seq->matchLength) { + /* Skip past the first srcSize of the match */ + seq->matchLength -= (U32)srcSize; + if (seq->matchLength < minMatch) { + /* The match is too short, omit it */ + if (rawSeqStore->pos + 1 < rawSeqStore->size) { + seq[1].litLength += seq[0].matchLength; + } + rawSeqStore->pos++; + } + return; + } + srcSize -= seq->matchLength; + seq->matchLength = 0; + rawSeqStore->pos++; + } +} + +// The only function that can update pos (i think, for now) +static rawSeq ldm_splitSequence(rawSeqStore_t* ldmSeqStore, U32 remainingBytes) { + rawSeq currSeq = ldmSeqStore->seq[ldmSeqStore->pos]; + /* No split */ + if (remainingBytes >= currSeq.litLength + currSeq.matchLength) { + ldmSeqStore->pos++; + return currSeq; + } + /* Need a split */ + if (remainingBytes <= currSeq.litLength) { + currSeq.offset = 0; + } else if (remainingBytes < currSeq.litLength + currSeq.matchLength) { + if (currSeq.matchLength < MINMATCH) { + currSeq.offset = 0; + } + } + + ldm_skipSequences(ldmSeqStore, remainingBytes, MINMATCH); + return currSeq; } /* Returns 1 if the rest of the block is just LDM literals */ -static int ldm_getNextMatch(U32* matchStartPosInBlock, U32* matchEndPosInBlock, U32 remainingBytes) { - int ret = ldm_splitSequence(); +static int ldm_getNextMatch(rawSeqStore_t* ldmSeqStore, + U32* matchStartPosInBlock, U32* matchEndPosInBlock, + U32 remainingBytes) { + int ret = ldm_splitSequence(ldmSeqStore, remainingBytes); } /* Adds an LDM if it's long enough */ From 84777059d24dc9a82b52bab7c68985e1b23ff3e3 Mon Sep 17 00:00:00 2001 From: senhuang42 Date: Sat, 26 Sep 2020 12:16:15 -0400 Subject: [PATCH 16/68] Implement ldm_getNextMatch() --- lib/compress/zstd_opt.c | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/lib/compress/zstd_opt.c b/lib/compress/zstd_opt.c index 6c8e8cc1d..c131e6f18 100644 --- a/lib/compress/zstd_opt.c +++ b/lib/compress/zstd_opt.c @@ -821,8 +821,20 @@ static rawSeq ldm_splitSequence(rawSeqStore_t* ldmSeqStore, U32 remainingBytes) /* Returns 1 if the rest of the block is just LDM literals */ static int ldm_getNextMatch(rawSeqStore_t* ldmSeqStore, U32* matchStartPosInBlock, U32* matchEndPosInBlock, + U32* matchOffset, U32 currPosInBlock, U32 remainingBytes) { - int ret = ldm_splitSequence(ldmSeqStore, remainingBytes); + rawSeq seq = ldm_splitSequence(ldmSeqStore, remainingBytes); + if (seq.offset == 0) { + // Don't use the LDM for the rest of the block (there is none) + *matchStartPosInBlock = UINT32_MAX; + *matchEndPosInBlock = UINT32_MAX; + return 1; + } + + *matchStartPosInBlock = currPosInBlock + seq.litLength; + *matchEndPosInBlock = *matchStartPosInBlock + seq.matchLength; + *matchOffset = seq.offset; + return 0; } /* Adds an LDM if it's long enough */ @@ -899,8 +911,9 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms, ZSTD_match_t* const matches = optStatePtr->matchTable; ZSTD_optimal_t lastSequence; - U32 matchStartPosInBlock = 0; - U32 matchEndPosInBlock = 0; + U32 ldmStartPosInBlock = 0; + U32 ldmEndPosInBlock = 0; + U32 ldmOffset = 0; ldm_getNextMatch(matchStartPosInBlock, matchEndPosInBlock, (U32)(iend - ip)); /* init */ From 766c4a8c2837addf5ede83c378e1e02982826caa Mon Sep 17 00:00:00 2001 From: senhuang42 Date: Sat, 26 Sep 2020 12:28:10 -0400 Subject: [PATCH 17/68] Implement part of ldm_maybeAddLdm() --- lib/compress/zstd_opt.c | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/lib/compress/zstd_opt.c b/lib/compress/zstd_opt.c index c131e6f18..9187edbef 100644 --- a/lib/compress/zstd_opt.c +++ b/lib/compress/zstd_opt.c @@ -838,8 +838,20 @@ static int ldm_getNextMatch(rawSeqStore_t* ldmSeqStore, } /* Adds an LDM if it's long enough */ -static void ldm_maybeAddLdm() { +static void ldm_maybeAddLdm(ZSTD_match_t* matches, U32 nbMatches, + U32 matchStartPosInBlock, U32 matchEndPosInBlock, + U32 matchOffset, U32 currPosInBlock) { + /* Check that current block position is not outside of the match */ + if (currPosInBlock < matchStartPosInBlock || currPosInBlock >= matchEndPosInBlock) + return; + U32 posDiff = currPosInBlock - matchStartPosInBlock; + assert(posDiff < matchEndPosInBlock - matchStartPosInBlock); + U32 matchLengthAdjusted = matchEndPosInBlock - matchStartPosInBlock - posDiff; + U32 matchOffsetAdjusted = matchOffset + posDiff; + if (matchLengthAdjusted >= matches[nbMatches-1]) { + + } } /* Updates the pos field in rawSeqStore */ @@ -849,12 +861,12 @@ static void ldm_maybeUpdateSeqStoreReadPos() { /* Wrapper function to call ldm functions as needed */ static void ldm_handleLdm(ZSTD_match_t* matches, int* nbMatches, - U32* matchStartPosInBlock, U32* matchEndPosInBlock, + U32* matchStartPosInBlock, U32* matchEndPosInBlock, U32* matchOffset, U32 currPosInBlock, U32 remainingBytes) { if (currPosInBlock >= matchEndPosInBlock) { int noMoreLdms = ldm_getNextMatch(matchStartPosInBlock, matchEndPosInBlock, remainingBytes); } - ldm_maybeAddLdm(matches, currPosInBlock, matchStartPosInBlock, matchEndPosInBlock); + ldm_maybeAddLdm(matches, *nbMatches, *matchStartPosInBlock, *matchEndPosInBlock, *matchOffset, currPosInBlock); } From dd3dd199bb74e097e1b697bf20c1e42dd43ffe59 Mon Sep 17 00:00:00 2001 From: senhuang42 Date: Sat, 26 Sep 2020 21:38:35 -0400 Subject: [PATCH 18/68] Get zstd to build with new functions and callsites, fix arguments --- lib/compress/zstd_opt.c | 41 +++++++++++++++++++++++++++++------------ 1 file changed, 29 insertions(+), 12 deletions(-) diff --git a/lib/compress/zstd_opt.c b/lib/compress/zstd_opt.c index 9187edbef..64d28b2fd 100644 --- a/lib/compress/zstd_opt.c +++ b/lib/compress/zstd_opt.c @@ -799,20 +799,25 @@ static void ldm_skipSequences(rawSeqStore_t* rawSeqStore, size_t srcSize, U32 co // The only function that can update pos (i think, for now) static rawSeq ldm_splitSequence(rawSeqStore_t* ldmSeqStore, U32 remainingBytes) { + printf("Split Sequence with remaining = %u : ", remainingBytes); rawSeq currSeq = ldmSeqStore->seq[ldmSeqStore->pos]; /* No split */ if (remainingBytes >= currSeq.litLength + currSeq.matchLength) { + printf("NO SPLIT\n", remainingBytes); ldmSeqStore->pos++; return currSeq; } /* Need a split */ if (remainingBytes <= currSeq.litLength) { + printf("SPLITTING: all remaining bytes were literals"); currSeq.offset = 0; } else if (remainingBytes < currSeq.litLength + currSeq.matchLength) { if (currSeq.matchLength < MINMATCH) { + printf("CurrSeq less than minmatch: all remaining bytes were literals"); currSeq.offset = 0; } } + printf("\n"); ldm_skipSequences(ldmSeqStore, remainingBytes, MINMATCH); return currSeq; @@ -838,7 +843,7 @@ static int ldm_getNextMatch(rawSeqStore_t* ldmSeqStore, } /* Adds an LDM if it's long enough */ -static void ldm_maybeAddLdm(ZSTD_match_t* matches, U32 nbMatches, +static void ldm_maybeAddLdm(ZSTD_match_t* matches, U32* nbMatches, U32 matchStartPosInBlock, U32 matchEndPosInBlock, U32 matchOffset, U32 currPosInBlock) { /* Check that current block position is not outside of the match */ @@ -849,8 +854,10 @@ static void ldm_maybeAddLdm(ZSTD_match_t* matches, U32 nbMatches, U32 matchLengthAdjusted = matchEndPosInBlock - matchStartPosInBlock - posDiff; U32 matchOffsetAdjusted = matchOffset + posDiff; - if (matchLengthAdjusted >= matches[nbMatches-1]) { - + if (matchLengthAdjusted >= matches[*nbMatches-1].len) { + matches[*nbMatches].len = matchLengthAdjusted; + matches[*nbMatches].off = matchOffsetAdjusted; + (*nbMatches)++; } } @@ -860,13 +867,15 @@ static void ldm_maybeUpdateSeqStoreReadPos() { } /* Wrapper function to call ldm functions as needed */ -static void ldm_handleLdm(ZSTD_match_t* matches, int* nbMatches, +static void ldm_handleLdm(rawSeqStore_t* ldmSeqStore, ZSTD_match_t* matches, U32* nbMatches, U32* matchStartPosInBlock, U32* matchEndPosInBlock, U32* matchOffset, U32 currPosInBlock, U32 remainingBytes) { - if (currPosInBlock >= matchEndPosInBlock) { - int noMoreLdms = ldm_getNextMatch(matchStartPosInBlock, matchEndPosInBlock, remainingBytes); + if (currPosInBlock >= *matchEndPosInBlock) { + int noMoreLdms = ldm_getNextMatch(ldmSeqStore, matchStartPosInBlock, + matchEndPosInBlock, matchOffset, + currPosInBlock, remainingBytes); } - ldm_maybeAddLdm(matches, *nbMatches, *matchStartPosInBlock, *matchEndPosInBlock, *matchOffset, currPosInBlock); + ldm_maybeAddLdm(matches, nbMatches, *matchStartPosInBlock, *matchEndPosInBlock, *matchOffset, currPosInBlock); } @@ -926,7 +935,9 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms, U32 ldmStartPosInBlock = 0; U32 ldmEndPosInBlock = 0; U32 ldmOffset = 0; - ldm_getNextMatch(matchStartPosInBlock, matchEndPosInBlock, (U32)(iend - ip)); + ldm_getNextMatch(&ms->ldmSeqStore, &ldmStartPosInBlock, + &ldmEndPosInBlock, &ldmOffset, + (U32)(ip-istart), (U32)(iend - ip)); /* init */ DEBUGLOG(5, "ZSTD_compressBlock_opt_generic: current=%u, prefix=%u, nextToUpdate=%u", @@ -942,8 +953,11 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms, /* find first match */ { U32 const litlen = (U32)(ip - anchor); U32 const ll0 = !litlen; - U32 const nbMatches = ZSTD_BtGetAllMatches(matches, ms, &nextToUpdate3, ip, iend, dictMode, rep, ll0, minMatch); - ldm_handleLdm(&nbMatches); + U32 nbMatches = ZSTD_BtGetAllMatches(matches, ms, &nextToUpdate3, ip, iend, dictMode, rep, ll0, minMatch); + ldm_handleLdm(&ms->ldmSeqStore, matches, + &nbMatches, &ldmStartPosInBlock, + &ldmEndPosInBlock, &ldmOffset, + (U32)(ip-istart), (U32)(iend-ip)); if (!nbMatches) { ip++; continue; } /* initialize opt[0] */ @@ -1056,10 +1070,13 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms, U32 const litlen = (opt[cur].mlen == 0) ? opt[cur].litlen : 0; U32 const previousPrice = opt[cur].price; U32 const basePrice = previousPrice + ZSTD_litLengthPrice(0, optStatePtr, optLevel); - U32 const nbMatches = ZSTD_BtGetAllMatches(matches, ms, &nextToUpdate3, inr, iend, dictMode, opt[cur].rep, ll0, minMatch); + U32 nbMatches = ZSTD_BtGetAllMatches(matches, ms, &nextToUpdate3, inr, iend, dictMode, opt[cur].rep, ll0, minMatch); U32 matchNb; - ldm_handleLdm(&nbMatches); + ldm_handleLdm(&ms->ldmSeqStore, matches, + &nbMatches, &ldmStartPosInBlock, + &ldmEndPosInBlock, &ldmOffset, + (U32)(inr-istart), (U32)(iend-inr)); if (!nbMatches) { DEBUGLOG(7, "rPos:%u : no match found", cur); continue; From 42395a70c2f70cc4d6c73ff3567943b62c2f246e Mon Sep 17 00:00:00 2001 From: senhuang42 Date: Sat, 26 Sep 2020 21:56:14 -0400 Subject: [PATCH 19/68] Add debug statements, flesh out functions --- lib/compress/zstd_opt.c | 46 ++++++++++++++++++++++++----------------- 1 file changed, 27 insertions(+), 19 deletions(-) diff --git a/lib/compress/zstd_opt.c b/lib/compress/zstd_opt.c index 64d28b2fd..0a17374ae 100644 --- a/lib/compress/zstd_opt.c +++ b/lib/compress/zstd_opt.c @@ -799,11 +799,12 @@ static void ldm_skipSequences(rawSeqStore_t* rawSeqStore, size_t srcSize, U32 co // The only function that can update pos (i think, for now) static rawSeq ldm_splitSequence(rawSeqStore_t* ldmSeqStore, U32 remainingBytes) { - printf("Split Sequence with remaining = %u : ", remainingBytes); rawSeq currSeq = ldmSeqStore->seq[ldmSeqStore->pos]; + printf("Current sequence: (of: %u, ml: %u, ll: %u)\n", currSeq.offset, currSeq.matchLength, currSeq.litLength); + printf("Split Sequence with remaining = %u : ", remainingBytes); /* No split */ if (remainingBytes >= currSeq.litLength + currSeq.matchLength) { - printf("NO SPLIT\n", remainingBytes); + printf("NO SPLIT\n"); ldmSeqStore->pos++; return currSeq; } @@ -820,6 +821,7 @@ static rawSeq ldm_splitSequence(rawSeqStore_t* ldmSeqStore, U32 remainingBytes) printf("\n"); ldm_skipSequences(ldmSeqStore, remainingBytes, MINMATCH); + printf("Sequence final: (of: %u, ml: %u, ll: %u)\n", currSeq.offset, currSeq.matchLength, currSeq.litLength); return currSeq; } @@ -850,22 +852,20 @@ static void ldm_maybeAddLdm(ZSTD_match_t* matches, U32* nbMatches, if (currPosInBlock < matchStartPosInBlock || currPosInBlock >= matchEndPosInBlock) return; U32 posDiff = currPosInBlock - matchStartPosInBlock; + if (posDiff > 0) + return; // dont deal with extra ldms for now assert(posDiff < matchEndPosInBlock - matchStartPosInBlock); U32 matchLengthAdjusted = matchEndPosInBlock - matchStartPosInBlock - posDiff; U32 matchOffsetAdjusted = matchOffset + posDiff; if (matchLengthAdjusted >= matches[*nbMatches-1].len) { + printf("Adding LDM with of: %u, ml: %u\n", matchOffsetAdjusted, matchLengthAdjusted); matches[*nbMatches].len = matchLengthAdjusted; matches[*nbMatches].off = matchOffsetAdjusted; (*nbMatches)++; } } -/* Updates the pos field in rawSeqStore */ -static void ldm_maybeUpdateSeqStoreReadPos() { - -} - /* Wrapper function to call ldm functions as needed */ static void ldm_handleLdm(rawSeqStore_t* ldmSeqStore, ZSTD_match_t* matches, U32* nbMatches, U32* matchStartPosInBlock, U32* matchEndPosInBlock, U32* matchOffset, @@ -875,7 +875,8 @@ static void ldm_handleLdm(rawSeqStore_t* ldmSeqStore, ZSTD_match_t* matches, U32 matchEndPosInBlock, matchOffset, currPosInBlock, remainingBytes); } - ldm_maybeAddLdm(matches, nbMatches, *matchStartPosInBlock, *matchEndPosInBlock, *matchOffset, currPosInBlock); + if (ldmSeqStore->pos <= ldmSeqStore->size) + ldm_maybeAddLdm(matches, nbMatches, *matchStartPosInBlock, *matchEndPosInBlock, *matchOffset, currPosInBlock); } @@ -935,9 +936,11 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms, U32 ldmStartPosInBlock = 0; U32 ldmEndPosInBlock = 0; U32 ldmOffset = 0; - ldm_getNextMatch(&ms->ldmSeqStore, &ldmStartPosInBlock, - &ldmEndPosInBlock, &ldmOffset, - (U32)(ip-istart), (U32)(iend - ip)); + if (ms->ldmSeqStore.size != 0) { + ldm_getNextMatch(&ms->ldmSeqStore, &ldmStartPosInBlock, + &ldmEndPosInBlock, &ldmOffset, + (U32)(ip-istart), (U32)(iend - ip)); + } /* init */ DEBUGLOG(5, "ZSTD_compressBlock_opt_generic: current=%u, prefix=%u, nextToUpdate=%u", @@ -954,10 +957,12 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms, { U32 const litlen = (U32)(ip - anchor); U32 const ll0 = !litlen; U32 nbMatches = ZSTD_BtGetAllMatches(matches, ms, &nextToUpdate3, ip, iend, dictMode, rep, ll0, minMatch); - ldm_handleLdm(&ms->ldmSeqStore, matches, - &nbMatches, &ldmStartPosInBlock, - &ldmEndPosInBlock, &ldmOffset, - (U32)(ip-istart), (U32)(iend-ip)); + if (ms->ldmSeqStore.size != 0) { + ldm_handleLdm(&ms->ldmSeqStore, matches, + &nbMatches, &ldmStartPosInBlock, + &ldmEndPosInBlock, &ldmOffset, + (U32)(ip-istart), (U32)(iend-ip)); + } if (!nbMatches) { ip++; continue; } /* initialize opt[0] */ @@ -1073,10 +1078,13 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms, U32 nbMatches = ZSTD_BtGetAllMatches(matches, ms, &nextToUpdate3, inr, iend, dictMode, opt[cur].rep, ll0, minMatch); U32 matchNb; - ldm_handleLdm(&ms->ldmSeqStore, matches, - &nbMatches, &ldmStartPosInBlock, - &ldmEndPosInBlock, &ldmOffset, - (U32)(inr-istart), (U32)(iend-inr)); + + if (ms->ldmSeqStore.size != 0) { + ldm_handleLdm(&ms->ldmSeqStore, matches, + &nbMatches, &ldmStartPosInBlock, + &ldmEndPosInBlock, &ldmOffset, + (U32)(inr-istart), (U32)(iend-inr)); + } if (!nbMatches) { DEBUGLOG(7, "rPos:%u : no match found", cur); continue; From 84009a076ad4f23de5eef76a84ac34be47ae5fef Mon Sep 17 00:00:00 2001 From: senhuang42 Date: Sat, 26 Sep 2020 22:14:21 -0400 Subject: [PATCH 20/68] Add re-copying of ldmSeqStore after processing --- lib/compress/zstd_ldm.c | 1 + lib/compress/zstd_opt.c | 22 +++++++++++++--------- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/lib/compress/zstd_ldm.c b/lib/compress/zstd_ldm.c index c68693bd6..42e6aa2d8 100644 --- a/lib/compress/zstd_ldm.c +++ b/lib/compress/zstd_ldm.c @@ -580,6 +580,7 @@ size_t ZSTD_ldm_blockCompress(rawSeqStore_t* rawSeqStore, size_t lastLLSize; ms->ldmSeqStore = *rawSeqStore; /* copy current seqStore */ lastLLSize = blockCompressor(ms, seqStore, rep, src, srcSize); + rawSeqStore->pos = ms->ldmSeqStore.pos; return lastLLSize; } diff --git a/lib/compress/zstd_opt.c b/lib/compress/zstd_opt.c index 0a17374ae..f938cce8e 100644 --- a/lib/compress/zstd_opt.c +++ b/lib/compress/zstd_opt.c @@ -806,6 +806,7 @@ static rawSeq ldm_splitSequence(rawSeqStore_t* ldmSeqStore, U32 remainingBytes) if (remainingBytes >= currSeq.litLength + currSeq.matchLength) { printf("NO SPLIT\n"); ldmSeqStore->pos++; + printf("pos is now: %u\n", ldmSeqStore->pos); return currSeq; } /* Need a split */ @@ -831,8 +832,9 @@ static int ldm_getNextMatch(rawSeqStore_t* ldmSeqStore, U32* matchOffset, U32 currPosInBlock, U32 remainingBytes) { rawSeq seq = ldm_splitSequence(ldmSeqStore, remainingBytes); - if (seq.offset == 0) { + if (seq.offset == 0 || ldmSeqStore->pos > ldmSeqStore->size) { // Don't use the LDM for the rest of the block (there is none) + printf("No ldm left in the block\n"); *matchStartPosInBlock = UINT32_MAX; *matchEndPosInBlock = UINT32_MAX; return 1; @@ -841,6 +843,7 @@ static int ldm_getNextMatch(rawSeqStore_t* ldmSeqStore, *matchStartPosInBlock = currPosInBlock + seq.litLength; *matchEndPosInBlock = *matchStartPosInBlock + seq.matchLength; *matchOffset = seq.offset; + printf("New match range in block is: (%u, %u) with of: %u where currPosInBlock: %u\n", *matchStartPosInBlock, *matchEndPosInBlock, *matchOffset, currPosInBlock); return 0; } @@ -860,6 +863,7 @@ static void ldm_maybeAddLdm(ZSTD_match_t* matches, U32* nbMatches, if (matchLengthAdjusted >= matches[*nbMatches-1].len) { printf("Adding LDM with of: %u, ml: %u\n", matchOffsetAdjusted, matchLengthAdjusted); + /* Add sifting */ matches[*nbMatches].len = matchLengthAdjusted; matches[*nbMatches].off = matchOffsetAdjusted; (*nbMatches)++; @@ -875,8 +879,7 @@ static void ldm_handleLdm(rawSeqStore_t* ldmSeqStore, ZSTD_match_t* matches, U32 matchEndPosInBlock, matchOffset, currPosInBlock, remainingBytes); } - if (ldmSeqStore->pos <= ldmSeqStore->size) - ldm_maybeAddLdm(matches, nbMatches, *matchStartPosInBlock, *matchEndPosInBlock, *matchOffset, currPosInBlock); + ldm_maybeAddLdm(matches, nbMatches, *matchStartPosInBlock, *matchEndPosInBlock, *matchOffset, currPosInBlock); } @@ -936,11 +939,7 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms, U32 ldmStartPosInBlock = 0; U32 ldmEndPosInBlock = 0; U32 ldmOffset = 0; - if (ms->ldmSeqStore.size != 0) { - ldm_getNextMatch(&ms->ldmSeqStore, &ldmStartPosInBlock, - &ldmEndPosInBlock, &ldmOffset, - (U32)(ip-istart), (U32)(iend - ip)); - } + /* init */ DEBUGLOG(5, "ZSTD_compressBlock_opt_generic: current=%u, prefix=%u, nextToUpdate=%u", @@ -948,7 +947,12 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms, assert(optLevel <= 2); ZSTD_rescaleFreqs(optStatePtr, (const BYTE*)src, srcSize, optLevel); ip += (ip==prefixStart); - + + if (ms->ldmSeqStore.size != 0) { + ldm_getNextMatch(&ms->ldmSeqStore, &ldmStartPosInBlock, + &ldmEndPosInBlock, &ldmOffset, + (U32)(ip-istart), (U32)(iend - ip)); + } /* Match Loop */ while (ip < ilimit) { U32 cur, last_pos = 0; From f8ce7cabc3c0eb416c6f31e31c431e9910bf2f44 Mon Sep 17 00:00:00 2001 From: senhuang42 Date: Sun, 27 Sep 2020 13:02:46 -0400 Subject: [PATCH 21/68] Added more debugging --- lib/compress/zstd_opt.c | 25 ++++++++++--------------- 1 file changed, 10 insertions(+), 15 deletions(-) diff --git a/lib/compress/zstd_opt.c b/lib/compress/zstd_opt.c index f938cce8e..8852fcbe5 100644 --- a/lib/compress/zstd_opt.c +++ b/lib/compress/zstd_opt.c @@ -850,7 +850,7 @@ static int ldm_getNextMatch(rawSeqStore_t* ldmSeqStore, /* Adds an LDM if it's long enough */ static void ldm_maybeAddLdm(ZSTD_match_t* matches, U32* nbMatches, U32 matchStartPosInBlock, U32 matchEndPosInBlock, - U32 matchOffset, U32 currPosInBlock) { + U32 matchOffset, U32 currPosInBlock, U32 curr) { /* Check that current block position is not outside of the match */ if (currPosInBlock < matchStartPosInBlock || currPosInBlock >= matchEndPosInBlock) return; @@ -862,10 +862,10 @@ static void ldm_maybeAddLdm(ZSTD_match_t* matches, U32* nbMatches, U32 matchOffsetAdjusted = matchOffset + posDiff; if (matchLengthAdjusted >= matches[*nbMatches-1].len) { - printf("Adding LDM with of: %u, ml: %u\n", matchOffsetAdjusted, matchLengthAdjusted); + printf("Adding LDM with of: %u, ml: %u @ currposinblock: %u, current: %u\n", matchOffsetAdjusted, matchLengthAdjusted, currPosInBlock, curr); /* Add sifting */ matches[*nbMatches].len = matchLengthAdjusted; - matches[*nbMatches].off = matchOffsetAdjusted; + matches[*nbMatches].off = matchOffsetAdjusted + ZSTD_REP_MOVE; (*nbMatches)++; } } @@ -873,13 +873,13 @@ static void ldm_maybeAddLdm(ZSTD_match_t* matches, U32* nbMatches, /* Wrapper function to call ldm functions as needed */ static void ldm_handleLdm(rawSeqStore_t* ldmSeqStore, ZSTD_match_t* matches, U32* nbMatches, U32* matchStartPosInBlock, U32* matchEndPosInBlock, U32* matchOffset, - U32 currPosInBlock, U32 remainingBytes) { + U32 currPosInBlock, U32 remainingBytes, U32 curr) { if (currPosInBlock >= *matchEndPosInBlock) { int noMoreLdms = ldm_getNextMatch(ldmSeqStore, matchStartPosInBlock, matchEndPosInBlock, matchOffset, currPosInBlock, remainingBytes); } - ldm_maybeAddLdm(matches, nbMatches, *matchStartPosInBlock, *matchEndPosInBlock, *matchOffset, currPosInBlock); + ldm_maybeAddLdm(matches, nbMatches, *matchStartPosInBlock, *matchEndPosInBlock, *matchOffset, currPosInBlock, curr); } @@ -947,12 +947,7 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms, assert(optLevel <= 2); ZSTD_rescaleFreqs(optStatePtr, (const BYTE*)src, srcSize, optLevel); ip += (ip==prefixStart); - - if (ms->ldmSeqStore.size != 0) { - ldm_getNextMatch(&ms->ldmSeqStore, &ldmStartPosInBlock, - &ldmEndPosInBlock, &ldmOffset, - (U32)(ip-istart), (U32)(iend - ip)); - } + /* Match Loop */ while (ip < ilimit) { U32 cur, last_pos = 0; @@ -963,9 +958,9 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms, U32 nbMatches = ZSTD_BtGetAllMatches(matches, ms, &nextToUpdate3, ip, iend, dictMode, rep, ll0, minMatch); if (ms->ldmSeqStore.size != 0) { ldm_handleLdm(&ms->ldmSeqStore, matches, - &nbMatches, &ldmStartPosInBlock, - &ldmEndPosInBlock, &ldmOffset, - (U32)(ip-istart), (U32)(iend-ip)); + &nbMatches, &ldmStartPosInBlock, + &ldmEndPosInBlock, &ldmOffset, + (U32)(ip-istart), (U32)(iend - ip), (U32)(ip-base)); } if (!nbMatches) { ip++; continue; } @@ -1087,7 +1082,7 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms, ldm_handleLdm(&ms->ldmSeqStore, matches, &nbMatches, &ldmStartPosInBlock, &ldmEndPosInBlock, &ldmOffset, - (U32)(inr-istart), (U32)(iend-inr)); + (U32)(inr-istart), (U32)(iend-inr), (U32)(ip-base)); } if (!nbMatches) { DEBUGLOG(7, "rPos:%u : no match found", cur); From 5df9b5e05fdd0b85b3d89e6524c49e8a1778ed38 Mon Sep 17 00:00:00 2001 From: senhuang42 Date: Sun, 27 Sep 2020 13:19:54 -0400 Subject: [PATCH 22/68] Add initial getNextMatch() in opt parser --- lib/compress/zstd_opt.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/compress/zstd_opt.c b/lib/compress/zstd_opt.c index 8852fcbe5..4ffcefe37 100644 --- a/lib/compress/zstd_opt.c +++ b/lib/compress/zstd_opt.c @@ -940,7 +940,10 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms, U32 ldmEndPosInBlock = 0; U32 ldmOffset = 0; - + if (ms->ldmSeqStore.size != 0) { + ldm_getNextMatch(&ms->ldmSeqStore, &ldmStartPosInBlock, + &ldmEndPosInBlock, &ldmOffset, (U32)(ip-istart), (U32)(iend-ip)); + } /* init */ DEBUGLOG(5, "ZSTD_compressBlock_opt_generic: current=%u, prefix=%u, nextToUpdate=%u", (U32)(ip - base), ms->window.dictLimit, ms->nextToUpdate); From f57c7e6bbfc1e61d92ba31d7ecf25d35744591d9 Mon Sep 17 00:00:00 2001 From: senhuang42 Date: Sun, 27 Sep 2020 13:25:27 -0400 Subject: [PATCH 23/68] Add base adjustment correction --- lib/compress/zstd_ldm.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/lib/compress/zstd_ldm.c b/lib/compress/zstd_ldm.c index 42e6aa2d8..5f44ce78a 100644 --- a/lib/compress/zstd_ldm.c +++ b/lib/compress/zstd_ldm.c @@ -579,8 +579,15 @@ size_t ZSTD_ldm_blockCompress(rawSeqStore_t* rawSeqStore, if (cParams->strategy >= ZSTD_btopt) { size_t lastLLSize; ms->ldmSeqStore = *rawSeqStore; /* copy current seqStore */ + const BYTE* const prevBase = (BYTE const*)ms->window.base; lastLLSize = blockCompressor(ms, seqStore, rep, src, srcSize); rawSeqStore->pos = ms->ldmSeqStore.pos; + ms->ldmSeqStore = *rawSeqStore; + if (prevBase != ms->window.base) { + int baseDiff = (int)(prevBase - ms->window.base); + printf("Bases were different, adjusting, diff = %d\n", baseDiff); + rawSeqStore->seq[rawSeqStore->pos].litLength += baseDiff; + } return lastLLSize; } From c8b8572b38f9d2c52c57a6a4d03381fb39e3ba7c Mon Sep 17 00:00:00 2001 From: senhuang42 Date: Sun, 27 Sep 2020 20:00:14 -0400 Subject: [PATCH 24/68] Adjustments to no longer segfault on nci --- lib/compress/zstd_compress.c | 2 +- lib/compress/zstd_opt.c | 56 +++++++++++++++++++++++++++--------- 2 files changed, 44 insertions(+), 14 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index b6edba207..037b5360a 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -2336,7 +2336,7 @@ static size_t ZSTD_buildSeqStore(ZSTD_CCtx* zc, const void* src, size_t srcSize) if (curr > ms->nextToUpdate + 384) ms->nextToUpdate = curr - MIN(192, (U32)(curr - ms->nextToUpdate - 384)); } - + printf("--NEW BLOCK--\n"); /* select and store sequences */ { ZSTD_dictMode_e const dictMode = ZSTD_matchState_dictMode(ms); size_t lastLLSize; diff --git a/lib/compress/zstd_opt.c b/lib/compress/zstd_opt.c index 4ffcefe37..c642de4da 100644 --- a/lib/compress/zstd_opt.c +++ b/lib/compress/zstd_opt.c @@ -768,6 +768,19 @@ FORCE_INLINE_TEMPLATE U32 ZSTD_BtGetAllMatches ( * LDM util functions *********************************/ +static void ldm_skipOvershotBytes(rawSeqStore_t* rawSeqStore, size_t posOvershoot) { + if (rawSeqStore->seq[rawSeqStore->pos].matchLength > posOvershoot) { + rawSeqStore->seq[rawSeqStore->pos].matchLength -= posOvershoot; + return; + } else { + rawSeqStore->seq[rawSeqStore->pos].matchLength = 0; + rawSeqStore->pos++; + } + + while (posOvershoot > 0 && rawSeqStore->pos < rawSeqStore->size) { + + } +} static void ldm_skipSequences(rawSeqStore_t* rawSeqStore, size_t srcSize, U32 const minMatch) { while (srcSize > 0 && rawSeqStore->pos < rawSeqStore->size) { @@ -780,8 +793,10 @@ static void ldm_skipSequences(rawSeqStore_t* rawSeqStore, size_t srcSize, U32 co srcSize -= seq->litLength; seq->litLength = 0; if (srcSize < seq->matchLength) { + printf("Splitting match: ml curr: %u ", seq->matchLength); /* Skip past the first srcSize of the match */ seq->matchLength -= (U32)srcSize; + printf("ml in store left: %u ", seq->matchLength); if (seq->matchLength < minMatch) { /* The match is too short, omit it */ if (rawSeqStore->pos + 1 < rawSeqStore->size) { @@ -818,6 +833,7 @@ static rawSeq ldm_splitSequence(rawSeqStore_t* ldmSeqStore, U32 remainingBytes) printf("CurrSeq less than minmatch: all remaining bytes were literals"); currSeq.offset = 0; } + currSeq.matchLength = remainingBytes - currSeq.litLength; } printf("\n"); @@ -830,7 +846,7 @@ static rawSeq ldm_splitSequence(rawSeqStore_t* ldmSeqStore, U32 remainingBytes) static int ldm_getNextMatch(rawSeqStore_t* ldmSeqStore, U32* matchStartPosInBlock, U32* matchEndPosInBlock, U32* matchOffset, U32 currPosInBlock, - U32 remainingBytes) { + U32 remainingBytes, U32 sbi) { rawSeq seq = ldm_splitSequence(ldmSeqStore, remainingBytes); if (seq.offset == 0 || ldmSeqStore->pos > ldmSeqStore->size) { // Don't use the LDM for the rest of the block (there is none) @@ -843,14 +859,14 @@ static int ldm_getNextMatch(rawSeqStore_t* ldmSeqStore, *matchStartPosInBlock = currPosInBlock + seq.litLength; *matchEndPosInBlock = *matchStartPosInBlock + seq.matchLength; *matchOffset = seq.offset; - printf("New match range in block is: (%u, %u) with of: %u where currPosInBlock: %u\n", *matchStartPosInBlock, *matchEndPosInBlock, *matchOffset, currPosInBlock); + printf("New match range in block is: (%u, %u) with of: %u where currPosInBlock: %u at adjusted absolute range: %u, %u\n", *matchStartPosInBlock, *matchEndPosInBlock, *matchOffset, currPosInBlock, *matchStartPosInBlock+sbi, *matchEndPosInBlock+sbi); return 0; } /* Adds an LDM if it's long enough */ static void ldm_maybeAddLdm(ZSTD_match_t* matches, U32* nbMatches, U32 matchStartPosInBlock, U32 matchEndPosInBlock, - U32 matchOffset, U32 currPosInBlock, U32 curr) { + U32 matchOffset, U32 currPosInBlock, U32 curr, U32 sbi) { /* Check that current block position is not outside of the match */ if (currPosInBlock < matchStartPosInBlock || currPosInBlock >= matchEndPosInBlock) return; @@ -861,25 +877,37 @@ static void ldm_maybeAddLdm(ZSTD_match_t* matches, U32* nbMatches, U32 matchLengthAdjusted = matchEndPosInBlock - matchStartPosInBlock - posDiff; U32 matchOffsetAdjusted = matchOffset + posDiff; - if (matchLengthAdjusted >= matches[*nbMatches-1].len) { - printf("Adding LDM with of: %u, ml: %u @ currposinblock: %u, current: %u\n", matchOffsetAdjusted, matchLengthAdjusted, currPosInBlock, curr); + if (*nbMatches == 0 || matchLengthAdjusted >= matches[*nbMatches-1].len) { + printf("Adding LDM with of: %u, ml: %u @ currposinblock: %u, current: %u, currposinblock+sbi: %u\n", matchOffsetAdjusted, matchLengthAdjusted, currPosInBlock, curr, currPosInBlock+sbi); /* Add sifting */ matches[*nbMatches].len = matchLengthAdjusted; matches[*nbMatches].off = matchOffsetAdjusted + ZSTD_REP_MOVE; (*nbMatches)++; + } else { + printf("too small ldm\n"); } } /* Wrapper function to call ldm functions as needed */ static void ldm_handleLdm(rawSeqStore_t* ldmSeqStore, ZSTD_match_t* matches, U32* nbMatches, U32* matchStartPosInBlock, U32* matchEndPosInBlock, U32* matchOffset, - U32 currPosInBlock, U32 remainingBytes, U32 curr) { + U32 currPosInBlock, U32 remainingBytes, U32 curr, U32 sbi) { if (currPosInBlock >= *matchEndPosInBlock) { - int noMoreLdms = ldm_getNextMatch(ldmSeqStore, matchStartPosInBlock, - matchEndPosInBlock, matchOffset, - currPosInBlock, remainingBytes); + printf("Went over match boundary: currPosInBlock: %u, %matchEndPosInBlock: %u\n", currPosInBlock, *matchEndPosInBlock); + if (currPosInBlock > *matchEndPosInBlock) { + U32 posOvershoot = currPosInBlock - *matchEndPosInBlock; + printf("Overshot position by: %u\n", posOvershoot); + ldm_skipSequences(ldmSeqStore, posOvershoot, MINMATCH); + int noMoreLdms = ldm_getNextMatch(ldmSeqStore, matchStartPosInBlock, + matchEndPosInBlock, matchOffset, + matchEndPosInBlock, remainingBytes, sbi); + } else { + int noMoreLdms = ldm_getNextMatch(ldmSeqStore, matchStartPosInBlock, + matchEndPosInBlock, matchOffset, + currPosInBlock, remainingBytes, sbi); + } } - ldm_maybeAddLdm(matches, nbMatches, *matchStartPosInBlock, *matchEndPosInBlock, *matchOffset, currPosInBlock, curr); + ldm_maybeAddLdm(matches, nbMatches, *matchStartPosInBlock, *matchEndPosInBlock, *matchOffset, currPosInBlock, curr, sbi); } @@ -931,6 +959,7 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms, U32 const sufficient_len = MIN(cParams->targetLength, ZSTD_OPT_NUM -1); U32 const minMatch = (cParams->minMatch == 3) ? 3 : 4; U32 nextToUpdate3 = ms->nextToUpdate; + U32 const sbi = (U32)(istart-base); ZSTD_optimal_t* const opt = optStatePtr->priceTable; ZSTD_match_t* const matches = optStatePtr->matchTable; @@ -940,9 +969,10 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms, U32 ldmEndPosInBlock = 0; U32 ldmOffset = 0; + printf("SBI for this block: %u\n", sbi); if (ms->ldmSeqStore.size != 0) { ldm_getNextMatch(&ms->ldmSeqStore, &ldmStartPosInBlock, - &ldmEndPosInBlock, &ldmOffset, (U32)(ip-istart), (U32)(iend-ip)); + &ldmEndPosInBlock, &ldmOffset, (U32)(ip-istart), (U32)(iend-ip), sbi); } /* init */ DEBUGLOG(5, "ZSTD_compressBlock_opt_generic: current=%u, prefix=%u, nextToUpdate=%u", @@ -963,7 +993,7 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms, ldm_handleLdm(&ms->ldmSeqStore, matches, &nbMatches, &ldmStartPosInBlock, &ldmEndPosInBlock, &ldmOffset, - (U32)(ip-istart), (U32)(iend - ip), (U32)(ip-base)); + (U32)(ip-istart), (U32)(iend - ip), (U32)(ip-base), sbi); } if (!nbMatches) { ip++; continue; } @@ -1085,7 +1115,7 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms, ldm_handleLdm(&ms->ldmSeqStore, matches, &nbMatches, &ldmStartPosInBlock, &ldmEndPosInBlock, &ldmOffset, - (U32)(inr-istart), (U32)(iend-inr), (U32)(ip-base)); + (U32)(inr-istart), (U32)(iend-inr), (U32)(inr-base), sbi); } if (!nbMatches) { DEBUGLOG(7, "rPos:%u : no match found", cur); From 9c3c7cd20e9cff361eb78fa4dd097e57929d542e Mon Sep 17 00:00:00 2001 From: senhuang42 Date: Sun, 27 Sep 2020 20:06:44 -0400 Subject: [PATCH 25/68] Fix function argument to getNextMatch() --- lib/compress/zstd_opt.c | 66 +++++++++++++++++++++++++++++++++-------- 1 file changed, 53 insertions(+), 13 deletions(-) diff --git a/lib/compress/zstd_opt.c b/lib/compress/zstd_opt.c index c642de4da..f1e39d7c3 100644 --- a/lib/compress/zstd_opt.c +++ b/lib/compress/zstd_opt.c @@ -768,7 +768,14 @@ FORCE_INLINE_TEMPLATE U32 ZSTD_BtGetAllMatches ( * LDM util functions *********************************/ -static void ldm_skipOvershotBytes(rawSeqStore_t* rawSeqStore, size_t posOvershoot) { +/* +static void ldm_skipOvershotBytes(rawSeqStore_t* rawSeqStore, U32 bytesOvershot) { + while (bytesOvershot > 0 && rawSeqStore->pos < rawSeqStore->size) { + rawSeq seq = rawSeqStore->seq[rawSeqStore->pos]; + if (bytesOvershot <= seq.litLength) { + seq->litLength -= bytesOvershot; + } + } if (rawSeqStore->seq[rawSeqStore->pos].matchLength > posOvershoot) { rawSeqStore->seq[rawSeqStore->pos].matchLength -= posOvershoot; return; @@ -780,14 +787,17 @@ static void ldm_skipOvershotBytes(rawSeqStore_t* rawSeqStore, size_t posOvershoo while (posOvershoot > 0 && rawSeqStore->pos < rawSeqStore->size) { } -} +}*/ static void ldm_skipSequences(rawSeqStore_t* rawSeqStore, size_t srcSize, U32 const minMatch) { while (srcSize > 0 && rawSeqStore->pos < rawSeqStore->size) { + printf("ldm_skipSequences(): %u remaining\n", srcSize); rawSeq* seq = rawSeqStore->seq + rawSeqStore->pos; + printf("ldm_skipSequences(): before: (of: %u, ml: %u, ll: %u)\n", seq->offset, seq->matchLength, seq->litLength); if (srcSize <= seq->litLength) { /* Skip past srcSize literals */ seq->litLength -= (U32)srcSize; + printf("ldm_skipSequences(): final: (of: %u, ml: %u, ll: %u)\n", seq->offset, seq->matchLength, seq->litLength); return; } srcSize -= seq->litLength; @@ -804,6 +814,7 @@ static void ldm_skipSequences(rawSeqStore_t* rawSeqStore, size_t srcSize, U32 co } rawSeqStore->pos++; } + printf("ldm_skipSequences(): final: (of: %u, ml: %u, ll: %u)\n", seq->offset, seq->matchLength, seq->litLength); return; } srcSize -= seq->matchLength; @@ -874,17 +885,46 @@ static void ldm_maybeAddLdm(ZSTD_match_t* matches, U32* nbMatches, if (posDiff > 0) return; // dont deal with extra ldms for now assert(posDiff < matchEndPosInBlock - matchStartPosInBlock); - U32 matchLengthAdjusted = matchEndPosInBlock - matchStartPosInBlock - posDiff; - U32 matchOffsetAdjusted = matchOffset + posDiff; + U32 candidateMatchLength = matchEndPosInBlock - matchStartPosInBlock - posDiff; + U32 candidateOffCode = matchOffset + posDiff + ZSTD_REP_MOVE; - if (*nbMatches == 0 || matchLengthAdjusted >= matches[*nbMatches-1].len) { - printf("Adding LDM with of: %u, ml: %u @ currposinblock: %u, current: %u, currposinblock+sbi: %u\n", matchOffsetAdjusted, matchLengthAdjusted, currPosInBlock, curr, currPosInBlock+sbi); - /* Add sifting */ - matches[*nbMatches].len = matchLengthAdjusted; - matches[*nbMatches].off = matchOffsetAdjusted + ZSTD_REP_MOVE; - (*nbMatches)++; - } else { - printf("too small ldm\n"); + if ((*nbMatches == 0 || candidateMatchLength >= matches[*nbMatches-1].len) && *nbMatches < ZSTD_OPT_NUM) { + printf("large enough, adding\n"); + /*matches[*nbMatches].len = candidateMatchLength; + matches[*nbMatches].off = candidateOffCode; + (*nbMatches)++;*/ + + if (*nbMatches == 0) { + matches[*nbMatches].len = candidateMatchLength; + matches[*nbMatches].off = candidateOffCode; + (*nbMatches)++; + } else { + if (candidateMatchLength == matches[*nbMatches-1].len) { + U32 candidateMatchIdx = *nbMatches; + matches[*nbMatches].len = candidateMatchLength; + matches[*nbMatches].off = candidateOffCode; + printf("Sifting...: idx: %u, len: %u, off: %u\n", candidateMatchIdx, candidateMatchLength, candidateOffCode); + //printf("Current best is...: idx: %u, len: %u, off: %u\n", *nbMatches-1, matches[*nbMatches-1].len, matches[*nbMatches-1].off); + if (candidateOffCode != matches[*nbMatches].off) + printf("DIFF: ldm: (len: %u, off: %u), best: (len: %u, off: %u)\n", candidateMatchLength, candidateOffCode, matches[*nbMatches-1].len, matches[*nbMatches-1].off); + //printf("Current best is...: idx: %u, len: %u, off: %u\n", *nbMatches-1, matches[*nbMatches-1].len, matches[*nbMatches-1].off); + while (candidateMatchIdx > 0 && + matches[candidateMatchIdx].off > matches[candidateMatchIdx - 1].off && + matches[candidateMatchIdx].len == matches[candidateMatchIdx - 1].len) { + //printf("Compared to: idx: %u, len: %u, off: %u", candidateMatchIdx - 1, matches[candidateMatchIdx - 1].len, matches[candidateMatchIdx - 1].off); + ZSTD_match_t tmp = matches[candidateMatchIdx - 1]; + matches[candidateMatchIdx - 1] = matches[candidateMatchIdx]; + matches[candidateMatchIdx] = tmp; + --candidateMatchIdx; + } + (*nbMatches)++; + } else { + printf("MATCHDIFF: ldm: (len: %u, off: %u), best: (len: %u, off: %u)\n", candidateMatchLength, candidateOffCode, matches[*nbMatches-1].len, matches[*nbMatches-1].off); + matches[*nbMatches].len = candidateMatchLength; + matches[*nbMatches].off = candidateOffCode; + (*nbMatches)++; + } + } } } @@ -900,7 +940,7 @@ static void ldm_handleLdm(rawSeqStore_t* ldmSeqStore, ZSTD_match_t* matches, U32 ldm_skipSequences(ldmSeqStore, posOvershoot, MINMATCH); int noMoreLdms = ldm_getNextMatch(ldmSeqStore, matchStartPosInBlock, matchEndPosInBlock, matchOffset, - matchEndPosInBlock, remainingBytes, sbi); + currPosInBlock, remainingBytes, sbi); } else { int noMoreLdms = ldm_getNextMatch(ldmSeqStore, matchStartPosInBlock, matchEndPosInBlock, matchOffset, From a2f2b58d04956d79a88fa4cafabda52a995952ca Mon Sep 17 00:00:00 2001 From: senhuang42 Date: Mon, 28 Sep 2020 10:29:17 -0400 Subject: [PATCH 26/68] Add a function ldm_voidSequences() --- lib/compress/zstd_opt.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/lib/compress/zstd_opt.c b/lib/compress/zstd_opt.c index f1e39d7c3..70ec8a941 100644 --- a/lib/compress/zstd_opt.c +++ b/lib/compress/zstd_opt.c @@ -789,6 +789,15 @@ static void ldm_skipOvershotBytes(rawSeqStore_t* rawSeqStore, U32 bytesOvershot) } }*/ + +static void ldm_voidSequences(rawSeqStore_t* ldmSeqStore, U32 overshotBytes) { + U32 posAdjustment; + U32 bytesAdjustment; + while (overshotBytes > 0 && ldmSeqStore->pos < ldmSeqStore->size) { + + } +} + static void ldm_skipSequences(rawSeqStore_t* rawSeqStore, size_t srcSize, U32 const minMatch) { while (srcSize > 0 && rawSeqStore->pos < rawSeqStore->size) { printf("ldm_skipSequences(): %u remaining\n", srcSize); From 28394b64f242dd6efcdda2dd3f4b8d926933fd97 Mon Sep 17 00:00:00 2001 From: senhuang42 Date: Mon, 28 Sep 2020 17:58:23 -0400 Subject: [PATCH 27/68] Add proper bounds check on adding ldms --- lib/compress/zstd_ldm.c | 8 ++++++++ lib/compress/zstd_opt.c | 35 ++++++++++++++++++----------------- 2 files changed, 26 insertions(+), 17 deletions(-) diff --git a/lib/compress/zstd_ldm.c b/lib/compress/zstd_ldm.c index 5f44ce78a..e31c6838d 100644 --- a/lib/compress/zstd_ldm.c +++ b/lib/compress/zstd_ldm.c @@ -562,6 +562,13 @@ static rawSeq maybeSplitSequence(rawSeqStore_t* rawSeqStore, return sequence; } +static void printSeqStore(rawSeqStore_t* rawSeqStore) { + printf("rawSeqStore: pos: %zu, bytesDiscarded: %zu\n", rawSeqStore->pos); + for (int i = 0; i < rawSeqStore->size; ++i) { + printf("(of:%u ml:%u ll: %u)\n", rawSeqStore->seq[i].offset, rawSeqStore->seq[i].matchLength, rawSeqStore->seq[i].litLength); + } +} + size_t ZSTD_ldm_blockCompress(rawSeqStore_t* rawSeqStore, ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], void const* src, size_t srcSize) @@ -578,6 +585,7 @@ size_t ZSTD_ldm_blockCompress(rawSeqStore_t* rawSeqStore, if (cParams->strategy >= ZSTD_btopt) { size_t lastLLSize; + printSeqStore(rawSeqStore); ms->ldmSeqStore = *rawSeqStore; /* copy current seqStore */ const BYTE* const prevBase = (BYTE const*)ms->window.base; lastLLSize = blockCompressor(ms, seqStore, rep, src, srcSize); diff --git a/lib/compress/zstd_opt.c b/lib/compress/zstd_opt.c index 70ec8a941..bc6436a1a 100644 --- a/lib/compress/zstd_opt.c +++ b/lib/compress/zstd_opt.c @@ -828,6 +828,7 @@ static void ldm_skipSequences(rawSeqStore_t* rawSeqStore, size_t srcSize, U32 co } srcSize -= seq->matchLength; seq->matchLength = 0; + printf("ldm_skipSequences(): final: (of: %u, ml: %u, ll: %u)\n", seq->offset, seq->matchLength, seq->litLength); rawSeqStore->pos++; } } @@ -867,10 +868,17 @@ static int ldm_getNextMatch(rawSeqStore_t* ldmSeqStore, U32* matchStartPosInBlock, U32* matchEndPosInBlock, U32* matchOffset, U32 currPosInBlock, U32 remainingBytes, U32 sbi) { - rawSeq seq = ldm_splitSequence(ldmSeqStore, remainingBytes); - if (seq.offset == 0 || ldmSeqStore->pos > ldmSeqStore->size) { + if (ldmSeqStore->pos >= ldmSeqStore->size) { // Don't use the LDM for the rest of the block (there is none) - printf("No ldm left in the block\n"); + printf("No ldm left in the block, pos max reached\n"); + *matchStartPosInBlock = UINT32_MAX; + *matchEndPosInBlock = UINT32_MAX; + return 1; + } + rawSeq seq = ldm_splitSequence(ldmSeqStore, remainingBytes); + if (seq.offset == 0) { + // Don't use the LDM for the rest of the block (there is none) + printf("No ldm left in the block, offset = 0\n"); *matchStartPosInBlock = UINT32_MAX; *matchEndPosInBlock = UINT32_MAX; return 1; @@ -898,11 +906,8 @@ static void ldm_maybeAddLdm(ZSTD_match_t* matches, U32* nbMatches, U32 candidateOffCode = matchOffset + posDiff + ZSTD_REP_MOVE; if ((*nbMatches == 0 || candidateMatchLength >= matches[*nbMatches-1].len) && *nbMatches < ZSTD_OPT_NUM) { - printf("large enough, adding\n"); - /*matches[*nbMatches].len = candidateMatchLength; - matches[*nbMatches].off = candidateOffCode; - (*nbMatches)++;*/ - + printf("large enough: curr: %u currposinblock: %u (ofcode: %u, ml: %u)\n", curr, currPosInBlock, candidateOffCode, candidateMatchLength); + if (*nbMatches == 0) { matches[*nbMatches].len = candidateMatchLength; matches[*nbMatches].off = candidateOffCode; @@ -947,14 +952,10 @@ static void ldm_handleLdm(rawSeqStore_t* ldmSeqStore, ZSTD_match_t* matches, U32 U32 posOvershoot = currPosInBlock - *matchEndPosInBlock; printf("Overshot position by: %u\n", posOvershoot); ldm_skipSequences(ldmSeqStore, posOvershoot, MINMATCH); - int noMoreLdms = ldm_getNextMatch(ldmSeqStore, matchStartPosInBlock, - matchEndPosInBlock, matchOffset, - currPosInBlock, remainingBytes, sbi); - } else { - int noMoreLdms = ldm_getNextMatch(ldmSeqStore, matchStartPosInBlock, - matchEndPosInBlock, matchOffset, - currPosInBlock, remainingBytes, sbi); - } + } + int noMoreLdms = ldm_getNextMatch(ldmSeqStore, matchStartPosInBlock, + matchEndPosInBlock, matchOffset, + currPosInBlock, remainingBytes, sbi); } ldm_maybeAddLdm(matches, nbMatches, *matchStartPosInBlock, *matchEndPosInBlock, *matchOffset, currPosInBlock, curr, sbi); } @@ -1018,7 +1019,7 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms, U32 ldmEndPosInBlock = 0; U32 ldmOffset = 0; - printf("SBI for this block: %u\n", sbi); + printf("SBI for this block: %u, base: %u\n", sbi, base); if (ms->ldmSeqStore.size != 0) { ldm_getNextMatch(&ms->ldmSeqStore, &ldmStartPosInBlock, &ldmEndPosInBlock, &ldmOffset, (U32)(ip-istart), (U32)(iend-ip), sbi); From 6ccd97fc96d4769fc4edcbd3dfdc66f38e214d2c Mon Sep 17 00:00:00 2001 From: senhuang42 Date: Tue, 29 Sep 2020 12:12:58 -0400 Subject: [PATCH 28/68] Fixed end of match boundary update issues --- lib/compress/zstd_compress.c | 4 +- lib/compress/zstd_compress_internal.h | 1 + lib/compress/zstd_ldm.c | 17 ++--- lib/compress/zstd_opt.c | 104 +++++++++++++++----------- lib/compress/zstdmt_compress.c | 2 +- 5 files changed, 72 insertions(+), 56 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 037b5360a..3c80ab860 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -2336,7 +2336,7 @@ static size_t ZSTD_buildSeqStore(ZSTD_CCtx* zc, const void* src, size_t srcSize) if (curr > ms->nextToUpdate + 384) ms->nextToUpdate = curr - MIN(192, (U32)(curr - ms->nextToUpdate - 384)); } - printf("--NEW BLOCK--\n"); + //printf("--NEW BLOCK--\n"); /* select and store sequences */ { ZSTD_dictMode_e const dictMode = ZSTD_matchState_dictMode(ms); size_t lastLLSize; @@ -2354,7 +2354,7 @@ static size_t ZSTD_buildSeqStore(ZSTD_CCtx* zc, const void* src, size_t srcSize) src, srcSize); assert(zc->externSeqStore.pos <= zc->externSeqStore.size); } else if (zc->appliedParams.ldmParams.enableLdm) { - rawSeqStore_t ldmSeqStore = {NULL, 0, 0, 0}; + rawSeqStore_t ldmSeqStore = {NULL, NULL, 0, 0, 0}; ldmSeqStore.seq = zc->ldmSequences; ldmSeqStore.capacity = zc->maxNbLdmSequences; diff --git a/lib/compress/zstd_compress_internal.h b/lib/compress/zstd_compress_internal.h index 524cbb025..53c20b12c 100644 --- a/lib/compress/zstd_compress_internal.h +++ b/lib/compress/zstd_compress_internal.h @@ -94,6 +94,7 @@ typedef struct { typedef struct { rawSeq* seq; /* The start of the sequences */ + BYTE const* base; size_t pos; /* The position where reading stopped. <= size. */ size_t size; /* The number of sequences. <= capacity. */ size_t capacity; /* The capacity starting from `seq` pointer */ diff --git a/lib/compress/zstd_ldm.c b/lib/compress/zstd_ldm.c index e31c6838d..9fcb9fc29 100644 --- a/lib/compress/zstd_ldm.c +++ b/lib/compress/zstd_ldm.c @@ -563,9 +563,9 @@ static rawSeq maybeSplitSequence(rawSeqStore_t* rawSeqStore, } static void printSeqStore(rawSeqStore_t* rawSeqStore) { - printf("rawSeqStore: pos: %zu, bytesDiscarded: %zu\n", rawSeqStore->pos); + printf("rawSeqStore: pos: %zu\n", rawSeqStore->pos); for (int i = 0; i < rawSeqStore->size; ++i) { - printf("(of:%u ml:%u ll: %u)\n", rawSeqStore->seq[i].offset, rawSeqStore->seq[i].matchLength, rawSeqStore->seq[i].litLength); + printf("pos %d (of:%u ml:%u ll: %u)\n", i, rawSeqStore->seq[i].offset, rawSeqStore->seq[i].matchLength, rawSeqStore->seq[i].litLength); } } @@ -582,20 +582,19 @@ size_t ZSTD_ldm_blockCompress(rawSeqStore_t* rawSeqStore, BYTE const* const iend = istart + srcSize; /* Input positions */ BYTE const* ip = istart; - + //printSeqStore(rawSeqStore); if (cParams->strategy >= ZSTD_btopt) { size_t lastLLSize; - printSeqStore(rawSeqStore); + //printSeqStore(rawSeqStore); ms->ldmSeqStore = *rawSeqStore; /* copy current seqStore */ - const BYTE* const prevBase = (BYTE const*)ms->window.base; + ms->ldmSeqStore.base = ms->window.base; lastLLSize = blockCompressor(ms, seqStore, rep, src, srcSize); - rawSeqStore->pos = ms->ldmSeqStore.pos; - ms->ldmSeqStore = *rawSeqStore; - if (prevBase != ms->window.base) { + *rawSeqStore = ms->ldmSeqStore; + /*if (prevBase != ms->window.base) { int baseDiff = (int)(prevBase - ms->window.base); printf("Bases were different, adjusting, diff = %d\n", baseDiff); rawSeqStore->seq[rawSeqStore->pos].litLength += baseDiff; - } + }*/ return lastLLSize; } diff --git a/lib/compress/zstd_opt.c b/lib/compress/zstd_opt.c index bc6436a1a..05fa4cae7 100644 --- a/lib/compress/zstd_opt.c +++ b/lib/compress/zstd_opt.c @@ -800,85 +800,85 @@ static void ldm_voidSequences(rawSeqStore_t* ldmSeqStore, U32 overshotBytes) { static void ldm_skipSequences(rawSeqStore_t* rawSeqStore, size_t srcSize, U32 const minMatch) { while (srcSize > 0 && rawSeqStore->pos < rawSeqStore->size) { - printf("ldm_skipSequences(): %u remaining\n", srcSize); + ////printf("ldm_skipSequences(): %u remaining\n", srcSize); rawSeq* seq = rawSeqStore->seq + rawSeqStore->pos; - printf("ldm_skipSequences(): before: (of: %u, ml: %u, ll: %u)\n", seq->offset, seq->matchLength, seq->litLength); + ////printf("ldm_skipSequences(): before: (of: %u, ml: %u, ll: %u)\n", seq->offset, seq->matchLength, seq->litLength); if (srcSize <= seq->litLength) { /* Skip past srcSize literals */ seq->litLength -= (U32)srcSize; - printf("ldm_skipSequences(): final: (of: %u, ml: %u, ll: %u)\n", seq->offset, seq->matchLength, seq->litLength); + ////printf("ldm_skipSequences(): seqstore final: (of: %u, ml: %u, ll: %u) at %u\n", seq->offset, seq->matchLength, seq->litLength, rawSeqStore->pos); return; } srcSize -= seq->litLength; seq->litLength = 0; if (srcSize < seq->matchLength) { - printf("Splitting match: ml curr: %u ", seq->matchLength); + ////printf("Splitting match: ml curr: %u ", seq->matchLength); /* Skip past the first srcSize of the match */ seq->matchLength -= (U32)srcSize; - printf("ml in store left: %u ", seq->matchLength); - if (seq->matchLength < minMatch) { - /* The match is too short, omit it */ + ////printf("ml in store left: %u ", seq->matchLength); + /*if (seq->matchLength < minMatch) { + // The match is too short, omit it if (rawSeqStore->pos + 1 < rawSeqStore->size) { seq[1].litLength += seq[0].matchLength; } rawSeqStore->pos++; - } - printf("ldm_skipSequences(): final: (of: %u, ml: %u, ll: %u)\n", seq->offset, seq->matchLength, seq->litLength); + }*/ + //printf("ldm_skipSequences(): seqstore final: (of: %u, ml: %u, ll: %u) at %u\n", seq->offset, seq->matchLength, seq->litLength, rawSeqStore->pos); return; } srcSize -= seq->matchLength; seq->matchLength = 0; - printf("ldm_skipSequences(): final: (of: %u, ml: %u, ll: %u)\n", seq->offset, seq->matchLength, seq->litLength); + //printf("ldm_skipSequences(): seqstore final: (of: %u, ml: %u, ll: %u) at %u\n", seq->offset, seq->matchLength, seq->litLength, rawSeqStore->pos); rawSeqStore->pos++; } } // The only function that can update pos (i think, for now) -static rawSeq ldm_splitSequence(rawSeqStore_t* ldmSeqStore, U32 remainingBytes) { +static rawSeq ldm_splitSequence(rawSeqStore_t* ldmSeqStore, U32* ldmSeqStoreBytesConsumed, U32 remainingBytes) { rawSeq currSeq = ldmSeqStore->seq[ldmSeqStore->pos]; - printf("Current sequence: (of: %u, ml: %u, ll: %u)\n", currSeq.offset, currSeq.matchLength, currSeq.litLength); - printf("Split Sequence with remaining = %u : ", remainingBytes); + //printf("ldm_splitSequence(): Current sequence: (of: %u, ml: %u, ll: %u) - remaining: %u\n", currSeq.offset, currSeq.matchLength, currSeq.litLength, remainingBytes); /* No split */ if (remainingBytes >= currSeq.litLength + currSeq.matchLength) { - printf("NO SPLIT\n"); + ////printf("NO SPLIT\n"); + *ldmSeqStoreBytesConsumed += currSeq.litLength + currSeq.matchLength; ldmSeqStore->pos++; - printf("pos is now: %u\n", ldmSeqStore->pos); + ////printf("pos is now: %u\n", ldmSeqStore->pos); return currSeq; } /* Need a split */ if (remainingBytes <= currSeq.litLength) { - printf("SPLITTING: all remaining bytes were literals"); + ////printf("SPLITTING: all remaining bytes were literals"); currSeq.offset = 0; } else if (remainingBytes < currSeq.litLength + currSeq.matchLength) { - if (currSeq.matchLength < MINMATCH) { - printf("CurrSeq less than minmatch: all remaining bytes were literals"); + /*if (currSeq.matchLength < MINMATCH) { + ////printf("CurrSeq less than minmatch: all remaining bytes were literals"); currSeq.offset = 0; - } + }*/ currSeq.matchLength = remainingBytes - currSeq.litLength; } - printf("\n"); - + ////printf("\n"); + *ldmSeqStoreBytesConsumed += remainingBytes; ldm_skipSequences(ldmSeqStore, remainingBytes, MINMATCH); - printf("Sequence final: (of: %u, ml: %u, ll: %u)\n", currSeq.offset, currSeq.matchLength, currSeq.litLength); + //printf("ldm_splitSequence(): Sequence final: (of: %u, ml: %u, ll: %u)\n", currSeq.offset, currSeq.matchLength, currSeq.litLength); return currSeq; } /* Returns 1 if the rest of the block is just LDM literals */ -static int ldm_getNextMatch(rawSeqStore_t* ldmSeqStore, +static int ldm_getNextMatch(rawSeqStore_t* ldmSeqStore, U32* ldmSeqStoreBytesConsumed, U32* matchStartPosInBlock, U32* matchEndPosInBlock, U32* matchOffset, U32 currPosInBlock, U32 remainingBytes, U32 sbi) { if (ldmSeqStore->pos >= ldmSeqStore->size) { // Don't use the LDM for the rest of the block (there is none) - printf("No ldm left in the block, pos max reached\n"); + //printf("No ldm left in the block, pos max reached\n"); *matchStartPosInBlock = UINT32_MAX; *matchEndPosInBlock = UINT32_MAX; return 1; } - rawSeq seq = ldm_splitSequence(ldmSeqStore, remainingBytes); + rawSeq seq = ldm_splitSequence(ldmSeqStore, ldmSeqStoreBytesConsumed, remainingBytes); if (seq.offset == 0) { // Don't use the LDM for the rest of the block (there is none) - printf("No ldm left in the block, offset = 0\n"); + ////printf("No ldm left in the block, offset = 0\n"); *matchStartPosInBlock = UINT32_MAX; *matchEndPosInBlock = UINT32_MAX; return 1; @@ -887,14 +887,14 @@ static int ldm_getNextMatch(rawSeqStore_t* ldmSeqStore, *matchStartPosInBlock = currPosInBlock + seq.litLength; *matchEndPosInBlock = *matchStartPosInBlock + seq.matchLength; *matchOffset = seq.offset; - printf("New match range in block is: (%u, %u) with of: %u where currPosInBlock: %u at adjusted absolute range: %u, %u\n", *matchStartPosInBlock, *matchEndPosInBlock, *matchOffset, currPosInBlock, *matchStartPosInBlock+sbi, *matchEndPosInBlock+sbi); + //printf("New match range in block is: (%u, %u) with of: %u where currPosInBlock: %u at adjusted absolute range: %u, %u\n", *matchStartPosInBlock, *matchEndPosInBlock, *matchOffset, currPosInBlock, *matchStartPosInBlock+sbi, *matchEndPosInBlock+sbi); return 0; } /* Adds an LDM if it's long enough */ static void ldm_maybeAddLdm(ZSTD_match_t* matches, U32* nbMatches, U32 matchStartPosInBlock, U32 matchEndPosInBlock, - U32 matchOffset, U32 currPosInBlock, U32 curr, U32 sbi) { + U32 matchOffset, U32 currPosInBlock, U32 curr, U32 sbi, rawSeqStore_t* rawSeqStore) { /* Check that current block position is not outside of the match */ if (currPosInBlock < matchStartPosInBlock || currPosInBlock >= matchEndPosInBlock) return; @@ -906,7 +906,8 @@ static void ldm_maybeAddLdm(ZSTD_match_t* matches, U32* nbMatches, U32 candidateOffCode = matchOffset + posDiff + ZSTD_REP_MOVE; if ((*nbMatches == 0 || candidateMatchLength >= matches[*nbMatches-1].len) && *nbMatches < ZSTD_OPT_NUM) { - printf("large enough: curr: %u currposinblock: %u (ofcode: %u, ml: %u)\n", curr, currPosInBlock, candidateOffCode, candidateMatchLength); + //printf("large enough: curr: %u currposinblock: %u (ofcode: %u, ml: %u) - range: (%u, %u) - (of: %u ml: %u ll: %u) @ pos: %u - \n", + // curr, currPosInBlock, candidateOffCode, candidateMatchLength, matchStartPosInBlock, matchEndPosInBlock, rawSeqStore->seq[rawSeqStore->pos].offset, rawSeqStore->seq[rawSeqStore->pos].matchLength, rawSeqStore->seq[rawSeqStore->pos].litLength, rawSeqStore->pos); if (*nbMatches == 0) { matches[*nbMatches].len = candidateMatchLength; @@ -917,15 +918,15 @@ static void ldm_maybeAddLdm(ZSTD_match_t* matches, U32* nbMatches, U32 candidateMatchIdx = *nbMatches; matches[*nbMatches].len = candidateMatchLength; matches[*nbMatches].off = candidateOffCode; - printf("Sifting...: idx: %u, len: %u, off: %u\n", candidateMatchIdx, candidateMatchLength, candidateOffCode); - //printf("Current best is...: idx: %u, len: %u, off: %u\n", *nbMatches-1, matches[*nbMatches-1].len, matches[*nbMatches-1].off); + ////printf("Sifting...: idx: %u, len: %u, off: %u\n", candidateMatchIdx, candidateMatchLength, candidateOffCode); + //////printf("Current best is...: idx: %u, len: %u, off: %u\n", *nbMatches-1, matches[*nbMatches-1].len, matches[*nbMatches-1].off); if (candidateOffCode != matches[*nbMatches].off) - printf("DIFF: ldm: (len: %u, off: %u), best: (len: %u, off: %u)\n", candidateMatchLength, candidateOffCode, matches[*nbMatches-1].len, matches[*nbMatches-1].off); - //printf("Current best is...: idx: %u, len: %u, off: %u\n", *nbMatches-1, matches[*nbMatches-1].len, matches[*nbMatches-1].off); + ////printf("DIFF: ldm: (len: %u, off: %u), best: (len: %u, off: %u)\n", candidateMatchLength, candidateOffCode, matches[*nbMatches-1].len, matches[*nbMatches-1].off); + //////printf("Current best is...: idx: %u, len: %u, off: %u\n", *nbMatches-1, matches[*nbMatches-1].len, matches[*nbMatches-1].off); while (candidateMatchIdx > 0 && matches[candidateMatchIdx].off > matches[candidateMatchIdx - 1].off && matches[candidateMatchIdx].len == matches[candidateMatchIdx - 1].len) { - //printf("Compared to: idx: %u, len: %u, off: %u", candidateMatchIdx - 1, matches[candidateMatchIdx - 1].len, matches[candidateMatchIdx - 1].off); + //////printf("Compared to: idx: %u, len: %u, off: %u", candidateMatchIdx - 1, matches[candidateMatchIdx - 1].len, matches[candidateMatchIdx - 1].off); ZSTD_match_t tmp = matches[candidateMatchIdx - 1]; matches[candidateMatchIdx - 1] = matches[candidateMatchIdx]; matches[candidateMatchIdx] = tmp; @@ -933,7 +934,7 @@ static void ldm_maybeAddLdm(ZSTD_match_t* matches, U32* nbMatches, } (*nbMatches)++; } else { - printf("MATCHDIFF: ldm: (len: %u, off: %u), best: (len: %u, off: %u)\n", candidateMatchLength, candidateOffCode, matches[*nbMatches-1].len, matches[*nbMatches-1].off); + ////printf("MATCHDIFF: ldm: (len: %u, off: %u), best: (len: %u, off: %u)\n", candidateMatchLength, candidateOffCode, matches[*nbMatches-1].len, matches[*nbMatches-1].off); matches[*nbMatches].len = candidateMatchLength; matches[*nbMatches].off = candidateOffCode; (*nbMatches)++; @@ -943,21 +944,22 @@ static void ldm_maybeAddLdm(ZSTD_match_t* matches, U32* nbMatches, } /* Wrapper function to call ldm functions as needed */ -static void ldm_handleLdm(rawSeqStore_t* ldmSeqStore, ZSTD_match_t* matches, U32* nbMatches, +static void ldm_handleLdm(rawSeqStore_t* ldmSeqStore, U32* ldmSeqStoreBytesConsumed, ZSTD_match_t* matches, U32* nbMatches, U32* matchStartPosInBlock, U32* matchEndPosInBlock, U32* matchOffset, U32 currPosInBlock, U32 remainingBytes, U32 curr, U32 sbi) { + ////printf("currPosInBlock: %u, curr: %u, called ldm_handleLdm()", currPosInBlock, curr); if (currPosInBlock >= *matchEndPosInBlock) { - printf("Went over match boundary: currPosInBlock: %u, %matchEndPosInBlock: %u\n", currPosInBlock, *matchEndPosInBlock); + //printf("Went over match boundary: currPosInBlock: %u, %matchEndPosInBlock: %u\n", currPosInBlock, *matchEndPosInBlock); if (currPosInBlock > *matchEndPosInBlock) { U32 posOvershoot = currPosInBlock - *matchEndPosInBlock; - printf("Overshot position by: %u\n", posOvershoot); + //printf("Overshot position by: %u\n", posOvershoot); ldm_skipSequences(ldmSeqStore, posOvershoot, MINMATCH); } - int noMoreLdms = ldm_getNextMatch(ldmSeqStore, matchStartPosInBlock, + int noMoreLdms = ldm_getNextMatch(ldmSeqStore, ldmSeqStoreBytesConsumed, matchStartPosInBlock, matchEndPosInBlock, matchOffset, currPosInBlock, remainingBytes, sbi); } - ldm_maybeAddLdm(matches, nbMatches, *matchStartPosInBlock, *matchEndPosInBlock, *matchOffset, currPosInBlock, curr, sbi); + ldm_maybeAddLdm(matches, nbMatches, *matchStartPosInBlock, *matchEndPosInBlock, *matchOffset, currPosInBlock, curr, sbi, ldmSeqStore); } @@ -1018,10 +1020,17 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms, U32 ldmStartPosInBlock = 0; U32 ldmEndPosInBlock = 0; U32 ldmOffset = 0; + U32 ldmSeqStoreBytesConsumed = 0; - printf("SBI for this block: %u, base: %u\n", sbi, base); + ////printf("SBI for this block: %u, base: %u\n", sbi, base); + if (ms->ldmSeqStore.size > 0 && ms->ldmSeqStore.base != base) { + int baseDiff = (int)(ms->ldmSeqStore.base - base); + //printf("Bases were different, adjusting, diff = %d\n", baseDiff); + ms->ldmSeqStore.seq[ms->ldmSeqStore.pos].litLength += baseDiff; + ms->ldmSeqStore.base = ms->window.base; + } if (ms->ldmSeqStore.size != 0) { - ldm_getNextMatch(&ms->ldmSeqStore, &ldmStartPosInBlock, + ldm_getNextMatch(&ms->ldmSeqStore, &ldmSeqStoreBytesConsumed, &ldmStartPosInBlock, &ldmEndPosInBlock, &ldmOffset, (U32)(ip-istart), (U32)(iend-ip), sbi); } /* init */ @@ -1040,7 +1049,7 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms, U32 const ll0 = !litlen; U32 nbMatches = ZSTD_BtGetAllMatches(matches, ms, &nextToUpdate3, ip, iend, dictMode, rep, ll0, minMatch); if (ms->ldmSeqStore.size != 0) { - ldm_handleLdm(&ms->ldmSeqStore, matches, + ldm_handleLdm(&ms->ldmSeqStore, &ldmSeqStoreBytesConsumed, matches, &nbMatches, &ldmStartPosInBlock, &ldmEndPosInBlock, &ldmOffset, (U32)(ip-istart), (U32)(iend - ip), (U32)(ip-base), sbi); @@ -1162,7 +1171,7 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms, if (ms->ldmSeqStore.size != 0) { - ldm_handleLdm(&ms->ldmSeqStore, matches, + ldm_handleLdm(&ms->ldmSeqStore, &ldmSeqStoreBytesConsumed, matches, &nbMatches, &ldmStartPosInBlock, &ldmEndPosInBlock, &ldmOffset, (U32)(inr-istart), (U32)(iend-inr), (U32)(inr-base), sbi); @@ -1281,6 +1290,13 @@ _shortestPath: /* cur, last_pos, best_mlen, best_off have to be set */ } } /* while (ip < ilimit) */ + int i = ms->ldmSeqStore.pos; + //printf("pos %d (of:%u ml:%u ll: %u)\n", i, ms->ldmSeqStore.seq[i].offset, ms->ldmSeqStore.seq[i].matchLength, ms->ldmSeqStore.seq[i].litLength); + //printf("matchend: %u bytesconsumed: %u\n", ldmEndPosInBlock, ldmSeqStoreBytesConsumed); + if (ldmEndPosInBlock < srcSize) { + //printf("Needs adjustment, endpos didn't reach end of block\n"); + ldm_skipSequences(&ms->ldmSeqStore, srcSize - ldmEndPosInBlock, MINMATCH); + } /* Return the last literals size */ return (size_t)(iend - anchor); } diff --git a/lib/compress/zstdmt_compress.c b/lib/compress/zstdmt_compress.c index baf6ef4ca..57015f156 100644 --- a/lib/compress/zstdmt_compress.c +++ b/lib/compress/zstdmt_compress.c @@ -277,7 +277,7 @@ static size_t ZSTDMT_sizeof_seqPool(ZSTDMT_seqPool* seqPool) static rawSeqStore_t bufferToSeq(buffer_t buffer) { - rawSeqStore_t seq = {NULL, 0, 0, 0}; + rawSeqStore_t seq = {NULL, NULL, 0, 0, 0}; seq.seq = (rawSeq*)buffer.start; seq.capacity = buffer.capacity / sizeof(rawSeq); return seq; From 78da2e1808ea40e68320dc045e996024daa242f9 Mon Sep 17 00:00:00 2001 From: senhuang42 Date: Tue, 29 Sep 2020 15:35:13 -0400 Subject: [PATCH 29/68] Fixed sifting algorithm --- lib/compress/zstd_opt.c | 137 ++++++++++++++++++++-------------------- 1 file changed, 67 insertions(+), 70 deletions(-) diff --git a/lib/compress/zstd_opt.c b/lib/compress/zstd_opt.c index 05fa4cae7..6cea5a38b 100644 --- a/lib/compress/zstd_opt.c +++ b/lib/compress/zstd_opt.c @@ -768,54 +768,24 @@ FORCE_INLINE_TEMPLATE U32 ZSTD_BtGetAllMatches ( * LDM util functions *********************************/ -/* -static void ldm_skipOvershotBytes(rawSeqStore_t* rawSeqStore, U32 bytesOvershot) { - while (bytesOvershot > 0 && rawSeqStore->pos < rawSeqStore->size) { - rawSeq seq = rawSeqStore->seq[rawSeqStore->pos]; - if (bytesOvershot <= seq.litLength) { - seq->litLength -= bytesOvershot; - } - } - if (rawSeqStore->seq[rawSeqStore->pos].matchLength > posOvershoot) { - rawSeqStore->seq[rawSeqStore->pos].matchLength -= posOvershoot; - return; - } else { - rawSeqStore->seq[rawSeqStore->pos].matchLength = 0; - rawSeqStore->pos++; - } - - while (posOvershoot > 0 && rawSeqStore->pos < rawSeqStore->size) { - - } -}*/ - - -static void ldm_voidSequences(rawSeqStore_t* ldmSeqStore, U32 overshotBytes) { - U32 posAdjustment; - U32 bytesAdjustment; - while (overshotBytes > 0 && ldmSeqStore->pos < ldmSeqStore->size) { - - } -} - static void ldm_skipSequences(rawSeqStore_t* rawSeqStore, size_t srcSize, U32 const minMatch) { while (srcSize > 0 && rawSeqStore->pos < rawSeqStore->size) { - ////printf("ldm_skipSequences(): %u remaining\n", srcSize); + //////printf("ldm_skipSequences(): %u remaining\n", srcSize); rawSeq* seq = rawSeqStore->seq + rawSeqStore->pos; - ////printf("ldm_skipSequences(): before: (of: %u, ml: %u, ll: %u)\n", seq->offset, seq->matchLength, seq->litLength); + //////printf("ldm_skipSequences(): before: (of: %u, ml: %u, ll: %u)\n", seq->offset, seq->matchLength, seq->litLength); if (srcSize <= seq->litLength) { /* Skip past srcSize literals */ seq->litLength -= (U32)srcSize; - ////printf("ldm_skipSequences(): seqstore final: (of: %u, ml: %u, ll: %u) at %u\n", seq->offset, seq->matchLength, seq->litLength, rawSeqStore->pos); + //////printf("ldm_skipSequences(): seqstore final: (of: %u, ml: %u, ll: %u) at %u\n", seq->offset, seq->matchLength, seq->litLength, rawSeqStore->pos); return; } srcSize -= seq->litLength; seq->litLength = 0; if (srcSize < seq->matchLength) { - ////printf("Splitting match: ml curr: %u ", seq->matchLength); + //////printf("Splitting match: ml curr: %u ", seq->matchLength); /* Skip past the first srcSize of the match */ seq->matchLength -= (U32)srcSize; - ////printf("ml in store left: %u ", seq->matchLength); + //////printf("ml in store left: %u ", seq->matchLength); /*if (seq->matchLength < minMatch) { // The match is too short, omit it if (rawSeqStore->pos + 1 < rawSeqStore->size) { @@ -823,12 +793,12 @@ static void ldm_skipSequences(rawSeqStore_t* rawSeqStore, size_t srcSize, U32 co } rawSeqStore->pos++; }*/ - //printf("ldm_skipSequences(): seqstore final: (of: %u, ml: %u, ll: %u) at %u\n", seq->offset, seq->matchLength, seq->litLength, rawSeqStore->pos); + ////printf("ldm_skipSequences(): seqstore final: (of: %u, ml: %u, ll: %u) at %u\n", seq->offset, seq->matchLength, seq->litLength, rawSeqStore->pos); return; } srcSize -= seq->matchLength; seq->matchLength = 0; - //printf("ldm_skipSequences(): seqstore final: (of: %u, ml: %u, ll: %u) at %u\n", seq->offset, seq->matchLength, seq->litLength, rawSeqStore->pos); + ////printf("ldm_skipSequences(): seqstore final: (of: %u, ml: %u, ll: %u) at %u\n", seq->offset, seq->matchLength, seq->litLength, rawSeqStore->pos); rawSeqStore->pos++; } } @@ -836,30 +806,30 @@ static void ldm_skipSequences(rawSeqStore_t* rawSeqStore, size_t srcSize, U32 co // The only function that can update pos (i think, for now) static rawSeq ldm_splitSequence(rawSeqStore_t* ldmSeqStore, U32* ldmSeqStoreBytesConsumed, U32 remainingBytes) { rawSeq currSeq = ldmSeqStore->seq[ldmSeqStore->pos]; - //printf("ldm_splitSequence(): Current sequence: (of: %u, ml: %u, ll: %u) - remaining: %u\n", currSeq.offset, currSeq.matchLength, currSeq.litLength, remainingBytes); + ////printf("ldm_splitSequence(): Current sequence: (of: %u, ml: %u, ll: %u) - remaining: %u\n", currSeq.offset, currSeq.matchLength, currSeq.litLength, remainingBytes); /* No split */ if (remainingBytes >= currSeq.litLength + currSeq.matchLength) { - ////printf("NO SPLIT\n"); + //////printf("NO SPLIT\n"); *ldmSeqStoreBytesConsumed += currSeq.litLength + currSeq.matchLength; ldmSeqStore->pos++; - ////printf("pos is now: %u\n", ldmSeqStore->pos); + //////printf("pos is now: %u\n", ldmSeqStore->pos); return currSeq; } /* Need a split */ if (remainingBytes <= currSeq.litLength) { - ////printf("SPLITTING: all remaining bytes were literals"); + //////printf("SPLITTING: all remaining bytes were literals"); currSeq.offset = 0; } else if (remainingBytes < currSeq.litLength + currSeq.matchLength) { /*if (currSeq.matchLength < MINMATCH) { - ////printf("CurrSeq less than minmatch: all remaining bytes were literals"); + //////printf("CurrSeq less than minmatch: all remaining bytes were literals"); currSeq.offset = 0; }*/ currSeq.matchLength = remainingBytes - currSeq.litLength; } - ////printf("\n"); + //////printf("\n"); *ldmSeqStoreBytesConsumed += remainingBytes; ldm_skipSequences(ldmSeqStore, remainingBytes, MINMATCH); - //printf("ldm_splitSequence(): Sequence final: (of: %u, ml: %u, ll: %u)\n", currSeq.offset, currSeq.matchLength, currSeq.litLength); + ////printf("ldm_splitSequence(): Sequence final: (of: %u, ml: %u, ll: %u)\n", currSeq.offset, currSeq.matchLength, currSeq.litLength); return currSeq; } @@ -870,7 +840,7 @@ static int ldm_getNextMatch(rawSeqStore_t* ldmSeqStore, U32* ldmSeqStoreBytesCon U32 remainingBytes, U32 sbi) { if (ldmSeqStore->pos >= ldmSeqStore->size) { // Don't use the LDM for the rest of the block (there is none) - //printf("No ldm left in the block, pos max reached\n"); + ////printf("No ldm left in the block, pos max reached\n"); *matchStartPosInBlock = UINT32_MAX; *matchEndPosInBlock = UINT32_MAX; return 1; @@ -878,7 +848,7 @@ static int ldm_getNextMatch(rawSeqStore_t* ldmSeqStore, U32* ldmSeqStoreBytesCon rawSeq seq = ldm_splitSequence(ldmSeqStore, ldmSeqStoreBytesConsumed, remainingBytes); if (seq.offset == 0) { // Don't use the LDM for the rest of the block (there is none) - ////printf("No ldm left in the block, offset = 0\n"); + //////printf("No ldm left in the block, offset = 0\n"); *matchStartPosInBlock = UINT32_MAX; *matchEndPosInBlock = UINT32_MAX; return 1; @@ -887,10 +857,32 @@ static int ldm_getNextMatch(rawSeqStore_t* ldmSeqStore, U32* ldmSeqStoreBytesCon *matchStartPosInBlock = currPosInBlock + seq.litLength; *matchEndPosInBlock = *matchStartPosInBlock + seq.matchLength; *matchOffset = seq.offset; - //printf("New match range in block is: (%u, %u) with of: %u where currPosInBlock: %u at adjusted absolute range: %u, %u\n", *matchStartPosInBlock, *matchEndPosInBlock, *matchOffset, currPosInBlock, *matchStartPosInBlock+sbi, *matchEndPosInBlock+sbi); + ////printf("New match range in block is: (%u, %u) with of: %u where currPosInBlock: %u at adjusted absolute range: %u, %u\n", *matchStartPosInBlock, *matchEndPosInBlock, *matchOffset, currPosInBlock, *matchStartPosInBlock+sbi, *matchEndPosInBlock+sbi); return 0; } +static void printMatches(ZSTD_match_t* matches, U32 nbMatches) { + for (int i = 0; i < nbMatches; ++i) { + //printf("(of: %u ml: %u) ", matches[i].off, matches[i].len); + } + //printf("\n"); +} +static void validateMatches(ZSTD_match_t* matches, U32 nbMatches) { + U32 prevLargestOffset = 0; + U32 prevLargestMatch = 0; + for (int i = 1; i < nbMatches; ++i) { + if (matches[i-1].len > matches[i].len) { + //printf("nope\n"); + exit(1); + } else if (matches[i-1].len == matches[i].len) { + if (matches[i-1].off < matches[i].off) { + //printf("nopeOff\n"); + exit(1); + } + } + } +} + /* Adds an LDM if it's long enough */ static void ldm_maybeAddLdm(ZSTD_match_t* matches, U32* nbMatches, U32 matchStartPosInBlock, U32 matchEndPosInBlock, @@ -903,43 +895,48 @@ static void ldm_maybeAddLdm(ZSTD_match_t* matches, U32* nbMatches, return; // dont deal with extra ldms for now assert(posDiff < matchEndPosInBlock - matchStartPosInBlock); U32 candidateMatchLength = matchEndPosInBlock - matchStartPosInBlock - posDiff; + if (candidateMatchLength < ZSTD_LDM_MINMATCH_MIN) + return; + U32 candidateOffCode = matchOffset + posDiff + ZSTD_REP_MOVE; if ((*nbMatches == 0 || candidateMatchLength >= matches[*nbMatches-1].len) && *nbMatches < ZSTD_OPT_NUM) { //printf("large enough: curr: %u currposinblock: %u (ofcode: %u, ml: %u) - range: (%u, %u) - (of: %u ml: %u ll: %u) @ pos: %u - \n", - // curr, currPosInBlock, candidateOffCode, candidateMatchLength, matchStartPosInBlock, matchEndPosInBlock, rawSeqStore->seq[rawSeqStore->pos].offset, rawSeqStore->seq[rawSeqStore->pos].matchLength, rawSeqStore->seq[rawSeqStore->pos].litLength, rawSeqStore->pos); + // curr, currPosInBlock, candidateOffCode, candidateMatchLength, matchStartPosInBlock, matchEndPosInBlock, rawSeqStore->seq[rawSeqStore->pos].offset, rawSeqStore->seq[rawSeqStore->pos].matchLength, rawSeqStore->seq[rawSeqStore->pos].litLength, rawSeqStore->pos); if (*nbMatches == 0) { + //printf("HERE1\n"); matches[*nbMatches].len = candidateMatchLength; matches[*nbMatches].off = candidateOffCode; (*nbMatches)++; } else { if (candidateMatchLength == matches[*nbMatches-1].len) { + //printf("HERE2\n"); U32 candidateMatchIdx = *nbMatches; matches[*nbMatches].len = candidateMatchLength; matches[*nbMatches].off = candidateOffCode; - ////printf("Sifting...: idx: %u, len: %u, off: %u\n", candidateMatchIdx, candidateMatchLength, candidateOffCode); - //////printf("Current best is...: idx: %u, len: %u, off: %u\n", *nbMatches-1, matches[*nbMatches-1].len, matches[*nbMatches-1].off); - if (candidateOffCode != matches[*nbMatches].off) - ////printf("DIFF: ldm: (len: %u, off: %u), best: (len: %u, off: %u)\n", candidateMatchLength, candidateOffCode, matches[*nbMatches-1].len, matches[*nbMatches-1].off); - //////printf("Current best is...: idx: %u, len: %u, off: %u\n", *nbMatches-1, matches[*nbMatches-1].len, matches[*nbMatches-1].off); - while (candidateMatchIdx > 0 && - matches[candidateMatchIdx].off > matches[candidateMatchIdx - 1].off && - matches[candidateMatchIdx].len == matches[candidateMatchIdx - 1].len) { - //////printf("Compared to: idx: %u, len: %u, off: %u", candidateMatchIdx - 1, matches[candidateMatchIdx - 1].len, matches[candidateMatchIdx - 1].off); - ZSTD_match_t tmp = matches[candidateMatchIdx - 1]; - matches[candidateMatchIdx - 1] = matches[candidateMatchIdx]; - matches[candidateMatchIdx] = tmp; - --candidateMatchIdx; + if (candidateOffCode != matches[*nbMatches-1].off) { + //printf("offsets not equal\n"); + while (candidateMatchIdx > 0 && + matches[candidateMatchIdx].off > matches[candidateMatchIdx - 1].off && + matches[candidateMatchIdx].len == matches[candidateMatchIdx - 1].len) { + //printf("sifting\n"); + ZSTD_match_t tmp = matches[candidateMatchIdx - 1]; + matches[candidateMatchIdx - 1] = matches[candidateMatchIdx]; + matches[candidateMatchIdx] = tmp; + --candidateMatchIdx; + } } (*nbMatches)++; } else { - ////printf("MATCHDIFF: ldm: (len: %u, off: %u), best: (len: %u, off: %u)\n", candidateMatchLength, candidateOffCode, matches[*nbMatches-1].len, matches[*nbMatches-1].off); + //printf("HERE3\n"); matches[*nbMatches].len = candidateMatchLength; matches[*nbMatches].off = candidateOffCode; (*nbMatches)++; } } + printMatches(matches, *nbMatches); + validateMatches(matches, *nbMatches); } } @@ -947,12 +944,12 @@ static void ldm_maybeAddLdm(ZSTD_match_t* matches, U32* nbMatches, static void ldm_handleLdm(rawSeqStore_t* ldmSeqStore, U32* ldmSeqStoreBytesConsumed, ZSTD_match_t* matches, U32* nbMatches, U32* matchStartPosInBlock, U32* matchEndPosInBlock, U32* matchOffset, U32 currPosInBlock, U32 remainingBytes, U32 curr, U32 sbi) { - ////printf("currPosInBlock: %u, curr: %u, called ldm_handleLdm()", currPosInBlock, curr); + //////printf("currPosInBlock: %u, curr: %u, called ldm_handleLdm()", currPosInBlock, curr); if (currPosInBlock >= *matchEndPosInBlock) { - //printf("Went over match boundary: currPosInBlock: %u, %matchEndPosInBlock: %u\n", currPosInBlock, *matchEndPosInBlock); + ////printf("Went over match boundary: currPosInBlock: %u, %matchEndPosInBlock: %u\n", currPosInBlock, *matchEndPosInBlock); if (currPosInBlock > *matchEndPosInBlock) { U32 posOvershoot = currPosInBlock - *matchEndPosInBlock; - //printf("Overshot position by: %u\n", posOvershoot); + ////printf("Overshot position by: %u\n", posOvershoot); ldm_skipSequences(ldmSeqStore, posOvershoot, MINMATCH); } int noMoreLdms = ldm_getNextMatch(ldmSeqStore, ldmSeqStoreBytesConsumed, matchStartPosInBlock, @@ -1022,10 +1019,10 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms, U32 ldmOffset = 0; U32 ldmSeqStoreBytesConsumed = 0; - ////printf("SBI for this block: %u, base: %u\n", sbi, base); + //////printf("SBI for this block: %u, base: %u\n", sbi, base); if (ms->ldmSeqStore.size > 0 && ms->ldmSeqStore.base != base) { int baseDiff = (int)(ms->ldmSeqStore.base - base); - //printf("Bases were different, adjusting, diff = %d\n", baseDiff); + ////printf("Bases were different, adjusting, diff = %d\n", baseDiff); ms->ldmSeqStore.seq[ms->ldmSeqStore.pos].litLength += baseDiff; ms->ldmSeqStore.base = ms->window.base; } @@ -1291,10 +1288,10 @@ _shortestPath: /* cur, last_pos, best_mlen, best_off have to be set */ } /* while (ip < ilimit) */ int i = ms->ldmSeqStore.pos; - //printf("pos %d (of:%u ml:%u ll: %u)\n", i, ms->ldmSeqStore.seq[i].offset, ms->ldmSeqStore.seq[i].matchLength, ms->ldmSeqStore.seq[i].litLength); - //printf("matchend: %u bytesconsumed: %u\n", ldmEndPosInBlock, ldmSeqStoreBytesConsumed); + ////printf("pos %d (of:%u ml:%u ll: %u)\n", i, ms->ldmSeqStore.seq[i].offset, ms->ldmSeqStore.seq[i].matchLength, ms->ldmSeqStore.seq[i].litLength); + ////printf("matchend: %u bytesconsumed: %u\n", ldmEndPosInBlock, ldmSeqStoreBytesConsumed); if (ldmEndPosInBlock < srcSize) { - //printf("Needs adjustment, endpos didn't reach end of block\n"); + ////printf("Needs adjustment, endpos didn't reach end of block\n"); ldm_skipSequences(&ms->ldmSeqStore, srcSize - ldmEndPosInBlock, MINMATCH); } /* Return the last literals size */ From ea92fb3a684c412eb81fab1e11fa11ece751f409 Mon Sep 17 00:00:00 2001 From: senhuang42 Date: Tue, 29 Sep 2020 16:35:06 -0400 Subject: [PATCH 30/68] Cleanups, add comments and explanations --- lib/compress/zstd_compress.c | 2 +- lib/compress/zstd_compress_internal.h | 2 +- lib/compress/zstd_ldm.c | 17 +- lib/compress/zstd_opt.c | 251 +++++++++++--------------- lib/compress/zstdmt_compress.c | 2 +- 5 files changed, 113 insertions(+), 161 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 3c80ab860..4958239cf 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -2336,7 +2336,7 @@ static size_t ZSTD_buildSeqStore(ZSTD_CCtx* zc, const void* src, size_t srcSize) if (curr > ms->nextToUpdate + 384) ms->nextToUpdate = curr - MIN(192, (U32)(curr - ms->nextToUpdate - 384)); } - //printf("--NEW BLOCK--\n"); + /* select and store sequences */ { ZSTD_dictMode_e const dictMode = ZSTD_matchState_dictMode(ms); size_t lastLLSize; diff --git a/lib/compress/zstd_compress_internal.h b/lib/compress/zstd_compress_internal.h index 53c20b12c..9eba078b4 100644 --- a/lib/compress/zstd_compress_internal.h +++ b/lib/compress/zstd_compress_internal.h @@ -94,7 +94,7 @@ typedef struct { typedef struct { rawSeq* seq; /* The start of the sequences */ - BYTE const* base; + BYTE const* base; /* The match state window base when LDMs were generated */ size_t pos; /* The position where reading stopped. <= size. */ size_t size; /* The number of sequences. <= capacity. */ size_t capacity; /* The capacity starting from `seq` pointer */ diff --git a/lib/compress/zstd_ldm.c b/lib/compress/zstd_ldm.c index 9fcb9fc29..743344036 100644 --- a/lib/compress/zstd_ldm.c +++ b/lib/compress/zstd_ldm.c @@ -562,13 +562,6 @@ static rawSeq maybeSplitSequence(rawSeqStore_t* rawSeqStore, return sequence; } -static void printSeqStore(rawSeqStore_t* rawSeqStore) { - printf("rawSeqStore: pos: %zu\n", rawSeqStore->pos); - for (int i = 0; i < rawSeqStore->size; ++i) { - printf("pos %d (of:%u ml:%u ll: %u)\n", i, rawSeqStore->seq[i].offset, rawSeqStore->seq[i].matchLength, rawSeqStore->seq[i].litLength); - } -} - size_t ZSTD_ldm_blockCompress(rawSeqStore_t* rawSeqStore, ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], void const* src, size_t srcSize) @@ -582,19 +575,13 @@ size_t ZSTD_ldm_blockCompress(rawSeqStore_t* rawSeqStore, BYTE const* const iend = istart + srcSize; /* Input positions */ BYTE const* ip = istart; - //printSeqStore(rawSeqStore); if (cParams->strategy >= ZSTD_btopt) { size_t lastLLSize; - //printSeqStore(rawSeqStore); - ms->ldmSeqStore = *rawSeqStore; /* copy current seqStore */ + ms->ldmSeqStore = *rawSeqStore; ms->ldmSeqStore.base = ms->window.base; lastLLSize = blockCompressor(ms, seqStore, rep, src, srcSize); + /* ldm seqstore will have changed during blockCompressor() call, make sure we copy those changes */ *rawSeqStore = ms->ldmSeqStore; - /*if (prevBase != ms->window.base) { - int baseDiff = (int)(prevBase - ms->window.base); - printf("Bases were different, adjusting, diff = %d\n", baseDiff); - rawSeqStore->seq[rawSeqStore->pos].litLength += baseDiff; - }*/ return lastLLSize; } diff --git a/lib/compress/zstd_opt.c b/lib/compress/zstd_opt.c index 6cea5a38b..075826b73 100644 --- a/lib/compress/zstd_opt.c +++ b/lib/compress/zstd_opt.c @@ -765,90 +765,69 @@ FORCE_INLINE_TEMPLATE U32 ZSTD_BtGetAllMatches ( } /*-******************************* -* LDM util functions +* LDM helper functions *********************************/ -static void ldm_skipSequences(rawSeqStore_t* rawSeqStore, size_t srcSize, U32 const minMatch) { - while (srcSize > 0 && rawSeqStore->pos < rawSeqStore->size) { - //////printf("ldm_skipSequences(): %u remaining\n", srcSize); - rawSeq* seq = rawSeqStore->seq + rawSeqStore->pos; - //////printf("ldm_skipSequences(): before: (of: %u, ml: %u, ll: %u)\n", seq->offset, seq->matchLength, seq->litLength); - if (srcSize <= seq->litLength) { +/* Skips past srcSize bytes in an ldm seqstore */ +static void ldm_skipBytesInSeqStore(rawSeqStore_t* ldmSeqStore, size_t bytesToSkip) { + while (bytesToSkip > 0 && ldmSeqStore->pos < ldmSeqStore->size) { + rawSeq* seq = ldmSeqStore->seq + ldmSeqStore->pos; + if (bytesToSkip <= seq->litLength) { /* Skip past srcSize literals */ - seq->litLength -= (U32)srcSize; - //////printf("ldm_skipSequences(): seqstore final: (of: %u, ml: %u, ll: %u) at %u\n", seq->offset, seq->matchLength, seq->litLength, rawSeqStore->pos); + seq->litLength -= (U32)bytesToSkip; return; } - srcSize -= seq->litLength; + bytesToSkip -= seq->litLength; seq->litLength = 0; - if (srcSize < seq->matchLength) { - //////printf("Splitting match: ml curr: %u ", seq->matchLength); - /* Skip past the first srcSize of the match */ - seq->matchLength -= (U32)srcSize; - //////printf("ml in store left: %u ", seq->matchLength); - /*if (seq->matchLength < minMatch) { - // The match is too short, omit it - if (rawSeqStore->pos + 1 < rawSeqStore->size) { - seq[1].litLength += seq[0].matchLength; - } - rawSeqStore->pos++; - }*/ - ////printf("ldm_skipSequences(): seqstore final: (of: %u, ml: %u, ll: %u) at %u\n", seq->offset, seq->matchLength, seq->litLength, rawSeqStore->pos); + if (bytesToSkip < seq->matchLength) { + seq->matchLength -= (U32)bytesToSkip; return; } - srcSize -= seq->matchLength; + bytesToSkip -= seq->matchLength; seq->matchLength = 0; - ////printf("ldm_skipSequences(): seqstore final: (of: %u, ml: %u, ll: %u) at %u\n", seq->offset, seq->matchLength, seq->litLength, rawSeqStore->pos); - rawSeqStore->pos++; + ldmSeqStore->pos++; } } -// The only function that can update pos (i think, for now) -static rawSeq ldm_splitSequence(rawSeqStore_t* ldmSeqStore, U32* ldmSeqStoreBytesConsumed, U32 remainingBytes) { +/* Splits a sequence if it's across the boundary. May update pos in the seq store too + * Pretty much the same function as maybeSplitSequence() in zstd_ldm.c + */ +static rawSeq ldm_splitSequenceAndUpdateSeqStore(rawSeqStore_t* ldmSeqStore, U32 remainingBytes) { rawSeq currSeq = ldmSeqStore->seq[ldmSeqStore->pos]; - ////printf("ldm_splitSequence(): Current sequence: (of: %u, ml: %u, ll: %u) - remaining: %u\n", currSeq.offset, currSeq.matchLength, currSeq.litLength, remainingBytes); - /* No split */ + /* Case where don't split the match*/ if (remainingBytes >= currSeq.litLength + currSeq.matchLength) { - //////printf("NO SPLIT\n"); - *ldmSeqStoreBytesConsumed += currSeq.litLength + currSeq.matchLength; ldmSeqStore->pos++; - //////printf("pos is now: %u\n", ldmSeqStore->pos); return currSeq; } /* Need a split */ if (remainingBytes <= currSeq.litLength) { - //////printf("SPLITTING: all remaining bytes were literals"); currSeq.offset = 0; } else if (remainingBytes < currSeq.litLength + currSeq.matchLength) { - /*if (currSeq.matchLength < MINMATCH) { - //////printf("CurrSeq less than minmatch: all remaining bytes were literals"); - currSeq.offset = 0; - }*/ currSeq.matchLength = remainingBytes - currSeq.litLength; } - //////printf("\n"); - *ldmSeqStoreBytesConsumed += remainingBytes; - ldm_skipSequences(ldmSeqStore, remainingBytes, MINMATCH); - ////printf("ldm_splitSequence(): Sequence final: (of: %u, ml: %u, ll: %u)\n", currSeq.offset, currSeq.matchLength, currSeq.litLength); + + /* After deriving currSeq which is the sequence before the block boundary, + * we now must skip past the remaining number of bytes unaccounted for, + * and update the entry at pos in the seqStore, which represents the second half + * of the sequence after the block boundary + */ + ldm_skipBytesInSeqStore(ldmSeqStore, remainingBytes); return currSeq; } -/* Returns 1 if the rest of the block is just LDM literals */ -static int ldm_getNextMatch(rawSeqStore_t* ldmSeqStore, U32* ldmSeqStoreBytesConsumed, +/* Fetch the next match in the ldm seq store */ +static void ldm_getNextMatch(rawSeqStore_t* ldmSeqStore, U32* matchStartPosInBlock, U32* matchEndPosInBlock, U32* matchOffset, U32 currPosInBlock, - U32 remainingBytes, U32 sbi) { + U32 remainingBytes) { + /* Setting match end position to MAX will ensure we never use an LDM during this block */ if (ldmSeqStore->pos >= ldmSeqStore->size) { - // Don't use the LDM for the rest of the block (there is none) - ////printf("No ldm left in the block, pos max reached\n"); *matchStartPosInBlock = UINT32_MAX; *matchEndPosInBlock = UINT32_MAX; return 1; } - rawSeq seq = ldm_splitSequence(ldmSeqStore, ldmSeqStoreBytesConsumed, remainingBytes); + rawSeq seq = ldm_splitSequenceAndUpdateSeqStore(ldmSeqStore, remainingBytes); if (seq.offset == 0) { - // Don't use the LDM for the rest of the block (there is none) - //////printf("No ldm left in the block, offset = 0\n"); *matchStartPosInBlock = UINT32_MAX; *matchEndPosInBlock = UINT32_MAX; return 1; @@ -857,106 +836,79 @@ static int ldm_getNextMatch(rawSeqStore_t* ldmSeqStore, U32* ldmSeqStoreBytesCon *matchStartPosInBlock = currPosInBlock + seq.litLength; *matchEndPosInBlock = *matchStartPosInBlock + seq.matchLength; *matchOffset = seq.offset; - ////printf("New match range in block is: (%u, %u) with of: %u where currPosInBlock: %u at adjusted absolute range: %u, %u\n", *matchStartPosInBlock, *matchEndPosInBlock, *matchOffset, currPosInBlock, *matchStartPosInBlock+sbi, *matchEndPosInBlock+sbi); return 0; } -static void printMatches(ZSTD_match_t* matches, U32 nbMatches) { - for (int i = 0; i < nbMatches; ++i) { - //printf("(of: %u ml: %u) ", matches[i].off, matches[i].len); - } - //printf("\n"); -} -static void validateMatches(ZSTD_match_t* matches, U32 nbMatches) { - U32 prevLargestOffset = 0; - U32 prevLargestMatch = 0; - for (int i = 1; i < nbMatches; ++i) { - if (matches[i-1].len > matches[i].len) { - //printf("nope\n"); - exit(1); - } else if (matches[i-1].len == matches[i].len) { - if (matches[i-1].off < matches[i].off) { - //printf("nopeOff\n"); - exit(1); - } - } - } -} - /* Adds an LDM if it's long enough */ static void ldm_maybeAddLdm(ZSTD_match_t* matches, U32* nbMatches, U32 matchStartPosInBlock, U32 matchEndPosInBlock, - U32 matchOffset, U32 currPosInBlock, U32 curr, U32 sbi, rawSeqStore_t* rawSeqStore) { + U32 matchOffset, U32 currPosInBlock) { /* Check that current block position is not outside of the match */ if (currPosInBlock < matchStartPosInBlock || currPosInBlock >= matchEndPosInBlock) return; + U32 posDiff = currPosInBlock - matchStartPosInBlock; + /* TODO: Next step will enable adding LDMs in the middle of a match */ if (posDiff > 0) - return; // dont deal with extra ldms for now - assert(posDiff < matchEndPosInBlock - matchStartPosInBlock); + return; + + /* Note: ZSTD_match_t actually contains offCode and matchLength (before subtracting MINMATCH) */ U32 candidateMatchLength = matchEndPosInBlock - matchStartPosInBlock - posDiff; if (candidateMatchLength < ZSTD_LDM_MINMATCH_MIN) return; - U32 candidateOffCode = matchOffset + posDiff + ZSTD_REP_MOVE; - if ((*nbMatches == 0 || candidateMatchLength >= matches[*nbMatches-1].len) && *nbMatches < ZSTD_OPT_NUM) { - //printf("large enough: curr: %u currposinblock: %u (ofcode: %u, ml: %u) - range: (%u, %u) - (of: %u ml: %u ll: %u) @ pos: %u - \n", - // curr, currPosInBlock, candidateOffCode, candidateMatchLength, matchStartPosInBlock, matchEndPosInBlock, rawSeqStore->seq[rawSeqStore->pos].offset, rawSeqStore->seq[rawSeqStore->pos].matchLength, rawSeqStore->seq[rawSeqStore->pos].litLength, rawSeqStore->pos); - - if (*nbMatches == 0) { - //printf("HERE1\n"); + if (*nbMatches == 0) { + matches[*nbMatches].len = candidateMatchLength; + matches[*nbMatches].off = candidateOffCode; + (*nbMatches)++; + } else if ((candidateMatchLength >= matches[*nbMatches-1].len) && *nbMatches < ZSTD_OPT_NUM) { + /* Maintain order of matches, which is first - increasing in matchlength, + * and secondly - decreasing in offCode. Since matches in ldm seq store are likely + * to be the longest match found, we simply start at the end of the array and sift + * the ldm match down as necessary. + */ + if (candidateMatchLength == matches[*nbMatches-1].len) { + U32 candidateMatchIdx = *nbMatches; + matches[*nbMatches].len = candidateMatchLength; + matches[*nbMatches].off = candidateOffCode; + if (candidateOffCode != matches[*nbMatches-1].off) { + while (candidateMatchIdx > 0 && + matches[candidateMatchIdx].off > matches[candidateMatchIdx - 1].off && + matches[candidateMatchIdx].len == matches[candidateMatchIdx - 1].len) { + ZSTD_match_t tmp = matches[candidateMatchIdx - 1]; + matches[candidateMatchIdx - 1] = matches[candidateMatchIdx]; + matches[candidateMatchIdx] = tmp; + --candidateMatchIdx; + } + } + (*nbMatches)++; + } else { matches[*nbMatches].len = candidateMatchLength; matches[*nbMatches].off = candidateOffCode; (*nbMatches)++; - } else { - if (candidateMatchLength == matches[*nbMatches-1].len) { - //printf("HERE2\n"); - U32 candidateMatchIdx = *nbMatches; - matches[*nbMatches].len = candidateMatchLength; - matches[*nbMatches].off = candidateOffCode; - if (candidateOffCode != matches[*nbMatches-1].off) { - //printf("offsets not equal\n"); - while (candidateMatchIdx > 0 && - matches[candidateMatchIdx].off > matches[candidateMatchIdx - 1].off && - matches[candidateMatchIdx].len == matches[candidateMatchIdx - 1].len) { - //printf("sifting\n"); - ZSTD_match_t tmp = matches[candidateMatchIdx - 1]; - matches[candidateMatchIdx - 1] = matches[candidateMatchIdx]; - matches[candidateMatchIdx] = tmp; - --candidateMatchIdx; - } - } - (*nbMatches)++; - } else { - //printf("HERE3\n"); - matches[*nbMatches].len = candidateMatchLength; - matches[*nbMatches].off = candidateOffCode; - (*nbMatches)++; - } } - printMatches(matches, *nbMatches); - validateMatches(matches, *nbMatches); } } -/* Wrapper function to call ldm functions as needed */ -static void ldm_handleLdm(rawSeqStore_t* ldmSeqStore, U32* ldmSeqStoreBytesConsumed, ZSTD_match_t* matches, U32* nbMatches, +/* Wrapper function to update and call ldm functions as necessary */ +static void ldm_handleLdm(rawSeqStore_t* ldmSeqStore, ZSTD_match_t* matches, U32* nbMatches, U32* matchStartPosInBlock, U32* matchEndPosInBlock, U32* matchOffset, - U32 currPosInBlock, U32 remainingBytes, U32 curr, U32 sbi) { - //////printf("currPosInBlock: %u, curr: %u, called ldm_handleLdm()", currPosInBlock, curr); + U32 currPosInBlock, U32 remainingBytes) { if (currPosInBlock >= *matchEndPosInBlock) { - ////printf("Went over match boundary: currPosInBlock: %u, %matchEndPosInBlock: %u\n", currPosInBlock, *matchEndPosInBlock); if (currPosInBlock > *matchEndPosInBlock) { + /* The position at which ldm_handleLdm() is called is not necessarily + * at the end of a match from the ldm seq store, and will often be some bytes + * over the end of an ldm match. As such, we need to correct for these "overshoots" + */ U32 posOvershoot = currPosInBlock - *matchEndPosInBlock; - ////printf("Overshot position by: %u\n", posOvershoot); - ldm_skipSequences(ldmSeqStore, posOvershoot, MINMATCH); + ldm_skipBytesInSeqStore(ldmSeqStore, posOvershoot); } - int noMoreLdms = ldm_getNextMatch(ldmSeqStore, ldmSeqStoreBytesConsumed, matchStartPosInBlock, - matchEndPosInBlock, matchOffset, - currPosInBlock, remainingBytes, sbi); + ldm_getNextMatch(ldmSeqStore, matchStartPosInBlock, + matchEndPosInBlock, matchOffset, + currPosInBlock, remainingBytes); } - ldm_maybeAddLdm(matches, nbMatches, *matchStartPosInBlock, *matchEndPosInBlock, *matchOffset, currPosInBlock, curr, sbi, ldmSeqStore); + ldm_maybeAddLdm(matches, nbMatches, *matchStartPosInBlock, *matchEndPosInBlock, *matchOffset, currPosInBlock); } @@ -1008,7 +960,6 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms, U32 const sufficient_len = MIN(cParams->targetLength, ZSTD_OPT_NUM -1); U32 const minMatch = (cParams->minMatch == 3) ? 3 : 4; U32 nextToUpdate3 = ms->nextToUpdate; - U32 const sbi = (U32)(istart-base); ZSTD_optimal_t* const opt = optStatePtr->priceTable; ZSTD_match_t* const matches = optStatePtr->matchTable; @@ -1017,18 +968,34 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms, U32 ldmStartPosInBlock = 0; U32 ldmEndPosInBlock = 0; U32 ldmOffset = 0; - U32 ldmSeqStoreBytesConsumed = 0; - //////printf("SBI for this block: %u, base: %u\n", sbi, base); - if (ms->ldmSeqStore.size > 0 && ms->ldmSeqStore.base != base) { - int baseDiff = (int)(ms->ldmSeqStore.base - base); - ////printf("Bases were different, adjusting, diff = %d\n", baseDiff); - ms->ldmSeqStore.seq[ms->ldmSeqStore.pos].litLength += baseDiff; - ms->ldmSeqStore.base = ms->window.base; - } - if (ms->ldmSeqStore.size != 0) { - ldm_getNextMatch(&ms->ldmSeqStore, &ldmSeqStoreBytesConsumed, &ldmStartPosInBlock, - &ldmEndPosInBlock, &ldmOffset, (U32)(ip-istart), (U32)(iend-ip), sbi); + /* It seems that if the ldm seqstore was calculated using a different base, we actually have to + * apply a correction to line it up with the block itself. This applies to multi-threaded cases + * where the block it can be the case that the LDMs were generated starting at a different + * block. By adding baseDiff to the litLength of seq, we pad out that base difference. + * + * For example, I found that when compressing dickens from silesia, the first LDM according to the seqStore is: + * (of:951277 ml:1362 ll: 952876) + * + * And as such, we might expect to insert the match in the 7th block at: + * pos_in_block 35373 == 952877 - 131072 * 7 + * I.e., at position 35373 the 7th block. + * + * Surprisingly, the correct place to insert is in the 8th block at + * pos_in_block 35373 = 1083948 - 131072 * 8 + * + * since the base shifts back 131072 bytes (1 block) after the first block. The consequence is that + * we should insert 35373 bytes into the 8th block, rather than 35373 bytes into the 7th block. + */ + if (ms->ldmSeqStore.size > 0) { + if (ms->ldmSeqStore.base != base) { + int baseDiff = (int)(ms->ldmSeqStore.base - base); + ms->ldmSeqStore.seq[ms->ldmSeqStore.pos].litLength += baseDiff; + ms->ldmSeqStore.base = ms->window.base; + } + ldm_getNextMatch(&ms->ldmSeqStore, &ldmStartPosInBlock, + &ldmEndPosInBlock, &ldmOffset, + (U32)(ip-istart), (U32)(iend-ip)); } /* init */ DEBUGLOG(5, "ZSTD_compressBlock_opt_generic: current=%u, prefix=%u, nextToUpdate=%u", @@ -1046,10 +1013,10 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms, U32 const ll0 = !litlen; U32 nbMatches = ZSTD_BtGetAllMatches(matches, ms, &nextToUpdate3, ip, iend, dictMode, rep, ll0, minMatch); if (ms->ldmSeqStore.size != 0) { - ldm_handleLdm(&ms->ldmSeqStore, &ldmSeqStoreBytesConsumed, matches, + ldm_handleLdm(&ms->ldmSeqStore, matches, &nbMatches, &ldmStartPosInBlock, &ldmEndPosInBlock, &ldmOffset, - (U32)(ip-istart), (U32)(iend - ip), (U32)(ip-base), sbi); + (U32)(ip-istart), (U32)(iend - ip)); } if (!nbMatches) { ip++; continue; } @@ -1168,10 +1135,10 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms, if (ms->ldmSeqStore.size != 0) { - ldm_handleLdm(&ms->ldmSeqStore, &ldmSeqStoreBytesConsumed, matches, + ldm_handleLdm(&ms->ldmSeqStore, matches, &nbMatches, &ldmStartPosInBlock, &ldmEndPosInBlock, &ldmOffset, - (U32)(inr-istart), (U32)(iend-inr), (U32)(inr-base), sbi); + (U32)(inr-istart), (U32)(iend-inr)); } if (!nbMatches) { DEBUGLOG(7, "rPos:%u : no match found", cur); @@ -1286,13 +1253,11 @@ _shortestPath: /* cur, last_pos, best_mlen, best_off have to be set */ ZSTD_setBasePrices(optStatePtr, optLevel); } } /* while (ip < ilimit) */ - - int i = ms->ldmSeqStore.pos; - ////printf("pos %d (of:%u ml:%u ll: %u)\n", i, ms->ldmSeqStore.seq[i].offset, ms->ldmSeqStore.seq[i].matchLength, ms->ldmSeqStore.seq[i].litLength); - ////printf("matchend: %u bytesconsumed: %u\n", ldmEndPosInBlock, ldmSeqStoreBytesConsumed); + if (ldmEndPosInBlock < srcSize) { - ////printf("Needs adjustment, endpos didn't reach end of block\n"); - ldm_skipSequences(&ms->ldmSeqStore, srcSize - ldmEndPosInBlock, MINMATCH); + /* This can occur if after adding the final match in an ldm seq store within this block, + ip goes to the end of the block without activating a check for ldm_getNextMatch */ + ldm_skipBytesInSeqStore(&ms->ldmSeqStore, srcSize - ldmEndPosInBlock); } /* Return the last literals size */ return (size_t)(iend - anchor); diff --git a/lib/compress/zstdmt_compress.c b/lib/compress/zstdmt_compress.c index 57015f156..6b3e8bee5 100644 --- a/lib/compress/zstdmt_compress.c +++ b/lib/compress/zstdmt_compress.c @@ -266,7 +266,7 @@ static void ZSTDMT_releaseBuffer(ZSTDMT_bufferPool* bufPool, buffer_t buf) /* ===== Seq Pool Wrapper ====== */ -static rawSeqStore_t kNullRawSeqStore = {NULL, 0, 0, 0}; +static rawSeqStore_t kNullRawSeqStore = {NULL, NULL, 0, 0, 0}; typedef ZSTDMT_bufferPool ZSTDMT_seqPool; From 724b94ed182ec55401f365f3912715fc3ba97352 Mon Sep 17 00:00:00 2001 From: senhuang42 Date: Tue, 29 Sep 2020 18:12:32 -0400 Subject: [PATCH 31/68] ldm_getNextMatch fixed return values --- lib/compress/zstd_opt.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/compress/zstd_opt.c b/lib/compress/zstd_opt.c index 075826b73..25168d6fe 100644 --- a/lib/compress/zstd_opt.c +++ b/lib/compress/zstd_opt.c @@ -824,19 +824,19 @@ static void ldm_getNextMatch(rawSeqStore_t* ldmSeqStore, if (ldmSeqStore->pos >= ldmSeqStore->size) { *matchStartPosInBlock = UINT32_MAX; *matchEndPosInBlock = UINT32_MAX; - return 1; + return; } rawSeq seq = ldm_splitSequenceAndUpdateSeqStore(ldmSeqStore, remainingBytes); if (seq.offset == 0) { *matchStartPosInBlock = UINT32_MAX; *matchEndPosInBlock = UINT32_MAX; - return 1; + return; } *matchStartPosInBlock = currPosInBlock + seq.litLength; *matchEndPosInBlock = *matchStartPosInBlock + seq.matchLength; *matchOffset = seq.offset; - return 0; + return; } /* Adds an LDM if it's long enough */ From bff5785fd58fd2197da4a8252fdae2a5a0379f33 Mon Sep 17 00:00:00 2001 From: senhuang42 Date: Wed, 30 Sep 2020 11:06:44 -0400 Subject: [PATCH 32/68] Address mixed variables C90 warning --- lib/compress/zstd_opt.c | 29 ++++++++++++++--------------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/lib/compress/zstd_opt.c b/lib/compress/zstd_opt.c index 25168d6fe..1c1700328 100644 --- a/lib/compress/zstd_opt.c +++ b/lib/compress/zstd_opt.c @@ -820,16 +820,18 @@ static void ldm_getNextMatch(rawSeqStore_t* ldmSeqStore, U32* matchStartPosInBlock, U32* matchEndPosInBlock, U32* matchOffset, U32 currPosInBlock, U32 remainingBytes) { + rawSeq seq; /* Setting match end position to MAX will ensure we never use an LDM during this block */ if (ldmSeqStore->pos >= ldmSeqStore->size) { - *matchStartPosInBlock = UINT32_MAX; - *matchEndPosInBlock = UINT32_MAX; + *matchStartPosInBlock = UINT_MAX; + *matchEndPosInBlock = UINT_MAX; return; } - rawSeq seq = ldm_splitSequenceAndUpdateSeqStore(ldmSeqStore, remainingBytes); + + seq = ldm_splitSequenceAndUpdateSeqStore(ldmSeqStore, remainingBytes); if (seq.offset == 0) { - *matchStartPosInBlock = UINT32_MAX; - *matchEndPosInBlock = UINT32_MAX; + *matchStartPosInBlock = UINT_MAX; + *matchEndPosInBlock = UINT_MAX; return; } @@ -843,21 +845,18 @@ static void ldm_getNextMatch(rawSeqStore_t* ldmSeqStore, static void ldm_maybeAddLdm(ZSTD_match_t* matches, U32* nbMatches, U32 matchStartPosInBlock, U32 matchEndPosInBlock, U32 matchOffset, U32 currPosInBlock) { - /* Check that current block position is not outside of the match */ - if (currPosInBlock < matchStartPosInBlock || currPosInBlock >= matchEndPosInBlock) - return; - U32 posDiff = currPosInBlock - matchStartPosInBlock; - /* TODO: Next step will enable adding LDMs in the middle of a match */ - if (posDiff > 0) - return; - /* Note: ZSTD_match_t actually contains offCode and matchLength (before subtracting MINMATCH) */ U32 candidateMatchLength = matchEndPosInBlock - matchStartPosInBlock - posDiff; - if (candidateMatchLength < ZSTD_LDM_MINMATCH_MIN) - return; U32 candidateOffCode = matchOffset + posDiff + ZSTD_REP_MOVE; + /* Ensure that current block position is not outside of the match */ + if (currPosInBlock < matchStartPosInBlock || + currPosInBlock >= matchEndPosInBlock || + posDiff > 0 || /* As a next evolution we can enable adding LDMs in the middle of a match */ + candidateMatchLength < ZSTD_LDM_MINMATCH_MIN) + return; + if (*nbMatches == 0) { matches[*nbMatches].len = candidateMatchLength; matches[*nbMatches].off = candidateOffCode; From 65f9cfeeec29724b5d48da088f7f325af1cfc01a Mon Sep 17 00:00:00 2001 From: senhuang42 Date: Wed, 30 Sep 2020 11:25:51 -0400 Subject: [PATCH 33/68] Add extra bounds check to prevent heap access after free ASAN error --- lib/compress/zstd_opt.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/compress/zstd_opt.c b/lib/compress/zstd_opt.c index 1c1700328..d6c728462 100644 --- a/lib/compress/zstd_opt.c +++ b/lib/compress/zstd_opt.c @@ -986,11 +986,11 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms, * since the base shifts back 131072 bytes (1 block) after the first block. The consequence is that * we should insert 35373 bytes into the 8th block, rather than 35373 bytes into the 7th block. */ - if (ms->ldmSeqStore.size > 0) { + if (ms->ldmSeqStore.size > 0 && ms->ldmSeqStore.pos != ms->ldmSeqStore.size) { if (ms->ldmSeqStore.base != base) { int baseDiff = (int)(ms->ldmSeqStore.base - base); ms->ldmSeqStore.seq[ms->ldmSeqStore.pos].litLength += baseDiff; - ms->ldmSeqStore.base = ms->window.base; + ms->ldmSeqStore.base = base; } ldm_getNextMatch(&ms->ldmSeqStore, &ldmStartPosInBlock, &ldmEndPosInBlock, &ldmOffset, From 4793ae3b8429dbd1d59410a62df0dff0990f6861 Mon Sep 17 00:00:00 2001 From: senhuang42 Date: Wed, 30 Sep 2020 12:54:45 -0400 Subject: [PATCH 34/68] Prevent duplicate LDMs from being inserted --- lib/compress/zstd_opt.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/compress/zstd_opt.c b/lib/compress/zstd_opt.c index d6c728462..088282ca2 100644 --- a/lib/compress/zstd_opt.c +++ b/lib/compress/zstd_opt.c @@ -868,6 +868,10 @@ static void ldm_maybeAddLdm(ZSTD_match_t* matches, U32* nbMatches, * the ldm match down as necessary. */ if (candidateMatchLength == matches[*nbMatches-1].len) { + if (candidateOffCode == matches[*nbMatches-1].off) { + /* No need to insert the match if it's the exact same */ + return; + } U32 candidateMatchIdx = *nbMatches; matches[*nbMatches].len = candidateMatchLength; matches[*nbMatches].off = candidateOffCode; From cfd2aec1b71cb8d34ab3926a2150be8147d869b9 Mon Sep 17 00:00:00 2001 From: senhuang42 Date: Wed, 30 Sep 2020 13:25:02 -0400 Subject: [PATCH 35/68] Add unit tests into playTests.sh --- tests/playTests.sh | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/tests/playTests.sh b/tests/playTests.sh index 7f75d8506..ee3ef42f0 100755 --- a/tests/playTests.sh +++ b/tests/playTests.sh @@ -1207,6 +1207,7 @@ roundTripTest -g1000K "1 --single-thread --long" roundTripTest -g517K "6 --single-thread --long" roundTripTest -g516K "16 --single-thread --long" roundTripTest -g518K "19 --single-thread --long" +roundTripTest -g2M "22 --single-thread --ultra --long" fileRoundTripTest -g5M "3 --single-thread --long" @@ -1216,6 +1217,7 @@ then println "\n===> zstdmt round-trip tests " roundTripTest -g4M "1 -T0" roundTripTest -g8M "3 -T2" + roundTripTest -g8M "3 -T0 --long" roundTripTest -g8000K "2 --threads=2" fileRoundTripTest -g4M "19 -T2 -B1M" @@ -1334,6 +1336,33 @@ roundTripTest -g1M -P50 "1 --single-thread --long=29" " --long=28 --memory=512MB roundTripTest -g1M -P50 "1 --single-thread --long=29" " --zstd=wlog=28 --memory=512MB" +println "\n===> zstd long distance matching with optimal parser compressed size tests " +optCSize16=$(datagen -g511K | zstd -16 -c | wc -c) +longCSize16=$(datagen -g511K | zstd -16 --long -c | wc -c) +optCSize19=$(datagen -g2M | zstd -19 -c | wc -c) +longCSize19=$(datagen -g2M | zstd -19 --long -c | wc -c) +optCSize19wlog23=$(datagen -g2M | zstd -19 -c --zstd=wlog=23 | wc -c) +longCSize19wlog23=$(datagen -g2M | zstd -19 -c --long=23 | wc -c) +optCSize19wlog27=$(datagen -g5M | zstd -19 -c --zstd=wlog=27 | wc -c) +longCSize19wlog27=$(datagen -g5M | zstd -19 -c --long=27 | wc -c) +optCSize22=$(datagen -g900K | zstd -22 --ultra -c | wc -c) +longCSize22=$(datagen -g900K | zstd -22 --ultra --long -c | wc -c) +if [ "$longCSize16" -gt "$optCSize16" ]; then + echo using --long on compression level 16 should not cause compressed size regression + exit 1 +elif [ "$longCSize19" -gt "$optCSize19" ]; then + echo using --long on compression level 19 should not cause compressed size regression + exit 1 +elif [ "$longCSize19wlog23" -gt "$optCSize19wlog23" ]; then + echo using --long on compression level 19 with wLog=23 should not cause compressed size regression + exit 1 +elif [ "$longCSize19wlog27" -gt "$optCSize19wlog27" ]; then + echo using --long on compression level 19 with wLog=27 should not cause compressed size regression + exit 1 +elif [ "$longCSize22" -gt "$optCSize22" ]; then + echo using --long on compression level 22 should not cause compressed size regression + exit 1 +fi if [ "$1" != "--test-large-data" ]; then From ef823e02996b25fcb4166b017d174ff788d95763 Mon Sep 17 00:00:00 2001 From: senhuang42 Date: Wed, 30 Sep 2020 17:20:47 -0400 Subject: [PATCH 36/68] Remove rawSeqStore.base and add rawSeqStore.posInSequence --- lib/compress/zstd_compress.c | 2 +- lib/compress/zstd_compress_internal.h | 2 +- lib/compress/zstdmt_compress.c | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 4958239cf..3a5ad0255 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -2354,7 +2354,7 @@ static size_t ZSTD_buildSeqStore(ZSTD_CCtx* zc, const void* src, size_t srcSize) src, srcSize); assert(zc->externSeqStore.pos <= zc->externSeqStore.size); } else if (zc->appliedParams.ldmParams.enableLdm) { - rawSeqStore_t ldmSeqStore = {NULL, NULL, 0, 0, 0}; + rawSeqStore_t ldmSeqStore = {NULL, 0, 0, 0, 0}; ldmSeqStore.seq = zc->ldmSequences; ldmSeqStore.capacity = zc->maxNbLdmSequences; diff --git a/lib/compress/zstd_compress_internal.h b/lib/compress/zstd_compress_internal.h index 9eba078b4..da5720e95 100644 --- a/lib/compress/zstd_compress_internal.h +++ b/lib/compress/zstd_compress_internal.h @@ -94,8 +94,8 @@ typedef struct { typedef struct { rawSeq* seq; /* The start of the sequences */ - BYTE const* base; /* The match state window base when LDMs were generated */ size_t pos; /* The position where reading stopped. <= size. */ + size_t posInSequence; /* The position to start at within the sequence when starting a new block */ size_t size; /* The number of sequences. <= capacity. */ size_t capacity; /* The capacity starting from `seq` pointer */ } rawSeqStore_t; diff --git a/lib/compress/zstdmt_compress.c b/lib/compress/zstdmt_compress.c index 6b3e8bee5..f789ae5bf 100644 --- a/lib/compress/zstdmt_compress.c +++ b/lib/compress/zstdmt_compress.c @@ -266,7 +266,7 @@ static void ZSTDMT_releaseBuffer(ZSTDMT_bufferPool* bufPool, buffer_t buf) /* ===== Seq Pool Wrapper ====== */ -static rawSeqStore_t kNullRawSeqStore = {NULL, NULL, 0, 0, 0}; +static rawSeqStore_t kNullRawSeqStore = {NULL, 0, 0, 0, 0}; typedef ZSTDMT_bufferPool ZSTDMT_seqPool; @@ -277,7 +277,7 @@ static size_t ZSTDMT_sizeof_seqPool(ZSTDMT_seqPool* seqPool) static rawSeqStore_t bufferToSeq(buffer_t buffer) { - rawSeqStore_t seq = {NULL, NULL, 0, 0, 0}; + rawSeqStore_t seq = {NULL, 0, 0, 0, 0}; seq.seq = (rawSeq*)buffer.start; seq.capacity = buffer.capacity / sizeof(rawSeq); return seq; From a1ef2db5b25e6f8a0564fcf6fd961d60015fe878 Mon Sep 17 00:00:00 2001 From: senhuang42 Date: Wed, 30 Sep 2020 19:14:31 -0400 Subject: [PATCH 37/68] Add ldm_calculateMatchRange() function --- lib/compress/zstd_compress_internal.h | 5 +- lib/compress/zstd_opt.c | 92 ++++++++++++++++++++------- 2 files changed, 71 insertions(+), 26 deletions(-) diff --git a/lib/compress/zstd_compress_internal.h b/lib/compress/zstd_compress_internal.h index da5720e95..770adefc1 100644 --- a/lib/compress/zstd_compress_internal.h +++ b/lib/compress/zstd_compress_internal.h @@ -94,8 +94,9 @@ typedef struct { typedef struct { rawSeq* seq; /* The start of the sequences */ - size_t pos; /* The position where reading stopped. <= size. */ - size_t posInSequence; /* The position to start at within the sequence when starting a new block */ + size_t pos; /* The index in this seqStore where reading stopped. <= size. */ + size_t posInSequence; /* The position within the rawSeq at index 'pos' where reading + stopped. */ size_t size; /* The number of sequences. <= capacity. */ size_t capacity; /* The capacity starting from `seq` pointer */ } rawSeqStore_t; diff --git a/lib/compress/zstd_opt.c b/lib/compress/zstd_opt.c index 088282ca2..9e38996e3 100644 --- a/lib/compress/zstd_opt.c +++ b/lib/compress/zstd_opt.c @@ -768,6 +768,67 @@ FORCE_INLINE_TEMPLATE U32 ZSTD_BtGetAllMatches ( * LDM helper functions *********************************/ +/* Moves forward in rawSeqStore by nbBytes bytes, which will updating the fields + * 'pos' and 'posInSequence' accordingly. + */ +static void ldm_moveForwardBytesInSeqStore(rawSeqStore_t* ldmSeqStore, size_t nbBytes) { + while (nbBytes && ldmSeqStore->pos < ldmSeqStore->size) { + rawSeq currSeq; + currSeq = ldmSeqStore->seq[ldmSeqStore->pos]; + + if (nbBytes <= currSeq.litLength) { + ldmSeqStore->posInSequence += nbBytes; + return; + } else { + nbBytes -= currSeq.litLength; + } + + if (nbBytes < currSeq.matchLength) { + ldmSeqStore->posInSequence += nbBytes; + return; + } else { + nbBytes -= currSeq.matchLength; + ldmSeqStore->pos++; + ldmSeqStore->posInSequence = 0; + } + } +} + +static void ldm_calculateMatchRange(rawSeqStore_t* ldmSeqStore, + U32* matchStartPosInBlock, U32* matchEndPosInBlock, + U32* matchOffset, U32 currPosInBlock, + U32 remainingBytes, U32 currBlockEndPos) { + rawSeq currSeq = ldmSeqStore->seq[ldmSeqStore->pos]; + U32 blockPosInSequence = ldmSeqStore->posInSequence + currPosInBlock; + U32 literalsBytesLeft = blockPosInSequence < currSeq.litLength ? + currSeq.litLength - blockPosInSequence : 0; + /* In this case, the match is further in the block than currPosInBlock */ + if (literalsBytesLeft) { + if (literalsBytesLeft >= remainingBytes) { + /* If there are more literal bytes than bytes remaining in block, no ldm */ + *matchStartPosInBlock = UINT_MAX; + *matchEndPosInBlock = UINT_MAX; + ldmSeqStore->pos++; + ldmSeqStore->posInSequence = 0; + return; + } + } + + *matchStartPosInBlock = currPosInBlock + literalsBytesLeft; + *matchEndPosInBlock = *matchStartPosInBlock + currSeq.matchLength; + *matchOffset = currSeq.offset; + + /* Match ends after the block ends, we can't use the whole match */ + if (*matchEndPosInBlock > currBlockEndPos) { + *matchEndPosInBlock = currBlockEndPos; + ldmSeqStore->posInSequence += (currBlockEndPos - currPosInBlock); + } else { + /* We can use the entire match */ + ldmSeqStore->posInSequence = 0; + ldmSeqStore->pos++; + } +} + /* Skips past srcSize bytes in an ldm seqstore */ static void ldm_skipBytesInSeqStore(rawSeqStore_t* ldmSeqStore, size_t bytesToSkip) { while (bytesToSkip > 0 && ldmSeqStore->pos < ldmSeqStore->size) { @@ -828,13 +889,14 @@ static void ldm_getNextMatch(rawSeqStore_t* ldmSeqStore, return; } - seq = ldm_splitSequenceAndUpdateSeqStore(ldmSeqStore, remainingBytes); + /*seq = ldm_splitSequenceAndUpdateSeqStore(ldmSeqStore, remainingBytes); if (seq.offset == 0) { *matchStartPosInBlock = UINT_MAX; *matchEndPosInBlock = UINT_MAX; return; - } + }*/ + ldm_calculateMatchRange(ldmSeqStore, matchStartPosInBlock, matchEndPosInBlock, matchOffset, remainingBytes); *matchStartPosInBlock = currPosInBlock + seq.litLength; *matchEndPosInBlock = *matchStartPosInBlock + seq.matchLength; *matchOffset = seq.offset; @@ -905,7 +967,7 @@ static void ldm_handleLdm(rawSeqStore_t* ldmSeqStore, ZSTD_match_t* matches, U32 * over the end of an ldm match. As such, we need to correct for these "overshoots" */ U32 posOvershoot = currPosInBlock - *matchEndPosInBlock; - ldm_skipBytesInSeqStore(ldmSeqStore, posOvershoot); + ldm_moveForwardBytesInSeqStore(ldmSeqStore, posOvershoot); } ldm_getNextMatch(ldmSeqStore, matchStartPosInBlock, matchEndPosInBlock, matchOffset, @@ -972,25 +1034,7 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms, U32 ldmEndPosInBlock = 0; U32 ldmOffset = 0; - /* It seems that if the ldm seqstore was calculated using a different base, we actually have to - * apply a correction to line it up with the block itself. This applies to multi-threaded cases - * where the block it can be the case that the LDMs were generated starting at a different - * block. By adding baseDiff to the litLength of seq, we pad out that base difference. - * - * For example, I found that when compressing dickens from silesia, the first LDM according to the seqStore is: - * (of:951277 ml:1362 ll: 952876) - * - * And as such, we might expect to insert the match in the 7th block at: - * pos_in_block 35373 == 952877 - 131072 * 7 - * I.e., at position 35373 the 7th block. - * - * Surprisingly, the correct place to insert is in the 8th block at - * pos_in_block 35373 = 1083948 - 131072 * 8 - * - * since the base shifts back 131072 bytes (1 block) after the first block. The consequence is that - * we should insert 35373 bytes into the 8th block, rather than 35373 bytes into the 7th block. - */ - if (ms->ldmSeqStore.size > 0 && ms->ldmSeqStore.pos != ms->ldmSeqStore.size) { + /*if (ms->ldmSeqStore.size > 0 && ms->ldmSeqStore.pos != ms->ldmSeqStore.size) { if (ms->ldmSeqStore.base != base) { int baseDiff = (int)(ms->ldmSeqStore.base - base); ms->ldmSeqStore.seq[ms->ldmSeqStore.pos].litLength += baseDiff; @@ -999,7 +1043,7 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms, ldm_getNextMatch(&ms->ldmSeqStore, &ldmStartPosInBlock, &ldmEndPosInBlock, &ldmOffset, (U32)(ip-istart), (U32)(iend-ip)); - } + }*/ /* init */ DEBUGLOG(5, "ZSTD_compressBlock_opt_generic: current=%u, prefix=%u, nextToUpdate=%u", (U32)(ip - base), ms->window.dictLimit, ms->nextToUpdate); @@ -1260,7 +1304,7 @@ _shortestPath: /* cur, last_pos, best_mlen, best_off have to be set */ if (ldmEndPosInBlock < srcSize) { /* This can occur if after adding the final match in an ldm seq store within this block, ip goes to the end of the block without activating a check for ldm_getNextMatch */ - ldm_skipBytesInSeqStore(&ms->ldmSeqStore, srcSize - ldmEndPosInBlock); + ldm_moveForwardBytesInSeqStore(ms->ldmSeqStore, srcSize - ldmEndPosInBlock); } /* Return the last literals size */ return (size_t)(iend - anchor); From 7348b40a87c61307c026ae5291202f22327059e9 Mon Sep 17 00:00:00 2001 From: senhuang42 Date: Wed, 30 Sep 2020 19:45:40 -0400 Subject: [PATCH 38/68] Adjustments to ldm_calculateMatchRange() to calculate bounds correctly --- lib/compress/zstd_opt.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/lib/compress/zstd_opt.c b/lib/compress/zstd_opt.c index 9e38996e3..5f7466b9b 100644 --- a/lib/compress/zstd_opt.c +++ b/lib/compress/zstd_opt.c @@ -799,29 +799,29 @@ static void ldm_calculateMatchRange(rawSeqStore_t* ldmSeqStore, U32* matchOffset, U32 currPosInBlock, U32 remainingBytes, U32 currBlockEndPos) { rawSeq currSeq = ldmSeqStore->seq[ldmSeqStore->pos]; - U32 blockPosInSequence = ldmSeqStore->posInSequence + currPosInBlock; - U32 literalsBytesLeft = blockPosInSequence < currSeq.litLength ? - currSeq.litLength - blockPosInSequence : 0; - /* In this case, the match is further in the block than currPosInBlock */ + U32 literalsBytesLeft = (ldmSeqStore->posInSequence < currSeq.litLength) ? + currSeq.litLength - ldmSeqStore->posInSequence : + 0; + /* In this case, the match is further in the block than currPosInBlock, and we are + currently in the literals section of the LDM */ if (literalsBytesLeft) { if (literalsBytesLeft >= remainingBytes) { /* If there are more literal bytes than bytes remaining in block, no ldm */ *matchStartPosInBlock = UINT_MAX; *matchEndPosInBlock = UINT_MAX; - ldmSeqStore->pos++; - ldmSeqStore->posInSequence = 0; + ldm_moveForwardBytesInSeqStore(ldmSeqStore, remainingBytes); return; } } - *matchStartPosInBlock = currPosInBlock + literalsBytesLeft; + *matchStartPosInBlock = currPosInBlock + currSeq.litLength; *matchEndPosInBlock = *matchStartPosInBlock + currSeq.matchLength; *matchOffset = currSeq.offset; /* Match ends after the block ends, we can't use the whole match */ if (*matchEndPosInBlock > currBlockEndPos) { *matchEndPosInBlock = currBlockEndPos; - ldmSeqStore->posInSequence += (currBlockEndPos - currPosInBlock); + ldm_moveForwardBytesInSeqStore(ldmSeqStore, currBlockEndPos - currPosInBlock); } else { /* We can use the entire match */ ldmSeqStore->posInSequence = 0; From 0718aa70df196b5854ce4e40c4d5abf8e846ad19 Mon Sep 17 00:00:00 2001 From: senhuang42 Date: Thu, 1 Oct 2020 10:12:21 -0400 Subject: [PATCH 39/68] Refactor existing functions to use posInSequence --- lib/compress/zstd_opt.c | 153 +++++++++++++++++++++------------------- 1 file changed, 81 insertions(+), 72 deletions(-) diff --git a/lib/compress/zstd_opt.c b/lib/compress/zstd_opt.c index 5f7466b9b..bafff8261 100644 --- a/lib/compress/zstd_opt.c +++ b/lib/compress/zstd_opt.c @@ -768,67 +768,6 @@ FORCE_INLINE_TEMPLATE U32 ZSTD_BtGetAllMatches ( * LDM helper functions *********************************/ -/* Moves forward in rawSeqStore by nbBytes bytes, which will updating the fields - * 'pos' and 'posInSequence' accordingly. - */ -static void ldm_moveForwardBytesInSeqStore(rawSeqStore_t* ldmSeqStore, size_t nbBytes) { - while (nbBytes && ldmSeqStore->pos < ldmSeqStore->size) { - rawSeq currSeq; - currSeq = ldmSeqStore->seq[ldmSeqStore->pos]; - - if (nbBytes <= currSeq.litLength) { - ldmSeqStore->posInSequence += nbBytes; - return; - } else { - nbBytes -= currSeq.litLength; - } - - if (nbBytes < currSeq.matchLength) { - ldmSeqStore->posInSequence += nbBytes; - return; - } else { - nbBytes -= currSeq.matchLength; - ldmSeqStore->pos++; - ldmSeqStore->posInSequence = 0; - } - } -} - -static void ldm_calculateMatchRange(rawSeqStore_t* ldmSeqStore, - U32* matchStartPosInBlock, U32* matchEndPosInBlock, - U32* matchOffset, U32 currPosInBlock, - U32 remainingBytes, U32 currBlockEndPos) { - rawSeq currSeq = ldmSeqStore->seq[ldmSeqStore->pos]; - U32 literalsBytesLeft = (ldmSeqStore->posInSequence < currSeq.litLength) ? - currSeq.litLength - ldmSeqStore->posInSequence : - 0; - /* In this case, the match is further in the block than currPosInBlock, and we are - currently in the literals section of the LDM */ - if (literalsBytesLeft) { - if (literalsBytesLeft >= remainingBytes) { - /* If there are more literal bytes than bytes remaining in block, no ldm */ - *matchStartPosInBlock = UINT_MAX; - *matchEndPosInBlock = UINT_MAX; - ldm_moveForwardBytesInSeqStore(ldmSeqStore, remainingBytes); - return; - } - } - - *matchStartPosInBlock = currPosInBlock + currSeq.litLength; - *matchEndPosInBlock = *matchStartPosInBlock + currSeq.matchLength; - *matchOffset = currSeq.offset; - - /* Match ends after the block ends, we can't use the whole match */ - if (*matchEndPosInBlock > currBlockEndPos) { - *matchEndPosInBlock = currBlockEndPos; - ldm_moveForwardBytesInSeqStore(ldmSeqStore, currBlockEndPos - currPosInBlock); - } else { - /* We can use the entire match */ - ldmSeqStore->posInSequence = 0; - ldmSeqStore->pos++; - } -} - /* Skips past srcSize bytes in an ldm seqstore */ static void ldm_skipBytesInSeqStore(rawSeqStore_t* ldmSeqStore, size_t bytesToSkip) { while (bytesToSkip > 0 && ldmSeqStore->pos < ldmSeqStore->size) { @@ -876,6 +815,79 @@ static rawSeq ldm_splitSequenceAndUpdateSeqStore(rawSeqStore_t* ldmSeqStore, U32 return currSeq; } +/* Moves forward in rawSeqStore by nbBytes bytes, which will updating the fields + * 'pos' and 'posInSequence' accordingly. + */ +static void ldm_moveForwardBytesInSeqStore(rawSeqStore_t* ldmSeqStore, size_t nbBytes) { + while (nbBytes && ldmSeqStore->pos < ldmSeqStore->size) { + rawSeq currSeq; + currSeq = ldmSeqStore->seq[ldmSeqStore->pos]; + + if (nbBytes <= currSeq.litLength) { + ldmSeqStore->posInSequence += nbBytes; + return; + } else { + ldmSeqStore->posInSequence += currSeq.litLength; + nbBytes -= currSeq.litLength; + } + + if (nbBytes < currSeq.matchLength) { + ldmSeqStore->posInSequence += nbBytes; + return; + } else { + nbBytes -= currSeq.matchLength; + /* We have moved through this entire sequence - move the read pos + forward to the next sequence, and reset posInSequence */ + ldmSeqStore->pos++; + ldmSeqStore->posInSequence = 0; + } + } +} + +/* Calculates the beginning and end of a match, and updates ldmSeqStore as + * necessary. + * posInSequence can be either within the literals section, or within a match. + * If + */ +static void ldm_calculateMatchRange(rawSeqStore_t* ldmSeqStore, + U32* matchStartPosInBlock, U32* matchEndPosInBlock, + U32* matchOffset, U32 currPosInBlock, + U32 blockBytesRemaining) { + rawSeq currSeq = ldmSeqStore->seq[ldmSeqStore->pos]; + U32 currBlockEndPos = currPosInBlock + blockBytesRemaining; + U32 literalsBytesRemaining = (ldmSeqStore->posInSequence < currSeq.litLength) ? + currSeq.litLength - ldmSeqStore->posInSequence : + 0; + + /* In this case, the match is further in the block than currPosInBlock, and we are + currently in the literals section of the LDM */ + if (literalsBytesRemaining) { + if (literalsBytesRemaining >= blockBytesRemaining) { + /* If there are more literal bytes than bytes remaining in block, no ldm */ + *matchStartPosInBlock = UINT_MAX; + *matchEndPosInBlock = UINT_MAX; + ldm_moveForwardBytesInSeqStore(ldmSeqStore, blockBytesRemaining); + return; + } + } + + /* Matches may be < MINMATCH by this process. In that case, we will reject them + when we are deciding whether or not to add the ldm */ + *matchStartPosInBlock = currPosInBlock + literalsBytesRemaining; + *matchEndPosInBlock = *matchStartPosInBlock + currSeq.matchLength; + *matchOffset = currSeq.offset; + + if (*matchEndPosInBlock > currBlockEndPos) { + /* Match ends after the block ends, we can't use the whole match */ + *matchEndPosInBlock = currBlockEndPos; + ldm_moveForwardBytesInSeqStore(ldmSeqStore, currBlockEndPos - currPosInBlock); + } else { + /* We can use the entire match */ + ldmSeqStore->posInSequence = 0; + ldmSeqStore->pos++; + } +} + /* Fetch the next match in the ldm seq store */ static void ldm_getNextMatch(rawSeqStore_t* ldmSeqStore, U32* matchStartPosInBlock, U32* matchEndPosInBlock, @@ -896,10 +908,7 @@ static void ldm_getNextMatch(rawSeqStore_t* ldmSeqStore, return; }*/ - ldm_calculateMatchRange(ldmSeqStore, matchStartPosInBlock, matchEndPosInBlock, matchOffset, remainingBytes); - *matchStartPosInBlock = currPosInBlock + seq.litLength; - *matchEndPosInBlock = *matchStartPosInBlock + seq.matchLength; - *matchOffset = seq.offset; + ldm_calculateMatchRange(ldmSeqStore, matchStartPosInBlock, matchEndPosInBlock, matchOffset, currPosInBlock, remainingBytes); return; } @@ -970,8 +979,8 @@ static void ldm_handleLdm(rawSeqStore_t* ldmSeqStore, ZSTD_match_t* matches, U32 ldm_moveForwardBytesInSeqStore(ldmSeqStore, posOvershoot); } ldm_getNextMatch(ldmSeqStore, matchStartPosInBlock, - matchEndPosInBlock, matchOffset, - currPosInBlock, remainingBytes); + matchEndPosInBlock, matchOffset, + currPosInBlock, remainingBytes); } ldm_maybeAddLdm(matches, nbMatches, *matchStartPosInBlock, *matchEndPosInBlock, *matchOffset, currPosInBlock); } @@ -1034,16 +1043,16 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms, U32 ldmEndPosInBlock = 0; U32 ldmOffset = 0; - /*if (ms->ldmSeqStore.size > 0 && ms->ldmSeqStore.pos != ms->ldmSeqStore.size) { - if (ms->ldmSeqStore.base != base) { + if (ms->ldmSeqStore.size > 0 && ms->ldmSeqStore.pos != ms->ldmSeqStore.size) { + /*if (ms->ldmSeqStore.base != base) { int baseDiff = (int)(ms->ldmSeqStore.base - base); ms->ldmSeqStore.seq[ms->ldmSeqStore.pos].litLength += baseDiff; ms->ldmSeqStore.base = base; - } + }*/ ldm_getNextMatch(&ms->ldmSeqStore, &ldmStartPosInBlock, &ldmEndPosInBlock, &ldmOffset, (U32)(ip-istart), (U32)(iend-ip)); - }*/ + } /* init */ DEBUGLOG(5, "ZSTD_compressBlock_opt_generic: current=%u, prefix=%u, nextToUpdate=%u", (U32)(ip - base), ms->window.dictLimit, ms->nextToUpdate); @@ -1304,7 +1313,7 @@ _shortestPath: /* cur, last_pos, best_mlen, best_off have to be set */ if (ldmEndPosInBlock < srcSize) { /* This can occur if after adding the final match in an ldm seq store within this block, ip goes to the end of the block without activating a check for ldm_getNextMatch */ - ldm_moveForwardBytesInSeqStore(ms->ldmSeqStore, srcSize - ldmEndPosInBlock); + ldm_moveForwardBytesInSeqStore(&ms->ldmSeqStore, srcSize - ldmEndPosInBlock); } /* Return the last literals size */ return (size_t)(iend - anchor); From 7dee62c2876b568e215aa63660c2d173d1e5ba1f Mon Sep 17 00:00:00 2001 From: senhuang42 Date: Thu, 1 Oct 2020 10:13:06 -0400 Subject: [PATCH 40/68] Reset ldmSeqStore after initStats_ultra() pass for btultra2 --- lib/compress/zstd_ldm.c | 1 - lib/compress/zstd_opt.c | 2 ++ 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/compress/zstd_ldm.c b/lib/compress/zstd_ldm.c index 743344036..60a6e5878 100644 --- a/lib/compress/zstd_ldm.c +++ b/lib/compress/zstd_ldm.c @@ -578,7 +578,6 @@ size_t ZSTD_ldm_blockCompress(rawSeqStore_t* rawSeqStore, if (cParams->strategy >= ZSTD_btopt) { size_t lastLLSize; ms->ldmSeqStore = *rawSeqStore; - ms->ldmSeqStore.base = ms->window.base; lastLLSize = blockCompressor(ms, seqStore, rep, src, srcSize); /* ldm seqstore will have changed during blockCompressor() call, make sure we copy those changes */ *rawSeqStore = ms->ldmSeqStore; diff --git a/lib/compress/zstd_opt.c b/lib/compress/zstd_opt.c index bafff8261..bb87e0580 100644 --- a/lib/compress/zstd_opt.c +++ b/lib/compress/zstd_opt.c @@ -1364,6 +1364,7 @@ ZSTD_initStats_ultra(ZSTD_matchState_t* ms, const void* src, size_t srcSize) { U32 tmpRep[ZSTD_REP_NUM]; /* updated rep codes will sink here */ + rawSeqStore_t tmpSeqStore = ms->ldmSeqStore; ZSTD_memcpy(tmpRep, rep, sizeof(tmpRep)); DEBUGLOG(4, "ZSTD_initStats_ultra (srcSize=%zu)", srcSize); @@ -1380,6 +1381,7 @@ ZSTD_initStats_ultra(ZSTD_matchState_t* ms, ms->window.dictLimit += (U32)srcSize; ms->window.lowLimit = ms->window.dictLimit; ms->nextToUpdate = ms->window.dictLimit; + ms->ldmSeqStore = tmpSeqStore; /* re-inforce weight of collected statistics */ ZSTD_upscaleStats(&ms->opt); From 37617e23d7ec76a60072df3d34f2e3f52d408edc Mon Sep 17 00:00:00 2001 From: senhuang42 Date: Thu, 1 Oct 2020 11:35:48 -0400 Subject: [PATCH 41/68] Correct matchLength calculation and remove unnecessary functions --- lib/compress/zstd_compress.c | 2 +- lib/compress/zstd_opt.c | 165 +++++++++++------------------------ 2 files changed, 52 insertions(+), 115 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 3a5ad0255..f298cd895 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -2336,7 +2336,7 @@ static size_t ZSTD_buildSeqStore(ZSTD_CCtx* zc, const void* src, size_t srcSize) if (curr > ms->nextToUpdate + 384) ms->nextToUpdate = curr - MIN(192, (U32)(curr - ms->nextToUpdate - 384)); } - + /* select and store sequences */ { ZSTD_dictMode_e const dictMode = ZSTD_matchState_dictMode(ms); size_t lastLLSize; diff --git a/lib/compress/zstd_opt.c b/lib/compress/zstd_opt.c index bb87e0580..7dfd1af64 100644 --- a/lib/compress/zstd_opt.c +++ b/lib/compress/zstd_opt.c @@ -768,54 +768,7 @@ FORCE_INLINE_TEMPLATE U32 ZSTD_BtGetAllMatches ( * LDM helper functions *********************************/ -/* Skips past srcSize bytes in an ldm seqstore */ -static void ldm_skipBytesInSeqStore(rawSeqStore_t* ldmSeqStore, size_t bytesToSkip) { - while (bytesToSkip > 0 && ldmSeqStore->pos < ldmSeqStore->size) { - rawSeq* seq = ldmSeqStore->seq + ldmSeqStore->pos; - if (bytesToSkip <= seq->litLength) { - /* Skip past srcSize literals */ - seq->litLength -= (U32)bytesToSkip; - return; - } - bytesToSkip -= seq->litLength; - seq->litLength = 0; - if (bytesToSkip < seq->matchLength) { - seq->matchLength -= (U32)bytesToSkip; - return; - } - bytesToSkip -= seq->matchLength; - seq->matchLength = 0; - ldmSeqStore->pos++; - } -} - -/* Splits a sequence if it's across the boundary. May update pos in the seq store too - * Pretty much the same function as maybeSplitSequence() in zstd_ldm.c - */ -static rawSeq ldm_splitSequenceAndUpdateSeqStore(rawSeqStore_t* ldmSeqStore, U32 remainingBytes) { - rawSeq currSeq = ldmSeqStore->seq[ldmSeqStore->pos]; - /* Case where don't split the match*/ - if (remainingBytes >= currSeq.litLength + currSeq.matchLength) { - ldmSeqStore->pos++; - return currSeq; - } - /* Need a split */ - if (remainingBytes <= currSeq.litLength) { - currSeq.offset = 0; - } else if (remainingBytes < currSeq.litLength + currSeq.matchLength) { - currSeq.matchLength = remainingBytes - currSeq.litLength; - } - - /* After deriving currSeq which is the sequence before the block boundary, - * we now must skip past the remaining number of bytes unaccounted for, - * and update the entry at pos in the seqStore, which represents the second half - * of the sequence after the block boundary - */ - ldm_skipBytesInSeqStore(ldmSeqStore, remainingBytes); - return currSeq; -} - -/* Moves forward in rawSeqStore by nbBytes bytes, which will updating the fields +/* Moves forward in rawSeqStore by nbBytes, which will update the fields * 'pos' and 'posInSequence' accordingly. */ static void ldm_moveForwardBytesInSeqStore(rawSeqStore_t* ldmSeqStore, size_t nbBytes) { @@ -844,37 +797,48 @@ static void ldm_moveForwardBytesInSeqStore(rawSeqStore_t* ldmSeqStore, size_t nb } } -/* Calculates the beginning and end of a match, and updates ldmSeqStore as - * necessary. - * posInSequence can be either within the literals section, or within a match. - * If +/* Calculates the beginning and end of a match, and updates 'pos' and 'posInSequence' + * of the ldmSeqStore. */ -static void ldm_calculateMatchRange(rawSeqStore_t* ldmSeqStore, - U32* matchStartPosInBlock, U32* matchEndPosInBlock, - U32* matchOffset, U32 currPosInBlock, - U32 blockBytesRemaining) { - rawSeq currSeq = ldmSeqStore->seq[ldmSeqStore->pos]; - U32 currBlockEndPos = currPosInBlock + blockBytesRemaining; - U32 literalsBytesRemaining = (ldmSeqStore->posInSequence < currSeq.litLength) ? - currSeq.litLength - ldmSeqStore->posInSequence : - 0; +static void ldm_calculateNextMatch(rawSeqStore_t* ldmSeqStore, + U32* matchStartPosInBlock, U32* matchEndPosInBlock, + U32* matchOffset, U32 currPosInBlock, + U32 blockBytesRemaining) { + rawSeq currSeq; + U32 currBlockEndPos; + U32 literalsBytesRemaining; + U32 matchBytesRemaining; - /* In this case, the match is further in the block than currPosInBlock, and we are - currently in the literals section of the LDM */ - if (literalsBytesRemaining) { - if (literalsBytesRemaining >= blockBytesRemaining) { - /* If there are more literal bytes than bytes remaining in block, no ldm */ - *matchStartPosInBlock = UINT_MAX; - *matchEndPosInBlock = UINT_MAX; - ldm_moveForwardBytesInSeqStore(ldmSeqStore, blockBytesRemaining); - return; - } + /* Setting match end position to MAX to ensure we never use an LDM during this block */ + if (ldmSeqStore->pos >= ldmSeqStore->size) { + *matchStartPosInBlock = UINT_MAX; + *matchEndPosInBlock = UINT_MAX; + return; + } + + /* Calculate appropriate bytes left in matchLength and litLength after adjusting + based on ldmSeqStore->posInSequence */ + currSeq = ldmSeqStore->seq[ldmSeqStore->pos]; + currBlockEndPos = currPosInBlock + blockBytesRemaining; + literalsBytesRemaining = (ldmSeqStore->posInSequence < currSeq.litLength) ? + currSeq.litLength - ldmSeqStore->posInSequence : + 0; + matchBytesRemaining = (literalsBytesRemaining == 0) ? + currSeq.matchLength - (ldmSeqStore->posInSequence - currSeq.litLength) : + currSeq.matchLength; + + /* If there are more literal bytes than bytes remaining in block, no ldm */ + if (literalsBytesRemaining >= blockBytesRemaining) { + *matchStartPosInBlock = UINT_MAX; + *matchEndPosInBlock = UINT_MAX; + ldm_moveForwardBytesInSeqStore(ldmSeqStore, blockBytesRemaining); + return; } /* Matches may be < MINMATCH by this process. In that case, we will reject them when we are deciding whether or not to add the ldm */ *matchStartPosInBlock = currPosInBlock + literalsBytesRemaining; - *matchEndPosInBlock = *matchStartPosInBlock + currSeq.matchLength; + *matchEndPosInBlock = *matchStartPosInBlock + matchBytesRemaining; *matchOffset = currSeq.offset; if (*matchEndPosInBlock > currBlockEndPos) { @@ -882,36 +846,12 @@ static void ldm_calculateMatchRange(rawSeqStore_t* ldmSeqStore, *matchEndPosInBlock = currBlockEndPos; ldm_moveForwardBytesInSeqStore(ldmSeqStore, currBlockEndPos - currPosInBlock); } else { - /* We can use the entire match */ + /* If we can use the whole match point the ldmSeqStore at the next match */ ldmSeqStore->posInSequence = 0; ldmSeqStore->pos++; } } -/* Fetch the next match in the ldm seq store */ -static void ldm_getNextMatch(rawSeqStore_t* ldmSeqStore, - U32* matchStartPosInBlock, U32* matchEndPosInBlock, - U32* matchOffset, U32 currPosInBlock, - U32 remainingBytes) { - rawSeq seq; - /* Setting match end position to MAX will ensure we never use an LDM during this block */ - if (ldmSeqStore->pos >= ldmSeqStore->size) { - *matchStartPosInBlock = UINT_MAX; - *matchEndPosInBlock = UINT_MAX; - return; - } - - /*seq = ldm_splitSequenceAndUpdateSeqStore(ldmSeqStore, remainingBytes); - if (seq.offset == 0) { - *matchStartPosInBlock = UINT_MAX; - *matchEndPosInBlock = UINT_MAX; - return; - }*/ - - ldm_calculateMatchRange(ldmSeqStore, matchStartPosInBlock, matchEndPosInBlock, matchOffset, currPosInBlock, remainingBytes); - return; -} - /* Adds an LDM if it's long enough */ static void ldm_maybeAddLdm(ZSTD_match_t* matches, U32* nbMatches, U32 matchStartPosInBlock, U32 matchEndPosInBlock, @@ -933,8 +873,8 @@ static void ldm_maybeAddLdm(ZSTD_match_t* matches, U32* nbMatches, matches[*nbMatches].off = candidateOffCode; (*nbMatches)++; } else if ((candidateMatchLength >= matches[*nbMatches-1].len) && *nbMatches < ZSTD_OPT_NUM) { - /* Maintain order of matches, which is first - increasing in matchlength, - * and secondly - decreasing in offCode. Since matches in ldm seq store are likely + /* Maintain order of matches, which is firstly - increasing in matchlength, + * and secondly - decreasing in offCode. Since matches from the ldm seq store are likely * to be the longest match found, we simply start at the end of the array and sift * the ldm match down as necessary. */ @@ -973,12 +913,12 @@ static void ldm_handleLdm(rawSeqStore_t* ldmSeqStore, ZSTD_match_t* matches, U32 if (currPosInBlock > *matchEndPosInBlock) { /* The position at which ldm_handleLdm() is called is not necessarily * at the end of a match from the ldm seq store, and will often be some bytes - * over the end of an ldm match. As such, we need to correct for these "overshoots" + * over beyond matchEndPosInBlock. As such, we need to correct for these "overshoots" */ U32 posOvershoot = currPosInBlock - *matchEndPosInBlock; ldm_moveForwardBytesInSeqStore(ldmSeqStore, posOvershoot); } - ldm_getNextMatch(ldmSeqStore, matchStartPosInBlock, + ldm_calculateNextMatch(ldmSeqStore, matchStartPosInBlock, matchEndPosInBlock, matchOffset, currPosInBlock, remainingBytes); } @@ -1043,16 +983,13 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms, U32 ldmEndPosInBlock = 0; U32 ldmOffset = 0; - if (ms->ldmSeqStore.size > 0 && ms->ldmSeqStore.pos != ms->ldmSeqStore.size) { - /*if (ms->ldmSeqStore.base != base) { - int baseDiff = (int)(ms->ldmSeqStore.base - base); - ms->ldmSeqStore.seq[ms->ldmSeqStore.pos].litLength += baseDiff; - ms->ldmSeqStore.base = base; - }*/ - ldm_getNextMatch(&ms->ldmSeqStore, &ldmStartPosInBlock, + /* Get first match from ldm seq store if long mode is enabled */ + if (ms->ldmSeqStore.size > 0 && ms->ldmSeqStore.pos < ms->ldmSeqStore.size) { + ldm_calculateNextMatch(&ms->ldmSeqStore, &ldmStartPosInBlock, &ldmEndPosInBlock, &ldmOffset, (U32)(ip-istart), (U32)(iend-ip)); } + /* init */ DEBUGLOG(5, "ZSTD_compressBlock_opt_generic: current=%u, prefix=%u, nextToUpdate=%u", (U32)(ip - base), ms->window.dictLimit, ms->nextToUpdate); @@ -1068,7 +1005,7 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms, { U32 const litlen = (U32)(ip - anchor); U32 const ll0 = !litlen; U32 nbMatches = ZSTD_BtGetAllMatches(matches, ms, &nextToUpdate3, ip, iend, dictMode, rep, ll0, minMatch); - if (ms->ldmSeqStore.size != 0) { + if (ms->ldmSeqStore.size != 0 && ms->ldmSeqStore.pos < ms->ldmSeqStore.size) { ldm_handleLdm(&ms->ldmSeqStore, matches, &nbMatches, &ldmStartPosInBlock, &ldmEndPosInBlock, &ldmOffset, @@ -1190,11 +1127,11 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms, U32 matchNb; - if (ms->ldmSeqStore.size != 0) { + if (ms->ldmSeqStore.size != 0 && ms->ldmSeqStore.pos < ms->ldmSeqStore.size) { ldm_handleLdm(&ms->ldmSeqStore, matches, - &nbMatches, &ldmStartPosInBlock, - &ldmEndPosInBlock, &ldmOffset, - (U32)(inr-istart), (U32)(iend-inr)); + &nbMatches, &ldmStartPosInBlock, + &ldmEndPosInBlock, &ldmOffset, + (U32)(inr-istart), (U32)(iend-inr)); } if (!nbMatches) { DEBUGLOG(7, "rPos:%u : no match found", cur); @@ -1312,7 +1249,7 @@ _shortestPath: /* cur, last_pos, best_mlen, best_off have to be set */ if (ldmEndPosInBlock < srcSize) { /* This can occur if after adding the final match in an ldm seq store within this block, - ip goes to the end of the block without activating a check for ldm_getNextMatch */ + ip goes to the end of the block without activating a check for ldm_calculateNextMatch */ ldm_moveForwardBytesInSeqStore(&ms->ldmSeqStore, srcSize - ldmEndPosInBlock); } /* Return the last literals size */ From 1a687b3fcb853b9437d33c853800192131b08f95 Mon Sep 17 00:00:00 2001 From: senhuang42 Date: Thu, 1 Oct 2020 11:40:52 -0400 Subject: [PATCH 42/68] Improve documentation of relevant structs --- lib/compress/zstd_compress_internal.h | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/lib/compress/zstd_compress_internal.h b/lib/compress/zstd_compress_internal.h index 770adefc1..954af31f9 100644 --- a/lib/compress/zstd_compress_internal.h +++ b/lib/compress/zstd_compress_internal.h @@ -82,23 +82,23 @@ typedef struct { } ZSTD_entropyCTables_t; typedef struct { - U32 off; - U32 len; + U32 off; /* Offset code for the match */ + U32 len; /* Raw length of match */ } ZSTD_match_t; typedef struct { - U32 offset; - U32 litLength; - U32 matchLength; + U32 offset; /* Offset of sequence */ + U32 litLength; /* Length of literals prior to match */ + U32 matchLength; /* Raw length of match */ } rawSeq; typedef struct { - rawSeq* seq; /* The start of the sequences */ - size_t pos; /* The index in this seqStore where reading stopped. <= size. */ - size_t posInSequence; /* The position within the rawSeq at index 'pos' where reading - stopped. */ - size_t size; /* The number of sequences. <= capacity. */ - size_t capacity; /* The capacity starting from `seq` pointer */ + rawSeq* seq; /* The start of the sequences */ + size_t pos; /* The index in seq where reading stopped. pos <= size. */ + size_t posInSequence; /* The position within the sequence at seq[pos] where reading + stopped. posInSequence <= seq[pos].litLength + seq[pos].matchLength */ + size_t size; /* The number of sequences. <= capacity. */ + size_t capacity; /* The capacity starting from `seq` pointer */ } rawSeqStore_t; typedef struct { From 10647924f1c0e5daa35a43c8a1af848d09185b09 Mon Sep 17 00:00:00 2001 From: senhuang42 Date: Thu, 1 Oct 2020 11:47:35 -0400 Subject: [PATCH 43/68] Make function descriptions more accurate --- lib/compress/zstd_compress.c | 2 +- lib/compress/zstd_compress_internal.h | 2 +- lib/compress/zstd_ldm.c | 3 +- lib/compress/zstd_opt.c | 49 ++++++++++++++------------- 4 files changed, 29 insertions(+), 27 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index f298cd895..3a5ad0255 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -2336,7 +2336,7 @@ static size_t ZSTD_buildSeqStore(ZSTD_CCtx* zc, const void* src, size_t srcSize) if (curr > ms->nextToUpdate + 384) ms->nextToUpdate = curr - MIN(192, (U32)(curr - ms->nextToUpdate - 384)); } - + /* select and store sequences */ { ZSTD_dictMode_e const dictMode = ZSTD_matchState_dictMode(ms); size_t lastLLSize; diff --git a/lib/compress/zstd_compress_internal.h b/lib/compress/zstd_compress_internal.h index 954af31f9..179e4f7bb 100644 --- a/lib/compress/zstd_compress_internal.h +++ b/lib/compress/zstd_compress_internal.h @@ -82,7 +82,7 @@ typedef struct { } ZSTD_entropyCTables_t; typedef struct { - U32 off; /* Offset code for the match */ + U32 off; /* Offset code (offset + ZSTD_REP_MOVE) for the match */ U32 len; /* Raw length of match */ } ZSTD_match_t; diff --git a/lib/compress/zstd_ldm.c b/lib/compress/zstd_ldm.c index 60a6e5878..341877858 100644 --- a/lib/compress/zstd_ldm.c +++ b/lib/compress/zstd_ldm.c @@ -579,8 +579,7 @@ size_t ZSTD_ldm_blockCompress(rawSeqStore_t* rawSeqStore, size_t lastLLSize; ms->ldmSeqStore = *rawSeqStore; lastLLSize = blockCompressor(ms, seqStore, rep, src, srcSize); - /* ldm seqstore will have changed during blockCompressor() call, make sure we copy those changes */ - *rawSeqStore = ms->ldmSeqStore; + *rawSeqStore = ms->ldmSeqStore; /* Persist changes to ldmSeqStore during blockCompressor() */ return lastLLSize; } diff --git a/lib/compress/zstd_opt.c b/lib/compress/zstd_opt.c index 7dfd1af64..7efae9b87 100644 --- a/lib/compress/zstd_opt.c +++ b/lib/compress/zstd_opt.c @@ -768,14 +768,13 @@ FORCE_INLINE_TEMPLATE U32 ZSTD_BtGetAllMatches ( * LDM helper functions *********************************/ -/* Moves forward in rawSeqStore by nbBytes, which will update the fields - * 'pos' and 'posInSequence' accordingly. +/* ldm_moveForwardBytesInSeqStore(): + * Moves forward in rawSeqStore by nbBytes, which will update the fields 'pos' and 'posInSequence'. */ static void ldm_moveForwardBytesInSeqStore(rawSeqStore_t* ldmSeqStore, size_t nbBytes) { while (nbBytes && ldmSeqStore->pos < ldmSeqStore->size) { rawSeq currSeq; currSeq = ldmSeqStore->seq[ldmSeqStore->pos]; - if (nbBytes <= currSeq.litLength) { ldmSeqStore->posInSequence += nbBytes; return; @@ -783,7 +782,6 @@ static void ldm_moveForwardBytesInSeqStore(rawSeqStore_t* ldmSeqStore, size_t nb ldmSeqStore->posInSequence += currSeq.litLength; nbBytes -= currSeq.litLength; } - if (nbBytes < currSeq.matchLength) { ldmSeqStore->posInSequence += nbBytes; return; @@ -797,13 +795,14 @@ static void ldm_moveForwardBytesInSeqStore(rawSeqStore_t* ldmSeqStore, size_t nb } } -/* Calculates the beginning and end of a match, and updates 'pos' and 'posInSequence' - * of the ldmSeqStore. +/* ldm_getNextMatchAndUpdateSeqStore(): + * Calculates the beginning and end of the next match in the current block. + * Updates 'pos' and 'posInSequence' of the ldmSeqStore. */ -static void ldm_calculateNextMatch(rawSeqStore_t* ldmSeqStore, - U32* matchStartPosInBlock, U32* matchEndPosInBlock, - U32* matchOffset, U32 currPosInBlock, - U32 blockBytesRemaining) { +static void ldm_getNextMatchAndUpdateSeqStore(rawSeqStore_t* ldmSeqStore, + U32* matchStartPosInBlock, U32* matchEndPosInBlock, + U32* matchOffset, U32 currPosInBlock, + U32 blockBytesRemaining) { rawSeq currSeq; U32 currBlockEndPos; U32 literalsBytesRemaining; @@ -827,7 +826,7 @@ static void ldm_calculateNextMatch(rawSeqStore_t* ldmSeqStore, currSeq.matchLength - (ldmSeqStore->posInSequence - currSeq.litLength) : currSeq.matchLength; - /* If there are more literal bytes than bytes remaining in block, no ldm */ + /* If there are more literal bytes than bytes remaining in block, no ldm is possible */ if (literalsBytesRemaining >= blockBytesRemaining) { *matchStartPosInBlock = UINT_MAX; *matchEndPosInBlock = UINT_MAX; @@ -852,7 +851,10 @@ static void ldm_calculateNextMatch(rawSeqStore_t* ldmSeqStore, } } -/* Adds an LDM if it's long enough */ +/* ldm_maybeAddLdm(): + * Adds a match if it's long enough, based on it's 'matchStartPosInBlock' + * and 'matchEndPosInBlock', into 'matches'. Maintains the correct ordering of 'matches' + */ static void ldm_maybeAddLdm(ZSTD_match_t* matches, U32* nbMatches, U32 matchStartPosInBlock, U32 matchEndPosInBlock, U32 matchOffset, U32 currPosInBlock) { @@ -865,7 +867,7 @@ static void ldm_maybeAddLdm(ZSTD_match_t* matches, U32* nbMatches, if (currPosInBlock < matchStartPosInBlock || currPosInBlock >= matchEndPosInBlock || posDiff > 0 || /* As a next evolution we can enable adding LDMs in the middle of a match */ - candidateMatchLength < ZSTD_LDM_MINMATCH_MIN) + candidateMatchLength < MINMATCH) return; if (*nbMatches == 0) { @@ -879,17 +881,18 @@ static void ldm_maybeAddLdm(ZSTD_match_t* matches, U32* nbMatches, * the ldm match down as necessary. */ if (candidateMatchLength == matches[*nbMatches-1].len) { + U32 candidateMatchIdx; if (candidateOffCode == matches[*nbMatches-1].off) { /* No need to insert the match if it's the exact same */ return; } - U32 candidateMatchIdx = *nbMatches; + candidateMatchIdx = *nbMatches; matches[*nbMatches].len = candidateMatchLength; matches[*nbMatches].off = candidateOffCode; if (candidateOffCode != matches[*nbMatches-1].off) { while (candidateMatchIdx > 0 && - matches[candidateMatchIdx].off > matches[candidateMatchIdx - 1].off && - matches[candidateMatchIdx].len == matches[candidateMatchIdx - 1].len) { + matches[candidateMatchIdx].off > matches[candidateMatchIdx - 1].off && + matches[candidateMatchIdx].len == matches[candidateMatchIdx - 1].len) { ZSTD_match_t tmp = matches[candidateMatchIdx - 1]; matches[candidateMatchIdx - 1] = matches[candidateMatchIdx]; matches[candidateMatchIdx] = tmp; @@ -918,9 +921,9 @@ static void ldm_handleLdm(rawSeqStore_t* ldmSeqStore, ZSTD_match_t* matches, U32 U32 posOvershoot = currPosInBlock - *matchEndPosInBlock; ldm_moveForwardBytesInSeqStore(ldmSeqStore, posOvershoot); } - ldm_calculateNextMatch(ldmSeqStore, matchStartPosInBlock, - matchEndPosInBlock, matchOffset, - currPosInBlock, remainingBytes); + ldm_getNextMatchAndUpdateSeqStore(ldmSeqStore, matchStartPosInBlock, + matchEndPosInBlock, matchOffset, + currPosInBlock, remainingBytes); } ldm_maybeAddLdm(matches, nbMatches, *matchStartPosInBlock, *matchEndPosInBlock, *matchOffset, currPosInBlock); } @@ -985,9 +988,9 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms, /* Get first match from ldm seq store if long mode is enabled */ if (ms->ldmSeqStore.size > 0 && ms->ldmSeqStore.pos < ms->ldmSeqStore.size) { - ldm_calculateNextMatch(&ms->ldmSeqStore, &ldmStartPosInBlock, - &ldmEndPosInBlock, &ldmOffset, - (U32)(ip-istart), (U32)(iend-ip)); + ldm_getNextMatchAndUpdateSeqStore(&ms->ldmSeqStore, &ldmStartPosInBlock, + &ldmEndPosInBlock, &ldmOffset, + (U32)(ip-istart), (U32)(iend-ip)); } /* init */ @@ -1249,7 +1252,7 @@ _shortestPath: /* cur, last_pos, best_mlen, best_off have to be set */ if (ldmEndPosInBlock < srcSize) { /* This can occur if after adding the final match in an ldm seq store within this block, - ip goes to the end of the block without activating a check for ldm_calculateNextMatch */ + ip reaches end of the block without calling ldm_getNextMatchAndUpdateSeqStore() */ ldm_moveForwardBytesInSeqStore(&ms->ldmSeqStore, srcSize - ldmEndPosInBlock); } /* Return the last literals size */ From 429dec4f42619f469c93f2e09373c3243a98bf40 Mon Sep 17 00:00:00 2001 From: senhuang42 Date: Thu, 1 Oct 2020 12:07:18 -0400 Subject: [PATCH 44/68] Add DEBUGLOG() calls in ldm helpers --- lib/compress/zstd_opt.c | 8 ++++++-- tests/playTests.sh | 5 ----- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/lib/compress/zstd_opt.c b/lib/compress/zstd_opt.c index 7efae9b87..a7618c872 100644 --- a/lib/compress/zstd_opt.c +++ b/lib/compress/zstd_opt.c @@ -849,6 +849,8 @@ static void ldm_getNextMatchAndUpdateSeqStore(rawSeqStore_t* ldmSeqStore, ldmSeqStore->posInSequence = 0; ldmSeqStore->pos++; } + DEBUGLOG(6, "ldm_getNextMatchAndUpdateSeqStore(): got an ldm that beginning at pos: %u, end at pos: %u, with offset: %u", + *matchStartPosInBlock, *matchEndPosInBlock, *matchOffset); } /* ldm_maybeAddLdm(): @@ -870,6 +872,8 @@ static void ldm_maybeAddLdm(ZSTD_match_t* matches, U32* nbMatches, candidateMatchLength < MINMATCH) return; + DEBUGLOG(6, "ldm_maybeAddLdm(): Adding ldm candidate match (offCode: %u matchLength %u) at block position=%u", + candidateOffCode, candidateMatchLength, currPosInBlock); if (*nbMatches == 0) { matches[*nbMatches].len = candidateMatchLength; matches[*nbMatches].off = candidateOffCode; @@ -985,14 +989,14 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms, U32 ldmStartPosInBlock = 0; U32 ldmEndPosInBlock = 0; U32 ldmOffset = 0; - + /* Get first match from ldm seq store if long mode is enabled */ if (ms->ldmSeqStore.size > 0 && ms->ldmSeqStore.pos < ms->ldmSeqStore.size) { ldm_getNextMatchAndUpdateSeqStore(&ms->ldmSeqStore, &ldmStartPosInBlock, &ldmEndPosInBlock, &ldmOffset, (U32)(ip-istart), (U32)(iend-ip)); } - + /* init */ DEBUGLOG(5, "ZSTD_compressBlock_opt_generic: current=%u, prefix=%u, nextToUpdate=%u", (U32)(ip - base), ms->window.dictLimit, ms->nextToUpdate); diff --git a/tests/playTests.sh b/tests/playTests.sh index ee3ef42f0..3959a8f3a 100755 --- a/tests/playTests.sh +++ b/tests/playTests.sh @@ -1343,8 +1343,6 @@ optCSize19=$(datagen -g2M | zstd -19 -c | wc -c) longCSize19=$(datagen -g2M | zstd -19 --long -c | wc -c) optCSize19wlog23=$(datagen -g2M | zstd -19 -c --zstd=wlog=23 | wc -c) longCSize19wlog23=$(datagen -g2M | zstd -19 -c --long=23 | wc -c) -optCSize19wlog27=$(datagen -g5M | zstd -19 -c --zstd=wlog=27 | wc -c) -longCSize19wlog27=$(datagen -g5M | zstd -19 -c --long=27 | wc -c) optCSize22=$(datagen -g900K | zstd -22 --ultra -c | wc -c) longCSize22=$(datagen -g900K | zstd -22 --ultra --long -c | wc -c) if [ "$longCSize16" -gt "$optCSize16" ]; then @@ -1356,9 +1354,6 @@ elif [ "$longCSize19" -gt "$optCSize19" ]; then elif [ "$longCSize19wlog23" -gt "$optCSize19wlog23" ]; then echo using --long on compression level 19 with wLog=23 should not cause compressed size regression exit 1 -elif [ "$longCSize19wlog27" -gt "$optCSize19wlog27" ]; then - echo using --long on compression level 19 with wLog=27 should not cause compressed size regression - exit 1 elif [ "$longCSize22" -gt "$optCSize22" ]; then echo using --long on compression level 22 should not cause compressed size regression exit 1 From c87d2e586619ff9d5b91021fab0f56b9dea0eecd Mon Sep 17 00:00:00 2001 From: senhuang42 Date: Thu, 1 Oct 2020 16:14:41 -0400 Subject: [PATCH 45/68] Prefix new static ldm helpers with ZSTD_opt --- lib/compress/zstd_opt.c | 99 +++++++++++++++++++++-------------------- 1 file changed, 51 insertions(+), 48 deletions(-) diff --git a/lib/compress/zstd_opt.c b/lib/compress/zstd_opt.c index a7618c872..fa3587bed 100644 --- a/lib/compress/zstd_opt.c +++ b/lib/compress/zstd_opt.c @@ -764,17 +764,17 @@ FORCE_INLINE_TEMPLATE U32 ZSTD_BtGetAllMatches ( } } -/*-******************************* -* LDM helper functions -*********************************/ +/************************* +* LDM helper functions * +*************************/ -/* ldm_moveForwardBytesInSeqStore(): +/* ZSTD_opt_skipBytesInLdmSeqStore(): * Moves forward in rawSeqStore by nbBytes, which will update the fields 'pos' and 'posInSequence'. */ -static void ldm_moveForwardBytesInSeqStore(rawSeqStore_t* ldmSeqStore, size_t nbBytes) { +static void ZSTD_opt_skipBytesInLdmSeqStore(rawSeqStore_t* ldmSeqStore, size_t nbBytes) { while (nbBytes && ldmSeqStore->pos < ldmSeqStore->size) { - rawSeq currSeq; - currSeq = ldmSeqStore->seq[ldmSeqStore->pos]; + rawSeq currSeq = ldmSeqStore->seq[ldmSeqStore->pos]; + assert(ldmSeqStore->posInSequence <= currSeq.matchLength + currSeq.litLength); if (nbBytes <= currSeq.litLength) { ldmSeqStore->posInSequence += nbBytes; return; @@ -795,14 +795,14 @@ static void ldm_moveForwardBytesInSeqStore(rawSeqStore_t* ldmSeqStore, size_t nb } } -/* ldm_getNextMatchAndUpdateSeqStore(): +/* ZSTD_opt_getNextLdmAndUpdateSeqStore(): * Calculates the beginning and end of the next match in the current block. * Updates 'pos' and 'posInSequence' of the ldmSeqStore. */ -static void ldm_getNextMatchAndUpdateSeqStore(rawSeqStore_t* ldmSeqStore, - U32* matchStartPosInBlock, U32* matchEndPosInBlock, - U32* matchOffset, U32 currPosInBlock, - U32 blockBytesRemaining) { +static void ZSTD_opt_getNextLdmAndUpdateSeqStore(rawSeqStore_t* ldmSeqStore, + U32* matchStartPosInBlock, U32* matchEndPosInBlock, + U32* matchOffset, U32 currPosInBlock, + U32 blockBytesRemaining) { rawSeq currSeq; U32 currBlockEndPos; U32 literalsBytesRemaining; @@ -814,10 +814,10 @@ static void ldm_getNextMatchAndUpdateSeqStore(rawSeqStore_t* ldmSeqStore, *matchEndPosInBlock = UINT_MAX; return; } - /* Calculate appropriate bytes left in matchLength and litLength after adjusting based on ldmSeqStore->posInSequence */ currSeq = ldmSeqStore->seq[ldmSeqStore->pos]; + assert(ldmSeqStore->posInSequence <= currSeq.litLength + currSeq.matchLength); currBlockEndPos = currPosInBlock + blockBytesRemaining; literalsBytesRemaining = (ldmSeqStore->posInSequence < currSeq.litLength) ? currSeq.litLength - ldmSeqStore->posInSequence : @@ -830,7 +830,7 @@ static void ldm_getNextMatchAndUpdateSeqStore(rawSeqStore_t* ldmSeqStore, if (literalsBytesRemaining >= blockBytesRemaining) { *matchStartPosInBlock = UINT_MAX; *matchEndPosInBlock = UINT_MAX; - ldm_moveForwardBytesInSeqStore(ldmSeqStore, blockBytesRemaining); + ZSTD_opt_skipBytesInLdmSeqStore(ldmSeqStore, blockBytesRemaining); return; } @@ -843,36 +843,37 @@ static void ldm_getNextMatchAndUpdateSeqStore(rawSeqStore_t* ldmSeqStore, if (*matchEndPosInBlock > currBlockEndPos) { /* Match ends after the block ends, we can't use the whole match */ *matchEndPosInBlock = currBlockEndPos; - ldm_moveForwardBytesInSeqStore(ldmSeqStore, currBlockEndPos - currPosInBlock); + ZSTD_opt_skipBytesInLdmSeqStore(ldmSeqStore, currBlockEndPos - currPosInBlock); } else { /* If we can use the whole match point the ldmSeqStore at the next match */ ldmSeqStore->posInSequence = 0; ldmSeqStore->pos++; } - DEBUGLOG(6, "ldm_getNextMatchAndUpdateSeqStore(): got an ldm that beginning at pos: %u, end at pos: %u, with offset: %u", + DEBUGLOG(6, "ZSTD_opt_getNextLdmAndUpdateSeqStore(): got an ldm that beginning at pos: %u, end at pos: %u, with offset: %u", *matchStartPosInBlock, *matchEndPosInBlock, *matchOffset); } -/* ldm_maybeAddLdm(): +/* ZSTD_opt_maybeAddLdm(): * Adds a match if it's long enough, based on it's 'matchStartPosInBlock' * and 'matchEndPosInBlock', into 'matches'. Maintains the correct ordering of 'matches' */ -static void ldm_maybeAddLdm(ZSTD_match_t* matches, U32* nbMatches, - U32 matchStartPosInBlock, U32 matchEndPosInBlock, - U32 matchOffset, U32 currPosInBlock) { +static void ZSTD_opt_maybeAddLdm(ZSTD_match_t* matches, U32* nbMatches, + U32 matchStartPosInBlock, U32 matchEndPosInBlock, + U32 matchOffset, U32 currPosInBlock) { U32 posDiff = currPosInBlock - matchStartPosInBlock; /* Note: ZSTD_match_t actually contains offCode and matchLength (before subtracting MINMATCH) */ U32 candidateMatchLength = matchEndPosInBlock - matchStartPosInBlock - posDiff; U32 candidateOffCode = matchOffset + posDiff + ZSTD_REP_MOVE; /* Ensure that current block position is not outside of the match */ - if (currPosInBlock < matchStartPosInBlock || - currPosInBlock >= matchEndPosInBlock || - posDiff > 0 || /* As a next evolution we can enable adding LDMs in the middle of a match */ - candidateMatchLength < MINMATCH) + if (currPosInBlock < matchStartPosInBlock + || currPosInBlock >= matchEndPosInBlock + || posDiff > 0 /* As a next evolution we can enable adding LDMs in the middle of a match */ + || candidateMatchLength < MINMATCH) { return; + } - DEBUGLOG(6, "ldm_maybeAddLdm(): Adding ldm candidate match (offCode: %u matchLength %u) at block position=%u", + DEBUGLOG(6, "ZSTD_opt_maybeAddLdm(): Adding ldm candidate match (offCode: %u matchLength %u) at block position=%u", candidateOffCode, candidateMatchLength, currPosInBlock); if (*nbMatches == 0) { matches[*nbMatches].len = candidateMatchLength; @@ -881,7 +882,7 @@ static void ldm_maybeAddLdm(ZSTD_match_t* matches, U32* nbMatches, } else if ((candidateMatchLength >= matches[*nbMatches-1].len) && *nbMatches < ZSTD_OPT_NUM) { /* Maintain order of matches, which is firstly - increasing in matchlength, * and secondly - decreasing in offCode. Since matches from the ldm seq store are likely - * to be the longest match found, we simply start at the end of the array and sift + * to be the longest match found, we simply start at the end of the array and bubble * the ldm match down as necessary. */ if (candidateMatchLength == matches[*nbMatches-1].len) { @@ -912,24 +913,26 @@ static void ldm_maybeAddLdm(ZSTD_match_t* matches, U32* nbMatches, } } -/* Wrapper function to update and call ldm functions as necessary */ -static void ldm_handleLdm(rawSeqStore_t* ldmSeqStore, ZSTD_match_t* matches, U32* nbMatches, - U32* matchStartPosInBlock, U32* matchEndPosInBlock, U32* matchOffset, - U32 currPosInBlock, U32 remainingBytes) { +/* ZSTD_opt_processLdm(): + * Wrapper function to update ldm seq store and call ldm functions as necessary. + */ +static void ZSTD_opt_processLdm(rawSeqStore_t* ldmSeqStore, ZSTD_match_t* matches, U32* nbMatches, + U32* matchStartPosInBlock, U32* matchEndPosInBlock, U32* matchOffset, + U32 currPosInBlock, U32 remainingBytes) { if (currPosInBlock >= *matchEndPosInBlock) { if (currPosInBlock > *matchEndPosInBlock) { - /* The position at which ldm_handleLdm() is called is not necessarily + /* The position at which ZSTD_opt_processLdm() is called is not necessarily * at the end of a match from the ldm seq store, and will often be some bytes * over beyond matchEndPosInBlock. As such, we need to correct for these "overshoots" */ U32 posOvershoot = currPosInBlock - *matchEndPosInBlock; - ldm_moveForwardBytesInSeqStore(ldmSeqStore, posOvershoot); + ZSTD_opt_skipBytesInLdmSeqStore(ldmSeqStore, posOvershoot); } - ldm_getNextMatchAndUpdateSeqStore(ldmSeqStore, matchStartPosInBlock, + ZSTD_opt_getNextLdmAndUpdateSeqStore(ldmSeqStore, matchStartPosInBlock, matchEndPosInBlock, matchOffset, currPosInBlock, remainingBytes); } - ldm_maybeAddLdm(matches, nbMatches, *matchStartPosInBlock, *matchEndPosInBlock, *matchOffset, currPosInBlock); + ZSTD_opt_maybeAddLdm(matches, nbMatches, *matchStartPosInBlock, *matchEndPosInBlock, *matchOffset, currPosInBlock); } @@ -992,11 +995,11 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms, /* Get first match from ldm seq store if long mode is enabled */ if (ms->ldmSeqStore.size > 0 && ms->ldmSeqStore.pos < ms->ldmSeqStore.size) { - ldm_getNextMatchAndUpdateSeqStore(&ms->ldmSeqStore, &ldmStartPosInBlock, - &ldmEndPosInBlock, &ldmOffset, - (U32)(ip-istart), (U32)(iend-ip)); + ZSTD_opt_getNextLdmAndUpdateSeqStore(&ms->ldmSeqStore, &ldmStartPosInBlock, + &ldmEndPosInBlock, &ldmOffset, + (U32)(ip-istart), (U32)(iend-ip)); } - + /* init */ DEBUGLOG(5, "ZSTD_compressBlock_opt_generic: current=%u, prefix=%u, nextToUpdate=%u", (U32)(ip - base), ms->window.dictLimit, ms->nextToUpdate); @@ -1013,10 +1016,10 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms, U32 const ll0 = !litlen; U32 nbMatches = ZSTD_BtGetAllMatches(matches, ms, &nextToUpdate3, ip, iend, dictMode, rep, ll0, minMatch); if (ms->ldmSeqStore.size != 0 && ms->ldmSeqStore.pos < ms->ldmSeqStore.size) { - ldm_handleLdm(&ms->ldmSeqStore, matches, - &nbMatches, &ldmStartPosInBlock, - &ldmEndPosInBlock, &ldmOffset, - (U32)(ip-istart), (U32)(iend - ip)); + ZSTD_opt_processLdm(&ms->ldmSeqStore, matches, + &nbMatches, &ldmStartPosInBlock, + &ldmEndPosInBlock, &ldmOffset, + (U32)(ip-istart), (U32)(iend - ip)); } if (!nbMatches) { ip++; continue; } @@ -1135,10 +1138,10 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms, if (ms->ldmSeqStore.size != 0 && ms->ldmSeqStore.pos < ms->ldmSeqStore.size) { - ldm_handleLdm(&ms->ldmSeqStore, matches, - &nbMatches, &ldmStartPosInBlock, - &ldmEndPosInBlock, &ldmOffset, - (U32)(inr-istart), (U32)(iend-inr)); + ZSTD_opt_processLdm(&ms->ldmSeqStore, matches, + &nbMatches, &ldmStartPosInBlock, + &ldmEndPosInBlock, &ldmOffset, + (U32)(inr-istart), (U32)(iend-inr)); } if (!nbMatches) { DEBUGLOG(7, "rPos:%u : no match found", cur); @@ -1256,8 +1259,8 @@ _shortestPath: /* cur, last_pos, best_mlen, best_off have to be set */ if (ldmEndPosInBlock < srcSize) { /* This can occur if after adding the final match in an ldm seq store within this block, - ip reaches end of the block without calling ldm_getNextMatchAndUpdateSeqStore() */ - ldm_moveForwardBytesInSeqStore(&ms->ldmSeqStore, srcSize - ldmEndPosInBlock); + ip reaches end of the block without calling ZSTD_opt_getNextLdmAndUpdateSeqStore() */ + ZSTD_opt_skipBytesInLdmSeqStore(&ms->ldmSeqStore, srcSize - ldmEndPosInBlock); } /* Return the last literals size */ return (size_t)(iend - anchor); From b8bfc4e63deb241f058f90e22aad5bb49a3ef262 Mon Sep 17 00:00:00 2001 From: senhuang42 Date: Fri, 2 Oct 2020 09:49:35 -0400 Subject: [PATCH 46/68] Add cSize regression test to fuzzer.c --- lib/compress/zstd_ldm.c | 2 ++ lib/compress/zstd_opt.c | 3 +++ tests/fuzzer.c | 38 ++++++++++++++++++++++++++++++++++++++ tests/playTests.sh | 2 +- 4 files changed, 44 insertions(+), 1 deletion(-) diff --git a/lib/compress/zstd_ldm.c b/lib/compress/zstd_ldm.c index 341877858..356549f36 100644 --- a/lib/compress/zstd_ldm.c +++ b/lib/compress/zstd_ldm.c @@ -575,6 +575,8 @@ size_t ZSTD_ldm_blockCompress(rawSeqStore_t* rawSeqStore, BYTE const* const iend = istart + srcSize; /* Input positions */ BYTE const* ip = istart; + + /* If using opt parser, use LDMs only as candidates rather than always accepting them */ if (cParams->strategy >= ZSTD_btopt) { size_t lastLLSize; ms->ldmSeqStore = *rawSeqStore; diff --git a/lib/compress/zstd_opt.c b/lib/compress/zstd_opt.c index fa3587bed..5211fc917 100644 --- a/lib/compress/zstd_opt.c +++ b/lib/compress/zstd_opt.c @@ -774,7 +774,9 @@ FORCE_INLINE_TEMPLATE U32 ZSTD_BtGetAllMatches ( static void ZSTD_opt_skipBytesInLdmSeqStore(rawSeqStore_t* ldmSeqStore, size_t nbBytes) { while (nbBytes && ldmSeqStore->pos < ldmSeqStore->size) { rawSeq currSeq = ldmSeqStore->seq[ldmSeqStore->pos]; + /* posInSequence necessarily must never represent a value beyond the sequence */ assert(ldmSeqStore->posInSequence <= currSeq.matchLength + currSeq.litLength); + if (nbBytes <= currSeq.litLength) { ldmSeqStore->posInSequence += nbBytes; return; @@ -782,6 +784,7 @@ static void ZSTD_opt_skipBytesInLdmSeqStore(rawSeqStore_t* ldmSeqStore, size_t n ldmSeqStore->posInSequence += currSeq.litLength; nbBytes -= currSeq.litLength; } + if (nbBytes < currSeq.matchLength) { ldmSeqStore->posInSequence += nbBytes; return; diff --git a/tests/fuzzer.c b/tests/fuzzer.c index 8b10078ac..82d2f119e 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -780,6 +780,44 @@ static int basicUnitTests(U32 const seed, double compressibility) } DISPLAYLEVEL(3, "OK \n"); + DISPLAYLEVEL(3, "test%3i : testing ldm no regressions in size for opt parser : ", testNb++); + { + size_t cSizeLdm; + size_t cSizeNoLdm; + ZSTD_CCtx* const cctx = ZSTD_createCCtx(); + + RDG_genBuffer(CNBuffer, CNBuffSize, 0.5, 0.5, seed); + + /* Enable checksum to verify round trip. */ + CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1)); + CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, 1)); + CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 19)); + + /* Round trip once with ldm. */ + cSizeLdm = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize); + CHECK_Z(cSizeLdm); + CHECK_Z(ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSizeLdm)); + + ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters); + CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1)); + CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, 0)); + CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 19)); + + /* Round trip once without ldm. */ + cSizeNoLdm = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize); + CHECK_Z(cSizeNoLdm); + CHECK_Z(ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSizeNoLdm)); + + if (cSizeLdm > cSizeNoLdm) { + DISPLAY("Using long mode should not cause regressions for btopt+\n"); + testResult = 1; + goto _output_error; + } + + ZSTD_freeCCtx(cctx); + } + DISPLAYLEVEL(3, "OK \n"); + /* Note: this test takes 0.5 seconds to run */ DISPLAYLEVEL(3, "test%3i : testing refPrefx vs refPrefx + ldm (size comparison) : ", testNb++); { diff --git a/tests/playTests.sh b/tests/playTests.sh index 3959a8f3a..49371040e 100755 --- a/tests/playTests.sh +++ b/tests/playTests.sh @@ -1217,7 +1217,7 @@ then println "\n===> zstdmt round-trip tests " roundTripTest -g4M "1 -T0" roundTripTest -g8M "3 -T2" - roundTripTest -g8M "3 -T0 --long" + roundTripTest -g8M "19 -T0 --long" roundTripTest -g8000K "2 --threads=2" fileRoundTripTest -g4M "19 -T2 -B1M" From d8b43a42023ee5bdbc1f093d4d418aad00b443cc Mon Sep 17 00:00:00 2001 From: senhuang42 Date: Fri, 2 Oct 2020 13:47:54 -0400 Subject: [PATCH 47/68] Add explicit conversion of size_t to U32 --- lib/compress/zstd_opt.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/compress/zstd_opt.c b/lib/compress/zstd_opt.c index 5211fc917..dcd9e1929 100644 --- a/lib/compress/zstd_opt.c +++ b/lib/compress/zstd_opt.c @@ -784,7 +784,7 @@ static void ZSTD_opt_skipBytesInLdmSeqStore(rawSeqStore_t* ldmSeqStore, size_t n ldmSeqStore->posInSequence += currSeq.litLength; nbBytes -= currSeq.litLength; } - + if (nbBytes < currSeq.matchLength) { ldmSeqStore->posInSequence += nbBytes; return; @@ -823,10 +823,10 @@ static void ZSTD_opt_getNextLdmAndUpdateSeqStore(rawSeqStore_t* ldmSeqStore, assert(ldmSeqStore->posInSequence <= currSeq.litLength + currSeq.matchLength); currBlockEndPos = currPosInBlock + blockBytesRemaining; literalsBytesRemaining = (ldmSeqStore->posInSequence < currSeq.litLength) ? - currSeq.litLength - ldmSeqStore->posInSequence : + currSeq.litLength - (U32)ldmSeqStore->posInSequence : 0; matchBytesRemaining = (literalsBytesRemaining == 0) ? - currSeq.matchLength - (ldmSeqStore->posInSequence - currSeq.litLength) : + currSeq.matchLength - ((U32)ldmSeqStore->posInSequence - currSeq.litLength) : currSeq.matchLength; /* If there are more literal bytes than bytes remaining in block, no ldm is possible */ From e96ea5d1478a8cbd29b8062907158fc826d24582 Mon Sep 17 00:00:00 2001 From: senhuang42 Date: Fri, 2 Oct 2020 15:27:06 -0400 Subject: [PATCH 48/68] Fix static analyze fuzzer.c error --- tests/fuzzer.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/fuzzer.c b/tests/fuzzer.c index 82d2f119e..d18cb48c7 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -811,7 +811,7 @@ static int basicUnitTests(U32 const seed, double compressibility) if (cSizeLdm > cSizeNoLdm) { DISPLAY("Using long mode should not cause regressions for btopt+\n"); testResult = 1; - goto _output_error; + goto _end; } ZSTD_freeCCtx(cctx); From 88f72ed942d6ebdca5e551ca292d4dcc0820514b Mon Sep 17 00:00:00 2001 From: senhuang42 Date: Fri, 2 Oct 2020 19:08:10 -0400 Subject: [PATCH 49/68] Correct incorrect offcode calculation --- lib/compress/zstd_opt.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/compress/zstd_opt.c b/lib/compress/zstd_opt.c index dcd9e1929..69e73cb47 100644 --- a/lib/compress/zstd_opt.c +++ b/lib/compress/zstd_opt.c @@ -866,7 +866,7 @@ static void ZSTD_opt_maybeAddLdm(ZSTD_match_t* matches, U32* nbMatches, U32 posDiff = currPosInBlock - matchStartPosInBlock; /* Note: ZSTD_match_t actually contains offCode and matchLength (before subtracting MINMATCH) */ U32 candidateMatchLength = matchEndPosInBlock - matchStartPosInBlock - posDiff; - U32 candidateOffCode = matchOffset + posDiff + ZSTD_REP_MOVE; + U32 candidateOffCode = matchOffset + ZSTD_REP_MOVE; /* Ensure that current block position is not outside of the match */ if (currPosInBlock < matchStartPosInBlock @@ -1146,6 +1146,7 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms, &ldmEndPosInBlock, &ldmOffset, (U32)(inr-istart), (U32)(iend-inr)); } + if (!nbMatches) { DEBUGLOG(7, "rPos:%u : no match found", cur); continue; From ddf8a3f1b91f91d16e00da9735169f4d1c0acdfe Mon Sep 17 00:00:00 2001 From: senhuang42 Date: Sun, 4 Oct 2020 21:46:47 -0400 Subject: [PATCH 50/68] Enable inclusion of mid-flight LDMs in opt parser --- lib/compress/zstd_opt.c | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/compress/zstd_opt.c b/lib/compress/zstd_opt.c index 69e73cb47..07b30f5f9 100644 --- a/lib/compress/zstd_opt.c +++ b/lib/compress/zstd_opt.c @@ -871,7 +871,6 @@ static void ZSTD_opt_maybeAddLdm(ZSTD_match_t* matches, U32* nbMatches, /* Ensure that current block position is not outside of the match */ if (currPosInBlock < matchStartPosInBlock || currPosInBlock >= matchEndPosInBlock - || posDiff > 0 /* As a next evolution we can enable adding LDMs in the middle of a match */ || candidateMatchLength < MINMATCH) { return; } From 031b7ec15fd4f0be2e62a67518ff6da9d7c19838 Mon Sep 17 00:00:00 2001 From: senhuang42 Date: Sun, 4 Oct 2020 21:47:25 -0400 Subject: [PATCH 51/68] Disable LDM minMatch adjustment when using opt parser --- lib/compress/zstd_ldm.c | 7 ------- 1 file changed, 7 deletions(-) diff --git a/lib/compress/zstd_ldm.c b/lib/compress/zstd_ldm.c index 356549f36..81dc3da19 100644 --- a/lib/compress/zstd_ldm.c +++ b/lib/compress/zstd_ldm.c @@ -27,13 +27,6 @@ void ZSTD_ldm_adjustParameters(ldmParams_t* params, DEBUGLOG(4, "ZSTD_ldm_adjustParameters"); if (!params->bucketSizeLog) params->bucketSizeLog = LDM_BUCKET_SIZE_LOG; if (!params->minMatchLength) params->minMatchLength = LDM_MIN_MATCH_LENGTH; - if (cParams->strategy >= ZSTD_btopt) { - /* Get out of the way of the optimal parser */ - U32 const minMatch = MAX(cParams->targetLength, params->minMatchLength); - assert(minMatch >= ZSTD_LDM_MINMATCH_MIN); - assert(minMatch <= ZSTD_LDM_MINMATCH_MAX); - params->minMatchLength = minMatch; - } if (params->hashLog == 0) { params->hashLog = MAX(ZSTD_HASHLOG_MIN, params->windowLog - LDM_HASH_RLOG); assert(params->hashLog <= ZSTD_HASHLOG_MAX); From 0325d878f230c2d394463b969f91d63bea8f66cc Mon Sep 17 00:00:00 2001 From: senhuang42 Date: Sun, 4 Oct 2020 21:48:39 -0400 Subject: [PATCH 52/68] Remove bubbling down matches with longer offCode and same matchLen --- lib/compress/zstd_opt.c | 35 ++++++----------------------------- 1 file changed, 6 insertions(+), 29 deletions(-) diff --git a/lib/compress/zstd_opt.c b/lib/compress/zstd_opt.c index 07b30f5f9..b737d6bdd 100644 --- a/lib/compress/zstd_opt.c +++ b/lib/compress/zstd_opt.c @@ -882,36 +882,13 @@ static void ZSTD_opt_maybeAddLdm(ZSTD_match_t* matches, U32* nbMatches, matches[*nbMatches].off = candidateOffCode; (*nbMatches)++; } else if ((candidateMatchLength >= matches[*nbMatches-1].len) && *nbMatches < ZSTD_OPT_NUM) { - /* Maintain order of matches, which is firstly - increasing in matchlength, - * and secondly - decreasing in offCode. Since matches from the ldm seq store are likely - * to be the longest match found, we simply start at the end of the array and bubble - * the ldm match down as necessary. - */ - if (candidateMatchLength == matches[*nbMatches-1].len) { - U32 candidateMatchIdx; - if (candidateOffCode == matches[*nbMatches-1].off) { - /* No need to insert the match if it's the exact same */ - return; - } - candidateMatchIdx = *nbMatches; - matches[*nbMatches].len = candidateMatchLength; - matches[*nbMatches].off = candidateOffCode; - if (candidateOffCode != matches[*nbMatches-1].off) { - while (candidateMatchIdx > 0 && - matches[candidateMatchIdx].off > matches[candidateMatchIdx - 1].off && - matches[candidateMatchIdx].len == matches[candidateMatchIdx - 1].len) { - ZSTD_match_t tmp = matches[candidateMatchIdx - 1]; - matches[candidateMatchIdx - 1] = matches[candidateMatchIdx]; - matches[candidateMatchIdx] = tmp; - --candidateMatchIdx; - } - } - (*nbMatches)++; - } else { - matches[*nbMatches].len = candidateMatchLength; - matches[*nbMatches].off = candidateOffCode; - (*nbMatches)++; + /* No need to insert the match if it's the exact same, or offCode is larger with same matchLen */ + if (candidateMatchLength == matches[*nbMatches-1].len && candidateOffCode >= matches[*nbMatches-1].off) { + return; } + matches[*nbMatches].len = candidateMatchLength; + matches[*nbMatches].off = candidateOffCode; + (*nbMatches)++; } } From 0731b94e7c326584fc9f4b02139925ad20bc5e25 Mon Sep 17 00:00:00 2001 From: senhuang42 Date: Mon, 5 Oct 2020 10:50:49 -0400 Subject: [PATCH 53/68] Use kNullRawSeqStore constant in zstdmt_compress.c --- lib/compress/zstdmt_compress.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/compress/zstdmt_compress.c b/lib/compress/zstdmt_compress.c index f789ae5bf..06462fe5a 100644 --- a/lib/compress/zstdmt_compress.c +++ b/lib/compress/zstdmt_compress.c @@ -277,7 +277,7 @@ static size_t ZSTDMT_sizeof_seqPool(ZSTDMT_seqPool* seqPool) static rawSeqStore_t bufferToSeq(buffer_t buffer) { - rawSeqStore_t seq = {NULL, 0, 0, 0, 0}; + rawSeqStore_t seq = kNullRawSeqStore; seq.seq = (rawSeq*)buffer.start; seq.capacity = buffer.capacity / sizeof(rawSeq); return seq; From a5500cf2af8810a124141b46cb623e42f59800a0 Mon Sep 17 00:00:00 2001 From: senhuang42 Date: Mon, 5 Oct 2020 18:10:10 -0400 Subject: [PATCH 54/68] Refactor separate ldm variables all into one struct --- lib/compress/zstd_ldm.c | 2 +- lib/compress/zstd_opt.c | 139 +++++++++++++++++++--------------------- 2 files changed, 67 insertions(+), 74 deletions(-) diff --git a/lib/compress/zstd_ldm.c b/lib/compress/zstd_ldm.c index 81dc3da19..623245567 100644 --- a/lib/compress/zstd_ldm.c +++ b/lib/compress/zstd_ldm.c @@ -574,7 +574,7 @@ size_t ZSTD_ldm_blockCompress(rawSeqStore_t* rawSeqStore, size_t lastLLSize; ms->ldmSeqStore = *rawSeqStore; lastLLSize = blockCompressor(ms, seqStore, rep, src, srcSize); - *rawSeqStore = ms->ldmSeqStore; /* Persist changes to ldmSeqStore during blockCompressor() */ + *rawSeqStore = ms->ldmSeqStore; return lastLLSize; } diff --git a/lib/compress/zstd_opt.c b/lib/compress/zstd_opt.c index b737d6bdd..472f9e8a8 100644 --- a/lib/compress/zstd_opt.c +++ b/lib/compress/zstd_opt.c @@ -768,10 +768,17 @@ FORCE_INLINE_TEMPLATE U32 ZSTD_BtGetAllMatches ( * LDM helper functions * *************************/ -/* ZSTD_opt_skipBytesInLdmSeqStore(): +typedef struct { + rawSeqStore_t* seqStore; /* Reference to struct containing external candidates */ + U32 startPosInBlock; /* Start position of the current match candidate */ + U32 endPosInBlock; /* End position of the current match candidate */ + U32 offset; /* Offset of the match candidate */ +} ZSTD_optLdm_t; + +/* ZSTD_opt_skipRawSeqStoreBytes(): * Moves forward in rawSeqStore by nbBytes, which will update the fields 'pos' and 'posInSequence'. */ -static void ZSTD_opt_skipBytesInLdmSeqStore(rawSeqStore_t* ldmSeqStore, size_t nbBytes) { +static void ZSTD_opt_skipRawSeqStoreBytes(rawSeqStore_t* ldmSeqStore, size_t nbBytes) { while (nbBytes && ldmSeqStore->pos < ldmSeqStore->size) { rawSeq currSeq = ldmSeqStore->seq[ldmSeqStore->pos]; /* posInSequence necessarily must never represent a value beyond the sequence */ @@ -798,84 +805,81 @@ static void ZSTD_opt_skipBytesInLdmSeqStore(rawSeqStore_t* ldmSeqStore, size_t n } } -/* ZSTD_opt_getNextLdmAndUpdateSeqStore(): +/* ZSTD_opt_getNextMatchAndUpdateSeqStore(): * Calculates the beginning and end of the next match in the current block. * Updates 'pos' and 'posInSequence' of the ldmSeqStore. */ -static void ZSTD_opt_getNextLdmAndUpdateSeqStore(rawSeqStore_t* ldmSeqStore, - U32* matchStartPosInBlock, U32* matchEndPosInBlock, - U32* matchOffset, U32 currPosInBlock, - U32 blockBytesRemaining) { +static void ZSTD_opt_getNextMatchAndUpdateSeqStore(ZSTD_optLdm_t* optLdm, U32 currPosInBlock, + U32 blockBytesRemaining) { rawSeq currSeq; U32 currBlockEndPos; U32 literalsBytesRemaining; U32 matchBytesRemaining; /* Setting match end position to MAX to ensure we never use an LDM during this block */ - if (ldmSeqStore->pos >= ldmSeqStore->size) { - *matchStartPosInBlock = UINT_MAX; - *matchEndPosInBlock = UINT_MAX; + if (optLdm->seqStore->pos >= optLdm->seqStore->size) { + optLdm->startPosInBlock = UINT_MAX; + optLdm->endPosInBlock = UINT_MAX; return; } /* Calculate appropriate bytes left in matchLength and litLength after adjusting based on ldmSeqStore->posInSequence */ - currSeq = ldmSeqStore->seq[ldmSeqStore->pos]; - assert(ldmSeqStore->posInSequence <= currSeq.litLength + currSeq.matchLength); + currSeq = optLdm->seqStore->seq[optLdm->seqStore->pos]; + assert(optLdm->seqStore->posInSequence <= currSeq.litLength + currSeq.matchLength); currBlockEndPos = currPosInBlock + blockBytesRemaining; - literalsBytesRemaining = (ldmSeqStore->posInSequence < currSeq.litLength) ? - currSeq.litLength - (U32)ldmSeqStore->posInSequence : + literalsBytesRemaining = (optLdm->seqStore->posInSequence < currSeq.litLength) ? + currSeq.litLength - (U32)optLdm->seqStore->posInSequence : 0; matchBytesRemaining = (literalsBytesRemaining == 0) ? - currSeq.matchLength - ((U32)ldmSeqStore->posInSequence - currSeq.litLength) : + currSeq.matchLength - ((U32)optLdm->seqStore->posInSequence - currSeq.litLength) : currSeq.matchLength; /* If there are more literal bytes than bytes remaining in block, no ldm is possible */ if (literalsBytesRemaining >= blockBytesRemaining) { - *matchStartPosInBlock = UINT_MAX; - *matchEndPosInBlock = UINT_MAX; - ZSTD_opt_skipBytesInLdmSeqStore(ldmSeqStore, blockBytesRemaining); + optLdm->startPosInBlock = UINT_MAX; + optLdm->endPosInBlock = UINT_MAX; + ZSTD_opt_skipRawSeqStoreBytes(optLdm->seqStore, blockBytesRemaining); return; } /* Matches may be < MINMATCH by this process. In that case, we will reject them when we are deciding whether or not to add the ldm */ - *matchStartPosInBlock = currPosInBlock + literalsBytesRemaining; - *matchEndPosInBlock = *matchStartPosInBlock + matchBytesRemaining; - *matchOffset = currSeq.offset; + optLdm->startPosInBlock = currPosInBlock + literalsBytesRemaining; + optLdm->endPosInBlock = optLdm->startPosInBlock + matchBytesRemaining; + optLdm->offset = currSeq.offset; - if (*matchEndPosInBlock > currBlockEndPos) { + if (optLdm->endPosInBlock > currBlockEndPos) { /* Match ends after the block ends, we can't use the whole match */ - *matchEndPosInBlock = currBlockEndPos; - ZSTD_opt_skipBytesInLdmSeqStore(ldmSeqStore, currBlockEndPos - currPosInBlock); + optLdm->endPosInBlock = currBlockEndPos; + ZSTD_opt_skipRawSeqStoreBytes(optLdm->seqStore, currBlockEndPos - currPosInBlock); } else { /* If we can use the whole match point the ldmSeqStore at the next match */ - ldmSeqStore->posInSequence = 0; - ldmSeqStore->pos++; + optLdm->seqStore->posInSequence = 0; + optLdm->seqStore->pos++; } - DEBUGLOG(6, "ZSTD_opt_getNextLdmAndUpdateSeqStore(): got an ldm that beginning at pos: %u, end at pos: %u, with offset: %u", - *matchStartPosInBlock, *matchEndPosInBlock, *matchOffset); + DEBUGLOG(6, "ZSTD_opt_getNextMatchAndUpdateSeqStore(): got an ldm that beginning at pos: %u, end at pos: %u, with offset: %u", + optLdm->startPosInBlock, optLdm->endPosInBlock, optLdm->offset); } -/* ZSTD_opt_maybeAddLdm(): +/* ZSTD_opt_maybeAddMatch(): * Adds a match if it's long enough, based on it's 'matchStartPosInBlock' * and 'matchEndPosInBlock', into 'matches'. Maintains the correct ordering of 'matches' */ -static void ZSTD_opt_maybeAddLdm(ZSTD_match_t* matches, U32* nbMatches, - U32 matchStartPosInBlock, U32 matchEndPosInBlock, - U32 matchOffset, U32 currPosInBlock) { - U32 posDiff = currPosInBlock - matchStartPosInBlock; +static void ZSTD_opt_maybeAddMatch(ZSTD_match_t* matches, U32* nbMatches, + ZSTD_optLdm_t* optLdm, U32 currPosInBlock) { + U32 posDiff = currPosInBlock - optLdm->startPosInBlock; /* Note: ZSTD_match_t actually contains offCode and matchLength (before subtracting MINMATCH) */ - U32 candidateMatchLength = matchEndPosInBlock - matchStartPosInBlock - posDiff; - U32 candidateOffCode = matchOffset + ZSTD_REP_MOVE; + U32 candidateMatchLength = optLdm->endPosInBlock - optLdm->startPosInBlock - posDiff; + U32 candidateOffCode = optLdm->offset + ZSTD_REP_MOVE; /* Ensure that current block position is not outside of the match */ - if (currPosInBlock < matchStartPosInBlock - || currPosInBlock >= matchEndPosInBlock + if (currPosInBlock < optLdm->startPosInBlock + || currPosInBlock >= optLdm->endPosInBlock || candidateMatchLength < MINMATCH) { return; } - DEBUGLOG(6, "ZSTD_opt_maybeAddLdm(): Adding ldm candidate match (offCode: %u matchLength %u) at block position=%u", + DEBUGLOG(6, "ZSTD_opt_maybeAddMatch(): Adding ldm candidate match (offCode: %u matchLength %u) at block position=%u", candidateOffCode, candidateMatchLength, currPosInBlock); if (*nbMatches == 0) { matches[*nbMatches].len = candidateMatchLength; @@ -892,29 +896,29 @@ static void ZSTD_opt_maybeAddLdm(ZSTD_match_t* matches, U32* nbMatches, } } -/* ZSTD_opt_processLdm(): +/* ZSTD_opt_processMatchCandidate(): * Wrapper function to update ldm seq store and call ldm functions as necessary. */ -static void ZSTD_opt_processLdm(rawSeqStore_t* ldmSeqStore, ZSTD_match_t* matches, U32* nbMatches, - U32* matchStartPosInBlock, U32* matchEndPosInBlock, U32* matchOffset, +static void ZSTD_opt_processMatchCandidate(ZSTD_optLdm_t* optLdm, ZSTD_match_t* matches, U32* nbMatches, U32 currPosInBlock, U32 remainingBytes) { - if (currPosInBlock >= *matchEndPosInBlock) { - if (currPosInBlock > *matchEndPosInBlock) { - /* The position at which ZSTD_opt_processLdm() is called is not necessarily + if (optLdm->seqStore->size == 0 && optLdm->seqStore->pos >= optLdm->seqStore->size) { + return; + } + + if (currPosInBlock >= optLdm->endPosInBlock) { + if (currPosInBlock > optLdm->endPosInBlock) { + /* The position at which ZSTD_opt_processMatchCandidate() is called is not necessarily * at the end of a match from the ldm seq store, and will often be some bytes * over beyond matchEndPosInBlock. As such, we need to correct for these "overshoots" */ - U32 posOvershoot = currPosInBlock - *matchEndPosInBlock; - ZSTD_opt_skipBytesInLdmSeqStore(ldmSeqStore, posOvershoot); + U32 posOvershoot = currPosInBlock - optLdm->endPosInBlock; + ZSTD_opt_skipRawSeqStoreBytes(optLdm->seqStore, posOvershoot); } - ZSTD_opt_getNextLdmAndUpdateSeqStore(ldmSeqStore, matchStartPosInBlock, - matchEndPosInBlock, matchOffset, - currPosInBlock, remainingBytes); + ZSTD_opt_getNextMatchAndUpdateSeqStore(optLdm, currPosInBlock, remainingBytes); } - ZSTD_opt_maybeAddLdm(matches, nbMatches, *matchStartPosInBlock, *matchEndPosInBlock, *matchOffset, currPosInBlock); + ZSTD_opt_maybeAddMatch(matches, nbMatches, optLdm, currPosInBlock); } - /*-******************************* * Optimal parser *********************************/ @@ -968,15 +972,12 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms, ZSTD_match_t* const matches = optStatePtr->matchTable; ZSTD_optimal_t lastSequence; - U32 ldmStartPosInBlock = 0; - U32 ldmEndPosInBlock = 0; - U32 ldmOffset = 0; + /* Make a copy so that ms->ldmSeqStore stays immutable during compressBlock() */ + ZSTD_optLdm_t optLdm = {&ms->ldmSeqStore, 0, 0, 0}; /* Get first match from ldm seq store if long mode is enabled */ if (ms->ldmSeqStore.size > 0 && ms->ldmSeqStore.pos < ms->ldmSeqStore.size) { - ZSTD_opt_getNextLdmAndUpdateSeqStore(&ms->ldmSeqStore, &ldmStartPosInBlock, - &ldmEndPosInBlock, &ldmOffset, - (U32)(ip-istart), (U32)(iend-ip)); + ZSTD_opt_getNextMatchAndUpdateSeqStore(&optLdm, (U32)(ip-istart), (U32)(iend-ip)); } /* init */ @@ -994,12 +995,8 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms, { U32 const litlen = (U32)(ip - anchor); U32 const ll0 = !litlen; U32 nbMatches = ZSTD_BtGetAllMatches(matches, ms, &nextToUpdate3, ip, iend, dictMode, rep, ll0, minMatch); - if (ms->ldmSeqStore.size != 0 && ms->ldmSeqStore.pos < ms->ldmSeqStore.size) { - ZSTD_opt_processLdm(&ms->ldmSeqStore, matches, - &nbMatches, &ldmStartPosInBlock, - &ldmEndPosInBlock, &ldmOffset, - (U32)(ip-istart), (U32)(iend - ip)); - } + ZSTD_opt_processMatchCandidate(&optLdm, matches, &nbMatches, + (U32)(ip-istart), (U32)(iend - ip)); if (!nbMatches) { ip++; continue; } /* initialize opt[0] */ @@ -1115,13 +1112,8 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms, U32 nbMatches = ZSTD_BtGetAllMatches(matches, ms, &nextToUpdate3, inr, iend, dictMode, opt[cur].rep, ll0, minMatch); U32 matchNb; - - if (ms->ldmSeqStore.size != 0 && ms->ldmSeqStore.pos < ms->ldmSeqStore.size) { - ZSTD_opt_processLdm(&ms->ldmSeqStore, matches, - &nbMatches, &ldmStartPosInBlock, - &ldmEndPosInBlock, &ldmOffset, - (U32)(inr-istart), (U32)(iend-inr)); - } + ZSTD_opt_processMatchCandidate(&optLdm, matches, &nbMatches, + (U32)(inr-istart), (U32)(iend-inr)); if (!nbMatches) { DEBUGLOG(7, "rPos:%u : no match found", cur); @@ -1236,12 +1228,13 @@ _shortestPath: /* cur, last_pos, best_mlen, best_off have to be set */ ZSTD_setBasePrices(optStatePtr, optLevel); } } /* while (ip < ilimit) */ - - if (ldmEndPosInBlock < srcSize) { + + if (optLdm.endPosInBlock < srcSize) { /* This can occur if after adding the final match in an ldm seq store within this block, ip reaches end of the block without calling ZSTD_opt_getNextLdmAndUpdateSeqStore() */ - ZSTD_opt_skipBytesInLdmSeqStore(&ms->ldmSeqStore, srcSize - ldmEndPosInBlock); + ZSTD_opt_skipRawSeqStoreBytes(optLdm.seqStore, srcSize - optLdm.endPosInBlock); } + /* Return the last literals size */ return (size_t)(iend - anchor); } From 0fac8e07e1d8e35288b35e349714aad83082a413 Mon Sep 17 00:00:00 2001 From: senhuang42 Date: Tue, 6 Oct 2020 13:43:13 -0400 Subject: [PATCH 55/68] Refactor usage of ms->ldmSeqStore so that it is not modified during compressBlock(), and simplify skipRawSeqStoreBytes --- lib/compress/zstd_ldm.c | 26 ++++++++++- lib/compress/zstd_opt.c | 100 ++++++++++++++++------------------------ 2 files changed, 64 insertions(+), 62 deletions(-) diff --git a/lib/compress/zstd_ldm.c b/lib/compress/zstd_ldm.c index 623245567..ae381c3e5 100644 --- a/lib/compress/zstd_ldm.c +++ b/lib/compress/zstd_ldm.c @@ -555,6 +555,26 @@ static rawSeq maybeSplitSequence(rawSeqStore_t* rawSeqStore, return sequence; } +/* ZSTD_ldm_skipRawSeqStoreBytes(): + * Moves forward in rawSeqStore by nbBytes, updating fields 'pos' and 'posInSequence'. + */ +static void ZSTD_ldm_skipRawSeqStoreBytes(rawSeqStore_t* rawSeqStore, size_t nbBytes) { + U32 currPos = rawSeqStore->posInSequence + nbBytes; + while (currPos && rawSeqStore->pos < rawSeqStore->size) { + rawSeq currSeq = rawSeqStore->seq[rawSeqStore->pos]; + if (currPos >= currSeq.litLength + currSeq.matchLength) { + currPos -= currSeq.litLength + currSeq.matchLength; + rawSeqStore->pos++; + if (currPos == 0) { + rawSeqStore->posInSequence = 0; + } + } else { + rawSeqStore->posInSequence = currPos; + break; + } + } +} + size_t ZSTD_ldm_blockCompress(rawSeqStore_t* rawSeqStore, ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], void const* src, size_t srcSize) @@ -574,7 +594,11 @@ size_t ZSTD_ldm_blockCompress(rawSeqStore_t* rawSeqStore, size_t lastLLSize; ms->ldmSeqStore = *rawSeqStore; lastLLSize = blockCompressor(ms, seqStore, rep, src, srcSize); - *rawSeqStore = ms->ldmSeqStore; + ZSTD_ldm_skipRawSeqStoreBytes(rawSeqStore, srcSize); + if (rawSeqStore->pos >= rawSeqStore->size) { + /* If we're done with rawSeqStore, invalidate the one in matchState as well */ + ms->ldmSeqStore.pos = rawSeqStore->size; + } return lastLLSize; } diff --git a/lib/compress/zstd_opt.c b/lib/compress/zstd_opt.c index 472f9e8a8..0baba0b18 100644 --- a/lib/compress/zstd_opt.c +++ b/lib/compress/zstd_opt.c @@ -768,39 +768,30 @@ FORCE_INLINE_TEMPLATE U32 ZSTD_BtGetAllMatches ( * LDM helper functions * *************************/ +/* Struct containing info needed to make decision about ldm inclusion */ typedef struct { - rawSeqStore_t* seqStore; /* Reference to struct containing external candidates */ + rawSeqStore_t seqStore; /* External match candidates store for this block */ U32 startPosInBlock; /* Start position of the current match candidate */ U32 endPosInBlock; /* End position of the current match candidate */ U32 offset; /* Offset of the match candidate */ } ZSTD_optLdm_t; -/* ZSTD_opt_skipRawSeqStoreBytes(): +/* ZSTD_optLdm_skipRawSeqStoreBytes(): * Moves forward in rawSeqStore by nbBytes, which will update the fields 'pos' and 'posInSequence'. */ -static void ZSTD_opt_skipRawSeqStoreBytes(rawSeqStore_t* ldmSeqStore, size_t nbBytes) { - while (nbBytes && ldmSeqStore->pos < ldmSeqStore->size) { - rawSeq currSeq = ldmSeqStore->seq[ldmSeqStore->pos]; - /* posInSequence necessarily must never represent a value beyond the sequence */ - assert(ldmSeqStore->posInSequence <= currSeq.matchLength + currSeq.litLength); - - if (nbBytes <= currSeq.litLength) { - ldmSeqStore->posInSequence += nbBytes; - return; +static void ZSTD_optLdm_skipRawSeqStoreBytes(rawSeqStore_t* rawSeqStore, size_t nbBytes) { + U32 currPos = rawSeqStore->posInSequence + nbBytes; + while (currPos && rawSeqStore->pos < rawSeqStore->size) { + rawSeq currSeq = rawSeqStore->seq[rawSeqStore->pos]; + if (currPos >= currSeq.litLength + currSeq.matchLength) { + currPos -= currSeq.litLength + currSeq.matchLength; + rawSeqStore->pos++; + if (currPos == 0) { + rawSeqStore->posInSequence = 0; + } } else { - ldmSeqStore->posInSequence += currSeq.litLength; - nbBytes -= currSeq.litLength; - } - - if (nbBytes < currSeq.matchLength) { - ldmSeqStore->posInSequence += nbBytes; - return; - } else { - nbBytes -= currSeq.matchLength; - /* We have moved through this entire sequence - move the read pos - forward to the next sequence, and reset posInSequence */ - ldmSeqStore->pos++; - ldmSeqStore->posInSequence = 0; + rawSeqStore->posInSequence = currPos; + break; } } } @@ -817,28 +808,28 @@ static void ZSTD_opt_getNextMatchAndUpdateSeqStore(ZSTD_optLdm_t* optLdm, U32 cu U32 matchBytesRemaining; /* Setting match end position to MAX to ensure we never use an LDM during this block */ - if (optLdm->seqStore->pos >= optLdm->seqStore->size) { + if (optLdm->seqStore.pos >= optLdm->seqStore.size || optLdm->seqStore.size == 0) { optLdm->startPosInBlock = UINT_MAX; optLdm->endPosInBlock = UINT_MAX; return; } /* Calculate appropriate bytes left in matchLength and litLength after adjusting based on ldmSeqStore->posInSequence */ - currSeq = optLdm->seqStore->seq[optLdm->seqStore->pos]; - assert(optLdm->seqStore->posInSequence <= currSeq.litLength + currSeq.matchLength); + currSeq = optLdm->seqStore.seq[optLdm->seqStore.pos]; + assert(optLdm->seqStore.posInSequence <= currSeq.litLength + currSeq.matchLength); currBlockEndPos = currPosInBlock + blockBytesRemaining; - literalsBytesRemaining = (optLdm->seqStore->posInSequence < currSeq.litLength) ? - currSeq.litLength - (U32)optLdm->seqStore->posInSequence : + literalsBytesRemaining = (optLdm->seqStore.posInSequence < currSeq.litLength) ? + currSeq.litLength - (U32)optLdm->seqStore.posInSequence : 0; matchBytesRemaining = (literalsBytesRemaining == 0) ? - currSeq.matchLength - ((U32)optLdm->seqStore->posInSequence - currSeq.litLength) : + currSeq.matchLength - ((U32)optLdm->seqStore.posInSequence - currSeq.litLength) : currSeq.matchLength; /* If there are more literal bytes than bytes remaining in block, no ldm is possible */ if (literalsBytesRemaining >= blockBytesRemaining) { optLdm->startPosInBlock = UINT_MAX; optLdm->endPosInBlock = UINT_MAX; - ZSTD_opt_skipRawSeqStoreBytes(optLdm->seqStore, blockBytesRemaining); + ZSTD_optLdm_skipRawSeqStoreBytes(&optLdm->seqStore, blockBytesRemaining); return; } @@ -851,21 +842,18 @@ static void ZSTD_opt_getNextMatchAndUpdateSeqStore(ZSTD_optLdm_t* optLdm, U32 cu if (optLdm->endPosInBlock > currBlockEndPos) { /* Match ends after the block ends, we can't use the whole match */ optLdm->endPosInBlock = currBlockEndPos; - ZSTD_opt_skipRawSeqStoreBytes(optLdm->seqStore, currBlockEndPos - currPosInBlock); + ZSTD_optLdm_skipRawSeqStoreBytes(&optLdm->seqStore, currBlockEndPos - currPosInBlock); } else { - /* If we can use the whole match point the ldmSeqStore at the next match */ - optLdm->seqStore->posInSequence = 0; - optLdm->seqStore->pos++; + /* Consume nb of bytes equal to size of sequence left */ + ZSTD_optLdm_skipRawSeqStoreBytes(&optLdm->seqStore, literalsBytesRemaining + matchBytesRemaining); } - DEBUGLOG(6, "ZSTD_opt_getNextMatchAndUpdateSeqStore(): got an ldm that beginning at pos: %u, end at pos: %u, with offset: %u", - optLdm->startPosInBlock, optLdm->endPosInBlock, optLdm->offset); } -/* ZSTD_opt_maybeAddMatch(): +/* ZSTD_optLdm_maybeAddMatch(): * Adds a match if it's long enough, based on it's 'matchStartPosInBlock' * and 'matchEndPosInBlock', into 'matches'. Maintains the correct ordering of 'matches' */ -static void ZSTD_opt_maybeAddMatch(ZSTD_match_t* matches, U32* nbMatches, +static void ZSTD_optLdm_maybeAddMatch(ZSTD_match_t* matches, U32* nbMatches, ZSTD_optLdm_t* optLdm, U32 currPosInBlock) { U32 posDiff = currPosInBlock - optLdm->startPosInBlock; /* Note: ZSTD_match_t actually contains offCode and matchLength (before subtracting MINMATCH) */ @@ -879,7 +867,7 @@ static void ZSTD_opt_maybeAddMatch(ZSTD_match_t* matches, U32* nbMatches, return; } - DEBUGLOG(6, "ZSTD_opt_maybeAddMatch(): Adding ldm candidate match (offCode: %u matchLength %u) at block position=%u", + DEBUGLOG(6, "ZSTD_optLdm_maybeAddMatch(): Adding ldm candidate match (offCode: %u matchLength %u) at block position=%u", candidateOffCode, candidateMatchLength, currPosInBlock); if (*nbMatches == 0) { matches[*nbMatches].len = candidateMatchLength; @@ -896,27 +884,27 @@ static void ZSTD_opt_maybeAddMatch(ZSTD_match_t* matches, U32* nbMatches, } } -/* ZSTD_opt_processMatchCandidate(): +/* ZSTD_optLdm_processMatchCandidate(): * Wrapper function to update ldm seq store and call ldm functions as necessary. */ -static void ZSTD_opt_processMatchCandidate(ZSTD_optLdm_t* optLdm, ZSTD_match_t* matches, U32* nbMatches, +static void ZSTD_optLdm_processMatchCandidate(ZSTD_optLdm_t* optLdm, ZSTD_match_t* matches, U32* nbMatches, U32 currPosInBlock, U32 remainingBytes) { - if (optLdm->seqStore->size == 0 && optLdm->seqStore->pos >= optLdm->seqStore->size) { + if (optLdm->seqStore.size == 0 && optLdm->seqStore.pos >= optLdm->seqStore.size) { return; } if (currPosInBlock >= optLdm->endPosInBlock) { if (currPosInBlock > optLdm->endPosInBlock) { - /* The position at which ZSTD_opt_processMatchCandidate() is called is not necessarily + /* The position at which ZSTD_optLdm_processMatchCandidate() is called is not necessarily * at the end of a match from the ldm seq store, and will often be some bytes * over beyond matchEndPosInBlock. As such, we need to correct for these "overshoots" */ U32 posOvershoot = currPosInBlock - optLdm->endPosInBlock; - ZSTD_opt_skipRawSeqStoreBytes(optLdm->seqStore, posOvershoot); + ZSTD_optLdm_skipRawSeqStoreBytes(&optLdm->seqStore, posOvershoot); } ZSTD_opt_getNextMatchAndUpdateSeqStore(optLdm, currPosInBlock, remainingBytes); } - ZSTD_opt_maybeAddMatch(matches, nbMatches, optLdm, currPosInBlock); + ZSTD_optLdm_maybeAddMatch(matches, nbMatches, optLdm, currPosInBlock); } /*-******************************* @@ -971,14 +959,10 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms, ZSTD_optimal_t* const opt = optStatePtr->priceTable; ZSTD_match_t* const matches = optStatePtr->matchTable; ZSTD_optimal_t lastSequence; - - /* Make a copy so that ms->ldmSeqStore stays immutable during compressBlock() */ - ZSTD_optLdm_t optLdm = {&ms->ldmSeqStore, 0, 0, 0}; + ZSTD_optLdm_t optLdm = {ms->ldmSeqStore, 0, 0, 0}; /* ms->ldmSeqStore itself is immutable in this function */ /* Get first match from ldm seq store if long mode is enabled */ - if (ms->ldmSeqStore.size > 0 && ms->ldmSeqStore.pos < ms->ldmSeqStore.size) { - ZSTD_opt_getNextMatchAndUpdateSeqStore(&optLdm, (U32)(ip-istart), (U32)(iend-ip)); - } + ZSTD_opt_getNextMatchAndUpdateSeqStore(&optLdm, (U32)(ip-istart), (U32)(iend-ip)); /* init */ DEBUGLOG(5, "ZSTD_compressBlock_opt_generic: current=%u, prefix=%u, nextToUpdate=%u", @@ -995,7 +979,7 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms, { U32 const litlen = (U32)(ip - anchor); U32 const ll0 = !litlen; U32 nbMatches = ZSTD_BtGetAllMatches(matches, ms, &nextToUpdate3, ip, iend, dictMode, rep, ll0, minMatch); - ZSTD_opt_processMatchCandidate(&optLdm, matches, &nbMatches, + ZSTD_optLdm_processMatchCandidate(&optLdm, matches, &nbMatches, (U32)(ip-istart), (U32)(iend - ip)); if (!nbMatches) { ip++; continue; } @@ -1112,7 +1096,7 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms, U32 nbMatches = ZSTD_BtGetAllMatches(matches, ms, &nextToUpdate3, inr, iend, dictMode, opt[cur].rep, ll0, minMatch); U32 matchNb; - ZSTD_opt_processMatchCandidate(&optLdm, matches, &nbMatches, + ZSTD_optLdm_processMatchCandidate(&optLdm, matches, &nbMatches, (U32)(inr-istart), (U32)(iend-inr)); if (!nbMatches) { @@ -1228,13 +1212,7 @@ _shortestPath: /* cur, last_pos, best_mlen, best_off have to be set */ ZSTD_setBasePrices(optStatePtr, optLevel); } } /* while (ip < ilimit) */ - - if (optLdm.endPosInBlock < srcSize) { - /* This can occur if after adding the final match in an ldm seq store within this block, - ip reaches end of the block without calling ZSTD_opt_getNextLdmAndUpdateSeqStore() */ - ZSTD_opt_skipRawSeqStoreBytes(optLdm.seqStore, srcSize - optLdm.endPosInBlock); - } - + /* Return the last literals size */ return (size_t)(iend - anchor); } From 0c515590d81efd2f6655a0f0c992ab33ff76bcdd Mon Sep 17 00:00:00 2001 From: senhuang42 Date: Tue, 6 Oct 2020 13:44:44 -0400 Subject: [PATCH 56/68] Replace offCode of largest match if ldm's offCode is superior --- lib/compress/zstd_opt.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/lib/compress/zstd_opt.c b/lib/compress/zstd_opt.c index 0baba0b18..80b8e28c9 100644 --- a/lib/compress/zstd_opt.c +++ b/lib/compress/zstd_opt.c @@ -874,8 +874,11 @@ static void ZSTD_optLdm_maybeAddMatch(ZSTD_match_t* matches, U32* nbMatches, matches[*nbMatches].off = candidateOffCode; (*nbMatches)++; } else if ((candidateMatchLength >= matches[*nbMatches-1].len) && *nbMatches < ZSTD_OPT_NUM) { - /* No need to insert the match if it's the exact same, or offCode is larger with same matchLen */ - if (candidateMatchLength == matches[*nbMatches-1].len && candidateOffCode >= matches[*nbMatches-1].off) { + if (candidateMatchLength == matches[*nbMatches-1].len) { + /* No need to insert match with same matchLength. At most, replace offCode if it is smaller. */ + if (candidateOffCode < matches[*nbMatches-1].off) { + matches[*nbMatches-1].off = candidateOffCode; + } return; } matches[*nbMatches].len = candidateMatchLength; From abce708a562b1c73baebadef469c86e590f3633a Mon Sep 17 00:00:00 2001 From: senhuang42 Date: Tue, 6 Oct 2020 16:16:41 -0400 Subject: [PATCH 57/68] Move posInSequence correction to correct location --- doc/zstd_manual.html | 105 +++++++++++++++++++++++++--------------- lib/compress/zstd_ldm.c | 12 +++-- lib/compress/zstd_opt.c | 18 ++++--- 3 files changed, 83 insertions(+), 52 deletions(-) diff --git a/doc/zstd_manual.html b/doc/zstd_manual.html index fe58f78cb..d894befb6 100644 --- a/doc/zstd_manual.html +++ b/doc/zstd_manual.html @@ -1,10 +1,10 @@ -zstd 1.4.5 Manual +zstd 1.4.6 Manual -

zstd 1.4.5 Manual

+

zstd 1.4.6 Manual

Contents

    @@ -24,19 +24,20 @@
  1. experimental API (static linking only)
  2. Frame size functions
  3. Memory management
    4. -
  4. Advanced compression functions
    5. -
  5. Advanced decompression functions
    6. -
  6. Advanced streaming functions
    7. -
  7. ! ZSTD_initCStream_usingDict() :
    8. -
  8. ! ZSTD_initCStream_advanced() :
    9. -
  9. ! ZSTD_initCStream_usingCDict() :
    10. -
  10. ! ZSTD_initCStream_usingCDict_advanced() :
    11. -
  11. This function is deprecated, and is equivalent to:
    12. +
  12. This API is temporary and is expected to change or disappear in the future!
    13. +
  13. Advanced compression functions
    14. +
  14. Advanced decompression functions
    15. +
  15. Advanced streaming functions
    16. +
  16. ! ZSTD_initCStream_usingDict() :
    17. +
  17. ! ZSTD_initCStream_advanced() :
    18. +
  18. ! ZSTD_initCStream_usingCDict() :
    19. +
  19. ! ZSTD_initCStream_usingCDict_advanced() :
  20. This function is deprecated, and is equivalent to:
    21. -
  21. Buffer-less and synchronous inner streaming functions
    22. -
  22. Buffer-less streaming compression (synchronous mode)
    23. -
  23. Buffer-less streaming decompression (synchronous mode)
    24. -
  24. Block level API
    25. +
  25. This function is deprecated, and is equivalent to:
    26. +
  26. Buffer-less and synchronous inner streaming functions
    27. +
  27. Buffer-less streaming compression (synchronous mode)
    28. +
  28. Buffer-less streaming decompression (synchronous mode)
    29. +
  29. Block level API

Introduction

@@ -72,8 +73,14 @@
 
 
Version

 
-
unsigned ZSTD_versionNumber(void);   /**< to check runtime library version */
-


+
unsigned ZSTD_versionNumber(void);
+
  Return runtime library version, the value is (MAJOR*100*100 + MINOR*100 + RELEASE). 
+


+
+
const char* ZSTD_versionString(void);
+
  Return runtime library version, like "1.4.5". Requires v1.3.0+. 
+


+
 
Simple API

 
 
size_t ZSTD_compress( void* dst, size_t dstCapacity,
@@ -308,16 +315,20 @@ size_t     ZSTD_freeDCtx(ZSTD_DCtx* dctx);
     ZSTD_c_dictIDFlag=202,   /* When applicable, dictionary's ID is written into frame header (default:1) */
 
     /* multi-threading parameters */
-    /* These parameters are only useful if multi-threading is enabled (compiled with build macro ZSTD_MULTITHREAD).
-     * They return an error otherwise. */
+    /* These parameters are only active if multi-threading is enabled (compiled with build macro ZSTD_MULTITHREAD).
+     * Otherwise, trying to set any other value than default (0) will be a no-op and return an error.
+     * In a situation where it's unknown if the linked library supports multi-threading or not,
+     * setting ZSTD_c_nbWorkers to any value >= 1 and consulting the return value provides a quick way to check this property.
+     */
     ZSTD_c_nbWorkers=400,    /* Select how many threads will be spawned to compress in parallel.
-                              * When nbWorkers >= 1, triggers asynchronous mode when used with ZSTD_compressStream*() :
+                              * When nbWorkers >= 1, triggers asynchronous mode when invoking ZSTD_compressStream*() :
                               * ZSTD_compressStream*() consumes input and flush output if possible, but immediately gives back control to caller,
-                              * while compression work is performed in parallel, within worker threads.
+                              * while compression is performed in parallel, within worker thread(s).
                               * (note : a strong exception to this rule is when first invocation of ZSTD_compressStream2() sets ZSTD_e_end :
                               *  in which case, ZSTD_compressStream2() delegates to ZSTD_compress2(), which is always a blocking call).
                               * More workers improve speed, but also increase memory usage.
-                              * Default value is `0`, aka "single-threaded mode" : no worker is spawned, compression is performed inside Caller's thread, all invocations are blocking */
+                              * Default value is `0`, aka "single-threaded mode" : no worker is spawned,
+                              * compression is performed inside Caller's thread, and all invocations are blocking */
     ZSTD_c_jobSize=401,      /* Size of a compression job. This value is enforced only when nbWorkers >= 1.
                               * Each compression job is completed in parallel, so this value can indirectly impact the nb of active threads.
                               * 0 means default, which is dynamically determined based on compression parameters.
@@ -346,6 +357,7 @@ size_t     ZSTD_freeDCtx(ZSTD_DCtx* dctx);
      * ZSTD_c_literalCompressionMode
      * ZSTD_c_targetCBlockSize
      * ZSTD_c_srcSizeHint
+     * ZSTD_c_enableDedicatedDictSearch
      * Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them.
      * note : never ever use experimentalParam? names directly;
      *        also, the enums values themselves are unstable and can still change.
@@ -356,7 +368,8 @@ size_t     ZSTD_freeDCtx(ZSTD_DCtx* dctx);
      ZSTD_c_experimentalParam4=1001,
      ZSTD_c_experimentalParam5=1002,
      ZSTD_c_experimentalParam6=1003,
-     ZSTD_c_experimentalParam7=1004
+     ZSTD_c_experimentalParam7=1004,
+     ZSTD_c_experimentalParam8=1005
 } ZSTD_cParameter;
 


 
typedef struct {
@@ -456,11 +469,13 @@ size_t     ZSTD_freeDCtx(ZSTD_DCtx* dctx);
      * At the time of this writing, they include :
      * ZSTD_d_format
      * ZSTD_d_stableOutBuffer
+     * ZSTD_d_forceIgnoreChecksum
      * Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them.
      * note : never ever use experimentalParam? names directly
      */
      ZSTD_d_experimentalParam1=1000,
-     ZSTD_d_experimentalParam2=1001
+     ZSTD_d_experimentalParam2=1001,
+     ZSTD_d_experimentalParam3=1002
 
 } ZSTD_dParameter;
 


@@ -591,8 +606,9 @@ size_t ZSTD_freeCStream(ZSTD_CStream* zcs);
   - Compression parameters cannot be changed once compression is started (save a list of exceptions in multi-threading mode)
   - output->pos must be <= dstCapacity, input->pos must be <= srcSize
   - output->pos and input->pos will be updated. They are guaranteed to remain below their respective limit.
+  - endOp must be a valid directive
   - When nbWorkers==0 (default), function is blocking : it completes its job before returning to caller.
-  - When nbWorkers>=1, function is non-blocking : it just acquires a copy of input, and distributes jobs to internal worker threads, flush whatever is available,
+  - When nbWorkers>=1, function is non-blocking : it copies a portion of input, distributes jobs to internal worker threads, flush to output whatever is available,
                                                   and then immediately returns, just indicating that there is some data remaining to be flushed.
                                                   The function nonetheless guarantees forward progress : it will return only after it reads or write at least 1+ byte.
   - Exception : if the first call requests a ZSTD_e_end directive and provides enough dstCapacity, the function delegates to ZSTD_compress2() which is always blocking.
@@ -951,6 +967,12 @@ size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
                                  * Decoder cannot recognise automatically this format, requiring this instruction. */
 } ZSTD_format_e;
 


+
typedef enum {
+    /* Note: this enum controls ZSTD_d_forceIgnoreChecksum */
+    ZSTD_d_validateChecksum = 0,
+    ZSTD_d_ignoreChecksum = 1
+} ZSTD_forceIgnoreChecksum_e;
+


 
typedef enum {
     /* Note: this enum and the behavior it controls are effectively internal
      * implementation details of the compressor. They are expected to continue
@@ -1148,7 +1170,10 @@ static ZSTD_customMem const ZSTD_defaultCMem = { NULL, NULL, NULL };  /**< t
  
 


 
-
Advanced compression functions

+
This API is temporary and is expected to change or disappear in the future!
 
+

+
+
Advanced compression functions

 
 
ZSTD_CDict* ZSTD_createCDict_byReference(const void* dictBuffer, size_t dictSize, int compressionLevel);
 
  Create a digested dictionary for compression
@@ -1264,8 +1289,10 @@ size_t ZSTD_freeCCtxParams(ZSTD_CCtx_params* params);
 
size_t ZSTD_CCtxParams_setParameter(ZSTD_CCtx_params* params, ZSTD_cParameter param, int value);
 
  Similar to ZSTD_CCtx_setParameter.
   Set one compression parameter, selected by enum ZSTD_cParameter.
-  Parameters must be applied to a ZSTD_CCtx using ZSTD_CCtx_setParametersUsingCCtxParams().
- @result : 0, or an error code (which can be tested with ZSTD_isError()).
+  Parameters must be applied to a ZSTD_CCtx using
+  ZSTD_CCtx_setParametersUsingCCtxParams().
+ @result : a code representing success or failure (which can be tested with
+           ZSTD_isError()).
  
 


 
@@ -1298,7 +1325,7 @@ size_t ZSTD_freeCCtxParams(ZSTD_CCtx_params* params);
  
 


 
-
Advanced decompression functions

+
Advanced decompression functions

 
 
unsigned ZSTD_isFrame(const void* buffer, size_t size);
 
  Tells if the content of `buffer` starts with a valid Frame Identifier.
@@ -1360,7 +1387,7 @@ size_t ZSTD_freeCCtxParams(ZSTD_CCtx_params* params);
  
 


 
-
Advanced streaming functions
  Warning : most of these functions are now redundant with the Advanced API.
+
Advanced streaming functions
  Warning : most of these functions are now redundant with the Advanced API.
   Once Advanced API reaches "stable" status,
   redundant functions will be deprecated, and then at some point removed.
 

@@ -1382,7 +1409,7 @@ ZSTD_initCStream_srcSize(ZSTD_CStream* zcs,
                          int compressionLevel,
                          unsigned long long pledgedSrcSize);
 


-
! ZSTD_initCStream_usingDict() :
 This function is deprecated, and is equivalent to:
+
! ZSTD_initCStream_usingDict() :
 This function is deprecated, and is equivalent to:
      ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
      ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel);
      ZSTD_CCtx_loadDictionary(zcs, dict, dictSize);
@@ -1395,7 +1422,7 @@ ZSTD_initCStream_srcSize(ZSTD_CStream* zcs,
  
 

 
-
! ZSTD_initCStream_advanced() :
 This function is deprecated, and is approximately equivalent to:
+
! ZSTD_initCStream_advanced() :
 This function is deprecated, and is approximately equivalent to:
      ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
      // Pseudocode: Set each zstd parameter and leave the rest as-is.
      for ((param, value) : params) {
@@ -1411,7 +1438,7 @@ ZSTD_initCStream_srcSize(ZSTD_CStream* zcs,
  
 

 
-
! ZSTD_initCStream_usingCDict() :
 This function is deprecated, and equivalent to:
+
! ZSTD_initCStream_usingCDict() :
 This function is deprecated, and equivalent to:
      ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
      ZSTD_CCtx_refCDict(zcs, cdict);
 
@@ -1420,7 +1447,7 @@ ZSTD_initCStream_srcSize(ZSTD_CStream* zcs,
  
 

 
-
! ZSTD_initCStream_usingCDict_advanced() :
   This function is DEPRECATED, and is approximately equivalent to:
+
! ZSTD_initCStream_usingCDict_advanced() :
   This function is DEPRECATED, and is approximately equivalent to:
      ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
      // Pseudocode: Set each zstd frame parameter and leave the rest as-is.
      for ((fParam, value) : fParams) {
@@ -1488,7 +1515,7 @@ ZSTD_initCStream_srcSize(ZSTD_CStream* zcs,
  */
 size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize);
 


-
This function is deprecated, and is equivalent to:
+
This function is deprecated, and is equivalent to:
      ZSTD_DCtx_reset(zds, ZSTD_reset_session_only);
      ZSTD_DCtx_refDDict(zds, ddict);
 
@@ -1497,7 +1524,7 @@ size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t di
  
 

 
-
This function is deprecated, and is equivalent to:
+
This function is deprecated, and is equivalent to:
      ZSTD_DCtx_reset(zds, ZSTD_reset_session_only);
 
  re-use decompression parameters from previous init; saves dictionary loading
@@ -1505,14 +1532,14 @@ size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t di
  
 

 
-
Buffer-less and synchronous inner streaming functions
+
Buffer-less and synchronous inner streaming functions
   This is an advanced API, giving full control over buffer management, for users which need direct control over memory.
   But it's also a complex one, with several restrictions, documented below.
   Prefer normal streaming API for an easier experience.
  
 

 
-
Buffer-less streaming compression (synchronous mode)
+
Buffer-less streaming compression (synchronous mode)
   A ZSTD_CCtx object is required to track streaming operations.
   Use ZSTD_createCCtx() / ZSTD_freeCCtx() to manage resource.
   ZSTD_CCtx object can be re-used multiple times within successive compression operations.
@@ -1548,7 +1575,7 @@ size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict);
 size_t ZSTD_compressBegin_usingCDict_advanced(ZSTD_CCtx* const cctx, const ZSTD_CDict* const cdict, ZSTD_frameParameters const fParams, unsigned long long const pledgedSrcSize);   /* compression parameters are already set within cdict. pledgedSrcSize must be correct. If srcSize is not known, use macro ZSTD_CONTENTSIZE_UNKNOWN */
 size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize); /**<  note: if pledgedSrcSize is not known, use ZSTD_CONTENTSIZE_UNKNOWN */
 


-
Buffer-less streaming decompression (synchronous mode)
+
Buffer-less streaming decompression (synchronous mode)
   A ZSTD_DCtx object is required to track streaming operations.
   Use ZSTD_createDCtx() / ZSTD_freeDCtx() to manage it.
   A ZSTD_DCtx object can be re-used multiple times.
@@ -1644,7 +1671,7 @@ size_t ZSTD_decodingBufferSize_min(unsigned long long windowSize, unsigned long
 
 
typedef enum { ZSTDnit_frameHeader, ZSTDnit_blockHeader, ZSTDnit_block, ZSTDnit_lastBlock, ZSTDnit_checksum, ZSTDnit_skippableFrame } ZSTD_nextInputType_e;
 


-
Block level API

+
Block level API

 
 
    Frame metadata cost is typically ~12 bytes, which can be non-negligible for very small blocks (< 100 bytes).
     But users will have to take in charge needed metadata to regenerate data, such as compressed and content sizes.
diff --git a/lib/compress/zstd_ldm.c b/lib/compress/zstd_ldm.c
index ae381c3e5..4bfaa5414 100644
--- a/lib/compress/zstd_ldm.c
+++ b/lib/compress/zstd_ldm.c
@@ -559,20 +559,22 @@ static rawSeq maybeSplitSequence(rawSeqStore_t* rawSeqStore,
  * Moves forward in rawSeqStore by nbBytes, updating fields 'pos' and 'posInSequence'.
  */
 static void ZSTD_ldm_skipRawSeqStoreBytes(rawSeqStore_t* rawSeqStore, size_t nbBytes) {
-    U32 currPos = rawSeqStore->posInSequence + nbBytes;
+    U32 currPos = (U32)(rawSeqStore->posInSequence + nbBytes);
     while (currPos && rawSeqStore->pos < rawSeqStore->size) {
         rawSeq currSeq = rawSeqStore->seq[rawSeqStore->pos];
         if (currPos >= currSeq.litLength + currSeq.matchLength) {
             currPos -= currSeq.litLength + currSeq.matchLength;
             rawSeqStore->pos++;
-            if (currPos == 0) {
-                rawSeqStore->posInSequence = 0;
-            }
         } else {
             rawSeqStore->posInSequence = currPos;
             break;
         }
     }
+    if (currPos == 0 || rawSeqStore->pos == rawSeqStore->size) {
+        rawSeqStore->posInSequence = 0;
+    }
+    assert(rawSeqStore->posInSequence <=
+           rawSeqStore->seq[rawSeqStore->pos].litLength + rawSeqStore->seq[rawSeqStore->pos].matchLength);
 }
 
 size_t ZSTD_ldm_blockCompress(rawSeqStore_t* rawSeqStore,
@@ -597,7 +599,7 @@ size_t ZSTD_ldm_blockCompress(rawSeqStore_t* rawSeqStore,
         ZSTD_ldm_skipRawSeqStoreBytes(rawSeqStore, srcSize);
         if (rawSeqStore->pos >= rawSeqStore->size) {
             /* If we're done with rawSeqStore, invalidate the one in matchState as well */
-            ms->ldmSeqStore.pos = rawSeqStore->size;
+            ms->ldmSeqStore.size = 0;
         }
         return lastLLSize;
     }
diff --git a/lib/compress/zstd_opt.c b/lib/compress/zstd_opt.c
index 80b8e28c9..30e07f889 100644
--- a/lib/compress/zstd_opt.c
+++ b/lib/compress/zstd_opt.c
@@ -780,20 +780,22 @@ typedef struct {
  * Moves forward in rawSeqStore by nbBytes, which will update the fields 'pos' and 'posInSequence'.
  */
 static void ZSTD_optLdm_skipRawSeqStoreBytes(rawSeqStore_t* rawSeqStore, size_t nbBytes) {
-    U32 currPos = rawSeqStore->posInSequence + nbBytes;
+    U32 currPos = (U32)(rawSeqStore->posInSequence + nbBytes);
     while (currPos && rawSeqStore->pos < rawSeqStore->size) {
         rawSeq currSeq = rawSeqStore->seq[rawSeqStore->pos];
         if (currPos >= currSeq.litLength + currSeq.matchLength) {
             currPos -= currSeq.litLength + currSeq.matchLength;
             rawSeqStore->pos++;
-            if (currPos == 0) {
-                rawSeqStore->posInSequence = 0;
-            }
         } else {
             rawSeqStore->posInSequence = currPos;
             break;
         }
     }
+    if (currPos == 0 || rawSeqStore->pos == rawSeqStore->size) {
+        rawSeqStore->posInSequence = 0;
+    }
+    assert(rawSeqStore->posInSequence <=
+           rawSeqStore->seq[rawSeqStore->pos].litLength + rawSeqStore->seq[rawSeqStore->pos].matchLength);
 }
 
 /* ZSTD_opt_getNextMatchAndUpdateSeqStore():
@@ -854,7 +856,7 @@ static void ZSTD_opt_getNextMatchAndUpdateSeqStore(ZSTD_optLdm_t* optLdm, U32 cu
  * and 'matchEndPosInBlock', into 'matches'. Maintains the correct ordering of 'matches'
  */
 static void ZSTD_optLdm_maybeAddMatch(ZSTD_match_t* matches, U32* nbMatches,
-                                   ZSTD_optLdm_t* optLdm, U32 currPosInBlock) {
+                                      ZSTD_optLdm_t* optLdm, U32 currPosInBlock) {
     U32 posDiff = currPosInBlock - optLdm->startPosInBlock;
     /* Note: ZSTD_match_t actually contains offCode and matchLength (before subtracting MINMATCH) */
     U32 candidateMatchLength = optLdm->endPosInBlock - optLdm->startPosInBlock - posDiff;
@@ -891,7 +893,7 @@ static void ZSTD_optLdm_maybeAddMatch(ZSTD_match_t* matches, U32* nbMatches,
  * Wrapper function to update ldm seq store and call ldm functions as necessary.
  */
 static void ZSTD_optLdm_processMatchCandidate(ZSTD_optLdm_t* optLdm, ZSTD_match_t* matches, U32* nbMatches,
-                                U32 currPosInBlock, U32 remainingBytes) {
+                                              U32 currPosInBlock, U32 remainingBytes) {
     if (optLdm->seqStore.size == 0 && optLdm->seqStore.pos >= optLdm->seqStore.size) {
         return;
     }
@@ -983,7 +985,7 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms,
             U32 const ll0 = !litlen;
             U32 nbMatches = ZSTD_BtGetAllMatches(matches, ms, &nextToUpdate3, ip, iend, dictMode, rep, ll0, minMatch);
             ZSTD_optLdm_processMatchCandidate(&optLdm, matches, &nbMatches,
-                                           (U32)(ip-istart), (U32)(iend - ip));
+                                              (U32)(ip-istart), (U32)(iend - ip));
             if (!nbMatches) { ip++; continue; }
 
             /* initialize opt[0] */
@@ -1100,7 +1102,7 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms,
                 U32 matchNb;
 
                 ZSTD_optLdm_processMatchCandidate(&optLdm, matches, &nbMatches,
-                                               (U32)(inr-istart), (U32)(iend-inr));
+                                                  (U32)(inr-istart), (U32)(iend-inr));
 
                 if (!nbMatches) {
                     DEBUGLOG(7, "rPos:%u : no match found", cur);

From a6165c1b2849ff7a5f79688225a5ff2f7962b218 Mon Sep 17 00:00:00 2001
From: senhuang42 
Date: Tue, 6 Oct 2020 19:43:48 -0400
Subject: [PATCH 58/68] Change matchState_t::ldmSeqStore to pointer

---
 doc/zstd_manual.html                  | 105 ++++++++++----------------
 lib/compress/zstd_compress.c          |   1 +
 lib/compress/zstd_compress_internal.h |   2 +-
 lib/compress/zstd_ldm.c               |  10 +--
 lib/compress/zstd_opt.c               |  27 ++++---
 5 files changed, 58 insertions(+), 87 deletions(-)

diff --git a/doc/zstd_manual.html b/doc/zstd_manual.html
index d894befb6..fe58f78cb 100644
--- a/doc/zstd_manual.html
+++ b/doc/zstd_manual.html
@@ -1,10 +1,10 @@
 
 
 
-zstd 1.4.6 Manual
+zstd 1.4.5 Manual
 
 
-
zstd 1.4.6 Manual

+
zstd 1.4.5 Manual

 

 
Contents

 
    
@@ -24,20 +24,19 @@
 
  1. experimental API (static linking only)
    2. 
 
  2. Frame size functions
    3. 
 
  3. Memory management
    4. 
-
  4. This API is temporary and is expected to change or disappear in the future!
    5. 
-
  5. Advanced compression functions
    6. 
-
  6. Advanced decompression functions
    7. 
-
  7. Advanced streaming functions
    8. 
-
  8. ! ZSTD_initCStream_usingDict() :
    9. 
-
  9. ! ZSTD_initCStream_advanced() :
    10. 
-
  10. ! ZSTD_initCStream_usingCDict() :
    11. 
-
  11. ! ZSTD_initCStream_usingCDict_advanced() :
    12. 
+
  12. Advanced compression functions
    13. 
+
  13. Advanced decompression functions
    14. 
+
  14. Advanced streaming functions
    15. 
+
  15. ! ZSTD_initCStream_usingDict() :
    16. 
+
  16. ! ZSTD_initCStream_advanced() :
    17. 
+
  17. ! ZSTD_initCStream_usingCDict() :
    18. 
+
  18. ! ZSTD_initCStream_usingCDict_advanced() :
    19. 
+
  19. This function is deprecated, and is equivalent to:
    20. 
 
  20. This function is deprecated, and is equivalent to:
    21. 
-
  21. This function is deprecated, and is equivalent to:
    22. 
-
  22. Buffer-less and synchronous inner streaming functions
    23. 
-
  23. Buffer-less streaming compression (synchronous mode)
    24. 
-
  24. Buffer-less streaming decompression (synchronous mode)
    25. 
-
  25. Block level API
    26. 
+
  26. Buffer-less and synchronous inner streaming functions
    27. 
+
  27. Buffer-less streaming compression (synchronous mode)
    28. 
+
  28. Buffer-less streaming decompression (synchronous mode)
    29. 
+
  29. Block level API
    30. 
 

 

 
Introduction
@@ -73,14 +72,8 @@
 
 
Version

 
-
unsigned ZSTD_versionNumber(void);
-
  Return runtime library version, the value is (MAJOR*100*100 + MINOR*100 + RELEASE). 
-


-
-
const char* ZSTD_versionString(void);
-
  Return runtime library version, like "1.4.5". Requires v1.3.0+. 
-


-
+
unsigned ZSTD_versionNumber(void);   /**< to check runtime library version */
+


 
Simple API

 
 
size_t ZSTD_compress( void* dst, size_t dstCapacity,
@@ -315,20 +308,16 @@ size_t     ZSTD_freeDCtx(ZSTD_DCtx* dctx);
     ZSTD_c_dictIDFlag=202,   /* When applicable, dictionary's ID is written into frame header (default:1) */
 
     /* multi-threading parameters */
-    /* These parameters are only active if multi-threading is enabled (compiled with build macro ZSTD_MULTITHREAD).
-     * Otherwise, trying to set any other value than default (0) will be a no-op and return an error.
-     * In a situation where it's unknown if the linked library supports multi-threading or not,
-     * setting ZSTD_c_nbWorkers to any value >= 1 and consulting the return value provides a quick way to check this property.
-     */
+    /* These parameters are only useful if multi-threading is enabled (compiled with build macro ZSTD_MULTITHREAD).
+     * They return an error otherwise. */
     ZSTD_c_nbWorkers=400,    /* Select how many threads will be spawned to compress in parallel.
-                              * When nbWorkers >= 1, triggers asynchronous mode when invoking ZSTD_compressStream*() :
+                              * When nbWorkers >= 1, triggers asynchronous mode when used with ZSTD_compressStream*() :
                               * ZSTD_compressStream*() consumes input and flush output if possible, but immediately gives back control to caller,
-                              * while compression is performed in parallel, within worker thread(s).
+                              * while compression work is performed in parallel, within worker threads.
                               * (note : a strong exception to this rule is when first invocation of ZSTD_compressStream2() sets ZSTD_e_end :
                               *  in which case, ZSTD_compressStream2() delegates to ZSTD_compress2(), which is always a blocking call).
                               * More workers improve speed, but also increase memory usage.
-                              * Default value is `0`, aka "single-threaded mode" : no worker is spawned,
-                              * compression is performed inside Caller's thread, and all invocations are blocking */
+                              * Default value is `0`, aka "single-threaded mode" : no worker is spawned, compression is performed inside Caller's thread, all invocations are blocking */
     ZSTD_c_jobSize=401,      /* Size of a compression job. This value is enforced only when nbWorkers >= 1.
                               * Each compression job is completed in parallel, so this value can indirectly impact the nb of active threads.
                               * 0 means default, which is dynamically determined based on compression parameters.
@@ -357,7 +346,6 @@ size_t     ZSTD_freeDCtx(ZSTD_DCtx* dctx);
      * ZSTD_c_literalCompressionMode
      * ZSTD_c_targetCBlockSize
      * ZSTD_c_srcSizeHint
-     * ZSTD_c_enableDedicatedDictSearch
      * Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them.
      * note : never ever use experimentalParam? names directly;
      *        also, the enums values themselves are unstable and can still change.
@@ -368,8 +356,7 @@ size_t     ZSTD_freeDCtx(ZSTD_DCtx* dctx);
      ZSTD_c_experimentalParam4=1001,
      ZSTD_c_experimentalParam5=1002,
      ZSTD_c_experimentalParam6=1003,
-     ZSTD_c_experimentalParam7=1004,
-     ZSTD_c_experimentalParam8=1005
+     ZSTD_c_experimentalParam7=1004
 } ZSTD_cParameter;
 


 
typedef struct {
@@ -469,13 +456,11 @@ size_t     ZSTD_freeDCtx(ZSTD_DCtx* dctx);
      * At the time of this writing, they include :
      * ZSTD_d_format
      * ZSTD_d_stableOutBuffer
-     * ZSTD_d_forceIgnoreChecksum
      * Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them.
      * note : never ever use experimentalParam? names directly
      */
      ZSTD_d_experimentalParam1=1000,
-     ZSTD_d_experimentalParam2=1001,
-     ZSTD_d_experimentalParam3=1002
+     ZSTD_d_experimentalParam2=1001
 
 } ZSTD_dParameter;
 


@@ -606,9 +591,8 @@ size_t ZSTD_freeCStream(ZSTD_CStream* zcs);
   - Compression parameters cannot be changed once compression is started (save a list of exceptions in multi-threading mode)
   - output->pos must be <= dstCapacity, input->pos must be <= srcSize
   - output->pos and input->pos will be updated. They are guaranteed to remain below their respective limit.
-  - endOp must be a valid directive
   - When nbWorkers==0 (default), function is blocking : it completes its job before returning to caller.
-  - When nbWorkers>=1, function is non-blocking : it copies a portion of input, distributes jobs to internal worker threads, flush to output whatever is available,
+  - When nbWorkers>=1, function is non-blocking : it just acquires a copy of input, and distributes jobs to internal worker threads, flush whatever is available,
                                                   and then immediately returns, just indicating that there is some data remaining to be flushed.
                                                   The function nonetheless guarantees forward progress : it will return only after it reads or write at least 1+ byte.
   - Exception : if the first call requests a ZSTD_e_end directive and provides enough dstCapacity, the function delegates to ZSTD_compress2() which is always blocking.
@@ -967,12 +951,6 @@ size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
                                  * Decoder cannot recognise automatically this format, requiring this instruction. */
 } ZSTD_format_e;
 


-
typedef enum {
-    /* Note: this enum controls ZSTD_d_forceIgnoreChecksum */
-    ZSTD_d_validateChecksum = 0,
-    ZSTD_d_ignoreChecksum = 1
-} ZSTD_forceIgnoreChecksum_e;
-


 
typedef enum {
     /* Note: this enum and the behavior it controls are effectively internal
      * implementation details of the compressor. They are expected to continue
@@ -1170,10 +1148,7 @@ static ZSTD_customMem const ZSTD_defaultCMem = { NULL, NULL, NULL };  /**< t
  
 


 
-
This API is temporary and is expected to change or disappear in the future!
 
-

-
-
Advanced compression functions

+
Advanced compression functions

 
 
ZSTD_CDict* ZSTD_createCDict_byReference(const void* dictBuffer, size_t dictSize, int compressionLevel);
 
  Create a digested dictionary for compression
@@ -1289,10 +1264,8 @@ size_t ZSTD_freeCCtxParams(ZSTD_CCtx_params* params);
 
size_t ZSTD_CCtxParams_setParameter(ZSTD_CCtx_params* params, ZSTD_cParameter param, int value);
 
  Similar to ZSTD_CCtx_setParameter.
   Set one compression parameter, selected by enum ZSTD_cParameter.
-  Parameters must be applied to a ZSTD_CCtx using
-  ZSTD_CCtx_setParametersUsingCCtxParams().
- @result : a code representing success or failure (which can be tested with
-           ZSTD_isError()).
+  Parameters must be applied to a ZSTD_CCtx using ZSTD_CCtx_setParametersUsingCCtxParams().
+ @result : 0, or an error code (which can be tested with ZSTD_isError()).
  
 


 
@@ -1325,7 +1298,7 @@ size_t ZSTD_freeCCtxParams(ZSTD_CCtx_params* params);
  
 


 
-
Advanced decompression functions

+
Advanced decompression functions

 
 
unsigned ZSTD_isFrame(const void* buffer, size_t size);
 
  Tells if the content of `buffer` starts with a valid Frame Identifier.
@@ -1387,7 +1360,7 @@ size_t ZSTD_freeCCtxParams(ZSTD_CCtx_params* params);
  
 


 
-
Advanced streaming functions
  Warning : most of these functions are now redundant with the Advanced API.
+
Advanced streaming functions
  Warning : most of these functions are now redundant with the Advanced API.
   Once Advanced API reaches "stable" status,
   redundant functions will be deprecated, and then at some point removed.
 

@@ -1409,7 +1382,7 @@ ZSTD_initCStream_srcSize(ZSTD_CStream* zcs,
                          int compressionLevel,
                          unsigned long long pledgedSrcSize);
 


-
! ZSTD_initCStream_usingDict() :
 This function is deprecated, and is equivalent to:
+
! ZSTD_initCStream_usingDict() :
 This function is deprecated, and is equivalent to:
      ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
      ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel);
      ZSTD_CCtx_loadDictionary(zcs, dict, dictSize);
@@ -1422,7 +1395,7 @@ ZSTD_initCStream_srcSize(ZSTD_CStream* zcs,
  
 

 
-
! ZSTD_initCStream_advanced() :
 This function is deprecated, and is approximately equivalent to:
+
! ZSTD_initCStream_advanced() :
 This function is deprecated, and is approximately equivalent to:
      ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
      // Pseudocode: Set each zstd parameter and leave the rest as-is.
      for ((param, value) : params) {
@@ -1438,7 +1411,7 @@ ZSTD_initCStream_srcSize(ZSTD_CStream* zcs,
  
 

 
-
! ZSTD_initCStream_usingCDict() :
 This function is deprecated, and equivalent to:
+
! ZSTD_initCStream_usingCDict() :
 This function is deprecated, and equivalent to:
      ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
      ZSTD_CCtx_refCDict(zcs, cdict);
 
@@ -1447,7 +1420,7 @@ ZSTD_initCStream_srcSize(ZSTD_CStream* zcs,
  
 

 
-
! ZSTD_initCStream_usingCDict_advanced() :
   This function is DEPRECATED, and is approximately equivalent to:
+
! ZSTD_initCStream_usingCDict_advanced() :
   This function is DEPRECATED, and is approximately equivalent to:
      ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
      // Pseudocode: Set each zstd frame parameter and leave the rest as-is.
      for ((fParam, value) : fParams) {
@@ -1515,7 +1488,7 @@ ZSTD_initCStream_srcSize(ZSTD_CStream* zcs,
  */
 size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize);
 


-
This function is deprecated, and is equivalent to:
+
This function is deprecated, and is equivalent to:
      ZSTD_DCtx_reset(zds, ZSTD_reset_session_only);
      ZSTD_DCtx_refDDict(zds, ddict);
 
@@ -1524,7 +1497,7 @@ size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t di
  
 

 
-
This function is deprecated, and is equivalent to:
+
This function is deprecated, and is equivalent to:
      ZSTD_DCtx_reset(zds, ZSTD_reset_session_only);
 
  re-use decompression parameters from previous init; saves dictionary loading
@@ -1532,14 +1505,14 @@ size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t di
  
 

 
-
Buffer-less and synchronous inner streaming functions
+
Buffer-less and synchronous inner streaming functions
   This is an advanced API, giving full control over buffer management, for users which need direct control over memory.
   But it's also a complex one, with several restrictions, documented below.
   Prefer normal streaming API for an easier experience.
  
 

 
-
Buffer-less streaming compression (synchronous mode)
+
Buffer-less streaming compression (synchronous mode)
   A ZSTD_CCtx object is required to track streaming operations.
   Use ZSTD_createCCtx() / ZSTD_freeCCtx() to manage resource.
   ZSTD_CCtx object can be re-used multiple times within successive compression operations.
@@ -1575,7 +1548,7 @@ size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict);
 size_t ZSTD_compressBegin_usingCDict_advanced(ZSTD_CCtx* const cctx, const ZSTD_CDict* const cdict, ZSTD_frameParameters const fParams, unsigned long long const pledgedSrcSize);   /* compression parameters are already set within cdict. pledgedSrcSize must be correct. If srcSize is not known, use macro ZSTD_CONTENTSIZE_UNKNOWN */
 size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize); /**<  note: if pledgedSrcSize is not known, use ZSTD_CONTENTSIZE_UNKNOWN */
 


-
Buffer-less streaming decompression (synchronous mode)
+
Buffer-less streaming decompression (synchronous mode)
   A ZSTD_DCtx object is required to track streaming operations.
   Use ZSTD_createDCtx() / ZSTD_freeDCtx() to manage it.
   A ZSTD_DCtx object can be re-used multiple times.
@@ -1671,7 +1644,7 @@ size_t ZSTD_decodingBufferSize_min(unsigned long long windowSize, unsigned long
 
 
typedef enum { ZSTDnit_frameHeader, ZSTDnit_blockHeader, ZSTDnit_block, ZSTDnit_lastBlock, ZSTDnit_checksum, ZSTDnit_skippableFrame } ZSTD_nextInputType_e;
 


-
Block level API

+
Block level API

 
 
    Frame metadata cost is typically ~12 bytes, which can be non-negligible for very small blocks (< 100 bytes).
     But users will have to take in charge needed metadata to regenerate data, such as compressed and content sizes.
diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 3a5ad0255..3109f1753 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -2371,6 +2371,7 @@ static size_t ZSTD_buildSeqStore(ZSTD_CCtx* zc, const void* src, size_t srcSize)
             assert(ldmSeqStore.pos == ldmSeqStore.size);
         } else {   /* not long range mode */
             ZSTD_blockCompressor const blockCompressor = ZSTD_selectBlockCompressor(zc->appliedParams.cParams.strategy, dictMode);
+            ms->ldmSeqStore = NULL;
             lastLLSize = blockCompressor(ms, &zc->seqStore, zc->blockState.nextCBlock->rep, src, srcSize);
         }
         {   const BYTE* const lastLiterals = (const BYTE*)src + srcSize - lastLLSize;
diff --git a/lib/compress/zstd_compress_internal.h b/lib/compress/zstd_compress_internal.h
index 179e4f7bb..f59370a29 100644
--- a/lib/compress/zstd_compress_internal.h
+++ b/lib/compress/zstd_compress_internal.h
@@ -167,7 +167,7 @@ struct ZSTD_matchState_t {
     optState_t opt;         /* optimal parser state */
     const ZSTD_matchState_t* dictMatchState;
     ZSTD_compressionParameters cParams;
-    rawSeqStore_t ldmSeqStore;
+    const rawSeqStore_t* ldmSeqStore;
 };
 
 typedef struct {
diff --git a/lib/compress/zstd_ldm.c b/lib/compress/zstd_ldm.c
index 4bfaa5414..cc338e697 100644
--- a/lib/compress/zstd_ldm.c
+++ b/lib/compress/zstd_ldm.c
@@ -573,8 +573,6 @@ static void ZSTD_ldm_skipRawSeqStoreBytes(rawSeqStore_t* rawSeqStore, size_t nbB
     if (currPos == 0 || rawSeqStore->pos == rawSeqStore->size) {
         rawSeqStore->posInSequence = 0;
     }
-    assert(rawSeqStore->posInSequence <=
-           rawSeqStore->seq[rawSeqStore->pos].litLength + rawSeqStore->seq[rawSeqStore->pos].matchLength);
 }
 
 size_t ZSTD_ldm_blockCompress(rawSeqStore_t* rawSeqStore,
@@ -591,20 +589,16 @@ size_t ZSTD_ldm_blockCompress(rawSeqStore_t* rawSeqStore,
     /* Input positions */
     BYTE const* ip = istart;
 
+    DEBUGLOG(5, "ZSTD_ldm_blockCompress: srcSize=%zu", srcSize);
     /* If using opt parser, use LDMs only as candidates rather than always accepting them */
     if (cParams->strategy >= ZSTD_btopt) {
         size_t lastLLSize;
-        ms->ldmSeqStore = *rawSeqStore;
+        ms->ldmSeqStore = rawSeqStore;
         lastLLSize = blockCompressor(ms, seqStore, rep, src, srcSize);
         ZSTD_ldm_skipRawSeqStoreBytes(rawSeqStore, srcSize);
-        if (rawSeqStore->pos >= rawSeqStore->size) {
-            /* If we're done with rawSeqStore, invalidate the one in matchState as well */
-            ms->ldmSeqStore.size = 0;
-        }
         return lastLLSize;
     }
 
-    DEBUGLOG(5, "ZSTD_ldm_blockCompress: srcSize=%zu", srcSize);
     assert(rawSeqStore->pos <= rawSeqStore->size);
     assert(rawSeqStore->size <= rawSeqStore->capacity);
     /* Loop through each sequence and apply the block compressor to the lits */
diff --git a/lib/compress/zstd_opt.c b/lib/compress/zstd_opt.c
index 30e07f889..29242523c 100644
--- a/lib/compress/zstd_opt.c
+++ b/lib/compress/zstd_opt.c
@@ -770,10 +770,10 @@ FORCE_INLINE_TEMPLATE U32 ZSTD_BtGetAllMatches (
 
 /* Struct containing info needed to make decision about ldm inclusion */
 typedef struct {
-    rawSeqStore_t seqStore;        /* External match candidates store for this block */
-    U32 startPosInBlock;           /* Start position of the current match candidate */
-    U32 endPosInBlock;             /* End position of the current match candidate */
-    U32 offset;                    /* Offset of the match candidate */
+    rawSeqStore_t seqStore;         /* External match candidates store for this block */
+    U32 startPosInBlock;            /* Start position of the current match candidate */
+    U32 endPosInBlock;              /* End position of the current match candidate */
+    U32 offset;                     /* Offset of the match candidate */
 } ZSTD_optLdm_t;
 
 /* ZSTD_optLdm_skipRawSeqStoreBytes():
@@ -794,8 +794,6 @@ static void ZSTD_optLdm_skipRawSeqStoreBytes(rawSeqStore_t* rawSeqStore, size_t
     if (currPos == 0 || rawSeqStore->pos == rawSeqStore->size) {
         rawSeqStore->posInSequence = 0;
     }
-    assert(rawSeqStore->posInSequence <=
-           rawSeqStore->seq[rawSeqStore->pos].litLength + rawSeqStore->seq[rawSeqStore->pos].matchLength);
 }
 
 /* ZSTD_opt_getNextMatchAndUpdateSeqStore():
@@ -810,7 +808,7 @@ static void ZSTD_opt_getNextMatchAndUpdateSeqStore(ZSTD_optLdm_t* optLdm, U32 cu
     U32 matchBytesRemaining;
 
     /* Setting match end position to MAX to ensure we never use an LDM during this block */
-    if (optLdm->seqStore.pos >= optLdm->seqStore.size || optLdm->seqStore.size == 0) {
+    if (optLdm->seqStore.size == 0 || optLdm->seqStore.pos >= optLdm->seqStore.size) {
         optLdm->startPosInBlock = UINT_MAX;
         optLdm->endPosInBlock = UINT_MAX;
         return;
@@ -894,7 +892,7 @@ static void ZSTD_optLdm_maybeAddMatch(ZSTD_match_t* matches, U32* nbMatches,
  */
 static void ZSTD_optLdm_processMatchCandidate(ZSTD_optLdm_t* optLdm, ZSTD_match_t* matches, U32* nbMatches,
                                               U32 currPosInBlock, U32 remainingBytes) {
-    if (optLdm->seqStore.size == 0 && optLdm->seqStore.pos >= optLdm->seqStore.size) {
+    if (optLdm->seqStore.size == 0 || optLdm->seqStore.pos >= optLdm->seqStore.size) {
         return;
     }
 
@@ -964,7 +962,14 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms,
     ZSTD_optimal_t* const opt = optStatePtr->priceTable;
     ZSTD_match_t* const matches = optStatePtr->matchTable;
     ZSTD_optimal_t lastSequence;
-    ZSTD_optLdm_t optLdm = {ms->ldmSeqStore, 0, 0, 0};  /* ms->ldmSeqStore itself is immutable in this function */
+    ZSTD_optLdm_t optLdm;
+
+    if (ms->ldmSeqStore) {
+        optLdm.seqStore = *ms->ldmSeqStore;
+        optLdm.endPosInBlock = optLdm.startPosInBlock = optLdm.offset = 0;
+    } else {
+        optLdm.seqStore.size = optLdm.seqStore.pos = 0;
+    }
 
     /* Get first match from ldm seq store if long mode is enabled */
     ZSTD_opt_getNextMatchAndUpdateSeqStore(&optLdm, (U32)(ip-istart), (U32)(iend-ip));
@@ -1217,7 +1222,7 @@ _shortestPath:   /* cur, last_pos, best_mlen, best_off have to be set */
             ZSTD_setBasePrices(optStatePtr, optLevel);
         }
     }   /* while (ip < ilimit) */
-    
+
     /* Return the last literals size */
     return (size_t)(iend - anchor);
 }
@@ -1267,7 +1272,6 @@ ZSTD_initStats_ultra(ZSTD_matchState_t* ms,
                const void* src, size_t srcSize)
 {
     U32 tmpRep[ZSTD_REP_NUM];  /* updated rep codes will sink here */
-    rawSeqStore_t tmpSeqStore = ms->ldmSeqStore;
     ZSTD_memcpy(tmpRep, rep, sizeof(tmpRep));
 
     DEBUGLOG(4, "ZSTD_initStats_ultra (srcSize=%zu)", srcSize);
@@ -1284,7 +1288,6 @@ ZSTD_initStats_ultra(ZSTD_matchState_t* ms,
     ms->window.dictLimit += (U32)srcSize;
     ms->window.lowLimit = ms->window.dictLimit;
     ms->nextToUpdate = ms->window.dictLimit;
-    ms->ldmSeqStore = tmpSeqStore;
 
     /* re-inforce weight of collected statistics */
     ZSTD_upscaleStats(&ms->opt);

From b9c8033cde4336fff66db993d954a283b6c0b1ae Mon Sep 17 00:00:00 2001
From: senhuang42 
Date: Wed, 7 Oct 2020 16:59:25 -0400
Subject: [PATCH 59/68] Define kNullRawSeqStore for every file

---
 lib/compress/zstd_compress.c          |  2 +-
 lib/compress/zstd_compress_internal.h |  2 ++
 lib/compress/zstd_opt.c               | 10 ++--------
 lib/compress/zstdmt_compress.c        |  2 --
 4 files changed, 5 insertions(+), 11 deletions(-)

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 3109f1753..25b63965c 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -2354,7 +2354,7 @@ static size_t ZSTD_buildSeqStore(ZSTD_CCtx* zc, const void* src, size_t srcSize)
                                        src, srcSize);
             assert(zc->externSeqStore.pos <= zc->externSeqStore.size);
         } else if (zc->appliedParams.ldmParams.enableLdm) {
-            rawSeqStore_t ldmSeqStore = {NULL, 0, 0, 0, 0};
+            rawSeqStore_t ldmSeqStore = kNullRawSeqStore;
 
             ldmSeqStore.seq = zc->ldmSequences;
             ldmSeqStore.capacity = zc->maxNbLdmSequences;
diff --git a/lib/compress/zstd_compress_internal.h b/lib/compress/zstd_compress_internal.h
index f59370a29..9b65b3c4a 100644
--- a/lib/compress/zstd_compress_internal.h
+++ b/lib/compress/zstd_compress_internal.h
@@ -101,6 +101,8 @@ typedef struct {
   size_t capacity;      /* The capacity starting from `seq` pointer */
 } rawSeqStore_t;
 
+UNUSED_ATTR static const rawSeqStore_t kNullRawSeqStore = {NULL, 0, 0, 0, 0};
+
 typedef struct {
     int price;
     U32 off;
diff --git a/lib/compress/zstd_opt.c b/lib/compress/zstd_opt.c
index 29242523c..8ba7aba38 100644
--- a/lib/compress/zstd_opt.c
+++ b/lib/compress/zstd_opt.c
@@ -964,14 +964,8 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms,
     ZSTD_optimal_t lastSequence;
     ZSTD_optLdm_t optLdm;
 
-    if (ms->ldmSeqStore) {
-        optLdm.seqStore = *ms->ldmSeqStore;
-        optLdm.endPosInBlock = optLdm.startPosInBlock = optLdm.offset = 0;
-    } else {
-        optLdm.seqStore.size = optLdm.seqStore.pos = 0;
-    }
-
-    /* Get first match from ldm seq store if long mode is enabled */
+    optLdm.seqStore = ms->ldmSeqStore ? *ms->ldmSeqStore : kNullRawSeqStore;
+    optLdm.endPosInBlock = optLdm.startPosInBlock = optLdm.offset = 0;
     ZSTD_opt_getNextMatchAndUpdateSeqStore(&optLdm, (U32)(ip-istart), (U32)(iend-ip));
 
     /* init */
diff --git a/lib/compress/zstdmt_compress.c b/lib/compress/zstdmt_compress.c
index 06462fe5a..e19184677 100644
--- a/lib/compress/zstdmt_compress.c
+++ b/lib/compress/zstdmt_compress.c
@@ -266,8 +266,6 @@ static void ZSTDMT_releaseBuffer(ZSTDMT_bufferPool* bufPool, buffer_t buf)
 
 /* =====   Seq Pool Wrapper   ====== */
 
-static rawSeqStore_t kNullRawSeqStore = {NULL, 0, 0, 0, 0};
-
 typedef ZSTDMT_bufferPool ZSTDMT_seqPool;
 
 static size_t ZSTDMT_sizeof_seqPool(ZSTDMT_seqPool* seqPool)

From cc907770bdadf49edaaebdfa31a82b3bffa1f46e Mon Sep 17 00:00:00 2001
From: Like Ma 
Date: Wed, 26 Aug 2020 03:10:06 +0800
Subject: [PATCH 60/68] Fix building on AIX 5.1

---
 lib/common/mem.h      | 6 +++++-
 lib/legacy/zstd_v01.c | 6 +++++-
 lib/legacy/zstd_v02.c | 6 +++++-
 lib/legacy/zstd_v03.c | 6 +++++-
 lib/legacy/zstd_v04.c | 6 +++++-
 lib/legacy/zstd_v05.c | 6 +++++-
 lib/legacy/zstd_v06.c | 6 +++++-
 lib/legacy/zstd_v07.c | 6 +++++-
 programs/timefn.h     | 6 +++++-
 9 files changed, 45 insertions(+), 9 deletions(-)

diff --git a/lib/common/mem.h b/lib/common/mem.h
index c8361ab10..4728ef767 100644
--- a/lib/common/mem.h
+++ b/lib/common/mem.h
@@ -45,7 +45,11 @@ extern "C" {
 *  Basic Types
 *****************************************************************/
 #if  !defined (__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) )
-# include 
+#  if defined(_AIX)
+#    include 
+#  else
+#    include  /* intptr_t */
+#  endif
   typedef   uint8_t BYTE;
   typedef  uint16_t U16;
   typedef   int16_t S16;
diff --git a/lib/legacy/zstd_v01.c b/lib/legacy/zstd_v01.c
index eb2362833..13115bec5 100644
--- a/lib/legacy/zstd_v01.c
+++ b/lib/legacy/zstd_v01.c
@@ -1280,7 +1280,11 @@ static size_t HUF_decompress (void* dst, size_t maxDstSize, const void* cSrc, si
 *  Basic Types
 *********************************************************/
 #if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L   /* C99 */
-# include 
+# if defined(_AIX)
+#  include 
+# else
+#  include  /* intptr_t */
+# endif
 typedef  uint8_t BYTE;
 typedef uint16_t U16;
 typedef  int16_t S16;
diff --git a/lib/legacy/zstd_v02.c b/lib/legacy/zstd_v02.c
index 32d45a6d4..9abb6d033 100644
--- a/lib/legacy/zstd_v02.c
+++ b/lib/legacy/zstd_v02.c
@@ -89,7 +89,11 @@ extern "C" {
 *  Basic Types
 *****************************************************************/
 #if defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
-# include 
+# if defined(_AIX)
+#  include 
+# else
+#  include  /* intptr_t */
+# endif
   typedef  uint8_t BYTE;
   typedef uint16_t U16;
   typedef  int16_t S16;
diff --git a/lib/legacy/zstd_v03.c b/lib/legacy/zstd_v03.c
index b541eae2a..a19cb205a 100644
--- a/lib/legacy/zstd_v03.c
+++ b/lib/legacy/zstd_v03.c
@@ -90,7 +90,11 @@ extern "C" {
 *  Basic Types
 *****************************************************************/
 #if defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
-# include 
+# if defined(_AIX)
+#  include 
+# else
+#  include  /* intptr_t */
+# endif
   typedef  uint8_t BYTE;
   typedef uint16_t U16;
   typedef  int16_t S16;
diff --git a/lib/legacy/zstd_v04.c b/lib/legacy/zstd_v04.c
index 19fda8980..77d52555b 100644
--- a/lib/legacy/zstd_v04.c
+++ b/lib/legacy/zstd_v04.c
@@ -52,7 +52,11 @@ extern "C" {
 *  Basic Types
 *****************************************************************/
 #if defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
-# include 
+# if defined(_AIX)
+#  include 
+# else
+#  include  /* intptr_t */
+# endif
   typedef  uint8_t BYTE;
   typedef uint16_t U16;
   typedef  int16_t S16;
diff --git a/lib/legacy/zstd_v05.c b/lib/legacy/zstd_v05.c
index 243d22256..ca8d5c9bf 100644
--- a/lib/legacy/zstd_v05.c
+++ b/lib/legacy/zstd_v05.c
@@ -80,7 +80,11 @@ extern "C" {
 *  Basic Types
 *****************************************************************/
 #if defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
-# include 
+# if defined(_AIX)
+#  include 
+# else
+#  include  /* intptr_t */
+# endif
   typedef  uint8_t BYTE;
   typedef uint16_t U16;
   typedef  int16_t S16;
diff --git a/lib/legacy/zstd_v06.c b/lib/legacy/zstd_v06.c
index c56f58277..c4ac7dba8 100644
--- a/lib/legacy/zstd_v06.c
+++ b/lib/legacy/zstd_v06.c
@@ -82,7 +82,11 @@ extern "C" {
 *  Basic Types
 *****************************************************************/
 #if  !defined (__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) )
-# include 
+# if defined(_AIX)
+#  include 
+# else
+#  include  /* intptr_t */
+# endif
   typedef  uint8_t BYTE;
   typedef uint16_t U16;
   typedef  int16_t S16;
diff --git a/lib/legacy/zstd_v07.c b/lib/legacy/zstd_v07.c
index 9f3a597f1..049ba4749 100644
--- a/lib/legacy/zstd_v07.c
+++ b/lib/legacy/zstd_v07.c
@@ -242,7 +242,11 @@ extern "C" {
 *  Basic Types
 *****************************************************************/
 #if  !defined (__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) )
-# include 
+# if defined(_AIX)
+#  include 
+# else
+#  include  /* intptr_t */
+# endif
   typedef  uint8_t BYTE;
   typedef uint16_t U16;
   typedef  int16_t S16;
diff --git a/programs/timefn.h b/programs/timefn.h
index eb3c13093..5d2818e8a 100644
--- a/programs/timefn.h
+++ b/programs/timefn.h
@@ -28,7 +28,11 @@ extern "C" {
 ******************************************/
 
 #if !defined (__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) )
-# include 
+# if defined(_AIX)
+#  include 
+# else
+#  include  /* intptr_t */
+# endif
   typedef uint64_t           PTime;  /* Precise Time */
 #else
   typedef unsigned long long PTime;  /* does not support compilers without long long support */

From d6911b86bea2132ffae3d3841d35c4215e6b4609 Mon Sep 17 00:00:00 2001
From: senhuang42 
Date: Wed, 7 Oct 2020 23:44:36 -0400
Subject: [PATCH 61/68] Require LDM matches to be strictly greater in length

---
 lib/compress/zstd_opt.c | 17 +++--------------
 1 file changed, 3 insertions(+), 14 deletions(-)

diff --git a/lib/compress/zstd_opt.c b/lib/compress/zstd_opt.c
index 8ba7aba38..e55c459de 100644
--- a/lib/compress/zstd_opt.c
+++ b/lib/compress/zstd_opt.c
@@ -867,20 +867,9 @@ static void ZSTD_optLdm_maybeAddMatch(ZSTD_match_t* matches, U32* nbMatches,
         return;
     }
 
-    DEBUGLOG(6, "ZSTD_optLdm_maybeAddMatch(): Adding ldm candidate match (offCode: %u matchLength %u) at block position=%u",
-             candidateOffCode, candidateMatchLength, currPosInBlock);
-    if (*nbMatches == 0) {
-        matches[*nbMatches].len = candidateMatchLength;
-        matches[*nbMatches].off = candidateOffCode;
-        (*nbMatches)++;
-    } else if ((candidateMatchLength >= matches[*nbMatches-1].len) && *nbMatches < ZSTD_OPT_NUM) {
-        if (candidateMatchLength == matches[*nbMatches-1].len) {
-            /* No need to insert match with same matchLength. At most, replace offCode if it is smaller. */
-            if (candidateOffCode < matches[*nbMatches-1].off) {
-                matches[*nbMatches-1].off = candidateOffCode;
-            }
-            return;
-        }
+    if (*nbMatches == 0 || ((candidateMatchLength > matches[*nbMatches-1].len) && *nbMatches < ZSTD_OPT_NUM)) {
+        DEBUGLOG(6, "ZSTD_optLdm_maybeAddMatch(): Adding ldm candidate match (offCode: %u matchLength %u) at block position=%u",
+                 candidateOffCode, candidateMatchLength, currPosInBlock);
         matches[*nbMatches].len = candidateMatchLength;
         matches[*nbMatches].off = candidateOffCode;
         (*nbMatches)++;

From 7083f790084be3f8b9b58c365e55c3a1df7de236 Mon Sep 17 00:00:00 2001
From: Nick Terrell 
Date: Mon, 5 Oct 2020 15:17:44 -0700
Subject: [PATCH 62/68] [bug] Fix dictContentType when reprocessing cdict

Conditions to trigger:
* CDict is loaded as raw content.
* CDict starts with the zstd dictionary magic number.
* The CDict is reprocessed (not attached or copied).
* The new API is used (streaming or `ZSTD_compress2()`).

Bug: The dictionary is loaded as a zstd dictionary, not a raw content
dictionary, because the dict content type is set to `ZSTD_dct_auto`.

Fix: Pass in the dictionary content type from cdict creation to the call
to `ZSTD_compress_insertDictionary()`.

Test: Added a test case that exposes the bug, and fixed the raw
content tests to not modify the `dictBuffer`, which makes all future
tests with the `dictBuffer` raw content, which doesn't seem intentional.
---
 lib/compress/zstd_compress.c |  4 ++-
 tests/fuzzer.c               | 65 +++++++++++++++++++++++-------------
 2 files changed, 45 insertions(+), 24 deletions(-)

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index ea4b04aab..9469fcf39 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -64,6 +64,7 @@ size_t ZSTD_compressBound(size_t srcSize) {
 struct ZSTD_CDict_s {
     const void* dictContent;
     size_t dictContentSize;
+    ZSTD_dictContentType_e dictContentType; /* The dictContentType the CDict was created with */
     U32* entropyWorkspace; /* entropy workspace of HUF_WORKSPACE_SIZE bytes */
     ZSTD_cwksp workspace;
     ZSTD_matchState_t matchState;
@@ -3188,7 +3189,7 @@ static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* cctx,
                 ZSTD_compress_insertDictionary(
                         cctx->blockState.prevCBlock, &cctx->blockState.matchState,
                         &cctx->ldmState, &cctx->workspace, &cctx->appliedParams, cdict->dictContent,
-                        cdict->dictContentSize, dictContentType, dtlm,
+                        cdict->dictContentSize, cdict->dictContentType, dtlm,
                         cctx->entropyWorkspace)
               : ZSTD_compress_insertDictionary(
                         cctx->blockState.prevCBlock, &cctx->blockState.matchState,
@@ -3460,6 +3461,7 @@ static size_t ZSTD_initCDict_internal(
         ZSTD_memcpy(internalBuffer, dictBuffer, dictSize);
     }
     cdict->dictContentSize = dictSize;
+    cdict->dictContentType = dictContentType;
 
     cdict->entropyWorkspace = (U32*)ZSTD_cwksp_reserve_object(&cdict->workspace, HUF_WORKSPACE_SIZE);
 
diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index 8b10078ac..27a7bde12 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -1971,34 +1971,53 @@ static int basicUnitTests(U32 const seed, double compressibility)
         }
         DISPLAYLEVEL(3, "OK \n");
 
-        DISPLAYLEVEL(3, "test%3i : Loading rawContent starting with dict header w/ ZSTD_dct_auto should fail : ", testNb++);
-        {
-            size_t ret;
-            MEM_writeLE32((char*)dictBuffer+2, ZSTD_MAGIC_DICTIONARY);
-            /* Either operation is allowed to fail, but one must fail. */
-            ret = ZSTD_CCtx_loadDictionary_advanced(
-                    cctx, (const char*)dictBuffer+2, dictSize-2, ZSTD_dlm_byRef, ZSTD_dct_auto);
-            if (!ZSTD_isError(ret)) {
-                ret = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100));
-                if (!ZSTD_isError(ret)) goto _output_error;
-            }
-        }
-        DISPLAYLEVEL(3, "OK \n");
+        {   char* rawDictBuffer = (char*)malloc(dictSize);
+            assert(rawDictBuffer);
+            memcpy(rawDictBuffer, (char*)dictBuffer + 2, dictSize - 2);
+            memset(rawDictBuffer + dictSize - 2, 0, 2);
+            MEM_writeLE32((char*)rawDictBuffer, ZSTD_MAGIC_DICTIONARY);
 
-        DISPLAYLEVEL(3, "test%3i : Loading rawContent starting with dict header w/ ZSTD_dct_rawContent should pass : ", testNb++);
-        {
-            size_t ret;
-            MEM_writeLE32((char*)dictBuffer+2, ZSTD_MAGIC_DICTIONARY);
-            ret = ZSTD_CCtx_loadDictionary_advanced(
-                    cctx, (const char*)dictBuffer+2, dictSize-2, ZSTD_dlm_byRef, ZSTD_dct_rawContent);
-            if (ZSTD_isError(ret)) goto _output_error;
-            ret = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100));
-            if (ZSTD_isError(ret)) goto _output_error;
+            DISPLAYLEVEL(3, "test%3i : Loading rawContent starting with dict header w/ ZSTD_dct_auto should fail : ", testNb++);
+            {
+                size_t ret;
+                /* Either operation is allowed to fail, but one must fail. */
+                ret = ZSTD_CCtx_loadDictionary_advanced(
+                        cctx, (const char*)rawDictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_auto);
+                if (!ZSTD_isError(ret)) {
+                    ret = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100));
+                    if (!ZSTD_isError(ret)) goto _output_error;
+                }
+            }
+            DISPLAYLEVEL(3, "OK \n");
+
+            DISPLAYLEVEL(3, "test%3i : Loading rawContent starting with dict header w/ ZSTD_dct_rawContent should pass : ", testNb++);
+            {
+                size_t ret;
+                ret = ZSTD_CCtx_loadDictionary_advanced(
+                        cctx, (const char*)rawDictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_rawContent);
+                if (ZSTD_isError(ret)) goto _output_error;
+                ret = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100));
+                if (ZSTD_isError(ret)) goto _output_error;
+            }
+            DISPLAYLEVEL(3, "OK \n");
+
+            DISPLAYLEVEL(3, "test%3i : Testing non-attached CDict with ZSTD_dct_rawContent : ", testNb++);
+            {   size_t const srcSize = MIN(CNBuffSize, 100);
+                ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
+                /* Force the dictionary to be reloaded in raw content mode */
+                CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_forceAttachDict, ZSTD_dictForceLoad));
+                CHECK_Z(ZSTD_CCtx_loadDictionary_advanced(cctx, rawDictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_rawContent));
+                cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, srcSize);
+                CHECK_Z(cSize);
+            }
+            DISPLAYLEVEL(3, "OK \n");
+
+            free(rawDictBuffer);
         }
-        DISPLAYLEVEL(3, "OK \n");
 
         DISPLAYLEVEL(3, "test%3i : ZSTD_CCtx_refCDict() then set parameters : ", testNb++);
         {   ZSTD_CDict* const cdict = ZSTD_createCDict(CNBuffer, dictSize, 1);
+            ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
             CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 1) );
             CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_hashLog, 12 ));
             CHECK_Z( ZSTD_CCtx_refCDict(cctx, cdict) );

From 012818df998993c0c20264c9c92c5bcdfd9fbe52 Mon Sep 17 00:00:00 2001
From: Nick Terrell 
Date: Mon, 5 Oct 2020 15:46:50 -0700
Subject: [PATCH 63/68] [refactor] Remove ZSTD_resetCStream_internal()

This function is only called in one place. It isn't a logical separation
of duties, and it was only obsfucating the code now, so inline it.
---
 lib/compress/zstd_compress.c | 47 +++++++++++-------------------------
 1 file changed, 14 insertions(+), 33 deletions(-)

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 9469fcf39..3e0339683 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -3793,34 +3793,6 @@ size_t ZSTD_CStreamOutSize(void)
     return ZSTD_compressBound(ZSTD_BLOCKSIZE_MAX) + ZSTD_blockHeaderSize + 4 /* 32-bits hash */ ;
 }
 
-static size_t ZSTD_resetCStream_internal(ZSTD_CStream* cctx,
-                    const void* const dict, size_t const dictSize, ZSTD_dictContentType_e const dictContentType,
-                    const ZSTD_CDict* const cdict,
-                    ZSTD_CCtx_params params, unsigned long long const pledgedSrcSize)
-{
-    DEBUGLOG(4, "ZSTD_resetCStream_internal");
-    /* Finalize the compression parameters */
-    params.cParams = ZSTD_getCParamsFromCCtxParams(¶ms, pledgedSrcSize, dictSize);
-    /* params are supposed to be fully validated at this point */
-    assert(!ZSTD_isError(ZSTD_checkCParams(params.cParams)));
-    assert(!((dict) && (cdict)));  /* either dict or cdict, not both */
-
-    FORWARD_IF_ERROR( ZSTD_compressBegin_internal(cctx,
-                                         dict, dictSize, dictContentType, ZSTD_dtlm_fast,
-                                         cdict,
-                                         ¶ms, pledgedSrcSize,
-                                         ZSTDb_buffered) , "");
-
-    cctx->inToCompress = 0;
-    cctx->inBuffPos = 0;
-    cctx->inBuffTarget = cctx->blockSize
-                      + (cctx->blockSize == pledgedSrcSize);   /* for small input: avoid automatic flush on reaching end of block, since it would require to add a 3-bytes null block to end frame */
-    cctx->outBuffContentSize = cctx->outBuffFlushedSize = 0;
-    cctx->streamStage = zcss_load;
-    cctx->frameEnded = 0;
-    return 0;   /* ready to go */
-}
-
 /* ZSTD_resetCStream():
  * pledgedSrcSize == 0 means "unknown" */
 size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pss)
@@ -4162,12 +4134,21 @@ size_t ZSTD_compressStream2( ZSTD_CCtx* cctx,
             cctx->appliedParams.nbWorkers = params.nbWorkers;
         } else
 #endif
-        {   FORWARD_IF_ERROR( ZSTD_resetCStream_internal(cctx,
-                            prefixDict.dict, prefixDict.dictSize, prefixDict.dictContentType,
-                            cctx->cdict,
-                            params, cctx->pledgedSrcSizePlusOne-1) , "");
-            assert(cctx->streamStage == zcss_load);
+        {   U64 const pledgedSrcSize = cctx->pledgedSrcSizePlusOne - 1;
+            assert(!ZSTD_isError(ZSTD_checkCParams(params.cParams)));
+            FORWARD_IF_ERROR( ZSTD_compressBegin_internal(cctx,
+                    prefixDict.dict, prefixDict.dictSize, prefixDict.dictContentType, ZSTD_dtlm_fast,
+                    cctx->cdict,
+                    ¶ms, pledgedSrcSize,
+                    ZSTDb_buffered) , "");
             assert(cctx->appliedParams.nbWorkers == 0);
+            cctx->inToCompress = 0;
+            cctx->inBuffPos = 0;
+            /* for small input: avoid automatic flush on reaching end of block, since it would require to add a 3-bytes null block to end frame */
+            cctx->inBuffTarget = cctx->blockSize + (cctx->blockSize == pledgedSrcSize);
+            cctx->outBuffContentSize = cctx->outBuffFlushedSize = 0;
+            cctx->streamStage = zcss_load;
+            cctx->frameEnded = 0;
     }   }
     /* end of transparent initialization stage */
 

From 48ef15fb47395dcb57900cd7c247f2dd5af2d5cd Mon Sep 17 00:00:00 2001
From: Nick Terrell 
Date: Mon, 5 Oct 2020 15:49:55 -0700
Subject: [PATCH 64/68] [minor improvement] Pass dictSize when selecting
 parameters

When selecting parameters in streaming compression with a dictionary use
the dictionary size to select the parameters.
---
 lib/compress/zstd_compress.c | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 3e0339683..b0f0e9eaa 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -4109,7 +4109,9 @@ size_t ZSTD_compressStream2( ZSTD_CCtx* cctx,
             params.compressionLevel = cctx->cdict->compressionLevel; /* let cdict take priority in terms of compression level */
         DEBUGLOG(4, "ZSTD_compressStream2 : transparent init stage");
         if (endOp == ZSTD_e_end) cctx->pledgedSrcSizePlusOne = input->size + 1;  /* auto-fix pledgedSrcSize */
-        params.cParams = ZSTD_getCParamsFromCCtxParams(¶ms, cctx->pledgedSrcSizePlusOne-1, 0 /*dictSize*/);
+        params.cParams = ZSTD_getCParamsFromCCtxParams(
+                ¶ms, cctx->pledgedSrcSizePlusOne-1,
+                cctx->prefixDict.dict ? cctx->prefixDict.dictSize : (cctx->cdict ? cctx->cdict->dictContentSize : 0));
 
 
 #ifdef ZSTD_MULTITHREAD

From fadaab8c7c00f6b1589c207e1623ffee7a21e9e7 Mon Sep 17 00:00:00 2001
From: Nick Terrell 
Date: Mon, 5 Oct 2020 15:38:10 -0700
Subject: [PATCH 65/68] [minor improvement] Pass 0 as the content size in the
 DDS

The DDS structure can't be copied into the working tables like the DMS.
So it doesn't need to account for the source size when sizing its
parameters, just the dictionary size.
---
 lib/compress/zstd_compress.c | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index b0f0e9eaa..09ec152f3 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -878,7 +878,6 @@ ZSTDLIB_API size_t ZSTD_CCtx_setPledgedSrcSize(ZSTD_CCtx* cctx, unsigned long lo
 
 static ZSTD_compressionParameters ZSTD_dedicatedDictSearch_getCParams(
         int const compressionLevel,
-        unsigned long long srcSizeHint,
         size_t const dictSize);
 static int ZSTD_dedicatedDictSearch_isSupported(
         const ZSTD_compressionParameters* cParams);
@@ -3559,7 +3558,7 @@ ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_advanced2(
 
     if (cctxParams.enableDedicatedDictSearch) {
         cParams = ZSTD_dedicatedDictSearch_getCParams(
-            cctxParams.compressionLevel, ZSTD_CONTENTSIZE_UNKNOWN, dictSize);
+            cctxParams.compressionLevel, dictSize);
         ZSTD_overrideCParams(&cParams, &cctxParams.cParams);
     } else {
         cParams = ZSTD_getCParamsFromCCtxParams(
@@ -4362,9 +4361,9 @@ static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEV
 },
 };
 
-static ZSTD_compressionParameters ZSTD_dedicatedDictSearch_getCParams(int const compressionLevel, unsigned long long srcSizeHint, size_t const dictSize)
+static ZSTD_compressionParameters ZSTD_dedicatedDictSearch_getCParams(int const compressionLevel, size_t const dictSize)
 {
-    ZSTD_compressionParameters cParams = ZSTD_getCParams_internal(compressionLevel, srcSizeHint, dictSize);
+    ZSTD_compressionParameters cParams = ZSTD_getCParams_internal(compressionLevel, 0, dictSize);
     switch (cParams.strategy) {
         case ZSTD_fast:
         case ZSTD_dfast:

From d5c688e8ae8959e1740fe3833251c88fca3e5e10 Mon Sep 17 00:00:00 2001
From: Nick Terrell 
Date: Thu, 1 Oct 2020 13:12:23 -0700
Subject: [PATCH 66/68] Fix ZSTD_adjustCParams_internal() to handle dictionary
 logic

Pass in the `ZSTD_cParamMode_e` to select how we define our cparams.
Based on the mode we either take the `dictSize` into account or we set
it to `0`. See the documentation for `ZSTD_cParamMode_e`.

Some of the modes currently share the same behavior. But they have
distinct modes because they are drastically different cases. E.g.
compression + reprocessing the dictionary and creating a cdict.

Additionally, when downsizing the hashLog and chainLog take the
(adjusted) dictionary size into account, since the size of the
dictionary gets added onto the window size.

Adds a simple test to ensure that we aren't downsizing too far.
---
 lib/compress/zstd_compress.c          | 160 +++++++++++++++++++-------
 lib/compress/zstd_compress_internal.h |  21 +++-
 lib/compress/zstdmt_compress.c        |   2 +-
 tests/fuzzer.c                        |  13 +++
 4 files changed, 155 insertions(+), 41 deletions(-)

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 09ec152f3..69ff6512d 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -1063,24 +1063,71 @@ U32 ZSTD_cycleLog(U32 hashLog, ZSTD_strategy strat)
     return hashLog - btScale;
 }
 
+/** ZSTD_dictAndWindowLog() :
+ *  Returns an adjusted window log that is large enough to fit the source and the dictionary.
+ *  The zstd format says that the entire dictionary is valid if one byte of the dictionary
+ *  is within the window. So the hashLog and chainLog should be large enough to reference both
+ *  the dictionary and the window. So we must use this adjusted dictAndWindowLog when downsizing
+ *  the hashLog and windowLog.
+ */
+static U32 ZSTD_dictAndWindowLog(U32 windowLog, U64 srcSize, U64 dictSize)
+{
+    const U64 maxWindowSize = 1ULL << ZSTD_WINDOWLOG_MAX;
+    /* No dictionary ==> No change */
+    if (dictSize == 0) {
+        return windowLog;
+    }
+    assert(srcSize != ZSTD_CONTENTSIZE_UNKNOWN); /* Handled in ZSTD_adjustCParams_internal() */
+    {
+        U64 const windowSize = 1ULL << windowLog;
+        U64 const dictAndWindowSize = dictSize + windowSize;
+        /* If the window size is already large enough to fit both the source and the dictionary
+         * then just use the window size. Otherwise adjust so that it fits the dictionary and
+         * the window.
+         */
+        if (windowSize >= dictSize + srcSize) {
+            return windowLog; /* Window size large enough already */
+        } else if (dictAndWindowSize >= maxWindowSize) {
+            return ZSTD_WINDOWLOG_MAX; /* Larger than max window log */
+        } else  {
+            return ZSTD_highbit32((U32)dictAndWindowSize - 1) + 1;
+        }
+    }
+}
+
 /** ZSTD_adjustCParams_internal() :
  *  optimize `cPar` for a specified input (`srcSize` and `dictSize`).
  *  mostly downsize to reduce memory consumption and initialization latency.
  * `srcSize` can be ZSTD_CONTENTSIZE_UNKNOWN when not known.
+ * `mode` is the mode for parameter adjustment. See docs for `ZSTD_cParamMode_e`.
  *  note : `srcSize==0` means 0!
  *  condition : cPar is presumed validated (can be checked using ZSTD_checkCParams()). */
 static ZSTD_compressionParameters
 ZSTD_adjustCParams_internal(ZSTD_compressionParameters cPar,
                             unsigned long long srcSize,
-                            size_t dictSize)
+                            size_t dictSize,
+                            ZSTD_cParamMode_e mode)
 {
-    static const U64 minSrcSize = 513; /* (1<<9) + 1 */
-    static const U64 maxWindowResize = 1ULL << (ZSTD_WINDOWLOG_MAX-1);
+    const U64 minSrcSize = 513; /* (1<<9) + 1 */
+    const U64 maxWindowResize = 1ULL << (ZSTD_WINDOWLOG_MAX-1);
     assert(ZSTD_checkCParams(cPar)==0);
 
     if (dictSize && srcSize == ZSTD_CONTENTSIZE_UNKNOWN)
         srcSize = minSrcSize;
 
+    switch (mode) {
+    case ZSTD_cpm_noAttachDict:
+    case ZSTD_cpm_unknown:
+    case ZSTD_cpm_createCDict:
+        break;
+    case ZSTD_cpm_attachDict:
+        dictSize = 0;
+        break;
+    default:
+        assert(0);
+        break;
+    }
+
     /* resize windowLog if input is small enough, to use less memory */
     if ( (srcSize < maxWindowResize)
       && (dictSize < maxWindowResize) )  {
@@ -1090,10 +1137,11 @@ ZSTD_adjustCParams_internal(ZSTD_compressionParameters cPar,
                             ZSTD_highbit32(tSize-1) + 1;
         if (cPar.windowLog > srcLog) cPar.windowLog = srcLog;
     }
-    if (cPar.hashLog > cPar.windowLog+1) cPar.hashLog = cPar.windowLog+1;
-    {   U32 const cycleLog = ZSTD_cycleLog(cPar.chainLog, cPar.strategy);
-        if (cycleLog > cPar.windowLog)
-            cPar.chainLog -= (cycleLog - cPar.windowLog);
+    {   U32 const dictAndWindowLog = ZSTD_dictAndWindowLog(cPar.windowLog, (U64)srcSize, (U64)dictSize);
+        U32 const cycleLog = ZSTD_cycleLog(cPar.chainLog, cPar.strategy);
+        if (cPar.hashLog > dictAndWindowLog+1) cPar.hashLog = dictAndWindowLog+1;
+        if (cycleLog > dictAndWindowLog)
+            cPar.chainLog -= (cycleLog - dictAndWindowLog);
     }
 
     if (cPar.windowLog < ZSTD_WINDOWLOG_ABSOLUTEMIN)
@@ -1109,11 +1157,11 @@ ZSTD_adjustCParams(ZSTD_compressionParameters cPar,
 {
     cPar = ZSTD_clampCParams(cPar);   /* resulting cPar is necessarily valid (all parameters within range) */
     if (srcSize == 0) srcSize = ZSTD_CONTENTSIZE_UNKNOWN;
-    return ZSTD_adjustCParams_internal(cPar, srcSize, dictSize);
+    return ZSTD_adjustCParams_internal(cPar, srcSize, dictSize, ZSTD_cpm_unknown);
 }
 
-static ZSTD_compressionParameters ZSTD_getCParams_internal(int compressionLevel, unsigned long long srcSizeHint, size_t dictSize);
-static ZSTD_parameters ZSTD_getParams_internal(int compressionLevel, unsigned long long srcSizeHint, size_t dictSize);
+static ZSTD_compressionParameters ZSTD_getCParams_internal(int compressionLevel, unsigned long long srcSizeHint, size_t dictSize, ZSTD_cParamMode_e mode);
+static ZSTD_parameters ZSTD_getParams_internal(int compressionLevel, unsigned long long srcSizeHint, size_t dictSize, ZSTD_cParamMode_e mode);
 
 static void ZSTD_overrideCParams(
               ZSTD_compressionParameters* cParams,
@@ -1129,18 +1177,18 @@ static void ZSTD_overrideCParams(
 }
 
 ZSTD_compressionParameters ZSTD_getCParamsFromCCtxParams(
-        const ZSTD_CCtx_params* CCtxParams, U64 srcSizeHint, size_t dictSize)
+        const ZSTD_CCtx_params* CCtxParams, U64 srcSizeHint, size_t dictSize, ZSTD_cParamMode_e mode)
 {
     ZSTD_compressionParameters cParams;
     if (srcSizeHint == ZSTD_CONTENTSIZE_UNKNOWN && CCtxParams->srcSizeHint > 0) {
       srcSizeHint = CCtxParams->srcSizeHint;
     }
-    cParams = ZSTD_getCParams_internal(CCtxParams->compressionLevel, srcSizeHint, dictSize);
+    cParams = ZSTD_getCParams_internal(CCtxParams->compressionLevel, srcSizeHint, dictSize, mode);
     if (CCtxParams->ldmParams.enableLdm) cParams.windowLog = ZSTD_LDM_DEFAULT_WINDOW_LOG;
     ZSTD_overrideCParams(&cParams, &CCtxParams->cParams);
     assert(!ZSTD_checkCParams(cParams));
     /* srcSizeHint == 0 means 0 */
-    return ZSTD_adjustCParams_internal(cParams, srcSizeHint, dictSize);
+    return ZSTD_adjustCParams_internal(cParams, srcSizeHint, dictSize, mode);
 }
 
 static size_t
@@ -1218,7 +1266,7 @@ static size_t ZSTD_estimateCCtxSize_usingCCtxParams_internal(
 size_t ZSTD_estimateCCtxSize_usingCCtxParams(const ZSTD_CCtx_params* params)
 {
     ZSTD_compressionParameters const cParams =
-                ZSTD_getCParamsFromCCtxParams(params, ZSTD_CONTENTSIZE_UNKNOWN, 0);
+                ZSTD_getCParamsFromCCtxParams(params, ZSTD_CONTENTSIZE_UNKNOWN, 0, ZSTD_cpm_noAttachDict);
 
     RETURN_ERROR_IF(params->nbWorkers > 0, GENERIC, "Estimate CCtx size is supported for single-threaded compression only.");
     /* estimateCCtxSize is for one-shot compression. So no buffers should
@@ -1236,7 +1284,7 @@ size_t ZSTD_estimateCCtxSize_usingCParams(ZSTD_compressionParameters cParams)
 
 static size_t ZSTD_estimateCCtxSize_internal(int compressionLevel)
 {
-    ZSTD_compressionParameters const cParams = ZSTD_getCParams_internal(compressionLevel, ZSTD_CONTENTSIZE_UNKNOWN, 0);
+    ZSTD_compressionParameters const cParams = ZSTD_getCParams_internal(compressionLevel, ZSTD_CONTENTSIZE_UNKNOWN, 0, ZSTD_cpm_noAttachDict);
     return ZSTD_estimateCCtxSize_usingCParams(cParams);
 }
 
@@ -1255,7 +1303,7 @@ size_t ZSTD_estimateCStreamSize_usingCCtxParams(const ZSTD_CCtx_params* params)
 {
     RETURN_ERROR_IF(params->nbWorkers > 0, GENERIC, "Estimate CCtx size is supported for single-threaded compression only.");
     {   ZSTD_compressionParameters const cParams =
-                ZSTD_getCParamsFromCCtxParams(params, ZSTD_CONTENTSIZE_UNKNOWN, 0);
+                ZSTD_getCParamsFromCCtxParams(params, ZSTD_CONTENTSIZE_UNKNOWN, 0, ZSTD_cpm_noAttachDict);
         size_t const blockSize = MIN(ZSTD_BLOCKSIZE_MAX, (size_t)1 << cParams.windowLog);
         size_t const inBuffSize = ((size_t)1 << cParams.windowLog) + blockSize;
         size_t const outBuffSize = ZSTD_compressBound(blockSize) + 1;
@@ -1274,7 +1322,7 @@ size_t ZSTD_estimateCStreamSize_usingCParams(ZSTD_compressionParameters cParams)
 
 static size_t ZSTD_estimateCStreamSize_internal(int compressionLevel)
 {
-    ZSTD_compressionParameters const cParams = ZSTD_getCParams_internal(compressionLevel, ZSTD_CONTENTSIZE_UNKNOWN, 0);
+    ZSTD_compressionParameters const cParams = ZSTD_getCParams_internal(compressionLevel, ZSTD_CONTENTSIZE_UNKNOWN, 0, ZSTD_cpm_noAttachDict);
     return ZSTD_estimateCStreamSize_usingCParams(cParams);
 }
 
@@ -1699,7 +1747,8 @@ ZSTD_resetCCtx_byAttachingCDict(ZSTD_CCtx* cctx,
             ZSTD_dedicatedDictSearch_revertCParams(&adjusted_cdict_cParams);
         }
 
-        params.cParams = ZSTD_adjustCParams_internal(adjusted_cdict_cParams, pledgedSrcSize, 0);
+        params.cParams = ZSTD_adjustCParams_internal(adjusted_cdict_cParams, pledgedSrcSize,
+                                                     cdict->dictContentSize, ZSTD_cpm_attachDict);
         params.cParams.windowLog = windowLog;
         FORWARD_IF_ERROR(ZSTD_resetCCtx_internal(cctx, params, pledgedSrcSize,
                                                  ZSTDcrp_makeClean, zbuff), "");
@@ -3235,7 +3284,7 @@ size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx,
 
 size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel)
 {
-    ZSTD_parameters const params = ZSTD_getParams_internal(compressionLevel, ZSTD_CONTENTSIZE_UNKNOWN, dictSize);
+    ZSTD_parameters const params = ZSTD_getParams_internal(compressionLevel, ZSTD_CONTENTSIZE_UNKNOWN, dictSize, ZSTD_cpm_noAttachDict);
     ZSTD_CCtx_params const cctxParams =
             ZSTD_assignParamsToCCtxParams(&cctx->requestedParams, ¶ms);
     DEBUGLOG(4, "ZSTD_compressBegin_usingDict (dictSize=%u)", (unsigned)dictSize);
@@ -3369,7 +3418,7 @@ size_t ZSTD_compress_usingDict(ZSTD_CCtx* cctx,
                          const void* dict, size_t dictSize,
                                int compressionLevel)
 {
-    ZSTD_parameters const params = ZSTD_getParams_internal(compressionLevel, srcSize, dict ? dictSize : 0);
+    ZSTD_parameters const params = ZSTD_getParams_internal(compressionLevel, srcSize, dict ? dictSize : 0, ZSTD_cpm_noAttachDict);
     ZSTD_CCtx_params cctxParams = ZSTD_assignParamsToCCtxParams(&cctx->requestedParams, ¶ms);
     DEBUGLOG(4, "ZSTD_compress_usingDict (srcSize=%u)", (unsigned)srcSize);
     assert(params.fParams.contentSizeFlag == 1);
@@ -3424,7 +3473,7 @@ size_t ZSTD_estimateCDictSize_advanced(
 
 size_t ZSTD_estimateCDictSize(size_t dictSize, int compressionLevel)
 {
-    ZSTD_compressionParameters const cParams = ZSTD_getCParams_internal(compressionLevel, ZSTD_CONTENTSIZE_UNKNOWN, dictSize);
+    ZSTD_compressionParameters const cParams = ZSTD_getCParams_internal(compressionLevel, ZSTD_CONTENTSIZE_UNKNOWN, dictSize, ZSTD_cpm_createCDict);
     return ZSTD_estimateCDictSize_advanced(dictSize, cParams, ZSTD_dlm_byCopy);
 }
 
@@ -3562,14 +3611,14 @@ ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_advanced2(
         ZSTD_overrideCParams(&cParams, &cctxParams.cParams);
     } else {
         cParams = ZSTD_getCParamsFromCCtxParams(
-            &cctxParams, ZSTD_CONTENTSIZE_UNKNOWN, dictSize);
+            &cctxParams, ZSTD_CONTENTSIZE_UNKNOWN, dictSize, ZSTD_cpm_createCDict);
     }
 
     if (!ZSTD_dedicatedDictSearch_isSupported(&cParams)) {
         /* Fall back to non-DDSS params */
         cctxParams.enableDedicatedDictSearch = 0;
         cParams = ZSTD_getCParamsFromCCtxParams(
-            &cctxParams, ZSTD_CONTENTSIZE_UNKNOWN, dictSize);
+            &cctxParams, ZSTD_CONTENTSIZE_UNKNOWN, dictSize, ZSTD_cpm_createCDict);
     }
 
     cctxParams.cParams = cParams;
@@ -3591,7 +3640,7 @@ ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_advanced2(
 
 ZSTD_CDict* ZSTD_createCDict(const void* dict, size_t dictSize, int compressionLevel)
 {
-    ZSTD_compressionParameters cParams = ZSTD_getCParams_internal(compressionLevel, ZSTD_CONTENTSIZE_UNKNOWN, dictSize);
+    ZSTD_compressionParameters cParams = ZSTD_getCParams_internal(compressionLevel, ZSTD_CONTENTSIZE_UNKNOWN, dictSize, ZSTD_cpm_createCDict);
     ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dict, dictSize,
                                                   ZSTD_dlm_byCopy, ZSTD_dct_auto,
                                                   cParams, ZSTD_defaultCMem);
@@ -3602,7 +3651,7 @@ ZSTD_CDict* ZSTD_createCDict(const void* dict, size_t dictSize, int compressionL
 
 ZSTD_CDict* ZSTD_createCDict_byReference(const void* dict, size_t dictSize, int compressionLevel)
 {
-    ZSTD_compressionParameters cParams = ZSTD_getCParams_internal(compressionLevel, ZSTD_CONTENTSIZE_UNKNOWN, dictSize);
+    ZSTD_compressionParameters cParams = ZSTD_getCParams_internal(compressionLevel, ZSTD_CONTENTSIZE_UNKNOWN, dictSize, ZSTD_cpm_createCDict);
     ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dict, dictSize,
                                      ZSTD_dlm_byRef, ZSTD_dct_auto,
                                      cParams, ZSTD_defaultCMem);
@@ -3792,6 +3841,14 @@ size_t ZSTD_CStreamOutSize(void)
     return ZSTD_compressBound(ZSTD_BLOCKSIZE_MAX) + ZSTD_blockHeaderSize + 4 /* 32-bits hash */ ;
 }
 
+static ZSTD_cParamMode_e ZSTD_getCParamMode(ZSTD_CDict const* cdict, ZSTD_CCtx_params const* params, U64 pledgedSrcSize)
+{
+    if (cdict != NULL && ZSTD_shouldAttachDict(cdict, params, pledgedSrcSize))
+        return ZSTD_cpm_attachDict;
+    else
+        return ZSTD_cpm_noAttachDict;
+}
+
 /* ZSTD_resetCStream():
  * pledgedSrcSize == 0 means "unknown" */
 size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pss)
@@ -4108,9 +4165,15 @@ size_t ZSTD_compressStream2( ZSTD_CCtx* cctx,
             params.compressionLevel = cctx->cdict->compressionLevel; /* let cdict take priority in terms of compression level */
         DEBUGLOG(4, "ZSTD_compressStream2 : transparent init stage");
         if (endOp == ZSTD_e_end) cctx->pledgedSrcSizePlusOne = input->size + 1;  /* auto-fix pledgedSrcSize */
-        params.cParams = ZSTD_getCParamsFromCCtxParams(
-                ¶ms, cctx->pledgedSrcSizePlusOne-1,
-                cctx->prefixDict.dict ? cctx->prefixDict.dictSize : (cctx->cdict ? cctx->cdict->dictContentSize : 0));
+        {
+            size_t const dictSize = prefixDict.dict
+                    ? prefixDict.dictSize
+                    : (cctx->cdict ? cctx->cdict->dictContentSize : 0);
+            ZSTD_cParamMode_e const mode = ZSTD_getCParamMode(cctx->cdict, ¶ms, cctx->pledgedSrcSizePlusOne - 1);
+            params.cParams = ZSTD_getCParamsFromCCtxParams(
+                    ¶ms, cctx->pledgedSrcSizePlusOne-1,
+                    dictSize, mode);
+        }
 
 
 #ifdef ZSTD_MULTITHREAD
@@ -4363,7 +4426,7 @@ static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEV
 
 static ZSTD_compressionParameters ZSTD_dedicatedDictSearch_getCParams(int const compressionLevel, size_t const dictSize)
 {
-    ZSTD_compressionParameters cParams = ZSTD_getCParams_internal(compressionLevel, 0, dictSize);
+    ZSTD_compressionParameters cParams = ZSTD_getCParams_internal(compressionLevel, 0, dictSize, ZSTD_cpm_createCDict);
     switch (cParams.strategy) {
         case ZSTD_fast:
         case ZSTD_dfast:
@@ -4412,15 +4475,34 @@ static void ZSTD_dedicatedDictSearch_revertCParams(
     }
 }
 
+static U64 ZSTD_getCParamRowSize(U64 srcSizeHint, size_t dictSize, ZSTD_cParamMode_e mode)
+{
+    switch (mode) {
+    case ZSTD_cpm_unknown:
+    case ZSTD_cpm_noAttachDict:
+    case ZSTD_cpm_createCDict:
+        break;
+    case ZSTD_cpm_attachDict:
+        dictSize = 0;
+        break;
+    default:
+        assert(0);
+        break;
+    }
+    {   int const unknown = srcSizeHint == ZSTD_CONTENTSIZE_UNKNOWN;
+        size_t const addedSize = unknown && dictSize > 0 ? 500 : 0;
+        return unknown && dictSize == 0 ? ZSTD_CONTENTSIZE_UNKNOWN : srcSizeHint+dictSize+addedSize;
+    }
+}
+
 /*! ZSTD_getCParams_internal() :
  * @return ZSTD_compressionParameters structure for a selected compression level, srcSize and dictSize.
  *  Note: srcSizeHint 0 means 0, use ZSTD_CONTENTSIZE_UNKNOWN for unknown.
- *        Use dictSize == 0 for unknown or unused. */
-static ZSTD_compressionParameters ZSTD_getCParams_internal(int compressionLevel, unsigned long long srcSizeHint, size_t dictSize)
+ *        Use dictSize == 0 for unknown or unused.
+ *  Note: `mode` controls how we treat the `dictSize`. See docs for `ZSTD_cParamMode_e`. */
+static ZSTD_compressionParameters ZSTD_getCParams_internal(int compressionLevel, unsigned long long srcSizeHint, size_t dictSize, ZSTD_cParamMode_e mode)
 {
-    int const unknown = srcSizeHint == ZSTD_CONTENTSIZE_UNKNOWN;
-    size_t const addedSize = unknown && dictSize > 0 ? 500 : 0;
-    U64 const rSize = unknown && dictSize == 0 ? ZSTD_CONTENTSIZE_UNKNOWN : srcSizeHint+dictSize+addedSize;
+    U64 const rSize = ZSTD_getCParamRowSize(srcSizeHint, dictSize, mode);
     U32 const tableID = (rSize <= 256 KB) + (rSize <= 128 KB) + (rSize <= 16 KB);
     int row = compressionLevel;
     DEBUGLOG(5, "ZSTD_getCParams_internal (cLevel=%i)", compressionLevel);
@@ -4430,7 +4512,7 @@ static ZSTD_compressionParameters ZSTD_getCParams_internal(int compressionLevel,
     {   ZSTD_compressionParameters cp = ZSTD_defaultCParameters[tableID][row];
         if (compressionLevel < 0) cp.targetLength = (unsigned)(-compressionLevel);   /* acceleration factor */
         /* refine parameters based on srcSize & dictSize */
-        return ZSTD_adjustCParams_internal(cp, srcSizeHint, dictSize);
+        return ZSTD_adjustCParams_internal(cp, srcSizeHint, dictSize, mode);
     }
 }
 
@@ -4440,16 +4522,16 @@ static ZSTD_compressionParameters ZSTD_getCParams_internal(int compressionLevel,
 ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, unsigned long long srcSizeHint, size_t dictSize)
 {
     if (srcSizeHint == 0) srcSizeHint = ZSTD_CONTENTSIZE_UNKNOWN;
-    return ZSTD_getCParams_internal(compressionLevel, srcSizeHint, dictSize);
+    return ZSTD_getCParams_internal(compressionLevel, srcSizeHint, dictSize, ZSTD_cpm_unknown);
 }
 
 /*! ZSTD_getParams() :
  *  same idea as ZSTD_getCParams()
  * @return a `ZSTD_parameters` structure (instead of `ZSTD_compressionParameters`).
  *  Fields of `ZSTD_frameParameters` are set to default values */
-static ZSTD_parameters ZSTD_getParams_internal(int compressionLevel, unsigned long long srcSizeHint, size_t dictSize) {
+static ZSTD_parameters ZSTD_getParams_internal(int compressionLevel, unsigned long long srcSizeHint, size_t dictSize, ZSTD_cParamMode_e mode) {
     ZSTD_parameters params;
-    ZSTD_compressionParameters const cParams = ZSTD_getCParams_internal(compressionLevel, srcSizeHint, dictSize);
+    ZSTD_compressionParameters const cParams = ZSTD_getCParams_internal(compressionLevel, srcSizeHint, dictSize, mode);
     DEBUGLOG(5, "ZSTD_getParams (cLevel=%i)", compressionLevel);
     ZSTD_memset(¶ms, 0, sizeof(params));
     params.cParams = cParams;
@@ -4463,5 +4545,5 @@ static ZSTD_parameters ZSTD_getParams_internal(int compressionLevel, unsigned lo
  *  Fields of `ZSTD_frameParameters` are set to default values */
 ZSTD_parameters ZSTD_getParams(int compressionLevel, unsigned long long srcSizeHint, size_t dictSize) {
     if (srcSizeHint == 0) srcSizeHint = ZSTD_CONTENTSIZE_UNKNOWN;
-    return ZSTD_getParams_internal(compressionLevel, srcSizeHint, dictSize);
+    return ZSTD_getParams_internal(compressionLevel, srcSizeHint, dictSize, ZSTD_cpm_unknown);
 }
diff --git a/lib/compress/zstd_compress_internal.h b/lib/compress/zstd_compress_internal.h
index 49ad87b1a..1fc9180aa 100644
--- a/lib/compress/zstd_compress_internal.h
+++ b/lib/compress/zstd_compress_internal.h
@@ -302,6 +302,25 @@ typedef enum {
     ZSTD_dedicatedDictSearch = 3
 } ZSTD_dictMode_e;
 
+typedef enum {
+    ZSTD_cpm_noAttachDict = 0,  /* Compression with ZSTD_noDict or ZSTD_extDict.
+                                 * In this mode we use both the srcSize and the dictSize
+                                 * when selecting and adjusting parameters.
+                                 */
+    ZSTD_cpm_attachDict = 1,    /* Compression with ZSTD_dictMatchState or ZSTD_dedicatedDictSearch.
+                                 * In this mode we only take the srcSize into account when selecting
+                                 * and adjusting parameters.
+                                 */
+    ZSTD_cpm_createCDict = 2,   /* Creating a CDict.
+                                 * In this mode we take both the source size and the dictionary size
+                                 * into account when selecting and adjusting the parameters.
+                                 */
+    ZSTD_cpm_unknown = 3,       /* ZSTD_getCParams, ZSTD_getParams, ZSTD_adjustParams.
+                                 * We don't know what these parameters are for. We default to the legacy
+                                 * behavior of taking both the source size and the dict size into account
+                                 * when selecting and adjusting parameters.
+                                 */
+} ZSTD_cParamMode_e;
 
 typedef size_t (*ZSTD_blockCompressor) (
         ZSTD_matchState_t* bs, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
@@ -1090,7 +1109,7 @@ void ZSTD_reset_compressedBlockState(ZSTD_compressedBlockState_t* bs);
  * Note: srcSizeHint == 0 means 0!
  */
 ZSTD_compressionParameters ZSTD_getCParamsFromCCtxParams(
-        const ZSTD_CCtx_params* CCtxParams, U64 srcSizeHint, size_t dictSize);
+        const ZSTD_CCtx_params* CCtxParams, U64 srcSizeHint, size_t dictSize, ZSTD_cParamMode_e mode);
 
 /*! ZSTD_initCStream_internal() :
  *  Private use only. Init streaming operation.
diff --git a/lib/compress/zstdmt_compress.c b/lib/compress/zstdmt_compress.c
index b809d602e..7bd8db484 100644
--- a/lib/compress/zstdmt_compress.c
+++ b/lib/compress/zstdmt_compress.c
@@ -1107,7 +1107,7 @@ void ZSTDMT_updateCParams_whileCompressing(ZSTDMT_CCtx* mtctx, const ZSTD_CCtx_p
     DEBUGLOG(5, "ZSTDMT_updateCParams_whileCompressing (level:%i)",
                 compressionLevel);
     mtctx->params.compressionLevel = compressionLevel;
-    {   ZSTD_compressionParameters cParams = ZSTD_getCParamsFromCCtxParams(cctxParams, ZSTD_CONTENTSIZE_UNKNOWN, 0);
+    {   ZSTD_compressionParameters cParams = ZSTD_getCParamsFromCCtxParams(cctxParams, ZSTD_CONTENTSIZE_UNKNOWN, 0, ZSTD_cpm_noAttachDict);
         cParams.windowLog = saved_wlog;
         mtctx->params.cParams = cParams;
     }
diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index 27a7bde12..838772c09 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -372,6 +372,19 @@ static int basicUnitTests(U32 const seed, double compressibility)
         DISPLAYLEVEL(3, "%u (OK) \n", vn);
     }
 
+    DISPLAYLEVEL(3, "test%3u : ZSTD_adjustCParams : ", testNb++);
+    {
+        ZSTD_compressionParameters params;
+        memset(¶ms, 0, sizeof(params));
+        params.windowLog = 10;
+        params.hashLog = 19;
+        params.chainLog = 19;
+        params = ZSTD_adjustCParams(params, 1000, 100000);
+        if (params.hashLog != 18) goto _output_error;
+        if (params.chainLog != 17) goto _output_error;
+    }
+    DISPLAYLEVEL(3, "OK \n");
+
     DISPLAYLEVEL(3, "test%3u : compress %u bytes : ", testNb++, (unsigned)CNBuffSize);
     {   ZSTD_CCtx* const cctx = ZSTD_createCCtx();
         if (cctx==NULL) goto _output_error;

From 7e6f91ed84e62513c2d12e7e40254704f593000c Mon Sep 17 00:00:00 2001
From: Nick Terrell 
Date: Mon, 12 Oct 2020 12:52:42 -0700
Subject: [PATCH 67/68] [minor] Improve docs and add an assert in response to
 review

---
 lib/compress/zstd_compress.c | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 69ff6512d..92010fd8a 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -1064,11 +1064,12 @@ U32 ZSTD_cycleLog(U32 hashLog, ZSTD_strategy strat)
 }
 
 /** ZSTD_dictAndWindowLog() :
- *  Returns an adjusted window log that is large enough to fit the source and the dictionary.
- *  The zstd format says that the entire dictionary is valid if one byte of the dictionary
- *  is within the window. So the hashLog and chainLog should be large enough to reference both
- *  the dictionary and the window. So we must use this adjusted dictAndWindowLog when downsizing
- *  the hashLog and windowLog.
+ * Returns an adjusted window log that is large enough to fit the source and the dictionary.
+ * The zstd format says that the entire dictionary is valid if one byte of the dictionary
+ * is within the window. So the hashLog and chainLog should be large enough to reference both
+ * the dictionary and the window. So we must use this adjusted dictAndWindowLog when downsizing
+ * the hashLog and windowLog.
+ * NOTE: srcSize must not be ZSTD_CONTENTSIZE_UNKNOWN.
  */
 static U32 ZSTD_dictAndWindowLog(U32 windowLog, U64 srcSize, U64 dictSize)
 {
@@ -1077,6 +1078,7 @@ static U32 ZSTD_dictAndWindowLog(U32 windowLog, U64 srcSize, U64 dictSize)
     if (dictSize == 0) {
         return windowLog;
     }
+    assert(windowLog <= ZSTD_WINDOWLOG_MAX);
     assert(srcSize != ZSTD_CONTENTSIZE_UNKNOWN); /* Handled in ZSTD_adjustCParams_internal() */
     {
         U64 const windowSize = 1ULL << windowLog;

From 43c0054405a5b10c82b4a1520b4be0a4ab1f1b7e Mon Sep 17 00:00:00 2001
From: Warner Losh 
Date: Tue, 13 Oct 2020 12:12:46 -0600
Subject: [PATCH 68/68] aarch64: use __ARM_NEON instead of __aarch64__ to
 control use of neon

There are compilation environments in aarch64 where NEON isn't
available. While these environments could define ZSTD_NO_INTRINSICS,
it's more fail-safe to use the more specific symbol to know if NEON
extensions are available.

__ARM_NEON is the proper symbol, defined in ARM C Language Extensions
Release 2.1 (https://developer.arm.com/documentation/ihi0053/d/). Some
sources suggest __ARM_NEON__, but that's the obsolete spelling from
prior versions of the standard.

Signed-off-by: Warner Losh 
---
 lib/common/zstd_internal.h | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/lib/common/zstd_internal.h b/lib/common/zstd_internal.h
index f57f8f83f..bfa98efaa 100644
--- a/lib/common/zstd_internal.h
+++ b/lib/common/zstd_internal.h
@@ -19,7 +19,7 @@
 /*-*************************************
 *  Dependencies
 ***************************************/
-#if !defined(ZSTD_NO_INTRINSICS) && defined(__aarch64__)
+#if !defined(ZSTD_NO_INTRINSICS) && defined(__ARM_NEON)
 #include 
 #endif
 #include "compiler.h"
@@ -242,7 +242,7 @@ static UNUSED_ATTR const U32 OF_defaultNormLog = OF_DEFAULTNORMLOG;
 *  Shared functions to include for inlining
 *********************************************/
 static void ZSTD_copy8(void* dst, const void* src) {
-#if !defined(ZSTD_NO_INTRINSICS) && defined(__aarch64__)
+#if !defined(ZSTD_NO_INTRINSICS) && defined(__ARM_NEON)
     vst1_u8((uint8_t*)dst, vld1_u8((const uint8_t*)src));
 #else
     ZSTD_memcpy(dst, src, 8);
@@ -251,7 +251,7 @@ static void ZSTD_copy8(void* dst, const void* src) {
 
 #define COPY8(d,s) { ZSTD_copy8(d,s); d+=8; s+=8; }
 static void ZSTD_copy16(void* dst, const void* src) {
-#if !defined(ZSTD_NO_INTRINSICS) && defined(__aarch64__)
+#if !defined(ZSTD_NO_INTRINSICS) && defined(__ARM_NEON)
     vst1q_u8((uint8_t*)dst, vld1q_u8((const uint8_t*)src));
 #else
     ZSTD_memcpy(dst, src, 16);