From 6309be677c21c7fef359569a3e747a3934ab35c3 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 15 Oct 2019 16:09:18 -0700 Subject: [PATCH 1/2] minor comments & refactoring --- programs/benchzstd.c | 39 +++++++++++++++++++-------------------- programs/benchzstd.h | 1 - 2 files changed, 19 insertions(+), 21 deletions(-) diff --git a/programs/benchzstd.c b/programs/benchzstd.c index 263dc0888..7439677c7 100644 --- a/programs/benchzstd.c +++ b/programs/benchzstd.c @@ -88,7 +88,7 @@ static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER; #endif #define DEBUGOUTPUT(...) { if (DEBUG) DISPLAY(__VA_ARGS__); } -#define EXM_THROW_INT(errorNum, ...) { \ +#define RETURN_ERROR_INT(errorNum, ...) { \ DEBUGOUTPUT("%s: %i: \n", __FILE__, __LINE__); \ DISPLAYLEVEL(1, "Error %i : ", errorNum); \ DISPLAYLEVEL(1, __VA_ARGS__); \ @@ -401,9 +401,9 @@ BMK_benchMemAdvancedNoAlloc( BMK_initCCtxArgs cctxprep; BMK_initDCtxArgs dctxprep; - cbp.benchFn = local_defaultCompress; + cbp.benchFn = local_defaultCompress; /* ZSTD_compress2 */ cbp.benchPayload = cctx; - cbp.initFn = local_initCCtx; + cbp.initFn = local_initCCtx; /* BMK_initCCtx */ cbp.initPayload = &cctxprep; cbp.errorFn = ZSTD_isError; cbp.blockCount = nbBlocks; @@ -534,8 +534,8 @@ BMK_benchMemAdvancedNoAlloc( if (u==srcSize-1) { /* should never happen */ DISPLAY("no difference detected\n"); } - } - } + } /* for (u=0; umode == BMK_both) && (crcOrig!=crcCheck)) */ } /* CRC Checking */ if (displayLevel == 1) { /* hidden display mode -q, used by python speed benchmark */ @@ -754,8 +754,7 @@ static int BMK_loadFiles(void* buffer, size_t bufferSize, size_t pos = 0, totalSize = 0; unsigned n; for (n=0; n bufferSize-pos) fileSize = bufferSize-pos, nbFiles=n; /* buffer too small - stop after this file */ - { size_t const readSize = fread(((char*)buffer)+pos, 1, (size_t)fileSize, f); - if (readSize != (size_t)fileSize) EXM_THROW_INT(11, "could not read %s", fileNamesTable[n]); - pos += readSize; - } - fileSizes[n] = (size_t)fileSize; - totalSize += (size_t)fileSize; - fclose(f); - } + { FILE* const f = fopen(fileNamesTable[n], "rb"); + if (f==NULL) RETURN_ERROR_INT(10, "impossible to open file %s", fileNamesTable[n]); + DISPLAYUPDATE(2, "Loading %s... \r", fileNamesTable[n]); + if (fileSize > bufferSize-pos) fileSize = bufferSize-pos, nbFiles=n; /* buffer too small - stop after this file */ + { size_t const readSize = fread(((char*)buffer)+pos, 1, (size_t)fileSize, f); + if (readSize != (size_t)fileSize) RETURN_ERROR_INT(11, "could not read %s", fileNamesTable[n]); + pos += readSize; + } + fileSizes[n] = (size_t)fileSize; + totalSize += (size_t)fileSize; + fclose(f); + } } - if (totalSize == 0) EXM_THROW_INT(12, "no data to bench"); + if (totalSize == 0) RETURN_ERROR_INT(12, "no data to bench"); return 0; } diff --git a/programs/benchzstd.h b/programs/benchzstd.h index 2c7627713..ef7d9fb11 100644 --- a/programs/benchzstd.h +++ b/programs/benchzstd.h @@ -205,7 +205,6 @@ BMK_benchOutcome_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize, - #endif /* BENCH_ZSTD_H_3242387 */ #if defined (__cplusplus) From 2d5201b0ab16b74e6aa1b8fa3079d3a5eb4f7eaf Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 16 Oct 2019 14:51:33 -0700 Subject: [PATCH 2/2] removed wildcopy8() which is no longer used, noticed by @davidbolvansky --- doc/zstd_manual.html | 27 +++++++++++++++++++++++++++ lib/common/zstd_internal.h | 14 -------------- 2 files changed, 27 insertions(+), 14 deletions(-) diff --git a/doc/zstd_manual.html b/doc/zstd_manual.html index 79b9d0231..7a1b457d3 100644 --- a/doc/zstd_manual.html +++ b/doc/zstd_manual.html @@ -866,6 +866,24 @@ size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
+
typedef struct {
+    unsigned int matchPos; /* Match pos in dst */
+    /* If seqDef.offset > 3, then this is seqDef.offset - 3
+     * If seqDef.offset < 3, then this is the corresponding repeat offset
+     * But if seqDef.offset < 3 and litLength == 0, this is the
+     *   repeat offset before the corresponding repeat offset
+     * And if seqDef.offset == 3 and litLength == 0, this is the
+     *   most recent repeat offset - 1
+     */
+    unsigned int offset;
+    unsigned int litLength; /* Literal length */
+    unsigned int matchLength; /* Match length */
+    /* 0 when seq not rep and seqDef.offset otherwise
+     * when litLength == 0 this will be <= 4, otherwise <= 3 like normal
+     */
+    unsigned int rep;
+} ZSTD_Sequence;
+

typedef struct {
     unsigned windowLog;       /**< largest match distance : larger == more compression, more memory needed during decompression */
     unsigned chainLog;        /**< fully searched segment : larger == more compression, slower, more memory (useless for fast) */
@@ -1001,6 +1019,15 @@ size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
            or an error code (if srcSize is too small)

+
size_t ZSTD_getSequences(ZSTD_CCtx* zc, ZSTD_Sequence* outSeqs,
+    size_t outSeqsSize, const void* src, size_t srcSize);
+
 Extract sequences from the sequence store
+ zc can be used to insert custom compression params.
+ This function invokes ZSTD_compress2
+ @return : number of sequences extracted
+ 
+

+

Memory management


 
 
size_t ZSTD_estimateCCtxSize(int compressionLevel);
diff --git a/lib/common/zstd_internal.h b/lib/common/zstd_internal.h
index f791c5b38..13420bd8a 100644
--- a/lib/common/zstd_internal.h
+++ b/lib/common/zstd_internal.h
@@ -247,20 +247,6 @@ void ZSTD_wildcopy(void* dst, const void* src, ptrdiff_t length, ZSTD_overlap_e
     }
 }
 
-/*! ZSTD_wildcopy8() :
- *  The same as ZSTD_wildcopy(), but it can only overwrite 8 bytes, and works for
- *  overlapping buffers that are at least 8 bytes apart.
- */
-MEM_STATIC void ZSTD_wildcopy8(void* dst, const void* src, ptrdiff_t length)
-{
-    const BYTE* ip = (const BYTE*)src;
-    BYTE* op = (BYTE*)dst;
-    BYTE* const oend = (BYTE*)op + length;
-    do {
-        COPY8(op, ip);
-    } while (op < oend);
-}
-
 
 /*-*******************************************
 *  Private declarations