ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

add the compressSequences() benchmark scenario

Browse Source
This commit is contained in:
Yann Collet
2024-12-20 10:36:59 -08:00
parent c050ae4fb8
commit 8ab04097ed
4 changed files with 52 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/compress/zstd_compress.c
+1 -1
View File
@@ -7058,7 +7058,7 @@ size_t ZSTD_compressSequences(ZSTD_CCtx* cctx,
size_t cSize = 0;
/* Transparent initialization stage, same as compressStream2() */
DEBUGLOG(4, "ZSTD_compressSequences (dstCapacity=%zu)", dstCapacity);
DEBUGLOG(4, "ZSTD_compressSequences (nbSeqs=%zu,dstCapacity=%zu)", inSeqsSize, dstCapacity);
assert(cctx != NULL);
FORWARD_IF_ERROR(ZSTD_CCtx_init_compressStream2(cctx, ZSTD_e_end, srcSize), "CCtx initialization failed");
lib/compress/zstd_compress_internal.h
-8
View File
@@ -741,14 +741,6 @@ ZSTD_storeSeqOnly(SeqStore_t* seqStorePtr,
U32 offBase,
size_t matchLength)
{
#if defined(DEBUGLEVEL) && (DEBUGLEVEL >= 6)
static const BYTE* g_start = NULL;
if (g_start==NULL) g_start = (const BYTE*)literals; /* note : index only works for compression within a single segment */
{ U32 const pos = (U32)((const BYTE*)literals - g_start);
DEBUGLOG(6, "Cpos%7u :%3u literals, match%4u bytes at offBase%7u",
pos, (U32)litLength, (U32)matchLength, (U32)offBase);
}
#endif
assert((size_t)(seqStorePtr->sequences - seqStorePtr->sequencesStart) < seqStorePtr->maxNbSeq);
/* update seqStorePtr->lit, so that we know how many literals were or will be copied */
assert(seqStorePtr->maxNbLit <= 128 KB);
lib/zstd.h
+3 -3
View File
@@ -1328,7 +1328,7 @@ typedef struct {
*
* Note: This field is optional. ZSTD_generateSequences() will calculate the value of
* 'rep', but repeat offsets do not necessarily need to be calculated from an external
* sequence provider's perspective. For example, ZSTD_compressSequences() does not
* sequence provider perspective. For example, ZSTD_compressSequences() does not
* use this 'rep' field at all (as of now).
*/
} ZSTD_Sequence;
@@ -1588,7 +1588,7 @@ ZSTDLIB_STATIC_API size_t ZSTD_sequenceBound(size_t srcSize);
* @param zc The compression context to be used for ZSTD_compress2(). Set any
* compression parameters you need on this context.
* @param outSeqs The output sequences buffer of size @p outSeqsSize
* @param outSeqsSize The size of the output sequences buffer.
* @param outSeqsCapacity The size of the output sequences buffer.
* ZSTD_sequenceBound(srcSize) is an upper bound on the number
* of sequences that can be generated.
* @param src The source buffer to generate sequences from of size @p srcSize.
@@ -1606,7 +1606,7 @@ ZSTDLIB_STATIC_API size_t ZSTD_sequenceBound(size_t srcSize);
ZSTD_DEPRECATED("For debugging only, will be replaced by ZSTD_extractSequences()")
ZSTDLIB_STATIC_API size_t
ZSTD_generateSequences(ZSTD_CCtx* zc,
ZSTD_Sequence* outSeqs, size_t outSeqsSize,
ZSTD_Sequence* outSeqs, size_t outSeqsCapacity,
const void* src, size_t srcSize);
/*! ZSTD_mergeBlockDelimiters() :
tests/fullbench.c
+48 -4
View File
@@ -12,6 +12,7 @@
/*_************************************
* Includes
**************************************/
#include <limits.h>
#define ZSTD_DISABLE_DEPRECATE_WARNINGS /* No deprecation warnings, we still bench some deprecated functions */
#include "util.h" /* Compiler options, UTIL_GetFileSize */
#include <stdlib.h> /* malloc */
@@ -272,7 +273,7 @@ local_ZSTD_decodeLiteralsHeader(const void* src, size_t srcSize, void* dst, size
return ZSTD_decodeLiteralsHeader(g_zdc, src, srcSize);
}
static PrepResult prepSequences(const void* src, size_t srcSize, int cLevel)
static PrepResult prepSequences1stBlock(const void* src, size_t srcSize, int cLevel)
{
PrepResult r = PREPRESULT_INIT;
size_t const dstCapacity = srcSize;
@@ -456,7 +457,6 @@ local_ZSTD_decompressStream(const void* src, size_t srcSize,
return buffOut.pos;
}
#ifndef ZSTD_DLL_IMPORT
static size_t local_ZSTD_compressContinue(const void* src, size_t srcSize,
void* dst, size_t dstCapacity,
void* payload)
@@ -523,11 +523,54 @@ static size_t local_ZSTD_decompressContinue(const void* src, size_t srcSize,
return regeneratedSize;
}
#endif
static PrepResult prepSequences(const void* src, size_t srcSize, int cLevel)
{
PrepResult r = PREPRESULT_INIT;
size_t const dstCapacity = ZSTD_compressBound(srcSize);
void* const dst = malloc(dstCapacity);
size_t const prepCapacity = dstCapacity * 4;
void* prepBuffer = malloc(prepCapacity);
void* sequencesStart = (char*)prepBuffer + 2*sizeof(unsigned);
ZSTD_Sequence* const seqs = sequencesStart;
size_t const seqsCapacity = prepCapacity / sizeof(ZSTD_Sequence);
size_t nbSeqs;
ZSTD_CCtx_reset(g_zcc, ZSTD_reset_session_and_parameters);
ZSTD_CCtx_setParameter(g_zcc, ZSTD_c_compressionLevel, cLevel);
nbSeqs = ZSTD_generateSequences(g_zcc, seqs, seqsCapacity, src, srcSize);
CONTROL(srcSize < UINT_MAX);
MEM_write32(prepBuffer, (U32)srcSize);
MEM_write32((char*)prepBuffer+4, (U32)nbSeqs);
memcpy(seqs + nbSeqs, src, srcSize);
r.prepBuffer = prepBuffer;
r.prepSize = 8 + sizeof(ZSTD_Sequence)*nbSeqs + srcSize;
r.dst = dst;
r.dstCapacity = dstCapacity;
return r;
}
static size_t local_compressSequences(const void* input, size_t inputSize,
void* dst, size_t dstCapacity,
void* payload)
{
const char* ip = input;
size_t srcSize = MEM_read32(ip);
size_t nbSeqs = MEM_read32(ip+=4);
const ZSTD_Sequence* seqs = (const ZSTD_Sequence*)(const void*)(ip+=4);
const void* src = (ip+=nbSeqs * sizeof(ZSTD_Sequence));
ZSTD_CCtx_reset(g_zcc, ZSTD_reset_session_and_parameters);
ZSTD_CCtx_setParameter(g_zcc, ZSTD_c_blockDelimiters, ZSTD_sf_explicitBlockDelimiters);
assert(8 + nbSeqs * sizeof(ZSTD_Sequence) + srcSize == inputSize); (void)inputSize;
(void)payload;
return ZSTD_compressSequences(g_zcc, dst, dstCapacity, seqs, nbSeqs, src, srcSize);
}
static PrepResult prepCopy(const void* src, size_t srcSize, int cLevel)
{
PrepResult r = PREPRESULT_INIT;
(void)cLevel;
r.prepSize = srcSize;
r.prepBuffer = malloc(srcSize);
CONTROL(r.prepBuffer != NULL);
@@ -577,10 +620,11 @@ static BenchScenario kScenarios[] = {
{ "compressStream2, continue", NULL, local_ZSTD_compressStream2_continue },
{ "compressStream2, -T2, continue", NULL, local_ZSTD_compress_generic_T2_continue },
{ "compressStream2, -T2, end", NULL, local_ZSTD_compress_generic_T2_end },
{ "compressSequences", prepSequences, local_compressSequences },
#ifndef ZSTD_DLL_IMPORT
{ "decodeLiteralsHeader (1st block)", prepLiterals, local_ZSTD_decodeLiteralsHeader },
{ "decodeLiteralsBlock (1st block)", prepLiterals, local_ZSTD_decodeLiteralsBlock },
{ "decodeSeqHeaders (1st block)", prepSequences, local_ZSTD_decodeSeqHeaders },
{ "decodeSeqHeaders (1st block)", prepSequences1stBlock, local_ZSTD_decodeSeqHeaders },
#endif
};
#define NB_SCENARIOS (sizeof(kScenarios) / sizeof(kScenarios[0]))