ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

added benchmark for ZSTD_convertBlockSequences_wBlockDelim()

Browse Source
This commit is contained in:
Yann Collet
2024-12-20 10:37:00 -08:00
parent d48e330ae1
commit 95ad9e47ff
3 changed files with 94 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/compress/zstd_compress.c
+18 -17
View File
@@ -3240,12 +3240,6 @@ static size_t ZSTD_fastSequenceLengthSum(ZSTD_Sequence const* seqBuf, size_t seq
return litLenSum + matchLenSum;
}
typedef struct {
U32 idx; /* Index in array of ZSTD_Sequence */
U32 posInSequence; /* Position within sequence at idx */
size_t posInSrc; /* Number of bytes given by sequences provided so far */
} ZSTD_SequencePosition;
static size_t
ZSTD_transferSequences_wBlockDelim(ZSTD_CCtx* cctx,
ZSTD_SequencePosition* seqPos,
@@ -7103,7 +7097,7 @@ size_t ZSTD_compressSequences(ZSTD_CCtx* cctx,
* @blockSize must be exactly correct.
*/
FORCE_INLINE_TEMPLATE size_t
ZSTD_convertSequences_wBlockDelim_internal(ZSTD_CCtx* cctx,
ZSTD_convertBlockSequences_wBlockDelim_internal(ZSTD_CCtx* cctx,
ZSTD_SequencePosition* seqPos,
const ZSTD_Sequence* const inSeqs, size_t nbSequences,
size_t blockSize,
@@ -7200,32 +7194,39 @@ ZSTD_convertSequences_wBlockDelim_internal(ZSTD_CCtx* cctx,
return litConsumed;
}
typedef size_t (*ZSTD_transferSequencesOnly_f) (ZSTD_CCtx* cctx,
/* for tests only */
void CCTX_resetSeqStore(ZSTD_CCtx* cctx)
{
cctx->seqStore.sequences = cctx->seqStore.sequencesStart;
cctx->seqStore.lit = cctx->seqStore.litStart;
}
typedef size_t (*ZSTD_convertBlockSequences_f) (ZSTD_CCtx* cctx,
ZSTD_SequencePosition* seqPos,
const ZSTD_Sequence* const inSeqs, size_t nbSequences,
size_t blockSize,
int const repcodeResolution);
static size_t
ZSTD_convertSequences_wBlockDelim(ZSTD_CCtx* cctx,
size_t
ZSTD_convertBlockSequences_wBlockDelim(ZSTD_CCtx* cctx,
ZSTD_SequencePosition* seqPos,
const ZSTD_Sequence* const inSeqs, size_t nbSequences,
size_t blockSize,
int const repcodeResolution)
{
return ZSTD_convertSequences_wBlockDelim_internal(cctx,
return ZSTD_convertBlockSequences_wBlockDelim_internal(cctx,
seqPos, inSeqs, nbSequences, blockSize,
repcodeResolution, 0);
}
static size_t
ZSTD_convertSequences_wBlockDelim_andCheckSequences(ZSTD_CCtx* cctx,
ZSTD_convertBlockSequences_wBlockDelim_andCheckSequences(ZSTD_CCtx* cctx,
ZSTD_SequencePosition* seqPos,
const ZSTD_Sequence* const inSeqs, size_t nbSequences,
size_t blockSize,
int const repcodeResolution)
{
return ZSTD_convertSequences_wBlockDelim_internal(cctx,
return ZSTD_convertBlockSequences_wBlockDelim_internal(cctx,
seqPos, inSeqs, nbSequences, blockSize,
repcodeResolution, 1);
}
@@ -7241,9 +7242,9 @@ ZSTD_compressSequencesAndLiterals_internal(ZSTD_CCtx* cctx,
ZSTD_SequencePosition seqPos = {0, 0, 0};
BYTE* op = (BYTE*)dst;
int const repcodeResolution = (cctx->appliedParams.searchForExternalRepcodes == ZSTD_ps_enable);
const ZSTD_transferSequencesOnly_f transfer = cctx->appliedParams.validateSequences ?
ZSTD_convertSequences_wBlockDelim_andCheckSequences
: ZSTD_convertSequences_wBlockDelim;
const ZSTD_convertBlockSequences_f convertBlockSequences = cctx->appliedParams.validateSequences ?
ZSTD_convertBlockSequences_wBlockDelim_andCheckSequences
: ZSTD_convertBlockSequences_wBlockDelim;
DEBUGLOG(4, "ZSTD_compressSequencesAndLiterals_internal: nbSeqs=%zu, litSize=%zu", nbSequences, litSize);
if (cctx->appliedParams.blockDelimiters == ZSTD_sf_noBlockDelimiters) {
@@ -7271,7 +7272,7 @@ ZSTD_compressSequencesAndLiterals_internal(ZSTD_CCtx* cctx,
assert(blockSize <= remaining);
ZSTD_resetSeqStore(&cctx->seqStore);
litConsumed = transfer(cctx,
litConsumed = convertBlockSequences(cctx,
&seqPos,
inSeqs, nbSequences,
blockSize,
lib/compress/zstd_compress_internal.h
+16
View File
@@ -1522,6 +1522,22 @@ size_t ZSTD_loadCEntropy(ZSTD_compressedBlockState_t* bs, void* workspace,
void ZSTD_reset_compressedBlockState(ZSTD_compressedBlockState_t* bs);
typedef struct {
U32 idx; /* Index in array of ZSTD_Sequence */
U32 posInSequence; /* Position within sequence at idx */
size_t posInSrc; /* Number of bytes given by sequences provided so far */
} ZSTD_SequencePosition;
size_t
ZSTD_convertBlockSequences_wBlockDelim(ZSTD_CCtx* cctx,
ZSTD_SequencePosition* seqPos,
const ZSTD_Sequence* const inSeqs, size_t nbSequences,
size_t blockSize,
int const repcodeResolution);
/* for tests only */
void CCTX_resetSeqStore(ZSTD_CCtx* cctx);
/* ==============================================================
* Private declarations
* These prototypes shall only be called from within lib/compress
tests/fullbench.c
+60
View File
@@ -21,6 +21,7 @@
#include <assert.h>
#include "mem.h" /* U32 */
#include "compress/zstd_compress_internal.h"
#ifndef ZSTD_DLL_IMPORT
#include "zstd_internal.h" /* ZSTD_decodeSeqHeaders, ZSTD_blockHeaderSize, ZSTD_getcBlockSize, blockType_e, KB, MB */
#include "decompress/zstd_decompress_internal.h" /* ZSTD_DCtx struct */
@@ -627,6 +628,64 @@ local_compressSequencesAndLiterals(const void* input, size_t inputSize,
return ZSTD_compressSequencesAndLiterals(g_zcc, dst, dstCapacity, seqs, nbSeqs, literals, nbLiterals, srcSize);
}
static PrepResult prepConvertSequences(const void* src, size_t srcSize, int cLevel)
{
PrepResult r = PREPRESULT_INIT;
size_t const prepCapacity = srcSize * 4;
void* prepBuffer = malloc(prepCapacity);
void* sequencesStart = (char*)prepBuffer + 2*sizeof(unsigned);
ZSTD_Sequence* const seqs = sequencesStart;
size_t const seqsCapacity = prepCapacity / sizeof(ZSTD_Sequence);
size_t totalNbSeqs, nbSeqs, blockSize=0;
ZSTD_CCtx_reset(g_zcc, ZSTD_reset_session_and_parameters);
ZSTD_CCtx_setParameter(g_zcc, ZSTD_c_compressionLevel, cLevel);
totalNbSeqs = ZSTD_generateSequences(g_zcc, seqs, seqsCapacity, src, srcSize);
CONTROL(!ZSTD_isError(totalNbSeqs));
/* find nb sequences in first block */
{ size_t n;
for (n=0; n<totalNbSeqs; n++) {
if (seqs[n].matchLength == 0) break;
blockSize += seqs[n].litLength + seqs[n].matchLength;
}
blockSize += seqs[n].litLength;
nbSeqs = n+1;
#if 0
printf("found %zu sequences representing a first block of size %zu\n", nbSeqs, blockSize);
#endif
}
/* generate benchmarked input */
CONTROL(blockSize < UINT_MAX);
MEM_write32(prepBuffer, (U32)blockSize);
MEM_write32((char*)prepBuffer+4, (U32)nbSeqs);
memcpy(seqs + nbSeqs, src, srcSize);
r.prepBuffer = prepBuffer;
r.prepSize = 8 + sizeof(ZSTD_Sequence)*nbSeqs + srcSize;
r.fixedOrigSize = blockSize;
return r;
}
static size_t
local_convertSequences(const void* input, size_t inputSize,
void* dst, size_t dstCapacity,
void* payload)
{
ZSTD_SequencePosition seqPos = { 0, 0 , 0 };
const char* ip = input;
size_t const blockSize = MEM_read32(ip);
size_t const nbSeqs = MEM_read32(ip+=4);
const ZSTD_Sequence* seqs = (const ZSTD_Sequence*)(const void*)(ip+=4);
ZSTD_CCtx_reset(g_zcc, ZSTD_reset_session_and_parameters);
CCTX_resetSeqStore(g_zcc);
ZSTD_CCtx_setParameter(g_zcc, ZSTD_c_blockDelimiters, ZSTD_sf_explicitBlockDelimiters);
# if 0 /* for tests */
ZSTD_CCtx_setParameter(g_zcc, ZSTD_c_repcodeResolution, ZSTD_ps_enable);
#endif
assert(8 + nbSeqs * sizeof(ZSTD_Sequence) == inputSize); (void)inputSize;
(void)dst; (void)dstCapacity;
(void)payload;
return ZSTD_convertBlockSequences_wBlockDelim(g_zcc, &seqPos, seqs, nbSeqs, blockSize, 0);
}
static PrepResult prepCopy(const void* src, size_t srcSize, int cLevel)
@@ -684,6 +743,7 @@ static BenchScenario kScenarios[] = {
{ "compressStream2, -T2, end", NULL, local_ZSTD_compress_generic_T2_end },
{ "compressSequences", prepSequences, local_compressSequences },
{ "compressSequencesAndLiterals", prepSequencesAndLiterals, local_compressSequencesAndLiterals },
{ "convertSequences (1st block)", prepConvertSequences, local_convertSequences },
#ifndef ZSTD_DLL_IMPORT
{ "decodeLiteralsHeader (1st block)", prepLiterals, local_ZSTD_decodeLiteralsHeader },
{ "decodeLiteralsBlock (1st block)", prepLiterals, local_ZSTD_decodeLiteralsBlock },