ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

improved speed of the Sequences converter

Browse Source
This commit is contained in:
Yann Collet
2024-12-20 10:37:00 -08:00
parent 95ad9e47ff
commit 12c47d3262
7 changed files with 116 additions and 179 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/Makefile
+3 -3
View File
@@ -26,6 +26,9 @@ export ZSTD_LEGACY_SUPPORT
DEBUGLEVEL ?= 2
export DEBUGLEVEL # transmit value to sub-makefiles
.PHONY: default
default: fullbench
LIBZSTD_MK_DIR := ../lib
include $(LIBZSTD_MK_DIR)/libzstd.mk
@@ -78,9 +81,6 @@ FUZZERTEST ?= -T200s
ZSTDRTTEST = --test-large-data
DECODECORPUS_TESTTIME ?= -T30
.PHONY: default
default: fullbench
.PHONY: all
all: fullbench fuzzer zstreamtest paramgrill datagen decodecorpus roundTripCrash poolTests
tests/fullbench.c
+5 -6
View File
@@ -623,9 +623,9 @@ local_compressSequencesAndLiterals(const void* input, size_t inputSize,
ZSTD_CCtx_setParameter(g_zcc, ZSTD_c_repcodeResolution, ZSTD_ps_enable);
#endif
assert(12 + nbSeqs * sizeof(ZSTD_Sequence) + nbLiterals == inputSize); (void)inputSize;
(void)payload;
(void)payload; (void)srcSize;
return ZSTD_compressSequencesAndLiterals(g_zcc, dst, dstCapacity, seqs, nbSeqs, literals, nbLiterals, srcSize);
return ZSTD_compressSequencesAndLiterals(g_zcc, dst, dstCapacity, seqs, nbSeqs, literals, nbLiterals);
}
static PrepResult prepConvertSequences(const void* src, size_t srcSize, int cLevel)
@@ -669,22 +669,21 @@ local_convertSequences(const void* input, size_t inputSize,
void* dst, size_t dstCapacity,
void* payload)
{
ZSTD_SequencePosition seqPos = { 0, 0 , 0 };
const char* ip = input;
size_t const blockSize = MEM_read32(ip);
size_t const nbSeqs = MEM_read32(ip+=4);
const ZSTD_Sequence* seqs = (const ZSTD_Sequence*)(const void*)(ip+=4);
ZSTD_CCtx_reset(g_zcc, ZSTD_reset_session_and_parameters);
CCTX_resetSeqStore(g_zcc);
ZSTD_resetSeqStore(&g_zcc->seqStore);
ZSTD_CCtx_setParameter(g_zcc, ZSTD_c_blockDelimiters, ZSTD_sf_explicitBlockDelimiters);
# if 0 /* for tests */
ZSTD_CCtx_setParameter(g_zcc, ZSTD_c_repcodeResolution, ZSTD_ps_enable);
#endif
assert(8 + nbSeqs * sizeof(ZSTD_Sequence) == inputSize); (void)inputSize;
(void)dst; (void)dstCapacity;
(void)payload;
(void)payload; (void)blockSize;
return ZSTD_convertBlockSequences_wBlockDelim(g_zcc, &seqPos, seqs, nbSeqs, blockSize, 0);
return ZSTD_convertBlockSequences(g_zcc, seqs, nbSeqs, 0);
}
tests/fuzzer.c
+3 -17
View File
@@ -3909,35 +3909,21 @@ static int basicUnitTests(U32 const seed, double compressibility)
FUZ_transferLiterals(litBuffer, decompressSize, CNBuffer, srcSize, seqs, nbSeqs);
/* not enough literals: must fail */
compressedSize = ZSTD_compressSequencesAndLiterals(cctx, dst, dstCapacity, seqs, nbSeqs, src, litSize-1, srcSize);
compressedSize = ZSTD_compressSequencesAndLiterals(cctx, dst, dstCapacity, seqs, nbSeqs, src, litSize-1);
if (!ZSTD_isError(compressedSize)) {
DISPLAY("ZSTD_compressSequencesAndLiterals() should have failed: not enough literals provided\n");
goto _output_error;
}
/* too many literals: must fail */
compressedSize = ZSTD_compressSequencesAndLiterals(cctx, dst, dstCapacity, seqs, nbSeqs, src, litSize+1, srcSize);
compressedSize = ZSTD_compressSequencesAndLiterals(cctx, dst, dstCapacity, seqs, nbSeqs, src, litSize+1);
if (!ZSTD_isError(compressedSize)) {
DISPLAY("ZSTD_compressSequencesAndLiterals() should have failed: too many literals provided\n");
goto _output_error;
}
/* too short srcSize: must fail */
compressedSize = ZSTD_compressSequencesAndLiterals(cctx, dst, dstCapacity, seqs, nbSeqs, src, litSize, srcSize-1);
if (!ZSTD_isError(compressedSize)) {
DISPLAY("ZSTD_compressSequencesAndLiterals() should have failed: srcSize is too short\n");
goto _output_error;
}
/* too large srcSize: must fail */
compressedSize = ZSTD_compressSequencesAndLiterals(cctx, dst, dstCapacity, seqs, nbSeqs, src, litSize, srcSize+1);
if (!ZSTD_isError(compressedSize)) {
DISPLAY("ZSTD_compressSequencesAndLiterals() should have failed: srcSize is too short\n");
goto _output_error;
}
/* correct amount of literals: should compress successfully */
compressedSize = ZSTD_compressSequencesAndLiterals(cctx, dst, dstCapacity, seqs, nbSeqs, litBuffer, litSize, srcSize);
compressedSize = ZSTD_compressSequencesAndLiterals(cctx, dst, dstCapacity, seqs, nbSeqs, litBuffer, litSize);
if (ZSTD_isError(compressedSize)) {
DISPLAY("Error in ZSTD_compressSequencesAndLiterals()\n");
goto _output_error;