ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

ZSTD_compressSequencesAndLiterals requires srcSize as parameter

Browse Source 
this makes it possible to adjust windowSize to its tightest.
This commit is contained in:
Yann Collet
2024-12-20 10:37:00 -08:00
parent ad023b392f
commit 0a54f6f288
5 changed files with 12 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files
doc/zstd_manual.html
+1 -1
View File
@@ -1421,7 +1421,7 @@ ZSTD_compressSequences(ZSTD_CCtx* cctx,
ZSTD_compressSequencesAndLiterals(ZSTD_CCtx* cctx,
void* dst, size_t dstCapacity,
const ZSTD_Sequence* inSeqs, size_t nbSequences,
const void* literals, size_t litSize);
const void* literals, size_t litSize, size_t srcSize);
</b><p> This is a variant of ZSTD_compressSequences() which,
instead of receiving (src,srcSize) as input parameter, receives (literals,litSize),
aka all the literals, already extracted and laid out into a single continuous buffer.
lib/compress/zstd_compress.c
+3 -3
View File
@@ -7291,7 +7291,7 @@ size_t
ZSTD_compressSequencesAndLiterals(ZSTD_CCtx* cctx,
void* dst, size_t dstCapacity,
const ZSTD_Sequence* inSeqs, size_t inSeqsSize,
const void* literals, size_t litSize)
const void* literals, size_t litSize, size_t srcSize)
{
BYTE* op = (BYTE*)dst;
size_t cSize = 0;
@@ -7299,7 +7299,7 @@ ZSTD_compressSequencesAndLiterals(ZSTD_CCtx* cctx,
/* Transparent initialization stage, same as compressStream2() */
DEBUGLOG(4, "ZSTD_compressSequencesAndLiterals (dstCapacity=%zu)", dstCapacity);
assert(cctx != NULL);
FORWARD_IF_ERROR(ZSTD_CCtx_init_compressStream2(cctx, ZSTD_e_continue, 0), "CCtx initialization failed");
FORWARD_IF_ERROR(ZSTD_CCtx_init_compressStream2(cctx, ZSTD_e_end, srcSize), "CCtx initialization failed");
if (cctx->appliedParams.blockDelimiters == ZSTD_sf_noBlockDelimiters) {
RETURN_ERROR(frameParameter_unsupported, "This mode is only compatible with explicit delimiters");
@@ -7310,7 +7310,7 @@ ZSTD_compressSequencesAndLiterals(ZSTD_CCtx* cctx,
/* Begin writing output, starting with frame header */
{ size_t const frameHeaderSize = ZSTD_writeFrameHeader(op, dstCapacity,
&cctx->appliedParams, ZSTD_CONTENTSIZE_UNKNOWN, cctx->dictID);
&cctx->appliedParams, srcSize, cctx->dictID);
op += frameHeaderSize;
assert(frameHeaderSize <= dstCapacity);
dstCapacity -= frameHeaderSize;
lib/zstd.h
+1 -2
View File
@@ -1675,7 +1675,6 @@ ZSTD_compressSequences(ZSTD_CCtx* cctx,
* but it also features the following limitations:
* - Only supports explicit delimiter mode
* - Not compatible with frame checksum, which must disabled
* - Does not write the content size in frame header
* - If any block is incompressible, will fail and return an error
* - @litSize must be == sum of all @.litLength fields in @inSeqs. Any discrepancy will generate an error.
* - the buffer @literals must be larger than @litSize by at least 8 bytes.
@@ -1685,7 +1684,7 @@ ZSTDLIB_STATIC_API size_t
ZSTD_compressSequencesAndLiterals(ZSTD_CCtx* cctx,
void* dst, size_t dstCapacity,
const ZSTD_Sequence* inSeqs, size_t nbSequences,
const void* literals, size_t litSize);
const void* literals, size_t litSize, size_t srcSize);
/*! ZSTD_writeSkippableFrame() :
tests/fullbench.c
+4 -4
View File
@@ -608,8 +608,8 @@ static PrepResult prepSequencesAndLiterals(const void* src, size_t srcSize, int
static size_t
local_compressSequencesAndLiterals(const void* input, size_t inputSize,
void* dst, size_t dstCapacity,
void* payload)
void* dst, size_t dstCapacity,
void* payload)
{
const char* ip = input;
size_t srcSize = MEM_read32(ip);
@@ -623,9 +623,9 @@ local_compressSequencesAndLiterals(const void* input, size_t inputSize,
ZSTD_CCtx_setParameter(g_zcc, ZSTD_c_repcodeResolution, ZSTD_ps_enable);
#endif
assert(12 + nbSeqs * sizeof(ZSTD_Sequence) + nbLiterals == inputSize); (void)inputSize;
(void)payload; (void)srcSize;
(void)payload;
return ZSTD_compressSequencesAndLiterals(g_zcc, dst, dstCapacity, seqs, nbSeqs, literals, nbLiterals);
return ZSTD_compressSequencesAndLiterals(g_zcc, dst, dstCapacity, seqs, nbSeqs, literals, nbLiterals, srcSize);
}
static PrepResult prepConvertSequences(const void* src, size_t srcSize, int cLevel)
tests/fuzzer.c
+3 -3
View File
@@ -3909,21 +3909,21 @@ static int basicUnitTests(U32 const seed, double compressibility)
FUZ_transferLiterals(litBuffer, decompressSize, CNBuffer, srcSize, seqs, nbSeqs);
/* not enough literals: must fail */
compressedSize = ZSTD_compressSequencesAndLiterals(cctx, dst, dstCapacity, seqs, nbSeqs, src, litSize-1);
compressedSize = ZSTD_compressSequencesAndLiterals(cctx, dst, dstCapacity, seqs, nbSeqs, src, litSize-1, srcSize);
if (!ZSTD_isError(compressedSize)) {
DISPLAY("ZSTD_compressSequencesAndLiterals() should have failed: not enough literals provided\n");
goto _output_error;
}
/* too many literals: must fail */
compressedSize = ZSTD_compressSequencesAndLiterals(cctx, dst, dstCapacity, seqs, nbSeqs, src, litSize+1);
compressedSize = ZSTD_compressSequencesAndLiterals(cctx, dst, dstCapacity, seqs, nbSeqs, src, litSize+1, srcSize);
if (!ZSTD_isError(compressedSize)) {
DISPLAY("ZSTD_compressSequencesAndLiterals() should have failed: too many literals provided\n");
goto _output_error;
}
/* correct amount of literals: should compress successfully */
compressedSize = ZSTD_compressSequencesAndLiterals(cctx, dst, dstCapacity, seqs, nbSeqs, litBuffer, litSize);
compressedSize = ZSTD_compressSequencesAndLiterals(cctx, dst, dstCapacity, seqs, nbSeqs, litBuffer, litSize, srcSize);
if (ZSTD_isError(compressedSize)) {
DISPLAY("Error in ZSTD_compressSequencesAndLiterals()\n");
goto _output_error;