ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Rename "External Matchfinder" to "Block-Level Sequence Producer" (#3484)

Browse Source 
* change "external matchfinder" to "external sequence producer"

* migrate contrib/ to new naming convention

* fix contrib build

* fix error message

* update debug strings

* fix def of invalid sequences in zstd.h

* nit

* update CHANGELOG

* fix .gitignore
This commit is contained in:
Elliot Gorokhovsky
2023-02-09 17:01:17 -05:00
committed by GitHub
parent 4aa3bc49da
commit ff42ed1582
20 changed files with 231 additions and 220 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/external_matchfinder.c
+9 -9
View File
@@ -17,8 +17,8 @@ static U32 const HLOG = 10;
static U32 const MLS = 4;
static U32 const BADIDX = 0xffffffff;
static size_t simpleExternalMatchFinder(
void* externalMatchState,
static size_t simpleSequenceProducer(
void* sequenceProducerState,
ZSTD_Sequence* outSeqs, size_t outSeqsCapacity,
const void* src, size_t srcSize,
const void* dict, size_t dictSize,
@@ -32,7 +32,7 @@ static size_t simpleExternalMatchFinder(
size_t seqCount = 0;
U32 hashTable[HSIZE];
(void)externalMatchState;
(void)sequenceProducerState;
(void)dict;
(void)dictSize;
(void)outSeqsCapacity;
@@ -80,15 +80,15 @@ static size_t simpleExternalMatchFinder(
return seqCount;
}
size_t zstreamExternalMatchFinder(
void* externalMatchState,
size_t zstreamSequenceProducer(
void* sequenceProducerState,
ZSTD_Sequence* outSeqs, size_t outSeqsCapacity,
const void* src, size_t srcSize,
const void* dict, size_t dictSize,
int compressionLevel,
size_t windowSize
) {
EMF_testCase const testCase = *((EMF_testCase*)externalMatchState);
EMF_testCase const testCase = *((EMF_testCase*)sequenceProducerState);
memset(outSeqs, 0, outSeqsCapacity);
switch (testCase) {
@@ -100,8 +100,8 @@ size_t zstreamExternalMatchFinder(
outSeqs[0].litLength = (U32)(srcSize);
return 1;
case EMF_LOTS_OF_SEQS:
return simpleExternalMatchFinder(
externalMatchState,
return simpleSequenceProducer(
sequenceProducerState,
outSeqs, outSeqsCapacity,
src, srcSize,
dict, dictSize,
@@ -135,6 +135,6 @@ size_t zstreamExternalMatchFinder(
return outSeqsCapacity + 1;
case EMF_BIG_ERROR:
default:
return ZSTD_EXTERNAL_MATCHFINDER_ERROR;
return ZSTD_SEQUENCE_PRODUCER_ERROR;
}
}
tests/external_matchfinder.h
+2 -2
View File
@@ -27,8 +27,8 @@ typedef enum {
EMF_INVALID_LAST_LITS = 8
} EMF_testCase;
size_t zstreamExternalMatchFinder(
void* externalMatchState,
size_t zstreamSequenceProducer(
void* sequenceProducerState,
ZSTD_Sequence* outSeqs, size_t outSeqsCapacity,
const void* src, size_t srcSize,
const void* dict, size_t dictSize,
tests/fuzz/Makefile
+2 -2
View File
@@ -35,8 +35,8 @@ PRGDIR = ../../programs
CONTRIBDIR = ../../contrib
# TODO(embg) make it possible to plug in an arbitrary matchfinder as a .o file
MATCHFINDER_DIR = $(CONTRIBDIR)/externalMatchfinder
MATCHFINDER_SRC = $(MATCHFINDER_DIR)/matchfinder.c
MATCHFINDER_DIR = $(CONTRIBDIR)/externalSequenceProducer
MATCHFINDER_SRC = $(MATCHFINDER_DIR)/sequence_producer.c
FUZZ_CPPFLAGS := -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/compress \
-I$(ZSTDDIR)/dictBuilder -I$(ZSTDDIR)/deprecated -I$(ZSTDDIR)/legacy \
tests/fuzz/zstd_helpers.c
+7 -7
View File
@@ -17,7 +17,7 @@
#include "fuzz_helpers.h"
#include "zstd.h"
#include "zdict.h"
#include "matchfinder.h"
#include "sequence_producer.h"
const int kMinClevel = -3;
const int kMaxClevel = 19;
@@ -71,13 +71,13 @@ ZSTD_parameters FUZZ_randomParams(size_t srcSize, FUZZ_dataProducer_t *producer)
return params;
}
static void setExternalMatchFinderParams(ZSTD_CCtx *cctx, FUZZ_dataProducer_t *producer) {
ZSTD_registerExternalMatchFinder(
static void setSequenceProducerParams(ZSTD_CCtx *cctx, FUZZ_dataProducer_t *producer) {
ZSTD_registerSequenceProducer(
cctx,
NULL,
simpleExternalMatchFinder
simpleSequenceProducer
);
setRand(cctx, ZSTD_c_enableMatchFinderFallback, 0, 1, producer);
setRand(cctx, ZSTD_c_enableSeqProducerFallback, 0, 1, producer);
FUZZ_ZASSERT(ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 0));
FUZZ_ZASSERT(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_disable));
}
@@ -138,9 +138,9 @@ void FUZZ_setRandomParameters(ZSTD_CCtx *cctx, size_t srcSize, FUZZ_dataProducer
}
if (FUZZ_dataProducer_uint32Range(producer, 0, 10) == 1) {
setExternalMatchFinderParams(cctx, producer);
setSequenceProducerParams(cctx, producer);
} else {
ZSTD_registerExternalMatchFinder(cctx, NULL, NULL);
ZSTD_registerSequenceProducer(cctx, NULL, NULL);
}
}
tests/zstreamtest.c
+23 -23
View File
@@ -40,7 +40,7 @@
#include "seqgen.h"
#include "util.h"
#include "timefn.h" /* UTIL_time_t, UTIL_clockSpanMicro, UTIL_getTime */
#include "external_matchfinder.h" /* zstreamExternalMatchFinder, EMF_testCase */
#include "external_matchfinder.h" /* zstreamSequenceProducer, EMF_testCase */
/*-************************************
* Constants
@@ -1856,14 +1856,14 @@ static int basicUnitTests(U32 seed, double compressibility, int bigTests)
}
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(3, "test%3i : External matchfinder API: ", testNb++);
DISPLAYLEVEL(3, "test%3i : Block-Level External Sequence Producer API: ", testNb++);
{
size_t const dstBufSize = ZSTD_compressBound(CNBufferSize);
BYTE* const dstBuf = (BYTE*)malloc(ZSTD_compressBound(dstBufSize));
size_t const checkBufSize = CNBufferSize;
BYTE* const checkBuf = (BYTE*)malloc(checkBufSize);
int enableFallback;
EMF_testCase externalMatchState;
EMF_testCase sequenceProducerState;
CHECK(dstBuf == NULL || checkBuf == NULL, "allocation failed");
@@ -1871,7 +1871,7 @@ static int basicUnitTests(U32 seed, double compressibility, int bigTests)
/* Reference external matchfinder outside the test loop to
* check that the reference is preserved across compressions */
ZSTD_registerExternalMatchFinder(zc, &externalMatchState, zstreamExternalMatchFinder);
ZSTD_registerSequenceProducer(zc, &sequenceProducerState, zstreamSequenceProducer);
for (enableFallback = 0; enableFallback <= 1; enableFallback++) {
size_t testCaseId;
@@ -1892,9 +1892,9 @@ static int basicUnitTests(U32 seed, double compressibility, int bigTests)
ZSTD_ErrorCode const errorCodes[] = {
ZSTD_error_no_error,
ZSTD_error_no_error,
ZSTD_error_externalMatchFinder_failed,
ZSTD_error_externalMatchFinder_failed,
ZSTD_error_externalMatchFinder_failed,
ZSTD_error_sequenceProducer_failed,
ZSTD_error_sequenceProducer_failed,
ZSTD_error_sequenceProducer_failed,
ZSTD_error_externalSequences_invalid,
ZSTD_error_externalSequences_invalid,
ZSTD_error_externalSequences_invalid,
@@ -1906,18 +1906,18 @@ static int basicUnitTests(U32 seed, double compressibility, int bigTests)
int const compressionShouldSucceed = (
(errorCodes[testCaseId] == ZSTD_error_no_error) ||
(enableFallback && errorCodes[testCaseId] == ZSTD_error_externalMatchFinder_failed)
(enableFallback && errorCodes[testCaseId] == ZSTD_error_sequenceProducer_failed)
);
int const testWithSequenceValidation = (
testCases[testCaseId] == EMF_INVALID_OFFSET
);
externalMatchState = testCases[testCaseId];
sequenceProducerState = testCases[testCaseId];
ZSTD_CCtx_reset(zc, ZSTD_reset_session_only);
CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_validateSequences, testWithSequenceValidation));
CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_enableMatchFinderFallback, enableFallback));
CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_enableSeqProducerFallback, enableFallback));
res = ZSTD_compress2(zc, dstBuf, dstBufSize, CNBuffer, CNBufferSize);
if (compressionShouldSucceed) {
@@ -1936,9 +1936,9 @@ static int basicUnitTests(U32 seed, double compressibility, int bigTests)
/* Test compression with external matchfinder + empty src buffer */
{
size_t res;
externalMatchState = EMF_ZERO_SEQS;
sequenceProducerState = EMF_ZERO_SEQS;
ZSTD_CCtx_reset(zc, ZSTD_reset_session_only);
CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_enableMatchFinderFallback, enableFallback));
CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_enableSeqProducerFallback, enableFallback));
res = ZSTD_compress2(zc, dstBuf, dstBufSize, CNBuffer, 0);
CHECK(ZSTD_isError(res), "EMF: Compression error: %s", ZSTD_getErrorName(res));
CHECK(ZSTD_decompress(checkBuf, checkBufSize, dstBuf, res) != 0, "EMF: Empty src round trip failed!");
@@ -1947,30 +1947,30 @@ static int basicUnitTests(U32 seed, double compressibility, int bigTests)
/* Test that reset clears the external matchfinder */
CHECK_Z(ZSTD_CCtx_reset(zc, ZSTD_reset_session_and_parameters));
externalMatchState = EMF_BIG_ERROR; /* ensure zstd will fail if the matchfinder wasn't cleared */
CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_enableMatchFinderFallback, 0));
sequenceProducerState = EMF_BIG_ERROR; /* ensure zstd will fail if the matchfinder wasn't cleared */
CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_enableSeqProducerFallback, 0));
CHECK_Z(ZSTD_compress2(zc, dstBuf, dstBufSize, CNBuffer, CNBufferSize));
/* Test that registering mFinder == NULL clears the external matchfinder */
ZSTD_CCtx_reset(zc, ZSTD_reset_session_and_parameters);
ZSTD_registerExternalMatchFinder(zc, &externalMatchState, zstreamExternalMatchFinder);
externalMatchState = EMF_BIG_ERROR; /* ensure zstd will fail if the matchfinder wasn't cleared */
CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_enableMatchFinderFallback, 0));
ZSTD_registerExternalMatchFinder(zc, NULL, NULL); /* clear the external matchfinder */
ZSTD_registerSequenceProducer(zc, &sequenceProducerState, zstreamSequenceProducer);
sequenceProducerState = EMF_BIG_ERROR; /* ensure zstd will fail if the matchfinder wasn't cleared */
CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_enableSeqProducerFallback, 0));
ZSTD_registerSequenceProducer(zc, NULL, NULL); /* clear the external matchfinder */
CHECK_Z(ZSTD_compress2(zc, dstBuf, dstBufSize, CNBuffer, CNBufferSize));
/* Test that external matchfinder doesn't interact with older APIs */
ZSTD_CCtx_reset(zc, ZSTD_reset_session_and_parameters);
ZSTD_registerExternalMatchFinder(zc, &externalMatchState, zstreamExternalMatchFinder);
externalMatchState = EMF_BIG_ERROR; /* ensure zstd will fail if the matchfinder is used */
CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_enableMatchFinderFallback, 0));
ZSTD_registerSequenceProducer(zc, &sequenceProducerState, zstreamSequenceProducer);
sequenceProducerState = EMF_BIG_ERROR; /* ensure zstd will fail if the matchfinder is used */
CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_enableSeqProducerFallback, 0));
CHECK_Z(ZSTD_compressCCtx(zc, dstBuf, dstBufSize, CNBuffer, CNBufferSize, 3));
/* Test that compression returns the correct error with LDM */
CHECK_Z(ZSTD_CCtx_reset(zc, ZSTD_reset_session_and_parameters));
{
size_t res;
ZSTD_registerExternalMatchFinder(zc, &externalMatchState, zstreamExternalMatchFinder);
ZSTD_registerSequenceProducer(zc, &sequenceProducerState, zstreamSequenceProducer);
CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable));
res = ZSTD_compress2(zc, dstBuf, dstBufSize, CNBuffer, CNBufferSize);
CHECK(!ZSTD_isError(res), "EMF: Should have raised an error!");
@@ -1985,7 +1985,7 @@ static int basicUnitTests(U32 seed, double compressibility, int bigTests)
CHECK_Z(ZSTD_CCtx_reset(zc, ZSTD_reset_session_and_parameters));
{
size_t res;
ZSTD_registerExternalMatchFinder(zc, &externalMatchState, zstreamExternalMatchFinder);
ZSTD_registerSequenceProducer(zc, &sequenceProducerState, zstreamSequenceProducer);
CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_nbWorkers, 1));
res = ZSTD_compress2(zc, dstBuf, dstBufSize, CNBuffer, CNBufferSize);
CHECK(!ZSTD_isError(res), "EMF: Should have raised an error!");