ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

merge dev

Browse Source
This commit is contained in:
Danielle Rozenblit
2023-01-23 08:12:19 -08:00
parent 638d502002 d9280afb7d
commit f75afb613f
44 changed files with 728 additions and 869 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/cli-tests/file-stat/compress-file-to-file.sh
+1
View File
@@ -3,6 +3,7 @@
set -e
datagen > file
chmod 642 file
zstd file -q --trace-file-stat -o file.zst
zstd -tq file.zst
tests/cli-tests/file-stat/compress-file-to-file.sh.stderr.exact
+11 -5
View File
@@ -6,12 +6,10 @@ Trace:FileStat: > UTIL_getFileSize(file)
Trace:FileStat: > UTIL_stat(file)
Trace:FileStat: < 1
Trace:FileStat: < 65537
Trace:FileStat: > UTIL_isDirectory(file)
Trace:FileStat: > UTIL_stat(file)
Trace:FileStat: < 1
Trace:FileStat: < 0
Trace:FileStat: > UTIL_stat(file)
Trace:FileStat: < 1
Trace:FileStat: > UTIL_isDirectoryStat()
Trace:FileStat: < 0
Trace:FileStat: > UTIL_stat(file)
Trace:FileStat: < 1
Trace:FileStat: > UTIL_isSameFile(file, file.zst)
@@ -32,5 +30,13 @@ Trace:FileStat: > UTIL_getFileSize(file)
Trace:FileStat: > UTIL_stat(file)
Trace:FileStat: < 1
Trace:FileStat: < 65537
Trace:FileStat: > UTIL_utime(file.zst)
Trace:FileStat: > UTIL_setFileStat(file.zst)
Trace:FileStat: > UTIL_stat(file.zst)
Trace:FileStat: < 1
Trace:FileStat: > UTIL_utime(file.zst)
Trace:FileStat: < 0
Trace:FileStat: > UTIL_chmod(file.zst, 0642)
Trace:FileStat: > chmod
Trace:FileStat: < 0
Trace:FileStat: < 0
Trace:FileStat: < 0
tests/cli-tests/file-stat/compress-file-to-stdout.sh.stderr.exact
+3 -3
View File
@@ -6,9 +6,9 @@ Trace:FileStat: > UTIL_getFileSize(file)
Trace:FileStat: > UTIL_stat(file)
Trace:FileStat: < 1
Trace:FileStat: < 65537
Trace:FileStat: > UTIL_isDirectory(file)
Trace:FileStat: > UTIL_stat(file)
Trace:FileStat: < 1
Trace:FileStat: > UTIL_stat(file)
Trace:FileStat: < 1
Trace:FileStat: > UTIL_isDirectoryStat()
Trace:FileStat: < 0
Trace:FileStat: > UTIL_stat(file)
Trace:FileStat: < 1
tests/cli-tests/file-stat/compress-stdin-to-file.sh.stderr.exact
-4
View File
@@ -6,10 +6,6 @@ Trace:FileStat: > UTIL_getFileSize(/*stdin*\)
Trace:FileStat: > UTIL_stat(/*stdin*\)
Trace:FileStat: < 0
Trace:FileStat: < -1
Trace:FileStat: > UTIL_isDirectory(/*stdin*\)
Trace:FileStat: > UTIL_stat(/*stdin*\)
Trace:FileStat: < 0
Trace:FileStat: < 0
Trace:FileStat: > UTIL_isSameFile(/*stdin*\, file.zst)
Trace:FileStat: > UTIL_stat(/*stdin*\)
Trace:FileStat: < 0
tests/cli-tests/file-stat/compress-stdin-to-stdout.sh.stderr.exact
-4
View File
@@ -6,10 +6,6 @@ Trace:FileStat: > UTIL_getFileSize(/*stdin*\)
Trace:FileStat: > UTIL_stat(/*stdin*\)
Trace:FileStat: < 0
Trace:FileStat: < -1
Trace:FileStat: > UTIL_isDirectory(/*stdin*\)
Trace:FileStat: > UTIL_stat(/*stdin*\)
Trace:FileStat: < 0
Trace:FileStat: < 0
Trace:FileStat: > UTIL_isRegularFile(/*stdout*\)
Trace:FileStat: > UTIL_stat(/*stdout*\)
Trace:FileStat: < 0
tests/cli-tests/file-stat/decompress-file-to-file.sh
+1
View File
@@ -3,5 +3,6 @@
set -e
datagen | zstd -q > file.zst
chmod 642 file.zst
zstd -dq --trace-file-stat file.zst
tests/cli-tests/file-stat/decompress-file-to-file.sh.stderr.exact
+11 -3
View File
@@ -7,11 +7,11 @@ Trace:FileStat: < 0
Trace:FileStat: > UTIL_isDirectory(file.zst)
Trace:FileStat: > UTIL_stat(file.zst)
Trace:FileStat: < 1
Trace:FileStat: > UTIL_isDirectoryStat()
Trace:FileStat: < 0
Trace:FileStat: < 0
Trace:FileStat: > UTIL_stat(file.zst)
Trace:FileStat: < 1
Trace:FileStat: > UTIL_stat(file.zst)
Trace:FileStat: < 1
Trace:FileStat: > UTIL_isSameFile(file.zst, file)
Trace:FileStat: > UTIL_stat(file.zst)
Trace:FileStat: < 1
@@ -26,5 +26,13 @@ Trace:FileStat: > UTIL_isRegularFile(file)
Trace:FileStat: > UTIL_stat(file)
Trace:FileStat: < 1
Trace:FileStat: < 1
Trace:FileStat: > UTIL_utime(file)
Trace:FileStat: > UTIL_setFileStat(file)
Trace:FileStat: > UTIL_stat(file)
Trace:FileStat: < 1
Trace:FileStat: > UTIL_utime(file)
Trace:FileStat: < 0
Trace:FileStat: > UTIL_chmod(file, 0642)
Trace:FileStat: > chmod
Trace:FileStat: < 0
Trace:FileStat: < 0
Trace:FileStat: < 0
tests/cli-tests/file-stat/decompress-file-to-stdout.sh.stderr.exact
+2
View File
@@ -5,6 +5,8 @@ Trace:FileStat: < 0
Trace:FileStat: > UTIL_isDirectory(file.zst)
Trace:FileStat: > UTIL_stat(file.zst)
Trace:FileStat: < 1
Trace:FileStat: > UTIL_isDirectoryStat()
Trace:FileStat: < 0
Trace:FileStat: < 0
Trace:FileStat: > UTIL_stat(file.zst)
Trace:FileStat: < 1
tests/decodecorpus.c
+3 -3
View File
@@ -460,7 +460,7 @@ static size_t writeHufHeader(U32* seed, HUF_CElt* hufTable, void* dst, size_t ds
}
/* Write table description header */
{ size_t const hSize = HUF_writeCTable (op, dstSize, hufTable, maxSymbolValue, huffLog);
{ size_t const hSize = HUF_writeCTable_wksp (op, dstSize, hufTable, maxSymbolValue, huffLog, WKSP, sizeof(WKSP));
if (hSize + 12 >= srcSize) return 0; /* not useful to try compression */
op += hSize;
}
@@ -564,10 +564,10 @@ static size_t writeLiteralsBlockCompressed(U32* seed, frame_t* frame, size_t con
sizeFormat == 0
? HUF_compress1X_usingCTable(
op, opend - op, LITERAL_BUFFER, litSize,
frame->stats.hufTable)
frame->stats.hufTable, /* flags */ 0)
: HUF_compress4X_usingCTable(
op, opend - op, LITERAL_BUFFER, litSize,
frame->stats.hufTable);
frame->stats.hufTable, /* flags */ 0);
CHECKERR(compressedSize);
/* this only occurs when it could not compress or similar */
} while (compressedSize <= 0);
tests/fullbench.c
+4 -3
View File
@@ -171,6 +171,7 @@ FORCE_NOINLINE size_t ZSTD_decodeLiteralsHeader(ZSTD_DCtx* dctx, void const* src
size_t lhSize, litSize, litCSize;
U32 const lhlCode = (istart[0] >> 2) & 3;
U32 const lhc = MEM_readLE32(istart);
int const flags = ZSTD_DCtx_get_bmi2(dctx) ? HUF_flags_bmi2 : 0;
switch(lhlCode)
{
case 0: case 1: default: /* note : default is impossible, since lhlCode into [0..3] */
@@ -195,16 +196,16 @@ FORCE_NOINLINE size_t ZSTD_decodeLiteralsHeader(ZSTD_DCtx* dctx, void const* src
RETURN_ERROR_IF(litSize > ZSTD_BLOCKSIZE_MAX, corruption_detected, "");
RETURN_ERROR_IF(litCSize + lhSize > srcSize, corruption_detected, "");
#ifndef HUF_FORCE_DECOMPRESS_X2
return HUF_readDTableX1_wksp_bmi2(
return HUF_readDTableX1_wksp(
dctx->entropy.hufTable,
istart+lhSize, litCSize,
dctx->workspace, sizeof(dctx->workspace),
ZSTD_DCtx_get_bmi2(dctx));
flags);
#else
return HUF_readDTableX2_wksp(
dctx->entropy.hufTable,
istart+lhSize, litCSize,
dctx->workspace, sizeof(dctx->workspace));
dctx->workspace, sizeof(dctx->workspace), flags);
#endif
}
}
tests/fuzz/Makefile
+22 -8
View File
@@ -34,9 +34,14 @@ ZSTDDIR = ../../lib
PRGDIR = ../../programs
CONTRIBDIR = ../../contrib
# TODO(embg) make it possible to plug in an arbitrary matchfinder as a .o file
MATCHFINDER_DIR = $(CONTRIBDIR)/externalMatchfinder
MATCHFINDER_SRC = $(MATCHFINDER_DIR)/matchfinder.c
FUZZ_CPPFLAGS := -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/compress \
-I$(ZSTDDIR)/dictBuilder -I$(ZSTDDIR)/deprecated -I$(ZSTDDIR)/legacy \
-I$(CONTRIBDIR)/seekable_format -I$(PRGDIR) -DZSTD_MULTITHREAD -DZSTD_LEGACY_SUPPORT=1 $(CPPFLAGS)
-I$(CONTRIBDIR)/seekable_format -I$(PRGDIR) -I$(MATCHFINDER_DIR) \
-DZSTD_MULTITHREAD -DZSTD_LEGACY_SUPPORT=1 $(CPPFLAGS)
FUZZ_EXTRA_FLAGS := -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \
-Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \
-Wstrict-prototypes -Wundef \
@@ -69,7 +74,8 @@ FUZZ_SRC := \
$(ZSTDCOMMON_SRC) \
$(ZSTDCOMP_SRC) \
$(ZSTDDICT_SRC) \
$(ZSTDLEGACY_SRC)
$(ZSTDLEGACY_SRC) \
$(MATCHFINDER_SRC)
FUZZ_SRC := $(sort $(wildcard $(FUZZ_SRC)))
FUZZ_D_OBJ1 := $(subst $(ZSTDDIR)/common/,d_lib_common_,$(FUZZ_SRC))
@@ -78,9 +84,10 @@ FUZZ_D_OBJ3 := $(subst $(ZSTDDIR)/decompress/,d_lib_decompress_,$(FUZZ_D_OBJ2))
FUZZ_D_OBJ4 := $(subst $(ZSTDDIR)/dictBuilder/,d_lib_dictBuilder_,$(FUZZ_D_OBJ3))
FUZZ_D_OBJ5 := $(subst $(ZSTDDIR)/legacy/,d_lib_legacy_,$(FUZZ_D_OBJ4))
FUZZ_D_OBJ6 := $(subst $(PRGDIR)/,d_prg_,$(FUZZ_D_OBJ5))
FUZZ_D_OBJ7 := $(subst $\./,d_fuzz_,$(FUZZ_D_OBJ6))
FUZZ_D_OBJ8 := $(FUZZ_D_OBJ7:.c=.o)
FUZZ_DECOMPRESS_OBJ := $(FUZZ_D_OBJ8:.S=.o)
FUZZ_D_OBJ7 := $(subst $(MATCHFINDER_DIR)/,d_matchfinder_,$(FUZZ_D_OBJ6))
FUZZ_D_OBJ8 := $(subst $\./,d_fuzz_,$(FUZZ_D_OBJ7))
FUZZ_D_OBJ9 := $(FUZZ_D_OBJ8:.c=.o)
FUZZ_DECOMPRESS_OBJ := $(FUZZ_D_OBJ9:.S=.o)
FUZZ_RT_OBJ1 := $(subst $(ZSTDDIR)/common/,rt_lib_common_,$(FUZZ_SRC))
FUZZ_RT_OBJ2 := $(subst $(ZSTDDIR)/compress/,rt_lib_compress_,$(FUZZ_RT_OBJ1))
@@ -88,9 +95,10 @@ FUZZ_RT_OBJ3 := $(subst $(ZSTDDIR)/decompress/,rt_lib_decompress_,$(FUZZ_RT_OBJ2
FUZZ_RT_OBJ4 := $(subst $(ZSTDDIR)/dictBuilder/,rt_lib_dictBuilder_,$(FUZZ_RT_OBJ3))
FUZZ_RT_OBJ5 := $(subst $(ZSTDDIR)/legacy/,rt_lib_legacy_,$(FUZZ_RT_OBJ4))
FUZZ_RT_OBJ6 := $(subst $(PRGDIR)/,rt_prg_,$(FUZZ_RT_OBJ5))
FUZZ_RT_OBJ7 := $(subst $\./,rt_fuzz_,$(FUZZ_RT_OBJ6))
FUZZ_RT_OBJ8 := $(FUZZ_RT_OBJ7:.c=.o)
FUZZ_ROUND_TRIP_OBJ := $(FUZZ_RT_OBJ8:.S=.o)
FUZZ_RT_OBJ7 := $(subst $(MATCHFINDER_DIR)/,rt_matchfinder_,$(FUZZ_RT_OBJ6))
FUZZ_RT_OBJ8 := $(subst $\./,rt_fuzz_,$(FUZZ_RT_OBJ7))
FUZZ_RT_OBJ9 := $(FUZZ_RT_OBJ8:.c=.o)
FUZZ_ROUND_TRIP_OBJ := $(FUZZ_RT_OBJ9:.S=.o)
.PHONY: default all clean cleanall
@@ -143,6 +151,9 @@ rt_prg_%.o: $(PRGDIR)/%.c
rt_fuzz_%.o: %.c
$(CC) $(FUZZ_CPPFLAGS) $(FUZZ_CFLAGS) $(FUZZ_ROUND_TRIP_FLAGS) $< -c -o $@
rt_matchfinder_%.o: $(MATCHFINDER_DIR)/%.c
$(CC) $(FUZZ_CPPFLAGS) $(FUZZ_CFLAGS) $(FUZZ_ROUND_TRIP_FLAGS) $< -c -o $@
d_lib_common_%.o: $(ZSTDDIR)/common/%.c
$(CC) $(FUZZ_CPPFLAGS) $(FUZZ_CFLAGS) $< -c -o $@
@@ -167,6 +178,9 @@ d_prg_%.o: $(PRGDIR)/%.c
d_fuzz_%.o: %.c
$(CC) $(FUZZ_CPPFLAGS) $(FUZZ_CFLAGS) $< -c -o $@
d_matchfinder_%.o: $(MATCHFINDER_DIR)/%.c
$(CC) $(FUZZ_CPPFLAGS) $(FUZZ_CFLAGS) $< -c -o $@
simple_round_trip: $(FUZZ_HEADERS) $(FUZZ_ROUND_TRIP_OBJ) rt_fuzz_simple_round_trip.o
$(CXX) $(FUZZ_TARGET_FLAGS) $(FUZZ_ROUND_TRIP_OBJ) rt_fuzz_simple_round_trip.o $(LIB_FUZZING_ENGINE) -o $@
tests/fuzz/huf_decompress.c
+10 -7
View File
@@ -13,8 +13,6 @@
* compares the result with the original, and calls abort() on corruption.
*/
#define HUF_STATIC_LINKING_ONLY
#include <stddef.h>
#include <stdlib.h>
#include <stdio.h>
@@ -30,7 +28,12 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size)
/* Select random parameters: #streams, X1 or X2 decoding, bmi2 */
int const streams = FUZZ_dataProducer_int32Range(producer, 0, 1);
int const symbols = FUZZ_dataProducer_int32Range(producer, 0, 1);
int const bmi2 = ZSTD_cpuid_bmi2(ZSTD_cpuid()) && FUZZ_dataProducer_int32Range(producer, 0, 1);
int const flags = 0
| (ZSTD_cpuid_bmi2(ZSTD_cpuid()) && FUZZ_dataProducer_int32Range(producer, 0, 1) ? HUF_flags_bmi2 : 0)
| (FUZZ_dataProducer_int32Range(producer, 0, 1) ? HUF_flags_optimalDepth : 0)
| (FUZZ_dataProducer_int32Range(producer, 0, 1) ? HUF_flags_preferRepeat : 0)
| (FUZZ_dataProducer_int32Range(producer, 0, 1) ? HUF_flags_suspectUncompressible : 0)
| (FUZZ_dataProducer_int32Range(producer, 0, 1) ? HUF_flags_disableAsm : 0);
/* Select a random cBufSize - it may be too small */
size_t const dBufSize = FUZZ_dataProducer_uint32Range(producer, 0, 8 * size + 500);
size_t const maxTableLog = FUZZ_dataProducer_uint32Range(producer, 1, HUF_TABLELOG_MAX);
@@ -42,18 +45,18 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size)
size = FUZZ_dataProducer_remainingBytes(producer);
if (symbols == 0) {
size_t const err = HUF_readDTableX1_wksp_bmi2(dt, src, size, wksp, wkspSize, bmi2);
size_t const err = HUF_readDTableX1_wksp(dt, src, size, wksp, wkspSize, flags);
if (ZSTD_isError(err))
goto _out;
} else {
size_t const err = HUF_readDTableX2_wksp_bmi2(dt, src, size, wksp, wkspSize, bmi2);
size_t const err = HUF_readDTableX2_wksp(dt, src, size, wksp, wkspSize, flags);
if (ZSTD_isError(err))
goto _out;
}
if (streams == 0)
HUF_decompress1X_usingDTable_bmi2(dBuf, dBufSize, src, size, dt, bmi2);
HUF_decompress1X_usingDTable(dBuf, dBufSize, src, size, dt, flags);
else
HUF_decompress4X_usingDTable_bmi2(dBuf, dBufSize, src, size, dt, bmi2);
HUF_decompress4X_usingDTable(dBuf, dBufSize, src, size, dt, flags);
_out:
free(dt);
tests/fuzz/huf_round_trip.c
+14 -13
View File
@@ -13,8 +13,6 @@
* compares the result with the original, and calls abort() on corruption.
*/
#define HUF_STATIC_LINKING_ONLY
#include <stddef.h>
#include <stdlib.h>
#include <stdio.h>
@@ -46,7 +44,12 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size)
/* Select random parameters: #streams, X1 or X2 decoding, bmi2 */
int const streams = FUZZ_dataProducer_int32Range(producer, 0, 1);
int const symbols = FUZZ_dataProducer_int32Range(producer, 0, 1);
int const bmi2 = ZSTD_cpuid_bmi2(ZSTD_cpuid()) && FUZZ_dataProducer_int32Range(producer, 0, 1);
int const flags = 0
| (ZSTD_cpuid_bmi2(ZSTD_cpuid()) && FUZZ_dataProducer_int32Range(producer, 0, 1) ? HUF_flags_bmi2 : 0)
| (FUZZ_dataProducer_int32Range(producer, 0, 1) ? HUF_flags_optimalDepth : 0)
| (FUZZ_dataProducer_int32Range(producer, 0, 1) ? HUF_flags_preferRepeat : 0)
| (FUZZ_dataProducer_int32Range(producer, 0, 1) ? HUF_flags_suspectUncompressible : 0)
| (FUZZ_dataProducer_int32Range(producer, 0, 1) ? HUF_flags_disableAsm : 0);
/* Select a random cBufSize - it may be too small */
size_t const cBufSize = FUZZ_dataProducer_uint32Range(producer, 0, 4 * size);
/* Select a random tableLog - we'll adjust it up later */
@@ -83,9 +86,7 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size)
HUF_DTable* dt = (HUF_DTable*)FUZZ_malloc(HUF_DTABLE_SIZE(tableLog) * sizeof(HUF_DTable));
dt[0] = tableLog * 0x01000001;
HUF_depth_mode depthMode = rand() & 1 ? HUF_depth_fast : HUF_depth_optimal;
tableLog = HUF_optimalTableLog(tableLog, size, maxSymbol, wksp, wkspSize, ct, count, depthMode);
tableLog = HUF_optimalTableLog(tableLog, size, maxSymbol, wksp, wkspSize, ct, count, flags);
FUZZ_ASSERT(tableLog <= 12);
tableLog = HUF_buildCTable_wksp(ct, count, maxSymbol, tableLog, wksp, wkspSize);
FUZZ_ZASSERT(tableLog);
@@ -96,11 +97,11 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size)
}
FUZZ_ZASSERT(tableSize);
if (symbols == 0) {
FUZZ_ZASSERT(HUF_readDTableX1_wksp_bmi2(dt, cBuf, tableSize, wksp, wkspSize, bmi2));
FUZZ_ZASSERT(HUF_readDTableX1_wksp(dt, cBuf, tableSize, wksp, wkspSize, flags));
} else {
size_t const ret = HUF_readDTableX2_wksp(dt, cBuf, tableSize, wksp, wkspSize);
size_t const ret = HUF_readDTableX2_wksp(dt, cBuf, tableSize, wksp, wkspSize, flags);
if (ERR_getErrorCode(ret) == ZSTD_error_tableLog_tooLarge) {
FUZZ_ZASSERT(HUF_readDTableX1_wksp_bmi2(dt, cBuf, tableSize, wksp, wkspSize, bmi2));
FUZZ_ZASSERT(HUF_readDTableX1_wksp(dt, cBuf, tableSize, wksp, wkspSize, flags));
} else {
FUZZ_ZASSERT(ret);
}
@@ -109,15 +110,15 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size)
size_t cSize;
size_t rSize;
if (streams == 0) {
cSize = HUF_compress1X_usingCTable_bmi2(cBuf, cBufSize, src, size, ct, bmi2);
cSize = HUF_compress1X_usingCTable(cBuf, cBufSize, src, size, ct, flags);
FUZZ_ZASSERT(cSize);
if (cSize != 0)
rSize = HUF_decompress1X_usingDTable_bmi2(rBuf, size, cBuf, cSize, dt, bmi2);
rSize = HUF_decompress1X_usingDTable(rBuf, size, cBuf, cSize, dt, flags);
} else {
cSize = HUF_compress4X_usingCTable_bmi2(cBuf, cBufSize, src, size, ct, bmi2);
cSize = HUF_compress4X_usingCTable(cBuf, cBufSize, src, size, ct, flags);
FUZZ_ZASSERT(cSize);
if (cSize != 0)
rSize = HUF_decompress4X_usingDTable_bmi2(rBuf, size, cBuf, cSize, dt, bmi2);
rSize = HUF_decompress4X_usingDTable(rBuf, size, cBuf, cSize, dt, flags);
}
if (cSize != 0) {
FUZZ_ZASSERT(rSize);
tests/fuzz/zstd_helpers.c
+18
View File
@@ -17,6 +17,7 @@
#include "fuzz_helpers.h"
#include "zstd.h"
#include "zdict.h"
#include "matchfinder.h"
const int kMinClevel = -3;
const int kMaxClevel = 19;
@@ -70,6 +71,17 @@ ZSTD_parameters FUZZ_randomParams(size_t srcSize, FUZZ_dataProducer_t *producer)
return params;
}
static void setExternalMatchFinderParams(ZSTD_CCtx *cctx, FUZZ_dataProducer_t *producer) {
ZSTD_registerExternalMatchFinder(
cctx,
NULL,
simpleExternalMatchFinder
);
setRand(cctx, ZSTD_c_enableMatchFinderFallback, 0, 1, producer);
FUZZ_ZASSERT(ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 0));
FUZZ_ZASSERT(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_disable));
}
void FUZZ_setRandomParameters(ZSTD_CCtx *cctx, size_t srcSize, FUZZ_dataProducer_t *producer)
{
ZSTD_compressionParameters cParams = FUZZ_randomCParams(srcSize, producer);
@@ -122,6 +134,12 @@ void FUZZ_setRandomParameters(ZSTD_CCtx *cctx, size_t srcSize, FUZZ_dataProducer
if (FUZZ_dataProducer_uint32Range(producer, 0, 1) == 0) {
setRand(cctx, ZSTD_c_targetCBlockSize, ZSTD_TARGETCBLOCKSIZE_MIN, ZSTD_TARGETCBLOCKSIZE_MAX, producer);
}
if (FUZZ_dataProducer_uint32Range(producer, 0, 10) == 1) {
setExternalMatchFinderParams(cctx, producer);
} else {
ZSTD_registerExternalMatchFinder(cctx, NULL, NULL);
}
}
FUZZ_dict_t FUZZ_train(void const* src, size_t srcSize, FUZZ_dataProducer_t *producer)
tests/fuzzer.c
+105
View File
@@ -2592,6 +2592,27 @@ static int basicUnitTests(U32 const seed, double compressibility)
}
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(3, "test%3d : bufferless api with cdict : ", testNb++);
{ ZSTD_CDict* const cdict = ZSTD_createCDict(dictBuffer, dictSize, 1);
ZSTD_DCtx* const dctx = ZSTD_createDCtx();
ZSTD_frameParameters const fParams = { 0, 1, 0 };
size_t cBlockSize;
cSize = 0;
CHECK_Z(ZSTD_compressBegin_usingCDict_advanced(cctx, cdict, fParams, ZSTD_CONTENTSIZE_UNKNOWN));
cBlockSize = ZSTD_compressContinue(cctx, (char*)compressedBuffer + cSize, compressedBufferSize - cSize, CNBuffer, 1000);
CHECK_Z(cBlockSize);
cSize += cBlockSize;
cBlockSize = ZSTD_compressEnd(cctx, (char*)compressedBuffer + cSize, compressedBufferSize - cSize, (char const*)CNBuffer + 2000, 1000);
CHECK_Z(cBlockSize);
cSize += cBlockSize;
CHECK_Z(ZSTD_decompress_usingDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, dictBuffer, dictSize));
ZSTD_freeCDict(cdict);
ZSTD_freeDCtx(dctx);
}
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(3, "test%3i : Building cdict w/ ZSTD_dct_fullDict on a good dictionary : ", testNb++);
{ ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize);
ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_fullDict, cParams, ZSTD_defaultCMem);
@@ -2832,6 +2853,90 @@ static int basicUnitTests(U32 const seed, double compressibility)
}
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(3, "test%3i : ZSTD_fast attach dictionary with hashLog = 25 and chainLog = 25 : ", testNb++);
{
ZSTD_CCtx_params* cctxParams = ZSTD_createCCtxParams();
ZSTD_customMem customMem = {NULL, NULL, NULL};
ZSTD_DCtx* dctx = ZSTD_createDCtx();
ZSTD_CDict* cdict;
CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_strategy, ZSTD_fast));
/* Set windowLog to 25 so hash/chain logs don't get sized down */
CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_windowLog, 25));
CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_hashLog, 25));
CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_chainLog, 25));
/* Set srcSizeHint to 2^25 so hash/chain logs don't get sized down */
CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_srcSizeHint, 1u << 25));
cdict = ZSTD_createCDict_advanced2(dictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_auto, cctxParams, customMem);
CHECK_Z(ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters));
CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_forceAttachDict, ZSTD_dictForceAttach));
CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1));
CHECK_Z(ZSTD_CCtx_refCDict(cctx, cdict));
cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize);
CHECK_Z(cSize);
CHECK_Z(ZSTD_decompress_usingDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, dictBuffer, dictSize));
ZSTD_freeCDict(cdict);
ZSTD_freeDCtx(dctx);
ZSTD_freeCCtxParams(cctxParams);
}
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(3, "test%3i : ZSTD_dfast attach dictionary with hashLog = 25 and chainLog = 25 : ", testNb++);
{
ZSTD_CCtx_params* cctxParams = ZSTD_createCCtxParams();
ZSTD_customMem customMem = {NULL, NULL, NULL};
ZSTD_DCtx* dctx = ZSTD_createDCtx();
ZSTD_CDict* cdict;
CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_strategy, ZSTD_dfast));
/* Set windowLog to 25 so hash/chain logs don't get sized down */
CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_windowLog, 25));
CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_hashLog, 25));
CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_chainLog, 25));
/* Set srcSizeHint to 2^25 so hash/chain logs don't get sized down */
CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_srcSizeHint, 1u << 25));
cdict = ZSTD_createCDict_advanced2(dictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_auto, cctxParams, customMem);
CHECK_Z(ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters));
CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_forceAttachDict, ZSTD_dictForceAttach));
CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1));
CHECK_Z(ZSTD_CCtx_refCDict(cctx, cdict));
cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize);
CHECK_Z(cSize);
CHECK_Z(ZSTD_decompress_usingDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, dictBuffer, dictSize));
ZSTD_freeCDict(cdict);
ZSTD_freeDCtx(dctx);
ZSTD_freeCCtxParams(cctxParams);
}
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(3, "test%3i : ZSTD_lazy attach dictionary with hashLog = 29 and searchLog = 4 : ", testNb++);
if (MEM_64bits()) {
ZSTD_CCtx_params* cctxParams = ZSTD_createCCtxParams();
ZSTD_customMem customMem = {NULL, NULL, NULL};
ZSTD_DCtx* dctx = ZSTD_createDCtx();
ZSTD_CDict* cdict;
CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_strategy, ZSTD_lazy));
/* Force enable row based match finder, and disable dedicated dict search. */
CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_useRowMatchFinder, ZSTD_ps_enable));
CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_enableDedicatedDictSearch, 0));
CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_searchLog, 4));
/* Set windowLog to 29 so hash/chain logs don't get sized down */
CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_windowLog, 29));
CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_hashLog, 29));
/* Set srcSizeHint to 2^29 so hash/chain logs don't get sized down */
CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_srcSizeHint, 1u << 29));
cdict = ZSTD_createCDict_advanced2(dictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_auto, cctxParams, customMem);
CHECK_Z(ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters));
CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_forceAttachDict, ZSTD_dictForceAttach));
CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1));
CHECK_Z(ZSTD_CCtx_refCDict(cctx, cdict));
cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize);
CHECK_Z(cSize);
CHECK_Z(ZSTD_decompress_usingDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, dictBuffer, dictSize));
ZSTD_freeCDict(cdict);
ZSTD_freeDCtx(dctx);
ZSTD_freeCCtxParams(cctxParams);
}
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(3, "test%3i : Dictionary with non-default repcodes : ", testNb++);
{ U32 u; for (u=0; u<nbSamples; u++) samplesSizes[u] = sampleUnitSize; }
dictSize = ZDICT_trainFromBuffer(dictBuffer, dictSize,
tests/playTests.sh
-10
View File
@@ -562,19 +562,9 @@ if [ "$isWindows" = false ] ; then
zstd -f -d tmp1.zst -o tmp1.out
assertFilePermissions tmp1.out 400
rm -f tmp1.zst tmp1.out
umask 0666
chmod 0666 tmp1 tmp2
println "test : respect umask when copying permissions in file -> file compression "
zstd -f tmp1 -o tmp1.zst
assertFilePermissions tmp1.zst 0
println "test : respect umask when copying permissions in file -> file decompression "
chmod 0666 tmp1.zst
zstd -f -d tmp1.zst -o tmp1.out
assertFilePermissions tmp1.out 0
rm -f tmp1.zst tmp1.out
println "test : respect umask when compressing from stdin input "
tests/test-zstd-versions.py
+12 -6
View File
@@ -85,18 +85,23 @@ def get_git_tags():
return tags
def create_dict(tag, dict_source_path):
def create_dict(tag, dict_source_path, fallback_tag=None):
dict_name = 'dict.' + tag
if not os.path.isfile(dict_name):
cFiles = glob.glob(dict_source_path + "/*.c")
hFiles = glob.glob(dict_source_path + "/*.h")
# Ensure the dictionary builder is deterministic
files = sorted(cFiles + hFiles)
if tag == 'v0.5.0':
result = execute('./dictBuilder.' + tag + ' ' + ' '.join(cFiles) + ' ' + ' '.join(hFiles) + ' -o ' + dict_name, print_output=False, param_shell=True)
result = execute('./dictBuilder.' + tag + ' ' + ' '.join(files) + ' -o ' + dict_name, print_output=False, param_shell=True)
else:
result = execute('./zstd.' + tag + ' -f --train ' + ' '.join(cFiles) + ' ' + ' '.join(hFiles) + ' -o ' + dict_name, print_output=False, param_shell=True)
if result == 0:
result = execute('./zstd.' + tag + ' -f --train ' + ' '.join(files) + ' -o ' + dict_name, print_output=False, param_shell=True)
if result == 0 and os.path.isfile(dict_name):
print(dict_name + ' created')
assert os.path.isfile(dict_name)
elif fallback_tag is not None:
fallback_dict_name = 'dict.' + fallback_tag
print('creating dictionary ' + dict_name + ' failed, falling back to ' + fallback_dict_name)
shutil.copy(fallback_dict_name, dict_name)
else:
raise RuntimeError('ERROR: creating of ' + dict_name + ' failed')
else:
@@ -272,10 +277,11 @@ if __name__ == '__main__':
print('Compress test.dat by all released zstd')
print('-----------------------------------------------')
create_dict(head, dict_source_path)
for tag in tags:
print(tag)
if tag >= 'v0.5.0':
create_dict(tag, dict_source_path)
create_dict(tag, dict_source_path, head)
dict_compress_sample(tag, test_dat)
remove_duplicates()
decompress_dict(tag)
tests/zstreamtest.c
+21
View File
@@ -1566,6 +1566,27 @@ static int basicUnitTests(U32 seed, double compressibility, int bigTests)
CHECK(!ZSTD_isError(ZSTD_CCtx_setParameter(zc, ZSTD_c_srcSizeHint, -1)), "Out of range doesn't error");
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(3, "test%3i : ZSTD_lazy compress with hashLog = 29 and searchLog = 4 : ", testNb++);
if (MEM_64bits()) {
ZSTD_outBuffer out = { compressedBuffer, compressedBufferSize, 0 };
ZSTD_inBuffer in = { CNBuffer, CNBufferSize, 0 };
CHECK_Z(ZSTD_CCtx_reset(zc, ZSTD_reset_session_and_parameters));
CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_strategy, ZSTD_lazy));
/* Force enable the row based match finder */
CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_useRowMatchFinder, ZSTD_ps_enable));
CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_searchLog, 4));
/* Set windowLog to 29 so the hashLog doesn't get sized down */
CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_windowLog, 29));
CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_hashLog, 29));
CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_checksumFlag, 1));
/* Compress with continue first so the hashLog doesn't get sized down */
CHECK_Z(ZSTD_compressStream2(zc, &out, &in, ZSTD_e_continue));
CHECK_Z(ZSTD_compressStream2(zc, &out, &in, ZSTD_e_end));
cSize = out.pos;
CHECK_Z(ZSTD_decompress(decodedBuffer, CNBufferSize, compressedBuffer, cSize));
}
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(3, "test%3i : Test offset == windowSize : ", testNb++);
{
int windowLog;