Yann Collet
@@ -3,42 +3,16 @@ zstd
|
||||
zstd32
|
||||
zstd-compress
|
||||
zstd-decompress
|
||||
fullbench
|
||||
fullbench32
|
||||
fuzzer
|
||||
fuzzer32
|
||||
zbufftest
|
||||
zbufftest32
|
||||
zstreamtest
|
||||
zstreamtest32
|
||||
datagen
|
||||
paramgrill
|
||||
paramgrill32
|
||||
roundTripCrash
|
||||
|
||||
# Object files
|
||||
*.o
|
||||
*.ko
|
||||
|
||||
# Libraries
|
||||
*.lib
|
||||
*.a
|
||||
|
||||
# Shared objects (inc. Windows DLLs)
|
||||
*.dll
|
||||
*.so
|
||||
*.so.*
|
||||
*.dylib
|
||||
|
||||
# Executables
|
||||
*.exe
|
||||
*.out
|
||||
*.app
|
||||
|
||||
# Visual solution files
|
||||
*.suo
|
||||
*.user
|
||||
|
||||
# Default result files
|
||||
dictionary
|
||||
grillResults.txt
|
||||
@@ -53,5 +27,4 @@ afl
|
||||
|
||||
# Misc files
|
||||
*.bat
|
||||
fileTests.sh
|
||||
dirTest*
|
||||
|
||||
+8
-128
@@ -22,13 +22,11 @@
|
||||
# - zstd homepage : http://www.zstd.net/
|
||||
# ##########################################################################
|
||||
# zstd : Command Line Utility, supporting gzip-like arguments
|
||||
# datagen : Synthetic and parametrable data generator, for tests
|
||||
# fuzzer : Test tool, to check zstd integrity on target platform
|
||||
# fuzzer32: Same as fuzzer, but forced to compile in 32-bits mode
|
||||
# zbufftest : Test tool, to check ZBUFF integrity on target platform
|
||||
# zbufftest32: Same as zbufftest, but forced to compile in 32-bits mode
|
||||
# fullbench : Precisely measure speed for each zstd inner function
|
||||
# fullbench32: Same as fullbench, but forced to compile in 32-bits mode
|
||||
# zstd32 : Same as zstd, but forced to compile in 32-bits mode
|
||||
# zstd_nolegacy : zstd without support of decompression of legacy versions
|
||||
# zstd-small: minimal zstd without dictBuilder and bench
|
||||
# zstd-compress: compressor-only version of zstd
|
||||
# zstd-decompress: decompressor-only version of zstd
|
||||
# ##########################################################################
|
||||
|
||||
DESTDIR?=
|
||||
@@ -56,7 +54,6 @@ ZSTDCOMMON_FILES := $(ZSTDDIR)/common/*.c
|
||||
ZSTDCOMP_FILES := $(ZSTDDIR)/compress/zstd_compress.c $(ZSTDDIR)/compress/fse_compress.c $(ZSTDDIR)/compress/huf_compress.c
|
||||
ZSTDDECOMP_FILES := $(ZSTDDIR)/decompress/huf_decompress.c
|
||||
ZSTD_FILES := $(ZSTDDECOMP_FILES) $(ZSTDCOMMON_FILES) $(ZSTDCOMP_FILES)
|
||||
ZBUFF_FILES := $(ZSTDDIR)/compress/zbuff_compress.c $(ZSTDDIR)/decompress/zbuff_decompress.c
|
||||
ZDICT_FILES := $(ZSTDDIR)/dictBuilder/*.c
|
||||
ZSTDDECOMP_O = $(ZSTDDIR)/decompress/zstd_decompress.o
|
||||
ZSTDDECOMP32_O = $(ZSTDDIR)/decompress/zstd_decompress32.o
|
||||
@@ -80,17 +77,12 @@ EXT =
|
||||
VOID = /dev/null
|
||||
endif
|
||||
|
||||
ZBUFFTEST = -T2mn
|
||||
FUZZERTEST= -T5mn
|
||||
ZSTDRTTEST= --test-large-data
|
||||
|
||||
.PHONY: default all all32 clean install uninstall test test32 test-all
|
||||
.PHONY: default all clean install uninstall
|
||||
|
||||
default: zstd
|
||||
|
||||
all: zstd fullbench fuzzer zbufftest zstreamtest paramgrill datagen
|
||||
|
||||
all32: cleano32 zstd32 fullbench32 fuzzer32 zbufftest32 zstreamtest32
|
||||
all: zstd
|
||||
|
||||
|
||||
$(ZSTDDECOMP_O): $(ZSTDDIR)/decompress/zstd_decompress.c
|
||||
@@ -136,58 +128,18 @@ zstd-decompress: $(ZSTDCOMMON_FILES) $(ZSTDDECOMP_FILES) \
|
||||
zstd-small: clean
|
||||
CFLAGS="-Os -s" $(MAKE) zstd-frugal
|
||||
|
||||
fullbench : $(ZSTDDECOMP_O) $(ZSTD_FILES) $(ZBUFF_FILES) datagen.c fullbench.c
|
||||
$(CC) $(FLAGS) $^ -o $@$(EXT)
|
||||
|
||||
fullbench32 : $(ZSTDDECOMP32_O) $(ZSTD_FILES) $(ZBUFF_FILES) datagen.c fullbench.c
|
||||
$(CC) -m32 $(FLAGS) $^ -o $@$(EXT)
|
||||
|
||||
fuzzer : CPPFLAGS += -I$(ZSTDDIR)/dictBuilder
|
||||
fuzzer : $(ZSTDDECOMP_O) $(ZSTD_FILES) $(ZDICT_FILES) datagen.c fuzzer.c
|
||||
$(CC) $(FLAGS) $^ -o $@$(EXT)
|
||||
|
||||
fuzzer32 : CPPFLAGS += -I$(ZSTDDIR)/dictBuilder
|
||||
fuzzer32 : $(ZSTDDECOMP32_O) $(ZSTD_FILES) $(ZDICT_FILES) datagen.c fuzzer.c
|
||||
$(CC) -m32 $(FLAGS) $^ -o $@$(EXT)
|
||||
|
||||
zbufftest : $(ZSTDDECOMP_O) $(ZSTD_FILES) $(ZBUFF_FILES) datagen.c zbufftest.c
|
||||
$(CC) $(FLAGS) $^ -o $@$(EXT)
|
||||
|
||||
zbufftest32 : $(ZSTDDECOMP32_O) $(ZSTD_FILES) $(ZBUFF_FILES) datagen.c zbufftest.c
|
||||
$(CC) -m32 $(FLAGS) $^ -o $@$(EXT)
|
||||
|
||||
zstreamtest : $(ZSTDDECOMP_O) $(ZSTD_FILES) datagen.c zstreamtest.c
|
||||
$(CC) $(FLAGS) $^ -o $@$(EXT)
|
||||
|
||||
zstreamtest32 : $(ZSTDDECOMP32_O) $(ZSTD_FILES) datagen.c zstreamtest.c
|
||||
$(CC) -m32 $(FLAGS) $^ -o $@$(EXT)
|
||||
|
||||
paramgrill : $(ZSTDDECOMP_O) $(ZSTD_FILES) datagen.c paramgrill.c
|
||||
$(CC) $(FLAGS) $^ -lm -o $@$(EXT)
|
||||
|
||||
datagen : datagen.c datagencli.c
|
||||
$(CC) $(FLAGS) $^ -o $@$(EXT)
|
||||
|
||||
roundTripCrash : $(ZSTDDECOMP_O) $(ZSTD_FILES) roundTripCrash.c
|
||||
$(CC) $(FLAGS) $^ -o $@$(EXT)
|
||||
|
||||
clean:
|
||||
$(MAKE) -C ../lib clean
|
||||
@rm -f core *.o tmp* result* *.gcda dictionary *.zst \
|
||||
zstd$(EXT) zstd32$(EXT) zstd-compress$(EXT) zstd-decompress$(EXT) \
|
||||
fullbench$(EXT) fullbench32$(EXT) \
|
||||
fuzzer$(EXT) fuzzer32$(EXT) zbufftest$(EXT) zbufftest32$(EXT) \
|
||||
datagen$(EXT) paramgrill$(EXT) roundTripCrash$(EXT)
|
||||
zstd$(EXT) zstd32$(EXT) zstd-compress$(EXT) zstd-decompress$(EXT)
|
||||
@echo Cleaning completed
|
||||
|
||||
cleano32:
|
||||
@rm -f ../lib/decompress/*.o
|
||||
|
||||
#----------------------------------------------------------------------------------
|
||||
#make install is validated only for Linux, OSX, kFreeBSD, Hurd and some BSD targets
|
||||
#----------------------------------------------------------------------------------
|
||||
ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD DragonFly))
|
||||
HOST_OS = POSIX
|
||||
install: zstd
|
||||
@echo Installing binaries
|
||||
@install -d -m 755 $(DESTDIR)$(BINDIR)/ $(DESTDIR)$(MANDIR)/
|
||||
@@ -208,76 +160,4 @@ uninstall:
|
||||
rm -f $(DESTDIR)$(MANDIR)/unzstd.1
|
||||
[ -f $(DESTDIR)$(MANDIR)/zstd.1 ] && rm -f $(DESTDIR)$(MANDIR)/zstd.1
|
||||
@echo zstd programs successfully uninstalled
|
||||
|
||||
valgrindTest: VALGRIND = valgrind --leak-check=full --error-exitcode=1
|
||||
valgrindTest: zstd datagen fuzzer fullbench zbufftest
|
||||
@echo "\n ---- valgrind tests : memory analyzer ----"
|
||||
$(VALGRIND) ./datagen -g50M > $(VOID)
|
||||
$(VALGRIND) ./zstd ; if [ $$? -eq 0 ] ; then echo "zstd without argument should have failed"; false; fi
|
||||
./datagen -g80 | $(VALGRIND) ./zstd - -c > $(VOID)
|
||||
./datagen -g16KB | $(VALGRIND) ./zstd -vf - -o $(VOID)
|
||||
./datagen -g2930KB | $(VALGRIND) ./zstd -5 -vf - -o tmp
|
||||
$(VALGRIND) ./zstd -vdf tmp -o $(VOID)
|
||||
./datagen -g64MB | $(VALGRIND) ./zstd -vf - -o $(VOID)
|
||||
@rm tmp
|
||||
$(VALGRIND) ./fuzzer -T1mn -t1
|
||||
$(VALGRIND) ./fullbench -i1
|
||||
$(VALGRIND) ./zbufftest -T1mn
|
||||
|
||||
endif
|
||||
|
||||
|
||||
ifneq (,$(filter MSYS%,$(shell uname)))
|
||||
HOST_OS = MSYS
|
||||
endif
|
||||
|
||||
|
||||
#------------------------------------------------------------------------
|
||||
#make tests validated only for MSYS, Linux, OSX, kFreeBSD and Hurd targets
|
||||
#------------------------------------------------------------------------
|
||||
ifneq (,$(filter $(HOST_OS),MSYS POSIX))
|
||||
zstd-playTests: datagen
|
||||
ZSTD=$(ZSTD) ./playTests.sh $(ZSTDRTTEST)
|
||||
|
||||
test: test-zstd test-fullbench test-fuzzer test-zbuff test-zstream
|
||||
|
||||
test32: test-zstd32 test-fullbench32 test-fuzzer32 test-zbuff32 test-zstream32
|
||||
|
||||
test-all: test test32 valgrindTest
|
||||
|
||||
test-zstd: ZSTD = ./zstd
|
||||
test-zstd: zstd zstd-playTests
|
||||
|
||||
test-zstd32: ZSTD = ./zstd32
|
||||
test-zstd32: zstd32 zstd-playTests
|
||||
|
||||
test-zstd_nolegacy: ZSTD = ./zstd
|
||||
test-zstd_nolegacy: zstd_nolegacy zstd-playTests
|
||||
|
||||
test-fullbench: fullbench datagen
|
||||
./fullbench -i1
|
||||
./fullbench -i1 -P0
|
||||
|
||||
test-fullbench32: fullbench32 datagen
|
||||
./fullbench32 -i1
|
||||
./fullbench32 -i1 -P0
|
||||
|
||||
test-fuzzer: fuzzer
|
||||
./fuzzer $(FUZZERTEST)
|
||||
|
||||
test-fuzzer32: fuzzer32
|
||||
./fuzzer32 $(FUZZERTEST)
|
||||
|
||||
test-zbuff: zbufftest
|
||||
./zbufftest $(ZBUFFTEST)
|
||||
|
||||
test-zbuff32: zbufftest32
|
||||
./zbufftest32 $(ZBUFFTEST)
|
||||
|
||||
test-zstream: zstreamtest
|
||||
./zstreamtest $(ZBUFFTEST)
|
||||
|
||||
test-zstream32: zstreamtest32
|
||||
./zstreamtest32 $(ZBUFFTEST)
|
||||
|
||||
endif
|
||||
|
||||
@@ -1,144 +0,0 @@
|
||||
/*
|
||||
datagencli.c
|
||||
compressible data command line generator
|
||||
Copyright (C) Yann Collet 2012-2015
|
||||
|
||||
GPL v2 License
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation; either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License along
|
||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
You can contact the author at :
|
||||
- ZSTD source repository : https://github.com/Cyan4973/zstd
|
||||
- Public forum : https://groups.google.com/forum/#!forum/lz4c
|
||||
*/
|
||||
|
||||
/*-************************************
|
||||
* Includes
|
||||
**************************************/
|
||||
#include "util.h" /* Compiler options */
|
||||
#include <stdio.h> /* fprintf, stderr */
|
||||
#include "datagen.h" /* RDG_generate */
|
||||
|
||||
|
||||
/*-************************************
|
||||
* Constants
|
||||
**************************************/
|
||||
#define KB *(1 <<10)
|
||||
#define MB *(1 <<20)
|
||||
#define GB *(1U<<30)
|
||||
|
||||
#define SIZE_DEFAULT ((64 KB) + 1)
|
||||
#define SEED_DEFAULT 0
|
||||
#define COMPRESSIBILITY_DEFAULT 50
|
||||
|
||||
|
||||
/*-************************************
|
||||
* Macros
|
||||
**************************************/
|
||||
#define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
|
||||
#define DISPLAYLEVEL(l, ...) if (displayLevel>=l) { DISPLAY(__VA_ARGS__); }
|
||||
static unsigned displayLevel = 2;
|
||||
|
||||
|
||||
/*-*******************************************************
|
||||
* Command line
|
||||
*********************************************************/
|
||||
static int usage(const char* programName)
|
||||
{
|
||||
DISPLAY( "Compressible data generator\n");
|
||||
DISPLAY( "Usage :\n");
|
||||
DISPLAY( " %s [args]\n", programName);
|
||||
DISPLAY( "\n");
|
||||
DISPLAY( "Arguments :\n");
|
||||
DISPLAY( " -g# : generate # data (default:%i)\n", SIZE_DEFAULT);
|
||||
DISPLAY( " -s# : Select seed (default:%i)\n", SEED_DEFAULT);
|
||||
DISPLAY( " -P# : Select compressibility in %% (default:%i%%)\n", COMPRESSIBILITY_DEFAULT);
|
||||
DISPLAY( " -h : display help and exit\n");
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
int main(int argc, const char** argv)
|
||||
{
|
||||
double proba = (double)COMPRESSIBILITY_DEFAULT / 100;
|
||||
double litProba = 0.0;
|
||||
U64 size = SIZE_DEFAULT;
|
||||
U32 seed = SEED_DEFAULT;
|
||||
const char* const programName = argv[0];
|
||||
|
||||
int argNb;
|
||||
for(argNb=1; argNb<argc; argNb++) {
|
||||
const char* argument = argv[argNb];
|
||||
|
||||
if(!argument) continue; /* Protection if argument empty */
|
||||
|
||||
/* Handle commands. Aggregated commands are allowed */
|
||||
if (*argument=='-') {
|
||||
argument++;
|
||||
while (*argument!=0) {
|
||||
switch(*argument)
|
||||
{
|
||||
case 'h':
|
||||
return usage(programName);
|
||||
case 'g':
|
||||
argument++;
|
||||
size=0;
|
||||
while ((*argument>='0') && (*argument<='9'))
|
||||
size *= 10, size += *argument++ - '0';
|
||||
if (*argument=='K') { size <<= 10; argument++; }
|
||||
if (*argument=='M') { size <<= 20; argument++; }
|
||||
if (*argument=='G') { size <<= 30; argument++; }
|
||||
if (*argument=='B') { argument++; }
|
||||
break;
|
||||
case 's':
|
||||
argument++;
|
||||
seed=0;
|
||||
while ((*argument>='0') && (*argument<='9'))
|
||||
seed *= 10, seed += *argument++ - '0';
|
||||
break;
|
||||
case 'P':
|
||||
argument++;
|
||||
proba=0.0;
|
||||
while ((*argument>='0') && (*argument<='9'))
|
||||
proba *= 10, proba += *argument++ - '0';
|
||||
if (proba>100.) proba=100.;
|
||||
proba /= 100.;
|
||||
break;
|
||||
case 'L': /* hidden argument : Literal distribution probability */
|
||||
argument++;
|
||||
litProba=0.;
|
||||
while ((*argument>='0') && (*argument<='9'))
|
||||
litProba *= 10, litProba += *argument++ - '0';
|
||||
if (litProba>100.) litProba=100.;
|
||||
litProba /= 100.;
|
||||
break;
|
||||
case 'v':
|
||||
displayLevel = 4;
|
||||
argument++;
|
||||
break;
|
||||
default:
|
||||
return usage(programName);
|
||||
}
|
||||
} } } /* for(argNb=1; argNb<argc; argNb++) */
|
||||
|
||||
DISPLAYLEVEL(4, "Data Generator \n");
|
||||
DISPLAYLEVEL(3, "Seed = %u \n", seed);
|
||||
if (proba!=COMPRESSIBILITY_DEFAULT) DISPLAYLEVEL(3, "Compressibility : %i%%\n", (U32)(proba*100));
|
||||
|
||||
RDG_genStdout(size, proba, litProba, seed);
|
||||
DISPLAYLEVEL(1, "\n");
|
||||
|
||||
return 0;
|
||||
}
|
||||
@@ -1,539 +0,0 @@
|
||||
/*
|
||||
fullbench.c - Detailed bench program for zstd
|
||||
Copyright (C) Yann Collet 2014-2016
|
||||
|
||||
GPL v2 License
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation; either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License along
|
||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
You can contact the author at :
|
||||
- zstd homepage : http://www.zstd.net
|
||||
*/
|
||||
|
||||
/*_************************************
|
||||
* Includes
|
||||
**************************************/
|
||||
#include "util.h" /* Compiler options, UTIL_GetFileSize */
|
||||
#include <stdlib.h> /* malloc */
|
||||
#include <stdio.h> /* fprintf, fopen, ftello64 */
|
||||
#include <time.h> /* clock_t, clock, CLOCKS_PER_SEC */
|
||||
|
||||
#include "mem.h"
|
||||
#include "zstd_internal.h" /* ZSTD_blockHeaderSize, blockType_e, KB, MB */
|
||||
#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_compressBegin, ZSTD_compressContinue, etc. */
|
||||
#include "zstd.h" /* ZSTD_VERSION_STRING */
|
||||
#define FSE_STATIC_LINKING_ONLY /* FSE_DTABLE_SIZE_U32 */
|
||||
#include "fse.h"
|
||||
#include "zbuff.h"
|
||||
#include "datagen.h"
|
||||
|
||||
|
||||
/*_************************************
|
||||
* Constants
|
||||
**************************************/
|
||||
#define PROGRAM_DESCRIPTION "Zstandard speed analyzer"
|
||||
#define AUTHOR "Yann Collet"
|
||||
#define WELCOME_MESSAGE "*** %s %s %i-bits, by %s (%s) ***\n", PROGRAM_DESCRIPTION, ZSTD_VERSION_STRING, (int)(sizeof(void*)*8), AUTHOR, __DATE__
|
||||
|
||||
#define NBLOOPS 6
|
||||
#define TIMELOOP_S 2
|
||||
|
||||
#define KNUTH 2654435761U
|
||||
#define MAX_MEM (1984 MB)
|
||||
|
||||
#define COMPRESSIBILITY_DEFAULT 0.50
|
||||
static const size_t g_sampleSize = 10000000;
|
||||
|
||||
|
||||
/*_************************************
|
||||
* Macros
|
||||
**************************************/
|
||||
#define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
|
||||
|
||||
|
||||
/*_************************************
|
||||
* Benchmark Parameters
|
||||
**************************************/
|
||||
static U32 g_nbIterations = NBLOOPS;
|
||||
static double g_compressibility = COMPRESSIBILITY_DEFAULT;
|
||||
|
||||
static void BMK_SetNbIterations(U32 nbLoops)
|
||||
{
|
||||
g_nbIterations = nbLoops;
|
||||
DISPLAY("- %i iterations -\n", g_nbIterations);
|
||||
}
|
||||
|
||||
|
||||
/*_*******************************************************
|
||||
* Private functions
|
||||
*********************************************************/
|
||||
static clock_t BMK_clockSpan( clock_t clockStart )
|
||||
{
|
||||
return clock() - clockStart; /* works even if overflow, span limited to <= ~30mn */
|
||||
}
|
||||
|
||||
|
||||
static size_t BMK_findMaxMem(U64 requiredMem)
|
||||
{
|
||||
size_t const step = 64 MB;
|
||||
void* testmem = NULL;
|
||||
|
||||
requiredMem = (((requiredMem >> 26) + 1) << 26);
|
||||
if (requiredMem > MAX_MEM) requiredMem = MAX_MEM;
|
||||
|
||||
requiredMem += step;
|
||||
do {
|
||||
testmem = malloc ((size_t)requiredMem);
|
||||
requiredMem -= step;
|
||||
} while (!testmem);
|
||||
|
||||
free (testmem);
|
||||
return (size_t) requiredMem;
|
||||
}
|
||||
|
||||
|
||||
/*_*******************************************************
|
||||
* Benchmark wrappers
|
||||
*********************************************************/
|
||||
typedef struct {
|
||||
blockType_e blockType;
|
||||
U32 unusedBits;
|
||||
U32 origSize;
|
||||
} blockProperties_t;
|
||||
|
||||
size_t local_ZSTD_compress(void* dst, size_t dstSize, void* buff2, const void* src, size_t srcSize)
|
||||
{
|
||||
(void)buff2;
|
||||
return ZSTD_compress(dst, dstSize, src, srcSize, 1);
|
||||
}
|
||||
|
||||
static size_t g_cSize = 0;
|
||||
size_t local_ZSTD_decompress(void* dst, size_t dstSize, void* buff2, const void* src, size_t srcSize)
|
||||
{
|
||||
(void)src; (void)srcSize;
|
||||
return ZSTD_decompress(dst, dstSize, buff2, g_cSize);
|
||||
}
|
||||
|
||||
static ZSTD_DCtx* g_zdc = NULL;
|
||||
extern size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* ctx, const void* src, size_t srcSize);
|
||||
size_t local_ZSTD_decodeLiteralsBlock(void* dst, size_t dstSize, void* buff2, const void* src, size_t srcSize)
|
||||
{
|
||||
(void)src; (void)srcSize; (void)dst; (void)dstSize;
|
||||
return ZSTD_decodeLiteralsBlock((ZSTD_DCtx*)g_zdc, buff2, g_cSize);
|
||||
}
|
||||
|
||||
extern size_t ZSTD_getcBlockSize(const void* src, size_t srcSize, blockProperties_t* bpPtr);
|
||||
extern size_t ZSTD_decodeSeqHeaders(int* nbSeq, FSE_DTable* DTableLL, FSE_DTable* DTableML, FSE_DTable* DTableOffb, U32 tableRepeatFlag, const void* src, size_t srcSize);
|
||||
size_t local_ZSTD_decodeSeqHeaders(void* dst, size_t dstSize, void* buff2, const void* src, size_t srcSize)
|
||||
{
|
||||
U32 DTableML[FSE_DTABLE_SIZE_U32(10)], DTableLL[FSE_DTABLE_SIZE_U32(10)], DTableOffb[FSE_DTABLE_SIZE_U32(9)]; /* MLFSELog, LLFSELog and OffFSELog are not public values */
|
||||
int nbSeq;
|
||||
(void)src; (void)srcSize; (void)dst; (void)dstSize;
|
||||
return ZSTD_decodeSeqHeaders(&nbSeq, DTableLL, DTableML, DTableOffb, 0, buff2, g_cSize);
|
||||
}
|
||||
|
||||
|
||||
static ZBUFF_CCtx* g_zbcc = NULL;
|
||||
size_t local_ZBUFF_compress(void* dst, size_t dstCapacity, void* buff2, const void* src, size_t srcSize)
|
||||
{
|
||||
size_t compressedSize;
|
||||
size_t srcRead = srcSize, dstWritten = dstCapacity;
|
||||
(void)buff2;
|
||||
ZBUFF_compressInit(g_zbcc, 1);
|
||||
ZBUFF_compressContinue(g_zbcc, dst, &dstWritten, src, &srcRead);
|
||||
compressedSize = dstWritten;
|
||||
dstWritten = dstCapacity-compressedSize;
|
||||
ZBUFF_compressEnd(g_zbcc, ((char*)dst)+compressedSize, &dstWritten);
|
||||
compressedSize += dstWritten;
|
||||
return compressedSize;
|
||||
}
|
||||
|
||||
static ZBUFF_DCtx* g_zbdc = NULL;
|
||||
static size_t local_ZBUFF_decompress(void* dst, size_t dstCapacity, void* buff2, const void* src, size_t srcSize)
|
||||
{
|
||||
size_t srcRead = g_cSize, dstWritten = dstCapacity;
|
||||
(void)src; (void)srcSize;
|
||||
ZBUFF_decompressInit(g_zbdc);
|
||||
ZBUFF_decompressContinue(g_zbdc, dst, &dstWritten, buff2, &srcRead);
|
||||
return dstWritten;
|
||||
}
|
||||
|
||||
static ZSTD_CCtx* g_zcc = NULL;
|
||||
size_t local_ZSTD_compressContinue(void* dst, size_t dstCapacity, void* buff2, const void* src, size_t srcSize)
|
||||
{
|
||||
(void)buff2;
|
||||
ZSTD_compressBegin(g_zcc, 1);
|
||||
return ZSTD_compressEnd(g_zcc, dst, dstCapacity, src, srcSize);
|
||||
}
|
||||
|
||||
size_t local_ZSTD_decompressContinue(void* dst, size_t dstCapacity, void* buff2, const void* src, size_t srcSize)
|
||||
{
|
||||
size_t regeneratedSize = 0;
|
||||
const BYTE* ip = (const BYTE*)buff2;
|
||||
const BYTE* const iend = ip + g_cSize;
|
||||
BYTE* op = (BYTE*)dst;
|
||||
size_t remainingCapacity = dstCapacity;
|
||||
|
||||
(void)src; (void)srcSize;
|
||||
ZSTD_decompressBegin(g_zdc);
|
||||
while (ip < iend) {
|
||||
size_t const iSize = ZSTD_nextSrcSizeToDecompress(g_zdc);
|
||||
size_t const decodedSize = ZSTD_decompressContinue(g_zdc, op, remainingCapacity, ip, iSize);
|
||||
ip += iSize;
|
||||
regeneratedSize += decodedSize;
|
||||
op += decodedSize;
|
||||
remainingCapacity -= decodedSize;
|
||||
}
|
||||
|
||||
return regeneratedSize;
|
||||
}
|
||||
|
||||
|
||||
/*_*******************************************************
|
||||
* Bench functions
|
||||
*********************************************************/
|
||||
static size_t benchMem(const void* src, size_t srcSize, U32 benchNb)
|
||||
{
|
||||
BYTE* dstBuff;
|
||||
size_t const dstBuffSize = ZSTD_compressBound(srcSize);
|
||||
void* buff2;
|
||||
const char* benchName;
|
||||
size_t (*benchFunction)(void* dst, size_t dstSize, void* verifBuff, const void* src, size_t srcSize);
|
||||
double bestTime = 100000000.;
|
||||
|
||||
/* Selection */
|
||||
switch(benchNb)
|
||||
{
|
||||
case 1:
|
||||
benchFunction = local_ZSTD_compress; benchName = "ZSTD_compress";
|
||||
break;
|
||||
case 2:
|
||||
benchFunction = local_ZSTD_decompress; benchName = "ZSTD_decompress";
|
||||
break;
|
||||
case 11:
|
||||
benchFunction = local_ZSTD_compressContinue; benchName = "ZSTD_compressContinue";
|
||||
break;
|
||||
case 12:
|
||||
benchFunction = local_ZSTD_decompressContinue; benchName = "ZSTD_decompressContinue";
|
||||
break;
|
||||
case 31:
|
||||
benchFunction = local_ZSTD_decodeLiteralsBlock; benchName = "ZSTD_decodeLiteralsBlock";
|
||||
break;
|
||||
case 32:
|
||||
benchFunction = local_ZSTD_decodeSeqHeaders; benchName = "ZSTD_decodeSeqHeaders";
|
||||
break;
|
||||
case 41:
|
||||
benchFunction = local_ZBUFF_compress; benchName = "ZBUFF_compressContinue";
|
||||
break;
|
||||
case 42:
|
||||
benchFunction = local_ZBUFF_decompress; benchName = "ZBUFF_decompressContinue";
|
||||
break;
|
||||
default :
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* Allocation */
|
||||
dstBuff = (BYTE*)malloc(dstBuffSize);
|
||||
buff2 = malloc(dstBuffSize);
|
||||
if ((!dstBuff) || (!buff2)) {
|
||||
DISPLAY("\nError: not enough memory!\n");
|
||||
free(dstBuff); free(buff2);
|
||||
return 12;
|
||||
}
|
||||
|
||||
/* Preparation */
|
||||
switch(benchNb)
|
||||
{
|
||||
case 2:
|
||||
g_cSize = ZSTD_compress(buff2, dstBuffSize, src, srcSize, 1);
|
||||
break;
|
||||
case 11 :
|
||||
if (g_zcc==NULL) g_zcc = ZSTD_createCCtx();
|
||||
break;
|
||||
case 12 :
|
||||
if (g_zdc==NULL) g_zdc = ZSTD_createDCtx();
|
||||
g_cSize = ZSTD_compress(buff2, dstBuffSize, src, srcSize, 1);
|
||||
break;
|
||||
case 31: /* ZSTD_decodeLiteralsBlock */
|
||||
if (g_zdc==NULL) g_zdc = ZSTD_createDCtx();
|
||||
{ blockProperties_t bp;
|
||||
ZSTD_frameParams zfp;
|
||||
size_t frameHeaderSize, skippedSize;
|
||||
g_cSize = ZSTD_compress(dstBuff, dstBuffSize, src, srcSize, 1);
|
||||
frameHeaderSize = ZSTD_getFrameParams(&zfp, dstBuff, ZSTD_frameHeaderSize_min);
|
||||
if (frameHeaderSize==0) frameHeaderSize = ZSTD_frameHeaderSize_min;
|
||||
ZSTD_getcBlockSize(dstBuff+frameHeaderSize, dstBuffSize, &bp); /* Get 1st block type */
|
||||
if (bp.blockType != bt_compressed) {
|
||||
DISPLAY("ZSTD_decodeLiteralsBlock : impossible to test on this sample (not compressible)\n");
|
||||
goto _cleanOut;
|
||||
}
|
||||
skippedSize = frameHeaderSize + ZSTD_blockHeaderSize;
|
||||
memcpy(buff2, dstBuff+skippedSize, g_cSize-skippedSize);
|
||||
srcSize = srcSize > 128 KB ? 128 KB : srcSize; /* speed relative to block */
|
||||
break;
|
||||
}
|
||||
case 32: /* ZSTD_decodeSeqHeaders */
|
||||
if (g_zdc==NULL) g_zdc = ZSTD_createDCtx();
|
||||
{ blockProperties_t bp;
|
||||
ZSTD_frameParams zfp;
|
||||
const BYTE* ip = dstBuff;
|
||||
const BYTE* iend;
|
||||
size_t frameHeaderSize, cBlockSize;
|
||||
ZSTD_compress(dstBuff, dstBuffSize, src, srcSize, 1); /* it would be better to use direct block compression here */
|
||||
g_cSize = ZSTD_compress(dstBuff, dstBuffSize, src, srcSize, 1);
|
||||
frameHeaderSize = ZSTD_getFrameParams(&zfp, dstBuff, ZSTD_frameHeaderSize_min);
|
||||
if (frameHeaderSize==0) frameHeaderSize = ZSTD_frameHeaderSize_min;
|
||||
ip += frameHeaderSize; /* Skip frame Header */
|
||||
cBlockSize = ZSTD_getcBlockSize(ip, dstBuffSize, &bp); /* Get 1st block type */
|
||||
if (bp.blockType != bt_compressed) {
|
||||
DISPLAY("ZSTD_decodeSeqHeaders : impossible to test on this sample (not compressible)\n");
|
||||
goto _cleanOut;
|
||||
}
|
||||
iend = ip + ZSTD_blockHeaderSize + cBlockSize; /* End of first block */
|
||||
ip += ZSTD_blockHeaderSize; /* skip block header */
|
||||
ip += ZSTD_decodeLiteralsBlock(g_zdc, ip, iend-ip); /* skip literal segment */
|
||||
g_cSize = iend-ip;
|
||||
memcpy(buff2, ip, g_cSize); /* copy rest of block (it starts by SeqHeader) */
|
||||
srcSize = srcSize > 128 KB ? 128 KB : srcSize; /* speed relative to block */
|
||||
break;
|
||||
}
|
||||
case 41 :
|
||||
if (g_zbcc==NULL) g_zbcc = ZBUFF_createCCtx();
|
||||
break;
|
||||
case 42 :
|
||||
if (g_zbdc==NULL) g_zbdc = ZBUFF_createDCtx();
|
||||
g_cSize = ZSTD_compress(buff2, dstBuffSize, src, srcSize, 1);
|
||||
break;
|
||||
|
||||
/* test functions */
|
||||
/* by convention, test functions can be added > 100 */
|
||||
|
||||
default : ;
|
||||
}
|
||||
|
||||
{ size_t i; for (i=0; i<dstBuffSize; i++) dstBuff[i]=(BYTE)i; } /* warming up memory */
|
||||
|
||||
{ U32 loopNb;
|
||||
for (loopNb = 1; loopNb <= g_nbIterations; loopNb++) {
|
||||
clock_t const timeLoop = TIMELOOP_S * CLOCKS_PER_SEC;
|
||||
clock_t clockStart;
|
||||
U32 nbRounds;
|
||||
size_t benchResult=0;
|
||||
double averageTime;
|
||||
|
||||
DISPLAY("%2i- %-30.30s : \r", loopNb, benchName);
|
||||
|
||||
clockStart = clock();
|
||||
while (clock() == clockStart);
|
||||
clockStart = clock();
|
||||
for (nbRounds=0; BMK_clockSpan(clockStart) < timeLoop; nbRounds++) {
|
||||
benchResult = benchFunction(dstBuff, dstBuffSize, buff2, src, srcSize);
|
||||
if (ZSTD_isError(benchResult)) { DISPLAY("ERROR ! %s() => %s !! \n", benchName, ZSTD_getErrorName(benchResult)); exit(1); }
|
||||
}
|
||||
averageTime = (((double)BMK_clockSpan(clockStart)) / CLOCKS_PER_SEC) / nbRounds;
|
||||
if (averageTime < bestTime) bestTime = averageTime;
|
||||
DISPLAY("%2i- %-30.30s : %7.1f MB/s (%9u)\r", loopNb, benchName, (double)srcSize / (1 MB) / bestTime, (U32)benchResult);
|
||||
}}
|
||||
DISPLAY("%2u\n", benchNb);
|
||||
|
||||
_cleanOut:
|
||||
free(dstBuff);
|
||||
free(buff2);
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
static int benchSample(U32 benchNb)
|
||||
{
|
||||
size_t const benchedSize = g_sampleSize;
|
||||
const char* name = "Sample 10MiB";
|
||||
|
||||
/* Allocation */
|
||||
void* origBuff = malloc(benchedSize);
|
||||
if (!origBuff) { DISPLAY("\nError: not enough memory!\n"); return 12; }
|
||||
|
||||
/* Fill buffer */
|
||||
RDG_genBuffer(origBuff, benchedSize, g_compressibility, 0.0, 0);
|
||||
|
||||
/* bench */
|
||||
DISPLAY("\r%79s\r", "");
|
||||
DISPLAY(" %s : \n", name);
|
||||
if (benchNb)
|
||||
benchMem(origBuff, benchedSize, benchNb);
|
||||
else
|
||||
for (benchNb=0; benchNb<100; benchNb++) benchMem(origBuff, benchedSize, benchNb);
|
||||
|
||||
free(origBuff);
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
static int benchFiles(const char** fileNamesTable, const int nbFiles, U32 benchNb)
|
||||
{
|
||||
/* Loop for each file */
|
||||
int fileIdx;
|
||||
for (fileIdx=0; fileIdx<nbFiles; fileIdx++) {
|
||||
const char* inFileName = fileNamesTable[fileIdx];
|
||||
FILE* inFile = fopen( inFileName, "rb" );
|
||||
U64 inFileSize;
|
||||
size_t benchedSize;
|
||||
void* origBuff;
|
||||
|
||||
/* Check file existence */
|
||||
if (inFile==NULL) { DISPLAY( "Pb opening %s\n", inFileName); return 11; }
|
||||
|
||||
/* Memory allocation & restrictions */
|
||||
inFileSize = UTIL_getFileSize(inFileName);
|
||||
benchedSize = BMK_findMaxMem(inFileSize*3) / 3;
|
||||
if ((U64)benchedSize > inFileSize) benchedSize = (size_t)inFileSize;
|
||||
if (benchedSize < inFileSize)
|
||||
DISPLAY("Not enough memory for '%s' full size; testing %u MB only...\n", inFileName, (U32)(benchedSize>>20));
|
||||
|
||||
/* Alloc */
|
||||
origBuff = malloc(benchedSize);
|
||||
if (!origBuff) { DISPLAY("\nError: not enough memory!\n"); fclose(inFile); return 12; }
|
||||
|
||||
/* Fill input buffer */
|
||||
DISPLAY("Loading %s... \r", inFileName);
|
||||
{
|
||||
size_t readSize = fread(origBuff, 1, benchedSize, inFile);
|
||||
fclose(inFile);
|
||||
if (readSize != benchedSize) {
|
||||
DISPLAY("\nError: problem reading file '%s' !! \n", inFileName);
|
||||
free(origBuff);
|
||||
return 13;
|
||||
} }
|
||||
|
||||
/* bench */
|
||||
DISPLAY("\r%79s\r", "");
|
||||
DISPLAY(" %s : \n", inFileName);
|
||||
if (benchNb)
|
||||
benchMem(origBuff, benchedSize, benchNb);
|
||||
else
|
||||
for (benchNb=0; benchNb<100; benchNb++) benchMem(origBuff, benchedSize, benchNb);
|
||||
|
||||
free(origBuff);
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
static int usage(const char* exename)
|
||||
{
|
||||
DISPLAY( "Usage :\n");
|
||||
DISPLAY( " %s [arg] file1 file2 ... fileX\n", exename);
|
||||
DISPLAY( "Arguments :\n");
|
||||
DISPLAY( " -H/-h : Help (this text + advanced options)\n");
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int usage_advanced(const char* exename)
|
||||
{
|
||||
usage(exename);
|
||||
DISPLAY( "\nAdvanced options :\n");
|
||||
DISPLAY( " -b# : test only function # \n");
|
||||
DISPLAY( " -i# : iteration loops [1-9](default : %i)\n", NBLOOPS);
|
||||
DISPLAY( " -P# : sample compressibility (default : %.1f%%)\n", COMPRESSIBILITY_DEFAULT * 100);
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int badusage(const char* exename)
|
||||
{
|
||||
DISPLAY("Wrong parameters\n");
|
||||
usage(exename);
|
||||
return 1;
|
||||
}
|
||||
|
||||
int main(int argc, const char** argv)
|
||||
{
|
||||
int i, filenamesStart=0, result;
|
||||
const char* exename = argv[0];
|
||||
const char* input_filename = NULL;
|
||||
U32 benchNb = 0, main_pause = 0;
|
||||
|
||||
DISPLAY(WELCOME_MESSAGE);
|
||||
if (argc<1) return badusage(exename);
|
||||
|
||||
for(i=1; i<argc; i++) {
|
||||
const char* argument = argv[i];
|
||||
if(!argument) continue; /* Protection if argument empty */
|
||||
|
||||
/* Commands (note : aggregated commands are allowed) */
|
||||
if (argument[0]=='-') {
|
||||
|
||||
while (argument[1]!=0) {
|
||||
argument++;
|
||||
|
||||
switch(argument[0])
|
||||
{
|
||||
/* Display help on usage */
|
||||
case 'h' :
|
||||
case 'H': return usage_advanced(exename);
|
||||
|
||||
/* Pause at the end (hidden option) */
|
||||
case 'p': main_pause = 1; break;
|
||||
|
||||
/* Select specific algorithm to bench */
|
||||
case 'b':
|
||||
benchNb = 0;
|
||||
while ((argument[1]>= '0') && (argument[1]<= '9')) {
|
||||
benchNb *= 10;
|
||||
benchNb += argument[1] - '0';
|
||||
argument++;
|
||||
}
|
||||
break;
|
||||
|
||||
/* Modify Nb Iterations */
|
||||
case 'i':
|
||||
if ((argument[1] >='0') && (argument[1] <='9')) {
|
||||
int iters = argument[1] - '0';
|
||||
BMK_SetNbIterations(iters);
|
||||
argument++;
|
||||
}
|
||||
break;
|
||||
|
||||
/* Select compressibility of synthetic sample */
|
||||
case 'P':
|
||||
{ U32 proba32 = 0;
|
||||
while ((argument[1]>= '0') && (argument[1]<= '9')) {
|
||||
proba32 *= 10;
|
||||
proba32 += argument[1] - '0';
|
||||
argument++;
|
||||
}
|
||||
g_compressibility = (double)proba32 / 100.;
|
||||
}
|
||||
break;
|
||||
|
||||
/* Unknown command */
|
||||
default : return badusage(exename);
|
||||
}
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
/* first provided filename is input */
|
||||
if (!input_filename) { input_filename=argument; filenamesStart=i; continue; }
|
||||
}
|
||||
|
||||
if (filenamesStart==0) /* no input file */
|
||||
result = benchSample(benchNb);
|
||||
else
|
||||
result = benchFiles(argv+filenamesStart, argc-filenamesStart, benchNb);
|
||||
|
||||
if (main_pause) { int unused; printf("press enter...\n"); unused = getchar(); (void)unused; }
|
||||
|
||||
return result;
|
||||
}
|
||||
@@ -1,898 +0,0 @@
|
||||
/*
|
||||
Fuzzer test tool for zstd
|
||||
Copyright (C) Yann Collet 2014-2016
|
||||
|
||||
GPL v2 License
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation; either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License along
|
||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
You can contact the author at :
|
||||
- ZSTD homepage : http://www.zstd.net
|
||||
*/
|
||||
|
||||
/*-************************************
|
||||
* Compiler specific
|
||||
**************************************/
|
||||
#ifdef _MSC_VER /* Visual Studio */
|
||||
# define _CRT_SECURE_NO_WARNINGS /* fgets */
|
||||
# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
|
||||
# pragma warning(disable : 4204) /* disable: C4204: non-constant aggregate initializer */
|
||||
#endif
|
||||
|
||||
|
||||
/*-************************************
|
||||
* Includes
|
||||
**************************************/
|
||||
#include <stdlib.h> /* free */
|
||||
#include <stdio.h> /* fgets, sscanf */
|
||||
#include <sys/timeb.h> /* timeb */
|
||||
#include <string.h> /* strcmp */
|
||||
#include <time.h> /* clock_t */
|
||||
#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_compressContinue, ZSTD_compressBlock */
|
||||
#include "zstd.h" /* ZSTD_VERSION_STRING */
|
||||
#include "error_public.h" /* ZSTD_getErrorCode */
|
||||
#include "zdict.h" /* ZDICT_trainFromBuffer */
|
||||
#include "datagen.h" /* RDG_genBuffer */
|
||||
#include "mem.h"
|
||||
#define XXH_STATIC_LINKING_ONLY
|
||||
#include "xxhash.h" /* XXH64 */
|
||||
|
||||
|
||||
/*-************************************
|
||||
* Constants
|
||||
**************************************/
|
||||
#define KB *(1U<<10)
|
||||
#define MB *(1U<<20)
|
||||
#define GB *(1U<<30)
|
||||
|
||||
static const U32 FUZ_compressibility_default = 50;
|
||||
static const U32 nbTestsDefault = 30000;
|
||||
|
||||
|
||||
/*-************************************
|
||||
* Display Macros
|
||||
**************************************/
|
||||
#define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
|
||||
#define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); }
|
||||
static U32 g_displayLevel = 2;
|
||||
|
||||
#define DISPLAYUPDATE(l, ...) if (g_displayLevel>=l) { \
|
||||
if ((FUZ_clockSpan(g_displayClock) > g_refreshRate) || (g_displayLevel>=4)) \
|
||||
{ g_displayClock = clock(); DISPLAY(__VA_ARGS__); \
|
||||
if (g_displayLevel>=4) fflush(stdout); } }
|
||||
static const clock_t g_refreshRate = CLOCKS_PER_SEC * 150 / 1000;
|
||||
static clock_t g_displayClock = 0;
|
||||
|
||||
|
||||
/*-*******************************************************
|
||||
* Fuzzer functions
|
||||
*********************************************************/
|
||||
#define MIN(a,b) ((a)<(b)?(a):(b))
|
||||
|
||||
static clock_t FUZ_clockSpan(clock_t cStart)
|
||||
{
|
||||
return clock() - cStart; /* works even when overflow; max span ~ 30mn */
|
||||
}
|
||||
|
||||
|
||||
#define FUZ_rotl32(x,r) ((x << r) | (x >> (32 - r)))
|
||||
static unsigned FUZ_rand(unsigned* src)
|
||||
{
|
||||
static const U32 prime1 = 2654435761U;
|
||||
static const U32 prime2 = 2246822519U;
|
||||
U32 rand32 = *src;
|
||||
rand32 *= prime1;
|
||||
rand32 += prime2;
|
||||
rand32 = FUZ_rotl32(rand32, 13);
|
||||
*src = rand32;
|
||||
return rand32 >> 5;
|
||||
}
|
||||
|
||||
|
||||
static unsigned FUZ_highbit32(U32 v32)
|
||||
{
|
||||
unsigned nbBits = 0;
|
||||
if (v32==0) return 0;
|
||||
while (v32) v32 >>= 1, nbBits++;
|
||||
return nbBits;
|
||||
}
|
||||
|
||||
|
||||
#define CHECK_V(var, fn) size_t const var = fn; if (ZSTD_isError(var)) goto _output_error
|
||||
#define CHECK(fn) { CHECK_V(err, fn); }
|
||||
#define CHECKPLUS(var, fn, more) { CHECK_V(var, fn); more; }
|
||||
static int basicUnitTests(U32 seed, double compressibility)
|
||||
{
|
||||
size_t const CNBuffSize = 5 MB;
|
||||
void* const CNBuffer = malloc(CNBuffSize);
|
||||
void* const compressedBuffer = malloc(ZSTD_compressBound(CNBuffSize));
|
||||
void* const decodedBuffer = malloc(CNBuffSize);
|
||||
int testResult = 0;
|
||||
U32 testNb=0;
|
||||
size_t cSize;
|
||||
|
||||
/* Create compressible noise */
|
||||
if (!CNBuffer || !compressedBuffer || !decodedBuffer) {
|
||||
DISPLAY("Not enough memory, aborting\n");
|
||||
testResult = 1;
|
||||
goto _end;
|
||||
}
|
||||
RDG_genBuffer(CNBuffer, CNBuffSize, compressibility, 0., seed);
|
||||
|
||||
/* Basic tests */
|
||||
DISPLAYLEVEL(4, "test%3i : ZSTD_getErrorName : ", testNb++);
|
||||
{ const char* errorString = ZSTD_getErrorName(0);
|
||||
DISPLAYLEVEL(4, "OK : %s \n", errorString);
|
||||
}
|
||||
|
||||
DISPLAYLEVEL(4, "test%3i : ZSTD_getErrorName with wrong value : ", testNb++);
|
||||
{ const char* errorString = ZSTD_getErrorName(499);
|
||||
DISPLAYLEVEL(4, "OK : %s \n", errorString);
|
||||
}
|
||||
|
||||
DISPLAYLEVEL(4, "test%3i : compress %u bytes : ", testNb++, (U32)CNBuffSize);
|
||||
CHECKPLUS(r, ZSTD_compress(compressedBuffer, ZSTD_compressBound(CNBuffSize),
|
||||
CNBuffer, CNBuffSize, 1),
|
||||
cSize=r );
|
||||
DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/CNBuffSize*100);
|
||||
|
||||
DISPLAYLEVEL(4, "test%3i : decompressed size test : ", testNb++);
|
||||
{ unsigned long long const rSize = ZSTD_getDecompressedSize(compressedBuffer, cSize);
|
||||
if (rSize != CNBuffSize) goto _output_error;
|
||||
}
|
||||
DISPLAYLEVEL(4, "OK \n");
|
||||
|
||||
DISPLAYLEVEL(4, "test%3i : decompress %u bytes : ", testNb++, (U32)CNBuffSize);
|
||||
{ size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize);
|
||||
if (r != CNBuffSize) goto _output_error; }
|
||||
DISPLAYLEVEL(4, "OK \n");
|
||||
|
||||
DISPLAYLEVEL(4, "test%3i : check decompressed result : ", testNb++);
|
||||
{ size_t u;
|
||||
for (u=0; u<CNBuffSize; u++) {
|
||||
if (((BYTE*)decodedBuffer)[u] != ((BYTE*)CNBuffer)[u]) goto _output_error;;
|
||||
} }
|
||||
DISPLAYLEVEL(4, "OK \n");
|
||||
|
||||
DISPLAYLEVEL(4, "test%3i : decompress with 1 missing byte : ", testNb++);
|
||||
{ size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize-1);
|
||||
if (!ZSTD_isError(r)) goto _output_error;
|
||||
if (ZSTD_getErrorCode((size_t)r) != ZSTD_error_srcSize_wrong) goto _output_error; }
|
||||
DISPLAYLEVEL(4, "OK \n");
|
||||
|
||||
DISPLAYLEVEL(4, "test%3i : decompress with 1 too much byte : ", testNb++);
|
||||
{ size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize+1);
|
||||
if (!ZSTD_isError(r)) goto _output_error;
|
||||
if (ZSTD_getErrorCode(r) != ZSTD_error_srcSize_wrong) goto _output_error; }
|
||||
DISPLAYLEVEL(4, "OK \n");
|
||||
|
||||
/* Dictionary and CCtx Duplication tests */
|
||||
{ ZSTD_CCtx* const ctxOrig = ZSTD_createCCtx();
|
||||
ZSTD_CCtx* const ctxDuplicated = ZSTD_createCCtx();
|
||||
ZSTD_DCtx* const dctx = ZSTD_createDCtx();
|
||||
static const size_t dictSize = 551;
|
||||
|
||||
DISPLAYLEVEL(4, "test%3i : copy context too soon : ", testNb++);
|
||||
{ size_t const copyResult = ZSTD_copyCCtx(ctxDuplicated, ctxOrig);
|
||||
if (!ZSTD_isError(copyResult)) goto _output_error; } /* error must be detected */
|
||||
DISPLAYLEVEL(4, "OK \n");
|
||||
|
||||
DISPLAYLEVEL(4, "test%3i : load dictionary into context : ", testNb++);
|
||||
CHECK( ZSTD_compressBegin_usingDict(ctxOrig, CNBuffer, dictSize, 2) );
|
||||
CHECK( ZSTD_copyCCtx(ctxDuplicated, ctxOrig) );
|
||||
DISPLAYLEVEL(4, "OK \n");
|
||||
|
||||
DISPLAYLEVEL(4, "test%3i : compress with flat dictionary : ", testNb++);
|
||||
cSize = 0;
|
||||
CHECKPLUS(r, ZSTD_compressEnd(ctxOrig, compressedBuffer, ZSTD_compressBound(CNBuffSize),
|
||||
(const char*)CNBuffer + dictSize, CNBuffSize - dictSize),
|
||||
cSize += r);
|
||||
DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/CNBuffSize*100);
|
||||
|
||||
DISPLAYLEVEL(4, "test%3i : frame built with flat dictionary should be decompressible : ", testNb++);
|
||||
CHECKPLUS(r, ZSTD_decompress_usingDict(dctx,
|
||||
decodedBuffer, CNBuffSize,
|
||||
compressedBuffer, cSize,
|
||||
CNBuffer, dictSize),
|
||||
if (r != CNBuffSize - dictSize) goto _output_error);
|
||||
DISPLAYLEVEL(4, "OK \n");
|
||||
|
||||
DISPLAYLEVEL(4, "test%3i : compress with duplicated context : ", testNb++);
|
||||
{ size_t const cSizeOrig = cSize;
|
||||
cSize = 0;
|
||||
CHECKPLUS(r, ZSTD_compressEnd(ctxDuplicated, compressedBuffer, ZSTD_compressBound(CNBuffSize),
|
||||
(const char*)CNBuffer + dictSize, CNBuffSize - dictSize),
|
||||
cSize += r);
|
||||
if (cSize != cSizeOrig) goto _output_error; /* should be identical ==> same size */
|
||||
}
|
||||
DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/CNBuffSize*100);
|
||||
|
||||
DISPLAYLEVEL(4, "test%3i : frame built with duplicated context should be decompressible : ", testNb++);
|
||||
CHECKPLUS(r, ZSTD_decompress_usingDict(dctx,
|
||||
decodedBuffer, CNBuffSize,
|
||||
compressedBuffer, cSize,
|
||||
CNBuffer, dictSize),
|
||||
if (r != CNBuffSize - dictSize) goto _output_error);
|
||||
DISPLAYLEVEL(4, "OK \n");
|
||||
|
||||
DISPLAYLEVEL(4, "test%3i : check content size on duplicated context : ", testNb++);
|
||||
{ size_t const testSize = CNBuffSize / 3;
|
||||
{ ZSTD_parameters p = ZSTD_getParams(2, testSize, dictSize);
|
||||
p.fParams.contentSizeFlag = 1;
|
||||
CHECK( ZSTD_compressBegin_advanced(ctxOrig, CNBuffer, dictSize, p, testSize-1) );
|
||||
}
|
||||
CHECK( ZSTD_copyCCtx(ctxDuplicated, ctxOrig) );
|
||||
|
||||
CHECKPLUS(r, ZSTD_compressContinue(ctxDuplicated, compressedBuffer, ZSTD_compressBound(testSize),
|
||||
(const char*)CNBuffer + dictSize, CNBuffSize - dictSize),
|
||||
cSize = r);
|
||||
{ ZSTD_frameParams fp;
|
||||
if (ZSTD_getFrameParams(&fp, compressedBuffer, cSize)) goto _output_error;
|
||||
if ((fp.frameContentSize != testSize) && (fp.frameContentSize != 0)) goto _output_error;
|
||||
} }
|
||||
DISPLAYLEVEL(4, "OK \n");
|
||||
|
||||
ZSTD_freeCCtx(ctxOrig);
|
||||
ZSTD_freeCCtx(ctxDuplicated);
|
||||
ZSTD_freeDCtx(dctx);
|
||||
}
|
||||
|
||||
/* Dictionary and dictBuilder tests */
|
||||
{ ZSTD_CCtx* const cctx = ZSTD_createCCtx();
|
||||
ZSTD_DCtx* const dctx = ZSTD_createDCtx();
|
||||
size_t dictSize = 16 KB;
|
||||
void* dictBuffer = malloc(dictSize);
|
||||
size_t const totalSampleSize = 1 MB;
|
||||
size_t const sampleUnitSize = 8 KB;
|
||||
U32 const nbSamples = (U32)(totalSampleSize / sampleUnitSize);
|
||||
size_t* const samplesSizes = (size_t*) malloc(nbSamples * sizeof(size_t));
|
||||
|
||||
if (dictBuffer==NULL || samplesSizes==NULL) {
|
||||
free(dictBuffer);
|
||||
free(samplesSizes);
|
||||
goto _output_error;
|
||||
}
|
||||
|
||||
DISPLAYLEVEL(4, "test%3i : dictBuilder : ", testNb++);
|
||||
{ U32 u; for (u=0; u<nbSamples; u++) samplesSizes[u] = sampleUnitSize; }
|
||||
dictSize = ZDICT_trainFromBuffer(dictBuffer, dictSize,
|
||||
CNBuffer, samplesSizes, nbSamples);
|
||||
if (ZDICT_isError(dictSize)) goto _output_error;
|
||||
DISPLAYLEVEL(4, "OK, created dictionary of size %u \n", (U32)dictSize);
|
||||
|
||||
DISPLAYLEVEL(4, "test%3i : compress with dictionary : ", testNb++);
|
||||
cSize = ZSTD_compress_usingDict(cctx, compressedBuffer, ZSTD_compressBound(CNBuffSize),
|
||||
CNBuffer, CNBuffSize,
|
||||
dictBuffer, dictSize, 4);
|
||||
if (ZSTD_isError(cSize)) goto _output_error;
|
||||
DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/CNBuffSize*100);
|
||||
|
||||
DISPLAYLEVEL(4, "test%3i : frame built with dictionary should be decompressible : ", testNb++);
|
||||
CHECKPLUS(r, ZSTD_decompress_usingDict(dctx,
|
||||
decodedBuffer, CNBuffSize,
|
||||
compressedBuffer, cSize,
|
||||
dictBuffer, dictSize),
|
||||
if (r != CNBuffSize) goto _output_error);
|
||||
DISPLAYLEVEL(4, "OK \n");
|
||||
|
||||
DISPLAYLEVEL(4, "test%3i : compress without dictID : ", testNb++);
|
||||
{ ZSTD_parameters p = ZSTD_getParams(3, CNBuffSize, dictSize);
|
||||
p.fParams.noDictIDFlag = 1;
|
||||
cSize = ZSTD_compress_advanced(cctx, compressedBuffer, ZSTD_compressBound(CNBuffSize),
|
||||
CNBuffer, CNBuffSize,
|
||||
dictBuffer, dictSize, p);
|
||||
if (ZSTD_isError(cSize)) goto _output_error;
|
||||
}
|
||||
DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/CNBuffSize*100);
|
||||
|
||||
DISPLAYLEVEL(4, "test%3i : frame built without dictID should be decompressible : ", testNb++);
|
||||
CHECKPLUS(r, ZSTD_decompress_usingDict(dctx,
|
||||
decodedBuffer, CNBuffSize,
|
||||
compressedBuffer, cSize,
|
||||
dictBuffer, dictSize),
|
||||
if (r != CNBuffSize) goto _output_error);
|
||||
DISPLAYLEVEL(4, "OK \n");
|
||||
|
||||
ZSTD_freeCCtx(cctx);
|
||||
ZSTD_freeDCtx(dctx);
|
||||
free(dictBuffer);
|
||||
free(samplesSizes);
|
||||
}
|
||||
|
||||
/* Decompression defense tests */
|
||||
DISPLAYLEVEL(4, "test%3i : Check input length for magic number : ", testNb++);
|
||||
{ size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, CNBuffer, 3);
|
||||
if (!ZSTD_isError(r)) goto _output_error;
|
||||
if (r != (size_t)-ZSTD_error_srcSize_wrong) goto _output_error; }
|
||||
DISPLAYLEVEL(4, "OK \n");
|
||||
|
||||
DISPLAYLEVEL(4, "test%3i : Check magic Number : ", testNb++);
|
||||
((char*)(CNBuffer))[0] = 1;
|
||||
{ size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, CNBuffer, 4);
|
||||
if (!ZSTD_isError(r)) goto _output_error; }
|
||||
DISPLAYLEVEL(4, "OK \n");
|
||||
|
||||
/* block API tests */
|
||||
{ ZSTD_CCtx* const cctx = ZSTD_createCCtx();
|
||||
ZSTD_DCtx* const dctx = ZSTD_createDCtx();
|
||||
static const size_t dictSize = 65 KB;
|
||||
static const size_t blockSize = 100 KB; /* won't cause pb with small dict size */
|
||||
size_t cSize2;
|
||||
|
||||
/* basic block compression */
|
||||
DISPLAYLEVEL(4, "test%3i : Block compression test : ", testNb++);
|
||||
CHECK( ZSTD_compressBegin(cctx, 5) );
|
||||
cSize = ZSTD_compressBlock(cctx, compressedBuffer, ZSTD_compressBound(blockSize), CNBuffer, blockSize);
|
||||
if (ZSTD_isError(cSize)) goto _output_error;
|
||||
DISPLAYLEVEL(4, "OK \n");
|
||||
|
||||
DISPLAYLEVEL(4, "test%3i : Block decompression test : ", testNb++);
|
||||
CHECK( ZSTD_decompressBegin(dctx) );
|
||||
{ CHECK_V(r, ZSTD_decompressBlock(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize) );
|
||||
if (r != blockSize) goto _output_error; }
|
||||
DISPLAYLEVEL(4, "OK \n");
|
||||
|
||||
/* dictionary block compression */
|
||||
DISPLAYLEVEL(4, "test%3i : Dictionary Block compression test : ", testNb++);
|
||||
CHECK( ZSTD_compressBegin_usingDict(cctx, CNBuffer, dictSize, 5) );
|
||||
cSize = ZSTD_compressBlock(cctx, compressedBuffer, ZSTD_compressBound(blockSize), (char*)CNBuffer+dictSize, blockSize);
|
||||
if (ZSTD_isError(cSize)) goto _output_error;
|
||||
cSize2 = ZSTD_compressBlock(cctx, (char*)compressedBuffer+cSize, ZSTD_compressBound(blockSize), (char*)CNBuffer+dictSize+blockSize, blockSize);
|
||||
if (ZSTD_isError(cSize2)) goto _output_error;
|
||||
memcpy((char*)compressedBuffer+cSize, (char*)CNBuffer+dictSize+blockSize, blockSize); /* fake non-compressed block */
|
||||
cSize2 = ZSTD_compressBlock(cctx, (char*)compressedBuffer+cSize+blockSize, ZSTD_compressBound(blockSize),
|
||||
(char*)CNBuffer+dictSize+2*blockSize, blockSize);
|
||||
if (ZSTD_isError(cSize2)) goto _output_error;
|
||||
DISPLAYLEVEL(4, "OK \n");
|
||||
|
||||
DISPLAYLEVEL(4, "test%3i : Dictionary Block decompression test : ", testNb++);
|
||||
CHECK( ZSTD_decompressBegin_usingDict(dctx, CNBuffer, dictSize) );
|
||||
{ CHECK_V( r, ZSTD_decompressBlock(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize) );
|
||||
if (r != blockSize) goto _output_error; }
|
||||
ZSTD_insertBlock(dctx, (char*)decodedBuffer+blockSize, blockSize); /* insert non-compressed block into dctx history */
|
||||
{ CHECK_V( r, ZSTD_decompressBlock(dctx, (char*)decodedBuffer+2*blockSize, CNBuffSize, (char*)compressedBuffer+cSize+blockSize, cSize2) );
|
||||
if (r != blockSize) goto _output_error; }
|
||||
DISPLAYLEVEL(4, "OK \n");
|
||||
|
||||
ZSTD_freeCCtx(cctx);
|
||||
ZSTD_freeDCtx(dctx);
|
||||
}
|
||||
|
||||
/* long rle test */
|
||||
{ size_t sampleSize = 0;
|
||||
DISPLAYLEVEL(4, "test%3i : Long RLE test : ", testNb++);
|
||||
RDG_genBuffer(CNBuffer, sampleSize, compressibility, 0., seed+1);
|
||||
memset((char*)CNBuffer+sampleSize, 'B', 256 KB - 1);
|
||||
sampleSize += 256 KB - 1;
|
||||
RDG_genBuffer((char*)CNBuffer+sampleSize, 96 KB, compressibility, 0., seed+2);
|
||||
sampleSize += 96 KB;
|
||||
cSize = ZSTD_compress(compressedBuffer, ZSTD_compressBound(sampleSize), CNBuffer, sampleSize, 1);
|
||||
if (ZSTD_isError(cSize)) goto _output_error;
|
||||
{ CHECK_V(regenSize, ZSTD_decompress(decodedBuffer, sampleSize, compressedBuffer, cSize));
|
||||
if (regenSize!=sampleSize) goto _output_error; }
|
||||
DISPLAYLEVEL(4, "OK \n");
|
||||
}
|
||||
|
||||
/* All zeroes test (test bug #137) */
|
||||
#define ZEROESLENGTH 100
|
||||
DISPLAYLEVEL(4, "test%3i : compress %u zeroes : ", testNb++, ZEROESLENGTH);
|
||||
memset(CNBuffer, 0, ZEROESLENGTH);
|
||||
{ CHECK_V(r, ZSTD_compress(compressedBuffer, ZSTD_compressBound(ZEROESLENGTH), CNBuffer, ZEROESLENGTH, 1) );
|
||||
cSize = r; }
|
||||
DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/ZEROESLENGTH*100);
|
||||
|
||||
DISPLAYLEVEL(4, "test%3i : decompress %u zeroes : ", testNb++, ZEROESLENGTH);
|
||||
{ CHECK_V(r, ZSTD_decompress(decodedBuffer, ZEROESLENGTH, compressedBuffer, cSize) );
|
||||
if (r != ZEROESLENGTH) goto _output_error; }
|
||||
DISPLAYLEVEL(4, "OK \n");
|
||||
|
||||
/* nbSeq limit test */
|
||||
#define _3BYTESTESTLENGTH 131000
|
||||
#define NB3BYTESSEQLOG 9
|
||||
#define NB3BYTESSEQ (1 << NB3BYTESSEQLOG)
|
||||
#define NB3BYTESSEQMASK (NB3BYTESSEQ-1)
|
||||
/* creates a buffer full of 3-bytes sequences */
|
||||
{ BYTE _3BytesSeqs[NB3BYTESSEQ][3];
|
||||
U32 rSeed = 1;
|
||||
|
||||
/* create batch of 3-bytes sequences */
|
||||
{ int i;
|
||||
for (i=0; i < NB3BYTESSEQ; i++) {
|
||||
_3BytesSeqs[i][0] = (BYTE)(FUZ_rand(&rSeed) & 255);
|
||||
_3BytesSeqs[i][1] = (BYTE)(FUZ_rand(&rSeed) & 255);
|
||||
_3BytesSeqs[i][2] = (BYTE)(FUZ_rand(&rSeed) & 255);
|
||||
} }
|
||||
|
||||
/* randomly fills CNBuffer with prepared 3-bytes sequences */
|
||||
{ int i;
|
||||
for (i=0; i < _3BYTESTESTLENGTH; i += 3) { /* note : CNBuffer size > _3BYTESTESTLENGTH+3 */
|
||||
U32 const id = FUZ_rand(&rSeed) & NB3BYTESSEQMASK;
|
||||
((BYTE*)CNBuffer)[i+0] = _3BytesSeqs[id][0];
|
||||
((BYTE*)CNBuffer)[i+1] = _3BytesSeqs[id][1];
|
||||
((BYTE*)CNBuffer)[i+2] = _3BytesSeqs[id][2];
|
||||
} } }
|
||||
DISPLAYLEVEL(4, "test%3i : compress lots 3-bytes sequences : ", testNb++);
|
||||
{ CHECK_V(r, ZSTD_compress(compressedBuffer, ZSTD_compressBound(_3BYTESTESTLENGTH),
|
||||
CNBuffer, _3BYTESTESTLENGTH, 19) );
|
||||
cSize = r; }
|
||||
DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/_3BYTESTESTLENGTH*100);
|
||||
|
||||
DISPLAYLEVEL(4, "test%3i : decompress lots 3-bytes sequence : ", testNb++);
|
||||
{ CHECK_V(r, ZSTD_decompress(decodedBuffer, _3BYTESTESTLENGTH, compressedBuffer, cSize) );
|
||||
if (r != _3BYTESTESTLENGTH) goto _output_error; }
|
||||
DISPLAYLEVEL(4, "OK \n");
|
||||
|
||||
_end:
|
||||
free(CNBuffer);
|
||||
free(compressedBuffer);
|
||||
free(decodedBuffer);
|
||||
return testResult;
|
||||
|
||||
_output_error:
|
||||
testResult = 1;
|
||||
DISPLAY("Error detected in Unit tests ! \n");
|
||||
goto _end;
|
||||
}
|
||||
|
||||
|
||||
static size_t findDiff(const void* buf1, const void* buf2, size_t max)
|
||||
{
|
||||
const BYTE* b1 = (const BYTE*)buf1;
|
||||
const BYTE* b2 = (const BYTE*)buf2;
|
||||
size_t u;
|
||||
for (u=0; u<max; u++) {
|
||||
if (b1[u] != b2[u]) break;
|
||||
}
|
||||
return u;
|
||||
}
|
||||
|
||||
|
||||
static size_t FUZ_rLogLength(U32* seed, U32 logLength)
|
||||
{
|
||||
size_t const lengthMask = ((size_t)1 << logLength) - 1;
|
||||
return (lengthMask+1) + (FUZ_rand(seed) & lengthMask);
|
||||
}
|
||||
|
||||
static size_t FUZ_randomLength(U32* seed, U32 maxLog)
|
||||
{
|
||||
U32 const logLength = FUZ_rand(seed) % maxLog;
|
||||
return FUZ_rLogLength(seed, logLength);
|
||||
}
|
||||
|
||||
#undef CHECK
|
||||
#define CHECK(cond, ...) if (cond) { DISPLAY("Error => "); DISPLAY(__VA_ARGS__); \
|
||||
DISPLAY(" (seed %u, test nb %u) \n", seed, testNb); goto _output_error; }
|
||||
|
||||
static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxDurationS, double compressibility)
|
||||
{
|
||||
static const U32 maxSrcLog = 23;
|
||||
static const U32 maxSampleLog = 22;
|
||||
size_t const srcBufferSize = (size_t)1<<maxSrcLog;
|
||||
size_t const dstBufferSize = (size_t)1<<maxSampleLog;
|
||||
size_t const cBufferSize = ZSTD_compressBound(dstBufferSize);
|
||||
BYTE* cNoiseBuffer[5];
|
||||
BYTE* srcBuffer; /* jumping pointer */
|
||||
BYTE* const cBuffer = (BYTE*) malloc (cBufferSize);
|
||||
BYTE* const dstBuffer = (BYTE*) malloc (dstBufferSize);
|
||||
BYTE* const mirrorBuffer = (BYTE*) malloc (dstBufferSize);
|
||||
ZSTD_CCtx* const refCtx = ZSTD_createCCtx();
|
||||
ZSTD_CCtx* const ctx = ZSTD_createCCtx();
|
||||
ZSTD_DCtx* const dctx = ZSTD_createDCtx();
|
||||
U32 result = 0;
|
||||
U32 testNb = 0;
|
||||
U32 coreSeed = seed, lseed = 0;
|
||||
clock_t const startClock = clock();
|
||||
clock_t const maxClockSpan = maxDurationS * CLOCKS_PER_SEC;
|
||||
|
||||
/* allocation */
|
||||
cNoiseBuffer[0] = (BYTE*)malloc (srcBufferSize);
|
||||
cNoiseBuffer[1] = (BYTE*)malloc (srcBufferSize);
|
||||
cNoiseBuffer[2] = (BYTE*)malloc (srcBufferSize);
|
||||
cNoiseBuffer[3] = (BYTE*)malloc (srcBufferSize);
|
||||
cNoiseBuffer[4] = (BYTE*)malloc (srcBufferSize);
|
||||
CHECK (!cNoiseBuffer[0] || !cNoiseBuffer[1] || !cNoiseBuffer[2] || !cNoiseBuffer[3] || !cNoiseBuffer[4]
|
||||
|| !dstBuffer || !mirrorBuffer || !cBuffer || !refCtx || !ctx || !dctx,
|
||||
"Not enough memory, fuzzer tests cancelled");
|
||||
|
||||
/* Create initial samples */
|
||||
RDG_genBuffer(cNoiseBuffer[0], srcBufferSize, 0.00, 0., coreSeed); /* pure noise */
|
||||
RDG_genBuffer(cNoiseBuffer[1], srcBufferSize, 0.05, 0., coreSeed); /* barely compressible */
|
||||
RDG_genBuffer(cNoiseBuffer[2], srcBufferSize, compressibility, 0., coreSeed);
|
||||
RDG_genBuffer(cNoiseBuffer[3], srcBufferSize, 0.95, 0., coreSeed); /* highly compressible */
|
||||
RDG_genBuffer(cNoiseBuffer[4], srcBufferSize, 1.00, 0., coreSeed); /* sparse content */
|
||||
srcBuffer = cNoiseBuffer[2];
|
||||
|
||||
/* catch up testNb */
|
||||
for (testNb=1; testNb < startTest; testNb++) FUZ_rand(&coreSeed);
|
||||
|
||||
/* main test loop */
|
||||
for ( ; (testNb <= nbTests) || (FUZ_clockSpan(startClock) < maxClockSpan); testNb++ ) {
|
||||
size_t sampleSize, maxTestSize, totalTestSize;
|
||||
size_t cSize, totalCSize, totalGenSize;
|
||||
XXH64_state_t xxhState;
|
||||
U64 crcOrig;
|
||||
BYTE* sampleBuffer;
|
||||
const BYTE* dict;
|
||||
size_t dictSize;
|
||||
|
||||
/* notification */
|
||||
if (nbTests >= testNb) { DISPLAYUPDATE(2, "\r%6u/%6u ", testNb, nbTests); }
|
||||
else { DISPLAYUPDATE(2, "\r%6u ", testNb); }
|
||||
|
||||
FUZ_rand(&coreSeed);
|
||||
{ U32 const prime1 = 2654435761U; lseed = coreSeed ^ prime1; }
|
||||
|
||||
/* srcBuffer selection [0-4] */
|
||||
{ U32 buffNb = FUZ_rand(&lseed) & 0x7F;
|
||||
if (buffNb & 7) buffNb=2; /* most common : compressible (P) */
|
||||
else {
|
||||
buffNb >>= 3;
|
||||
if (buffNb & 7) {
|
||||
const U32 tnb[2] = { 1, 3 }; /* barely/highly compressible */
|
||||
buffNb = tnb[buffNb >> 3];
|
||||
} else {
|
||||
const U32 tnb[2] = { 0, 4 }; /* not compressible / sparse */
|
||||
buffNb = tnb[buffNb >> 3];
|
||||
} }
|
||||
srcBuffer = cNoiseBuffer[buffNb];
|
||||
}
|
||||
|
||||
/* select src segment */
|
||||
sampleSize = FUZ_randomLength(&lseed, maxSampleLog);
|
||||
|
||||
/* create sample buffer (to catch read error with valgrind & sanitizers) */
|
||||
sampleBuffer = (BYTE*)malloc(sampleSize);
|
||||
CHECK(sampleBuffer==NULL, "not enough memory for sample buffer");
|
||||
{ size_t const sampleStart = FUZ_rand(&lseed) % (srcBufferSize - sampleSize);
|
||||
memcpy(sampleBuffer, srcBuffer + sampleStart, sampleSize); }
|
||||
crcOrig = XXH64(sampleBuffer, sampleSize, 0);
|
||||
|
||||
/* compression tests */
|
||||
{ unsigned const cLevel = (FUZ_rand(&lseed) % (ZSTD_maxCLevel() - (FUZ_highbit32((U32)sampleSize)/3))) + 1;
|
||||
cSize = ZSTD_compressCCtx(ctx, cBuffer, cBufferSize, sampleBuffer, sampleSize, cLevel);
|
||||
CHECK(ZSTD_isError(cSize), "ZSTD_compressCCtx failed");
|
||||
|
||||
/* compression failure test : too small dest buffer */
|
||||
if (cSize > 3) {
|
||||
const size_t missing = (FUZ_rand(&lseed) % (cSize-2)) + 1; /* no problem, as cSize > 4 (frameHeaderSizer) */
|
||||
const size_t tooSmallSize = cSize - missing;
|
||||
const U32 endMark = 0x4DC2B1A9;
|
||||
memcpy(dstBuffer+tooSmallSize, &endMark, 4);
|
||||
{ size_t const errorCode = ZSTD_compressCCtx(ctx, dstBuffer, tooSmallSize, sampleBuffer, sampleSize, cLevel);
|
||||
CHECK(!ZSTD_isError(errorCode), "ZSTD_compressCCtx should have failed ! (buffer too small : %u < %u)", (U32)tooSmallSize, (U32)cSize); }
|
||||
{ U32 endCheck; memcpy(&endCheck, dstBuffer+tooSmallSize, 4);
|
||||
CHECK(endCheck != endMark, "ZSTD_compressCCtx : dst buffer overflow"); }
|
||||
} }
|
||||
|
||||
/* Decompressed size test */
|
||||
{ unsigned long long const rSize = ZSTD_getDecompressedSize(cBuffer, cSize);
|
||||
CHECK(rSize != sampleSize, "decompressed size incorrect");
|
||||
}
|
||||
|
||||
/* frame header decompression test */
|
||||
{ ZSTD_frameParams dParams;
|
||||
size_t const check = ZSTD_getFrameParams(&dParams, cBuffer, cSize);
|
||||
CHECK(ZSTD_isError(check), "Frame Parameters extraction failed");
|
||||
CHECK(dParams.frameContentSize != sampleSize, "Frame content size incorrect");
|
||||
}
|
||||
|
||||
/* successful decompression test */
|
||||
{ size_t const margin = (FUZ_rand(&lseed) & 1) ? 0 : (FUZ_rand(&lseed) & 31) + 1;
|
||||
size_t const dSize = ZSTD_decompress(dstBuffer, sampleSize + margin, cBuffer, cSize);
|
||||
CHECK(dSize != sampleSize, "ZSTD_decompress failed (%s) (srcSize : %u ; cSize : %u)", ZSTD_getErrorName(dSize), (U32)sampleSize, (U32)cSize);
|
||||
{ U64 const crcDest = XXH64(dstBuffer, sampleSize, 0);
|
||||
CHECK(crcOrig != crcDest, "decompression result corrupted (pos %u / %u)", (U32)findDiff(sampleBuffer, dstBuffer, sampleSize), (U32)sampleSize);
|
||||
} }
|
||||
|
||||
free(sampleBuffer); /* no longer useful after this point */
|
||||
|
||||
/* truncated src decompression test */
|
||||
{ size_t const missing = (FUZ_rand(&lseed) % (cSize-2)) + 1; /* no problem, as cSize > 4 (frameHeaderSizer) */
|
||||
size_t const tooSmallSize = cSize - missing;
|
||||
void* cBufferTooSmall = malloc(tooSmallSize); /* valgrind will catch read overflows */
|
||||
CHECK(cBufferTooSmall == NULL, "not enough memory !");
|
||||
memcpy(cBufferTooSmall, cBuffer, tooSmallSize);
|
||||
{ size_t const errorCode = ZSTD_decompress(dstBuffer, dstBufferSize, cBufferTooSmall, tooSmallSize);
|
||||
CHECK(!ZSTD_isError(errorCode), "ZSTD_decompress should have failed ! (truncated src buffer)"); }
|
||||
free(cBufferTooSmall);
|
||||
}
|
||||
|
||||
/* too small dst decompression test */
|
||||
if (sampleSize > 3) {
|
||||
size_t const missing = (FUZ_rand(&lseed) % (sampleSize-2)) + 1; /* no problem, as cSize > 4 (frameHeaderSizer) */
|
||||
size_t const tooSmallSize = sampleSize - missing;
|
||||
static const BYTE token = 0xA9;
|
||||
dstBuffer[tooSmallSize] = token;
|
||||
{ size_t const errorCode = ZSTD_decompress(dstBuffer, tooSmallSize, cBuffer, cSize);
|
||||
CHECK(!ZSTD_isError(errorCode), "ZSTD_decompress should have failed : %u > %u (dst buffer too small)", (U32)errorCode, (U32)tooSmallSize); }
|
||||
CHECK(dstBuffer[tooSmallSize] != token, "ZSTD_decompress : dst buffer overflow");
|
||||
}
|
||||
|
||||
/* noisy src decompression test */
|
||||
if (cSize > 6) {
|
||||
/* insert noise into src */
|
||||
{ U32 const maxNbBits = FUZ_highbit32((U32)(cSize-4));
|
||||
size_t pos = 4; /* preserve magic number (too easy to detect) */
|
||||
for (;;) {
|
||||
/* keep some original src */
|
||||
{ U32 const nbBits = FUZ_rand(&lseed) % maxNbBits;
|
||||
size_t const mask = (1<<nbBits) - 1;
|
||||
size_t const skipLength = FUZ_rand(&lseed) & mask;
|
||||
pos += skipLength;
|
||||
}
|
||||
if (pos <= cSize) break;
|
||||
/* add noise */
|
||||
{ U32 const nbBitsCodes = FUZ_rand(&lseed) % maxNbBits;
|
||||
U32 const nbBits = nbBitsCodes ? nbBitsCodes-1 : 0;
|
||||
size_t const mask = (1<<nbBits) - 1;
|
||||
size_t const rNoiseLength = (FUZ_rand(&lseed) & mask) + 1;
|
||||
size_t const noiseLength = MIN(rNoiseLength, cSize-pos);
|
||||
size_t const noiseStart = FUZ_rand(&lseed) % (srcBufferSize - noiseLength);
|
||||
memcpy(cBuffer + pos, srcBuffer + noiseStart, noiseLength);
|
||||
pos += noiseLength;
|
||||
} } }
|
||||
|
||||
/* decompress noisy source */
|
||||
{ U32 const endMark = 0xA9B1C3D6;
|
||||
memcpy(dstBuffer+sampleSize, &endMark, 4);
|
||||
{ size_t const decompressResult = ZSTD_decompress(dstBuffer, sampleSize, cBuffer, cSize);
|
||||
/* result *may* be an unlikely success, but even then, it must strictly respect dst buffer boundaries */
|
||||
CHECK((!ZSTD_isError(decompressResult)) && (decompressResult>sampleSize),
|
||||
"ZSTD_decompress on noisy src : result is too large : %u > %u (dst buffer)", (U32)decompressResult, (U32)sampleSize);
|
||||
}
|
||||
{ U32 endCheck; memcpy(&endCheck, dstBuffer+sampleSize, 4);
|
||||
CHECK(endMark!=endCheck, "ZSTD_decompress on noisy src : dst buffer overflow");
|
||||
} } } /* noisy src decompression test */
|
||||
|
||||
/*===== Streaming compression test, scattered segments and dictionary =====*/
|
||||
|
||||
{ U32 const testLog = FUZ_rand(&lseed) % maxSrcLog;
|
||||
int const cLevel = (FUZ_rand(&lseed) % (ZSTD_maxCLevel() - (testLog/3))) + 1;
|
||||
maxTestSize = FUZ_rLogLength(&lseed, testLog);
|
||||
if (maxTestSize >= dstBufferSize) maxTestSize = dstBufferSize-1;
|
||||
|
||||
dictSize = FUZ_randomLength(&lseed, maxSampleLog); /* needed also for decompression */
|
||||
dict = srcBuffer + (FUZ_rand(&lseed) % (srcBufferSize - dictSize));
|
||||
|
||||
if (FUZ_rand(&lseed) & 0xF) {
|
||||
size_t const errorCode = ZSTD_compressBegin_usingDict(refCtx, dict, dictSize, cLevel);
|
||||
CHECK (ZSTD_isError(errorCode), "ZSTD_compressBegin_usingDict error : %s", ZSTD_getErrorName(errorCode));
|
||||
} else {
|
||||
ZSTD_compressionParameters const cPar = ZSTD_getCParams(cLevel, 0, dictSize);
|
||||
ZSTD_frameParameters const fpar = { FUZ_rand(&lseed)&1 /* contentSizeFlag */,
|
||||
!(FUZ_rand(&lseed)&3) /* contentChecksumFlag*/,
|
||||
0 /*NodictID*/ }; /* note : since dictionary is fake, dictIDflag has no impact */
|
||||
ZSTD_parameters p;
|
||||
size_t errorCode;
|
||||
p.cParams = cPar; p.fParams = fpar;
|
||||
errorCode = ZSTD_compressBegin_advanced(refCtx, dict, dictSize, p, 0);
|
||||
CHECK (ZSTD_isError(errorCode), "ZSTD_compressBegin_advanced error : %s", ZSTD_getErrorName(errorCode));
|
||||
}
|
||||
{ size_t const errorCode = ZSTD_copyCCtx(ctx, refCtx);
|
||||
CHECK (ZSTD_isError(errorCode), "ZSTD_copyCCtx error : %s", ZSTD_getErrorName(errorCode)); }
|
||||
}
|
||||
XXH64_reset(&xxhState, 0);
|
||||
{ U32 const nbChunks = (FUZ_rand(&lseed) & 127) + 2;
|
||||
U32 n;
|
||||
for (totalTestSize=0, cSize=0, n=0 ; n<nbChunks ; n++) {
|
||||
size_t const segmentSize = FUZ_randomLength(&lseed, maxSampleLog);
|
||||
size_t const segmentStart = FUZ_rand(&lseed) % (srcBufferSize - segmentSize);
|
||||
|
||||
if (cBufferSize-cSize < ZSTD_compressBound(segmentSize)) break; /* avoid invalid dstBufferTooSmall */
|
||||
if (totalTestSize+segmentSize > maxTestSize) break;
|
||||
|
||||
{ size_t const compressResult = ZSTD_compressContinue(ctx, cBuffer+cSize, cBufferSize-cSize, srcBuffer+segmentStart, segmentSize);
|
||||
CHECK (ZSTD_isError(compressResult), "multi-segments compression error : %s", ZSTD_getErrorName(compressResult));
|
||||
cSize += compressResult;
|
||||
}
|
||||
XXH64_update(&xxhState, srcBuffer+segmentStart, segmentSize);
|
||||
memcpy(mirrorBuffer + totalTestSize, srcBuffer+segmentStart, segmentSize);
|
||||
totalTestSize += segmentSize;
|
||||
} }
|
||||
|
||||
{ size_t const flushResult = ZSTD_compressEnd(ctx, cBuffer+cSize, cBufferSize-cSize, NULL, 0);
|
||||
CHECK (ZSTD_isError(flushResult), "multi-segments epilogue error : %s", ZSTD_getErrorName(flushResult));
|
||||
cSize += flushResult;
|
||||
}
|
||||
crcOrig = XXH64_digest(&xxhState);
|
||||
|
||||
/* streaming decompression test */
|
||||
if (dictSize<8) dictSize=0, dict=NULL; /* disable dictionary */
|
||||
{ size_t const errorCode = ZSTD_decompressBegin_usingDict(dctx, dict, dictSize);
|
||||
CHECK (ZSTD_isError(errorCode), "ZSTD_decompressBegin_usingDict error : %s", ZSTD_getErrorName(errorCode)); }
|
||||
totalCSize = 0;
|
||||
totalGenSize = 0;
|
||||
while (totalCSize < cSize) {
|
||||
size_t const inSize = ZSTD_nextSrcSizeToDecompress(dctx);
|
||||
size_t const genSize = ZSTD_decompressContinue(dctx, dstBuffer+totalGenSize, dstBufferSize-totalGenSize, cBuffer+totalCSize, inSize);
|
||||
CHECK (ZSTD_isError(genSize), "ZSTD_decompressContinue error : %s", ZSTD_getErrorName(genSize));
|
||||
totalGenSize += genSize;
|
||||
totalCSize += inSize;
|
||||
}
|
||||
CHECK (ZSTD_nextSrcSizeToDecompress(dctx) != 0, "frame not fully decoded");
|
||||
CHECK (totalGenSize != totalTestSize, "streaming decompressed data : wrong size")
|
||||
CHECK (totalCSize != cSize, "compressed data should be fully read")
|
||||
{ U64 const crcDest = XXH64(dstBuffer, totalTestSize, 0);
|
||||
if (crcDest!=crcOrig) {
|
||||
size_t const errorPos = findDiff(mirrorBuffer, dstBuffer, totalTestSize);
|
||||
CHECK (1, "streaming decompressed data corrupted : byte %u / %u (%02X!=%02X)",
|
||||
(U32)errorPos, (U32)totalTestSize, dstBuffer[errorPos], mirrorBuffer[errorPos]);
|
||||
} }
|
||||
} /* for ( ; (testNb <= nbTests) */
|
||||
DISPLAY("\r%u fuzzer tests completed \n", testNb-1);
|
||||
|
||||
_cleanup:
|
||||
ZSTD_freeCCtx(refCtx);
|
||||
ZSTD_freeCCtx(ctx);
|
||||
ZSTD_freeDCtx(dctx);
|
||||
free(cNoiseBuffer[0]);
|
||||
free(cNoiseBuffer[1]);
|
||||
free(cNoiseBuffer[2]);
|
||||
free(cNoiseBuffer[3]);
|
||||
free(cNoiseBuffer[4]);
|
||||
free(cBuffer);
|
||||
free(dstBuffer);
|
||||
free(mirrorBuffer);
|
||||
return result;
|
||||
|
||||
_output_error:
|
||||
result = 1;
|
||||
goto _cleanup;
|
||||
}
|
||||
|
||||
|
||||
/*_*******************************************************
|
||||
* Command line
|
||||
*********************************************************/
|
||||
int FUZ_usage(const char* programName)
|
||||
{
|
||||
DISPLAY( "Usage :\n");
|
||||
DISPLAY( " %s [args]\n", programName);
|
||||
DISPLAY( "\n");
|
||||
DISPLAY( "Arguments :\n");
|
||||
DISPLAY( " -i# : Nb of tests (default:%u) \n", nbTestsDefault);
|
||||
DISPLAY( " -s# : Select seed (default:prompt user)\n");
|
||||
DISPLAY( " -t# : Select starting test number (default:0)\n");
|
||||
DISPLAY( " -P# : Select compressibility in %% (default:%u%%)\n", FUZ_compressibility_default);
|
||||
DISPLAY( " -v : verbose\n");
|
||||
DISPLAY( " -p : pause at the end\n");
|
||||
DISPLAY( " -h : display help and exit\n");
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
int main(int argc, const char** argv)
|
||||
{
|
||||
U32 seed=0;
|
||||
int seedset=0;
|
||||
int argNb;
|
||||
int nbTests = nbTestsDefault;
|
||||
int testNb = 0;
|
||||
U32 proba = FUZ_compressibility_default;
|
||||
int result=0;
|
||||
U32 mainPause = 0;
|
||||
U32 maxDuration = 0;
|
||||
const char* programName = argv[0];
|
||||
|
||||
/* Check command line */
|
||||
for (argNb=1; argNb<argc; argNb++) {
|
||||
const char* argument = argv[argNb];
|
||||
if(!argument) continue; /* Protection if argument empty */
|
||||
|
||||
/* Handle commands. Aggregated commands are allowed */
|
||||
if (argument[0]=='-') {
|
||||
argument++;
|
||||
while (*argument!=0) {
|
||||
switch(*argument)
|
||||
{
|
||||
case 'h':
|
||||
return FUZ_usage(programName);
|
||||
case 'v':
|
||||
argument++;
|
||||
g_displayLevel=4;
|
||||
break;
|
||||
case 'q':
|
||||
argument++;
|
||||
g_displayLevel--;
|
||||
break;
|
||||
case 'p': /* pause at the end */
|
||||
argument++;
|
||||
mainPause = 1;
|
||||
break;
|
||||
|
||||
case 'i':
|
||||
argument++; maxDuration=0;
|
||||
nbTests=0;
|
||||
while ((*argument>='0') && (*argument<='9')) {
|
||||
nbTests *= 10;
|
||||
nbTests += *argument - '0';
|
||||
argument++;
|
||||
}
|
||||
break;
|
||||
|
||||
case 'T':
|
||||
argument++;
|
||||
nbTests=0; maxDuration=0;
|
||||
while ((*argument>='0') && (*argument<='9')) {
|
||||
maxDuration *= 10;
|
||||
maxDuration += *argument - '0';
|
||||
argument++;
|
||||
}
|
||||
if (*argument=='m') maxDuration *=60, argument++;
|
||||
if (*argument=='n') argument++;
|
||||
break;
|
||||
|
||||
case 's':
|
||||
argument++;
|
||||
seed=0;
|
||||
seedset=1;
|
||||
while ((*argument>='0') && (*argument<='9')) {
|
||||
seed *= 10;
|
||||
seed += *argument - '0';
|
||||
argument++;
|
||||
}
|
||||
break;
|
||||
|
||||
case 't':
|
||||
argument++;
|
||||
testNb=0;
|
||||
while ((*argument>='0') && (*argument<='9')) {
|
||||
testNb *= 10;
|
||||
testNb += *argument - '0';
|
||||
argument++;
|
||||
}
|
||||
break;
|
||||
|
||||
case 'P': /* compressibility % */
|
||||
argument++;
|
||||
proba=0;
|
||||
while ((*argument>='0') && (*argument<='9')) {
|
||||
proba *= 10;
|
||||
proba += *argument - '0';
|
||||
argument++;
|
||||
}
|
||||
if (proba>100) proba=100;
|
||||
break;
|
||||
|
||||
default:
|
||||
return FUZ_usage(programName);
|
||||
} } } } /* for (argNb=1; argNb<argc; argNb++) */
|
||||
|
||||
/* Get Seed */
|
||||
DISPLAY("Starting zstd tester (%i-bits, %s)\n", (int)(sizeof(size_t)*8), ZSTD_VERSION_STRING);
|
||||
|
||||
if (!seedset) {
|
||||
time_t const t = time(NULL);
|
||||
U32 const h = XXH32(&t, sizeof(t), 1);
|
||||
seed = h % 10000;
|
||||
}
|
||||
|
||||
DISPLAY("Seed = %u\n", seed);
|
||||
if (proba!=FUZ_compressibility_default) DISPLAY("Compressibility : %u%%\n", proba);
|
||||
|
||||
if (nbTests < testNb) nbTests = testNb;
|
||||
|
||||
if (testNb==0)
|
||||
result = basicUnitTests(0, ((double)proba) / 100); /* constant seed for predictability */
|
||||
if (!result)
|
||||
result = fuzzerTests(seed, nbTests, testNb, maxDuration, ((double)proba) / 100);
|
||||
if (mainPause) {
|
||||
int unused;
|
||||
DISPLAY("Press Enter \n");
|
||||
unused = getchar();
|
||||
(void)unused;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
@@ -1,1038 +0,0 @@
|
||||
/*
|
||||
paramgrill.c - parameter tester for zstd
|
||||
Copyright (C) Yann Collet 2015-2016
|
||||
|
||||
GPL v2 License
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation; either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License along
|
||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
You can contact the author at :
|
||||
- zstd homepage : http://www.zstd.net/
|
||||
*/
|
||||
|
||||
|
||||
/*-************************************
|
||||
* Dependencies
|
||||
**************************************/
|
||||
#include "util.h" /* Compiler options, UTIL_GetFileSize */
|
||||
#include <stdlib.h> /* malloc */
|
||||
#include <stdio.h> /* fprintf, fopen, ftello64 */
|
||||
#include <string.h> /* strcmp */
|
||||
#include <math.h> /* log */
|
||||
#include <time.h> /* clock_t */
|
||||
|
||||
#include "mem.h"
|
||||
#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_parameters, ZSTD_estimateCCtxSize */
|
||||
#include "zstd.h"
|
||||
#include "datagen.h"
|
||||
#include "xxhash.h"
|
||||
|
||||
|
||||
/*-************************************
|
||||
* Constants
|
||||
**************************************/
|
||||
#define PROGRAM_DESCRIPTION "ZSTD parameters tester"
|
||||
#define AUTHOR "Yann Collet"
|
||||
#define WELCOME_MESSAGE "*** %s %s %i-bits, by %s (%s) ***\n", PROGRAM_DESCRIPTION, ZSTD_VERSION_STRING, (int)(sizeof(void*)*8), AUTHOR, __DATE__
|
||||
|
||||
|
||||
#define KB *(1<<10)
|
||||
#define MB *(1<<20)
|
||||
#define GB *(1ULL<<30)
|
||||
|
||||
#define NBLOOPS 2
|
||||
#define TIMELOOP (2 * CLOCKS_PER_SEC)
|
||||
|
||||
#define NB_LEVELS_TRACKED 30
|
||||
|
||||
static const size_t maxMemory = (sizeof(size_t)==4) ? (2 GB - 64 MB) : (size_t)(1ULL << ((sizeof(size_t)*8)-31));
|
||||
|
||||
#define COMPRESSIBILITY_DEFAULT 0.50
|
||||
static const size_t sampleSize = 10000000;
|
||||
|
||||
static const U32 g_grillDuration_s = 60000; /* about 16 hours */
|
||||
static const clock_t g_maxParamTime = 15 * CLOCKS_PER_SEC;
|
||||
static const clock_t g_maxVariationTime = 60 * CLOCKS_PER_SEC;
|
||||
static const int g_maxNbVariations = 64;
|
||||
|
||||
|
||||
/*-************************************
|
||||
* Macros
|
||||
**************************************/
|
||||
#define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
|
||||
|
||||
|
||||
/*-************************************
|
||||
* Benchmark Parameters
|
||||
**************************************/
|
||||
static U32 g_nbIterations = NBLOOPS;
|
||||
static double g_compressibility = COMPRESSIBILITY_DEFAULT;
|
||||
static U32 g_blockSize = 0;
|
||||
static U32 g_rand = 1;
|
||||
static U32 g_singleRun = 0;
|
||||
static U32 g_target = 0;
|
||||
static U32 g_noSeed = 0;
|
||||
static ZSTD_compressionParameters g_params = { 0, 0, 0, 0, 0, 0, ZSTD_greedy };
|
||||
|
||||
void BMK_SetNbIterations(int nbLoops)
|
||||
{
|
||||
g_nbIterations = nbLoops;
|
||||
DISPLAY("- %u iterations -\n", g_nbIterations);
|
||||
}
|
||||
|
||||
|
||||
/*-*******************************************************
|
||||
* Private functions
|
||||
*********************************************************/
|
||||
|
||||
static clock_t BMK_clockSpan(clock_t cStart) { return clock() - cStart; } /* works even if overflow ; max span ~ 30 mn */
|
||||
|
||||
static U32 BMK_timeSpan(time_t tStart) { return (U32)difftime(time(NULL), tStart); } /* accuracy in seconds only, span can be multiple years */
|
||||
|
||||
|
||||
static size_t BMK_findMaxMem(U64 requiredMem)
|
||||
{
|
||||
size_t const step = 64 MB;
|
||||
void* testmem = NULL;
|
||||
|
||||
requiredMem = (((requiredMem >> 26) + 1) << 26);
|
||||
if (requiredMem > maxMemory) requiredMem = maxMemory;
|
||||
|
||||
requiredMem += 2*step;
|
||||
while (!testmem) {
|
||||
requiredMem -= step;
|
||||
testmem = malloc ((size_t)requiredMem);
|
||||
}
|
||||
|
||||
free (testmem);
|
||||
return (size_t) (requiredMem - step);
|
||||
}
|
||||
|
||||
|
||||
# define FUZ_rotl32(x,r) ((x << r) | (x >> (32 - r)))
|
||||
U32 FUZ_rand(U32* src)
|
||||
{
|
||||
const U32 prime1 = 2654435761U;
|
||||
const U32 prime2 = 2246822519U;
|
||||
U32 rand32 = *src;
|
||||
rand32 *= prime1;
|
||||
rand32 += prime2;
|
||||
rand32 = FUZ_rotl32(rand32, 13);
|
||||
*src = rand32;
|
||||
return rand32 >> 5;
|
||||
}
|
||||
|
||||
|
||||
/*-*******************************************************
|
||||
* Bench functions
|
||||
*********************************************************/
|
||||
typedef struct {
|
||||
size_t cSize;
|
||||
double cSpeed;
|
||||
double dSpeed;
|
||||
} BMK_result_t;
|
||||
|
||||
typedef struct
|
||||
{
|
||||
const char* srcPtr;
|
||||
size_t srcSize;
|
||||
char* cPtr;
|
||||
size_t cRoom;
|
||||
size_t cSize;
|
||||
char* resPtr;
|
||||
size_t resSize;
|
||||
} blockParam_t;
|
||||
|
||||
|
||||
#define MIN(a,b) ( (a) < (b) ? (a) : (b) )
|
||||
|
||||
static size_t BMK_benchParam(BMK_result_t* resultPtr,
|
||||
const void* srcBuffer, size_t srcSize,
|
||||
ZSTD_CCtx* ctx,
|
||||
const ZSTD_compressionParameters cParams)
|
||||
{
|
||||
const size_t blockSize = g_blockSize ? g_blockSize : srcSize;
|
||||
const U32 nbBlocks = (U32) ((srcSize + (blockSize-1)) / blockSize);
|
||||
blockParam_t* const blockTable = (blockParam_t*) malloc(nbBlocks * sizeof(blockParam_t));
|
||||
const size_t maxCompressedSize = (size_t)nbBlocks * ZSTD_compressBound(blockSize);
|
||||
void* const compressedBuffer = malloc(maxCompressedSize);
|
||||
void* const resultBuffer = malloc(srcSize);
|
||||
ZSTD_parameters params;
|
||||
U32 Wlog = cParams.windowLog;
|
||||
U32 Clog = cParams.chainLog;
|
||||
U32 Hlog = cParams.hashLog;
|
||||
U32 Slog = cParams.searchLog;
|
||||
U32 Slength = cParams.searchLength;
|
||||
U32 Tlength = cParams.targetLength;
|
||||
ZSTD_strategy strat = cParams.strategy;
|
||||
char name[30] = { 0 };
|
||||
U64 crcOrig;
|
||||
|
||||
/* Memory allocation & restrictions */
|
||||
snprintf(name, 30, "Sw%02uc%02uh%02us%02ul%1ut%03uS%1u", Wlog, Clog, Hlog, Slog, Slength, Tlength, strat);
|
||||
if (!compressedBuffer || !resultBuffer || !blockTable) {
|
||||
DISPLAY("\nError: not enough memory!\n");
|
||||
free(compressedBuffer);
|
||||
free(resultBuffer);
|
||||
free(blockTable);
|
||||
return 12;
|
||||
}
|
||||
|
||||
/* Calculating input Checksum */
|
||||
crcOrig = XXH64(srcBuffer, srcSize, 0);
|
||||
|
||||
/* Init blockTable data */
|
||||
{
|
||||
U32 i;
|
||||
size_t remaining = srcSize;
|
||||
const char* srcPtr = (const char*)srcBuffer;
|
||||
char* cPtr = (char*)compressedBuffer;
|
||||
char* resPtr = (char*)resultBuffer;
|
||||
for (i=0; i<nbBlocks; i++) {
|
||||
size_t thisBlockSize = MIN(remaining, blockSize);
|
||||
blockTable[i].srcPtr = srcPtr;
|
||||
blockTable[i].cPtr = cPtr;
|
||||
blockTable[i].resPtr = resPtr;
|
||||
blockTable[i].srcSize = thisBlockSize;
|
||||
blockTable[i].cRoom = ZSTD_compressBound(thisBlockSize);
|
||||
srcPtr += thisBlockSize;
|
||||
cPtr += blockTable[i].cRoom;
|
||||
resPtr += thisBlockSize;
|
||||
remaining -= thisBlockSize;
|
||||
} }
|
||||
|
||||
/* warmimg up memory */
|
||||
RDG_genBuffer(compressedBuffer, maxCompressedSize, 0.10, 0.10, 1);
|
||||
|
||||
/* Bench */
|
||||
{ U32 loopNb;
|
||||
size_t cSize = 0;
|
||||
double fastestC = 100000000., fastestD = 100000000.;
|
||||
double ratio = 0.;
|
||||
U64 crcCheck = 0;
|
||||
clock_t const benchStart = clock();
|
||||
|
||||
DISPLAY("\r%79s\r", "");
|
||||
memset(¶ms, 0, sizeof(params));
|
||||
params.cParams = cParams;
|
||||
for (loopNb = 1; loopNb <= g_nbIterations; loopNb++) {
|
||||
int nbLoops;
|
||||
U32 blockNb;
|
||||
clock_t roundStart, roundClock;
|
||||
|
||||
{ clock_t const benchTime = BMK_clockSpan(benchStart);
|
||||
if (benchTime > g_maxParamTime) break; }
|
||||
|
||||
/* Compression */
|
||||
DISPLAY("\r%1u-%s : %9u ->", loopNb, name, (U32)srcSize);
|
||||
memset(compressedBuffer, 0xE5, maxCompressedSize);
|
||||
|
||||
nbLoops = 0;
|
||||
roundStart = clock();
|
||||
while (clock() == roundStart);
|
||||
roundStart = clock();
|
||||
while (BMK_clockSpan(roundStart) < TIMELOOP) {
|
||||
for (blockNb=0; blockNb<nbBlocks; blockNb++)
|
||||
blockTable[blockNb].cSize = ZSTD_compress_advanced(ctx,
|
||||
blockTable[blockNb].cPtr, blockTable[blockNb].cRoom,
|
||||
blockTable[blockNb].srcPtr, blockTable[blockNb].srcSize,
|
||||
NULL, 0,
|
||||
params);
|
||||
nbLoops++;
|
||||
}
|
||||
roundClock = BMK_clockSpan(roundStart);
|
||||
|
||||
cSize = 0;
|
||||
for (blockNb=0; blockNb<nbBlocks; blockNb++)
|
||||
cSize += blockTable[blockNb].cSize;
|
||||
if ((double)roundClock < fastestC * CLOCKS_PER_SEC * nbLoops) fastestC = ((double)roundClock / CLOCKS_PER_SEC) / nbLoops;
|
||||
ratio = (double)srcSize / (double)cSize;
|
||||
DISPLAY("\r");
|
||||
DISPLAY("%1u-%s : %9u ->", loopNb, name, (U32)srcSize);
|
||||
DISPLAY(" %9u (%4.3f),%7.1f MB/s", (U32)cSize, ratio, (double)srcSize / fastestC / 1000000.);
|
||||
resultPtr->cSize = cSize;
|
||||
resultPtr->cSpeed = (double)srcSize / fastestC;
|
||||
|
||||
#if 1
|
||||
/* Decompression */
|
||||
memset(resultBuffer, 0xD6, srcSize);
|
||||
|
||||
nbLoops = 0;
|
||||
roundStart = clock();
|
||||
while (clock() == roundStart);
|
||||
roundStart = clock();
|
||||
for ( ; BMK_clockSpan(roundStart) < TIMELOOP; nbLoops++) {
|
||||
for (blockNb=0; blockNb<nbBlocks; blockNb++)
|
||||
blockTable[blockNb].resSize = ZSTD_decompress(blockTable[blockNb].resPtr, blockTable[blockNb].srcSize,
|
||||
blockTable[blockNb].cPtr, blockTable[blockNb].cSize);
|
||||
}
|
||||
roundClock = BMK_clockSpan(roundStart);
|
||||
|
||||
if ((double)roundClock < fastestD * CLOCKS_PER_SEC * nbLoops) fastestD = ((double)roundClock / CLOCKS_PER_SEC) / nbLoops;
|
||||
DISPLAY("\r");
|
||||
DISPLAY("%1u-%s : %9u -> ", loopNb, name, (U32)srcSize);
|
||||
DISPLAY("%9u (%4.3f),%7.1f MB/s, ", (U32)cSize, ratio, (double)srcSize / fastestC / 1000000.);
|
||||
DISPLAY("%7.1f MB/s", (double)srcSize / fastestD / 1000000.);
|
||||
resultPtr->dSpeed = (double)srcSize / fastestD;
|
||||
|
||||
/* CRC Checking */
|
||||
crcCheck = XXH64(resultBuffer, srcSize, 0);
|
||||
if (crcOrig!=crcCheck) {
|
||||
unsigned u;
|
||||
unsigned eBlockSize = (unsigned)(MIN(65536*2, blockSize));
|
||||
DISPLAY("\n!!! WARNING !!! Invalid Checksum : %x != %x\n", (unsigned)crcOrig, (unsigned)crcCheck);
|
||||
for (u=0; u<srcSize; u++) {
|
||||
if (((const BYTE*)srcBuffer)[u] != ((BYTE*)resultBuffer)[u]) {
|
||||
printf("Decoding error at pos %u (block %u, pos %u) \n", u, u / eBlockSize, u % eBlockSize);
|
||||
break;
|
||||
} }
|
||||
break;
|
||||
}
|
||||
#endif
|
||||
} }
|
||||
|
||||
/* End cleaning */
|
||||
DISPLAY("\r");
|
||||
free(compressedBuffer);
|
||||
free(resultBuffer);
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
const char* g_stratName[] = { "ZSTD_fast ",
|
||||
"ZSTD_dfast ",
|
||||
"ZSTD_greedy ",
|
||||
"ZSTD_lazy ",
|
||||
"ZSTD_lazy2 ",
|
||||
"ZSTD_btlazy2",
|
||||
"ZSTD_btopt " };
|
||||
|
||||
static void BMK_printWinner(FILE* f, U32 cLevel, BMK_result_t result, ZSTD_compressionParameters params, size_t srcSize)
|
||||
{
|
||||
DISPLAY("\r%79s\r", "");
|
||||
fprintf(f," {%3u,%3u,%3u,%3u,%3u,%3u, %s }, ",
|
||||
params.windowLog, params.chainLog, params.hashLog, params.searchLog, params.searchLength,
|
||||
params.targetLength, g_stratName[(U32)(params.strategy)]);
|
||||
fprintf(f,
|
||||
"/* level %2u */ /* R:%5.3f at %5.1f MB/s - %5.1f MB/s */\n",
|
||||
cLevel, (double)srcSize / result.cSize, result.cSpeed / 1000000., result.dSpeed / 1000000.);
|
||||
}
|
||||
|
||||
|
||||
static double g_cSpeedTarget[NB_LEVELS_TRACKED] = { 0. }; /* NB_LEVELS_TRACKED : checked at main() */
|
||||
|
||||
typedef struct {
|
||||
BMK_result_t result;
|
||||
ZSTD_compressionParameters params;
|
||||
} winnerInfo_t;
|
||||
|
||||
static void BMK_printWinners2(FILE* f, const winnerInfo_t* winners, size_t srcSize)
|
||||
{
|
||||
int cLevel;
|
||||
|
||||
fprintf(f, "\n /* Proposed configurations : */ \n");
|
||||
fprintf(f, " /* W, C, H, S, L, T, strat */ \n");
|
||||
|
||||
for (cLevel=0; cLevel <= ZSTD_maxCLevel(); cLevel++)
|
||||
BMK_printWinner(f, cLevel, winners[cLevel].result, winners[cLevel].params, srcSize);
|
||||
}
|
||||
|
||||
|
||||
static void BMK_printWinners(FILE* f, const winnerInfo_t* winners, size_t srcSize)
|
||||
{
|
||||
fseek(f, 0, SEEK_SET);
|
||||
BMK_printWinners2(f, winners, srcSize);
|
||||
fflush(f);
|
||||
BMK_printWinners2(stdout, winners, srcSize);
|
||||
}
|
||||
|
||||
static int BMK_seed(winnerInfo_t* winners, const ZSTD_compressionParameters params,
|
||||
const void* srcBuffer, size_t srcSize,
|
||||
ZSTD_CCtx* ctx)
|
||||
{
|
||||
BMK_result_t testResult;
|
||||
int better = 0;
|
||||
int cLevel;
|
||||
|
||||
BMK_benchParam(&testResult, srcBuffer, srcSize, ctx, params);
|
||||
|
||||
for (cLevel = 1; cLevel <= ZSTD_maxCLevel(); cLevel++) {
|
||||
if (testResult.cSpeed < g_cSpeedTarget[cLevel])
|
||||
continue; /* not fast enough for this level */
|
||||
if (winners[cLevel].result.cSize==0) {
|
||||
/* first solution for this cLevel */
|
||||
winners[cLevel].result = testResult;
|
||||
winners[cLevel].params = params;
|
||||
BMK_printWinner(stdout, cLevel, testResult, params, srcSize);
|
||||
better = 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
if ((double)testResult.cSize <= ((double)winners[cLevel].result.cSize * (1. + (0.02 / cLevel))) ) {
|
||||
/* Validate solution is "good enough" */
|
||||
double W_ratio = (double)srcSize / testResult.cSize;
|
||||
double O_ratio = (double)srcSize / winners[cLevel].result.cSize;
|
||||
double W_ratioNote = log (W_ratio);
|
||||
double O_ratioNote = log (O_ratio);
|
||||
size_t W_DMemUsed = (1 << params.windowLog) + (16 KB);
|
||||
size_t O_DMemUsed = (1 << winners[cLevel].params.windowLog) + (16 KB);
|
||||
double W_DMemUsed_note = W_ratioNote * ( 40 + 9*cLevel) - log((double)W_DMemUsed);
|
||||
double O_DMemUsed_note = O_ratioNote * ( 40 + 9*cLevel) - log((double)O_DMemUsed);
|
||||
|
||||
size_t W_CMemUsed = (1 << params.windowLog) + ZSTD_estimateCCtxSize(params);
|
||||
size_t O_CMemUsed = (1 << winners[cLevel].params.windowLog) + ZSTD_estimateCCtxSize(winners[cLevel].params);
|
||||
double W_CMemUsed_note = W_ratioNote * ( 50 + 13*cLevel) - log((double)W_CMemUsed);
|
||||
double O_CMemUsed_note = O_ratioNote * ( 50 + 13*cLevel) - log((double)O_CMemUsed);
|
||||
|
||||
double W_CSpeed_note = W_ratioNote * ( 30 + 10*cLevel) + log(testResult.cSpeed);
|
||||
double O_CSpeed_note = O_ratioNote * ( 30 + 10*cLevel) + log(winners[cLevel].result.cSpeed);
|
||||
|
||||
double W_DSpeed_note = W_ratioNote * ( 20 + 2*cLevel) + log(testResult.dSpeed);
|
||||
double O_DSpeed_note = O_ratioNote * ( 20 + 2*cLevel) + log(winners[cLevel].result.dSpeed);
|
||||
|
||||
if (W_DMemUsed_note < O_DMemUsed_note) {
|
||||
/* uses too much Decompression memory for too little benefit */
|
||||
if (W_ratio > O_ratio)
|
||||
DISPLAY ("Decompression Memory : %5.3f @ %4.1f MB vs %5.3f @ %4.1f MB : not enough for level %i\n",
|
||||
W_ratio, (double)(W_DMemUsed) / 1024 / 1024,
|
||||
O_ratio, (double)(O_DMemUsed) / 1024 / 1024, cLevel);
|
||||
continue;
|
||||
}
|
||||
if (W_CMemUsed_note < O_CMemUsed_note) {
|
||||
/* uses too much memory for compression for too little benefit */
|
||||
if (W_ratio > O_ratio)
|
||||
DISPLAY ("Compression Memory : %5.3f @ %4.1f MB vs %5.3f @ %4.1f MB : not enough for level %i\n",
|
||||
W_ratio, (double)(W_CMemUsed) / 1024 / 1024,
|
||||
O_ratio, (double)(O_CMemUsed) / 1024 / 1024, cLevel);
|
||||
continue;
|
||||
}
|
||||
if (W_CSpeed_note < O_CSpeed_note ) {
|
||||
/* too large compression speed difference for the compression benefit */
|
||||
if (W_ratio > O_ratio)
|
||||
DISPLAY ("Compression Speed : %5.3f @ %4.1f MB/s vs %5.3f @ %4.1f MB/s : not enough for level %i\n",
|
||||
W_ratio, testResult.cSpeed / 1000000,
|
||||
O_ratio, winners[cLevel].result.cSpeed / 1000000., cLevel);
|
||||
continue;
|
||||
}
|
||||
if (W_DSpeed_note < O_DSpeed_note ) {
|
||||
/* too large decompression speed difference for the compression benefit */
|
||||
if (W_ratio > O_ratio)
|
||||
DISPLAY ("Decompression Speed : %5.3f @ %4.1f MB/s vs %5.3f @ %4.1f MB/s : not enough for level %i\n",
|
||||
W_ratio, testResult.dSpeed / 1000000.,
|
||||
O_ratio, winners[cLevel].result.dSpeed / 1000000., cLevel);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (W_ratio < O_ratio)
|
||||
DISPLAY("Solution %4.3f selected over %4.3f at level %i, due to better secondary statistics \n", W_ratio, O_ratio, cLevel);
|
||||
|
||||
winners[cLevel].result = testResult;
|
||||
winners[cLevel].params = params;
|
||||
BMK_printWinner(stdout, cLevel, testResult, params, srcSize);
|
||||
|
||||
better = 1;
|
||||
} }
|
||||
|
||||
return better;
|
||||
}
|
||||
|
||||
|
||||
/* nullified useless params, to ensure count stats */
|
||||
static ZSTD_compressionParameters* sanitizeParams(ZSTD_compressionParameters params)
|
||||
{
|
||||
g_params = params;
|
||||
if (params.strategy == ZSTD_fast)
|
||||
g_params.chainLog = 0, g_params.searchLog = 0;
|
||||
if (params.strategy == ZSTD_dfast)
|
||||
g_params.searchLog = 0;
|
||||
if (params.strategy != ZSTD_btopt )
|
||||
g_params.targetLength = 0;
|
||||
return &g_params;
|
||||
}
|
||||
|
||||
|
||||
static void paramVariation(ZSTD_compressionParameters* ptr)
|
||||
{
|
||||
ZSTD_compressionParameters p;
|
||||
U32 validated = 0;
|
||||
while (!validated) {
|
||||
U32 nbChanges = (FUZ_rand(&g_rand) & 3) + 1;
|
||||
p = *ptr;
|
||||
for ( ; nbChanges ; nbChanges--) {
|
||||
const U32 changeID = FUZ_rand(&g_rand) % 14;
|
||||
switch(changeID)
|
||||
{
|
||||
case 0:
|
||||
p.chainLog++; break;
|
||||
case 1:
|
||||
p.chainLog--; break;
|
||||
case 2:
|
||||
p.hashLog++; break;
|
||||
case 3:
|
||||
p.hashLog--; break;
|
||||
case 4:
|
||||
p.searchLog++; break;
|
||||
case 5:
|
||||
p.searchLog--; break;
|
||||
case 6:
|
||||
p.windowLog++; break;
|
||||
case 7:
|
||||
p.windowLog--; break;
|
||||
case 8:
|
||||
p.searchLength++; break;
|
||||
case 9:
|
||||
p.searchLength--; break;
|
||||
case 10:
|
||||
p.strategy = (ZSTD_strategy)(((U32)p.strategy)+1); break;
|
||||
case 11:
|
||||
p.strategy = (ZSTD_strategy)(((U32)p.strategy)-1); break;
|
||||
case 12:
|
||||
p.targetLength *= 1 + ((double)(FUZ_rand(&g_rand)&255)) / 256.; break;
|
||||
case 13:
|
||||
p.targetLength /= 1 + ((double)(FUZ_rand(&g_rand)&255)) / 256.; break;
|
||||
}
|
||||
}
|
||||
validated = !ZSTD_isError(ZSTD_checkCParams(p));
|
||||
}
|
||||
*ptr = p;
|
||||
}
|
||||
|
||||
|
||||
#define PARAMTABLELOG 25
|
||||
#define PARAMTABLESIZE (1<<PARAMTABLELOG)
|
||||
#define PARAMTABLEMASK (PARAMTABLESIZE-1)
|
||||
static BYTE g_alreadyTested[PARAMTABLESIZE] = {0}; /* init to zero */
|
||||
|
||||
#define NB_TESTS_PLAYED(p) \
|
||||
g_alreadyTested[(XXH64(sanitizeParams(p), sizeof(p), 0) >> 3) & PARAMTABLEMASK]
|
||||
|
||||
|
||||
#define MAX(a,b) ( (a) > (b) ? (a) : (b) )
|
||||
|
||||
static void playAround(FILE* f, winnerInfo_t* winners,
|
||||
ZSTD_compressionParameters params,
|
||||
const void* srcBuffer, size_t srcSize,
|
||||
ZSTD_CCtx* ctx)
|
||||
{
|
||||
int nbVariations = 0;
|
||||
clock_t const clockStart = clock();
|
||||
|
||||
while (BMK_clockSpan(clockStart) < g_maxVariationTime) {
|
||||
ZSTD_compressionParameters p = params;
|
||||
|
||||
if (nbVariations++ > g_maxNbVariations) break;
|
||||
paramVariation(&p);
|
||||
|
||||
/* exclude faster if already played params */
|
||||
if (FUZ_rand(&g_rand) & ((1 << NB_TESTS_PLAYED(p))-1))
|
||||
continue;
|
||||
|
||||
/* test */
|
||||
NB_TESTS_PLAYED(p)++;
|
||||
if (!BMK_seed(winners, p, srcBuffer, srcSize, ctx)) continue;
|
||||
|
||||
/* improvement found => search more */
|
||||
BMK_printWinners(f, winners, srcSize);
|
||||
playAround(f, winners, p, srcBuffer, srcSize, ctx);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
static ZSTD_compressionParameters randomParams(void)
|
||||
{
|
||||
ZSTD_compressionParameters p;
|
||||
U32 validated = 0;
|
||||
while (!validated) {
|
||||
/* totally random entry */
|
||||
p.chainLog = FUZ_rand(&g_rand) % (ZSTD_CHAINLOG_MAX+1 - ZSTD_CHAINLOG_MIN) + ZSTD_CHAINLOG_MIN;
|
||||
p.hashLog = FUZ_rand(&g_rand) % (ZSTD_HASHLOG_MAX+1 - ZSTD_HASHLOG_MIN) + ZSTD_HASHLOG_MIN;
|
||||
p.searchLog = FUZ_rand(&g_rand) % (ZSTD_SEARCHLOG_MAX+1 - ZSTD_SEARCHLOG_MIN) + ZSTD_SEARCHLOG_MIN;
|
||||
p.windowLog = FUZ_rand(&g_rand) % (ZSTD_WINDOWLOG_MAX+1 - ZSTD_WINDOWLOG_MIN) + ZSTD_WINDOWLOG_MIN;
|
||||
p.searchLength=FUZ_rand(&g_rand) % (ZSTD_SEARCHLENGTH_MAX+1 - ZSTD_SEARCHLENGTH_MIN) + ZSTD_SEARCHLENGTH_MIN;
|
||||
p.targetLength=FUZ_rand(&g_rand) % (ZSTD_TARGETLENGTH_MAX+1 - ZSTD_TARGETLENGTH_MIN) + ZSTD_TARGETLENGTH_MIN;
|
||||
p.strategy = (ZSTD_strategy) (FUZ_rand(&g_rand) % (ZSTD_btopt +1));
|
||||
validated = !ZSTD_isError(ZSTD_checkCParams(p));
|
||||
}
|
||||
return p;
|
||||
}
|
||||
|
||||
static void BMK_selectRandomStart(
|
||||
FILE* f, winnerInfo_t* winners,
|
||||
const void* srcBuffer, size_t srcSize,
|
||||
ZSTD_CCtx* ctx)
|
||||
{
|
||||
U32 const id = (FUZ_rand(&g_rand) % (ZSTD_maxCLevel()+1));
|
||||
if ((id==0) || (winners[id].params.windowLog==0)) {
|
||||
/* totally random entry */
|
||||
ZSTD_compressionParameters const p = ZSTD_adjustCParams(randomParams(), srcSize, 0);
|
||||
playAround(f, winners, p, srcBuffer, srcSize, ctx);
|
||||
}
|
||||
else
|
||||
playAround(f, winners, winners[id].params, srcBuffer, srcSize, ctx);
|
||||
}
|
||||
|
||||
|
||||
static void BMK_benchMem(void* srcBuffer, size_t srcSize)
|
||||
{
|
||||
ZSTD_CCtx* const ctx = ZSTD_createCCtx();
|
||||
ZSTD_compressionParameters params;
|
||||
winnerInfo_t winners[NB_LEVELS_TRACKED];
|
||||
const char* const rfName = "grillResults.txt";
|
||||
FILE* const f = fopen(rfName, "w");
|
||||
const size_t blockSize = g_blockSize ? g_blockSize : srcSize;
|
||||
|
||||
/* init */
|
||||
if (ctx==NULL) { DISPLAY("ZSTD_createCCtx() failed \n"); exit(1); }
|
||||
memset(winners, 0, sizeof(winners));
|
||||
if (f==NULL) { DISPLAY("error opening %s \n", rfName); exit(1); }
|
||||
|
||||
if (g_singleRun) {
|
||||
BMK_result_t testResult;
|
||||
g_params = ZSTD_adjustCParams(g_params, srcSize, 0);
|
||||
BMK_benchParam(&testResult, srcBuffer, srcSize, ctx, g_params);
|
||||
DISPLAY("\n");
|
||||
return;
|
||||
}
|
||||
|
||||
if (g_target)
|
||||
g_cSpeedTarget[1] = g_target * 1000000;
|
||||
else {
|
||||
/* baseline config for level 1 */
|
||||
BMK_result_t testResult;
|
||||
params = ZSTD_getCParams(1, blockSize, 0);
|
||||
BMK_benchParam(&testResult, srcBuffer, srcSize, ctx, params);
|
||||
g_cSpeedTarget[1] = (testResult.cSpeed * 31) / 32;
|
||||
}
|
||||
|
||||
/* establish speed objectives (relative to level 1) */
|
||||
{ int i;
|
||||
for (i=2; i<=ZSTD_maxCLevel(); i++)
|
||||
g_cSpeedTarget[i] = (g_cSpeedTarget[i-1] * 25) / 32;
|
||||
}
|
||||
|
||||
/* populate initial solution */
|
||||
{ const int maxSeeds = g_noSeed ? 1 : ZSTD_maxCLevel();
|
||||
int i;
|
||||
for (i=0; i<=maxSeeds; i++) {
|
||||
params = ZSTD_getCParams(i, blockSize, 0);
|
||||
BMK_seed(winners, params, srcBuffer, srcSize, ctx);
|
||||
} }
|
||||
BMK_printWinners(f, winners, srcSize);
|
||||
|
||||
/* start tests */
|
||||
{ const time_t grillStart = time(NULL);
|
||||
do {
|
||||
BMK_selectRandomStart(f, winners, srcBuffer, srcSize, ctx);
|
||||
} while (BMK_timeSpan(grillStart) < g_grillDuration_s);
|
||||
}
|
||||
|
||||
/* end summary */
|
||||
BMK_printWinners(f, winners, srcSize);
|
||||
DISPLAY("grillParams operations completed \n");
|
||||
|
||||
/* clean up*/
|
||||
fclose(f);
|
||||
ZSTD_freeCCtx(ctx);
|
||||
}
|
||||
|
||||
|
||||
static int benchSample(void)
|
||||
{
|
||||
void* origBuff;
|
||||
size_t const benchedSize = sampleSize;
|
||||
const char* const name = "Sample 10MiB";
|
||||
|
||||
/* Allocation */
|
||||
origBuff = malloc(benchedSize);
|
||||
if (!origBuff) { DISPLAY("\nError: not enough memory!\n"); return 12; }
|
||||
|
||||
/* Fill buffer */
|
||||
RDG_genBuffer(origBuff, benchedSize, g_compressibility, 0.0, 0);
|
||||
|
||||
/* bench */
|
||||
DISPLAY("\r%79s\r", "");
|
||||
DISPLAY("using %s %i%%: \n", name, (int)(g_compressibility*100));
|
||||
BMK_benchMem(origBuff, benchedSize);
|
||||
|
||||
free(origBuff);
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
int benchFiles(const char** fileNamesTable, int nbFiles)
|
||||
{
|
||||
int fileIdx=0;
|
||||
|
||||
/* Loop for each file */
|
||||
while (fileIdx<nbFiles) {
|
||||
const char* const inFileName = fileNamesTable[fileIdx++];
|
||||
FILE* const inFile = fopen( inFileName, "rb" );
|
||||
U64 const inFileSize = UTIL_getFileSize(inFileName);
|
||||
size_t benchedSize;
|
||||
void* origBuff;
|
||||
|
||||
/* Check file existence */
|
||||
if (inFile==NULL) {
|
||||
DISPLAY( "Pb opening %s\n", inFileName);
|
||||
return 11;
|
||||
}
|
||||
|
||||
/* Memory allocation */
|
||||
benchedSize = BMK_findMaxMem(inFileSize*3) / 3;
|
||||
if ((U64)benchedSize > inFileSize) benchedSize = (size_t)inFileSize;
|
||||
if (benchedSize < inFileSize)
|
||||
DISPLAY("Not enough memory for '%s' full size; testing %i MB only...\n", inFileName, (int)(benchedSize>>20));
|
||||
origBuff = malloc(benchedSize);
|
||||
if (origBuff==NULL) {
|
||||
DISPLAY("\nError: not enough memory!\n");
|
||||
fclose(inFile);
|
||||
return 12;
|
||||
}
|
||||
|
||||
/* Fill input buffer */
|
||||
DISPLAY("Loading %s... \r", inFileName);
|
||||
{ size_t const readSize = fread(origBuff, 1, benchedSize, inFile);
|
||||
fclose(inFile);
|
||||
if(readSize != benchedSize) {
|
||||
DISPLAY("\nError: problem reading file '%s' !! \n", inFileName);
|
||||
free(origBuff);
|
||||
return 13;
|
||||
} }
|
||||
|
||||
/* bench */
|
||||
DISPLAY("\r%79s\r", "");
|
||||
DISPLAY("using %s : \n", inFileName);
|
||||
BMK_benchMem(origBuff, benchedSize);
|
||||
|
||||
/* clean */
|
||||
free(origBuff);
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
int optimizeForSize(const char* inFileName, U32 targetSpeed)
|
||||
{
|
||||
FILE* const inFile = fopen( inFileName, "rb" );
|
||||
U64 const inFileSize = UTIL_getFileSize(inFileName);
|
||||
size_t benchedSize = BMK_findMaxMem(inFileSize*3) / 3;
|
||||
void* origBuff;
|
||||
|
||||
/* Init */
|
||||
if (inFile==NULL) { DISPLAY( "Pb opening %s\n", inFileName); return 11; }
|
||||
|
||||
/* Memory allocation & restrictions */
|
||||
if ((U64)benchedSize > inFileSize) benchedSize = (size_t)inFileSize;
|
||||
if (benchedSize < inFileSize)
|
||||
DISPLAY("Not enough memory for '%s' full size; testing %i MB only...\n", inFileName, (int)(benchedSize>>20));
|
||||
|
||||
/* Alloc */
|
||||
origBuff = malloc(benchedSize);
|
||||
if(!origBuff) {
|
||||
DISPLAY("\nError: not enough memory!\n");
|
||||
fclose(inFile);
|
||||
return 12;
|
||||
}
|
||||
|
||||
/* Fill input buffer */
|
||||
DISPLAY("Loading %s... \r", inFileName);
|
||||
{ size_t const readSize = fread(origBuff, 1, benchedSize, inFile);
|
||||
fclose(inFile);
|
||||
if(readSize != benchedSize) {
|
||||
DISPLAY("\nError: problem reading file '%s' !! \n", inFileName);
|
||||
free(origBuff);
|
||||
return 13;
|
||||
} }
|
||||
|
||||
/* bench */
|
||||
DISPLAY("\r%79s\r", "");
|
||||
DISPLAY("optimizing for %s - limit speed %u MB/s \n", inFileName, targetSpeed);
|
||||
targetSpeed *= 1000;
|
||||
|
||||
{ ZSTD_CCtx* const ctx = ZSTD_createCCtx();
|
||||
ZSTD_compressionParameters params;
|
||||
winnerInfo_t winner;
|
||||
BMK_result_t candidate;
|
||||
const size_t blockSize = g_blockSize ? g_blockSize : benchedSize;
|
||||
|
||||
/* init */
|
||||
if (ctx==NULL) { DISPLAY("\n ZSTD_createCCtx error \n"); free(origBuff); return 14;}
|
||||
memset(&winner, 0, sizeof(winner));
|
||||
winner.result.cSize = (size_t)(-1);
|
||||
|
||||
/* find best solution from default params */
|
||||
{ const int maxSeeds = g_noSeed ? 1 : ZSTD_maxCLevel();
|
||||
int i;
|
||||
for (i=1; i<=maxSeeds; i++) {
|
||||
params = ZSTD_getCParams(i, blockSize, 0);
|
||||
BMK_benchParam(&candidate, origBuff, benchedSize, ctx, params);
|
||||
if (candidate.cSpeed < targetSpeed)
|
||||
break;
|
||||
if ( (candidate.cSize < winner.result.cSize)
|
||||
| ((candidate.cSize == winner.result.cSize) & (candidate.cSpeed > winner.result.cSpeed)) )
|
||||
{
|
||||
winner.params = params;
|
||||
winner.result = candidate;
|
||||
BMK_printWinner(stdout, i, winner.result, winner.params, benchedSize);
|
||||
} }
|
||||
}
|
||||
BMK_printWinner(stdout, 99, winner.result, winner.params, benchedSize);
|
||||
|
||||
/* start tests */
|
||||
{ time_t const grillStart = time(NULL);
|
||||
do {
|
||||
params = winner.params;
|
||||
paramVariation(¶ms);
|
||||
if ((FUZ_rand(&g_rand) & 15) == 3) params = randomParams();
|
||||
|
||||
/* exclude faster if already played set of params */
|
||||
if (FUZ_rand(&g_rand) & ((1 << NB_TESTS_PLAYED(params))-1)) continue;
|
||||
|
||||
/* test */
|
||||
NB_TESTS_PLAYED(params)++;
|
||||
BMK_benchParam(&candidate, origBuff, benchedSize, ctx, params);
|
||||
|
||||
/* improvement found => new winner */
|
||||
if ( (candidate.cSpeed > targetSpeed)
|
||||
& ( (candidate.cSize < winner.result.cSize)
|
||||
| ((candidate.cSize == winner.result.cSize) & (candidate.cSpeed > winner.result.cSpeed)) ) )
|
||||
{
|
||||
winner.params = params;
|
||||
winner.result = candidate;
|
||||
BMK_printWinner(stdout, 99, winner.result, winner.params, benchedSize);
|
||||
}
|
||||
} while (BMK_timeSpan(grillStart) < g_grillDuration_s);
|
||||
}
|
||||
|
||||
/* end summary */
|
||||
BMK_printWinner(stdout, 99, winner.result, winner.params, benchedSize);
|
||||
DISPLAY("grillParams size - optimizer completed \n");
|
||||
|
||||
/* clean up*/
|
||||
ZSTD_freeCCtx(ctx);
|
||||
}
|
||||
|
||||
free(origBuff);
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
static int usage(const char* exename)
|
||||
{
|
||||
DISPLAY( "Usage :\n");
|
||||
DISPLAY( " %s [arg] file\n", exename);
|
||||
DISPLAY( "Arguments :\n");
|
||||
DISPLAY( " file : path to the file used as reference (if none, generates a compressible sample)\n");
|
||||
DISPLAY( " -H/-h : Help (this text + advanced options)\n");
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int usage_advanced(void)
|
||||
{
|
||||
DISPLAY( "\nAdvanced options :\n");
|
||||
DISPLAY( " -T# : set level 1 speed objective \n");
|
||||
DISPLAY( " -B# : cut input into blocks of size # (default : single block) \n");
|
||||
DISPLAY( " -i# : iteration loops [1-9](default : %i) \n", NBLOOPS);
|
||||
DISPLAY( " -O# : find Optimized parameters for # target speed (default : 0) \n");
|
||||
DISPLAY( " -S : Single run \n");
|
||||
DISPLAY( " -P# : generated sample compressibility (default : %.1f%%) \n", COMPRESSIBILITY_DEFAULT * 100);
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int badusage(const char* exename)
|
||||
{
|
||||
DISPLAY("Wrong parameters\n");
|
||||
usage(exename);
|
||||
return 1;
|
||||
}
|
||||
|
||||
int main(int argc, const char** argv)
|
||||
{
|
||||
int i,
|
||||
filenamesStart=0,
|
||||
result;
|
||||
const char* exename=argv[0];
|
||||
const char* input_filename=0;
|
||||
U32 optimizer = 0;
|
||||
U32 main_pause = 0;
|
||||
U32 targetSpeed = 0;
|
||||
|
||||
/* checks */
|
||||
if (NB_LEVELS_TRACKED <= ZSTD_maxCLevel()) {
|
||||
DISPLAY("Error : NB_LEVELS_TRACKED <= ZSTD_maxCLevel() \n");
|
||||
exit(1);
|
||||
}
|
||||
|
||||
/* Welcome message */
|
||||
DISPLAY(WELCOME_MESSAGE);
|
||||
|
||||
if (argc<1) { badusage(exename); return 1; }
|
||||
|
||||
for(i=1; i<argc; i++) {
|
||||
const char* argument = argv[i];
|
||||
|
||||
if(!argument) continue; /* Protection if argument empty */
|
||||
|
||||
if(!strcmp(argument,"--no-seed")) { g_noSeed = 1; continue; }
|
||||
|
||||
/* Decode command (note : aggregated commands are allowed) */
|
||||
if (argument[0]=='-') {
|
||||
argument++;
|
||||
|
||||
while (argument[0]!=0) {
|
||||
|
||||
switch(argument[0])
|
||||
{
|
||||
/* Display help on usage */
|
||||
case 'h' :
|
||||
case 'H': usage(exename); usage_advanced(); return 0;
|
||||
|
||||
/* Pause at the end (hidden option) */
|
||||
case 'p': main_pause = 1; argument++; break;
|
||||
|
||||
/* Modify Nb Iterations */
|
||||
case 'i':
|
||||
argument++;
|
||||
if ((argument[0] >='0') & (argument[0] <='9'))
|
||||
g_nbIterations = *argument++ - '0';
|
||||
break;
|
||||
|
||||
/* Sample compressibility (when no file provided) */
|
||||
case 'P':
|
||||
argument++;
|
||||
{ U32 proba32 = 0;
|
||||
while ((argument[0]>= '0') & (argument[0]<= '9'))
|
||||
proba32 = (proba32*10) + (*argument++ - '0');
|
||||
g_compressibility = (double)proba32 / 100.;
|
||||
}
|
||||
break;
|
||||
|
||||
case 'O':
|
||||
argument++;
|
||||
optimizer=1;
|
||||
targetSpeed = 0;
|
||||
while ((*argument >= '0') & (*argument <= '9'))
|
||||
targetSpeed = (targetSpeed*10) + (*argument++ - '0');
|
||||
break;
|
||||
|
||||
/* Run Single conf */
|
||||
case 'S':
|
||||
g_singleRun = 1;
|
||||
argument++;
|
||||
g_params = ZSTD_getCParams(2, g_blockSize, 0);
|
||||
for ( ; ; ) {
|
||||
switch(*argument)
|
||||
{
|
||||
case 'w':
|
||||
g_params.windowLog = 0;
|
||||
argument++;
|
||||
while ((*argument>= '0') && (*argument<='9'))
|
||||
g_params.windowLog *= 10, g_params.windowLog += *argument++ - '0';
|
||||
continue;
|
||||
case 'c':
|
||||
g_params.chainLog = 0;
|
||||
argument++;
|
||||
while ((*argument>= '0') && (*argument<='9'))
|
||||
g_params.chainLog *= 10, g_params.chainLog += *argument++ - '0';
|
||||
continue;
|
||||
case 'h':
|
||||
g_params.hashLog = 0;
|
||||
argument++;
|
||||
while ((*argument>= '0') && (*argument<='9'))
|
||||
g_params.hashLog *= 10, g_params.hashLog += *argument++ - '0';
|
||||
continue;
|
||||
case 's':
|
||||
g_params.searchLog = 0;
|
||||
argument++;
|
||||
while ((*argument>= '0') && (*argument<='9'))
|
||||
g_params.searchLog *= 10, g_params.searchLog += *argument++ - '0';
|
||||
continue;
|
||||
case 'l': /* search length */
|
||||
g_params.searchLength = 0;
|
||||
argument++;
|
||||
while ((*argument>= '0') && (*argument<='9'))
|
||||
g_params.searchLength *= 10, g_params.searchLength += *argument++ - '0';
|
||||
continue;
|
||||
case 't': /* target length */
|
||||
g_params.targetLength = 0;
|
||||
argument++;
|
||||
while ((*argument>= '0') && (*argument<='9'))
|
||||
g_params.targetLength *= 10, g_params.targetLength += *argument++ - '0';
|
||||
continue;
|
||||
case 'S': /* strategy */
|
||||
argument++;
|
||||
while ((*argument>= '0') && (*argument<='9'))
|
||||
g_params.strategy = (ZSTD_strategy)(*argument++ - '0');
|
||||
continue;
|
||||
case 'L':
|
||||
{ int cLevel = 0;
|
||||
argument++;
|
||||
while ((*argument>= '0') && (*argument<='9'))
|
||||
cLevel *= 10, cLevel += *argument++ - '0';
|
||||
g_params = ZSTD_getCParams(cLevel, g_blockSize, 0);
|
||||
continue;
|
||||
}
|
||||
default : ;
|
||||
}
|
||||
break;
|
||||
}
|
||||
break;
|
||||
|
||||
/* target level1 speed objective, in MB/s */
|
||||
case 'T':
|
||||
argument++;
|
||||
g_target = 0;
|
||||
while ((*argument >= '0') && (*argument <= '9'))
|
||||
g_target = (g_target*10) + (*argument++ - '0');
|
||||
break;
|
||||
|
||||
/* cut input into blocks */
|
||||
case 'B':
|
||||
g_blockSize = 0;
|
||||
argument++;
|
||||
while ((*argument >='0') & (*argument <='9'))
|
||||
g_blockSize = (g_blockSize*10) + (*argument++ - '0');
|
||||
if (*argument=='K') g_blockSize<<=10, argument++; /* allows using KB notation */
|
||||
if (*argument=='M') g_blockSize<<=20, argument++;
|
||||
if (*argument=='B') argument++;
|
||||
DISPLAY("using %u KB block size \n", g_blockSize>>10);
|
||||
break;
|
||||
|
||||
/* Unknown command */
|
||||
default : return badusage(exename);
|
||||
}
|
||||
}
|
||||
continue;
|
||||
} /* if (argument[0]=='-') */
|
||||
|
||||
/* first provided filename is input */
|
||||
if (!input_filename) { input_filename=argument; filenamesStart=i; continue; }
|
||||
}
|
||||
|
||||
if (filenamesStart==0)
|
||||
result = benchSample();
|
||||
else {
|
||||
if (optimizer)
|
||||
result = optimizeForSize(input_filename, targetSpeed);
|
||||
else
|
||||
result = benchFiles(argv+filenamesStart, argc-filenamesStart);
|
||||
}
|
||||
|
||||
if (main_pause) { int unused; printf("press enter...\n"); unused = getchar(); (void)unused; }
|
||||
|
||||
return result;
|
||||
}
|
||||
@@ -1,272 +0,0 @@
|
||||
#!/bin/sh -e
|
||||
|
||||
die() {
|
||||
$ECHO "$@" 1>&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
roundTripTest() {
|
||||
if [ -n "$3" ]; then
|
||||
local c="$3"
|
||||
local p="$2"
|
||||
else
|
||||
local c="$2"
|
||||
fi
|
||||
|
||||
rm -f tmp1 tmp2
|
||||
$ECHO "roundTripTest: ./datagen $1 $p | $ZSTD -v$c | $ZSTD -d"
|
||||
./datagen $1 $p | $MD5SUM > tmp1
|
||||
./datagen $1 $p | $ZSTD -v$c | $ZSTD -d | $MD5SUM > tmp2
|
||||
diff -q tmp1 tmp2
|
||||
}
|
||||
|
||||
isWindows=false
|
||||
ECHO="echo"
|
||||
INTOVOID="/dev/null"
|
||||
case "$OS" in
|
||||
Windows*)
|
||||
isWindows=true
|
||||
ECHO="echo -e"
|
||||
INTOVOID="nul"
|
||||
;;
|
||||
esac
|
||||
|
||||
MD5SUM="md5sum"
|
||||
if [ "$TRAVIS_OS_NAME" = "osx" ]; then
|
||||
MD5SUM="md5 -r"
|
||||
fi
|
||||
|
||||
$ECHO "\nStarting playTests.sh isWindows=$isWindows TRAVIS_OS_NAME=$TRAVIS_OS_NAME"
|
||||
|
||||
[ -n "$ZSTD" ] || die "ZSTD variable must be defined!"
|
||||
|
||||
file $ZSTD
|
||||
$ECHO "\n**** simple tests **** "
|
||||
|
||||
./datagen > tmp
|
||||
$ZSTD -f tmp # trivial compression case, creates tmp.zst
|
||||
$ZSTD -df tmp.zst # trivial decompression case (overwrites tmp)
|
||||
$ECHO "test : too large compression level (must fail)"
|
||||
$ZSTD -99 -f tmp # too large compression level, automatic sized down
|
||||
$ECHO "test : compress to stdout"
|
||||
$ZSTD tmp -c > tmpCompressed
|
||||
$ZSTD tmp --stdout > tmpCompressed # long command format
|
||||
$ECHO "test : null-length file roundtrip"
|
||||
$ECHO -n '' | $ZSTD - --stdout | $ZSTD -d --stdout
|
||||
$ECHO "test : decompress file with wrong suffix (must fail)"
|
||||
$ZSTD -d tmpCompressed && die "wrong suffix error not detected!"
|
||||
$ZSTD -df tmp && die "should have refused : wrong extension"
|
||||
$ECHO "test : decompress into stdout"
|
||||
$ZSTD -d tmpCompressed -c > tmpResult # decompression using stdout
|
||||
$ZSTD --decompress tmpCompressed -c > tmpResult
|
||||
$ZSTD --decompress tmpCompressed --stdout > tmpResult
|
||||
$ECHO "test : decompress from stdin into stdout"
|
||||
$ZSTD -dc < tmp.zst > $INTOVOID # combine decompression, stdin & stdout
|
||||
$ZSTD -dc - < tmp.zst > $INTOVOID
|
||||
$ZSTD -d < tmp.zst > $INTOVOID # implicit stdout when stdin is used
|
||||
$ZSTD -d - < tmp.zst > $INTOVOID
|
||||
$ECHO "test : overwrite protection"
|
||||
$ZSTD -q tmp && die "overwrite check failed!"
|
||||
$ECHO "test : force overwrite"
|
||||
$ZSTD -q -f tmp
|
||||
$ZSTD -q --force tmp
|
||||
$ECHO "test : file removal"
|
||||
$ZSTD -f --rm tmp
|
||||
ls tmp && die "tmp should no longer be present"
|
||||
$ZSTD -f -d --rm tmp.zst
|
||||
ls tmp.zst && die "tmp.zst should no longer be present"
|
||||
rm tmp
|
||||
$ZSTD -f tmp && die "tmp not present : should have failed"
|
||||
ls tmp.zst && die "tmp.zst should not be created"
|
||||
|
||||
|
||||
$ECHO "\n**** Pass-Through mode **** "
|
||||
$ECHO "Hello world !" | $ZSTD -df
|
||||
$ECHO "Hello world !" | $ZSTD -dcf
|
||||
|
||||
|
||||
$ECHO "\n**** frame concatenation **** "
|
||||
|
||||
$ECHO "hello " > hello.tmp
|
||||
$ECHO "world!" > world.tmp
|
||||
cat hello.tmp world.tmp > helloworld.tmp
|
||||
$ZSTD -c hello.tmp > hello.zstd
|
||||
$ZSTD -c world.tmp > world.zstd
|
||||
cat hello.zstd world.zstd > helloworld.zstd
|
||||
$ZSTD -dc helloworld.zstd > result.tmp
|
||||
cat result.tmp
|
||||
sdiff helloworld.tmp result.tmp
|
||||
$ECHO "frame concatenation without checksum"
|
||||
$ZSTD -c hello.tmp > hello.zstd --no-check
|
||||
$ZSTD -c world.tmp > world.zstd --no-check
|
||||
cat hello.zstd world.zstd > helloworld.zstd
|
||||
$ZSTD -dc helloworld.zstd > result.tmp
|
||||
cat result.tmp
|
||||
sdiff helloworld.tmp result.tmp
|
||||
rm ./*.tmp ./*.zstd
|
||||
$ECHO "frame concatenation tests completed"
|
||||
|
||||
|
||||
if [ "$isWindows" = false ] ; then
|
||||
$ECHO "\n**** flush write error test **** "
|
||||
|
||||
$ECHO "$ECHO foo | $ZSTD > /dev/full"
|
||||
$ECHO foo | $ZSTD > /dev/full && die "write error not detected!"
|
||||
$ECHO "$ECHO foo | $ZSTD | $ZSTD -d > /dev/full"
|
||||
$ECHO foo | $ZSTD | $ZSTD -d > /dev/full && die "write error not detected!"
|
||||
fi
|
||||
|
||||
|
||||
$ECHO "\n**** test sparse file support **** "
|
||||
|
||||
./datagen -g5M -P100 > tmpSparse
|
||||
$ZSTD tmpSparse -c | $ZSTD -dv -o tmpSparseRegen
|
||||
diff -s tmpSparse tmpSparseRegen
|
||||
$ZSTD tmpSparse -c | $ZSTD -dv --sparse -c > tmpOutSparse
|
||||
diff -s tmpSparse tmpOutSparse
|
||||
$ZSTD tmpSparse -c | $ZSTD -dv --no-sparse -c > tmpOutNoSparse
|
||||
diff -s tmpSparse tmpOutNoSparse
|
||||
ls -ls tmpSparse*
|
||||
./datagen -s1 -g1200007 -P100 | $ZSTD | $ZSTD -dv --sparse -c > tmpSparseOdd # Odd size file (to not finish on an exact nb of blocks)
|
||||
./datagen -s1 -g1200007 -P100 | diff -s - tmpSparseOdd
|
||||
ls -ls tmpSparseOdd
|
||||
$ECHO "\n Sparse Compatibility with Console :"
|
||||
$ECHO "Hello World 1 !" | $ZSTD | $ZSTD -d -c
|
||||
$ECHO "Hello World 2 !" | $ZSTD | $ZSTD -d | cat
|
||||
$ECHO "\n Sparse Compatibility with Append :"
|
||||
./datagen -P100 -g1M > tmpSparse1M
|
||||
cat tmpSparse1M tmpSparse1M > tmpSparse2M
|
||||
$ZSTD -v -f tmpSparse1M -o tmpSparseCompressed
|
||||
$ZSTD -d -v -f tmpSparseCompressed -o tmpSparseRegenerated
|
||||
$ZSTD -d -v -f tmpSparseCompressed -c >> tmpSparseRegenerated
|
||||
ls -ls tmpSparse*
|
||||
diff tmpSparse2M tmpSparseRegenerated
|
||||
rm tmpSparse*
|
||||
|
||||
|
||||
$ECHO "\n**** multiple files tests **** "
|
||||
|
||||
./datagen -s1 > tmp1 2> $INTOVOID
|
||||
./datagen -s2 -g100K > tmp2 2> $INTOVOID
|
||||
./datagen -s3 -g1M > tmp3 2> $INTOVOID
|
||||
$ECHO "compress tmp* : "
|
||||
$ZSTD -f tmp*
|
||||
ls -ls tmp*
|
||||
rm tmp1 tmp2 tmp3
|
||||
$ECHO "decompress tmp* : "
|
||||
$ZSTD -df *.zst
|
||||
ls -ls tmp*
|
||||
$ECHO "compress tmp* into stdout > tmpall : "
|
||||
$ZSTD -c tmp1 tmp2 tmp3 > tmpall
|
||||
ls -ls tmp*
|
||||
$ECHO "decompress tmpall* into stdout > tmpdec : "
|
||||
cp tmpall tmpall2
|
||||
$ZSTD -dc tmpall* > tmpdec
|
||||
ls -ls tmp*
|
||||
$ECHO "compress multiple files including a missing one (notHere) : "
|
||||
$ZSTD -f tmp1 notHere tmp2 && die "missing file not detected!"
|
||||
|
||||
|
||||
$ECHO "\n**** dictionary tests **** "
|
||||
|
||||
./datagen > tmpDict
|
||||
./datagen -g1M | $MD5SUM > tmp1
|
||||
./datagen -g1M | $ZSTD -D tmpDict | $ZSTD -D tmpDict -dvq | $MD5SUM > tmp2
|
||||
diff -q tmp1 tmp2
|
||||
$ECHO "- Create first dictionary"
|
||||
$ZSTD --train *.c -o tmpDict
|
||||
cp zstdcli.c tmp
|
||||
$ZSTD -f tmp -D tmpDict
|
||||
$ZSTD -d tmp.zst -D tmpDict -of result
|
||||
diff zstdcli.c result
|
||||
$ECHO "- Create second (different) dictionary"
|
||||
$ZSTD --train *.c *.h -o tmpDictC
|
||||
$ZSTD -d tmp.zst -D tmpDictC -of result && die "wrong dictionary not detected!"
|
||||
$ECHO "- Create dictionary with short dictID"
|
||||
$ZSTD --train *.c --dictID 1 -o tmpDict1
|
||||
cmp tmpDict tmpDict1 && die "dictionaries should have different ID !"
|
||||
$ECHO "- Compress without dictID"
|
||||
$ZSTD -f tmp -D tmpDict1 --no-dictID
|
||||
$ZSTD -d tmp.zst -D tmpDict -of result
|
||||
diff zstdcli.c result
|
||||
$ECHO "- Compress multiple files with dictionary"
|
||||
rm -rf dirTestDict
|
||||
mkdir dirTestDict
|
||||
cp *.c dirTestDict
|
||||
cp *.h dirTestDict
|
||||
cat dirTestDict/* | $MD5SUM > tmph1 # note : we expect same file order to generate same hash
|
||||
$ZSTD -f dirTestDict/* -D tmpDictC
|
||||
$ZSTD -d dirTestDict/*.zst -D tmpDictC -c | $MD5SUM > tmph2
|
||||
diff -q tmph1 tmph2
|
||||
rm -rf dirTestDict
|
||||
rm tmp*
|
||||
|
||||
|
||||
$ECHO "\n**** integrity tests **** "
|
||||
|
||||
$ECHO "test one file (tmp1.zst) "
|
||||
./datagen > tmp1
|
||||
$ZSTD tmp1
|
||||
$ZSTD -t tmp1.zst
|
||||
$ZSTD --test tmp1.zst
|
||||
$ECHO "test multiple files (*.zst) "
|
||||
$ZSTD -t *.zst
|
||||
$ECHO "test bad files (*) "
|
||||
$ZSTD -t * && die "bad files not detected !"
|
||||
$ZSTD -t tmp1 && die "bad file not detected !"
|
||||
cp tmp1 tmp2.zst
|
||||
$ZSTD -t tmp2.zst && die "bad file not detected !"
|
||||
./datagen -g0 > tmp3
|
||||
$ZSTD -t tmp3 && die "bad file not detected !" # detects 0-sized files as bad
|
||||
$ECHO "test --rm and --test combined "
|
||||
$ZSTD -t --rm tmp1.zst
|
||||
ls -ls tmp1.zst # check file is still present
|
||||
|
||||
|
||||
$ECHO "\n**** zstd round-trip tests **** "
|
||||
|
||||
roundTripTest
|
||||
roundTripTest -g15K # TableID==3
|
||||
roundTripTest -g127K # TableID==2
|
||||
roundTripTest -g255K # TableID==1
|
||||
roundTripTest -g513K # TableID==0
|
||||
roundTripTest -g512K 6 # greedy, hash chain
|
||||
roundTripTest -g512K 16 # btlazy2
|
||||
roundTripTest -g512K 19 # btopt
|
||||
|
||||
rm tmp*
|
||||
|
||||
if [ "$1" != "--test-large-data" ]; then
|
||||
$ECHO "Skipping large data tests"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
roundTripTest -g270000000 1
|
||||
roundTripTest -g270000000 2
|
||||
roundTripTest -g270000000 3
|
||||
|
||||
roundTripTest -g140000000 -P60 4
|
||||
roundTripTest -g140000000 -P60 5
|
||||
roundTripTest -g140000000 -P60 6
|
||||
|
||||
roundTripTest -g70000000 -P70 7
|
||||
roundTripTest -g70000000 -P70 8
|
||||
roundTripTest -g70000000 -P70 9
|
||||
|
||||
roundTripTest -g35000000 -P75 10
|
||||
roundTripTest -g35000000 -P75 11
|
||||
roundTripTest -g35000000 -P75 12
|
||||
|
||||
roundTripTest -g18000000 -P80 13
|
||||
roundTripTest -g18000000 -P80 14
|
||||
roundTripTest -g18000000 -P80 15
|
||||
roundTripTest -g18000000 -P80 16
|
||||
roundTripTest -g18000000 -P80 17
|
||||
|
||||
roundTripTest -g50000000 -P94 18
|
||||
roundTripTest -g50000000 -P94 19
|
||||
|
||||
roundTripTest -g99000000 -P99 20
|
||||
roundTripTest -g6000000000 -P99 1
|
||||
|
||||
rm tmp*
|
||||
@@ -1,193 +0,0 @@
|
||||
/*
|
||||
roundTripCrash
|
||||
Copyright (C) Yann Collet 2013-2016
|
||||
|
||||
GPL v2 License
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation; either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License along
|
||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
You can contact the author at :
|
||||
- zstd homepage : http://www.zstd.net
|
||||
*/
|
||||
/*
|
||||
This program takes a file in input,
|
||||
performs a zstd round-trip test (compression - decompress)
|
||||
compares the result with original
|
||||
and generates a crash (double free) on corruption detection.
|
||||
*/
|
||||
|
||||
/*===========================================
|
||||
* Dependencies
|
||||
*==========================================*/
|
||||
#include <stddef.h> /* size_t */
|
||||
#include <stdlib.h> /* malloc, free, exit */
|
||||
#include <stdio.h> /* fprintf */
|
||||
#include <sys/types.h> /* stat */
|
||||
#include <sys/stat.h> /* stat */
|
||||
#include "xxhash.h"
|
||||
#include "zstd.h"
|
||||
|
||||
/*===========================================
|
||||
* Macros
|
||||
*==========================================*/
|
||||
#define MIN(a,b) ( (a) < (b) ? (a) : (b) )
|
||||
|
||||
/** roundTripTest() :
|
||||
* Compresses `srcBuff` into `compressedBuff`,
|
||||
* then decompresses `compressedBuff` into `resultBuff`.
|
||||
* Compression level used is derived from first content byte.
|
||||
* @return : result of decompression, which should be == `srcSize`
|
||||
* or an error code if either compression or decompression fails.
|
||||
* Note : `compressedBuffCapacity` should be `>= ZSTD_compressBound(srcSize)`
|
||||
* for compression to be guaranteed to work */
|
||||
static size_t roundTripTest(void* resultBuff, size_t resultBuffCapacity,
|
||||
void* compressedBuff, size_t compressedBuffCapacity,
|
||||
const void* srcBuff, size_t srcBuffSize)
|
||||
{
|
||||
static const int maxClevel = 19;
|
||||
size_t const hashLength = MIN(128, srcBuffSize);
|
||||
unsigned const h32 = XXH32(srcBuff, hashLength, 0);
|
||||
int const cLevel = h32 % maxClevel;
|
||||
size_t const cSize = ZSTD_compress(compressedBuff, compressedBuffCapacity, srcBuff, srcBuffSize, cLevel);
|
||||
if (ZSTD_isError(cSize)) {
|
||||
fprintf(stderr, "Compression error : %s \n", ZSTD_getErrorName(cSize));
|
||||
return cSize;
|
||||
}
|
||||
return ZSTD_decompress(resultBuff, resultBuffCapacity, compressedBuff, cSize);
|
||||
}
|
||||
|
||||
|
||||
static size_t checkBuffers(const void* buff1, const void* buff2, size_t buffSize)
|
||||
{
|
||||
const char* ip1 = (const char*)buff1;
|
||||
const char* ip2 = (const char*)buff2;
|
||||
size_t pos;
|
||||
|
||||
for (pos=0; pos<buffSize; pos++)
|
||||
if (ip1[pos]!=ip2[pos])
|
||||
break;
|
||||
|
||||
return pos;
|
||||
}
|
||||
|
||||
|
||||
static void roundTripCheck(const void* srcBuff, size_t srcBuffSize)
|
||||
{
|
||||
size_t const cBuffSize = ZSTD_compressBound(srcBuffSize);
|
||||
void* cBuff = malloc(cBuffSize);
|
||||
void* rBuff = malloc(cBuffSize);
|
||||
#define CRASH { free(cBuff); free(cBuff); } /* double free, to crash program */
|
||||
|
||||
if (!cBuff || !rBuff) {
|
||||
fprintf(stderr, "not enough memory ! \n");
|
||||
exit (1);
|
||||
}
|
||||
|
||||
{ size_t const result = roundTripTest(rBuff, cBuffSize, cBuff, cBuffSize, srcBuff, srcBuffSize);
|
||||
if (ZSTD_isError(result)) {
|
||||
fprintf(stderr, "roundTripTest error : %s \n", ZSTD_getErrorName(result));
|
||||
CRASH;
|
||||
}
|
||||
if (result != srcBuffSize) {
|
||||
fprintf(stderr, "Incorrect regenerated size : %u != %u\n", (unsigned)result, (unsigned)srcBuffSize);
|
||||
CRASH;
|
||||
}
|
||||
if (checkBuffers(srcBuff, rBuff, srcBuffSize) != srcBuffSize) {
|
||||
fprintf(stderr, "Silent decoding corruption !!!");
|
||||
CRASH;
|
||||
}
|
||||
}
|
||||
|
||||
free(cBuff);
|
||||
free(rBuff);
|
||||
}
|
||||
|
||||
|
||||
static size_t getFileSize(const char* infilename)
|
||||
{
|
||||
int r;
|
||||
#if defined(_MSC_VER)
|
||||
struct _stat64 statbuf;
|
||||
r = _stat64(infilename, &statbuf);
|
||||
if (r || !(statbuf.st_mode & S_IFREG)) return 0; /* No good... */
|
||||
#else
|
||||
struct stat statbuf;
|
||||
r = stat(infilename, &statbuf);
|
||||
if (r || !S_ISREG(statbuf.st_mode)) return 0; /* No good... */
|
||||
#endif
|
||||
return (size_t)statbuf.st_size;
|
||||
}
|
||||
|
||||
|
||||
static int isDirectory(const char* infilename)
|
||||
{
|
||||
int r;
|
||||
#if defined(_MSC_VER)
|
||||
struct _stat64 statbuf;
|
||||
r = _stat64(infilename, &statbuf);
|
||||
if (!r && (statbuf.st_mode & _S_IFDIR)) return 1;
|
||||
#else
|
||||
struct stat statbuf;
|
||||
r = stat(infilename, &statbuf);
|
||||
if (!r && S_ISDIR(statbuf.st_mode)) return 1;
|
||||
#endif
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
/** loadFile() :
|
||||
* requirement : `buffer` size >= `fileSize` */
|
||||
static void loadFile(void* buffer, const char* fileName, size_t fileSize)
|
||||
{
|
||||
FILE* const f = fopen(fileName, "rb");
|
||||
if (isDirectory(fileName)) {
|
||||
fprintf(stderr, "Ignoring %s directory \n", fileName);
|
||||
exit(2);
|
||||
}
|
||||
if (f==NULL) {
|
||||
fprintf(stderr, "Impossible to open %s \n", fileName);
|
||||
exit(3);
|
||||
}
|
||||
{ size_t const readSize = fread(buffer, 1, fileSize, f);
|
||||
if (readSize != fileSize) {
|
||||
fprintf(stderr, "Error reading %s \n", fileName);
|
||||
exit(5);
|
||||
} }
|
||||
fclose(f);
|
||||
}
|
||||
|
||||
|
||||
static void fileCheck(const char* fileName)
|
||||
{
|
||||
size_t const fileSize = getFileSize(fileName);
|
||||
void* buffer = malloc(fileSize);
|
||||
if (!buffer) {
|
||||
fprintf(stderr, "not enough memory \n");
|
||||
exit(4);
|
||||
}
|
||||
loadFile(buffer, fileName, fileSize);
|
||||
roundTripCheck(buffer, fileSize);
|
||||
free (buffer);
|
||||
}
|
||||
|
||||
int main(int argCount, const char** argv) {
|
||||
if (argCount < 2) {
|
||||
fprintf(stderr, "Error : no argument : need input file \n");
|
||||
exit(9);
|
||||
}
|
||||
fileCheck(argv[1]);
|
||||
fprintf(stderr, "no pb detected\n");
|
||||
return 0;
|
||||
}
|
||||
@@ -1,645 +0,0 @@
|
||||
/*
|
||||
Fuzzer test tool for zstd_buffered
|
||||
Copyright (C) Yann Collet 2015-2016
|
||||
|
||||
GPL v2 License
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation; either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License along
|
||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
You can contact the author at :
|
||||
- ZSTD homepage : https://www.zstd.net/
|
||||
*/
|
||||
|
||||
/*-************************************
|
||||
* Compiler specific
|
||||
**************************************/
|
||||
#ifdef _MSC_VER /* Visual Studio */
|
||||
# define _CRT_SECURE_NO_WARNINGS /* fgets */
|
||||
# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
|
||||
# pragma warning(disable : 4146) /* disable: C4146: minus unsigned expression */
|
||||
#endif
|
||||
|
||||
|
||||
/*-************************************
|
||||
* Includes
|
||||
**************************************/
|
||||
#include <stdlib.h> /* free */
|
||||
#include <stdio.h> /* fgets, sscanf */
|
||||
#include <sys/timeb.h> /* timeb */
|
||||
#include <string.h> /* strcmp */
|
||||
#include "mem.h"
|
||||
#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_maxCLevel */
|
||||
#include "zstd.h" /* ZSTD_compressBound */
|
||||
#define ZBUFF_STATIC_LINKING_ONLY
|
||||
#include "zbuff.h" /* ZBUFF_createCCtx_advanced */
|
||||
#include "datagen.h" /* RDG_genBuffer */
|
||||
#define XXH_STATIC_LINKING_ONLY
|
||||
#include "xxhash.h" /* XXH64_* */
|
||||
|
||||
|
||||
/*-************************************
|
||||
* Constants
|
||||
**************************************/
|
||||
#define KB *(1U<<10)
|
||||
#define MB *(1U<<20)
|
||||
#define GB *(1U<<30)
|
||||
|
||||
static const U32 nbTestsDefault = 10000;
|
||||
#define COMPRESSIBLE_NOISE_LENGTH (10 MB)
|
||||
#define FUZ_COMPRESSIBILITY_DEFAULT 50
|
||||
static const U32 prime1 = 2654435761U;
|
||||
static const U32 prime2 = 2246822519U;
|
||||
|
||||
|
||||
|
||||
/*-************************************
|
||||
* Display Macros
|
||||
**************************************/
|
||||
#define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
|
||||
#define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); }
|
||||
static U32 g_displayLevel = 2;
|
||||
|
||||
#define DISPLAYUPDATE(l, ...) if (g_displayLevel>=l) { \
|
||||
if ((FUZ_GetMilliSpan(g_displayTime) > g_refreshRate) || (g_displayLevel>=4)) \
|
||||
{ g_displayTime = FUZ_GetMilliStart(); DISPLAY(__VA_ARGS__); \
|
||||
if (g_displayLevel>=4) fflush(stdout); } }
|
||||
static const U32 g_refreshRate = 150;
|
||||
static U32 g_displayTime = 0;
|
||||
|
||||
static U32 g_testTime = 0;
|
||||
|
||||
|
||||
/*-*******************************************************
|
||||
* Fuzzer functions
|
||||
*********************************************************/
|
||||
#define MAX(a,b) ((a)>(b)?(a):(b))
|
||||
|
||||
static U32 FUZ_GetMilliStart(void)
|
||||
{
|
||||
struct timeb tb;
|
||||
U32 nCount;
|
||||
ftime( &tb );
|
||||
nCount = (U32) (((tb.time & 0xFFFFF) * 1000) + tb.millitm);
|
||||
return nCount;
|
||||
}
|
||||
|
||||
|
||||
static U32 FUZ_GetMilliSpan(U32 nTimeStart)
|
||||
{
|
||||
U32 const nCurrent = FUZ_GetMilliStart();
|
||||
U32 nSpan = nCurrent - nTimeStart;
|
||||
if (nTimeStart > nCurrent)
|
||||
nSpan += 0x100000 * 1000;
|
||||
return nSpan;
|
||||
}
|
||||
|
||||
/*! FUZ_rand() :
|
||||
@return : a 27 bits random value, from a 32-bits `seed`.
|
||||
`seed` is also modified */
|
||||
# define FUZ_rotl32(x,r) ((x << r) | (x >> (32 - r)))
|
||||
unsigned int FUZ_rand(unsigned int* seedPtr)
|
||||
{
|
||||
U32 rand32 = *seedPtr;
|
||||
rand32 *= prime1;
|
||||
rand32 += prime2;
|
||||
rand32 = FUZ_rotl32(rand32, 13);
|
||||
*seedPtr = rand32;
|
||||
return rand32 >> 5;
|
||||
}
|
||||
|
||||
|
||||
/*
|
||||
static unsigned FUZ_highbit32(U32 v32)
|
||||
{
|
||||
unsigned nbBits = 0;
|
||||
if (v32==0) return 0;
|
||||
for ( ; v32 ; v32>>=1) nbBits++;
|
||||
return nbBits;
|
||||
}
|
||||
*/
|
||||
|
||||
static void* ZBUFF_allocFunction(void* opaque, size_t size)
|
||||
{
|
||||
void* address = malloc(size);
|
||||
(void)opaque;
|
||||
/* DISPLAYLEVEL(4, "alloc %p, %d opaque=%p \n", address, (int)size, opaque); */
|
||||
return address;
|
||||
}
|
||||
|
||||
static void ZBUFF_freeFunction(void* opaque, void* address)
|
||||
{
|
||||
(void)opaque;
|
||||
/* if (address) DISPLAYLEVEL(4, "free %p opaque=%p \n", address, opaque); */
|
||||
free(address);
|
||||
}
|
||||
|
||||
static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem customMem)
|
||||
{
|
||||
int testResult = 0;
|
||||
size_t CNBufferSize = COMPRESSIBLE_NOISE_LENGTH;
|
||||
void* CNBuffer = malloc(CNBufferSize);
|
||||
size_t const skippableFrameSize = 11;
|
||||
size_t const compressedBufferSize = (8 + skippableFrameSize) + ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH);
|
||||
void* compressedBuffer = malloc(compressedBufferSize);
|
||||
size_t const decodedBufferSize = CNBufferSize;
|
||||
void* decodedBuffer = malloc(decodedBufferSize);
|
||||
size_t cSize, readSize, readSkipSize, genSize;
|
||||
U32 testNb=0;
|
||||
ZBUFF_CCtx* zc = ZBUFF_createCCtx_advanced(customMem);
|
||||
ZBUFF_DCtx* zd = ZBUFF_createDCtx_advanced(customMem);
|
||||
|
||||
/* Create compressible test buffer */
|
||||
if (!CNBuffer || !compressedBuffer || !decodedBuffer || !zc || !zd) {
|
||||
DISPLAY("Not enough memory, aborting\n");
|
||||
goto _output_error;
|
||||
}
|
||||
RDG_genBuffer(CNBuffer, CNBufferSize, compressibility, 0., seed);
|
||||
|
||||
/* generate skippable frame */
|
||||
MEM_writeLE32(compressedBuffer, ZSTD_MAGIC_SKIPPABLE_START);
|
||||
MEM_writeLE32(((char*)compressedBuffer)+4, (U32)skippableFrameSize);
|
||||
cSize = skippableFrameSize + 8;
|
||||
|
||||
/* Basic compression test */
|
||||
DISPLAYLEVEL(4, "test%3i : compress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH);
|
||||
ZBUFF_compressInitDictionary(zc, CNBuffer, 128 KB, 1);
|
||||
readSize = CNBufferSize;
|
||||
genSize = compressedBufferSize;
|
||||
{ size_t const r = ZBUFF_compressContinue(zc, ((char*)compressedBuffer)+cSize, &genSize, CNBuffer, &readSize);
|
||||
if (ZBUFF_isError(r)) goto _output_error; }
|
||||
if (readSize != CNBufferSize) goto _output_error; /* entire input should be consumed */
|
||||
cSize += genSize;
|
||||
genSize = compressedBufferSize - cSize;
|
||||
{ size_t const r = ZBUFF_compressEnd(zc, ((char*)compressedBuffer)+cSize, &genSize);
|
||||
if (r != 0) goto _output_error; } /*< error, or some data not flushed */
|
||||
cSize += genSize;
|
||||
DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/COMPRESSIBLE_NOISE_LENGTH*100);
|
||||
|
||||
/* skippable frame test */
|
||||
DISPLAYLEVEL(4, "test%3i : decompress skippable frame : ", testNb++);
|
||||
ZBUFF_decompressInitDictionary(zd, CNBuffer, 128 KB);
|
||||
readSkipSize = cSize;
|
||||
genSize = CNBufferSize;
|
||||
{ size_t const r = ZBUFF_decompressContinue(zd, decodedBuffer, &genSize, compressedBuffer, &readSkipSize);
|
||||
if (r != 0) goto _output_error; }
|
||||
if (genSize != 0) goto _output_error; /* skippable frame len is 0 */
|
||||
DISPLAYLEVEL(4, "OK \n");
|
||||
|
||||
/* Basic decompression test */
|
||||
DISPLAYLEVEL(4, "test%3i : decompress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH);
|
||||
ZBUFF_decompressInitDictionary(zd, CNBuffer, 128 KB);
|
||||
readSize = cSize - readSkipSize;
|
||||
genSize = CNBufferSize;
|
||||
{ size_t const r = ZBUFF_decompressContinue(zd, decodedBuffer, &genSize, ((char*)compressedBuffer)+readSkipSize, &readSize);
|
||||
if (r != 0) goto _output_error; } /* should reach end of frame == 0; otherwise, some data left, or an error */
|
||||
if (genSize != CNBufferSize) goto _output_error; /* should regenerate the same amount */
|
||||
if (readSize+readSkipSize != cSize) goto _output_error; /* should have read the entire frame */
|
||||
DISPLAYLEVEL(4, "OK \n");
|
||||
|
||||
/* check regenerated data is byte exact */
|
||||
DISPLAYLEVEL(4, "test%3i : check decompressed result : ", testNb++);
|
||||
{ size_t i;
|
||||
for (i=0; i<CNBufferSize; i++) {
|
||||
if (((BYTE*)decodedBuffer)[i] != ((BYTE*)CNBuffer)[i]) goto _output_error;;
|
||||
} }
|
||||
DISPLAYLEVEL(4, "OK \n");
|
||||
|
||||
/* Byte-by-byte decompression test */
|
||||
DISPLAYLEVEL(4, "test%3i : decompress byte-by-byte : ", testNb++);
|
||||
{ size_t r, pIn=0, pOut=0;
|
||||
do
|
||||
{ ZBUFF_decompressInitDictionary(zd, CNBuffer, 128 KB);
|
||||
r = 1;
|
||||
while (r) {
|
||||
size_t inS = 1;
|
||||
size_t outS = 1;
|
||||
r = ZBUFF_decompressContinue(zd, ((BYTE*)decodedBuffer)+pOut, &outS, ((BYTE*)compressedBuffer)+pIn, &inS);
|
||||
pIn += inS;
|
||||
pOut += outS;
|
||||
}
|
||||
readSize = pIn;
|
||||
genSize = pOut;
|
||||
} while (genSize==0);
|
||||
}
|
||||
if (genSize != CNBufferSize) goto _output_error; /* should regenerate the same amount */
|
||||
if (readSize != cSize) goto _output_error; /* should have read the entire frame */
|
||||
DISPLAYLEVEL(4, "OK \n");
|
||||
|
||||
/* check regenerated data is byte exact */
|
||||
DISPLAYLEVEL(4, "test%3i : check decompressed result : ", testNb++);
|
||||
{ size_t i;
|
||||
for (i=0; i<CNBufferSize; i++) {
|
||||
if (((BYTE*)decodedBuffer)[i] != ((BYTE*)CNBuffer)[i]) goto _output_error;;
|
||||
} }
|
||||
DISPLAYLEVEL(4, "OK \n");
|
||||
|
||||
_end:
|
||||
ZBUFF_freeCCtx(zc);
|
||||
ZBUFF_freeDCtx(zd);
|
||||
free(CNBuffer);
|
||||
free(compressedBuffer);
|
||||
free(decodedBuffer);
|
||||
return testResult;
|
||||
|
||||
_output_error:
|
||||
testResult = 1;
|
||||
DISPLAY("Error detected in Unit tests ! \n");
|
||||
goto _end;
|
||||
}
|
||||
|
||||
|
||||
static size_t findDiff(const void* buf1, const void* buf2, size_t max)
|
||||
{
|
||||
const BYTE* b1 = (const BYTE*)buf1;
|
||||
const BYTE* b2 = (const BYTE*)buf2;
|
||||
size_t u;
|
||||
for (u=0; u<max; u++) {
|
||||
if (b1[u] != b2[u]) break;
|
||||
}
|
||||
return u;
|
||||
}
|
||||
|
||||
static size_t FUZ_rLogLength(U32* seed, U32 logLength)
|
||||
{
|
||||
size_t const lengthMask = ((size_t)1 << logLength) - 1;
|
||||
return (lengthMask+1) + (FUZ_rand(seed) & lengthMask);
|
||||
}
|
||||
|
||||
static size_t FUZ_randomLength(U32* seed, U32 maxLog)
|
||||
{
|
||||
U32 const logLength = FUZ_rand(seed) % maxLog;
|
||||
return FUZ_rLogLength(seed, logLength);
|
||||
}
|
||||
|
||||
#define MIN(a,b) ( (a) < (b) ? (a) : (b) )
|
||||
|
||||
#define CHECK(cond, ...) if (cond) { DISPLAY("Error => "); DISPLAY(__VA_ARGS__); \
|
||||
DISPLAY(" (seed %u, test nb %u) \n", seed, testNb); goto _output_error; }
|
||||
|
||||
static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibility)
|
||||
{
|
||||
static const U32 maxSrcLog = 24;
|
||||
static const U32 maxSampleLog = 19;
|
||||
BYTE* cNoiseBuffer[5];
|
||||
size_t srcBufferSize = (size_t)1<<maxSrcLog;
|
||||
BYTE* copyBuffer;
|
||||
size_t copyBufferSize= srcBufferSize + (1<<maxSampleLog);
|
||||
BYTE* cBuffer;
|
||||
size_t cBufferSize = ZSTD_compressBound(srcBufferSize);
|
||||
BYTE* dstBuffer;
|
||||
size_t dstBufferSize = srcBufferSize;
|
||||
U32 result = 0;
|
||||
U32 testNb = 0;
|
||||
U32 coreSeed = seed;
|
||||
ZBUFF_CCtx* zc;
|
||||
ZBUFF_DCtx* zd;
|
||||
U32 startTime = FUZ_GetMilliStart();
|
||||
|
||||
/* allocations */
|
||||
zc = ZBUFF_createCCtx();
|
||||
zd = ZBUFF_createDCtx();
|
||||
cNoiseBuffer[0] = (BYTE*)malloc (srcBufferSize);
|
||||
cNoiseBuffer[1] = (BYTE*)malloc (srcBufferSize);
|
||||
cNoiseBuffer[2] = (BYTE*)malloc (srcBufferSize);
|
||||
cNoiseBuffer[3] = (BYTE*)malloc (srcBufferSize);
|
||||
cNoiseBuffer[4] = (BYTE*)malloc (srcBufferSize);
|
||||
copyBuffer= (BYTE*)malloc (copyBufferSize);
|
||||
dstBuffer = (BYTE*)malloc (dstBufferSize);
|
||||
cBuffer = (BYTE*)malloc (cBufferSize);
|
||||
CHECK (!cNoiseBuffer[0] || !cNoiseBuffer[1] || !cNoiseBuffer[2] || !cNoiseBuffer[3] || !cNoiseBuffer[4] ||
|
||||
!copyBuffer || !dstBuffer || !cBuffer || !zc || !zd,
|
||||
"Not enough memory, fuzzer tests cancelled");
|
||||
|
||||
/* Create initial samples */
|
||||
RDG_genBuffer(cNoiseBuffer[0], srcBufferSize, 0.00, 0., coreSeed); /* pure noise */
|
||||
RDG_genBuffer(cNoiseBuffer[1], srcBufferSize, 0.05, 0., coreSeed); /* barely compressible */
|
||||
RDG_genBuffer(cNoiseBuffer[2], srcBufferSize, compressibility, 0., coreSeed);
|
||||
RDG_genBuffer(cNoiseBuffer[3], srcBufferSize, 0.95, 0., coreSeed); /* highly compressible */
|
||||
RDG_genBuffer(cNoiseBuffer[4], srcBufferSize, 1.00, 0., coreSeed); /* sparse content */
|
||||
memset(copyBuffer, 0x65, copyBufferSize); /* make copyBuffer considered initialized */
|
||||
|
||||
/* catch up testNb */
|
||||
for (testNb=1; testNb < startTest; testNb++)
|
||||
FUZ_rand(&coreSeed);
|
||||
|
||||
/* test loop */
|
||||
for ( ; (testNb <= nbTests) || (FUZ_GetMilliSpan(startTime) < g_testTime) ; testNb++ ) {
|
||||
U32 lseed;
|
||||
const BYTE* srcBuffer;
|
||||
const BYTE* dict;
|
||||
size_t maxTestSize, dictSize;
|
||||
size_t cSize, totalTestSize, totalCSize, totalGenSize;
|
||||
size_t errorCode;
|
||||
U32 n, nbChunks;
|
||||
XXH64_state_t xxhState;
|
||||
U64 crcOrig;
|
||||
|
||||
/* init */
|
||||
DISPLAYUPDATE(2, "\r%6u", testNb);
|
||||
if (nbTests >= testNb) DISPLAYUPDATE(2, "/%6u ", nbTests);
|
||||
FUZ_rand(&coreSeed);
|
||||
lseed = coreSeed ^ prime1;
|
||||
|
||||
/* states full reset (unsynchronized) */
|
||||
/* some issues only happen when reusing states in a specific sequence of parameters */
|
||||
if ((FUZ_rand(&lseed) & 0xFF) == 131) { ZBUFF_freeCCtx(zc); zc = ZBUFF_createCCtx(); }
|
||||
if ((FUZ_rand(&lseed) & 0xFF) == 132) { ZBUFF_freeDCtx(zd); zd = ZBUFF_createDCtx(); }
|
||||
|
||||
/* srcBuffer selection [0-4] */
|
||||
{ U32 buffNb = FUZ_rand(&lseed) & 0x7F;
|
||||
if (buffNb & 7) buffNb=2; /* most common : compressible (P) */
|
||||
else {
|
||||
buffNb >>= 3;
|
||||
if (buffNb & 7) {
|
||||
const U32 tnb[2] = { 1, 3 }; /* barely/highly compressible */
|
||||
buffNb = tnb[buffNb >> 3];
|
||||
} else {
|
||||
const U32 tnb[2] = { 0, 4 }; /* not compressible / sparse */
|
||||
buffNb = tnb[buffNb >> 3];
|
||||
} }
|
||||
srcBuffer = cNoiseBuffer[buffNb];
|
||||
}
|
||||
|
||||
/* compression init */
|
||||
{ U32 const testLog = FUZ_rand(&lseed) % maxSrcLog;
|
||||
U32 const cLevel = (FUZ_rand(&lseed) % (ZSTD_maxCLevel() - (testLog/3))) + 1;
|
||||
maxTestSize = FUZ_rLogLength(&lseed, testLog);
|
||||
dictSize = (FUZ_rand(&lseed)==1) ? FUZ_randomLength(&lseed, maxSampleLog) : 0;
|
||||
/* random dictionary selection */
|
||||
{ size_t const dictStart = FUZ_rand(&lseed) % (srcBufferSize - dictSize);
|
||||
dict = srcBuffer + dictStart;
|
||||
}
|
||||
{ ZSTD_parameters params = ZSTD_getParams(cLevel, 0, dictSize);
|
||||
params.fParams.checksumFlag = FUZ_rand(&lseed) & 1;
|
||||
params.fParams.noDictIDFlag = FUZ_rand(&lseed) & 1;
|
||||
{ size_t const initError = ZBUFF_compressInit_advanced(zc, dict, dictSize, params, 0);
|
||||
CHECK (ZBUFF_isError(initError),"init error : %s", ZBUFF_getErrorName(initError));
|
||||
} } }
|
||||
|
||||
/* multi-segments compression test */
|
||||
XXH64_reset(&xxhState, 0);
|
||||
nbChunks = (FUZ_rand(&lseed) & 127) + 2;
|
||||
for (n=0, cSize=0, totalTestSize=0 ; (n<nbChunks) && (totalTestSize < maxTestSize) ; n++) {
|
||||
/* compress random chunk into random size dst buffer */
|
||||
{ size_t readChunkSize = FUZ_randomLength(&lseed, maxSampleLog);
|
||||
size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog);
|
||||
size_t dstBuffSize = MIN(cBufferSize - cSize, randomDstSize);
|
||||
size_t const srcStart = FUZ_rand(&lseed) % (srcBufferSize - readChunkSize);
|
||||
|
||||
size_t const compressionError = ZBUFF_compressContinue(zc, cBuffer+cSize, &dstBuffSize, srcBuffer+srcStart, &readChunkSize);
|
||||
CHECK (ZBUFF_isError(compressionError), "compression error : %s", ZBUFF_getErrorName(compressionError));
|
||||
|
||||
XXH64_update(&xxhState, srcBuffer+srcStart, readChunkSize);
|
||||
memcpy(copyBuffer+totalTestSize, srcBuffer+srcStart, readChunkSize);
|
||||
cSize += dstBuffSize;
|
||||
totalTestSize += readChunkSize;
|
||||
}
|
||||
|
||||
/* random flush operation, to mess around */
|
||||
if ((FUZ_rand(&lseed) & 15) == 0) {
|
||||
size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog);
|
||||
size_t dstBuffSize = MIN(cBufferSize - cSize, randomDstSize);
|
||||
size_t const flushError = ZBUFF_compressFlush(zc, cBuffer+cSize, &dstBuffSize);
|
||||
CHECK (ZBUFF_isError(flushError), "flush error : %s", ZBUFF_getErrorName(flushError));
|
||||
cSize += dstBuffSize;
|
||||
} }
|
||||
|
||||
/* final frame epilogue */
|
||||
{ size_t remainingToFlush = (size_t)(-1);
|
||||
while (remainingToFlush) {
|
||||
size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog);
|
||||
size_t dstBuffSize = MIN(cBufferSize - cSize, randomDstSize);
|
||||
U32 const enoughDstSize = dstBuffSize >= remainingToFlush;
|
||||
remainingToFlush = ZBUFF_compressEnd(zc, cBuffer+cSize, &dstBuffSize);
|
||||
CHECK (ZBUFF_isError(remainingToFlush), "flush error : %s", ZBUFF_getErrorName(remainingToFlush));
|
||||
CHECK (enoughDstSize && remainingToFlush, "ZBUFF_compressEnd() not fully flushed (%u remaining), but enough space available", (U32)remainingToFlush);
|
||||
cSize += dstBuffSize;
|
||||
} }
|
||||
crcOrig = XXH64_digest(&xxhState);
|
||||
|
||||
/* multi - fragments decompression test */
|
||||
ZBUFF_decompressInitDictionary(zd, dict, dictSize);
|
||||
errorCode = 1;
|
||||
for (totalCSize = 0, totalGenSize = 0 ; errorCode ; ) {
|
||||
size_t readCSrcSize = FUZ_randomLength(&lseed, maxSampleLog);
|
||||
size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog);
|
||||
size_t dstBuffSize = MIN(dstBufferSize - totalGenSize, randomDstSize);
|
||||
errorCode = ZBUFF_decompressContinue(zd, dstBuffer+totalGenSize, &dstBuffSize, cBuffer+totalCSize, &readCSrcSize);
|
||||
CHECK (ZBUFF_isError(errorCode), "decompression error : %s", ZBUFF_getErrorName(errorCode));
|
||||
totalGenSize += dstBuffSize;
|
||||
totalCSize += readCSrcSize;
|
||||
}
|
||||
CHECK (errorCode != 0, "frame not fully decoded");
|
||||
CHECK (totalGenSize != totalTestSize, "decompressed data : wrong size")
|
||||
CHECK (totalCSize != cSize, "compressed data should be fully read")
|
||||
{ U64 const crcDest = XXH64(dstBuffer, totalTestSize, 0);
|
||||
if (crcDest!=crcOrig) findDiff(copyBuffer, dstBuffer, totalTestSize);
|
||||
CHECK (crcDest!=crcOrig, "decompressed data corrupted"); }
|
||||
|
||||
/*===== noisy/erroneous src decompression test =====*/
|
||||
|
||||
/* add some noise */
|
||||
{ U32 const nbNoiseChunks = (FUZ_rand(&lseed) & 7) + 2;
|
||||
U32 nn; for (nn=0; nn<nbNoiseChunks; nn++) {
|
||||
size_t const randomNoiseSize = FUZ_randomLength(&lseed, maxSampleLog);
|
||||
size_t const noiseSize = MIN((cSize/3) , randomNoiseSize);
|
||||
size_t const noiseStart = FUZ_rand(&lseed) % (srcBufferSize - noiseSize);
|
||||
size_t const cStart = FUZ_rand(&lseed) % (cSize - noiseSize);
|
||||
memcpy(cBuffer+cStart, srcBuffer+noiseStart, noiseSize);
|
||||
} }
|
||||
|
||||
/* try decompression on noisy data */
|
||||
ZBUFF_decompressInit(zd);
|
||||
totalCSize = 0;
|
||||
totalGenSize = 0;
|
||||
while ( (totalCSize < cSize) && (totalGenSize < dstBufferSize) ) {
|
||||
size_t readCSrcSize = FUZ_randomLength(&lseed, maxSampleLog);
|
||||
size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog);
|
||||
size_t dstBuffSize = MIN(dstBufferSize - totalGenSize, randomDstSize);
|
||||
size_t const decompressError = ZBUFF_decompressContinue(zd, dstBuffer+totalGenSize, &dstBuffSize, cBuffer+totalCSize, &readCSrcSize);
|
||||
if (ZBUFF_isError(decompressError)) break; /* error correctly detected */
|
||||
totalGenSize += dstBuffSize;
|
||||
totalCSize += readCSrcSize;
|
||||
} }
|
||||
DISPLAY("\r%u fuzzer tests completed \n", testNb);
|
||||
|
||||
_cleanup:
|
||||
ZBUFF_freeCCtx(zc);
|
||||
ZBUFF_freeDCtx(zd);
|
||||
free(cNoiseBuffer[0]);
|
||||
free(cNoiseBuffer[1]);
|
||||
free(cNoiseBuffer[2]);
|
||||
free(cNoiseBuffer[3]);
|
||||
free(cNoiseBuffer[4]);
|
||||
free(copyBuffer);
|
||||
free(cBuffer);
|
||||
free(dstBuffer);
|
||||
return result;
|
||||
|
||||
_output_error:
|
||||
result = 1;
|
||||
goto _cleanup;
|
||||
}
|
||||
|
||||
|
||||
/*-*******************************************************
|
||||
* Command line
|
||||
*********************************************************/
|
||||
int FUZ_usage(const char* programName)
|
||||
{
|
||||
DISPLAY( "Usage :\n");
|
||||
DISPLAY( " %s [args]\n", programName);
|
||||
DISPLAY( "\n");
|
||||
DISPLAY( "Arguments :\n");
|
||||
DISPLAY( " -i# : Nb of tests (default:%u) \n", nbTestsDefault);
|
||||
DISPLAY( " -s# : Select seed (default:prompt user)\n");
|
||||
DISPLAY( " -t# : Select starting test number (default:0)\n");
|
||||
DISPLAY( " -P# : Select compressibility in %% (default:%i%%)\n", FUZ_COMPRESSIBILITY_DEFAULT);
|
||||
DISPLAY( " -v : verbose\n");
|
||||
DISPLAY( " -p : pause at the end\n");
|
||||
DISPLAY( " -h : display help and exit\n");
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
int main(int argc, const char** argv)
|
||||
{
|
||||
U32 seed=0;
|
||||
int seedset=0;
|
||||
int argNb;
|
||||
int nbTests = nbTestsDefault;
|
||||
int testNb = 0;
|
||||
int proba = FUZ_COMPRESSIBILITY_DEFAULT;
|
||||
int result=0;
|
||||
U32 mainPause = 0;
|
||||
const char* programName = argv[0];
|
||||
ZSTD_customMem customMem = { ZBUFF_allocFunction, ZBUFF_freeFunction, NULL };
|
||||
ZSTD_customMem customNULL = { NULL, NULL, NULL };
|
||||
|
||||
/* Check command line */
|
||||
for(argNb=1; argNb<argc; argNb++) {
|
||||
const char* argument = argv[argNb];
|
||||
if(!argument) continue; /* Protection if argument empty */
|
||||
|
||||
/* Parsing commands. Aggregated commands are allowed */
|
||||
if (argument[0]=='-') {
|
||||
argument++;
|
||||
|
||||
while (*argument!=0) {
|
||||
switch(*argument)
|
||||
{
|
||||
case 'h':
|
||||
return FUZ_usage(programName);
|
||||
case 'v':
|
||||
argument++;
|
||||
g_displayLevel=4;
|
||||
break;
|
||||
case 'q':
|
||||
argument++;
|
||||
g_displayLevel--;
|
||||
break;
|
||||
case 'p': /* pause at the end */
|
||||
argument++;
|
||||
mainPause = 1;
|
||||
break;
|
||||
|
||||
case 'i':
|
||||
argument++;
|
||||
nbTests=0; g_testTime=0;
|
||||
while ((*argument>='0') && (*argument<='9')) {
|
||||
nbTests *= 10;
|
||||
nbTests += *argument - '0';
|
||||
argument++;
|
||||
}
|
||||
break;
|
||||
|
||||
case 'T':
|
||||
argument++;
|
||||
nbTests=0; g_testTime=0;
|
||||
while ((*argument>='0') && (*argument<='9')) {
|
||||
g_testTime *= 10;
|
||||
g_testTime += *argument - '0';
|
||||
argument++;
|
||||
}
|
||||
if (*argument=='m') g_testTime *=60, argument++;
|
||||
if (*argument=='n') argument++;
|
||||
g_testTime *= 1000;
|
||||
break;
|
||||
|
||||
case 's':
|
||||
argument++;
|
||||
seed=0;
|
||||
seedset=1;
|
||||
while ((*argument>='0') && (*argument<='9')) {
|
||||
seed *= 10;
|
||||
seed += *argument - '0';
|
||||
argument++;
|
||||
}
|
||||
break;
|
||||
|
||||
case 't':
|
||||
argument++;
|
||||
testNb=0;
|
||||
while ((*argument>='0') && (*argument<='9')) {
|
||||
testNb *= 10;
|
||||
testNb += *argument - '0';
|
||||
argument++;
|
||||
}
|
||||
break;
|
||||
|
||||
case 'P': /* compressibility % */
|
||||
argument++;
|
||||
proba=0;
|
||||
while ((*argument>='0') && (*argument<='9')) {
|
||||
proba *= 10;
|
||||
proba += *argument - '0';
|
||||
argument++;
|
||||
}
|
||||
if (proba<0) proba=0;
|
||||
if (proba>100) proba=100;
|
||||
break;
|
||||
|
||||
default:
|
||||
return FUZ_usage(programName);
|
||||
}
|
||||
} } } /* for(argNb=1; argNb<argc; argNb++) */
|
||||
|
||||
/* Get Seed */
|
||||
DISPLAY("Starting zstd_buffered tester (%i-bits, %s)\n", (int)(sizeof(size_t)*8), ZSTD_VERSION_STRING);
|
||||
|
||||
if (!seedset) seed = FUZ_GetMilliStart() % 10000;
|
||||
DISPLAY("Seed = %u\n", seed);
|
||||
if (proba!=FUZ_COMPRESSIBILITY_DEFAULT) DISPLAY("Compressibility : %i%%\n", proba);
|
||||
|
||||
if (nbTests<=0) nbTests=1;
|
||||
|
||||
if (testNb==0) {
|
||||
result = basicUnitTests(0, ((double)proba) / 100, customNULL); /* constant seed for predictability */
|
||||
if (!result) {
|
||||
DISPLAYLEVEL(4, "Unit tests using customMem :\n")
|
||||
result = basicUnitTests(0, ((double)proba) / 100, customMem); /* use custom memory allocation functions */
|
||||
} }
|
||||
|
||||
if (!result)
|
||||
result = fuzzerTests(seed, nbTests, testNb, ((double)proba) / 100);
|
||||
|
||||
if (mainPause) {
|
||||
int unused;
|
||||
DISPLAY("Press Enter \n");
|
||||
unused = getchar();
|
||||
(void)unused;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
@@ -1,659 +0,0 @@
|
||||
/*
|
||||
Fuzzer test tool for zstd streaming API
|
||||
Copyright (C) Yann Collet 2016
|
||||
|
||||
GPL v2 License
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation; either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License along
|
||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
You can contact the author at :
|
||||
- ZSTD homepage : https://www.zstd.net/
|
||||
*/
|
||||
|
||||
/*-************************************
|
||||
* Compiler specific
|
||||
**************************************/
|
||||
#ifdef _MSC_VER /* Visual Studio */
|
||||
# define _CRT_SECURE_NO_WARNINGS /* fgets */
|
||||
# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
|
||||
# pragma warning(disable : 4146) /* disable: C4146: minus unsigned expression */
|
||||
#endif
|
||||
|
||||
|
||||
/*-************************************
|
||||
* Includes
|
||||
**************************************/
|
||||
#include <stdlib.h> /* free */
|
||||
#include <stdio.h> /* fgets, sscanf */
|
||||
#include <sys/timeb.h> /* timeb */
|
||||
#include <string.h> /* strcmp */
|
||||
#include "mem.h"
|
||||
#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_maxCLevel */
|
||||
#include "zstd.h" /* ZSTD_compressBound */
|
||||
#include "datagen.h" /* RDG_genBuffer */
|
||||
#define XXH_STATIC_LINKING_ONLY
|
||||
#include "xxhash.h" /* XXH64_* */
|
||||
|
||||
|
||||
/*-************************************
|
||||
* Constants
|
||||
**************************************/
|
||||
#define KB *(1U<<10)
|
||||
#define MB *(1U<<20)
|
||||
#define GB *(1U<<30)
|
||||
|
||||
static const U32 nbTestsDefault = 10000;
|
||||
#define COMPRESSIBLE_NOISE_LENGTH (10 MB)
|
||||
#define FUZ_COMPRESSIBILITY_DEFAULT 50
|
||||
static const U32 prime1 = 2654435761U;
|
||||
static const U32 prime2 = 2246822519U;
|
||||
|
||||
|
||||
|
||||
/*-************************************
|
||||
* Display Macros
|
||||
**************************************/
|
||||
#define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
|
||||
#define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); }
|
||||
static U32 g_displayLevel = 2;
|
||||
|
||||
#define DISPLAYUPDATE(l, ...) if (g_displayLevel>=l) { \
|
||||
if ((FUZ_GetMilliSpan(g_displayTime) > g_refreshRate) || (g_displayLevel>=4)) \
|
||||
{ g_displayTime = FUZ_GetMilliStart(); DISPLAY(__VA_ARGS__); \
|
||||
if (g_displayLevel>=4) fflush(stdout); } }
|
||||
static const U32 g_refreshRate = 150;
|
||||
static U32 g_displayTime = 0;
|
||||
|
||||
static U32 g_testTime = 0;
|
||||
|
||||
|
||||
/*-*******************************************************
|
||||
* Fuzzer functions
|
||||
*********************************************************/
|
||||
#define MAX(a,b) ((a)>(b)?(a):(b))
|
||||
|
||||
static U32 FUZ_GetMilliStart(void)
|
||||
{
|
||||
struct timeb tb;
|
||||
U32 nCount;
|
||||
ftime( &tb );
|
||||
nCount = (U32) (((tb.time & 0xFFFFF) * 1000) + tb.millitm);
|
||||
return nCount;
|
||||
}
|
||||
|
||||
|
||||
static U32 FUZ_GetMilliSpan(U32 nTimeStart)
|
||||
{
|
||||
U32 const nCurrent = FUZ_GetMilliStart();
|
||||
U32 nSpan = nCurrent - nTimeStart;
|
||||
if (nTimeStart > nCurrent)
|
||||
nSpan += 0x100000 * 1000;
|
||||
return nSpan;
|
||||
}
|
||||
|
||||
/*! FUZ_rand() :
|
||||
@return : a 27 bits random value, from a 32-bits `seed`.
|
||||
`seed` is also modified */
|
||||
# define FUZ_rotl32(x,r) ((x << r) | (x >> (32 - r)))
|
||||
unsigned int FUZ_rand(unsigned int* seedPtr)
|
||||
{
|
||||
U32 rand32 = *seedPtr;
|
||||
rand32 *= prime1;
|
||||
rand32 += prime2;
|
||||
rand32 = FUZ_rotl32(rand32, 13);
|
||||
*seedPtr = rand32;
|
||||
return rand32 >> 5;
|
||||
}
|
||||
|
||||
|
||||
/*
|
||||
static unsigned FUZ_highbit32(U32 v32)
|
||||
{
|
||||
unsigned nbBits = 0;
|
||||
if (v32==0) return 0;
|
||||
for ( ; v32 ; v32>>=1) nbBits++;
|
||||
return nbBits;
|
||||
}
|
||||
*/
|
||||
|
||||
static void* allocFunction(void* opaque, size_t size)
|
||||
{
|
||||
void* address = malloc(size);
|
||||
(void)opaque;
|
||||
return address;
|
||||
}
|
||||
|
||||
static void freeFunction(void* opaque, void* address)
|
||||
{
|
||||
(void)opaque;
|
||||
free(address);
|
||||
}
|
||||
|
||||
static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem customMem)
|
||||
{
|
||||
int testResult = 0;
|
||||
size_t CNBufferSize = COMPRESSIBLE_NOISE_LENGTH;
|
||||
void* CNBuffer = malloc(CNBufferSize);
|
||||
size_t const skippableFrameSize = 11;
|
||||
size_t const compressedBufferSize = (8 + skippableFrameSize) + ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH);
|
||||
void* compressedBuffer = malloc(compressedBufferSize);
|
||||
size_t const decodedBufferSize = CNBufferSize;
|
||||
void* decodedBuffer = malloc(decodedBufferSize);
|
||||
size_t cSize;
|
||||
U32 testNb=0;
|
||||
ZSTD_CStream* zc = ZSTD_createCStream_advanced(customMem);
|
||||
ZSTD_DStream* zd = ZSTD_createDStream_advanced(customMem);
|
||||
ZSTD_inBuffer inBuff;
|
||||
ZSTD_outBuffer outBuff;
|
||||
|
||||
/* Create compressible test buffer */
|
||||
if (!CNBuffer || !compressedBuffer || !decodedBuffer || !zc || !zd) {
|
||||
DISPLAY("Not enough memory, aborting\n");
|
||||
goto _output_error;
|
||||
}
|
||||
RDG_genBuffer(CNBuffer, CNBufferSize, compressibility, 0., seed);
|
||||
|
||||
/* generate skippable frame */
|
||||
MEM_writeLE32(compressedBuffer, ZSTD_MAGIC_SKIPPABLE_START);
|
||||
MEM_writeLE32(((char*)compressedBuffer)+4, (U32)skippableFrameSize);
|
||||
cSize = skippableFrameSize + 8;
|
||||
|
||||
/* Basic compression test */
|
||||
DISPLAYLEVEL(4, "test%3i : compress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH);
|
||||
ZSTD_initCStream_usingDict(zc, CNBuffer, 128 KB, 1);
|
||||
outBuff.dst = (char*)(compressedBuffer)+cSize;
|
||||
outBuff.size = compressedBufferSize;
|
||||
outBuff.pos = 0;
|
||||
inBuff.src = CNBuffer;
|
||||
inBuff.size = CNBufferSize;
|
||||
inBuff.pos = 0;
|
||||
{ size_t const r = ZSTD_compressStream(zc, &outBuff, &inBuff);
|
||||
if (ZSTD_isError(r)) goto _output_error; }
|
||||
if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */
|
||||
{ size_t const r = ZSTD_endStream(zc, &outBuff);
|
||||
if (r != 0) goto _output_error; } /*< error, or some data not flushed */
|
||||
cSize += outBuff.pos;
|
||||
DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/COMPRESSIBLE_NOISE_LENGTH*100);
|
||||
|
||||
/* skippable frame test */
|
||||
DISPLAYLEVEL(4, "test%3i : decompress skippable frame : ", testNb++);
|
||||
ZSTD_initDStream_usingDict(zd, CNBuffer, 128 KB);
|
||||
inBuff.src = compressedBuffer;
|
||||
inBuff.size = cSize;
|
||||
inBuff.pos = 0;
|
||||
outBuff.dst = decodedBuffer;
|
||||
outBuff.size = CNBufferSize;
|
||||
outBuff.pos = 0;
|
||||
{ size_t const r = ZSTD_decompressStream(zd, &outBuff, &inBuff);
|
||||
if (r != 0) goto _output_error; }
|
||||
if (outBuff.pos != 0) goto _output_error; /* skippable frame len is 0 */
|
||||
DISPLAYLEVEL(4, "OK \n");
|
||||
|
||||
/* Basic decompression test */
|
||||
DISPLAYLEVEL(4, "test%3i : decompress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH);
|
||||
ZSTD_initDStream_usingDict(zd, CNBuffer, 128 KB);
|
||||
{ size_t const r = ZSTD_decompressStream(zd, &outBuff, &inBuff);
|
||||
if (r != 0) goto _output_error; } /* should reach end of frame == 0; otherwise, some data left, or an error */
|
||||
if (outBuff.pos != CNBufferSize) goto _output_error; /* should regenerate the same amount */
|
||||
if (inBuff.pos != inBuff.size) goto _output_error; /* should have read the entire frame */
|
||||
DISPLAYLEVEL(4, "OK \n");
|
||||
|
||||
/* check regenerated data is byte exact */
|
||||
DISPLAYLEVEL(4, "test%3i : check decompressed result : ", testNb++);
|
||||
{ size_t i;
|
||||
for (i=0; i<CNBufferSize; i++) {
|
||||
if (((BYTE*)decodedBuffer)[i] != ((BYTE*)CNBuffer)[i]) goto _output_error;
|
||||
} }
|
||||
DISPLAYLEVEL(4, "OK \n");
|
||||
|
||||
/* Byte-by-byte decompression test */
|
||||
DISPLAYLEVEL(4, "test%3i : decompress byte-by-byte : ", testNb++);
|
||||
{ size_t r = 1;
|
||||
ZSTD_initDStream_usingDict(zd, CNBuffer, 128 KB);
|
||||
inBuff.src = compressedBuffer;
|
||||
outBuff.dst = decodedBuffer;
|
||||
inBuff.pos = 0;
|
||||
outBuff.pos = 0;
|
||||
while (r) { /* skippable frame */
|
||||
inBuff.size = inBuff.pos + 1;
|
||||
outBuff.size = outBuff.pos + 1;
|
||||
r = ZSTD_decompressStream(zd, &outBuff, &inBuff);
|
||||
if (ZSTD_isError(r)) goto _output_error;
|
||||
}
|
||||
ZSTD_initDStream_usingDict(zd, CNBuffer, 128 KB);
|
||||
r=1;
|
||||
while (r) { /* normal frame */
|
||||
inBuff.size = inBuff.pos + 1;
|
||||
outBuff.size = outBuff.pos + 1;
|
||||
r = ZSTD_decompressStream(zd, &outBuff, &inBuff);
|
||||
if (ZSTD_isError(r)) goto _output_error;
|
||||
}
|
||||
}
|
||||
if (outBuff.pos != CNBufferSize) goto _output_error; /* should regenerate the same amount */
|
||||
if (inBuff.pos != cSize) goto _output_error; /* should have read the entire frame */
|
||||
DISPLAYLEVEL(4, "OK \n");
|
||||
|
||||
/* check regenerated data is byte exact */
|
||||
DISPLAYLEVEL(4, "test%3i : check decompressed result : ", testNb++);
|
||||
{ size_t i;
|
||||
for (i=0; i<CNBufferSize; i++) {
|
||||
if (((BYTE*)decodedBuffer)[i] != ((BYTE*)CNBuffer)[i]) goto _output_error;;
|
||||
} }
|
||||
DISPLAYLEVEL(4, "OK \n");
|
||||
|
||||
_end:
|
||||
ZSTD_freeCStream(zc);
|
||||
ZSTD_freeDStream(zd);
|
||||
free(CNBuffer);
|
||||
free(compressedBuffer);
|
||||
free(decodedBuffer);
|
||||
return testResult;
|
||||
|
||||
_output_error:
|
||||
testResult = 1;
|
||||
DISPLAY("Error detected in Unit tests ! \n");
|
||||
goto _end;
|
||||
}
|
||||
|
||||
|
||||
static size_t findDiff(const void* buf1, const void* buf2, size_t max)
|
||||
{
|
||||
const BYTE* b1 = (const BYTE*)buf1;
|
||||
const BYTE* b2 = (const BYTE*)buf2;
|
||||
size_t u;
|
||||
for (u=0; u<max; u++) {
|
||||
if (b1[u] != b2[u]) break;
|
||||
}
|
||||
return u;
|
||||
}
|
||||
|
||||
static size_t FUZ_rLogLength(U32* seed, U32 logLength)
|
||||
{
|
||||
size_t const lengthMask = ((size_t)1 << logLength) - 1;
|
||||
return (lengthMask+1) + (FUZ_rand(seed) & lengthMask);
|
||||
}
|
||||
|
||||
static size_t FUZ_randomLength(U32* seed, U32 maxLog)
|
||||
{
|
||||
U32 const logLength = FUZ_rand(seed) % maxLog;
|
||||
return FUZ_rLogLength(seed, logLength);
|
||||
}
|
||||
|
||||
#define MIN(a,b) ( (a) < (b) ? (a) : (b) )
|
||||
|
||||
#define CHECK(cond, ...) if (cond) { DISPLAY("Error => "); DISPLAY(__VA_ARGS__); \
|
||||
DISPLAY(" (seed %u, test nb %u) \n", seed, testNb); goto _output_error; }
|
||||
|
||||
static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibility)
|
||||
{
|
||||
static const U32 maxSrcLog = 24;
|
||||
static const U32 maxSampleLog = 19;
|
||||
BYTE* cNoiseBuffer[5];
|
||||
size_t srcBufferSize = (size_t)1<<maxSrcLog;
|
||||
BYTE* copyBuffer;
|
||||
size_t copyBufferSize= srcBufferSize + (1<<maxSampleLog);
|
||||
BYTE* cBuffer;
|
||||
size_t cBufferSize = ZSTD_compressBound(srcBufferSize);
|
||||
BYTE* dstBuffer;
|
||||
size_t dstBufferSize = srcBufferSize;
|
||||
U32 result = 0;
|
||||
U32 testNb = 0;
|
||||
U32 coreSeed = seed;
|
||||
ZSTD_CStream* zc;
|
||||
ZSTD_DStream* zd;
|
||||
U32 startTime = FUZ_GetMilliStart();
|
||||
|
||||
/* allocations */
|
||||
zc = ZSTD_createCStream();
|
||||
zd = ZSTD_createDStream();
|
||||
cNoiseBuffer[0] = (BYTE*)malloc (srcBufferSize);
|
||||
cNoiseBuffer[1] = (BYTE*)malloc (srcBufferSize);
|
||||
cNoiseBuffer[2] = (BYTE*)malloc (srcBufferSize);
|
||||
cNoiseBuffer[3] = (BYTE*)malloc (srcBufferSize);
|
||||
cNoiseBuffer[4] = (BYTE*)malloc (srcBufferSize);
|
||||
copyBuffer= (BYTE*)malloc (copyBufferSize);
|
||||
dstBuffer = (BYTE*)malloc (dstBufferSize);
|
||||
cBuffer = (BYTE*)malloc (cBufferSize);
|
||||
CHECK (!cNoiseBuffer[0] || !cNoiseBuffer[1] || !cNoiseBuffer[2] || !cNoiseBuffer[3] || !cNoiseBuffer[4] ||
|
||||
!copyBuffer || !dstBuffer || !cBuffer || !zc || !zd,
|
||||
"Not enough memory, fuzzer tests cancelled");
|
||||
|
||||
/* Create initial samples */
|
||||
RDG_genBuffer(cNoiseBuffer[0], srcBufferSize, 0.00, 0., coreSeed); /* pure noise */
|
||||
RDG_genBuffer(cNoiseBuffer[1], srcBufferSize, 0.05, 0., coreSeed); /* barely compressible */
|
||||
RDG_genBuffer(cNoiseBuffer[2], srcBufferSize, compressibility, 0., coreSeed);
|
||||
RDG_genBuffer(cNoiseBuffer[3], srcBufferSize, 0.95, 0., coreSeed); /* highly compressible */
|
||||
RDG_genBuffer(cNoiseBuffer[4], srcBufferSize, 1.00, 0., coreSeed); /* sparse content */
|
||||
memset(copyBuffer, 0x65, copyBufferSize); /* make copyBuffer considered initialized */
|
||||
|
||||
/* catch up testNb */
|
||||
for (testNb=1; testNb < startTest; testNb++)
|
||||
FUZ_rand(&coreSeed);
|
||||
|
||||
/* test loop */
|
||||
for ( ; (testNb <= nbTests) || (FUZ_GetMilliSpan(startTime) < g_testTime) ; testNb++ ) {
|
||||
U32 lseed;
|
||||
const BYTE* srcBuffer;
|
||||
const BYTE* dict;
|
||||
size_t maxTestSize, dictSize;
|
||||
size_t cSize, totalTestSize, totalGenSize;
|
||||
U32 n, nbChunks;
|
||||
XXH64_state_t xxhState;
|
||||
U64 crcOrig;
|
||||
|
||||
/* init */
|
||||
DISPLAYUPDATE(2, "\r%6u", testNb);
|
||||
if (nbTests >= testNb) DISPLAYUPDATE(2, "/%6u ", nbTests);
|
||||
FUZ_rand(&coreSeed);
|
||||
lseed = coreSeed ^ prime1;
|
||||
|
||||
/* states full reset (unsynchronized) */
|
||||
/* some issues only happen when reusing states in a specific sequence of parameters */
|
||||
if ((FUZ_rand(&lseed) & 0xFF) == 131) { ZSTD_freeCStream(zc); zc = ZSTD_createCStream(); }
|
||||
if ((FUZ_rand(&lseed) & 0xFF) == 132) { ZSTD_freeDStream(zd); zd = ZSTD_createDStream(); }
|
||||
|
||||
/* srcBuffer selection [0-4] */
|
||||
{ U32 buffNb = FUZ_rand(&lseed) & 0x7F;
|
||||
if (buffNb & 7) buffNb=2; /* most common : compressible (P) */
|
||||
else {
|
||||
buffNb >>= 3;
|
||||
if (buffNb & 7) {
|
||||
const U32 tnb[2] = { 1, 3 }; /* barely/highly compressible */
|
||||
buffNb = tnb[buffNb >> 3];
|
||||
} else {
|
||||
const U32 tnb[2] = { 0, 4 }; /* not compressible / sparse */
|
||||
buffNb = tnb[buffNb >> 3];
|
||||
} }
|
||||
srcBuffer = cNoiseBuffer[buffNb];
|
||||
}
|
||||
|
||||
/* compression init */
|
||||
{ U32 const testLog = FUZ_rand(&lseed) % maxSrcLog;
|
||||
U32 const cLevel = (FUZ_rand(&lseed) % (ZSTD_maxCLevel() - (testLog/3))) + 1;
|
||||
maxTestSize = FUZ_rLogLength(&lseed, testLog);
|
||||
dictSize = (FUZ_rand(&lseed)==1) ? FUZ_randomLength(&lseed, maxSampleLog) : 0;
|
||||
/* random dictionary selection */
|
||||
{ size_t const dictStart = FUZ_rand(&lseed) % (srcBufferSize - dictSize);
|
||||
dict = srcBuffer + dictStart;
|
||||
}
|
||||
{ ZSTD_parameters params = ZSTD_getParams(cLevel, 0, dictSize);
|
||||
params.fParams.checksumFlag = FUZ_rand(&lseed) & 1;
|
||||
params.fParams.noDictIDFlag = FUZ_rand(&lseed) & 1;
|
||||
{ size_t const initError = ZSTD_initCStream_advanced(zc, dict, dictSize, params, 0);
|
||||
CHECK (ZSTD_isError(initError),"init error : %s", ZSTD_getErrorName(initError));
|
||||
} } }
|
||||
|
||||
/* multi-segments compression test */
|
||||
XXH64_reset(&xxhState, 0);
|
||||
nbChunks = (FUZ_rand(&lseed) & 127) + 2;
|
||||
{ ZSTD_outBuffer outBuff = { cBuffer, cBufferSize, 0 } ;
|
||||
for (n=0, cSize=0, totalTestSize=0 ; (n<nbChunks) && (totalTestSize < maxTestSize) ; n++) {
|
||||
/* compress random chunk into random size dst buffer */
|
||||
{ size_t const readChunkSize = FUZ_randomLength(&lseed, maxSampleLog);
|
||||
size_t const srcStart = FUZ_rand(&lseed) % (srcBufferSize - readChunkSize);
|
||||
size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog);
|
||||
size_t const dstBuffSize = MIN(cBufferSize - cSize, randomDstSize);
|
||||
ZSTD_inBuffer inBuff = { srcBuffer+srcStart, readChunkSize, 0 };
|
||||
outBuff.size = outBuff.pos + dstBuffSize;
|
||||
|
||||
{ size_t const compressionError = ZSTD_compressStream(zc, &outBuff, &inBuff);
|
||||
CHECK (ZSTD_isError(compressionError), "compression error : %s", ZSTD_getErrorName(compressionError)); }
|
||||
|
||||
XXH64_update(&xxhState, srcBuffer+srcStart, inBuff.pos);
|
||||
memcpy(copyBuffer+totalTestSize, srcBuffer+srcStart, inBuff.pos);
|
||||
totalTestSize += inBuff.pos;
|
||||
}
|
||||
|
||||
/* random flush operation, to mess around */
|
||||
if ((FUZ_rand(&lseed) & 15) == 0) {
|
||||
size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog);
|
||||
size_t const adjustedDstSize = MIN(cBufferSize - cSize, randomDstSize);
|
||||
outBuff.size = outBuff.pos + adjustedDstSize;
|
||||
{ size_t const flushError = ZSTD_flushStream(zc, &outBuff);
|
||||
CHECK (ZSTD_isError(flushError), "flush error : %s", ZSTD_getErrorName(flushError));
|
||||
} } }
|
||||
|
||||
/* final frame epilogue */
|
||||
{ size_t remainingToFlush = (size_t)(-1);
|
||||
while (remainingToFlush) {
|
||||
size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog);
|
||||
size_t const adjustedDstSize = MIN(cBufferSize - cSize, randomDstSize);
|
||||
U32 const enoughDstSize = (adjustedDstSize >= remainingToFlush);
|
||||
outBuff.size = outBuff.pos + adjustedDstSize;
|
||||
remainingToFlush = ZSTD_endStream(zc, &outBuff);
|
||||
CHECK (ZSTD_isError(remainingToFlush), "flush error : %s", ZSTD_getErrorName(remainingToFlush));
|
||||
CHECK (enoughDstSize && remainingToFlush, "ZSTD_endStream() not fully flushed (%u remaining), but enough space available", (U32)remainingToFlush);
|
||||
} }
|
||||
crcOrig = XXH64_digest(&xxhState);
|
||||
cSize = outBuff.pos;
|
||||
}
|
||||
|
||||
/* multi - fragments decompression test */
|
||||
ZSTD_initDStream_usingDict(zd, dict, dictSize);
|
||||
{ size_t decompressionResult = 1;
|
||||
ZSTD_inBuffer inBuff = { cBuffer, cSize, 0 };
|
||||
ZSTD_outBuffer outBuff= { dstBuffer, dstBufferSize, 0 };
|
||||
for (totalGenSize = 0 ; decompressionResult ; ) {
|
||||
size_t const readCSrcSize = FUZ_randomLength(&lseed, maxSampleLog);
|
||||
size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog);
|
||||
size_t const dstBuffSize = MIN(dstBufferSize - totalGenSize, randomDstSize);
|
||||
inBuff.size = inBuff.pos + readCSrcSize;
|
||||
outBuff.size = inBuff.pos + dstBuffSize;
|
||||
decompressionResult = ZSTD_decompressStream(zd, &outBuff, &inBuff);
|
||||
CHECK (ZSTD_isError(decompressionResult), "decompression error : %s", ZSTD_getErrorName(decompressionResult));
|
||||
}
|
||||
CHECK (decompressionResult != 0, "frame not fully decoded");
|
||||
CHECK (outBuff.pos != totalTestSize, "decompressed data : wrong size")
|
||||
CHECK (inBuff.pos != cSize, "compressed data should be fully read")
|
||||
{ U64 const crcDest = XXH64(dstBuffer, totalTestSize, 0);
|
||||
if (crcDest!=crcOrig) findDiff(copyBuffer, dstBuffer, totalTestSize);
|
||||
CHECK (crcDest!=crcOrig, "decompressed data corrupted");
|
||||
} }
|
||||
|
||||
/*===== noisy/erroneous src decompression test =====*/
|
||||
|
||||
/* add some noise */
|
||||
{ U32 const nbNoiseChunks = (FUZ_rand(&lseed) & 7) + 2;
|
||||
U32 nn; for (nn=0; nn<nbNoiseChunks; nn++) {
|
||||
size_t const randomNoiseSize = FUZ_randomLength(&lseed, maxSampleLog);
|
||||
size_t const noiseSize = MIN((cSize/3) , randomNoiseSize);
|
||||
size_t const noiseStart = FUZ_rand(&lseed) % (srcBufferSize - noiseSize);
|
||||
size_t const cStart = FUZ_rand(&lseed) % (cSize - noiseSize);
|
||||
memcpy(cBuffer+cStart, srcBuffer+noiseStart, noiseSize);
|
||||
} }
|
||||
|
||||
/* try decompression on noisy data */
|
||||
ZSTD_initDStream(zd);
|
||||
{ ZSTD_inBuffer inBuff = { cBuffer, cSize, 0 };
|
||||
ZSTD_outBuffer outBuff= { dstBuffer, dstBufferSize, 0 };
|
||||
while (outBuff.pos < dstBufferSize) {
|
||||
size_t const randomCSrcSize = FUZ_randomLength(&lseed, maxSampleLog);
|
||||
size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog);
|
||||
size_t const adjustedDstSize = MIN(dstBufferSize - outBuff.pos, randomDstSize);
|
||||
outBuff.size = outBuff.pos + adjustedDstSize;
|
||||
inBuff.size = inBuff.pos + randomCSrcSize;
|
||||
{ size_t const decompressError = ZSTD_decompressStream(zd, &outBuff, &inBuff);
|
||||
if (ZSTD_isError(decompressError)) break; /* error correctly detected */
|
||||
} } } }
|
||||
DISPLAY("\r%u fuzzer tests completed \n", testNb);
|
||||
|
||||
_cleanup:
|
||||
ZSTD_freeCStream(zc);
|
||||
ZSTD_freeDStream(zd);
|
||||
free(cNoiseBuffer[0]);
|
||||
free(cNoiseBuffer[1]);
|
||||
free(cNoiseBuffer[2]);
|
||||
free(cNoiseBuffer[3]);
|
||||
free(cNoiseBuffer[4]);
|
||||
free(copyBuffer);
|
||||
free(cBuffer);
|
||||
free(dstBuffer);
|
||||
return result;
|
||||
|
||||
_output_error:
|
||||
result = 1;
|
||||
goto _cleanup;
|
||||
}
|
||||
|
||||
|
||||
/*-*******************************************************
|
||||
* Command line
|
||||
*********************************************************/
|
||||
int FUZ_usage(const char* programName)
|
||||
{
|
||||
DISPLAY( "Usage :\n");
|
||||
DISPLAY( " %s [args]\n", programName);
|
||||
DISPLAY( "\n");
|
||||
DISPLAY( "Arguments :\n");
|
||||
DISPLAY( " -i# : Nb of tests (default:%u) \n", nbTestsDefault);
|
||||
DISPLAY( " -s# : Select seed (default:prompt user)\n");
|
||||
DISPLAY( " -t# : Select starting test number (default:0)\n");
|
||||
DISPLAY( " -P# : Select compressibility in %% (default:%i%%)\n", FUZ_COMPRESSIBILITY_DEFAULT);
|
||||
DISPLAY( " -v : verbose\n");
|
||||
DISPLAY( " -p : pause at the end\n");
|
||||
DISPLAY( " -h : display help and exit\n");
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
int main(int argc, const char** argv)
|
||||
{
|
||||
U32 seed=0;
|
||||
int seedset=0;
|
||||
int argNb;
|
||||
int nbTests = nbTestsDefault;
|
||||
int testNb = 0;
|
||||
int proba = FUZ_COMPRESSIBILITY_DEFAULT;
|
||||
int result=0;
|
||||
U32 mainPause = 0;
|
||||
const char* programName = argv[0];
|
||||
ZSTD_customMem customMem = { allocFunction, freeFunction, NULL };
|
||||
ZSTD_customMem customNULL = { NULL, NULL, NULL };
|
||||
|
||||
/* Check command line */
|
||||
for(argNb=1; argNb<argc; argNb++) {
|
||||
const char* argument = argv[argNb];
|
||||
if(!argument) continue; /* Protection if argument empty */
|
||||
|
||||
/* Parsing commands. Aggregated commands are allowed */
|
||||
if (argument[0]=='-') {
|
||||
argument++;
|
||||
|
||||
while (*argument!=0) {
|
||||
switch(*argument)
|
||||
{
|
||||
case 'h':
|
||||
return FUZ_usage(programName);
|
||||
case 'v':
|
||||
argument++;
|
||||
g_displayLevel=4;
|
||||
break;
|
||||
case 'q':
|
||||
argument++;
|
||||
g_displayLevel--;
|
||||
break;
|
||||
case 'p': /* pause at the end */
|
||||
argument++;
|
||||
mainPause = 1;
|
||||
break;
|
||||
|
||||
case 'i':
|
||||
argument++;
|
||||
nbTests=0; g_testTime=0;
|
||||
while ((*argument>='0') && (*argument<='9')) {
|
||||
nbTests *= 10;
|
||||
nbTests += *argument - '0';
|
||||
argument++;
|
||||
}
|
||||
break;
|
||||
|
||||
case 'T':
|
||||
argument++;
|
||||
nbTests=0; g_testTime=0;
|
||||
while ((*argument>='0') && (*argument<='9')) {
|
||||
g_testTime *= 10;
|
||||
g_testTime += *argument - '0';
|
||||
argument++;
|
||||
}
|
||||
if (*argument=='m') g_testTime *=60, argument++;
|
||||
if (*argument=='n') argument++;
|
||||
g_testTime *= 1000;
|
||||
break;
|
||||
|
||||
case 's':
|
||||
argument++;
|
||||
seed=0;
|
||||
seedset=1;
|
||||
while ((*argument>='0') && (*argument<='9')) {
|
||||
seed *= 10;
|
||||
seed += *argument - '0';
|
||||
argument++;
|
||||
}
|
||||
break;
|
||||
|
||||
case 't':
|
||||
argument++;
|
||||
testNb=0;
|
||||
while ((*argument>='0') && (*argument<='9')) {
|
||||
testNb *= 10;
|
||||
testNb += *argument - '0';
|
||||
argument++;
|
||||
}
|
||||
break;
|
||||
|
||||
case 'P': /* compressibility % */
|
||||
argument++;
|
||||
proba=0;
|
||||
while ((*argument>='0') && (*argument<='9')) {
|
||||
proba *= 10;
|
||||
proba += *argument - '0';
|
||||
argument++;
|
||||
}
|
||||
if (proba<0) proba=0;
|
||||
if (proba>100) proba=100;
|
||||
break;
|
||||
|
||||
default:
|
||||
return FUZ_usage(programName);
|
||||
}
|
||||
} } } /* for(argNb=1; argNb<argc; argNb++) */
|
||||
|
||||
/* Get Seed */
|
||||
DISPLAY("Starting zstd_buffered tester (%i-bits, %s)\n", (int)(sizeof(size_t)*8), ZSTD_VERSION_STRING);
|
||||
|
||||
if (!seedset) seed = FUZ_GetMilliStart() % 10000;
|
||||
DISPLAY("Seed = %u\n", seed);
|
||||
if (proba!=FUZ_COMPRESSIBILITY_DEFAULT) DISPLAY("Compressibility : %i%%\n", proba);
|
||||
|
||||
if (nbTests<=0) nbTests=1;
|
||||
|
||||
if (testNb==0) {
|
||||
result = basicUnitTests(0, ((double)proba) / 100, customNULL); /* constant seed for predictability */
|
||||
if (!result) {
|
||||
DISPLAYLEVEL(4, "Unit tests using customMem :\n")
|
||||
result = basicUnitTests(0, ((double)proba) / 100, customMem); /* use custom memory allocation functions */
|
||||
} }
|
||||
|
||||
if (!result)
|
||||
result = fuzzerTests(seed, nbTests, testNb, ((double)proba) / 100);
|
||||
|
||||
if (mainPause) {
|
||||
int unused;
|
||||
DISPLAY("Press Enter \n");
|
||||
unused = getchar();
|
||||
(void)unused;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
Reference in New Issue
Block a user