ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'refs/remotes/Cyan4973/dev070' into dev070

Browse Source
This commit is contained in:
inikep
2016-06-02 22:09:09 +02:00
parent 614fddee6a 923938edde
commit b74a468fad
16 changed files with 160 additions and 130 deletions
Download Patch File Download Diff File Expand all files Collapse all files
programs/Makefile
+1 -1
View File
@@ -40,7 +40,7 @@ ZSTDDIR = ../lib
CPPFLAGS= -I$(ZSTDDIR)/common -I$(ZSTDDIR)/dictBuilder -DXXH_NAMESPACE=ZSTD_
CFLAGS ?= -O3 # -falign-loops=32 # not always beneficial
CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wstrict-aliasing=1 -Wswitch-enum -Wstrict-prototypes -Wundef
CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes -Wundef
FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) $(MOREFLAGS)
programs/bench.c
-2
View File
@@ -220,7 +220,6 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
{ ZSTD_parameters params;
params.cParams = ZSTD_getCParams(cLevel, blockSize, dictBufferSize);
params.fParams.contentSizeFlag = 1;
ZSTD_adjustCParams(&params.cParams, blockSize, dictBufferSize);
{ size_t const initResult = ZSTD_compressBegin_advanced(refCtx, dictBuffer, dictBufferSize, params, blockSize);
if (ZSTD_isError(initResult)) break;
} }
@@ -505,4 +504,3 @@ int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles,
BMK_benchFileTable(fileNamesTable, nbFiles, dictFileName, cLevel, cLevelLast);
return 0;
}
programs/fileio.c
+3
View File
@@ -135,6 +135,8 @@ static U32 g_sparseFileSupport = 1; /* 0 : no sparse allowed; 1: auto (file ye
void FIO_setSparseWrite(unsigned sparse) { g_sparseFileSupport=sparse; }
static U32 g_dictIDFlag = 1;
void FIO_setDictIDFlag(unsigned dictIDFlag) { g_dictIDFlag = dictIDFlag; }
static U32 g_checksumFlag = 0;
void FIO_setChecksumFlag(unsigned checksumFlag) { g_checksumFlag = checksumFlag; }
/*-*************************************
@@ -313,6 +315,7 @@ static int FIO_compressFilename_internal(cRess_t ress,
memset(&params, 0, sizeof(params));
params.cParams = ZSTD_getCParams(cLevel, fileSize, ress.dictBufferSize);
params.fParams.contentSizeFlag = 1;
params.fParams.checksumFlag = g_checksumFlag;
params.fParams.noDictIDFlag = !g_dictIDFlag;
if (g_maxWLog) if (params.cParams.windowLog > g_maxWLog) params.cParams.windowLog = g_maxWLog;
{ size_t const errorCode = ZBUFF_compressInit_advanced(ress.ctx, ress.dictBuffer, ress.dictBufferSize, params, fileSize);
programs/fileio.h
+1
View File
@@ -49,6 +49,7 @@ void FIO_setNotificationLevel(unsigned level);
void FIO_setMaxWLog(unsigned maxWLog); /**< if `maxWLog` == 0, no max enforced */
void FIO_setSparseWrite(unsigned sparse); /**< 0: no sparse; 1: disable on stdout; 2: always enabled */
void FIO_setDictIDFlag(unsigned dictIDFlag);
void FIO_setChecksumFlag(unsigned checksumFlag);
/*-*************************************
programs/paramgrill.c
+15 -17
View File
@@ -600,22 +600,22 @@ static void playAround(FILE* f, winnerInfo_t* winners,
}
static void potentialRandomParams(ZSTD_compressionParameters* p, U32 inverseChance)
static ZSTD_compressionParameters randomParams(void)
{
U32 chance = (FUZ_rand(&g_rand) % (inverseChance+1));
ZSTD_compressionParameters p;
U32 validated = 0;
if (!chance)
while (!validated) {
/* totally random entry */
p->chainLog = FUZ_rand(&g_rand) % (ZSTD_CHAINLOG_MAX+1 - ZSTD_CHAINLOG_MIN) + ZSTD_CHAINLOG_MIN;
p->hashLog = FUZ_rand(&g_rand) % (ZSTD_HASHLOG_MAX+1 - ZSTD_HASHLOG_MIN) + ZSTD_HASHLOG_MIN;
p->searchLog = FUZ_rand(&g_rand) % (ZSTD_SEARCHLOG_MAX+1 - ZSTD_SEARCHLOG_MIN) + ZSTD_SEARCHLOG_MIN;
p->windowLog = FUZ_rand(&g_rand) % (ZSTD_WINDOWLOG_MAX+1 - ZSTD_WINDOWLOG_MIN) + ZSTD_WINDOWLOG_MIN;
p->searchLength=FUZ_rand(&g_rand) % (ZSTD_SEARCHLENGTH_MAX+1 - ZSTD_SEARCHLENGTH_MIN) + ZSTD_SEARCHLENGTH_MIN;
p->targetLength=FUZ_rand(&g_rand) % (ZSTD_TARGETLENGTH_MAX+1 - ZSTD_TARGETLENGTH_MIN) + ZSTD_TARGETLENGTH_MIN;
p->strategy = (ZSTD_strategy) (FUZ_rand(&g_rand) % (ZSTD_btopt +1));
validated = !ZSTD_isError(ZSTD_checkCParams(*p));
p.chainLog = FUZ_rand(&g_rand) % (ZSTD_CHAINLOG_MAX+1 - ZSTD_CHAINLOG_MIN) + ZSTD_CHAINLOG_MIN;
p.hashLog = FUZ_rand(&g_rand) % (ZSTD_HASHLOG_MAX+1 - ZSTD_HASHLOG_MIN) + ZSTD_HASHLOG_MIN;
p.searchLog = FUZ_rand(&g_rand) % (ZSTD_SEARCHLOG_MAX+1 - ZSTD_SEARCHLOG_MIN) + ZSTD_SEARCHLOG_MIN;
p.windowLog = FUZ_rand(&g_rand) % (ZSTD_WINDOWLOG_MAX+1 - ZSTD_WINDOWLOG_MIN) + ZSTD_WINDOWLOG_MIN;
p.searchLength=FUZ_rand(&g_rand) % (ZSTD_SEARCHLENGTH_MAX+1 - ZSTD_SEARCHLENGTH_MIN) + ZSTD_SEARCHLENGTH_MIN;
p.targetLength=FUZ_rand(&g_rand) % (ZSTD_TARGETLENGTH_MAX+1 - ZSTD_TARGETLENGTH_MIN) + ZSTD_TARGETLENGTH_MIN;
p.strategy = (ZSTD_strategy) (FUZ_rand(&g_rand) % (ZSTD_btopt +1));
validated = !ZSTD_isError(ZSTD_checkCParams(p));
}
return p;
}
static void BMK_selectRandomStart(
@@ -623,12 +623,10 @@ static void BMK_selectRandomStart(
const void* srcBuffer, size_t srcSize,
ZSTD_CCtx* ctx)
{
U32 id = (FUZ_rand(&g_rand) % (ZSTD_maxCLevel()+1));
U32 const id = (FUZ_rand(&g_rand) % (ZSTD_maxCLevel()+1));
if ((id==0) || (winners[id].params.windowLog==0)) {
/* totally random entry */
ZSTD_compressionParameters p;
potentialRandomParams(&p, 1);
ZSTD_adjustCParams(&p, srcSize, 0);
ZSTD_compressionParameters const p = ZSTD_adjustCParams(randomParams(), srcSize, 0);
playAround(f, winners, p, srcBuffer, srcSize, ctx);
}
else
@@ -649,7 +647,7 @@ static void BMK_benchMem(void* srcBuffer, size_t srcSize)
if (g_singleRun) {
BMK_result_t testResult;
ZSTD_adjustCParams(&g_params, srcSize, 0);
g_params = ZSTD_adjustCParams(g_params, srcSize, 0);
BMK_benchParam(&testResult, srcBuffer, srcSize, ctx, g_params);
DISPLAY("\n");
return;
@@ -861,7 +859,7 @@ int optimizeForSize(char* inFileName)
do {
params = winner.params;
paramVariation(&params);
potentialRandomParams(&params, 16);
if ((FUZ_rand(&g_rand) & 15) == 1) params = randomParams();
/* exclude faster if already played set of params */
if (FUZ_rand(&g_rand) & ((1 << NB_TESTS_PLAYED(params))-1)) continue;
programs/zbufftest.c
+16 -10
View File
@@ -214,7 +214,7 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
/* Byte-by-byte decompression test */
DISPLAYLEVEL(4, "test%3i : decompress byte-by-byte : ", testNb++);
{ size_t r, pIn=0, pOut=0;
do
do
{ ZBUFF_decompressInitDictionary(zd, CNBuffer, 128 KB);
r = 1;
while (r) {
@@ -348,8 +348,8 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compres
FUZ_rand(&coreSeed);
lseed = coreSeed ^ prime1;
/* state total reset */
/* some problems only happen when states are re-used in a specific order */
/* states full reset (unsynchronized) */
/* some issues only happen when reusing states in a specific sequence of parameters */
if ((FUZ_rand(&lseed) & 0xFF) == 131) { ZBUFF_freeCCtx(zc); zc = ZBUFF_createCCtx(); }
if ((FUZ_rand(&lseed) & 0xFF) == 132) { ZBUFF_freeDCtx(zd); zd = ZBUFF_createDCtx(); }
@@ -372,15 +372,21 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compres
{ U32 const testLog = FUZ_rand(&lseed) % maxSrcLog;
U32 const cLevel = (FUZ_rand(&lseed) % (ZSTD_maxCLevel() - (testLog/3))) + 1;
maxTestSize = FUZ_rLogLength(&lseed, testLog);
dictSize = (FUZ_rand(&lseed)==1) ? FUZ_randomLength(&lseed, maxSampleLog) : 0;
/* random dictionary selection */
{ size_t dictStart;
dictSize = (FUZ_rand(&lseed)==1) ? FUZ_randomLength(&lseed, maxSampleLog) : 0;
dictStart = FUZ_rand(&lseed) % (srcBufferSize - dictSize);
dict = srcBuffer + dictStart;
{ size_t const dictStart = FUZ_rand(&lseed) % (srcBufferSize - dictSize);
dict = srcBuffer + dictStart;
}
{ size_t const initError = ZBUFF_compressInitDictionary(zc, dict, dictSize, cLevel);
CHECK (ZBUFF_isError(initError),"init error : %s", ZBUFF_getErrorName(initError));
} }
{ ZSTD_compressionParameters const cPar = ZSTD_getCParams(cLevel, 0, dictSize);
U32 const checksum = FUZ_rand(&lseed) & 1;
U32 const noDictIDFlag = FUZ_rand(&lseed) & 1;
ZSTD_frameParameters const fPar = { 0, checksum, noDictIDFlag };
ZSTD_parameters params;
params.cParams = cPar;
params.fParams = fPar;
{ size_t const initError = ZBUFF_compressInit_advanced(zc, dict, dictSize, params, 0);
CHECK (ZBUFF_isError(initError),"init error : %s", ZBUFF_getErrorName(initError));
} } }
/* multi-segments compression test */
XXH64_reset(&xxhState, 0);
programs/zstd.1
+8 -1
View File
@@ -52,7 +52,7 @@ It also features a very fast decoder, with speed > 500 MB/s per core.
use `file` as Dictionary to compress or decompress FILE(s)
.TP
.B \-o file
save result into `file` (only possible with a single input FILE)
save result into `file` (only possible with a single INPUT-FILE)
.TP
.BR \-f ", " --force
overwrite output without prompting
@@ -71,6 +71,13 @@ It also features a very fast decoder, with speed > 500 MB/s per core.
.TP
.BR \-c ", " --stdout
force write to standard output, even if it is the console
.TP
.BR \-C ", " --check
add integrity check computed from uncompressed data
.TP
.BR \-t ", " --test
Test the integrity of compressed files. This option is equivalent to \fB--decompress --stdout > /dev/null\fR.
No files are created or removed.
.SH DICTIONARY
.PP
programs/zstdcli.c
+6 -1
View File
@@ -134,9 +134,10 @@ static int usage_advanced(const char* programName)
#ifndef ZSTD_NOCOMPRESS
DISPLAY( "--ultra : enable ultra modes (requires more memory to decompress)\n");
DISPLAY( "--no-dictID:don't write dictID into header (dictionary compression)\n");
DISPLAY( "--check : enable integrity check\n");
#endif
#ifndef ZSTD_NODECOMPRESS
DISPLAY( " -t : test compressed file integrity \n");
DISPLAY( "--test : test compressed file integrity \n");
DISPLAY( "--[no-]sparse : sparse mode (default:enabled on file, disabled on stdout)\n");
#endif
#ifndef ZSTD_NODICT
@@ -240,6 +241,7 @@ int main(int argCount, const char** argv)
if (!strcmp(argument, "--quiet")) { displayLevel--; continue; }
if (!strcmp(argument, "--stdout")) { forceStdout=1; outFileName=stdoutmark; displayLevel=1; continue; }
if (!strcmp(argument, "--ultra")) { FIO_setMaxWLog(0); continue; }
if (!strcmp(argument, "--check")) { FIO_setChecksumFlag(2); continue; }
if (!strcmp(argument, "--no-dictID")) { FIO_setDictIDFlag(0); continue; }
if (!strcmp(argument, "--sparse")) { FIO_setSparseWrite(2); continue; }
if (!strcmp(argument, "--no-sparse")) { FIO_setSparseWrite(0); continue; }
@@ -303,6 +305,9 @@ int main(int argCount, const char** argv)
/* keep source file (default anyway, so useless; for gzip/xz compatibility) */
case 'k': argument++; break;
/* Checksum */
case 'C': argument++; FIO_setChecksumFlag(2); break;
/* test compressed file */
case 't': decode=1; outFileName=nulmark; argument++; break;