ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'refs/remotes/Cyan4973/dev060' into repcodes

Browse Source 
# Conflicts:
#	lib/zstd_decompress.c
#	programs/bench.c
This commit is contained in:
inikep
2016-03-18 11:14:58 +01:00
parent eca2ec0b3b 1d1ae40258
commit 5b15865413
19 changed files with 721 additions and 633 deletions
Download Patch File Download Diff File Expand all files Collapse all files
programs/Makefile
+2 -2
View File
@@ -112,11 +112,11 @@ zstd-frugal: $(ZSTD_FILES) $(ZSTDDIR)/zbuff.c zstdcli.c fileio.c
zstd-small: clean
CFLAGS="-Os -s" $(MAKE) zstd-frugal
fullbench : $(ZSTD_FILES) \
fullbench : $(ZSTD_FILES) $(ZSTDDIR)/zbuff.c \
datagen.c fullbench.c
$(CC) $(FLAGS) $^ -o $@$(EXT)
fullbench32: $(ZSTD_FILES) \
fullbench32: $(ZSTD_FILES) $(ZSTDDIR)/zbuff.c \
datagen.c fullbench.c
$(CC) -m32 $(FLAGS) $^ -o $@$(EXT)
programs/bench.c
+59 -77
View File
@@ -40,11 +40,6 @@
# define _LARGEFILE64_SOURCE
#endif
/* S_ISREG & gettimeofday() are not supported by MSVC */
#if defined(_MSC_VER) || defined(_WIN32)
# define BMK_LEGACY_TIMER 1
#endif
/* *************************************
* Includes
@@ -54,12 +49,17 @@
#include <stdio.h> /* fprintf, fopen, ftello64 */
#include <sys/types.h> /* stat64 */
#include <sys/stat.h> /* stat64 */
#include <time.h> /* clock_t, clock, CLOCKS_PER_SEC */
/* Use ftime() if gettimeofday() is not available */
#if defined(BMK_LEGACY_TIMER)
# include <sys/timeb.h> /* timeb, ftime */
/* sleep : posix - windows - others */
#if !defined(_WIN32) && (defined(__unix__) || defined(__unix) || (defined(__APPLE__) && defined(__MACH__)))
# include <unistd.h>
# define BMK_sleep(s) sleep(s)
#elif defined(_WIN32)
# include <windows.h>
# define BMK_sleep(s) Sleep(1000*s)
#else
# include <sys/time.h> /* gettimeofday */
# define BMK_sleep(s) /* disabled */
#endif
#include "mem.h"
@@ -88,7 +88,10 @@
# define ZSTD_VERSION ""
#endif
#define NBLOOPS 3
#define NBLOOPS 3
#define TIMELOOP_S 1
#define ACTIVEPERIOD_S 70
#define COOLPERIOD_S 10
#define KB *(1 <<10)
#define MB *(1 <<20)
@@ -127,15 +130,15 @@ static U32 g_displayLevel = 2; /* 0 : no display; 1: errors; 2 : + result
/* *************************************
* Benchmark Parameters
***************************************/
static int nbIterations = NBLOOPS;
static U32 g_nbIterations = NBLOOPS;
static size_t g_blockSize = 0;
void BMK_setNotificationLevel(unsigned level) { g_displayLevel=level; }
void BMK_SetNbIterations(int nbLoops)
void BMK_SetNbIterations(unsigned nbLoops)
{
nbIterations = nbLoops;
DISPLAYLEVEL(2, "- %i iterations -\n", nbIterations);
g_nbIterations = nbLoops;
DISPLAYLEVEL(2, "- %i iterations -\n", g_nbIterations);
}
void BMK_SetBlockSize(size_t blockSize)
@@ -148,44 +151,11 @@ void BMK_SetBlockSize(size_t blockSize)
/* ********************************************************
* Private functions
**********************************************************/
#if defined(BMK_LEGACY_TIMER)
static int BMK_GetMilliStart(void)
static clock_t BMK_clockSpan( clock_t clockStart )
{
/* Based on Legacy ftime()
* Rolls over every ~ 12.1 days (0x100000/24/60/60)
* Use GetMilliSpan to correct for rollover */
struct timeb tb;
int nCount;
ftime( &tb );
nCount = (int) (tb.millitm + (tb.time & 0xfffff) * 1000);
return nCount;
return clock() - clockStart; /* works even if overflow, span limited to <= ~30mn */
}
#else
static int BMK_GetMilliStart(void)
{
/* Based on newer gettimeofday()
* Use GetMilliSpan to correct for rollover */
struct timeval tv;
int nCount;
gettimeofday(&tv, NULL);
nCount = (int) (tv.tv_usec/1000 + (tv.tv_sec & 0xfffff) * 1000);
return nCount;
}
#endif
static int BMK_GetMilliSpan( int nTimeStart )
{
int nSpan = BMK_GetMilliStart() - nTimeStart;
if ( nSpan < 0 )
nSpan += 0x100000 * 1000;
return nSpan;
}
static U64 BMK_getFileSize(const char* infilename)
{
@@ -259,14 +229,14 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
/* Init blockTable data */
{
U32 fileNb;
const char* srcPtr = (const char*)srcBuffer;
char* cPtr = (char*)compressedBuffer;
char* resPtr = (char*)resultBuffer;
U32 fileNb;
for (fileNb=0; fileNb<nbFiles; fileNb++) {
size_t remaining = fileSizes[fileNb];
U32 nbBlocksforThisFile = (U32)((remaining + (blockSize-1)) / blockSize);
U32 blockEnd = nbBlocks + nbBlocksforThisFile;
U32 const nbBlocksforThisFile = (U32)((remaining + (blockSize-1)) / blockSize);
U32 const blockEnd = nbBlocks + nbBlocksforThisFile;
for ( ; nbBlocks<blockEnd; nbBlocks++) {
size_t thisBlockSize = MIN(remaining, blockSize);
blockTable[nbBlocks].srcPtr = srcPtr;
@@ -283,32 +253,40 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
/* warmimg up memory */
// int timeloop = additionalParam ? additionalParam : 2500;
int timeloop = 2500;
kSlotNew = additionalParam;
RDG_genBuffer(compressedBuffer, maxCompressedSize, 0.10, 0.50, 1);
/* Bench */
{
int loopNb;
U32 loopNb;
double fastestC = 100000000., fastestD = 100000000.;
double ratio = 0.;
U64 crcCheck = 0;
clock_t coolTime = clock();
DISPLAYLEVEL(2, "\r%79s\r", "");
for (loopNb = 1; loopNb <= nbIterations; loopNb++) {
for (loopNb = 1; loopNb <= (g_nbIterations + !g_nbIterations); loopNb++) {
int nbLoops;
int milliTime;
U32 blockNb;
clock_t clockStart, clockSpan;
clock_t const clockLoop = g_nbIterations ? TIMELOOP_S * CLOCKS_PER_SEC : 10;
/* overheat protection */
if (BMK_clockSpan(coolTime) > ACTIVEPERIOD_S * CLOCKS_PER_SEC) {
DISPLAY("\rcooling down ... \r");
BMK_sleep(COOLPERIOD_S);
coolTime = clock();
}
/* Compression */
DISPLAYLEVEL(2, "%2i-%-17.17s :%10u ->\r", loopNb, displayName, (U32)srcSize);
memset(compressedBuffer, 0xE5, maxCompressedSize);
memset(compressedBuffer, 0xE5, maxCompressedSize); /* warm up and erase result buffer */
nbLoops = 0;
milliTime = BMK_GetMilliStart();
while (BMK_GetMilliStart() == milliTime);
milliTime = BMK_GetMilliStart();
while (BMK_GetMilliSpan(milliTime) < timeloop) {
clockStart = clock();
while (clock() == clockStart);
clockStart = clock();
while (BMK_clockSpan(clockStart) < clockLoop) {
ZSTD_compressBegin_advanced(refCtx, dictBuffer, dictBufferSize, ZSTD_getParams(cLevel, MAX(dictBufferSize, largestBlockSize)));
for (blockNb=0; blockNb<nbBlocks; blockNb++) {
size_t rSize = ZSTD_compress_usingPreparedCCtx(ctx, refCtx,
@@ -319,25 +297,27 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
}
nbLoops++;
}
milliTime = BMK_GetMilliSpan(milliTime);
clockSpan = BMK_clockSpan(clockStart);
cSize = 0;
for (blockNb=0; blockNb<nbBlocks; blockNb++)
cSize += blockTable[blockNb].cSize;
if ((double)milliTime < fastestC*nbLoops) fastestC = (double)milliTime / nbLoops;
if ((double)clockSpan < fastestC*nbLoops) fastestC = (double)clockSpan / nbLoops;
ratio = (double)srcSize / (double)cSize;
DISPLAYLEVEL(2, "%2i-%-17.17s :%10i ->%10i (%5.3f),%6.1f MB/s\r", loopNb, displayName, (int)srcSize, (int)cSize, ratio, (double)srcSize / fastestC / 1000.);
DISPLAYLEVEL(2, "%2i-%-17.17s :%10u ->%10u (%5.3f),%6.1f MB/s\r",
loopNb, displayName, (U32)srcSize, (U32)cSize, ratio,
(double)srcSize / 1000000. / (fastestC / CLOCKS_PER_SEC) );
#if 1
/* Decompression */
memset(resultBuffer, 0xD6, srcSize); /* warm result buffer */
nbLoops = 0;
milliTime = BMK_GetMilliStart();
while (BMK_GetMilliStart() == milliTime);
milliTime = BMK_GetMilliStart();
clockStart = clock();
while (clock() == clockStart);
clockStart = clock();
for ( ; BMK_GetMilliSpan(milliTime) < timeloop; nbLoops++) {
for ( ; BMK_clockSpan(clockStart) < clockLoop; nbLoops++) {
ZSTD_decompressBegin_usingDict(refDCtx, dictBuffer, dictBufferSize);
for (blockNb=0; blockNb<nbBlocks; blockNb++) {
size_t regenSize = ZSTD_decompress_usingPreparedDCtx(dctx, refDCtx,
@@ -351,9 +331,12 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
blockTable[blockNb].resSize = regenSize;
} }
milliTime = BMK_GetMilliSpan(milliTime);
if ((double)milliTime < fastestD*nbLoops) fastestD = (double)milliTime / nbLoops;
DISPLAYLEVEL(2, "%2i-%-17.17s :%10i ->%10i (%5.3f),%6.1f MB/s ,%6.1f MB/s\r", loopNb, displayName, (int)srcSize, (int)cSize, ratio, (double)srcSize / fastestC / 1000., (double)srcSize / fastestD / 1000.);
clockSpan = BMK_clockSpan(clockStart);
if ((double)clockSpan < fastestD*nbLoops) fastestD = (double)clockSpan / nbLoops;
DISPLAYLEVEL(2, "%2i-%-17.17s :%10u ->%10u (%5.3f),%6.1f MB/s ,%6.1f MB/s\r",
loopNb, displayName, (U32)srcSize, (U32)cSize, ratio,
(double)srcSize / 1000000. / (fastestC / CLOCKS_PER_SEC),
(double)srcSize / 1000000. / (fastestD / CLOCKS_PER_SEC) );
/* CRC Checking */
_findError:
@@ -439,7 +422,7 @@ static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize,
memset(&total, 0, sizeof(total));
if (g_displayLevel == 1 && !additionalParam)
DISPLAY("bench %s: input %u bytes, %i iterations, %u KB blocks\n", ZSTD_VERSION, (U32)benchedSize, nbIterations, (U32)(g_blockSize>>10));
DISPLAY("bench %s: input %u bytes, %i iterations, %u KB blocks\n", ZSTD_VERSION, (U32)benchedSize, g_nbIterations, (U32)(g_blockSize>>10));
if (cLevelLast < cLevel) cLevelLast = cLevel;
@@ -480,12 +463,11 @@ static U64 BMK_getTotalFileSize(const char** fileNamesTable, unsigned nbFiles)
static void BMK_loadFiles(void* buffer, size_t bufferSize,
size_t* fileSizes,
const char** fileNamesTable, unsigned nbFiles)
const char** fileNamesTable, unsigned const nbFiles)
{
BYTE* buff = (BYTE*)buffer;
size_t pos = 0;
unsigned n;
unsigned n;
for (n=0; n<nbFiles; n++) {
size_t readSize;
U64 fileSize = BMK_getFileSize(fileNamesTable[n]);
@@ -493,7 +475,7 @@ static void BMK_loadFiles(void* buffer, size_t bufferSize,
if (f==NULL) EXM_THROW(10, "impossible to open file %s", fileNamesTable[n]);
DISPLAYLEVEL(2, "Loading %s... \r", fileNamesTable[n]);
if (fileSize > bufferSize-pos) fileSize = bufferSize-pos;
readSize = fread(buff+pos, 1, (size_t)fileSize, f);
readSize = fread(((char*)buffer)+pos, 1, (size_t)fileSize, f);
if (readSize != (size_t)fileSize) EXM_THROW(11, "could not read %s", fileNamesTable[n]);
pos += readSize;
fileSizes[n] = (size_t)fileSize;
@@ -577,7 +559,7 @@ static void BMK_syntheticTest(int cLevel, int cLevelLast, int additionalParam, d
int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles,
const char* dictFileName, int cLevel, int cLevelLast, int additionalParam)
{
double compressibility = (double)g_compressibilityDefault / 100;
double const compressibility = (double)g_compressibilityDefault / 100;
if (nbFiles == 0)
BMK_syntheticTest(cLevel, cLevelLast, additionalParam, compressibility);
programs/bench.h
+1 -1
View File
@@ -30,7 +30,7 @@ int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles,
const char* dictFileName, int cLevel, int cLevelLast, int additionalParam);
/* Set Parameters */
void BMK_SetNbIterations(int nbLoops);
void BMK_SetNbIterations(unsigned nbLoops);
void BMK_SetBlockSize(size_t blockSize);
void BMK_setNotificationLevel(unsigned level);
programs/fileio.c
+42 -20
View File
@@ -64,7 +64,7 @@
#include <sys/stat.h> /* stat64 */
#include "mem.h"
#include "fileio.h"
#include "zstd_static.h" /* ZSTD_magicNumber */
#include "zstd_static.h" /* ZSTD_magicNumber, ZSTD_frameHeaderSize_max */
#include "zbuff_static.h"
#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1)
@@ -126,6 +126,7 @@
#define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
#define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); }
static U32 g_displayLevel = 2; /* 0 : no display; 1: errors; 2 : + result + interaction + warnings; 3 : + progression; 4 : + information */
void FIO_setNotificationLevel(unsigned level) { g_displayLevel=level; }
#define DISPLAYUPDATE(l, ...) if (g_displayLevel>=l) { \
if ((FIO_GetMilliSpan(g_time) > refreshRate) || (g_displayLevel>=4)) \
@@ -142,7 +143,8 @@ static clock_t g_time = 0;
***************************************/
static U32 g_overwrite = 0;
void FIO_overwriteMode(void) { g_overwrite=1; }
void FIO_setNotificationLevel(unsigned level) { g_displayLevel=level; }
static U32 g_maxWLog = 23;
void FIO_setMaxWLog(unsigned maxWLog) { g_maxWLog = maxWLog; }
/*-*************************************
@@ -239,10 +241,10 @@ static FILE* FIO_openDstFile(const char* dstFileName)
}
/*!FIO_loadFile
* creates a buffer, pointed by *bufferPtr,
* loads "filename" content into it
* up to MAX_DICT_SIZE bytes
/*! FIO_loadFile() :
* creates a buffer, pointed by `*bufferPtr`,
* loads `filename` content into it,
* up to MAX_DICT_SIZE bytes
*/
static size_t FIO_loadFile(void** bufferPtr, const char* fileName)
{
@@ -274,7 +276,7 @@ static size_t FIO_loadFile(void** bufferPtr, const char* fileName)
}
/* **********************************************************************
/*-**********************************************************************
* Compression
************************************************************************/
typedef struct {
@@ -321,7 +323,7 @@ static void FIO_freeCResources(cRess_t ress)
/*! FIO_compressFilename_internal() :
* same as FIO_compressFilename_extRess(), with ress.desFile already opened
* same as FIO_compressFilename_extRess(), with `ress.desFile` already opened.
* @return : 0 : compression completed correctly,
* 1 : missing or pb opening srcFileName
*/
@@ -335,10 +337,14 @@ static int FIO_compressFilename_internal(cRess_t ress,
U64 compressedfilesize = 0;
size_t dictSize = ress.dictBufferSize;
size_t sizeCheck, errorCode;
ZSTD_parameters params;
/* init */
filesize = MAX(FIO_getFileSize(srcFileName),dictSize);
errorCode = ZBUFF_compressInit_advanced(ress.ctx, ress.dictBuffer, ress.dictBufferSize, ZSTD_getParams(cLevel, filesize));
params = ZSTD_getParams(cLevel, filesize);
params.srcSize = filesize;
if (g_maxWLog) if (params.windowLog > g_maxWLog) params.windowLog = g_maxWLog;
errorCode = ZBUFF_compressInit_advanced(ress.ctx, ress.dictBuffer, ress.dictBufferSize, params);
if (ZBUFF_isError(errorCode)) EXM_THROW(21, "Error initializing compression : %s", ZBUFF_getErrorName(errorCode));
/* Main compression loop */
@@ -389,7 +395,7 @@ static int FIO_compressFilename_internal(cRess_t ress,
/*! FIO_compressFilename_internal() :
* same as FIO_compressFilename_extRess(), with ress.desFile already opened
* same as FIO_compressFilename_extRess(), with ress.destFile already opened (typically stdout)
* @return : 0 : compression completed correctly,
* 1 : missing or pb opening srcFileName
*/
@@ -427,6 +433,7 @@ static int FIO_compressFilename_extRess(cRess_t ress,
if (ress.dstFile==0) { fclose(ress.srcFile); return 1; }
result = FIO_compressFilename_internal(ress, dstFileName, srcFileName, cLevel);
if (result!=0) remove(dstFileName); /* remove operation artefact */
fclose(ress.srcFile); /* no pb to expect : only reading */
if (fclose(ress.dstFile)) EXM_THROW(28, "Write error : cannot properly close %s", dstFileName);
@@ -482,7 +489,7 @@ int FIO_compressMultipleFilenames(const char** inFileNamesTable, unsigned nbFile
ress.dstFile = stdout;
for (u=0; u<nbFiles; u++)
missed_files += FIO_compressFilename_srcFile(ress, stdoutmark,
inFileNamesTable[u], compressionLevel);
inFileNamesTable[u], compressionLevel);
if (fclose(ress.dstFile)) EXM_THROW(29, "Write error : cannot properly close %s", stdoutmark);
} else {
for (u=0; u<nbFiles; u++) {
@@ -547,35 +554,46 @@ static void FIO_freeDResources(dRess_t ress)
}
/** FIO_decompressFrame() :
@return : size of decoded frame
*/
unsigned long long FIO_decompressFrame(dRess_t ress,
FILE* foutput, FILE* finput, size_t alreadyLoaded)
{
U64 frameSize = 0;
size_t readSize=alreadyLoaded;
size_t readSize;
ZBUFF_decompressInitDictionary(ress.dctx, ress.dictBuffer, ress.dictBufferSize);
/* Complete Header loading */
{ size_t const toLoad = ZSTD_frameHeaderSize_max - alreadyLoaded; /* assumption : alreadyLoaded <= ZSTD_frameHeaderSize_max */
size_t const checkSize = fread(((char*)ress.srcBuffer) + alreadyLoaded, 1, toLoad, finput);
if (checkSize != toLoad) EXM_THROW(32, "Read error");
}
readSize = ZSTD_frameHeaderSize_max;
/* Main decompression Loop */
ZBUFF_decompressInitDictionary(ress.dctx, ress.dictBuffer, ress.dictBufferSize);
while (1) {
/* Decode */
size_t sizeCheck;
size_t inSize=readSize, decodedSize=ress.dstBufferSize;
size_t toRead = ZBUFF_decompressContinue(ress.dctx, ress.dstBuffer, &decodedSize, ress.srcBuffer, &inSize);
if (ZBUFF_isError(toRead)) EXM_THROW(36, "Decoding error : %s", ZBUFF_getErrorName(toRead));
readSize -= inSize;
/* Write block */
sizeCheck = fwrite(ress.dstBuffer, 1, decodedSize, foutput);
if (sizeCheck != decodedSize) EXM_THROW(37, "Write error : unable to write data block to destination file");
{ size_t const sizeCheck = fwrite(ress.dstBuffer, 1, decodedSize, foutput);
if (sizeCheck != decodedSize) EXM_THROW(37, "Write error : unable to write data block into destination"); }
frameSize += decodedSize;
DISPLAYUPDATE(2, "\rDecoded : %u MB... ", (U32)(frameSize>>20) );
if (toRead == 0) break;
if (toRead == 0) break; /* end of frame */
if (readSize) EXM_THROW(38, "Decoding error : should consume entire input");
/* Fill input buffer */
if (toRead > ress.srcBufferSize) EXM_THROW(34, "too large block");
readSize = fread(ress.srcBuffer, 1, toRead, finput);
if (readSize != toRead) EXM_THROW(35, "Read error");
if (readSize != toRead)
EXM_THROW(35, "Read error");
}
return frameSize;
@@ -633,13 +651,17 @@ static int FIO_decompressSrcFile(dRess_t ress, const char* srcFileName)
static int FIO_decompressFile_extRess(dRess_t ress,
const char* dstFileName, const char* srcFileName)
{
int result;
ress.dstFile = FIO_openDstFile(dstFileName);
if (ress.dstFile==0) return 1;
FIO_decompressSrcFile(ress, srcFileName);
result = FIO_decompressSrcFile(ress, srcFileName);
if (result != 0) {
remove(dstFileName);
}
if (fclose(ress.dstFile)) EXM_THROW(38, "Write error : cannot properly close %s", dstFileName);
return 0;
return result;
}
programs/fileio.h
+4 -3
View File
@@ -41,14 +41,15 @@ extern "C" {
#endif
/* *************************************
/*-*************************************
* Parameters
***************************************/
void FIO_overwriteMode(void);
void FIO_setNotificationLevel(unsigned level);
void FIO_setMaxWLog(unsigned maxWLog); /**< if `maxWLog` == 0, no max enforced */
/* *************************************
/*-*************************************
* Single File functions
***************************************/
/** FIO_compressFilename() :
@@ -60,7 +61,7 @@ int FIO_compressFilename (const char* outfilename, const char* infilename, const
int FIO_decompressFilename (const char* outfilename, const char* infilename, const char* dictFileName);
/* *************************************
/*-*************************************
* Multiple File functions
***************************************/
/** FIO_compressMultipleFilenames() :
programs/fullbench.c
+213 -208
View File
@@ -1,6 +1,6 @@
/*
fullbench.c - Detailed bench program for zstd
Copyright (C) Yann Collet 2014-2015
Copyright (C) Yann Collet 2014-2016
GPL v2 License
@@ -19,11 +19,10 @@
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
You can contact the author at :
- zstd source repository : https://github.com/Cyan4973/zstd
- ztsd public forum : https://groups.google.com/forum/#!forum/lz4c
- zstd homepage : http://www.zstd.net
*/
/**************************************
/*_************************************
* Compiler Options
**************************************/
/* Disable some Visual warning messages */
@@ -38,13 +37,8 @@
# define _LARGEFILE64_SOURCE
#endif
/* S_ISREG & gettimeofday() are not supported by MSVC */
#if defined(_MSC_VER) || defined(_WIN32)
# define BMK_LEGACY_TIMER 1
#endif
/**************************************
/*_************************************
* Includes
**************************************/
#include <stdlib.h> /* malloc */
@@ -52,21 +46,16 @@
#include <sys/types.h> /* stat64 */
#include <sys/stat.h> /* stat64 */
#include <string.h> /* strcmp */
/* Use ftime() if gettimeofday() is not available on your target */
#if defined(BMK_LEGACY_TIMER)
# include <sys/timeb.h> /* timeb, ftime */
#else
# include <sys/time.h> /* gettimeofday */
#endif
#include <time.h> /* clock_t, clock, CLOCKS_PER_SEC */
#include "mem.h"
#include "zstd_static.h"
#include "fse_static.h"
#include "zbuff.h"
#include "datagen.h"
/**************************************
/*_************************************
* Compiler Options
**************************************/
/* S_ISREG & gettimeofday() are not supported by MSVC */
@@ -75,7 +64,7 @@
#endif
/**************************************
/*_************************************
* Constants
**************************************/
#define PROGRAM_DESCRIPTION "Zstandard speed analyzer"
@@ -90,98 +79,63 @@
#define MB *(1<<20)
#define NBLOOPS 6
#define TIMELOOP 2500
#define TIMELOOP_S 2
#define KNUTH 2654435761U
#define MAX_MEM (1984 MB)
#define COMPRESSIBILITY_DEFAULT 0.50
static const size_t sampleSize = 10000000;
static const size_t g_sampleSize = 10000000;
/**************************************
/*_************************************
* Macros
**************************************/
#define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
/**************************************
/*_************************************
* Benchmark Parameters
**************************************/
static int nbIterations = NBLOOPS;
static U32 g_nbIterations = NBLOOPS;
static double g_compressibility = COMPRESSIBILITY_DEFAULT;
void BMK_SetNbIterations(int nbLoops)
static void BMK_SetNbIterations(U32 nbLoops)
{
nbIterations = nbLoops;
DISPLAY("- %i iterations -\n", nbIterations);
g_nbIterations = nbLoops;
DISPLAY("- %i iterations -\n", g_nbIterations);
}
/*********************************************************
/*_*******************************************************
* Private functions
*********************************************************/
#if defined(BMK_LEGACY_TIMER)
static int BMK_GetMilliStart(void)
static clock_t BMK_clockSpan( clock_t clockStart )
{
/* Based on Legacy ftime()
* Rolls over every ~ 12.1 days (0x100000/24/60/60)
* Use GetMilliSpan to correct for rollover */
struct timeb tb;
int nCount;
ftime( &tb );
nCount = (int) (tb.millitm + (tb.time & 0xfffff) * 1000);
return nCount;
}
#else
static int BMK_GetMilliStart(void)
{
/* Based on newer gettimeofday()
* Use GetMilliSpan to correct for rollover */
struct timeval tv;
int nCount;
gettimeofday(&tv, NULL);
nCount = (int) (tv.tv_usec/1000 + (tv.tv_sec & 0xfffff) * 1000);
return nCount;
}
#endif
static int BMK_GetMilliSpan( int nTimeStart )
{
int nSpan = BMK_GetMilliStart() - nTimeStart;
if ( nSpan < 0 )
nSpan += 0x100000 * 1000;
return nSpan;
return clock() - clockStart; /* works even if overflow, span limited to <= ~30mn */
}
static size_t BMK_findMaxMem(U64 requiredMem)
{
size_t step = 64 MB;
BYTE* testmem=NULL;
const size_t step = 64 MB;
void* testmem = NULL;
requiredMem = (((requiredMem >> 26) + 1) << 26);
if (requiredMem > MAX_MEM) requiredMem = MAX_MEM;
requiredMem += 2*step;
while (!testmem)
{
requiredMem += step;
do {
testmem = malloc ((size_t)requiredMem);
requiredMem -= step;
testmem = (BYTE*) malloc ((size_t)requiredMem);
}
} while (!testmem);
free (testmem);
return (size_t) (requiredMem - step);
return (size_t) requiredMem;
}
static U64 BMK_GetFileSize(char* infilename)
static U64 BMK_GetFileSize(const char* infilename)
{
int r;
#if defined(_MSC_VER)
@@ -196,42 +150,39 @@ static U64 BMK_GetFileSize(char* infilename)
}
/*********************************************************
/*_*******************************************************
* Benchmark wrappers
*********************************************************/
typedef enum { bt_compressed, bt_raw, bt_rle, bt_end } blockType_t;
typedef struct
{
typedef struct {
blockType_t blockType;
U32 unusedBits;
U32 origSize;
} blockProperties_t;
static size_t g_cSize = 0;
static ZSTD_DCtx* g_dctxPtr = NULL;
extern size_t ZSTD_getcBlockSize(const void* src, size_t srcSize, blockProperties_t* bpPtr);
extern size_t ZSTD_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* dumpsLengthPtr, FSE_DTable* DTableLL, FSE_DTable* DTableML, FSE_DTable* DTableOffb, const void* src, size_t srcSize);
size_t local_ZSTD_compress(void* dst, size_t dstSize, void* buff2, const void* src, size_t srcSize)
{
(void)buff2;
return ZSTD_compress(dst, dstSize, src, srcSize, 1);
}
static size_t g_cSize = 0;
size_t local_ZSTD_decompress(void* dst, size_t dstSize, void* buff2, const void* src, size_t srcSize)
{
(void)src; (void)srcSize;
return ZSTD_decompress(dst, dstSize, buff2, g_cSize);
}
static ZSTD_DCtx* g_zdc = NULL;
extern size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* ctx, const void* src, size_t srcSize);
size_t local_ZSTD_decodeLiteralsBlock(void* dst, size_t dstSize, void* buff2, const void* src, size_t srcSize)
{
(void)src; (void)srcSize; (void)dst; (void)dstSize;
return ZSTD_decodeLiteralsBlock((ZSTD_DCtx*)g_dctxPtr, buff2, g_cSize);
return ZSTD_decodeLiteralsBlock((ZSTD_DCtx*)g_zdc, buff2, g_cSize);
}
extern size_t ZSTD_getcBlockSize(const void* src, size_t srcSize, blockProperties_t* bpPtr);
extern size_t ZSTD_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* dumpsLengthPtr, FSE_DTable* DTableLL, FSE_DTable* DTableML, FSE_DTable* DTableOffb, const void* src, size_t srcSize);
size_t local_ZSTD_decodeSeqHeaders(void* dst, size_t dstSize, void* buff2, const void* src, size_t srcSize)
{
U32 DTableML[FSE_DTABLE_SIZE_U32(10)], DTableLL[FSE_DTABLE_SIZE_U32(10)], DTableOffb[FSE_DTABLE_SIZE_U32(9)]; /* MLFSELog, LLFSELog and OffFSELog are not public values */
@@ -243,20 +194,76 @@ size_t local_ZSTD_decodeSeqHeaders(void* dst, size_t dstSize, void* buff2, const
}
static ZBUFF_CCtx* g_zbcc = NULL;
size_t local_ZBUFF_compress(void* dst, size_t dstCapacity, void* buff2, const void* src, size_t srcSize)
{
size_t compressedSize;
size_t srcRead = srcSize, dstWritten = dstCapacity;
(void)buff2;
ZBUFF_compressInit(g_zbcc, 1);
ZBUFF_compressContinue(g_zbcc, dst, &dstWritten, src, &srcRead);
compressedSize = dstWritten;
dstWritten = dstCapacity-compressedSize;
ZBUFF_compressEnd(g_zbcc, ((char*)dst)+compressedSize, &dstWritten);
compressedSize += dstWritten;
return compressedSize;
}
/*********************************************************
static ZBUFF_DCtx* g_zbdc = NULL;
static size_t local_ZBUFF_decompress(void* dst, size_t dstCapacity, void* buff2, const void* src, size_t srcSize)
{
size_t srcRead = g_cSize, dstWritten = dstCapacity;
(void)src; (void)srcSize;
ZBUFF_decompressInit(g_zbdc);
ZBUFF_decompressContinue(g_zbdc, dst, &dstWritten, buff2, &srcRead);
return dstWritten;
}
static ZSTD_CCtx* g_zcc = NULL;
size_t local_ZSTD_compressContinue(void* dst, size_t dstCapacity, void* buff2, const void* src, size_t srcSize)
{
size_t compressedSize;
(void)buff2;
ZSTD_compressBegin(g_zcc, 1);
compressedSize = ZSTD_compressContinue(g_zcc, dst, dstCapacity, src, srcSize);
compressedSize += ZSTD_compressEnd(g_zcc, ((char*)dst)+compressedSize, dstCapacity-compressedSize);
return compressedSize;
}
size_t local_ZSTD_decompressContinue(void* dst, size_t dstCapacity, void* buff2, const void* src, size_t srcSize)
{
size_t regeneratedSize = 0;
const BYTE* ip = (const BYTE*)buff2;
const BYTE* const iend = ip + g_cSize;
BYTE* op = (BYTE*)dst;
size_t remainingCapacity = dstCapacity;
(void)src; (void)srcSize;
ZSTD_decompressBegin(g_zdc);
while (ip < iend) {
size_t const iSize = ZSTD_nextSrcSizeToDecompress(g_zdc);
size_t const decodedSize = ZSTD_decompressContinue(g_zdc, op, remainingCapacity, ip, iSize);
ip += iSize;
regeneratedSize += decodedSize;
op += decodedSize;
remainingCapacity -= decodedSize;
}
return regeneratedSize;
}
/*_*******************************************************
* Bench functions
*********************************************************/
size_t benchMem(void* src, size_t srcSize, U32 benchNb)
static size_t benchMem(const void* src, size_t srcSize, U32 benchNb)
{
BYTE* dstBuff;
size_t dstBuffSize;
BYTE* buff2;
int loopNb;
const char* benchName;
size_t (*benchFunction)(void* dst, size_t dstSize, void* verifBuff, const void* src, size_t srcSize);
double bestTime = 100000000.;
size_t errorCode = 0;
/* Selection */
switch(benchNb)
@@ -264,15 +271,27 @@ size_t benchMem(void* src, size_t srcSize, U32 benchNb)
case 1:
benchFunction = local_ZSTD_compress; benchName = "ZSTD_compress";
break;
case 11:
case 2:
benchFunction = local_ZSTD_decompress; benchName = "ZSTD_decompress";
break;
case 11:
benchFunction = local_ZSTD_compressContinue; benchName = "ZSTD_compressContinue";
break;
case 12:
benchFunction = local_ZSTD_decompressContinue; benchName = "ZSTD_decompressContinue";
break;
case 31:
benchFunction = local_ZSTD_decodeLiteralsBlock; benchName = "ZSTD_decodeLiteralsBlock";
break;
case 32:
benchFunction = local_ZSTD_decodeSeqHeaders; benchName = "ZSTD_decodeSeqHeaders";
break;
case 41:
benchFunction = local_ZBUFF_compress; benchName = "ZBUFF_compressContinue";
break;
case 42:
benchFunction = local_ZBUFF_decompress; benchName = "ZBUFF_decompressContinue";
break;
default :
return 0;
}
@@ -281,9 +300,7 @@ size_t benchMem(void* src, size_t srcSize, U32 benchNb)
dstBuffSize = ZSTD_compressBound(srcSize);
dstBuff = (BYTE*)malloc(dstBuffSize);
buff2 = (BYTE*)malloc(dstBuffSize);
g_dctxPtr = ZSTD_createDCtx();
if ((!dstBuff) || (!buff2))
{
if ((!dstBuff) || (!buff2)) {
DISPLAY("\nError: not enough memory!\n");
free(dstBuff); free(buff2);
return 12;
@@ -292,45 +309,66 @@ size_t benchMem(void* src, size_t srcSize, U32 benchNb)
/* Preparation */
switch(benchNb)
{
case 11:
case 2:
g_cSize = ZSTD_compress(buff2, dstBuffSize, src, srcSize, 1);
break;
case 11 :
if (g_zcc==NULL) g_zcc = ZSTD_createCCtx();
break;
case 12 :
if (g_zdc==NULL) g_zdc = ZSTD_createDCtx();
g_cSize = ZSTD_compress(buff2, dstBuffSize, src, srcSize, 1);
break;
case 31: /* ZSTD_decodeLiteralsBlock */
{
blockProperties_t bp;
if (g_zdc==NULL) g_zdc = ZSTD_createDCtx();
{ blockProperties_t bp;
ZSTD_frameParams zfp;
size_t frameHeaderSize, skippedSize;
g_cSize = ZSTD_compress(dstBuff, dstBuffSize, src, srcSize, 1);
ZSTD_getcBlockSize(dstBuff+4, dstBuffSize, &bp); /* Get 1st block type */
if (bp.blockType != bt_compressed)
{
frameHeaderSize = ZSTD_getFrameParams(&zfp, dstBuff, ZSTD_frameHeaderSize_min);
if (frameHeaderSize==0) frameHeaderSize = ZSTD_frameHeaderSize_min;
ZSTD_getcBlockSize(dstBuff+frameHeaderSize, dstBuffSize, &bp); /* Get 1st block type */
if (bp.blockType != bt_compressed) {
DISPLAY("ZSTD_decodeLiteralsBlock : impossible to test on this sample (not compressible)\n");
goto _cleanOut;
}
memcpy(buff2, dstBuff+8, g_cSize-8);
skippedSize = frameHeaderSize + 3 /* ZSTD_blockHeaderSize */;
memcpy(buff2, dstBuff+skippedSize, g_cSize-skippedSize);
srcSize = srcSize > 128 KB ? 128 KB : srcSize; /* speed relative to block */
break;
}
case 32: /* ZSTD_decodeSeqHeaders */
{
blockProperties_t bp;
if (g_zdc==NULL) g_zdc = ZSTD_createDCtx();
{ blockProperties_t bp;
ZSTD_frameParams zfp;
const BYTE* ip = dstBuff;
const BYTE* iend;
size_t blockSize;
size_t frameHeaderSize, cBlockSize;
ZSTD_compress(dstBuff, dstBuffSize, src, srcSize, 1); /* it would be better to use direct block compression here */
ip += 5; /* Skip frame Header */
blockSize = ZSTD_getcBlockSize(ip, dstBuffSize, &bp); /* Get 1st block type */
if (bp.blockType != bt_compressed)
{
g_cSize = ZSTD_compress(dstBuff, dstBuffSize, src, srcSize, 1);
frameHeaderSize = ZSTD_getFrameParams(&zfp, dstBuff, ZSTD_frameHeaderSize_min);
if (frameHeaderSize==0) frameHeaderSize = ZSTD_frameHeaderSize_min;
ip += frameHeaderSize; /* Skip frame Header */
cBlockSize = ZSTD_getcBlockSize(ip, dstBuffSize, &bp); /* Get 1st block type */
if (bp.blockType != bt_compressed) {
DISPLAY("ZSTD_decodeSeqHeaders : impossible to test on this sample (not compressible)\n");
goto _cleanOut;
}
iend = ip + 3 + blockSize; /* End of first block */
ip += 3; /* skip block header */
ip += ZSTD_decodeLiteralsBlock(g_dctxPtr, ip, iend-ip); /* skip literal segment */
iend = ip + 3 /* ZSTD_blockHeaderSize */ + cBlockSize; /* End of first block */
ip += 3 /* ZSTD_blockHeaderSize */; /* skip block header */
ip += ZSTD_decodeLiteralsBlock(g_zdc, ip, iend-ip); /* skip literal segment */
g_cSize = iend-ip;
memcpy(buff2, ip, g_cSize); /* copy rest of block (it starts by SeqHeader) */
srcSize = srcSize > 128 KB ? 128 KB : srcSize; /* speed relative to block */
break;
}
case 41 :
if (g_zbcc==NULL) g_zbcc = ZBUFF_createCCtx();
break;
case 42 :
if (g_zbdc==NULL) g_zbdc = ZBUFF_createDCtx();
g_cSize = ZSTD_compress(buff2, dstBuffSize, src, srcSize, 1);
break;
/* test functions */
/* by convention, test functions can be added > 100 */
@@ -340,53 +378,44 @@ size_t benchMem(void* src, size_t srcSize, U32 benchNb)
{ size_t i; for (i=0; i<dstBuffSize; i++) dstBuff[i]=(BYTE)i; } /* warming up memory */
for (loopNb = 1; loopNb <= nbIterations; loopNb++)
{
{ U32 loopNb;
for (loopNb = 1; loopNb <= g_nbIterations; loopNb++) {
clock_t const timeLoop = TIMELOOP_S * CLOCKS_PER_SEC;
clock_t clockStart;
U32 nbRounds;
size_t benchResult=0;
double averageTime;
int milliTime;
U32 nbRounds=0;
DISPLAY("%2i- %-30.30s : \r", loopNb, benchName);
milliTime = BMK_GetMilliStart();
while(BMK_GetMilliStart() == milliTime);
milliTime = BMK_GetMilliStart();
while(BMK_GetMilliSpan(milliTime) < TIMELOOP)
{
errorCode = benchFunction(dstBuff, dstBuffSize, buff2, src, srcSize);
if (ZSTD_isError(errorCode)) { DISPLAY("ERROR ! %s() => %s !! \n", benchName, ZSTD_getErrorName(errorCode)); exit(1); }
nbRounds++;
clockStart = clock();
while (clock() == clockStart);
clockStart = clock();
for (nbRounds=0; BMK_clockSpan(clockStart) < timeLoop; nbRounds++) {
benchResult = benchFunction(dstBuff, dstBuffSize, buff2, src, srcSize);
if (ZSTD_isError(benchResult)) { DISPLAY("ERROR ! %s() => %s !! \n", benchName, ZSTD_getErrorName(benchResult)); exit(1); }
}
milliTime = BMK_GetMilliSpan(milliTime);
averageTime = (double)milliTime / nbRounds;
averageTime = (((double)BMK_clockSpan(clockStart)) / CLOCKS_PER_SEC) / nbRounds;
if (averageTime < bestTime) bestTime = averageTime;
DISPLAY("%2i- %-30.30s : %7.1f MB/s (%9u)\r", loopNb, benchName, (double)srcSize / bestTime / 1000., (U32)errorCode);
}
DISPLAY("%2u- %-30.30s : %7.1f MB/s (%9u)\n", benchNb, benchName, (double)srcSize / bestTime / 1000., (U32)errorCode);
DISPLAY("%2i- %-30.30s : %7.1f MB/s (%9u)\r", loopNb, benchName, (double)srcSize / (1 MB) / bestTime, (U32)benchResult);
}}
DISPLAY("%2u\n", benchNb);
_cleanOut:
free(dstBuff);
free(buff2);
ZSTD_freeDCtx(g_dctxPtr);
return 0;
}
int benchSample(U32 benchNb)
static int benchSample(U32 benchNb)
{
char* origBuff;
size_t benchedSize = sampleSize;
size_t const benchedSize = g_sampleSize;
const char* name = "Sample 10MiB";
/* Allocation */
origBuff = (char*) malloc((size_t)benchedSize);
if(!origBuff)
{
DISPLAY("\nError: not enough memory!\n");
return 12;
}
void* origBuff = malloc(benchedSize);
if (!origBuff) { DISPLAY("\nError: not enough memory!\n"); return 12; }
/* Fill buffer */
RDG_genBuffer(origBuff, benchedSize, g_compressibility, 0.0, 0);
@@ -404,58 +433,41 @@ int benchSample(U32 benchNb)
}
int benchFiles(char** fileNamesTable, int nbFiles, U32 benchNb)
static int benchFiles(const char** fileNamesTable, const int nbFiles, U32 benchNb)
{
int fileIdx=0;
/* Loop for each file */
while (fileIdx<nbFiles)
{
FILE* inFile;
char* inFileName;
int fileIdx;
for (fileIdx=0; fileIdx<nbFiles; fileIdx++) {
const char* inFileName = fileNamesTable[fileIdx];
FILE* inFile = fopen( inFileName, "rb" );
U64 inFileSize;
size_t benchedSize;
size_t readSize;
char* origBuff;
void* origBuff;
/* Check file existence */
inFileName = fileNamesTable[fileIdx++];
inFile = fopen( inFileName, "rb" );
if (inFile==NULL)
{
DISPLAY( "Pb opening %s\n", inFileName);
return 11;
}
if (inFile==NULL) { DISPLAY( "Pb opening %s\n", inFileName); return 11; }
/* Memory allocation & restrictions */
inFileSize = BMK_GetFileSize(inFileName);
benchedSize = (size_t) BMK_findMaxMem(inFileSize*3) / 3;
benchedSize = BMK_findMaxMem(inFileSize*3) / 3;
if ((U64)benchedSize > inFileSize) benchedSize = (size_t)inFileSize;
if (benchedSize < inFileSize)
{
DISPLAY("Not enough memory for '%s' full size; testing %i MB only...\n", inFileName, (int)(benchedSize>>20));
}
DISPLAY("Not enough memory for '%s' full size; testing %u MB only...\n", inFileName, (U32)(benchedSize>>20));
/* Alloc */
origBuff = (char*) malloc((size_t)benchedSize);
if(!origBuff)
{
DISPLAY("\nError: not enough memory!\n");
fclose(inFile);
return 12;
}
origBuff = malloc(benchedSize);
if (!origBuff) { DISPLAY("\nError: not enough memory!\n"); fclose(inFile); return 12; }
/* Fill input buffer */
DISPLAY("Loading %s... \r", inFileName);
readSize = fread(origBuff, 1, benchedSize, inFile);
fclose(inFile);
if(readSize != benchedSize)
{
DISPLAY("\nError: problem reading file '%s' !! \n", inFileName);
free(origBuff);
return 13;
}
size_t readSize = fread(origBuff, 1, benchedSize, inFile);
fclose(inFile);
if (readSize != benchedSize) {
DISPLAY("\nError: problem reading file '%s' !! \n", inFileName);
free(origBuff);
return 13;
} }
/* bench */
DISPLAY("\r%79s\r", "");
@@ -464,13 +476,15 @@ int benchFiles(char** fileNamesTable, int nbFiles, U32 benchNb)
benchMem(origBuff, benchedSize, benchNb);
else
for (benchNb=0; benchNb<100; benchNb++) benchMem(origBuff, benchedSize, benchNb);
free(origBuff);
}
return 0;
}
int usage(char* exename)
static int usage(const char* exename)
{
DISPLAY( "Usage :\n");
DISPLAY( " %s [arg] file1 file2 ... fileX\n", exename);
@@ -479,8 +493,9 @@ int usage(char* exename)
return 0;
}
int usage_advanced(void)
static int usage_advanced(const char* exename)
{
usage(exename);
DISPLAY( "\nAdvanced options :\n");
DISPLAY( " -b# : test only function # \n");
DISPLAY( " -i# : iteration loops [1-9](default : %i)\n", NBLOOPS);
@@ -488,45 +503,38 @@ int usage_advanced(void)
return 0;
}
int badusage(char* exename)
static int badusage(const char* exename)
{
DISPLAY("Wrong parameters\n");
usage(exename);
return 0;
return 1;
}
int main(int argc, char** argv)
int main(int argc, const char** argv)
{
int i,
filenamesStart=0,
result;
char* exename=argv[0];
char* input_filename=0;
int i, filenamesStart=0, result;
const char* exename = argv[0];
const char* input_filename = NULL;
U32 benchNb = 0, main_pause = 0;
// Welcome message
DISPLAY(WELCOME_MESSAGE);
if (argc<1) return badusage(exename);
if (argc<1) { badusage(exename); return 1; }
for(i=1; i<argc; i++) {
const char* argument = argv[i];
if(!argument) continue; /* Protection if argument empty */
for(i=1; i<argc; i++)
{
char* argument = argv[i];
/* Commands (note : aggregated commands are allowed) */
if (argument[0]=='-') {
if(!argument) continue; // Protection if argument empty
// Decode command (note : aggregated commands are allowed)
if (argument[0]=='-')
{
while (argument[1]!=0)
{
argument ++;
while (argument[1]!=0) {
argument++;
switch(argument[0])
{
/* Display help on usage */
case 'h' :
case 'H': usage(exename); usage_advanced(); return 0;
case 'H': return usage_advanced(exename);
/* Pause at the end (hidden option) */
case 'p': main_pause = 1; break;
@@ -534,8 +542,7 @@ int main(int argc, char** argv)
/* Select specific algorithm to bench */
case 'b':
benchNb = 0;
while ((argument[1]>= '0') && (argument[1]<= '9'))
{
while ((argument[1]>= '0') && (argument[1]<= '9')) {
benchNb *= 10;
benchNb += argument[1] - '0';
argument++;
@@ -544,20 +551,17 @@ int main(int argc, char** argv)
/* Modify Nb Iterations */
case 'i':
if ((argument[1] >='0') && (argument[1] <='9'))
{
if ((argument[1] >='0') && (argument[1] <='9')) {
int iters = argument[1] - '0';
BMK_SetNbIterations(iters);
argument++;
}
break;
/* Select specific algorithm to bench */
/* Select compressibility of synthetic sample */
case 'P':
{
U32 proba32 = 0;
while ((argument[1]>= '0') && (argument[1]<= '9'))
{
{ U32 proba32 = 0;
while ((argument[1]>= '0') && (argument[1]<= '9')) {
proba32 *= 10;
proba32 += argument[1] - '0';
argument++;
@@ -567,7 +571,7 @@ int main(int argc, char** argv)
break;
/* Unknown command */
default : badusage(exename); return 1;
default : return badusage(exename);
}
}
continue;
@@ -577,9 +581,10 @@ int main(int argc, char** argv)
if (!input_filename) { input_filename=argument; filenamesStart=i; continue; }
}
if (filenamesStart==0)
if (filenamesStart==0) /* no input file */
result = benchSample(benchNb);
else result = benchFiles(argv+filenamesStart, argc-filenamesStart, benchNb);
else
result = benchFiles(argv+filenamesStart, argc-filenamesStart, benchNb);
if (main_pause) { int unused; printf("press enter...\n"); unused = getchar(); (void)unused; }
programs/fuzzer.c
+8 -2
View File
@@ -513,8 +513,7 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibilit
CHECK(ZSTD_isError(cSize), "ZSTD_compressCCtx failed");
/* compression failure test : too small dest buffer */
if (cSize > 3)
{
if (cSize > 3) {
const size_t missing = (FUZ_rand(&lseed) % (cSize-2)) + 1; /* no problem, as cSize > 4 (frameHeaderSizer) */
const size_t tooSmallSize = cSize - missing;
static const U32 endMark = 0x4DC2B1A9;
@@ -526,6 +525,13 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibilit
CHECK(endCheck != endMark, "ZSTD_compressCCtx : dst buffer overflow");
}
/* decompression header test */
{ ZSTD_frameParams dParams;
size_t const check = ZSTD_getFrameParams(&dParams, cBuffer, cSize);
CHECK(ZSTD_isError(check), "Frame Parameters extraction failed");
CHECK(dParams.frameContentSize != sampleSize, "Frame content size incorrect");
}
/* successfull decompression tests*/
dSupSize = (FUZ_rand(&lseed) & 1) ? 0 : (FUZ_rand(&lseed) & 31) + 1;
dSize = ZSTD_decompress(dstBuffer, sampleSize + dSupSize, cBuffer, cSize);
programs/playTests.sh
+4 -1
View File
@@ -36,7 +36,10 @@ $ZSTD --decompress tmpCompressed --stdout > tmpResult
$ZSTD -q tmp && die "overwrite check failed!"
$ZSTD -q -f tmp
$ZSTD -q --force tmp
$ZSTD -df tmp && die "should have refused : wrong extension"
cp tmp tmp2.zst
$ZSTD -df tmp2.zst && die "should have failed : wrong format"
rm tmp2.zst
echo "\n**** frame concatenation **** "
programs/zstdcli.c
+2
View File
@@ -132,6 +132,7 @@ static int usage_advanced(const char* programName)
DISPLAY( " -v : verbose mode\n");
DISPLAY( " -q : suppress warnings; specify twice to suppress errors too\n");
DISPLAY( " -c : force write to standard output, even if it is the console\n");
DISPLAY( "--ultra : enable ultra modes (requires more memory to decompress)\n");
#ifndef ZSTD_NODICT
DISPLAY( "Dictionary builder :\n");
DISPLAY( "--train : create a dictionary from a training set of files \n");
@@ -220,6 +221,7 @@ int main(int argCount, const char** argv)
if (!strcmp(argument, "--train")) { dictBuild=1; outFileName=g_defaultDictName; continue; }
if (!strcmp(argument, "--maxdict")) { nextArgumentIsMaxDict=1; continue; }
if (!strcmp(argument, "--keep")) { continue; } /* does nothing, since preserving input is default; for gzip/xz compatibility */
if (!strcmp(argument, "--ultra")) { FIO_setMaxWLog(0); continue; }
/* '-' means stdin/stdout */
if (!strcmp(argument, "-")){