ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

bench: fixed Visual warning regarding struct initialization

Browse Source 
also :
removed dependency on zstdmt_compress.h
removed several unused macros
fileio : small code refactoring to reduce some variable scope
This commit is contained in:
Yann Collet
2017-10-19 11:56:14 -07:00
parent d3b9547aa4
commit eac42534fe
2 changed files with 93 additions and 104 deletions
Download Patch File Download Diff File Expand all files Collapse all files
programs/bench.c
+26 -25
View File
@@ -41,7 +41,6 @@
#include "zstd.h"
#include "datagen.h" /* RDG_genBuffer */
#include "xxhash.h"
#include "zstdmt_compress.h"
/* *************************************
@@ -189,7 +188,6 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
size_t const maxCompressedSize = ZSTD_compressBound(srcSize) + (maxNbBlocks * 1024); /* add some room for safety */
void* const compressedBuffer = malloc(maxCompressedSize);
void* resultBuffer = malloc(srcSize);
ZSTDMT_CCtx* const mtctx = ZSTDMT_createCCtx(g_nbThreads);
ZSTD_CCtx* const ctx = ZSTD_createCCtx();
ZSTD_DCtx* const dctx = ZSTD_createDCtx();
size_t const loadedCompressedSize = srcSize;
@@ -286,7 +284,6 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
if (!cCompleted) { /* still some time to do compression tests */
U64 const clockLoop = g_nbSeconds ? TIMELOOP_MICROSEC : 1;
U32 nbLoops = 0;
ZSTD_CDict* cdict = NULL;
ZSTD_CCtx_setParameter(ctx, ZSTD_p_nbThreads, g_nbThreads);
ZSTD_CCtx_setParameter(ctx, ZSTD_p_compressionLevel, cLevel);
ZSTD_CCtx_setParameter(ctx, ZSTD_p_enableLongDistanceMatching, g_ldmFlag);
@@ -308,27 +305,31 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
do {
U32 blockNb;
for (blockNb=0; blockNb<nbBlocks; blockNb++) {
size_t rSize;
ZSTD_outBuffer out = { blockTable[blockNb].cPtr, blockTable[blockNb].cRoom, 0 };
ZSTD_inBuffer in = { blockTable[blockNb].srcPtr, blockTable[blockNb].srcSize, 0 };
size_t cError = 1;
while (cError) {
cError = ZSTD_compress_generic(ctx,
size_t moreToFlush = 1;
ZSTD_outBuffer out;
ZSTD_inBuffer in;
in.src = blockTable[blockNb].srcPtr;
in.size = blockTable[blockNb].srcSize;
in.pos = 0;
out.dst = blockTable[blockNb].cPtr;
out.size = blockTable[blockNb].cRoom;
out.pos = 0;
while (moreToFlush) {
moreToFlush = ZSTD_compress_generic(ctx,
&out, &in, ZSTD_e_end);
if (ZSTD_isError(cError))
if (ZSTD_isError(moreToFlush))
EXM_THROW(1, "ZSTD_compress_generic() error : %s",
ZSTD_getErrorName(cError));
ZSTD_getErrorName(moreToFlush));
}
rSize = out.pos;
blockTable[blockNb].cSize = rSize;
blockTable[blockNb].cSize = out.pos;
}
nbLoops++;
} while (UTIL_clockSpanMicro(clockStart) < clockLoop);
ZSTD_freeCDict(cdict);
{ U64 const clockSpanMicro = UTIL_clockSpanMicro(clockStart);
if (clockSpanMicro < fastestC*nbLoops) fastestC = clockSpanMicro / nbLoops;
totalCTime += clockSpanMicro;
cCompleted = (totalCTime >= maxTime);
{ U64 const loopDuration = UTIL_clockSpanMicro(clockStart);
if (loopDuration < fastestC*nbLoops)
fastestC = loopDuration / nbLoops;
totalCTime += loopDuration;
cCompleted = (totalCTime >= maxTime); /* end compression tests */
} }
cSize = 0;
@@ -336,15 +337,15 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
ratio = (double)srcSize / (double)cSize;
markNb = (markNb+1) % NB_MARKS;
DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.3f),%6.1f MB/s\r",
marks[markNb], displayName, (U32)srcSize, (U32)cSize, ratio,
(double)srcSize / fastestC );
marks[markNb], displayName, (U32)srcSize, (U32)cSize,
ratio, (double)srcSize / fastestC );
} else { /* g_decodeOnly */
memcpy(compressedBuffer, srcBuffer, loadedCompressedSize);
}
#if 0 /* disable decompression test */
dCompleted=1;
(void)totalDTime; (void)fastestD; (void)crcOrig; /* unused when decompression disabled */
(void)totalDTime; (void)fastestD; (void)crcOrig; /* unused when decompression disabled */
#else
/* Decompression */
if (!dCompleted) memset(resultBuffer, 0xD6, srcSize); /* warm result buffer */
@@ -374,9 +375,10 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
nbLoops++;
} while (UTIL_clockSpanMicro(clockStart) < clockLoop);
ZSTD_freeDDict(ddict);
{ U64 const clockSpanMicro = UTIL_clockSpanMicro(clockStart);
if (clockSpanMicro < fastestD*nbLoops) fastestD = clockSpanMicro / nbLoops;
totalDTime += clockSpanMicro;
{ U64 const loopDuration = UTIL_clockSpanMicro(clockStart);
if (loopDuration < fastestD*nbLoops)
fastestD = loopDuration / nbLoops;
totalDTime += loopDuration;
dCompleted = (totalDTime >= maxTime);
} }
@@ -439,7 +441,6 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
free(blockTable);
free(compressedBuffer);
free(resultBuffer);
ZSTDMT_freeCCtx(mtctx);
ZSTD_freeCCtx(ctx);
ZSTD_freeDCtx(dctx);
return 0;
programs/fileio.c
+67 -79
View File
@@ -42,9 +42,6 @@
#include "fileio.h"
#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_magicNumber, ZSTD_frameHeaderSize_max */
#include "zstd.h"
#ifdef ZSTD_MULTITHREAD
# include "zstdmt_compress.h"
#endif
#if defined(ZSTD_GZCOMPRESS) || defined(ZSTD_GZDECOMPRESS)
# include <zlib.h>
# if !defined(z_const)
@@ -70,18 +67,6 @@
#define MB *(1<<20)
#define GB *(1U<<30)
#define _1BIT 0x01
#define _2BITS 0x03
#define _3BITS 0x07
#define _4BITS 0x0F
#define _6BITS 0x3F
#define _8BITS 0xFF
#define BLOCKSIZE (128 KB)
#define ROLLBUFFERSIZE (BLOCKSIZE*8*64)
#define FIO_FRAMEHEADERSIZE 5 /* as a define, because needed to allocated table on stack */
#define DICTSIZE_MAX (32 MB) /* protection against large input (attack scenario) */
#define FNSPACE 30
@@ -122,14 +107,14 @@ static clock_t g_time = 0;
# define ZSTD_DEBUG 0
#endif
#define DEBUGLOG(l,...) if (l<=ZSTD_DEBUG) DISPLAY(__VA_ARGS__);
#define EXM_THROW(error, ...) \
{ \
DISPLAYLEVEL(1, "zstd: "); \
DEBUGLOG(1, "Error defined at %s, line %i : \n", __FILE__, __LINE__); \
DISPLAYLEVEL(1, "error %i : ", error); \
DISPLAYLEVEL(1, __VA_ARGS__); \
DISPLAYLEVEL(1, " \n"); \
exit(error); \
#define EXM_THROW(error, ...) \
{ \
DISPLAYLEVEL(1, "zstd: "); \
DEBUGLOG(1, "Error defined at %s, line %i : \n", __FILE__, __LINE__); \
DISPLAYLEVEL(1, "error %i : ", error); \
DISPLAYLEVEL(1, __VA_ARGS__); \
DISPLAYLEVEL(1, " \n"); \
exit(error); \
}
#define CHECK(f) { \
@@ -273,86 +258,88 @@ static int FIO_remove(const char* path)
}
/** FIO_openSrcFile() :
* condition : `dstFileName` must be non-NULL.
* @result : FILE* to `dstFileName`, or NULL if it fails */
* condition : `srcFileName` must be non-NULL.
* @result : FILE* to `srcFileName`, or NULL if it fails */
static FILE* FIO_openSrcFile(const char* srcFileName)
{
FILE* f;
assert(srcFileName != NULL);
if (!strcmp (srcFileName, stdinmark)) {
DISPLAYLEVEL(4,"Using stdin for input\n");
f = stdin;
SET_BINARY_MODE(stdin);
} else {
if (!UTIL_isRegularFile(srcFileName)) {
DISPLAYLEVEL(1, "zstd: %s is not a regular file -- ignored \n",
srcFileName);
return NULL;
}
f = fopen(srcFileName, "rb");
if ( f==NULL )
DISPLAYLEVEL(1, "zstd: %s: %s \n", srcFileName, strerror(errno));
return stdin;
}
return f;
if (!UTIL_isRegularFile(srcFileName)) {
DISPLAYLEVEL(1, "zstd: %s is not a regular file -- ignored \n",
srcFileName);
return NULL;
}
{ FILE* const f = fopen(srcFileName, "rb");
if (f == NULL)
DISPLAYLEVEL(1, "zstd: %s: %s \n", srcFileName, strerror(errno));
return f;
}
}
/** FIO_openDstFile() :
* condition : `dstFileName` must be non-NULL.
* condition : `dstFileName` must be non-NULL.
* @result : FILE* to `dstFileName`, or NULL if it fails */
static FILE* FIO_openDstFile(const char* dstFileName)
{
FILE* f;
assert(dstFileName != NULL);
if (!strcmp (dstFileName, stdoutmark)) {
DISPLAYLEVEL(4,"Using stdout for output\n");
f = stdout;
SET_BINARY_MODE(stdout);
if (g_sparseFileSupport==1) {
g_sparseFileSupport = 0;
DISPLAYLEVEL(4, "Sparse File Support is automatically disabled on stdout ; try --sparse \n");
}
} else {
if (g_sparseFileSupport == 1) {
g_sparseFileSupport = ZSTD_SPARSE_DEFAULT;
}
if (strcmp (dstFileName, nulmark)) {
/* Check if destination file already exists */
f = fopen( dstFileName, "rb" );
if (f != 0) { /* dst file exists, prompt for overwrite authorization */
fclose(f);
if (!g_overwrite) {
if (g_displayLevel <= 1) {
/* No interaction possible */
DISPLAY("zstd: %s already exists; not overwritten \n",
dstFileName);
return NULL;
}
DISPLAY("zstd: %s already exists; do you wish to overwrite (y/N) ? ",
dstFileName);
{ int ch = getchar();
if ((ch!='Y') && (ch!='y')) {
DISPLAY(" not overwritten \n");
return NULL;
}
/* flush rest of input line */
while ((ch!=EOF) && (ch!='\n')) ch = getchar();
} }
/* need to unlink */
FIO_remove(dstFileName);
} }
f = fopen( dstFileName, "wb" );
if (f==NULL) DISPLAYLEVEL(1, "zstd: %s: %s\n", dstFileName, strerror(errno));
return stdout;
}
return f;
if (g_sparseFileSupport == 1) {
g_sparseFileSupport = ZSTD_SPARSE_DEFAULT;
}
if (strcmp (dstFileName, nulmark)) { /* not /dev/null */
/* Check if destination file already exists */
FILE* const fCheck = fopen( dstFileName, "rb" );
if (fCheck != NULL) { /* dst file exists, authorization prompt */
fclose(fCheck);
if (!g_overwrite) {
if (g_displayLevel <= 1) {
/* No interaction possible */
DISPLAY("zstd: %s already exists; not overwritten \n",
dstFileName);
return NULL;
}
DISPLAY("zstd: %s already exists; overwrite (y/N) ? ",
dstFileName);
{ int ch = getchar();
if ((ch!='Y') && (ch!='y')) {
DISPLAY(" not overwritten \n");
return NULL;
}
/* flush rest of input line */
while ((ch!=EOF) && (ch!='\n')) ch = getchar();
} }
/* need to unlink */
FIO_remove(dstFileName);
} }
{ FILE* const f = fopen( dstFileName, "wb" );
if (f == NULL)
DISPLAYLEVEL(1, "zstd: %s: %s\n", dstFileName, strerror(errno));
return f;
}
}
/*! FIO_createDictBuffer() :
* creates a buffer, pointed by `*bufferPtr`,
* loads `filename` content into it, up to DICTSIZE_MAX bytes.
* @return : loaded size
* @return : loaded size
* if fileName==NULL, returns 0 and a NULL pointer
*/
static size_t FIO_createDictBuffer(void** bufferPtr, const char* fileName)
@@ -360,12 +347,13 @@ static size_t FIO_createDictBuffer(void** bufferPtr, const char* fileName)
FILE* fileHandle;
U64 fileSize;
assert(bufferPtr != NULL);
*bufferPtr = NULL;
if (fileName == NULL) return 0;
DISPLAYLEVEL(4,"Loading %s as dictionary \n", fileName);
fileHandle = fopen(fileName, "rb");
if (fileHandle==0) EXM_THROW(31, "%s: %s", fileName, strerror(errno));
if (fileHandle==NULL) EXM_THROW(31, "%s: %s", fileName, strerror(errno));
fileSize = UTIL_getFileSize(fileName);
if (fileSize > DICTSIZE_MAX) {
EXM_THROW(32, "Dictionary file %s is too large (> %u MB)",
@@ -412,13 +400,13 @@ static cRess_t FIO_createCResources(const char* dictFileName, int cLevel,
if (!ress.srcBuffer || !ress.dstBuffer)
EXM_THROW(31, "allocation error : not enough memory");
/* dictionary */
/* Advances parameters, including dictionary */
{ void* dictBuffer;
size_t const dictBuffSize = FIO_createDictBuffer(&dictBuffer, dictFileName); /* works with dictFileName==NULL */
if (dictFileName && (dictBuffer==NULL))
EXM_THROW(32, "allocation error : can't create dictBuffer");
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_contentSizeFlag, 1) ); /* always enable content size, when available (note: supposed to be default anyway) */
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_contentSizeFlag, 1) ); /* always enable content size when available (note: supposed to be default) */
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_dictIDFlag, g_dictIDFlag) );
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_checksumFlag, g_checksumFlag) );
/* compression level */
@@ -446,7 +434,7 @@ static cRess_t FIO_createCResources(const char* dictFileName, int cLevel,
DISPLAYLEVEL(5,"set nb threads = %u \n", g_nbThreads);
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_nbThreads, g_nbThreads) );
/* dictionary */
CHECK( ZSTD_CCtx_setPledgedSrcSize(ress.cctx, srcSize) ); /* just to load dictionary with good compression parameters */
CHECK( ZSTD_CCtx_setPledgedSrcSize(ress.cctx, srcSize) ); /* just for dictionary loading, using good compression parameters */
CHECK( ZSTD_CCtx_loadDictionary(ress.cctx, dictBuffer, dictBuffSize) );
CHECK( ZSTD_CCtx_setPledgedSrcSize(ress.cctx, ZSTD_CONTENTSIZE_UNKNOWN) ); /* reset */