ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

bench: fixed Visual warning regarding struct initialization

Browse Source 
also :
removed dependency on zstdmt_compress.h
removed several unused macros
fileio : small code refactoring to reduce some variable scope
This commit is contained in:
Yann Collet
2017-10-19 11:56:14 -07:00
parent d3b9547aa4
commit eac42534fe
2 changed files with 93 additions and 104 deletions
Download Patch File Download Diff File Expand all files Collapse all files
programs/bench.c
+26 -25
View File
@@ -41,7 +41,6 @@
#include "zstd.h" #include "zstd.h"
#include "datagen.h" /* RDG_genBuffer */ #include "datagen.h" /* RDG_genBuffer */
#include "xxhash.h" #include "xxhash.h"
#include "zstdmt_compress.h"
/* ************************************* /* *************************************
@@ -189,7 +188,6 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
size_t const maxCompressedSize = ZSTD_compressBound(srcSize) + (maxNbBlocks * 1024); /* add some room for safety */ size_t const maxCompressedSize = ZSTD_compressBound(srcSize) + (maxNbBlocks * 1024); /* add some room for safety */
void* const compressedBuffer = malloc(maxCompressedSize); void* const compressedBuffer = malloc(maxCompressedSize);
void* resultBuffer = malloc(srcSize); void* resultBuffer = malloc(srcSize);
ZSTDMT_CCtx* const mtctx = ZSTDMT_createCCtx(g_nbThreads);
ZSTD_CCtx* const ctx = ZSTD_createCCtx(); ZSTD_CCtx* const ctx = ZSTD_createCCtx();
ZSTD_DCtx* const dctx = ZSTD_createDCtx(); ZSTD_DCtx* const dctx = ZSTD_createDCtx();
size_t const loadedCompressedSize = srcSize; size_t const loadedCompressedSize = srcSize;
@@ -286,7 +284,6 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
if (!cCompleted) { /* still some time to do compression tests */ if (!cCompleted) { /* still some time to do compression tests */
U64 const clockLoop = g_nbSeconds ? TIMELOOP_MICROSEC : 1; U64 const clockLoop = g_nbSeconds ? TIMELOOP_MICROSEC : 1;
U32 nbLoops = 0; U32 nbLoops = 0;
ZSTD_CDict* cdict = NULL;
ZSTD_CCtx_setParameter(ctx, ZSTD_p_nbThreads, g_nbThreads); ZSTD_CCtx_setParameter(ctx, ZSTD_p_nbThreads, g_nbThreads);
ZSTD_CCtx_setParameter(ctx, ZSTD_p_compressionLevel, cLevel); ZSTD_CCtx_setParameter(ctx, ZSTD_p_compressionLevel, cLevel);
ZSTD_CCtx_setParameter(ctx, ZSTD_p_enableLongDistanceMatching, g_ldmFlag); ZSTD_CCtx_setParameter(ctx, ZSTD_p_enableLongDistanceMatching, g_ldmFlag);
@@ -308,27 +305,31 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
do { do {
U32 blockNb; U32 blockNb;
for (blockNb=0; blockNb<nbBlocks; blockNb++) { for (blockNb=0; blockNb<nbBlocks; blockNb++) {
size_t rSize; size_t moreToFlush = 1;
ZSTD_outBuffer out = { blockTable[blockNb].cPtr, blockTable[blockNb].cRoom, 0 }; ZSTD_outBuffer out;
ZSTD_inBuffer in = { blockTable[blockNb].srcPtr, blockTable[blockNb].srcSize, 0 }; ZSTD_inBuffer in;
size_t cError = 1; in.src = blockTable[blockNb].srcPtr;
while (cError) { in.size = blockTable[blockNb].srcSize;
cError = ZSTD_compress_generic(ctx, in.pos = 0;
out.dst = blockTable[blockNb].cPtr;
out.size = blockTable[blockNb].cRoom;
out.pos = 0;
while (moreToFlush) {
moreToFlush = ZSTD_compress_generic(ctx,
&out, &in, ZSTD_e_end); &out, &in, ZSTD_e_end);
if (ZSTD_isError(cError)) if (ZSTD_isError(moreToFlush))
EXM_THROW(1, "ZSTD_compress_generic() error : %s", EXM_THROW(1, "ZSTD_compress_generic() error : %s",
ZSTD_getErrorName(cError)); ZSTD_getErrorName(moreToFlush));
} }
rSize = out.pos; blockTable[blockNb].cSize = out.pos;
blockTable[blockNb].cSize = rSize;
} }
nbLoops++; nbLoops++;
} while (UTIL_clockSpanMicro(clockStart) < clockLoop); } while (UTIL_clockSpanMicro(clockStart) < clockLoop);
ZSTD_freeCDict(cdict); { U64 const loopDuration = UTIL_clockSpanMicro(clockStart);
{ U64 const clockSpanMicro = UTIL_clockSpanMicro(clockStart); if (loopDuration < fastestC*nbLoops)
if (clockSpanMicro < fastestC*nbLoops) fastestC = clockSpanMicro / nbLoops; fastestC = loopDuration / nbLoops;
totalCTime += clockSpanMicro; totalCTime += loopDuration;
cCompleted = (totalCTime >= maxTime); cCompleted = (totalCTime >= maxTime); /* end compression tests */
} } } }
cSize = 0; cSize = 0;
@@ -336,15 +337,15 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
ratio = (double)srcSize / (double)cSize; ratio = (double)srcSize / (double)cSize;
markNb = (markNb+1) % NB_MARKS; markNb = (markNb+1) % NB_MARKS;
DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.3f),%6.1f MB/s\r", DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.3f),%6.1f MB/s\r",
marks[markNb], displayName, (U32)srcSize, (U32)cSize, ratio, marks[markNb], displayName, (U32)srcSize, (U32)cSize,
(double)srcSize / fastestC ); ratio, (double)srcSize / fastestC );
} else { /* g_decodeOnly */ } else { /* g_decodeOnly */
memcpy(compressedBuffer, srcBuffer, loadedCompressedSize); memcpy(compressedBuffer, srcBuffer, loadedCompressedSize);
} }
#if 0 /* disable decompression test */ #if 0 /* disable decompression test */
dCompleted=1; dCompleted=1;
(void)totalDTime; (void)fastestD; (void)crcOrig; /* unused when decompression disabled */ (void)totalDTime; (void)fastestD; (void)crcOrig; /* unused when decompression disabled */
#else #else
/* Decompression */ /* Decompression */
if (!dCompleted) memset(resultBuffer, 0xD6, srcSize); /* warm result buffer */ if (!dCompleted) memset(resultBuffer, 0xD6, srcSize); /* warm result buffer */
@@ -374,9 +375,10 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
nbLoops++; nbLoops++;
} while (UTIL_clockSpanMicro(clockStart) < clockLoop); } while (UTIL_clockSpanMicro(clockStart) < clockLoop);
ZSTD_freeDDict(ddict); ZSTD_freeDDict(ddict);
{ U64 const clockSpanMicro = UTIL_clockSpanMicro(clockStart); { U64 const loopDuration = UTIL_clockSpanMicro(clockStart);
if (clockSpanMicro < fastestD*nbLoops) fastestD = clockSpanMicro / nbLoops; if (loopDuration < fastestD*nbLoops)
totalDTime += clockSpanMicro; fastestD = loopDuration / nbLoops;
totalDTime += loopDuration;
dCompleted = (totalDTime >= maxTime); dCompleted = (totalDTime >= maxTime);
} } } }
@@ -439,7 +441,6 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
free(blockTable); free(blockTable);
free(compressedBuffer); free(compressedBuffer);
free(resultBuffer); free(resultBuffer);
ZSTDMT_freeCCtx(mtctx);
ZSTD_freeCCtx(ctx); ZSTD_freeCCtx(ctx);
ZSTD_freeDCtx(dctx); ZSTD_freeDCtx(dctx);
return 0; return 0;
programs/fileio.c
+67 -79
View File
@@ -42,9 +42,6 @@
#include "fileio.h" #include "fileio.h"
#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_magicNumber, ZSTD_frameHeaderSize_max */ #define ZSTD_STATIC_LINKING_ONLY /* ZSTD_magicNumber, ZSTD_frameHeaderSize_max */
#include "zstd.h" #include "zstd.h"
#ifdef ZSTD_MULTITHREAD
# include "zstdmt_compress.h"
#endif
#if defined(ZSTD_GZCOMPRESS) || defined(ZSTD_GZDECOMPRESS) #if defined(ZSTD_GZCOMPRESS) || defined(ZSTD_GZDECOMPRESS)
# include <zlib.h> # include <zlib.h>
# if !defined(z_const) # if !defined(z_const)
@@ -70,18 +67,6 @@
#define MB *(1<<20) #define MB *(1<<20)
#define GB *(1U<<30) #define GB *(1U<<30)
#define _1BIT 0x01
#define _2BITS 0x03
#define _3BITS 0x07
#define _4BITS 0x0F
#define _6BITS 0x3F
#define _8BITS 0xFF
#define BLOCKSIZE (128 KB)
#define ROLLBUFFERSIZE (BLOCKSIZE*8*64)
#define FIO_FRAMEHEADERSIZE 5 /* as a define, because needed to allocated table on stack */
#define DICTSIZE_MAX (32 MB) /* protection against large input (attack scenario) */ #define DICTSIZE_MAX (32 MB) /* protection against large input (attack scenario) */
#define FNSPACE 30 #define FNSPACE 30
@@ -122,14 +107,14 @@ static clock_t g_time = 0;
# define ZSTD_DEBUG 0 # define ZSTD_DEBUG 0
#endif #endif
#define DEBUGLOG(l,...) if (l<=ZSTD_DEBUG) DISPLAY(__VA_ARGS__); #define DEBUGLOG(l,...) if (l<=ZSTD_DEBUG) DISPLAY(__VA_ARGS__);
#define EXM_THROW(error, ...) \ #define EXM_THROW(error, ...) \
{ \ { \
DISPLAYLEVEL(1, "zstd: "); \ DISPLAYLEVEL(1, "zstd: "); \
DEBUGLOG(1, "Error defined at %s, line %i : \n", __FILE__, __LINE__); \ DEBUGLOG(1, "Error defined at %s, line %i : \n", __FILE__, __LINE__); \
DISPLAYLEVEL(1, "error %i : ", error); \ DISPLAYLEVEL(1, "error %i : ", error); \
DISPLAYLEVEL(1, __VA_ARGS__); \ DISPLAYLEVEL(1, __VA_ARGS__); \
DISPLAYLEVEL(1, " \n"); \ DISPLAYLEVEL(1, " \n"); \
exit(error); \ exit(error); \
} }
#define CHECK(f) { \ #define CHECK(f) { \
@@ -273,86 +258,88 @@ static int FIO_remove(const char* path)
} }
/** FIO_openSrcFile() : /** FIO_openSrcFile() :
* condition : `dstFileName` must be non-NULL. * condition : `srcFileName` must be non-NULL.
* @result : FILE* to `dstFileName`, or NULL if it fails */ * @result : FILE* to `srcFileName`, or NULL if it fails */
static FILE* FIO_openSrcFile(const char* srcFileName) static FILE* FIO_openSrcFile(const char* srcFileName)
{ {
FILE* f; assert(srcFileName != NULL);
if (!strcmp (srcFileName, stdinmark)) { if (!strcmp (srcFileName, stdinmark)) {
DISPLAYLEVEL(4,"Using stdin for input\n"); DISPLAYLEVEL(4,"Using stdin for input\n");
f = stdin;
SET_BINARY_MODE(stdin); SET_BINARY_MODE(stdin);
} else { return stdin;
if (!UTIL_isRegularFile(srcFileName)) {
DISPLAYLEVEL(1, "zstd: %s is not a regular file -- ignored \n",
srcFileName);
return NULL;
}
f = fopen(srcFileName, "rb");
if ( f==NULL )
DISPLAYLEVEL(1, "zstd: %s: %s \n", srcFileName, strerror(errno));
} }
return f; if (!UTIL_isRegularFile(srcFileName)) {
DISPLAYLEVEL(1, "zstd: %s is not a regular file -- ignored \n",
srcFileName);
return NULL;
}
{ FILE* const f = fopen(srcFileName, "rb");
if (f == NULL)
DISPLAYLEVEL(1, "zstd: %s: %s \n", srcFileName, strerror(errno));
return f;
}
} }
/** FIO_openDstFile() : /** FIO_openDstFile() :
* condition : `dstFileName` must be non-NULL. * condition : `dstFileName` must be non-NULL.
* @result : FILE* to `dstFileName`, or NULL if it fails */ * @result : FILE* to `dstFileName`, or NULL if it fails */
static FILE* FIO_openDstFile(const char* dstFileName) static FILE* FIO_openDstFile(const char* dstFileName)
{ {
FILE* f; assert(dstFileName != NULL);
if (!strcmp (dstFileName, stdoutmark)) { if (!strcmp (dstFileName, stdoutmark)) {
DISPLAYLEVEL(4,"Using stdout for output\n"); DISPLAYLEVEL(4,"Using stdout for output\n");
f = stdout;
SET_BINARY_MODE(stdout); SET_BINARY_MODE(stdout);
if (g_sparseFileSupport==1) { if (g_sparseFileSupport==1) {
g_sparseFileSupport = 0; g_sparseFileSupport = 0;
DISPLAYLEVEL(4, "Sparse File Support is automatically disabled on stdout ; try --sparse \n"); DISPLAYLEVEL(4, "Sparse File Support is automatically disabled on stdout ; try --sparse \n");
} }
} else { return stdout;
if (g_sparseFileSupport == 1) {
g_sparseFileSupport = ZSTD_SPARSE_DEFAULT;
}
if (strcmp (dstFileName, nulmark)) {
/* Check if destination file already exists */
f = fopen( dstFileName, "rb" );
if (f != 0) { /* dst file exists, prompt for overwrite authorization */
fclose(f);
if (!g_overwrite) {
if (g_displayLevel <= 1) {
/* No interaction possible */
DISPLAY("zstd: %s already exists; not overwritten \n",
dstFileName);
return NULL;
}
DISPLAY("zstd: %s already exists; do you wish to overwrite (y/N) ? ",
dstFileName);
{ int ch = getchar();
if ((ch!='Y') && (ch!='y')) {
DISPLAY(" not overwritten \n");
return NULL;
}
/* flush rest of input line */
while ((ch!=EOF) && (ch!='\n')) ch = getchar();
} }
/* need to unlink */
FIO_remove(dstFileName);
} }
f = fopen( dstFileName, "wb" );
if (f==NULL) DISPLAYLEVEL(1, "zstd: %s: %s\n", dstFileName, strerror(errno));
} }
return f; if (g_sparseFileSupport == 1) {
g_sparseFileSupport = ZSTD_SPARSE_DEFAULT;
}
if (strcmp (dstFileName, nulmark)) { /* not /dev/null */
/* Check if destination file already exists */
FILE* const fCheck = fopen( dstFileName, "rb" );
if (fCheck != NULL) { /* dst file exists, authorization prompt */
fclose(fCheck);
if (!g_overwrite) {
if (g_displayLevel <= 1) {
/* No interaction possible */
DISPLAY("zstd: %s already exists; not overwritten \n",
dstFileName);
return NULL;
}
DISPLAY("zstd: %s already exists; overwrite (y/N) ? ",
dstFileName);
{ int ch = getchar();
if ((ch!='Y') && (ch!='y')) {
DISPLAY(" not overwritten \n");
return NULL;
}
/* flush rest of input line */
while ((ch!=EOF) && (ch!='\n')) ch = getchar();
} }
/* need to unlink */
FIO_remove(dstFileName);
} }
{ FILE* const f = fopen( dstFileName, "wb" );
if (f == NULL)
DISPLAYLEVEL(1, "zstd: %s: %s\n", dstFileName, strerror(errno));
return f;
}
} }
/*! FIO_createDictBuffer() : /*! FIO_createDictBuffer() :
* creates a buffer, pointed by `*bufferPtr`, * creates a buffer, pointed by `*bufferPtr`,
* loads `filename` content into it, up to DICTSIZE_MAX bytes. * loads `filename` content into it, up to DICTSIZE_MAX bytes.
* @return : loaded size * @return : loaded size
* if fileName==NULL, returns 0 and a NULL pointer * if fileName==NULL, returns 0 and a NULL pointer
*/ */
static size_t FIO_createDictBuffer(void** bufferPtr, const char* fileName) static size_t FIO_createDictBuffer(void** bufferPtr, const char* fileName)
@@ -360,12 +347,13 @@ static size_t FIO_createDictBuffer(void** bufferPtr, const char* fileName)
FILE* fileHandle; FILE* fileHandle;
U64 fileSize; U64 fileSize;
assert(bufferPtr != NULL);
*bufferPtr = NULL; *bufferPtr = NULL;
if (fileName == NULL) return 0; if (fileName == NULL) return 0;
DISPLAYLEVEL(4,"Loading %s as dictionary \n", fileName); DISPLAYLEVEL(4,"Loading %s as dictionary \n", fileName);
fileHandle = fopen(fileName, "rb"); fileHandle = fopen(fileName, "rb");
if (fileHandle==0) EXM_THROW(31, "%s: %s", fileName, strerror(errno)); if (fileHandle==NULL) EXM_THROW(31, "%s: %s", fileName, strerror(errno));
fileSize = UTIL_getFileSize(fileName); fileSize = UTIL_getFileSize(fileName);
if (fileSize > DICTSIZE_MAX) { if (fileSize > DICTSIZE_MAX) {
EXM_THROW(32, "Dictionary file %s is too large (> %u MB)", EXM_THROW(32, "Dictionary file %s is too large (> %u MB)",
@@ -412,13 +400,13 @@ static cRess_t FIO_createCResources(const char* dictFileName, int cLevel,
if (!ress.srcBuffer || !ress.dstBuffer) if (!ress.srcBuffer || !ress.dstBuffer)
EXM_THROW(31, "allocation error : not enough memory"); EXM_THROW(31, "allocation error : not enough memory");
/* dictionary */ /* Advances parameters, including dictionary */
{ void* dictBuffer; { void* dictBuffer;
size_t const dictBuffSize = FIO_createDictBuffer(&dictBuffer, dictFileName); /* works with dictFileName==NULL */ size_t const dictBuffSize = FIO_createDictBuffer(&dictBuffer, dictFileName); /* works with dictFileName==NULL */
if (dictFileName && (dictBuffer==NULL)) if (dictFileName && (dictBuffer==NULL))
EXM_THROW(32, "allocation error : can't create dictBuffer"); EXM_THROW(32, "allocation error : can't create dictBuffer");
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_contentSizeFlag, 1) ); /* always enable content size, when available (note: supposed to be default anyway) */ CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_contentSizeFlag, 1) ); /* always enable content size when available (note: supposed to be default) */
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_dictIDFlag, g_dictIDFlag) ); CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_dictIDFlag, g_dictIDFlag) );
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_checksumFlag, g_checksumFlag) ); CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_checksumFlag, g_checksumFlag) );
/* compression level */ /* compression level */
@@ -446,7 +434,7 @@ static cRess_t FIO_createCResources(const char* dictFileName, int cLevel,
DISPLAYLEVEL(5,"set nb threads = %u \n", g_nbThreads); DISPLAYLEVEL(5,"set nb threads = %u \n", g_nbThreads);
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_nbThreads, g_nbThreads) ); CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_nbThreads, g_nbThreads) );
/* dictionary */ /* dictionary */
CHECK( ZSTD_CCtx_setPledgedSrcSize(ress.cctx, srcSize) ); /* just to load dictionary with good compression parameters */ CHECK( ZSTD_CCtx_setPledgedSrcSize(ress.cctx, srcSize) ); /* just for dictionary loading, using good compression parameters */
CHECK( ZSTD_CCtx_loadDictionary(ress.cctx, dictBuffer, dictBuffSize) ); CHECK( ZSTD_CCtx_loadDictionary(ress.cctx, dictBuffer, dictBuffSize) );
CHECK( ZSTD_CCtx_setPledgedSrcSize(ress.cctx, ZSTD_CONTENTSIZE_UNKNOWN) ); /* reset */ CHECK( ZSTD_CCtx_setPledgedSrcSize(ress.cctx, ZSTD_CONTENTSIZE_UNKNOWN) ); /* reset */