From d3b9547aa4cab7421946ef4010a998de34703da8 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 18 Oct 2017 17:01:53 -0700 Subject: [PATCH 1/3] IO and bench : ZSTD_NEWAPI is the only remaining code path removed the other 2 code paths (single thread, and ZSTDMT ones) keeping only the new advanced API, for easier code coverage. It shall also fix identified issue with Visual Studio which doesn't have ZSTD_NEWAPI defined. --- programs/Makefile | 3 +- programs/bench.c | 39 -------------- programs/fileio.c | 135 +++++++++++----------------------------------- 3 files changed, 33 insertions(+), 144 deletions(-) diff --git a/programs/Makefile b/programs/Makefile index 1b21281b4..e1c5f50b6 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -37,8 +37,7 @@ endif CPPFLAGS+= -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/compress \ -I$(ZSTDDIR)/dictBuilder \ - -DXXH_NAMESPACE=ZSTD_ \ - -DZSTD_NEWAPI + -DXXH_NAMESPACE=ZSTD_ CFLAGS ?= -O3 DEBUGFLAGS+=-Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \ -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \ diff --git a/programs/bench.c b/programs/bench.c index efec1892a..5b67d5c76 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -287,7 +287,6 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, U64 const clockLoop = g_nbSeconds ? TIMELOOP_MICROSEC : 1; U32 nbLoops = 0; ZSTD_CDict* cdict = NULL; -#ifdef ZSTD_NEWAPI ZSTD_CCtx_setParameter(ctx, ZSTD_p_nbThreads, g_nbThreads); ZSTD_CCtx_setParameter(ctx, ZSTD_p_compressionLevel, cLevel); ZSTD_CCtx_setParameter(ctx, ZSTD_p_enableLongDistanceMatching, g_ldmFlag); @@ -306,25 +305,10 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, ZSTD_CCtx_setParameter(ctx, ZSTD_p_targetLength, comprParams->targetLength); ZSTD_CCtx_setParameter(ctx, ZSTD_p_compressionStrategy, comprParams->strategy); ZSTD_CCtx_loadDictionary(ctx, dictBuffer, dictBufferSize); -#else - size_t const avgSize = MIN(blockSize, (srcSize / nbFiles)); - ZSTD_parameters zparams = ZSTD_getParams(cLevel, avgSize, dictBufferSize); - ZSTD_customMem const cmem = { NULL, NULL, NULL }; - if (comprParams->windowLog) zparams.cParams.windowLog = comprParams->windowLog; - if (comprParams->chainLog) zparams.cParams.chainLog = comprParams->chainLog; - if (comprParams->hashLog) zparams.cParams.hashLog = comprParams->hashLog; - if (comprParams->searchLog) zparams.cParams.searchLog = comprParams->searchLog; - if (comprParams->searchLength) zparams.cParams.searchLength = comprParams->searchLength; - if (comprParams->targetLength) zparams.cParams.targetLength = comprParams->targetLength; - if (comprParams->strategy) zparams.cParams.strategy = comprParams->strategy; - cdict = ZSTD_createCDict_advanced(dictBuffer, dictBufferSize, ZSTD_dlm_byRef, ZSTD_dm_auto, zparams.cParams, cmem); - if (cdict==NULL) EXM_THROW(1, "ZSTD_createCDict_advanced() allocation failure"); -#endif do { U32 blockNb; for (blockNb=0; blockNbwindowLog) ); - CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_chainLog, comprParams->chainLog) ); - CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_hashLog, comprParams->hashLog) ); - CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_searchLog, comprParams->searchLog) ); - CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_minMatch, comprParams->searchLength) ); - CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_targetLength, comprParams->targetLength) ); - CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_compressionStrategy, (U32)comprParams->strategy) ); - /* multi-threading */ - DISPLAYLEVEL(5,"set nb threads = %u \n", g_nbThreads); - CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_nbThreads, g_nbThreads) ); - /* dictionary */ - CHECK( ZSTD_CCtx_setPledgedSrcSize(ress.cctx, srcSize) ); /* just to load dictionary with good compression parameters */ - CHECK( ZSTD_CCtx_loadDictionary(ress.cctx, dictBuffer, dictBuffSize) ); - CHECK( ZSTD_CCtx_setPledgedSrcSize(ress.cctx, ZSTD_CONTENTSIZE_UNKNOWN) ); /* reset */ + CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_contentSizeFlag, 1) ); /* always enable content size, when available (note: supposed to be default anyway) */ + CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_dictIDFlag, g_dictIDFlag) ); + CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_checksumFlag, g_checksumFlag) ); + /* compression level */ + CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_compressionLevel, cLevel) ); + /* long distance matching */ + CHECK( ZSTD_CCtx_setParameter( + ress.cctx, ZSTD_p_enableLongDistanceMatching, g_ldmFlag) ); + CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_ldmHashLog, g_ldmHashLog) ); + CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_ldmMinMatch, g_ldmMinMatch) ); + if (g_ldmBucketSizeLog != FIO_LDM_PARAM_NOTSET) { + CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_ldmBucketSizeLog, g_ldmBucketSizeLog) ); } -#elif defined(ZSTD_MULTITHREAD) - { ZSTD_parameters params = ZSTD_getParams(cLevel, srcSize, dictBuffSize); - params.fParams.checksumFlag = g_checksumFlag; - params.fParams.noDictIDFlag = !g_dictIDFlag; - if (comprParams->windowLog) params.cParams.windowLog = comprParams->windowLog; - if (comprParams->chainLog) params.cParams.chainLog = comprParams->chainLog; - if (comprParams->hashLog) params.cParams.hashLog = comprParams->hashLog; - if (comprParams->searchLog) params.cParams.searchLog = comprParams->searchLog; - if (comprParams->searchLength) params.cParams.searchLength = comprParams->searchLength; - if (comprParams->targetLength) params.cParams.targetLength = comprParams->targetLength; - if (comprParams->strategy) params.cParams.strategy = (ZSTD_strategy) comprParams->strategy; - CHECK( ZSTDMT_initCStream_advanced(ress.cctx, dictBuffer, dictBuffSize, params, srcSize) ); - ZSTDMT_setMTCtxParameter(ress.cctx, ZSTDMT_p_sectionSize, g_blockSize); + if (g_ldmHashEveryLog != FIO_LDM_PARAM_NOTSET) { + CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_ldmHashEveryLog, g_ldmHashEveryLog) ); } -#else - { ZSTD_parameters params = ZSTD_getParams(cLevel, srcSize, dictBuffSize); - params.fParams.checksumFlag = g_checksumFlag; - params.fParams.noDictIDFlag = !g_dictIDFlag; - if (comprParams->windowLog) params.cParams.windowLog = comprParams->windowLog; - if (comprParams->chainLog) params.cParams.chainLog = comprParams->chainLog; - if (comprParams->hashLog) params.cParams.hashLog = comprParams->hashLog; - if (comprParams->searchLog) params.cParams.searchLog = comprParams->searchLog; - if (comprParams->searchLength) params.cParams.searchLength = comprParams->searchLength; - if (comprParams->targetLength) params.cParams.targetLength = comprParams->targetLength; - if (comprParams->strategy) params.cParams.strategy = (ZSTD_strategy) comprParams->strategy; - CHECK( ZSTD_initCStream_advanced(ress.cctx, dictBuffer, dictBuffSize, params, srcSize) ); - } -#endif + /* compression parameters */ + CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_windowLog, comprParams->windowLog) ); + CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_chainLog, comprParams->chainLog) ); + CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_hashLog, comprParams->hashLog) ); + CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_searchLog, comprParams->searchLog) ); + CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_minMatch, comprParams->searchLength) ); + CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_targetLength, comprParams->targetLength) ); + CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_compressionStrategy, (U32)comprParams->strategy) ); + /* multi-threading */ + DISPLAYLEVEL(5,"set nb threads = %u \n", g_nbThreads); + CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_nbThreads, g_nbThreads) ); + /* dictionary */ + CHECK( ZSTD_CCtx_setPledgedSrcSize(ress.cctx, srcSize) ); /* just to load dictionary with good compression parameters */ + CHECK( ZSTD_CCtx_loadDictionary(ress.cctx, dictBuffer, dictBuffSize) ); + CHECK( ZSTD_CCtx_setPledgedSrcSize(ress.cctx, ZSTD_CONTENTSIZE_UNKNOWN) ); /* reset */ + free(dictBuffer); } @@ -509,11 +460,7 @@ static void FIO_freeCResources(cRess_t ress) { free(ress.srcBuffer); free(ress.dstBuffer); -#if !defined(ZSTD_NEWAPI) && defined(ZSTD_MULTITHREAD) - ZSTDMT_freeCCtx(ress.cctx); -#else ZSTD_freeCStream(ress.cctx); /* never fails */ -#endif } @@ -814,14 +761,8 @@ static int FIO_compressFilename_internal(cRess_t ress, } /* init */ -#ifdef ZSTD_NEWAPI - if (fileSize!=UTIL_FILESIZE_UNKNOWN) /* when src is stdin, fileSize==0, but is effectively unknown */ + if (fileSize != UTIL_FILESIZE_UNKNOWN) ZSTD_CCtx_setPledgedSrcSize(ress.cctx, fileSize); -#elif defined(ZSTD_MULTITHREAD) - CHECK( ZSTDMT_resetCStream(ress.cctx, (fileSize==UTIL_FILESIZE_UNKNOWN) ? ZSTD_CONTENTSIZE_UNKNOWN : fileSize) ); -#else - CHECK( ZSTD_resetCStream(ress.cctx, (fileSize==UTIL_FILESIZE_UNKNOWN) ? ZSTD_CONTENTSIZE_UNKNOWN : fileSize) ); -#endif /* Main compression loop */ while (1) { @@ -833,14 +774,8 @@ static int FIO_compressFilename_internal(cRess_t ress, while (inBuff.pos != inBuff.size) { ZSTD_outBuffer outBuff = { ress.dstBuffer, ress.dstBufferSize, 0 }; -#ifdef ZSTD_NEWAPI CHECK( ZSTD_compress_generic(ress.cctx, &outBuff, &inBuff, ZSTD_e_continue) ); -#elif defined(ZSTD_MULTITHREAD) - CHECK( ZSTDMT_compressStream(ress.cctx, &outBuff, &inBuff) ); -#else - CHECK( ZSTD_compressStream(ress.cctx, &outBuff, &inBuff) ); -#endif /* Write compressed stream */ if (outBuff.pos) { @@ -871,21 +806,15 @@ static int FIO_compressFilename_internal(cRess_t ress, { size_t result = 1; while (result != 0) { ZSTD_outBuffer outBuff = { ress.dstBuffer, ress.dstBufferSize, 0 }; -#ifdef ZSTD_NEWAPI ZSTD_inBuffer inBuff = { NULL, 0, 0 }; result = ZSTD_compress_generic(ress.cctx, &outBuff, &inBuff, ZSTD_e_end); -#elif defined(ZSTD_MULTITHREAD) - result = ZSTDMT_endStream(ress.cctx, &outBuff); -#else - result = ZSTD_endStream(ress.cctx, &outBuff); -#endif if (ZSTD_isError(result)) { EXM_THROW(26, "Compression error during frame end : %s", ZSTD_getErrorName(result)); } { size_t const sizeCheck = fwrite(ress.dstBuffer, 1, outBuff.pos, dstFile); - if (sizeCheck!=outBuff.pos) + if (sizeCheck != outBuff.pos) EXM_THROW(27, "Write error : cannot write frame end into %s", dstFileName); } compressedfilesize += outBuff.pos; From eac42534fe4f631606c2d604d5db0b6613cf01cd Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Thu, 19 Oct 2017 11:56:14 -0700 Subject: [PATCH 2/3] bench: fixed Visual warning regarding struct initialization also : removed dependency on zstdmt_compress.h removed several unused macros fileio : small code refactoring to reduce some variable scope --- programs/bench.c | 51 ++++++++-------- programs/fileio.c | 146 +++++++++++++++++++++------------------------- 2 files changed, 93 insertions(+), 104 deletions(-) diff --git a/programs/bench.c b/programs/bench.c index 5b67d5c76..8ecae9ebd 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -41,7 +41,6 @@ #include "zstd.h" #include "datagen.h" /* RDG_genBuffer */ #include "xxhash.h" -#include "zstdmt_compress.h" /* ************************************* @@ -189,7 +188,6 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, size_t const maxCompressedSize = ZSTD_compressBound(srcSize) + (maxNbBlocks * 1024); /* add some room for safety */ void* const compressedBuffer = malloc(maxCompressedSize); void* resultBuffer = malloc(srcSize); - ZSTDMT_CCtx* const mtctx = ZSTDMT_createCCtx(g_nbThreads); ZSTD_CCtx* const ctx = ZSTD_createCCtx(); ZSTD_DCtx* const dctx = ZSTD_createDCtx(); size_t const loadedCompressedSize = srcSize; @@ -286,7 +284,6 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, if (!cCompleted) { /* still some time to do compression tests */ U64 const clockLoop = g_nbSeconds ? TIMELOOP_MICROSEC : 1; U32 nbLoops = 0; - ZSTD_CDict* cdict = NULL; ZSTD_CCtx_setParameter(ctx, ZSTD_p_nbThreads, g_nbThreads); ZSTD_CCtx_setParameter(ctx, ZSTD_p_compressionLevel, cLevel); ZSTD_CCtx_setParameter(ctx, ZSTD_p_enableLongDistanceMatching, g_ldmFlag); @@ -308,27 +305,31 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, do { U32 blockNb; for (blockNb=0; blockNb= maxTime); + { U64 const loopDuration = UTIL_clockSpanMicro(clockStart); + if (loopDuration < fastestC*nbLoops) + fastestC = loopDuration / nbLoops; + totalCTime += loopDuration; + cCompleted = (totalCTime >= maxTime); /* end compression tests */ } } cSize = 0; @@ -336,15 +337,15 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, ratio = (double)srcSize / (double)cSize; markNb = (markNb+1) % NB_MARKS; DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.3f),%6.1f MB/s\r", - marks[markNb], displayName, (U32)srcSize, (U32)cSize, ratio, - (double)srcSize / fastestC ); + marks[markNb], displayName, (U32)srcSize, (U32)cSize, + ratio, (double)srcSize / fastestC ); } else { /* g_decodeOnly */ memcpy(compressedBuffer, srcBuffer, loadedCompressedSize); } #if 0 /* disable decompression test */ dCompleted=1; - (void)totalDTime; (void)fastestD; (void)crcOrig; /* unused when decompression disabled */ + (void)totalDTime; (void)fastestD; (void)crcOrig; /* unused when decompression disabled */ #else /* Decompression */ if (!dCompleted) memset(resultBuffer, 0xD6, srcSize); /* warm result buffer */ @@ -374,9 +375,10 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, nbLoops++; } while (UTIL_clockSpanMicro(clockStart) < clockLoop); ZSTD_freeDDict(ddict); - { U64 const clockSpanMicro = UTIL_clockSpanMicro(clockStart); - if (clockSpanMicro < fastestD*nbLoops) fastestD = clockSpanMicro / nbLoops; - totalDTime += clockSpanMicro; + { U64 const loopDuration = UTIL_clockSpanMicro(clockStart); + if (loopDuration < fastestD*nbLoops) + fastestD = loopDuration / nbLoops; + totalDTime += loopDuration; dCompleted = (totalDTime >= maxTime); } } @@ -439,7 +441,6 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, free(blockTable); free(compressedBuffer); free(resultBuffer); - ZSTDMT_freeCCtx(mtctx); ZSTD_freeCCtx(ctx); ZSTD_freeDCtx(dctx); return 0; diff --git a/programs/fileio.c b/programs/fileio.c index c8eab0679..7efa3c656 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -42,9 +42,6 @@ #include "fileio.h" #define ZSTD_STATIC_LINKING_ONLY /* ZSTD_magicNumber, ZSTD_frameHeaderSize_max */ #include "zstd.h" -#ifdef ZSTD_MULTITHREAD -# include "zstdmt_compress.h" -#endif #if defined(ZSTD_GZCOMPRESS) || defined(ZSTD_GZDECOMPRESS) # include # if !defined(z_const) @@ -70,18 +67,6 @@ #define MB *(1<<20) #define GB *(1U<<30) -#define _1BIT 0x01 -#define _2BITS 0x03 -#define _3BITS 0x07 -#define _4BITS 0x0F -#define _6BITS 0x3F -#define _8BITS 0xFF - -#define BLOCKSIZE (128 KB) -#define ROLLBUFFERSIZE (BLOCKSIZE*8*64) - -#define FIO_FRAMEHEADERSIZE 5 /* as a define, because needed to allocated table on stack */ - #define DICTSIZE_MAX (32 MB) /* protection against large input (attack scenario) */ #define FNSPACE 30 @@ -122,14 +107,14 @@ static clock_t g_time = 0; # define ZSTD_DEBUG 0 #endif #define DEBUGLOG(l,...) if (l<=ZSTD_DEBUG) DISPLAY(__VA_ARGS__); -#define EXM_THROW(error, ...) \ -{ \ - DISPLAYLEVEL(1, "zstd: "); \ - DEBUGLOG(1, "Error defined at %s, line %i : \n", __FILE__, __LINE__); \ - DISPLAYLEVEL(1, "error %i : ", error); \ - DISPLAYLEVEL(1, __VA_ARGS__); \ - DISPLAYLEVEL(1, " \n"); \ - exit(error); \ +#define EXM_THROW(error, ...) \ +{ \ + DISPLAYLEVEL(1, "zstd: "); \ + DEBUGLOG(1, "Error defined at %s, line %i : \n", __FILE__, __LINE__); \ + DISPLAYLEVEL(1, "error %i : ", error); \ + DISPLAYLEVEL(1, __VA_ARGS__); \ + DISPLAYLEVEL(1, " \n"); \ + exit(error); \ } #define CHECK(f) { \ @@ -273,86 +258,88 @@ static int FIO_remove(const char* path) } /** FIO_openSrcFile() : - * condition : `dstFileName` must be non-NULL. - * @result : FILE* to `dstFileName`, or NULL if it fails */ + * condition : `srcFileName` must be non-NULL. + * @result : FILE* to `srcFileName`, or NULL if it fails */ static FILE* FIO_openSrcFile(const char* srcFileName) { - FILE* f; - + assert(srcFileName != NULL); if (!strcmp (srcFileName, stdinmark)) { DISPLAYLEVEL(4,"Using stdin for input\n"); - f = stdin; SET_BINARY_MODE(stdin); - } else { - if (!UTIL_isRegularFile(srcFileName)) { - DISPLAYLEVEL(1, "zstd: %s is not a regular file -- ignored \n", - srcFileName); - return NULL; - } - f = fopen(srcFileName, "rb"); - if ( f==NULL ) - DISPLAYLEVEL(1, "zstd: %s: %s \n", srcFileName, strerror(errno)); + return stdin; } - return f; + if (!UTIL_isRegularFile(srcFileName)) { + DISPLAYLEVEL(1, "zstd: %s is not a regular file -- ignored \n", + srcFileName); + return NULL; + } + + { FILE* const f = fopen(srcFileName, "rb"); + if (f == NULL) + DISPLAYLEVEL(1, "zstd: %s: %s \n", srcFileName, strerror(errno)); + return f; + } } /** FIO_openDstFile() : - * condition : `dstFileName` must be non-NULL. + * condition : `dstFileName` must be non-NULL. * @result : FILE* to `dstFileName`, or NULL if it fails */ static FILE* FIO_openDstFile(const char* dstFileName) { - FILE* f; - + assert(dstFileName != NULL); if (!strcmp (dstFileName, stdoutmark)) { DISPLAYLEVEL(4,"Using stdout for output\n"); - f = stdout; SET_BINARY_MODE(stdout); if (g_sparseFileSupport==1) { g_sparseFileSupport = 0; DISPLAYLEVEL(4, "Sparse File Support is automatically disabled on stdout ; try --sparse \n"); } - } else { - if (g_sparseFileSupport == 1) { - g_sparseFileSupport = ZSTD_SPARSE_DEFAULT; - } - if (strcmp (dstFileName, nulmark)) { - /* Check if destination file already exists */ - f = fopen( dstFileName, "rb" ); - if (f != 0) { /* dst file exists, prompt for overwrite authorization */ - fclose(f); - if (!g_overwrite) { - if (g_displayLevel <= 1) { - /* No interaction possible */ - DISPLAY("zstd: %s already exists; not overwritten \n", - dstFileName); - return NULL; - } - DISPLAY("zstd: %s already exists; do you wish to overwrite (y/N) ? ", - dstFileName); - { int ch = getchar(); - if ((ch!='Y') && (ch!='y')) { - DISPLAY(" not overwritten \n"); - return NULL; - } - /* flush rest of input line */ - while ((ch!=EOF) && (ch!='\n')) ch = getchar(); - } } - /* need to unlink */ - FIO_remove(dstFileName); - } } - f = fopen( dstFileName, "wb" ); - if (f==NULL) DISPLAYLEVEL(1, "zstd: %s: %s\n", dstFileName, strerror(errno)); + return stdout; } - return f; + if (g_sparseFileSupport == 1) { + g_sparseFileSupport = ZSTD_SPARSE_DEFAULT; + } + + if (strcmp (dstFileName, nulmark)) { /* not /dev/null */ + /* Check if destination file already exists */ + FILE* const fCheck = fopen( dstFileName, "rb" ); + if (fCheck != NULL) { /* dst file exists, authorization prompt */ + fclose(fCheck); + if (!g_overwrite) { + if (g_displayLevel <= 1) { + /* No interaction possible */ + DISPLAY("zstd: %s already exists; not overwritten \n", + dstFileName); + return NULL; + } + DISPLAY("zstd: %s already exists; overwrite (y/N) ? ", + dstFileName); + { int ch = getchar(); + if ((ch!='Y') && (ch!='y')) { + DISPLAY(" not overwritten \n"); + return NULL; + } + /* flush rest of input line */ + while ((ch!=EOF) && (ch!='\n')) ch = getchar(); + } } + /* need to unlink */ + FIO_remove(dstFileName); + } } + + { FILE* const f = fopen( dstFileName, "wb" ); + if (f == NULL) + DISPLAYLEVEL(1, "zstd: %s: %s\n", dstFileName, strerror(errno)); + return f; + } } /*! FIO_createDictBuffer() : * creates a buffer, pointed by `*bufferPtr`, * loads `filename` content into it, up to DICTSIZE_MAX bytes. - * @return : loaded size + * @return : loaded size * if fileName==NULL, returns 0 and a NULL pointer */ static size_t FIO_createDictBuffer(void** bufferPtr, const char* fileName) @@ -360,12 +347,13 @@ static size_t FIO_createDictBuffer(void** bufferPtr, const char* fileName) FILE* fileHandle; U64 fileSize; + assert(bufferPtr != NULL); *bufferPtr = NULL; if (fileName == NULL) return 0; DISPLAYLEVEL(4,"Loading %s as dictionary \n", fileName); fileHandle = fopen(fileName, "rb"); - if (fileHandle==0) EXM_THROW(31, "%s: %s", fileName, strerror(errno)); + if (fileHandle==NULL) EXM_THROW(31, "%s: %s", fileName, strerror(errno)); fileSize = UTIL_getFileSize(fileName); if (fileSize > DICTSIZE_MAX) { EXM_THROW(32, "Dictionary file %s is too large (> %u MB)", @@ -412,13 +400,13 @@ static cRess_t FIO_createCResources(const char* dictFileName, int cLevel, if (!ress.srcBuffer || !ress.dstBuffer) EXM_THROW(31, "allocation error : not enough memory"); - /* dictionary */ + /* Advances parameters, including dictionary */ { void* dictBuffer; size_t const dictBuffSize = FIO_createDictBuffer(&dictBuffer, dictFileName); /* works with dictFileName==NULL */ if (dictFileName && (dictBuffer==NULL)) EXM_THROW(32, "allocation error : can't create dictBuffer"); - CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_contentSizeFlag, 1) ); /* always enable content size, when available (note: supposed to be default anyway) */ + CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_contentSizeFlag, 1) ); /* always enable content size when available (note: supposed to be default) */ CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_dictIDFlag, g_dictIDFlag) ); CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_checksumFlag, g_checksumFlag) ); /* compression level */ @@ -446,7 +434,7 @@ static cRess_t FIO_createCResources(const char* dictFileName, int cLevel, DISPLAYLEVEL(5,"set nb threads = %u \n", g_nbThreads); CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_nbThreads, g_nbThreads) ); /* dictionary */ - CHECK( ZSTD_CCtx_setPledgedSrcSize(ress.cctx, srcSize) ); /* just to load dictionary with good compression parameters */ + CHECK( ZSTD_CCtx_setPledgedSrcSize(ress.cctx, srcSize) ); /* just for dictionary loading, using good compression parameters */ CHECK( ZSTD_CCtx_loadDictionary(ress.cctx, dictBuffer, dictBuffSize) ); CHECK( ZSTD_CCtx_setPledgedSrcSize(ress.cctx, ZSTD_CONTENTSIZE_UNKNOWN) ); /* reset */ From 91535d71ec58b9ef9090c5b4ed39a31243a5ced4 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Thu, 19 Oct 2017 12:09:34 -0700 Subject: [PATCH 3/3] fixed missing zstdmt_compress.h dependency we lose a warning message : when a job size is chosen < minimum job size for multithreading, it is automatically resized to minimum size. If this information is really useful, it should be present in zstd.h now. --- programs/fileio.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index 7efa3c656..a8e33994c 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -203,10 +203,6 @@ static U32 g_blockSize = 0; void FIO_setBlockSize(unsigned blockSize) { if (blockSize && g_nbThreads==1) DISPLAYLEVEL(2, "Setting block size is useless in single-thread mode \n"); -#ifdef ZSTD_MULTITHREAD - if (blockSize-1 < ZSTDMT_SECTION_SIZE_MIN-1) /* intentional underflow */ - DISPLAYLEVEL(2, "Note : minimum block size is %u KB \n", (ZSTDMT_SECTION_SIZE_MIN>>10)); -#endif g_blockSize = blockSize; } #define FIO_OVERLAP_LOG_NOTSET 9999