ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #896 from facebook/simplifyio

Browse Source 
fileio and bench : ZSTD_NEWAPI as the only code path
This commit is contained in:
Yann Collet
2017-10-19 13:59:59 -07:00
committed by GitHub
parent 9d377c6e88 91535d71ec
commit ca78554028
3 changed files with 124 additions and 250 deletions
Download Patch File Download Diff File Expand all files Collapse all files
programs/Makefile
+1 -2
View File
@@ -37,8 +37,7 @@ endif
CPPFLAGS+= -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/compress \ CPPFLAGS+= -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/compress \
-I$(ZSTDDIR)/dictBuilder \ -I$(ZSTDDIR)/dictBuilder \
-DXXH_NAMESPACE=ZSTD_ \ -DXXH_NAMESPACE=ZSTD_
-DZSTD_NEWAPI
CFLAGS ?= -O3 CFLAGS ?= -O3
DEBUGFLAGS+=-Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \ DEBUGFLAGS+=-Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \
-Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \ -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \
programs/bench.c
+25 -63
View File
@@ -41,7 +41,6 @@
#include "zstd.h" #include "zstd.h"
#include "datagen.h" /* RDG_genBuffer */ #include "datagen.h" /* RDG_genBuffer */
#include "xxhash.h" #include "xxhash.h"
#include "zstdmt_compress.h"
/* ************************************* /* *************************************
@@ -189,7 +188,6 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
size_t const maxCompressedSize = ZSTD_compressBound(srcSize) + (maxNbBlocks * 1024); /* add some room for safety */ size_t const maxCompressedSize = ZSTD_compressBound(srcSize) + (maxNbBlocks * 1024); /* add some room for safety */
void* const compressedBuffer = malloc(maxCompressedSize); void* const compressedBuffer = malloc(maxCompressedSize);
void* resultBuffer = malloc(srcSize); void* resultBuffer = malloc(srcSize);
ZSTDMT_CCtx* const mtctx = ZSTDMT_createCCtx(g_nbThreads);
ZSTD_CCtx* const ctx = ZSTD_createCCtx(); ZSTD_CCtx* const ctx = ZSTD_createCCtx();
ZSTD_DCtx* const dctx = ZSTD_createDCtx(); ZSTD_DCtx* const dctx = ZSTD_createDCtx();
size_t const loadedCompressedSize = srcSize; size_t const loadedCompressedSize = srcSize;
@@ -286,8 +284,6 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
if (!cCompleted) { /* still some time to do compression tests */ if (!cCompleted) { /* still some time to do compression tests */
U64 const clockLoop = g_nbSeconds ? TIMELOOP_MICROSEC : 1; U64 const clockLoop = g_nbSeconds ? TIMELOOP_MICROSEC : 1;
U32 nbLoops = 0; U32 nbLoops = 0;
ZSTD_CDict* cdict = NULL;
#ifdef ZSTD_NEWAPI
ZSTD_CCtx_setParameter(ctx, ZSTD_p_nbThreads, g_nbThreads); ZSTD_CCtx_setParameter(ctx, ZSTD_p_nbThreads, g_nbThreads);
ZSTD_CCtx_setParameter(ctx, ZSTD_p_compressionLevel, cLevel); ZSTD_CCtx_setParameter(ctx, ZSTD_p_compressionLevel, cLevel);
ZSTD_CCtx_setParameter(ctx, ZSTD_p_enableLongDistanceMatching, g_ldmFlag); ZSTD_CCtx_setParameter(ctx, ZSTD_p_enableLongDistanceMatching, g_ldmFlag);
@@ -306,68 +302,34 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
ZSTD_CCtx_setParameter(ctx, ZSTD_p_targetLength, comprParams->targetLength); ZSTD_CCtx_setParameter(ctx, ZSTD_p_targetLength, comprParams->targetLength);
ZSTD_CCtx_setParameter(ctx, ZSTD_p_compressionStrategy, comprParams->strategy); ZSTD_CCtx_setParameter(ctx, ZSTD_p_compressionStrategy, comprParams->strategy);
ZSTD_CCtx_loadDictionary(ctx, dictBuffer, dictBufferSize); ZSTD_CCtx_loadDictionary(ctx, dictBuffer, dictBufferSize);
#else
size_t const avgSize = MIN(blockSize, (srcSize / nbFiles));
ZSTD_parameters zparams = ZSTD_getParams(cLevel, avgSize, dictBufferSize);
ZSTD_customMem const cmem = { NULL, NULL, NULL };
if (comprParams->windowLog) zparams.cParams.windowLog = comprParams->windowLog;
if (comprParams->chainLog) zparams.cParams.chainLog = comprParams->chainLog;
if (comprParams->hashLog) zparams.cParams.hashLog = comprParams->hashLog;
if (comprParams->searchLog) zparams.cParams.searchLog = comprParams->searchLog;
if (comprParams->searchLength) zparams.cParams.searchLength = comprParams->searchLength;
if (comprParams->targetLength) zparams.cParams.targetLength = comprParams->targetLength;
if (comprParams->strategy) zparams.cParams.strategy = comprParams->strategy;
cdict = ZSTD_createCDict_advanced(dictBuffer, dictBufferSize, ZSTD_dlm_byRef, ZSTD_dm_auto, zparams.cParams, cmem);
if (cdict==NULL) EXM_THROW(1, "ZSTD_createCDict_advanced() allocation failure");
#endif
do { do {
U32 blockNb; U32 blockNb;
for (blockNb=0; blockNb<nbBlocks; blockNb++) { for (blockNb=0; blockNb<nbBlocks; blockNb++) {
size_t rSize; size_t moreToFlush = 1;
#ifdef ZSTD_NEWAPI ZSTD_outBuffer out;
ZSTD_outBuffer out = { blockTable[blockNb].cPtr, blockTable[blockNb].cRoom, 0 }; ZSTD_inBuffer in;
ZSTD_inBuffer in = { blockTable[blockNb].srcPtr, blockTable[blockNb].srcSize, 0 }; in.src = blockTable[blockNb].srcPtr;
size_t cError = 1; in.size = blockTable[blockNb].srcSize;
while (cError) { in.pos = 0;
cError = ZSTD_compress_generic(ctx, out.dst = blockTable[blockNb].cPtr;
out.size = blockTable[blockNb].cRoom;
out.pos = 0;
while (moreToFlush) {
moreToFlush = ZSTD_compress_generic(ctx,
&out, &in, ZSTD_e_end); &out, &in, ZSTD_e_end);
if (ZSTD_isError(cError)) if (ZSTD_isError(moreToFlush))
EXM_THROW(1, "ZSTD_compress_generic() error : %s", EXM_THROW(1, "ZSTD_compress_generic() error : %s",
ZSTD_getErrorName(cError)); ZSTD_getErrorName(moreToFlush));
} }
rSize = out.pos; blockTable[blockNb].cSize = out.pos;
#else /* ! ZSTD_NEWAPI */
if (dictBufferSize) {
rSize = ZSTD_compress_usingCDict(ctx,
blockTable[blockNb].cPtr, blockTable[blockNb].cRoom,
blockTable[blockNb].srcPtr,blockTable[blockNb].srcSize,
cdict);
} else {
# ifdef ZSTD_MULTITHREAD /* note : limitation : MT single-pass does not support compression with dictionary */
rSize = ZSTDMT_compressCCtx(mtctx,
blockTable[blockNb].cPtr, blockTable[blockNb].cRoom,
blockTable[blockNb].srcPtr,blockTable[blockNb].srcSize,
cLevel);
# else
rSize = ZSTD_compress_advanced (ctx,
blockTable[blockNb].cPtr, blockTable[blockNb].cRoom,
blockTable[blockNb].srcPtr,blockTable[blockNb].srcSize,
NULL, 0, zparams);
# endif
}
if (ZSTD_isError(rSize))
EXM_THROW(1, "ZSTD_compress_usingCDict() failed : %s",
ZSTD_getErrorName(rSize));
#endif /* ZSTD_NEWAPI */
blockTable[blockNb].cSize = rSize;
} }
nbLoops++; nbLoops++;
} while (UTIL_clockSpanMicro(clockStart) < clockLoop); } while (UTIL_clockSpanMicro(clockStart) < clockLoop);
ZSTD_freeCDict(cdict); { U64 const loopDuration = UTIL_clockSpanMicro(clockStart);
{ U64 const clockSpanMicro = UTIL_clockSpanMicro(clockStart); if (loopDuration < fastestC*nbLoops)
if (clockSpanMicro < fastestC*nbLoops) fastestC = clockSpanMicro / nbLoops; fastestC = loopDuration / nbLoops;
totalCTime += clockSpanMicro; totalCTime += loopDuration;
cCompleted = (totalCTime >= maxTime); cCompleted = (totalCTime >= maxTime); /* end compression tests */
} } } }
cSize = 0; cSize = 0;
@@ -375,8 +337,8 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
ratio = (double)srcSize / (double)cSize; ratio = (double)srcSize / (double)cSize;
markNb = (markNb+1) % NB_MARKS; markNb = (markNb+1) % NB_MARKS;
DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.3f),%6.1f MB/s\r", DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.3f),%6.1f MB/s\r",
marks[markNb], displayName, (U32)srcSize, (U32)cSize, ratio, marks[markNb], displayName, (U32)srcSize, (U32)cSize,
(double)srcSize / fastestC ); ratio, (double)srcSize / fastestC );
} else { /* g_decodeOnly */ } else { /* g_decodeOnly */
memcpy(compressedBuffer, srcBuffer, loadedCompressedSize); memcpy(compressedBuffer, srcBuffer, loadedCompressedSize);
} }
@@ -413,9 +375,10 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
nbLoops++; nbLoops++;
} while (UTIL_clockSpanMicro(clockStart) < clockLoop); } while (UTIL_clockSpanMicro(clockStart) < clockLoop);
ZSTD_freeDDict(ddict); ZSTD_freeDDict(ddict);
{ U64 const clockSpanMicro = UTIL_clockSpanMicro(clockStart); { U64 const loopDuration = UTIL_clockSpanMicro(clockStart);
if (clockSpanMicro < fastestD*nbLoops) fastestD = clockSpanMicro / nbLoops; if (loopDuration < fastestD*nbLoops)
totalDTime += clockSpanMicro; fastestD = loopDuration / nbLoops;
totalDTime += loopDuration;
dCompleted = (totalDTime >= maxTime); dCompleted = (totalDTime >= maxTime);
} } } }
@@ -478,7 +441,6 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
free(blockTable); free(blockTable);
free(compressedBuffer); free(compressedBuffer);
free(resultBuffer); free(resultBuffer);
ZSTDMT_freeCCtx(mtctx);
ZSTD_freeCCtx(ctx); ZSTD_freeCCtx(ctx);
ZSTD_freeDCtx(dctx); ZSTD_freeDCtx(dctx);
return 0; return 0;
programs/fileio.c
+30 -117
View File
@@ -42,9 +42,6 @@
#include "fileio.h" #include "fileio.h"
#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_magicNumber, ZSTD_frameHeaderSize_max */ #define ZSTD_STATIC_LINKING_ONLY /* ZSTD_magicNumber, ZSTD_frameHeaderSize_max */
#include "zstd.h" #include "zstd.h"
#ifdef ZSTD_MULTITHREAD
# include "zstdmt_compress.h"
#endif
#if defined(ZSTD_GZCOMPRESS) || defined(ZSTD_GZDECOMPRESS) #if defined(ZSTD_GZCOMPRESS) || defined(ZSTD_GZDECOMPRESS)
# include <zlib.h> # include <zlib.h>
# if !defined(z_const) # if !defined(z_const)
@@ -70,18 +67,6 @@
#define MB *(1<<20) #define MB *(1<<20)
#define GB *(1U<<30) #define GB *(1U<<30)
#define _1BIT 0x01
#define _2BITS 0x03
#define _3BITS 0x07
#define _4BITS 0x0F
#define _6BITS 0x3F
#define _8BITS 0xFF
#define BLOCKSIZE (128 KB)
#define ROLLBUFFERSIZE (BLOCKSIZE*8*64)
#define FIO_FRAMEHEADERSIZE 5 /* as a define, because needed to allocated table on stack */
#define DICTSIZE_MAX (32 MB) /* protection against large input (attack scenario) */ #define DICTSIZE_MAX (32 MB) /* protection against large input (attack scenario) */
#define FNSPACE 30 #define FNSPACE 30
@@ -218,10 +203,6 @@ static U32 g_blockSize = 0;
void FIO_setBlockSize(unsigned blockSize) { void FIO_setBlockSize(unsigned blockSize) {
if (blockSize && g_nbThreads==1) if (blockSize && g_nbThreads==1)
DISPLAYLEVEL(2, "Setting block size is useless in single-thread mode \n"); DISPLAYLEVEL(2, "Setting block size is useless in single-thread mode \n");
#ifdef ZSTD_MULTITHREAD
if (blockSize-1 < ZSTDMT_SECTION_SIZE_MIN-1) /* intentional underflow */
DISPLAYLEVEL(2, "Note : minimum block size is %u KB \n", (ZSTDMT_SECTION_SIZE_MIN>>10));
#endif
g_blockSize = blockSize; g_blockSize = blockSize;
} }
#define FIO_OVERLAP_LOG_NOTSET 9999 #define FIO_OVERLAP_LOG_NOTSET 9999
@@ -273,54 +254,55 @@ static int FIO_remove(const char* path)
} }
/** FIO_openSrcFile() : /** FIO_openSrcFile() :
* condition : `dstFileName` must be non-NULL. * condition : `srcFileName` must be non-NULL.
* @result : FILE* to `dstFileName`, or NULL if it fails */ * @result : FILE* to `srcFileName`, or NULL if it fails */
static FILE* FIO_openSrcFile(const char* srcFileName) static FILE* FIO_openSrcFile(const char* srcFileName)
{ {
FILE* f; assert(srcFileName != NULL);
if (!strcmp (srcFileName, stdinmark)) { if (!strcmp (srcFileName, stdinmark)) {
DISPLAYLEVEL(4,"Using stdin for input\n"); DISPLAYLEVEL(4,"Using stdin for input\n");
f = stdin;
SET_BINARY_MODE(stdin); SET_BINARY_MODE(stdin);
} else { return stdin;
}
if (!UTIL_isRegularFile(srcFileName)) { if (!UTIL_isRegularFile(srcFileName)) {
DISPLAYLEVEL(1, "zstd: %s is not a regular file -- ignored \n", DISPLAYLEVEL(1, "zstd: %s is not a regular file -- ignored \n",
srcFileName); srcFileName);
return NULL; return NULL;
} }
f = fopen(srcFileName, "rb");
{ FILE* const f = fopen(srcFileName, "rb");
if (f == NULL) if (f == NULL)
DISPLAYLEVEL(1, "zstd: %s: %s \n", srcFileName, strerror(errno)); DISPLAYLEVEL(1, "zstd: %s: %s \n", srcFileName, strerror(errno));
}
return f; return f;
} }
}
/** FIO_openDstFile() : /** FIO_openDstFile() :
* condition : `dstFileName` must be non-NULL. * condition : `dstFileName` must be non-NULL.
* @result : FILE* to `dstFileName`, or NULL if it fails */ * @result : FILE* to `dstFileName`, or NULL if it fails */
static FILE* FIO_openDstFile(const char* dstFileName) static FILE* FIO_openDstFile(const char* dstFileName)
{ {
FILE* f; assert(dstFileName != NULL);
if (!strcmp (dstFileName, stdoutmark)) { if (!strcmp (dstFileName, stdoutmark)) {
DISPLAYLEVEL(4,"Using stdout for output\n"); DISPLAYLEVEL(4,"Using stdout for output\n");
f = stdout;
SET_BINARY_MODE(stdout); SET_BINARY_MODE(stdout);
if (g_sparseFileSupport==1) { if (g_sparseFileSupport==1) {
g_sparseFileSupport = 0; g_sparseFileSupport = 0;
DISPLAYLEVEL(4, "Sparse File Support is automatically disabled on stdout ; try --sparse \n"); DISPLAYLEVEL(4, "Sparse File Support is automatically disabled on stdout ; try --sparse \n");
} }
} else { return stdout;
}
if (g_sparseFileSupport == 1) { if (g_sparseFileSupport == 1) {
g_sparseFileSupport = ZSTD_SPARSE_DEFAULT; g_sparseFileSupport = ZSTD_SPARSE_DEFAULT;
} }
if (strcmp (dstFileName, nulmark)) {
if (strcmp (dstFileName, nulmark)) { /* not /dev/null */
/* Check if destination file already exists */ /* Check if destination file already exists */
f = fopen( dstFileName, "rb" ); FILE* const fCheck = fopen( dstFileName, "rb" );
if (f != 0) { /* dst file exists, prompt for overwrite authorization */ if (fCheck != NULL) { /* dst file exists, authorization prompt */
fclose(f); fclose(fCheck);
if (!g_overwrite) { if (!g_overwrite) {
if (g_displayLevel <= 1) { if (g_displayLevel <= 1) {
/* No interaction possible */ /* No interaction possible */
@@ -328,7 +310,7 @@ static FILE* FIO_openDstFile(const char* dstFileName)
dstFileName); dstFileName);
return NULL; return NULL;
} }
DISPLAY("zstd: %s already exists; do you wish to overwrite (y/N) ? ", DISPLAY("zstd: %s already exists; overwrite (y/N) ? ",
dstFileName); dstFileName);
{ int ch = getchar(); { int ch = getchar();
if ((ch!='Y') && (ch!='y')) { if ((ch!='Y') && (ch!='y')) {
@@ -341,12 +323,13 @@ static FILE* FIO_openDstFile(const char* dstFileName)
/* need to unlink */ /* need to unlink */
FIO_remove(dstFileName); FIO_remove(dstFileName);
} } } }
f = fopen( dstFileName, "wb" );
if (f==NULL) DISPLAYLEVEL(1, "zstd: %s: %s\n", dstFileName, strerror(errno));
}
{ FILE* const f = fopen( dstFileName, "wb" );
if (f == NULL)
DISPLAYLEVEL(1, "zstd: %s: %s\n", dstFileName, strerror(errno));
return f; return f;
} }
}
/*! FIO_createDictBuffer() : /*! FIO_createDictBuffer() :
@@ -360,12 +343,13 @@ static size_t FIO_createDictBuffer(void** bufferPtr, const char* fileName)
FILE* fileHandle; FILE* fileHandle;
U64 fileSize; U64 fileSize;
assert(bufferPtr != NULL);
*bufferPtr = NULL; *bufferPtr = NULL;
if (fileName == NULL) return 0; if (fileName == NULL) return 0;
DISPLAYLEVEL(4,"Loading %s as dictionary \n", fileName); DISPLAYLEVEL(4,"Loading %s as dictionary \n", fileName);
fileHandle = fopen(fileName, "rb"); fileHandle = fopen(fileName, "rb");
if (fileHandle==0) EXM_THROW(31, "%s: %s", fileName, strerror(errno)); if (fileHandle==NULL) EXM_THROW(31, "%s: %s", fileName, strerror(errno));
fileSize = UTIL_getFileSize(fileName); fileSize = UTIL_getFileSize(fileName);
if (fileSize > DICTSIZE_MAX) { if (fileSize > DICTSIZE_MAX) {
EXM_THROW(32, "Dictionary file %s is too large (> %u MB)", EXM_THROW(32, "Dictionary file %s is too large (> %u MB)",
@@ -393,11 +377,7 @@ typedef struct {
size_t srcBufferSize; size_t srcBufferSize;
void* dstBuffer; void* dstBuffer;
size_t dstBufferSize; size_t dstBufferSize;
#if !defined(ZSTD_NEWAPI) && defined(ZSTD_MULTITHREAD)
ZSTDMT_CCtx* cctx;
#else
ZSTD_CStream* cctx; ZSTD_CStream* cctx;
#endif
} cRess_t; } cRess_t;
static cRess_t FIO_createCResources(const char* dictFileName, int cLevel, static cRess_t FIO_createCResources(const char* dictFileName, int cLevel,
@@ -406,24 +386,9 @@ static cRess_t FIO_createCResources(const char* dictFileName, int cLevel,
cRess_t ress; cRess_t ress;
memset(&ress, 0, sizeof(ress)); memset(&ress, 0, sizeof(ress));
#ifdef ZSTD_NEWAPI
ress.cctx = ZSTD_createCCtx(); ress.cctx = ZSTD_createCCtx();
if (ress.cctx == NULL) if (ress.cctx == NULL)
EXM_THROW(30, "allocation error : can't create ZSTD_CCtx"); EXM_THROW(30, "allocation error : can't create ZSTD_CCtx");
#elif defined(ZSTD_MULTITHREAD)
ress.cctx = ZSTDMT_createCCtx(g_nbThreads);
if (ress.cctx == NULL)
EXM_THROW(30, "allocation error : can't create ZSTDMT_CCtx");
if ((cLevel==ZSTD_maxCLevel()) && (g_overlapLog==FIO_OVERLAP_LOG_NOTSET))
/* use complete window for overlap */
ZSTDMT_setMTCtxParameter(ress.cctx, ZSTDMT_p_overlapSectionLog, 9);
if (g_overlapLog != FIO_OVERLAP_LOG_NOTSET)
ZSTDMT_setMTCtxParameter(ress.cctx, ZSTDMT_p_overlapSectionLog, g_overlapLog);
#else
ress.cctx = ZSTD_createCStream();
if (ress.cctx == NULL)
EXM_THROW(30, "allocation error : can't create ZSTD_CStream");
#endif
ress.srcBufferSize = ZSTD_CStreamInSize(); ress.srcBufferSize = ZSTD_CStreamInSize();
ress.srcBuffer = malloc(ress.srcBufferSize); ress.srcBuffer = malloc(ress.srcBufferSize);
ress.dstBufferSize = ZSTD_CStreamOutSize(); ress.dstBufferSize = ZSTD_CStreamOutSize();
@@ -431,15 +396,13 @@ static cRess_t FIO_createCResources(const char* dictFileName, int cLevel,
if (!ress.srcBuffer || !ress.dstBuffer) if (!ress.srcBuffer || !ress.dstBuffer)
EXM_THROW(31, "allocation error : not enough memory"); EXM_THROW(31, "allocation error : not enough memory");
/* dictionary */ /* Advances parameters, including dictionary */
{ void* dictBuffer; { void* dictBuffer;
size_t const dictBuffSize = FIO_createDictBuffer(&dictBuffer, dictFileName); /* works with dictFileName==NULL */ size_t const dictBuffSize = FIO_createDictBuffer(&dictBuffer, dictFileName); /* works with dictFileName==NULL */
if (dictFileName && (dictBuffer==NULL)) if (dictFileName && (dictBuffer==NULL))
EXM_THROW(32, "allocation error : can't create dictBuffer"); EXM_THROW(32, "allocation error : can't create dictBuffer");
#ifdef ZSTD_NEWAPI CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_contentSizeFlag, 1) ); /* always enable content size when available (note: supposed to be default) */
{ /* frame parameters */
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_contentSizeFlag, 1) );
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_dictIDFlag, g_dictIDFlag) ); CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_dictIDFlag, g_dictIDFlag) );
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_checksumFlag, g_checksumFlag) ); CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_checksumFlag, g_checksumFlag) );
/* compression level */ /* compression level */
@@ -467,38 +430,10 @@ static cRess_t FIO_createCResources(const char* dictFileName, int cLevel,
DISPLAYLEVEL(5,"set nb threads = %u \n", g_nbThreads); DISPLAYLEVEL(5,"set nb threads = %u \n", g_nbThreads);
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_nbThreads, g_nbThreads) ); CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_nbThreads, g_nbThreads) );
/* dictionary */ /* dictionary */
CHECK( ZSTD_CCtx_setPledgedSrcSize(ress.cctx, srcSize) ); /* just to load dictionary with good compression parameters */ CHECK( ZSTD_CCtx_setPledgedSrcSize(ress.cctx, srcSize) ); /* just for dictionary loading, using good compression parameters */
CHECK( ZSTD_CCtx_loadDictionary(ress.cctx, dictBuffer, dictBuffSize) ); CHECK( ZSTD_CCtx_loadDictionary(ress.cctx, dictBuffer, dictBuffSize) );
CHECK( ZSTD_CCtx_setPledgedSrcSize(ress.cctx, ZSTD_CONTENTSIZE_UNKNOWN) ); /* reset */ CHECK( ZSTD_CCtx_setPledgedSrcSize(ress.cctx, ZSTD_CONTENTSIZE_UNKNOWN) ); /* reset */
}
#elif defined(ZSTD_MULTITHREAD)
{ ZSTD_parameters params = ZSTD_getParams(cLevel, srcSize, dictBuffSize);
params.fParams.checksumFlag = g_checksumFlag;
params.fParams.noDictIDFlag = !g_dictIDFlag;
if (comprParams->windowLog) params.cParams.windowLog = comprParams->windowLog;
if (comprParams->chainLog) params.cParams.chainLog = comprParams->chainLog;
if (comprParams->hashLog) params.cParams.hashLog = comprParams->hashLog;
if (comprParams->searchLog) params.cParams.searchLog = comprParams->searchLog;
if (comprParams->searchLength) params.cParams.searchLength = comprParams->searchLength;
if (comprParams->targetLength) params.cParams.targetLength = comprParams->targetLength;
if (comprParams->strategy) params.cParams.strategy = (ZSTD_strategy) comprParams->strategy;
CHECK( ZSTDMT_initCStream_advanced(ress.cctx, dictBuffer, dictBuffSize, params, srcSize) );
ZSTDMT_setMTCtxParameter(ress.cctx, ZSTDMT_p_sectionSize, g_blockSize);
}
#else
{ ZSTD_parameters params = ZSTD_getParams(cLevel, srcSize, dictBuffSize);
params.fParams.checksumFlag = g_checksumFlag;
params.fParams.noDictIDFlag = !g_dictIDFlag;
if (comprParams->windowLog) params.cParams.windowLog = comprParams->windowLog;
if (comprParams->chainLog) params.cParams.chainLog = comprParams->chainLog;
if (comprParams->hashLog) params.cParams.hashLog = comprParams->hashLog;
if (comprParams->searchLog) params.cParams.searchLog = comprParams->searchLog;
if (comprParams->searchLength) params.cParams.searchLength = comprParams->searchLength;
if (comprParams->targetLength) params.cParams.targetLength = comprParams->targetLength;
if (comprParams->strategy) params.cParams.strategy = (ZSTD_strategy) comprParams->strategy;
CHECK( ZSTD_initCStream_advanced(ress.cctx, dictBuffer, dictBuffSize, params, srcSize) );
}
#endif
free(dictBuffer); free(dictBuffer);
} }
@@ -509,11 +444,7 @@ static void FIO_freeCResources(cRess_t ress)
{ {
free(ress.srcBuffer); free(ress.srcBuffer);
free(ress.dstBuffer); free(ress.dstBuffer);
#if !defined(ZSTD_NEWAPI) && defined(ZSTD_MULTITHREAD)
ZSTDMT_freeCCtx(ress.cctx);
#else
ZSTD_freeCStream(ress.cctx); /* never fails */ ZSTD_freeCStream(ress.cctx); /* never fails */
#endif
} }
@@ -814,14 +745,8 @@ static int FIO_compressFilename_internal(cRess_t ress,
} }
/* init */ /* init */
#ifdef ZSTD_NEWAPI if (fileSize != UTIL_FILESIZE_UNKNOWN)
if (fileSize!=UTIL_FILESIZE_UNKNOWN) /* when src is stdin, fileSize==0, but is effectively unknown */
ZSTD_CCtx_setPledgedSrcSize(ress.cctx, fileSize); ZSTD_CCtx_setPledgedSrcSize(ress.cctx, fileSize);
#elif defined(ZSTD_MULTITHREAD)
CHECK( ZSTDMT_resetCStream(ress.cctx, (fileSize==UTIL_FILESIZE_UNKNOWN) ? ZSTD_CONTENTSIZE_UNKNOWN : fileSize) );
#else
CHECK( ZSTD_resetCStream(ress.cctx, (fileSize==UTIL_FILESIZE_UNKNOWN) ? ZSTD_CONTENTSIZE_UNKNOWN : fileSize) );
#endif
/* Main compression loop */ /* Main compression loop */
while (1) { while (1) {
@@ -833,14 +758,8 @@ static int FIO_compressFilename_internal(cRess_t ress,
while (inBuff.pos != inBuff.size) { while (inBuff.pos != inBuff.size) {
ZSTD_outBuffer outBuff = { ress.dstBuffer, ress.dstBufferSize, 0 }; ZSTD_outBuffer outBuff = { ress.dstBuffer, ress.dstBufferSize, 0 };
#ifdef ZSTD_NEWAPI
CHECK( ZSTD_compress_generic(ress.cctx, CHECK( ZSTD_compress_generic(ress.cctx,
&outBuff, &inBuff, ZSTD_e_continue) ); &outBuff, &inBuff, ZSTD_e_continue) );
#elif defined(ZSTD_MULTITHREAD)
CHECK( ZSTDMT_compressStream(ress.cctx, &outBuff, &inBuff) );
#else
CHECK( ZSTD_compressStream(ress.cctx, &outBuff, &inBuff) );
#endif
/* Write compressed stream */ /* Write compressed stream */
if (outBuff.pos) { if (outBuff.pos) {
@@ -871,15 +790,9 @@ static int FIO_compressFilename_internal(cRess_t ress,
{ size_t result = 1; { size_t result = 1;
while (result != 0) { while (result != 0) {
ZSTD_outBuffer outBuff = { ress.dstBuffer, ress.dstBufferSize, 0 }; ZSTD_outBuffer outBuff = { ress.dstBuffer, ress.dstBufferSize, 0 };
#ifdef ZSTD_NEWAPI
ZSTD_inBuffer inBuff = { NULL, 0, 0 }; ZSTD_inBuffer inBuff = { NULL, 0, 0 };
result = ZSTD_compress_generic(ress.cctx, result = ZSTD_compress_generic(ress.cctx,
&outBuff, &inBuff, ZSTD_e_end); &outBuff, &inBuff, ZSTD_e_end);
#elif defined(ZSTD_MULTITHREAD)
result = ZSTDMT_endStream(ress.cctx, &outBuff);
#else
result = ZSTD_endStream(ress.cctx, &outBuff);
#endif
if (ZSTD_isError(result)) { if (ZSTD_isError(result)) {
EXM_THROW(26, "Compression error during frame end : %s", EXM_THROW(26, "Compression error during frame end : %s",
ZSTD_getErrorName(result)); ZSTD_getErrorName(result));