Yann Collet
+48
-20
@@ -86,14 +86,13 @@ static clock_t g_time = 0;
|
||||
#ifndef DEBUG
|
||||
# define DEBUG 0
|
||||
#endif
|
||||
#define DEBUGOUTPUT(...) if (DEBUG) DISPLAY(__VA_ARGS__);
|
||||
#define EXM_THROW(error, ...) \
|
||||
{ \
|
||||
DEBUGOUTPUT("%s: %i: \n", __FILE__, __LINE__); \
|
||||
DISPLAYLEVEL(1, "Error %i : ", error); \
|
||||
DISPLAYLEVEL(1, __VA_ARGS__); \
|
||||
DISPLAYLEVEL(1, " \n"); \
|
||||
exit(error); \
|
||||
#define DEBUGOUTPUT(...) { if (DEBUG) DISPLAY(__VA_ARGS__); }
|
||||
#define EXM_THROW(error, ...) { \
|
||||
DEBUGOUTPUT("%s: %i: \n", __FILE__, __LINE__); \
|
||||
DISPLAYLEVEL(1, "Error %i : ", error); \
|
||||
DISPLAYLEVEL(1, __VA_ARGS__); \
|
||||
DISPLAYLEVEL(1, " \n"); \
|
||||
exit(error); \
|
||||
}
|
||||
|
||||
|
||||
@@ -159,7 +158,6 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
|
||||
const ZSTD_compressionParameters* comprParams)
|
||||
{
|
||||
size_t const blockSize = ((g_blockSize>=32 && !g_decodeOnly) ? g_blockSize : srcSize) + (!srcSize) /* avoid div by 0 */ ;
|
||||
size_t const avgSize = MIN(blockSize, (srcSize / nbFiles));
|
||||
U32 const maxNbBlocks = (U32) ((srcSize + (blockSize-1)) / blockSize) + nbFiles;
|
||||
blockParam_t* const blockTable = (blockParam_t*) malloc(maxNbBlocks * sizeof(blockParam_t));
|
||||
size_t const maxCompressedSize = ZSTD_compressBound(srcSize) + (maxNbBlocks * 1024); /* add some room for safety */
|
||||
@@ -262,42 +260,72 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
|
||||
UTIL_getTime(&clockStart);
|
||||
|
||||
if (!cCompleted) { /* still some time to do compression tests */
|
||||
ZSTD_customMem const cmem = { NULL, NULL, NULL };
|
||||
U64 const clockLoop = g_nbSeconds ? TIMELOOP_MICROSEC : 1;
|
||||
U32 nbLoops = 0;
|
||||
ZSTD_CDict* cdict = NULL;
|
||||
#ifdef ZSTD_NEWAPI
|
||||
ZSTD_CCtx_setParameter(ctx, ZSTD_p_nbThreads, g_nbThreads);
|
||||
ZSTD_CCtx_setParameter(ctx, ZSTD_p_compressionLevel, cLevel);
|
||||
ZSTD_CCtx_setParameter(ctx, ZSTD_p_windowLog, comprParams->windowLog);
|
||||
ZSTD_CCtx_setParameter(ctx, ZSTD_p_chainLog, comprParams->chainLog);
|
||||
ZSTD_CCtx_setParameter(ctx, ZSTD_p_searchLog, comprParams->searchLog);
|
||||
ZSTD_CCtx_setParameter(ctx, ZSTD_p_minMatch, comprParams->searchLength);
|
||||
ZSTD_CCtx_setParameter(ctx, ZSTD_p_targetLength, comprParams->targetLength);
|
||||
ZSTD_CCtx_setParameter(ctx, ZSTD_p_compressionStrategy, comprParams->strategy);
|
||||
ZSTD_CCtx_loadDictionary(ctx, dictBuffer, dictBufferSize);
|
||||
#else
|
||||
size_t const avgSize = MIN(blockSize, (srcSize / nbFiles));
|
||||
ZSTD_parameters zparams = ZSTD_getParams(cLevel, avgSize, dictBufferSize);
|
||||
ZSTD_CDict* cdict;
|
||||
ZSTD_customMem const cmem = { NULL, NULL, NULL };
|
||||
if (comprParams->windowLog) zparams.cParams.windowLog = comprParams->windowLog;
|
||||
if (comprParams->chainLog) zparams.cParams.chainLog = comprParams->chainLog;
|
||||
if (comprParams->hashLog) zparams.cParams.hashLog = comprParams->hashLog;
|
||||
if (comprParams->searchLog) zparams.cParams.searchLog = comprParams->searchLog;
|
||||
if (comprParams->searchLength) zparams.cParams.searchLength = comprParams->searchLength;
|
||||
if (comprParams->targetLength) zparams.cParams.targetLength = comprParams->targetLength;
|
||||
if (comprParams->strategy) zparams.cParams.strategy = (ZSTD_strategy)(comprParams->strategy - 1);
|
||||
cdict = ZSTD_createCDict_advanced(dictBuffer, dictBufferSize, 1, zparams.cParams, cmem);
|
||||
if (comprParams->strategy) zparams.cParams.strategy = comprParams->strategy;
|
||||
cdict = ZSTD_createCDict_advanced(dictBuffer, dictBufferSize, 1 /*byRef*/, ZSTD_dm_auto, zparams.cParams, cmem);
|
||||
if (cdict==NULL) EXM_THROW(1, "ZSTD_createCDict_advanced() allocation failure");
|
||||
#endif
|
||||
do {
|
||||
U32 blockNb;
|
||||
size_t rSize;
|
||||
for (blockNb=0; blockNb<nbBlocks; blockNb++) {
|
||||
size_t rSize;
|
||||
#ifdef ZSTD_NEWAPI
|
||||
ZSTD_outBuffer out = { blockTable[blockNb].cPtr, blockTable[blockNb].cRoom, 0 };
|
||||
ZSTD_inBuffer in = { blockTable[blockNb].srcPtr, blockTable[blockNb].srcSize, 0 };
|
||||
size_t cError = 1;
|
||||
while (cError) {
|
||||
cError = ZSTD_compress_generic(ctx,
|
||||
&out, &in, ZSTD_e_end);
|
||||
if (ZSTD_isError(cError))
|
||||
EXM_THROW(1, "ZSTD_compress_generic() error : %s",
|
||||
ZSTD_getErrorName(cError));
|
||||
}
|
||||
rSize = out.pos;
|
||||
#else /* ! ZSTD_NEWAPI */
|
||||
if (dictBufferSize) {
|
||||
rSize = ZSTD_compress_usingCDict(ctx,
|
||||
blockTable[blockNb].cPtr, blockTable[blockNb].cRoom,
|
||||
blockTable[blockNb].srcPtr,blockTable[blockNb].srcSize,
|
||||
cdict);
|
||||
} else {
|
||||
#ifdef ZSTD_MULTITHREAD /* note : limitation : MT single-pass does not support compression with dictionary */
|
||||
# ifdef ZSTD_MULTITHREAD /* note : limitation : MT single-pass does not support compression with dictionary */
|
||||
rSize = ZSTDMT_compressCCtx(mtctx,
|
||||
blockTable[blockNb].cPtr, blockTable[blockNb].cRoom,
|
||||
blockTable[blockNb].srcPtr,blockTable[blockNb].srcSize,
|
||||
cLevel);
|
||||
#else
|
||||
# else
|
||||
rSize = ZSTD_compress_advanced (ctx,
|
||||
blockTable[blockNb].cPtr, blockTable[blockNb].cRoom,
|
||||
blockTable[blockNb].srcPtr,blockTable[blockNb].srcSize, NULL, 0, zparams);
|
||||
#endif
|
||||
blockTable[blockNb].srcPtr,blockTable[blockNb].srcSize,
|
||||
NULL, 0, zparams);
|
||||
# endif
|
||||
}
|
||||
if (ZSTD_isError(rSize)) EXM_THROW(1, "ZSTD_compress_usingCDict() failed : %s", ZSTD_getErrorName(rSize));
|
||||
if (ZSTD_isError(rSize))
|
||||
EXM_THROW(1, "ZSTD_compress_usingCDict() failed : %s",
|
||||
ZSTD_getErrorName(rSize));
|
||||
#endif /* ZSTD_NEWAPI */
|
||||
blockTable[blockNb].cSize = rSize;
|
||||
}
|
||||
nbLoops++;
|
||||
@@ -383,7 +411,7 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
|
||||
}
|
||||
pos = (U32)(u - bacc);
|
||||
bNb = pos / (128 KB);
|
||||
DISPLAY("(block %u, sub %u, pos %u) \n", segNb, bNb, pos);
|
||||
DISPLAY("(sample %u, block %u, pos %u) \n", segNb, bNb, pos);
|
||||
if (u>5) {
|
||||
int n;
|
||||
for (n=-5; n<0; n++) DISPLAY("%02X ", ((const BYTE*)srcBuffer)[u+n]);
|
||||
|
||||
+135
-89
@@ -77,10 +77,7 @@
|
||||
#define BLOCKSIZE (128 KB)
|
||||
#define ROLLBUFFERSIZE (BLOCKSIZE*8*64)
|
||||
|
||||
#define FIO_FRAMEHEADERSIZE 5 /* as a define, because needed to allocated table on stack */
|
||||
#define FSE_CHECKSUM_SEED 0
|
||||
|
||||
#define CACHELINE 64
|
||||
#define FIO_FRAMEHEADERSIZE 5 /* as a define, because needed to allocated table on stack */
|
||||
|
||||
#define DICTSIZE_MAX (32 MB) /* protection against large input (attack scenario) */
|
||||
|
||||
@@ -93,7 +90,7 @@
|
||||
#define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
|
||||
#define DISPLAYOUT(...) fprintf(stdout, __VA_ARGS__)
|
||||
#define DISPLAYLEVEL(l, ...) { if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); } }
|
||||
static int g_displayLevel = 2; /* 0 : no display; 1: errors; 2 : + result + interaction + warnings; 3 : + progression; 4 : + information */
|
||||
static int g_displayLevel = 2; /* 0 : no display; 1: errors; 2: + result + interaction + warnings; 3: + progression; 4: + information */
|
||||
void FIO_setNotificationLevel(unsigned level) { g_displayLevel=level; }
|
||||
|
||||
#define DISPLAYUPDATE(l, ...) { if (g_displayLevel>=l) { \
|
||||
@@ -103,10 +100,35 @@ void FIO_setNotificationLevel(unsigned level) { g_displayLevel=level; }
|
||||
static const clock_t refreshRate = CLOCKS_PER_SEC * 15 / 100;
|
||||
static clock_t g_time = 0;
|
||||
|
||||
#undef MIN
|
||||
#undef MIN /* in case it would be already defined */
|
||||
#define MIN(a,b) ((a) < (b) ? (a) : (b))
|
||||
|
||||
|
||||
/*-*************************************
|
||||
* Errors
|
||||
***************************************/
|
||||
#ifndef ZSTD_DEBUG
|
||||
# define ZSTD_DEBUG 0
|
||||
#endif
|
||||
#define DEBUGLOG(l,...) if (l<=ZSTD_DEBUG) DISPLAY(__VA_ARGS__);
|
||||
#define EXM_THROW(error, ...) \
|
||||
{ \
|
||||
DISPLAYLEVEL(1, "zstd: "); \
|
||||
DEBUGLOG(1, "Error defined at %s, line %i : \n", __FILE__, __LINE__); \
|
||||
DISPLAYLEVEL(1, "error %i : ", error); \
|
||||
DISPLAYLEVEL(1, __VA_ARGS__); \
|
||||
DISPLAYLEVEL(1, " \n"); \
|
||||
exit(error); \
|
||||
}
|
||||
|
||||
#define CHECK(f) { \
|
||||
size_t const err = f; \
|
||||
if (ZSTD_isError(err)) { \
|
||||
DEBUGLOG(1, "%s \n", #f); \
|
||||
EXM_THROW(11, "%s", ZSTD_getErrorName(err)); \
|
||||
} }
|
||||
|
||||
|
||||
/* ************************************************************
|
||||
* Avoid fseek()'s 2GiB barrier with MSVC, MacOS, *BSD, MinGW
|
||||
***************************************************************/
|
||||
@@ -146,7 +168,7 @@ static FIO_compressionType_t g_compressionType = FIO_zstdCompression;
|
||||
void FIO_setCompressionType(FIO_compressionType_t compressionType) { g_compressionType = compressionType; }
|
||||
static U32 g_overwrite = 0;
|
||||
void FIO_overwriteMode(void) { g_overwrite=1; }
|
||||
static U32 g_sparseFileSupport = 1; /* 0 : no sparse allowed; 1: auto (file yes, stdout no); 2: force sparse */
|
||||
static U32 g_sparseFileSupport = 1; /* 0: no sparse allowed; 1: auto (file yes, stdout no); 2: force sparse */
|
||||
void FIO_setSparseWrite(unsigned sparse) { g_sparseFileSupport=sparse; }
|
||||
static U32 g_dictIDFlag = 1;
|
||||
void FIO_setDictIDFlag(unsigned dictIDFlag) { g_dictIDFlag = dictIDFlag; }
|
||||
@@ -182,23 +204,6 @@ void FIO_setOverlapLog(unsigned overlapLog){
|
||||
}
|
||||
|
||||
|
||||
/*-*************************************
|
||||
* Exceptions
|
||||
***************************************/
|
||||
#ifndef DEBUG
|
||||
# define DEBUG 0
|
||||
#endif
|
||||
#define DEBUGOUTPUT(...) if (DEBUG) DISPLAY(__VA_ARGS__);
|
||||
#define EXM_THROW(error, ...) \
|
||||
{ \
|
||||
DEBUGOUTPUT("Error defined at %s, line %i : \n", __FILE__, __LINE__); \
|
||||
DISPLAYLEVEL(1, "Error %i : ", error); \
|
||||
DISPLAYLEVEL(1, __VA_ARGS__); \
|
||||
DISPLAYLEVEL(1, " \n"); \
|
||||
exit(error); \
|
||||
}
|
||||
|
||||
|
||||
/*-*************************************
|
||||
* Functions
|
||||
***************************************/
|
||||
@@ -226,12 +231,14 @@ static FILE* FIO_openSrcFile(const char* srcFileName)
|
||||
f = stdin;
|
||||
SET_BINARY_MODE(stdin);
|
||||
} else {
|
||||
if (!UTIL_isRegFile(srcFileName)) {
|
||||
DISPLAYLEVEL(1, "zstd: %s is not a regular file -- ignored \n", srcFileName);
|
||||
if (!UTIL_isRegularFile(srcFileName)) {
|
||||
DISPLAYLEVEL(1, "zstd: %s is not a regular file -- ignored \n",
|
||||
srcFileName);
|
||||
return NULL;
|
||||
}
|
||||
f = fopen(srcFileName, "rb");
|
||||
if ( f==NULL ) DISPLAYLEVEL(1, "zstd: %s: %s \n", srcFileName, strerror(errno));
|
||||
if ( f==NULL )
|
||||
DISPLAYLEVEL(1, "zstd: %s: %s \n", srcFileName, strerror(errno));
|
||||
}
|
||||
|
||||
return f;
|
||||
@@ -256,26 +263,28 @@ static FILE* FIO_openDstFile(const char* dstFileName)
|
||||
if (g_sparseFileSupport == 1) {
|
||||
g_sparseFileSupport = ZSTD_SPARSE_DEFAULT;
|
||||
}
|
||||
if (strcmp (dstFileName, nulmark)) { /* Check if destination file already exists */
|
||||
if (strcmp (dstFileName, nulmark)) {
|
||||
/* Check if destination file already exists */
|
||||
f = fopen( dstFileName, "rb" );
|
||||
if (f != 0) { /* dest file exists, prompt for overwrite authorization */
|
||||
if (f != 0) { /* dst file exists, prompt for overwrite authorization */
|
||||
fclose(f);
|
||||
if (!g_overwrite) {
|
||||
if (g_displayLevel <= 1) {
|
||||
/* No interaction possible */
|
||||
DISPLAY("zstd: %s already exists; not overwritten \n", dstFileName);
|
||||
DISPLAY("zstd: %s already exists; not overwritten \n",
|
||||
dstFileName);
|
||||
return NULL;
|
||||
}
|
||||
DISPLAY("zstd: %s already exists; do you wish to overwrite (y/N) ? ", dstFileName);
|
||||
DISPLAY("zstd: %s already exists; do you wish to overwrite (y/N) ? ",
|
||||
dstFileName);
|
||||
{ int ch = getchar();
|
||||
if ((ch!='Y') && (ch!='y')) {
|
||||
DISPLAY(" not overwritten \n");
|
||||
return NULL;
|
||||
}
|
||||
while ((ch!=EOF) && (ch!='\n')) ch = getchar(); /* flush rest of input line */
|
||||
}
|
||||
}
|
||||
|
||||
/* flush rest of input line */
|
||||
while ((ch!=EOF) && (ch!='\n')) ch = getchar();
|
||||
} }
|
||||
/* need to unlink */
|
||||
FIO_remove(dstFileName);
|
||||
} }
|
||||
@@ -303,11 +312,13 @@ static size_t FIO_createDictBuffer(void** bufferPtr, const char* fileName)
|
||||
|
||||
DISPLAYLEVEL(4,"Loading %s as dictionary \n", fileName);
|
||||
fileHandle = fopen(fileName, "rb");
|
||||
if (fileHandle==0) EXM_THROW(31, "zstd: %s: %s", fileName, strerror(errno));
|
||||
if (fileHandle==0) EXM_THROW(31, "%s: %s", fileName, strerror(errno));
|
||||
fileSize = UTIL_getFileSize(fileName);
|
||||
if (fileSize > DICTSIZE_MAX) EXM_THROW(32, "Dictionary file %s is too large (> %u MB)", fileName, DICTSIZE_MAX >> 20); /* avoid extreme cases */
|
||||
if (fileSize > DICTSIZE_MAX)
|
||||
EXM_THROW(32, "Dictionary file %s is too large (> %u MB)",
|
||||
fileName, DICTSIZE_MAX >> 20); /* avoid extreme cases */
|
||||
*bufferPtr = malloc((size_t)fileSize);
|
||||
if (*bufferPtr==NULL) EXM_THROW(34, "zstd: %s", strerror(errno));
|
||||
if (*bufferPtr==NULL) EXM_THROW(34, "%s", strerror(errno));
|
||||
{ size_t const readSize = fread(*bufferPtr, 1, (size_t)fileSize, fileHandle);
|
||||
if (readSize!=fileSize) EXM_THROW(35, "Error reading dictionary file %s", fileName); }
|
||||
fclose(fileHandle);
|
||||
@@ -326,7 +337,7 @@ typedef struct {
|
||||
size_t srcBufferSize;
|
||||
void* dstBuffer;
|
||||
size_t dstBufferSize;
|
||||
#ifdef ZSTD_MULTITHREAD
|
||||
#if !defined(ZSTD_NEWAPI) && defined(ZSTD_MULTITHREAD)
|
||||
ZSTDMT_CCtx* cctx;
|
||||
#else
|
||||
ZSTD_CStream* cctx;
|
||||
@@ -334,15 +345,19 @@ typedef struct {
|
||||
} cRess_t;
|
||||
|
||||
static cRess_t FIO_createCResources(const char* dictFileName, int cLevel,
|
||||
U64 srcSize, int srcRegFile,
|
||||
U64 srcSize, int srcIsRegularFile,
|
||||
ZSTD_compressionParameters* comprParams) {
|
||||
cRess_t ress;
|
||||
memset(&ress, 0, sizeof(ress));
|
||||
|
||||
#ifdef ZSTD_MULTITHREAD
|
||||
#ifdef ZSTD_NEWAPI
|
||||
ress.cctx = ZSTD_createCCtx();
|
||||
if (ress.cctx == NULL)
|
||||
EXM_THROW(30, "allocation error : can't create ZSTD_CCtx");
|
||||
#elif defined(ZSTD_MULTITHREAD)
|
||||
ress.cctx = ZSTDMT_createCCtx(g_nbThreads);
|
||||
if (ress.cctx == NULL)
|
||||
EXM_THROW(30, "zstd: allocation error : can't create ZSTD_CStream");
|
||||
EXM_THROW(30, "allocation error : can't create ZSTDMT_CCtx");
|
||||
if ((cLevel==ZSTD_maxCLevel()) && (g_overlapLog==FIO_OVERLAP_LOG_NOTSET))
|
||||
/* use complete window for overlap */
|
||||
ZSTDMT_setMTCtxParameter(ress.cctx, ZSTDMT_p_overlapSectionLog, 9);
|
||||
@@ -351,22 +366,44 @@ static cRess_t FIO_createCResources(const char* dictFileName, int cLevel,
|
||||
#else
|
||||
ress.cctx = ZSTD_createCStream();
|
||||
if (ress.cctx == NULL)
|
||||
EXM_THROW(30, "zstd: allocation error : can't create ZSTD_CStream");
|
||||
EXM_THROW(30, "allocation error : can't create ZSTD_CStream");
|
||||
#endif
|
||||
ress.srcBufferSize = ZSTD_CStreamInSize();
|
||||
ress.srcBuffer = malloc(ress.srcBufferSize);
|
||||
ress.dstBufferSize = ZSTD_CStreamOutSize();
|
||||
ress.dstBuffer = malloc(ress.dstBufferSize);
|
||||
if (!ress.srcBuffer || !ress.dstBuffer)
|
||||
EXM_THROW(31, "zstd: allocation error : not enough memory");
|
||||
EXM_THROW(31, "allocation error : not enough memory");
|
||||
|
||||
/* dictionary */
|
||||
{ void* dictBuffer;
|
||||
size_t const dictBuffSize = FIO_createDictBuffer(&dictBuffer, dictFileName); /* works with dictFileName==NULL */
|
||||
if (dictFileName && (dictBuffer==NULL))
|
||||
EXM_THROW(32, "zstd: allocation error : can't create dictBuffer");
|
||||
EXM_THROW(32, "allocation error : can't create dictBuffer");
|
||||
|
||||
#ifdef ZSTD_NEWAPI
|
||||
{ /* frame parameters */
|
||||
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_contentSizeFlag, srcIsRegularFile) );
|
||||
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_dictIDFlag, g_dictIDFlag) );
|
||||
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_checksumFlag, g_checksumFlag) );
|
||||
CHECK( ZSTD_CCtx_setPledgedSrcSize(ress.cctx, srcSize) );
|
||||
/* compression parameters */
|
||||
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_compressionLevel, cLevel) );
|
||||
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_windowLog, comprParams->windowLog) );
|
||||
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_chainLog, comprParams->chainLog) );
|
||||
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_hashLog, comprParams->hashLog) );
|
||||
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_searchLog, comprParams->searchLog) );
|
||||
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_minMatch, comprParams->searchLength) );
|
||||
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_targetLength, comprParams->targetLength) );
|
||||
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_compressionStrategy, (U32)comprParams->strategy) );
|
||||
/* multi-threading */
|
||||
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_nbThreads, g_nbThreads) );
|
||||
/* dictionary */
|
||||
CHECK( ZSTD_CCtx_loadDictionary(ress.cctx, dictBuffer, dictBuffSize) );
|
||||
}
|
||||
#elif defined(ZSTD_MULTITHREAD)
|
||||
{ ZSTD_parameters params = ZSTD_getParams(cLevel, srcSize, dictBuffSize);
|
||||
params.fParams.contentSizeFlag = srcRegFile;
|
||||
params.fParams.contentSizeFlag = srcIsRegularFile;
|
||||
params.fParams.checksumFlag = g_checksumFlag;
|
||||
params.fParams.noDictIDFlag = !g_dictIDFlag;
|
||||
if (comprParams->windowLog) params.cParams.windowLog = comprParams->windowLog;
|
||||
@@ -376,15 +413,24 @@ static cRess_t FIO_createCResources(const char* dictFileName, int cLevel,
|
||||
if (comprParams->searchLength) params.cParams.searchLength = comprParams->searchLength;
|
||||
if (comprParams->targetLength) params.cParams.targetLength = comprParams->targetLength;
|
||||
if (comprParams->strategy) params.cParams.strategy = (ZSTD_strategy) comprParams->strategy;
|
||||
#ifdef ZSTD_MULTITHREAD
|
||||
{ size_t const errorCode = ZSTDMT_initCStream_advanced(ress.cctx, dictBuffer, dictBuffSize, params, srcSize);
|
||||
if (ZSTD_isError(errorCode)) EXM_THROW(33, "Error initializing CStream : %s", ZSTD_getErrorName(errorCode));
|
||||
ZSTDMT_setMTCtxParameter(ress.cctx, ZSTDMT_p_sectionSize, g_blockSize);
|
||||
CHECK( ZSTDMT_initCStream_advanced(ress.cctx, dictBuffer, dictBuffSize, params, srcSize) );
|
||||
ZSTDMT_setMTCtxParameter(ress.cctx, ZSTDMT_p_sectionSize, g_blockSize);
|
||||
}
|
||||
#else
|
||||
{ size_t const errorCode = ZSTD_initCStream_advanced(ress.cctx, dictBuffer, dictBuffSize, params, srcSize);
|
||||
if (ZSTD_isError(errorCode)) EXM_THROW(33, "Error initializing CStream : %s", ZSTD_getErrorName(errorCode));
|
||||
{ ZSTD_parameters params = ZSTD_getParams(cLevel, srcSize, dictBuffSize);
|
||||
params.fParams.contentSizeFlag = srcIsRegularFile;
|
||||
params.fParams.checksumFlag = g_checksumFlag;
|
||||
params.fParams.noDictIDFlag = !g_dictIDFlag;
|
||||
if (comprParams->windowLog) params.cParams.windowLog = comprParams->windowLog;
|
||||
if (comprParams->chainLog) params.cParams.chainLog = comprParams->chainLog;
|
||||
if (comprParams->hashLog) params.cParams.hashLog = comprParams->hashLog;
|
||||
if (comprParams->searchLog) params.cParams.searchLog = comprParams->searchLog;
|
||||
if (comprParams->searchLength) params.cParams.searchLength = comprParams->searchLength;
|
||||
if (comprParams->targetLength) params.cParams.targetLength = comprParams->targetLength;
|
||||
if (comprParams->strategy) params.cParams.strategy = (ZSTD_strategy) comprParams->strategy;
|
||||
CHECK( ZSTD_initCStream_advanced(ress.cctx, dictBuffer, dictBuffSize, params, srcSize) );
|
||||
}
|
||||
#endif
|
||||
} }
|
||||
free(dictBuffer);
|
||||
}
|
||||
|
||||
@@ -395,7 +441,7 @@ static void FIO_freeCResources(cRess_t ress)
|
||||
{
|
||||
free(ress.srcBuffer);
|
||||
free(ress.dstBuffer);
|
||||
#ifdef ZSTD_MULTITHREAD
|
||||
#if !defined(ZSTD_NEWAPI) && defined(ZSTD_MULTITHREAD)
|
||||
ZSTDMT_freeCCtx(ress.cctx);
|
||||
#else
|
||||
ZSTD_freeCStream(ress.cctx); /* never fails */
|
||||
@@ -700,41 +746,40 @@ static int FIO_compressFilename_internal(cRess_t ress,
|
||||
}
|
||||
|
||||
/* init */
|
||||
#ifdef ZSTD_MULTITHREAD
|
||||
{ size_t const resetError = ZSTDMT_resetCStream(ress.cctx, fileSize);
|
||||
#ifdef ZSTD_NEWAPI
|
||||
/* nothing, reset is implied */
|
||||
#elif defined(ZSTD_MULTITHREAD)
|
||||
CHECK( ZSTDMT_resetCStream(ress.cctx, fileSize) );
|
||||
#else
|
||||
{ size_t const resetError = ZSTD_resetCStream(ress.cctx, fileSize);
|
||||
CHECK( ZSTD_resetCStream(ress.cctx, fileSize) );
|
||||
#endif
|
||||
if (ZSTD_isError(resetError))
|
||||
EXM_THROW(21, "Error initializing compression : %s",
|
||||
ZSTD_getErrorName(resetError));
|
||||
}
|
||||
|
||||
/* Main compression loop */
|
||||
while (1) {
|
||||
/* Fill input Buffer */
|
||||
size_t const inSize = fread(ress.srcBuffer, (size_t)1, ress.srcBufferSize, srcFile);
|
||||
ZSTD_inBuffer inBuff = { ress.srcBuffer, inSize, 0 };
|
||||
if (inSize==0) break;
|
||||
readsize += inSize;
|
||||
|
||||
{ ZSTD_inBuffer inBuff = { ress.srcBuffer, inSize, 0 };
|
||||
while (inBuff.pos != inBuff.size) {
|
||||
ZSTD_outBuffer outBuff = { ress.dstBuffer, ress.dstBufferSize, 0 };
|
||||
#ifdef ZSTD_MULTITHREAD
|
||||
size_t const result = ZSTDMT_compressStream(ress.cctx, &outBuff, &inBuff);
|
||||
while (inBuff.pos != inBuff.size) {
|
||||
ZSTD_outBuffer outBuff = { ress.dstBuffer, ress.dstBufferSize, 0 };
|
||||
#ifdef ZSTD_NEWAPI
|
||||
CHECK( ZSTD_compress_generic(ress.cctx,
|
||||
&outBuff, &inBuff, ZSTD_e_continue) );
|
||||
#elif defined(ZSTD_MULTITHREAD)
|
||||
CHECK( ZSTDMT_compressStream(ress.cctx, &outBuff, &inBuff) );
|
||||
#else
|
||||
size_t const result = ZSTD_compressStream(ress.cctx, &outBuff, &inBuff);
|
||||
CHECK( ZSTD_compressStream(ress.cctx, &outBuff, &inBuff) );
|
||||
#endif
|
||||
if (ZSTD_isError(result))
|
||||
EXM_THROW(23, "Compression error : %s ", ZSTD_getErrorName(result));
|
||||
|
||||
/* Write compressed stream */
|
||||
if (outBuff.pos) {
|
||||
size_t const sizeCheck = fwrite(ress.dstBuffer, 1, outBuff.pos, dstFile);
|
||||
if (sizeCheck!=outBuff.pos)
|
||||
EXM_THROW(25, "Write error : cannot write compressed block into %s", dstFileName);
|
||||
compressedfilesize += outBuff.pos;
|
||||
} } }
|
||||
/* Write compressed stream */
|
||||
if (outBuff.pos) {
|
||||
size_t const sizeCheck = fwrite(ress.dstBuffer, 1, outBuff.pos, dstFile);
|
||||
if (sizeCheck!=outBuff.pos)
|
||||
EXM_THROW(25, "Write error : cannot write compressed block into %s", dstFileName);
|
||||
compressedfilesize += outBuff.pos;
|
||||
} }
|
||||
if (g_nbThreads > 1) {
|
||||
if (!fileSize)
|
||||
DISPLAYUPDATE(2, "\rRead : %u MB", (U32)(readsize>>20))
|
||||
@@ -757,12 +802,18 @@ static int FIO_compressFilename_internal(cRess_t ress,
|
||||
{ size_t result = 1;
|
||||
while (result!=0) { /* note : is there any possibility of endless loop ? */
|
||||
ZSTD_outBuffer outBuff = { ress.dstBuffer, ress.dstBufferSize, 0 };
|
||||
#ifdef ZSTD_MULTITHREAD
|
||||
#ifdef ZSTD_NEWAPI
|
||||
ZSTD_inBuffer inBuff = { NULL, 0, 0};
|
||||
result = ZSTD_compress_generic(ress.cctx,
|
||||
&outBuff, &inBuff, ZSTD_e_end);
|
||||
#elif defined(ZSTD_MULTITHREAD)
|
||||
result = ZSTDMT_endStream(ress.cctx, &outBuff);
|
||||
#else
|
||||
result = ZSTD_endStream(ress.cctx, &outBuff);
|
||||
#endif
|
||||
if (ZSTD_isError(result)) EXM_THROW(26, "Compression error during frame end : %s", ZSTD_getErrorName(result));
|
||||
if (ZSTD_isError(result))
|
||||
EXM_THROW(26, "Compression error during frame end : %s",
|
||||
ZSTD_getErrorName(result));
|
||||
{ size_t const sizeCheck = fwrite(ress.dstBuffer, 1, outBuff.pos, dstFile);
|
||||
if (sizeCheck!=outBuff.pos) EXM_THROW(27, "Write error : cannot write frame end into %s", dstFileName); }
|
||||
compressedfilesize += outBuff.pos;
|
||||
@@ -851,9 +902,9 @@ int FIO_compressFilename(const char* dstFileName, const char* srcFileName,
|
||||
{
|
||||
clock_t const start = clock();
|
||||
U64 const srcSize = UTIL_getFileSize(srcFileName);
|
||||
int const regFile = UTIL_isRegFile(srcFileName);
|
||||
int const isRegularFile = UTIL_isRegularFile(srcFileName);
|
||||
|
||||
cRess_t const ress = FIO_createCResources(dictFileName, compressionLevel, srcSize, regFile, comprParams);
|
||||
cRess_t const ress = FIO_createCResources(dictFileName, compressionLevel, srcSize, isRegularFile, comprParams);
|
||||
int const result = FIO_compressFilename_dstFile(ress, dstFileName, srcFileName, compressionLevel);
|
||||
|
||||
double const seconds = (double)(clock() - start) / CLOCKS_PER_SEC;
|
||||
@@ -1098,8 +1149,8 @@ int FIO_compressMultipleFilenames(const char** inFileNamesTable, unsigned nbFile
|
||||
char* dstFileName = (char*)malloc(FNSPACE);
|
||||
size_t const suffixSize = suffix ? strlen(suffix) : 0;
|
||||
U64 const srcSize = (nbFiles != 1) ? 0 : UTIL_getFileSize(inFileNamesTable[0]) ;
|
||||
int const regFile = (nbFiles != 1) ? 0 : UTIL_isRegFile(inFileNamesTable[0]);
|
||||
cRess_t ress = FIO_createCResources(dictFileName, compressionLevel, srcSize, regFile, comprParams);
|
||||
int const isRegularFile = (nbFiles != 1) ? 0 : UTIL_isRegularFile(inFileNamesTable[0]);
|
||||
cRess_t ress = FIO_createCResources(dictFileName, compressionLevel, srcSize, isRegularFile, comprParams);
|
||||
|
||||
/* init */
|
||||
if (dstFileName==NULL)
|
||||
@@ -1177,9 +1228,7 @@ static dRess_t FIO_createDResources(const char* dictFileName)
|
||||
/* dictionary */
|
||||
{ void* dictBuffer;
|
||||
size_t const dictBufferSize = FIO_createDictBuffer(&dictBuffer, dictFileName);
|
||||
size_t const initError = ZSTD_initDStream_usingDict(ress.dctx, dictBuffer, dictBufferSize);
|
||||
if (ZSTD_isError(initError))
|
||||
EXM_THROW(61, "ZSTD_initDStream_usingDict error : %s", ZSTD_getErrorName(initError));
|
||||
CHECK( ZSTD_initDStream_usingDict(ress.dctx, dictBuffer, dictBufferSize) );
|
||||
free(dictBuffer);
|
||||
}
|
||||
|
||||
@@ -1188,10 +1237,7 @@ static dRess_t FIO_createDResources(const char* dictFileName)
|
||||
|
||||
static void FIO_freeDResources(dRess_t ress)
|
||||
{
|
||||
size_t const errorCode = ZSTD_freeDStream(ress.dctx);
|
||||
if (ZSTD_isError(errorCode))
|
||||
EXM_THROW(69, "Error : can't free ZSTD_DStream context resource : %s",
|
||||
ZSTD_getErrorName(errorCode));
|
||||
CHECK( ZSTD_freeDStream(ress.dctx) );
|
||||
free(ress.srcBuffer);
|
||||
free(ress.dstBuffer);
|
||||
}
|
||||
|
||||
+1
-1
@@ -208,7 +208,7 @@ UTIL_STATIC int UTIL_getFileStat(const char* infilename, stat_t *statbuf)
|
||||
}
|
||||
|
||||
|
||||
UTIL_STATIC int UTIL_isRegFile(const char* infilename)
|
||||
UTIL_STATIC int UTIL_isRegularFile(const char* infilename)
|
||||
{
|
||||
stat_t statbuf;
|
||||
return UTIL_getFileStat(infilename, &statbuf); /* Only need to know whether it is a regular file */
|
||||
|
||||
Binary file not shown.
Binary file not shown.
+3
-3
@@ -1,5 +1,5 @@
|
||||
.
|
||||
.TH "ZSTD" "1" "May 2017" "zstd 1.3.0" "User Commands"
|
||||
.TH "ZSTD" "1" "June 2017" "zstd 1.3.0" "User Commands"
|
||||
.
|
||||
.SH "NAME"
|
||||
\fBzstd\fR \- zstd, zstdmt, unzstd, zstdcat \- Compress or decompress \.zst files
|
||||
@@ -235,8 +235,8 @@ benchmark file(s) using multiple compression levels, from \fB\-b#\fR to \fB\-e#\
|
||||
minimum evaluation time, in seconds (default : 3s), benchmark mode only
|
||||
.
|
||||
.TP
|
||||
\fB\-B#\fR
|
||||
cut file into independent blocks of size # (default: no block)
|
||||
\fB\-B#\fR, \fB\-\-block\-size=#\fR
|
||||
cut file(s) into independent blocks of size # (default: no block)
|
||||
.
|
||||
.TP
|
||||
\fB\-\-priority=rt\fR
|
||||
|
||||
+4
-4
@@ -234,8 +234,8 @@ BENCHMARK
|
||||
benchmark file(s) using multiple compression levels, from `-b#` to `-e#` (inclusive)
|
||||
* `-i#`:
|
||||
minimum evaluation time, in seconds (default : 3s), benchmark mode only
|
||||
* `-B#`:
|
||||
cut file into independent blocks of size # (default: no block)
|
||||
* `-B#`, `--block-size=#`:
|
||||
cut file(s) into independent blocks of size # (default: no block)
|
||||
* `--priority=rt`:
|
||||
set process priority to real-time
|
||||
|
||||
@@ -255,8 +255,8 @@ The list of available _options_:
|
||||
Specify a strategy used by a match finder.
|
||||
|
||||
There are 8 strategies numbered from 0 to 7, from faster to stronger:
|
||||
0=ZSTD\_fast, 1=ZSTD\_dfast, 2=ZSTD\_greedy, 3=ZSTD\_lazy,
|
||||
4=ZSTD\_lazy2, 5=ZSTD\_btlazy2, 6=ZSTD\_btopt, 7=ZSTD\_btultra.
|
||||
1=ZSTD\_fast, 2=ZSTD\_dfast, 3=ZSTD\_greedy, 4=ZSTD\_lazy,
|
||||
5=ZSTD\_lazy2, 6=ZSTD\_btlazy2, 7=ZSTD\_btopt, 8=ZSTD\_btultra.
|
||||
|
||||
- `windowLog`=_wlog_, `wlog`=_wlog_:
|
||||
Specify the maximum number of bits for a match distance.
|
||||
|
||||
+1
-1
@@ -300,7 +300,7 @@ static unsigned parseCompressionParameters(const char* stringPtr, ZSTD_compressi
|
||||
if (longCommandWArg(&stringPtr, "searchLog=") || longCommandWArg(&stringPtr, "slog=")) { params->searchLog = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
|
||||
if (longCommandWArg(&stringPtr, "searchLength=") || longCommandWArg(&stringPtr, "slen=")) { params->searchLength = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
|
||||
if (longCommandWArg(&stringPtr, "targetLength=") || longCommandWArg(&stringPtr, "tlen=")) { params->targetLength = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
|
||||
if (longCommandWArg(&stringPtr, "strategy=") || longCommandWArg(&stringPtr, "strat=")) { params->strategy = (ZSTD_strategy)(1 + readU32FromChar(&stringPtr)); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
|
||||
if (longCommandWArg(&stringPtr, "strategy=") || longCommandWArg(&stringPtr, "strat=")) { params->strategy = (ZSTD_strategy)(readU32FromChar(&stringPtr)); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
|
||||
if (longCommandWArg(&stringPtr, "overlapLog=") || longCommandWArg(&stringPtr, "ovlog=")) { g_overlapLog = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
|
||||
return 0;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user