ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'dev' into benchfn

Browse Source
This commit is contained in:
Yann Collet
2019-04-10 11:57:05 -07:00
parent 90c0462d63 f86d4bd1d5
commit 094c000904
84 changed files with 4136 additions and 2229 deletions
Download Patch File Download Diff File Expand all files Collapse all files
programs/Makefile
+1 -1
View File
@@ -51,7 +51,7 @@ endif
CFLAGS ?= -O3
DEBUGFLAGS+=-Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \
-Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \
-Wstrict-prototypes -Wundef -Wpointer-arith -Wformat-security \
-Wstrict-prototypes -Wundef -Wpointer-arith \
-Wvla -Wformat=2 -Winit-self -Wfloat-equal -Wwrite-strings \
-Wredundant-decls -Wmissing-prototypes -Wc++-compat
CFLAGS += $(DEBUGFLAGS) $(MOREFLAGS)
programs/benchzstd.c
+3 -1
View File
@@ -135,7 +135,8 @@ BMK_advancedParams_t BMK_initAdvancedParams(void) {
0, /* ldmMinMatch */
0, /* ldmHashLog */
0, /* ldmBuckSizeLog */
0 /* ldmHashRateLog */
0, /* ldmHashRateLog */
ZSTD_lcm_auto /* literalCompressionMode */
};
return res;
}
@@ -184,6 +185,7 @@ BMK_initCCtx(ZSTD_CCtx* ctx,
CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_searchLog, (int)comprParams->searchLog));
CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_minMatch, (int)comprParams->minMatch));
CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_targetLength, (int)comprParams->targetLength));
CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_literalCompressionMode, (int)adv->literalCompressionMode));
CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_strategy, comprParams->strategy));
CHECK_Z(ZSTD_CCtx_loadDictionary(ctx, dictBuffer, dictBufferSize));
}
programs/benchzstd.h
+1
View File
@@ -116,6 +116,7 @@ typedef struct {
int ldmHashLog;
int ldmBucketSizeLog;
int ldmHashRateLog;
ZSTD_literalCompressionMode_e literalCompressionMode;
} BMK_advancedParams_t;
/* returns default parameters used by nonAdvanced functions */
programs/fileio.c
+23 -24
View File
@@ -24,7 +24,7 @@
* Includes
***************************************/
#include "platform.h" /* Large Files support, SET_BINARY_MODE */
#include "util.h" /* UTIL_getFileSize, UTIL_isRegularFile */
#include "util.h" /* UTIL_getFileSize, UTIL_isRegularFile, UTIL_isSameFile */
#include <stdio.h> /* fprintf, fopen, fread, _fileno, stdin, stdout */
#include <stdlib.h> /* malloc, free */
#include <string.h> /* strcmp, strlen */
@@ -296,6 +296,7 @@ struct FIO_prefs_s {
int ldmMinMatch;
int ldmBucketSizeLog;
int ldmHashRateLog;
ZSTD_literalCompressionMode_e literalCompressionMode;
/* IO preferences */
U32 removeSrcFile;
@@ -339,6 +340,7 @@ FIO_prefs_t* FIO_createPreferences(void)
ret->ldmMinMatch = 0;
ret->ldmBucketSizeLog = FIO_LDM_PARAM_NOTSET;
ret->ldmHashRateLog = FIO_LDM_PARAM_NOTSET;
ret->literalCompressionMode = ZSTD_lcm_auto;
return ret;
}
@@ -406,6 +408,12 @@ void FIO_setRsyncable(FIO_prefs_t* const prefs, int rsyncable) {
prefs->rsyncable = rsyncable;
}
void FIO_setLiteralCompressionMode(
FIO_prefs_t* const prefs,
ZSTD_literalCompressionMode_e mode) {
prefs->literalCompressionMode = mode;
}
void FIO_setAdaptMin(FIO_prefs_t* const prefs, int minCLevel)
{
#ifndef ZSTD_NOCOMPRESS
@@ -507,28 +515,10 @@ static FILE* FIO_openDstFile(FIO_prefs_t* const prefs, const char* srcFileName,
return stdout;
}
/* ensure dst is not the same file as src */
if (srcFileName != NULL) {
#ifdef _MSC_VER
/* note : Visual does not support file identification by inode.
* The following work-around is limited to detecting exact name repetition only,
* aka `filename` is considered different from `subdir/../filename` */
if (!strcmp(srcFileName, dstFileName)) {
DISPLAYLEVEL(1, "zstd: Refusing to open a output file which will overwrite the input file \n");
return NULL;
}
#else
stat_t srcStat;
stat_t dstStat;
if (UTIL_getFileStat(srcFileName, &srcStat)
&& UTIL_getFileStat(dstFileName, &dstStat)) {
if (srcStat.st_dev == dstStat.st_dev
&& srcStat.st_ino == dstStat.st_ino) {
DISPLAYLEVEL(1, "zstd: Refusing to open a output file which will overwrite the input file \n");
return NULL;
}
}
#endif
/* ensure dst is not the same as src */
if (srcFileName != NULL && UTIL_isSameFile(srcFileName, dstFileName)) {
DISPLAYLEVEL(1, "zstd: Refusing to open an output file which will overwrite the input file \n");
return NULL;
}
if (prefs->sparseFileSupport == 1) {
@@ -620,6 +610,7 @@ typedef struct {
size_t srcBufferSize;
void* dstBuffer;
size_t dstBufferSize;
const char* dictFileName;
ZSTD_CStream* cctx;
} cRess_t;
@@ -647,6 +638,7 @@ static cRess_t FIO_createCResources(FIO_prefs_t* const prefs,
size_t const dictBuffSize = FIO_createDictBuffer(&dictBuffer, dictFileName); /* works with dictFileName==NULL */
if (dictFileName && (dictBuffer==NULL))
EXM_THROW(32, "allocation error : can't create dictBuffer");
ress.dictFileName = dictFileName;
if (prefs->adaptiveMode && !prefs->ldmFlag && !comprParams.windowLog)
comprParams.windowLog = ADAPT_WINDOWLOG_DEFAULT;
@@ -674,6 +666,7 @@ static cRess_t FIO_createCResources(FIO_prefs_t* const prefs,
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_minMatch, (int)comprParams.minMatch) );
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_targetLength, (int)comprParams.targetLength) );
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_strategy, comprParams.strategy) );
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_literalCompressionMode, (int)prefs->literalCompressionMode) );
/* multi-threading */
#ifdef ZSTD_MULTITHREAD
DISPLAYLEVEL(5,"set nb workers = %u \n", prefs->nbWorkers);
@@ -1299,12 +1292,18 @@ FIO_compressFilename_srcFile(FIO_prefs_t* const prefs,
{
int result;
/* File check */
/* ensure src is not a directory */
if (UTIL_isDirectory(srcFileName)) {
DISPLAYLEVEL(1, "zstd: %s is a directory -- ignored \n", srcFileName);
return 1;
}
/* ensure src is not the same as dict (if present) */
if (ress.dictFileName != NULL && UTIL_isSameFile(srcFileName, ress.dictFileName)) {
DISPLAYLEVEL(1, "zstd: cannot use %s as an input file and dictionary \n", srcFileName);
return 1;
}
ress.srcFile = FIO_openSrcFile(srcFileName);
if (ress.srcFile == NULL) return 1; /* srcFile could not be opened */
programs/fileio.h
+3
View File
@@ -71,6 +71,9 @@ void FIO_setOverlapLog(FIO_prefs_t* const prefs, int overlapLog);
void FIO_setRemoveSrcFile(FIO_prefs_t* const prefs, unsigned flag);
void FIO_setSparseWrite(FIO_prefs_t* const prefs, unsigned sparse); /**< 0: no sparse; 1: disable on stdout; 2: always enabled */
void FIO_setRsyncable(FIO_prefs_t* const prefs, int rsyncable);
void FIO_setLiteralCompressionMode(
FIO_prefs_t* const prefs,
ZSTD_literalCompressionMode_e mode);
void FIO_setNoProgress(unsigned noProgress);
void FIO_setNotificationLevel(int level);
programs/util.c
+20 -1
View File
@@ -87,6 +87,23 @@ U32 UTIL_isDirectory(const char* infilename)
return 0;
}
int UTIL_isSameFile(const char* file1, const char* file2)
{
#if defined(_MSC_VER)
/* note : Visual does not support file identification by inode.
* The following work-around is limited to detecting exact name repetition only,
* aka `filename` is considered different from `subdir/../filename` */
return !strcmp(file1, file2);
#else
stat_t file1Stat;
stat_t file2Stat;
return UTIL_getFileStat(file1, &file1Stat)
&& UTIL_getFileStat(file2, &file2Stat)
&& (file1Stat.st_dev == file2Stat.st_dev)
&& (file1Stat.st_ino == file2Stat.st_ino);
#endif
}
U32 UTIL_isLink(const char* infilename)
{
/* macro guards, as defined in : https://linux.die.net/man/2/lstat */
@@ -95,7 +112,9 @@ U32 UTIL_isLink(const char* infilename)
|| (defined(_XOPEN_SOURCE) && (_XOPEN_SOURCE >= 500)) \
|| (defined(_XOPEN_SOURCE) && defined(_XOPEN_SOURCE_EXTENDED)) \
|| (defined(_POSIX_C_SOURCE) && (_POSIX_C_SOURCE >= 200112L)) \
|| (defined(__APPLE__) && defined(__MACH__))
|| (defined(__APPLE__) && defined(__MACH__)) \
|| defined(__OpenBSD__) \
|| defined(__FreeBSD__)
int r;
stat_t statbuf;
r = lstat(infilename, &statbuf);
programs/util.h
+1
View File
@@ -174,6 +174,7 @@ int UTIL_isRegularFile(const char* infilename);
int UTIL_setFileStat(const char* filename, stat_t* statbuf);
U32 UTIL_isDirectory(const char* infilename);
int UTIL_getFileStat(const char* infilename, stat_t* statbuf);
int UTIL_isSameFile(const char* file1, const char* file2);
U32 UTIL_isLink(const char* infilename);
#define UTIL_FILESIZE_UNKNOWN ((U64)(-1))
programs/zstd.1
+1 -1
View File
@@ -164,7 +164,7 @@ keep source file(s) after successful compression or decompression\. This is the
.
.TP
\fB\-r\fR
operate recursively on dictionaries
operate recursively on directories
.
.TP
\fB\-\-format=FORMAT\fR
programs/zstd.1.md
+1 -1
View File
@@ -178,7 +178,7 @@ the last one takes effect.
keep source file(s) after successful compression or decompression.
This is the default behavior.
* `-r`:
operate recursively on dictionaries
operate recursively on directories
* `--format=FORMAT`:
compress and decompress in other formats. If compiled with
support, zstd can compress to or decompress from other compression algorithm
programs/zstdcli.c
+9 -1
View File
@@ -148,6 +148,7 @@ static int usage_advanced(const char* programName)
#endif
DISPLAY( "--no-dictID : don't write dictID into header (dictionary compression)\n");
DISPLAY( "--[no-]check : integrity check (default: enabled) \n");
DISPLAY( "--[no-]compress-literals : force (un)compressed literals \n");
#endif
#ifdef UTIL_HAS_CREATEFILELIST
DISPLAY( " -r : operate recursively on directories \n");
@@ -569,6 +570,7 @@ int main(int argCount, const char* argv[])
#ifndef ZSTD_NOBENCH
BMK_advancedParams_t benchParams = BMK_initAdvancedParams();
#endif
ZSTD_literalCompressionMode_e literalCompressionMode = ZSTD_lcm_auto;
/* init */
@@ -661,6 +663,8 @@ int main(int argCount, const char* argv[])
if (!strcmp(argument, "--format=lz4")) { suffix = LZ4_EXTENSION; FIO_setCompressionType(prefs, FIO_lz4Compression); continue; }
#endif
if (!strcmp(argument, "--rsyncable")) { rsyncable = 1; continue; }
if (!strcmp(argument, "--compress-literals")) { literalCompressionMode = ZSTD_lcm_huffman; continue; }
if (!strcmp(argument, "--no-compress-literals")) { literalCompressionMode = ZSTD_lcm_uncompressed; continue; }
if (!strcmp(argument, "--no-progress")) { FIO_setNoProgress(1); continue; }
/* long commands with arguments */
@@ -953,6 +957,8 @@ int main(int argCount, const char* argv[])
filenameTable[fileNamesNb++] = filenameTable[u];
}
}
if (fileNamesNb == 0 && filenameIdx > 0)
CLEAN_RETURN(1);
filenameIdx = fileNamesNb;
}
if (recursive) { /* at this stage, filenameTable is a list of paths, which can contain both files and directories */
@@ -995,6 +1001,7 @@ int main(int argCount, const char* argv[])
if (g_ldmHashRateLog != LDM_PARAM_DEFAULT) {
benchParams.ldmHashRateLog = g_ldmHashRateLog;
}
benchParams.literalCompressionMode = literalCompressionMode;
if (cLevel > ZSTD_maxCLevel()) cLevel = ZSTD_maxCLevel();
if (cLevelLast > ZSTD_maxCLevel()) cLevelLast = ZSTD_maxCLevel();
@@ -1108,6 +1115,7 @@ int main(int argCount, const char* argv[])
FIO_setAdaptMin(prefs, adaptMin);
FIO_setAdaptMax(prefs, adaptMax);
FIO_setRsyncable(prefs, rsyncable);
FIO_setLiteralCompressionMode(prefs, literalCompressionMode);
if (adaptMin > cLevel) cLevel = adaptMin;
if (adaptMax < cLevel) cLevel = adaptMax;
@@ -1116,7 +1124,7 @@ int main(int argCount, const char* argv[])
else
operationResult = FIO_compressMultipleFilenames(prefs, filenameTable, filenameIdx, outFileName, suffix, dictFileName, cLevel, compressionParams);
#else
(void)suffix; (void)adapt; (void)rsyncable; (void)ultra; (void)cLevel; (void)ldmFlag; /* not used when ZSTD_NOCOMPRESS set */
(void)suffix; (void)adapt; (void)rsyncable; (void)ultra; (void)cLevel; (void)ldmFlag; (void)literalCompressionMode; /* not used when ZSTD_NOCOMPRESS set */
DISPLAY("Compression not supported \n");
#endif
} else { /* decompression or test */