ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'dev' of github.com:Cyan4973/zstd into dev

Browse Source
This commit is contained in:
Yann Collet
2016-09-22 15:57:56 +02:00
parent 51f4d566c2 97b378a6f8
commit 21412bb3f6
8 changed files with 80 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+2 -1
View File
@@ -79,7 +79,8 @@ Once you have the repository cloned, there are multiple ways provided to build Z
#### Makefile
If your system is compatible with `make`, you can simply run `make` at the root directory.
If your system is compatible with a standard `make` (or `gmake`) binary generator,
you can simply run it at the root directory.
It will generate `zstd` within root directory.
Other available options include :
lib/compress/zstd_compress.c
+4 -5
View File
@@ -322,13 +322,12 @@ static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc,
* Duplicate an existing context `srcCCtx` into another one `dstCCtx`.
* Only works during stage ZSTDcs_init (i.e. after creation, but before first call to ZSTD_compressContinue()).
* @return : 0, or an error code */
size_t ZSTD_copyCCtx(ZSTD_CCtx* dstCCtx, const ZSTD_CCtx* srcCCtx)
size_t ZSTD_copyCCtx(ZSTD_CCtx* dstCCtx, const ZSTD_CCtx* srcCCtx, unsigned long long pledgedSrcSize)
{
if (srcCCtx->stage!=ZSTDcs_init) return ERROR(stage_wrong);
memcpy(&dstCCtx->customMem, &srcCCtx->customMem, sizeof(ZSTD_customMem));
ZSTD_resetCCtx_advanced(dstCCtx, srcCCtx->params, srcCCtx->frameContentSize, ZSTDcrp_noMemset);
dstCCtx->params.fParams.contentSizeFlag = 0; /* content size different from the one set during srcCCtx init */
ZSTD_resetCCtx_advanced(dstCCtx, srcCCtx->params, pledgedSrcSize, ZSTDcrp_noMemset);
/* copy tables */
{ size_t const chainSize = (srcCCtx->params.cParams.strategy == ZSTD_fast) ? 0 : (1 << srcCCtx->params.cParams.chainLog);
@@ -2730,7 +2729,7 @@ ZSTD_CDict* ZSTD_createCDict(const void* dict, size_t dictSize, int compressionL
size_t ZSTD_freeCDict(ZSTD_CDict* cdict)
{
if (cdict==NULL) return 0; /* support free on NULL */
{ ZSTD_customMem cMem = cdict->refContext->customMem;
{ ZSTD_customMem const cMem = cdict->refContext->customMem;
ZSTD_freeCCtx(cdict->refContext);
ZSTD_free(cdict->dictContent, cMem);
ZSTD_free(cdict, cMem);
@@ -2740,7 +2739,7 @@ size_t ZSTD_freeCDict(ZSTD_CDict* cdict)
size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict, U64 pledgedSrcSize)
{
if (cdict->dictContentSize) CHECK_F(ZSTD_copyCCtx(cctx, cdict->refContext))
if (cdict->dictContentSize) CHECK_F(ZSTD_copyCCtx(cctx, cdict->refContext, pledgedSrcSize))
else CHECK_F(ZSTD_compressBegin_advanced(cctx, NULL, 0, cdict->refContext->params, pledgedSrcSize));
return 0;
}
lib/dictBuilder/zdict.c
+1 -1
View File
@@ -563,7 +563,7 @@ static void ZDICT_countEStats(EStats_ress_t esr, ZSTD_parameters params,
size_t cSize;
if (srcSize > blockSizeMax) srcSize = blockSizeMax; /* protection vs large samples */
{ size_t const errorCode = ZSTD_copyCCtx(esr.zc, esr.ref);
{ size_t const errorCode = ZSTD_copyCCtx(esr.zc, esr.ref, 0);
if (ZSTD_isError(errorCode)) { DISPLAYLEVEL(1, "warning : ZSTD_copyCCtx failed \n"); return; }
}
cSize = ZSTD_compressBlock(esr.zc, esr.workPlace, ZSTD_BLOCKSIZE_ABSOLUTEMAX, src, srcSize);
lib/zstd.h
+1 -1
View File
@@ -447,7 +447,7 @@ ZSTDLIB_API size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds);
ZSTDLIB_API size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel);
ZSTDLIB_API size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel);
ZSTDLIB_API size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize);
ZSTDLIB_API size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx);
ZSTDLIB_API size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize);
ZSTDLIB_API size_t ZSTD_compressContinue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
ZSTDLIB_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
programs/fileio.c
+7 -5
View File
@@ -255,7 +255,7 @@ typedef struct {
FILE* srcFile;
} cRess_t;
static cRess_t FIO_createCResources(const char* dictFileName, int cLevel)
static cRess_t FIO_createCResources(const char* dictFileName, int cLevel, U64 srcSize)
{
cRess_t ress;
memset(&ress, 0, sizeof(ress));
@@ -272,11 +272,11 @@ static cRess_t FIO_createCResources(const char* dictFileName, int cLevel)
{ void* dictBuffer;
size_t const dictBuffSize = FIO_loadFile(&dictBuffer, dictFileName);
if (dictFileName && (dictBuffer==NULL)) EXM_THROW(32, "zstd: allocation error : can't create dictBuffer");
{ ZSTD_parameters params = ZSTD_getParams(cLevel, 0, dictBuffSize);
{ ZSTD_parameters params = ZSTD_getParams(cLevel, srcSize, dictBuffSize);
params.fParams.contentSizeFlag = 1;
params.fParams.checksumFlag = g_checksumFlag;
params.fParams.noDictIDFlag = !g_dictIDFlag;
{ size_t const errorCode = ZSTD_initCStream_advanced(ress.cctx, dictBuffer, dictBuffSize, params, 0);
{ size_t const errorCode = ZSTD_initCStream_advanced(ress.cctx, dictBuffer, dictBuffSize, params, srcSize);
if (ZSTD_isError(errorCode)) EXM_THROW(33, "Error initializing CStream : %s", ZSTD_getErrorName(errorCode));
} }
free(dictBuffer);
@@ -408,8 +408,9 @@ int FIO_compressFilename(const char* dstFileName, const char* srcFileName,
const char* dictFileName, int compressionLevel)
{
clock_t const start = clock();
U64 const srcSize = UTIL_getFileSize(srcFileName);
cRess_t const ress = FIO_createCResources(dictFileName, compressionLevel);
cRess_t const ress = FIO_createCResources(dictFileName, compressionLevel, srcSize);
int const result = FIO_compressFilename_dstFile(ress, dstFileName, srcFileName);
double const seconds = (double)(clock() - start) / CLOCKS_PER_SEC;
@@ -428,7 +429,8 @@ int FIO_compressMultipleFilenames(const char** inFileNamesTable, unsigned nbFile
size_t dfnSize = FNSPACE;
char* dstFileName = (char*)malloc(FNSPACE);
size_t const suffixSize = suffix ? strlen(suffix) : 0;
cRess_t ress = FIO_createCResources(dictFileName, compressionLevel);
U64 const srcSize = (nbFiles != 1) ? 0 : UTIL_getFileSize(inFileNamesTable[0]) ;
cRess_t ress = FIO_createCResources(dictFileName, compressionLevel, srcSize);
/* init */
if (dstFileName==NULL) EXM_THROW(27, "FIO_compressMultipleFilenames : allocation error for dstFileName");
programs/zstdcli.c
+35 -27
View File
@@ -23,7 +23,7 @@
#endif
#ifndef ZSTDCLI_CLEVEL_MAX
# define ZSTDCLI_CLEVEL_MAX 19
# define ZSTDCLI_CLEVEL_MAX 19 /* when not using --ultra */
#endif
@@ -51,13 +51,11 @@
#if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) || defined(__CYGWIN__)
# include <io.h> /* _isatty */
# define IS_CONSOLE(stdStream) _isatty(_fileno(stdStream))
#elif defined(_POSIX_C_SOURCE) || defined(_XOPEN_SOURCE) || defined(_POSIX_SOURCE) || (defined(__APPLE__) && defined(__MACH__)) /* https://sourceforge.net/p/predef/wiki/OperatingSystems/ */
# include <unistd.h> /* isatty */
# define IS_CONSOLE(stdStream) isatty(fileno(stdStream))
#else
# if defined(_POSIX_C_SOURCE) || defined(_XOPEN_SOURCE) || defined(_POSIX_SOURCE) || (defined(__APPLE__) && defined(__MACH__)) /* https://sourceforge.net/p/predef/wiki/OperatingSystems/ */
# include <unistd.h> /* isatty */
# define IS_CONSOLE(stdStream) isatty(fileno(stdStream))
# else
# define IS_CONSOLE(stdStream) 0
# endif
# define IS_CONSOLE(stdStream) 0
#endif
@@ -195,7 +193,7 @@ static unsigned readU32FromChar(const char** stringPtr)
#define CLEAN_RETURN(i) { operationResult = (i); goto _end; }
int main(int argCount, char** argv)
int main(int argCount, const char* argv[])
{
int argNb,
bench=0,
@@ -210,7 +208,8 @@ int main(int argCount, char** argv)
nextArgumentIsMaxDict=0,
nextArgumentIsDictID=0,
nextArgumentIsFile=0,
ultra=0;
ultra=0,
lastCommand = 0;
int cLevel = ZSTDCLI_CLEVEL_DEFAULT;
int cLevelLast = 1;
unsigned recursive = 0;
@@ -219,13 +218,12 @@ int main(int argCount, char** argv)
const char* programName = argv[0];
const char* outFileName = NULL;
const char* dictFileName = NULL;
char* dynNameSpace = NULL;
unsigned maxDictSize = g_defaultMaxDictSize;
unsigned dictID = 0;
int dictCLevel = g_defaultDictCLevel;
unsigned dictSelect = g_defaultSelectivityLevel;
#ifdef UTIL_HAS_CREATEFILELIST
const char** fileNamesTable = NULL;
const char** extendedFileList = NULL;
char* fileNamesBuf = NULL;
unsigned fileNamesNb;
#endif
@@ -271,8 +269,8 @@ int main(int argCount, char** argv)
if (!strcmp(argument, "--no-sparse")) { FIO_setSparseWrite(0); continue; }
if (!strcmp(argument, "--test")) { testmode=1; decode=1; continue; }
if (!strcmp(argument, "--train")) { dictBuild=1; outFileName=g_defaultDictName; continue; }
if (!strcmp(argument, "--maxdict")) { nextArgumentIsMaxDict=1; continue; }
if (!strcmp(argument, "--dictID")) { nextArgumentIsDictID=1; continue; }
if (!strcmp(argument, "--maxdict")) { nextArgumentIsMaxDict=1; lastCommand=1; continue; }
if (!strcmp(argument, "--dictID")) { nextArgumentIsDictID=1; lastCommand=1; continue; }
if (!strcmp(argument, "--keep")) { FIO_setRemoveSrcFile(0); continue; }
if (!strcmp(argument, "--rm")) { FIO_setRemoveSrcFile(1); continue; }
@@ -290,6 +288,10 @@ int main(int argCount, char** argv)
argument++;
while (argument[0]!=0) {
if (lastCommand) {
DISPLAY("error : command must be followed by argument \n");
return 1;
}
#ifndef ZSTD_NOCOMPRESS
/* compression Level */
if ((*argument>='0') && (*argument<='9')) {
@@ -312,7 +314,7 @@ int main(int argCount, char** argv)
case 'c': forceStdout=1; outFileName=stdoutmark; argument++; break;
/* Use file content as dictionary */
case 'D': nextEntryIsDictionary = 1; argument++; break;
case 'D': nextEntryIsDictionary = 1; lastCommand = 1; argument++; break;
/* Overwrite */
case 'f': FIO_overwriteMode(); forceStdout=1; argument++; break;
@@ -333,7 +335,7 @@ int main(int argCount, char** argv)
case 't': testmode=1; decode=1; argument++; break;
/* destination file name */
case 'o': nextArgumentIsOutFileName=1; argument++; break;
case 'o': nextArgumentIsOutFileName=1; lastCommand=1; argument++; break;
#ifdef UTIL_HAS_CREATEFILELIST
/* recursive */
@@ -397,6 +399,7 @@ int main(int argCount, char** argv)
if (nextArgumentIsMaxDict) {
nextArgumentIsMaxDict = 0;
lastCommand = 0;
maxDictSize = readU32FromChar(&argument);
if (toupper(*argument)=='K') maxDictSize <<= 10;
if (toupper(*argument)=='M') maxDictSize <<= 20;
@@ -405,6 +408,7 @@ int main(int argCount, char** argv)
if (nextArgumentIsDictID) {
nextArgumentIsDictID = 0;
lastCommand = 0;
dictID = readU32FromChar(&argument);
continue;
}
@@ -413,12 +417,14 @@ int main(int argCount, char** argv)
if (nextEntryIsDictionary) {
nextEntryIsDictionary = 0;
lastCommand = 0;
dictFileName = argument;
continue;
}
if (nextArgumentIsOutFileName) {
nextArgumentIsOutFileName = 0;
lastCommand = 0;
outFileName = argument;
if (!strcmp(outFileName, "-")) outFileName = stdoutmark;
continue;
@@ -428,17 +434,19 @@ int main(int argCount, char** argv)
filenameTable[filenameIdx++] = argument;
}
if (lastCommand) { DISPLAY("error : command must be followed by argument \n"); return 1; } /* forgotten argument */
/* Welcome message (if verbose) */
DISPLAYLEVEL(3, WELCOME_MESSAGE);
#ifdef UTIL_HAS_CREATEFILELIST
if (recursive) {
fileNamesTable = UTIL_createFileList(filenameTable, filenameIdx, &fileNamesBuf, &fileNamesNb);
if (fileNamesTable) {
if (recursive) { /* at this stage, filenameTable is a list of paths, which can contain both files and directories */
extendedFileList = UTIL_createFileList(filenameTable, filenameIdx, &fileNamesBuf, &fileNamesNb);
if (extendedFileList) {
unsigned u;
for (u=0; u<fileNamesNb; u++) DISPLAYLEVEL(4, "%u %s\n", u, fileNamesTable[u]);
for (u=0; u<fileNamesNb; u++) DISPLAYLEVEL(4, "%u %s\n", u, extendedFileList[u]);
free((void*)filenameTable);
filenameTable = fileNamesTable;
filenameTable = extendedFileList;
filenameIdx = fileNamesNb;
}
}
@@ -495,15 +503,16 @@ int main(int argCount, char** argv)
/* IO Stream/File */
FIO_setNotificationLevel(displayLevel);
#ifndef ZSTD_NOCOMPRESS
if (!decode) {
if (filenameIdx==1 && outFileName)
#ifndef ZSTD_NOCOMPRESS
if ((filenameIdx==1) && outFileName)
operationResult = FIO_compressFilename(outFileName, filenameTable[0], dictFileName, cLevel);
else
operationResult = FIO_compressMultipleFilenames(filenameTable, filenameIdx, outFileName ? outFileName : ZSTD_EXTENSION, dictFileName, cLevel);
} else
#else
DISPLAY("Compression not supported\n");
#endif
{ /* decompression */
} else { /* decompression */
#ifndef ZSTD_NODECOMPRESS
if (testmode) { outFileName=nulmark; FIO_setRemoveSrcFile(0); } /* test mode */
if (filenameIdx==1 && outFileName)
@@ -517,10 +526,9 @@ int main(int argCount, char** argv)
_end:
if (main_pause) waitEnter();
free(dynNameSpace);
#ifdef UTIL_HAS_CREATEFILELIST
if (fileNamesTable)
UTIL_freeFileList(fileNamesTable, fileNamesBuf);
if (extendedFileList)
UTIL_freeFileList(extendedFileList, fileNamesBuf);
else
#endif
free((void*)filenameTable);
tests/fuzzer.c
+8 -8
View File
@@ -173,13 +173,13 @@ static int basicUnitTests(U32 seed, double compressibility)
static const size_t dictSize = 551;
DISPLAYLEVEL(4, "test%3i : copy context too soon : ", testNb++);
{ size_t const copyResult = ZSTD_copyCCtx(ctxDuplicated, ctxOrig);
{ size_t const copyResult = ZSTD_copyCCtx(ctxDuplicated, ctxOrig, 0);
if (!ZSTD_isError(copyResult)) goto _output_error; } /* error must be detected */
DISPLAYLEVEL(4, "OK \n");
DISPLAYLEVEL(4, "test%3i : load dictionary into context : ", testNb++);
CHECK( ZSTD_compressBegin_usingDict(ctxOrig, CNBuffer, dictSize, 2) );
CHECK( ZSTD_copyCCtx(ctxDuplicated, ctxOrig) );
CHECK( ZSTD_copyCCtx(ctxDuplicated, ctxOrig, CNBuffSize - dictSize) );
DISPLAYLEVEL(4, "OK \n");
DISPLAYLEVEL(4, "test%3i : compress with flat dictionary : ", testNb++);
@@ -221,10 +221,10 @@ static int basicUnitTests(U32 seed, double compressibility)
p.fParams.contentSizeFlag = 1;
CHECK( ZSTD_compressBegin_advanced(ctxOrig, CNBuffer, dictSize, p, testSize-1) );
}
CHECK( ZSTD_copyCCtx(ctxDuplicated, ctxOrig) );
CHECK( ZSTD_copyCCtx(ctxDuplicated, ctxOrig, testSize) );
CHECKPLUS(r, ZSTD_compressContinue(ctxDuplicated, compressedBuffer, ZSTD_compressBound(testSize),
(const char*)CNBuffer + dictSize, CNBuffSize - dictSize),
CHECKPLUS(r, ZSTD_compressEnd(ctxDuplicated, compressedBuffer, ZSTD_compressBound(testSize),
(const char*)CNBuffer + dictSize, testSize),
cSize = r);
{ ZSTD_frameParams fp;
if (ZSTD_getFrameParams(&fp, compressedBuffer, cSize)) goto _output_error;
@@ -674,9 +674,9 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxD
errorCode = ZSTD_compressBegin_advanced(refCtx, dict, dictSize, p, 0);
CHECK (ZSTD_isError(errorCode), "ZSTD_compressBegin_advanced error : %s", ZSTD_getErrorName(errorCode));
}
{ size_t const errorCode = ZSTD_copyCCtx(ctx, refCtx);
CHECK (ZSTD_isError(errorCode), "ZSTD_copyCCtx error : %s", ZSTD_getErrorName(errorCode)); }
}
{ size_t const errorCode = ZSTD_copyCCtx(ctx, refCtx, 0);
CHECK (ZSTD_isError(errorCode), "ZSTD_copyCCtx error : %s", ZSTD_getErrorName(errorCode));
} }
XXH64_reset(&xxhState, 0);
{ U32 const nbChunks = (FUZ_rand(&lseed) & 127) + 2;
U32 n;
tests/playTests.sh
+22 -3
View File
@@ -54,6 +54,17 @@ $ZSTD -99 -f tmp # too large compression level, automatic sized down
$ECHO "test : compress to stdout"
$ZSTD tmp -c > tmpCompressed
$ZSTD tmp --stdout > tmpCompressed # long command format
$ECHO "test : compress to named file"
rm tmpCompressed
$ZSTD tmp -o tmpCompressed
ls tmpCompressed # must work
$ECHO "test : -o must be followed by filename (must fail)"
$ZSTD tmp -of tmpCompressed && die "-o must be followed by filename "
$ECHO "test : force write, correct order"
$ZSTD tmp -fo tmpCompressed
$ECHO "test : forgotten argument"
cp tmp tmp2
$ZSTD tmp2 -fo && die "-o must be followed by filename "
$ECHO "test : implied stdout when input is stdin"
$ECHO bob | $ZSTD | $ZSTD -d
$ECHO "test : null-length file roundtrip"
@@ -183,18 +194,26 @@ $ECHO "- Create first dictionary"
$ZSTD --train *.c ../programs/*.c -o tmpDict
cp $TESTFILE tmp
$ZSTD -f tmp -D tmpDict
$ZSTD -d tmp.zst -D tmpDict -of result
$ZSTD -d tmp.zst -D tmpDict -fo result
diff $TESTFILE result
$ECHO "- Create second (different) dictionary"
$ZSTD --train *.c ../programs/*.c ../programs/*.h -o tmpDictC
$ZSTD -d tmp.zst -D tmpDictC -of result && die "wrong dictionary not detected!"
$ZSTD -d tmp.zst -D tmpDictC -fo result && die "wrong dictionary not detected!"
$ECHO "- Create dictionary with short dictID"
$ZSTD --train *.c ../programs/*.c --dictID 1 -o tmpDict1
cmp tmpDict tmpDict1 && die "dictionaries should have different ID !"
$ECHO "- Create dictionary with wrong dictID parameter order (must fail)"
$ZSTD --train *.c ../programs/*.c --dictID -o 1 tmpDict1 && die "wrong order : --dictID must be followed by argument "
$ECHO "- Create dictionary with size limit"
$ZSTD --train *.c ../programs/*.c -o tmpDict2 --maxdict 4K -v
$ECHO "- Create dictionary with wrong parameter order (must fail)"
$ZSTD --train *.c ../programs/*.c -o tmpDict2 --maxdict -v 4K && die "wrong order : --maxdict must be followed by argument "
$ECHO "- Compress without dictID"
$ZSTD -f tmp -D tmpDict1 --no-dictID
$ZSTD -d tmp.zst -D tmpDict -of result
$ZSTD -d tmp.zst -D tmpDict -fo result
diff $TESTFILE result
$ECHO "- Compress with wrong argument order (must fail)"
$ZSTD tmp -Df tmpDict1 -c > /dev/null && die "-D must be followed by dictionary name "
$ECHO "- Compress multiple files with dictionary"
rm -rf dirTestDict
mkdir dirTestDict