diff --git a/README.md b/README.md index 85d5ac324..53609a146 100644 --- a/README.md +++ b/README.md @@ -79,7 +79,8 @@ Once you have the repository cloned, there are multiple ways provided to build Z #### Makefile -If your system is compatible with `make`, you can simply run `make` at the root directory. +If your system is compatible with a standard `make` (or `gmake`) binary generator, +you can simply run it at the root directory. It will generate `zstd` within root directory. Other available options include : diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index f5b712fc3..298278c99 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -322,13 +322,12 @@ static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc, * Duplicate an existing context `srcCCtx` into another one `dstCCtx`. * Only works during stage ZSTDcs_init (i.e. after creation, but before first call to ZSTD_compressContinue()). * @return : 0, or an error code */ -size_t ZSTD_copyCCtx(ZSTD_CCtx* dstCCtx, const ZSTD_CCtx* srcCCtx) +size_t ZSTD_copyCCtx(ZSTD_CCtx* dstCCtx, const ZSTD_CCtx* srcCCtx, unsigned long long pledgedSrcSize) { if (srcCCtx->stage!=ZSTDcs_init) return ERROR(stage_wrong); memcpy(&dstCCtx->customMem, &srcCCtx->customMem, sizeof(ZSTD_customMem)); - ZSTD_resetCCtx_advanced(dstCCtx, srcCCtx->params, srcCCtx->frameContentSize, ZSTDcrp_noMemset); - dstCCtx->params.fParams.contentSizeFlag = 0; /* content size different from the one set during srcCCtx init */ + ZSTD_resetCCtx_advanced(dstCCtx, srcCCtx->params, pledgedSrcSize, ZSTDcrp_noMemset); /* copy tables */ { size_t const chainSize = (srcCCtx->params.cParams.strategy == ZSTD_fast) ? 0 : (1 << srcCCtx->params.cParams.chainLog); @@ -2730,7 +2729,7 @@ ZSTD_CDict* ZSTD_createCDict(const void* dict, size_t dictSize, int compressionL size_t ZSTD_freeCDict(ZSTD_CDict* cdict) { if (cdict==NULL) return 0; /* support free on NULL */ - { ZSTD_customMem cMem = cdict->refContext->customMem; + { ZSTD_customMem const cMem = cdict->refContext->customMem; ZSTD_freeCCtx(cdict->refContext); ZSTD_free(cdict->dictContent, cMem); ZSTD_free(cdict, cMem); @@ -2740,7 +2739,7 @@ size_t ZSTD_freeCDict(ZSTD_CDict* cdict) size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict, U64 pledgedSrcSize) { - if (cdict->dictContentSize) CHECK_F(ZSTD_copyCCtx(cctx, cdict->refContext)) + if (cdict->dictContentSize) CHECK_F(ZSTD_copyCCtx(cctx, cdict->refContext, pledgedSrcSize)) else CHECK_F(ZSTD_compressBegin_advanced(cctx, NULL, 0, cdict->refContext->params, pledgedSrcSize)); return 0; } diff --git a/lib/dictBuilder/zdict.c b/lib/dictBuilder/zdict.c index cfabb20ba..8a38aadeb 100644 --- a/lib/dictBuilder/zdict.c +++ b/lib/dictBuilder/zdict.c @@ -563,7 +563,7 @@ static void ZDICT_countEStats(EStats_ress_t esr, ZSTD_parameters params, size_t cSize; if (srcSize > blockSizeMax) srcSize = blockSizeMax; /* protection vs large samples */ - { size_t const errorCode = ZSTD_copyCCtx(esr.zc, esr.ref); + { size_t const errorCode = ZSTD_copyCCtx(esr.zc, esr.ref, 0); if (ZSTD_isError(errorCode)) { DISPLAYLEVEL(1, "warning : ZSTD_copyCCtx failed \n"); return; } } cSize = ZSTD_compressBlock(esr.zc, esr.workPlace, ZSTD_BLOCKSIZE_ABSOLUTEMAX, src, srcSize); diff --git a/lib/zstd.h b/lib/zstd.h index 31171d04d..d7eb9c01f 100644 --- a/lib/zstd.h +++ b/lib/zstd.h @@ -447,7 +447,7 @@ ZSTDLIB_API size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds); ZSTDLIB_API size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel); ZSTDLIB_API size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel); ZSTDLIB_API size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize); -ZSTDLIB_API size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx); +ZSTDLIB_API size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize); ZSTDLIB_API size_t ZSTD_compressContinue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); ZSTDLIB_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); diff --git a/programs/fileio.c b/programs/fileio.c index 7dee7c11b..56f22fe75 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -255,7 +255,7 @@ typedef struct { FILE* srcFile; } cRess_t; -static cRess_t FIO_createCResources(const char* dictFileName, int cLevel) +static cRess_t FIO_createCResources(const char* dictFileName, int cLevel, U64 srcSize) { cRess_t ress; memset(&ress, 0, sizeof(ress)); @@ -272,11 +272,11 @@ static cRess_t FIO_createCResources(const char* dictFileName, int cLevel) { void* dictBuffer; size_t const dictBuffSize = FIO_loadFile(&dictBuffer, dictFileName); if (dictFileName && (dictBuffer==NULL)) EXM_THROW(32, "zstd: allocation error : can't create dictBuffer"); - { ZSTD_parameters params = ZSTD_getParams(cLevel, 0, dictBuffSize); + { ZSTD_parameters params = ZSTD_getParams(cLevel, srcSize, dictBuffSize); params.fParams.contentSizeFlag = 1; params.fParams.checksumFlag = g_checksumFlag; params.fParams.noDictIDFlag = !g_dictIDFlag; - { size_t const errorCode = ZSTD_initCStream_advanced(ress.cctx, dictBuffer, dictBuffSize, params, 0); + { size_t const errorCode = ZSTD_initCStream_advanced(ress.cctx, dictBuffer, dictBuffSize, params, srcSize); if (ZSTD_isError(errorCode)) EXM_THROW(33, "Error initializing CStream : %s", ZSTD_getErrorName(errorCode)); } } free(dictBuffer); @@ -408,8 +408,9 @@ int FIO_compressFilename(const char* dstFileName, const char* srcFileName, const char* dictFileName, int compressionLevel) { clock_t const start = clock(); + U64 const srcSize = UTIL_getFileSize(srcFileName); - cRess_t const ress = FIO_createCResources(dictFileName, compressionLevel); + cRess_t const ress = FIO_createCResources(dictFileName, compressionLevel, srcSize); int const result = FIO_compressFilename_dstFile(ress, dstFileName, srcFileName); double const seconds = (double)(clock() - start) / CLOCKS_PER_SEC; @@ -428,7 +429,8 @@ int FIO_compressMultipleFilenames(const char** inFileNamesTable, unsigned nbFile size_t dfnSize = FNSPACE; char* dstFileName = (char*)malloc(FNSPACE); size_t const suffixSize = suffix ? strlen(suffix) : 0; - cRess_t ress = FIO_createCResources(dictFileName, compressionLevel); + U64 const srcSize = (nbFiles != 1) ? 0 : UTIL_getFileSize(inFileNamesTable[0]) ; + cRess_t ress = FIO_createCResources(dictFileName, compressionLevel, srcSize); /* init */ if (dstFileName==NULL) EXM_THROW(27, "FIO_compressMultipleFilenames : allocation error for dstFileName"); diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 14571344f..6d1d9f649 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -23,7 +23,7 @@ #endif #ifndef ZSTDCLI_CLEVEL_MAX -# define ZSTDCLI_CLEVEL_MAX 19 +# define ZSTDCLI_CLEVEL_MAX 19 /* when not using --ultra */ #endif @@ -51,13 +51,11 @@ #if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) || defined(__CYGWIN__) # include /* _isatty */ # define IS_CONSOLE(stdStream) _isatty(_fileno(stdStream)) +#elif defined(_POSIX_C_SOURCE) || defined(_XOPEN_SOURCE) || defined(_POSIX_SOURCE) || (defined(__APPLE__) && defined(__MACH__)) /* https://sourceforge.net/p/predef/wiki/OperatingSystems/ */ +# include /* isatty */ +# define IS_CONSOLE(stdStream) isatty(fileno(stdStream)) #else -# if defined(_POSIX_C_SOURCE) || defined(_XOPEN_SOURCE) || defined(_POSIX_SOURCE) || (defined(__APPLE__) && defined(__MACH__)) /* https://sourceforge.net/p/predef/wiki/OperatingSystems/ */ -# include /* isatty */ -# define IS_CONSOLE(stdStream) isatty(fileno(stdStream)) -# else -# define IS_CONSOLE(stdStream) 0 -# endif +# define IS_CONSOLE(stdStream) 0 #endif @@ -195,7 +193,7 @@ static unsigned readU32FromChar(const char** stringPtr) #define CLEAN_RETURN(i) { operationResult = (i); goto _end; } -int main(int argCount, char** argv) +int main(int argCount, const char* argv[]) { int argNb, bench=0, @@ -210,7 +208,8 @@ int main(int argCount, char** argv) nextArgumentIsMaxDict=0, nextArgumentIsDictID=0, nextArgumentIsFile=0, - ultra=0; + ultra=0, + lastCommand = 0; int cLevel = ZSTDCLI_CLEVEL_DEFAULT; int cLevelLast = 1; unsigned recursive = 0; @@ -219,13 +218,12 @@ int main(int argCount, char** argv) const char* programName = argv[0]; const char* outFileName = NULL; const char* dictFileName = NULL; - char* dynNameSpace = NULL; unsigned maxDictSize = g_defaultMaxDictSize; unsigned dictID = 0; int dictCLevel = g_defaultDictCLevel; unsigned dictSelect = g_defaultSelectivityLevel; #ifdef UTIL_HAS_CREATEFILELIST - const char** fileNamesTable = NULL; + const char** extendedFileList = NULL; char* fileNamesBuf = NULL; unsigned fileNamesNb; #endif @@ -271,8 +269,8 @@ int main(int argCount, char** argv) if (!strcmp(argument, "--no-sparse")) { FIO_setSparseWrite(0); continue; } if (!strcmp(argument, "--test")) { testmode=1; decode=1; continue; } if (!strcmp(argument, "--train")) { dictBuild=1; outFileName=g_defaultDictName; continue; } - if (!strcmp(argument, "--maxdict")) { nextArgumentIsMaxDict=1; continue; } - if (!strcmp(argument, "--dictID")) { nextArgumentIsDictID=1; continue; } + if (!strcmp(argument, "--maxdict")) { nextArgumentIsMaxDict=1; lastCommand=1; continue; } + if (!strcmp(argument, "--dictID")) { nextArgumentIsDictID=1; lastCommand=1; continue; } if (!strcmp(argument, "--keep")) { FIO_setRemoveSrcFile(0); continue; } if (!strcmp(argument, "--rm")) { FIO_setRemoveSrcFile(1); continue; } @@ -290,6 +288,10 @@ int main(int argCount, char** argv) argument++; while (argument[0]!=0) { + if (lastCommand) { + DISPLAY("error : command must be followed by argument \n"); + return 1; + } #ifndef ZSTD_NOCOMPRESS /* compression Level */ if ((*argument>='0') && (*argument<='9')) { @@ -312,7 +314,7 @@ int main(int argCount, char** argv) case 'c': forceStdout=1; outFileName=stdoutmark; argument++; break; /* Use file content as dictionary */ - case 'D': nextEntryIsDictionary = 1; argument++; break; + case 'D': nextEntryIsDictionary = 1; lastCommand = 1; argument++; break; /* Overwrite */ case 'f': FIO_overwriteMode(); forceStdout=1; argument++; break; @@ -333,7 +335,7 @@ int main(int argCount, char** argv) case 't': testmode=1; decode=1; argument++; break; /* destination file name */ - case 'o': nextArgumentIsOutFileName=1; argument++; break; + case 'o': nextArgumentIsOutFileName=1; lastCommand=1; argument++; break; #ifdef UTIL_HAS_CREATEFILELIST /* recursive */ @@ -397,6 +399,7 @@ int main(int argCount, char** argv) if (nextArgumentIsMaxDict) { nextArgumentIsMaxDict = 0; + lastCommand = 0; maxDictSize = readU32FromChar(&argument); if (toupper(*argument)=='K') maxDictSize <<= 10; if (toupper(*argument)=='M') maxDictSize <<= 20; @@ -405,6 +408,7 @@ int main(int argCount, char** argv) if (nextArgumentIsDictID) { nextArgumentIsDictID = 0; + lastCommand = 0; dictID = readU32FromChar(&argument); continue; } @@ -413,12 +417,14 @@ int main(int argCount, char** argv) if (nextEntryIsDictionary) { nextEntryIsDictionary = 0; + lastCommand = 0; dictFileName = argument; continue; } if (nextArgumentIsOutFileName) { nextArgumentIsOutFileName = 0; + lastCommand = 0; outFileName = argument; if (!strcmp(outFileName, "-")) outFileName = stdoutmark; continue; @@ -428,17 +434,19 @@ int main(int argCount, char** argv) filenameTable[filenameIdx++] = argument; } + if (lastCommand) { DISPLAY("error : command must be followed by argument \n"); return 1; } /* forgotten argument */ + /* Welcome message (if verbose) */ DISPLAYLEVEL(3, WELCOME_MESSAGE); #ifdef UTIL_HAS_CREATEFILELIST - if (recursive) { - fileNamesTable = UTIL_createFileList(filenameTable, filenameIdx, &fileNamesBuf, &fileNamesNb); - if (fileNamesTable) { + if (recursive) { /* at this stage, filenameTable is a list of paths, which can contain both files and directories */ + extendedFileList = UTIL_createFileList(filenameTable, filenameIdx, &fileNamesBuf, &fileNamesNb); + if (extendedFileList) { unsigned u; - for (u=0; u tmpCompressed $ZSTD tmp --stdout > tmpCompressed # long command format +$ECHO "test : compress to named file" +rm tmpCompressed +$ZSTD tmp -o tmpCompressed +ls tmpCompressed # must work +$ECHO "test : -o must be followed by filename (must fail)" +$ZSTD tmp -of tmpCompressed && die "-o must be followed by filename " +$ECHO "test : force write, correct order" +$ZSTD tmp -fo tmpCompressed +$ECHO "test : forgotten argument" +cp tmp tmp2 +$ZSTD tmp2 -fo && die "-o must be followed by filename " $ECHO "test : implied stdout when input is stdin" $ECHO bob | $ZSTD | $ZSTD -d $ECHO "test : null-length file roundtrip" @@ -183,18 +194,26 @@ $ECHO "- Create first dictionary" $ZSTD --train *.c ../programs/*.c -o tmpDict cp $TESTFILE tmp $ZSTD -f tmp -D tmpDict -$ZSTD -d tmp.zst -D tmpDict -of result +$ZSTD -d tmp.zst -D tmpDict -fo result diff $TESTFILE result $ECHO "- Create second (different) dictionary" $ZSTD --train *.c ../programs/*.c ../programs/*.h -o tmpDictC -$ZSTD -d tmp.zst -D tmpDictC -of result && die "wrong dictionary not detected!" +$ZSTD -d tmp.zst -D tmpDictC -fo result && die "wrong dictionary not detected!" $ECHO "- Create dictionary with short dictID" $ZSTD --train *.c ../programs/*.c --dictID 1 -o tmpDict1 cmp tmpDict tmpDict1 && die "dictionaries should have different ID !" +$ECHO "- Create dictionary with wrong dictID parameter order (must fail)" +$ZSTD --train *.c ../programs/*.c --dictID -o 1 tmpDict1 && die "wrong order : --dictID must be followed by argument " +$ECHO "- Create dictionary with size limit" +$ZSTD --train *.c ../programs/*.c -o tmpDict2 --maxdict 4K -v +$ECHO "- Create dictionary with wrong parameter order (must fail)" +$ZSTD --train *.c ../programs/*.c -o tmpDict2 --maxdict -v 4K && die "wrong order : --maxdict must be followed by argument " $ECHO "- Compress without dictID" $ZSTD -f tmp -D tmpDict1 --no-dictID -$ZSTD -d tmp.zst -D tmpDict -of result +$ZSTD -d tmp.zst -D tmpDict -fo result diff $TESTFILE result +$ECHO "- Compress with wrong argument order (must fail)" +$ZSTD tmp -Df tmpDict1 -c > /dev/null && die "-D must be followed by dictionary name " $ECHO "- Compress multiple files with dictionary" rm -rf dirTestDict mkdir dirTestDict