ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

initial commit

Browse Source
This commit is contained in:
Danielle Rozenblit
2023-02-09 07:37:37 -08:00
parent dc39409a03
commit 610c8b9e33
385 changed files with 7518 additions and 4039 deletions
Download Patch File Download Diff File Expand all files Collapse all files
programs/Makefile
+8 -5
View File
@@ -1,5 +1,5 @@
# ################################################################
# Copyright (c) Yann Collet, Facebook, Inc.
# Copyright (c) Meta Platforms, Inc. and affiliates.
# All rights reserved.
#
# This source code is licensed under both the BSD-style license (found in the
@@ -231,9 +231,12 @@ zstd-dll : zstd
## zstd-pgo: zstd executable optimized with PGO.
.PHONY: zstd-pgo
zstd-pgo : LLVM_PROFDATA?=llvm-profdata
zstd-pgo : PROF_GENERATE_FLAGS=-fprofile-generate $(if $(findstring gcc,$(CC)),-fprofile-dir=.)
zstd-pgo : PROF_USE_FLAGS=-fprofile-use $(if $(findstring gcc,$(CC)),-fprofile-dir=. -Werror=missing-profile -Wno-error=coverage-mismatch)
zstd-pgo :
$(MAKE) clean HASH_DIR=$(HASH_DIR)
$(MAKE) zstd HASH_DIR=$(HASH_DIR) MOREFLAGS=-fprofile-generate
$(MAKE) zstd HASH_DIR=$(HASH_DIR) MOREFLAGS="$(PROF_GENERATE_FLAGS)"
./zstd -b19i1 $(PROFILE_WITH)
./zstd -b16i1 $(PROFILE_WITH)
./zstd -b9i2 $(PROFILE_WITH)
@@ -245,8 +248,8 @@ ifndef BUILD_DIR
else
$(RM) zstd $(BUILD_DIR)/zstd $(BUILD_DIR)/*.o
endif
case $(CC) in *clang*) if ! [ -e default.profdata ]; then llvm-profdata merge -output=default.profdata default*.profraw; fi ;; esac
$(MAKE) zstd HASH_DIR=$(HASH_DIR) MOREFLAGS=-fprofile-use
case $(CC) in *clang*) if ! [ -e default.profdata ]; then $(LLVM_PROFDATA) merge -output=default.profdata default*.profraw; fi ;; esac
$(MAKE) zstd HASH_DIR=$(HASH_DIR) MOREFLAGS="$(PROF_USE_FLAGS)"
## zstd-small: minimal target, supporting only zstd compression and decompression. no bench. no legacy. no other format.
CLEAN += zstd-small zstd-frugal
@@ -276,7 +279,7 @@ generate_res: $(RES64_FILE) $(RES32_FILE)
ifneq (,$(filter Windows%,$(OS)))
RC ?= windres
# http://stackoverflow.com/questions/708238/how-do-i-add-an-icon-to-a-mingw-gcc-compiled-executable
# https://stackoverflow.com/questions/708238/how-do-i-add-an-icon-to-a-mingw-gcc-compiled-executable
$(RES64_FILE): windres/zstd.rc
$(RC) -o $@ -I ../lib -I windres -i $< -O coff -F pe-x86-64
$(RES32_FILE): windres/zstd.rc
programs/README.md
+1 -1
View File
@@ -276,7 +276,7 @@ compression speed (for lower levels) with minimal change in compression ratio.
The below table illustrates this on the [Silesia compression corpus].
[Silesia compression corpus]: http://sun.aei.polsl.pl/~sdeor/index.php?page=silesia
[Silesia compression corpus]: https://sun.aei.polsl.pl//~sdeor/index.php?page=silesia
| Method | Compression ratio | Compression speed | Decompression speed |
|:-------|------------------:|------------------:|---------------------:|
programs/benchfn.c
+4 -4
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) Yann Collet, Facebook, Inc.
* Copyright (c) Meta Platforms, Inc. and affiliates.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
@@ -229,9 +229,9 @@ BMK_runOutcome_t BMK_benchTimedFn(BMK_timedFnState_t* cont,
cont->timeSpent_ns += (unsigned long long)loopDuration_ns;
/* estimate nbLoops for next run to last approximately 1 second */
if (loopDuration_ns > (runBudget_ns / 50)) {
if (loopDuration_ns > ((double)runBudget_ns / 50)) {
double const fastestRun_ns = MIN(bestRunTime.nanoSecPerRun, newRunTime.nanoSecPerRun);
cont->nbLoops = (unsigned)(runBudget_ns / fastestRun_ns) + 1;
cont->nbLoops = (unsigned)((double)runBudget_ns / fastestRun_ns) + 1;
} else {
/* previous run was too short : blindly increase workload by x multiplier */
const unsigned multiplier = 10;
@@ -239,7 +239,7 @@ BMK_runOutcome_t BMK_benchTimedFn(BMK_timedFnState_t* cont,
cont->nbLoops *= multiplier;
}
if(loopDuration_ns < runTimeMin_ns) {
if(loopDuration_ns < (double)runTimeMin_ns) {
/* don't report results for which benchmark run time was too small : increased risks of rounding errors */
assert(completed == 0);
continue;
programs/benchfn.h
+1 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) Yann Collet, Facebook, Inc.
* Copyright (c) Meta Platforms, Inc. and affiliates.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
programs/benchzstd.c
+18 -16
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) Yann Collet, Facebook, Inc.
* Copyright (c) Meta Platforms, Inc. and affiliates.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
@@ -327,26 +327,31 @@ BMK_benchMemAdvancedNoAlloc(
/* init */
memset(&benchResult, 0, sizeof(benchResult));
if (strlen(displayName)>17) displayName += strlen(displayName) - 17; /* display last 17 characters */
if (adv->mode == BMK_decodeOnly) { /* benchmark only decompression : source must be already compressed */
if (adv->mode == BMK_decodeOnly) {
/* benchmark only decompression : source must be already compressed */
const char* srcPtr = (const char*)srcBuffer;
U64 totalDSize64 = 0;
U32 fileNb;
for (fileNb=0; fileNb<nbFiles; fileNb++) {
U64 const fSize64 = ZSTD_findDecompressedSize(srcPtr, fileSizes[fileNb]);
if (fSize64==0) RETURN_ERROR(32, BMK_benchOutcome_t, "Impossible to determine original size ");
if (fSize64 == ZSTD_CONTENTSIZE_UNKNOWN) {
RETURN_ERROR(32, BMK_benchOutcome_t, "Decompressed size cannot be determined: cannot benchmark");
}
if (fSize64 == ZSTD_CONTENTSIZE_ERROR) {
RETURN_ERROR(32, BMK_benchOutcome_t, "Error while trying to assess decompressed size: data may be invalid");
}
totalDSize64 += fSize64;
srcPtr += fileSizes[fileNb];
}
{ size_t const decodedSize = (size_t)totalDSize64;
assert((U64)decodedSize == totalDSize64); /* check overflow */
free(*resultBufferPtr);
if (totalDSize64 > decodedSize) { /* size_t overflow */
RETURN_ERROR(32, BMK_benchOutcome_t, "decompressed size is too large for local system");
}
*resultBufferPtr = malloc(decodedSize);
if (!(*resultBufferPtr)) {
RETURN_ERROR(33, BMK_benchOutcome_t, "not enough memory");
}
if (totalDSize64 > decodedSize) { /* size_t overflow */
free(*resultBufferPtr);
RETURN_ERROR(32, BMK_benchOutcome_t, "original size is too large");
RETURN_ERROR(33, BMK_benchOutcome_t, "allocation error: not enough memory");
}
cSize = srcSize;
srcSize = decodedSize;
@@ -387,12 +392,9 @@ BMK_benchMemAdvancedNoAlloc(
RDG_genBuffer(compressedBuffer, maxCompressedSize, 0.10, 0.50, 1);
}
#if defined(UTIL_TIME_USES_C90_CLOCK)
if (adv->nbWorkers > 1) {
OUTPUTLEVEL(2, "Warning : time measurements restricted to C90 clock_t. \n")
OUTPUTLEVEL(2, "Warning : using C90 clock_t leads to incorrect measurements in multithreading mode. \n")
if (!UTIL_support_MT_measurements() && adv->nbWorkers > 1) {
OUTPUTLEVEL(2, "Warning : time measurements may be incorrect in multithreading mode... \n")
}
#endif
/* Bench */
{ U64 const crcOrig = (adv->mode == BMK_decodeOnly) ? 0 : XXH64(srcBuffer, srcSize, 0);
@@ -449,7 +451,7 @@ BMK_benchMemAdvancedNoAlloc(
BMK_runOutcome_t const cOutcome = BMK_benchTimedFn( timeStateCompress, cbp);
if (!BMK_isSuccessful_runOutcome(cOutcome)) {
return BMK_benchOutcome_error();
RETURN_ERROR(30, BMK_benchOutcome_t, "compression error");
}
{ BMK_runTime_t const cResult = BMK_extract_runTime(cOutcome);
@@ -477,7 +479,7 @@ BMK_benchMemAdvancedNoAlloc(
BMK_runOutcome_t const dOutcome = BMK_benchTimedFn(timeStateDecompress, dbp);
if(!BMK_isSuccessful_runOutcome(dOutcome)) {
return BMK_benchOutcome_error();
RETURN_ERROR(30, BMK_benchOutcome_t, "decompression error");
}
{ BMK_runTime_t const dResult = BMK_extract_runTime(dOutcome);
@@ -601,7 +603,7 @@ BMK_benchOutcome_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize,
void* resultBuffer = srcSize ? malloc(srcSize) : NULL;
int allocationincomplete = !srcPtrs || !srcSizes || !cPtrs ||
int const allocationincomplete = !srcPtrs || !srcSizes || !cPtrs ||
!cSizes || !cCapacities || !resPtrs || !resSizes ||
!timeStateCompress || !timeStateDecompress ||
!cctx || !dctx ||
programs/benchzstd.h
+1 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) Yann Collet, Facebook, Inc.
* Copyright (c) Meta Platforms, Inc. and affiliates.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
programs/datagen.c
+1 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) Yann Collet, Facebook, Inc.
* Copyright (c) Meta Platforms, Inc. and affiliates.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
programs/datagen.h
+1 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) Yann Collet, Facebook, Inc.
* Copyright (c) Meta Platforms, Inc. and affiliates.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
programs/dibio.c
+5 -6
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) Yann Collet, Facebook, Inc.
* Copyright (c) Meta Platforms, Inc. and affiliates.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
@@ -274,21 +274,20 @@ static fileStats DiB_fileStats(const char** fileNamesTable, int nbFiles, size_t
int n;
memset(&fs, 0, sizeof(fs));
// We assume that if chunking is requested, the chunk size is < SAMPLESIZE_MAX
/* We assume that if chunking is requested, the chunk size is < SAMPLESIZE_MAX */
assert( chunkSize <= SAMPLESIZE_MAX );
for (n=0; n<nbFiles; n++) {
S64 const fileSize = DiB_getFileSize(fileNamesTable[n]);
// TODO: is there a minimum sample size? What if the file is 1-byte?
/* TODO: is there a minimum sample size? What if the file is 1-byte? */
if (fileSize == 0) {
DISPLAYLEVEL(3, "Sample file '%s' has zero size, skipping...\n", fileNamesTable[n]);
continue;
}
/* the case where we are breaking up files in sample chunks */
if (chunkSize > 0)
{
// TODO: is there a minimum sample size? Can we have a 1-byte sample?
if (chunkSize > 0) {
/* TODO: is there a minimum sample size? Can we have a 1-byte sample? */
fs.nbSamples += (int)((fileSize + chunkSize-1) / chunkSize);
fs.totalSizeToLoad += fileSize;
}
programs/dibio.h
+1 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) Yann Collet, Facebook, Inc.
* Copyright (c) Meta Platforms, Inc. and affiliates.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
programs/fileio.c
+448 -253
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) Yann Collet, Facebook, Inc.
* Copyright (c) Meta Platforms, Inc. and affiliates.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
@@ -28,6 +28,7 @@
#include <stdio.h> /* fprintf, open, fdopen, fread, _fileno, stdin, stdout */
#include <stdlib.h> /* malloc, free */
#include <string.h> /* strcmp, strlen */
#include <time.h> /* clock_t, to measure process time */
#include <fcntl.h> /* O_WRONLY */
#include <assert.h>
#include <errno.h> /* errno */
@@ -40,6 +41,10 @@
# include <io.h>
#endif
#if (PLATFORM_POSIX_VERSION > 0)
# include <sys/mman.h>
#endif
#include "fileio.h"
#include "fileio_asyncio.h"
#include "fileio_common.h"
@@ -113,11 +118,15 @@ char const* FIO_lzmaVersion(void)
#define FNSPACE 30
/* Default file permissions 0666 (modulated by umask) */
/* Temporary restricted file permissions are used when we're going to
* chmod/chown at the end of the operation. */
#if !defined(_WIN32)
/* These macros aren't defined on windows. */
#define DEFAULT_FILE_PERMISSIONS (S_IRUSR|S_IWUSR|S_IRGRP|S_IWGRP|S_IROTH|S_IWOTH)
#define TEMPORARY_FILE_PERMISSIONS (S_IRUSR|S_IWUSR)
#else
#define DEFAULT_FILE_PERMISSIONS (0666)
#define TEMPORARY_FILE_PERMISSIONS (0600)
#endif
/*-************************************
@@ -249,6 +258,18 @@ struct FIO_ctx_s {
size_t totalBytesOutput;
};
static int FIO_shouldDisplayFileSummary(FIO_ctx_t const* fCtx)
{
return fCtx->nbFilesTotal <= 1 || g_display_prefs.displayLevel >= 3;
}
static int FIO_shouldDisplayMultipleFileSummary(FIO_ctx_t const* fCtx)
{
int const shouldDisplay = (fCtx->nbFilesProcessed >= 1 && fCtx->nbFilesTotal > 1);
assert(shouldDisplay || FIO_shouldDisplayFileSummary(fCtx) || fCtx->nbFilesProcessed == 0);
return shouldDisplay;
}
/*-*************************************
* Parameters: Initialization
@@ -345,7 +366,7 @@ void FIO_setDictIDFlag(FIO_prefs_t* const prefs, int dictIDFlag) { prefs->dictID
void FIO_setChecksumFlag(FIO_prefs_t* const prefs, int checksumFlag) { prefs->checksumFlag = checksumFlag; }
void FIO_setRemoveSrcFile(FIO_prefs_t* const prefs, unsigned flag) { prefs->removeSrcFile = (flag>0); }
void FIO_setRemoveSrcFile(FIO_prefs_t* const prefs, int flag) { prefs->removeSrcFile = (flag!=0); }
void FIO_setMemLimit(FIO_prefs_t* const prefs, unsigned memLimit) { prefs->memLimit = memLimit; }
@@ -518,26 +539,26 @@ static int FIO_removeFile(const char* path)
/** FIO_openSrcFile() :
* condition : `srcFileName` must be non-NULL. `prefs` may be NULL.
* @result : FILE* to `srcFileName`, or NULL if it fails */
static FILE* FIO_openSrcFile(const FIO_prefs_t* const prefs, const char* srcFileName)
static FILE* FIO_openSrcFile(const FIO_prefs_t* const prefs, const char* srcFileName, stat_t* statbuf)
{
stat_t statbuf;
int allowBlockDevices = prefs != NULL ? prefs->allowBlockDevices : 0;
assert(srcFileName != NULL);
assert(statbuf != NULL);
if (!strcmp (srcFileName, stdinmark)) {
DISPLAYLEVEL(4,"Using stdin for input \n");
SET_BINARY_MODE(stdin);
return stdin;
}
if (!UTIL_stat(srcFileName, &statbuf)) {
if (!UTIL_stat(srcFileName, statbuf)) {
DISPLAYLEVEL(1, "zstd: can't stat %s : %s -- ignored \n",
srcFileName, strerror(errno));
return NULL;
}
if (!UTIL_isRegularFileStat(&statbuf)
&& !UTIL_isFIFOStat(&statbuf)
&& !(allowBlockDevices && UTIL_isBlockDevStat(&statbuf))
if (!UTIL_isRegularFileStat(statbuf)
&& !UTIL_isFIFOStat(statbuf)
&& !(allowBlockDevices && UTIL_isBlockDevStat(statbuf))
) {
DISPLAYLEVEL(1, "zstd: %s is not a regular file -- ignored \n",
srcFileName);
@@ -595,7 +616,7 @@ FIO_openDstFile(FIO_ctx_t* fCtx, FIO_prefs_t* const prefs,
if (!prefs->overwrite) {
if (g_display_prefs.displayLevel <= 1) {
/* No interaction possible */
DISPLAY("zstd: %s already exists; not overwritten \n",
DISPLAYLEVEL(1, "zstd: %s already exists; not overwritten \n",
dstFileName);
return NULL;
}
@@ -649,23 +670,23 @@ FIO_openDstFile(FIO_ctx_t* fCtx, FIO_prefs_t* const prefs,
* @return : loaded size
* if fileName==NULL, returns 0 and a NULL pointer
*/
static size_t FIO_createDictBuffer(void** bufferPtr, const char* fileName, FIO_prefs_t* const prefs)
static size_t FIO_createDictBuffer(void** bufferPtr, const char* fileName, FIO_prefs_t* const prefs, stat_t* dictFileStat)
{
FILE* fileHandle;
U64 fileSize;
stat_t statbuf;
assert(bufferPtr != NULL);
assert(dictFileStat != NULL);
*bufferPtr = NULL;
if (fileName == NULL) return 0;
DISPLAYLEVEL(4,"Loading %s as dictionary \n", fileName);
if (!UTIL_stat(fileName, &statbuf)) {
if (!UTIL_stat(fileName, dictFileStat)) {
EXM_THROW(31, "Stat failed on dictionary file %s: %s", fileName, strerror(errno));
}
if (!UTIL_isRegularFileStat(&statbuf)) {
if (!UTIL_isRegularFileStat(dictFileStat)) {
EXM_THROW(32, "Dictionary %s must be a regular file.", fileName);
}
@@ -675,7 +696,7 @@ static size_t FIO_createDictBuffer(void** bufferPtr, const char* fileName, FIO_p
EXM_THROW(33, "Couldn't open dictionary %s: %s", fileName, strerror(errno));
}
fileSize = UTIL_getFileSizeStat(&statbuf);
fileSize = UTIL_getFileSizeStat(dictFileStat);
{
size_t const dictSizeMax = prefs->patchFromMode ? prefs->memLimit : DICTSIZE_MAX;
if (fileSize > dictSizeMax) {
@@ -695,6 +716,54 @@ static size_t FIO_createDictBuffer(void** bufferPtr, const char* fileName, FIO_p
return (size_t)fileSize;
}
/*! FIO_createDictBuffer() :
* creates a buffer, pointed by `*bufferPtr` using mmap,
* loads entire `filename` content into it.
* @return : loaded size
* if fileName==NULL, returns 0 and a NULL pointer
*/
static size_t FIO_createDictBufferMMap(void** bufferPtr, const char* fileName, FIO_prefs_t* const prefs, stat_t* dictFileStat)
{
int fileHandle;
U64 fileSize;
assert(bufferPtr != NULL);
assert(dictFileStat != NULL);
*bufferPtr = NULL;
if (fileName == NULL) return 0;
DISPLAYLEVEL(4,"Loading %s as dictionary \n", fileName);
if (!UTIL_stat(fileName, dictFileStat)) {
EXM_THROW(31, "Stat failed on dictionary file %s: %s", fileName, strerror(errno));
}
if (!UTIL_isRegularFileStat(dictFileStat)) {
EXM_THROW(32, "Dictionary %s must be a regular file.", fileName);
}
fileHandle = open(fileName, O_RDONLY);
if (fileHandle == -1) {
EXM_THROW(33, "Couldn't open dictionary %s: %s", fileName, strerror(errno));
}
fileSize = UTIL_getFileSizeStat(dictFileStat);
{
size_t const dictSizeMax = prefs->patchFromMode ? prefs->memLimit : DICTSIZE_MAX;
if (fileSize > dictSizeMax) {
EXM_THROW(34, "Dictionary file %s is too large (> %u bytes)",
fileName, (unsigned)dictSizeMax); /* avoid extreme cases */
}
}
*bufferPtr = mmap(NULL, (size_t)fileSize, PROT_READ, MAP_PRIVATE, fileHandle, 0);
close(fileHandle);
return (size_t)fileSize;
}
/* FIO_checkFilenameCollisions() :
@@ -706,7 +775,7 @@ int FIO_checkFilenameCollisions(const char** filenameTable, unsigned nbFiles) {
filenameTableSorted = (const char**) malloc(sizeof(char*) * nbFiles);
if (!filenameTableSorted) {
DISPLAY("Unable to malloc new str array, not checking for name collisions\n");
DISPLAYLEVEL(1, "Allocation error during filename collision checking \n");
return 1;
}
@@ -723,7 +792,7 @@ int FIO_checkFilenameCollisions(const char** filenameTable, unsigned nbFiles) {
prevElem = filenameTableSorted[0];
for (u = 1; u < nbFiles; ++u) {
if (strcmp(prevElem, filenameTableSorted[u]) == 0) {
DISPLAY("WARNING: Two files have same filename: %s\n", prevElem);
DISPLAYLEVEL(2, "WARNING: Two files have same filename: %s\n", prevElem);
}
prevElem = filenameTableSorted[u];
}
@@ -806,45 +875,89 @@ static void FIO_adjustMemLimitForPatchFromMode(FIO_prefs_t* const prefs,
FIO_setMemLimit(prefs, (unsigned)maxSize);
}
/* FIO_removeMultiFilesWarning() :
/* FIO_multiFilesConcatWarning() :
* This function handles logic when processing multiple files with -o or -c, displaying the appropriate warnings/prompts.
* Returns 1 if the console should abort, 0 if console should proceed.
* This function handles logic when processing multiple files with -o, displaying the appropriate warnings/prompts.
*
* If -f is specified, or there is just 1 file, zstd will always proceed as usual.
* If --rm is specified, there will be a prompt asking for user confirmation.
* If -f is specified with --rm, zstd will proceed as usual
* If -q is specified with --rm, zstd will abort pre-emptively
* If neither flag is specified, zstd will prompt the user for confirmation to proceed.
* If --rm is not specified, then zstd will print a warning to the user (which can be silenced with -q).
* However, if the output is stdout, we will always abort rather than displaying the warning prompt.
* If output is stdout or test mode is active, check that `--rm` disabled.
*
* If there is just 1 file to process, zstd will proceed as usual.
* If each file get processed into its own separate destination file, proceed as usual.
*
* When multiple files are processed into a single output,
* display a warning message, then disable --rm if it's set.
*
* If -f is specified or if output is stdout, just proceed.
* If output is set with -o, prompt for confirmation.
*/
static int FIO_removeMultiFilesWarning(FIO_ctx_t* const fCtx, const FIO_prefs_t* const prefs, const char* outFileName, int displayLevelCutoff)
static int FIO_multiFilesConcatWarning(const FIO_ctx_t* fCtx, FIO_prefs_t* prefs, const char* outFileName, int displayLevelCutoff)
{
int error = 0;
if (fCtx->nbFilesTotal > 1 && !prefs->overwrite) {
if (g_display_prefs.displayLevel <= displayLevelCutoff) {
if (prefs->removeSrcFile) {
DISPLAYLEVEL(1, "zstd: Aborting... not deleting files and processing into dst: %s\n", outFileName);
error = 1;
}
} else {
if (!strcmp(outFileName, stdoutmark)) {
DISPLAYLEVEL(2, "zstd: WARNING: all input files will be processed and concatenated into stdout. \n");
} else {
DISPLAYLEVEL(2, "zstd: WARNING: all input files will be processed and concatenated into a single output file: %s \n", outFileName);
}
DISPLAYLEVEL(2, "The concatenated output CANNOT regenerate the original directory tree. \n")
if (prefs->removeSrcFile) {
if (fCtx->hasStdoutOutput) {
DISPLAYLEVEL(1, "Aborting. Use -f if you really want to delete the files and output to stdout\n");
error = 1;
} else {
error = g_display_prefs.displayLevel > displayLevelCutoff && UTIL_requireUserConfirmation("This is a destructive operation. Proceed? (y/n): ", "Aborting...", "yY", fCtx->hasStdinInput);
}
}
}
if (fCtx->hasStdoutOutput) {
if (prefs->removeSrcFile)
/* this should not happen ; hard fail, to protect user's data
* note: this should rather be an assert(), but we want to be certain that user's data will not be wiped out in case it nonetheless happen */
EXM_THROW(43, "It's not allowed to remove input files when processed output is piped to stdout. "
"This scenario is not supposed to be possible. "
"This is a programming error. File an issue for it to be fixed.");
}
return error;
if (prefs->testMode) {
if (prefs->removeSrcFile)
/* this should not happen ; hard fail, to protect user's data
* note: this should rather be an assert(), but we want to be certain that user's data will not be wiped out in case it nonetheless happen */
EXM_THROW(43, "Test mode shall not remove input files! "
"This scenario is not supposed to be possible. "
"This is a programming error. File an issue for it to be fixed.");
return 0;
}
if (fCtx->nbFilesTotal == 1) return 0;
assert(fCtx->nbFilesTotal > 1);
if (!outFileName) return 0;
if (fCtx->hasStdoutOutput) {
DISPLAYLEVEL(2, "zstd: WARNING: all input files will be processed and concatenated into stdout. \n");
} else {
DISPLAYLEVEL(2, "zstd: WARNING: all input files will be processed and concatenated into a single output file: %s \n", outFileName);
}
DISPLAYLEVEL(2, "The concatenated output CANNOT regenerate original file names nor directory structure. \n")
/* multi-input into single output : --rm is not allowed */
if (prefs->removeSrcFile) {
DISPLAYLEVEL(2, "Since it's a destructive operation, input files will not be removed. \n");
prefs->removeSrcFile = 0;
}
if (fCtx->hasStdoutOutput) return 0;
if (prefs->overwrite) return 0;
/* multiple files concatenated into single destination file using -o without -f */
if (g_display_prefs.displayLevel <= displayLevelCutoff) {
/* quiet mode => no prompt => fail automatically */
DISPLAYLEVEL(1, "Concatenating multiple processed inputs into a single output loses file metadata. \n");
DISPLAYLEVEL(1, "Aborting. \n");
return 1;
}
/* normal mode => prompt */
return UTIL_requireUserConfirmation("Proceed? (y/n): ", "Aborting...", "yY", fCtx->hasStdinInput);
}
static ZSTD_inBuffer setInBuffer(const void* buf, size_t s, size_t pos)
{
ZSTD_inBuffer i;
i.src = buf;
i.size = s;
i.pos = pos;
return i;
}
static ZSTD_outBuffer setOutBuffer(void* buf, size_t s, size_t pos)
{
ZSTD_outBuffer o;
o.dst = buf;
o.size = s;
o.pos = pos;
return o;
}
#ifndef ZSTD_NOCOMPRESS
@@ -856,9 +969,11 @@ typedef struct {
void* dictBuffer;
size_t dictBufferSize;
const char* dictFileName;
stat_t dictFileStat;
ZSTD_CStream* cctx;
WritePoolCtx_t *writeCtx;
ReadPoolCtx_t *readCtx;
int mmapDict;
} cRess_t;
/** ZSTD_cycleLog() :
@@ -899,11 +1014,14 @@ static void FIO_adjustParamsForPatchFromMode(FIO_prefs_t* const prefs,
static cRess_t FIO_createCResources(FIO_prefs_t* const prefs,
const char* dictFileName, unsigned long long const maxSrcFileSize,
int cLevel, ZSTD_compressionParameters comprParams) {
U64 const dictSize = UTIL_getFileSize(dictFileName);
int const mmapDict = prefs->patchFromMode && PLATFORM_POSIX_VERSION < 1 && dictSize > prefs->memLimit;
cRess_t ress;
memset(&ress, 0, sizeof(ress));
DISPLAYLEVEL(6, "FIO_createCResources \n");
ress.cctx = ZSTD_createCCtx();
ress.mmapDict = mmapDict;
if (ress.cctx == NULL)
EXM_THROW(30, "allocation error (%s): can't create ZSTD_CCtx",
strerror(errno));
@@ -912,9 +1030,14 @@ static cRess_t FIO_createCResources(FIO_prefs_t* const prefs,
* because of memLimit check inside it */
if (prefs->patchFromMode) {
unsigned long long const ssSize = (unsigned long long)prefs->streamSrcSize;
FIO_adjustParamsForPatchFromMode(prefs, &comprParams, UTIL_getFileSize(dictFileName), ssSize > 0 ? ssSize : maxSrcFileSize, cLevel);
FIO_adjustParamsForPatchFromMode(prefs, &comprParams, dictSize, ssSize > 0 ? ssSize : maxSrcFileSize, cLevel);
}
if (!mmapDict) {
ress.dictBufferSize = FIO_createDictBuffer(&ress.dictBuffer, dictFileName, prefs, &ress.dictFileStat); /* works with dictFileName==NULL */
} else {
ress.dictBufferSize = FIO_createDictBufferMMap(&ress.dictBuffer, dictFileName, prefs, &ress.dictFileStat);
}
ress.dictBufferSize = FIO_createDictBuffer(&ress.dictBuffer, dictFileName, prefs); /* works with dictFileName==NULL */
ress.writeCtx = AIO_WritePool_create(prefs, ZSTD_CStreamOutSize());
ress.readCtx = AIO_ReadPool_create(prefs, ZSTD_CStreamInSize());
@@ -980,7 +1103,11 @@ static cRess_t FIO_createCResources(FIO_prefs_t* const prefs,
static void FIO_freeCResources(const cRess_t* const ress)
{
free(ress->dictBuffer);
if (!ress->mmapDict) {
free(ress->dictBuffer);
} else {
munmap(ress->dictBuffer, ress->dictBufferSize);
}
AIO_WritePool_free(ress->writeCtx);
AIO_ReadPool_free(ress->readCtx);
ZSTD_freeCStream(ress->cctx); /* never fails */
@@ -1006,7 +1133,7 @@ FIO_compressGzFrame(const cRess_t* ress, /* buffers & handlers are used, but no
{ int const ret = deflateInit2(&strm, compressionLevel, Z_DEFLATED,
15 /* maxWindowLogSize */ + 16 /* gzip only */,
8, Z_DEFAULT_STRATEGY); /* see http://www.zlib.net/manual.html */
8, Z_DEFAULT_STRATEGY); /* see https://www.zlib.net/manual.html */
if (ret != Z_OK) {
EXM_THROW(71, "zstd: %s: deflateInit2 error %d \n", srcFileName, ret);
} }
@@ -1044,14 +1171,16 @@ FIO_compressGzFrame(const cRess_t* ress, /* buffers & handlers are used, but no
strm.avail_out = (uInt)writeJob->bufferSize;
} }
if (srcFileSize == UTIL_FILESIZE_UNKNOWN) {
DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%% ",
(unsigned)(inFileSize>>20),
(double)outFileSize/inFileSize*100)
DISPLAYUPDATE_PROGRESS(
"\rRead : %u MB ==> %.2f%% ",
(unsigned)(inFileSize>>20),
(double)outFileSize/(double)inFileSize*100)
} else {
DISPLAYUPDATE(2, "\rRead : %u / %u MB ==> %.2f%% ",
(unsigned)(inFileSize>>20), (unsigned)(srcFileSize>>20),
(double)outFileSize/inFileSize*100);
} }
DISPLAYUPDATE_PROGRESS(
"\rRead : %u / %u MB ==> %.2f%% ",
(unsigned)(inFileSize>>20), (unsigned)(srcFileSize>>20),
(double)outFileSize/(double)inFileSize*100);
} }
while (1) {
int const ret = deflate(&strm, Z_FINISH);
@@ -1141,13 +1270,13 @@ FIO_compressLzmaFrame(cRess_t* ress,
strm.avail_out = writeJob->bufferSize;
} }
if (srcFileSize == UTIL_FILESIZE_UNKNOWN)
DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%%",
DISPLAYUPDATE_PROGRESS("\rRead : %u MB ==> %.2f%%",
(unsigned)(inFileSize>>20),
(double)outFileSize/inFileSize*100)
(double)outFileSize/(double)inFileSize*100)
else
DISPLAYUPDATE(2, "\rRead : %u / %u MB ==> %.2f%%",
DISPLAYUPDATE_PROGRESS("\rRead : %u / %u MB ==> %.2f%%",
(unsigned)(inFileSize>>20), (unsigned)(srcFileSize>>20),
(double)outFileSize/inFileSize*100);
(double)outFileSize/(double)inFileSize*100);
if (ret == LZMA_STREAM_END) break;
}
@@ -1225,13 +1354,13 @@ FIO_compressLz4Frame(cRess_t* ress,
srcFileName, LZ4F_getErrorName(outSize));
outFileSize += outSize;
if (srcFileSize == UTIL_FILESIZE_UNKNOWN) {
DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%%",
DISPLAYUPDATE_PROGRESS("\rRead : %u MB ==> %.2f%%",
(unsigned)(inFileSize>>20),
(double)outFileSize/inFileSize*100)
(double)outFileSize/(double)inFileSize*100)
} else {
DISPLAYUPDATE(2, "\rRead : %u / %u MB ==> %.2f%%",
DISPLAYUPDATE_PROGRESS("\rRead : %u / %u MB ==> %.2f%%",
(unsigned)(inFileSize>>20), (unsigned)(srcFileSize>>20),
(double)outFileSize/inFileSize*100);
(double)outFileSize/(double)inFileSize*100);
}
/* Write Block */
@@ -1263,7 +1392,6 @@ FIO_compressLz4Frame(cRess_t* ress,
}
#endif
static unsigned long long
FIO_compressZstdFrame(FIO_ctx_t* const fCtx,
FIO_prefs_t* const prefs,
@@ -1287,6 +1415,9 @@ FIO_compressZstdFrame(FIO_ctx_t* const fCtx,
unsigned inputPresented = 0;
unsigned inputBlocked = 0;
unsigned lastJobID = 0;
UTIL_time_t lastAdaptTime = UTIL_getTime();
U64 const adaptEveryMicro = REFRESH_RATE;
UTIL_HumanReadableSize_t const file_hrs = UTIL_makeHumanReadableSize(fileSize);
DISPLAYLEVEL(6, "compression using zstd format \n");
@@ -1324,7 +1455,7 @@ FIO_compressZstdFrame(FIO_ctx_t* const fCtx,
size_t stillToFlush;
/* Fill input Buffer */
size_t const inSize = AIO_ReadPool_fillBuffer(ress.readCtx, ZSTD_CStreamInSize());
ZSTD_inBuffer inBuff = { ress.readCtx->srcBuffer, ress.readCtx->srcBufferLoaded, 0 };
ZSTD_inBuffer inBuff = setInBuffer( ress.readCtx->srcBuffer, ress.readCtx->srcBufferLoaded, 0 );
DISPLAYLEVEL(6, "fread %u bytes from source \n", (unsigned)inSize);
*readsize += inSize;
@@ -1336,7 +1467,7 @@ FIO_compressZstdFrame(FIO_ctx_t* const fCtx,
|| (directive == ZSTD_e_end && stillToFlush != 0) ) {
size_t const oldIPos = inBuff.pos;
ZSTD_outBuffer outBuff= { writeJob->buffer, writeJob->bufferSize, 0 };
ZSTD_outBuffer outBuff = setOutBuffer( writeJob->buffer, writeJob->bufferSize, 0 );
size_t const toFlushNow = ZSTD_toFlushNow(ress.cctx);
CHECK_V(stillToFlush, ZSTD_compressStream2(ress.cctx, &outBuff, &inBuff, directive));
AIO_ReadPool_consumeBytes(ress.readCtx, inBuff.pos - oldIPos);
@@ -1355,131 +1486,137 @@ FIO_compressZstdFrame(FIO_ctx_t* const fCtx,
compressedfilesize += outBuff.pos;
}
/* display notification; and adapt compression level */
if (READY_FOR_UPDATE()) {
/* adaptive mode : statistics measurement and speed correction */
if (prefs->adaptiveMode && UTIL_clockSpanMicro(lastAdaptTime) > adaptEveryMicro) {
ZSTD_frameProgression const zfp = ZSTD_getFrameProgression(ress.cctx);
lastAdaptTime = UTIL_getTime();
/* check output speed */
if (zfp.currentJobID > 1) { /* only possible if nbWorkers >= 1 */
unsigned long long newlyProduced = zfp.produced - previous_zfp_update.produced;
unsigned long long newlyFlushed = zfp.flushed - previous_zfp_update.flushed;
assert(zfp.produced >= previous_zfp_update.produced);
assert(prefs->nbWorkers >= 1);
/* test if compression is blocked
* either because output is slow and all buffers are full
* or because input is slow and no job can start while waiting for at least one buffer to be filled.
* note : exclude starting part, since currentJobID > 1 */
if ( (zfp.consumed == previous_zfp_update.consumed) /* no data compressed : no data available, or no more buffer to compress to, OR compression is really slow (compression of a single block is slower than update rate)*/
&& (zfp.nbActiveWorkers == 0) /* confirmed : no compression ongoing */
) {
DISPLAYLEVEL(6, "all buffers full : compression stopped => slow down \n")
speedChange = slower;
}
previous_zfp_update = zfp;
if ( (newlyProduced > (newlyFlushed * 9 / 8)) /* compression produces more data than output can flush (though production can be spiky, due to work unit : (N==4)*block sizes) */
&& (flushWaiting == 0) /* flush speed was never slowed by lack of production, so it's operating at max capacity */
) {
DISPLAYLEVEL(6, "compression faster than flush (%llu > %llu), and flushed was never slowed down by lack of production => slow down \n", newlyProduced, newlyFlushed);
speedChange = slower;
}
flushWaiting = 0;
}
/* course correct only if there is at least one new job completed */
if (zfp.currentJobID > lastJobID) {
DISPLAYLEVEL(6, "compression level adaptation check \n")
/* check input speed */
if (zfp.currentJobID > (unsigned)(prefs->nbWorkers+1)) { /* warm up period, to fill all workers */
if (inputBlocked <= 0) {
DISPLAYLEVEL(6, "input is never blocked => input is slower than ingestion \n");
speedChange = slower;
} else if (speedChange == noChange) {
unsigned long long newlyIngested = zfp.ingested - previous_zfp_correction.ingested;
unsigned long long newlyConsumed = zfp.consumed - previous_zfp_correction.consumed;
unsigned long long newlyProduced = zfp.produced - previous_zfp_correction.produced;
unsigned long long newlyFlushed = zfp.flushed - previous_zfp_correction.flushed;
previous_zfp_correction = zfp;
assert(inputPresented > 0);
DISPLAYLEVEL(6, "input blocked %u/%u(%.2f) - ingested:%u vs %u:consumed - flushed:%u vs %u:produced \n",
inputBlocked, inputPresented, (double)inputBlocked/inputPresented*100,
(unsigned)newlyIngested, (unsigned)newlyConsumed,
(unsigned)newlyFlushed, (unsigned)newlyProduced);
if ( (inputBlocked > inputPresented / 8) /* input is waiting often, because input buffers is full : compression or output too slow */
&& (newlyFlushed * 33 / 32 > newlyProduced) /* flush everything that is produced */
&& (newlyIngested * 33 / 32 > newlyConsumed) /* input speed as fast or faster than compression speed */
) {
DISPLAYLEVEL(6, "recommend faster as in(%llu) >= (%llu)comp(%llu) <= out(%llu) \n",
newlyIngested, newlyConsumed, newlyProduced, newlyFlushed);
speedChange = faster;
}
}
inputBlocked = 0;
inputPresented = 0;
}
if (speedChange == slower) {
DISPLAYLEVEL(6, "slower speed , higher compression \n")
compressionLevel ++;
if (compressionLevel > ZSTD_maxCLevel()) compressionLevel = ZSTD_maxCLevel();
if (compressionLevel > prefs->maxAdaptLevel) compressionLevel = prefs->maxAdaptLevel;
compressionLevel += (compressionLevel == 0); /* skip 0 */
ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_compressionLevel, compressionLevel);
}
if (speedChange == faster) {
DISPLAYLEVEL(6, "faster speed , lighter compression \n")
compressionLevel --;
if (compressionLevel < prefs->minAdaptLevel) compressionLevel = prefs->minAdaptLevel;
compressionLevel -= (compressionLevel == 0); /* skip 0 */
ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_compressionLevel, compressionLevel);
}
speedChange = noChange;
lastJobID = zfp.currentJobID;
} /* if (zfp.currentJobID > lastJobID) */
} /* if (prefs->adaptiveMode && UTIL_clockSpanMicro(lastAdaptTime) > adaptEveryMicro) */
/* display notification */
if (SHOULD_DISPLAY_PROGRESS() && READY_FOR_UPDATE()) {
ZSTD_frameProgression const zfp = ZSTD_getFrameProgression(ress.cctx);
double const cShare = (double)zfp.produced / (double)(zfp.consumed + !zfp.consumed/*avoid div0*/) * 100;
UTIL_HumanReadableSize_t const buffered_hrs = UTIL_makeHumanReadableSize(zfp.ingested - zfp.consumed);
UTIL_HumanReadableSize_t const consumed_hrs = UTIL_makeHumanReadableSize(zfp.consumed);
UTIL_HumanReadableSize_t const produced_hrs = UTIL_makeHumanReadableSize(zfp.produced);
DELAY_NEXT_UPDATE();
/* display progress notifications */
DISPLAY_PROGRESS("\r%79s\r", ""); /* Clear out the current displayed line */
if (g_display_prefs.displayLevel >= 3) {
DISPLAYUPDATE(3, "\r(L%i) Buffered:%5.*f%s - Consumed:%5.*f%s - Compressed:%5.*f%s => %.2f%% ",
compressionLevel,
buffered_hrs.precision, buffered_hrs.value, buffered_hrs.suffix,
consumed_hrs.precision, consumed_hrs.value, consumed_hrs.suffix,
produced_hrs.precision, produced_hrs.value, produced_hrs.suffix,
cShare );
} else if (g_display_prefs.displayLevel >= 2 || g_display_prefs.progressSetting == FIO_ps_always) {
/* Verbose progress update */
DISPLAY_PROGRESS(
"(L%i) Buffered:%5.*f%s - Consumed:%5.*f%s - Compressed:%5.*f%s => %.2f%% ",
compressionLevel,
buffered_hrs.precision, buffered_hrs.value, buffered_hrs.suffix,
consumed_hrs.precision, consumed_hrs.value, consumed_hrs.suffix,
produced_hrs.precision, produced_hrs.value, produced_hrs.suffix,
cShare );
} else {
/* Require level 2 or forcibly displayed progress counter for summarized updates */
DISPLAYLEVEL(1, "\r%79s\r", ""); /* Clear out the current displayed line */
if (fCtx->nbFilesTotal > 1) {
size_t srcFileNameSize = strlen(srcFileName);
/* Ensure that the string we print is roughly the same size each time */
if (srcFileNameSize > 18) {
const char* truncatedSrcFileName = srcFileName + srcFileNameSize - 15;
DISPLAYLEVEL(1, "Compress: %u/%u files. Current: ...%s ",
DISPLAY_PROGRESS("Compress: %u/%u files. Current: ...%s ",
fCtx->currFileIdx+1, fCtx->nbFilesTotal, truncatedSrcFileName);
} else {
DISPLAYLEVEL(1, "Compress: %u/%u files. Current: %*s ",
DISPLAY_PROGRESS("Compress: %u/%u files. Current: %*s ",
fCtx->currFileIdx+1, fCtx->nbFilesTotal, (int)(18-srcFileNameSize), srcFileName);
}
}
DISPLAYLEVEL(1, "Read:%6.*f%4s ", consumed_hrs.precision, consumed_hrs.value, consumed_hrs.suffix);
DISPLAY_PROGRESS("Read:%6.*f%4s ", consumed_hrs.precision, consumed_hrs.value, consumed_hrs.suffix);
if (fileSize != UTIL_FILESIZE_UNKNOWN)
DISPLAYLEVEL(2, "/%6.*f%4s", file_hrs.precision, file_hrs.value, file_hrs.suffix);
DISPLAYLEVEL(1, " ==> %2.f%%", cShare);
DELAY_NEXT_UPDATE();
DISPLAY_PROGRESS("/%6.*f%4s", file_hrs.precision, file_hrs.value, file_hrs.suffix);
DISPLAY_PROGRESS(" ==> %2.f%%", cShare);
}
/* adaptive mode : statistics measurement and speed correction */
if (prefs->adaptiveMode) {
/* check output speed */
if (zfp.currentJobID > 1) { /* only possible if nbWorkers >= 1 */
unsigned long long newlyProduced = zfp.produced - previous_zfp_update.produced;
unsigned long long newlyFlushed = zfp.flushed - previous_zfp_update.flushed;
assert(zfp.produced >= previous_zfp_update.produced);
assert(prefs->nbWorkers >= 1);
/* test if compression is blocked
* either because output is slow and all buffers are full
* or because input is slow and no job can start while waiting for at least one buffer to be filled.
* note : exclude starting part, since currentJobID > 1 */
if ( (zfp.consumed == previous_zfp_update.consumed) /* no data compressed : no data available, or no more buffer to compress to, OR compression is really slow (compression of a single block is slower than update rate)*/
&& (zfp.nbActiveWorkers == 0) /* confirmed : no compression ongoing */
) {
DISPLAYLEVEL(6, "all buffers full : compression stopped => slow down \n")
speedChange = slower;
}
previous_zfp_update = zfp;
if ( (newlyProduced > (newlyFlushed * 9 / 8)) /* compression produces more data than output can flush (though production can be spiky, due to work unit : (N==4)*block sizes) */
&& (flushWaiting == 0) /* flush speed was never slowed by lack of production, so it's operating at max capacity */
) {
DISPLAYLEVEL(6, "compression faster than flush (%llu > %llu), and flushed was never slowed down by lack of production => slow down \n", newlyProduced, newlyFlushed);
speedChange = slower;
}
flushWaiting = 0;
}
/* course correct only if there is at least one new job completed */
if (zfp.currentJobID > lastJobID) {
DISPLAYLEVEL(6, "compression level adaptation check \n")
/* check input speed */
if (zfp.currentJobID > (unsigned)(prefs->nbWorkers+1)) { /* warm up period, to fill all workers */
if (inputBlocked <= 0) {
DISPLAYLEVEL(6, "input is never blocked => input is slower than ingestion \n");
speedChange = slower;
} else if (speedChange == noChange) {
unsigned long long newlyIngested = zfp.ingested - previous_zfp_correction.ingested;
unsigned long long newlyConsumed = zfp.consumed - previous_zfp_correction.consumed;
unsigned long long newlyProduced = zfp.produced - previous_zfp_correction.produced;
unsigned long long newlyFlushed = zfp.flushed - previous_zfp_correction.flushed;
previous_zfp_correction = zfp;
assert(inputPresented > 0);
DISPLAYLEVEL(6, "input blocked %u/%u(%.2f) - ingested:%u vs %u:consumed - flushed:%u vs %u:produced \n",
inputBlocked, inputPresented, (double)inputBlocked/inputPresented*100,
(unsigned)newlyIngested, (unsigned)newlyConsumed,
(unsigned)newlyFlushed, (unsigned)newlyProduced);
if ( (inputBlocked > inputPresented / 8) /* input is waiting often, because input buffers is full : compression or output too slow */
&& (newlyFlushed * 33 / 32 > newlyProduced) /* flush everything that is produced */
&& (newlyIngested * 33 / 32 > newlyConsumed) /* input speed as fast or faster than compression speed */
) {
DISPLAYLEVEL(6, "recommend faster as in(%llu) >= (%llu)comp(%llu) <= out(%llu) \n",
newlyIngested, newlyConsumed, newlyProduced, newlyFlushed);
speedChange = faster;
}
}
inputBlocked = 0;
inputPresented = 0;
}
if (speedChange == slower) {
DISPLAYLEVEL(6, "slower speed , higher compression \n")
compressionLevel ++;
if (compressionLevel > ZSTD_maxCLevel()) compressionLevel = ZSTD_maxCLevel();
if (compressionLevel > prefs->maxAdaptLevel) compressionLevel = prefs->maxAdaptLevel;
compressionLevel += (compressionLevel == 0); /* skip 0 */
ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_compressionLevel, compressionLevel);
}
if (speedChange == faster) {
DISPLAYLEVEL(6, "faster speed , lighter compression \n")
compressionLevel --;
if (compressionLevel < prefs->minAdaptLevel) compressionLevel = prefs->minAdaptLevel;
compressionLevel -= (compressionLevel == 0); /* skip 0 */
ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_compressionLevel, compressionLevel);
}
speedChange = noChange;
lastJobID = zfp.currentJobID;
} /* if (zfp.currentJobID > lastJobID) */
} /* if (g_adaptiveMode) */
} /* if (READY_FOR_UPDATE()) */
} /* if (SHOULD_DISPLAY_PROGRESS() && READY_FOR_UPDATE()) */
} /* while ((inBuff.pos != inBuff.size) */
} while (directive != ZSTD_e_end);
@@ -1555,20 +1692,18 @@ FIO_compressFilename_internal(FIO_ctx_t* const fCtx,
/* Status */
fCtx->totalBytesInput += (size_t)readsize;
fCtx->totalBytesOutput += (size_t)compressedfilesize;
DISPLAYLEVEL(2, "\r%79s\r", "");
if (g_display_prefs.displayLevel >= 2 &&
!fCtx->hasStdoutOutput &&
(g_display_prefs.displayLevel >= 3 || fCtx->nbFilesTotal <= 1)) {
DISPLAY_PROGRESS("\r%79s\r", "");
if (FIO_shouldDisplayFileSummary(fCtx)) {
UTIL_HumanReadableSize_t hr_isize = UTIL_makeHumanReadableSize((U64) readsize);
UTIL_HumanReadableSize_t hr_osize = UTIL_makeHumanReadableSize((U64) compressedfilesize);
if (readsize == 0) {
DISPLAYLEVEL(2,"%-20s : (%6.*f%s => %6.*f%s, %s) \n",
DISPLAY_SUMMARY("%-20s : (%6.*f%s => %6.*f%s, %s) \n",
srcFileName,
hr_isize.precision, hr_isize.value, hr_isize.suffix,
hr_osize.precision, hr_osize.value, hr_osize.suffix,
dstFileName);
} else {
DISPLAYLEVEL(2,"%-20s :%6.2f%% (%6.*f%s => %6.*f%s, %s) \n",
DISPLAY_SUMMARY("%-20s :%6.2f%% (%6.*f%s => %6.*f%s, %s) \n",
srcFileName,
(double)compressedfilesize / (double)readsize * 100,
hr_isize.precision, hr_isize.value, hr_isize.suffix,
@@ -1604,27 +1739,27 @@ static int FIO_compressFilename_dstFile(FIO_ctx_t* const fCtx,
cRess_t ress,
const char* dstFileName,
const char* srcFileName,
const stat_t* srcFileStat,
int compressionLevel)
{
int closeDstFile = 0;
int result;
stat_t statbuf;
int transferMTime = 0;
int transferStat = 0;
FILE *dstFile;
assert(AIO_ReadPool_getFile(ress.readCtx) != NULL);
if (AIO_WritePool_getFile(ress.writeCtx) == NULL) {
int dstFilePermissions = DEFAULT_FILE_PERMISSIONS;
int dstFileInitialPermissions = DEFAULT_FILE_PERMISSIONS;
if ( strcmp (srcFileName, stdinmark)
&& strcmp (dstFileName, stdoutmark)
&& UTIL_stat(srcFileName, &statbuf)
&& UTIL_isRegularFileStat(&statbuf) ) {
dstFilePermissions = statbuf.st_mode;
transferMTime = 1;
&& UTIL_isRegularFileStat(srcFileStat) ) {
transferStat = 1;
dstFileInitialPermissions = TEMPORARY_FILE_PERMISSIONS;
}
closeDstFile = 1;
DISPLAYLEVEL(6, "FIO_compressFilename_dstFile: opening dst: %s \n", dstFileName);
dstFile = FIO_openDstFile(fCtx, prefs, srcFileName, dstFileName, dstFilePermissions);
dstFile = FIO_openDstFile(fCtx, prefs, srcFileName, dstFileName, dstFileInitialPermissions);
if (dstFile==NULL) return 1; /* could not open dstFileName */
AIO_WritePool_setFile(ress.writeCtx, dstFile);
/* Must only be added after FIO_openDstFile() succeeds.
@@ -1644,8 +1779,8 @@ static int FIO_compressFilename_dstFile(FIO_ctx_t* const fCtx,
DISPLAYLEVEL(1, "zstd: %s: %s \n", dstFileName, strerror(errno));
result=1;
}
if (transferMTime) {
UTIL_utime(dstFileName, &statbuf);
if (transferStat) {
UTIL_setFileStat(dstFileName, srcFileStat);
}
if ( (result != 0) /* operation failure */
&& strcmp(dstFileName, stdoutmark) /* special case : don't remove() stdout */
@@ -1687,18 +1822,26 @@ FIO_compressFilename_srcFile(FIO_ctx_t* const fCtx,
{
int result;
FILE* srcFile;
stat_t srcFileStat;
U64 fileSize = UTIL_FILESIZE_UNKNOWN;
DISPLAYLEVEL(6, "FIO_compressFilename_srcFile: %s \n", srcFileName);
/* ensure src is not a directory */
if (UTIL_isDirectory(srcFileName)) {
DISPLAYLEVEL(1, "zstd: %s is a directory -- ignored \n", srcFileName);
return 1;
}
if (strcmp(srcFileName, stdinmark)) {
if (UTIL_stat(srcFileName, &srcFileStat)) {
/* failure to stat at all is handled during opening */
/* ensure src is not the same as dict (if present) */
if (ress.dictFileName != NULL && UTIL_isSameFile(srcFileName, ress.dictFileName)) {
DISPLAYLEVEL(1, "zstd: cannot use %s as an input file and dictionary \n", srcFileName);
return 1;
/* ensure src is not a directory */
if (UTIL_isDirectoryStat(&srcFileStat)) {
DISPLAYLEVEL(1, "zstd: %s is a directory -- ignored \n", srcFileName);
return 1;
}
/* ensure src is not the same as dict (if present) */
if (ress.dictFileName != NULL && UTIL_isSameFileStat(srcFileName, ress.dictFileName, &srcFileStat, &ress.dictFileStat)) {
DISPLAYLEVEL(1, "zstd: cannot use %s as an input file and dictionary \n", srcFileName);
return 1;
}
}
}
/* Check if "srcFile" is compressed. Only done if --exclude-compressed flag is used
@@ -1710,16 +1853,30 @@ FIO_compressFilename_srcFile(FIO_ctx_t* const fCtx,
return 0;
}
srcFile = FIO_openSrcFile(prefs, srcFileName);
srcFile = FIO_openSrcFile(prefs, srcFileName, &srcFileStat);
if (srcFile == NULL) return 1; /* srcFile could not be opened */
/* Don't use AsyncIO for small files */
if (strcmp(srcFileName, stdinmark)) /* Stdin doesn't have stats */
fileSize = UTIL_getFileSizeStat(&srcFileStat);
if(fileSize != UTIL_FILESIZE_UNKNOWN && fileSize < ZSTD_BLOCKSIZE_MAX * 3) {
AIO_ReadPool_setAsync(ress.readCtx, 0);
AIO_WritePool_setAsync(ress.writeCtx, 0);
} else {
AIO_ReadPool_setAsync(ress.readCtx, 1);
AIO_WritePool_setAsync(ress.writeCtx, 1);
}
AIO_ReadPool_setFile(ress.readCtx, srcFile);
result = FIO_compressFilename_dstFile(fCtx, prefs, ress, dstFileName, srcFileName, compressionLevel);
result = FIO_compressFilename_dstFile(
fCtx, prefs, ress,
dstFileName, srcFileName,
&srcFileStat, compressionLevel);
AIO_ReadPool_closeFile(ress.readCtx);
if ( prefs->removeSrcFile /* --rm */
&& result == 0 /* success */
&& strcmp(srcFileName, stdinmark) /* exception : don't erase stdin */
if ( prefs->removeSrcFile /* --rm */
&& result == 0 /* success */
&& strcmp(srcFileName, stdinmark) /* exception : don't erase stdin */
) {
/* We must clear the handler, since after this point calling it would
* delete both the source and destination files.
@@ -1741,7 +1898,8 @@ checked_index(const char* options[], size_t length, size_t index) {
#define INDEX(options, index) checked_index((options), sizeof(options) / sizeof(char*), (size_t)(index))
void FIO_displayCompressionParameters(const FIO_prefs_t* prefs) {
void FIO_displayCompressionParameters(const FIO_prefs_t* prefs)
{
static const char* formatOptions[5] = {ZSTD_EXTENSION, GZ_EXTENSION, XZ_EXTENSION,
LZMA_EXTENSION, LZ4_EXTENSION};
static const char* sparseOptions[3] = {" --no-sparse", "", " --sparse"};
@@ -1870,7 +2028,7 @@ int FIO_compressMultipleFilenames(FIO_ctx_t* const fCtx,
assert(outFileName != NULL || suffix != NULL);
if (outFileName != NULL) { /* output into a single destination (stdout typically) */
FILE *dstFile;
if (FIO_removeMultiFilesWarning(fCtx, prefs, outFileName, 1 /* displayLevelCutoff */)) {
if (FIO_multiFilesConcatWarning(fCtx, prefs, outFileName, 1 /* displayLevelCutoff */)) {
FIO_freeCResources(&ress);
return 1;
}
@@ -1917,16 +2075,23 @@ int FIO_compressMultipleFilenames(FIO_ctx_t* const fCtx,
FIO_checkFilenameCollisions(inFileNamesTable , (unsigned)fCtx->nbFilesTotal);
}
if (fCtx->nbFilesProcessed >= 1 && fCtx->nbFilesTotal > 1 && fCtx->totalBytesInput != 0) {
if (FIO_shouldDisplayMultipleFileSummary(fCtx)) {
UTIL_HumanReadableSize_t hr_isize = UTIL_makeHumanReadableSize((U64) fCtx->totalBytesInput);
UTIL_HumanReadableSize_t hr_osize = UTIL_makeHumanReadableSize((U64) fCtx->totalBytesOutput);
DISPLAYLEVEL(2, "\r%79s\r", "");
DISPLAYLEVEL(2, "%3d files compressed :%.2f%% (%6.*f%4s => %6.*f%4s)\n",
fCtx->nbFilesProcessed,
(double)fCtx->totalBytesOutput/((double)fCtx->totalBytesInput)*100,
hr_isize.precision, hr_isize.value, hr_isize.suffix,
hr_osize.precision, hr_osize.value, hr_osize.suffix);
DISPLAY_PROGRESS("\r%79s\r", "");
if (fCtx->totalBytesInput == 0) {
DISPLAY_SUMMARY("%3d files compressed : (%6.*f%4s => %6.*f%4s)\n",
fCtx->nbFilesProcessed,
hr_isize.precision, hr_isize.value, hr_isize.suffix,
hr_osize.precision, hr_osize.value, hr_osize.suffix);
} else {
DISPLAY_SUMMARY("%3d files compressed : %.2f%% (%6.*f%4s => %6.*f%4s)\n",
fCtx->nbFilesProcessed,
(double)fCtx->totalBytesOutput/((double)fCtx->totalBytesInput)*100,
hr_isize.precision, hr_isize.value, hr_isize.suffix,
hr_osize.precision, hr_osize.value, hr_osize.suffix);
}
}
FIO_freeCResources(&ress);
@@ -1950,11 +2115,13 @@ typedef struct {
static dRess_t FIO_createDResources(FIO_prefs_t* const prefs, const char* dictFileName)
{
U64 const dictSize = UTIL_getFileSize(dictFileName);
int const mmapDict = prefs->patchFromMode && PLATFORM_POSIX_VERSION < 1 && dictSize > prefs->memLimit;
dRess_t ress;
memset(&ress, 0, sizeof(ress));
if (prefs->patchFromMode)
FIO_adjustMemLimitForPatchFromMode(prefs, UTIL_getFileSize(dictFileName), 0 /* just use the dict size */);
FIO_adjustMemLimitForPatchFromMode(prefs, dictSize, 0 /* just use the dict size */);
/* Allocation */
ress.dctx = ZSTD_createDStream();
@@ -1965,9 +2132,23 @@ static dRess_t FIO_createDResources(FIO_prefs_t* const prefs, const char* dictFi
/* dictionary */
{ void* dictBuffer;
size_t const dictBufferSize = FIO_createDictBuffer(&dictBuffer, dictFileName, prefs);
CHECK( ZSTD_initDStream_usingDict(ress.dctx, dictBuffer, dictBufferSize) );
free(dictBuffer);
stat_t statbuf;
size_t dictBufferSize;
if (!mmapDict) {
dictBufferSize = FIO_createDictBuffer(&dictBuffer, dictFileName, prefs, &statbuf);
} else {
dictBufferSize = FIO_createDictBufferMMap(&dictBuffer, dictFileName, prefs, &statbuf);
}
CHECK( ZSTD_DCtx_reset(ress.dctx, ZSTD_reset_session_only) );
CHECK( ZSTD_DCtx_loadDictionary(ress.dctx, dictBuffer, dictBufferSize) );
if (!mmapDict) {
free(dictBuffer);
} else {
munmap(dictBuffer, dictBufferSize);
}
}
ress.writeCtx = AIO_WritePool_create(prefs, ZSTD_DStreamOutSize());
@@ -2064,10 +2245,9 @@ FIO_decompressZstdFrame(FIO_ctx_t* const fCtx, dRess_t* ress,
/* Main decompression Loop */
while (1) {
ZSTD_inBuffer inBuff = { ress->readCtx->srcBuffer, ress->readCtx->srcBufferLoaded, 0 };
ZSTD_outBuffer outBuff= { writeJob->buffer, writeJob->bufferSize, 0 };
ZSTD_inBuffer inBuff = setInBuffer( ress->readCtx->srcBuffer, ress->readCtx->srcBufferLoaded, 0 );
ZSTD_outBuffer outBuff= setOutBuffer( writeJob->buffer, writeJob->bufferSize, 0 );
size_t const readSizeHint = ZSTD_decompressStream(ress->dctx, &outBuff, &inBuff);
const int displayLevel = (g_display_prefs.progressSetting == FIO_ps_always) ? 1 : 2;
UTIL_HumanReadableSize_t const hrs = UTIL_makeHumanReadableSize(alreadyDecoded+frameSize);
if (ZSTD_isError(readSizeHint)) {
DISPLAYLEVEL(1, "%s : Decoding error (36) : %s \n",
@@ -2085,14 +2265,15 @@ FIO_decompressZstdFrame(FIO_ctx_t* const fCtx, dRess_t* ress,
size_t srcFileNameSize = strlen(srcFileName);
if (srcFileNameSize > 18) {
const char* truncatedSrcFileName = srcFileName + srcFileNameSize - 15;
DISPLAYUPDATE(displayLevel, "\rDecompress: %2u/%2u files. Current: ...%s : %.*f%s... ",
fCtx->currFileIdx+1, fCtx->nbFilesTotal, truncatedSrcFileName, hrs.precision, hrs.value, hrs.suffix);
DISPLAYUPDATE_PROGRESS(
"\rDecompress: %2u/%2u files. Current: ...%s : %.*f%s... ",
fCtx->currFileIdx+1, fCtx->nbFilesTotal, truncatedSrcFileName, hrs.precision, hrs.value, hrs.suffix);
} else {
DISPLAYUPDATE(displayLevel, "\rDecompress: %2u/%2u files. Current: %s : %.*f%s... ",
DISPLAYUPDATE_PROGRESS("\rDecompress: %2u/%2u files. Current: %s : %.*f%s... ",
fCtx->currFileIdx+1, fCtx->nbFilesTotal, srcFileName, hrs.precision, hrs.value, hrs.suffix);
}
} else {
DISPLAYUPDATE(displayLevel, "\r%-20.20s : %.*f%s... ",
DISPLAYUPDATE_PROGRESS("\r%-20.20s : %.*f%s... ",
srcFileName, hrs.precision, hrs.value, hrs.suffix);
}
@@ -2134,7 +2315,7 @@ FIO_decompressGzFrame(dRess_t* ress, const char* srcFileName)
strm.opaque = Z_NULL;
strm.next_in = 0;
strm.avail_in = 0;
/* see http://www.zlib.net/manual.html */
/* see https://www.zlib.net/manual.html */
if (inflateInit2(&strm, 15 /* maxWindowLogSize */ + 16 /* gzip only */) != Z_OK)
return FIO_ERROR_FRAME_DECODING;
@@ -2307,7 +2488,7 @@ FIO_decompressLz4Frame(dRess_t* ress, const char* srcFileName)
AIO_WritePool_enqueueAndReacquireWriteJob(&writeJob);
filesize += decodedBytes;
hrs = UTIL_makeHumanReadableSize(filesize);
DISPLAYUPDATE(2, "\rDecompressed : %.*f%s ", hrs.precision, hrs.value, hrs.suffix);
DISPLAYUPDATE_PROGRESS("\rDecompressed : %.*f%s ", hrs.precision, hrs.value, hrs.suffix);
}
if (!nextToLoad) break;
@@ -2415,13 +2596,9 @@ static int FIO_decompressFrames(FIO_ctx_t* const fCtx,
/* Final Status */
fCtx->totalBytesOutput += (size_t)filesize;
DISPLAYLEVEL(2, "\r%79s\r", "");
/* No status message in pipe mode (stdin - stdout) or multi-files mode */
if ((g_display_prefs.displayLevel >= 2 && fCtx->nbFilesTotal <= 1) ||
g_display_prefs.displayLevel >= 3 ||
g_display_prefs.progressSetting == FIO_ps_always) {
DISPLAYLEVEL(1, "\r%-20s: %llu bytes \n", srcFileName, filesize);
}
DISPLAY_PROGRESS("\r%79s\r", "");
if (FIO_shouldDisplayFileSummary(fCtx))
DISPLAY_SUMMARY("%-20s: %llu bytes \n", srcFileName, filesize);
return 0;
}
@@ -2435,22 +2612,22 @@ static int FIO_decompressFrames(FIO_ctx_t* const fCtx,
static int FIO_decompressDstFile(FIO_ctx_t* const fCtx,
FIO_prefs_t* const prefs,
dRess_t ress,
const char* dstFileName, const char* srcFileName)
const char* dstFileName,
const char* srcFileName,
const stat_t* srcFileStat)
{
int result;
stat_t statbuf;
int releaseDstFile = 0;
int transferMTime = 0;
int transferStat = 0;
if ((AIO_WritePool_getFile(ress.writeCtx) == NULL) && (prefs->testMode == 0)) {
FILE *dstFile;
int dstFilePermissions = DEFAULT_FILE_PERMISSIONS;
if ( strcmp(srcFileName, stdinmark) /* special case : don't transfer permissions from stdin */
&& strcmp(dstFileName, stdoutmark)
&& UTIL_stat(srcFileName, &statbuf)
&& UTIL_isRegularFileStat(&statbuf) ) {
dstFilePermissions = statbuf.st_mode;
transferMTime = 1;
&& UTIL_isRegularFileStat(srcFileStat) ) {
transferStat = 1;
dstFilePermissions = TEMPORARY_FILE_PERMISSIONS;
}
releaseDstFile = 1;
@@ -2475,8 +2652,8 @@ static int FIO_decompressDstFile(FIO_ctx_t* const fCtx,
result = 1;
}
if (transferMTime) {
UTIL_utime(dstFileName, &statbuf);
if (transferStat) {
UTIL_setFileStat(dstFileName, srcFileStat);
}
if ( (result != 0) /* operation failure */
@@ -2498,18 +2675,32 @@ static int FIO_decompressDstFile(FIO_ctx_t* const fCtx,
static int FIO_decompressSrcFile(FIO_ctx_t* const fCtx, FIO_prefs_t* const prefs, dRess_t ress, const char* dstFileName, const char* srcFileName)
{
FILE* srcFile;
stat_t srcFileStat;
int result;
U64 fileSize = UTIL_FILESIZE_UNKNOWN;
if (UTIL_isDirectory(srcFileName)) {
DISPLAYLEVEL(1, "zstd: %s is a directory -- ignored \n", srcFileName);
return 1;
}
srcFile = FIO_openSrcFile(prefs, srcFileName);
srcFile = FIO_openSrcFile(prefs, srcFileName, &srcFileStat);
if (srcFile==NULL) return 1;
/* Don't use AsyncIO for small files */
if (strcmp(srcFileName, stdinmark)) /* Stdin doesn't have stats */
fileSize = UTIL_getFileSizeStat(&srcFileStat);
if(fileSize != UTIL_FILESIZE_UNKNOWN && fileSize < ZSTD_BLOCKSIZE_MAX * 3) {
AIO_ReadPool_setAsync(ress.readCtx, 0);
AIO_WritePool_setAsync(ress.writeCtx, 0);
} else {
AIO_ReadPool_setAsync(ress.readCtx, 1);
AIO_WritePool_setAsync(ress.writeCtx, 1);
}
AIO_ReadPool_setFile(ress.readCtx, srcFile);
result = FIO_decompressDstFile(fCtx, prefs, ress, dstFileName, srcFileName);
result = FIO_decompressDstFile(fCtx, prefs, ress, dstFileName, srcFileName, &srcFileStat);
AIO_ReadPool_setFile(ress.readCtx, NULL);
@@ -2686,7 +2877,7 @@ FIO_decompressMultipleFilenames(FIO_ctx_t* const fCtx,
dRess_t ress = FIO_createDResources(prefs, dictFileName);
if (outFileName) {
if (FIO_removeMultiFilesWarning(fCtx, prefs, outFileName, 1 /* displayLevelCutoff */)) {
if (FIO_multiFilesConcatWarning(fCtx, prefs, outFileName, 1 /* displayLevelCutoff */)) {
FIO_freeDResources(ress);
return 1;
}
@@ -2730,8 +2921,11 @@ FIO_decompressMultipleFilenames(FIO_ctx_t* const fCtx,
FIO_checkFilenameCollisions(srcNamesTable , (unsigned)fCtx->nbFilesTotal);
}
if (fCtx->nbFilesProcessed >= 1 && fCtx->nbFilesTotal > 1 && fCtx->totalBytesOutput != 0)
DISPLAYLEVEL(2, "%d files decompressed : %6zu bytes total \n", fCtx->nbFilesProcessed, fCtx->totalBytesOutput);
if (FIO_shouldDisplayMultipleFileSummary(fCtx)) {
DISPLAY_PROGRESS("\r%79s\r", "");
DISPLAY_SUMMARY("%d files decompressed : %6llu bytes total \n",
fCtx->nbFilesProcessed, (unsigned long long)fCtx->totalBytesOutput);
}
FIO_freeDResources(ress);
return error;
@@ -2759,7 +2953,7 @@ typedef enum {
info_frame_error=1,
info_not_zstd=2,
info_file_error=3,
info_truncated_input=4,
info_truncated_input=4
} InfoError;
#define ERROR_IF(c,n,...) { \
@@ -2871,10 +3065,11 @@ static InfoError
getFileInfo_fileConfirmed(fileInfo_t* info, const char* inFileName)
{
InfoError status;
FILE* const srcFile = FIO_openSrcFile(NULL, inFileName);
stat_t srcFileStat;
FILE* const srcFile = FIO_openSrcFile(NULL, inFileName, &srcFileStat);
ERROR_IF(srcFile == NULL, info_file_error, "Error: could not open source file %s", inFileName);
info->compressedSize = UTIL_getFileSize(inFileName);
info->compressedSize = UTIL_getFileSizeStat(&srcFileStat);
status = FIO_analyzeFrames(info, srcFile);
fclose(srcFile);
@@ -3010,7 +3205,7 @@ int FIO_listMultipleFiles(unsigned numFiles, const char** filenameTable, int dis
} }
if (numFiles == 0) {
if (!IS_CONSOLE(stdin)) {
if (!UTIL_isConsole(stdin)) {
DISPLAYLEVEL(1, "zstd: --list does not support reading from standard input \n");
}
DISPLAYLEVEL(1, "No files given \n");
programs/fileio.h
+2 -2
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) Yann Collet, Facebook, Inc.
* Copyright (c) Meta Platforms, Inc. and affiliates.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
@@ -86,7 +86,7 @@ void FIO_setLdmMinMatch(FIO_prefs_t* const prefs, int ldmMinMatch);
void FIO_setMemLimit(FIO_prefs_t* const prefs, unsigned memLimit);
void FIO_setNbWorkers(FIO_prefs_t* const prefs, int nbWorkers);
void FIO_setOverlapLog(FIO_prefs_t* const prefs, int overlapLog);
void FIO_setRemoveSrcFile(FIO_prefs_t* const prefs, unsigned flag);
void FIO_setRemoveSrcFile(FIO_prefs_t* const prefs, int flag);
void FIO_setSparseWrite(FIO_prefs_t* const prefs, int sparse); /**< 0: no sparse; 1: disable on stdout; 2: always enabled */
void FIO_setRsyncable(FIO_prefs_t* const prefs, int rsyncable);
void FIO_setStreamSrcSize(FIO_prefs_t* const prefs, size_t streamSrcSize);
programs/fileio_asyncio.c
+70 -29
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) Facebook, Inc.
* Copyright (c) Meta Platforms, Inc. and affiliates.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
@@ -140,7 +140,7 @@ int AIO_supported(void) {
}
/* ***********************************
* General IoPool implementation
* Generic IoPool implementation
*************************************/
static IOJob_t *AIO_IOPool_createIoJob(IOPoolCtx_t *ctx, size_t bufferSize) {
@@ -163,20 +163,22 @@ static IOJob_t *AIO_IOPool_createIoJob(IOPoolCtx_t *ctx, size_t bufferSize) {
* Displays warning if asyncio is requested but MT isn't available. */
static void AIO_IOPool_createThreadPool(IOPoolCtx_t* ctx, const FIO_prefs_t* prefs) {
ctx->threadPool = NULL;
ctx->threadPoolActive = 0;
if(prefs->asyncIO) {
if (ZSTD_pthread_mutex_init(&ctx->ioJobsMutex, NULL))
EXM_THROW(102,"Failed creating write availableJobs mutex");
EXM_THROW(102,"Failed creating ioJobsMutex mutex");
/* We want MAX_IO_JOBS-2 queue items because we need to always have 1 free buffer to
* decompress into and 1 buffer that's actively written to disk and owned by the writing thread. */
assert(MAX_IO_JOBS >= 2);
ctx->threadPool = POOL_create(1, MAX_IO_JOBS - 2);
ctx->threadPoolActive = 1;
if (!ctx->threadPool)
EXM_THROW(104, "Failed creating writer thread pool");
EXM_THROW(104, "Failed creating I/O thread pool");
}
}
/* AIO_IOPool_init:
* Allocates and sets and a new write pool including its included availableJobs. */
* Allocates and sets and a new I/O thread pool including its included availableJobs. */
static void AIO_IOPool_init(IOPoolCtx_t* ctx, const FIO_prefs_t* prefs, POOL_function poolFunction, size_t bufferSize) {
int i;
AIO_IOPool_createThreadPool(ctx, prefs);
@@ -192,27 +194,59 @@ static void AIO_IOPool_init(IOPoolCtx_t* ctx, const FIO_prefs_t* prefs, POOL_fun
}
/* AIO_IOPool_threadPoolActive:
* Check if current operation uses thread pool.
* Note that in some cases we have a thread pool initialized but choose not to use it. */
static int AIO_IOPool_threadPoolActive(IOPoolCtx_t* ctx) {
return ctx->threadPool && ctx->threadPoolActive;
}
/* AIO_IOPool_lockJobsMutex:
* Locks the IO jobs mutex if threading is active */
static void AIO_IOPool_lockJobsMutex(IOPoolCtx_t* ctx) {
if(AIO_IOPool_threadPoolActive(ctx))
ZSTD_pthread_mutex_lock(&ctx->ioJobsMutex);
}
/* AIO_IOPool_unlockJobsMutex:
* Unlocks the IO jobs mutex if threading is active */
static void AIO_IOPool_unlockJobsMutex(IOPoolCtx_t* ctx) {
if(AIO_IOPool_threadPoolActive(ctx))
ZSTD_pthread_mutex_unlock(&ctx->ioJobsMutex);
}
/* AIO_IOPool_releaseIoJob:
* Releases an acquired job back to the pool. Doesn't execute the job. */
static void AIO_IOPool_releaseIoJob(IOJob_t* job) {
IOPoolCtx_t* const ctx = (IOPoolCtx_t *) job->ctx;
if(ctx->threadPool)
ZSTD_pthread_mutex_lock(&ctx->ioJobsMutex);
AIO_IOPool_lockJobsMutex(ctx);
assert(ctx->availableJobsCount < ctx->totalIoJobs);
ctx->availableJobs[ctx->availableJobsCount++] = job;
if(ctx->threadPool)
ZSTD_pthread_mutex_unlock(&ctx->ioJobsMutex);
AIO_IOPool_unlockJobsMutex(ctx);
}
/* AIO_IOPool_join:
* Waits for all tasks in the pool to finish executing. */
static void AIO_IOPool_join(IOPoolCtx_t* ctx) {
if(ctx->threadPool)
if(AIO_IOPool_threadPoolActive(ctx))
POOL_joinJobs(ctx->threadPool);
}
/* AIO_IOPool_setThreaded:
* Allows (de)activating threaded mode, to be used when the expected overhead
* of threading costs more than the expected gains. */
static void AIO_IOPool_setThreaded(IOPoolCtx_t* ctx, int threaded) {
assert(threaded == 0 || threaded == 1);
assert(ctx != NULL);
if(ctx->threadPoolActive != threaded) {
AIO_IOPool_join(ctx);
ctx->threadPoolActive = threaded;
}
}
/* AIO_IOPool_free:
* Release a previously allocated write thread pool. Makes sure all takss are done and released. */
* Release a previously allocated IO thread pool. Makes sure all tasks are done and released. */
static void AIO_IOPool_destroy(IOPoolCtx_t* ctx) {
int i;
if(ctx->threadPool) {
@@ -236,12 +270,10 @@ static void AIO_IOPool_destroy(IOPoolCtx_t* ctx) {
static IOJob_t* AIO_IOPool_acquireJob(IOPoolCtx_t* ctx) {
IOJob_t *job;
assert(ctx->file != NULL || ctx->prefs->testMode);
if(ctx->threadPool)
ZSTD_pthread_mutex_lock(&ctx->ioJobsMutex);
AIO_IOPool_lockJobsMutex(ctx);
assert(ctx->availableJobsCount > 0);
job = (IOJob_t*) ctx->availableJobs[--ctx->availableJobsCount];
if(ctx->threadPool)
ZSTD_pthread_mutex_unlock(&ctx->ioJobsMutex);
AIO_IOPool_unlockJobsMutex(ctx);
job->usedBufferSize = 0;
job->file = ctx->file;
job->offset = 0;
@@ -251,8 +283,7 @@ static IOJob_t* AIO_IOPool_acquireJob(IOPoolCtx_t* ctx) {
/* AIO_IOPool_setFile:
* Sets the destination file for future files in the pool.
* Requires completion of all queues write jobs and release of all otherwise acquired jobs.
* Also requires ending of sparse write if a previous file was used in sparse mode. */
* Requires completion of all queued jobs and release of all otherwise acquired jobs. */
static void AIO_IOPool_setFile(IOPoolCtx_t* ctx, FILE* file) {
assert(ctx!=NULL);
AIO_IOPool_join(ctx);
@@ -269,7 +300,7 @@ static FILE* AIO_IOPool_getFile(const IOPoolCtx_t* ctx) {
* The queued job shouldn't be used directly after queueing it. */
static void AIO_IOPool_enqueueJob(IOJob_t* job) {
IOPoolCtx_t* const ctx = (IOPoolCtx_t *)job->ctx;
if(ctx->threadPool)
if(AIO_IOPool_threadPoolActive(ctx))
POOL_add(ctx->threadPool, ctx->poolFunction, job);
else
ctx->poolFunction(job);
@@ -300,8 +331,7 @@ void AIO_WritePool_enqueueAndReacquireWriteJob(IOJob_t **job) {
* Blocks on completion of all current write jobs before executing. */
void AIO_WritePool_sparseWriteEnd(WritePoolCtx_t* ctx) {
assert(ctx != NULL);
if(ctx->base.threadPool)
POOL_joinJobs(ctx->base.threadPool);
AIO_IOPool_join(&ctx->base);
AIO_fwriteSparseEnd(ctx->base.prefs, ctx->base.file, ctx->storedSkips);
ctx->storedSkips = 0;
}
@@ -368,6 +398,13 @@ void AIO_WritePool_free(WritePoolCtx_t* ctx) {
free(ctx);
}
/* AIO_WritePool_setAsync:
* Allows (de)activating async mode, to be used when the expected overhead
* of asyncio costs more than the expected gains. */
void AIO_WritePool_setAsync(WritePoolCtx_t* ctx, int async) {
AIO_IOPool_setThreaded(&ctx->base, async);
}
/* ***********************************
* ReadPool implementation
@@ -383,14 +420,13 @@ static void AIO_ReadPool_releaseAllCompletedJobs(ReadPoolCtx_t* ctx) {
static void AIO_ReadPool_addJobToCompleted(IOJob_t* job) {
ReadPoolCtx_t* const ctx = (ReadPoolCtx_t *)job->ctx;
if(ctx->base.threadPool)
ZSTD_pthread_mutex_lock(&ctx->base.ioJobsMutex);
AIO_IOPool_lockJobsMutex(&ctx->base);
assert(ctx->completedJobsCount < MAX_IO_JOBS);
ctx->completedJobs[ctx->completedJobsCount++] = job;
if(ctx->base.threadPool) {
if(AIO_IOPool_threadPoolActive(&ctx->base)) {
ZSTD_pthread_cond_signal(&ctx->jobCompletedCond);
ZSTD_pthread_mutex_unlock(&ctx->base.ioJobsMutex);
}
AIO_IOPool_unlockJobsMutex(&ctx->base);
}
/* AIO_ReadPool_findNextWaitingOffsetCompletedJob_locked:
@@ -426,8 +462,7 @@ static size_t AIO_ReadPool_numReadsInFlight(ReadPoolCtx_t* ctx) {
* Would block. */
static IOJob_t* AIO_ReadPool_getNextCompletedJob(ReadPoolCtx_t* ctx) {
IOJob_t *job = NULL;
if (ctx->base.threadPool)
ZSTD_pthread_mutex_lock(&ctx->base.ioJobsMutex);
AIO_IOPool_lockJobsMutex(&ctx->base);
job = AIO_ReadPool_findNextWaitingOffsetCompletedJob_locked(ctx);
@@ -443,8 +478,7 @@ static IOJob_t* AIO_ReadPool_getNextCompletedJob(ReadPoolCtx_t* ctx) {
ctx->waitingOnOffset += job->usedBufferSize;
}
if (ctx->base.threadPool)
ZSTD_pthread_mutex_unlock(&ctx->base.ioJobsMutex);
AIO_IOPool_unlockJobsMutex(&ctx->base);
return job;
}
@@ -524,7 +558,7 @@ ReadPoolCtx_t* AIO_ReadPool_create(const FIO_prefs_t* prefs, size_t bufferSize)
if(ctx->base.threadPool)
if (ZSTD_pthread_cond_init(&ctx->jobCompletedCond, NULL))
EXM_THROW(103,"Failed creating write jobCompletedCond mutex");
EXM_THROW(103,"Failed creating jobCompletedCond cond");
return ctx;
}
@@ -620,3 +654,10 @@ int AIO_ReadPool_closeFile(ReadPoolCtx_t* ctx) {
AIO_ReadPool_setFile(ctx, NULL);
return fclose(file);
}
/* AIO_ReadPool_setAsync:
* Allows (de)activating async mode, to be used when the expected overhead
* of asyncio costs more than the expected gains. */
void AIO_ReadPool_setAsync(ReadPoolCtx_t* ctx, int async) {
AIO_IOPool_setThreaded(&ctx->base, async);
}
programs/fileio_asyncio.h
+23 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) Facebook, Inc.
* Copyright (c) Meta Platforms, Inc. and affiliates.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
@@ -8,6 +8,17 @@
* You may select, at your option, one of the above-listed licenses.
*/
/*
* FileIO AsyncIO exposes read/write IO pools that allow doing IO asynchronously.
* Current implementation relies on having one thread that reads and one that
* writes.
* Each IO pool supports up to `MAX_IO_JOBS` that can be enqueued for work, but
* are performed serially by the appropriate worker thread.
* Most systems exposes better primitives to perform asynchronous IO, such as
* io_uring on newer linux systems. The API is built in such a way that in the
* future we could replace the threads with better solutions when available.
*/
#ifndef ZSTD_FILEIO_ASYNCIO_H
#define ZSTD_FILEIO_ASYNCIO_H
@@ -27,6 +38,7 @@ extern "C" {
typedef struct {
/* These struct fields should be set only on creation and not changed afterwards */
POOL_ctx* threadPool;
int threadPoolActive;
int totalIoJobs;
const FIO_prefs_t* prefs;
POOL_function poolFunction;
@@ -136,6 +148,11 @@ WritePoolCtx_t* AIO_WritePool_create(const FIO_prefs_t* prefs, size_t bufferSize
* Frees and releases a writePool and its resources. Closes destination file. */
void AIO_WritePool_free(WritePoolCtx_t* ctx);
/* AIO_WritePool_setAsync:
* Allows (de)activating async mode, to be used when the expected overhead
* of asyncio costs more than the expected gains. */
void AIO_WritePool_setAsync(WritePoolCtx_t* ctx, int async);
/* AIO_ReadPool_create:
* Allocates and sets and a new readPool including its included jobs.
* bufferSize should be set to the maximal buffer we want to read at a time, will also be used
@@ -146,6 +163,11 @@ ReadPoolCtx_t* AIO_ReadPool_create(const FIO_prefs_t* prefs, size_t bufferSize);
* Frees and releases a readPool and its resources. Closes source file. */
void AIO_ReadPool_free(ReadPoolCtx_t* ctx);
/* AIO_ReadPool_setAsync:
* Allows (de)activating async mode, to be used when the expected overhead
* of asyncio costs more than the expected gains. */
void AIO_ReadPool_setAsync(ReadPoolCtx_t* ctx, int async);
/* AIO_ReadPool_consumeBytes:
* Consumes byes from srcBuffer's beginning and updates srcBufferLoaded accordingly. */
void AIO_ReadPool_consumeBytes(ReadPoolCtx_t *ctx, size_t n);
programs/fileio_common.h
+12 -4
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) Facebook, Inc.
* Copyright (c) Meta Platforms, Inc. and affiliates.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
@@ -38,16 +38,24 @@ extern FIO_display_prefs_t g_display_prefs;
extern UTIL_time_t g_displayClock;
#define REFRESH_RATE ((U64)(SEC_TO_MICRO / 6))
#define READY_FOR_UPDATE() ((g_display_prefs.progressSetting != FIO_ps_never) && UTIL_clockSpanMicro(g_displayClock) > REFRESH_RATE)
#define READY_FOR_UPDATE() (UTIL_clockSpanMicro(g_displayClock) > REFRESH_RATE || g_display_prefs.displayLevel >= 4)
#define DELAY_NEXT_UPDATE() { g_displayClock = UTIL_getTime(); }
#define DISPLAYUPDATE(l, ...) { \
if (g_display_prefs.displayLevel>=l && (g_display_prefs.progressSetting != FIO_ps_never)) { \
if (READY_FOR_UPDATE() || (g_display_prefs.displayLevel>=4)) { \
if (READY_FOR_UPDATE()) { \
DELAY_NEXT_UPDATE(); \
DISPLAY(__VA_ARGS__); \
if (g_display_prefs.displayLevel>=4) fflush(stderr); \
} } }
#define SHOULD_DISPLAY_SUMMARY() \
(g_display_prefs.displayLevel >= 2 || g_display_prefs.progressSetting == FIO_ps_always)
#define SHOULD_DISPLAY_PROGRESS() \
(g_display_prefs.progressSetting != FIO_ps_never && SHOULD_DISPLAY_SUMMARY())
#define DISPLAY_PROGRESS(...) { if (SHOULD_DISPLAY_PROGRESS()) { DISPLAYLEVEL(1, __VA_ARGS__); }}
#define DISPLAYUPDATE_PROGRESS(...) { if (SHOULD_DISPLAY_PROGRESS()) { DISPLAYUPDATE(1, __VA_ARGS__); }}
#define DISPLAY_SUMMARY(...) { if (SHOULD_DISPLAY_SUMMARY()) { DISPLAYLEVEL(1, __VA_ARGS__); } }
#undef MIN /* in case it would be already defined */
#define MIN(a,b) ((a) < (b) ? (a) : (b))
@@ -114,4 +122,4 @@ extern UTIL_time_t g_displayClock;
#if defined (__cplusplus)
}
#endif
#endif //ZSTD_FILEIO_COMMON_H
#endif /* ZSTD_FILEIO_COMMON_H */
programs/fileio_types.h
+1 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) Facebook, Inc.
* Copyright (c) Meta Platforms, Inc. and affiliates.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
programs/platform.h
+9 -5
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) Przemyslaw Skibinski, Yann Collet, Facebook, Inc.
* Copyright (c) Meta Platforms, Inc. and affiliates.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
@@ -33,7 +33,7 @@ extern "C" {
/* **************************************
* Detect 64-bit OS
* http://nadeausoftware.com/articles/2012/02/c_c_tip_how_detect_processor_type_using_compiler_predefined_macros
* https://nadeausoftware.com/articles/2012/02/c_c_tip_how_detect_processor_type_using_compiler_predefined_macros
****************************************/
#if defined __ia64 || defined _M_IA64 /* Intel Itanium */ \
|| defined __powerpc64__ || defined __ppc64__ || defined __PPC64__ /* POWER 64-bit */ \
@@ -80,7 +80,7 @@ extern "C" {
* note: it's better to use unistd.h's _POSIX_VERSION whenever possible */
# define PLATFORM_POSIX_VERSION 200112L
/* try to determine posix version through official unistd.h's _POSIX_VERSION (http://pubs.opengroup.org/onlinepubs/7908799/xsh/unistd.h.html).
/* try to determine posix version through official unistd.h's _POSIX_VERSION (https://pubs.opengroup.org/onlinepubs/7908799/xsh/unistd.h.html).
* note : there is no simple way to know in advance if <unistd.h> is present or not on target system,
* Posix specification mandates its presence and its content, but target system must respect this spec.
* It's necessary to _not_ #include <unistd.h> whenever target OS is not unix-like
@@ -127,6 +127,10 @@ extern "C" {
/*-*********************************************
* Detect if isatty() and fileno() are available
*
* Note: Use UTIL_isConsole() for the zstd CLI
* instead, as it allows faking is console for
* testing.
************************************************/
#if (defined(__linux__) && (PLATFORM_POSIX_VERSION > 1)) \
|| (PLATFORM_POSIX_VERSION >= 200112L) \
@@ -192,13 +196,13 @@ static __inline int IS_CONSOLE(FILE* stdStream) {
#ifndef ZSTD_SETPRIORITY_SUPPORT
/* mandates presence of <sys/resource.h> and support for setpriority() : http://man7.org/linux/man-pages/man2/setpriority.2.html */
/* mandates presence of <sys/resource.h> and support for setpriority() : https://man7.org/linux/man-pages/man2/setpriority.2.html */
# define ZSTD_SETPRIORITY_SUPPORT (PLATFORM_POSIX_VERSION >= 200112L)
#endif
#ifndef ZSTD_NANOSLEEP_SUPPORT
/* mandates support of nanosleep() within <time.h> : http://man7.org/linux/man-pages/man2/nanosleep.2.html */
/* mandates support of nanosleep() within <time.h> : https://man7.org/linux/man-pages/man2/nanosleep.2.html */
# if (defined(__linux__) && (PLATFORM_POSIX_VERSION >= 199309L)) \
|| (PLATFORM_POSIX_VERSION >= 200112L)
# define ZSTD_NANOSLEEP_SUPPORT 1
programs/timefn.c
+72 -73
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) Yann Collet, Facebook, Inc.
* Copyright (c) Meta Platforms, Inc. and affiliates.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
@@ -12,7 +12,8 @@
/* === Dependencies === */
#include "timefn.h"
#include "platform.h" /* set _POSIX_C_SOURCE */
#include <time.h> /* CLOCK_MONOTONIC, TIME_UTC */
/*-****************************************
* Time functions
@@ -20,12 +21,11 @@
#if defined(_WIN32) /* Windows */
#include <windows.h> /* LARGE_INTEGER */
#include <stdlib.h> /* abort */
#include <stdio.h> /* perror */
UTIL_time_t UTIL_getTime(void) { UTIL_time_t x; QueryPerformanceCounter(&x); return x; }
PTime UTIL_getSpanTimeMicro(UTIL_time_t clockStart, UTIL_time_t clockEnd)
UTIL_time_t UTIL_getTime(void)
{
static LARGE_INTEGER ticksPerSecond;
static int init = 0;
@@ -36,30 +36,20 @@ PTime UTIL_getSpanTimeMicro(UTIL_time_t clockStart, UTIL_time_t clockEnd)
}
init = 1;
}
return 1000000ULL*(clockEnd.QuadPart - clockStart.QuadPart)/ticksPerSecond.QuadPart;
}
PTime UTIL_getSpanTimeNano(UTIL_time_t clockStart, UTIL_time_t clockEnd)
{
static LARGE_INTEGER ticksPerSecond;
static int init = 0;
if (!init) {
if (!QueryPerformanceFrequency(&ticksPerSecond)) {
perror("timefn::QueryPerformanceFrequency");
abort();
}
init = 1;
{ UTIL_time_t r;
LARGE_INTEGER x;
QueryPerformanceCounter(&x);
r.t = (PTime)(x.QuadPart * 1000000000ULL / ticksPerSecond.QuadPart);
return r;
}
return 1000000000ULL*(clockEnd.QuadPart - clockStart.QuadPart)/ticksPerSecond.QuadPart;
}
#elif defined(__APPLE__) && defined(__MACH__)
UTIL_time_t UTIL_getTime(void) { return mach_absolute_time(); }
#include <mach/mach_time.h> /* mach_timebase_info_data_t, mach_timebase_info, mach_absolute_time */
PTime UTIL_getSpanTimeMicro(UTIL_time_t clockStart, UTIL_time_t clockEnd)
UTIL_time_t UTIL_getTime(void)
{
static mach_timebase_info_data_t rate;
static int init = 0;
@@ -67,23 +57,39 @@ PTime UTIL_getSpanTimeMicro(UTIL_time_t clockStart, UTIL_time_t clockEnd)
mach_timebase_info(&rate);
init = 1;
}
return (((clockEnd - clockStart) * (PTime)rate.numer) / ((PTime)rate.denom))/1000ULL;
{ UTIL_time_t r;
r.t = mach_absolute_time() * (PTime)rate.numer / (PTime)rate.denom;
return r;
}
}
PTime UTIL_getSpanTimeNano(UTIL_time_t clockStart, UTIL_time_t clockEnd)
/* POSIX.1-2001 (optional) */
#elif defined(CLOCK_MONOTONIC)
#include <stdlib.h> /* abort */
#include <stdio.h> /* perror */
UTIL_time_t UTIL_getTime(void)
{
static mach_timebase_info_data_t rate;
static int init = 0;
if (!init) {
mach_timebase_info(&rate);
init = 1;
/* time must be initialized, othersize it may fail msan test.
* No good reason, likely a limitation of timespec_get() for some target */
struct timespec time = { 0, 0 };
if (clock_gettime(CLOCK_MONOTONIC, &time) != 0) {
perror("timefn::clock_gettime(CLOCK_MONOTONIC)");
abort();
}
{ UTIL_time_t r;
r.t = (PTime)time.tv_sec * 1000000000ULL + (PTime)time.tv_nsec;
return r;
}
return ((clockEnd - clockStart) * (PTime)rate.numer) / ((PTime)rate.denom);
}
/* C11 requires timespec_get, but FreeBSD 11 lacks it, while still claiming C11 compliance.
Android also lacks it but does define TIME_UTC. */
/* C11 requires support of timespec_get().
* However, FreeBSD 11 claims C11 compliance while lacking timespec_get().
* Double confirm timespec_get() support by checking the definition of TIME_UTC.
* However, some versions of Android manage to simultanously define TIME_UTC
* and lack timespec_get() support... */
#elif (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) /* C11 */) \
&& defined(TIME_UTC) && !defined(__ANDROID__)
@@ -94,65 +100,49 @@ UTIL_time_t UTIL_getTime(void)
{
/* time must be initialized, othersize it may fail msan test.
* No good reason, likely a limitation of timespec_get() for some target */
UTIL_time_t time = UTIL_TIME_INITIALIZER;
struct timespec time = { 0, 0 };
if (timespec_get(&time, TIME_UTC) != TIME_UTC) {
perror("timefn::timespec_get");
perror("timefn::timespec_get(TIME_UTC)");
abort();
}
return time;
{ UTIL_time_t r;
r.t = (PTime)time.tv_sec * 1000000000ULL + (PTime)time.tv_nsec;
return r;
}
}
static UTIL_time_t UTIL_getSpanTime(UTIL_time_t begin, UTIL_time_t end)
#else /* relies on standard C90 (note : clock_t produces wrong measurements for multi-threaded workloads) */
UTIL_time_t UTIL_getTime(void)
{
UTIL_time_t diff;
if (end.tv_nsec < begin.tv_nsec) {
diff.tv_sec = (end.tv_sec - 1) - begin.tv_sec;
diff.tv_nsec = (end.tv_nsec + 1000000000ULL) - begin.tv_nsec;
} else {
diff.tv_sec = end.tv_sec - begin.tv_sec;
diff.tv_nsec = end.tv_nsec - begin.tv_nsec;
}
return diff;
UTIL_time_t r;
r.t = (PTime)clock() * 1000000000ULL / CLOCKS_PER_SEC;
return r;
}
#define TIME_MT_MEASUREMENTS_NOT_SUPPORTED
#endif
/* ==== Common functions, valid for all time API ==== */
PTime UTIL_getSpanTimeNano(UTIL_time_t clockStart, UTIL_time_t clockEnd)
{
return clockEnd.t - clockStart.t;
}
PTime UTIL_getSpanTimeMicro(UTIL_time_t begin, UTIL_time_t end)
{
UTIL_time_t const diff = UTIL_getSpanTime(begin, end);
PTime micro = 0;
micro += 1000000ULL * diff.tv_sec;
micro += diff.tv_nsec / 1000ULL;
return micro;
return UTIL_getSpanTimeNano(begin, end) / 1000ULL;
}
PTime UTIL_getSpanTimeNano(UTIL_time_t begin, UTIL_time_t end)
{
UTIL_time_t const diff = UTIL_getSpanTime(begin, end);
PTime nano = 0;
nano += 1000000000ULL * diff.tv_sec;
nano += diff.tv_nsec;
return nano;
}
#else /* relies on standard C90 (note : clock_t measurements can be wrong when using multi-threading) */
UTIL_time_t UTIL_getTime(void) { return clock(); }
PTime UTIL_getSpanTimeMicro(UTIL_time_t clockStart, UTIL_time_t clockEnd) { return 1000000ULL * (clockEnd - clockStart) / CLOCKS_PER_SEC; }
PTime UTIL_getSpanTimeNano(UTIL_time_t clockStart, UTIL_time_t clockEnd) { return 1000000000ULL * (clockEnd - clockStart) / CLOCKS_PER_SEC; }
#endif
/* returns time span in microseconds */
PTime UTIL_clockSpanMicro(UTIL_time_t clockStart )
{
UTIL_time_t const clockEnd = UTIL_getTime();
return UTIL_getSpanTimeMicro(clockStart, clockEnd);
}
/* returns time span in microseconds */
PTime UTIL_clockSpanNano(UTIL_time_t clockStart )
{
UTIL_time_t const clockEnd = UTIL_getTime();
@@ -167,3 +157,12 @@ void UTIL_waitForNextTick(void)
clockEnd = UTIL_getTime();
} while (UTIL_getSpanTimeNano(clockStart, clockEnd) == 0);
}
int UTIL_support_MT_measurements(void)
{
# if defined(TIME_MT_MEASUREMENTS_NOT_SUPPORTED)
return 0;
# else
return 1;
# endif
}
programs/timefn.h
+22 -42
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) Yann Collet, Facebook, Inc.
* Copyright (c) Meta Platforms, Inc. and affiliates.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
@@ -16,71 +16,51 @@ extern "C" {
#endif
/*-****************************************
* Dependencies
******************************************/
#include <time.h> /* clock_t, clock, CLOCKS_PER_SEC */
/*-****************************************
* Local Types
* Types
******************************************/
#if !defined (__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) )
# if defined(_AIX)
# include <inttypes.h>
# else
# include <stdint.h> /* intptr_t */
# include <stdint.h> /* uint64_t */
# endif
typedef uint64_t PTime; /* Precise Time */
#else
typedef unsigned long long PTime; /* does not support compilers without long long support */
#endif
/* UTIL_time_t contains a nanosecond time counter.
* The absolute value is not meaningful.
* It's only valid to compute the difference between 2 measurements. */
typedef struct { PTime t; } UTIL_time_t;
#define UTIL_TIME_INITIALIZER { 0 }
/*-****************************************
* Time functions
******************************************/
#if defined(_WIN32) /* Windows */
#include <windows.h> /* LARGE_INTEGER */
typedef LARGE_INTEGER UTIL_time_t;
#define UTIL_TIME_INITIALIZER { { 0, 0 } }
#elif defined(__APPLE__) && defined(__MACH__)
#include <mach/mach_time.h>
typedef PTime UTIL_time_t;
#define UTIL_TIME_INITIALIZER 0
/* C11 requires timespec_get, but FreeBSD 11 lacks it, while still claiming C11 compliance.
Android also lacks it but does define TIME_UTC. */
#elif (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) /* C11 */) \
&& defined(TIME_UTC) && !defined(__ANDROID__)
typedef struct timespec UTIL_time_t;
#define UTIL_TIME_INITIALIZER { 0, 0 }
#else /* relies on standard C90 (note : clock_t measurements can be wrong when using multi-threading) */
#define UTIL_TIME_USES_C90_CLOCK
typedef clock_t UTIL_time_t;
#define UTIL_TIME_INITIALIZER 0
#endif
UTIL_time_t UTIL_getTime(void);
PTime UTIL_getSpanTimeMicro(UTIL_time_t clockStart, UTIL_time_t clockEnd);
PTime UTIL_getSpanTimeNano(UTIL_time_t clockStart, UTIL_time_t clockEnd);
#define SEC_TO_MICRO ((PTime)1000000)
PTime UTIL_clockSpanMicro(UTIL_time_t clockStart);
/* Timer resolution can be low on some platforms.
* To improve accuracy, it's recommended to wait for a new tick
* before starting benchmark measurements */
void UTIL_waitForNextTick(void);
/* tells if timefn will return correct time measurements
* in presence of multi-threaded workload.
* note : this is not the case if only C90 clock_t measurements are available */
int UTIL_support_MT_measurements(void);
PTime UTIL_getSpanTimeNano(UTIL_time_t clockStart, UTIL_time_t clockEnd);
PTime UTIL_clockSpanNano(UTIL_time_t clockStart);
void UTIL_waitForNextTick(void);
PTime UTIL_getSpanTimeMicro(UTIL_time_t clockStart, UTIL_time_t clockEnd);
PTime UTIL_clockSpanMicro(UTIL_time_t clockStart);
#define SEC_TO_MICRO ((PTime)1000000) /* nb of microseconds in a second */
#if defined (__cplusplus)
programs/util.c
+201 -36
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) Przemyslaw Skibinski, Yann Collet, Facebook, Inc.
* Copyright (c) Meta Platforms, Inc. and affiliates.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
@@ -66,6 +66,27 @@ extern "C" {
#define UTIL_DISPLAY(...) fprintf(stderr, __VA_ARGS__)
#define UTIL_DISPLAYLEVEL(l, ...) { if (g_utilDisplayLevel>=l) { UTIL_DISPLAY(__VA_ARGS__); } }
static int g_traceDepth = 0;
int g_traceFileStat = 0;
#define UTIL_TRACE_CALL(...) \
{ \
if (g_traceFileStat) { \
UTIL_DISPLAY("Trace:FileStat: %*s> ", g_traceDepth, ""); \
UTIL_DISPLAY(__VA_ARGS__); \
UTIL_DISPLAY("\n"); \
++g_traceDepth; \
} \
}
#define UTIL_TRACE_RET(ret) \
{ \
if (g_traceFileStat) { \
--g_traceDepth; \
UTIL_DISPLAY("Trace:FileStat: %*s< %d\n", g_traceDepth, "", (ret)); \
} \
}
/* A modified version of realloc().
* If UTIL_realloc() fails the original block is freed.
*/
@@ -100,7 +121,7 @@ int UTIL_requireUserConfirmation(const char* prompt, const char* abortMsg,
ch = getchar();
result = 0;
if (strchr(acceptableLetters, ch) == NULL) {
UTIL_DISPLAY("%s", abortMsg);
UTIL_DISPLAY("%s \n", abortMsg);
result = 1;
}
/* flush the rest */
@@ -121,21 +142,34 @@ int UTIL_requireUserConfirmation(const char* prompt, const char* abortMsg,
* Functions
***************************************/
void UTIL_traceFileStat(void)
{
g_traceFileStat = 1;
}
int UTIL_stat(const char* filename, stat_t* statbuf)
{
int ret;
UTIL_TRACE_CALL("UTIL_stat(%s)", filename);
#if defined(_MSC_VER)
return !_stat64(filename, statbuf);
ret = !_stat64(filename, statbuf);
#elif defined(__MINGW32__) && defined (__MSVCRT__)
return !_stati64(filename, statbuf);
ret = !_stati64(filename, statbuf);
#else
return !stat(filename, statbuf);
ret = !stat(filename, statbuf);
#endif
UTIL_TRACE_RET(ret);
return ret;
}
int UTIL_isRegularFile(const char* infilename)
{
stat_t statbuf;
return UTIL_stat(infilename, &statbuf) && UTIL_isRegularFileStat(&statbuf);
int ret;
UTIL_TRACE_CALL("UTIL_isRegularFile(%s)", infilename);
ret = UTIL_stat(infilename, &statbuf) && UTIL_isRegularFileStat(&statbuf);
UTIL_TRACE_RET(ret);
return ret;
}
int UTIL_isRegularFileStat(const stat_t* statbuf)
@@ -151,71 +185,114 @@ int UTIL_isRegularFileStat(const stat_t* statbuf)
int UTIL_chmod(char const* filename, const stat_t* statbuf, mode_t permissions)
{
stat_t localStatBuf;
UTIL_TRACE_CALL("UTIL_chmod(%s, %#4o)", filename, (unsigned)permissions);
if (statbuf == NULL) {
if (!UTIL_stat(filename, &localStatBuf)) return 0;
if (!UTIL_stat(filename, &localStatBuf)) {
UTIL_TRACE_RET(0);
return 0;
}
statbuf = &localStatBuf;
}
if (!UTIL_isRegularFileStat(statbuf)) return 0; /* pretend success, but don't change anything */
return chmod(filename, permissions);
if (!UTIL_isRegularFileStat(statbuf)) {
UTIL_TRACE_RET(0);
return 0; /* pretend success, but don't change anything */
}
UTIL_TRACE_CALL("chmod");
{
int const ret = chmod(filename, permissions);
UTIL_TRACE_RET(ret);
UTIL_TRACE_RET(ret);
return ret;
}
}
/* set access and modification times */
int UTIL_utime(const char* filename, const stat_t *statbuf)
{
int ret;
UTIL_TRACE_CALL("UTIL_utime(%s)", filename);
/* We check that st_mtime is a macro here in order to give us confidence
* that struct stat has a struct timespec st_mtim member. We need this
* check because there are some platforms that claim to be POSIX 2008
* compliant but which do not have st_mtim... */
#if (PLATFORM_POSIX_VERSION >= 200809L) && defined(st_mtime)
/* (atime, mtime) */
struct timespec timebuf[2] = { {0, UTIME_NOW} };
timebuf[1] = statbuf->st_mtim;
ret = utimensat(AT_FDCWD, filename, timebuf, 0);
{
/* (atime, mtime) */
struct timespec timebuf[2] = { {0, UTIME_NOW} };
timebuf[1] = statbuf->st_mtim;
ret = utimensat(AT_FDCWD, filename, timebuf, 0);
}
#else
struct utimbuf timebuf;
timebuf.actime = time(NULL);
timebuf.modtime = statbuf->st_mtime;
ret = utime(filename, &timebuf);
{
struct utimbuf timebuf;
timebuf.actime = time(NULL);
timebuf.modtime = statbuf->st_mtime;
ret = utime(filename, &timebuf);
}
#endif
errno = 0;
UTIL_TRACE_RET(ret);
return ret;
}
int UTIL_setFileStat(const char *filename, const stat_t *statbuf)
{
int res = 0;
stat_t curStatBuf;
if (!UTIL_stat(filename, &curStatBuf) || !UTIL_isRegularFileStat(&curStatBuf))
UTIL_TRACE_CALL("UTIL_setFileStat(%s)", filename);
if (!UTIL_stat(filename, &curStatBuf) || !UTIL_isRegularFileStat(&curStatBuf)) {
UTIL_TRACE_RET(-1);
return -1;
}
/* set access and modification times */
res += UTIL_utime(filename, statbuf);
/* Mimic gzip's behavior:
*
* "Change the group first, then the permissions, then the owner.
* That way, the permissions will be correct on systems that allow
* users to give away files, without introducing a security hole.
* Security depends on permissions not containing the setuid or
* setgid bits." */
#if !defined(_WIN32)
res += chown(filename, statbuf->st_uid, statbuf->st_gid); /* Copy ownership */
res += chown(filename, -1, statbuf->st_gid); /* Apply group ownership */
#endif
res += UTIL_chmod(filename, &curStatBuf, statbuf->st_mode & 07777); /* Copy file permissions */
res += UTIL_chmod(filename, &curStatBuf, statbuf->st_mode & 0777); /* Copy file permissions */
#if !defined(_WIN32)
res += chown(filename, statbuf->st_uid, -1); /* Apply user ownership */
#endif
errno = 0;
UTIL_TRACE_RET(-res);
return -res; /* number of errors is returned */
}
int UTIL_isDirectory(const char* infilename)
{
stat_t statbuf;
return UTIL_stat(infilename, &statbuf) && UTIL_isDirectoryStat(&statbuf);
int ret;
UTIL_TRACE_CALL("UTIL_isDirectory(%s)", infilename);
ret = UTIL_stat(infilename, &statbuf) && UTIL_isDirectoryStat(&statbuf);
UTIL_TRACE_RET(ret);
return ret;
}
int UTIL_isDirectoryStat(const stat_t* statbuf)
{
int ret;
UTIL_TRACE_CALL("UTIL_isDirectoryStat()");
#if defined(_MSC_VER)
return (statbuf->st_mode & _S_IFDIR) != 0;
ret = (statbuf->st_mode & _S_IFDIR) != 0;
#else
return S_ISDIR(statbuf->st_mode) != 0;
ret = S_ISDIR(statbuf->st_mode) != 0;
#endif
UTIL_TRACE_RET(ret);
return ret;
}
int UTIL_compareStr(const void *p1, const void *p2) {
@@ -224,33 +301,68 @@ int UTIL_compareStr(const void *p1, const void *p2) {
int UTIL_isSameFile(const char* fName1, const char* fName2)
{
int ret;
assert(fName1 != NULL); assert(fName2 != NULL);
UTIL_TRACE_CALL("UTIL_isSameFile(%s, %s)", fName1, fName2);
#if defined(_MSC_VER) || defined(_WIN32)
/* note : Visual does not support file identification by inode.
* inode does not work on Windows, even with a posix layer, like msys2.
* The following work-around is limited to detecting exact name repetition only,
* aka `filename` is considered different from `subdir/../filename` */
return !strcmp(fName1, fName2);
ret = !strcmp(fName1, fName2);
#else
{ stat_t file1Stat;
stat_t file2Stat;
return UTIL_stat(fName1, &file1Stat)
ret = UTIL_stat(fName1, &file1Stat)
&& UTIL_stat(fName2, &file2Stat)
&& (file1Stat.st_dev == file2Stat.st_dev)
&& (file1Stat.st_ino == file2Stat.st_ino);
&& UTIL_isSameFileStat(fName1, fName2, &file1Stat, &file2Stat);
}
#endif
UTIL_TRACE_RET(ret);
return ret;
}
int UTIL_isSameFileStat(
const char* fName1, const char* fName2,
const stat_t* file1Stat, const stat_t* file2Stat)
{
int ret;
assert(fName1 != NULL); assert(fName2 != NULL);
UTIL_TRACE_CALL("UTIL_isSameFileStat(%s, %s)", fName1, fName2);
#if defined(_MSC_VER) || defined(_WIN32)
/* note : Visual does not support file identification by inode.
* inode does not work on Windows, even with a posix layer, like msys2.
* The following work-around is limited to detecting exact name repetition only,
* aka `filename` is considered different from `subdir/../filename` */
(void)file1Stat;
(void)file2Stat;
ret = !strcmp(fName1, fName2);
#else
{
ret = (file1Stat->st_dev == file2Stat->st_dev)
&& (file1Stat->st_ino == file2Stat->st_ino);
}
#endif
UTIL_TRACE_RET(ret);
return ret;
}
/* UTIL_isFIFO : distinguish named pipes */
int UTIL_isFIFO(const char* infilename)
{
UTIL_TRACE_CALL("UTIL_isFIFO(%s)", infilename);
/* macro guards, as defined in : https://linux.die.net/man/2/lstat */
#if PLATFORM_POSIX_VERSION >= 200112L
stat_t statbuf;
if (UTIL_stat(infilename, &statbuf) && UTIL_isFIFOStat(&statbuf)) return 1;
{
stat_t statbuf;
if (UTIL_stat(infilename, &statbuf) && UTIL_isFIFOStat(&statbuf)) {
UTIL_TRACE_RET(1);
return 1;
}
}
#endif
(void)infilename;
UTIL_TRACE_RET(0);
return 0;
}
@@ -278,21 +390,69 @@ int UTIL_isBlockDevStat(const stat_t* statbuf)
int UTIL_isLink(const char* infilename)
{
UTIL_TRACE_CALL("UTIL_isLink(%s)", infilename);
/* macro guards, as defined in : https://linux.die.net/man/2/lstat */
#if PLATFORM_POSIX_VERSION >= 200112L
stat_t statbuf;
int const r = lstat(infilename, &statbuf);
if (!r && S_ISLNK(statbuf.st_mode)) return 1;
{
stat_t statbuf;
int const r = lstat(infilename, &statbuf);
if (!r && S_ISLNK(statbuf.st_mode)) {
UTIL_TRACE_RET(1);
return 1;
}
}
#endif
(void)infilename;
UTIL_TRACE_RET(0);
return 0;
}
static int g_fakeStdinIsConsole = 0;
static int g_fakeStderrIsConsole = 0;
static int g_fakeStdoutIsConsole = 0;
int UTIL_isConsole(FILE* file)
{
int ret;
UTIL_TRACE_CALL("UTIL_isConsole(%d)", fileno(file));
if (file == stdin && g_fakeStdinIsConsole)
ret = 1;
else if (file == stderr && g_fakeStderrIsConsole)
ret = 1;
else if (file == stdout && g_fakeStdoutIsConsole)
ret = 1;
else
ret = IS_CONSOLE(file);
UTIL_TRACE_RET(ret);
return ret;
}
void UTIL_fakeStdinIsConsole(void)
{
g_fakeStdinIsConsole = 1;
}
void UTIL_fakeStdoutIsConsole(void)
{
g_fakeStdoutIsConsole = 1;
}
void UTIL_fakeStderrIsConsole(void)
{
g_fakeStderrIsConsole = 1;
}
U64 UTIL_getFileSize(const char* infilename)
{
stat_t statbuf;
if (!UTIL_stat(infilename, &statbuf)) return UTIL_FILESIZE_UNKNOWN;
return UTIL_getFileSizeStat(&statbuf);
UTIL_TRACE_CALL("UTIL_getFileSize(%s)", infilename);
if (!UTIL_stat(infilename, &statbuf)) {
UTIL_TRACE_RET(-1);
return UTIL_FILESIZE_UNKNOWN;
}
{
U64 const size = UTIL_getFileSizeStat(&statbuf);
UTIL_TRACE_RET((int)size);
return size;
}
}
U64 UTIL_getFileSizeStat(const stat_t* statbuf)
@@ -369,11 +529,16 @@ U64 UTIL_getTotalFileSize(const char* const * fileNamesTable, unsigned nbFiles)
{
U64 total = 0;
unsigned n;
UTIL_TRACE_CALL("UTIL_getTotalFileSize(%u)", nbFiles);
for (n=0; n<nbFiles; n++) {
U64 const size = UTIL_getFileSize(fileNamesTable[n]);
if (size == UTIL_FILESIZE_UNKNOWN) return UTIL_FILESIZE_UNKNOWN;
if (size == UTIL_FILESIZE_UNKNOWN) {
UTIL_TRACE_RET(-1);
return UTIL_FILESIZE_UNKNOWN;
}
total += size;
}
UTIL_TRACE_RET((int)total);
return total;
}
programs/util.h
+21 -2
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) Przemyslaw Skibinski, Yann Collet, Facebook, Inc.
* Copyright (c) Meta Platforms, Inc. and affiliates.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
@@ -171,10 +171,30 @@ int UTIL_chmod(char const* filename, const stat_t* statbuf, mode_t permissions);
int UTIL_isRegularFile(const char* infilename);
int UTIL_isDirectory(const char* infilename);
int UTIL_isSameFile(const char* file1, const char* file2);
int UTIL_isSameFileStat(const char* file1, const char* file2, const stat_t* file1Stat, const stat_t* file2Stat);
int UTIL_isCompressedFile(const char* infilename, const char *extensionList[]);
int UTIL_isLink(const char* infilename);
int UTIL_isFIFO(const char* infilename);
/**
* Returns with the given file descriptor is a console.
* Allows faking whether stdin/stdout/stderr is a console
* using UTIL_fake*IsConsole().
*/
int UTIL_isConsole(FILE* file);
/**
* Pretends that stdin/stdout/stderr is a console for testing.
*/
void UTIL_fakeStdinIsConsole(void);
void UTIL_fakeStdoutIsConsole(void);
void UTIL_fakeStderrIsConsole(void);
/**
* Emit traces for functions that read, or modify file metadata.
*/
void UTIL_traceFileStat(void);
#define UTIL_FILESIZE_UNKNOWN ((U64)(-1))
U64 UTIL_getFileSize(const char* infilename);
U64 UTIL_getTotalFileSize(const char* const * fileNamesTable, unsigned nbFiles);
@@ -248,7 +268,6 @@ UTIL_mergeFileNamesTable(FileNamesTable* table1, FileNamesTable* table2);
/*! UTIL_expandFNT() :
* read names from @fnt, and expand those corresponding to directories
* update @fnt, now containing only file names,
* @return : 0 in case of success, 1 if error
* note : in case of error, @fnt[0] is NULL
*/
void UTIL_expandFNT(FileNamesTable** fnt, int followLinks);
programs/windres/verrsrc.h
+1 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) Yann Collet, Facebook, Inc.
* Copyright (c) Meta Platforms, Inc. and affiliates.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
programs/windres/zstd.rc
+2 -2
View File
@@ -32,11 +32,11 @@ BEGIN
BEGIN
BLOCK "040904B0"
BEGIN
VALUE "CompanyName", "Yann Collet, Facebook, Inc."
VALUE "CompanyName", "Meta Platforms, Inc."
VALUE "FileDescription", "Zstandard - Fast and efficient compression algorithm"
VALUE "FileVersion", ZSTD_VERSION_STRING
VALUE "InternalName", "zstd.exe"
VALUE "LegalCopyright", "Copyright (c) 2013-present, Yann Collet, Facebook, Inc."
VALUE "LegalCopyright", "Copyright (c) Meta Platforms, Inc. and affiliates."
VALUE "OriginalFilename", "zstd.exe"
VALUE "ProductName", "Zstandard"
VALUE "ProductVersion", ZSTD_VERSION_STRING
programs/zstd.1
+74 -41
View File
@@ -1,5 +1,5 @@
.
.TH "ZSTD" "1" "August 2022" "zstd 1.5.3" "User Commands"
.TH "ZSTD" "1" "December 2022" "zstd 1.5.3" "User Commands"
.
.SH "NAME"
\fBzstd\fR \- zstd, zstdmt, unzstd, zstdcat \- Compress or decompress \.zst files
@@ -17,10 +17,10 @@
\fBzstdcat\fR is equivalent to \fBzstd \-dcf\fR
.
.SH "DESCRIPTION"
\fBzstd\fR is a fast lossless compression algorithm and data compression tool, with command line syntax similar to \fBgzip (1)\fR and \fBxz (1)\fR\. It is based on the \fBLZ77\fR family, with further FSE & huff0 entropy stages\. \fBzstd\fR offers highly configurable compression speed, from fast modes at > 200 MB/s per core, to strong modes with excellent compression ratios\. It also features a very fast decoder, with speeds > 500 MB/s per core\.
\fBzstd\fR is a fast lossless compression algorithm and data compression tool, with command line syntax similar to \fBgzip\fR(1) and \fBxz\fR(1)\. It is based on the \fBLZ77\fR family, with further FSE & huff0 entropy stages\. \fBzstd\fR offers highly configurable compression speed, from fast modes at > 200 MB/s per core, to strong modes with excellent compression ratios\. It also features a very fast decoder, with speeds > 500 MB/s per core\.
.
.P
\fBzstd\fR command line syntax is generally similar to gzip, but features the following differences :
\fBzstd\fR command line syntax is generally similar to gzip, but features the following differences:
.
.IP "\(bu" 4
Source files are preserved by default\. It\'s possible to remove them automatically by using the \fB\-\-rm\fR command\.
@@ -34,10 +34,13 @@ When compressing a single file, \fBzstd\fR displays progress notifications and r
.IP "\(bu" 4
\fBzstd\fR does not accept input from console, though it does accept \fBstdin\fR when it\'s not the console\.
.
.IP "\(bu" 4
\fBzstd\fR does not store the input\'s filename or attributes, only its contents\.
.
.IP "" 0
.
.P
\fBzstd\fR processes each \fIfile\fR according to the selected operation mode\. If no \fIfiles\fR are given or \fIfile\fR is \fB\-\fR, \fBzstd\fR reads from standard input and writes the processed data to standard output\. \fBzstd\fR will refuse to write compressed data to standard output if it is a terminal : it will display an error message and skip the \fIfile\fR\. Similarly, \fBzstd\fR will refuse to read compressed data from standard input if it is a terminal\.
\fBzstd\fR processes each \fIfile\fR according to the selected operation mode\. If no \fIfiles\fR are given or \fIfile\fR is \fB\-\fR, \fBzstd\fR reads from standard input and writes the processed data to standard output\. \fBzstd\fR will refuse to write compressed data to standard output if it is a terminal: it will display an error message and skip the file\. Similarly, \fBzstd\fR will refuse to read compressed data from standard input if it is a terminal\.
.
.P
Unless \fB\-\-stdout\fR or \fB\-o\fR is specified, \fIfiles\fR are written to a new file whose name is derived from the source \fIfile\fR name:
@@ -50,12 +53,12 @@ When decompressing, the \fB\.zst\fR suffix is removed from the source filename t
.
.IP "" 0
.
.SS "Concatenation with \.zst files"
.SS "Concatenation with \.zst Files"
It is possible to concatenate multiple \fB\.zst\fR files\. \fBzstd\fR will decompress such agglomerated file as if it was a single \fB\.zst\fR file\.
.
.SH "OPTIONS"
.
.SS "Integer suffixes and special values"
.SS "Integer Suffixes and Special Values"
In most places where an integer argument is expected, an optional suffix is supported to easily indicate large integers\. There must be no space between the integer and the suffix\.
.
.TP
@@ -66,7 +69,7 @@ Multiply the integer by 1,024 (2^10)\. \fBKi\fR, \fBK\fR, and \fBKB\fR are accep
\fBMiB\fR
Multiply the integer by 1,048,576 (2^20)\. \fBMi\fR, \fBM\fR, and \fBMB\fR are accepted as synonyms for \fBMiB\fR\.
.
.SS "Operation mode"
.SS "Operation Mode"
If multiple operation mode options are given, the last one takes effect\.
.
.TP
@@ -83,20 +86,20 @@ Test the integrity of compressed \fIfiles\fR\. This option is equivalent to \fB\
.
.TP
\fB\-b#\fR
Benchmark file(s) using compression level #
Benchmark file(s) using compression level \fI#\fR\. See \fIBENCHMARK\fR below for a description of this operation\.
.
.TP
\fB\-\-train FILEs\fR
Use FILEs as a training set to create a dictionary\. The training set should contain a lot of small files (> 100)\.
\fB\-\-train FILES\fR
Use \fIFILES\fR as a training set to create a dictionary\. The training set should contain a lot of small files (> 100)\. See \fIDICTIONARY BUILDER\fR below for a description of this operation\.
.
.TP
\fB\-l\fR, \fB\-\-list\fR
Display information related to a zstd compressed file, such as size, ratio, and checksum\. Some of these fields may not be available\. This command\'s output can be augmented with the \fB\-v\fR modifier\.
.
.SS "Operation modifiers"
.SS "Operation Modifiers"
.
.IP "\(bu" 4
\fB\-#\fR: \fB#\fR compression level [1\-19] (default: 3)
\fB\-#\fR: selects \fB#\fR compression level [1\-19] (default: 3)
.
.IP "\(bu" 4
\fB\-\-ultra\fR: unlocks high compression levels 20+ (maximum 22), using a lot more memory\. Note that decompression will also require more memory when using these levels\.
@@ -108,13 +111,22 @@ Display information related to a zstd compressed file, such as size, ratio, and
\fB\-T#\fR, \fB\-\-threads=#\fR: Compress using \fB#\fR working threads (default: 1)\. If \fB#\fR is 0, attempt to detect and use the number of physical CPU cores\. In all cases, the nb of threads is capped to \fBZSTDMT_NBWORKERS_MAX\fR, which is either 64 in 32\-bit mode, or 256 for 64\-bit environments\. This modifier does nothing if \fBzstd\fR is compiled without multithread support\.
.
.IP "\(bu" 4
\fB\-\-single\-thread\fR: Use a single thread for both I/O and compression\. As compression is serialized with I/O, this can be slightly slower\. Single\-thread mode features significantly lower memory usage, which can be useful for systems with limited amount of memory, such as 32\-bit systems\. Note 1 : this mode is the only available one when multithread support is disabled\. Note 2 : this mode is different from \fB\-T1\fR, which spawns 1 compression thread in parallel with I/O\. Final compressed result is also slightly different from \fB\-T1\fR\.
\fB\-\-single\-thread\fR: Use a single thread for both I/O and compression\. As compression is serialized with I/O, this can be slightly slower\. Single\-thread mode features significantly lower memory usage, which can be useful for systems with limited amount of memory, such as 32\-bit systems\.
.
.IP
Note 1: this mode is the only available one when multithread support is disabled\.
.
.IP
Note 2: this mode is different from \fB\-T1\fR, which spawns 1 compression thread in parallel with I/O\. Final compressed result is also slightly different from \fB\-T1\fR\.
.
.IP "\(bu" 4
\fB\-\-auto\-threads={physical,logical} (default: physical)\fR: When using a default amount of threads via \fB\-T0\fR, choose the default based on the number of detected physical or logical cores\.
.
.IP "\(bu" 4
\fB\-\-adapt[=min=#,max=#]\fR : \fBzstd\fR will dynamically adapt compression level to perceived I/O conditions\. Compression level adaptation can be observed live by using command \fB\-v\fR\. Adaptation can be constrained between supplied \fBmin\fR and \fBmax\fR levels\. The feature works when combined with multi\-threading and \fB\-\-long\fR mode\. It does not work with \fB\-\-single\-thread\fR\. It sets window size to 8 MB by default (can be changed manually, see \fBwlog\fR)\. Due to the chaotic nature of dynamic adaptation, compressed result is not reproducible\. \fInote\fR : at the time of this writing, \fB\-\-adapt\fR can remain stuck at low speed when combined with multiple worker threads (>=2)\.
\fB\-\-adapt[=min=#,max=#]\fR: \fBzstd\fR will dynamically adapt compression level to perceived I/O conditions\. Compression level adaptation can be observed live by using command \fB\-v\fR\. Adaptation can be constrained between supplied \fBmin\fR and \fBmax\fR levels\. The feature works when combined with multi\-threading and \fB\-\-long\fR mode\. It does not work with \fB\-\-single\-thread\fR\. It sets window size to 8 MiB by default (can be changed manually, see \fBwlog\fR)\. Due to the chaotic nature of dynamic adaptation, compressed result is not reproducible\.
.
.IP
\fINote\fR: at the time of this writing, \fB\-\-adapt\fR can remain stuck at low speed when combined with multiple worker threads (>=2)\.
.
.IP "\(bu" 4
\fB\-\-long[=#]\fR: enables long distance matching with \fB#\fR \fBwindowLog\fR, if \fB#\fR is not present it defaults to \fB27\fR\. This increases the window size (\fBwindowLog\fR) and memory usage for both the compressor and decompressor\. This setting is designed to improve the compression ratio for files with long matches at a large distance\.
@@ -126,40 +138,49 @@ Note: If \fBwindowLog\fR is set to larger than 27, \fB\-\-long=windowLog\fR or \
\fB\-D DICT\fR: use \fBDICT\fR as Dictionary to compress or decompress FILE(s)
.
.IP "\(bu" 4
\fB\-\-patch\-from FILE\fR: Specify the file to be used as a reference point for zstd\'s diff engine\. This is effectively dictionary compression with some convenient parameter selection, namely that windowSize > srcSize\.
\fB\-\-patch\-from FILE\fR: Specify the file to be used as a reference point for zstd\'s diff engine\. This is effectively dictionary compression with some convenient parameter selection, namely that \fIwindowSize\fR > \fIsrcSize\fR\.
.
.IP
Note: cannot use both this and \-D together Note: \fB\-\-long\fR mode will be automatically activated if chainLog < fileLog (fileLog being the windowLog required to cover the whole file)\. You can also manually force it\. Note: for all levels, you can use \-\-patch\-from in \-\-single\-thread mode to improve compression ratio at the cost of speed Note: for level 19, you can get increased compression ratio at the cost of speed by specifying \fB\-\-zstd=targetLength=\fR to be something large (i\.e\. 4096), and by setting a large \fB\-\-zstd=chainLog=\fR
Note: cannot use both this and \fB\-D\fR together\.
.
.IP
Note: \fB\-\-long\fR mode will be automatically activated if \fIchainLog\fR < \fIfileLog\fR (\fIfileLog\fR being the \fIwindowLog\fR required to cover the whole file)\. You can also manually force it\.
.
.IP
Note: for all levels, you can use \fB\-\-patch\-from\fR in \fB\-\-single\-thread\fR mode to improve compression ratio at the cost of speed\.
.
.IP
Note: for level 19, you can get increased compression ratio at the cost of speed by specifying \fB\-\-zstd=targetLength=\fR to be something large (i\.e\. 4096), and by setting a large \fB\-\-zstd=chainLog=\fR\.
.
.IP "\(bu" 4
\fB\-\-rsyncable\fR : \fBzstd\fR will periodically synchronize the compression state to make the compressed file more rsync\-friendly\. There is a negligible impact to compression ratio, and the faster compression levels will see a small compression speed hit\. This feature does not work with \fB\-\-single\-thread\fR\. You probably don\'t want to use it with long range mode, since it will decrease the effectiveness of the synchronization points, but your mileage may vary\.
\fB\-\-rsyncable\fR: \fBzstd\fR will periodically synchronize the compression state to make the compressed file more rsync\-friendly\. There is a negligible impact to compression ratio, and the faster compression levels will see a small compression speed hit\. This feature does not work with \fB\-\-single\-thread\fR\. You probably don\'t want to use it with long range mode, since it will decrease the effectiveness of the synchronization points, but your mileage may vary\.
.
.IP "\(bu" 4
\fB\-C\fR, \fB\-\-[no\-]check\fR: add integrity check computed from uncompressed data (default: enabled)
.
.IP "\(bu" 4
\fB\-\-[no\-]content\-size\fR: enable / disable whether or not the original size of the file is placed in the header of the compressed file\. The default option is \-\-content\-size (meaning that the original size will be placed in the header)\.
\fB\-\-[no\-]content\-size\fR: enable / disable whether or not the original size of the file is placed in the header of the compressed file\. The default option is \fB\-\-content\-size\fR (meaning that the original size will be placed in the header)\.
.
.IP "\(bu" 4
\fB\-\-no\-dictID\fR: do not store dictionary ID within frame header (dictionary compression)\. The decoder will have to rely on implicit knowledge about which dictionary to use, it won\'t be able to check if it\'s correct\.
.
.IP "\(bu" 4
\fB\-M#\fR, \fB\-\-memory=#\fR: Set a memory usage limit\. By default, \fBzstd\fR uses 128 MB for decompression as the maximum amount of memory the decompressor is allowed to use, but you can override this manually if need be in either direction (i\.e\. you can increase or decrease it)\.
\fB\-M#\fR, \fB\-\-memory=#\fR: Set a memory usage limit\. By default, \fBzstd\fR uses 128 MiB for decompression as the maximum amount of memory the decompressor is allowed to use, but you can override this manually if need be in either direction (i\.e\. you can increase or decrease it)\.
.
.IP
This is also used during compression when using with \-\-patch\-from=\. In this case, this parameter overrides that maximum size allowed for a dictionary\. (128 MB)\.
This is also used during compression when using with \fB\-\-patch\-from=\fR\. In this case, this parameter overrides that maximum size allowed for a dictionary\. (128 MiB)\.
.
.IP
Additionally, this can be used to limit memory for dictionary training\. This parameter overrides the default limit of 2 GB\. zstd will load training samples up to the memory limit and ignore the rest\.
Additionally, this can be used to limit memory for dictionary training\. This parameter overrides the default limit of 2 GiB\. zstd will load training samples up to the memory limit and ignore the rest\.
.
.IP "\(bu" 4
\fB\-\-stream\-size=#\fR : Sets the pledged source size of input coming from a stream\. This value must be exact, as it will be included in the produced frame header\. Incorrect stream sizes will cause an error\. This information will be used to better optimize compression parameters, resulting in better and potentially faster compression, especially for smaller source sizes\.
\fB\-\-stream\-size=#\fR: Sets the pledged source size of input coming from a stream\. This value must be exact, as it will be included in the produced frame header\. Incorrect stream sizes will cause an error\. This information will be used to better optimize compression parameters, resulting in better and potentially faster compression, especially for smaller source sizes\.
.
.IP "\(bu" 4
\fB\-\-size\-hint=#\fR: When handling input from a stream, \fBzstd\fR must guess how large the source size will be when optimizing compression parameters\. If the stream size is relatively small, this guess may be a poor one, resulting in a higher compression ratio than expected\. This feature allows for controlling the guess when needed\. Exact guesses result in better compression ratios\. Overestimates result in slightly degraded compression ratios, while underestimates may result in significant degradation\.
.
.IP "\(bu" 4
\fB\-o FILE\fR: save result into \fBFILE\fR
\fB\-o FILE\fR: save result into \fBFILE\fR\.
.
.IP "\(bu" 4
\fB\-f\fR, \fB\-\-force\fR: disable input and output checks\. Allows overwriting existing files, input from console, output to stdout, operating on links, block devices, etc\. During decompression and when the output destination is stdout, pass\-through unrecognized formats as\-is\.
@@ -171,10 +192,10 @@ Additionally, this can be used to limit memory for dictionary training\. This pa
\fB\-\-[no\-]sparse\fR: enable / disable sparse FS support, to make files with many zeroes smaller on disk\. Creating sparse files may save disk space and speed up decompression by reducing the amount of disk I/O\. default: enabled when output is into a file, and disabled when output is stdout\. This setting overrides default and can force sparse mode over stdout\.
.
.IP "\(bu" 4
\fB\-\-[no\-]pass\-through\fR enable / disable passing through uncompressed files as\-is\. During decompression when pass\-through is enabled, unrecognized formats will be copied as\-is from the input to the output\. By default, pass\-through will occur when the output destination is stdout and the force (\-f) option is set\.
\fB\-\-[no\-]pass\-through\fR enable / disable passing through uncompressed files as\-is\. During decompression when pass\-through is enabled, unrecognized formats will be copied as\-is from the input to the output\. By default, pass\-through will occur when the output destination is stdout and the force (\fB\-f\fR) option is set\.
.
.IP "\(bu" 4
\fB\-\-rm\fR: remove source file(s) after successful compression or decompression\. If used in combination with \-o, will trigger a confirmation prompt (which can be silenced with \-f), as this is a destructive operation\.
\fB\-\-rm\fR: remove source file(s) after successful compression or decompression\. If used in combination with \fB\-o\fR, will trigger a confirmation prompt (which can be silenced with \fB\-f\fR), as this is a destructive operation\.
.
.IP "\(bu" 4
\fB\-k\fR, \fB\-\-keep\fR: keep source file(s) after successful compression or decompression\. This is the default behavior\.
@@ -201,7 +222,7 @@ If input directory contains "\.\.", the files in this directory will be ignored\
\fB\-h\fR/\fB\-H\fR, \fB\-\-help\fR: display help/long help and exit
.
.IP "\(bu" 4
\fB\-V\fR, \fB\-\-version\fR: display version number and exit\. Advanced : \fB\-vV\fR also displays supported formats\. \fB\-vvV\fR also displays POSIX support\. \fB\-q\fR will only display the version number, suitable for machine reading\.
\fB\-V\fR, \fB\-\-version\fR: display version number and exit\. Advanced: \fB\-vV\fR also displays supported formats\. \fB\-vvV\fR also displays POSIX support\. \fB\-q\fR will only display the version number, suitable for machine reading\.
.
.IP "\(bu" 4
\fB\-v\fR, \fB\-\-verbose\fR: verbose mode, display more information
@@ -213,14 +234,14 @@ If input directory contains "\.\.", the files in this directory will be ignored\
\fB\-\-no\-progress\fR: do not display the progress bar, but keep all other messages\.
.
.IP "\(bu" 4
\fB\-\-show\-default\-cparams\fR: Shows the default compression parameters that will be used for a particular src file\. If the provided src file is not a regular file (e\.g\. named pipe), the cli will just output the default parameters\. That is, the parameters that are used when the src size is unknown\.
\fB\-\-show\-default\-cparams\fR: shows the default compression parameters that will be used for a particular input file, based on the provided compression level and the input size\. If the provided file is not a regular file (e\.g\. a pipe), this flag will output the parameters used for inputs of unknown size\.
.
.IP "\(bu" 4
\fB\-\-\fR: All arguments after \fB\-\-\fR are treated as files
.
.IP "" 0
.
.SS "gzip Operation modifiers"
.SS "gzip Operation Modifiers"
When invoked via a \fBgzip\fR symlink, \fBzstd\fR will support further options that intend to mimic the \fBgzip\fR behavior:
.
.TP
@@ -231,7 +252,7 @@ do not store the original filename and timestamps when compressing a file\. This
\fB\-\-best\fR
alias to the option \fB\-9\fR\.
.
.SS "Interactions with Environment Variables"
.SS "Environment Variables"
Employing environment variables to set parameters has security implications\. Therefore, this avenue is intentionally limited\. Only \fBZSTD_CLEVEL\fR and \fBZSTD_NBTHREADS\fR are currently supported\. They set the compression level and number of threads to use during compression, respectively\.
.
.P
@@ -251,7 +272,7 @@ They can both be overridden by corresponding command line arguments: \fB\-#\fR f
Use FILEs as training set to create a dictionary\. The training set should ideally contain a lot of samples (> 100), and weight typically 100x the target dictionary size (for example, ~10 MB for a 100 KB dictionary)\. \fB\-\-train\fR can be combined with \fB\-r\fR to indicate a directory rather than listing all the files, which can be useful to circumvent shell expansion limits\.
.
.IP
Since dictionary compression is mostly effective for small files, the expectation is that the training set will only contain small files\. In the case where some samples happen to be large, only the first 128 KB of these samples will be used for training\.
Since dictionary compression is mostly effective for small files, the expectation is that the training set will only contain small files\. In the case where some samples happen to be large, only the first 128 KiB of these samples will be used for training\.
.
.IP
\fB\-\-train\fR supports multithreading if \fBzstd\fR is compiled with threading support (default)\. Additional advanced parameters can be specified with \fB\-\-train\-fastcover\fR\. The legacy dictionary builder can be accessed with \fB\-\-train\-legacy\fR\. The slower cover dictionary builder can be accessed with \fB\-\-train\-cover\fR\. Default \fB\-\-train\fR is equivalent to \fB\-\-train\-fastcover=d=8,steps=4\fR\.
@@ -281,7 +302,10 @@ In situations where the training set is larger than maximum memory, the CLI will
.
.TP
\fB\-\-dictID=#\fR
A dictionary ID is a locally unique ID\. The decoder will use this value to verify it is using the right dictionary\. By default, zstd will create a 4\-bytes random number ID\. It\'s possible to provide an explicit number ID instead\. It\'s up to the dictionary manager to not assign twice the same ID to 2 different dictionaries\. Note that short numbers have an advantage : an ID < 256 will only need 1 byte in the compressed frame header, and an ID < 65536 will only need 2 bytes\. This compares favorably to 4 bytes default\.
A dictionary ID is a locally unique ID\. The decoder will use this value to verify it is using the right dictionary\. By default, zstd will create a 4\-bytes random number ID\. It\'s possible to provide an explicit number ID instead\. It\'s up to the dictionary manager to not assign twice the same ID to 2 different dictionaries\. Note that short numbers have an advantage: an ID < 256 will only need 1 byte in the compressed frame header, and an ID < 65536 will only need 2 bytes\. This compares favorably to 4 bytes default\.
.
.IP
Note that RFC8878 reserves IDs less than 32768 and greater than or equal to 2^31, so they should not be used in public\.
.
.TP
\fB\-\-train\-cover[=k#,d=#,steps=#,split=#,shrink[=#]]\fR
@@ -366,7 +390,7 @@ cut file(s) into independent chunks of size # (default: no chunking)
set process priority to real\-time
.
.P
\fBOutput Format:\fR CompressionLevel#Filename : InputSize \-> OutputSize (CompressionRatio), CompressionSpeed, DecompressionSpeed
\fBOutput Format:\fR CompressionLevel#Filename: InputSize \-> OutputSize (CompressionRatio), CompressionSpeed, DecompressionSpeed
.
.P
\fBMethodology:\fR For both compression and decompression speed, the entire input is compressed/decompressed in\-memory to measure speed\. A run lasts at least 1 sec, so when files are small, they are compressed/decompressed several times per run, in order to improve measurement accuracy\.
@@ -377,14 +401,14 @@ set process priority to real\-time
Specify the size of each compression job\. This parameter is only available when multi\-threading is enabled\. Each compression job is run in parallel, so this value indirectly impacts the nb of active threads\. Default job size varies depending on compression level (generally \fB4 * windowSize\fR)\. \fB\-B#\fR makes it possible to manually select a custom size\. Note that job size must respect a minimum value which is enforced transparently\. This minimum is either 512 KB, or \fBoverlapSize\fR, whichever is largest\. Different job sizes will lead to non\-identical compressed frames\.
.
.SS "\-\-zstd[=options]:"
\fBzstd\fR provides 22 predefined compression levels\. The selected or default predefined compression level can be changed with advanced compression options\. The \fIoptions\fR are provided as a comma\-separated list\. You may specify only the options you want to change and the rest will be taken from the selected or default compression level\. The list of available \fIoptions\fR:
\fBzstd\fR provides 22 predefined regular compression levels plus the fast levels\. This compression level is translated internally into a number of specific parameters that actually control the behavior of the compressor\. (You can see the result of this translation with \fB\-\-show\-default\-cparams\fR\.) These specific parameters can be overridden with advanced compression options\. The \fIoptions\fR are provided as a comma\-separated list\. You may specify only the options you want to change and the rest will be taken from the selected or default compression level\. The list of available \fIoptions\fR:
.
.TP
\fBstrategy\fR=\fIstrat\fR, \fBstrat\fR=\fIstrat\fR
Specify a strategy used by a match finder\.
.
.IP
There are 9 strategies numbered from 1 to 9, from faster to stronger: 1=ZSTD_fast, 2=ZSTD_dfast, 3=ZSTD_greedy, 4=ZSTD_lazy, 5=ZSTD_lazy2, 6=ZSTD_btlazy2, 7=ZSTD_btopt, 8=ZSTD_btultra, 9=ZSTD_btultra2\.
There are 9 strategies numbered from 1 to 9, from fastest to strongest: 1=\fBZSTD_fast\fR, 2=\fBZSTD_dfast\fR, 3=\fBZSTD_greedy\fR, 4=\fBZSTD_lazy\fR, 5=\fBZSTD_lazy2\fR, 6=\fBZSTD_btlazy2\fR, 7=\fBZSTD_btopt\fR, 8=\fBZSTD_btultra\fR, 9=\fBZSTD_btultra2\fR\.
.
.TP
\fBwindowLog\fR=\fIwlog\fR, \fBwlog\fR=\fIwlog\fR
@@ -404,17 +428,17 @@ Specify the maximum number of bits for a hash table\.
Bigger hash tables cause fewer collisions which usually makes compression faster, but requires more memory during compression\.
.
.IP
The minimum \fIhlog\fR is 6 (64 B) and the maximum is 30 (1 GiB)\.
The minimum \fIhlog\fR is 6 (64 entries / 256 B) and the maximum is 30 (1B entries / 4 GiB)\.
.
.TP
\fBchainLog\fR=\fIclog\fR, \fBclog\fR=\fIclog\fR
Specify the maximum number of bits for a hash chain or a binary tree\.
Specify the maximum number of bits for the secondary search structure, whose form depends on the selected \fBstrategy\fR\.
.
.IP
Higher numbers of bits increases the chance to find a match which usually improves compression ratio\. It also slows down compression speed and increases memory requirements for compression\. This option is ignored for the ZSTD_fast strategy\.
Higher numbers of bits increases the chance to find a match which usually improves compression ratio\. It also slows down compression speed and increases memory requirements for compression\. This option is ignored for the \fBZSTD_fast\fR \fBstrategy\fR, which only has the primary hash table\.
.
.IP
The minimum \fIclog\fR is 6 (64 B) and the maximum is 29 (524 Mib) on 32\-bit platforms and 30 (1 Gib) on 64\-bit platforms\.
The minimum \fIclog\fR is 6 (64 entries / 256 B) and the maximum is 29 (512M entries / 2 GiB) on 32\-bit platforms and 30 (1B entries / 4 GiB) on 64\-bit platforms\.
.
.TP
\fBsearchLog\fR=\fIslog\fR, \fBslog\fR=\fIslog\fR
@@ -441,20 +465,23 @@ The minimum \fImml\fR is 3 and the maximum is 7\.
The impact of this field vary depending on selected strategy\.
.
.IP
For ZSTD_btopt, ZSTD_btultra and ZSTD_btultra2, it specifies the minimum match length that causes match finder to stop searching\. A larger \fBtargetLength\fR usually improves compression ratio but decreases compression speed\. t For ZSTD_fast, it triggers ultra\-fast mode when > 0\. The value represents the amount of data skipped between match sampling\. Impact is reversed : a larger \fBtargetLength\fR increases compression speed but decreases compression ratio\.
For \fBZSTD_btopt\fR, \fBZSTD_btultra\fR and \fBZSTD_btultra2\fR, it specifies the minimum match length that causes match finder to stop searching\. A larger \fBtargetLength\fR usually improves compression ratio but decreases compression speed\.
.
.IP
For \fBZSTD_fast\fR, it triggers ultra\-fast mode when > 0\. The value represents the amount of data skipped between match sampling\. Impact is reversed: a larger \fBtargetLength\fR increases compression speed but decreases compression ratio\.
.
.IP
For all other strategies, this field has no impact\.
.
.IP
The minimum \fItlen\fR is 0 and the maximum is 128 Kib\.
The minimum \fItlen\fR is 0 and the maximum is 128 KiB\.
.
.TP
\fBoverlapLog\fR=\fIovlog\fR, \fBovlog\fR=\fIovlog\fR
Determine \fBoverlapSize\fR, amount of data reloaded from previous job\. This parameter is only available when multithreading is enabled\. Reloading more data improves compression ratio, but decreases speed\.
.
.IP
The minimum \fIovlog\fR is 0, and the maximum is 9\. 1 means "no overlap", hence completely independent jobs\. 9 means "full overlap", meaning up to \fBwindowSize\fR is reloaded from previous job\. Reducing \fIovlog\fR by 1 reduces the reloaded amount by a factor 2\. For example, 8 means "windowSize/2", and 6 means "windowSize/8"\. Value 0 is special and means "default" : \fIovlog\fR is automatically determined by \fBzstd\fR\. In which case, \fIovlog\fR will range from 6 to 9, depending on selected \fIstrat\fR\.
The minimum \fIovlog\fR is 0, and the maximum is 9\. 1 means "no overlap", hence completely independent jobs\. 9 means "full overlap", meaning up to \fBwindowSize\fR is reloaded from previous job\. Reducing \fIovlog\fR by 1 reduces the reloaded amount by a factor 2\. For example, 8 means "windowSize/2", and 6 means "windowSize/8"\. Value 0 is special and means "default": \fIovlog\fR is automatically determined by \fBzstd\fR\. In which case, \fIovlog\fR will range from 6 to 9, depending on selected \fIstrat\fR\.
.
.TP
\fBldmHashLog\fR=\fIlhlog\fR, \fBlhlog\fR=\fIlhlog\fR
@@ -514,6 +541,12 @@ The following parameters sets advanced compression options to something similar
.P
\fB\-\-zstd\fR=wlog=23,clog=23,hlog=22,slog=6,mml=3,tlen=48,strat=6
.
.SH "SEE ALSO"
\fBzstdgrep\fR(1), \fBzstdless\fR(1), \fBgzip\fR(1), \fBxz\fR(1)
.
.P
The \fIzstandard\fR format is specified in Y\. Collet, "Zstandard Compression and the \'application/zstd\' Media Type", https://www\.ietf\.org/rfc/rfc8878\.txt, Internet RFC 8878 (February 2021)\.
.
.SH "BUGS"
Report bugs at: https://github\.com/facebook/zstd/issues
.
programs/zstd.1.md
+83 -63
View File
@@ -4,7 +4,7 @@ zstd(1) -- zstd, zstdmt, unzstd, zstdcat - Compress or decompress .zst files
SYNOPSIS
--------
`zstd` [*OPTIONS*] [-|_INPUT-FILE_] [-o _OUTPUT-FILE_]
`zstd` [<OPTIONS>] [-|<INPUT-FILE>] [-o <OUTPUT-FILE>]
`zstdmt` is equivalent to `zstd -T0`
@@ -16,7 +16,7 @@ SYNOPSIS
DESCRIPTION
-----------
`zstd` is a fast lossless compression algorithm and data compression tool,
with command line syntax similar to `gzip (1)` and `xz (1)`.
with command line syntax similar to `gzip`(1) and `xz`(1).
It is based on the **LZ77** family, with further FSE & huff0 entropy stages.
`zstd` offers highly configurable compression speed,
from fast modes at > 200 MB/s per core,
@@ -24,7 +24,7 @@ to strong modes with excellent compression ratios.
It also features a very fast decoder, with speeds > 500 MB/s per core.
`zstd` command line syntax is generally similar to gzip,
but features the following differences :
but features the following differences:
- Source files are preserved by default.
It's possible to remove them automatically by using the `--rm` command.
@@ -35,12 +35,13 @@ but features the following differences :
Use `-q` to turn it off.
- `zstd` does not accept input from console,
though it does accept `stdin` when it's not the console.
- `zstd` does not store the input's filename or attributes, only its contents.
`zstd` processes each _file_ according to the selected operation mode.
If no _files_ are given or _file_ is `-`, `zstd` reads from standard input
and writes the processed data to standard output.
`zstd` will refuse to write compressed data to standard output
if it is a terminal : it will display an error message and skip the _file_.
if it is a terminal: it will display an error message and skip the file.
Similarly, `zstd` will refuse to read compressed data from standard input
if it is a terminal.
@@ -52,14 +53,15 @@ whose name is derived from the source _file_ name:
* When decompressing, the `.zst` suffix is removed from the source filename to
get the target filename
### Concatenation with .zst files
### Concatenation with .zst Files
It is possible to concatenate multiple `.zst` files. `zstd` will decompress
such agglomerated file as if it was a single `.zst` file.
OPTIONS
-------
### Integer suffixes and special values
### Integer Suffixes and Special Values
In most places where an integer argument is expected,
an optional suffix is supported to easily indicate large integers.
There must be no space between the integer and the suffix.
@@ -71,7 +73,8 @@ There must be no space between the integer and the suffix.
Multiply the integer by 1,048,576 (2\^20).
`Mi`, `M`, and `MB` are accepted as synonyms for `MiB`.
### Operation mode
### Operation Mode
If multiple operation mode options are given,
the last one takes effect.
@@ -88,19 +91,21 @@ the last one takes effect.
decompressed data is discarded and checksummed for errors.
No files are created or removed.
* `-b#`:
Benchmark file(s) using compression level #
* `--train FILEs`:
Use FILEs as a training set to create a dictionary.
Benchmark file(s) using compression level _#_.
See _BENCHMARK_ below for a description of this operation.
* `--train FILES`:
Use _FILES_ as a training set to create a dictionary.
The training set should contain a lot of small files (> 100).
See _DICTIONARY BUILDER_ below for a description of this operation.
* `-l`, `--list`:
Display information related to a zstd compressed file, such as size, ratio, and checksum.
Some of these fields may not be available.
This command's output can be augmented with the `-v` modifier.
### Operation modifiers
### Operation Modifiers
* `-#`:
`#` compression level \[1-19] (default: 3)
selects `#` compression level \[1-19\] (default: 3)
* `--ultra`:
unlocks high compression levels 20+ (maximum 22), using a lot more memory.
Note that decompression will also require more memory when using these levels.
@@ -122,21 +127,24 @@ the last one takes effect.
As compression is serialized with I/O, this can be slightly slower.
Single-thread mode features significantly lower memory usage,
which can be useful for systems with limited amount of memory, such as 32-bit systems.
Note 1 : this mode is the only available one when multithread support is disabled.
Note 2 : this mode is different from `-T1`, which spawns 1 compression thread in parallel with I/O.
Note 1: this mode is the only available one when multithread support is disabled.
Note 2: this mode is different from `-T1`, which spawns 1 compression thread in parallel with I/O.
Final compressed result is also slightly different from `-T1`.
* `--auto-threads={physical,logical} (default: physical)`:
When using a default amount of threads via `-T0`, choose the default based on the number
of detected physical or logical cores.
* `--adapt[=min=#,max=#]` :
* `--adapt[=min=#,max=#]`:
`zstd` will dynamically adapt compression level to perceived I/O conditions.
Compression level adaptation can be observed live by using command `-v`.
Adaptation can be constrained between supplied `min` and `max` levels.
The feature works when combined with multi-threading and `--long` mode.
It does not work with `--single-thread`.
It sets window size to 8 MB by default (can be changed manually, see `wlog`).
It sets window size to 8 MiB by default (can be changed manually, see `wlog`).
Due to the chaotic nature of dynamic adaptation, compressed result is not reproducible.
_note_ : at the time of this writing, `--adapt` can remain stuck at low speed
_Note_: at the time of this writing, `--adapt` can remain stuck at low speed
when combined with multiple worker threads (>=2).
* `--long[=#]`:
enables long distance matching with `#` `windowLog`, if `#` is not
@@ -153,18 +161,21 @@ the last one takes effect.
* `--patch-from FILE`:
Specify the file to be used as a reference point for zstd's diff engine.
This is effectively dictionary compression with some convenient parameter
selection, namely that windowSize > srcSize.
selection, namely that _windowSize_ > _srcSize_.
Note: cannot use both this and -D together
Note: `--long` mode will be automatically activated if chainLog < fileLog
(fileLog being the windowLog required to cover the whole file). You
Note: cannot use both this and `-D` together.
Note: `--long` mode will be automatically activated if _chainLog_ < _fileLog_
(_fileLog_ being the _windowLog_ required to cover the whole file). You
can also manually force it.
Note: for all levels, you can use --patch-from in --single-thread mode
to improve compression ratio at the cost of speed
Note: for all levels, you can use `--patch-from` in `--single-thread` mode
to improve compression ratio at the cost of speed.
Note: for level 19, you can get increased compression ratio at the cost
of speed by specifying `--zstd=targetLength=` to be something large
(i.e. 4096), and by setting a large `--zstd=chainLog=`
* `--rsyncable` :
(i.e. 4096), and by setting a large `--zstd=chainLog=`.
* `--rsyncable`:
`zstd` will periodically synchronize the compression state to make the
compressed file more rsync-friendly. There is a negligible impact to
compression ratio, and the faster compression levels will see a small
@@ -177,24 +188,24 @@ the last one takes effect.
* `--[no-]content-size`:
enable / disable whether or not the original size of the file is placed in
the header of the compressed file. The default option is
--content-size (meaning that the original size will be placed in the header).
`--content-size` (meaning that the original size will be placed in the header).
* `--no-dictID`:
do not store dictionary ID within frame header (dictionary compression).
The decoder will have to rely on implicit knowledge about which dictionary to use,
it won't be able to check if it's correct.
* `-M#`, `--memory=#`:
Set a memory usage limit. By default, `zstd` uses 128 MB for decompression
Set a memory usage limit. By default, `zstd` uses 128 MiB for decompression
as the maximum amount of memory the decompressor is allowed to use, but you can
override this manually if need be in either direction (i.e. you can increase or
decrease it).
This is also used during compression when using with --patch-from=. In this case,
this parameter overrides that maximum size allowed for a dictionary. (128 MB).
This is also used during compression when using with `--patch-from=`. In this case,
this parameter overrides that maximum size allowed for a dictionary. (128 MiB).
Additionally, this can be used to limit memory for dictionary training. This parameter
overrides the default limit of 2 GB. zstd will load training samples up to the memory limit
overrides the default limit of 2 GiB. zstd will load training samples up to the memory limit
and ignore the rest.
* `--stream-size=#` :
* `--stream-size=#`:
Sets the pledged source size of input coming from a stream. This value must be exact, as it
will be included in the produced frame header. Incorrect stream sizes will cause an error.
This information will be used to better optimize compression parameters, resulting in
@@ -207,7 +218,7 @@ the last one takes effect.
Exact guesses result in better compression ratios. Overestimates result in slightly
degraded compression ratios, while underestimates may result in significant degradation.
* `-o FILE`:
save result into `FILE`
save result into `FILE`.
* `-f`, `--force`:
disable input and output checks. Allows overwriting existing files, input
from console, output to stdout, operating on links, block devices, etc.
@@ -227,11 +238,13 @@ the last one takes effect.
enable / disable passing through uncompressed files as-is. During
decompression when pass-through is enabled, unrecognized formats will be
copied as-is from the input to the output. By default, pass-through will
occur when the output destination is stdout and the force (-f) option is
occur when the output destination is stdout and the force (`-f`) option is
set.
* `--rm`:
remove source file(s) after successful compression or decompression. If used in combination with
-o, will trigger a confirmation prompt (which can be silenced with -f), as this is a destructive operation.
remove source file(s) after successful compression or decompression.
This command is silently ignored if output is `stdout`.
If used in combination with `-o`,
triggers a confirmation prompt (which can be silenced with `-f`), as this is a destructive operation.
* `-k`, `--keep`:
keep source file(s) after successful compression or decompression.
This is the default behavior.
@@ -270,7 +283,7 @@ the last one takes effect.
display help/long help and exit
* `-V`, `--version`:
display version number and exit.
Advanced : `-vV` also displays supported formats.
Advanced: `-vV` also displays supported formats.
`-vvV` also displays POSIX support.
`-q` will only display the version number, suitable for machine reading.
* `-v`, `--verbose`:
@@ -281,15 +294,13 @@ the last one takes effect.
* `--no-progress`:
do not display the progress bar, but keep all other messages.
* `--show-default-cparams`:
Shows the default compression parameters that will be used for a
particular src file. If the provided src file is not a regular file
(e.g. named pipe), the cli will just output the default parameters.
That is, the parameters that are used when the src size is unknown.
shows the default compression parameters that will be used for a particular input file, based on the provided compression level and the input size.
If the provided file is not a regular file (e.g. a pipe), this flag will output the parameters used for inputs of unknown size.
* `--`:
All arguments after `--` are treated as files
### gzip Operation modifiers
### gzip Operation Modifiers
When invoked via a `gzip` symlink, `zstd` will support further
options that intend to mimic the `gzip` behavior:
@@ -300,7 +311,7 @@ options that intend to mimic the `gzip` behavior:
alias to the option `-9`.
### Interactions with Environment Variables
### Environment Variables
Employing environment variables to set parameters has security implications.
Therefore, this avenue is intentionally limited.
@@ -341,7 +352,7 @@ Compression of small files similar to the sample set will be greatly improved.
Since dictionary compression is mostly effective for small files,
the expectation is that the training set will only contain small files.
In the case where some samples happen to be large,
only the first 128 KB of these samples will be used for training.
only the first 128 KiB of these samples will be used for training.
`--train` supports multithreading if `zstd` is compiled with threading support (default).
Additional advanced parameters can be specified with `--train-fastcover`.
@@ -389,11 +400,13 @@ Compression of small files similar to the sample set will be greatly improved.
It's possible to provide an explicit number ID instead.
It's up to the dictionary manager to not assign twice the same ID to
2 different dictionaries.
Note that short numbers have an advantage :
Note that short numbers have an advantage:
an ID < 256 will only need 1 byte in the compressed frame header,
and an ID < 65536 will only need 2 bytes.
This compares favorably to 4 bytes default.
Note that RFC8878 reserves IDs less than 32768 and greater than or equal to 2\^31, so they should not be used in public.
* `--train-cover[=k#,d=#,steps=#,split=#,shrink[=#]]`:
Select parameters for the default dictionary builder algorithm named cover.
If _d_ is not specified, then it tries _d_ = 6 and _d_ = 8.
@@ -482,7 +495,7 @@ BENCHMARK
* `--priority=rt`:
set process priority to real-time
**Output Format:** CompressionLevel#Filename : InputSize -> OutputSize (CompressionRatio), CompressionSpeed, DecompressionSpeed
**Output Format:** CompressionLevel#Filename: InputSize -> OutputSize (CompressionRatio), CompressionSpeed, DecompressionSpeed
**Methodology:** For both compression and decompression speed, the entire input is compressed/decompressed in-memory to measure speed. A run lasts at least 1 sec, so when files are small, they are compressed/decompressed several times per run, in order to improve measurement accuracy.
@@ -499,9 +512,10 @@ This minimum is either 512 KB, or `overlapSize`, whichever is largest.
Different job sizes will lead to non-identical compressed frames.
### --zstd[=options]:
`zstd` provides 22 predefined compression levels.
The selected or default predefined compression level can be changed with
advanced compression options.
`zstd` provides 22 predefined regular compression levels plus the fast levels.
This compression level is translated internally into a number of specific parameters that actually control the behavior of the compressor.
(You can see the result of this translation with `--show-default-cparams`.)
These specific parameters can be overridden with advanced compression options.
The _options_ are provided as a comma-separated list.
You may specify only the options you want to change and the rest will be
taken from the selected or default compression level.
@@ -510,10 +524,10 @@ The list of available _options_:
- `strategy`=_strat_, `strat`=_strat_:
Specify a strategy used by a match finder.
There are 9 strategies numbered from 1 to 9, from faster to stronger:
1=ZSTD\_fast, 2=ZSTD\_dfast, 3=ZSTD\_greedy,
4=ZSTD\_lazy, 5=ZSTD\_lazy2, 6=ZSTD\_btlazy2,
7=ZSTD\_btopt, 8=ZSTD\_btultra, 9=ZSTD\_btultra2.
There are 9 strategies numbered from 1 to 9, from fastest to strongest:
1=`ZSTD_fast`, 2=`ZSTD_dfast`, 3=`ZSTD_greedy`,
4=`ZSTD_lazy`, 5=`ZSTD_lazy2`, 6=`ZSTD_btlazy2`,
7=`ZSTD_btopt`, 8=`ZSTD_btultra`, 9=`ZSTD_btultra2`.
- `windowLog`=_wlog_, `wlog`=_wlog_:
Specify the maximum number of bits for a match distance.
@@ -533,19 +547,20 @@ The list of available _options_:
Bigger hash tables cause fewer collisions which usually makes compression
faster, but requires more memory during compression.
The minimum _hlog_ is 6 (64 B) and the maximum is 30 (1 GiB).
The minimum _hlog_ is 6 (64 entries / 256 B) and the maximum is 30 (1B entries / 4 GiB).
- `chainLog`=_clog_, `clog`=_clog_:
Specify the maximum number of bits for a hash chain or a binary tree.
Specify the maximum number of bits for the secondary search structure,
whose form depends on the selected `strategy`.
Higher numbers of bits increases the chance to find a match which usually
improves compression ratio.
It also slows down compression speed and increases memory requirements for
compression.
This option is ignored for the ZSTD_fast strategy.
This option is ignored for the `ZSTD_fast` `strategy`, which only has the primary hash table.
The minimum _clog_ is 6 (64 B) and the maximum is 29 (524 Mib) on 32-bit platforms
and 30 (1 Gib) on 64-bit platforms.
The minimum _clog_ is 6 (64 entries / 256 B) and the maximum is 29 (512M entries / 2 GiB) on 32-bit platforms
and 30 (1B entries / 4 GiB) on 64-bit platforms.
- `searchLog`=_slog_, `slog`=_slog_:
Specify the maximum number of searches in a hash chain or a binary tree
@@ -567,19 +582,19 @@ The list of available _options_:
- `targetLength`=_tlen_, `tlen`=_tlen_:
The impact of this field vary depending on selected strategy.
For ZSTD\_btopt, ZSTD\_btultra and ZSTD\_btultra2, it specifies
For `ZSTD_btopt`, `ZSTD_btultra` and `ZSTD_btultra2`, it specifies
the minimum match length that causes match finder to stop searching.
A larger `targetLength` usually improves compression ratio
but decreases compression speed.
t
For ZSTD\_fast, it triggers ultra-fast mode when > 0.
For `ZSTD_fast`, it triggers ultra-fast mode when > 0.
The value represents the amount of data skipped between match sampling.
Impact is reversed : a larger `targetLength` increases compression speed
Impact is reversed: a larger `targetLength` increases compression speed
but decreases compression ratio.
For all other strategies, this field has no impact.
The minimum _tlen_ is 0 and the maximum is 128 Kib.
The minimum _tlen_ is 0 and the maximum is 128 KiB.
- `overlapLog`=_ovlog_, `ovlog`=_ovlog_:
Determine `overlapSize`, amount of data reloaded from previous job.
@@ -591,7 +606,7 @@ t
9 means "full overlap", meaning up to `windowSize` is reloaded from previous job.
Reducing _ovlog_ by 1 reduces the reloaded amount by a factor 2.
For example, 8 means "windowSize/2", and 6 means "windowSize/8".
Value 0 is special and means "default" : _ovlog_ is automatically determined by `zstd`.
Value 0 is special and means "default": _ovlog_ is automatically determined by `zstd`.
In which case, _ovlog_ will range from 6 to 9, depending on selected _strat_.
- `ldmHashLog`=_lhlog_, `lhlog`=_lhlog_:
@@ -641,6 +656,11 @@ similar to predefined level 19 for files bigger than 256 KB:
`--zstd`=wlog=23,clog=23,hlog=22,slog=6,mml=3,tlen=48,strat=6
SEE ALSO
--------
`zstdgrep`(1), `zstdless`(1), `gzip`(1), `xz`(1)
The <zstandard> format is specified in Y. Collet, "Zstandard Compression and the 'application/zstd' Media Type", https://www.ietf.org/rfc/rfc8878.txt, Internet RFC 8878 (February 2021).
BUGS
----
programs/zstdcli.c
+185 -142
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) Yann Collet, Facebook, Inc.
* Copyright (c) Meta Platforms, Inc. and affiliates.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
@@ -27,8 +27,8 @@
/*-************************************
* Dependencies
**************************************/
#include "platform.h" /* IS_CONSOLE, PLATFORM_POSIX_VERSION */
#include "util.h" /* UTIL_HAS_CREATEFILELIST, UTIL_createFileList */
#include "platform.h" /* PLATFORM_POSIX_VERSION */
#include "util.h" /* UTIL_HAS_CREATEFILELIST, UTIL_createFileList, UTIL_isConsole */
#include <stdlib.h> /* getenv */
#include <string.h> /* strcmp, strlen */
#include <stdio.h> /* fprintf(), stdin, stdout, stderr */
@@ -52,12 +52,12 @@
/*-************************************
* Constants
**************************************/
#define COMPRESSOR_NAME "zstd command line interface"
#define COMPRESSOR_NAME "Zstandard CLI"
#ifndef ZSTD_VERSION
# define ZSTD_VERSION "v" ZSTD_VERSION_STRING
#endif
#define AUTHOR "Yann Collet"
#define WELCOME_MESSAGE "*** %s %i-bits %s, by %s ***\n", COMPRESSOR_NAME, (int)(sizeof(size_t)*8), ZSTD_VERSION, AUTHOR
#define WELCOME_MESSAGE "*** %s (%i-bit) %s, by %s ***\n", COMPRESSOR_NAME, (int)(sizeof(size_t)*8), ZSTD_VERSION, AUTHOR
#define ZSTD_ZSTDMT "zstdmt"
#define ZSTD_UNZSTD "unzstd"
@@ -143,160 +143,174 @@ static int exeNameMatch(const char* exeName, const char* test)
*/
static void usage(FILE* f, const char* programName)
{
DISPLAY_F(f, "Usage: %s [OPTION]... [FILE]... [-o file]\n", programName);
DISPLAY_F(f, "Compress or uncompress FILEs (with no FILE or when FILE is `-`, read from standard input).\n\n");
DISPLAY_F(f, " -o file result stored into `file` (only 1 output file)\n");
#ifndef ZSTD_NOCOMPRESS
DISPLAY_F(f, " -1 .. -%d compression level (faster .. better; default: %d)\n", ZSTDCLI_CLEVEL_MAX, ZSTDCLI_CLEVEL_DEFAULT);
#endif
#ifndef ZSTD_NODECOMPRESS
DISPLAY_F(f, " -d, --decompress decompression\n");
#endif
DISPLAY_F(f, " -f, --force disable input and output checks. Allows overwriting existing files,\n");
DISPLAY_F(f, " input from console, output to stdout, operating on links,\n");
DISPLAY_F(f, " block devices, etc. During decompression and when the output\n");
DISPLAY_F(f, " destination is stdout, pass-through unrecognized formats as-is.\n");
DISPLAY_F(f, " --rm remove source file(s) after successful de/compression\n");
DISPLAY_F(f, " -k, --keep preserve source file(s) (default) \n");
DISPLAY_F(f, "Compress or decompress the INPUT file(s); reads from STDIN if INPUT is `-` or not provided.\n\n");
DISPLAY_F(f, "Usage: %s [OPTIONS...] [INPUT... | -] [-o OUTPUT]\n\n", programName);
DISPLAY_F(f, "Options:\n");
DISPLAY_F(f, " -o OUTPUT Write output to a single file, OUTPUT.\n");
DISPLAY_F(f, " -k, --keep Preserve INPUT file(s). [Default] \n");
DISPLAY_F(f, " --rm Remove INPUT file(s) after successful (de)compression.\n");
#ifdef ZSTD_GZCOMPRESS
if (exeNameMatch(programName, ZSTD_GZ)) { /* behave like gzip */
DISPLAY_F(f, " -n, --no-name do not store original filename when compressing\n");
DISPLAY_F(f, " -n, --no-name Do not store original filename when compressing.\n\n");
}
#endif
DISPLAY_F(f, " -D DICT use DICT as Dictionary for compression or decompression\n");
DISPLAY_F(f, " -h display usage and exit\n");
DISPLAY_F(f, " -H,--help display long help and exit\n");
DISPLAY_F(f, "\n");
#ifndef ZSTD_NOCOMPRESS
DISPLAY_F(f, " -# Desired compression level, where `#` is a number between 1 and %d;\n", ZSTDCLI_CLEVEL_MAX);
DISPLAY_F(f, " lower numbers provide faster compression, higher numbers yield\n");
DISPLAY_F(f, " better compression ratios. [Default: %d]\n\n", ZSTDCLI_CLEVEL_DEFAULT);
#endif
#ifndef ZSTD_NODECOMPRESS
DISPLAY_F(f, " -d, --decompress Perform decompression.\n");
#endif
DISPLAY_F(f, " -D DICT Use DICT as the dictionary for compression or decompression.\n\n");
DISPLAY_F(f, " -f, --force Disable input and output checks. Allows overwriting existing files,\n");
DISPLAY_F(f, " receiving input from the console, printing ouput to STDOUT, and\n");
DISPLAY_F(f, " operating on links, block devices, etc. Unrecognized formats will be\n");
DISPLAY_F(f, " passed-through through as-is.\n\n");
DISPLAY_F(f, " -h Display short usage and exit.\n");
DISPLAY_F(f, " -H, --help Display full help and exit.\n");
DISPLAY_F(f, " -V, --version Display the program version and exit.\n");
DISPLAY_F(f, "\n");
}
static void usage_advanced(const char* programName)
{
DISPLAYOUT(WELCOME_MESSAGE);
DISPLAYOUT("\n");
usage(stdout, programName);
DISPLAYOUT("Advanced options :\n");
DISPLAYOUT(" -V, --version display Version number and exit\n");
DISPLAYOUT("Advanced options:\n");
DISPLAYOUT(" -c, --stdout Write to STDOUT (even if it is a console) and keep the INPUT file(s).\n\n");
DISPLAYOUT(" -c, --stdout write to standard output (even if it is the console), keep original file\n");
DISPLAYOUT(" -v, --verbose verbose mode; specify multiple times to increase verbosity\n");
DISPLAYOUT(" -q, --quiet suppress warnings; specify twice to suppress errors too\n");
DISPLAYOUT(" --[no-]progress forcibly display, or never display the progress counter\n");
DISPLAYOUT(" note: any (de)compressed output to terminal will mix with progress counter text\n");
DISPLAYOUT(" -v, --verbose Enable verbose output; pass multiple times to increase verbosity.\n");
DISPLAYOUT(" -q, --quiet Suppress warnings; pass twice to suppress errors.\n");
#ifndef ZSTD_NOTRACE
DISPLAYOUT(" --trace LOG Log tracing information to LOG.\n");
#endif
DISPLAYOUT("\n");
DISPLAYOUT(" --[no-]progress Forcibly show/hide the progress counter. NOTE: Any (de)compressed\n");
DISPLAYOUT(" output to terminal will mix with progress counter text.\n\n");
#ifdef UTIL_HAS_CREATEFILELIST
DISPLAYOUT(" -r operate recursively on directories\n");
DISPLAYOUT(" --filelist FILE read list of files to operate upon from FILE\n");
DISPLAYOUT(" --output-dir-flat DIR : processed files are stored into DIR\n");
DISPLAYOUT(" -r Operate recursively on directories.\n");
DISPLAYOUT(" --filelist LIST Read a list of files to operate on from LIST.\n");
DISPLAYOUT(" --output-dir-flat DIR Store processed files in DIR.\n");
#endif
#ifdef UTIL_HAS_MIRRORFILELIST
DISPLAYOUT(" --output-dir-mirror DIR : processed files are stored into DIR respecting original directory structure\n");
DISPLAYOUT(" --output-dir-mirror DIR Store processed files in DIR, respecting original directory structure.\n");
#endif
if (AIO_supported())
DISPLAYOUT(" --[no-]asyncio use asynchronous IO (default: enabled)\n");
DISPLAYOUT(" --[no-]asyncio Use asynchronous IO. [Default: Enabled]\n");
DISPLAYOUT("\n");
#ifndef ZSTD_NOCOMPRESS
DISPLAYOUT(" --[no-]check during compression, add XXH64 integrity checksum to frame (default: enabled)\n");
DISPLAYOUT(" --[no-]check Add XXH64 integrity checksums during compression. [Default: Add, Validate]\n");
#ifndef ZSTD_NODECOMPRESS
DISPLAYOUT(" if specified with -d, decompressor will ignore/validate checksums in compressed frame (default: validate)\n");
DISPLAYOUT(" If `-d` is present, ignore/validate checksums during decompression.\n");
#endif
#else
#ifdef ZSTD_NOCOMPRESS
DISPLAYOUT(" --[no-]check during decompression, ignore/validate checksums in compressed frame (default: validate)");
DISPLAYOUT(" --[no-]check Ignore/validate checksums during decompression. [Default: Validate]");
#endif
DISPLAYOUT("\n");
#endif /* ZSTD_NOCOMPRESS */
#ifndef ZSTD_NOTRACE
DISPLAYOUT(" --trace FILE log tracing information to FILE\n");
#endif
DISPLAYOUT(" -- all arguments after \"--\" are treated as files\n");
DISPLAYOUT("\n");
DISPLAYOUT(" -- Treat remaining arguments after `--` as files.\n");
#ifndef ZSTD_NOCOMPRESS
DISPLAYOUT("\n");
DISPLAYOUT("Advanced compression options :\n");
DISPLAYOUT(" --ultra enable levels beyond %i, up to %i (requires more memory)\n", ZSTDCLI_CLEVEL_MAX, ZSTD_maxCLevel());
DISPLAYOUT(" --fast[=#] switch to very fast compression levels (default: %u)\n", 1);
DISPLAYOUT("Advanced compression options:\n");
DISPLAYOUT(" --ultra Enable levels beyond %i, up to %i; requires more memory.\n", ZSTDCLI_CLEVEL_MAX, ZSTD_maxCLevel());
DISPLAYOUT(" --fast[=#] Use to very fast compression levels. [Default: %u]\n", 1);
#ifdef ZSTD_GZCOMPRESS
if (exeNameMatch(programName, ZSTD_GZ)) { /* behave like gzip */
DISPLAYOUT(" --best compatibility alias for -9 \n");
DISPLAYOUT(" --no-name do not store original filename when compressing\n");
DISPLAYOUT(" --best Compatibility alias for `-9`.\n");
}
#endif
DISPLAYOUT(" --long[=#] enable long distance matching with given window log (default: %u)\n", g_defaultMaxWindowLog);
DISPLAYOUT(" --patch-from=FILE : specify the file to be used as a reference point for zstd's diff engine. \n");
DISPLAYOUT(" --adapt dynamically adapt compression level to I/O conditions\n");
DISPLAYOUT(" --adapt Dynamically adapt compression level to I/O conditions.\n");
DISPLAYOUT(" --long[=#] Enable long distance matching with window log #. [Default: %u]\n", g_defaultMaxWindowLog);
DISPLAYOUT(" --patch-from=REF Use REF as the reference point for Zstandard's diff engine. \n\n");
# ifdef ZSTD_MULTITHREAD
DISPLAYOUT(" -T# spawn # compression threads (default: 1, 0==# cores) \n");
DISPLAYOUT(" -B# select size of each job (default: 0==automatic) \n");
DISPLAYOUT(" --single-thread use a single thread for both I/O and compression (result slightly different than -T1) \n");
DISPLAYOUT(" --auto-threads={physical,logical} : use either physical cores or logical cores as default when specifying -T0 (default: physical)\n");
DISPLAYOUT(" --rsyncable compress using a rsync-friendly method (-B sets block size) \n");
DISPLAYOUT(" -T# Spawn # compression threads. [Default: 1; pass 0 for core count.]\n");
DISPLAYOUT(" --single-thread Share a single thread for I/O and compression (slightly different than `-T1`).\n");
DISPLAYOUT(" --auto-threads={physical|logical}\n");
DISPLAYOUT(" Use physical/logical cores when using `-T0`. [Default: Physical]\n\n");
DISPLAYOUT(" -B# Set job size to #. [Default: 0 (automatic)]\n");
DISPLAYOUT(" --rsyncable Compress using a rsync-friendly method (`-B` sets block size). \n");
DISPLAYOUT("\n");
# endif
DISPLAYOUT(" --exclude-compressed : only compress files that are not already compressed \n");
DISPLAYOUT(" --stream-size=# specify size of streaming input from `stdin` \n");
DISPLAYOUT(" --size-hint=# optimize compression parameters for streaming input of approximately this size \n");
DISPLAYOUT(" --target-compressed-block-size=# : generate compressed block of approximately targeted size \n");
DISPLAYOUT(" --no-dictID don't write dictID into header (dictionary compression only)\n");
DISPLAYOUT(" --[no-]compress-literals : force (un)compressed literals\n");
DISPLAYOUT(" --[no-]row-match-finder : force enable/disable usage of fast row-based matchfinder for greedy, lazy, and lazy2 strategies\n");
DISPLAYOUT(" --exclude-compressed Only compress files that are not already compressed.\n\n");
DISPLAYOUT(" --format=zstd compress files to the .zst format (default)\n");
DISPLAYOUT(" --stream-size=# Specify size of streaming input from STDIN.\n");
DISPLAYOUT(" --size-hint=# Optimize compression parameters for streaming input of approximately size #.\n");
DISPLAYOUT(" --target-compressed-block-size=#\n");
DISPLAYOUT(" Generate compressed blocks of approximately # size.\n\n");
DISPLAYOUT(" --no-dictID Don't write `dictID` into the header (dictionary compression only).\n");
DISPLAYOUT(" --[no-]compress-literals Force (un)compressed literals.\n");
DISPLAYOUT(" --[no-]row-match-finder Explicitly enable/disable the fast, row-based matchfinder for\n");
DISPLAYOUT(" the 'greedy', 'lazy', and 'lazy2' strategies.\n");
DISPLAYOUT("\n");
DISPLAYOUT(" --format=zstd Compress files to the `.zst` format. [Default]\n");
#ifdef ZSTD_GZCOMPRESS
DISPLAYOUT(" --format=gzip compress files to the .gz format\n");
DISPLAYOUT(" --format=gzip Compress files to the `.gz` format.\n");
#endif
#ifdef ZSTD_LZMACOMPRESS
DISPLAYOUT(" --format=xz compress files to the .xz format\n");
DISPLAYOUT(" --format=lzma compress files to the .lzma format\n");
DISPLAYOUT(" --format=xz Compress files to the `.xz` format.\n");
DISPLAYOUT(" --format=lzma Compress files to the `.lzma` format.\n");
#endif
#ifdef ZSTD_LZ4COMPRESS
DISPLAYOUT( " --format=lz4 compress files to the .lz4 format\n");
DISPLAYOUT( " --format=lz4 Compress files to the `.lz4` format.\n");
#endif
#endif /* !ZSTD_NOCOMPRESS */
#ifndef ZSTD_NODECOMPRESS
DISPLAYOUT("\n");
DISPLAYOUT("Advanced decompression options :\n");
DISPLAYOUT(" -l print information about zstd compressed files\n");
DISPLAYOUT(" --test test compressed file integrity\n");
DISPLAYOUT(" -M# Set a memory usage limit for decompression\n");
DISPLAYOUT("Advanced decompression options:\n");
DISPLAYOUT(" -l Print information about Zstandard-compressed files.\n");
DISPLAYOUT(" --test Test compressed file integrity.\n");
DISPLAYOUT(" -M# Set the memory usage limit to # megabytes.\n");
# if ZSTD_SPARSE_DEFAULT
DISPLAYOUT(" --[no-]sparse sparse mode (default: enabled on file, disabled on stdout)\n");
DISPLAYOUT(" --[no-]sparse Enable sparse mode. [Default: Enabled for files, disabled for STDOUT.]\n");
# else
DISPLAYOUT(" --[no-]sparse sparse mode (default: disabled)\n");
DISPLAYOUT(" --[no-]sparse Enable sparse mode. [Default: Disabled]\n");
# endif
{
char const* passThroughDefault = "disabled";
char const* passThroughDefault = "Disabled";
if (exeNameMatch(programName, ZSTD_CAT) ||
exeNameMatch(programName, ZSTD_ZCAT) ||
exeNameMatch(programName, ZSTD_GZCAT)) {
passThroughDefault = "enabled";
passThroughDefault = "Enabled";
}
DISPLAYOUT(" --[no-]pass-through : passes through uncompressed files as-is (default: %s)\n", passThroughDefault);
DISPLAYOUT(" --[no-]pass-through Pass through uncompressed files as-is. [Default: %s]\n", passThroughDefault);
}
#endif /* ZSTD_NODECOMPRESS */
#ifndef ZSTD_NODICT
DISPLAYOUT("\n");
DISPLAYOUT("Dictionary builder :\n");
DISPLAYOUT(" --train ## create a dictionary from a training set of files\n");
DISPLAYOUT(" --train-cover[=k=#,d=#,steps=#,split=#,shrink[=#]] : use the cover algorithm with optional args\n");
DISPLAYOUT(" --train-fastcover[=k=#,d=#,f=#,steps=#,split=#,accel=#,shrink[=#]] : use the fast cover algorithm with optional args\n");
DISPLAYOUT(" --train-legacy[=s=#] : use the legacy algorithm with selectivity (default: %u)\n", g_defaultSelectivityLevel);
DISPLAYOUT(" -o DICT DICT is dictionary name (default: %s)\n", g_defaultDictName);
DISPLAYOUT(" --maxdict=# limit dictionary to specified size (default: %u)\n", g_defaultMaxDictSize);
DISPLAYOUT(" --dictID=# force dictionary ID to specified value (default: random)\n");
DISPLAYOUT("Dictionary builder:\n");
DISPLAYOUT(" --train Create a dictionary from a training set of files.\n\n");
DISPLAYOUT(" --train-cover[=k=#,d=#,steps=#,split=#,shrink[=#]]\n");
DISPLAYOUT(" Use the cover algorithm (with optional arguments).\n");
DISPLAYOUT(" --train-fastcover[=k=#,d=#,f=#,steps=#,split=#,accel=#,shrink[=#]]\n");
DISPLAYOUT(" Use the fast cover algorithm (with optional arguments).\n\n");
DISPLAYOUT(" --train-legacy[=s=#] Use the legacy algorithm with selectivity #. [Default: %u]\n", g_defaultSelectivityLevel);
DISPLAYOUT(" -o NAME Use NAME as dictionary name. [Default: %s]\n", g_defaultDictName);
DISPLAYOUT(" --maxdict=# Limit dictionary to specified size #. [Default: %u]\n", g_defaultMaxDictSize);
DISPLAYOUT(" --dictID=# Force dictionary ID to #. [Default: Random]\n");
#endif
#ifndef ZSTD_NOBENCH
DISPLAYOUT("\n");
DISPLAYOUT("Benchmark options : \n");
DISPLAYOUT(" -b# benchmark file(s), using # compression level (default: %d)\n", ZSTDCLI_CLEVEL_DEFAULT);
DISPLAYOUT(" -e# test all compression levels successively from -b# to -e# (default: 1)\n");
DISPLAYOUT(" -i# minimum evaluation time in seconds (default: 3s)\n");
DISPLAYOUT(" -B# cut file into independent chunks of size # (default: no chunking)\n");
DISPLAYOUT(" -S output one benchmark result per input file (default: consolidated result)\n");
DISPLAYOUT(" --priority=rt set process priority to real-time\n");
DISPLAYOUT("Benchmark options:\n");
DISPLAYOUT(" -b# Perform benchmarking with compression level #. [Default: %d]\n", ZSTDCLI_CLEVEL_DEFAULT);
DISPLAYOUT(" -e# Test all compression levels up to #; starting level is `-b#`. [Default: 1]\n");
DISPLAYOUT(" -i# Set the minimum evaluation to time # seconds. [Default: 3]\n");
DISPLAYOUT(" -B# Cut file into independent chunks of size #. [Default: No chunking]\n");
DISPLAYOUT(" -S Output one benchmark result per input file. [Default: Consolidated result]\n");
DISPLAYOUT(" --priority=rt Set process priority to real-time.\n");
#endif
}
@@ -325,7 +339,7 @@ static const char* lastNameFromPath(const char* path)
static void errorOut(const char* msg)
{
DISPLAY("%s \n", msg); exit(1);
DISPLAYLEVEL(1, "%s \n", msg); exit(1);
}
/*! readU32FromCharChecked() :
@@ -772,13 +786,13 @@ static unsigned init_nbThreads(void) {
} else { \
argNb++; \
if (argNb >= argCount) { \
DISPLAY("error: missing command argument \n"); \
DISPLAYLEVEL(1, "error: missing command argument \n"); \
CLEAN_RETURN(1); \
} \
ptr = argv[argNb]; \
assert(ptr != NULL); \
if (ptr[0]=='-') { \
DISPLAY("error: command cannot be separated from its argument by another command \n"); \
DISPLAYLEVEL(1, "error: command cannot be separated from its argument by another command \n"); \
CLEAN_RETURN(1); \
} } }
@@ -824,7 +838,6 @@ int main(int argCount, const char* argv[])
ldmFlag = 0,
main_pause = 0,
adapt = 0,
useRowMatchFinder = 0,
adaptMin = MINCLEVEL,
adaptMax = MAXCLEVEL,
rsyncable = 0,
@@ -836,7 +849,10 @@ int main(int argCount, const char* argv[])
defaultLogicalCores = 0,
showDefaultCParams = 0,
ultra=0,
contentSize=1;
contentSize=1,
removeSrcFile=0;
ZSTD_paramSwitch_e useRowMatchFinder = ZSTD_ps_auto;
FIO_compressionType_t cType = FIO_zstdCompression;
unsigned nbWorkers = 0;
double compressibility = 0.5;
unsigned bench_nbSeconds = 3; /* would be better if this value was synchronized from bench */
@@ -844,6 +860,7 @@ int main(int argCount, const char* argv[])
FIO_prefs_t* const prefs = FIO_createPreferences();
FIO_ctx_t* const fCtx = FIO_createContext();
FIO_progressSetting_e progress = FIO_ps_auto;
zstd_operation_mode operation = zom_compress;
ZSTD_compressionParameters compressionParams;
int cLevel = init_cLevel();
@@ -883,7 +900,7 @@ int main(int argCount, const char* argv[])
(void)recursive; (void)cLevelLast; /* not used when ZSTD_NOBENCH set */
(void)memLimit;
assert(argCount >= 1);
if ((filenames==NULL) || (file_of_names==NULL)) { DISPLAY("zstd: allocation error \n"); exit(1); }
if ((filenames==NULL) || (file_of_names==NULL)) { DISPLAYLEVEL(1, "zstd: allocation error \n"); exit(1); }
programName = lastNameFromPath(programName);
#ifdef ZSTD_MULTITHREAD
nbWorkers = init_nbThreads();
@@ -895,17 +912,17 @@ int main(int argCount, const char* argv[])
if (exeNameMatch(programName, ZSTD_CAT)) { operation=zom_decompress; FIO_overwriteMode(prefs); forceStdout=1; followLinks=1; FIO_setPassThroughFlag(prefs, 1); outFileName=stdoutmark; g_displayLevel=1; } /* supports multiple formats */
if (exeNameMatch(programName, ZSTD_ZCAT)) { operation=zom_decompress; FIO_overwriteMode(prefs); forceStdout=1; followLinks=1; FIO_setPassThroughFlag(prefs, 1); outFileName=stdoutmark; g_displayLevel=1; } /* behave like zcat, also supports multiple formats */
if (exeNameMatch(programName, ZSTD_GZ)) { /* behave like gzip */
suffix = GZ_EXTENSION; FIO_setCompressionType(prefs, FIO_gzipCompression); FIO_setRemoveSrcFile(prefs, 1);
suffix = GZ_EXTENSION; cType = FIO_gzipCompression; removeSrcFile=1;
dictCLevel = cLevel = 6; /* gzip default is -6 */
}
if (exeNameMatch(programName, ZSTD_GUNZIP)) { operation=zom_decompress; FIO_setRemoveSrcFile(prefs, 1); } /* behave like gunzip, also supports multiple formats */
if (exeNameMatch(programName, ZSTD_GUNZIP)) { operation=zom_decompress; removeSrcFile=1; } /* behave like gunzip, also supports multiple formats */
if (exeNameMatch(programName, ZSTD_GZCAT)) { operation=zom_decompress; FIO_overwriteMode(prefs); forceStdout=1; followLinks=1; FIO_setPassThroughFlag(prefs, 1); outFileName=stdoutmark; g_displayLevel=1; } /* behave like gzcat, also supports multiple formats */
if (exeNameMatch(programName, ZSTD_LZMA)) { suffix = LZMA_EXTENSION; FIO_setCompressionType(prefs, FIO_lzmaCompression); FIO_setRemoveSrcFile(prefs, 1); } /* behave like lzma */
if (exeNameMatch(programName, ZSTD_UNLZMA)) { operation=zom_decompress; FIO_setCompressionType(prefs, FIO_lzmaCompression); FIO_setRemoveSrcFile(prefs, 1); } /* behave like unlzma, also supports multiple formats */
if (exeNameMatch(programName, ZSTD_XZ)) { suffix = XZ_EXTENSION; FIO_setCompressionType(prefs, FIO_xzCompression); FIO_setRemoveSrcFile(prefs, 1); } /* behave like xz */
if (exeNameMatch(programName, ZSTD_UNXZ)) { operation=zom_decompress; FIO_setCompressionType(prefs, FIO_xzCompression); FIO_setRemoveSrcFile(prefs, 1); } /* behave like unxz, also supports multiple formats */
if (exeNameMatch(programName, ZSTD_LZ4)) { suffix = LZ4_EXTENSION; FIO_setCompressionType(prefs, FIO_lz4Compression); } /* behave like lz4 */
if (exeNameMatch(programName, ZSTD_UNLZ4)) { operation=zom_decompress; FIO_setCompressionType(prefs, FIO_lz4Compression); } /* behave like unlz4, also supports multiple formats */
if (exeNameMatch(programName, ZSTD_LZMA)) { suffix = LZMA_EXTENSION; cType = FIO_lzmaCompression; removeSrcFile=1; } /* behave like lzma */
if (exeNameMatch(programName, ZSTD_UNLZMA)) { operation=zom_decompress; cType = FIO_lzmaCompression; removeSrcFile=1; } /* behave like unlzma, also supports multiple formats */
if (exeNameMatch(programName, ZSTD_XZ)) { suffix = XZ_EXTENSION; cType = FIO_xzCompression; removeSrcFile=1; } /* behave like xz */
if (exeNameMatch(programName, ZSTD_UNXZ)) { operation=zom_decompress; cType = FIO_xzCompression; removeSrcFile=1; } /* behave like unxz, also supports multiple formats */
if (exeNameMatch(programName, ZSTD_LZ4)) { suffix = LZ4_EXTENSION; cType = FIO_lz4Compression; } /* behave like lz4 */
if (exeNameMatch(programName, ZSTD_UNLZ4)) { operation=zom_decompress; cType = FIO_lz4Compression; } /* behave like unlz4, also supports multiple formats */
memset(&compressionParams, 0, sizeof(compressionParams));
/* init crash handler */
@@ -942,7 +959,7 @@ int main(int argCount, const char* argv[])
if (!strcmp(argument, "--help")) { usage_advanced(programName); CLEAN_RETURN(0); }
if (!strcmp(argument, "--verbose")) { g_displayLevel++; continue; }
if (!strcmp(argument, "--quiet")) { g_displayLevel--; continue; }
if (!strcmp(argument, "--stdout")) { forceStdout=1; outFileName=stdoutmark; FIO_setRemoveSrcFile(prefs, 0); g_displayLevel-=(g_displayLevel==2); continue; }
if (!strcmp(argument, "--stdout")) { forceStdout=1; outFileName=stdoutmark; removeSrcFile=0; continue; }
if (!strcmp(argument, "--ultra")) { ultra=1; continue; }
if (!strcmp(argument, "--check")) { FIO_setChecksumFlag(prefs, 2); continue; }
if (!strcmp(argument, "--no-check")) { FIO_setChecksumFlag(prefs, 0); continue; }
@@ -955,38 +972,42 @@ int main(int argCount, const char* argv[])
if (!strcmp(argument, "--no-asyncio")) { FIO_setAsyncIOFlag(prefs, 0); continue;}
if (!strcmp(argument, "--train")) { operation=zom_train; if (outFileName==NULL) outFileName=g_defaultDictName; continue; }
if (!strcmp(argument, "--no-dictID")) { FIO_setDictIDFlag(prefs, 0); continue; }
if (!strcmp(argument, "--keep")) { FIO_setRemoveSrcFile(prefs, 0); continue; }
if (!strcmp(argument, "--rm")) { FIO_setRemoveSrcFile(prefs, 1); continue; }
if (!strcmp(argument, "--keep")) { removeSrcFile=0; continue; }
if (!strcmp(argument, "--rm")) { removeSrcFile=1; continue; }
if (!strcmp(argument, "--priority=rt")) { setRealTimePrio = 1; continue; }
if (!strcmp(argument, "--show-default-cparams")) { showDefaultCParams = 1; continue; }
if (!strcmp(argument, "--content-size")) { contentSize = 1; continue; }
if (!strcmp(argument, "--no-content-size")) { contentSize = 0; continue; }
if (!strcmp(argument, "--adapt")) { adapt = 1; continue; }
if (!strcmp(argument, "--no-row-match-finder")) { useRowMatchFinder = 1; continue; }
if (!strcmp(argument, "--row-match-finder")) { useRowMatchFinder = 2; continue; }
if (!strcmp(argument, "--no-row-match-finder")) { useRowMatchFinder = ZSTD_ps_disable; continue; }
if (!strcmp(argument, "--row-match-finder")) { useRowMatchFinder = ZSTD_ps_enable; continue; }
if (longCommandWArg(&argument, "--adapt=")) { adapt = 1; if (!parseAdaptParameters(argument, &adaptMin, &adaptMax)) { badusage(programName); CLEAN_RETURN(1); } continue; }
if (!strcmp(argument, "--single-thread")) { nbWorkers = 0; singleThread = 1; continue; }
if (!strcmp(argument, "--format=zstd")) { suffix = ZSTD_EXTENSION; FIO_setCompressionType(prefs, FIO_zstdCompression); continue; }
if (!strcmp(argument, "--format=zstd")) { suffix = ZSTD_EXTENSION; cType = FIO_zstdCompression; continue; }
#ifdef ZSTD_GZCOMPRESS
if (!strcmp(argument, "--format=gzip")) { suffix = GZ_EXTENSION; FIO_setCompressionType(prefs, FIO_gzipCompression); continue; }
if (!strcmp(argument, "--format=gzip")) { suffix = GZ_EXTENSION; cType = FIO_gzipCompression; continue; }
if (exeNameMatch(programName, ZSTD_GZ)) { /* behave like gzip */
if (!strcmp(argument, "--best")) { dictCLevel = cLevel = 9; continue; }
if (!strcmp(argument, "--no-name")) { /* ignore for now */; continue; }
}
#endif
#ifdef ZSTD_LZMACOMPRESS
if (!strcmp(argument, "--format=lzma")) { suffix = LZMA_EXTENSION; FIO_setCompressionType(prefs, FIO_lzmaCompression); continue; }
if (!strcmp(argument, "--format=xz")) { suffix = XZ_EXTENSION; FIO_setCompressionType(prefs, FIO_xzCompression); continue; }
if (!strcmp(argument, "--format=lzma")) { suffix = LZMA_EXTENSION; cType = FIO_lzmaCompression; continue; }
if (!strcmp(argument, "--format=xz")) { suffix = XZ_EXTENSION; cType = FIO_xzCompression; continue; }
#endif
#ifdef ZSTD_LZ4COMPRESS
if (!strcmp(argument, "--format=lz4")) { suffix = LZ4_EXTENSION; FIO_setCompressionType(prefs, FIO_lz4Compression); continue; }
if (!strcmp(argument, "--format=lz4")) { suffix = LZ4_EXTENSION; cType = FIO_lz4Compression; continue; }
#endif
if (!strcmp(argument, "--rsyncable")) { rsyncable = 1; continue; }
if (!strcmp(argument, "--compress-literals")) { literalCompressionMode = ZSTD_ps_enable; continue; }
if (!strcmp(argument, "--no-compress-literals")) { literalCompressionMode = ZSTD_ps_disable; continue; }
if (!strcmp(argument, "--no-progress")) { FIO_setProgressSetting(FIO_ps_never); continue; }
if (!strcmp(argument, "--progress")) { FIO_setProgressSetting(FIO_ps_always); continue; }
if (!strcmp(argument, "--no-progress")) { progress = FIO_ps_never; continue; }
if (!strcmp(argument, "--progress")) { progress = FIO_ps_always; continue; }
if (!strcmp(argument, "--exclude-compressed")) { FIO_setExcludeCompressedFile(prefs, 1); continue; }
if (!strcmp(argument, "--fake-stdin-is-console")) { UTIL_fakeStdinIsConsole(); continue; }
if (!strcmp(argument, "--fake-stdout-is-console")) { UTIL_fakeStdoutIsConsole(); continue; }
if (!strcmp(argument, "--fake-stderr-is-console")) { UTIL_fakeStderrIsConsole(); continue; }
if (!strcmp(argument, "--trace-file-stat")) { UTIL_traceFileStat(); continue; }
/* long commands with arguments */
#ifndef ZSTD_NODICT
@@ -1031,14 +1052,14 @@ int main(int argCount, const char* argv[])
if (longCommandWArg(&argument, "--block-size")) { NEXT_TSIZE(blockSize); continue; }
if (longCommandWArg(&argument, "--maxdict")) { NEXT_UINT32(maxDictSize); continue; }
if (longCommandWArg(&argument, "--dictID")) { NEXT_UINT32(dictID); continue; }
if (longCommandWArg(&argument, "--zstd=")) { if (!parseCompressionParameters(argument, &compressionParams)) { badusage(programName); CLEAN_RETURN(1); } continue; }
if (longCommandWArg(&argument, "--zstd=")) { if (!parseCompressionParameters(argument, &compressionParams)) { badusage(programName); CLEAN_RETURN(1); } ; cType = FIO_zstdCompression; continue; }
if (longCommandWArg(&argument, "--stream-size")) { NEXT_TSIZE(streamSrcSize); continue; }
if (longCommandWArg(&argument, "--target-compressed-block-size")) { NEXT_TSIZE(targetCBlockSize); continue; }
if (longCommandWArg(&argument, "--size-hint")) { NEXT_TSIZE(srcSizeHint); continue; }
if (longCommandWArg(&argument, "--output-dir-flat")) {
NEXT_FIELD(outDirName);
if (strlen(outDirName) == 0) {
DISPLAY("error: output dir cannot be empty string (did you mean to pass '.' instead?)\n");
DISPLAYLEVEL(1, "error: output dir cannot be empty string (did you mean to pass '.' instead?)\n");
CLEAN_RETURN(1);
}
continue;
@@ -1054,7 +1075,7 @@ int main(int argCount, const char* argv[])
if (longCommandWArg(&argument, "--output-dir-mirror")) {
NEXT_FIELD(outMirroredDirName);
if (strlen(outMirroredDirName) == 0) {
DISPLAY("error: output dir cannot be empty string (did you mean to pass '.' instead?)\n");
DISPLAYLEVEL(1, "error: output dir cannot be empty string (did you mean to pass '.' instead?)\n");
CLEAN_RETURN(1);
}
continue;
@@ -1149,7 +1170,7 @@ int main(int argCount, const char* argv[])
operation=zom_decompress; argument++; break;
/* Force stdout, even if stdout==console */
case 'c': forceStdout=1; outFileName=stdoutmark; FIO_setRemoveSrcFile(prefs, 0); argument++; break;
case 'c': forceStdout=1; outFileName=stdoutmark; removeSrcFile=0; argument++; break;
/* do not store filename - gzip compatibility - nothing to do */
case 'n': argument++; break;
@@ -1167,7 +1188,7 @@ int main(int argCount, const char* argv[])
case 'q': g_displayLevel--; argument++; break;
/* keep source file (default) */
case 'k': FIO_setRemoveSrcFile(prefs, 0); argument++; break;
case 'k': removeSrcFile=0; argument++; break;
/* Checksum */
case 'C': FIO_setChecksumFlag(prefs, 2); argument++; break;
@@ -1330,7 +1351,7 @@ int main(int argCount, const char* argv[])
int const ret = FIO_listMultipleFiles((unsigned)filenames->tableSize, filenames->fileNames, g_displayLevel);
CLEAN_RETURN(ret);
#else
DISPLAY("file information is not supported \n");
DISPLAYLEVEL(1, "file information is not supported \n");
CLEAN_RETURN(1);
#endif
}
@@ -1338,6 +1359,10 @@ int main(int argCount, const char* argv[])
/* Check if benchmark is selected */
if (operation==zom_bench) {
#ifndef ZSTD_NOBENCH
if (cType != FIO_zstdCompression) {
DISPLAYLEVEL(1, "benchmark mode is only compatible with zstd format \n");
CLEAN_RETURN(1);
}
benchParams.blockSize = blockSize;
benchParams.nbWorkers = (int)nbWorkers;
benchParams.realTime = (unsigned)setRealTimePrio;
@@ -1345,7 +1370,7 @@ int main(int argCount, const char* argv[])
benchParams.ldmFlag = ldmFlag;
benchParams.ldmMinMatch = (int)g_ldmMinMatch;
benchParams.ldmHashLog = (int)g_ldmHashLog;
benchParams.useRowMatchFinder = useRowMatchFinder;
benchParams.useRowMatchFinder = (int)useRowMatchFinder;
if (g_ldmBucketSizeLog != LDM_PARAM_DEFAULT) {
benchParams.ldmBucketSizeLog = (int)g_ldmBucketSizeLog;
}
@@ -1366,15 +1391,18 @@ int main(int argCount, const char* argv[])
int c;
DISPLAYLEVEL(3, "Benchmarking %s \n", filenames->fileNames[i]);
for(c = cLevel; c <= cLevelLast; c++) {
BMK_benchFilesAdvanced(&filenames->fileNames[i], 1, dictFileName, c, &compressionParams, g_displayLevel, &benchParams);
BMK_benchOutcome_t const bo = BMK_benchFilesAdvanced(&filenames->fileNames[i], 1, dictFileName, c, &compressionParams, g_displayLevel, &benchParams);
if (!BMK_isSuccessful_benchOutcome(bo)) return 1;
} }
} else {
for(; cLevel <= cLevelLast; cLevel++) {
BMK_benchFilesAdvanced(filenames->fileNames, (unsigned)filenames->tableSize, dictFileName, cLevel, &compressionParams, g_displayLevel, &benchParams);
BMK_benchOutcome_t const bo = BMK_benchFilesAdvanced(filenames->fileNames, (unsigned)filenames->tableSize, dictFileName, cLevel, &compressionParams, g_displayLevel, &benchParams);
if (!BMK_isSuccessful_benchOutcome(bo)) return 1;
} }
} else {
for(; cLevel <= cLevelLast; cLevel++) {
BMK_syntheticTest(cLevel, compressibility, &compressionParams, g_displayLevel, &benchParams);
BMK_benchOutcome_t const bo = BMK_syntheticTest(cLevel, compressibility, &compressionParams, g_displayLevel, &benchParams);
if (!BMK_isSuccessful_benchOutcome(bo)) return 1;
} }
#else
@@ -1416,7 +1444,7 @@ int main(int argCount, const char* argv[])
}
#ifndef ZSTD_NODECOMPRESS
if (operation==zom_test) { FIO_setTestMode(prefs, 1); outFileName=nulmark; FIO_setRemoveSrcFile(prefs, 0); } /* test mode */
if (operation==zom_test) { FIO_setTestMode(prefs, 1); outFileName=nulmark; removeSrcFile=0; } /* test mode */
#endif
/* No input filename ==> use stdin and stdout */
@@ -1437,12 +1465,12 @@ int main(int argCount, const char* argv[])
/* Check if input/output defined as console; trigger an error in this case */
if (!forceStdin
&& (UTIL_searchFileNamesTable(filenames, stdinmark) != -1)
&& IS_CONSOLE(stdin) ) {
&& UTIL_isConsole(stdin) ) {
DISPLAYLEVEL(1, "stdin is a console, aborting\n");
CLEAN_RETURN(1);
}
if ( (!outFileName || !strcmp(outFileName, stdoutmark))
&& IS_CONSOLE(stdout)
&& UTIL_isConsole(stdout)
&& (UTIL_searchFileNamesTable(filenames, stdinmark) != -1)
&& !forceStdout
&& operation!=zom_decompress ) {
@@ -1461,25 +1489,35 @@ int main(int argCount, const char* argv[])
if (showDefaultCParams) {
if (operation == zom_decompress) {
DISPLAY("error : can't use --show-default-cparams in decompression mode \n");
DISPLAYLEVEL(1, "error : can't use --show-default-cparams in decompression mode \n");
CLEAN_RETURN(1);
}
}
if (dictFileName != NULL && patchFromDictFileName != NULL) {
DISPLAY("error : can't use -D and --patch-from=# at the same time \n");
DISPLAYLEVEL(1, "error : can't use -D and --patch-from=# at the same time \n");
CLEAN_RETURN(1);
}
if (patchFromDictFileName != NULL && filenames->tableSize > 1) {
DISPLAY("error : can't use --patch-from=# on multiple files \n");
DISPLAYLEVEL(1, "error : can't use --patch-from=# on multiple files \n");
CLEAN_RETURN(1);
}
/* No status message in pipe mode (stdin - stdout) */
/* No status message by default when output is stdout */
hasStdout = outFileName && !strcmp(outFileName,stdoutmark);
if (hasStdout && (g_displayLevel==2)) g_displayLevel=1;
if ((hasStdout || !IS_CONSOLE(stderr)) && (g_displayLevel==2)) g_displayLevel=1;
/* when stderr is not the console, do not pollute it with progress updates (unless requested) */
if (!UTIL_isConsole(stderr) && (progress!=FIO_ps_always)) progress=FIO_ps_never;
FIO_setProgressSetting(progress);
/* don't remove source files when output is stdout */;
if (hasStdout && removeSrcFile) {
DISPLAYLEVEL(3, "Note: src files are not removed when output is stdout \n");
removeSrcFile = 0;
}
FIO_setRemoveSrcFile(prefs, removeSrcFile);
/* IO Stream/File */
FIO_setHasStdoutOutput(fCtx, hasStdout);
@@ -1499,6 +1537,7 @@ int main(int argCount, const char* argv[])
FIO_setMemLimit(prefs, memLimit);
if (operation==zom_compress) {
#ifndef ZSTD_NOCOMPRESS
FIO_setCompressionType(prefs, cType);
FIO_setContentSize(prefs, contentSize);
FIO_setNbWorkers(prefs, (int)nbWorkers);
FIO_setBlockSize(prefs, (int)blockSize);
@@ -1509,7 +1548,7 @@ int main(int argCount, const char* argv[])
if (g_ldmBucketSizeLog != LDM_PARAM_DEFAULT) FIO_setLdmBucketSizeLog(prefs, (int)g_ldmBucketSizeLog);
if (g_ldmHashRateLog != LDM_PARAM_DEFAULT) FIO_setLdmHashRateLog(prefs, (int)g_ldmHashRateLog);
FIO_setAdaptiveMode(prefs, adapt);
FIO_setUseRowMatchFinder(prefs, useRowMatchFinder);
FIO_setUseRowMatchFinder(prefs, (int)useRowMatchFinder);
FIO_setAdaptMin(prefs, adaptMin);
FIO_setAdaptMax(prefs, adaptMax);
FIO_setRsyncable(prefs, rsyncable);
@@ -1543,8 +1582,12 @@ int main(int argCount, const char* argv[])
else
operationResult = FIO_compressMultipleFilenames(fCtx, prefs, filenames->fileNames, outMirroredDirName, outDirName, outFileName, suffix, dictFileName, cLevel, compressionParams);
#else
(void)contentSize; (void)suffix; (void)adapt; (void)rsyncable; (void)ultra; (void)cLevel; (void)ldmFlag; (void)literalCompressionMode; (void)targetCBlockSize; (void)streamSrcSize; (void)srcSizeHint; (void)ZSTD_strategyMap; (void)useRowMatchFinder; /* not used when ZSTD_NOCOMPRESS set */
DISPLAY("Compression not supported \n");
/* these variables are only used when compression mode is enabled */
(void)contentSize; (void)suffix; (void)adapt; (void)rsyncable;
(void)ultra; (void)cLevel; (void)ldmFlag; (void)literalCompressionMode;
(void)targetCBlockSize; (void)streamSrcSize; (void)srcSizeHint;
(void)ZSTD_strategyMap; (void)useRowMatchFinder; (void)cType;
DISPLAYLEVEL(1, "Compression not supported \n");
#endif
} else { /* decompression or test */
#ifndef ZSTD_NODECOMPRESS
@@ -1554,7 +1597,7 @@ int main(int argCount, const char* argv[])
operationResult = FIO_decompressMultipleFilenames(fCtx, prefs, filenames->fileNames, outMirroredDirName, outDirName, outFileName, dictFileName);
}
#else
DISPLAY("Decompression not supported \n");
DISPLAYLEVEL(1, "Decompression not supported \n");
#endif
}
programs/zstdcli_trace.c
+1 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) Facebook, Inc.
* Copyright (c) Meta Platforms, Inc. and affiliates.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
programs/zstdcli_trace.h
+1 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) Facebook, Inc.
* Copyright (c) Meta Platforms, Inc. and affiliates.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
programs/zstdgrep.1.md
+5 -5
View File
@@ -4,16 +4,16 @@ zstdgrep(1) -- print lines matching a pattern in zstandard-compressed files
SYNOPSIS
--------
`zstdgrep` [*grep-flags*] [--] _pattern_ [_files_ ...]
`zstdgrep` [<grep-flags>] [--] <pattern> [<files> ...]
DESCRIPTION
-----------
`zstdgrep` runs `grep (1)` on files, or `stdin` if no files argument is given, after decompressing them with `zstdcat (1)`.
`zstdgrep` runs `grep`(1) on files, or `stdin` if no files argument is given, after decompressing them with `zstdcat`(1).
The grep-flags and pattern arguments are passed on to `grep (1)`. If an `-e` flag is found in the `grep-flags`, `zstdgrep` will not look for a pattern argument.
The <grep-flags> and <pattern> arguments are passed on to `grep`(1). If an `-e` flag is found in the <grep-flags>, `zstdgrep` will not look for a <pattern> argument.
Note that modern `grep` alternatives such as `ripgrep` (`rg`) support `zstd`-compressed files out of the box,
Note that modern `grep` alternatives such as `ripgrep` (`rg`(1)) support `zstd`-compressed files out of the box,
and can prove better alternatives than `zstdgrep` notably for unsupported complex pattern searches.
Note though that such alternatives may also feature some minor command line differences.
@@ -23,7 +23,7 @@ In case of missing arguments or missing pattern, 1 will be returned, otherwise 0
SEE ALSO
--------
`zstd (1)`
`zstd`(1)
AUTHORS
-------
programs/zstdless.1.md
+3 -3
View File
@@ -4,13 +4,13 @@ zstdless(1) -- view zstandard-compressed files
SYNOPSIS
--------
`zstdless` [*flags*] [_file_ ...]
`zstdless` [<flags>] [<file> ...]
DESCRIPTION
-----------
`zstdless` runs `less (1)` on files or stdin, if no files argument is given, after decompressing them with `zstdcat (1)`.
`zstdless` runs `less`(1) on files or stdin, if no <file> argument is given, after decompressing them with `zstdcat`(1).
SEE ALSO
--------
`zstd (1)`
`zstd`(1)