CI failure fixes
This commit is contained in:
Danielle Rozenblit
+22
-14
@@ -129,10 +129,10 @@ SET_CACHE_DIRECTORY = \
|
||||
|
||||
|
||||
.PHONY: all
|
||||
all: zstd
|
||||
all: zstd zstd-compress zstd-decompress zstd-small
|
||||
|
||||
.PHONY: allVariants
|
||||
allVariants: zstd zstd-compress zstd-decompress zstd-small zstd-frugal zstd-nolegacy zstd-dictBuilder
|
||||
allVariants: all zstd-frugal zstd-nolegacy zstd-dictBuilder
|
||||
|
||||
.PHONY: zstd # must always be run
|
||||
zstd : CPPFLAGS += $(THREAD_CPP) $(ZLIBCPP) $(LZMACPP) $(LZ4CPP)
|
||||
@@ -180,11 +180,13 @@ zstd : $(BUILD_DIR)/zstd
|
||||
endif # BUILD_DIR
|
||||
|
||||
|
||||
CLEAN += zstd
|
||||
.PHONY: zstd-release
|
||||
zstd-release: DEBUGFLAGS := -DBACKTRACE_ENABLE=0
|
||||
zstd-release: DEBUGFLAGS_LD :=
|
||||
zstd-release: zstd
|
||||
|
||||
CLEAN += zstd32
|
||||
zstd32 : CPPFLAGS += $(THREAD_CPP)
|
||||
zstd32 : LDFLAGS += $(THREAD_LD)
|
||||
zstd32 : CPPFLAGS += -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT)
|
||||
@@ -194,7 +196,8 @@ endif
|
||||
zstd32 : $(ZSTDLIB_FULL_SRC) $(ZSTD_CLI_SRC)
|
||||
$(CC) -m32 $(FLAGS) $^ -o $@$(EXT)
|
||||
|
||||
## zstd-nolegacy: same scope as zstd, with just support of legacy formats removed
|
||||
## zstd-nolegacy: same scope as zstd, with removed support of legacy formats
|
||||
CLEAN += zstd-nolegacy
|
||||
zstd-nolegacy : LDFLAGS += $(THREAD_LD) $(ZLIBLD) $(LZMALD) $(LZ4LD) $(DEBUGFLAGS_LD)
|
||||
zstd-nolegacy : CPPFLAGS += -UZSTD_LEGACY_SUPPORT -DZSTD_LEGACY_SUPPORT=0
|
||||
zstd-nolegacy : $(ZSTDLIB_CORE_SRC) $(ZDICT_SRC) $(ZSTD_CLI_OBJ)
|
||||
@@ -229,33 +232,42 @@ zstd-dll : zstd
|
||||
## zstd-pgo: zstd executable optimized with PGO.
|
||||
.PHONY: zstd-pgo
|
||||
zstd-pgo :
|
||||
$(MAKE) clean
|
||||
$(MAKE) zstd MOREFLAGS=-fprofile-generate
|
||||
$(MAKE) clean HASH_DIR=$(HASH_DIR)
|
||||
$(MAKE) zstd HASH_DIR=$(HASH_DIR) MOREFLAGS=-fprofile-generate
|
||||
./zstd -b19i1 $(PROFILE_WITH)
|
||||
./zstd -b16i1 $(PROFILE_WITH)
|
||||
./zstd -b9i2 $(PROFILE_WITH)
|
||||
./zstd -b $(PROFILE_WITH)
|
||||
./zstd -b7i2 $(PROFILE_WITH)
|
||||
./zstd -b5 $(PROFILE_WITH)
|
||||
$(RM) zstd *.o
|
||||
ifndef BUILD_DIR
|
||||
$(RM) zstd obj/$(HASH_DIR)/zstd obj/$(HASH_DIR)/*.o
|
||||
else
|
||||
$(RM) zstd $(BUILD_DIR)/zstd $(BUILD_DIR)/*.o
|
||||
endif
|
||||
case $(CC) in *clang*) if ! [ -e default.profdata ]; then llvm-profdata merge -output=default.profdata default*.profraw; fi ;; esac
|
||||
$(MAKE) zstd MOREFLAGS=-fprofile-use
|
||||
$(MAKE) zstd HASH_DIR=$(HASH_DIR) MOREFLAGS=-fprofile-use
|
||||
|
||||
## zstd-small: minimal target, supporting only zstd compression and decompression. no bench. no legacy. no other format.
|
||||
CLEAN += zstd-small zstd-frugal
|
||||
zstd-small: CFLAGS = -Os -Wl,-s
|
||||
zstd-frugal zstd-small: $(ZSTDLIB_CORE_SRC) zstdcli.c util.c timefn.c fileio.c fileio_asyncio.c
|
||||
$(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_NOTRACE -UZSTD_LEGACY_SUPPORT -DZSTD_LEGACY_SUPPORT=0 $^ -o $@$(EXT)
|
||||
|
||||
CLEAN += zstd-decompress
|
||||
zstd-decompress: $(ZSTDLIB_COMMON_SRC) $(ZSTDLIB_DECOMPRESS_SRC) zstdcli.c util.c timefn.c fileio.c fileio_asyncio.c
|
||||
$(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_NOCOMPRESS -DZSTD_NOTRACE -UZSTD_LEGACY_SUPPORT -DZSTD_LEGACY_SUPPORT=0 $^ -o $@$(EXT)
|
||||
|
||||
CLEAN += zstd-compress
|
||||
zstd-compress: $(ZSTDLIB_COMMON_SRC) $(ZSTDLIB_COMPRESS_SRC) zstdcli.c util.c timefn.c fileio.c fileio_asyncio.c
|
||||
$(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_NODECOMPRESS -DZSTD_NOTRACE -UZSTD_LEGACY_SUPPORT -DZSTD_LEGACY_SUPPORT=0 $^ -o $@$(EXT)
|
||||
|
||||
## zstd-dictBuilder: executable supporting dictionary creation and compression (only)
|
||||
CLEAN += zstd-dictBuilder
|
||||
zstd-dictBuilder: $(ZSTDLIB_COMMON_SRC) $(ZSTDLIB_COMPRESS_SRC) $(ZDICT_SRC) zstdcli.c util.c timefn.c fileio.c fileio_asyncio.c dibio.c
|
||||
$(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODECOMPRESS -DZSTD_NOTRACE $^ -o $@$(EXT)
|
||||
|
||||
CLEAN += zstdmt
|
||||
zstdmt: zstd
|
||||
ln -sf zstd zstdmt
|
||||
|
||||
@@ -273,12 +285,8 @@ endif
|
||||
|
||||
.PHONY: clean
|
||||
clean:
|
||||
$(RM) core *.o tmp* result* *.gcda dictionary *.zst \
|
||||
zstd$(EXT) zstd32$(EXT) zstd-dll$(EXT) \
|
||||
zstd-compress$(EXT) zstd-decompress$(EXT) \
|
||||
zstd-small$(EXT) zstd-frugal$(EXT) zstd-nolegacy$(EXT) zstd4$(EXT) \
|
||||
zstd-dictBuilder$(EXT) \
|
||||
*.gcda default*.profraw default.profdata have_zlib$(EXT)
|
||||
$(RM) $(CLEAN) core *.o tmp* result* dictionary *.zst \
|
||||
*.gcda default*.profraw default.profdata have_zlib
|
||||
$(RM) -r obj/*
|
||||
@echo Cleaning completed
|
||||
|
||||
@@ -339,7 +347,7 @@ ifneq (,$(filter $(UNAME),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD D
|
||||
|
||||
HAVE_COLORNEVER = $(shell echo a | egrep --color=never a > /dev/null 2> /dev/null && echo 1 || echo 0)
|
||||
EGREP_OPTIONS ?=
|
||||
ifeq ($HAVE_COLORNEVER, 1)
|
||||
ifeq ($(HAVE_COLORNEVER), 1)
|
||||
EGREP_OPTIONS += --color=never
|
||||
endif
|
||||
EGREP = egrep $(EGREP_OPTIONS)
|
||||
|
||||
+24
-8
@@ -290,6 +290,7 @@ FIO_prefs_t* FIO_createPreferences(void)
|
||||
ret->excludeCompressedFiles = 0;
|
||||
ret->allowBlockDevices = 0;
|
||||
ret->asyncIO = AIO_supported();
|
||||
ret->passThrough = -1;
|
||||
return ret;
|
||||
}
|
||||
|
||||
@@ -338,7 +339,7 @@ void FIO_setCompressionType(FIO_prefs_t* const prefs, FIO_compressionType_t comp
|
||||
|
||||
void FIO_overwriteMode(FIO_prefs_t* const prefs) { prefs->overwrite = 1; }
|
||||
|
||||
void FIO_setSparseWrite(FIO_prefs_t* const prefs, unsigned sparse) { prefs->sparseFileSupport = sparse; }
|
||||
void FIO_setSparseWrite(FIO_prefs_t* const prefs, int sparse) { prefs->sparseFileSupport = sparse; }
|
||||
|
||||
void FIO_setDictIDFlag(FIO_prefs_t* const prefs, int dictIDFlag) { prefs->dictIDFlag = dictIDFlag; }
|
||||
|
||||
@@ -371,7 +372,7 @@ void FIO_setOverlapLog(FIO_prefs_t* const prefs, int overlapLog){
|
||||
prefs->overlapLog = overlapLog;
|
||||
}
|
||||
|
||||
void FIO_setAdaptiveMode(FIO_prefs_t* const prefs, unsigned adapt) {
|
||||
void FIO_setAdaptiveMode(FIO_prefs_t* const prefs, int adapt) {
|
||||
if ((adapt>0) && (prefs->nbWorkers==0))
|
||||
EXM_THROW(1, "Adaptive mode is not compatible with single thread mode \n");
|
||||
prefs->adaptiveMode = adapt;
|
||||
@@ -453,7 +454,7 @@ void FIO_setContentSize(FIO_prefs_t* const prefs, int value)
|
||||
prefs->contentSize = value != 0;
|
||||
}
|
||||
|
||||
void FIO_setAsyncIOFlag(FIO_prefs_t* const prefs, unsigned value) {
|
||||
void FIO_setAsyncIOFlag(FIO_prefs_t* const prefs, int value) {
|
||||
#ifdef ZSTD_MULTITHREAD
|
||||
prefs->asyncIO = value;
|
||||
#else
|
||||
@@ -463,6 +464,10 @@ void FIO_setAsyncIOFlag(FIO_prefs_t* const prefs, unsigned value) {
|
||||
#endif
|
||||
}
|
||||
|
||||
void FIO_setPassThroughFlag(FIO_prefs_t* const prefs, int value) {
|
||||
prefs->passThrough = (value != 0);
|
||||
}
|
||||
|
||||
/* FIO_ctx_t functions */
|
||||
|
||||
void FIO_setHasStdoutOutput(FIO_ctx_t* const fCtx, int value) {
|
||||
@@ -1306,7 +1311,7 @@ FIO_compressZstdFrame(FIO_ctx_t* const fCtx,
|
||||
windowLog = ZSTD_WINDOWLOG_LIMIT_DEFAULT;
|
||||
} else {
|
||||
const ZSTD_compressionParameters cParams = ZSTD_getCParams(compressionLevel, fileSize, 0);
|
||||
windowLog = cParams.windowLog;
|
||||
windowLog = (int)cParams.windowLog;
|
||||
}
|
||||
}
|
||||
windowSize = UTIL_makeHumanReadableSize(MAX(1ULL, MIN(1ULL << windowLog, pledgedSrcSize)));
|
||||
@@ -1726,14 +1731,15 @@ FIO_compressFilename_srcFile(FIO_ctx_t* const fCtx,
|
||||
return result;
|
||||
}
|
||||
|
||||
static const char* checked_index(const char* options[], size_t length, size_t index) {
|
||||
static const char*
|
||||
checked_index(const char* options[], size_t length, size_t index) {
|
||||
assert(index < length);
|
||||
/* Necessary to avoid warnings since -O3 will omit the above `assert` */
|
||||
(void) length;
|
||||
return options[index];
|
||||
}
|
||||
|
||||
#define INDEX(options, index) checked_index((options), sizeof(options) / sizeof(char*), (index))
|
||||
#define INDEX(options, index) checked_index((options), sizeof(options) / sizeof(char*), (size_t)(index))
|
||||
|
||||
void FIO_displayCompressionParameters(const FIO_prefs_t* prefs) {
|
||||
static const char* formatOptions[5] = {ZSTD_EXTENSION, GZ_EXTENSION, XZ_EXTENSION,
|
||||
@@ -2335,6 +2341,16 @@ static int FIO_decompressFrames(FIO_ctx_t* const fCtx,
|
||||
{
|
||||
unsigned readSomething = 0;
|
||||
unsigned long long filesize = 0;
|
||||
int passThrough = prefs->passThrough;
|
||||
|
||||
if (passThrough == -1) {
|
||||
/* If pass-through mode is not explicitly enabled or disabled,
|
||||
* default to the legacy behavior of enabling it if we are writing
|
||||
* to stdout with the overwrite flag enabled.
|
||||
*/
|
||||
passThrough = prefs->overwrite && !strcmp(dstFileName, stdoutmark);
|
||||
}
|
||||
assert(passThrough == 0 || passThrough == 1);
|
||||
|
||||
/* for each frame */
|
||||
for ( ; ; ) {
|
||||
@@ -2352,7 +2368,7 @@ static int FIO_decompressFrames(FIO_ctx_t* const fCtx,
|
||||
}
|
||||
readSomething = 1; /* there is at least 1 byte in srcFile */
|
||||
if (ress.readCtx->srcBufferLoaded < toRead) { /* not enough input to check magic number */
|
||||
if ((prefs->overwrite) && !strcmp (dstFileName, stdoutmark)) { /* pass-through mode */
|
||||
if (passThrough) {
|
||||
return FIO_passThrough(&ress);
|
||||
}
|
||||
DISPLAYLEVEL(1, "zstd: %s: unknown header \n", srcFileName);
|
||||
@@ -2390,7 +2406,7 @@ static int FIO_decompressFrames(FIO_ctx_t* const fCtx,
|
||||
DISPLAYLEVEL(1, "zstd: %s: lz4 file cannot be uncompressed (zstd compiled without HAVE_LZ4) -- ignored \n", srcFileName);
|
||||
return 1;
|
||||
#endif
|
||||
} else if ((prefs->overwrite) && !strcmp (dstFileName, stdoutmark)) { /* pass-through mode */
|
||||
} else if (passThrough) {
|
||||
return FIO_passThrough(&ress);
|
||||
} else {
|
||||
DISPLAYLEVEL(1, "zstd: %s: unsupported format \n", srcFileName);
|
||||
|
||||
+6
-4
@@ -12,8 +12,9 @@
|
||||
#ifndef FILEIO_H_23981798732
|
||||
#define FILEIO_H_23981798732
|
||||
|
||||
#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_compressionParameters */
|
||||
#include "fileio_types.h"
|
||||
#include "util.h" /* FileNamesTable */
|
||||
#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_compressionParameters */
|
||||
#include "../lib/zstd.h" /* ZSTD_* */
|
||||
|
||||
#if defined (__cplusplus)
|
||||
@@ -70,7 +71,7 @@ void FIO_freeContext(FIO_ctx_t* const fCtx);
|
||||
/* FIO_prefs_t functions */
|
||||
void FIO_setCompressionType(FIO_prefs_t* const prefs, FIO_compressionType_t compressionType);
|
||||
void FIO_overwriteMode(FIO_prefs_t* const prefs);
|
||||
void FIO_setAdaptiveMode(FIO_prefs_t* const prefs, unsigned adapt);
|
||||
void FIO_setAdaptiveMode(FIO_prefs_t* const prefs, int adapt);
|
||||
void FIO_setAdaptMin(FIO_prefs_t* const prefs, int minCLevel);
|
||||
void FIO_setAdaptMax(FIO_prefs_t* const prefs, int maxCLevel);
|
||||
void FIO_setUseRowMatchFinder(FIO_prefs_t* const prefs, int useRowMatchFinder);
|
||||
@@ -86,7 +87,7 @@ void FIO_setMemLimit(FIO_prefs_t* const prefs, unsigned memLimit);
|
||||
void FIO_setNbWorkers(FIO_prefs_t* const prefs, int nbWorkers);
|
||||
void FIO_setOverlapLog(FIO_prefs_t* const prefs, int overlapLog);
|
||||
void FIO_setRemoveSrcFile(FIO_prefs_t* const prefs, unsigned flag);
|
||||
void FIO_setSparseWrite(FIO_prefs_t* const prefs, unsigned sparse); /**< 0: no sparse; 1: disable on stdout; 2: always enabled */
|
||||
void FIO_setSparseWrite(FIO_prefs_t* const prefs, int sparse); /**< 0: no sparse; 1: disable on stdout; 2: always enabled */
|
||||
void FIO_setRsyncable(FIO_prefs_t* const prefs, int rsyncable);
|
||||
void FIO_setStreamSrcSize(FIO_prefs_t* const prefs, size_t streamSrcSize);
|
||||
void FIO_setTargetCBlockSize(FIO_prefs_t* const prefs, size_t targetCBlockSize);
|
||||
@@ -103,7 +104,8 @@ void FIO_setAllowBlockDevices(FIO_prefs_t* const prefs, int allowBlockDevices);
|
||||
void FIO_setPatchFromMode(FIO_prefs_t* const prefs, int value);
|
||||
void FIO_setContentSize(FIO_prefs_t* const prefs, int value);
|
||||
void FIO_displayCompressionParameters(const FIO_prefs_t* prefs);
|
||||
void FIO_setAsyncIOFlag(FIO_prefs_t* const prefs, unsigned value);
|
||||
void FIO_setAsyncIOFlag(FIO_prefs_t* const prefs, int value);
|
||||
void FIO_setPassThroughFlag(FIO_prefs_t* const prefs, int value);
|
||||
|
||||
/* FIO_ctx_t functions */
|
||||
void FIO_setNbFilesTotal(FIO_ctx_t* const fCtx, int value);
|
||||
|
||||
@@ -34,13 +34,13 @@ typedef struct FIO_prefs_s {
|
||||
|
||||
/* Algorithm preferences */
|
||||
FIO_compressionType_t compressionType;
|
||||
U32 sparseFileSupport; /* 0: no sparse allowed; 1: auto (file yes, stdout no); 2: force sparse */
|
||||
int sparseFileSupport; /* 0: no sparse allowed; 1: auto (file yes, stdout no); 2: force sparse */
|
||||
int dictIDFlag;
|
||||
int checksumFlag;
|
||||
int blockSize;
|
||||
int overlapLog;
|
||||
U32 adaptiveMode;
|
||||
U32 useRowMatchFinder;
|
||||
int adaptiveMode;
|
||||
int useRowMatchFinder;
|
||||
int rsyncable;
|
||||
int minAdaptLevel;
|
||||
int maxAdaptLevel;
|
||||
@@ -56,9 +56,9 @@ typedef struct FIO_prefs_s {
|
||||
ZSTD_paramSwitch_e literalCompressionMode;
|
||||
|
||||
/* IO preferences */
|
||||
U32 removeSrcFile;
|
||||
U32 overwrite;
|
||||
U32 asyncIO;
|
||||
int removeSrcFile;
|
||||
int overwrite;
|
||||
int asyncIO;
|
||||
|
||||
/* Computation resources preferences */
|
||||
unsigned memLimit;
|
||||
@@ -68,6 +68,7 @@ typedef struct FIO_prefs_s {
|
||||
int patchFromMode;
|
||||
int contentSize;
|
||||
int allowBlockDevices;
|
||||
int passThrough;
|
||||
} FIO_prefs_t;
|
||||
|
||||
#endif /* FILEIO_TYPES_HEADER */
|
||||
|
||||
+5
-2
@@ -1,5 +1,5 @@
|
||||
.
|
||||
.TH "ZSTD" "1" "June 2022" "zstd 1.5.3" "User Commands"
|
||||
.TH "ZSTD" "1" "August 2022" "zstd 1.5.3" "User Commands"
|
||||
.
|
||||
.SH "NAME"
|
||||
\fBzstd\fR \- zstd, zstdmt, unzstd, zstdcat \- Compress or decompress \.zst files
|
||||
@@ -162,7 +162,7 @@ Additionally, this can be used to limit memory for dictionary training\. This pa
|
||||
\fB\-o FILE\fR: save result into \fBFILE\fR
|
||||
.
|
||||
.IP "\(bu" 4
|
||||
\fB\-f\fR, \fB\-\-force\fR: disable input and output checks\. Allows overwriting existing files, input from console, output to stdout, operating on links, block devices, etc\.
|
||||
\fB\-f\fR, \fB\-\-force\fR: disable input and output checks\. Allows overwriting existing files, input from console, output to stdout, operating on links, block devices, etc\. During decompression and when the output destination is stdout, pass\-through unrecognized formats as\-is\.
|
||||
.
|
||||
.IP "\(bu" 4
|
||||
\fB\-c\fR, \fB\-\-stdout\fR: write to standard output (even if it is the console); keep original files unchanged\.
|
||||
@@ -171,6 +171,9 @@ Additionally, this can be used to limit memory for dictionary training\. This pa
|
||||
\fB\-\-[no\-]sparse\fR: enable / disable sparse FS support, to make files with many zeroes smaller on disk\. Creating sparse files may save disk space and speed up decompression by reducing the amount of disk I/O\. default: enabled when output is into a file, and disabled when output is stdout\. This setting overrides default and can force sparse mode over stdout\.
|
||||
.
|
||||
.IP "\(bu" 4
|
||||
\fB\-\-[no\-]pass\-through\fR enable / disable passing through uncompressed files as\-is\. During decompression when pass\-through is enabled, unrecognized formats will be copied as\-is from the input to the output\. By default, pass\-through will occur when the output destination is stdout and the force (\-f) option is set\.
|
||||
.
|
||||
.IP "\(bu" 4
|
||||
\fB\-\-rm\fR: remove source file(s) after successful compression or decompression\. If used in combination with \-o, will trigger a confirmation prompt (which can be silenced with \-f), as this is a destructive operation\.
|
||||
.
|
||||
.IP "\(bu" 4
|
||||
|
||||
@@ -211,6 +211,8 @@ the last one takes effect.
|
||||
* `-f`, `--force`:
|
||||
disable input and output checks. Allows overwriting existing files, input
|
||||
from console, output to stdout, operating on links, block devices, etc.
|
||||
During decompression and when the output destination is stdout, pass-through
|
||||
unrecognized formats as-is.
|
||||
* `-c`, `--stdout`:
|
||||
write to standard output (even if it is the console); keep original files unchanged.
|
||||
* `--[no-]sparse`:
|
||||
@@ -221,6 +223,12 @@ the last one takes effect.
|
||||
default: enabled when output is into a file,
|
||||
and disabled when output is stdout.
|
||||
This setting overrides default and can force sparse mode over stdout.
|
||||
* `--[no-]pass-through`
|
||||
enable / disable passing through uncompressed files as-is. During
|
||||
decompression when pass-through is enabled, unrecognized formats will be
|
||||
copied as-is from the input to the output. By default, pass-through will
|
||||
occur when the output destination is stdout and the force (-f) option is
|
||||
set.
|
||||
* `--rm`:
|
||||
remove source file(s) after successful compression or decompression. If used in combination with
|
||||
-o, will trigger a confirmation prompt (which can be silenced with -f), as this is a destructive operation.
|
||||
|
||||
+41
-15
@@ -154,7 +154,8 @@ static void usage(FILE* f, const char* programName)
|
||||
#endif
|
||||
DISPLAY_F(f, " -f, --force disable input and output checks. Allows overwriting existing files,\n");
|
||||
DISPLAY_F(f, " input from console, output to stdout, operating on links,\n");
|
||||
DISPLAY_F(f, " block devices, etc.\n");
|
||||
DISPLAY_F(f, " block devices, etc. During decompression and when the output\n");
|
||||
DISPLAY_F(f, " destination is stdout, pass-through unrecognized formats as-is.\n");
|
||||
DISPLAY_F(f, " --rm remove source file(s) after successful de/compression\n");
|
||||
DISPLAY_F(f, " -k, --keep preserve source file(s) (default) \n");
|
||||
#ifdef ZSTD_GZCOMPRESS
|
||||
@@ -264,6 +265,15 @@ static void usage_advanced(const char* programName)
|
||||
# else
|
||||
DISPLAYOUT(" --[no-]sparse sparse mode (default: disabled)\n");
|
||||
# endif
|
||||
{
|
||||
char const* passThroughDefault = "disabled";
|
||||
if (exeNameMatch(programName, ZSTD_CAT) ||
|
||||
exeNameMatch(programName, ZSTD_ZCAT) ||
|
||||
exeNameMatch(programName, ZSTD_GZCAT)) {
|
||||
passThroughDefault = "enabled";
|
||||
}
|
||||
DISPLAYOUT(" --[no-]pass-through : passes through uncompressed files as-is (default: %s)\n", passThroughDefault);
|
||||
}
|
||||
#endif /* ZSTD_NODECOMPRESS */
|
||||
|
||||
#ifndef ZSTD_NODICT
|
||||
@@ -776,6 +786,18 @@ static unsigned init_nbThreads(void) {
|
||||
const char* __nb; \
|
||||
NEXT_FIELD(__nb); \
|
||||
val32 = readU32FromChar(&__nb); \
|
||||
if(*__nb != 0) { \
|
||||
errorOut("error: only numeric values with optional suffixes K, KB, KiB, M, MB, MiB are allowed"); \
|
||||
} \
|
||||
}
|
||||
|
||||
#define NEXT_TSIZE(valTsize) { \
|
||||
const char* __nb; \
|
||||
NEXT_FIELD(__nb); \
|
||||
valTsize = readSizeTFromChar(&__nb); \
|
||||
if(*__nb != 0) { \
|
||||
errorOut("error: only numeric values with optional suffixes K, KB, KiB, M, MB, MiB are allowed"); \
|
||||
} \
|
||||
}
|
||||
|
||||
typedef enum { zom_compress, zom_decompress, zom_test, zom_bench, zom_train, zom_list } zstd_operation_mode;
|
||||
@@ -801,7 +823,6 @@ int main(int argCount, const char* argv[])
|
||||
hasStdout = 0,
|
||||
ldmFlag = 0,
|
||||
main_pause = 0,
|
||||
nbWorkers = 0,
|
||||
adapt = 0,
|
||||
useRowMatchFinder = 0,
|
||||
adaptMin = MINCLEVEL,
|
||||
@@ -816,6 +837,7 @@ int main(int argCount, const char* argv[])
|
||||
showDefaultCParams = 0,
|
||||
ultra=0,
|
||||
contentSize=1;
|
||||
unsigned nbWorkers = 0;
|
||||
double compressibility = 0.5;
|
||||
unsigned bench_nbSeconds = 3; /* would be better if this value was synchronized from bench */
|
||||
size_t blockSize = 0;
|
||||
@@ -870,14 +892,14 @@ int main(int argCount, const char* argv[])
|
||||
/* preset behaviors */
|
||||
if (exeNameMatch(programName, ZSTD_ZSTDMT)) nbWorkers=0, singleThread=0;
|
||||
if (exeNameMatch(programName, ZSTD_UNZSTD)) operation=zom_decompress;
|
||||
if (exeNameMatch(programName, ZSTD_CAT)) { operation=zom_decompress; FIO_overwriteMode(prefs); forceStdout=1; followLinks=1; outFileName=stdoutmark; g_displayLevel=1; } /* supports multiple formats */
|
||||
if (exeNameMatch(programName, ZSTD_ZCAT)) { operation=zom_decompress; FIO_overwriteMode(prefs); forceStdout=1; followLinks=1; outFileName=stdoutmark; g_displayLevel=1; } /* behave like zcat, also supports multiple formats */
|
||||
if (exeNameMatch(programName, ZSTD_CAT)) { operation=zom_decompress; FIO_overwriteMode(prefs); forceStdout=1; followLinks=1; FIO_setPassThroughFlag(prefs, 1); outFileName=stdoutmark; g_displayLevel=1; } /* supports multiple formats */
|
||||
if (exeNameMatch(programName, ZSTD_ZCAT)) { operation=zom_decompress; FIO_overwriteMode(prefs); forceStdout=1; followLinks=1; FIO_setPassThroughFlag(prefs, 1); outFileName=stdoutmark; g_displayLevel=1; } /* behave like zcat, also supports multiple formats */
|
||||
if (exeNameMatch(programName, ZSTD_GZ)) { /* behave like gzip */
|
||||
suffix = GZ_EXTENSION; FIO_setCompressionType(prefs, FIO_gzipCompression); FIO_setRemoveSrcFile(prefs, 1);
|
||||
dictCLevel = cLevel = 6; /* gzip default is -6 */
|
||||
}
|
||||
if (exeNameMatch(programName, ZSTD_GUNZIP)) { operation=zom_decompress; FIO_setRemoveSrcFile(prefs, 1); } /* behave like gunzip, also supports multiple formats */
|
||||
if (exeNameMatch(programName, ZSTD_GZCAT)) { operation=zom_decompress; FIO_overwriteMode(prefs); forceStdout=1; followLinks=1; outFileName=stdoutmark; g_displayLevel=1; } /* behave like gzcat, also supports multiple formats */
|
||||
if (exeNameMatch(programName, ZSTD_GZCAT)) { operation=zom_decompress; FIO_overwriteMode(prefs); forceStdout=1; followLinks=1; FIO_setPassThroughFlag(prefs, 1); outFileName=stdoutmark; g_displayLevel=1; } /* behave like gzcat, also supports multiple formats */
|
||||
if (exeNameMatch(programName, ZSTD_LZMA)) { suffix = LZMA_EXTENSION; FIO_setCompressionType(prefs, FIO_lzmaCompression); FIO_setRemoveSrcFile(prefs, 1); } /* behave like lzma */
|
||||
if (exeNameMatch(programName, ZSTD_UNLZMA)) { operation=zom_decompress; FIO_setCompressionType(prefs, FIO_lzmaCompression); FIO_setRemoveSrcFile(prefs, 1); } /* behave like unlzma, also supports multiple formats */
|
||||
if (exeNameMatch(programName, ZSTD_XZ)) { suffix = XZ_EXTENSION; FIO_setCompressionType(prefs, FIO_xzCompression); FIO_setRemoveSrcFile(prefs, 1); } /* behave like xz */
|
||||
@@ -926,6 +948,8 @@ int main(int argCount, const char* argv[])
|
||||
if (!strcmp(argument, "--no-check")) { FIO_setChecksumFlag(prefs, 0); continue; }
|
||||
if (!strcmp(argument, "--sparse")) { FIO_setSparseWrite(prefs, 2); continue; }
|
||||
if (!strcmp(argument, "--no-sparse")) { FIO_setSparseWrite(prefs, 0); continue; }
|
||||
if (!strcmp(argument, "--pass-through")) { FIO_setPassThroughFlag(prefs, 1); continue; }
|
||||
if (!strcmp(argument, "--no-pass-through")) { FIO_setPassThroughFlag(prefs, 0); continue; }
|
||||
if (!strcmp(argument, "--test")) { operation=zom_test; continue; }
|
||||
if (!strcmp(argument, "--asyncio")) { FIO_setAsyncIOFlag(prefs, 1); continue;}
|
||||
if (!strcmp(argument, "--no-asyncio")) { FIO_setAsyncIOFlag(prefs, 0); continue;}
|
||||
@@ -1004,13 +1028,13 @@ int main(int argCount, const char* argv[])
|
||||
if (longCommandWArg(&argument, "--memlimit")) { NEXT_UINT32(memLimit); continue; }
|
||||
if (longCommandWArg(&argument, "--memory")) { NEXT_UINT32(memLimit); continue; }
|
||||
if (longCommandWArg(&argument, "--memlimit-decompress")) { NEXT_UINT32(memLimit); continue; }
|
||||
if (longCommandWArg(&argument, "--block-size=")) { blockSize = readSizeTFromChar(&argument); continue; }
|
||||
if (longCommandWArg(&argument, "--block-size")) { NEXT_TSIZE(blockSize); continue; }
|
||||
if (longCommandWArg(&argument, "--maxdict")) { NEXT_UINT32(maxDictSize); continue; }
|
||||
if (longCommandWArg(&argument, "--dictID")) { NEXT_UINT32(dictID); continue; }
|
||||
if (longCommandWArg(&argument, "--zstd=")) { if (!parseCompressionParameters(argument, &compressionParams)) { badusage(programName); CLEAN_RETURN(1); } continue; }
|
||||
if (longCommandWArg(&argument, "--stream-size=")) { streamSrcSize = readSizeTFromChar(&argument); continue; }
|
||||
if (longCommandWArg(&argument, "--target-compressed-block-size=")) { targetCBlockSize = readSizeTFromChar(&argument); continue; }
|
||||
if (longCommandWArg(&argument, "--size-hint=")) { srcSizeHint = readSizeTFromChar(&argument); continue; }
|
||||
if (longCommandWArg(&argument, "--stream-size")) { NEXT_TSIZE(streamSrcSize); continue; }
|
||||
if (longCommandWArg(&argument, "--target-compressed-block-size")) { NEXT_TSIZE(targetCBlockSize); continue; }
|
||||
if (longCommandWArg(&argument, "--size-hint")) { NEXT_TSIZE(srcSizeHint); continue; }
|
||||
if (longCommandWArg(&argument, "--output-dir-flat")) {
|
||||
NEXT_FIELD(outDirName);
|
||||
if (strlen(outDirName) == 0) {
|
||||
@@ -1051,6 +1075,8 @@ int main(int argCount, const char* argv[])
|
||||
/* Invalid character following --long */
|
||||
badusage(programName);
|
||||
CLEAN_RETURN(1);
|
||||
} else {
|
||||
ldmWindowLog = g_defaultMaxWindowLog;
|
||||
}
|
||||
/* Only set windowLog if not already set by --zstd */
|
||||
if (compressionParams.windowLog == 0)
|
||||
@@ -1200,7 +1226,7 @@ int main(int argCount, const char* argv[])
|
||||
/* nb of threads (hidden option) */
|
||||
case 'T':
|
||||
argument++;
|
||||
nbWorkers = (int)readU32FromChar(&argument);
|
||||
nbWorkers = readU32FromChar(&argument);
|
||||
break;
|
||||
|
||||
/* Dictionary Selection level */
|
||||
@@ -1246,10 +1272,10 @@ int main(int argCount, const char* argv[])
|
||||
if ((nbWorkers==0) && (!singleThread)) {
|
||||
/* automatically set # workers based on # of reported cpus */
|
||||
if (defaultLogicalCores) {
|
||||
nbWorkers = UTIL_countLogicalCores();
|
||||
nbWorkers = (unsigned)UTIL_countLogicalCores();
|
||||
DISPLAYLEVEL(3, "Note: %d logical core(s) detected \n", nbWorkers);
|
||||
} else {
|
||||
nbWorkers = UTIL_countPhysicalCores();
|
||||
nbWorkers = (unsigned)UTIL_countPhysicalCores();
|
||||
DISPLAYLEVEL(3, "Note: %d physical core(s) detected \n", nbWorkers);
|
||||
}
|
||||
}
|
||||
@@ -1313,7 +1339,7 @@ int main(int argCount, const char* argv[])
|
||||
if (operation==zom_bench) {
|
||||
#ifndef ZSTD_NOBENCH
|
||||
benchParams.blockSize = blockSize;
|
||||
benchParams.nbWorkers = nbWorkers;
|
||||
benchParams.nbWorkers = (int)nbWorkers;
|
||||
benchParams.realTime = (unsigned)setRealTimePrio;
|
||||
benchParams.nbSeconds = bench_nbSeconds;
|
||||
benchParams.ldmFlag = ldmFlag;
|
||||
@@ -1474,7 +1500,7 @@ int main(int argCount, const char* argv[])
|
||||
if (operation==zom_compress) {
|
||||
#ifndef ZSTD_NOCOMPRESS
|
||||
FIO_setContentSize(prefs, contentSize);
|
||||
FIO_setNbWorkers(prefs, nbWorkers);
|
||||
FIO_setNbWorkers(prefs, (int)nbWorkers);
|
||||
FIO_setBlockSize(prefs, (int)blockSize);
|
||||
if (g_overlapLog!=OVERLAP_LOG_DEFAULT) FIO_setOverlapLog(prefs, (int)g_overlapLog);
|
||||
FIO_setLdmFlag(prefs, (unsigned)ldmFlag);
|
||||
@@ -1482,7 +1508,7 @@ int main(int argCount, const char* argv[])
|
||||
FIO_setLdmMinMatch(prefs, (int)g_ldmMinMatch);
|
||||
if (g_ldmBucketSizeLog != LDM_PARAM_DEFAULT) FIO_setLdmBucketSizeLog(prefs, (int)g_ldmBucketSizeLog);
|
||||
if (g_ldmHashRateLog != LDM_PARAM_DEFAULT) FIO_setLdmHashRateLog(prefs, (int)g_ldmHashRateLog);
|
||||
FIO_setAdaptiveMode(prefs, (unsigned)adapt);
|
||||
FIO_setAdaptiveMode(prefs, adapt);
|
||||
FIO_setUseRowMatchFinder(prefs, useRowMatchFinder);
|
||||
FIO_setAdaptMin(prefs, adaptMin);
|
||||
FIO_setAdaptMax(prefs, adaptMax);
|
||||
|
||||
+11
-2
@@ -1,17 +1,26 @@
|
||||
.TH "ZSTDGREP" "1" "April 2022" "zstd 1.5.2" "User Commands"
|
||||
.
|
||||
.TH "ZSTDGREP" "1" "August 2022" "zstd 1.5.3" "User Commands"
|
||||
.
|
||||
.SH "NAME"
|
||||
\fBzstdgrep\fR \- print lines matching a pattern in zstandard\-compressed files
|
||||
.
|
||||
.SH "SYNOPSIS"
|
||||
\fBzstdgrep\fR [\fIgrep\-flags\fR] [\-\-] \fIpattern\fR [\fIfiles\fR \|\.\|\.\|\.]
|
||||
\fBzstdgrep\fR [\fIgrep\-flags\fR] [\-\-] \fIpattern\fR [\fIfiles\fR \.\.\.]
|
||||
.
|
||||
.SH "DESCRIPTION"
|
||||
\fBzstdgrep\fR runs \fBgrep (1)\fR on files, or \fBstdin\fR if no files argument is given, after decompressing them with \fBzstdcat (1)\fR\.
|
||||
.
|
||||
.P
|
||||
The grep\-flags and pattern arguments are passed on to \fBgrep (1)\fR\. If an \fB\-e\fR flag is found in the \fBgrep\-flags\fR, \fBzstdgrep\fR will not look for a pattern argument\.
|
||||
.
|
||||
.P
|
||||
Note that modern \fBgrep\fR alternatives such as \fBripgrep\fR (\fBrg\fR) support \fBzstd\fR\-compressed files out of the box, and can prove better alternatives than \fBzstdgrep\fR notably for unsupported complex pattern searches\. Note though that such alternatives may also feature some minor command line differences\.
|
||||
.
|
||||
.SH "EXIT STATUS"
|
||||
In case of missing arguments or missing pattern, 1 will be returned, otherwise 0\.
|
||||
.
|
||||
.SH "SEE ALSO"
|
||||
\fBzstd (1)\fR
|
||||
.
|
||||
.SH "AUTHORS"
|
||||
Thomas Klausner \fIwiz@NetBSD\.org\fR
|
||||
|
||||
+7
-2
@@ -1,9 +1,14 @@
|
||||
.TH "ZSTDLESS" "1" "April 2022" "zstd 1.5.2" "User Commands"
|
||||
.
|
||||
.TH "ZSTDLESS" "1" "August 2022" "zstd 1.5.3" "User Commands"
|
||||
.
|
||||
.SH "NAME"
|
||||
\fBzstdless\fR \- view zstandard\-compressed files
|
||||
.
|
||||
.SH "SYNOPSIS"
|
||||
\fBzstdless\fR [\fIflags\fR] [\fIfile\fR \|\.\|\.\|\.]
|
||||
\fBzstdless\fR [\fIflags\fR] [\fIfile\fR \.\.\.]
|
||||
.
|
||||
.SH "DESCRIPTION"
|
||||
\fBzstdless\fR runs \fBless (1)\fR on files or stdin, if no files argument is given, after decompressing them with \fBzstdcat (1)\fR\.
|
||||
.
|
||||
.SH "SEE ALSO"
|
||||
\fBzstd (1)\fR
|
||||
|
||||
Reference in New Issue
Block a user