ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'upstream/longRangeMatcher' into ldm-integrate

Browse Source
This commit is contained in:
Stella Lau
2017-09-01 10:19:38 -07:00
parent 8081becadc 9ab7614d23
commit 17d8e0bdcc
199 changed files with 6486 additions and 1738 deletions
Download Patch File Download Diff File Expand all files Collapse all files
programs/.gitignore
+4
View File
@@ -1,8 +1,12 @@
# local binary (Makefile)
zstd
zstd32
zstd4
zstd-compress
zstd-decompress
zstd-frugal
zstd-small
zstd-nolegacy
# Object files
*.o
programs/Makefile
+31 -20
View File
@@ -1,12 +1,10 @@
# ##########################################################################
# ################################################################
# Copyright (c) 2015-present, Yann Collet, Facebook, Inc.
# All rights reserved.
#
# This Makefile is validated for Linux, macOS, *BSD, Hurd, Solaris, MSYS2 targets
#
# This source code is licensed under the BSD-style license found in the
# LICENSE file in the root directory of this source tree. An additional grant
# of patent rights can be found in the PATENTS file in the same directory.
# This source code is licensed under both the BSD-style license (found in the
# LICENSE file in the root directory of this source tree) and the GPLv2 (found
# in the COPYING file in the root directory of this source tree).
# ##########################################################################
# zstd : Command Line Utility, supporting gzip-like arguments
# zstd32 : Same as zstd, but forced to compile in 32-bits mode
@@ -42,7 +40,7 @@ CPPFLAGS+= -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/compress \
-DZSTD_NEWAPI \
-DXXH_NAMESPACE=ZSTD_ # because xxhash.o already compiled with this macro from library
CFLAGS ?= -O3
DEBUGFLAGS = -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \
DEBUGFLAGS= -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \
-Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \
-Wstrict-prototypes -Wundef -Wpointer-arith -Wformat-security \
-Wvla -Wformat=2 -Winit-self -Wfloat-equal -Wwrite-strings \
@@ -68,8 +66,7 @@ endif
else
endif
ZSTDLIB_FILES := $(wildcard $(ZSTD_FILES)) $(wildcard $(ZSTDLEGACY_FILES)) $(wildcard $(ZDICT_FILES))
ZSTDLIB_OBJ := $(patsubst %.c,%.o,$(ZSTDLIB_FILES))
ZSTDLIB_FILES := $(sort $(wildcard $(ZSTD_FILES)) $(wildcard $(ZSTDLEGACY_FILES)) $(wildcard $(ZDICT_FILES)))
# Define *.exe as extension for Windows systems
ifneq (,$(filter Windows%,$(OS)))
@@ -132,12 +129,15 @@ else
LZ4_MSG := $(NO_LZ4_MSG)
endif
.PHONY: default all clean clean_decomp_o install uninstall generate_res
.PHONY: default
default: zstd-release
.PHONY: all
all: zstd
.PHONY: allVariants
allVariants: zstd zstd-compress zstd-decompress zstd-small zstd-nolegacy
$(ZSTDDECOMP_O): CFLAGS += $(ALIGN_LOOP)
zstd zstd4 : CPPFLAGS += $(THREAD_CPP) $(ZLIBCPP) $(LZMACPP)
@@ -154,8 +154,9 @@ zstd zstd4 : $(ZSTDLIB_FILES) zstdcli.o fileio.o bench.o datagen.o dibio.o
ifneq (,$(filter Windows%,$(OS)))
windres/generate_res.bat
endif
$(CC) $(FLAGS) $^ $(RES_FILE) -o zstd$(EXT) $(LDFLAGS)
$(CC) $(FLAGS) $^ $(RES_FILE) -o $@$(EXT) $(LDFLAGS)
.PHONY: zstd-release
zstd-release: DEBUGFLAGS :=
zstd-release: zstd
@@ -166,8 +167,8 @@ ifneq (,$(filter Windows%,$(OS)))
endif
$(CC) -m32 $(FLAGS) $^ $(RES32_FILE) -o $@$(EXT)
zstd-nolegacy : clean_decomp_o
$(MAKE) zstd ZSTD_LEGACY_SUPPORT=0
zstd-nolegacy : $(ZSTD_FILES) $(ZDICT_FILES) zstdcli.o fileio.c bench.o datagen.o dibio.o
$(CC) $(FLAGS) $^ -o $@$(EXT) $(LDFLAGS)
zstd-nomt : THREAD_CPP :=
zstd-nomt : THREAD_LD :=
@@ -198,9 +199,9 @@ zstd-pgo : clean zstd
$(MAKE) zstd MOREFLAGS=-fprofile-use
# minimal target, with only zstd compression and decompression. no bench. no legacy.
zstd-small: CFLAGS = "-Os -s"
zstd-small: CFLAGS = -Os -s
zstd-frugal zstd-small: $(ZSTD_FILES) zstdcli.c fileio.c
$(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT $^ -o zstd$(EXT)
$(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT $^ -o $@$(EXT)
zstd-decompress: $(ZSTDCOMMON_FILES) $(ZSTDDECOMP_FILES) zstdcli.c fileio.c
$(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_NOCOMPRESS $^ -o $@$(EXT)
@@ -208,34 +209,37 @@ zstd-decompress: $(ZSTDCOMMON_FILES) $(ZSTDDECOMP_FILES) zstdcli.c fileio.c
zstd-compress: $(ZSTDCOMMON_FILES) $(ZSTDCOMP_FILES) zstdcli.c fileio.c
$(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_NODECOMPRESS $^ -o $@$(EXT)
# zstd is now built with Multi-threading by default
# zstd is now built with multithreading enabled y default
zstdmt: zstd
.PHONY: generate_res
generate_res:
windres/generate_res.bat
.PHONY: clean
clean:
$(MAKE) -C $(ZSTDDIR) clean
@$(RM) $(ZSTDDIR)/decompress/*.o $(ZSTDDIR)/decompress/zstd_decompress.gcda
@$(RM) core *.o tmp* result* *.gcda dictionary *.zst \
zstd$(EXT) zstd32$(EXT) zstd-compress$(EXT) zstd-decompress$(EXT) \
zstd-small$(EXT) zstd-frugal$(EXT) zstd-nolegacy$(EXT) zstd4$(EXT) \
*.gcda default.profraw have_zlib$(EXT)
@echo Cleaning completed
clean_decomp_o:
@$(RM) $(ZSTDDECOMP_O)
MD2ROFF = ronn
MD2ROFF_FLAGS = --roff --warnings --manual="User Commands" --organization="zstd $(ZSTD_VERSION)"
zstd.1: zstd.1.md
cat $^ | $(MD2ROFF) $(MD2ROFF_FLAGS) | sed -n '/^\.\\\".*/!p' > $@
.PHONY: man
man: zstd.1
.PHONY: clean-man
clean-man:
rm zstd.1
.PHONY: preview-man
preview-man: clean-man man
man ./zstd.1
@@ -244,6 +248,10 @@ preview-man: clean-man man
#-----------------------------------------------------------------------------
ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS))
.PHONY: list
list:
@$(MAKE) -pRrq -f $(lastword $(MAKEFILE_LIST)) : 2>/dev/null | awk -v RS= -F: '/^# File/,/^# Finished Make data base/ {if ($$1 !~ "^[#.]") {print $$1}}' | sort | egrep -v -e '^[^[:alnum:]]' -e '^$@$$' | xargs
ifneq (,$(filter $(shell uname),SunOS))
INSTALL ?= ginstall
else
@@ -264,6 +272,7 @@ INSTALL_PROGRAM ?= $(INSTALL) -m 755
INSTALL_SCRIPT ?= $(INSTALL) -m 755
INSTALL_MAN ?= $(INSTALL) -m 644
.PHONY: install
install: zstd
@echo Installing binaries
@$(INSTALL) -d -m 755 $(DESTDIR)$(BINDIR)/ $(DESTDIR)$(MANDIR)/
@@ -279,6 +288,7 @@ install: zstd
@ln -sf zstd.1 $(DESTDIR)$(MANDIR)/unzstd.1
@echo zstd installation completed
.PHONY: uninstall
uninstall:
@$(RM) $(DESTDIR)$(BINDIR)/zstdgrep
@$(RM) $(DESTDIR)$(BINDIR)/zstdless
@@ -289,4 +299,5 @@ uninstall:
@$(RM) $(DESTDIR)$(MANDIR)/unzstd.1
@$(RM) $(DESTDIR)$(MANDIR)/zstd.1
@echo zstd programs successfully uninstalled
endif
programs/README.md
+28 -17
View File
@@ -3,44 +3,54 @@ Command Line Interface for Zstandard library
Command Line Interface (CLI) can be created using the `make` command without any additional parameters.
There are however other Makefile targets that create different variations of CLI:
- `zstd` : default CLI supporting gzip-like arguments; includes dictionary builder, benchmark, and support for decompression of legacy zstd versions
- `zstd32` : Same as `zstd`, but forced to compile in 32-bits mode
- `zstd_nolegacy` : Same as `zstd` except of support for decompression of legacy zstd versions
- `zstd-small` : CLI optimized for minimal size; without dictionary builder, benchmark, and support for decompression of legacy zstd versions
- `zstd-compress` : compressor-only version of CLI; without dictionary builder, benchmark, and support for decompression of legacy zstd versions
- `zstd-decompress` : decompressor-only version of CLI; without dictionary builder, benchmark, and support for decompression of legacy zstd versions
- `zstd` : default CLI supporting gzip-like arguments; includes dictionary builder, benchmark, and support for decompression of legacy zstd formats
- `zstd_nolegacy` : Same as `zstd` but without support for legacy zstd formats
- `zstd-small` : CLI optimized for minimal size; no dictionary builder, no benchmark, and no support for legacy zstd formats
- `zstd-compress` : version of CLI which can only compress into zstd format
- `zstd-decompress` : version of CLI which can only decompress zstd format
#### Compilation variables
`zstd` tries to detect and use the following features automatically :
`zstd` scope can be altered by modifying the following compilation variables :
- __HAVE_THREAD__ : multithreading is automatically enabled when `pthread` is detected.
It's possible to disable multithread support, by either compiling `zstd-nomt` target or using HAVE_THREAD=0 variable.
It's possible to disable multithread support, by setting HAVE_THREAD=0 .
Example : make zstd HAVE_THREAD=0
It's also possible to force compilation with multithread support, using HAVE_THREAD=1.
In which case, linking stage will fail if `pthread` library cannot be found.
This might be useful to prevent silent feature disabling.
- __HAVE_ZLIB__ : `zstd` can compress and decompress files in `.gz` format.
This is done through command `--format=gzip`.
This is ordered through command `--format=gzip`.
Alternatively, symlinks named `gzip` or `gunzip` will mimic intended behavior.
`.gz` support is automatically enabled when `zlib` library is detected at build time.
It's possible to disable `.gz` support, by either compiling `zstd-nogz` target or using HAVE_ZLIB=0 variable.
It's possible to disable `.gz` support, by setting HAVE_ZLIB=0.
Example : make zstd HAVE_ZLIB=0
It's also possible to force compilation with zlib support, using HAVE_ZLIB=1.
In which case, linking stage will fail if `zlib` library cannot be found.
This might be useful to prevent silent feature disabling.
- __HAVE_LZMA__ : `zstd` can compress and decompress files in `.xz` and `.lzma` formats.
This is done through commands `--format=xz` and `--format=lzma` respectively.
This is ordered through commands `--format=xz` and `--format=lzma` respectively.
Alternatively, symlinks named `xz`, `unxz`, `lzma`, or `unlzma` will mimic intended behavior.
`.xz` and `.lzma` support is automatically enabled when `lzma` library is detected at build time.
It's possible to disable `.xz` and `.lzma` support, by either compiling `zstd-noxz` target or using HAVE_LZMA=0 variable.
It's possible to disable `.xz` and `.lzma` support, by setting HAVE_LZMA=0 .
Example : make zstd HAVE_LZMA=0
It's also possible to force compilation with lzma support, using HAVE_LZMA=1.
In which case, linking stage will fail if `lzma` library cannot be found.
This might be useful to prevent silent feature disabling.
- __ZSTD_LEGACY_SUPPORT__ : `zstd` can decompress files compressed by older versions of `zstd`.
Starting v0.8.0, all versions of `zstd` produce frames compliant with the [specification](../doc/zstd_compression_format.md), and are therefore compatible.
But older versions (< v0.8.0) produced different, incompatible, frames.
By default, `zstd` supports decoding legacy formats >= v0.4.0 (`ZSTD_LEGACY_SUPPORT=4`).
This can be altered by modifying this compilation variable.
`ZSTD_LEGACY_SUPPORT=1` means "support all formats >= v0.1.0".
`ZSTD_LEGACY_SUPPORT=2` means "support all formats >= v0.2.0", and so on.
`ZSTD_LEGACY_SUPPORT=0` means _DO NOT_ support any legacy format.
if `ZSTD_LEGACY_SUPPORT >= 8`, it's the same as `0`, since there is no legacy format after `7`.
Note : `zstd` only supports decoding older formats, and cannot generate any legacy format.
#### Aggregation of parameters
CLI supports aggregation of parameters i.e. `-b1`, `-e18`, and `-i1` can be joined into `-b1e18i1`.
@@ -61,7 +71,7 @@ will rely more and more on previously decoded content to compress the rest of th
Usage of the dictionary builder and created dictionaries with CLI:
1. Create the dictionary : `zstd --train FullPathToTrainingSet/* -o dictionaryName`
1. Create the dictionary : `zstd --train PathToTrainingSet/* -o dictionaryName`
2. Compress with the dictionary: `zstd FILE -D dictionaryName`
3. Decompress with the dictionary: `zstd --decompress FILE.zst -D dictionaryName`
@@ -70,8 +80,8 @@ Usage of the dictionary builder and created dictionaries with CLI:
CLI includes in-memory compression benchmark module for zstd.
The benchmark is conducted using given filenames. The files are read into memory and joined together.
It makes benchmark more precise as it eliminates I/O overhead.
Many filenames can be supplied as multiple parameters, parameters with wildcards or
names of directories can be used as parameters with the `-r` option.
Multiple filenames can be supplied, as multiple parameters, with wildcards,
or names of directories can be used as parameters with `-r` option.
The benchmark measures ratio, compressed size, compression and decompression speed.
One can select compression levels starting from `-b` and ending with `-e`.
@@ -101,13 +111,14 @@ Advanced arguments :
-v : verbose mode; specify multiple times to increase verbosity
-q : suppress warnings; specify twice to suppress errors too
-c : force write to standard output, even if it is the console
-l : print information about zstd compressed files
--ultra : enable levels beyond 19, up to 22 (requires more memory)
-T# : use # threads for compression (default:1)
-B# : select size of each job (default:0==automatic)
--no-dictID : don't write dictID into header (dictionary compression)
--[no-]check : integrity check (default:enabled)
-r : operate recursively on directories
--format=gzip : compress files to the .gz format
--format=xz : compress files to the .xz format
--format=lzma : compress files to the .lzma format
--test : test compressed file integrity
--[no-]sparse : sparse mode (default:disabled)
-M# : Set a memory usage limit for decompression
programs/bench.c
+4 -4
View File
@@ -1,10 +1,10 @@
/**
/*
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under the BSD-style license found in the
* LICENSE file in the root directory of this source tree. An additional grant
* of patent rights can be found in the PATENTS file in the same directory.
* This source code is licensed under both the BSD-style license (found in the
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
* in the COPYING file in the root directory of this source tree).
*/
programs/bench.h
+4 -4
View File
@@ -1,10 +1,10 @@
/**
/*
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under the BSD-style license found in the
* LICENSE file in the root directory of this source tree. An additional grant
* of patent rights can be found in the PATENTS file in the same directory.
* This source code is licensed under both the BSD-style license (found in the
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
* in the COPYING file in the root directory of this source tree).
*/
programs/datagen.c
+4 -4
View File
@@ -1,10 +1,10 @@
/**
/*
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under the BSD-style license found in the
* LICENSE file in the root directory of this source tree. An additional grant
* of patent rights can be found in the PATENTS file in the same directory.
* This source code is licensed under both the BSD-style license (found in the
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
* in the COPYING file in the root directory of this source tree).
*/
programs/datagen.h
+6 -4
View File
@@ -1,11 +1,13 @@
/**
/*
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under the BSD-style license found in the
* LICENSE file in the root directory of this source tree. An additional grant
* of patent rights can be found in the PATENTS file in the same directory.
* This source code is licensed under both the BSD-style license (found in the
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
* in the COPYING file in the root directory of this source tree).
*/
#ifndef DATAGEN_H
#define DATAGEN_H
programs/dibio.c
+4 -4
View File
@@ -1,10 +1,10 @@
/**
/*
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under the BSD-style license found in the
* LICENSE file in the root directory of this source tree. An additional grant
* of patent rights can be found in the PATENTS file in the same directory.
* This source code is licensed under both the BSD-style license (found in the
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
* in the COPYING file in the root directory of this source tree).
*/
programs/dibio.h
+4 -4
View File
@@ -1,10 +1,10 @@
/**
/*
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under the BSD-style license found in the
* LICENSE file in the root directory of this source tree. An additional grant
* of patent rights can be found in the PATENTS file in the same directory.
* This source code is licensed under both the BSD-style license (found in the
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
* in the COPYING file in the root directory of this source tree).
*/
/* This library is designed for a single-threaded console application.
programs/fileio.c
+234 -233
View File
@@ -1,10 +1,10 @@
/**
/*
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under the BSD-style license found in the
* LICENSE file in the root directory of this source tree. An additional grant
* of patent rights can be found in the PATENTS file in the same directory.
* This source code is licensed under both the BSD-style license (found in the
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
* in the COPYING file in the root directory of this source tree).
*/
@@ -932,230 +932,6 @@ int FIO_compressFilename(const char* dstFileName, const char* srcFileName,
return result;
}
typedef struct {
int numActualFrames;
int numSkippableFrames;
unsigned long long decompressedSize;
int decompUnavailable;
unsigned long long compressedSize;
int usesCheck;
} fileInfo_t;
/*
* Reads information from file, stores in *info
* if successful, returns 0, returns 1 for frame analysis error, returns 2 for file not compressed with zstd
* returns 3 for cases in which file could not be opened.
*/
static int getFileInfo(fileInfo_t* info, const char* inFileName){
int detectError = 0;
FILE* const srcFile = FIO_openSrcFile(inFileName);
if (srcFile == NULL) {
DISPLAY("Error: could not open source file %s\n", inFileName);
return 3;
}
info->compressedSize = (unsigned long long)UTIL_getFileSize(inFileName);
/* begin analyzing frame */
for ( ; ; ) {
BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX];
size_t const numBytesRead = fread(headerBuffer, 1, sizeof(headerBuffer), srcFile);
if (numBytesRead < ZSTD_frameHeaderSize_min) {
if (feof(srcFile) && numBytesRead == 0 && info->compressedSize > 0) {
break;
}
else if (feof(srcFile)) {
DISPLAY("Error: reached end of file with incomplete frame\n");
detectError = 2;
break;
}
else {
DISPLAY("Error: did not reach end of file but ran out of frames\n");
detectError = 1;
break;
}
}
{
U32 const magicNumber = MEM_readLE32(headerBuffer);
if (magicNumber == ZSTD_MAGICNUMBER) {
U64 const frameContentSize = ZSTD_getFrameContentSize(headerBuffer, numBytesRead);
if (frameContentSize == ZSTD_CONTENTSIZE_ERROR || frameContentSize == ZSTD_CONTENTSIZE_UNKNOWN) {
info->decompUnavailable = 1;
}
else {
info->decompressedSize += frameContentSize;
}
{
/* move to the end of the frame header */
size_t const headerSize = ZSTD_frameHeaderSize(headerBuffer, numBytesRead);
if (ZSTD_isError(headerSize)) {
DISPLAY("Error: could not determine frame header size\n");
detectError = 1;
break;
}
{
int const ret = fseek(srcFile, ((long)headerSize)-((long)numBytesRead), SEEK_CUR);
if (ret != 0) {
DISPLAY("Error: could not move to end of frame header\n");
detectError = 1;
break;
}
}
}
/* skip the rest of the blocks in the frame */
{
int lastBlock = 0;
do {
BYTE blockHeaderBuffer[3];
U32 blockHeader;
int blockSize;
size_t const readBytes = fread(blockHeaderBuffer, 1, 3, srcFile);
if (readBytes != 3) {
DISPLAY("There was a problem reading the block header\n");
detectError = 1;
break;
}
blockHeader = MEM_readLE24(blockHeaderBuffer);
lastBlock = blockHeader & 1;
blockSize = blockHeader >> 3;
{
int const ret = fseek(srcFile, blockSize, SEEK_CUR);
if (ret != 0) {
DISPLAY("Error: could not skip to end of block\n");
detectError = 1;
break;
}
}
} while (lastBlock != 1);
if (detectError) {
break;
}
}
{
/* check if checksum is used */
BYTE const frameHeaderDescriptor = headerBuffer[4];
int const contentChecksumFlag = (frameHeaderDescriptor & (1 << 2)) >> 2;
if (contentChecksumFlag) {
int const ret = fseek(srcFile, 4, SEEK_CUR);
info->usesCheck = 1;
if (ret != 0) {
DISPLAY("Error: could not skip past checksum\n");
detectError = 1;
break;
}
}
}
info->numActualFrames++;
}
else if (magicNumber == ZSTD_MAGIC_SKIPPABLE_START) {
BYTE frameSizeBuffer[4];
size_t const readBytes = fread(frameSizeBuffer, 1, 4, srcFile);
if (readBytes != 4) {
DISPLAY("There was an error reading skippable frame size");
detectError = 1;
break;
}
{
U32 const frameSize = MEM_readLE32(frameSizeBuffer);
int const ret = LONG_SEEK(srcFile, frameSize, SEEK_CUR);
if (ret != 0) {
DISPLAY("Error: could not find end of skippable frame\n");
detectError = 1;
break;
}
}
info->numSkippableFrames++;
}
else {
detectError = 2;
break;
}
}
}
fclose(srcFile);
return detectError;
}
static void displayInfo(const char* inFileName, fileInfo_t* info, int displayLevel){
double const compressedSizeMB = (double)info->compressedSize/(1 MB);
double const decompressedSizeMB = (double)info->decompressedSize/(1 MB);
double const ratio = (info->compressedSize == 0) ? 0 : ((double)info->decompressedSize)/info->compressedSize;
const char* const checkString = (info->usesCheck ? "XXH64" : "None");
if (displayLevel <= 2) {
if (!info->decompUnavailable) {
DISPLAYOUT("Skippable Non-Skippable Compressed Uncompressed Ratio Check Filename\n");
DISPLAYOUT("%9d %13d %7.2f MB %9.2f MB %5.3f %5s %s\n",
info->numSkippableFrames, info->numActualFrames, compressedSizeMB, decompressedSizeMB,
ratio, checkString, inFileName);
}
else {
DISPLAYOUT("Skippable Non-Skippable Compressed Check Filename\n");
DISPLAYOUT("%9d %13d %7.2f MB %5s %s\n",
info->numSkippableFrames, info->numActualFrames, compressedSizeMB, checkString, inFileName);
}
}
else{
DISPLAYOUT("# Zstandard Frames: %d\n", info->numActualFrames);
DISPLAYOUT("# Skippable Frames: %d\n", info->numSkippableFrames);
DISPLAYOUT("Compressed Size: %.2f MB (%llu B)\n", compressedSizeMB, info->compressedSize);
if (!info->decompUnavailable) {
DISPLAYOUT("Decompressed Size: %.2f MB (%llu B)\n", decompressedSizeMB, info->decompressedSize);
DISPLAYOUT("Ratio: %.4f\n", ratio);
}
DISPLAYOUT("Check: %s\n", checkString);
DISPLAYOUT("\n");
}
}
static int FIO_listFile(const char* inFileName, int displayLevel, unsigned fileNo, unsigned numFiles){
/* initialize info to avoid warnings */
fileInfo_t info;
memset(&info, 0, sizeof(info));
DISPLAYOUT("%s (%u/%u):\n", inFileName, fileNo, numFiles);
{
int const error = getFileInfo(&info, inFileName);
if (error == 1) {
/* display error, but provide output */
DISPLAY("An error occurred with getting file info\n");
}
else if (error == 2) {
DISPLAYOUT("File %s not compressed with zstd\n", inFileName);
if (displayLevel > 2) {
DISPLAYOUT("\n");
}
return 1;
}
else if (error == 3) {
/* error occurred with opening the file */
if (displayLevel > 2) {
DISPLAYOUT("\n");
}
return 1;
}
displayInfo(inFileName, &info, displayLevel);
return error;
}
}
int FIO_listMultipleFiles(unsigned numFiles, const char** filenameTable, int displayLevel){
if (numFiles == 0) {
DISPLAYOUT("No files given\n");
return 0;
}
DISPLAYOUT("===========================================\n");
DISPLAYOUT("Printing information about compressed files\n");
DISPLAYOUT("===========================================\n");
DISPLAYOUT("Number of files listed: %u\n", numFiles);
{
int error = 0;
unsigned u;
for (u=0; u<numFiles;u++) {
error |= FIO_listFile(filenameTable[u], displayLevel, u+1, numFiles);
}
return error;
}
}
int FIO_compressMultipleFilenames(const char** inFileNamesTable, unsigned nbFiles,
const char* suffix,
@@ -1201,10 +977,8 @@ int FIO_compressMultipleFilenames(const char** inFileNamesTable, unsigned nbFile
missed_files += FIO_compressFilename_dstFile(ress, dstFileName, inFileNamesTable[u], compressionLevel);
} }
/* Close & Free */
FIO_freeCResources(ress);
free(dstFileName);
return missed_files;
}
@@ -1215,8 +989,8 @@ int FIO_compressMultipleFilenames(const char** inFileNamesTable, unsigned nbFile
#ifndef ZSTD_NODECOMPRESS
/* **************************************************************************
* Decompression
****************************************************************************/
* Decompression
***************************************************************************/
typedef struct {
void* srcBuffer;
size_t srcBufferLoaded;
@@ -1623,7 +1397,7 @@ static unsigned long long FIO_decompressLz4Frame(dRess_t* ress,
/* Write Block */
if (decodedBytes) {
if (fwrite(ress->dstBuffer, 1, decodedBytes, ress->dstFile) != decodedBytes) {
DISPLAYLEVEL(1, "zstd: %s \n", strerr(errno));
DISPLAYLEVEL(1, "zstd: %s \n", strerror(errno));
decodingError = 1; break;
}
filesize += decodedBytes;
@@ -1885,4 +1659,231 @@ int FIO_decompressMultipleFilenames(const char** srcNamesTable, unsigned nbFiles
return missingFiles + skippedFiles;
}
/* **************************************************************************
* .zst file info (--list command)
***************************************************************************/
typedef struct {
int numActualFrames;
int numSkippableFrames;
unsigned long long decompressedSize;
int decompUnavailable;
unsigned long long compressedSize;
int usesCheck;
} fileInfo_t;
/** getFileInfo() :
* Reads information from file, stores in *info
* @return : 0 if successful
* 1 for frame analysis error
* 2 for file not compressed with zstd
* 3 for cases in which file could not be opened.
*/
static int getFileInfo(fileInfo_t* info, const char* inFileName){
int detectError = 0;
FILE* const srcFile = FIO_openSrcFile(inFileName);
if (srcFile == NULL) {
DISPLAY("Error: could not open source file %s\n", inFileName);
return 3;
}
info->compressedSize = (unsigned long long)UTIL_getFileSize(inFileName);
/* begin analyzing frame */
for ( ; ; ) {
BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX];
size_t const numBytesRead = fread(headerBuffer, 1, sizeof(headerBuffer), srcFile);
if (numBytesRead < ZSTD_frameHeaderSize_min) {
if (feof(srcFile) && numBytesRead == 0 && info->compressedSize > 0) {
break;
}
else if (feof(srcFile)) {
DISPLAY("Error: reached end of file with incomplete frame\n");
detectError = 2;
break;
}
else {
DISPLAY("Error: did not reach end of file but ran out of frames\n");
detectError = 1;
break;
}
}
{ U32 const magicNumber = MEM_readLE32(headerBuffer);
/* Zstandard frame */
if (magicNumber == ZSTD_MAGICNUMBER) {
U64 const frameContentSize = ZSTD_getFrameContentSize(headerBuffer, numBytesRead);
if (frameContentSize == ZSTD_CONTENTSIZE_ERROR || frameContentSize == ZSTD_CONTENTSIZE_UNKNOWN) {
info->decompUnavailable = 1;
} else {
info->decompressedSize += frameContentSize;
}
/* move to the end of the frame header */
{ size_t const headerSize = ZSTD_frameHeaderSize(headerBuffer, numBytesRead);
if (ZSTD_isError(headerSize)) {
DISPLAY("Error: could not determine frame header size\n");
detectError = 1;
break;
}
{ int const ret = fseek(srcFile, ((long)headerSize)-((long)numBytesRead), SEEK_CUR);
if (ret != 0) {
DISPLAY("Error: could not move to end of frame header\n");
detectError = 1;
break;
} } }
/* skip the rest of the blocks in the frame */
{ int lastBlock = 0;
do {
BYTE blockHeaderBuffer[3];
size_t const readBytes = fread(blockHeaderBuffer, 1, 3, srcFile);
if (readBytes != 3) {
DISPLAY("There was a problem reading the block header\n");
detectError = 1;
break;
}
{ U32 const blockHeader = MEM_readLE24(blockHeaderBuffer);
U32 const blockTypeID = (blockHeader >> 1) & 3;
U32 const isRLE = (blockTypeID == 1);
U32 const isWrongBlock = (blockTypeID == 3);
long const blockSize = isRLE ? 1 : (long)(blockHeader >> 3);
if (isWrongBlock) {
DISPLAY("Error: unsupported block type \n");
detectError = 1;
break;
}
lastBlock = blockHeader & 1;
{ int const ret = fseek(srcFile, blockSize, SEEK_CUR);
if (ret != 0) {
DISPLAY("Error: could not skip to end of block\n");
detectError = 1;
break;
} } }
} while (lastBlock != 1);
if (detectError) break;
}
/* check if checksum is used */
{ BYTE const frameHeaderDescriptor = headerBuffer[4];
int const contentChecksumFlag = (frameHeaderDescriptor & (1 << 2)) >> 2;
if (contentChecksumFlag) {
int const ret = fseek(srcFile, 4, SEEK_CUR);
info->usesCheck = 1;
if (ret != 0) {
DISPLAY("Error: could not skip past checksum\n");
detectError = 1;
break;
} } }
info->numActualFrames++;
}
/* Skippable frame */
else if ((magicNumber & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) {
U32 const frameSize = MEM_readLE32(headerBuffer + 4);
long const seek = (long)(8 + frameSize - numBytesRead);
int const ret = LONG_SEEK(srcFile, seek, SEEK_CUR);
if (ret != 0) {
DISPLAY("Error: could not find end of skippable frame\n");
detectError = 1;
break;
}
info->numSkippableFrames++;
}
/* unknown content */
else {
detectError = 2;
break;
}
}
} /* end analyzing frame */
fclose(srcFile);
return detectError;
}
static void displayInfo(const char* inFileName, fileInfo_t* info, int displayLevel){
unsigned const unit = info->compressedSize < (1 MB) ? (1 KB) : (1 MB);
const char* const unitStr = info->compressedSize < (1 MB) ? "KB" : "MB";
double const compressedSizeUnit = (double)info->compressedSize / unit;
double const decompressedSizeUnit = (double)info->decompressedSize / unit;
double const ratio = (info->compressedSize == 0) ? 0 : ((double)info->decompressedSize)/info->compressedSize;
const char* const checkString = (info->usesCheck ? "XXH64" : "None");
if (displayLevel <= 2) {
if (!info->decompUnavailable) {
DISPLAYOUT("Skippable Non-Skippable Compressed Uncompressed Ratio Check Filename\n");
DISPLAYOUT("%9d %13d %7.2f %2s %9.2f %2s %5.3f %5s %s\n",
info->numSkippableFrames, info->numActualFrames,
compressedSizeUnit, unitStr, decompressedSizeUnit, unitStr,
ratio, checkString, inFileName);
} else {
DISPLAYOUT("Skippable Non-Skippable Compressed Check Filename\n");
DISPLAYOUT("%9d %13d %7.2f MB %5s %s\n",
info->numSkippableFrames, info->numActualFrames,
compressedSizeUnit, checkString, inFileName);
}
} else {
DISPLAYOUT("# Zstandard Frames: %d\n", info->numActualFrames);
DISPLAYOUT("# Skippable Frames: %d\n", info->numSkippableFrames);
DISPLAYOUT("Compressed Size: %.2f %2s (%llu B)\n",
compressedSizeUnit, unitStr, info->compressedSize);
if (!info->decompUnavailable) {
DISPLAYOUT("Decompressed Size: %.2f %2s (%llu B)\n",
decompressedSizeUnit, unitStr, info->decompressedSize);
DISPLAYOUT("Ratio: %.4f\n", ratio);
}
DISPLAYOUT("Check: %s\n", checkString);
DISPLAYOUT("\n");
}
}
static int FIO_listFile(const char* inFileName, int displayLevel, unsigned fileNo, unsigned numFiles){
/* initialize info to avoid warnings */
fileInfo_t info;
memset(&info, 0, sizeof(info));
DISPLAYOUT("%s (%u/%u):\n", inFileName, fileNo, numFiles);
{
int const error = getFileInfo(&info, inFileName);
if (error == 1) {
/* display error, but provide output */
DISPLAY("An error occurred with getting file info\n");
}
else if (error == 2) {
DISPLAYOUT("File %s not compressed with zstd\n", inFileName);
if (displayLevel > 2) {
DISPLAYOUT("\n");
}
return 1;
}
else if (error == 3) {
/* error occurred with opening the file */
if (displayLevel > 2) {
DISPLAYOUT("\n");
}
return 1;
}
displayInfo(inFileName, &info, displayLevel);
return error;
}
}
int FIO_listMultipleFiles(unsigned numFiles, const char** filenameTable, int displayLevel){
if (numFiles == 0) {
DISPLAYOUT("No files given\n");
return 0;
}
DISPLAYOUT("===========================================\n");
DISPLAYOUT("Printing information about compressed files\n");
DISPLAYOUT("===========================================\n");
DISPLAYOUT("Number of files listed: %u\n", numFiles);
{
int error = 0;
unsigned u;
for (u=0; u<numFiles;u++) {
error |= FIO_listFile(filenameTable[u], displayLevel, u+1, numFiles);
}
return error;
}
}
#endif /* #ifndef ZSTD_NODECOMPRESS */
programs/fileio.h
+4 -4
View File
@@ -1,10 +1,10 @@
/**
/*
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under the BSD-style license found in the
* LICENSE file in the root directory of this source tree. An additional grant
* of patent rights can be found in the PATENTS file in the same directory.
* This source code is licensed under both the BSD-style license (found in the
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
* in the COPYING file in the root directory of this source tree).
*/
programs/platform.h
+4 -6
View File
@@ -1,12 +1,10 @@
/**
* platform.h - compiler and OS detection
*
/*
* Copyright (c) 2016-present, Przemyslaw Skibinski, Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under the BSD-style license found in the
* LICENSE file in the root directory of this source tree. An additional grant
* of patent rights can be found in the PATENTS file in the same directory.
* This source code is licensed under both the BSD-style license (found in the
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
* in the COPYING file in the root directory of this source tree).
*/
#ifndef PLATFORM_H_MODULE
programs/util.h
+17 -15
View File
@@ -1,12 +1,10 @@
/**
* util.h - utility functions
*
/*
* Copyright (c) 2016-present, Przemyslaw Skibinski, Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under the BSD-style license found in the
* LICENSE file in the root directory of this source tree. An additional grant
* of patent rights can be found in the PATENTS file in the same directory.
* This source code is licensed under both the BSD-style license (found in the
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
* in the COPYING file in the root directory of this source tree).
*/
#ifndef UTIL_H_MODULE
@@ -107,13 +105,21 @@ extern "C" {
#endif
/*-****************************************
* Console log
******************************************/
static int g_utilDisplayLevel;
#define UTIL_DISPLAY(...) fprintf(stderr, __VA_ARGS__)
#define UTIL_DISPLAYLEVEL(l, ...) { if (g_utilDisplayLevel>=l) { UTIL_DISPLAY(__VA_ARGS__); } }
/*-****************************************
* Time functions
******************************************/
#if defined(_WIN32) /* Windows */
typedef LARGE_INTEGER UTIL_freq_t;
typedef LARGE_INTEGER UTIL_time_t;
UTIL_STATIC void UTIL_initTimer(UTIL_freq_t* ticksPerSecond) { if (!QueryPerformanceFrequency(ticksPerSecond)) fprintf(stderr, "ERROR: QueryPerformance not present\n"); }
UTIL_STATIC void UTIL_initTimer(UTIL_freq_t* ticksPerSecond) { if (!QueryPerformanceFrequency(ticksPerSecond)) UTIL_DISPLAYLEVEL(1, "ERROR: QueryPerformance not present\n"); }
UTIL_STATIC void UTIL_getTime(UTIL_time_t* x) { QueryPerformanceCounter(x); }
UTIL_STATIC U64 UTIL_getSpanTimeMicro(UTIL_freq_t ticksPerSecond, UTIL_time_t clockStart, UTIL_time_t clockEnd) { return 1000000ULL*(clockEnd.QuadPart - clockStart.QuadPart)/ticksPerSecond.QuadPart; }
UTIL_STATIC U64 UTIL_getSpanTimeNano(UTIL_freq_t ticksPerSecond, UTIL_time_t clockStart, UTIL_time_t clockEnd) { return 1000000000ULL*(clockEnd.QuadPart - clockStart.QuadPart)/ticksPerSecond.QuadPart; }
@@ -286,10 +292,6 @@ UTIL_STATIC void *UTIL_realloc(void *ptr, size_t size)
return NULL;
}
static int g_utilDisplayLevel;
#define UTIL_DISPLAY(...) fprintf(stderr, __VA_ARGS__)
#define UTIL_DISPLAYLEVEL(l, ...) { if (g_utilDisplayLevel>=l) { UTIL_DISPLAY(__VA_ARGS__); } }
#ifdef _WIN32
# define UTIL_HAS_CREATEFILELIST
@@ -311,7 +313,7 @@ UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_
hFile=FindFirstFileA(path, &cFile);
if (hFile == INVALID_HANDLE_VALUE) {
fprintf(stderr, "Cannot open directory '%s'\n", dirName);
UTIL_DISPLAYLEVEL(1, "Cannot open directory '%s'\n", dirName);
return 0;
}
free(path);
@@ -365,7 +367,7 @@ UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_
int dirLength, fnameLength, pathLength, nbFiles = 0;
if (!(dir = opendir(dirName))) {
fprintf(stderr, "Cannot open directory '%s': %s\n", dirName, strerror(errno));
UTIL_DISPLAYLEVEL(1, "Cannot open directory '%s': %s\n", dirName, strerror(errno));
return 0;
}
@@ -410,7 +412,7 @@ UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_
}
if (errno != 0) {
fprintf(stderr, "readdir(%s) error: %s\n", dirName, strerror(errno));
UTIL_DISPLAYLEVEL(1, "readdir(%s) error: %s\n", dirName, strerror(errno));
free(*bufStart);
*bufStart = NULL;
}
@@ -423,7 +425,7 @@ UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_
UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_t* pos, char** bufEnd, int followLinks)
{
(void)bufStart; (void)bufEnd; (void)pos;
fprintf(stderr, "Directory %s ignored (compiled without _WIN32 or _POSIX_C_SOURCE)\n", dirName);
UTIL_DISPLAYLEVEL(1, "Directory %s ignored (compiled without _WIN32 or _POSIX_C_SOURCE)\n", dirName);
return 0;
}
programs/zstd.1
+2 -2
View File
@@ -1,5 +1,5 @@
.
.TH "ZSTD" "1" "July 2017" "zstd 1.3.1" "User Commands"
.TH "ZSTD" "1" "August 2017" "zstd 1.3.1" "User Commands"
.
.SH "NAME"
\fBzstd\fR \- zstd, zstdmt, unzstd, zstdcat \- Compress or decompress \.zst files
@@ -149,7 +149,7 @@ display help/long help and exit
.
.TP
\fB\-V\fR, \fB\-\-version\fR
display version number and exit
display version number and exit\. Advanced : \fB\-vV\fR also displays supported formats\. \fB\-vvV\fR also displays POSIX support\.
.
.TP
\fB\-v\fR
programs/zstd.1.md
+3 -1
View File
@@ -140,7 +140,9 @@ the last one takes effect.
* `-h`/`-H`, `--help`:
display help/long help and exit
* `-V`, `--version`:
display version number and exit
display version number and exit.
Advanced : `-vV` also displays supported formats.
`-vvV` also displays POSIX support.
* `-v`:
verbose mode
* `-q`, `--quiet`:
programs/zstdcli.c
+69 -38
View File
@@ -1,10 +1,10 @@
/**
/*
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under the BSD-style license found in the
* LICENSE file in the root directory of this source tree. An additional grant
* of patent rights can be found in the PATENTS file in the same directory.
* This source code is licensed under both the BSD-style license (found in the
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
* in the COPYING file in the root directory of this source tree).
*/
@@ -16,7 +16,7 @@
#endif
#ifndef ZSTDCLI_CLEVEL_MAX
# define ZSTDCLI_CLEVEL_MAX 19 /* when not using --ultra */
# define ZSTDCLI_CLEVEL_MAX 19 /* without using --ultra */
#endif
@@ -26,14 +26,15 @@
**************************************/
#include "platform.h" /* IS_CONSOLE, PLATFORM_POSIX_VERSION */
#include "util.h" /* UTIL_HAS_CREATEFILELIST, UTIL_createFileList */
#include <stdio.h> /* fprintf(), stdin, stdout, stderr */
#include <string.h> /* strcmp, strlen */
#include <errno.h> /* errno */
#include "fileio.h"
#include "fileio.h" /* stdinmark, stdoutmark, ZSTD_EXTENSION */
#ifndef ZSTD_NOBENCH
# include "bench.h" /* BMK_benchFiles, BMK_SetNbSeconds */
#endif
#ifndef ZSTD_NODICT
# include "dibio.h"
# include "dibio.h" /* ZDICT_cover_params_t, DiB_trainFromFiles() */
#endif
#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_maxCLevel */
#include "zstd.h" /* ZSTD_VERSION_STRING */
@@ -64,7 +65,7 @@
#define MB *(1 <<20)
#define GB *(1U<<30)
#define DEFAULT_DISPLAY_LEVEL 2
#define DISPLAY_LEVEL_DEFAULT 2
static const char* g_defaultDictName = "dictionary";
static const unsigned g_defaultMaxDictSize = 110 KB;
@@ -79,7 +80,7 @@ static U32 g_overlapLog = OVERLAP_LOG_DEFAULT;
**************************************/
#define DISPLAY(...) fprintf(g_displayOut, __VA_ARGS__)
#define DISPLAYLEVEL(l, ...) { if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); } }
static int g_displayLevel = DEFAULT_DISPLAY_LEVEL; /* 0 : no display, 1: errors, 2 : + result + interaction + warnings, 3 : + progression, 4 : + information */
static int g_displayLevel = DISPLAY_LEVEL_DEFAULT; /* 0 : no display, 1: errors, 2 : + result + interaction + warnings, 3 : + progression, 4 : + information */
static FILE* g_displayOut;
@@ -88,12 +89,12 @@ static FILE* g_displayOut;
**************************************/
static int usage(const char* programName)
{
DISPLAY( "Usage :\n");
DISPLAY( " %s [args] [FILE(s)] [-o file]\n", programName);
DISPLAY( "Usage : \n");
DISPLAY( " %s [args] [FILE(s)] [-o file] \n", programName);
DISPLAY( "\n");
DISPLAY( "FILE : a filename\n");
DISPLAY( "FILE : a filename \n");
DISPLAY( " with no FILE, or when FILE is - , read standard input\n");
DISPLAY( "Arguments :\n");
DISPLAY( "Arguments : \n");
#ifndef ZSTD_NOCOMPRESS
DISPLAY( " -# : # compression level (1-%d, default:%d) \n", ZSTDCLI_CLEVEL_MAX, ZSTDCLI_CLEVEL_DEFAULT);
#endif
@@ -105,7 +106,7 @@ static int usage(const char* programName)
DISPLAY( " -f : overwrite output without prompting and (de)compress links \n");
DISPLAY( "--rm : remove source file(s) after successful de/compression \n");
DISPLAY( " -k : preserve source file(s) (default) \n");
DISPLAY( " -h/-H : display help/long help and exit\n");
DISPLAY( " -h/-H : display help/long help and exit \n");
return 0;
}
@@ -114,14 +115,15 @@ static int usage_advanced(const char* programName)
DISPLAY(WELCOME_MESSAGE);
usage(programName);
DISPLAY( "\n");
DISPLAY( "Advanced arguments :\n");
DISPLAY( " -V : display Version number and exit\n");
DISPLAY( "Advanced arguments : \n");
DISPLAY( " -V : display Version number and exit \n");
DISPLAY( " -v : verbose mode; specify multiple times to increase verbosity\n");
DISPLAY( " -q : suppress warnings; specify twice to suppress errors too\n");
DISPLAY( " -c : force write to standard output, even if it is the console\n");
DISPLAY( " -l : print information about zstd compressed files.\n");
DISPLAY( " -l : print information about zstd compressed files \n");
#ifndef ZSTD_NOCOMPRESS
DISPLAY( "--ultra : enable levels beyond %i, up to %i (requires more memory)\n", ZSTDCLI_CLEVEL_MAX, ZSTD_maxCLevel());
DISPLAY( "--long : enable long distance matching\n");
#ifdef ZSTD_MULTITHREAD
DISPLAY( " -T# : use # threads for compression (default:1) \n");
DISPLAY( " -B# : select size of each job (default:0==automatic) \n");
@@ -151,12 +153,10 @@ static int usage_advanced(const char* programName)
#endif
#endif
DISPLAY( " -M# : Set a memory usage limit for decompression \n");
DISPLAY( "--list : list information about a zstd compressed file \n");
DISPLAY( "--long : enable long distance matching\n");
DISPLAY( "-- : All arguments after \"--\" are treated as files \n");
#ifndef ZSTD_NODICT
DISPLAY( "\n");
DISPLAY( "Dictionary builder :\n");
DISPLAY( "Dictionary builder : \n");
DISPLAY( "--train ## : create a dictionary from a training set of files \n");
DISPLAY( "--train-cover[=k=#,d=#,steps=#] : use the cover algorithm with optional args\n");
DISPLAY( "--train-legacy[=s=#] : use the legacy algorithm with selectivity (default: %u)\n", g_defaultSelectivityLevel);
@@ -166,12 +166,12 @@ static int usage_advanced(const char* programName)
#endif
#ifndef ZSTD_NOBENCH
DISPLAY( "\n");
DISPLAY( "Benchmark arguments :\n");
DISPLAY( "Benchmark arguments : \n");
DISPLAY( " -b# : benchmark file(s), using # compression level (default : 1) \n");
DISPLAY( " -e# : test all compression levels from -bX to # (default: 1)\n");
DISPLAY( " -i# : minimum evaluation time in seconds (default : 3s)\n");
DISPLAY( " -i# : minimum evaluation time in seconds (default : 3s) \n");
DISPLAY( " -B# : cut file into independent blocks of size # (default: no block)\n");
DISPLAY( "--priority=rt : set process priority to real-time\n");
DISPLAY( "--priority=rt : set process priority to real-time \n");
#endif
return 0;
}
@@ -314,6 +314,35 @@ static unsigned parseCompressionParameters(const char* stringPtr, ZSTD_compressi
return 1;
}
static void printVersion(void)
{
DISPLAY(WELCOME_MESSAGE);
/* format support */
DISPLAYLEVEL(3, "*** supports: zstd");
#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>0) && (ZSTD_LEGACY_SUPPORT<8)
DISPLAYLEVEL(3, ", zstd legacy v0.%d+", ZSTD_LEGACY_SUPPORT);
#endif
#ifdef ZSTD_GZCOMPRESS
DISPLAYLEVEL(3, ", gzip");
#endif
#ifdef ZSTD_LZ4COMPRESS
DISPLAYLEVEL(3, ", lz4");
#endif
#ifdef ZSTD_LZMACOMPRESS
DISPLAYLEVEL(3, ", lzma, xz ");
#endif
DISPLAYLEVEL(3, "\n");
/* posix support */
#ifdef _POSIX_C_SOURCE
DISPLAYLEVEL(4, "_POSIX_C_SOURCE defined: %ldL\n", (long) _POSIX_C_SOURCE);
#endif
#ifdef _POSIX_VERSION
DISPLAYLEVEL(4, "_POSIX_VERSION defined: %ldL \n", (long) _POSIX_VERSION);
#endif
#ifdef PLATFORM_POSIX_VERSION
DISPLAYLEVEL(4, "PLATFORM_POSIX_VERSION defined: %ldL\n", (long) PLATFORM_POSIX_VERSION);
#endif
}
typedef enum { zom_compress, zom_decompress, zom_test, zom_bench, zom_train, zom_list } zstd_operation_mode;
@@ -367,7 +396,7 @@ int main(int argCount, const char* argv[])
/* init */
(void)recursive; (void)cLevelLast; /* not used when ZSTD_NOBENCH set */
(void)dictCLevel; (void)dictSelect; (void)dictID; (void)maxDictSize; /* not used when ZSTD_NODICT set */
(void)ultra; (void)cLevel; /* not used when ZSTD_NOCOMPRESS set */
(void)ultra; (void)cLevel; (void)ldmFlag; /* not used when ZSTD_NOCOMPRESS set */
(void)memLimit; /* not used when ZSTD_NODECOMPRESS set */
if (filenameTable==NULL) { DISPLAY("zstd: %s \n", strerror(errno)); exit(1); }
filenameTable[0] = stdinmark;
@@ -420,6 +449,7 @@ int main(int argCount, const char* argv[])
if (!strcmp(argument, "--quiet")) { g_displayLevel--; continue; }
if (!strcmp(argument, "--stdout")) { forceStdout=1; outFileName=stdoutmark; g_displayLevel-=(g_displayLevel==2); continue; }
if (!strcmp(argument, "--ultra")) { ultra=1; continue; }
if (!strcmp(argument, "--long")) { ldmFlag = 1; continue; }
if (!strcmp(argument, "--check")) { FIO_setChecksumFlag(2); continue; }
if (!strcmp(argument, "--no-check")) { FIO_setChecksumFlag(0); continue; }
if (!strcmp(argument, "--sparse")) { FIO_setSparseWrite(2); continue; }
@@ -442,7 +472,6 @@ int main(int argCount, const char* argv[])
#ifdef ZSTD_LZ4COMPRESS
if (!strcmp(argument, "--format=lz4")) { suffix = LZ4_EXTENSION; FIO_setCompressionType(FIO_lz4Compression); continue; }
#endif
if (!strcmp(argument, "--long")) { ldmFlag = 1; continue; }
/* long commands with arguments */
#ifndef ZSTD_NODICT
@@ -495,7 +524,7 @@ int main(int argCount, const char* argv[])
switch(argument[0])
{
/* Display help */
case 'V': g_displayOut=stdout; DISPLAY(WELCOME_MESSAGE); CLEAN_RETURN(0); /* Version Only */
case 'V': g_displayOut=stdout; printVersion(); CLEAN_RETURN(0); /* Version Only */
case 'H':
case 'h': g_displayOut=stdout; CLEAN_RETURN(usage_advanced(programName));
@@ -638,24 +667,18 @@ int main(int argCount, const char* argv[])
filenameTable[filenameIdx++] = argument;
}
if (lastCommand) { DISPLAY("error : command must be followed by argument \n"); CLEAN_RETURN(1); } /* forgotten argument */
if (lastCommand) { /* forgotten argument */
DISPLAY("error : command must be followed by argument \n");
CLEAN_RETURN(1);
}
/* Welcome message (if verbose) */
DISPLAYLEVEL(3, WELCOME_MESSAGE);
#ifdef _POSIX_C_SOURCE
DISPLAYLEVEL(4, "_POSIX_C_SOURCE defined: %ldL\n", (long) _POSIX_C_SOURCE);
#endif
#ifdef _POSIX_VERSION
DISPLAYLEVEL(4, "_POSIX_VERSION defined: %ldL\n", (long) _POSIX_VERSION);
#endif
#ifdef PLATFORM_POSIX_VERSION
DISPLAYLEVEL(4, "PLATFORM_POSIX_VERSION defined: %ldL\n", (long) PLATFORM_POSIX_VERSION);
#endif
if (nbThreads == 0) {
/* try to guess */
nbThreads = UTIL_countPhysicalCores();
DISPLAYLEVEL(3, "Note: %d physical core(s) detected\n", nbThreads);
DISPLAYLEVEL(3, "Note: %d physical core(s) detected \n", nbThreads);
}
g_utilDisplayLevel = g_displayLevel;
@@ -682,10 +705,17 @@ int main(int argCount, const char* argv[])
}
}
#endif
if (operation == zom_list) {
#ifndef ZSTD_NODECOMPRESS
int const ret = FIO_listMultipleFiles(filenameIdx, filenameTable, g_displayLevel);
CLEAN_RETURN(ret);
#else
DISPLAY("file information is not supported \n");
CLEAN_RETURN(1);
#endif
}
/* Check if benchmark is selected */
if (operation==zom_bench) {
#ifndef ZSTD_NOBENCH
@@ -696,7 +726,7 @@ int main(int argCount, const char* argv[])
BMK_setLdmFlag(ldmFlag);
BMK_benchFiles(filenameTable, filenameIdx, dictFileName, cLevel, cLevelLast, &compressionParams, setRealTimePrio);
#endif
(void)bench_nbSeconds;
(void)bench_nbSeconds; (void)blockSize; (void)setRealTimePrio;
goto _end;
}
@@ -768,6 +798,7 @@ int main(int argCount, const char* argv[])
else
operationResult = FIO_compressMultipleFilenames(filenameTable, filenameIdx, outFileName ? outFileName : suffix, dictFileName, cLevel, &compressionParams);
#else
(void)suffix;
DISPLAY("Compression not supported\n");
#endif
} else { /* decompression or test */