From 2238312c2f5f593a938c5410f9396a99254eb38a Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 2 Dec 2016 11:36:11 -0800 Subject: [PATCH 1/6] fix dict loading --- NEWS | 3 ++- lib/decompress/huf_decompress.c | 4 +++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/NEWS b/NEWS index 79ceec771..eaae710ce 100644 --- a/NEWS +++ b/NEWS @@ -1,8 +1,9 @@ v1.1.2 Improved : faster decompression speed at ultra compression settings and in 32-bits mode cli : new : preserve file attributes, by Przemyslaw Skibinski -cli : fixed : status displays total amount decoded when stream/file consists of multiple appended frames (like pzstd) +cli : fixed : status displays total amount decoded, even for file consisting of multiple frames (like pzstd) API : changed : zbuff prototypes now generate deprecation warnings +Changed : reduced stack memory use v1.1.1 New : command -M#, --memory=, --memlimit=, --memlimit-decompress= to limit allowed memory consumption diff --git a/lib/decompress/huf_decompress.c b/lib/decompress/huf_decompress.c index d212dd88e..a342dfb1e 100644 --- a/lib/decompress/huf_decompress.c +++ b/lib/decompress/huf_decompress.c @@ -358,13 +358,15 @@ typedef struct { U16 sequence; BYTE nbBits; BYTE length; } HUF_DEltX4; /* doubl typedef struct { BYTE symbol; BYTE weight; } sortedSymbol_t; +/* HUF_fillDTableX4Level2() : + * `rankValOrigin` must be a table of at least (HUF_TABLELOG_MAX + 1) U32 */ static void HUF_fillDTableX4Level2(HUF_DEltX4* DTable, U32 sizeLog, const U32 consumed, const U32* rankValOrigin, const int minWeight, const sortedSymbol_t* sortedSymbols, const U32 sortedListSize, U32 nbBitsBaseline, U16 baseSeq) { HUF_DEltX4 DElt; - U32 rankVal[HUF_TABLELOG_ABSOLUTEMAX + 1]; + U32 rankVal[HUF_TABLELOG_MAX + 1]; /* get pre-calculated rankVal */ memcpy(rankVal, rankValOrigin, sizeof(rankVal)); From 5bd4237beb789b6b1eeaea62a5db364a0272d80e Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 2 Dec 2016 12:40:57 -0800 Subject: [PATCH 2/6] minor refactor --- programs/Makefile | 22 ++++++++--------- programs/fileio.c | 60 ++++++++++++++++++++++------------------------- 2 files changed, 39 insertions(+), 43 deletions(-) diff --git a/programs/Makefile b/programs/Makefile index e44f11264..f0727b0e6 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -27,13 +27,13 @@ else ALIGN_LOOP = endif -CPPFLAGS= -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/dictBuilder -CFLAGS ?= -O3 -CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wstrict-aliasing=1 \ +CPPFLAGS+= -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/dictBuilder +CFLAGS ?= -O3 +CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wstrict-aliasing=1 \ -Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes -Wundef \ -Wpointer-arith -CFLAGS += $(MOREFLAGS) -FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) +CFLAGS += $(MOREFLAGS) +FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) ZSTDCOMMON_FILES := $(ZSTDDIR)/common/*.c @@ -82,7 +82,7 @@ zstd : $(ZSTDDECOMP_O) $(ZSTD_FILES) $(ZSTDLEGACY_FILES) $(ZDICT_FILES) \ ifneq (,$(filter Windows%,$(OS))) windres\generate_res.bat endif - $(CC) $(FLAGS) $^ $(RES_FILE) -o $@$(EXT) $(LDFLAGS) + $(CC) $(FLAGS) $^ $(RES_FILE) -o $@$(EXT) $(LDFLAGS) zstd32 : CPPFLAGS += -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT) @@ -91,7 +91,7 @@ zstd32 : $(ZSTDDIR)/decompress/zstd_decompress.c $(ZSTD_FILES) $(ZSTDLEGACY_FILE ifneq (,$(filter Windows%,$(OS))) windres\generate_res.bat endif - $(CC) -m32 $(FLAGS) $^ $(RES32_FILE) -o $@$(EXT) + $(CC) -m32 $(FLAGS) $^ $(RES32_FILE) -o $@$(EXT) zstd-nolegacy : clean_decomp_o @@ -110,23 +110,23 @@ zstd-pgo : clean zstd $(MAKE) zstd MOREFLAGS=-fprofile-use zstd-frugal: $(ZSTDDECOMP_O) $(ZSTD_FILES) zstdcli.c fileio.c - $(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT $^ -o zstd$(EXT) + $(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT $^ -o zstd$(EXT) zstd-small: clean_decomp_o ZSTD_LEGACY_SUPPORT=0 CFLAGS="-Os -s" $(MAKE) zstd-frugal zstd-decompress-clean: $(ZSTDDECOMP_O) $(ZSTDCOMMON_FILES) $(ZSTDDECOMP_FILES) zstdcli.c fileio.c - $(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_NOCOMPRESS $^ -o zstd-decompress$(EXT) + $(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_NOCOMPRESS $^ -o zstd-decompress$(EXT) zstd-decompress: clean_decomp_o ZSTD_LEGACY_SUPPORT=0 $(MAKE) zstd-decompress-clean zstd-compress: $(ZSTDCOMMON_FILES) $(ZSTDCOMP_FILES) zstdcli.c fileio.c - $(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_NODECOMPRESS $^ -o $@$(EXT) + $(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_NODECOMPRESS $^ -o $@$(EXT) gzstd: clean_decomp_o ifeq ($(shell ld -lz 2>/dev/null && echo -n true),true) - $(MAKE) zstd MOREFLAGS=-DZSTD_GZDECOMPRESS LDFLAGS="-lz" + CPPFLAGS=-DZSTD_GZDECOMPRESS LDFLAGS="-lz" $(MAKE) zstd else $(MAKE) zstd endif diff --git a/programs/fileio.c b/programs/fileio.c index 3e30c1e5e..446799af8 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -7,7 +7,6 @@ * of patent rights can be found in the PATENTS file in the same directory. */ - /* ************************************* * Compiler Options ***************************************/ @@ -587,7 +586,7 @@ static void FIO_fwriteSparseEnd(FILE* file, unsigned storedSkips) @return : size of decoded frame */ unsigned long long FIO_decompressFrame(dRess_t ress, - FILE* foutput, FILE* finput, size_t alreadyLoaded, + FILE* finput, size_t alreadyLoaded, U64 alreadyDecoded) { U64 frameSize = 0; @@ -610,7 +609,7 @@ unsigned long long FIO_decompressFrame(dRess_t ress, if (ZSTD_isError(readSizeHint)) EXM_THROW(36, "Decoding error : %s", ZSTD_getErrorName(readSizeHint)); /* Write block */ - storedSkips = FIO_fwriteSparse(foutput, ress.dstBuffer, outBuff.pos, storedSkips); + storedSkips = FIO_fwriteSparse(ress.dstFile, ress.dstBuffer, outBuff.pos, storedSkips); frameSize += outBuff.pos; DISPLAYUPDATE(2, "\rDecoded : %u MB... ", (U32)((alreadyDecoded+frameSize)>>20) ); @@ -623,7 +622,7 @@ unsigned long long FIO_decompressFrame(dRess_t ress, if (readSize < toRead) EXM_THROW(39, "Read error : premature end"); } } - FIO_fwriteSparseEnd(foutput, storedSkips); + FIO_fwriteSparseEnd(ress.dstFile, storedSkips); return frameSize; } @@ -650,47 +649,45 @@ static unsigned FIO_passThrough(FILE* foutput, FILE* finput, void* buffer, size_ return 0; } - #ifdef ZSTD_GZDECOMPRESS -static size_t FIO_decompressGzFile(dRess_t ress, size_t headBufSize, const char* srcFileName, FILE* srcFile) +static unsigned long long FIO_decompressGzFrame(dRess_t ress, FILE* srcFile, const char* srcFileName, size_t alreadyLoaded) { - int ret; - unsigned char in[1]; unsigned char* headBuf = (unsigned char*)ress.srcBuffer; - size_t decompBytes, outFileSize = 0; + unsigned long long outFileSize = 0; z_stream strm; - + strm.zalloc = Z_NULL; strm.zfree = Z_NULL; strm.opaque = Z_NULL; strm.next_in = 0; strm.avail_in = Z_NULL; - if (inflateInit2(&strm, 15 + 16) != Z_OK) return 0; - + if (inflateInit2(&strm, 15 /* maxWindowLogSize */ + 16 /* gzip only */) != Z_OK) return 0; /* see http://www.zlib.net/manual.html */ + strm.next_out = ress.dstBuffer; strm.avail_out = ress.dstBufferSize; for ( ; ; ) { - if (headBufSize) { - headBufSize--; + unsigned char in[1]; + if (alreadyLoaded) { + alreadyLoaded--; in[0] = *headBuf++; } else { if (fread(in, 1, 1, srcFile) == 0) break; } strm.next_in = in; strm.avail_in = 1; - ret = inflate(&strm, Z_NO_FLUSH); - if (ret == Z_STREAM_END) break; - if (ret != Z_OK) { DISPLAY("zstd: %s: inflate error %d \n", srcFileName, ret); return 0; } - - decompBytes = ress.dstBufferSize - strm.avail_out; - if (decompBytes) { - if (fwrite(ress.dstBuffer, 1, decompBytes, ress.dstFile) != (size_t)decompBytes) EXM_THROW(31, "Write error : cannot write to output file"); - outFileSize += decompBytes; - strm.next_out = ress.dstBuffer; - strm.avail_out = ress.dstBufferSize; + { int const ret = inflate(&strm, Z_NO_FLUSH); + if (ret == Z_STREAM_END) break; + if (ret != Z_OK) { DISPLAY("zstd: %s: inflate error %d \n", srcFileName, ret); return 0; } } - } + + { size_t const decompBytes = ress.dstBufferSize - strm.avail_out; + if (decompBytes) { + if (fwrite(ress.dstBuffer, 1, decompBytes, ress.dstFile) != decompBytes) EXM_THROW(31, "Write error : cannot write to output file"); + outFileSize += decompBytes; + strm.next_out = ress.dstBuffer; + strm.avail_out = ress.dstBufferSize; + } } } inflateEnd(&strm); return outFileSize; @@ -705,10 +702,9 @@ static size_t FIO_decompressGzFile(dRess_t ress, size_t headBufSize, const char* */ static int FIO_decompressSrcFile(dRess_t ress, const char* srcFileName) { - unsigned long long filesize = 0; - FILE* const dstFile = ress.dstFile; FILE* srcFile; unsigned readSomething = 0; + unsigned long long filesize = 0; if (UTIL_isDirectory(srcFileName)) { DISPLAYLEVEL(1, "zstd: %s is a directory -- ignored \n", srcFileName); @@ -732,17 +728,17 @@ static int FIO_decompressSrcFile(dRess_t ress, const char* srcFileName) if (sizeCheck != toRead) { DISPLAY("zstd: %s: unknown header \n", srcFileName); fclose(srcFile); return 1; } /* srcFileName is empty */ if (buf[0] == 31 && buf[1] == 139) { /* gz header */ #ifdef ZSTD_GZDECOMPRESS - size_t const result = FIO_decompressGzFile(ress, toRead, srcFileName, srcFile); + unsigned long long const result = FIO_decompressGzFrame(ress, srcFile, srcFileName, toRead); if (result == 0) return 1; filesize += result; #else - DISPLAYLEVEL(1, "zstd: %s: gzip file cannot be uncompressed -- ignored (zstd compiled without ZSTD_GZDECOMPRESS) \n", srcFileName); + DISPLAYLEVEL(1, "zstd: %s: gzip file cannot be uncompressed (zstd compiled without ZSTD_GZDECOMPRESS) -- ignored \n", srcFileName); return 1; #endif } else { if (!ZSTD_isFrame(ress.srcBuffer, toRead)) { if ((g_overwrite) && !strcmp (srcFileName, stdinmark)) { /* pass-through mode */ - unsigned const result = FIO_passThrough(dstFile, srcFile, ress.srcBuffer, ress.srcBufferSize); + unsigned const result = FIO_passThrough(ress.dstFile, srcFile, ress.srcBuffer, ress.srcBufferSize); if (fclose(srcFile)) EXM_THROW(32, "zstd: %s close error", srcFileName); /* error should never happen */ return result; } else { @@ -750,7 +746,7 @@ static int FIO_decompressSrcFile(dRess_t ress, const char* srcFileName) fclose(srcFile); return 1; } } - filesize += FIO_decompressFrame(ress, dstFile, srcFile, toRead, filesize); + filesize += FIO_decompressFrame(ress, srcFile, toRead, filesize); } } @@ -771,7 +767,7 @@ static int FIO_decompressSrcFile(dRess_t ress, const char* srcFileName) 1 : operation aborted (src not available, dst already taken, etc.) */ static int FIO_decompressDstFile(dRess_t ress, - const char* dstFileName, const char* srcFileName) + const char* dstFileName, const char* srcFileName) { int result; stat_t statbuf; From 743b33f57eb3fdd9f3dee1a056db4d323cf30b77 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 2 Dec 2016 15:18:57 -0800 Subject: [PATCH 3/6] fix zstdcat --- NEWS | 4 ++- programs/fileio.c | 10 +++--- programs/fileio.h | 2 +- programs/zstd.1 | 2 +- programs/zstdcli.c | 79 +++++++++++++++++++++++++--------------------- tests/playTests.sh | 6 ++-- 6 files changed, 57 insertions(+), 46 deletions(-) diff --git a/NEWS b/NEWS index eaae710ce..0b3dd92a5 100644 --- a/NEWS +++ b/NEWS @@ -1,7 +1,9 @@ v1.1.2 Improved : faster decompression speed at ultra compression settings and in 32-bits mode -cli : new : preserve file attributes, by Przemyslaw Skibinski +cli : new : gzstd, experimental version able to decode .gz files, by Przemyslaw Skibinski +cli : new : preserve file attributes cli : fixed : status displays total amount decoded, even for file consisting of multiple frames (like pzstd) +cli : fixed : zstdcat API : changed : zbuff prototypes now generate deprecation warnings Changed : reduced stack memory use diff --git a/programs/fileio.c b/programs/fileio.c index 446799af8..dff4eae05 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -700,7 +700,7 @@ static unsigned long long FIO_decompressGzFrame(dRess_t ress, FILE* srcFile, con @return : 0 : OK 1 : operation not started */ -static int FIO_decompressSrcFile(dRess_t ress, const char* srcFileName) +static int FIO_decompressSrcFile(dRess_t ress, const char* dstFileName, const char* srcFileName) { FILE* srcFile; unsigned readSomething = 0; @@ -737,7 +737,7 @@ static int FIO_decompressSrcFile(dRess_t ress, const char* srcFileName) #endif } else { if (!ZSTD_isFrame(ress.srcBuffer, toRead)) { - if ((g_overwrite) && !strcmp (srcFileName, stdinmark)) { /* pass-through mode */ + if ((g_overwrite) && !strcmp (dstFileName, stdoutmark)) { /* pass-through mode */ unsigned const result = FIO_passThrough(ress.dstFile, srcFile, ress.srcBuffer, ress.srcBufferSize); if (fclose(srcFile)) EXM_THROW(32, "zstd: %s close error", srcFileName); /* error should never happen */ return result; @@ -777,7 +777,7 @@ static int FIO_decompressDstFile(dRess_t ress, if (ress.dstFile==0) return 1; if (strcmp (srcFileName, stdinmark) && UTIL_getFileStat(srcFileName, &statbuf)) stat_result = 1; - result = FIO_decompressSrcFile(ress, srcFileName); + result = FIO_decompressSrcFile(ress, dstFileName, srcFileName); if (fclose(ress.dstFile)) EXM_THROW(38, "Write error : cannot properly close %s", dstFileName); @@ -814,12 +814,12 @@ int FIO_decompressMultipleFilenames(const char** srcNamesTable, unsigned nbFiles if (suffix==NULL) EXM_THROW(70, "zstd: decompression: unknown dst"); /* should never happen */ - if (!strcmp(suffix, stdoutmark) || !strcmp(suffix, nulmark)) { + if (!strcmp(suffix, stdoutmark) || !strcmp(suffix, nulmark)) { /* special cases : -c or -t */ unsigned u; ress.dstFile = FIO_openDstFile(suffix); if (ress.dstFile == 0) EXM_THROW(71, "cannot open %s", suffix); for (u=0; u tmp1 +$ZSTD -dcf tmp1 $ECHO "\n**** frame concatenation **** " From efaf104b2d2c29dedfb853c7b318a88755e921a6 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 2 Dec 2016 15:24:40 -0800 Subject: [PATCH 4/6] added zstdless --- NEWS | 1 + programs/Makefile | 9 +++++---- programs/zstdless | 1 + 3 files changed, 7 insertions(+), 4 deletions(-) create mode 100755 programs/zstdless diff --git a/NEWS b/NEWS index 0b3dd92a5..69b24e635 100644 --- a/NEWS +++ b/NEWS @@ -2,6 +2,7 @@ v1.1.2 Improved : faster decompression speed at ultra compression settings and in 32-bits mode cli : new : gzstd, experimental version able to decode .gz files, by Przemyslaw Skibinski cli : new : preserve file attributes +cli : new : added zstdless cli : fixed : status displays total amount decoded, even for file consisting of multiple frames (like pzstd) cli : fixed : zstdcat API : changed : zbuff prototypes now generate deprecation warnings diff --git a/programs/Makefile b/programs/Makefile index f0727b0e6..c78c369e2 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -153,9 +153,10 @@ ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD Dr install: zstd @echo Installing binaries @install -d -m 755 $(DESTDIR)$(BINDIR)/ $(DESTDIR)$(MANDIR)/ - @install -m 755 zstd$(EXT) $(DESTDIR)$(BINDIR)/zstd$(EXT) - @ln -sf zstd$(EXT) $(DESTDIR)$(BINDIR)/zstdcat - @ln -sf zstd$(EXT) $(DESTDIR)$(BINDIR)/unzstd + @install -m 755 zstd $(DESTDIR)$(BINDIR)/zstd + @ln -sf zstd $(DESTDIR)$(BINDIR)/zstdcat + @ln -sf zstd $(DESTDIR)$(BINDIR)/unzstd + @install -m 755 zstdless $(DESTDIR)$(BINDIR)/zstdless @echo Installing man pages @install -m 644 zstd.1 $(DESTDIR)$(MANDIR)/zstd.1 @ln -sf zstd.1 $(DESTDIR)$(MANDIR)/zstdcat.1 @@ -165,7 +166,7 @@ install: zstd uninstall: @$(RM) $(DESTDIR)$(BINDIR)/zstdcat @$(RM) $(DESTDIR)$(BINDIR)/unzstd - @$(RM) $(DESTDIR)$(BINDIR)/zstd$(EXT) + @$(RM) $(DESTDIR)$(BINDIR)/zstd @$(RM) $(DESTDIR)$(MANDIR)/zstdcat.1 @$(RM) $(DESTDIR)$(MANDIR)/unzstd.1 @$(RM) $(DESTDIR)$(MANDIR)/zstd.1 diff --git a/programs/zstdless b/programs/zstdless new file mode 100755 index 000000000..ab021405c --- /dev/null +++ b/programs/zstdless @@ -0,0 +1 @@ +zstdcat $@ | less From db85a6e09a7f82ca090ab00130c99c65465ec1fd Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 2 Dec 2016 15:57:07 -0800 Subject: [PATCH 5/6] added zstdgrep --- programs/Makefile | 3 ++ programs/zstdgrep | 124 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 127 insertions(+) create mode 100755 programs/zstdgrep diff --git a/programs/Makefile b/programs/Makefile index c78c369e2..44eafbf63 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -157,6 +157,7 @@ install: zstd @ln -sf zstd $(DESTDIR)$(BINDIR)/zstdcat @ln -sf zstd $(DESTDIR)$(BINDIR)/unzstd @install -m 755 zstdless $(DESTDIR)$(BINDIR)/zstdless + @install -m 755 zstdgrep $(DESTDIR)$(BINDIR)/zstdgrep @echo Installing man pages @install -m 644 zstd.1 $(DESTDIR)$(MANDIR)/zstd.1 @ln -sf zstd.1 $(DESTDIR)$(MANDIR)/zstdcat.1 @@ -164,6 +165,8 @@ install: zstd @echo zstd installation completed uninstall: + @$(RM) $(DESTDIR)$(BINDIR)/zstdgrep + @$(RM) $(DESTDIR)$(BINDIR)/zstdless @$(RM) $(DESTDIR)$(BINDIR)/zstdcat @$(RM) $(DESTDIR)$(BINDIR)/unzstd @$(RM) $(DESTDIR)$(BINDIR)/zstd diff --git a/programs/zstdgrep b/programs/zstdgrep new file mode 100755 index 000000000..9f871c03f --- /dev/null +++ b/programs/zstdgrep @@ -0,0 +1,124 @@ +#!/bin/sh +# +# Copyright (c) 2003 Thomas Klausner. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR +# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES +# OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. +# IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, +# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT +# NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +grep=grep +zcat=zstdcat + +endofopts=0 +pattern_found=0 +grep_args="" +hyphen=0 +silent=0 + +prg=$(basename $0) + +# handle being called 'zegrep' or 'zfgrep' +case ${prg} in + *zegrep) + grep_args="-E";; + *zfgrep) + grep_args="-F";; +esac + +# skip all options and pass them on to grep taking care of options +# with arguments, and if -e was supplied + +while [ $# -gt 0 -a ${endofopts} -eq 0 ] +do + case $1 in + # from GNU grep-2.5.1 -- keep in sync! + -[ABCDXdefm]) + if [ $# -lt 2 ] + then + echo "${prg}: missing argument for $1 flag" >&2 + exit 1 + fi + case $1 in + -e) + pattern="$2" + pattern_found=1 + shift 2 + break + ;; + *) + ;; + esac + grep_args="${grep_args} $1 $2" + shift 2 + ;; + --) + shift + endofopts=1 + ;; + -) + hyphen=1 + shift + ;; + -h) + silent=1 + shift + ;; + -*) + grep_args="${grep_args} $1" + shift + ;; + *) + # pattern to grep for + endofopts=1 + ;; + esac +done + +# if no -e option was found, take next argument as grep-pattern +if [ ${pattern_found} -lt 1 ] +then + if [ $# -ge 1 ]; then + pattern="$1" + shift + elif [ ${hyphen} -gt 0 ]; then + pattern="-" + else + echo "${prg}: missing pattern" >&2 + exit 1 + fi +fi + +# call grep ... +if [ $# -lt 1 ] +then + # ... on stdin + ${zcat} -fq - | ${grep} ${grep_args} -- "${pattern}" - +else + # ... on all files given on the command line + if [ ${silent} -lt 1 -a $# -gt 1 ]; then + grep_args="-H ${grep_args}" + fi + while [ $# -gt 0 ] + do + ${zcat} -fq -- "$1" | ${grep} --label="${1}" ${grep_args} -- "${pattern}" - + shift + done +fi + +exit 0 From 9ffbeea87507215a8c6846f179325ecb4662b5d6 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 2 Dec 2016 18:37:38 -0800 Subject: [PATCH 6/6] API : changed : streaming decompression : implicit reset on starting new frames --- NEWS | 3 ++- lib/decompress/zstd_decompress.c | 3 ++- lib/zstd.h | 4 ++-- tests/zstreamtest.c | 19 +++++++++++++++---- 4 files changed, 21 insertions(+), 8 deletions(-) diff --git a/NEWS b/NEWS index 69b24e635..580889542 100644 --- a/NEWS +++ b/NEWS @@ -2,10 +2,11 @@ v1.1.2 Improved : faster decompression speed at ultra compression settings and in 32-bits mode cli : new : gzstd, experimental version able to decode .gz files, by Przemyslaw Skibinski cli : new : preserve file attributes -cli : new : added zstdless +cli : new : added zstdless and zstdgrep tools cli : fixed : status displays total amount decoded, even for file consisting of multiple frames (like pzstd) cli : fixed : zstdcat API : changed : zbuff prototypes now generate deprecation warnings +API : changed : streaming decompression implicit reset on starting new frame Changed : reduced stack memory use v1.1.1 diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index 3106fa014..7c4f54bd3 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -1959,7 +1959,8 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB switch(zds->stage) { case zdss_init : - return ERROR(init_missing); + ZSTD_resetDStream(zds); /* transparent reset on starting decoding a new frame */ + /* fall-through */ case zdss_loadHeader : { size_t const hSize = ZSTD_getFrameParams(&zds->fParams, zds->headerBuffer, zds->lhSize); diff --git a/lib/zstd.h b/lib/zstd.h index 607612026..b7a8bb989 100644 --- a/lib/zstd.h +++ b/lib/zstd.h @@ -299,8 +299,8 @@ ZSTDLIB_API size_t ZSTD_CStreamOutSize(void); /**< recommended size for output * If `output.pos < output.size`, decoder has flushed everything it could. * @return : 0 when a frame is completely decoded and fully flushed, * an error code, which can be tested using ZSTD_isError(), -* any other value > 0, which means there is still some work to do to complete the frame. -* The return value is a suggested next input size (just an hint, to help latency). +* any other value > 0, which means there is still some decoding to do to complete current frame. +* The return value is a suggested next input size (a hint to improve latency) that will never load more than the current frame. * *******************************************************************************/ /*===== Streaming decompression functions =====*/ diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c index aa119e636..c21b9de9b 100644 --- a/tests/zstreamtest.c +++ b/tests/zstreamtest.c @@ -130,7 +130,7 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo U32 testNb=0; ZSTD_CStream* zc = ZSTD_createCStream_advanced(customMem); ZSTD_DStream* zd = ZSTD_createDStream_advanced(customMem); - ZSTD_inBuffer inBuff; + ZSTD_inBuffer inBuff, inBuff2; ZSTD_outBuffer outBuff; /* Create compressible test buffer */ @@ -183,12 +183,22 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo DISPLAYLEVEL(4, "OK \n"); /* Basic decompression test */ + inBuff2 = inBuff; DISPLAYLEVEL(4, "test%3i : decompress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH); ZSTD_initDStream_usingDict(zd, CNBuffer, 128 KB); { size_t const r = ZSTD_setDStreamParameter(zd, ZSTDdsp_maxWindowSize, 1000000000); /* large limit */ if (ZSTD_isError(r)) goto _output_error; } - { size_t const r = ZSTD_decompressStream(zd, &outBuff, &inBuff); - if (r != 0) goto _output_error; } /* should reach end of frame == 0; otherwise, some data left, or an error */ + { size_t const remaining = ZSTD_decompressStream(zd, &outBuff, &inBuff); + if (remaining != 0) goto _output_error; } /* should reach end of frame == 0; otherwise, some data left, or an error */ + if (outBuff.pos != CNBufferSize) goto _output_error; /* should regenerate the same amount */ + if (inBuff.pos != inBuff.size) goto _output_error; /* should have read the entire frame */ + DISPLAYLEVEL(4, "OK \n"); + + /* Re-use without init */ + DISPLAYLEVEL(4, "test%3i : decompress again without init (re-use previous settings): ", testNb++); + outBuff.pos = 0; + { size_t const remaining = ZSTD_decompressStream(zd, &outBuff, &inBuff2); + if (remaining != 0) goto _output_error; } /* should reach end of frame == 0; otherwise, some data left, or an error */ if (outBuff.pos != CNBufferSize) goto _output_error; /* should regenerate the same amount */ if (inBuff.pos != inBuff.size) goto _output_error; /* should have read the entire frame */ DISPLAYLEVEL(4, "OK \n"); @@ -553,8 +563,9 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compres /* multi - fragments decompression test */ if (!dictSize /* don't reset if dictionary : could be different */ && (FUZ_rand(&lseed) & 1)) { CHECK (ZSTD_isError(ZSTD_resetDStream(zd)), "ZSTD_resetDStream failed"); - } else + } else { ZSTD_initDStream_usingDict(zd, dict, dictSize); + } { size_t decompressionResult = 1; ZSTD_inBuffer inBuff = { cBuffer, cSize, 0 }; ZSTD_outBuffer outBuff= { dstBuffer, dstBufferSize, 0 };