diff --git a/NEWS b/NEWS index 79ceec771..580889542 100644 --- a/NEWS +++ b/NEWS @@ -1,8 +1,13 @@ v1.1.2 Improved : faster decompression speed at ultra compression settings and in 32-bits mode -cli : new : preserve file attributes, by Przemyslaw Skibinski -cli : fixed : status displays total amount decoded when stream/file consists of multiple appended frames (like pzstd) +cli : new : gzstd, experimental version able to decode .gz files, by Przemyslaw Skibinski +cli : new : preserve file attributes +cli : new : added zstdless and zstdgrep tools +cli : fixed : status displays total amount decoded, even for file consisting of multiple frames (like pzstd) +cli : fixed : zstdcat API : changed : zbuff prototypes now generate deprecation warnings +API : changed : streaming decompression implicit reset on starting new frame +Changed : reduced stack memory use v1.1.1 New : command -M#, --memory=, --memlimit=, --memlimit-decompress= to limit allowed memory consumption diff --git a/lib/decompress/huf_decompress.c b/lib/decompress/huf_decompress.c index d212dd88e..a342dfb1e 100644 --- a/lib/decompress/huf_decompress.c +++ b/lib/decompress/huf_decompress.c @@ -358,13 +358,15 @@ typedef struct { U16 sequence; BYTE nbBits; BYTE length; } HUF_DEltX4; /* doubl typedef struct { BYTE symbol; BYTE weight; } sortedSymbol_t; +/* HUF_fillDTableX4Level2() : + * `rankValOrigin` must be a table of at least (HUF_TABLELOG_MAX + 1) U32 */ static void HUF_fillDTableX4Level2(HUF_DEltX4* DTable, U32 sizeLog, const U32 consumed, const U32* rankValOrigin, const int minWeight, const sortedSymbol_t* sortedSymbols, const U32 sortedListSize, U32 nbBitsBaseline, U16 baseSeq) { HUF_DEltX4 DElt; - U32 rankVal[HUF_TABLELOG_ABSOLUTEMAX + 1]; + U32 rankVal[HUF_TABLELOG_MAX + 1]; /* get pre-calculated rankVal */ memcpy(rankVal, rankValOrigin, sizeof(rankVal)); diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index 3106fa014..7c4f54bd3 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -1959,7 +1959,8 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB switch(zds->stage) { case zdss_init : - return ERROR(init_missing); + ZSTD_resetDStream(zds); /* transparent reset on starting decoding a new frame */ + /* fall-through */ case zdss_loadHeader : { size_t const hSize = ZSTD_getFrameParams(&zds->fParams, zds->headerBuffer, zds->lhSize); diff --git a/lib/zstd.h b/lib/zstd.h index 607612026..b7a8bb989 100644 --- a/lib/zstd.h +++ b/lib/zstd.h @@ -299,8 +299,8 @@ ZSTDLIB_API size_t ZSTD_CStreamOutSize(void); /**< recommended size for output * If `output.pos < output.size`, decoder has flushed everything it could. * @return : 0 when a frame is completely decoded and fully flushed, * an error code, which can be tested using ZSTD_isError(), -* any other value > 0, which means there is still some work to do to complete the frame. -* The return value is a suggested next input size (just an hint, to help latency). +* any other value > 0, which means there is still some decoding to do to complete current frame. +* The return value is a suggested next input size (a hint to improve latency) that will never load more than the current frame. * *******************************************************************************/ /*===== Streaming decompression functions =====*/ diff --git a/programs/Makefile b/programs/Makefile index e44f11264..44eafbf63 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -27,13 +27,13 @@ else ALIGN_LOOP = endif -CPPFLAGS= -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/dictBuilder -CFLAGS ?= -O3 -CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wstrict-aliasing=1 \ +CPPFLAGS+= -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/dictBuilder +CFLAGS ?= -O3 +CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wstrict-aliasing=1 \ -Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes -Wundef \ -Wpointer-arith -CFLAGS += $(MOREFLAGS) -FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) +CFLAGS += $(MOREFLAGS) +FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) ZSTDCOMMON_FILES := $(ZSTDDIR)/common/*.c @@ -82,7 +82,7 @@ zstd : $(ZSTDDECOMP_O) $(ZSTD_FILES) $(ZSTDLEGACY_FILES) $(ZDICT_FILES) \ ifneq (,$(filter Windows%,$(OS))) windres\generate_res.bat endif - $(CC) $(FLAGS) $^ $(RES_FILE) -o $@$(EXT) $(LDFLAGS) + $(CC) $(FLAGS) $^ $(RES_FILE) -o $@$(EXT) $(LDFLAGS) zstd32 : CPPFLAGS += -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT) @@ -91,7 +91,7 @@ zstd32 : $(ZSTDDIR)/decompress/zstd_decompress.c $(ZSTD_FILES) $(ZSTDLEGACY_FILE ifneq (,$(filter Windows%,$(OS))) windres\generate_res.bat endif - $(CC) -m32 $(FLAGS) $^ $(RES32_FILE) -o $@$(EXT) + $(CC) -m32 $(FLAGS) $^ $(RES32_FILE) -o $@$(EXT) zstd-nolegacy : clean_decomp_o @@ -110,23 +110,23 @@ zstd-pgo : clean zstd $(MAKE) zstd MOREFLAGS=-fprofile-use zstd-frugal: $(ZSTDDECOMP_O) $(ZSTD_FILES) zstdcli.c fileio.c - $(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT $^ -o zstd$(EXT) + $(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT $^ -o zstd$(EXT) zstd-small: clean_decomp_o ZSTD_LEGACY_SUPPORT=0 CFLAGS="-Os -s" $(MAKE) zstd-frugal zstd-decompress-clean: $(ZSTDDECOMP_O) $(ZSTDCOMMON_FILES) $(ZSTDDECOMP_FILES) zstdcli.c fileio.c - $(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_NOCOMPRESS $^ -o zstd-decompress$(EXT) + $(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_NOCOMPRESS $^ -o zstd-decompress$(EXT) zstd-decompress: clean_decomp_o ZSTD_LEGACY_SUPPORT=0 $(MAKE) zstd-decompress-clean zstd-compress: $(ZSTDCOMMON_FILES) $(ZSTDCOMP_FILES) zstdcli.c fileio.c - $(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_NODECOMPRESS $^ -o $@$(EXT) + $(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_NODECOMPRESS $^ -o $@$(EXT) gzstd: clean_decomp_o ifeq ($(shell ld -lz 2>/dev/null && echo -n true),true) - $(MAKE) zstd MOREFLAGS=-DZSTD_GZDECOMPRESS LDFLAGS="-lz" + CPPFLAGS=-DZSTD_GZDECOMPRESS LDFLAGS="-lz" $(MAKE) zstd else $(MAKE) zstd endif @@ -153,9 +153,11 @@ ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD Dr install: zstd @echo Installing binaries @install -d -m 755 $(DESTDIR)$(BINDIR)/ $(DESTDIR)$(MANDIR)/ - @install -m 755 zstd$(EXT) $(DESTDIR)$(BINDIR)/zstd$(EXT) - @ln -sf zstd$(EXT) $(DESTDIR)$(BINDIR)/zstdcat - @ln -sf zstd$(EXT) $(DESTDIR)$(BINDIR)/unzstd + @install -m 755 zstd $(DESTDIR)$(BINDIR)/zstd + @ln -sf zstd $(DESTDIR)$(BINDIR)/zstdcat + @ln -sf zstd $(DESTDIR)$(BINDIR)/unzstd + @install -m 755 zstdless $(DESTDIR)$(BINDIR)/zstdless + @install -m 755 zstdgrep $(DESTDIR)$(BINDIR)/zstdgrep @echo Installing man pages @install -m 644 zstd.1 $(DESTDIR)$(MANDIR)/zstd.1 @ln -sf zstd.1 $(DESTDIR)$(MANDIR)/zstdcat.1 @@ -163,9 +165,11 @@ install: zstd @echo zstd installation completed uninstall: + @$(RM) $(DESTDIR)$(BINDIR)/zstdgrep + @$(RM) $(DESTDIR)$(BINDIR)/zstdless @$(RM) $(DESTDIR)$(BINDIR)/zstdcat @$(RM) $(DESTDIR)$(BINDIR)/unzstd - @$(RM) $(DESTDIR)$(BINDIR)/zstd$(EXT) + @$(RM) $(DESTDIR)$(BINDIR)/zstd @$(RM) $(DESTDIR)$(MANDIR)/zstdcat.1 @$(RM) $(DESTDIR)$(MANDIR)/unzstd.1 @$(RM) $(DESTDIR)$(MANDIR)/zstd.1 diff --git a/programs/fileio.c b/programs/fileio.c index 3e30c1e5e..dff4eae05 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -7,7 +7,6 @@ * of patent rights can be found in the PATENTS file in the same directory. */ - /* ************************************* * Compiler Options ***************************************/ @@ -587,7 +586,7 @@ static void FIO_fwriteSparseEnd(FILE* file, unsigned storedSkips) @return : size of decoded frame */ unsigned long long FIO_decompressFrame(dRess_t ress, - FILE* foutput, FILE* finput, size_t alreadyLoaded, + FILE* finput, size_t alreadyLoaded, U64 alreadyDecoded) { U64 frameSize = 0; @@ -610,7 +609,7 @@ unsigned long long FIO_decompressFrame(dRess_t ress, if (ZSTD_isError(readSizeHint)) EXM_THROW(36, "Decoding error : %s", ZSTD_getErrorName(readSizeHint)); /* Write block */ - storedSkips = FIO_fwriteSparse(foutput, ress.dstBuffer, outBuff.pos, storedSkips); + storedSkips = FIO_fwriteSparse(ress.dstFile, ress.dstBuffer, outBuff.pos, storedSkips); frameSize += outBuff.pos; DISPLAYUPDATE(2, "\rDecoded : %u MB... ", (U32)((alreadyDecoded+frameSize)>>20) ); @@ -623,7 +622,7 @@ unsigned long long FIO_decompressFrame(dRess_t ress, if (readSize < toRead) EXM_THROW(39, "Read error : premature end"); } } - FIO_fwriteSparseEnd(foutput, storedSkips); + FIO_fwriteSparseEnd(ress.dstFile, storedSkips); return frameSize; } @@ -650,47 +649,45 @@ static unsigned FIO_passThrough(FILE* foutput, FILE* finput, void* buffer, size_ return 0; } - #ifdef ZSTD_GZDECOMPRESS -static size_t FIO_decompressGzFile(dRess_t ress, size_t headBufSize, const char* srcFileName, FILE* srcFile) +static unsigned long long FIO_decompressGzFrame(dRess_t ress, FILE* srcFile, const char* srcFileName, size_t alreadyLoaded) { - int ret; - unsigned char in[1]; unsigned char* headBuf = (unsigned char*)ress.srcBuffer; - size_t decompBytes, outFileSize = 0; + unsigned long long outFileSize = 0; z_stream strm; - + strm.zalloc = Z_NULL; strm.zfree = Z_NULL; strm.opaque = Z_NULL; strm.next_in = 0; strm.avail_in = Z_NULL; - if (inflateInit2(&strm, 15 + 16) != Z_OK) return 0; - + if (inflateInit2(&strm, 15 /* maxWindowLogSize */ + 16 /* gzip only */) != Z_OK) return 0; /* see http://www.zlib.net/manual.html */ + strm.next_out = ress.dstBuffer; strm.avail_out = ress.dstBufferSize; for ( ; ; ) { - if (headBufSize) { - headBufSize--; + unsigned char in[1]; + if (alreadyLoaded) { + alreadyLoaded--; in[0] = *headBuf++; } else { if (fread(in, 1, 1, srcFile) == 0) break; } strm.next_in = in; strm.avail_in = 1; - ret = inflate(&strm, Z_NO_FLUSH); - if (ret == Z_STREAM_END) break; - if (ret != Z_OK) { DISPLAY("zstd: %s: inflate error %d \n", srcFileName, ret); return 0; } - - decompBytes = ress.dstBufferSize - strm.avail_out; - if (decompBytes) { - if (fwrite(ress.dstBuffer, 1, decompBytes, ress.dstFile) != (size_t)decompBytes) EXM_THROW(31, "Write error : cannot write to output file"); - outFileSize += decompBytes; - strm.next_out = ress.dstBuffer; - strm.avail_out = ress.dstBufferSize; + { int const ret = inflate(&strm, Z_NO_FLUSH); + if (ret == Z_STREAM_END) break; + if (ret != Z_OK) { DISPLAY("zstd: %s: inflate error %d \n", srcFileName, ret); return 0; } } - } + + { size_t const decompBytes = ress.dstBufferSize - strm.avail_out; + if (decompBytes) { + if (fwrite(ress.dstBuffer, 1, decompBytes, ress.dstFile) != decompBytes) EXM_THROW(31, "Write error : cannot write to output file"); + outFileSize += decompBytes; + strm.next_out = ress.dstBuffer; + strm.avail_out = ress.dstBufferSize; + } } } inflateEnd(&strm); return outFileSize; @@ -703,12 +700,11 @@ static size_t FIO_decompressGzFile(dRess_t ress, size_t headBufSize, const char* @return : 0 : OK 1 : operation not started */ -static int FIO_decompressSrcFile(dRess_t ress, const char* srcFileName) +static int FIO_decompressSrcFile(dRess_t ress, const char* dstFileName, const char* srcFileName) { - unsigned long long filesize = 0; - FILE* const dstFile = ress.dstFile; FILE* srcFile; unsigned readSomething = 0; + unsigned long long filesize = 0; if (UTIL_isDirectory(srcFileName)) { DISPLAYLEVEL(1, "zstd: %s is a directory -- ignored \n", srcFileName); @@ -732,17 +728,17 @@ static int FIO_decompressSrcFile(dRess_t ress, const char* srcFileName) if (sizeCheck != toRead) { DISPLAY("zstd: %s: unknown header \n", srcFileName); fclose(srcFile); return 1; } /* srcFileName is empty */ if (buf[0] == 31 && buf[1] == 139) { /* gz header */ #ifdef ZSTD_GZDECOMPRESS - size_t const result = FIO_decompressGzFile(ress, toRead, srcFileName, srcFile); + unsigned long long const result = FIO_decompressGzFrame(ress, srcFile, srcFileName, toRead); if (result == 0) return 1; filesize += result; #else - DISPLAYLEVEL(1, "zstd: %s: gzip file cannot be uncompressed -- ignored (zstd compiled without ZSTD_GZDECOMPRESS) \n", srcFileName); + DISPLAYLEVEL(1, "zstd: %s: gzip file cannot be uncompressed (zstd compiled without ZSTD_GZDECOMPRESS) -- ignored \n", srcFileName); return 1; #endif } else { if (!ZSTD_isFrame(ress.srcBuffer, toRead)) { - if ((g_overwrite) && !strcmp (srcFileName, stdinmark)) { /* pass-through mode */ - unsigned const result = FIO_passThrough(dstFile, srcFile, ress.srcBuffer, ress.srcBufferSize); + if ((g_overwrite) && !strcmp (dstFileName, stdoutmark)) { /* pass-through mode */ + unsigned const result = FIO_passThrough(ress.dstFile, srcFile, ress.srcBuffer, ress.srcBufferSize); if (fclose(srcFile)) EXM_THROW(32, "zstd: %s close error", srcFileName); /* error should never happen */ return result; } else { @@ -750,7 +746,7 @@ static int FIO_decompressSrcFile(dRess_t ress, const char* srcFileName) fclose(srcFile); return 1; } } - filesize += FIO_decompressFrame(ress, dstFile, srcFile, toRead, filesize); + filesize += FIO_decompressFrame(ress, srcFile, toRead, filesize); } } @@ -771,7 +767,7 @@ static int FIO_decompressSrcFile(dRess_t ress, const char* srcFileName) 1 : operation aborted (src not available, dst already taken, etc.) */ static int FIO_decompressDstFile(dRess_t ress, - const char* dstFileName, const char* srcFileName) + const char* dstFileName, const char* srcFileName) { int result; stat_t statbuf; @@ -781,7 +777,7 @@ static int FIO_decompressDstFile(dRess_t ress, if (ress.dstFile==0) return 1; if (strcmp (srcFileName, stdinmark) && UTIL_getFileStat(srcFileName, &statbuf)) stat_result = 1; - result = FIO_decompressSrcFile(ress, srcFileName); + result = FIO_decompressSrcFile(ress, dstFileName, srcFileName); if (fclose(ress.dstFile)) EXM_THROW(38, "Write error : cannot properly close %s", dstFileName); @@ -818,12 +814,12 @@ int FIO_decompressMultipleFilenames(const char** srcNamesTable, unsigned nbFiles if (suffix==NULL) EXM_THROW(70, "zstd: decompression: unknown dst"); /* should never happen */ - if (!strcmp(suffix, stdoutmark) || !strcmp(suffix, nulmark)) { + if (!strcmp(suffix, stdoutmark) || !strcmp(suffix, nulmark)) { /* special cases : -c or -t */ unsigned u; ress.dstFile = FIO_openDstFile(suffix); if (ress.dstFile == 0) EXM_THROW(71, "cannot open %s", suffix); for (u=0; u&2 + exit 1 + fi + case $1 in + -e) + pattern="$2" + pattern_found=1 + shift 2 + break + ;; + *) + ;; + esac + grep_args="${grep_args} $1 $2" + shift 2 + ;; + --) + shift + endofopts=1 + ;; + -) + hyphen=1 + shift + ;; + -h) + silent=1 + shift + ;; + -*) + grep_args="${grep_args} $1" + shift + ;; + *) + # pattern to grep for + endofopts=1 + ;; + esac +done + +# if no -e option was found, take next argument as grep-pattern +if [ ${pattern_found} -lt 1 ] +then + if [ $# -ge 1 ]; then + pattern="$1" + shift + elif [ ${hyphen} -gt 0 ]; then + pattern="-" + else + echo "${prg}: missing pattern" >&2 + exit 1 + fi +fi + +# call grep ... +if [ $# -lt 1 ] +then + # ... on stdin + ${zcat} -fq - | ${grep} ${grep_args} -- "${pattern}" - +else + # ... on all files given on the command line + if [ ${silent} -lt 1 -a $# -gt 1 ]; then + grep_args="-H ${grep_args}" + fi + while [ $# -gt 0 ] + do + ${zcat} -fq -- "$1" | ${grep} --label="${1}" ${grep_args} -- "${pattern}" - + shift + done +fi + +exit 0 diff --git a/programs/zstdless b/programs/zstdless new file mode 100755 index 000000000..ab021405c --- /dev/null +++ b/programs/zstdless @@ -0,0 +1 @@ +zstdcat $@ | less diff --git a/tests/playTests.sh b/tests/playTests.sh index c539d567f..dfca1de87 100755 --- a/tests/playTests.sh +++ b/tests/playTests.sh @@ -100,8 +100,10 @@ ls tmp.zst && die "tmp.zst should not be created" $ECHO "\n**** Pass-Through mode **** " -$ECHO "Hello world !" | $ZSTD -df -$ECHO "Hello world !" | $ZSTD -dcf +$ECHO "Hello world 1!" | $ZSTD -df +$ECHO "Hello world 2!" | $ZSTD -dcf +$ECHO "Hello world 3!" > tmp1 +$ZSTD -dcf tmp1 $ECHO "\n**** frame concatenation **** " diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c index aa119e636..c21b9de9b 100644 --- a/tests/zstreamtest.c +++ b/tests/zstreamtest.c @@ -130,7 +130,7 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo U32 testNb=0; ZSTD_CStream* zc = ZSTD_createCStream_advanced(customMem); ZSTD_DStream* zd = ZSTD_createDStream_advanced(customMem); - ZSTD_inBuffer inBuff; + ZSTD_inBuffer inBuff, inBuff2; ZSTD_outBuffer outBuff; /* Create compressible test buffer */ @@ -183,12 +183,22 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo DISPLAYLEVEL(4, "OK \n"); /* Basic decompression test */ + inBuff2 = inBuff; DISPLAYLEVEL(4, "test%3i : decompress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH); ZSTD_initDStream_usingDict(zd, CNBuffer, 128 KB); { size_t const r = ZSTD_setDStreamParameter(zd, ZSTDdsp_maxWindowSize, 1000000000); /* large limit */ if (ZSTD_isError(r)) goto _output_error; } - { size_t const r = ZSTD_decompressStream(zd, &outBuff, &inBuff); - if (r != 0) goto _output_error; } /* should reach end of frame == 0; otherwise, some data left, or an error */ + { size_t const remaining = ZSTD_decompressStream(zd, &outBuff, &inBuff); + if (remaining != 0) goto _output_error; } /* should reach end of frame == 0; otherwise, some data left, or an error */ + if (outBuff.pos != CNBufferSize) goto _output_error; /* should regenerate the same amount */ + if (inBuff.pos != inBuff.size) goto _output_error; /* should have read the entire frame */ + DISPLAYLEVEL(4, "OK \n"); + + /* Re-use without init */ + DISPLAYLEVEL(4, "test%3i : decompress again without init (re-use previous settings): ", testNb++); + outBuff.pos = 0; + { size_t const remaining = ZSTD_decompressStream(zd, &outBuff, &inBuff2); + if (remaining != 0) goto _output_error; } /* should reach end of frame == 0; otherwise, some data left, or an error */ if (outBuff.pos != CNBufferSize) goto _output_error; /* should regenerate the same amount */ if (inBuff.pos != inBuff.size) goto _output_error; /* should have read the entire frame */ DISPLAYLEVEL(4, "OK \n"); @@ -553,8 +563,9 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compres /* multi - fragments decompression test */ if (!dictSize /* don't reset if dictionary : could be different */ && (FUZ_rand(&lseed) & 1)) { CHECK (ZSTD_isError(ZSTD_resetDStream(zd)), "ZSTD_resetDStream failed"); - } else + } else { ZSTD_initDStream_usingDict(zd, dict, dictSize); + } { size_t decompressionResult = 1; ZSTD_inBuffer inBuff = { cBuffer, cSize, 0 }; ZSTD_outBuffer outBuff= { dstBuffer, dstBufferSize, 0 };