From 0e14675df2550f9850d62ab8f5e0b66554afe7d9 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Wed, 30 Nov 2016 13:34:21 +0100 Subject: [PATCH 01/13] fileio.c: detect .gz files --- programs/fileio.c | 28 +++++++++++++++++++++++----- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index 71593ac90..a11f92868 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -74,6 +74,7 @@ #define MAX_DICT_SIZE (8 MB) /* protection against large input (attack scenario) */ #define FNSPACE 30 +#define GZ_EXTENSION ".gz" /*-************************************* @@ -658,11 +659,20 @@ static int FIO_decompressSrcFile(dRess_t ress, const char* srcFileName) FILE* const dstFile = ress.dstFile; FILE* srcFile; unsigned readSomething = 0; + size_t const suffixSize = strlen(GZ_EXTENSION); + size_t const sfnSize = strlen(srcFileName); + const char* const suffixPtr = srcFileName + sfnSize - suffixSize; + + if (sfnSize > suffixSize && strcmp(suffixPtr, GZ_EXTENSION) == 0) { + DISPLAYLEVEL(1, "zstd: %s: gz file cannot be uncompressed -- ignored \n", srcFileName); + return 1; + } if (UTIL_isDirectory(srcFileName)) { DISPLAYLEVEL(1, "zstd: %s is a directory -- ignored \n", srcFileName); return 1; } + srcFile = FIO_openSrcFile(srcFileName); if (srcFile==0) return 1; @@ -763,6 +773,7 @@ int FIO_decompressMultipleFilenames(const char** srcNamesTable, unsigned nbFiles if (fclose(ress.dstFile)) EXM_THROW(72, "Write error : cannot properly close stdout"); } else { size_t const suffixSize = strlen(suffix); + size_t const gzSuffixSize = strlen(GZ_EXTENSION); size_t dfnSize = FNSPACE; unsigned u; char* dstFileName = (char*)malloc(FNSPACE); @@ -771,6 +782,7 @@ int FIO_decompressMultipleFilenames(const char** srcNamesTable, unsigned nbFiles const char* const srcFileName = srcNamesTable[u]; size_t const sfnSize = strlen(srcFileName); const char* const suffixPtr = srcFileName + sfnSize - suffixSize; + const char* const gzSuffixPtr = srcFileName + sfnSize - gzSuffixSize; if (dfnSize+suffixSize <= sfnSize+1) { free(dstFileName); dfnSize = sfnSize + 20; @@ -778,12 +790,18 @@ int FIO_decompressMultipleFilenames(const char** srcNamesTable, unsigned nbFiles if (dstFileName==NULL) EXM_THROW(74, "not enough memory for dstFileName"); } if (sfnSize <= suffixSize || strcmp(suffixPtr, suffix) != 0) { - DISPLAYLEVEL(1, "zstd: %s: unknown suffix (%4s expected) -- ignored \n", srcFileName, suffix); - skippedFiles++; - continue; + if (sfnSize <= gzSuffixSize || strcmp(gzSuffixPtr, GZ_EXTENSION) != 0) { + DISPLAYLEVEL(1, "zstd: %s: unknown suffix (%4s expected) -- ignored \n", srcFileName, suffix); + skippedFiles++; + continue; + } else { + memcpy(dstFileName, srcFileName, sfnSize - gzSuffixSize); + dstFileName[sfnSize-gzSuffixSize] = '\0'; + } + } else { + memcpy(dstFileName, srcFileName, sfnSize - suffixSize); + dstFileName[sfnSize-suffixSize] = '\0'; } - memcpy(dstFileName, srcFileName, sfnSize - suffixSize); - dstFileName[sfnSize-suffixSize] = '\0'; missingFiles += FIO_decompressDstFile(ress, dstFileName, srcFileName); } From abfb51f5f2241ca69a55dd676029b8b9618dd29c Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Wed, 30 Nov 2016 15:05:54 +0100 Subject: [PATCH 02/13] gzstd: decompresses .gz files --- programs/Makefile | 5 ++- programs/fileio.c | 85 +++++++++++++++++++++++++++++------------------ 2 files changed, 57 insertions(+), 33 deletions(-) diff --git a/programs/Makefile b/programs/Makefile index b72cebdb1..4f5a00d5b 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -82,7 +82,7 @@ zstd : $(ZSTDDECOMP_O) $(ZSTD_FILES) $(ZSTDLEGACY_FILES) $(ZDICT_FILES) \ ifneq (,$(filter Windows%,$(OS))) windres\generate_res.bat endif - $(CC) $(FLAGS) $^ $(RES_FILE) -o $@$(EXT) + $(CC) $(FLAGS) $^ $(RES_FILE) -o $@$(EXT) $(LDFLAGS) zstd32 : CPPFLAGS += -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT) @@ -124,6 +124,9 @@ zstd-decompress: clean_decomp_o zstd-compress: $(ZSTDCOMMON_FILES) $(ZSTDCOMP_FILES) zstdcli.c fileio.c $(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_NODECOMPRESS $^ -o $@$(EXT) +gzstd: clean_decomp_o + CFLAGS+=-DZSTD_GZDECOMPRESS LDFLAGS+=-lz $(MAKE) zstd + generate_res: windres\generate_res.bat diff --git a/programs/fileio.c b/programs/fileio.c index a11f92868..733abe01d 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -34,6 +34,9 @@ #include "fileio.h" #define ZSTD_STATIC_LINKING_ONLY /* ZSTD_magicNumber, ZSTD_frameHeaderSize_max */ #include "zstd.h" +#ifdef ZSTD_GZDECOMPRESS +#include "zlib.h" +#endif /*-************************************* @@ -657,56 +660,74 @@ static int FIO_decompressSrcFile(dRess_t ress, const char* srcFileName) { unsigned long long filesize = 0; FILE* const dstFile = ress.dstFile; - FILE* srcFile; unsigned readSomething = 0; size_t const suffixSize = strlen(GZ_EXTENSION); size_t const sfnSize = strlen(srcFileName); const char* const suffixPtr = srcFileName + sfnSize - suffixSize; - if (sfnSize > suffixSize && strcmp(suffixPtr, GZ_EXTENSION) == 0) { - DISPLAYLEVEL(1, "zstd: %s: gz file cannot be uncompressed -- ignored \n", srcFileName); - return 1; - } - if (UTIL_isDirectory(srcFileName)) { DISPLAYLEVEL(1, "zstd: %s is a directory -- ignored \n", srcFileName); return 1; } - srcFile = FIO_openSrcFile(srcFileName); - if (srcFile==0) return 1; + if (sfnSize <= suffixSize || strcmp(suffixPtr, GZ_EXTENSION) != 0) { + FILE* srcFile = FIO_openSrcFile(srcFileName); + if (srcFile==0) return 1; - /* for each frame */ - for ( ; ; ) { - /* check magic number -> version */ - size_t const toRead = 4; - size_t const sizeCheck = fread(ress.srcBuffer, (size_t)1, toRead, srcFile); - if (sizeCheck==0) { - if (readSomething==0) { DISPLAY("zstd: %s: unexpected end of file \n", srcFileName); fclose(srcFile); return 1; } /* srcFileName is empty */ - break; /* no more input */ + /* for each frame */ + for ( ; ; ) { + /* check magic number -> version */ + size_t const toRead = 4; + size_t const sizeCheck = fread(ress.srcBuffer, (size_t)1, toRead, srcFile); + if (sizeCheck==0) { + if (readSomething==0) { DISPLAY("zstd: %s: unexpected end of file \n", srcFileName); fclose(srcFile); return 1; } /* srcFileName is empty */ + break; /* no more input */ + } + readSomething = 1; /* there is at least >= 4 bytes in srcFile */ + if (sizeCheck != toRead) { DISPLAY("zstd: %s: unknown header \n", srcFileName); fclose(srcFile); return 1; } /* srcFileName is empty */ + if (!ZSTD_isFrame(ress.srcBuffer, toRead)) { + if ((g_overwrite) && !strcmp (srcFileName, stdinmark)) { /* pass-through mode */ + unsigned const result = FIO_passThrough(dstFile, srcFile, ress.srcBuffer, ress.srcBufferSize); + if (fclose(srcFile)) EXM_THROW(32, "zstd: %s close error", srcFileName); /* error should never happen */ + return result; + } else { + DISPLAYLEVEL(1, "zstd: %s: not in zstd format \n", srcFileName); + fclose(srcFile); + return 1; + } } + filesize += FIO_decompressFrame(ress, dstFile, srcFile, toRead, filesize); } - readSomething = 1; /* there is at least >= 4 bytes in srcFile */ - if (sizeCheck != toRead) { DISPLAY("zstd: %s: unknown header \n", srcFileName); fclose(srcFile); return 1; } /* srcFileName is empty */ - if (!ZSTD_isFrame(ress.srcBuffer, toRead)) { - if ((g_overwrite) && !strcmp (srcFileName, stdinmark)) { /* pass-through mode */ - unsigned const result = FIO_passThrough(dstFile, srcFile, ress.srcBuffer, ress.srcBufferSize); - if (fclose(srcFile)) EXM_THROW(32, "zstd: %s close error", srcFileName); /* error should never happen */ - return result; - } else { - DISPLAYLEVEL(1, "zstd: %s: not in zstd format \n", srcFileName); - fclose(srcFile); - return 1; - } } - filesize += FIO_decompressFrame(ress, dstFile, srcFile, toRead, filesize); + /* Close file */ + if (fclose(srcFile)) EXM_THROW(33, "zstd: %s close error", srcFileName); /* error should never happen */ + } else { +#ifndef ZSTD_GZDECOMPRESS + DISPLAYLEVEL(1, "zstd: %s: gz file cannot be uncompressed (zstd compiled without ZSTD_GZDECOMPRESS) -- ignored \n", srcFileName); + return 1; +#else + int readBytes; + gzFile gzSrcFile = gzopen(srcFileName, "rb"); + if (gzSrcFile == NULL) { DISPLAY("zstd: %s: gzopen error \n", srcFileName); return 1; } + + do { + readBytes = gzread(gzSrcFile, ress.dstBuffer, ress.dstBufferSize); + if (readBytes < 0) { DISPLAY("zstd: %s: gzread error \n", srcFileName); return 1; } + if (readBytes > 0) { + size_t const sizeCheck = fwrite(ress.dstBuffer, 1, readBytes, dstFile); + if (sizeCheck != (size_t)readBytes) EXM_THROW(34, "Write error : cannot write to output file"); + } + filesize += readBytes; + } while ((size_t)readBytes == ress.dstBufferSize); + + if (gzclose(gzSrcFile) != Z_OK) { DISPLAY("zstd: %s: gzclose error \n", srcFileName); return 1; } +#endif } /* Final Status */ DISPLAYLEVEL(2, "\r%79s\r", ""); DISPLAYLEVEL(2, "%-20s: %llu bytes \n", srcFileName, filesize); - /* Close */ - if (fclose(srcFile)) EXM_THROW(33, "zstd: %s close error", srcFileName); /* error should never happen */ - if (g_removeSrcFile) { if (remove(srcFileName)) EXM_THROW(34, "zstd: %s: %s", srcFileName, strerror(errno)); }; + /* Remove source file */ + if (g_removeSrcFile) { if (remove(srcFileName)) EXM_THROW(35, "zstd: %s: %s", srcFileName, strerror(errno)); }; return 0; } From 0efaf7e7b1fee82e4fda3f5b78dbfa27856c0869 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Wed, 30 Nov 2016 15:20:24 +0100 Subject: [PATCH 03/13] added test-gzstd --- .travis.yml | 4 ++-- tests/Makefile | 8 ++++++++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index c001152f5..148a98f1b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,7 +12,7 @@ matrix: os: linux sudo: false - - env: Ubu=12.04cont Cmd="make zlibwrapper && make clean && make -C tests test-zstd-nolegacy && make clean && make clean && make cmaketest && make clean && make -C contrib/pzstd googletest pzstd tests check && make -C contrib/pzstd clean" + - env: Ubu=12.04cont Cmd="make zlibwrapper && make clean && make -C tests test-zstd-nolegacy && make clean && make cmaketest && make clean && make -C contrib/pzstd googletest pzstd tests check && make -C contrib/pzstd clean" os: linux sudo: false language: cpp @@ -37,7 +37,7 @@ matrix: # Standard Ubuntu 12.04 LTS Server Edition 64 bit - - env: Ubu=12.04 Cmd="make -C programs zstd-small && make -C programs zstd-decompress && make -C programs zstd-compress && make -C programs clean && make -C tests versionsTest" + - env: Ubu=12.04 Cmd="make -C programs zstd-small zstd-decompress zstd-compress && make -C tests test-gzstd && make -C programs clean && make -C tests versionsTest" os: linux sudo: required diff --git a/tests/Makefile b/tests/Makefile index 7e5c66ad8..ca899d2f8 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -76,6 +76,9 @@ zstd32: zstd-nolegacy: $(MAKE) -C $(PRGDIR) $@ +gzstd: + $(MAKE) -C $(PRGDIR) $@ + fullbench : $(ZSTD_FILES) $(PRGDIR)/datagen.c fullbench.c $(CC) $(FLAGS) $^ -o $@$(EXT) @@ -190,6 +193,11 @@ test-zstd32: zstd32 zstd-playTests test-zstd-nolegacy: ZSTD = $(PRGDIR)/zstd test-zstd-nolegacy: zstd-nolegacy zstd-playTests +test-gzstd: gzstd + gzip README.md test-zstd-speed.py + $(PRGDIR)/zstd -d README.md.gz -o README2.md + $(PRGDIR)/zstd -d README.md.gz test-zstd-speed.py.gz + test-fullbench: fullbench datagen $(QEMU_SYS) ./fullbench -i1 $(QEMU_SYS) ./fullbench -i1 -P0 From 166830ed0a87471203c5ecba576be24f6e1fd72e Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Wed, 30 Nov 2016 16:43:07 +0100 Subject: [PATCH 04/13] autodetect -lz --- programs/Makefile | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/programs/Makefile b/programs/Makefile index 4f5a00d5b..e44f11264 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -125,7 +125,11 @@ zstd-compress: $(ZSTDCOMMON_FILES) $(ZSTDCOMP_FILES) zstdcli.c fileio.c $(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_NODECOMPRESS $^ -o $@$(EXT) gzstd: clean_decomp_o - CFLAGS+=-DZSTD_GZDECOMPRESS LDFLAGS+=-lz $(MAKE) zstd +ifeq ($(shell ld -lz 2>/dev/null && echo -n true),true) + $(MAKE) zstd MOREFLAGS=-DZSTD_GZDECOMPRESS LDFLAGS="-lz" +else + $(MAKE) zstd +endif generate_res: windres\generate_res.bat From dc2fe75732eea8cb4bb2b2dd5641094dafcaa84f Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Thu, 1 Dec 2016 11:56:20 +0100 Subject: [PATCH 05/13] gzread.c: improved comments --- zlibWrapper/gzread.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/zlibWrapper/gzread.c b/zlibWrapper/gzread.c index ad33ebc62..51ffef3e6 100644 --- a/zlibWrapper/gzread.c +++ b/zlibWrapper/gzread.c @@ -138,8 +138,8 @@ local int gz_look(state) single byte is sufficient indication that it is not a gzip file) */ //printf("strm->next_in[0]=%d strm->next_in[1]=%d\n", strm->next_in[0], strm->next_in[1]); if (strm->avail_in > 1 && - ((strm->next_in[0] == 31 && strm->next_in[1] == 139) - || (strm->next_in[0] == 40 && strm->next_in[1] == 181))) { // zstd + ((strm->next_in[0] == 31 && strm->next_in[1] == 139) /* gz header */ + || (strm->next_in[0] == 40 && strm->next_in[1] == 181))) { /* zstd header */ inflateReset(strm); state.state->how = GZIP; state.state->direct = 0; From 19aad42ee1e696433a8a84944099a1505d348cb9 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Thu, 1 Dec 2016 11:56:31 +0100 Subject: [PATCH 06/13] added FIO_decompressGzFile --- programs/fileio.c | 71 ++++++++++++++++++++++++++++++----------------- 1 file changed, 45 insertions(+), 26 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index 733abe01d..e3c8a3de8 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -651,6 +651,30 @@ static unsigned FIO_passThrough(FILE* foutput, FILE* finput, void* buffer, size_ } +#ifdef ZSTD_GZDECOMPRESS +static unsigned long long FIO_decompressGzFile(dRess_t ress, const char* srcFileName, gzFile gzSrcFile) +{ + unsigned long long filesize = 0; + int readBytes; + if (gzSrcFile == NULL) { DISPLAY("zstd: %s: gzopen error \n", srcFileName); return 0; } + + do { + readBytes = gzread(gzSrcFile, ress.dstBuffer, ress.dstBufferSize); + if (readBytes < 0) { DISPLAY("zstd: %s: gzread error \n", srcFileName); return 0; } + if (readBytes > 0) { + size_t const sizeCheck = fwrite(ress.dstBuffer, 1, readBytes, ress.dstFile); + if (sizeCheck != (size_t)readBytes) EXM_THROW(34, "Write error : cannot write to output file"); + } + filesize += readBytes; + } while ((size_t)readBytes == ress.dstBufferSize); + + if (gzclose(gzSrcFile) != Z_OK) { DISPLAY("zstd: %s: gzclose error \n", srcFileName); return 0; } + + return filesize; +} +#endif + + /** FIO_decompressSrcFile() : Decompression `srcFileName` into `ress.dstFile` @return : 0 : OK @@ -679,23 +703,30 @@ static int FIO_decompressSrcFile(dRess_t ress, const char* srcFileName) /* check magic number -> version */ size_t const toRead = 4; size_t const sizeCheck = fread(ress.srcBuffer, (size_t)1, toRead, srcFile); + const BYTE* buf = (const BYTE*)ress.srcBuffer; if (sizeCheck==0) { if (readSomething==0) { DISPLAY("zstd: %s: unexpected end of file \n", srcFileName); fclose(srcFile); return 1; } /* srcFileName is empty */ break; /* no more input */ } readSomething = 1; /* there is at least >= 4 bytes in srcFile */ if (sizeCheck != toRead) { DISPLAY("zstd: %s: unknown header \n", srcFileName); fclose(srcFile); return 1; } /* srcFileName is empty */ - if (!ZSTD_isFrame(ress.srcBuffer, toRead)) { - if ((g_overwrite) && !strcmp (srcFileName, stdinmark)) { /* pass-through mode */ - unsigned const result = FIO_passThrough(dstFile, srcFile, ress.srcBuffer, ress.srcBufferSize); - if (fclose(srcFile)) EXM_THROW(32, "zstd: %s close error", srcFileName); /* error should never happen */ - return result; - } else { - DISPLAYLEVEL(1, "zstd: %s: not in zstd format \n", srcFileName); - fclose(srcFile); - return 1; - } } - filesize += FIO_decompressFrame(ress, dstFile, srcFile, toRead, filesize); + if (buf[0] == 31 && buf[1] == 139) { /* gz header */ + unsigned long long result = FIO_decompressGzFile(ress, srcFileName, gzdopen(fileno(srcFile), "rb")); + if (result == 0) return 1; + filesize += result; + } else { + if (!ZSTD_isFrame(ress.srcBuffer, toRead)) { + if ((g_overwrite) && !strcmp (srcFileName, stdinmark)) { /* pass-through mode */ + unsigned const result = FIO_passThrough(dstFile, srcFile, ress.srcBuffer, ress.srcBufferSize); + if (fclose(srcFile)) EXM_THROW(32, "zstd: %s close error", srcFileName); /* error should never happen */ + return result; + } else { + DISPLAYLEVEL(1, "zstd: %s: not in zstd format \n", srcFileName); + fclose(srcFile); + return 1; + } } + filesize += FIO_decompressFrame(ress, dstFile, srcFile, toRead, filesize); + } } /* Close file */ if (fclose(srcFile)) EXM_THROW(33, "zstd: %s close error", srcFileName); /* error should never happen */ @@ -704,21 +735,9 @@ static int FIO_decompressSrcFile(dRess_t ress, const char* srcFileName) DISPLAYLEVEL(1, "zstd: %s: gz file cannot be uncompressed (zstd compiled without ZSTD_GZDECOMPRESS) -- ignored \n", srcFileName); return 1; #else - int readBytes; - gzFile gzSrcFile = gzopen(srcFileName, "rb"); - if (gzSrcFile == NULL) { DISPLAY("zstd: %s: gzopen error \n", srcFileName); return 1; } - - do { - readBytes = gzread(gzSrcFile, ress.dstBuffer, ress.dstBufferSize); - if (readBytes < 0) { DISPLAY("zstd: %s: gzread error \n", srcFileName); return 1; } - if (readBytes > 0) { - size_t const sizeCheck = fwrite(ress.dstBuffer, 1, readBytes, dstFile); - if (sizeCheck != (size_t)readBytes) EXM_THROW(34, "Write error : cannot write to output file"); - } - filesize += readBytes; - } while ((size_t)readBytes == ress.dstBufferSize); - - if (gzclose(gzSrcFile) != Z_OK) { DISPLAY("zstd: %s: gzclose error \n", srcFileName); return 1; } + unsigned long long result = FIO_decompressGzFile(ress, srcFileName, gzopen(srcFileName, "rb")); + if (result == 0) return 1; + filesize += result; #endif } From daaf7545390c080b434aae6fc05828b35a651e69 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Thu, 1 Dec 2016 13:29:19 +0100 Subject: [PATCH 07/13] detect stream with ungetc --- programs/fileio.c | 36 +++++++++++++++++++++--------------- 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index e3c8a3de8..535308e58 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -664,8 +664,8 @@ static unsigned long long FIO_decompressGzFile(dRess_t ress, const char* srcFile if (readBytes > 0) { size_t const sizeCheck = fwrite(ress.dstBuffer, 1, readBytes, ress.dstFile); if (sizeCheck != (size_t)readBytes) EXM_THROW(34, "Write error : cannot write to output file"); + filesize += readBytes; } - filesize += readBytes; } while ((size_t)readBytes == ress.dstBufferSize); if (gzclose(gzSrcFile) != Z_OK) { DISPLAY("zstd: %s: gzclose error \n", srcFileName); return 0; } @@ -700,21 +700,27 @@ static int FIO_decompressSrcFile(dRess_t ress, const char* srcFileName) /* for each frame */ for ( ; ; ) { - /* check magic number -> version */ - size_t const toRead = 4; - size_t const sizeCheck = fread(ress.srcBuffer, (size_t)1, toRead, srcFile); - const BYTE* buf = (const BYTE*)ress.srcBuffer; - if (sizeCheck==0) { - if (readSomething==0) { DISPLAY("zstd: %s: unexpected end of file \n", srcFileName); fclose(srcFile); return 1; } /* srcFileName is empty */ - break; /* no more input */ + if (srcFile == stdin) { + int c = getc(srcFile); + if (c < 0) break; /* no more input */ + c = ungetc(c, srcFile); /* only one pushback is guaranteed */ + if (c == 31) { /* 31,139 = gz header */ + unsigned long long result = FIO_decompressGzFile(ress, srcFileName, gzdopen(fileno(srcFile), "rb")); + printf("result=%d\n", (int)result); + if (result == 0) return 1; + filesize += result; + continue; + } } - readSomething = 1; /* there is at least >= 4 bytes in srcFile */ - if (sizeCheck != toRead) { DISPLAY("zstd: %s: unknown header \n", srcFileName); fclose(srcFile); return 1; } /* srcFileName is empty */ - if (buf[0] == 31 && buf[1] == 139) { /* gz header */ - unsigned long long result = FIO_decompressGzFile(ress, srcFileName, gzdopen(fileno(srcFile), "rb")); - if (result == 0) return 1; - filesize += result; - } else { + /* check magic number -> version */ + { size_t const toRead = 4; + size_t const sizeCheck = fread(ress.srcBuffer, (size_t)1, toRead, srcFile); + if (sizeCheck==0) { + if (readSomething==0) { DISPLAY("zstd: %s: unexpected end of file \n", srcFileName); fclose(srcFile); return 1; } /* srcFileName is empty */ + break; /* no more input */ + } + readSomething = 1; /* there is at least >= 4 bytes in srcFile */ + if (sizeCheck != toRead) { DISPLAY("zstd: %s: unknown header \n", srcFileName); fclose(srcFile); return 1; } /* srcFileName is empty */ if (!ZSTD_isFrame(ress.srcBuffer, toRead)) { if ((g_overwrite) && !strcmp (srcFileName, stdinmark)) { /* pass-through mode */ unsigned const result = FIO_passThrough(dstFile, srcFile, ress.srcBuffer, ress.srcBufferSize); From 4b504f131aff79cf9a427c970de239191c8455ee Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Fri, 2 Dec 2016 13:11:39 +0100 Subject: [PATCH 08/13] added gzip_open, gzip_close, gzip_read --- programs/fileio.c | 163 ++++++++++++++++++++++++++++++++++------------ 1 file changed, 121 insertions(+), 42 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index 535308e58..975a7cfdc 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -652,23 +652,107 @@ static unsigned FIO_passThrough(FILE* foutput, FILE* finput, void* buffer, size_ #ifdef ZSTD_GZDECOMPRESS -static unsigned long long FIO_decompressGzFile(dRess_t ress, const char* srcFileName, gzFile gzSrcFile) +typedef struct gzipContext_s { + FILE *file; + int err; + char *msg; + z_stream strm; +} *gzipContext; + + +static gzipContext gzip_open(const char *path, int fd, const char *mode) { + gzipContext gzip = malloc(sizeof(struct gzipContext_s)); + if (gzip == NULL) + return NULL; + + gzip->file = (path == NULL) ? fdopen(fd, mode) : fopen(path, mode); + if (gzip->file == NULL) { + free(gzip); + return NULL; + } + + gzip->strm.zalloc = Z_NULL; + gzip->strm.zfree = Z_NULL; + gzip->strm.opaque = Z_NULL; + gzip->strm.next_in = 0; + gzip->strm.avail_in = Z_NULL; + if (inflateInit2(&(gzip->strm), 15 + 16) != Z_OK) { + fclose(gzip->file); + free(gzip); + return NULL; + } + gzip->err = 0; + gzip->msg = ""; + return gzip; +} + + +static int gzip_close(gzipContext gzip) +{ + if (gzip == NULL) return Z_STREAM_ERROR; + + inflateEnd(&(gzip->strm)); + fclose(gzip->file); + free(gzip); + return Z_OK; +} + + +int gzip_read(gzipContext gzip, dRess_t ress, size_t headBufSize) { + int ret; + unsigned readBytes; + unsigned char in[1]; + unsigned char* headBuf = (unsigned char*)ress.srcBuffer; + z_stream *strm; + + if (gzip == NULL || gzip->err) + return 0; + strm = &(gzip->strm); + strm->next_out = ress.dstBuffer; + strm->avail_out = ress.dstBufferSize; + + do { + if (headBufSize) { + headBufSize--; + in[0] = *headBuf++; + } else { + readBytes = fread(in, 1, 1, gzip->file); + if (readBytes == 0) + break; + } + strm->next_in = in; + strm->avail_in = 1; + ret = inflate(strm, Z_NO_FLUSH); + if (ret == Z_DATA_ERROR) { + gzip->err = Z_DATA_ERROR; + gzip->msg = strm->msg; + return 0; + } + if (ret == Z_STREAM_END) + inflateReset(strm); + } while (strm->avail_out); + + return ress.dstBufferSize - strm->avail_out; +} + + +static unsigned long long FIO_decompressGzFile(dRess_t ress, size_t headBufSize, const char* srcFileName, gzipContext gzipSrcFile) { unsigned long long filesize = 0; int readBytes; - if (gzSrcFile == NULL) { DISPLAY("zstd: %s: gzopen error \n", srcFileName); return 0; } + if (gzipSrcFile == NULL) { DISPLAY("zstd: %s: FIO_decompressGzFile error \n", srcFileName); return 0; } - do { - readBytes = gzread(gzSrcFile, ress.dstBuffer, ress.dstBufferSize); - if (readBytes < 0) { DISPLAY("zstd: %s: gzread error \n", srcFileName); return 0; } - if (readBytes > 0) { - size_t const sizeCheck = fwrite(ress.dstBuffer, 1, readBytes, ress.dstFile); - if (sizeCheck != (size_t)readBytes) EXM_THROW(34, "Write error : cannot write to output file"); - filesize += readBytes; - } - } while ((size_t)readBytes == ress.dstBufferSize); + for ( ; ; ) { + readBytes = gzip_read(gzipSrcFile, ress, headBufSize); + printf("readBytes=%d dstBufferSize=%d\n", (int)readBytes, (int)ress.dstBufferSize); + if (readBytes < 0) EXM_THROW(33, "zstd: %s: gzip_read error: %s \n", srcFileName, gzipSrcFile->msg); + if (readBytes == 0) break; + + if (fwrite(ress.dstBuffer, 1, readBytes, ress.dstFile) != (size_t)readBytes) EXM_THROW(34, "Write error : cannot write to output file"); + filesize += readBytes; + } - if (gzclose(gzSrcFile) != Z_OK) { DISPLAY("zstd: %s: gzclose error \n", srcFileName); return 0; } + if (gzip_close(gzipSrcFile) != Z_OK) { DISPLAY("zstd: %s: gzip_close error \n", srcFileName); return 0; } return filesize; } @@ -700,48 +784,43 @@ static int FIO_decompressSrcFile(dRess_t ress, const char* srcFileName) /* for each frame */ for ( ; ; ) { - if (srcFile == stdin) { - int c = getc(srcFile); - if (c < 0) break; /* no more input */ - c = ungetc(c, srcFile); /* only one pushback is guaranteed */ - if (c == 31) { /* 31,139 = gz header */ - unsigned long long result = FIO_decompressGzFile(ress, srcFileName, gzdopen(fileno(srcFile), "rb")); - printf("result=%d\n", (int)result); - if (result == 0) return 1; - filesize += result; - continue; - } - } /* check magic number -> version */ { size_t const toRead = 4; size_t const sizeCheck = fread(ress.srcBuffer, (size_t)1, toRead, srcFile); + const BYTE* buf = (const BYTE*)ress.srcBuffer; if (sizeCheck==0) { if (readSomething==0) { DISPLAY("zstd: %s: unexpected end of file \n", srcFileName); fclose(srcFile); return 1; } /* srcFileName is empty */ break; /* no more input */ } readSomething = 1; /* there is at least >= 4 bytes in srcFile */ if (sizeCheck != toRead) { DISPLAY("zstd: %s: unknown header \n", srcFileName); fclose(srcFile); return 1; } /* srcFileName is empty */ - if (!ZSTD_isFrame(ress.srcBuffer, toRead)) { - if ((g_overwrite) && !strcmp (srcFileName, stdinmark)) { /* pass-through mode */ - unsigned const result = FIO_passThrough(dstFile, srcFile, ress.srcBuffer, ress.srcBufferSize); - if (fclose(srcFile)) EXM_THROW(32, "zstd: %s close error", srcFileName); /* error should never happen */ - return result; - } else { - DISPLAYLEVEL(1, "zstd: %s: not in zstd format \n", srcFileName); - fclose(srcFile); - return 1; - } } - filesize += FIO_decompressFrame(ress, dstFile, srcFile, toRead, filesize); + if (buf[0] == 31 && buf[1] == 139) { /* gz header */ + unsigned long long result = FIO_decompressGzFile(ress, toRead, srcFileName, gzip_open(NULL, fileno(srcFile), "rb")); + if (result == 0) return 1; + filesize += result; + } else { + if (!ZSTD_isFrame(ress.srcBuffer, toRead)) { + if ((g_overwrite) && !strcmp (srcFileName, stdinmark)) { /* pass-through mode */ + unsigned const result = FIO_passThrough(dstFile, srcFile, ress.srcBuffer, ress.srcBufferSize); + if (fclose(srcFile)) EXM_THROW(32, "zstd: %s close error", srcFileName); /* error should never happen */ + return result; + } else { + DISPLAYLEVEL(1, "zstd: %s: not in zstd format \n", srcFileName); + fclose(srcFile); + return 1; + } } + filesize += FIO_decompressFrame(ress, dstFile, srcFile, toRead, filesize); + } } } /* Close file */ if (fclose(srcFile)) EXM_THROW(33, "zstd: %s close error", srcFileName); /* error should never happen */ } else { #ifndef ZSTD_GZDECOMPRESS - DISPLAYLEVEL(1, "zstd: %s: gz file cannot be uncompressed (zstd compiled without ZSTD_GZDECOMPRESS) -- ignored \n", srcFileName); + DISPLAYLEVEL(1, "zstd: %s: gzip file cannot be uncompressed (zstd compiled without ZSTD_GZDECOMPRESS) -- ignored \n", srcFileName); return 1; #else - unsigned long long result = FIO_decompressGzFile(ress, srcFileName, gzopen(srcFileName, "rb")); + unsigned long long result = FIO_decompressGzFile(ress, 0, srcFileName, gzip_open(srcFileName, 0, "rb")); if (result == 0) return 1; filesize += result; #endif @@ -819,7 +898,7 @@ int FIO_decompressMultipleFilenames(const char** srcNamesTable, unsigned nbFiles if (fclose(ress.dstFile)) EXM_THROW(72, "Write error : cannot properly close stdout"); } else { size_t const suffixSize = strlen(suffix); - size_t const gzSuffixSize = strlen(GZ_EXTENSION); + size_t const gzipSuffixSize = strlen(GZ_EXTENSION); size_t dfnSize = FNSPACE; unsigned u; char* dstFileName = (char*)malloc(FNSPACE); @@ -828,7 +907,7 @@ int FIO_decompressMultipleFilenames(const char** srcNamesTable, unsigned nbFiles const char* const srcFileName = srcNamesTable[u]; size_t const sfnSize = strlen(srcFileName); const char* const suffixPtr = srcFileName + sfnSize - suffixSize; - const char* const gzSuffixPtr = srcFileName + sfnSize - gzSuffixSize; + const char* const gzipSuffixPtr = srcFileName + sfnSize - gzipSuffixSize; if (dfnSize+suffixSize <= sfnSize+1) { free(dstFileName); dfnSize = sfnSize + 20; @@ -836,13 +915,13 @@ int FIO_decompressMultipleFilenames(const char** srcNamesTable, unsigned nbFiles if (dstFileName==NULL) EXM_THROW(74, "not enough memory for dstFileName"); } if (sfnSize <= suffixSize || strcmp(suffixPtr, suffix) != 0) { - if (sfnSize <= gzSuffixSize || strcmp(gzSuffixPtr, GZ_EXTENSION) != 0) { + if (sfnSize <= gzipSuffixSize || strcmp(gzipSuffixPtr, GZ_EXTENSION) != 0) { DISPLAYLEVEL(1, "zstd: %s: unknown suffix (%4s expected) -- ignored \n", srcFileName, suffix); skippedFiles++; continue; } else { - memcpy(dstFileName, srcFileName, sfnSize - gzSuffixSize); - dstFileName[sfnSize-gzSuffixSize] = '\0'; + memcpy(dstFileName, srcFileName, sfnSize - gzipSuffixSize); + dstFileName[sfnSize-gzipSuffixSize] = '\0'; } } else { memcpy(dstFileName, srcFileName, sfnSize - suffixSize); From b0f2ef21190ebf25ea7f18cbc86845a343b81ab6 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Fri, 2 Dec 2016 13:50:29 +0100 Subject: [PATCH 09/13] improved gzip_* functions --- programs/fileio.c | 57 ++++++++++++++++++++++------------------------- 1 file changed, 27 insertions(+), 30 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index 975a7cfdc..16ae337b9 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -653,31 +653,26 @@ static unsigned FIO_passThrough(FILE* foutput, FILE* finput, void* buffer, size_ #ifdef ZSTD_GZDECOMPRESS typedef struct gzipContext_s { - FILE *file; int err; char *msg; z_stream strm; } *gzipContext; -static gzipContext gzip_open(const char *path, int fd, const char *mode) { - gzipContext gzip = malloc(sizeof(struct gzipContext_s)); +static gzipContext gzip_create(void) +{ + gzipContext gzip; + + gzip = malloc(sizeof(struct gzipContext_s)); if (gzip == NULL) return NULL; - gzip->file = (path == NULL) ? fdopen(fd, mode) : fopen(path, mode); - if (gzip->file == NULL) { - free(gzip); - return NULL; - } - gzip->strm.zalloc = Z_NULL; gzip->strm.zfree = Z_NULL; gzip->strm.opaque = Z_NULL; gzip->strm.next_in = 0; gzip->strm.avail_in = Z_NULL; if (inflateInit2(&(gzip->strm), 15 + 16) != Z_OK) { - fclose(gzip->file); free(gzip); return NULL; } @@ -687,18 +682,18 @@ static gzipContext gzip_open(const char *path, int fd, const char *mode) { } -static int gzip_close(gzipContext gzip) +static int gzip_free(gzipContext gzip) { if (gzip == NULL) return Z_STREAM_ERROR; inflateEnd(&(gzip->strm)); - fclose(gzip->file); free(gzip); return Z_OK; } -int gzip_read(gzipContext gzip, dRess_t ress, size_t headBufSize) { +static int gzip_decompress(gzipContext gzip, FILE* file, dRess_t ress, size_t headBufSize) +{ int ret; unsigned readBytes; unsigned char in[1]; @@ -716,7 +711,7 @@ int gzip_read(gzipContext gzip, dRess_t ress, size_t headBufSize) { headBufSize--; in[0] = *headBuf++; } else { - readBytes = fread(in, 1, 1, gzip->file); + readBytes = fread(in, 1, 1, file); if (readBytes == 0) break; } @@ -736,23 +731,25 @@ int gzip_read(gzipContext gzip, dRess_t ress, size_t headBufSize) { } -static unsigned long long FIO_decompressGzFile(dRess_t ress, size_t headBufSize, const char* srcFileName, gzipContext gzipSrcFile) +static unsigned long long FIO_decompressGzFile(dRess_t ress, size_t headBufSize, const char* srcFileName, FILE* srcFile) { unsigned long long filesize = 0; int readBytes; - if (gzipSrcFile == NULL) { DISPLAY("zstd: %s: FIO_decompressGzFile error \n", srcFileName); return 0; } + gzipContext gzipCtx = gzip_create(); + + if (gzipCtx == NULL) { DISPLAY("zstd: %s: gzip_create error \n", srcFileName); return 0; } for ( ; ; ) { - readBytes = gzip_read(gzipSrcFile, ress, headBufSize); - printf("readBytes=%d dstBufferSize=%d\n", (int)readBytes, (int)ress.dstBufferSize); - if (readBytes < 0) EXM_THROW(33, "zstd: %s: gzip_read error: %s \n", srcFileName, gzipSrcFile->msg); + readBytes = gzip_decompress(gzipCtx, srcFile, ress, headBufSize); + printf("headBufSize=%d readBytes=%d dstBufferSize=%d\n", (int)headBufSize, (int)readBytes, (int)ress.dstBufferSize); + if (readBytes < 0) EXM_THROW(30, "zstd: %s: gzip_decompress error: %s \n", srcFileName, gzipCtx->msg); if (readBytes == 0) break; - if (fwrite(ress.dstBuffer, 1, readBytes, ress.dstFile) != (size_t)readBytes) EXM_THROW(34, "Write error : cannot write to output file"); + if (fwrite(ress.dstBuffer, 1, readBytes, ress.dstFile) != (size_t)readBytes) EXM_THROW(31, "Write error : cannot write to output file"); filesize += readBytes; } - if (gzip_close(gzipSrcFile) != Z_OK) { DISPLAY("zstd: %s: gzip_close error \n", srcFileName); return 0; } + if (gzip_free(gzipCtx) != Z_OK) { DISPLAY("zstd: %s: gzip_free error \n", srcFileName); return 0; } return filesize; } @@ -768,6 +765,7 @@ static int FIO_decompressSrcFile(dRess_t ress, const char* srcFileName) { unsigned long long filesize = 0; FILE* const dstFile = ress.dstFile; + FILE* srcFile; unsigned readSomething = 0; size_t const suffixSize = strlen(GZ_EXTENSION); size_t const sfnSize = strlen(srcFileName); @@ -778,10 +776,10 @@ static int FIO_decompressSrcFile(dRess_t ress, const char* srcFileName) return 1; } - if (sfnSize <= suffixSize || strcmp(suffixPtr, GZ_EXTENSION) != 0) { - FILE* srcFile = FIO_openSrcFile(srcFileName); - if (srcFile==0) return 1; + srcFile = FIO_openSrcFile(srcFileName); + if (srcFile==0) return 1; + if (sfnSize <= suffixSize || strcmp(suffixPtr, GZ_EXTENSION) != 0) { /* for each frame */ for ( ; ; ) { /* check magic number -> version */ @@ -795,7 +793,7 @@ static int FIO_decompressSrcFile(dRess_t ress, const char* srcFileName) readSomething = 1; /* there is at least >= 4 bytes in srcFile */ if (sizeCheck != toRead) { DISPLAY("zstd: %s: unknown header \n", srcFileName); fclose(srcFile); return 1; } /* srcFileName is empty */ if (buf[0] == 31 && buf[1] == 139) { /* gz header */ - unsigned long long result = FIO_decompressGzFile(ress, toRead, srcFileName, gzip_open(NULL, fileno(srcFile), "rb")); + unsigned long long result = FIO_decompressGzFile(ress, toRead, srcFileName, srcFile); if (result == 0) return 1; filesize += result; } else { @@ -813,14 +811,12 @@ static int FIO_decompressSrcFile(dRess_t ress, const char* srcFileName) } } } - /* Close file */ - if (fclose(srcFile)) EXM_THROW(33, "zstd: %s close error", srcFileName); /* error should never happen */ } else { #ifndef ZSTD_GZDECOMPRESS DISPLAYLEVEL(1, "zstd: %s: gzip file cannot be uncompressed (zstd compiled without ZSTD_GZDECOMPRESS) -- ignored \n", srcFileName); return 1; #else - unsigned long long result = FIO_decompressGzFile(ress, 0, srcFileName, gzip_open(srcFileName, 0, "rb")); + unsigned long long result = FIO_decompressGzFile(ress, 0, srcFileName, srcFile); if (result == 0) return 1; filesize += result; #endif @@ -830,8 +826,9 @@ static int FIO_decompressSrcFile(dRess_t ress, const char* srcFileName) DISPLAYLEVEL(2, "\r%79s\r", ""); DISPLAYLEVEL(2, "%-20s: %llu bytes \n", srcFileName, filesize); - /* Remove source file */ - if (g_removeSrcFile) { if (remove(srcFileName)) EXM_THROW(35, "zstd: %s: %s", srcFileName, strerror(errno)); }; + /* Close file */ + if (fclose(srcFile)) EXM_THROW(33, "zstd: %s close error", srcFileName); /* error should never happen */ + if (g_removeSrcFile) { if (remove(srcFileName)) EXM_THROW(34, "zstd: %s: %s", srcFileName, strerror(errno)); }; return 0; } From c5eebca12814a08a1a8bb6e848df74b98e4d0c89 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Fri, 2 Dec 2016 15:01:31 +0100 Subject: [PATCH 10/13] rewritten FIO_decompressGzFile --- programs/fileio.c | 191 +++++++++++++++------------------------------- 1 file changed, 63 insertions(+), 128 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index 16ae337b9..317483a96 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -652,106 +652,48 @@ static unsigned FIO_passThrough(FILE* foutput, FILE* finput, void* buffer, size_ #ifdef ZSTD_GZDECOMPRESS -typedef struct gzipContext_s { - int err; - char *msg; - z_stream strm; -} *gzipContext; - - -static gzipContext gzip_create(void) -{ - gzipContext gzip; - - gzip = malloc(sizeof(struct gzipContext_s)); - if (gzip == NULL) - return NULL; - - gzip->strm.zalloc = Z_NULL; - gzip->strm.zfree = Z_NULL; - gzip->strm.opaque = Z_NULL; - gzip->strm.next_in = 0; - gzip->strm.avail_in = Z_NULL; - if (inflateInit2(&(gzip->strm), 15 + 16) != Z_OK) { - free(gzip); - return NULL; - } - gzip->err = 0; - gzip->msg = ""; - return gzip; -} - - -static int gzip_free(gzipContext gzip) -{ - if (gzip == NULL) return Z_STREAM_ERROR; - - inflateEnd(&(gzip->strm)); - free(gzip); - return Z_OK; -} - - -static int gzip_decompress(gzipContext gzip, FILE* file, dRess_t ress, size_t headBufSize) +static size_t FIO_decompressGzFile(dRess_t ress, size_t headBufSize, const char* srcFileName, FILE* srcFile) { int ret; - unsigned readBytes; unsigned char in[1]; unsigned char* headBuf = (unsigned char*)ress.srcBuffer; - z_stream *strm; + size_t decompBytes, outFileSize = 0; + z_stream strm; + + strm.zalloc = Z_NULL; + strm.zfree = Z_NULL; + strm.opaque = Z_NULL; + strm.next_in = 0; + strm.avail_in = Z_NULL; + if (inflateInit2(&strm, 15 + 16) != Z_OK) return 0; + + strm.next_out = ress.dstBuffer; + strm.avail_out = ress.dstBufferSize; - if (gzip == NULL || gzip->err) - return 0; - strm = &(gzip->strm); - strm->next_out = ress.dstBuffer; - strm->avail_out = ress.dstBufferSize; - - do { + for ( ; ; ) { if (headBufSize) { headBufSize--; in[0] = *headBuf++; } else { - readBytes = fread(in, 1, 1, file); - if (readBytes == 0) - break; + if (fread(in, 1, 1, srcFile) == 0) break; } - strm->next_in = in; - strm->avail_in = 1; - ret = inflate(strm, Z_NO_FLUSH); - if (ret == Z_DATA_ERROR) { - gzip->err = Z_DATA_ERROR; - gzip->msg = strm->msg; - return 0; + strm.next_in = in; + strm.avail_in = 1; + ret = inflate(&strm, Z_NO_FLUSH); + if (ret == Z_STREAM_END) break; + if (ret != Z_OK) { DISPLAY("zstd: %s: inflate error %d \n", srcFileName, ret); return 0; } + + decompBytes = ress.dstBufferSize - strm.avail_out; + if (decompBytes) { + if (fwrite(ress.dstBuffer, 1, decompBytes, ress.dstFile) != (size_t)decompBytes) EXM_THROW(31, "Write error : cannot write to output file"); + outFileSize += decompBytes; + strm.next_out = ress.dstBuffer; + strm.avail_out = ress.dstBufferSize; } - if (ret == Z_STREAM_END) - inflateReset(strm); - } while (strm->avail_out); - - return ress.dstBufferSize - strm->avail_out; -} - - -static unsigned long long FIO_decompressGzFile(dRess_t ress, size_t headBufSize, const char* srcFileName, FILE* srcFile) -{ - unsigned long long filesize = 0; - int readBytes; - gzipContext gzipCtx = gzip_create(); - - if (gzipCtx == NULL) { DISPLAY("zstd: %s: gzip_create error \n", srcFileName); return 0; } - - for ( ; ; ) { - readBytes = gzip_decompress(gzipCtx, srcFile, ress, headBufSize); - printf("headBufSize=%d readBytes=%d dstBufferSize=%d\n", (int)headBufSize, (int)readBytes, (int)ress.dstBufferSize); - if (readBytes < 0) EXM_THROW(30, "zstd: %s: gzip_decompress error: %s \n", srcFileName, gzipCtx->msg); - if (readBytes == 0) break; - - if (fwrite(ress.dstBuffer, 1, readBytes, ress.dstFile) != (size_t)readBytes) EXM_THROW(31, "Write error : cannot write to output file"); - filesize += readBytes; } - if (gzip_free(gzipCtx) != Z_OK) { DISPLAY("zstd: %s: gzip_free error \n", srcFileName); return 0; } - - return filesize; + inflateEnd(&strm); + return outFileSize; } #endif @@ -767,9 +709,6 @@ static int FIO_decompressSrcFile(dRess_t ress, const char* srcFileName) FILE* const dstFile = ress.dstFile; FILE* srcFile; unsigned readSomething = 0; - size_t const suffixSize = strlen(GZ_EXTENSION); - size_t const sfnSize = strlen(srcFileName); - const char* const suffixPtr = srcFileName + sfnSize - suffixSize; if (UTIL_isDirectory(srcFileName)) { DISPLAYLEVEL(1, "zstd: %s is a directory -- ignored \n", srcFileName); @@ -779,47 +718,43 @@ static int FIO_decompressSrcFile(dRess_t ress, const char* srcFileName) srcFile = FIO_openSrcFile(srcFileName); if (srcFile==0) return 1; - if (sfnSize <= suffixSize || strcmp(suffixPtr, GZ_EXTENSION) != 0) { - /* for each frame */ - for ( ; ; ) { - /* check magic number -> version */ - { size_t const toRead = 4; - size_t const sizeCheck = fread(ress.srcBuffer, (size_t)1, toRead, srcFile); - const BYTE* buf = (const BYTE*)ress.srcBuffer; - if (sizeCheck==0) { - if (readSomething==0) { DISPLAY("zstd: %s: unexpected end of file \n", srcFileName); fclose(srcFile); return 1; } /* srcFileName is empty */ - break; /* no more input */ - } - readSomething = 1; /* there is at least >= 4 bytes in srcFile */ - if (sizeCheck != toRead) { DISPLAY("zstd: %s: unknown header \n", srcFileName); fclose(srcFile); return 1; } /* srcFileName is empty */ - if (buf[0] == 31 && buf[1] == 139) { /* gz header */ - unsigned long long result = FIO_decompressGzFile(ress, toRead, srcFileName, srcFile); - if (result == 0) return 1; - filesize += result; - } else { - if (!ZSTD_isFrame(ress.srcBuffer, toRead)) { - if ((g_overwrite) && !strcmp (srcFileName, stdinmark)) { /* pass-through mode */ - unsigned const result = FIO_passThrough(dstFile, srcFile, ress.srcBuffer, ress.srcBufferSize); - if (fclose(srcFile)) EXM_THROW(32, "zstd: %s close error", srcFileName); /* error should never happen */ - return result; - } else { - DISPLAYLEVEL(1, "zstd: %s: not in zstd format \n", srcFileName); - fclose(srcFile); - return 1; - } } - filesize += FIO_decompressFrame(ress, dstFile, srcFile, toRead, filesize); - } + /* for each frame */ + for ( ; ; ) { + /* check magic number -> version */ + { size_t const toRead = 4; + size_t const sizeCheck = fread(ress.srcBuffer, (size_t)1, toRead, srcFile); + const BYTE* buf = (const BYTE*)ress.srcBuffer; + if (sizeCheck==0) { + if (readSomething==0) { DISPLAY("zstd: %s: unexpected end of file \n", srcFileName); fclose(srcFile); return 1; } /* srcFileName is empty */ + break; /* no more input */ } - } - } else { -#ifndef ZSTD_GZDECOMPRESS - DISPLAYLEVEL(1, "zstd: %s: gzip file cannot be uncompressed (zstd compiled without ZSTD_GZDECOMPRESS) -- ignored \n", srcFileName); - return 1; + readSomething = 1; /* there is at least >= 4 bytes in srcFile */ + if (sizeCheck != toRead) { DISPLAY("zstd: %s: unknown header \n", srcFileName); fclose(srcFile); return 1; } /* srcFileName is empty */ + if (buf[0] == 31 && buf[1] == 139) { /* gz header */ +#ifdef ZSTD_GZDECOMPRESS + size_t const result = FIO_decompressGzFile(ress, toRead, srcFileName, srcFile); + printf("result=%d\n", (int)result); + if (result == 0) return 1; + filesize += result; #else - unsigned long long result = FIO_decompressGzFile(ress, 0, srcFileName, srcFile); - if (result == 0) return 1; - filesize += result; + DISPLAYLEVEL(1, "zstd: %s: gzip file cannot be uncompressed (zstd compiled without ZSTD_GZDECOMPRESS) -- ignored \n", srcFileName); + return 1; #endif + } else { + if (!ZSTD_isFrame(ress.srcBuffer, toRead)) { + if ((g_overwrite) && !strcmp (srcFileName, stdinmark)) { /* pass-through mode */ + unsigned const result = FIO_passThrough(dstFile, srcFile, ress.srcBuffer, ress.srcBufferSize); + if (fclose(srcFile)) EXM_THROW(32, "zstd: %s close error", srcFileName); /* error should never happen */ + return result; + } else { + DISPLAYLEVEL(1, "zstd: %s: not in zstd format \n", srcFileName); + fclose(srcFile); + return 1; + } } + filesize += FIO_decompressFrame(ress, dstFile, srcFile, toRead, filesize); + } + printf("filesize=%d\n", (int)filesize); + } } /* Final Status */ From 4e49580407d2b4ed9a4864837235438948929b67 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Fri, 2 Dec 2016 15:19:00 +0100 Subject: [PATCH 11/13] removed testing artifacts --- programs/fileio.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index 317483a96..06728ae31 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -733,11 +733,10 @@ static int FIO_decompressSrcFile(dRess_t ress, const char* srcFileName) if (buf[0] == 31 && buf[1] == 139) { /* gz header */ #ifdef ZSTD_GZDECOMPRESS size_t const result = FIO_decompressGzFile(ress, toRead, srcFileName, srcFile); - printf("result=%d\n", (int)result); if (result == 0) return 1; filesize += result; #else - DISPLAYLEVEL(1, "zstd: %s: gzip file cannot be uncompressed (zstd compiled without ZSTD_GZDECOMPRESS) -- ignored \n", srcFileName); + DISPLAYLEVEL(1, "zstd: %s: gzip file cannot be uncompressed -- ignored (zstd compiled without ZSTD_GZDECOMPRESS) \n", srcFileName); return 1; #endif } else { @@ -753,7 +752,6 @@ static int FIO_decompressSrcFile(dRess_t ress, const char* srcFileName) } } filesize += FIO_decompressFrame(ress, dstFile, srcFile, toRead, filesize); } - printf("filesize=%d\n", (int)filesize); } } From 821bf1febc8d2772ed093fa5e540471e9c260c5e Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Fri, 2 Dec 2016 16:13:41 +0100 Subject: [PATCH 12/13] fixed Doxygen trailing comment --- lib/common/huf.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/common/huf.h b/lib/common/huf.h index 35d6033ea..9427ae8cb 100644 --- a/lib/common/huf.h +++ b/lib/common/huf.h @@ -74,7 +74,7 @@ size_t HUF_decompress(void* dst, size_t originalSize, /* *** Tool functions *** */ -#define HUF_BLOCKSIZE_MAX (128 * 1024) /*< maximum input size for a single block compressed with HUF_compress */ +#define HUF_BLOCKSIZE_MAX (128 * 1024) /**< maximum input size for a single block compressed with HUF_compress */ size_t HUF_compressBound(size_t size); /**< maximum compressed size (worst case) */ /* Error Management */ From 690753ea1aa8d82b7c9149920c3b31de3a6071ae Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Fri, 2 Dec 2016 16:20:16 +0100 Subject: [PATCH 13/13] improved formatting in FIO_decompressSrcFile --- programs/fileio.c | 55 +++++++++++++++++++++++------------------------ 1 file changed, 27 insertions(+), 28 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index 06728ae31..3e30c1e5e 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -721,37 +721,36 @@ static int FIO_decompressSrcFile(dRess_t ress, const char* srcFileName) /* for each frame */ for ( ; ; ) { /* check magic number -> version */ - { size_t const toRead = 4; - size_t const sizeCheck = fread(ress.srcBuffer, (size_t)1, toRead, srcFile); - const BYTE* buf = (const BYTE*)ress.srcBuffer; - if (sizeCheck==0) { - if (readSomething==0) { DISPLAY("zstd: %s: unexpected end of file \n", srcFileName); fclose(srcFile); return 1; } /* srcFileName is empty */ - break; /* no more input */ - } - readSomething = 1; /* there is at least >= 4 bytes in srcFile */ - if (sizeCheck != toRead) { DISPLAY("zstd: %s: unknown header \n", srcFileName); fclose(srcFile); return 1; } /* srcFileName is empty */ - if (buf[0] == 31 && buf[1] == 139) { /* gz header */ + size_t const toRead = 4; + size_t const sizeCheck = fread(ress.srcBuffer, (size_t)1, toRead, srcFile); + const BYTE* buf = (const BYTE*)ress.srcBuffer; + if (sizeCheck==0) { + if (readSomething==0) { DISPLAY("zstd: %s: unexpected end of file \n", srcFileName); fclose(srcFile); return 1; } /* srcFileName is empty */ + break; /* no more input */ + } + readSomething = 1; /* there is at least >= 4 bytes in srcFile */ + if (sizeCheck != toRead) { DISPLAY("zstd: %s: unknown header \n", srcFileName); fclose(srcFile); return 1; } /* srcFileName is empty */ + if (buf[0] == 31 && buf[1] == 139) { /* gz header */ #ifdef ZSTD_GZDECOMPRESS - size_t const result = FIO_decompressGzFile(ress, toRead, srcFileName, srcFile); - if (result == 0) return 1; - filesize += result; + size_t const result = FIO_decompressGzFile(ress, toRead, srcFileName, srcFile); + if (result == 0) return 1; + filesize += result; #else - DISPLAYLEVEL(1, "zstd: %s: gzip file cannot be uncompressed -- ignored (zstd compiled without ZSTD_GZDECOMPRESS) \n", srcFileName); - return 1; + DISPLAYLEVEL(1, "zstd: %s: gzip file cannot be uncompressed -- ignored (zstd compiled without ZSTD_GZDECOMPRESS) \n", srcFileName); + return 1; #endif - } else { - if (!ZSTD_isFrame(ress.srcBuffer, toRead)) { - if ((g_overwrite) && !strcmp (srcFileName, stdinmark)) { /* pass-through mode */ - unsigned const result = FIO_passThrough(dstFile, srcFile, ress.srcBuffer, ress.srcBufferSize); - if (fclose(srcFile)) EXM_THROW(32, "zstd: %s close error", srcFileName); /* error should never happen */ - return result; - } else { - DISPLAYLEVEL(1, "zstd: %s: not in zstd format \n", srcFileName); - fclose(srcFile); - return 1; - } } - filesize += FIO_decompressFrame(ress, dstFile, srcFile, toRead, filesize); - } + } else { + if (!ZSTD_isFrame(ress.srcBuffer, toRead)) { + if ((g_overwrite) && !strcmp (srcFileName, stdinmark)) { /* pass-through mode */ + unsigned const result = FIO_passThrough(dstFile, srcFile, ress.srcBuffer, ress.srcBufferSize); + if (fclose(srcFile)) EXM_THROW(32, "zstd: %s close error", srcFileName); /* error should never happen */ + return result; + } else { + DISPLAYLEVEL(1, "zstd: %s: not in zstd format \n", srcFileName); + fclose(srcFile); + return 1; + } } + filesize += FIO_decompressFrame(ress, dstFile, srcFile, toRead, filesize); } }