From 901435e9ef324b7e639bb9027ad544677b338f00 Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Mon, 5 Jun 2017 14:45:31 -0700 Subject: [PATCH 01/54] setup basic functions for adding --list functionality --- programs/fileio.c | 5 +++++ programs/fileio.h | 1 + programs/zstdcli.c | 17 ++++++++++++++--- 3 files changed, 20 insertions(+), 3 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index 44152b0b1..466faf385 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -862,6 +862,11 @@ int FIO_compressFilename(const char* dstFileName, const char* srcFileName, return result; } +int FIO_listFile(const char* infilename){ + DISPLAY("FILE DETECTED: %s\n", infilename); + DISPLAY("working\n"); + return 0; +} int FIO_compressMultipleFilenames(const char** inFileNamesTable, unsigned nbFiles, const char* suffix, diff --git a/programs/fileio.h b/programs/fileio.h index 65da98d7f..4ca1a2b38 100644 --- a/programs/fileio.h +++ b/programs/fileio.h @@ -70,6 +70,7 @@ int FIO_compressFilename (const char* outfilename, const char* infilename, const @return : 0 == ok; 1 == pb with src file. */ int FIO_decompressFilename (const char* outfilename, const char* infilename, const char* dictFileName); +int FIO_listFile(const char* infilename); /*-************************************* * Multiple File functions diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 32fef9993..be0a447bb 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -117,6 +117,7 @@ static int usage_advanced(const char* programName) DISPLAY( " -v : verbose mode; specify multiple times to increase verbosity\n"); DISPLAY( " -q : suppress warnings; specify twice to suppress errors too\n"); DISPLAY( " -c : force write to standard output, even if it is the console\n"); + DISPLAY( " -l : print information about zstd compressed files.\n"); #ifndef ZSTD_NOCOMPRESS DISPLAY( "--ultra : enable levels beyond %i, up to %i (requires more memory)\n", ZSTDCLI_CLEVEL_MAX, ZSTD_maxCLevel()); #ifdef ZSTD_MULTITHREAD @@ -310,7 +311,7 @@ static unsigned parseCompressionParameters(const char* stringPtr, ZSTD_compressi } -typedef enum { zom_compress, zom_decompress, zom_test, zom_bench, zom_train } zstd_operation_mode; +typedef enum { zom_compress, zom_decompress, zom_test, zom_bench, zom_train, zom_list } zstd_operation_mode; #define CLEAN_RETURN(i) { operationResult = (i); goto _end; } @@ -531,7 +532,7 @@ int main(int argCount, const char* argv[]) argument++; memLimit = readU32FromChar(&argument); break; - + case 'l': operation=zom_list; argument++; break; #ifdef UTIL_HAS_CREATEFILELIST /* recursive */ case 'r': recursive=1; argument++; break; @@ -672,7 +673,17 @@ int main(int argCount, const char* argv[]) } } #endif - + if(operation==zom_list){ + DISPLAY("===========================================\n"); + DISPLAY("Printing information about compressed files\n"); + DISPLAY("===========================================\n"); + unsigned u; + DISPLAY("Number of files listed: %d\n", filenameIdx); + for(u=0; u Date: Mon, 5 Jun 2017 15:00:06 -0700 Subject: [PATCH 02/54] added in check suffix check to ensure file was compressed with zstd --- programs/fileio.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/programs/fileio.c b/programs/fileio.c index 466faf385..7a8907ab9 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -864,7 +864,11 @@ int FIO_compressFilename(const char* dstFileName, const char* srcFileName, int FIO_listFile(const char* infilename){ DISPLAY("FILE DETECTED: %s\n", infilename); - DISPLAY("working\n"); + const char* const suffixPtr = strrchr(infilename, '.'); + if(!suffixPtr || strcmp(suffixPtr, ZSTD_EXTENSION)){ + DISPLAYLEVEL(1, "file %s was not compressed with zstd -- ignoring\n", infilename); + return 1; + } return 0; } From 0f06f4f2663269161820a9dfa9a9fdfa11adf78e Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Tue, 6 Jun 2017 09:21:42 -0700 Subject: [PATCH 03/54] added display for compressed size --- programs/fileio.c | 12 +++++++++++- programs/fileio.h | 2 +- programs/zstdcli.c | 2 +- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index 7a8907ab9..c836020c9 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -862,13 +862,23 @@ int FIO_compressFilename(const char* dstFileName, const char* srcFileName, return result; } -int FIO_listFile(const char* infilename){ +int FIO_listFile(const char* infilename, int displayLevel){ DISPLAY("FILE DETECTED: %s\n", infilename); const char* const suffixPtr = strrchr(infilename, '.'); if(!suffixPtr || strcmp(suffixPtr, ZSTD_EXTENSION)){ DISPLAYLEVEL(1, "file %s was not compressed with zstd -- ignoring\n", infilename); return 1; } + else{ + U64 const compSize = UTIL_getFileSize(infilename); + if(displayLevel<=2){ + DISPLAY("Skippable Non-Skippable Compressed Uncompressed Ratio Check Filename\n"); + DISPLAY(" %7.2f MB\n", (double)compSize/(1 MB)); + } + else{ + DISPLAY("Compressed Size: %.2f MB (%llu B)\n", (double)compSize/(1 MB), compSize); + } + } return 0; } diff --git a/programs/fileio.h b/programs/fileio.h index 4ca1a2b38..b28ba1de7 100644 --- a/programs/fileio.h +++ b/programs/fileio.h @@ -70,7 +70,7 @@ int FIO_compressFilename (const char* outfilename, const char* infilename, const @return : 0 == ok; 1 == pb with src file. */ int FIO_decompressFilename (const char* outfilename, const char* infilename, const char* dictFileName); -int FIO_listFile(const char* infilename); +int FIO_listFile(const char* infilename, int displayLevel); /*-************************************* * Multiple File functions diff --git a/programs/zstdcli.c b/programs/zstdcli.c index be0a447bb..74f2606bc 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -680,7 +680,7 @@ int main(int argCount, const char* argv[]) unsigned u; DISPLAY("Number of files listed: %d\n", filenameIdx); for(u=0; u Date: Mon, 12 Jun 2017 10:58:34 -0700 Subject: [PATCH 04/54] added line spacing for clarity --- programs/fileio.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index c836020c9..a7812d73a 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -867,18 +867,20 @@ int FIO_listFile(const char* infilename, int displayLevel){ const char* const suffixPtr = strrchr(infilename, '.'); if(!suffixPtr || strcmp(suffixPtr, ZSTD_EXTENSION)){ DISPLAYLEVEL(1, "file %s was not compressed with zstd -- ignoring\n", infilename); - return 1; + DISPLAY("\n"); + return 1; } else{ - U64 const compSize = UTIL_getFileSize(infilename); + U64 const compSize = UTIL_getFileSize(infilename); if(displayLevel<=2){ - DISPLAY("Skippable Non-Skippable Compressed Uncompressed Ratio Check Filename\n"); - DISPLAY(" %7.2f MB\n", (double)compSize/(1 MB)); + DISPLAY("Skippable Non-Skippable Compressed Uncompressed Ratio Check Filename\n"); + DISPLAY(" %7.2f MB\n", (double)compSize/(1 MB)); } else{ DISPLAY("Compressed Size: %.2f MB (%llu B)\n", (double)compSize/(1 MB), compSize); } } + DISPLAY("\n"); return 0; } From 786b7cac27634e87a1cebdf3055e762fd15180f0 Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Mon, 12 Jun 2017 13:46:39 -0700 Subject: [PATCH 05/54] added code to analyze the first frame header --- programs/fileio.c | 70 ++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 63 insertions(+), 7 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index a7812d73a..bf92f6b11 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -862,22 +862,78 @@ int FIO_compressFilename(const char* dstFileName, const char* srcFileName, return result; } -int FIO_listFile(const char* infilename, int displayLevel){ - DISPLAY("FILE DETECTED: %s\n", infilename); - const char* const suffixPtr = strrchr(infilename, '.'); +typedef struct { + int numActualFrames; + int numSkippableFrames; + U64 decompressedSize; + U64 compressedSize; + int usesCheck; +} fileInfo_t; + +/* + * Reads information from file, stores in *info + * if successful, returns 0, otherwise returns 1 + */ +int getFileInfo(fileInfo_t* info, const char* inFileName){ + FILE* srcFile = FIO_openSrcFile(inFileName); + if(srcFile==NULL){ + return 1; + } + info->compressedSize = UTIL_getFileSize(inFileName); + BYTE* frameHeaderBuffer = malloc(ZSTD_frameHeaderSize_max); + fread(frameHeaderBuffer, ZSTD_frameHeaderSize_max, 1, srcFile); + info->decompressedSize = 0; + info->numActualFrames = 0; + info-> numSkippableFrames = 0; + + /* begin analyzing frame */ + + info->decompressedSize += ZSTD_getFrameContentSize((void*)frameHeaderBuffer, ZSTD_FRAMEHEADERSIZE_MAX); + U32 magicNumber = frameHeaderBuffer[0] + (frameHeaderBuffer[1] << 8) + (frameHeaderBuffer[2] << 16) + (frameHeaderBuffer[3] << 24); + if(magicNumber==ZSTD_MAGICNUMBER){ + info->numActualFrames++; + } + else if(magicNumber==ZSTD_MAGIC_SKIPPABLE_START){ + info->numSkippableFrames++; + } + const int checksumBitMask = 4; + if(frameHeaderBuffer[1] & checksumBitMask){ + DISPLAY("Uses checksum\n"); + info->usesCheck = 1; + } + return 0; +} + +int FIO_listFile(const char* inFileName, int displayLevel){ + DISPLAY("FILE DETECTED: %s\n", inFileName); + const char* const suffixPtr = strrchr(inFileName, '.'); if(!suffixPtr || strcmp(suffixPtr, ZSTD_EXTENSION)){ - DISPLAYLEVEL(1, "file %s was not compressed with zstd -- ignoring\n", infilename); + DISPLAYLEVEL(1, "file %s was not compressed with zstd -- ignoring\n", inFileName); DISPLAY("\n"); return 1; } else{ - U64 const compSize = UTIL_getFileSize(infilename); + fileInfo_t* info = malloc(sizeof(fileInfo_t)); + int error = getFileInfo(info, inFileName); + if(error==1){ + DISPLAY("An error occurred with getting file info\n"); + exit(1); + } if(displayLevel<=2){ DISPLAY("Skippable Non-Skippable Compressed Uncompressed Ratio Check Filename\n"); - DISPLAY(" %7.2f MB\n", (double)compSize/(1 MB)); + DISPLAY(" %7.2f MB\n", (double)info->compressedSize/(1 MB)); } else{ - DISPLAY("Compressed Size: %.2f MB (%llu B)\n", (double)compSize/(1 MB), compSize); + DISPLAY("Compressed Size: %.2f MB (%llu B)\n", (double)info->compressedSize/(1 MB), info->compressedSize); + if(info->decompressedSize!=ZSTD_CONTENTSIZE_ERROR && info->decompressedSize!=ZSTD_CONTENTSIZE_UNKNOWN){ + DISPLAY("Decompressed Size: %.2f MB (%llu B)\n", (double)info->decompressedSize/(1 MB), info->decompressedSize); + } + else if(info->decompressedSize==ZSTD_CONTENTSIZE_ERROR){ + DISPLAY("Decompressed Size: There was an error with getting the decompressed size\n"); + } + else{ + DISPLAY("Decompressed Size: N/A\n"); + } } } DISPLAY("\n"); From 9cb602ee25736e81a004fd6caa667ab101510e41 Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Mon, 12 Jun 2017 15:22:48 -0700 Subject: [PATCH 06/54] added in logic for parsing through blocks/frames --- programs/fileio.c | 101 ++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 85 insertions(+), 16 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index bf92f6b11..a2d9b8f30 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -870,6 +870,8 @@ typedef struct { int usesCheck; } fileInfo_t; + + /* * Reads information from file, stores in *info * if successful, returns 0, otherwise returns 1 @@ -880,26 +882,93 @@ int getFileInfo(fileInfo_t* info, const char* inFileName){ return 1; } info->compressedSize = UTIL_getFileSize(inFileName); - BYTE* frameHeaderBuffer = malloc(ZSTD_frameHeaderSize_max); - fread(frameHeaderBuffer, ZSTD_frameHeaderSize_max, 1, srcFile); info->decompressedSize = 0; info->numActualFrames = 0; info-> numSkippableFrames = 0; - /* begin analyzing frame */ - - info->decompressedSize += ZSTD_getFrameContentSize((void*)frameHeaderBuffer, ZSTD_FRAMEHEADERSIZE_MAX); - U32 magicNumber = frameHeaderBuffer[0] + (frameHeaderBuffer[1] << 8) + (frameHeaderBuffer[2] << 16) + (frameHeaderBuffer[3] << 24); - if(magicNumber==ZSTD_MAGICNUMBER){ - info->numActualFrames++; - } - else if(magicNumber==ZSTD_MAGIC_SKIPPABLE_START){ - info->numSkippableFrames++; - } - const int checksumBitMask = 4; - if(frameHeaderBuffer[1] & checksumBitMask){ - DISPLAY("Uses checksum\n"); - info->usesCheck = 1; + while(1){ + BYTE magicNumberBuffer[4]; + int numBytesRead = fread(magicNumberBuffer, 1, 4, srcFile); + if(numBytesRead != 4) break; + U32 magicNumber = MEM_readLE32(magicNumberBuffer); + if(magicNumber==ZSTD_MAGICNUMBER){ + DISPLAY("Hello\n"); + int frameContentSizeBytes=0; + info->numActualFrames++; + BYTE frameHeaderDescriptor; + fread(&frameHeaderDescriptor, 1, 1, srcFile); + + /* calculate actual frame header size */ + int frameContentSizeFlag = frameHeaderDescriptor >> 6; + int singleSegmentFlag = (frameHeaderDescriptor & (1 << 5)) >> 5; + int contentChecksumFlag = (frameHeaderDescriptor & (1 << 2)) >> 2; + int dictionaryIDFlag = frameHeaderDescriptor & 3; + if(frameContentSizeFlag!=0){ + frameContentSizeBytes = 1 << frameContentSizeFlag; + } + else if(singleSegmentFlag){ + frameContentSizeBytes = 1; + } + int windowDescriptorBytes = singleSegmentFlag ? 0 : 1; + int dictionaryIDBytes = dictionaryIDFlag ? 1 << (dictionaryIDFlag - 1): 0; + int totalFrameHeaderBytes = 4 + 1 + windowDescriptorBytes + frameContentSizeBytes + dictionaryIDBytes; + + /* reset to beginning of from and read entire header */ + fseek(srcFile, -5, SEEK_CUR); + BYTE* frameHeader = malloc(totalFrameHeaderBytes); + fread(frameHeader, totalFrameHeaderBytes, 1, srcFile); + + /* get decompressed file size */ + info->decompressedSize = ZSTD_getFrameContentSize(frameHeader, totalFrameHeaderBytes); + + /* check if checksum is used */ + if(contentChecksumFlag){ + info->usesCheck = 1; + } + + /* skip the rest of the blocks in the frame */ + int lastBlock = 0; + do{ + BYTE blockHeaderBuffer[3]; + fread(blockHeaderBuffer, 3, 1, srcFile); + U32 blockHeader = MEM_readLE24(blockHeaderBuffer); + lastBlock = blockHeader & 1; + int blockSize = (blockHeader - (blockHeader & 7)) >> 3; + fseek(srcFile, blockSize, SEEK_CUR); + }while(lastBlock != 1); + if(contentChecksumFlag){ + fseek(srcFile, 4, SEEK_CUR); + } + } + else if(magicNumber==ZSTD_MAGIC_SKIPPABLE_START){ + info->numSkippableFrames++; + BYTE frameSizeBuffer[4]; + fread(frameSizeBuffer, 4, 1, srcFile); + long frameSize = MEM_readLE32(frameSizeBuffer); + fseek(srcFile, frameSize, SEEK_CUR); + } + + + // int isSkippableFrame = 0; + // /* analyze the first 18 bytes (magic number + frame header) of the frame */ + // BYTE* frameHeaderBuffer = malloc(ZSTD_frameHeaderSize_max); + // fread(frameHeaderBuffer, ZSTD_frameHeaderSize_max, 1, srcFile); + // info->decompressedSize += ZSTD_getFrameContentSize((void*)frameHeaderBuffer, ZSTD_FRAMEHEADERSIZE_MAX); + // U32 magicNumber = frameHeaderBuffer[0] + (frameHeaderBuffer[1] << 8) + (frameHeaderBuffer[2] << 16) + (frameHeaderBuffer[3] << 24); + // if(magicNumber==ZSTD_MAGICNUMBER){ + // info->numActualFrames++; + // } + // else if(magicNumber==ZSTD_MAGIC_SKIPPABLE_START){ + // info->numSkippableFrames++; + // isSkippableFrame = 1; + // } + // const int checksumBitMask = 4; + // if(frameHeaderBuffer[1] & checksumBitMask){ + // DISPLAY("Uses checksum\n"); + // info->usesCheck = 1; + // } + + } return 0; } From 6996bd25985d13d7e4fe8274865353b0a951acd7 Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Mon, 12 Jun 2017 15:24:50 -0700 Subject: [PATCH 07/54] removed useless lines --- programs/fileio.c | 22 ---------------------- 1 file changed, 22 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index a2d9b8f30..2e8e4a6f6 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -892,7 +892,6 @@ int getFileInfo(fileInfo_t* info, const char* inFileName){ if(numBytesRead != 4) break; U32 magicNumber = MEM_readLE32(magicNumberBuffer); if(magicNumber==ZSTD_MAGICNUMBER){ - DISPLAY("Hello\n"); int frameContentSizeBytes=0; info->numActualFrames++; BYTE frameHeaderDescriptor; @@ -948,27 +947,6 @@ int getFileInfo(fileInfo_t* info, const char* inFileName){ fseek(srcFile, frameSize, SEEK_CUR); } - - // int isSkippableFrame = 0; - // /* analyze the first 18 bytes (magic number + frame header) of the frame */ - // BYTE* frameHeaderBuffer = malloc(ZSTD_frameHeaderSize_max); - // fread(frameHeaderBuffer, ZSTD_frameHeaderSize_max, 1, srcFile); - // info->decompressedSize += ZSTD_getFrameContentSize((void*)frameHeaderBuffer, ZSTD_FRAMEHEADERSIZE_MAX); - // U32 magicNumber = frameHeaderBuffer[0] + (frameHeaderBuffer[1] << 8) + (frameHeaderBuffer[2] << 16) + (frameHeaderBuffer[3] << 24); - // if(magicNumber==ZSTD_MAGICNUMBER){ - // info->numActualFrames++; - // } - // else if(magicNumber==ZSTD_MAGIC_SKIPPABLE_START){ - // info->numSkippableFrames++; - // isSkippableFrame = 1; - // } - // const int checksumBitMask = 4; - // if(frameHeaderBuffer[1] & checksumBitMask){ - // DISPLAY("Uses checksum\n"); - // info->usesCheck = 1; - // } - - } return 0; } From dcd6ba6dc64f56ff80150de6a904492d372c3c32 Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Mon, 12 Jun 2017 15:40:47 -0700 Subject: [PATCH 08/54] incremented decompressedSize instead of setting value --- programs/fileio.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/programs/fileio.c b/programs/fileio.c index 2e8e4a6f6..d5da78479 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -918,7 +918,7 @@ int getFileInfo(fileInfo_t* info, const char* inFileName){ fread(frameHeader, totalFrameHeaderBytes, 1, srcFile); /* get decompressed file size */ - info->decompressedSize = ZSTD_getFrameContentSize(frameHeader, totalFrameHeaderBytes); + info->decompressedSize += ZSTD_getFrameContentSize(frameHeader, totalFrameHeaderBytes); /* check if checksum is used */ if(contentChecksumFlag){ From 6e0204470a745df9e9bbee053ca5a4e78bdf7ce6 Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Mon, 12 Jun 2017 15:51:59 -0700 Subject: [PATCH 09/54] displayed decompressed size --- programs/fileio.c | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index d5da78479..d29589544 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -866,6 +866,7 @@ typedef struct { int numActualFrames; int numSkippableFrames; U64 decompressedSize; + int canComputeDecompSize; U64 compressedSize; int usesCheck; } fileInfo_t; @@ -885,6 +886,7 @@ int getFileInfo(fileInfo_t* info, const char* inFileName){ info->decompressedSize = 0; info->numActualFrames = 0; info-> numSkippableFrames = 0; + info->canComputeDecompSize = 1; /* begin analyzing frame */ while(1){ BYTE magicNumberBuffer[4]; @@ -918,7 +920,13 @@ int getFileInfo(fileInfo_t* info, const char* inFileName){ fread(frameHeader, totalFrameHeaderBytes, 1, srcFile); /* get decompressed file size */ - info->decompressedSize += ZSTD_getFrameContentSize(frameHeader, totalFrameHeaderBytes); + U64 additional = ZSTD_getFrameContentSize(frameHeader, totalFrameHeaderBytes); + if(additional!=ZSTD_CONTENTSIZE_UNKNOWN && additional!=ZSTD_CONTENTSIZE_ERROR){ + info->decompressedSize += additional; + } + else{ + info->canComputeDecompSize = 0; + } /* check if checksum is used */ if(contentChecksumFlag){ @@ -966,9 +974,14 @@ int FIO_listFile(const char* inFileName, int displayLevel){ DISPLAY("An error occurred with getting file info\n"); exit(1); } + + double compressedSizeMB = (double)info->compressedSize/(1 MB); + double decompressedSizeMB = (double)info->decompressedSize/(1 MB); + DISPLAY("%d %d\n", info->canComputeDecompSize, info->usesCheck); + if(displayLevel<=2){ DISPLAY("Skippable Non-Skippable Compressed Uncompressed Ratio Check Filename\n"); - DISPLAY(" %7.2f MB\n", (double)info->compressedSize/(1 MB)); + DISPLAY(" %7.2f MB %7.2f MB\n", compressedSizeMB, decompressedSizeMB); } else{ DISPLAY("Compressed Size: %.2f MB (%llu B)\n", (double)info->compressedSize/(1 MB), info->compressedSize); From f3d6ab28a250445f78f01bbf92bc2de01ad4fe86 Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Mon, 12 Jun 2017 15:59:28 -0700 Subject: [PATCH 10/54] formated information for verbose output --- programs/fileio.c | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index d29589544..b8dfd898e 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -977,22 +977,21 @@ int FIO_listFile(const char* inFileName, int displayLevel){ double compressedSizeMB = (double)info->compressedSize/(1 MB); double decompressedSizeMB = (double)info->decompressedSize/(1 MB); - DISPLAY("%d %d\n", info->canComputeDecompSize, info->usesCheck); if(displayLevel<=2){ DISPLAY("Skippable Non-Skippable Compressed Uncompressed Ratio Check Filename\n"); DISPLAY(" %7.2f MB %7.2f MB\n", compressedSizeMB, decompressedSizeMB); } else{ - DISPLAY("Compressed Size: %.2f MB (%llu B)\n", (double)info->compressedSize/(1 MB), info->compressedSize); - if(info->decompressedSize!=ZSTD_CONTENTSIZE_ERROR && info->decompressedSize!=ZSTD_CONTENTSIZE_UNKNOWN){ - DISPLAY("Decompressed Size: %.2f MB (%llu B)\n", (double)info->decompressedSize/(1 MB), info->decompressedSize); + DISPLAY("# Zstandard Frames: %d\n", info->numActualFrames); + DISPLAY("# Skippable Frames: %d\n", info->numSkippableFrames); + DISPLAY("Compressed Size: %.2f MB (%llu B)\n", compressedSizeMB, info->compressedSize); + if(info->canComputeDecompSize){ + DISPLAY("Decompressed Size: %.2f MB (%llu B)\n", decompressedSizeMB, info->decompressedSize); + DISPLAY("Ratio: %.4f\n", compressedSizeMB/decompressedSizeMB); } - else if(info->decompressedSize==ZSTD_CONTENTSIZE_ERROR){ - DISPLAY("Decompressed Size: There was an error with getting the decompressed size\n"); - } - else{ - DISPLAY("Decompressed Size: N/A\n"); + if(info->usesCheck){ + DISPLAY("Check: XXH64\n"); } } } From 173a7397170ffe2c1633cc319140b5142afe3579 Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Mon, 12 Jun 2017 16:14:04 -0700 Subject: [PATCH 11/54] code refactorings with explicit conversion --- programs/fileio.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index b8dfd898e..29029238e 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -890,7 +890,7 @@ int getFileInfo(fileInfo_t* info, const char* inFileName){ /* begin analyzing frame */ while(1){ BYTE magicNumberBuffer[4]; - int numBytesRead = fread(magicNumberBuffer, 1, 4, srcFile); + size_t numBytesRead = fread(magicNumberBuffer, 1, 4, srcFile); if(numBytesRead != 4) break; U32 magicNumber = MEM_readLE32(magicNumberBuffer); if(magicNumber==ZSTD_MAGICNUMBER){ @@ -916,7 +916,7 @@ int getFileInfo(fileInfo_t* info, const char* inFileName){ /* reset to beginning of from and read entire header */ fseek(srcFile, -5, SEEK_CUR); - BYTE* frameHeader = malloc(totalFrameHeaderBytes); + BYTE* frameHeader = (BYTE*)malloc(totalFrameHeaderBytes); fread(frameHeader, totalFrameHeaderBytes, 1, srcFile); /* get decompressed file size */ @@ -968,7 +968,7 @@ int FIO_listFile(const char* inFileName, int displayLevel){ return 1; } else{ - fileInfo_t* info = malloc(sizeof(fileInfo_t)); + fileInfo_t* info = (fileInfo_t*)malloc(sizeof(fileInfo_t)); int error = getFileInfo(info, inFileName); if(error==1){ DISPLAY("An error occurred with getting file info\n"); From 6e33c74133a78673f6fa5729b80ab423b0d38f9c Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Mon, 12 Jun 2017 16:29:00 -0700 Subject: [PATCH 12/54] formatting issues resolved --- programs/fileio.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index 29029238e..73912be5a 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -985,9 +985,9 @@ int FIO_listFile(const char* inFileName, int displayLevel){ else{ DISPLAY("# Zstandard Frames: %d\n", info->numActualFrames); DISPLAY("# Skippable Frames: %d\n", info->numSkippableFrames); - DISPLAY("Compressed Size: %.2f MB (%llu B)\n", compressedSizeMB, info->compressedSize); + DISPLAY("Compressed Size: %.2f MB (%lu B)\n", compressedSizeMB, info->compressedSize); if(info->canComputeDecompSize){ - DISPLAY("Decompressed Size: %.2f MB (%llu B)\n", decompressedSizeMB, info->decompressedSize); + DISPLAY("Decompressed Size: %.2f MB (%lu B)\n", decompressedSizeMB, info->decompressedSize); DISPLAY("Ratio: %.4f\n", compressedSizeMB/decompressedSizeMB); } if(info->usesCheck){ From 618a7b66e2977edc113516e0d362d517934e606e Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Mon, 12 Jun 2017 16:53:50 -0700 Subject: [PATCH 13/54] refactoring for C90 syntax --- programs/fileio.c | 66 ++++++++++++++++++++++++++++++++-------------- programs/zstdcli.c | 8 +++--- 2 files changed, 50 insertions(+), 24 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index 73912be5a..e49d0386f 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -891,36 +891,54 @@ int getFileInfo(fileInfo_t* info, const char* inFileName){ while(1){ BYTE magicNumberBuffer[4]; size_t numBytesRead = fread(magicNumberBuffer, 1, 4, srcFile); + U32 magicNumber; if(numBytesRead != 4) break; - U32 magicNumber = MEM_readLE32(magicNumberBuffer); + magicNumber = MEM_readLE32(magicNumberBuffer); if(magicNumber==ZSTD_MAGICNUMBER){ - int frameContentSizeBytes=0; - info->numActualFrames++; BYTE frameHeaderDescriptor; - fread(&frameHeaderDescriptor, 1, 1, srcFile); - + int frameContentSizeFlag; + int singleSegmentFlag; + int contentChecksumFlag; + int dictionaryIDFlag; + int frameContentSizeBytes=0; + int windowDescriptorBytes; + int dictionaryIDBytes; + int totalFrameHeaderBytes; + BYTE* frameHeader; + U64 additional; + int lastBlock; + size_t readBytes = fread(&frameHeaderDescriptor, 1, 1, srcFile); + info->numActualFrames++; + if(readBytes != 1){ + DISPLAY("There was an error with reading frame header descriptor\n"); + exit(1); + } /* calculate actual frame header size */ - int frameContentSizeFlag = frameHeaderDescriptor >> 6; - int singleSegmentFlag = (frameHeaderDescriptor & (1 << 5)) >> 5; - int contentChecksumFlag = (frameHeaderDescriptor & (1 << 2)) >> 2; - int dictionaryIDFlag = frameHeaderDescriptor & 3; + frameContentSizeFlag = frameHeaderDescriptor >> 6; + singleSegmentFlag = (frameHeaderDescriptor & (1 << 5)) >> 5; + contentChecksumFlag = (frameHeaderDescriptor & (1 << 2)) >> 2; + dictionaryIDFlag = frameHeaderDescriptor & 3; if(frameContentSizeFlag!=0){ frameContentSizeBytes = 1 << frameContentSizeFlag; } else if(singleSegmentFlag){ frameContentSizeBytes = 1; } - int windowDescriptorBytes = singleSegmentFlag ? 0 : 1; - int dictionaryIDBytes = dictionaryIDFlag ? 1 << (dictionaryIDFlag - 1): 0; - int totalFrameHeaderBytes = 4 + 1 + windowDescriptorBytes + frameContentSizeBytes + dictionaryIDBytes; + windowDescriptorBytes = singleSegmentFlag ? 0 : 1; + dictionaryIDBytes = dictionaryIDFlag ? 1 << (dictionaryIDFlag - 1): 0; + totalFrameHeaderBytes = 4 + 1 + windowDescriptorBytes + frameContentSizeBytes + dictionaryIDBytes; /* reset to beginning of from and read entire header */ fseek(srcFile, -5, SEEK_CUR); - BYTE* frameHeader = (BYTE*)malloc(totalFrameHeaderBytes); - fread(frameHeader, totalFrameHeaderBytes, 1, srcFile); + frameHeader = (BYTE*)malloc(totalFrameHeaderBytes); + readBytes = fread(frameHeader, 1, totalFrameHeaderBytes, srcFile); + if(readBytes != (size_t)totalFrameHeaderBytes){ + DISPLAY("There was an error reading the frame header\n"); + exit(1); + } /* get decompressed file size */ - U64 additional = ZSTD_getFrameContentSize(frameHeader, totalFrameHeaderBytes); + additional = ZSTD_getFrameContentSize(frameHeader, totalFrameHeaderBytes); if(additional!=ZSTD_CONTENTSIZE_UNKNOWN && additional!=ZSTD_CONTENTSIZE_ERROR){ info->decompressedSize += additional; } @@ -934,10 +952,14 @@ int getFileInfo(fileInfo_t* info, const char* inFileName){ } /* skip the rest of the blocks in the frame */ - int lastBlock = 0; + lastBlock = 0; do{ BYTE blockHeaderBuffer[3]; - fread(blockHeaderBuffer, 3, 1, srcFile); + readBytes = fread(blockHeaderBuffer, 1, 3, srcFile); + if(readBytes != 3){ + DISPLAY("There was a problem reading the block header\n"); + exit(1); + } U32 blockHeader = MEM_readLE24(blockHeaderBuffer); lastBlock = blockHeader & 1; int blockSize = (blockHeader - (blockHeader & 7)) >> 3; @@ -950,7 +972,11 @@ int getFileInfo(fileInfo_t* info, const char* inFileName){ else if(magicNumber==ZSTD_MAGIC_SKIPPABLE_START){ info->numSkippableFrames++; BYTE frameSizeBuffer[4]; - fread(frameSizeBuffer, 4, 1, srcFile); + size_t readBytes = fread(frameSizeBuffer, 1, 4, srcFile); + if(readBytes != 4){ + DISPLAY("There was an error reading skippable frame size"); + exit(1); + } long frameSize = MEM_readLE32(frameSizeBuffer); fseek(srcFile, frameSize, SEEK_CUR); } @@ -985,9 +1011,9 @@ int FIO_listFile(const char* inFileName, int displayLevel){ else{ DISPLAY("# Zstandard Frames: %d\n", info->numActualFrames); DISPLAY("# Skippable Frames: %d\n", info->numSkippableFrames); - DISPLAY("Compressed Size: %.2f MB (%lu B)\n", compressedSizeMB, info->compressedSize); + DISPLAY("Compressed Size: %.2f MB (%llu B)\n", compressedSizeMB, info->compressedSize); if(info->canComputeDecompSize){ - DISPLAY("Decompressed Size: %.2f MB (%lu B)\n", decompressedSizeMB, info->decompressedSize); + DISPLAY("Decompressed Size: %.2f MB (%llu B)\n", decompressedSizeMB, info->decompressedSize); DISPLAY("Ratio: %.4f\n", compressedSizeMB/decompressedSizeMB); } if(info->usesCheck){ diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 74f2606bc..91bce8bb8 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -674,13 +674,13 @@ int main(int argCount, const char* argv[]) } #endif if(operation==zom_list){ - DISPLAY("===========================================\n"); - DISPLAY("Printing information about compressed files\n"); - DISPLAY("===========================================\n"); unsigned u; + DISPLAY("===========================================\n"); + DISPLAY("Printing information about compressed files\n"); + DISPLAY("===========================================\n"); DISPLAY("Number of files listed: %d\n", filenameIdx); for(u=0; u Date: Mon, 12 Jun 2017 17:09:08 -0700 Subject: [PATCH 14/54] more refactoring for C90 --- programs/fileio.c | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index e49d0386f..497e202d1 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -955,14 +955,16 @@ int getFileInfo(fileInfo_t* info, const char* inFileName){ lastBlock = 0; do{ BYTE blockHeaderBuffer[3]; + U32 blockHeader; + int blockSize; readBytes = fread(blockHeaderBuffer, 1, 3, srcFile); if(readBytes != 3){ DISPLAY("There was a problem reading the block header\n"); exit(1); } - U32 blockHeader = MEM_readLE24(blockHeaderBuffer); + blockHeader = MEM_readLE24(blockHeaderBuffer); lastBlock = blockHeader & 1; - int blockSize = (blockHeader - (blockHeader & 7)) >> 3; + blockSize = (blockHeader - (blockHeader & 7)) >> 3; fseek(srcFile, blockSize, SEEK_CUR); }while(lastBlock != 1); if(contentChecksumFlag){ @@ -970,14 +972,15 @@ int getFileInfo(fileInfo_t* info, const char* inFileName){ } } else if(magicNumber==ZSTD_MAGIC_SKIPPABLE_START){ - info->numSkippableFrames++; BYTE frameSizeBuffer[4]; + long frameSize; + info->numSkippableFrames++; size_t readBytes = fread(frameSizeBuffer, 1, 4, srcFile); if(readBytes != 4){ DISPLAY("There was an error reading skippable frame size"); exit(1); } - long frameSize = MEM_readLE32(frameSizeBuffer); + frameSize = MEM_readLE32(frameSizeBuffer); fseek(srcFile, frameSize, SEEK_CUR); } @@ -986,14 +989,16 @@ int getFileInfo(fileInfo_t* info, const char* inFileName){ } int FIO_listFile(const char* inFileName, int displayLevel){ - DISPLAY("FILE DETECTED: %s\n", inFileName); const char* const suffixPtr = strrchr(inFileName, '.'); + DISPLAY("FILE DETECTED: %s\n", inFileName); if(!suffixPtr || strcmp(suffixPtr, ZSTD_EXTENSION)){ DISPLAYLEVEL(1, "file %s was not compressed with zstd -- ignoring\n", inFileName); DISPLAY("\n"); return 1; } else{ + double compressedSizeMB; + double decompressedSizeMB; fileInfo_t* info = (fileInfo_t*)malloc(sizeof(fileInfo_t)); int error = getFileInfo(info, inFileName); if(error==1){ @@ -1001,8 +1006,8 @@ int FIO_listFile(const char* inFileName, int displayLevel){ exit(1); } - double compressedSizeMB = (double)info->compressedSize/(1 MB); - double decompressedSizeMB = (double)info->decompressedSize/(1 MB); + compressedSizeMB = (double)info->compressedSize/(1 MB); + decompressedSizeMB = (double)info->decompressedSize/(1 MB); if(displayLevel<=2){ DISPLAY("Skippable Non-Skippable Compressed Uncompressed Ratio Check Filename\n"); From c828b521119b55d12fccc8ae14a949b3b8bf66ed Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Mon, 12 Jun 2017 17:19:53 -0700 Subject: [PATCH 15/54] yet more refactoring --- programs/fileio.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/programs/fileio.c b/programs/fileio.c index 497e202d1..bf8d6d61b 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -974,8 +974,8 @@ int getFileInfo(fileInfo_t* info, const char* inFileName){ else if(magicNumber==ZSTD_MAGIC_SKIPPABLE_START){ BYTE frameSizeBuffer[4]; long frameSize; - info->numSkippableFrames++; size_t readBytes = fread(frameSizeBuffer, 1, 4, srcFile); + info->numSkippableFrames++; if(readBytes != 4){ DISPLAY("There was an error reading skippable frame size"); exit(1); From a56dcbfcf8d41724414f3cc662d6fcbc6242e3d7 Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Mon, 12 Jun 2017 17:47:33 -0700 Subject: [PATCH 16/54] changed print format for non-verbose version --- programs/fileio.c | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index bf8d6d61b..dadb8ecb3 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -1010,8 +1010,28 @@ int FIO_listFile(const char* inFileName, int displayLevel){ decompressedSizeMB = (double)info->decompressedSize/(1 MB); if(displayLevel<=2){ - DISPLAY("Skippable Non-Skippable Compressed Uncompressed Ratio Check Filename\n"); - DISPLAY(" %7.2f MB %7.2f MB\n", compressedSizeMB, decompressedSizeMB); + if(info->usesCheck && info->canComputeDecompSize){ + DISPLAY("Skippable Non-Skippable Compressed Uncompressed Ratio Check Filename\n"); + DISPLAY("%9d %13d %7.2f MB %7.2f MB %5.3f XXH64 %s\n", + info->numSkippableFrames, info->numActualFrames, compressedSizeMB, decompressedSizeMB, + compressedSizeMB/decompressedSizeMB, inFileName); + } + else if(!info->usesCheck){ + DISPLAY("Skippable Non-Skippable Compressed Uncompressed Ratio Check Filename\n"); + DISPLAY("%9d %13d %7.2f MB %7.2f MB %5.3f %s\n", + info->numSkippableFrames, info->numActualFrames, compressedSizeMB, decompressedSizeMB, + compressedSizeMB/decompressedSizeMB, inFileName); + } + else if(!info->canComputeDecompSize){ + DISPLAY("Skippable Non-Skippable Compressed Uncompressed Ratio Check Filename\n"); + DISPLAY("%9d %13d %7.2f MB XXH64 %s\n", + info->numSkippableFrames, info->numActualFrames, compressedSizeMB, inFileName); + } + else{ + DISPLAY("Skippable Non-Skippable Filename\n"); + DISPLAY("%9d %13d %7.2f MB %s\n", + info->numSkippableFrames, info->numActualFrames, compressedSizeMB, inFileName); + } } else{ DISPLAY("# Zstandard Frames: %d\n", info->numActualFrames); From 81fa33b55a8db2256c26391b3531de16bc1f23bf Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Tue, 13 Jun 2017 10:06:49 -0700 Subject: [PATCH 17/54] cleaning up code --- programs/fileio.c | 162 +++++++++++++++++++++++----------------------- 1 file changed, 81 insertions(+), 81 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index dadb8ecb3..ca08c7f9d 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -872,6 +872,23 @@ typedef struct { } fileInfo_t; +int calcFrameHeaderSize(BYTE frameHeaderDescriptor){ + int frameContentSizeBytes = 0; + int windowDescriptorBytes; + int dictionaryIDBytes; + int frameContentSizeFlag = frameHeaderDescriptor >> 6; + int singleSegmentFlag = (frameHeaderDescriptor & (1 << 5)) >> 5; + int dictionaryIDFlag = frameHeaderDescriptor & 3; + if(frameContentSizeFlag!=0){ + frameContentSizeBytes = 1 << frameContentSizeFlag; + } + else if(singleSegmentFlag){ + frameContentSizeBytes = 1; + } + windowDescriptorBytes = singleSegmentFlag ? 0 : 1; + dictionaryIDBytes = dictionaryIDFlag ? 1 << (dictionaryIDFlag - 1): 0; + return 4 + 1 + windowDescriptorBytes + frameContentSizeBytes + dictionaryIDBytes; +} /* * Reads information from file, stores in *info @@ -896,17 +913,9 @@ int getFileInfo(fileInfo_t* info, const char* inFileName){ magicNumber = MEM_readLE32(magicNumberBuffer); if(magicNumber==ZSTD_MAGICNUMBER){ BYTE frameHeaderDescriptor; - int frameContentSizeFlag; - int singleSegmentFlag; - int contentChecksumFlag; - int dictionaryIDFlag; - int frameContentSizeBytes=0; - int windowDescriptorBytes; - int dictionaryIDBytes; int totalFrameHeaderBytes; BYTE* frameHeader; - U64 additional; - int lastBlock; + int lastBlock = 0; size_t readBytes = fread(&frameHeaderDescriptor, 1, 1, srcFile); info->numActualFrames++; if(readBytes != 1){ @@ -914,19 +923,7 @@ int getFileInfo(fileInfo_t* info, const char* inFileName){ exit(1); } /* calculate actual frame header size */ - frameContentSizeFlag = frameHeaderDescriptor >> 6; - singleSegmentFlag = (frameHeaderDescriptor & (1 << 5)) >> 5; - contentChecksumFlag = (frameHeaderDescriptor & (1 << 2)) >> 2; - dictionaryIDFlag = frameHeaderDescriptor & 3; - if(frameContentSizeFlag!=0){ - frameContentSizeBytes = 1 << frameContentSizeFlag; - } - else if(singleSegmentFlag){ - frameContentSizeBytes = 1; - } - windowDescriptorBytes = singleSegmentFlag ? 0 : 1; - dictionaryIDBytes = dictionaryIDFlag ? 1 << (dictionaryIDFlag - 1): 0; - totalFrameHeaderBytes = 4 + 1 + windowDescriptorBytes + frameContentSizeBytes + dictionaryIDBytes; + totalFrameHeaderBytes = calcFrameHeaderSize(frameHeaderDescriptor); /* reset to beginning of from and read entire header */ fseek(srcFile, -5, SEEK_CUR); @@ -938,21 +935,17 @@ int getFileInfo(fileInfo_t* info, const char* inFileName){ } /* get decompressed file size */ - additional = ZSTD_getFrameContentSize(frameHeader, totalFrameHeaderBytes); - if(additional!=ZSTD_CONTENTSIZE_UNKNOWN && additional!=ZSTD_CONTENTSIZE_ERROR){ - info->decompressedSize += additional; - } - else{ - info->canComputeDecompSize = 0; - } - - /* check if checksum is used */ - if(contentChecksumFlag){ - info->usesCheck = 1; + { + U64 additional = ZSTD_getFrameContentSize(frameHeader, totalFrameHeaderBytes); + if(additional!=ZSTD_CONTENTSIZE_UNKNOWN && additional!=ZSTD_CONTENTSIZE_ERROR){ + info->decompressedSize += additional; + } + else{ + info->canComputeDecompSize = 0; + } } /* skip the rest of the blocks in the frame */ - lastBlock = 0; do{ BYTE blockHeaderBuffer[3]; U32 blockHeader; @@ -967,8 +960,15 @@ int getFileInfo(fileInfo_t* info, const char* inFileName){ blockSize = (blockHeader - (blockHeader & 7)) >> 3; fseek(srcFile, blockSize, SEEK_CUR); }while(lastBlock != 1); - if(contentChecksumFlag){ - fseek(srcFile, 4, SEEK_CUR); + { + /* check if checksum is used */ + int contentChecksumFlag = (frameHeaderDescriptor & (1 << 2)) >> 2; + if(contentChecksumFlag){ + info->usesCheck = 1; + } + if(contentChecksumFlag){ + fseek(srcFile, 4, SEEK_CUR); + } } } else if(magicNumber==ZSTD_MAGIC_SKIPPABLE_START){ @@ -987,64 +987,64 @@ int getFileInfo(fileInfo_t* info, const char* inFileName){ } return 0; } +void displayInfo(const char* inFileName, fileInfo_t* info, int displayLevel){ + double compressedSizeMB = (double)info->compressedSize/(1 MB); + double decompressedSizeMB = (double)info->decompressedSize/(1 MB); + + if(displayLevel<=2){ + if(info->usesCheck && info->canComputeDecompSize){ + DISPLAY("Skippable Non-Skippable Compressed Uncompressed Ratio Check Filename\n"); + DISPLAY("%9d %13d %7.2f MB %7.2f MB %5.3f XXH64 %s\n", + info->numSkippableFrames, info->numActualFrames, compressedSizeMB, decompressedSizeMB, + compressedSizeMB/decompressedSizeMB, inFileName); + } + else if(!info->usesCheck){ + DISPLAY("Skippable Non-Skippable Compressed Uncompressed Ratio Check Filename\n"); + DISPLAY("%9d %13d %7.2f MB %7.2f MB %5.3f %s\n", + info->numSkippableFrames, info->numActualFrames, compressedSizeMB, decompressedSizeMB, + compressedSizeMB/decompressedSizeMB, inFileName); + } + else if(!info->canComputeDecompSize){ + DISPLAY("Skippable Non-Skippable Compressed Uncompressed Ratio Check Filename\n"); + DISPLAY("%9d %13d %7.2f MB XXH64 %s\n", + info->numSkippableFrames, info->numActualFrames, compressedSizeMB, inFileName); + } + else{ + DISPLAY("Skippable Non-Skippable Filename\n"); + DISPLAY("%9d %13d %7.2f MB %s\n", + info->numSkippableFrames, info->numActualFrames, compressedSizeMB, inFileName); + } + } + else{ + DISPLAY("# Zstandard Frames: %d\n", info->numActualFrames); + DISPLAY("# Skippable Frames: %d\n", info->numSkippableFrames); + DISPLAY("Compressed Size: %.2f MB (%llu B)\n", compressedSizeMB, info->compressedSize); + if(info->canComputeDecompSize){ + DISPLAY("Decompressed Size: %.2f MB (%llu B)\n", decompressedSizeMB, info->decompressedSize); + DISPLAY("Ratio: %.4f\n", compressedSizeMB/decompressedSizeMB); + } + if(info->usesCheck){ + DISPLAY("Check: XXH64\n"); + } + } + +} int FIO_listFile(const char* inFileName, int displayLevel){ const char* const suffixPtr = strrchr(inFileName, '.'); - DISPLAY("FILE DETECTED: %s\n", inFileName); + DISPLAY("File: %s\n", inFileName); if(!suffixPtr || strcmp(suffixPtr, ZSTD_EXTENSION)){ - DISPLAYLEVEL(1, "file %s was not compressed with zstd -- ignoring\n", inFileName); - DISPLAY("\n"); + DISPLAYLEVEL(1, "file %s was not compressed with zstd -- ignoring\n\n", inFileName); return 1; } else{ - double compressedSizeMB; - double decompressedSizeMB; fileInfo_t* info = (fileInfo_t*)malloc(sizeof(fileInfo_t)); int error = getFileInfo(info, inFileName); if(error==1){ DISPLAY("An error occurred with getting file info\n"); exit(1); } - - compressedSizeMB = (double)info->compressedSize/(1 MB); - decompressedSizeMB = (double)info->decompressedSize/(1 MB); - - if(displayLevel<=2){ - if(info->usesCheck && info->canComputeDecompSize){ - DISPLAY("Skippable Non-Skippable Compressed Uncompressed Ratio Check Filename\n"); - DISPLAY("%9d %13d %7.2f MB %7.2f MB %5.3f XXH64 %s\n", - info->numSkippableFrames, info->numActualFrames, compressedSizeMB, decompressedSizeMB, - compressedSizeMB/decompressedSizeMB, inFileName); - } - else if(!info->usesCheck){ - DISPLAY("Skippable Non-Skippable Compressed Uncompressed Ratio Check Filename\n"); - DISPLAY("%9d %13d %7.2f MB %7.2f MB %5.3f %s\n", - info->numSkippableFrames, info->numActualFrames, compressedSizeMB, decompressedSizeMB, - compressedSizeMB/decompressedSizeMB, inFileName); - } - else if(!info->canComputeDecompSize){ - DISPLAY("Skippable Non-Skippable Compressed Uncompressed Ratio Check Filename\n"); - DISPLAY("%9d %13d %7.2f MB XXH64 %s\n", - info->numSkippableFrames, info->numActualFrames, compressedSizeMB, inFileName); - } - else{ - DISPLAY("Skippable Non-Skippable Filename\n"); - DISPLAY("%9d %13d %7.2f MB %s\n", - info->numSkippableFrames, info->numActualFrames, compressedSizeMB, inFileName); - } - } - else{ - DISPLAY("# Zstandard Frames: %d\n", info->numActualFrames); - DISPLAY("# Skippable Frames: %d\n", info->numSkippableFrames); - DISPLAY("Compressed Size: %.2f MB (%llu B)\n", compressedSizeMB, info->compressedSize); - if(info->canComputeDecompSize){ - DISPLAY("Decompressed Size: %.2f MB (%llu B)\n", decompressedSizeMB, info->decompressedSize); - DISPLAY("Ratio: %.4f\n", compressedSizeMB/decompressedSizeMB); - } - if(info->usesCheck){ - DISPLAY("Check: XXH64\n"); - } - } + displayInfo(inFileName, info, displayLevel); } DISPLAY("\n"); return 0; From ebd60349f52b58197328b84016bd365e2e2d1bfd Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Wed, 14 Jun 2017 13:11:18 -0700 Subject: [PATCH 18/54] changed U64 to unsigned long to get rid of build test errors --- programs/fileio.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index ca08c7f9d..9d905062f 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -865,9 +865,9 @@ int FIO_compressFilename(const char* dstFileName, const char* srcFileName, typedef struct { int numActualFrames; int numSkippableFrames; - U64 decompressedSize; + unsigned long decompressedSize; int canComputeDecompSize; - U64 compressedSize; + unsigned long compressedSize; int usesCheck; } fileInfo_t; @@ -1018,9 +1018,9 @@ void displayInfo(const char* inFileName, fileInfo_t* info, int displayLevel){ else{ DISPLAY("# Zstandard Frames: %d\n", info->numActualFrames); DISPLAY("# Skippable Frames: %d\n", info->numSkippableFrames); - DISPLAY("Compressed Size: %.2f MB (%llu B)\n", compressedSizeMB, info->compressedSize); + DISPLAY("Compressed Size: %.2f MB (%lu B)\n", compressedSizeMB, info->compressedSize); if(info->canComputeDecompSize){ - DISPLAY("Decompressed Size: %.2f MB (%llu B)\n", decompressedSizeMB, info->decompressedSize); + DISPLAY("Decompressed Size: %.2f MB (%lu B)\n", decompressedSizeMB, info->decompressedSize); DISPLAY("Ratio: %.4f\n", compressedSizeMB/decompressedSizeMB); } if(info->usesCheck){ From 607ee9f4bcad9a72425392c55934be5237042e81 Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Wed, 14 Jun 2017 13:23:36 -0700 Subject: [PATCH 19/54] print message when no files are given, rearrange code for readability --- programs/zstdcli.c | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 91bce8bb8..f8612f297 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -674,13 +674,19 @@ int main(int argCount, const char* argv[]) } #endif if(operation==zom_list){ - unsigned u; + if(filenameIdx==0){ + DISPLAY("No files given\n"); + CLEAN_RETURN(0); + } DISPLAY("===========================================\n"); DISPLAY("Printing information about compressed files\n"); DISPLAY("===========================================\n"); DISPLAY("Number of files listed: %d\n", filenameIdx); - for(u=0; u Date: Wed, 14 Jun 2017 13:26:19 -0700 Subject: [PATCH 20/54] changing formatting again --- programs/fileio.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index 9d905062f..2708176c1 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -865,9 +865,9 @@ int FIO_compressFilename(const char* dstFileName, const char* srcFileName, typedef struct { int numActualFrames; int numSkippableFrames; - unsigned long decompressedSize; + unsigned long long decompressedSize; int canComputeDecompSize; - unsigned long compressedSize; + unsigned long long compressedSize; int usesCheck; } fileInfo_t; @@ -899,7 +899,7 @@ int getFileInfo(fileInfo_t* info, const char* inFileName){ if(srcFile==NULL){ return 1; } - info->compressedSize = UTIL_getFileSize(inFileName); + info->compressedSize = (unsigned long long)UTIL_getFileSize(inFileName); info->decompressedSize = 0; info->numActualFrames = 0; info-> numSkippableFrames = 0; @@ -1018,9 +1018,9 @@ void displayInfo(const char* inFileName, fileInfo_t* info, int displayLevel){ else{ DISPLAY("# Zstandard Frames: %d\n", info->numActualFrames); DISPLAY("# Skippable Frames: %d\n", info->numSkippableFrames); - DISPLAY("Compressed Size: %.2f MB (%lu B)\n", compressedSizeMB, info->compressedSize); + DISPLAY("Compressed Size: %.2f MB (%llu B)\n", compressedSizeMB, info->compressedSize); if(info->canComputeDecompSize){ - DISPLAY("Decompressed Size: %.2f MB (%lu B)\n", decompressedSizeMB, info->decompressedSize); + DISPLAY("Decompressed Size: %.2f MB (%llu B)\n", decompressedSizeMB, info->decompressedSize); DISPLAY("Ratio: %.4f\n", compressedSizeMB/decompressedSizeMB); } if(info->usesCheck){ From e20899252904b8e18b9c5b0a6f944ae289b89910 Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Thu, 15 Jun 2017 12:27:32 -0700 Subject: [PATCH 21/54] cleaning up code --- programs/fileio.c | 89 ++++++++++++++++++++++++++-------------------- programs/zstdcli.c | 1 + 2 files changed, 51 insertions(+), 39 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index 2708176c1..8af3c5a77 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -91,6 +91,7 @@ * Macros ***************************************/ #define DISPLAY(...) fprintf(stderr, __VA_ARGS__) +#define DISPLAYOUT(...) fprintf(stdout, __VA_ARGS__) #define DISPLAYLEVEL(l, ...) { if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); } } static int g_displayLevel = 2; /* 0 : no display; 1: errors; 2 : + result + interaction + warnings; 3 : + progression; 4 : + information */ void FIO_setNotificationLevel(unsigned level) { g_displayLevel=level; } @@ -872,21 +873,13 @@ typedef struct { } fileInfo_t; -int calcFrameHeaderSize(BYTE frameHeaderDescriptor){ - int frameContentSizeBytes = 0; - int windowDescriptorBytes; - int dictionaryIDBytes; - int frameContentSizeFlag = frameHeaderDescriptor >> 6; - int singleSegmentFlag = (frameHeaderDescriptor & (1 << 5)) >> 5; - int dictionaryIDFlag = frameHeaderDescriptor & 3; - if(frameContentSizeFlag!=0){ - frameContentSizeBytes = 1 << frameContentSizeFlag; - } - else if(singleSegmentFlag){ - frameContentSizeBytes = 1; - } - windowDescriptorBytes = singleSegmentFlag ? 0 : 1; - dictionaryIDBytes = dictionaryIDFlag ? 1 << (dictionaryIDFlag - 1): 0; +static int calcFrameHeaderSize(BYTE frameHeaderDescriptor){ + const int frameContentSizeFlag = frameHeaderDescriptor >> 6; + const int singleSegmentFlag = (frameHeaderDescriptor & (1 << 5)) >> 5; + const int dictionaryIDFlag = frameHeaderDescriptor & 3; + const int windowDescriptorBytes = singleSegmentFlag ? 0 : 1; + const int frameContentSizeBytes = (frameContentSizeFlag != 0) ? (1 << frameContentSizeFlag) : (singleSegmentFlag ? 1 : 0); + const int dictionaryIDBytes = dictionaryIDFlag ? 1 << (dictionaryIDFlag - 1): 0; return 4 + 1 + windowDescriptorBytes + frameContentSizeBytes + dictionaryIDBytes; } @@ -894,50 +887,67 @@ int calcFrameHeaderSize(BYTE frameHeaderDescriptor){ * Reads information from file, stores in *info * if successful, returns 0, otherwise returns 1 */ -int getFileInfo(fileInfo_t* info, const char* inFileName){ - FILE* srcFile = FIO_openSrcFile(inFileName); - if(srcFile==NULL){ +static int getFileInfo(fileInfo_t* info, const char* inFileName){ + FILE* const srcFile = FIO_openSrcFile(inFileName); + if (srcFile == NULL) { + DISPLAY("Error: could not open source file %s\n", inFileName); return 1; } info->compressedSize = (unsigned long long)UTIL_getFileSize(inFileName); info->decompressedSize = 0; info->numActualFrames = 0; - info-> numSkippableFrames = 0; + info->numSkippableFrames = 0; info->canComputeDecompSize = 1; /* begin analyzing frame */ - while(1){ + for( ; ; ){ BYTE magicNumberBuffer[4]; size_t numBytesRead = fread(magicNumberBuffer, 1, 4, srcFile); U32 magicNumber; - if(numBytesRead != 4) break; + if (numBytesRead != 4) break; magicNumber = MEM_readLE32(magicNumberBuffer); - if(magicNumber==ZSTD_MAGICNUMBER){ + if (magicNumber==ZSTD_MAGICNUMBER) { BYTE frameHeaderDescriptor; int totalFrameHeaderBytes; BYTE* frameHeader; int lastBlock = 0; size_t readBytes = fread(&frameHeaderDescriptor, 1, 1, srcFile); info->numActualFrames++; - if(readBytes != 1){ - DISPLAY("There was an error with reading frame header descriptor\n"); - exit(1); + if (readBytes != 1) { + DISPLAY("Error: could not read frame header descriptor\n"); + fclose(srcFile); + + return 1; } /* calculate actual frame header size */ totalFrameHeaderBytes = calcFrameHeaderSize(frameHeaderDescriptor); /* reset to beginning of from and read entire header */ - fseek(srcFile, -5, SEEK_CUR); + { + int returnVal = fseek(srcFile, -5, SEEK_CUR); + if (returnVal!=0) { + DISPLAY("Error: could not reset to the beginning of the frame header\n"); + fclose(srcFile); + return 1; + } + } frameHeader = (BYTE*)malloc(totalFrameHeaderBytes); + if (frameHeader==NULL) { + DISPLAY("Error: could not allocate space for frameHeader\n"); + fclose(srcFile); + return 1; + } readBytes = fread(frameHeader, 1, totalFrameHeaderBytes, srcFile); - if(readBytes != (size_t)totalFrameHeaderBytes){ - DISPLAY("There was an error reading the frame header\n"); - exit(1); + if (readBytes != (size_t)totalFrameHeaderBytes) { + DISPLAY("Error: could not read frame header\n"); + fclose(srcFile); + free(frameHeader); + return 1; } /* get decompressed file size */ { U64 additional = ZSTD_getFrameContentSize(frameHeader, totalFrameHeaderBytes); - if(additional!=ZSTD_CONTENTSIZE_UNKNOWN && additional!=ZSTD_CONTENTSIZE_ERROR){ + if (additional!=ZSTD_CONTENTSIZE_UNKNOWN && additional!=ZSTD_CONTENTSIZE_ERROR) { info->decompressedSize += additional; } else{ @@ -951,7 +961,7 @@ int getFileInfo(fileInfo_t* info, const char* inFileName){ U32 blockHeader; int blockSize; readBytes = fread(blockHeaderBuffer, 1, 3, srcFile); - if(readBytes != 3){ + if (readBytes != 3) { DISPLAY("There was a problem reading the block header\n"); exit(1); } @@ -959,37 +969,38 @@ int getFileInfo(fileInfo_t* info, const char* inFileName){ lastBlock = blockHeader & 1; blockSize = (blockHeader - (blockHeader & 7)) >> 3; fseek(srcFile, blockSize, SEEK_CUR); - }while(lastBlock != 1); + }while (lastBlock != 1); { /* check if checksum is used */ int contentChecksumFlag = (frameHeaderDescriptor & (1 << 2)) >> 2; - if(contentChecksumFlag){ + if (contentChecksumFlag) { info->usesCheck = 1; } - if(contentChecksumFlag){ + if (contentChecksumFlag) { fseek(srcFile, 4, SEEK_CUR); } } + free(frameHeader); } - else if(magicNumber==ZSTD_MAGIC_SKIPPABLE_START){ + else if (magicNumber == ZSTD_MAGIC_SKIPPABLE_START) { BYTE frameSizeBuffer[4]; long frameSize; size_t readBytes = fread(frameSizeBuffer, 1, 4, srcFile); info->numSkippableFrames++; - if(readBytes != 4){ + if (readBytes != 4) { DISPLAY("There was an error reading skippable frame size"); exit(1); } frameSize = MEM_readLE32(frameSizeBuffer); fseek(srcFile, frameSize, SEEK_CUR); } - } + fclose(srcFile); return 0; } void displayInfo(const char* inFileName, fileInfo_t* info, int displayLevel){ - double compressedSizeMB = (double)info->compressedSize/(1 MB); - double decompressedSizeMB = (double)info->decompressedSize/(1 MB); + double const compressedSizeMB = (double)info->compressedSize/(1 MB); + double const decompressedSizeMB = (double)info->decompressedSize/(1 MB); if(displayLevel<=2){ if(info->usesCheck && info->canComputeDecompSize){ diff --git a/programs/zstdcli.c b/programs/zstdcli.c index f8612f297..6de33ce09 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -674,6 +674,7 @@ int main(int argCount, const char* argv[]) } #endif if(operation==zom_list){ + g_displayOut = stdout; if(filenameIdx==0){ DISPLAY("No files given\n"); CLEAN_RETURN(0); From a9b77c83e58fab8992ce9318cb11033e34a86d52 Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Thu, 15 Jun 2017 14:13:28 -0700 Subject: [PATCH 22/54] cleaning up code for analyzing frames --- lib/decompress/zstd_decompress.c | 2 +- lib/zstd.h | 5 + programs/fileio.c | 170 +++++++++++++++---------------- 3 files changed, 90 insertions(+), 87 deletions(-) diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index ea64edeed..af8b98d7f 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -204,7 +204,7 @@ unsigned ZSTD_isFrame(const void* buffer, size_t size) /** ZSTD_frameHeaderSize() : * srcSize must be >= ZSTD_frameHeaderSize_prefix. * @return : size of the Frame Header */ -static size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize) +size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize) { if (srcSize < ZSTD_frameHeaderSize_prefix) return ERROR(srcSize_wrong); { BYTE const fhd = ((const BYTE*)src)[4]; diff --git a/lib/zstd.h b/lib/zstd.h index b0aae3f73..53acdcdd0 100644 --- a/lib/zstd.h +++ b/lib/zstd.h @@ -461,6 +461,11 @@ ZSTDLIB_API unsigned long long ZSTD_getFrameContentSize(const void *src, size_t * however it does mean that all frame data must be present and valid. */ ZSTDLIB_API unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize); +/*! ZSTD_frameHeaderSize() : +* `src` should point to the start of a ZSTD frame +* `srcSize` must be >= ZSTD_frameHeaderSize_prefix. +* @return : size of the Frame Header */ +size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize); /*************************************** * Context memory usage diff --git a/programs/fileio.c b/programs/fileio.c index 8af3c5a77..4f92e16e8 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -872,17 +872,6 @@ typedef struct { int usesCheck; } fileInfo_t; - -static int calcFrameHeaderSize(BYTE frameHeaderDescriptor){ - const int frameContentSizeFlag = frameHeaderDescriptor >> 6; - const int singleSegmentFlag = (frameHeaderDescriptor & (1 << 5)) >> 5; - const int dictionaryIDFlag = frameHeaderDescriptor & 3; - const int windowDescriptorBytes = singleSegmentFlag ? 0 : 1; - const int frameContentSizeBytes = (frameContentSizeFlag != 0) ? (1 << frameContentSizeFlag) : (singleSegmentFlag ? 1 : 0); - const int dictionaryIDBytes = dictionaryIDFlag ? 1 << (dictionaryIDFlag - 1): 0; - return 4 + 1 + windowDescriptorBytes + frameContentSizeBytes + dictionaryIDBytes; -} - /* * Reads information from file, stores in *info * if successful, returns 0, otherwise returns 1 @@ -900,78 +889,79 @@ static int getFileInfo(fileInfo_t* info, const char* inFileName){ info->canComputeDecompSize = 1; /* begin analyzing frame */ for( ; ; ){ - BYTE magicNumberBuffer[4]; - size_t numBytesRead = fread(magicNumberBuffer, 1, 4, srcFile); - U32 magicNumber; - if (numBytesRead != 4) break; - magicNumber = MEM_readLE32(magicNumberBuffer); - if (magicNumber==ZSTD_MAGICNUMBER) { - BYTE frameHeaderDescriptor; - int totalFrameHeaderBytes; - BYTE* frameHeader; - int lastBlock = 0; - size_t readBytes = fread(&frameHeaderDescriptor, 1, 1, srcFile); - info->numActualFrames++; - if (readBytes != 1) { - DISPLAY("Error: could not read frame header descriptor\n"); + BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX]; + size_t const numBytesRead = fread(headerBuffer, 1, sizeof(headerBuffer), srcFile); + if (numBytesRead < ZSTD_frameHeaderSize_min) { + if(feof(srcFile)){ + DISPLAY("ran out of files\n"); + break; + } + else{ + DISPLAY("Error: did not reach end of file but ran out of frames\n"); fclose(srcFile); - return 1; } - /* calculate actual frame header size */ - totalFrameHeaderBytes = calcFrameHeaderSize(frameHeaderDescriptor); + } + U32 const magicNumber = MEM_readLE32(headerBuffer); + if (magicNumber == ZSTD_MAGICNUMBER) { + U64 const frameContentSize = ZSTD_getFrameContentSize(headerBuffer, numBytesRead); + if (frameContentSize == ZSTD_CONTENTSIZE_ERROR || frameContentSize == ZSTD_CONTENTSIZE_UNKNOWN) { + info->canComputeDecompSize = 0; + DISPLAY("could not compute decompressed size\n"); + } + else { + info->decompressedSize += frameContentSize; + } + size_t const headerSize = ZSTD_frameHeaderSize(headerBuffer, numBytesRead); + if (ZSTD_isError(headerSize)) { + fclose(srcFile); + DISPLAY("Error: could not determine frame header size\n"); + return 1; + } - /* reset to beginning of from and read entire header */ { - int returnVal = fseek(srcFile, -5, SEEK_CUR); - if (returnVal!=0) { - DISPLAY("Error: could not reset to the beginning of the frame header\n"); + /* go back to the beginning of the frame */ + int const ret = fseek(srcFile, -numBytesRead, SEEK_CUR); + if (ret != 0) { + DISPLAY("Error: could not rewind to beginning of frame\n"); fclose(srcFile); return 1; } } - frameHeader = (BYTE*)malloc(totalFrameHeaderBytes); - if (frameHeader==NULL) { - DISPLAY("Error: could not allocate space for frameHeader\n"); - fclose(srcFile); - return 1; - } - readBytes = fread(frameHeader, 1, totalFrameHeaderBytes, srcFile); - if (readBytes != (size_t)totalFrameHeaderBytes) { - DISPLAY("Error: could not read frame header\n"); - fclose(srcFile); - free(frameHeader); - return 1; - } - /* get decompressed file size */ { - U64 additional = ZSTD_getFrameContentSize(frameHeader, totalFrameHeaderBytes); - if (additional!=ZSTD_CONTENTSIZE_UNKNOWN && additional!=ZSTD_CONTENTSIZE_ERROR) { - info->decompressedSize += additional; - } - else{ - info->canComputeDecompSize = 0; + /* skip frame header */ + int const ret = fseek(srcFile, headerSize, SEEK_CUR); + if (ret != 0) { + DISPLAY("Error: could not skip header\n"); + fclose(srcFile); + return 1; } } /* skip the rest of the blocks in the frame */ - do{ - BYTE blockHeaderBuffer[3]; - U32 blockHeader; - int blockSize; - readBytes = fread(blockHeaderBuffer, 1, 3, srcFile); - if (readBytes != 3) { - DISPLAY("There was a problem reading the block header\n"); - exit(1); - } - blockHeader = MEM_readLE24(blockHeaderBuffer); - lastBlock = blockHeader & 1; - blockSize = (blockHeader - (blockHeader & 7)) >> 3; - fseek(srcFile, blockSize, SEEK_CUR); - }while (lastBlock != 1); + { + int lastBlock = 0; + int readBytes = 0; + do{ + BYTE blockHeaderBuffer[3]; + U32 blockHeader; + int blockSize; + readBytes = fread(blockHeaderBuffer, 1, 3, srcFile); + if (readBytes != 3) { + DISPLAY("There was a problem reading the block header\n"); + fclose(srcFile); + return 1; + } + blockHeader = MEM_readLE24(blockHeaderBuffer); + lastBlock = blockHeader & 1; + blockSize = (blockHeader - (blockHeader & 7)) >> 3; + fseek(srcFile, blockSize, SEEK_CUR); + }while (lastBlock != 1); + } { /* check if checksum is used */ + BYTE frameHeaderDescriptor = headerBuffer[4]; int contentChecksumFlag = (frameHeaderDescriptor & (1 << 2)) >> 2; if (contentChecksumFlag) { info->usesCheck = 1; @@ -980,7 +970,7 @@ static int getFileInfo(fileInfo_t* info, const char* inFileName){ fseek(srcFile, 4, SEEK_CUR); } } - free(frameHeader); + info->numActualFrames++; } else if (magicNumber == ZSTD_MAGIC_SKIPPABLE_START) { BYTE frameSizeBuffer[4]; @@ -992,7 +982,16 @@ static int getFileInfo(fileInfo_t* info, const char* inFileName){ exit(1); } frameSize = MEM_readLE32(frameSizeBuffer); + { + int const ret = fseek(srcFile, frameSize, SEEK_CUR); + if (ret != 0) { + DISPLAY("Error: could not find end of skippable frame\n"); + fclose(srcFile); + return 1; + } + } fseek(srcFile, frameSize, SEEK_CUR); + info->numSkippableFrames++; } } fclose(srcFile); @@ -1004,51 +1003,50 @@ void displayInfo(const char* inFileName, fileInfo_t* info, int displayLevel){ if(displayLevel<=2){ if(info->usesCheck && info->canComputeDecompSize){ - DISPLAY("Skippable Non-Skippable Compressed Uncompressed Ratio Check Filename\n"); - DISPLAY("%9d %13d %7.2f MB %7.2f MB %5.3f XXH64 %s\n", + DISPLAYOUT("Skippable Non-Skippable Compressed Uncompressed Ratio Check Filename\n"); + DISPLAYOUT("%9d %13d %7.2f MB %7.2f MB %5.3f XXH64 %s\n", info->numSkippableFrames, info->numActualFrames, compressedSizeMB, decompressedSizeMB, compressedSizeMB/decompressedSizeMB, inFileName); } else if(!info->usesCheck){ - DISPLAY("Skippable Non-Skippable Compressed Uncompressed Ratio Check Filename\n"); - DISPLAY("%9d %13d %7.2f MB %7.2f MB %5.3f %s\n", + DISPLAYOUT("Skippable Non-Skippable Compressed Uncompressed Ratio Check Filename\n"); + DISPLAYOUT("%9d %13d %7.2f MB %7.2f MB %5.3f %s\n", info->numSkippableFrames, info->numActualFrames, compressedSizeMB, decompressedSizeMB, compressedSizeMB/decompressedSizeMB, inFileName); } else if(!info->canComputeDecompSize){ - DISPLAY("Skippable Non-Skippable Compressed Uncompressed Ratio Check Filename\n"); - DISPLAY("%9d %13d %7.2f MB XXH64 %s\n", + DISPLAYOUT("Skippable Non-Skippable Compressed Uncompressed Ratio Check Filename\n"); + DISPLAYOUT("%9d %13d %7.2f MB XXH64 %s\n", info->numSkippableFrames, info->numActualFrames, compressedSizeMB, inFileName); } else{ - DISPLAY("Skippable Non-Skippable Filename\n"); - DISPLAY("%9d %13d %7.2f MB %s\n", + DISPLAYOUT("Skippable Non-Skippable Filename\n"); + DISPLAYOUT("%9d %13d %7.2f MB %s\n", info->numSkippableFrames, info->numActualFrames, compressedSizeMB, inFileName); } } else{ - DISPLAY("# Zstandard Frames: %d\n", info->numActualFrames); - DISPLAY("# Skippable Frames: %d\n", info->numSkippableFrames); - DISPLAY("Compressed Size: %.2f MB (%llu B)\n", compressedSizeMB, info->compressedSize); - if(info->canComputeDecompSize){ - DISPLAY("Decompressed Size: %.2f MB (%llu B)\n", decompressedSizeMB, info->decompressedSize); - DISPLAY("Ratio: %.4f\n", compressedSizeMB/decompressedSizeMB); + DISPLAYOUT("# Zstandard Frames: %d\n", info->numActualFrames); + DISPLAYOUT("# Skippable Frames: %d\n", info->numSkippableFrames); + DISPLAYOUT("Compressed Size: %.2f MB (%llu B)\n", compressedSizeMB, info->compressedSize); + if (info->canComputeDecompSize) { + DISPLAYOUT("Decompressed Size: %.2f MB (%llu B)\n", decompressedSizeMB, info->decompressedSize); + DISPLAYOUT("Ratio: %.4f\n", compressedSizeMB/decompressedSizeMB); } - if(info->usesCheck){ - DISPLAY("Check: XXH64\n"); + if (info->usesCheck) { + DISPLAYOUT("Check: XXH64\n"); } } - } int FIO_listFile(const char* inFileName, int displayLevel){ const char* const suffixPtr = strrchr(inFileName, '.'); DISPLAY("File: %s\n", inFileName); - if(!suffixPtr || strcmp(suffixPtr, ZSTD_EXTENSION)){ + if (!suffixPtr || strcmp(suffixPtr, ZSTD_EXTENSION)) { DISPLAYLEVEL(1, "file %s was not compressed with zstd -- ignoring\n\n", inFileName); return 1; } - else{ + else { fileInfo_t* info = (fileInfo_t*)malloc(sizeof(fileInfo_t)); int error = getFileInfo(info, inFileName); if(error==1){ From 4495e9a82616a5953bb209dd7692afca99fbd3e8 Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Thu, 15 Jun 2017 15:02:54 -0700 Subject: [PATCH 23/54] allow analysis even if file does not have zst extension --- programs/fileio.c | 23 +++++++++-------------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index 4f92e16e8..e6671308c 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -914,8 +914,8 @@ static int getFileInfo(fileInfo_t* info, const char* inFileName){ } size_t const headerSize = ZSTD_frameHeaderSize(headerBuffer, numBytesRead); if (ZSTD_isError(headerSize)) { - fclose(srcFile); DISPLAY("Error: could not determine frame header size\n"); + fclose(srcFile); return 1; } @@ -1037,25 +1037,20 @@ void displayInfo(const char* inFileName, fileInfo_t* info, int displayLevel){ DISPLAYOUT("Check: XXH64\n"); } } + + DISPLAYOUT("\n"); } int FIO_listFile(const char* inFileName, int displayLevel){ - const char* const suffixPtr = strrchr(inFileName, '.'); DISPLAY("File: %s\n", inFileName); - if (!suffixPtr || strcmp(suffixPtr, ZSTD_EXTENSION)) { - DISPLAYLEVEL(1, "file %s was not compressed with zstd -- ignoring\n\n", inFileName); + fileInfo_t* info = (fileInfo_t*)malloc(sizeof(fileInfo_t)); + int error = getFileInfo(info, inFileName); + if (error == 1) { + DISPLAY("An error occurred with getting file info\n"); return 1; } - else { - fileInfo_t* info = (fileInfo_t*)malloc(sizeof(fileInfo_t)); - int error = getFileInfo(info, inFileName); - if(error==1){ - DISPLAY("An error occurred with getting file info\n"); - exit(1); - } - displayInfo(inFileName, info, displayLevel); - } - DISPLAY("\n"); + displayInfo(inFileName, info, displayLevel); + free(info); return 0; } From 9437cc74980e7f5168eb485d31fd64f054f945cd Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Thu, 15 Jun 2017 15:07:54 -0700 Subject: [PATCH 24/54] added --list command --- programs/zstdcli.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 6de33ce09..c71e80bdd 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -149,6 +149,7 @@ static int usage_advanced(const char* programName) #endif #endif DISPLAY( " -M# : Set a memory usage limit for decompression \n"); + DISPLAY( "--list : list information about a zstd compressed file \n"); DISPLAY( "-- : All arguments after \"--\" are treated as files \n"); #ifndef ZSTD_NODICT DISPLAY( "\n"); @@ -402,6 +403,7 @@ int main(int argCount, const char* argv[]) if (argument[1]=='-') { /* long commands (--long-word) */ if (!strcmp(argument, "--")) { nextArgumentsAreFiles=1; continue; } /* only file names allowed from now on */ + if (!strcmp(argument, "--list")) { operation=zom_list; continue; } if (!strcmp(argument, "--compress")) { operation=zom_compress; continue; } if (!strcmp(argument, "--decompress")) { operation=zom_decompress; continue; } if (!strcmp(argument, "--uncompress")) { operation=zom_decompress; continue; } From 96cabf72330f1b466407059a5f0981a24571fac1 Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Thu, 15 Jun 2017 15:25:20 -0700 Subject: [PATCH 25/54] cleaned up if statements --- programs/fileio.c | 27 ++++++++------------------- 1 file changed, 8 insertions(+), 19 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index e6671308c..523b6174e 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -893,7 +893,6 @@ static int getFileInfo(fileInfo_t* info, const char* inFileName){ size_t const numBytesRead = fread(headerBuffer, 1, sizeof(headerBuffer), srcFile); if (numBytesRead < ZSTD_frameHeaderSize_min) { if(feof(srcFile)){ - DISPLAY("ran out of files\n"); break; } else{ @@ -1000,29 +999,19 @@ static int getFileInfo(fileInfo_t* info, const char* inFileName){ void displayInfo(const char* inFileName, fileInfo_t* info, int displayLevel){ double const compressedSizeMB = (double)info->compressedSize/(1 MB); double const decompressedSizeMB = (double)info->decompressedSize/(1 MB); + const char* checkString = (info->usesCheck ? "XXH64" : "None"); if(displayLevel<=2){ - if(info->usesCheck && info->canComputeDecompSize){ + if (info->canComputeDecompSize) { DISPLAYOUT("Skippable Non-Skippable Compressed Uncompressed Ratio Check Filename\n"); - DISPLAYOUT("%9d %13d %7.2f MB %7.2f MB %5.3f XXH64 %s\n", + DISPLAYOUT("%9d %13d %7.2f MB %9.2f MB %5.3f %s %s\n", info->numSkippableFrames, info->numActualFrames, compressedSizeMB, decompressedSizeMB, - compressedSizeMB/decompressedSizeMB, inFileName); + compressedSizeMB/decompressedSizeMB, checkString, inFileName); } - else if(!info->usesCheck){ - DISPLAYOUT("Skippable Non-Skippable Compressed Uncompressed Ratio Check Filename\n"); - DISPLAYOUT("%9d %13d %7.2f MB %7.2f MB %5.3f %s\n", - info->numSkippableFrames, info->numActualFrames, compressedSizeMB, decompressedSizeMB, - compressedSizeMB/decompressedSizeMB, inFileName); - } - else if(!info->canComputeDecompSize){ - DISPLAYOUT("Skippable Non-Skippable Compressed Uncompressed Ratio Check Filename\n"); - DISPLAYOUT("%9d %13d %7.2f MB XXH64 %s\n", - info->numSkippableFrames, info->numActualFrames, compressedSizeMB, inFileName); - } - else{ - DISPLAYOUT("Skippable Non-Skippable Filename\n"); - DISPLAYOUT("%9d %13d %7.2f MB %s\n", - info->numSkippableFrames, info->numActualFrames, compressedSizeMB, inFileName); + else { + DISPLAYOUT("Skippable Non-Skippable Compressed Check Filename\n"); + DISPLAYOUT("%9d %13d %7.2f MB %s %s\n", + info->numSkippableFrames, info->numActualFrames, compressedSizeMB, checkString, inFileName); } } else{ From 8b3ff7c9bcb91bd15e994e6fda151b1fda0a7f0a Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Thu, 15 Jun 2017 15:35:37 -0700 Subject: [PATCH 26/54] changed info to static allocation --- programs/fileio.c | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index 523b6174e..aff1edd41 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -1000,7 +1000,6 @@ void displayInfo(const char* inFileName, fileInfo_t* info, int displayLevel){ double const compressedSizeMB = (double)info->compressedSize/(1 MB); double const decompressedSizeMB = (double)info->decompressedSize/(1 MB); const char* checkString = (info->usesCheck ? "XXH64" : "None"); - if(displayLevel<=2){ if (info->canComputeDecompSize) { DISPLAYOUT("Skippable Non-Skippable Compressed Uncompressed Ratio Check Filename\n"); @@ -1031,15 +1030,14 @@ void displayInfo(const char* inFileName, fileInfo_t* info, int displayLevel){ } int FIO_listFile(const char* inFileName, int displayLevel){ - DISPLAY("File: %s\n", inFileName); - fileInfo_t* info = (fileInfo_t*)malloc(sizeof(fileInfo_t)); - int error = getFileInfo(info, inFileName); + DISPLAYOUT("File: %s\n", inFileName); + fileInfo_t info; + int const error = getFileInfo(&info, inFileName); if (error == 1) { DISPLAY("An error occurred with getting file info\n"); return 1; } - displayInfo(inFileName, info, displayLevel); - free(info); + displayInfo(inFileName, &info, displayLevel); return 0; } From e7f02fc58ad25c55ff78b0fea9cb5ad40978f0c0 Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Thu, 15 Jun 2017 15:53:02 -0700 Subject: [PATCH 27/54] closed file at end of function, created another variable to keep exit points simpler --- programs/fileio.c | 75 +++++++++++++++++++++++++---------------------- 1 file changed, 40 insertions(+), 35 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index aff1edd41..09ef1791d 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -887,6 +887,7 @@ static int getFileInfo(fileInfo_t* info, const char* inFileName){ info->numActualFrames = 0; info->numSkippableFrames = 0; info->canComputeDecompSize = 1; + int detectError = 0; /* begin analyzing frame */ for( ; ; ){ BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX]; @@ -897,8 +898,8 @@ static int getFileInfo(fileInfo_t* info, const char* inFileName){ } else{ DISPLAY("Error: did not reach end of file but ran out of frames\n"); - fclose(srcFile); - return 1; + detectError = 1; + break; } } U32 const magicNumber = MEM_readLE32(headerBuffer); @@ -906,7 +907,6 @@ static int getFileInfo(fileInfo_t* info, const char* inFileName){ U64 const frameContentSize = ZSTD_getFrameContentSize(headerBuffer, numBytesRead); if (frameContentSize == ZSTD_CONTENTSIZE_ERROR || frameContentSize == ZSTD_CONTENTSIZE_UNKNOWN) { info->canComputeDecompSize = 0; - DISPLAY("could not compute decompressed size\n"); } else { info->decompressedSize += frameContentSize; @@ -914,34 +914,24 @@ static int getFileInfo(fileInfo_t* info, const char* inFileName){ size_t const headerSize = ZSTD_frameHeaderSize(headerBuffer, numBytesRead); if (ZSTD_isError(headerSize)) { DISPLAY("Error: could not determine frame header size\n"); - fclose(srcFile); - return 1; + detectError = 1; + break; } { - /* go back to the beginning of the frame */ - int const ret = fseek(srcFile, -numBytesRead, SEEK_CUR); + /* move to the end of the frame header */ + int const ret = fseek(srcFile, headerSize-numBytesRead, SEEK_CUR); if (ret != 0) { - DISPLAY("Error: could not rewind to beginning of frame\n"); - fclose(srcFile); - return 1; - } - } - - { - /* skip frame header */ - int const ret = fseek(srcFile, headerSize, SEEK_CUR); - if (ret != 0) { - DISPLAY("Error: could not skip header\n"); - fclose(srcFile); - return 1; + DISPLAY("Error: could not move to end of frame header\n"); + detectError = 1; + break; } } /* skip the rest of the blocks in the frame */ { int lastBlock = 0; - int readBytes = 0; + size_t readBytes = 0; do{ BYTE blockHeaderBuffer[3]; U32 blockHeader; @@ -949,44 +939,59 @@ static int getFileInfo(fileInfo_t* info, const char* inFileName){ readBytes = fread(blockHeaderBuffer, 1, 3, srcFile); if (readBytes != 3) { DISPLAY("There was a problem reading the block header\n"); - fclose(srcFile); - return 1; + detectError = 1; + break; } blockHeader = MEM_readLE24(blockHeaderBuffer); lastBlock = blockHeader & 1; blockSize = (blockHeader - (blockHeader & 7)) >> 3; - fseek(srcFile, blockSize, SEEK_CUR); - }while (lastBlock != 1); + { + int const ret = fseek(srcFile, blockSize, SEEK_CUR); + if (ret != 0) { + DISPLAY("Error: could not skip to end of block\n"); + detectError = 1; + break; + } + } + } while (lastBlock != 1); + + if (detectError) { + break; + } } { /* check if checksum is used */ - BYTE frameHeaderDescriptor = headerBuffer[4]; - int contentChecksumFlag = (frameHeaderDescriptor & (1 << 2)) >> 2; + BYTE const frameHeaderDescriptor = headerBuffer[4]; + int const contentChecksumFlag = (frameHeaderDescriptor & (1 << 2)) >> 2; if (contentChecksumFlag) { info->usesCheck = 1; } if (contentChecksumFlag) { - fseek(srcFile, 4, SEEK_CUR); + int const ret = fseek(srcFile, 4, SEEK_CUR); + if (ret != 0) { + DISPLAY("Error: could not skip past checksum\n"); + detectError = 1; + break; + } } } info->numActualFrames++; } else if (magicNumber == ZSTD_MAGIC_SKIPPABLE_START) { BYTE frameSizeBuffer[4]; - long frameSize; size_t readBytes = fread(frameSizeBuffer, 1, 4, srcFile); - info->numSkippableFrames++; if (readBytes != 4) { DISPLAY("There was an error reading skippable frame size"); - exit(1); + detectError = 1; + break; } - frameSize = MEM_readLE32(frameSizeBuffer); + long const frameSize = MEM_readLE32(frameSizeBuffer); { int const ret = fseek(srcFile, frameSize, SEEK_CUR); if (ret != 0) { DISPLAY("Error: could not find end of skippable frame\n"); - fclose(srcFile); - return 1; + detectError = 1; + break; } } fseek(srcFile, frameSize, SEEK_CUR); @@ -994,7 +999,7 @@ static int getFileInfo(fileInfo_t* info, const char* inFileName){ } } fclose(srcFile); - return 0; + return detectError; } void displayInfo(const char* inFileName, fileInfo_t* info, int displayLevel){ double const compressedSizeMB = (double)info->compressedSize/(1 MB); From b766211e10fd3e148d38e0c1c24043c65a817002 Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Thu, 15 Jun 2017 15:55:49 -0700 Subject: [PATCH 28/54] made another function static --- programs/fileio.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index 09ef1791d..23c8badc6 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -954,7 +954,7 @@ static int getFileInfo(fileInfo_t* info, const char* inFileName){ } } } while (lastBlock != 1); - + if (detectError) { break; } @@ -1001,7 +1001,8 @@ static int getFileInfo(fileInfo_t* info, const char* inFileName){ fclose(srcFile); return detectError; } -void displayInfo(const char* inFileName, fileInfo_t* info, int displayLevel){ + +static void displayInfo(const char* inFileName, fileInfo_t* info, int displayLevel){ double const compressedSizeMB = (double)info->compressedSize/(1 MB); double const decompressedSizeMB = (double)info->decompressedSize/(1 MB); const char* checkString = (info->usesCheck ? "XXH64" : "None"); From 54e0b6c66b78b0f9035ec06e959428cd200289c1 Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Thu, 15 Jun 2017 16:01:52 -0700 Subject: [PATCH 29/54] got rid of imlpicit precision loss --- programs/fileio.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/programs/fileio.c b/programs/fileio.c index 23c8badc6..e76f93791 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -920,7 +920,7 @@ static int getFileInfo(fileInfo_t* info, const char* inFileName){ { /* move to the end of the frame header */ - int const ret = fseek(srcFile, headerSize-numBytesRead, SEEK_CUR); + int const ret = fseek(srcFile, (long)(headerSize-numBytesRead), SEEK_CUR); if (ret != 0) { DISPLAY("Error: could not move to end of frame header\n"); detectError = 1; From e49afae2ab6f720333fb47fda66333c53f6d736d Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Thu, 15 Jun 2017 16:03:40 -0700 Subject: [PATCH 30/54] changed implicit precision loss again --- programs/fileio.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/programs/fileio.c b/programs/fileio.c index e76f93791..5a1bef7a8 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -920,7 +920,7 @@ static int getFileInfo(fileInfo_t* info, const char* inFileName){ { /* move to the end of the frame header */ - int const ret = fseek(srcFile, (long)(headerSize-numBytesRead), SEEK_CUR); + int const ret = fseek(srcFile, ((long)headerSize)-((long)numBytesRead), SEEK_CUR); if (ret != 0) { DISPLAY("Error: could not move to end of frame header\n"); detectError = 1; From 0757eae6ff55c6dd09ff3320ad232804ea2ae86f Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Thu, 15 Jun 2017 16:12:04 -0700 Subject: [PATCH 31/54] rearranging code to allow for consts --- programs/fileio.c | 158 +++++++++++++++++++++++----------------------- 1 file changed, 79 insertions(+), 79 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index 5a1bef7a8..cf329ddef 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -877,6 +877,7 @@ typedef struct { * if successful, returns 0, otherwise returns 1 */ static int getFileInfo(fileInfo_t* info, const char* inFileName){ + int detectError = 0; FILE* const srcFile = FIO_openSrcFile(inFileName); if (srcFile == NULL) { DISPLAY("Error: could not open source file %s\n", inFileName); @@ -887,7 +888,6 @@ static int getFileInfo(fileInfo_t* info, const char* inFileName){ info->numActualFrames = 0; info->numSkippableFrames = 0; info->canComputeDecompSize = 1; - int detectError = 0; /* begin analyzing frame */ for( ; ; ){ BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX]; @@ -902,100 +902,100 @@ static int getFileInfo(fileInfo_t* info, const char* inFileName){ break; } } - U32 const magicNumber = MEM_readLE32(headerBuffer); - if (magicNumber == ZSTD_MAGICNUMBER) { - U64 const frameContentSize = ZSTD_getFrameContentSize(headerBuffer, numBytesRead); - if (frameContentSize == ZSTD_CONTENTSIZE_ERROR || frameContentSize == ZSTD_CONTENTSIZE_UNKNOWN) { - info->canComputeDecompSize = 0; - } - else { - info->decompressedSize += frameContentSize; - } - size_t const headerSize = ZSTD_frameHeaderSize(headerBuffer, numBytesRead); - if (ZSTD_isError(headerSize)) { - DISPLAY("Error: could not determine frame header size\n"); - detectError = 1; - break; - } - - { - /* move to the end of the frame header */ - int const ret = fseek(srcFile, ((long)headerSize)-((long)numBytesRead), SEEK_CUR); - if (ret != 0) { - DISPLAY("Error: could not move to end of frame header\n"); - detectError = 1; - break; + { + U32 const magicNumber = MEM_readLE32(headerBuffer); + if (magicNumber == ZSTD_MAGICNUMBER) { + U64 const frameContentSize = ZSTD_getFrameContentSize(headerBuffer, numBytesRead); + if (frameContentSize == ZSTD_CONTENTSIZE_ERROR || frameContentSize == ZSTD_CONTENTSIZE_UNKNOWN) { + info->canComputeDecompSize = 0; } - } - - /* skip the rest of the blocks in the frame */ - { - int lastBlock = 0; - size_t readBytes = 0; - do{ - BYTE blockHeaderBuffer[3]; - U32 blockHeader; - int blockSize; - readBytes = fread(blockHeaderBuffer, 1, 3, srcFile); - if (readBytes != 3) { - DISPLAY("There was a problem reading the block header\n"); + else { + info->decompressedSize += frameContentSize; + } + { + /* move to the end of the frame header */ + size_t const headerSize = ZSTD_frameHeaderSize(headerBuffer, numBytesRead); + if (ZSTD_isError(headerSize)) { + DISPLAY("Error: could not determine frame header size\n"); detectError = 1; break; } - blockHeader = MEM_readLE24(blockHeaderBuffer); - lastBlock = blockHeader & 1; - blockSize = (blockHeader - (blockHeader & 7)) >> 3; - { - int const ret = fseek(srcFile, blockSize, SEEK_CUR); + int const ret = fseek(srcFile, ((long)headerSize)-((long)numBytesRead), SEEK_CUR); + if (ret != 0) { + DISPLAY("Error: could not move to end of frame header\n"); + detectError = 1; + break; + } + } + + /* skip the rest of the blocks in the frame */ + { + int lastBlock = 0; + size_t readBytes = 0; + do{ + BYTE blockHeaderBuffer[3]; + U32 blockHeader; + int blockSize; + readBytes = fread(blockHeaderBuffer, 1, 3, srcFile); + if (readBytes != 3) { + DISPLAY("There was a problem reading the block header\n"); + detectError = 1; + break; + } + blockHeader = MEM_readLE24(blockHeaderBuffer); + lastBlock = blockHeader & 1; + blockSize = (blockHeader - (blockHeader & 7)) >> 3; + { + int const ret = fseek(srcFile, blockSize, SEEK_CUR); + if (ret != 0) { + DISPLAY("Error: could not skip to end of block\n"); + detectError = 1; + break; + } + } + } while (lastBlock != 1); + + if (detectError) { + break; + } + } + { + /* check if checksum is used */ + BYTE const frameHeaderDescriptor = headerBuffer[4]; + int const contentChecksumFlag = (frameHeaderDescriptor & (1 << 2)) >> 2; + if (contentChecksumFlag) { + info->usesCheck = 1; + } + if (contentChecksumFlag) { + int const ret = fseek(srcFile, 4, SEEK_CUR); if (ret != 0) { - DISPLAY("Error: could not skip to end of block\n"); + DISPLAY("Error: could not skip past checksum\n"); detectError = 1; break; } } - } while (lastBlock != 1); - - if (detectError) { + } + info->numActualFrames++; + } + else if (magicNumber == ZSTD_MAGIC_SKIPPABLE_START) { + BYTE frameSizeBuffer[4]; + size_t readBytes = fread(frameSizeBuffer, 1, 4, srcFile); + if (readBytes != 4) { + DISPLAY("There was an error reading skippable frame size"); + detectError = 1; break; } - } - { - /* check if checksum is used */ - BYTE const frameHeaderDescriptor = headerBuffer[4]; - int const contentChecksumFlag = (frameHeaderDescriptor & (1 << 2)) >> 2; - if (contentChecksumFlag) { - info->usesCheck = 1; - } - if (contentChecksumFlag) { - int const ret = fseek(srcFile, 4, SEEK_CUR); + { + long const frameSize = MEM_readLE32(frameSizeBuffer); + int const ret = fseek(srcFile, frameSize, SEEK_CUR); if (ret != 0) { - DISPLAY("Error: could not skip past checksum\n"); + DISPLAY("Error: could not find end of skippable frame\n"); detectError = 1; break; } } + info->numSkippableFrames++; } - info->numActualFrames++; - } - else if (magicNumber == ZSTD_MAGIC_SKIPPABLE_START) { - BYTE frameSizeBuffer[4]; - size_t readBytes = fread(frameSizeBuffer, 1, 4, srcFile); - if (readBytes != 4) { - DISPLAY("There was an error reading skippable frame size"); - detectError = 1; - break; - } - long const frameSize = MEM_readLE32(frameSizeBuffer); - { - int const ret = fseek(srcFile, frameSize, SEEK_CUR); - if (ret != 0) { - DISPLAY("Error: could not find end of skippable frame\n"); - detectError = 1; - break; - } - } - fseek(srcFile, frameSize, SEEK_CUR); - info->numSkippableFrames++; } } fclose(srcFile); @@ -1036,8 +1036,8 @@ static void displayInfo(const char* inFileName, fileInfo_t* info, int displayLev } int FIO_listFile(const char* inFileName, int displayLevel){ - DISPLAYOUT("File: %s\n", inFileName); fileInfo_t info; + DISPLAYOUT("File: %s\n", inFileName); int const error = getFileInfo(&info, inFileName); if (error == 1) { DISPLAY("An error occurred with getting file info\n"); From acaefb531bb0549cb11fac143ad349ccf40efc99 Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Thu, 15 Jun 2017 16:27:38 -0700 Subject: [PATCH 32/54] refactoring brackets to allow for const values --- programs/fileio.c | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index cf329ddef..bb3f13d8e 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -920,11 +920,13 @@ static int getFileInfo(fileInfo_t* info, const char* inFileName){ detectError = 1; break; } - int const ret = fseek(srcFile, ((long)headerSize)-((long)numBytesRead), SEEK_CUR); - if (ret != 0) { - DISPLAY("Error: could not move to end of frame header\n"); - detectError = 1; - break; + { + int const ret = fseek(srcFile, ((long)headerSize)-((long)numBytesRead), SEEK_CUR); + if (ret != 0) { + DISPLAY("Error: could not move to end of frame header\n"); + detectError = 1; + break; + } } } @@ -1038,10 +1040,12 @@ static void displayInfo(const char* inFileName, fileInfo_t* info, int displayLev int FIO_listFile(const char* inFileName, int displayLevel){ fileInfo_t info; DISPLAYOUT("File: %s\n", inFileName); - int const error = getFileInfo(&info, inFileName); - if (error == 1) { - DISPLAY("An error occurred with getting file info\n"); - return 1; + { + int const error = getFileInfo(&info, inFileName); + if (error == 1) { + DISPLAY("An error occurred with getting file info\n"); + return 1; + } } displayInfo(inFileName, &info, displayLevel); return 0; From ddd1ab710c58f8fc8613ede1e2217bc5fadc360c Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Thu, 15 Jun 2017 16:53:03 -0700 Subject: [PATCH 33/54] added tests in playTest.sh --- tests/playTests.sh | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/tests/playTests.sh b/tests/playTests.sh index e69574588..6b794f6d4 100755 --- a/tests/playTests.sh +++ b/tests/playTests.sh @@ -537,6 +537,29 @@ fi rm tmp* +$ECHO "\n**** zstd --list/-l single frame tests ****" +./datagen > tmp1 +./datagen > tmp2 +./datagen > tmp3 +./datagen > tmp4 +$ZSTD tmp* +$ZSTD -l tmp* +$ZSTD -lv tmp* +$ZSTD --list tmp* +$ZSTD --list -v tmp* + +$ECHO "\n**** zstd --list/-l multiple frame tests ****" +cat tmp1.zst tmp2.zst > tmp12.zst +cat tmp3.zst tmp4.zst > tmp34.zst +cat tmp12.zst tmp34.zst > tmp1234.zst +cat tmp12.zst tmp4.zst > tmp124.zst +$ZSTD -l tmp* +$ZSTD -lv tmp* +$ZSTD --list tmp* +$ZSTD --list -v tmp* + +rm tmp* + if [ "$1" != "--test-large-data" ]; then $ECHO "Skipping large data tests" exit 0 From 60a2e55e2e2c7ffeb978d944a31696f2b7c5bcc1 Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Thu, 15 Jun 2017 17:00:59 -0700 Subject: [PATCH 34/54] added case for when file is not compressed with zstd (incorrect magic number) --- programs/fileio.c | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index bb3f13d8e..908e0edb6 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -874,7 +874,7 @@ typedef struct { /* * Reads information from file, stores in *info - * if successful, returns 0, otherwise returns 1 + * if successful, returns 0, returns 1 for frame analysis error, returns 2 for file not compressed with zstd */ static int getFileInfo(fileInfo_t* info, const char* inFileName){ int detectError = 0; @@ -998,6 +998,10 @@ static int getFileInfo(fileInfo_t* info, const char* inFileName){ } info->numSkippableFrames++; } + else { + detectError = 2; + break; + } } } fclose(srcFile); @@ -1043,8 +1047,12 @@ int FIO_listFile(const char* inFileName, int displayLevel){ { int const error = getFileInfo(&info, inFileName); if (error == 1) { + /* display error, but provide output */ DISPLAY("An error occurred with getting file info\n"); - return 1; + } + else if (error == 2) { + DISPLAYOUT("File %s not compressed with zstd\n\n", inFileName); + return 0; } } displayInfo(inFileName, &info, displayLevel); From b316691f9126711a2a68f7e0661194afd2b3fdf8 Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Thu, 15 Jun 2017 17:03:59 -0700 Subject: [PATCH 35/54] added const --- programs/fileio.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/programs/fileio.c b/programs/fileio.c index 908e0edb6..d807cffc1 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -981,7 +981,7 @@ static int getFileInfo(fileInfo_t* info, const char* inFileName){ } else if (magicNumber == ZSTD_MAGIC_SKIPPABLE_START) { BYTE frameSizeBuffer[4]; - size_t readBytes = fread(frameSizeBuffer, 1, 4, srcFile); + size_t const readBytes = fread(frameSizeBuffer, 1, 4, srcFile); if (readBytes != 4) { DISPLAY("There was an error reading skippable frame size"); detectError = 1; From d3b34e4a2516418c4589812fc2e98f6f9bd461d8 Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Thu, 15 Jun 2017 17:12:41 -0700 Subject: [PATCH 36/54] changed spacing on if statements --- programs/fileio.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index d807cffc1..8db61a6fc 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -893,7 +893,7 @@ static int getFileInfo(fileInfo_t* info, const char* inFileName){ BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX]; size_t const numBytesRead = fread(headerBuffer, 1, sizeof(headerBuffer), srcFile); if (numBytesRead < ZSTD_frameHeaderSize_min) { - if(feof(srcFile)){ + if (feof(srcFile)) { break; } else{ @@ -1012,7 +1012,7 @@ static void displayInfo(const char* inFileName, fileInfo_t* info, int displayLev double const compressedSizeMB = (double)info->compressedSize/(1 MB); double const decompressedSizeMB = (double)info->decompressedSize/(1 MB); const char* checkString = (info->usesCheck ? "XXH64" : "None"); - if(displayLevel<=2){ + if (displayLevel <= 2) { if (info->canComputeDecompSize) { DISPLAYOUT("Skippable Non-Skippable Compressed Uncompressed Ratio Check Filename\n"); DISPLAYOUT("%9d %13d %7.2f MB %9.2f MB %5.3f %s %s\n", From 0d7c4d766ab9358b58e193b579578b6442e2f8fd Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Thu, 15 Jun 2017 17:30:18 -0700 Subject: [PATCH 37/54] initialized info manually to avoid error --- programs/fileio.c | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index 8db61a6fc..5ab3c5be0 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -884,10 +884,6 @@ static int getFileInfo(fileInfo_t* info, const char* inFileName){ return 1; } info->compressedSize = (unsigned long long)UTIL_getFileSize(inFileName); - info->decompressedSize = 0; - info->numActualFrames = 0; - info->numSkippableFrames = 0; - info->canComputeDecompSize = 1; /* begin analyzing frame */ for( ; ; ){ BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX]; @@ -1014,10 +1010,11 @@ static void displayInfo(const char* inFileName, fileInfo_t* info, int displayLev const char* checkString = (info->usesCheck ? "XXH64" : "None"); if (displayLevel <= 2) { if (info->canComputeDecompSize) { + double const ratio = (info->decompressedSize == 0) ? 0.0 : compressedSizeMB/decompressedSizeMB; DISPLAYOUT("Skippable Non-Skippable Compressed Uncompressed Ratio Check Filename\n"); DISPLAYOUT("%9d %13d %7.2f MB %9.2f MB %5.3f %s %s\n", info->numSkippableFrames, info->numActualFrames, compressedSizeMB, decompressedSizeMB, - compressedSizeMB/decompressedSizeMB, checkString, inFileName); + ratio, checkString, inFileName); } else { DISPLAYOUT("Skippable Non-Skippable Compressed Check Filename\n"); @@ -1043,6 +1040,14 @@ static void displayInfo(const char* inFileName, fileInfo_t* info, int displayLev int FIO_listFile(const char* inFileName, int displayLevel){ fileInfo_t info; + + /* initialize info to avoid warnings */ + info.numActualFrames = 0; + info.numSkippableFrames = 0; + info.decompressedSize = 0; + info.canComputeDecompSize = 1; + info.compressedSize = 0; + info.usesCheck = 0; DISPLAYOUT("File: %s\n", inFileName); { int const error = getFileInfo(&info, inFileName); From b3e62446c67ff0b76eaf37be717f4c5a9c880260 Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Thu, 15 Jun 2017 17:46:49 -0700 Subject: [PATCH 38/54] added in correct error return from main in zstdcli.c --- programs/fileio.c | 6 +++--- programs/zstdcli.c | 7 ++++--- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index 5ab3c5be0..2f51b95cf 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -1057,11 +1057,11 @@ int FIO_listFile(const char* inFileName, int displayLevel){ } else if (error == 2) { DISPLAYOUT("File %s not compressed with zstd\n\n", inFileName); - return 0; + return 1; } + displayInfo(inFileName, &info, displayLevel); + return error; } - displayInfo(inFileName, &info, displayLevel); - return 0; } int FIO_compressMultipleFilenames(const char** inFileNamesTable, unsigned nbFiles, diff --git a/programs/zstdcli.c b/programs/zstdcli.c index c71e80bdd..a55640351 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -675,7 +675,7 @@ int main(int argCount, const char* argv[]) } } #endif - if(operation==zom_list){ + if (operation == zom_list) { g_displayOut = stdout; if(filenameIdx==0){ DISPLAY("No files given\n"); @@ -686,12 +686,13 @@ int main(int argCount, const char* argv[]) DISPLAY("===========================================\n"); DISPLAY("Number of files listed: %d\n", filenameIdx); { + int error = 0; unsigned u; for(u=0; u Date: Thu, 15 Jun 2017 18:08:22 -0700 Subject: [PATCH 39/54] fixing more spacing on loops --- programs/fileio.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index 2f51b95cf..df492a9ef 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -885,7 +885,7 @@ static int getFileInfo(fileInfo_t* info, const char* inFileName){ } info->compressedSize = (unsigned long long)UTIL_getFileSize(inFileName); /* begin analyzing frame */ - for( ; ; ){ + for ( ; ; ) { BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX]; size_t const numBytesRead = fread(headerBuffer, 1, sizeof(headerBuffer), srcFile); if (numBytesRead < ZSTD_frameHeaderSize_min) { @@ -930,7 +930,7 @@ static int getFileInfo(fileInfo_t* info, const char* inFileName){ { int lastBlock = 0; size_t readBytes = 0; - do{ + do { BYTE blockHeaderBuffer[3]; U32 blockHeader; int blockSize; From 55b9b28dad3b3aa98bfa81def73039e1cf38472e Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Mon, 19 Jun 2017 09:55:55 -0700 Subject: [PATCH 40/54] made style changes, added description in man file --- programs/fileio.c | 7 +++---- programs/zstd.1.md | 4 ++++ 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index df492a9ef..f46986d0a 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -929,12 +929,11 @@ static int getFileInfo(fileInfo_t* info, const char* inFileName){ /* skip the rest of the blocks in the frame */ { int lastBlock = 0; - size_t readBytes = 0; do { BYTE blockHeaderBuffer[3]; U32 blockHeader; int blockSize; - readBytes = fread(blockHeaderBuffer, 1, 3, srcFile); + size_t readBytes = fread(blockHeaderBuffer, 1, 3, srcFile); if (readBytes != 3) { DISPLAY("There was a problem reading the block header\n"); detectError = 1; @@ -984,8 +983,8 @@ static int getFileInfo(fileInfo_t* info, const char* inFileName){ break; } { - long const frameSize = MEM_readLE32(frameSizeBuffer); - int const ret = fseek(srcFile, frameSize, SEEK_CUR); + U32 const frameSize = MEM_readLE32(frameSizeBuffer); + int const ret = fseek(srcFile, (long)frameSize, SEEK_CUR); if (ret != 0) { DISPLAY("Error: could not find end of skippable frame\n"); detectError = 1; diff --git a/programs/zstd.1.md b/programs/zstd.1.md index 2f5a6747e..a5ae27cb7 100644 --- a/programs/zstd.1.md +++ b/programs/zstd.1.md @@ -93,6 +93,10 @@ the last one takes effect. * `--train FILEs`: Use FILEs as a training set to create a dictionary. The training set should contain a lot of small files (> 100). +* `-l`, `--list`: + Display information related to a zstd compressed file, such as size, ratio, and checksum. + Some of these fields may not be available. + This command can be augmented with the `-v` modifier. ### Operation modifiers From cb2dbe64349ceb134ebd29a7852aed64393748a1 Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Tue, 20 Jun 2017 09:35:21 -0700 Subject: [PATCH 41/54] changed fileInfo initialization, updated error code --- programs/fileio.c | 18 ++++++------------ programs/zstdcli.c | 4 ++-- 2 files changed, 8 insertions(+), 14 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index f46986d0a..780ddbf13 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -867,7 +867,7 @@ typedef struct { int numActualFrames; int numSkippableFrames; unsigned long long decompressedSize; - int canComputeDecompSize; + int decompUnavailable; unsigned long long compressedSize; int usesCheck; } fileInfo_t; @@ -903,7 +903,7 @@ static int getFileInfo(fileInfo_t* info, const char* inFileName){ if (magicNumber == ZSTD_MAGICNUMBER) { U64 const frameContentSize = ZSTD_getFrameContentSize(headerBuffer, numBytesRead); if (frameContentSize == ZSTD_CONTENTSIZE_ERROR || frameContentSize == ZSTD_CONTENTSIZE_UNKNOWN) { - info->canComputeDecompSize = 0; + info->decompUnavailable = 1; } else { info->decompressedSize += frameContentSize; @@ -1008,7 +1008,7 @@ static void displayInfo(const char* inFileName, fileInfo_t* info, int displayLev double const decompressedSizeMB = (double)info->decompressedSize/(1 MB); const char* checkString = (info->usesCheck ? "XXH64" : "None"); if (displayLevel <= 2) { - if (info->canComputeDecompSize) { + if (!info->decompUnavailable) { double const ratio = (info->decompressedSize == 0) ? 0.0 : compressedSizeMB/decompressedSizeMB; DISPLAYOUT("Skippable Non-Skippable Compressed Uncompressed Ratio Check Filename\n"); DISPLAYOUT("%9d %13d %7.2f MB %9.2f MB %5.3f %s %s\n", @@ -1025,7 +1025,7 @@ static void displayInfo(const char* inFileName, fileInfo_t* info, int displayLev DISPLAYOUT("# Zstandard Frames: %d\n", info->numActualFrames); DISPLAYOUT("# Skippable Frames: %d\n", info->numSkippableFrames); DISPLAYOUT("Compressed Size: %.2f MB (%llu B)\n", compressedSizeMB, info->compressedSize); - if (info->canComputeDecompSize) { + if (!info->decompUnavailable) { DISPLAYOUT("Decompressed Size: %.2f MB (%llu B)\n", decompressedSizeMB, info->decompressedSize); DISPLAYOUT("Ratio: %.4f\n", compressedSizeMB/decompressedSizeMB); } @@ -1038,15 +1038,9 @@ static void displayInfo(const char* inFileName, fileInfo_t* info, int displayLev } int FIO_listFile(const char* inFileName, int displayLevel){ - fileInfo_t info; - /* initialize info to avoid warnings */ - info.numActualFrames = 0; - info.numSkippableFrames = 0; - info.decompressedSize = 0; - info.canComputeDecompSize = 1; - info.compressedSize = 0; - info.usesCheck = 0; + fileInfo_t info; + memset(&info, 0, sizeof(info)); DISPLAYOUT("File: %s\n", inFileName); { int const error = getFileInfo(&info, inFileName); diff --git a/programs/zstdcli.c b/programs/zstdcli.c index a55640351..2b0419b0c 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -688,8 +688,8 @@ int main(int argCount, const char* argv[]) { int error = 0; unsigned u; - for(u=0; u Date: Tue, 20 Jun 2017 10:45:06 -0700 Subject: [PATCH 42/54] change test to list info on .zst files --- tests/playTests.sh | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/playTests.sh b/tests/playTests.sh index 6b794f6d4..07d63e300 100755 --- a/tests/playTests.sh +++ b/tests/playTests.sh @@ -543,20 +543,20 @@ $ECHO "\n**** zstd --list/-l single frame tests ****" ./datagen > tmp3 ./datagen > tmp4 $ZSTD tmp* -$ZSTD -l tmp* -$ZSTD -lv tmp* -$ZSTD --list tmp* -$ZSTD --list -v tmp* +$ZSTD -l *.zst +$ZSTD -lv *.zst +$ZSTD --list *.zst +$ZSTD --list -v *.zst $ECHO "\n**** zstd --list/-l multiple frame tests ****" cat tmp1.zst tmp2.zst > tmp12.zst cat tmp3.zst tmp4.zst > tmp34.zst cat tmp12.zst tmp34.zst > tmp1234.zst cat tmp12.zst tmp4.zst > tmp124.zst -$ZSTD -l tmp* -$ZSTD -lv tmp* -$ZSTD --list tmp* -$ZSTD --list -v tmp* +$ZSTD -l *.zst +$ZSTD -lv *.zst +$ZSTD --list *.zst +$ZSTD --list -v *.zst rm tmp* From bed0dc95bcf01417f0c75dc4403ba750566c034c Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Tue, 20 Jun 2017 11:47:24 -0700 Subject: [PATCH 43/54] changed displayInfo alignment and const --- programs/fileio.c | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index 780ddbf13..f49273271 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -1006,18 +1006,18 @@ static int getFileInfo(fileInfo_t* info, const char* inFileName){ static void displayInfo(const char* inFileName, fileInfo_t* info, int displayLevel){ double const compressedSizeMB = (double)info->compressedSize/(1 MB); double const decompressedSizeMB = (double)info->decompressedSize/(1 MB); - const char* checkString = (info->usesCheck ? "XXH64" : "None"); + const char* const checkString = (info->usesCheck ? "XXH64" : "None"); if (displayLevel <= 2) { if (!info->decompUnavailable) { double const ratio = (info->decompressedSize == 0) ? 0.0 : compressedSizeMB/decompressedSizeMB; DISPLAYOUT("Skippable Non-Skippable Compressed Uncompressed Ratio Check Filename\n"); - DISPLAYOUT("%9d %13d %7.2f MB %9.2f MB %5.3f %s %s\n", + DISPLAYOUT("%9d %13d %7.2f MB %9.2f MB %5.3f %5s %s\n", info->numSkippableFrames, info->numActualFrames, compressedSizeMB, decompressedSizeMB, ratio, checkString, inFileName); } else { DISPLAYOUT("Skippable Non-Skippable Compressed Check Filename\n"); - DISPLAYOUT("%9d %13d %7.2f MB %s %s\n", + DISPLAYOUT("%9d %13d %7.2f MB %5s %s\n", info->numSkippableFrames, info->numActualFrames, compressedSizeMB, checkString, inFileName); } } @@ -1029,12 +1029,9 @@ static void displayInfo(const char* inFileName, fileInfo_t* info, int displayLev DISPLAYOUT("Decompressed Size: %.2f MB (%llu B)\n", decompressedSizeMB, info->decompressedSize); DISPLAYOUT("Ratio: %.4f\n", compressedSizeMB/decompressedSizeMB); } - if (info->usesCheck) { - DISPLAYOUT("Check: XXH64\n"); - } + DISPLAYOUT("Check: %s\n", checkString); + DISPLAYOUT("\n"); } - - DISPLAYOUT("\n"); } int FIO_listFile(const char* inFileName, int displayLevel){ @@ -1049,7 +1046,10 @@ int FIO_listFile(const char* inFileName, int displayLevel){ DISPLAY("An error occurred with getting file info\n"); } else if (error == 2) { - DISPLAYOUT("File %s not compressed with zstd\n\n", inFileName); + DISPLAYOUT("File %s not compressed with zstd\n", inFileName); + if (displayLevel > 2) { + DISPLAYOUT("\n"); + } return 1; } displayInfo(inFileName, &info, displayLevel); From b07d0af2baa2197a12e30af855c1ea2cbdb4f58e Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Tue, 20 Jun 2017 11:54:44 -0700 Subject: [PATCH 44/54] added additional tests for error detection --- tests/playTests.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/playTests.sh b/tests/playTests.sh index 07d63e300..0fa6a7219 100755 --- a/tests/playTests.sh +++ b/tests/playTests.sh @@ -558,6 +558,12 @@ $ZSTD -lv *.zst $ZSTD --list *.zst $ZSTD --list -v *.zst +$ECHO "\n**** zstd --list/-l error detection tests ****" +!$ZSTD -l tmp1 tmp1.zst +!$ZSTD --list tmp* +!$ZSTD -lv tmp1* +!$ZSTD --list -v tmp2 tmp23.zst + rm tmp* if [ "$1" != "--test-large-data" ]; then From cea55a9217530b1b2888836c334194c960c77523 Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Tue, 20 Jun 2017 11:56:11 -0700 Subject: [PATCH 45/54] size_t const value --- programs/fileio.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/programs/fileio.c b/programs/fileio.c index f49273271..342325465 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -933,7 +933,7 @@ static int getFileInfo(fileInfo_t* info, const char* inFileName){ BYTE blockHeaderBuffer[3]; U32 blockHeader; int blockSize; - size_t readBytes = fread(blockHeaderBuffer, 1, 3, srcFile); + size_t const readBytes = fread(blockHeaderBuffer, 1, 3, srcFile); if (readBytes != 3) { DISPLAY("There was a problem reading the block header\n"); detectError = 1; From 72e0ac9929b97f2711d08f9155c1211f7aed634c Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Tue, 20 Jun 2017 11:58:27 -0700 Subject: [PATCH 46/54] minor update to logic --- programs/fileio.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/programs/fileio.c b/programs/fileio.c index 342325465..533b2250a 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -941,7 +941,7 @@ static int getFileInfo(fileInfo_t* info, const char* inFileName){ } blockHeader = MEM_readLE24(blockHeaderBuffer); lastBlock = blockHeader & 1; - blockSize = (blockHeader - (blockHeader & 7)) >> 3; + blockSize = blockHeader >> 3; { int const ret = fseek(srcFile, blockSize, SEEK_CUR); if (ret != 0) { From c594507871efe3d89a6cfe26d69444e7c13f2a31 Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Tue, 20 Jun 2017 11:59:45 -0700 Subject: [PATCH 47/54] removed double if statements for same condition --- programs/fileio.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index 533b2250a..3964f0f97 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -960,11 +960,9 @@ static int getFileInfo(fileInfo_t* info, const char* inFileName){ /* check if checksum is used */ BYTE const frameHeaderDescriptor = headerBuffer[4]; int const contentChecksumFlag = (frameHeaderDescriptor & (1 << 2)) >> 2; - if (contentChecksumFlag) { - info->usesCheck = 1; - } if (contentChecksumFlag) { int const ret = fseek(srcFile, 4, SEEK_CUR); + info->usesCheck = 1; if (ret != 0) { DISPLAY("Error: could not skip past checksum\n"); detectError = 1; From bda57754a2bed5845c583906f6def2486988e1fd Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Tue, 20 Jun 2017 12:43:10 -0700 Subject: [PATCH 48/54] moved signifcant operations to fileio.c --- programs/fileio.c | 22 +++++++++++++++++++++- programs/fileio.h | 2 +- programs/zstdcli.c | 19 ++----------------- 3 files changed, 24 insertions(+), 19 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index 3964f0f97..059823952 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -1032,7 +1032,8 @@ static void displayInfo(const char* inFileName, fileInfo_t* info, int displayLev } } -int FIO_listFile(const char* inFileName, int displayLevel){ + +static int FIO_listFile(const char* inFileName, int displayLevel){ /* initialize info to avoid warnings */ fileInfo_t info; memset(&info, 0, sizeof(info)); @@ -1055,6 +1056,25 @@ int FIO_listFile(const char* inFileName, int displayLevel){ } } +int FIO_listMultipleFiles(unsigned numFiles, const char** filenameTable, int displayLevel){ + if (numFiles == 0) { + DISPLAYOUT("No files given\n"); + return 0; + } + DISPLAYOUT("===========================================\n"); + DISPLAYOUT("Printing information about compressed files\n"); + DISPLAYOUT("===========================================\n"); + DISPLAYOUT("Number of files listed: %u\n", numFiles); + { + int error = 0; + unsigned u; + for (u=0; u Date: Tue, 20 Jun 2017 13:26:25 -0700 Subject: [PATCH 49/54] added progress metric to display --- programs/fileio.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index 059823952..93ec3f181 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -1033,11 +1033,11 @@ static void displayInfo(const char* inFileName, fileInfo_t* info, int displayLev } -static int FIO_listFile(const char* inFileName, int displayLevel){ +static int FIO_listFile(const char* inFileName, int displayLevel, unsigned fileNo, unsigned numFiles){ /* initialize info to avoid warnings */ fileInfo_t info; memset(&info, 0, sizeof(info)); - DISPLAYOUT("File: %s\n", inFileName); + DISPLAYOUT("%s (%u/%u):\n", inFileName, fileNo, numFiles); { int const error = getFileInfo(&info, inFileName); if (error == 1) { @@ -1069,7 +1069,7 @@ int FIO_listMultipleFiles(unsigned numFiles, const char** filenameTable, int dis int error = 0; unsigned u; for (u=0; u Date: Tue, 20 Jun 2017 13:44:05 -0700 Subject: [PATCH 50/54] changed to use LONG_SEEK --- programs/fileio.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/programs/fileio.c b/programs/fileio.c index 93ec3f181..37ec7f815 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -982,7 +982,7 @@ static int getFileInfo(fileInfo_t* info, const char* inFileName){ } { U32 const frameSize = MEM_readLE32(frameSizeBuffer); - int const ret = fseek(srcFile, (long)frameSize, SEEK_CUR); + int const ret = LONG_SEEK(srcFile, frameSize, SEEK_CUR); if (ret != 0) { DISPLAY("Error: could not find end of skippable frame\n"); detectError = 1; From 076560290346c3f83137c6f10f3d5a33efc234aa Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Tue, 20 Jun 2017 14:04:46 -0700 Subject: [PATCH 51/54] fixed error where extremely small files were not being detected as not compressed with zstd --- programs/fileio.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index 37ec7f815..bf65ba528 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -889,10 +889,15 @@ static int getFileInfo(fileInfo_t* info, const char* inFileName){ BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX]; size_t const numBytesRead = fread(headerBuffer, 1, sizeof(headerBuffer), srcFile); if (numBytesRead < ZSTD_frameHeaderSize_min) { - if (feof(srcFile)) { + if (feof(srcFile) && numBytesRead == 0) { break; } - else{ + else if (feof(srcFile)) { + DISPLAY("Error: reached end of file with incomplete frame\n"); + detectError = 2; + break; + } + else { DISPLAY("Error: did not reach end of file but ran out of frames\n"); detectError = 1; break; From 58c19b42026e80e4a09e4f2473045c66e2a87066 Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Tue, 20 Jun 2017 14:14:53 -0700 Subject: [PATCH 52/54] spacing matters for test scripts --- tests/playTests.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/playTests.sh b/tests/playTests.sh index 0fa6a7219..7792a5690 100755 --- a/tests/playTests.sh +++ b/tests/playTests.sh @@ -559,10 +559,10 @@ $ZSTD --list *.zst $ZSTD --list -v *.zst $ECHO "\n**** zstd --list/-l error detection tests ****" -!$ZSTD -l tmp1 tmp1.zst -!$ZSTD --list tmp* -!$ZSTD -lv tmp1* -!$ZSTD --list -v tmp2 tmp23.zst +! $ZSTD -l tmp1 tmp1.zst +! $ZSTD --list tmp* +! $ZSTD -lv tmp1* +! $ZSTD --list -v tmp2 tmp23.zst rm tmp* From a73c2a444aa13ef1d2932ae13f565fea71382fa5 Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Tue, 20 Jun 2017 14:33:08 -0700 Subject: [PATCH 53/54] added tests with null files, changed condition to check that the file is more than 0 bytes --- programs/fileio.c | 2 +- tests/playTests.sh | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/programs/fileio.c b/programs/fileio.c index bf65ba528..a5db88a4c 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -889,7 +889,7 @@ static int getFileInfo(fileInfo_t* info, const char* inFileName){ BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX]; size_t const numBytesRead = fread(headerBuffer, 1, sizeof(headerBuffer), srcFile); if (numBytesRead < ZSTD_frameHeaderSize_min) { - if (feof(srcFile) && numBytesRead == 0) { + if (feof(srcFile) && numBytesRead == 0 && info->compressedSize > 0) { break; } else if (feof(srcFile)) { diff --git a/tests/playTests.sh b/tests/playTests.sh index 7792a5690..21f79c715 100755 --- a/tests/playTests.sh +++ b/tests/playTests.sh @@ -564,6 +564,14 @@ $ECHO "\n**** zstd --list/-l error detection tests ****" ! $ZSTD -lv tmp1* ! $ZSTD --list -v tmp2 tmp23.zst +$ECHO "\n**** zstd --list/-l test with null files ****" +./datagen -g0 > tmp5 +$ZSTD tmp5 +! $ZSTD -l tmp5* +! $ZSTD -lv tmp5* +! $ZSTD --list tmp5* +! $ZSTD --list -v tmp5* + rm tmp* if [ "$1" != "--test-large-data" ]; then From db3606e24989be34395559c42862f4130e5e45f3 Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Tue, 20 Jun 2017 17:43:36 -0700 Subject: [PATCH 54/54] added test for files with no frame content size --- tests/playTests.sh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/playTests.sh b/tests/playTests.sh index 21f79c715..4eb794b54 100755 --- a/tests/playTests.sh +++ b/tests/playTests.sh @@ -572,6 +572,14 @@ $ZSTD tmp5 ! $ZSTD --list tmp5* ! $ZSTD --list -v tmp5* +$ECHO "\n**** zstd --list/-l test with no frame content size ****" +echo -n '' > tmp6 +$ZSTD tmp6 +$ZSTD -l tmp6.zst +$ZSTD -lv tmp6.zst +$ZSTD --list tmp6.zst +$ZSTD --list -v tmp6.zst + rm tmp* if [ "$1" != "--test-large-data" ]; then