From 901435e9ef324b7e639bb9027ad544677b338f00 Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Mon, 5 Jun 2017 14:45:31 -0700 Subject: [PATCH 01/20] setup basic functions for adding --list functionality --- programs/fileio.c | 5 +++++ programs/fileio.h | 1 + programs/zstdcli.c | 17 ++++++++++++++--- 3 files changed, 20 insertions(+), 3 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index 44152b0b1..466faf385 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -862,6 +862,11 @@ int FIO_compressFilename(const char* dstFileName, const char* srcFileName, return result; } +int FIO_listFile(const char* infilename){ + DISPLAY("FILE DETECTED: %s\n", infilename); + DISPLAY("working\n"); + return 0; +} int FIO_compressMultipleFilenames(const char** inFileNamesTable, unsigned nbFiles, const char* suffix, diff --git a/programs/fileio.h b/programs/fileio.h index 65da98d7f..4ca1a2b38 100644 --- a/programs/fileio.h +++ b/programs/fileio.h @@ -70,6 +70,7 @@ int FIO_compressFilename (const char* outfilename, const char* infilename, const @return : 0 == ok; 1 == pb with src file. */ int FIO_decompressFilename (const char* outfilename, const char* infilename, const char* dictFileName); +int FIO_listFile(const char* infilename); /*-************************************* * Multiple File functions diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 32fef9993..be0a447bb 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -117,6 +117,7 @@ static int usage_advanced(const char* programName) DISPLAY( " -v : verbose mode; specify multiple times to increase verbosity\n"); DISPLAY( " -q : suppress warnings; specify twice to suppress errors too\n"); DISPLAY( " -c : force write to standard output, even if it is the console\n"); + DISPLAY( " -l : print information about zstd compressed files.\n"); #ifndef ZSTD_NOCOMPRESS DISPLAY( "--ultra : enable levels beyond %i, up to %i (requires more memory)\n", ZSTDCLI_CLEVEL_MAX, ZSTD_maxCLevel()); #ifdef ZSTD_MULTITHREAD @@ -310,7 +311,7 @@ static unsigned parseCompressionParameters(const char* stringPtr, ZSTD_compressi } -typedef enum { zom_compress, zom_decompress, zom_test, zom_bench, zom_train } zstd_operation_mode; +typedef enum { zom_compress, zom_decompress, zom_test, zom_bench, zom_train, zom_list } zstd_operation_mode; #define CLEAN_RETURN(i) { operationResult = (i); goto _end; } @@ -531,7 +532,7 @@ int main(int argCount, const char* argv[]) argument++; memLimit = readU32FromChar(&argument); break; - + case 'l': operation=zom_list; argument++; break; #ifdef UTIL_HAS_CREATEFILELIST /* recursive */ case 'r': recursive=1; argument++; break; @@ -672,7 +673,17 @@ int main(int argCount, const char* argv[]) } } #endif - + if(operation==zom_list){ + DISPLAY("===========================================\n"); + DISPLAY("Printing information about compressed files\n"); + DISPLAY("===========================================\n"); + unsigned u; + DISPLAY("Number of files listed: %d\n", filenameIdx); + for(u=0; u Date: Mon, 5 Jun 2017 15:00:06 -0700 Subject: [PATCH 02/20] added in check suffix check to ensure file was compressed with zstd --- programs/fileio.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/programs/fileio.c b/programs/fileio.c index 466faf385..7a8907ab9 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -864,7 +864,11 @@ int FIO_compressFilename(const char* dstFileName, const char* srcFileName, int FIO_listFile(const char* infilename){ DISPLAY("FILE DETECTED: %s\n", infilename); - DISPLAY("working\n"); + const char* const suffixPtr = strrchr(infilename, '.'); + if(!suffixPtr || strcmp(suffixPtr, ZSTD_EXTENSION)){ + DISPLAYLEVEL(1, "file %s was not compressed with zstd -- ignoring\n", infilename); + return 1; + } return 0; } From 0f06f4f2663269161820a9dfa9a9fdfa11adf78e Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Tue, 6 Jun 2017 09:21:42 -0700 Subject: [PATCH 03/20] added display for compressed size --- programs/fileio.c | 12 +++++++++++- programs/fileio.h | 2 +- programs/zstdcli.c | 2 +- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index 7a8907ab9..c836020c9 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -862,13 +862,23 @@ int FIO_compressFilename(const char* dstFileName, const char* srcFileName, return result; } -int FIO_listFile(const char* infilename){ +int FIO_listFile(const char* infilename, int displayLevel){ DISPLAY("FILE DETECTED: %s\n", infilename); const char* const suffixPtr = strrchr(infilename, '.'); if(!suffixPtr || strcmp(suffixPtr, ZSTD_EXTENSION)){ DISPLAYLEVEL(1, "file %s was not compressed with zstd -- ignoring\n", infilename); return 1; } + else{ + U64 const compSize = UTIL_getFileSize(infilename); + if(displayLevel<=2){ + DISPLAY("Skippable Non-Skippable Compressed Uncompressed Ratio Check Filename\n"); + DISPLAY(" %7.2f MB\n", (double)compSize/(1 MB)); + } + else{ + DISPLAY("Compressed Size: %.2f MB (%llu B)\n", (double)compSize/(1 MB), compSize); + } + } return 0; } diff --git a/programs/fileio.h b/programs/fileio.h index 4ca1a2b38..b28ba1de7 100644 --- a/programs/fileio.h +++ b/programs/fileio.h @@ -70,7 +70,7 @@ int FIO_compressFilename (const char* outfilename, const char* infilename, const @return : 0 == ok; 1 == pb with src file. */ int FIO_decompressFilename (const char* outfilename, const char* infilename, const char* dictFileName); -int FIO_listFile(const char* infilename); +int FIO_listFile(const char* infilename, int displayLevel); /*-************************************* * Multiple File functions diff --git a/programs/zstdcli.c b/programs/zstdcli.c index be0a447bb..74f2606bc 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -680,7 +680,7 @@ int main(int argCount, const char* argv[]) unsigned u; DISPLAY("Number of files listed: %d\n", filenameIdx); for(u=0; u Date: Mon, 12 Jun 2017 10:58:34 -0700 Subject: [PATCH 04/20] added line spacing for clarity --- programs/fileio.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index c836020c9..a7812d73a 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -867,18 +867,20 @@ int FIO_listFile(const char* infilename, int displayLevel){ const char* const suffixPtr = strrchr(infilename, '.'); if(!suffixPtr || strcmp(suffixPtr, ZSTD_EXTENSION)){ DISPLAYLEVEL(1, "file %s was not compressed with zstd -- ignoring\n", infilename); - return 1; + DISPLAY("\n"); + return 1; } else{ - U64 const compSize = UTIL_getFileSize(infilename); + U64 const compSize = UTIL_getFileSize(infilename); if(displayLevel<=2){ - DISPLAY("Skippable Non-Skippable Compressed Uncompressed Ratio Check Filename\n"); - DISPLAY(" %7.2f MB\n", (double)compSize/(1 MB)); + DISPLAY("Skippable Non-Skippable Compressed Uncompressed Ratio Check Filename\n"); + DISPLAY(" %7.2f MB\n", (double)compSize/(1 MB)); } else{ DISPLAY("Compressed Size: %.2f MB (%llu B)\n", (double)compSize/(1 MB), compSize); } } + DISPLAY("\n"); return 0; } From 786b7cac27634e87a1cebdf3055e762fd15180f0 Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Mon, 12 Jun 2017 13:46:39 -0700 Subject: [PATCH 05/20] added code to analyze the first frame header --- programs/fileio.c | 70 ++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 63 insertions(+), 7 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index a7812d73a..bf92f6b11 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -862,22 +862,78 @@ int FIO_compressFilename(const char* dstFileName, const char* srcFileName, return result; } -int FIO_listFile(const char* infilename, int displayLevel){ - DISPLAY("FILE DETECTED: %s\n", infilename); - const char* const suffixPtr = strrchr(infilename, '.'); +typedef struct { + int numActualFrames; + int numSkippableFrames; + U64 decompressedSize; + U64 compressedSize; + int usesCheck; +} fileInfo_t; + +/* + * Reads information from file, stores in *info + * if successful, returns 0, otherwise returns 1 + */ +int getFileInfo(fileInfo_t* info, const char* inFileName){ + FILE* srcFile = FIO_openSrcFile(inFileName); + if(srcFile==NULL){ + return 1; + } + info->compressedSize = UTIL_getFileSize(inFileName); + BYTE* frameHeaderBuffer = malloc(ZSTD_frameHeaderSize_max); + fread(frameHeaderBuffer, ZSTD_frameHeaderSize_max, 1, srcFile); + info->decompressedSize = 0; + info->numActualFrames = 0; + info-> numSkippableFrames = 0; + + /* begin analyzing frame */ + + info->decompressedSize += ZSTD_getFrameContentSize((void*)frameHeaderBuffer, ZSTD_FRAMEHEADERSIZE_MAX); + U32 magicNumber = frameHeaderBuffer[0] + (frameHeaderBuffer[1] << 8) + (frameHeaderBuffer[2] << 16) + (frameHeaderBuffer[3] << 24); + if(magicNumber==ZSTD_MAGICNUMBER){ + info->numActualFrames++; + } + else if(magicNumber==ZSTD_MAGIC_SKIPPABLE_START){ + info->numSkippableFrames++; + } + const int checksumBitMask = 4; + if(frameHeaderBuffer[1] & checksumBitMask){ + DISPLAY("Uses checksum\n"); + info->usesCheck = 1; + } + return 0; +} + +int FIO_listFile(const char* inFileName, int displayLevel){ + DISPLAY("FILE DETECTED: %s\n", inFileName); + const char* const suffixPtr = strrchr(inFileName, '.'); if(!suffixPtr || strcmp(suffixPtr, ZSTD_EXTENSION)){ - DISPLAYLEVEL(1, "file %s was not compressed with zstd -- ignoring\n", infilename); + DISPLAYLEVEL(1, "file %s was not compressed with zstd -- ignoring\n", inFileName); DISPLAY("\n"); return 1; } else{ - U64 const compSize = UTIL_getFileSize(infilename); + fileInfo_t* info = malloc(sizeof(fileInfo_t)); + int error = getFileInfo(info, inFileName); + if(error==1){ + DISPLAY("An error occurred with getting file info\n"); + exit(1); + } if(displayLevel<=2){ DISPLAY("Skippable Non-Skippable Compressed Uncompressed Ratio Check Filename\n"); - DISPLAY(" %7.2f MB\n", (double)compSize/(1 MB)); + DISPLAY(" %7.2f MB\n", (double)info->compressedSize/(1 MB)); } else{ - DISPLAY("Compressed Size: %.2f MB (%llu B)\n", (double)compSize/(1 MB), compSize); + DISPLAY("Compressed Size: %.2f MB (%llu B)\n", (double)info->compressedSize/(1 MB), info->compressedSize); + if(info->decompressedSize!=ZSTD_CONTENTSIZE_ERROR && info->decompressedSize!=ZSTD_CONTENTSIZE_UNKNOWN){ + DISPLAY("Decompressed Size: %.2f MB (%llu B)\n", (double)info->decompressedSize/(1 MB), info->decompressedSize); + } + else if(info->decompressedSize==ZSTD_CONTENTSIZE_ERROR){ + DISPLAY("Decompressed Size: There was an error with getting the decompressed size\n"); + } + else{ + DISPLAY("Decompressed Size: N/A\n"); + } } } DISPLAY("\n"); From 9cb602ee25736e81a004fd6caa667ab101510e41 Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Mon, 12 Jun 2017 15:22:48 -0700 Subject: [PATCH 06/20] added in logic for parsing through blocks/frames --- programs/fileio.c | 101 ++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 85 insertions(+), 16 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index bf92f6b11..a2d9b8f30 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -870,6 +870,8 @@ typedef struct { int usesCheck; } fileInfo_t; + + /* * Reads information from file, stores in *info * if successful, returns 0, otherwise returns 1 @@ -880,26 +882,93 @@ int getFileInfo(fileInfo_t* info, const char* inFileName){ return 1; } info->compressedSize = UTIL_getFileSize(inFileName); - BYTE* frameHeaderBuffer = malloc(ZSTD_frameHeaderSize_max); - fread(frameHeaderBuffer, ZSTD_frameHeaderSize_max, 1, srcFile); info->decompressedSize = 0; info->numActualFrames = 0; info-> numSkippableFrames = 0; - /* begin analyzing frame */ - - info->decompressedSize += ZSTD_getFrameContentSize((void*)frameHeaderBuffer, ZSTD_FRAMEHEADERSIZE_MAX); - U32 magicNumber = frameHeaderBuffer[0] + (frameHeaderBuffer[1] << 8) + (frameHeaderBuffer[2] << 16) + (frameHeaderBuffer[3] << 24); - if(magicNumber==ZSTD_MAGICNUMBER){ - info->numActualFrames++; - } - else if(magicNumber==ZSTD_MAGIC_SKIPPABLE_START){ - info->numSkippableFrames++; - } - const int checksumBitMask = 4; - if(frameHeaderBuffer[1] & checksumBitMask){ - DISPLAY("Uses checksum\n"); - info->usesCheck = 1; + while(1){ + BYTE magicNumberBuffer[4]; + int numBytesRead = fread(magicNumberBuffer, 1, 4, srcFile); + if(numBytesRead != 4) break; + U32 magicNumber = MEM_readLE32(magicNumberBuffer); + if(magicNumber==ZSTD_MAGICNUMBER){ + DISPLAY("Hello\n"); + int frameContentSizeBytes=0; + info->numActualFrames++; + BYTE frameHeaderDescriptor; + fread(&frameHeaderDescriptor, 1, 1, srcFile); + + /* calculate actual frame header size */ + int frameContentSizeFlag = frameHeaderDescriptor >> 6; + int singleSegmentFlag = (frameHeaderDescriptor & (1 << 5)) >> 5; + int contentChecksumFlag = (frameHeaderDescriptor & (1 << 2)) >> 2; + int dictionaryIDFlag = frameHeaderDescriptor & 3; + if(frameContentSizeFlag!=0){ + frameContentSizeBytes = 1 << frameContentSizeFlag; + } + else if(singleSegmentFlag){ + frameContentSizeBytes = 1; + } + int windowDescriptorBytes = singleSegmentFlag ? 0 : 1; + int dictionaryIDBytes = dictionaryIDFlag ? 1 << (dictionaryIDFlag - 1): 0; + int totalFrameHeaderBytes = 4 + 1 + windowDescriptorBytes + frameContentSizeBytes + dictionaryIDBytes; + + /* reset to beginning of from and read entire header */ + fseek(srcFile, -5, SEEK_CUR); + BYTE* frameHeader = malloc(totalFrameHeaderBytes); + fread(frameHeader, totalFrameHeaderBytes, 1, srcFile); + + /* get decompressed file size */ + info->decompressedSize = ZSTD_getFrameContentSize(frameHeader, totalFrameHeaderBytes); + + /* check if checksum is used */ + if(contentChecksumFlag){ + info->usesCheck = 1; + } + + /* skip the rest of the blocks in the frame */ + int lastBlock = 0; + do{ + BYTE blockHeaderBuffer[3]; + fread(blockHeaderBuffer, 3, 1, srcFile); + U32 blockHeader = MEM_readLE24(blockHeaderBuffer); + lastBlock = blockHeader & 1; + int blockSize = (blockHeader - (blockHeader & 7)) >> 3; + fseek(srcFile, blockSize, SEEK_CUR); + }while(lastBlock != 1); + if(contentChecksumFlag){ + fseek(srcFile, 4, SEEK_CUR); + } + } + else if(magicNumber==ZSTD_MAGIC_SKIPPABLE_START){ + info->numSkippableFrames++; + BYTE frameSizeBuffer[4]; + fread(frameSizeBuffer, 4, 1, srcFile); + long frameSize = MEM_readLE32(frameSizeBuffer); + fseek(srcFile, frameSize, SEEK_CUR); + } + + + // int isSkippableFrame = 0; + // /* analyze the first 18 bytes (magic number + frame header) of the frame */ + // BYTE* frameHeaderBuffer = malloc(ZSTD_frameHeaderSize_max); + // fread(frameHeaderBuffer, ZSTD_frameHeaderSize_max, 1, srcFile); + // info->decompressedSize += ZSTD_getFrameContentSize((void*)frameHeaderBuffer, ZSTD_FRAMEHEADERSIZE_MAX); + // U32 magicNumber = frameHeaderBuffer[0] + (frameHeaderBuffer[1] << 8) + (frameHeaderBuffer[2] << 16) + (frameHeaderBuffer[3] << 24); + // if(magicNumber==ZSTD_MAGICNUMBER){ + // info->numActualFrames++; + // } + // else if(magicNumber==ZSTD_MAGIC_SKIPPABLE_START){ + // info->numSkippableFrames++; + // isSkippableFrame = 1; + // } + // const int checksumBitMask = 4; + // if(frameHeaderBuffer[1] & checksumBitMask){ + // DISPLAY("Uses checksum\n"); + // info->usesCheck = 1; + // } + + } return 0; } From 6996bd25985d13d7e4fe8274865353b0a951acd7 Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Mon, 12 Jun 2017 15:24:50 -0700 Subject: [PATCH 07/20] removed useless lines --- programs/fileio.c | 22 ---------------------- 1 file changed, 22 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index a2d9b8f30..2e8e4a6f6 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -892,7 +892,6 @@ int getFileInfo(fileInfo_t* info, const char* inFileName){ if(numBytesRead != 4) break; U32 magicNumber = MEM_readLE32(magicNumberBuffer); if(magicNumber==ZSTD_MAGICNUMBER){ - DISPLAY("Hello\n"); int frameContentSizeBytes=0; info->numActualFrames++; BYTE frameHeaderDescriptor; @@ -948,27 +947,6 @@ int getFileInfo(fileInfo_t* info, const char* inFileName){ fseek(srcFile, frameSize, SEEK_CUR); } - - // int isSkippableFrame = 0; - // /* analyze the first 18 bytes (magic number + frame header) of the frame */ - // BYTE* frameHeaderBuffer = malloc(ZSTD_frameHeaderSize_max); - // fread(frameHeaderBuffer, ZSTD_frameHeaderSize_max, 1, srcFile); - // info->decompressedSize += ZSTD_getFrameContentSize((void*)frameHeaderBuffer, ZSTD_FRAMEHEADERSIZE_MAX); - // U32 magicNumber = frameHeaderBuffer[0] + (frameHeaderBuffer[1] << 8) + (frameHeaderBuffer[2] << 16) + (frameHeaderBuffer[3] << 24); - // if(magicNumber==ZSTD_MAGICNUMBER){ - // info->numActualFrames++; - // } - // else if(magicNumber==ZSTD_MAGIC_SKIPPABLE_START){ - // info->numSkippableFrames++; - // isSkippableFrame = 1; - // } - // const int checksumBitMask = 4; - // if(frameHeaderBuffer[1] & checksumBitMask){ - // DISPLAY("Uses checksum\n"); - // info->usesCheck = 1; - // } - - } return 0; } From dcd6ba6dc64f56ff80150de6a904492d372c3c32 Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Mon, 12 Jun 2017 15:40:47 -0700 Subject: [PATCH 08/20] incremented decompressedSize instead of setting value --- programs/fileio.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/programs/fileio.c b/programs/fileio.c index 2e8e4a6f6..d5da78479 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -918,7 +918,7 @@ int getFileInfo(fileInfo_t* info, const char* inFileName){ fread(frameHeader, totalFrameHeaderBytes, 1, srcFile); /* get decompressed file size */ - info->decompressedSize = ZSTD_getFrameContentSize(frameHeader, totalFrameHeaderBytes); + info->decompressedSize += ZSTD_getFrameContentSize(frameHeader, totalFrameHeaderBytes); /* check if checksum is used */ if(contentChecksumFlag){ From 6e0204470a745df9e9bbee053ca5a4e78bdf7ce6 Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Mon, 12 Jun 2017 15:51:59 -0700 Subject: [PATCH 09/20] displayed decompressed size --- programs/fileio.c | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index d5da78479..d29589544 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -866,6 +866,7 @@ typedef struct { int numActualFrames; int numSkippableFrames; U64 decompressedSize; + int canComputeDecompSize; U64 compressedSize; int usesCheck; } fileInfo_t; @@ -885,6 +886,7 @@ int getFileInfo(fileInfo_t* info, const char* inFileName){ info->decompressedSize = 0; info->numActualFrames = 0; info-> numSkippableFrames = 0; + info->canComputeDecompSize = 1; /* begin analyzing frame */ while(1){ BYTE magicNumberBuffer[4]; @@ -918,7 +920,13 @@ int getFileInfo(fileInfo_t* info, const char* inFileName){ fread(frameHeader, totalFrameHeaderBytes, 1, srcFile); /* get decompressed file size */ - info->decompressedSize += ZSTD_getFrameContentSize(frameHeader, totalFrameHeaderBytes); + U64 additional = ZSTD_getFrameContentSize(frameHeader, totalFrameHeaderBytes); + if(additional!=ZSTD_CONTENTSIZE_UNKNOWN && additional!=ZSTD_CONTENTSIZE_ERROR){ + info->decompressedSize += additional; + } + else{ + info->canComputeDecompSize = 0; + } /* check if checksum is used */ if(contentChecksumFlag){ @@ -966,9 +974,14 @@ int FIO_listFile(const char* inFileName, int displayLevel){ DISPLAY("An error occurred with getting file info\n"); exit(1); } + + double compressedSizeMB = (double)info->compressedSize/(1 MB); + double decompressedSizeMB = (double)info->decompressedSize/(1 MB); + DISPLAY("%d %d\n", info->canComputeDecompSize, info->usesCheck); + if(displayLevel<=2){ DISPLAY("Skippable Non-Skippable Compressed Uncompressed Ratio Check Filename\n"); - DISPLAY(" %7.2f MB\n", (double)info->compressedSize/(1 MB)); + DISPLAY(" %7.2f MB %7.2f MB\n", compressedSizeMB, decompressedSizeMB); } else{ DISPLAY("Compressed Size: %.2f MB (%llu B)\n", (double)info->compressedSize/(1 MB), info->compressedSize); From f3d6ab28a250445f78f01bbf92bc2de01ad4fe86 Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Mon, 12 Jun 2017 15:59:28 -0700 Subject: [PATCH 10/20] formated information for verbose output --- programs/fileio.c | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index d29589544..b8dfd898e 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -977,22 +977,21 @@ int FIO_listFile(const char* inFileName, int displayLevel){ double compressedSizeMB = (double)info->compressedSize/(1 MB); double decompressedSizeMB = (double)info->decompressedSize/(1 MB); - DISPLAY("%d %d\n", info->canComputeDecompSize, info->usesCheck); if(displayLevel<=2){ DISPLAY("Skippable Non-Skippable Compressed Uncompressed Ratio Check Filename\n"); DISPLAY(" %7.2f MB %7.2f MB\n", compressedSizeMB, decompressedSizeMB); } else{ - DISPLAY("Compressed Size: %.2f MB (%llu B)\n", (double)info->compressedSize/(1 MB), info->compressedSize); - if(info->decompressedSize!=ZSTD_CONTENTSIZE_ERROR && info->decompressedSize!=ZSTD_CONTENTSIZE_UNKNOWN){ - DISPLAY("Decompressed Size: %.2f MB (%llu B)\n", (double)info->decompressedSize/(1 MB), info->decompressedSize); + DISPLAY("# Zstandard Frames: %d\n", info->numActualFrames); + DISPLAY("# Skippable Frames: %d\n", info->numSkippableFrames); + DISPLAY("Compressed Size: %.2f MB (%llu B)\n", compressedSizeMB, info->compressedSize); + if(info->canComputeDecompSize){ + DISPLAY("Decompressed Size: %.2f MB (%llu B)\n", decompressedSizeMB, info->decompressedSize); + DISPLAY("Ratio: %.4f\n", compressedSizeMB/decompressedSizeMB); } - else if(info->decompressedSize==ZSTD_CONTENTSIZE_ERROR){ - DISPLAY("Decompressed Size: There was an error with getting the decompressed size\n"); - } - else{ - DISPLAY("Decompressed Size: N/A\n"); + if(info->usesCheck){ + DISPLAY("Check: XXH64\n"); } } } From 173a7397170ffe2c1633cc319140b5142afe3579 Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Mon, 12 Jun 2017 16:14:04 -0700 Subject: [PATCH 11/20] code refactorings with explicit conversion --- programs/fileio.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index b8dfd898e..29029238e 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -890,7 +890,7 @@ int getFileInfo(fileInfo_t* info, const char* inFileName){ /* begin analyzing frame */ while(1){ BYTE magicNumberBuffer[4]; - int numBytesRead = fread(magicNumberBuffer, 1, 4, srcFile); + size_t numBytesRead = fread(magicNumberBuffer, 1, 4, srcFile); if(numBytesRead != 4) break; U32 magicNumber = MEM_readLE32(magicNumberBuffer); if(magicNumber==ZSTD_MAGICNUMBER){ @@ -916,7 +916,7 @@ int getFileInfo(fileInfo_t* info, const char* inFileName){ /* reset to beginning of from and read entire header */ fseek(srcFile, -5, SEEK_CUR); - BYTE* frameHeader = malloc(totalFrameHeaderBytes); + BYTE* frameHeader = (BYTE*)malloc(totalFrameHeaderBytes); fread(frameHeader, totalFrameHeaderBytes, 1, srcFile); /* get decompressed file size */ @@ -968,7 +968,7 @@ int FIO_listFile(const char* inFileName, int displayLevel){ return 1; } else{ - fileInfo_t* info = malloc(sizeof(fileInfo_t)); + fileInfo_t* info = (fileInfo_t*)malloc(sizeof(fileInfo_t)); int error = getFileInfo(info, inFileName); if(error==1){ DISPLAY("An error occurred with getting file info\n"); From 6e33c74133a78673f6fa5729b80ab423b0d38f9c Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Mon, 12 Jun 2017 16:29:00 -0700 Subject: [PATCH 12/20] formatting issues resolved --- programs/fileio.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index 29029238e..73912be5a 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -985,9 +985,9 @@ int FIO_listFile(const char* inFileName, int displayLevel){ else{ DISPLAY("# Zstandard Frames: %d\n", info->numActualFrames); DISPLAY("# Skippable Frames: %d\n", info->numSkippableFrames); - DISPLAY("Compressed Size: %.2f MB (%llu B)\n", compressedSizeMB, info->compressedSize); + DISPLAY("Compressed Size: %.2f MB (%lu B)\n", compressedSizeMB, info->compressedSize); if(info->canComputeDecompSize){ - DISPLAY("Decompressed Size: %.2f MB (%llu B)\n", decompressedSizeMB, info->decompressedSize); + DISPLAY("Decompressed Size: %.2f MB (%lu B)\n", decompressedSizeMB, info->decompressedSize); DISPLAY("Ratio: %.4f\n", compressedSizeMB/decompressedSizeMB); } if(info->usesCheck){ From 618a7b66e2977edc113516e0d362d517934e606e Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Mon, 12 Jun 2017 16:53:50 -0700 Subject: [PATCH 13/20] refactoring for C90 syntax --- programs/fileio.c | 66 ++++++++++++++++++++++++++++++++-------------- programs/zstdcli.c | 8 +++--- 2 files changed, 50 insertions(+), 24 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index 73912be5a..e49d0386f 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -891,36 +891,54 @@ int getFileInfo(fileInfo_t* info, const char* inFileName){ while(1){ BYTE magicNumberBuffer[4]; size_t numBytesRead = fread(magicNumberBuffer, 1, 4, srcFile); + U32 magicNumber; if(numBytesRead != 4) break; - U32 magicNumber = MEM_readLE32(magicNumberBuffer); + magicNumber = MEM_readLE32(magicNumberBuffer); if(magicNumber==ZSTD_MAGICNUMBER){ - int frameContentSizeBytes=0; - info->numActualFrames++; BYTE frameHeaderDescriptor; - fread(&frameHeaderDescriptor, 1, 1, srcFile); - + int frameContentSizeFlag; + int singleSegmentFlag; + int contentChecksumFlag; + int dictionaryIDFlag; + int frameContentSizeBytes=0; + int windowDescriptorBytes; + int dictionaryIDBytes; + int totalFrameHeaderBytes; + BYTE* frameHeader; + U64 additional; + int lastBlock; + size_t readBytes = fread(&frameHeaderDescriptor, 1, 1, srcFile); + info->numActualFrames++; + if(readBytes != 1){ + DISPLAY("There was an error with reading frame header descriptor\n"); + exit(1); + } /* calculate actual frame header size */ - int frameContentSizeFlag = frameHeaderDescriptor >> 6; - int singleSegmentFlag = (frameHeaderDescriptor & (1 << 5)) >> 5; - int contentChecksumFlag = (frameHeaderDescriptor & (1 << 2)) >> 2; - int dictionaryIDFlag = frameHeaderDescriptor & 3; + frameContentSizeFlag = frameHeaderDescriptor >> 6; + singleSegmentFlag = (frameHeaderDescriptor & (1 << 5)) >> 5; + contentChecksumFlag = (frameHeaderDescriptor & (1 << 2)) >> 2; + dictionaryIDFlag = frameHeaderDescriptor & 3; if(frameContentSizeFlag!=0){ frameContentSizeBytes = 1 << frameContentSizeFlag; } else if(singleSegmentFlag){ frameContentSizeBytes = 1; } - int windowDescriptorBytes = singleSegmentFlag ? 0 : 1; - int dictionaryIDBytes = dictionaryIDFlag ? 1 << (dictionaryIDFlag - 1): 0; - int totalFrameHeaderBytes = 4 + 1 + windowDescriptorBytes + frameContentSizeBytes + dictionaryIDBytes; + windowDescriptorBytes = singleSegmentFlag ? 0 : 1; + dictionaryIDBytes = dictionaryIDFlag ? 1 << (dictionaryIDFlag - 1): 0; + totalFrameHeaderBytes = 4 + 1 + windowDescriptorBytes + frameContentSizeBytes + dictionaryIDBytes; /* reset to beginning of from and read entire header */ fseek(srcFile, -5, SEEK_CUR); - BYTE* frameHeader = (BYTE*)malloc(totalFrameHeaderBytes); - fread(frameHeader, totalFrameHeaderBytes, 1, srcFile); + frameHeader = (BYTE*)malloc(totalFrameHeaderBytes); + readBytes = fread(frameHeader, 1, totalFrameHeaderBytes, srcFile); + if(readBytes != (size_t)totalFrameHeaderBytes){ + DISPLAY("There was an error reading the frame header\n"); + exit(1); + } /* get decompressed file size */ - U64 additional = ZSTD_getFrameContentSize(frameHeader, totalFrameHeaderBytes); + additional = ZSTD_getFrameContentSize(frameHeader, totalFrameHeaderBytes); if(additional!=ZSTD_CONTENTSIZE_UNKNOWN && additional!=ZSTD_CONTENTSIZE_ERROR){ info->decompressedSize += additional; } @@ -934,10 +952,14 @@ int getFileInfo(fileInfo_t* info, const char* inFileName){ } /* skip the rest of the blocks in the frame */ - int lastBlock = 0; + lastBlock = 0; do{ BYTE blockHeaderBuffer[3]; - fread(blockHeaderBuffer, 3, 1, srcFile); + readBytes = fread(blockHeaderBuffer, 1, 3, srcFile); + if(readBytes != 3){ + DISPLAY("There was a problem reading the block header\n"); + exit(1); + } U32 blockHeader = MEM_readLE24(blockHeaderBuffer); lastBlock = blockHeader & 1; int blockSize = (blockHeader - (blockHeader & 7)) >> 3; @@ -950,7 +972,11 @@ int getFileInfo(fileInfo_t* info, const char* inFileName){ else if(magicNumber==ZSTD_MAGIC_SKIPPABLE_START){ info->numSkippableFrames++; BYTE frameSizeBuffer[4]; - fread(frameSizeBuffer, 4, 1, srcFile); + size_t readBytes = fread(frameSizeBuffer, 1, 4, srcFile); + if(readBytes != 4){ + DISPLAY("There was an error reading skippable frame size"); + exit(1); + } long frameSize = MEM_readLE32(frameSizeBuffer); fseek(srcFile, frameSize, SEEK_CUR); } @@ -985,9 +1011,9 @@ int FIO_listFile(const char* inFileName, int displayLevel){ else{ DISPLAY("# Zstandard Frames: %d\n", info->numActualFrames); DISPLAY("# Skippable Frames: %d\n", info->numSkippableFrames); - DISPLAY("Compressed Size: %.2f MB (%lu B)\n", compressedSizeMB, info->compressedSize); + DISPLAY("Compressed Size: %.2f MB (%llu B)\n", compressedSizeMB, info->compressedSize); if(info->canComputeDecompSize){ - DISPLAY("Decompressed Size: %.2f MB (%lu B)\n", decompressedSizeMB, info->decompressedSize); + DISPLAY("Decompressed Size: %.2f MB (%llu B)\n", decompressedSizeMB, info->decompressedSize); DISPLAY("Ratio: %.4f\n", compressedSizeMB/decompressedSizeMB); } if(info->usesCheck){ diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 74f2606bc..91bce8bb8 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -674,13 +674,13 @@ int main(int argCount, const char* argv[]) } #endif if(operation==zom_list){ - DISPLAY("===========================================\n"); - DISPLAY("Printing information about compressed files\n"); - DISPLAY("===========================================\n"); unsigned u; + DISPLAY("===========================================\n"); + DISPLAY("Printing information about compressed files\n"); + DISPLAY("===========================================\n"); DISPLAY("Number of files listed: %d\n", filenameIdx); for(u=0; u Date: Mon, 12 Jun 2017 17:09:08 -0700 Subject: [PATCH 14/20] more refactoring for C90 --- programs/fileio.c | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index e49d0386f..497e202d1 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -955,14 +955,16 @@ int getFileInfo(fileInfo_t* info, const char* inFileName){ lastBlock = 0; do{ BYTE blockHeaderBuffer[3]; + U32 blockHeader; + int blockSize; readBytes = fread(blockHeaderBuffer, 1, 3, srcFile); if(readBytes != 3){ DISPLAY("There was a problem reading the block header\n"); exit(1); } - U32 blockHeader = MEM_readLE24(blockHeaderBuffer); + blockHeader = MEM_readLE24(blockHeaderBuffer); lastBlock = blockHeader & 1; - int blockSize = (blockHeader - (blockHeader & 7)) >> 3; + blockSize = (blockHeader - (blockHeader & 7)) >> 3; fseek(srcFile, blockSize, SEEK_CUR); }while(lastBlock != 1); if(contentChecksumFlag){ @@ -970,14 +972,15 @@ int getFileInfo(fileInfo_t* info, const char* inFileName){ } } else if(magicNumber==ZSTD_MAGIC_SKIPPABLE_START){ - info->numSkippableFrames++; BYTE frameSizeBuffer[4]; + long frameSize; + info->numSkippableFrames++; size_t readBytes = fread(frameSizeBuffer, 1, 4, srcFile); if(readBytes != 4){ DISPLAY("There was an error reading skippable frame size"); exit(1); } - long frameSize = MEM_readLE32(frameSizeBuffer); + frameSize = MEM_readLE32(frameSizeBuffer); fseek(srcFile, frameSize, SEEK_CUR); } @@ -986,14 +989,16 @@ int getFileInfo(fileInfo_t* info, const char* inFileName){ } int FIO_listFile(const char* inFileName, int displayLevel){ - DISPLAY("FILE DETECTED: %s\n", inFileName); const char* const suffixPtr = strrchr(inFileName, '.'); + DISPLAY("FILE DETECTED: %s\n", inFileName); if(!suffixPtr || strcmp(suffixPtr, ZSTD_EXTENSION)){ DISPLAYLEVEL(1, "file %s was not compressed with zstd -- ignoring\n", inFileName); DISPLAY("\n"); return 1; } else{ + double compressedSizeMB; + double decompressedSizeMB; fileInfo_t* info = (fileInfo_t*)malloc(sizeof(fileInfo_t)); int error = getFileInfo(info, inFileName); if(error==1){ @@ -1001,8 +1006,8 @@ int FIO_listFile(const char* inFileName, int displayLevel){ exit(1); } - double compressedSizeMB = (double)info->compressedSize/(1 MB); - double decompressedSizeMB = (double)info->decompressedSize/(1 MB); + compressedSizeMB = (double)info->compressedSize/(1 MB); + decompressedSizeMB = (double)info->decompressedSize/(1 MB); if(displayLevel<=2){ DISPLAY("Skippable Non-Skippable Compressed Uncompressed Ratio Check Filename\n"); From c828b521119b55d12fccc8ae14a949b3b8bf66ed Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Mon, 12 Jun 2017 17:19:53 -0700 Subject: [PATCH 15/20] yet more refactoring --- programs/fileio.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/programs/fileio.c b/programs/fileio.c index 497e202d1..bf8d6d61b 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -974,8 +974,8 @@ int getFileInfo(fileInfo_t* info, const char* inFileName){ else if(magicNumber==ZSTD_MAGIC_SKIPPABLE_START){ BYTE frameSizeBuffer[4]; long frameSize; - info->numSkippableFrames++; size_t readBytes = fread(frameSizeBuffer, 1, 4, srcFile); + info->numSkippableFrames++; if(readBytes != 4){ DISPLAY("There was an error reading skippable frame size"); exit(1); From a56dcbfcf8d41724414f3cc662d6fcbc6242e3d7 Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Mon, 12 Jun 2017 17:47:33 -0700 Subject: [PATCH 16/20] changed print format for non-verbose version --- programs/fileio.c | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index bf8d6d61b..dadb8ecb3 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -1010,8 +1010,28 @@ int FIO_listFile(const char* inFileName, int displayLevel){ decompressedSizeMB = (double)info->decompressedSize/(1 MB); if(displayLevel<=2){ - DISPLAY("Skippable Non-Skippable Compressed Uncompressed Ratio Check Filename\n"); - DISPLAY(" %7.2f MB %7.2f MB\n", compressedSizeMB, decompressedSizeMB); + if(info->usesCheck && info->canComputeDecompSize){ + DISPLAY("Skippable Non-Skippable Compressed Uncompressed Ratio Check Filename\n"); + DISPLAY("%9d %13d %7.2f MB %7.2f MB %5.3f XXH64 %s\n", + info->numSkippableFrames, info->numActualFrames, compressedSizeMB, decompressedSizeMB, + compressedSizeMB/decompressedSizeMB, inFileName); + } + else if(!info->usesCheck){ + DISPLAY("Skippable Non-Skippable Compressed Uncompressed Ratio Check Filename\n"); + DISPLAY("%9d %13d %7.2f MB %7.2f MB %5.3f %s\n", + info->numSkippableFrames, info->numActualFrames, compressedSizeMB, decompressedSizeMB, + compressedSizeMB/decompressedSizeMB, inFileName); + } + else if(!info->canComputeDecompSize){ + DISPLAY("Skippable Non-Skippable Compressed Uncompressed Ratio Check Filename\n"); + DISPLAY("%9d %13d %7.2f MB XXH64 %s\n", + info->numSkippableFrames, info->numActualFrames, compressedSizeMB, inFileName); + } + else{ + DISPLAY("Skippable Non-Skippable Filename\n"); + DISPLAY("%9d %13d %7.2f MB %s\n", + info->numSkippableFrames, info->numActualFrames, compressedSizeMB, inFileName); + } } else{ DISPLAY("# Zstandard Frames: %d\n", info->numActualFrames); From 81fa33b55a8db2256c26391b3531de16bc1f23bf Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Tue, 13 Jun 2017 10:06:49 -0700 Subject: [PATCH 17/20] cleaning up code --- programs/fileio.c | 162 +++++++++++++++++++++++----------------------- 1 file changed, 81 insertions(+), 81 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index dadb8ecb3..ca08c7f9d 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -872,6 +872,23 @@ typedef struct { } fileInfo_t; +int calcFrameHeaderSize(BYTE frameHeaderDescriptor){ + int frameContentSizeBytes = 0; + int windowDescriptorBytes; + int dictionaryIDBytes; + int frameContentSizeFlag = frameHeaderDescriptor >> 6; + int singleSegmentFlag = (frameHeaderDescriptor & (1 << 5)) >> 5; + int dictionaryIDFlag = frameHeaderDescriptor & 3; + if(frameContentSizeFlag!=0){ + frameContentSizeBytes = 1 << frameContentSizeFlag; + } + else if(singleSegmentFlag){ + frameContentSizeBytes = 1; + } + windowDescriptorBytes = singleSegmentFlag ? 0 : 1; + dictionaryIDBytes = dictionaryIDFlag ? 1 << (dictionaryIDFlag - 1): 0; + return 4 + 1 + windowDescriptorBytes + frameContentSizeBytes + dictionaryIDBytes; +} /* * Reads information from file, stores in *info @@ -896,17 +913,9 @@ int getFileInfo(fileInfo_t* info, const char* inFileName){ magicNumber = MEM_readLE32(magicNumberBuffer); if(magicNumber==ZSTD_MAGICNUMBER){ BYTE frameHeaderDescriptor; - int frameContentSizeFlag; - int singleSegmentFlag; - int contentChecksumFlag; - int dictionaryIDFlag; - int frameContentSizeBytes=0; - int windowDescriptorBytes; - int dictionaryIDBytes; int totalFrameHeaderBytes; BYTE* frameHeader; - U64 additional; - int lastBlock; + int lastBlock = 0; size_t readBytes = fread(&frameHeaderDescriptor, 1, 1, srcFile); info->numActualFrames++; if(readBytes != 1){ @@ -914,19 +923,7 @@ int getFileInfo(fileInfo_t* info, const char* inFileName){ exit(1); } /* calculate actual frame header size */ - frameContentSizeFlag = frameHeaderDescriptor >> 6; - singleSegmentFlag = (frameHeaderDescriptor & (1 << 5)) >> 5; - contentChecksumFlag = (frameHeaderDescriptor & (1 << 2)) >> 2; - dictionaryIDFlag = frameHeaderDescriptor & 3; - if(frameContentSizeFlag!=0){ - frameContentSizeBytes = 1 << frameContentSizeFlag; - } - else if(singleSegmentFlag){ - frameContentSizeBytes = 1; - } - windowDescriptorBytes = singleSegmentFlag ? 0 : 1; - dictionaryIDBytes = dictionaryIDFlag ? 1 << (dictionaryIDFlag - 1): 0; - totalFrameHeaderBytes = 4 + 1 + windowDescriptorBytes + frameContentSizeBytes + dictionaryIDBytes; + totalFrameHeaderBytes = calcFrameHeaderSize(frameHeaderDescriptor); /* reset to beginning of from and read entire header */ fseek(srcFile, -5, SEEK_CUR); @@ -938,21 +935,17 @@ int getFileInfo(fileInfo_t* info, const char* inFileName){ } /* get decompressed file size */ - additional = ZSTD_getFrameContentSize(frameHeader, totalFrameHeaderBytes); - if(additional!=ZSTD_CONTENTSIZE_UNKNOWN && additional!=ZSTD_CONTENTSIZE_ERROR){ - info->decompressedSize += additional; - } - else{ - info->canComputeDecompSize = 0; - } - - /* check if checksum is used */ - if(contentChecksumFlag){ - info->usesCheck = 1; + { + U64 additional = ZSTD_getFrameContentSize(frameHeader, totalFrameHeaderBytes); + if(additional!=ZSTD_CONTENTSIZE_UNKNOWN && additional!=ZSTD_CONTENTSIZE_ERROR){ + info->decompressedSize += additional; + } + else{ + info->canComputeDecompSize = 0; + } } /* skip the rest of the blocks in the frame */ - lastBlock = 0; do{ BYTE blockHeaderBuffer[3]; U32 blockHeader; @@ -967,8 +960,15 @@ int getFileInfo(fileInfo_t* info, const char* inFileName){ blockSize = (blockHeader - (blockHeader & 7)) >> 3; fseek(srcFile, blockSize, SEEK_CUR); }while(lastBlock != 1); - if(contentChecksumFlag){ - fseek(srcFile, 4, SEEK_CUR); + { + /* check if checksum is used */ + int contentChecksumFlag = (frameHeaderDescriptor & (1 << 2)) >> 2; + if(contentChecksumFlag){ + info->usesCheck = 1; + } + if(contentChecksumFlag){ + fseek(srcFile, 4, SEEK_CUR); + } } } else if(magicNumber==ZSTD_MAGIC_SKIPPABLE_START){ @@ -987,64 +987,64 @@ int getFileInfo(fileInfo_t* info, const char* inFileName){ } return 0; } +void displayInfo(const char* inFileName, fileInfo_t* info, int displayLevel){ + double compressedSizeMB = (double)info->compressedSize/(1 MB); + double decompressedSizeMB = (double)info->decompressedSize/(1 MB); + + if(displayLevel<=2){ + if(info->usesCheck && info->canComputeDecompSize){ + DISPLAY("Skippable Non-Skippable Compressed Uncompressed Ratio Check Filename\n"); + DISPLAY("%9d %13d %7.2f MB %7.2f MB %5.3f XXH64 %s\n", + info->numSkippableFrames, info->numActualFrames, compressedSizeMB, decompressedSizeMB, + compressedSizeMB/decompressedSizeMB, inFileName); + } + else if(!info->usesCheck){ + DISPLAY("Skippable Non-Skippable Compressed Uncompressed Ratio Check Filename\n"); + DISPLAY("%9d %13d %7.2f MB %7.2f MB %5.3f %s\n", + info->numSkippableFrames, info->numActualFrames, compressedSizeMB, decompressedSizeMB, + compressedSizeMB/decompressedSizeMB, inFileName); + } + else if(!info->canComputeDecompSize){ + DISPLAY("Skippable Non-Skippable Compressed Uncompressed Ratio Check Filename\n"); + DISPLAY("%9d %13d %7.2f MB XXH64 %s\n", + info->numSkippableFrames, info->numActualFrames, compressedSizeMB, inFileName); + } + else{ + DISPLAY("Skippable Non-Skippable Filename\n"); + DISPLAY("%9d %13d %7.2f MB %s\n", + info->numSkippableFrames, info->numActualFrames, compressedSizeMB, inFileName); + } + } + else{ + DISPLAY("# Zstandard Frames: %d\n", info->numActualFrames); + DISPLAY("# Skippable Frames: %d\n", info->numSkippableFrames); + DISPLAY("Compressed Size: %.2f MB (%llu B)\n", compressedSizeMB, info->compressedSize); + if(info->canComputeDecompSize){ + DISPLAY("Decompressed Size: %.2f MB (%llu B)\n", decompressedSizeMB, info->decompressedSize); + DISPLAY("Ratio: %.4f\n", compressedSizeMB/decompressedSizeMB); + } + if(info->usesCheck){ + DISPLAY("Check: XXH64\n"); + } + } + +} int FIO_listFile(const char* inFileName, int displayLevel){ const char* const suffixPtr = strrchr(inFileName, '.'); - DISPLAY("FILE DETECTED: %s\n", inFileName); + DISPLAY("File: %s\n", inFileName); if(!suffixPtr || strcmp(suffixPtr, ZSTD_EXTENSION)){ - DISPLAYLEVEL(1, "file %s was not compressed with zstd -- ignoring\n", inFileName); - DISPLAY("\n"); + DISPLAYLEVEL(1, "file %s was not compressed with zstd -- ignoring\n\n", inFileName); return 1; } else{ - double compressedSizeMB; - double decompressedSizeMB; fileInfo_t* info = (fileInfo_t*)malloc(sizeof(fileInfo_t)); int error = getFileInfo(info, inFileName); if(error==1){ DISPLAY("An error occurred with getting file info\n"); exit(1); } - - compressedSizeMB = (double)info->compressedSize/(1 MB); - decompressedSizeMB = (double)info->decompressedSize/(1 MB); - - if(displayLevel<=2){ - if(info->usesCheck && info->canComputeDecompSize){ - DISPLAY("Skippable Non-Skippable Compressed Uncompressed Ratio Check Filename\n"); - DISPLAY("%9d %13d %7.2f MB %7.2f MB %5.3f XXH64 %s\n", - info->numSkippableFrames, info->numActualFrames, compressedSizeMB, decompressedSizeMB, - compressedSizeMB/decompressedSizeMB, inFileName); - } - else if(!info->usesCheck){ - DISPLAY("Skippable Non-Skippable Compressed Uncompressed Ratio Check Filename\n"); - DISPLAY("%9d %13d %7.2f MB %7.2f MB %5.3f %s\n", - info->numSkippableFrames, info->numActualFrames, compressedSizeMB, decompressedSizeMB, - compressedSizeMB/decompressedSizeMB, inFileName); - } - else if(!info->canComputeDecompSize){ - DISPLAY("Skippable Non-Skippable Compressed Uncompressed Ratio Check Filename\n"); - DISPLAY("%9d %13d %7.2f MB XXH64 %s\n", - info->numSkippableFrames, info->numActualFrames, compressedSizeMB, inFileName); - } - else{ - DISPLAY("Skippable Non-Skippable Filename\n"); - DISPLAY("%9d %13d %7.2f MB %s\n", - info->numSkippableFrames, info->numActualFrames, compressedSizeMB, inFileName); - } - } - else{ - DISPLAY("# Zstandard Frames: %d\n", info->numActualFrames); - DISPLAY("# Skippable Frames: %d\n", info->numSkippableFrames); - DISPLAY("Compressed Size: %.2f MB (%llu B)\n", compressedSizeMB, info->compressedSize); - if(info->canComputeDecompSize){ - DISPLAY("Decompressed Size: %.2f MB (%llu B)\n", decompressedSizeMB, info->decompressedSize); - DISPLAY("Ratio: %.4f\n", compressedSizeMB/decompressedSizeMB); - } - if(info->usesCheck){ - DISPLAY("Check: XXH64\n"); - } - } + displayInfo(inFileName, info, displayLevel); } DISPLAY("\n"); return 0; From ebd60349f52b58197328b84016bd365e2e2d1bfd Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Wed, 14 Jun 2017 13:11:18 -0700 Subject: [PATCH 18/20] changed U64 to unsigned long to get rid of build test errors --- programs/fileio.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index ca08c7f9d..9d905062f 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -865,9 +865,9 @@ int FIO_compressFilename(const char* dstFileName, const char* srcFileName, typedef struct { int numActualFrames; int numSkippableFrames; - U64 decompressedSize; + unsigned long decompressedSize; int canComputeDecompSize; - U64 compressedSize; + unsigned long compressedSize; int usesCheck; } fileInfo_t; @@ -1018,9 +1018,9 @@ void displayInfo(const char* inFileName, fileInfo_t* info, int displayLevel){ else{ DISPLAY("# Zstandard Frames: %d\n", info->numActualFrames); DISPLAY("# Skippable Frames: %d\n", info->numSkippableFrames); - DISPLAY("Compressed Size: %.2f MB (%llu B)\n", compressedSizeMB, info->compressedSize); + DISPLAY("Compressed Size: %.2f MB (%lu B)\n", compressedSizeMB, info->compressedSize); if(info->canComputeDecompSize){ - DISPLAY("Decompressed Size: %.2f MB (%llu B)\n", decompressedSizeMB, info->decompressedSize); + DISPLAY("Decompressed Size: %.2f MB (%lu B)\n", decompressedSizeMB, info->decompressedSize); DISPLAY("Ratio: %.4f\n", compressedSizeMB/decompressedSizeMB); } if(info->usesCheck){ From 607ee9f4bcad9a72425392c55934be5237042e81 Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Wed, 14 Jun 2017 13:23:36 -0700 Subject: [PATCH 19/20] print message when no files are given, rearrange code for readability --- programs/zstdcli.c | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 91bce8bb8..f8612f297 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -674,13 +674,19 @@ int main(int argCount, const char* argv[]) } #endif if(operation==zom_list){ - unsigned u; + if(filenameIdx==0){ + DISPLAY("No files given\n"); + CLEAN_RETURN(0); + } DISPLAY("===========================================\n"); DISPLAY("Printing information about compressed files\n"); DISPLAY("===========================================\n"); DISPLAY("Number of files listed: %d\n", filenameIdx); - for(u=0; u Date: Wed, 14 Jun 2017 13:26:19 -0700 Subject: [PATCH 20/20] changing formatting again --- programs/fileio.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index 9d905062f..2708176c1 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -865,9 +865,9 @@ int FIO_compressFilename(const char* dstFileName, const char* srcFileName, typedef struct { int numActualFrames; int numSkippableFrames; - unsigned long decompressedSize; + unsigned long long decompressedSize; int canComputeDecompSize; - unsigned long compressedSize; + unsigned long long compressedSize; int usesCheck; } fileInfo_t; @@ -899,7 +899,7 @@ int getFileInfo(fileInfo_t* info, const char* inFileName){ if(srcFile==NULL){ return 1; } - info->compressedSize = UTIL_getFileSize(inFileName); + info->compressedSize = (unsigned long long)UTIL_getFileSize(inFileName); info->decompressedSize = 0; info->numActualFrames = 0; info-> numSkippableFrames = 0; @@ -1018,9 +1018,9 @@ void displayInfo(const char* inFileName, fileInfo_t* info, int displayLevel){ else{ DISPLAY("# Zstandard Frames: %d\n", info->numActualFrames); DISPLAY("# Skippable Frames: %d\n", info->numSkippableFrames); - DISPLAY("Compressed Size: %.2f MB (%lu B)\n", compressedSizeMB, info->compressedSize); + DISPLAY("Compressed Size: %.2f MB (%llu B)\n", compressedSizeMB, info->compressedSize); if(info->canComputeDecompSize){ - DISPLAY("Decompressed Size: %.2f MB (%lu B)\n", decompressedSizeMB, info->decompressedSize); + DISPLAY("Decompressed Size: %.2f MB (%llu B)\n", decompressedSizeMB, info->decompressedSize); DISPLAY("Ratio: %.4f\n", compressedSizeMB/decompressedSizeMB); } if(info->usesCheck){