From e20899252904b8e18b9c5b0a6f944ae289b89910 Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Thu, 15 Jun 2017 12:27:32 -0700 Subject: [PATCH 01/12] cleaning up code --- programs/fileio.c | 89 ++++++++++++++++++++++++++-------------------- programs/zstdcli.c | 1 + 2 files changed, 51 insertions(+), 39 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index 2708176c1..8af3c5a77 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -91,6 +91,7 @@ * Macros ***************************************/ #define DISPLAY(...) fprintf(stderr, __VA_ARGS__) +#define DISPLAYOUT(...) fprintf(stdout, __VA_ARGS__) #define DISPLAYLEVEL(l, ...) { if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); } } static int g_displayLevel = 2; /* 0 : no display; 1: errors; 2 : + result + interaction + warnings; 3 : + progression; 4 : + information */ void FIO_setNotificationLevel(unsigned level) { g_displayLevel=level; } @@ -872,21 +873,13 @@ typedef struct { } fileInfo_t; -int calcFrameHeaderSize(BYTE frameHeaderDescriptor){ - int frameContentSizeBytes = 0; - int windowDescriptorBytes; - int dictionaryIDBytes; - int frameContentSizeFlag = frameHeaderDescriptor >> 6; - int singleSegmentFlag = (frameHeaderDescriptor & (1 << 5)) >> 5; - int dictionaryIDFlag = frameHeaderDescriptor & 3; - if(frameContentSizeFlag!=0){ - frameContentSizeBytes = 1 << frameContentSizeFlag; - } - else if(singleSegmentFlag){ - frameContentSizeBytes = 1; - } - windowDescriptorBytes = singleSegmentFlag ? 0 : 1; - dictionaryIDBytes = dictionaryIDFlag ? 1 << (dictionaryIDFlag - 1): 0; +static int calcFrameHeaderSize(BYTE frameHeaderDescriptor){ + const int frameContentSizeFlag = frameHeaderDescriptor >> 6; + const int singleSegmentFlag = (frameHeaderDescriptor & (1 << 5)) >> 5; + const int dictionaryIDFlag = frameHeaderDescriptor & 3; + const int windowDescriptorBytes = singleSegmentFlag ? 0 : 1; + const int frameContentSizeBytes = (frameContentSizeFlag != 0) ? (1 << frameContentSizeFlag) : (singleSegmentFlag ? 1 : 0); + const int dictionaryIDBytes = dictionaryIDFlag ? 1 << (dictionaryIDFlag - 1): 0; return 4 + 1 + windowDescriptorBytes + frameContentSizeBytes + dictionaryIDBytes; } @@ -894,50 +887,67 @@ int calcFrameHeaderSize(BYTE frameHeaderDescriptor){ * Reads information from file, stores in *info * if successful, returns 0, otherwise returns 1 */ -int getFileInfo(fileInfo_t* info, const char* inFileName){ - FILE* srcFile = FIO_openSrcFile(inFileName); - if(srcFile==NULL){ +static int getFileInfo(fileInfo_t* info, const char* inFileName){ + FILE* const srcFile = FIO_openSrcFile(inFileName); + if (srcFile == NULL) { + DISPLAY("Error: could not open source file %s\n", inFileName); return 1; } info->compressedSize = (unsigned long long)UTIL_getFileSize(inFileName); info->decompressedSize = 0; info->numActualFrames = 0; - info-> numSkippableFrames = 0; + info->numSkippableFrames = 0; info->canComputeDecompSize = 1; /* begin analyzing frame */ - while(1){ + for( ; ; ){ BYTE magicNumberBuffer[4]; size_t numBytesRead = fread(magicNumberBuffer, 1, 4, srcFile); U32 magicNumber; - if(numBytesRead != 4) break; + if (numBytesRead != 4) break; magicNumber = MEM_readLE32(magicNumberBuffer); - if(magicNumber==ZSTD_MAGICNUMBER){ + if (magicNumber==ZSTD_MAGICNUMBER) { BYTE frameHeaderDescriptor; int totalFrameHeaderBytes; BYTE* frameHeader; int lastBlock = 0; size_t readBytes = fread(&frameHeaderDescriptor, 1, 1, srcFile); info->numActualFrames++; - if(readBytes != 1){ - DISPLAY("There was an error with reading frame header descriptor\n"); - exit(1); + if (readBytes != 1) { + DISPLAY("Error: could not read frame header descriptor\n"); + fclose(srcFile); + + return 1; } /* calculate actual frame header size */ totalFrameHeaderBytes = calcFrameHeaderSize(frameHeaderDescriptor); /* reset to beginning of from and read entire header */ - fseek(srcFile, -5, SEEK_CUR); + { + int returnVal = fseek(srcFile, -5, SEEK_CUR); + if (returnVal!=0) { + DISPLAY("Error: could not reset to the beginning of the frame header\n"); + fclose(srcFile); + return 1; + } + } frameHeader = (BYTE*)malloc(totalFrameHeaderBytes); + if (frameHeader==NULL) { + DISPLAY("Error: could not allocate space for frameHeader\n"); + fclose(srcFile); + return 1; + } readBytes = fread(frameHeader, 1, totalFrameHeaderBytes, srcFile); - if(readBytes != (size_t)totalFrameHeaderBytes){ - DISPLAY("There was an error reading the frame header\n"); - exit(1); + if (readBytes != (size_t)totalFrameHeaderBytes) { + DISPLAY("Error: could not read frame header\n"); + fclose(srcFile); + free(frameHeader); + return 1; } /* get decompressed file size */ { U64 additional = ZSTD_getFrameContentSize(frameHeader, totalFrameHeaderBytes); - if(additional!=ZSTD_CONTENTSIZE_UNKNOWN && additional!=ZSTD_CONTENTSIZE_ERROR){ + if (additional!=ZSTD_CONTENTSIZE_UNKNOWN && additional!=ZSTD_CONTENTSIZE_ERROR) { info->decompressedSize += additional; } else{ @@ -951,7 +961,7 @@ int getFileInfo(fileInfo_t* info, const char* inFileName){ U32 blockHeader; int blockSize; readBytes = fread(blockHeaderBuffer, 1, 3, srcFile); - if(readBytes != 3){ + if (readBytes != 3) { DISPLAY("There was a problem reading the block header\n"); exit(1); } @@ -959,37 +969,38 @@ int getFileInfo(fileInfo_t* info, const char* inFileName){ lastBlock = blockHeader & 1; blockSize = (blockHeader - (blockHeader & 7)) >> 3; fseek(srcFile, blockSize, SEEK_CUR); - }while(lastBlock != 1); + }while (lastBlock != 1); { /* check if checksum is used */ int contentChecksumFlag = (frameHeaderDescriptor & (1 << 2)) >> 2; - if(contentChecksumFlag){ + if (contentChecksumFlag) { info->usesCheck = 1; } - if(contentChecksumFlag){ + if (contentChecksumFlag) { fseek(srcFile, 4, SEEK_CUR); } } + free(frameHeader); } - else if(magicNumber==ZSTD_MAGIC_SKIPPABLE_START){ + else if (magicNumber == ZSTD_MAGIC_SKIPPABLE_START) { BYTE frameSizeBuffer[4]; long frameSize; size_t readBytes = fread(frameSizeBuffer, 1, 4, srcFile); info->numSkippableFrames++; - if(readBytes != 4){ + if (readBytes != 4) { DISPLAY("There was an error reading skippable frame size"); exit(1); } frameSize = MEM_readLE32(frameSizeBuffer); fseek(srcFile, frameSize, SEEK_CUR); } - } + fclose(srcFile); return 0; } void displayInfo(const char* inFileName, fileInfo_t* info, int displayLevel){ - double compressedSizeMB = (double)info->compressedSize/(1 MB); - double decompressedSizeMB = (double)info->decompressedSize/(1 MB); + double const compressedSizeMB = (double)info->compressedSize/(1 MB); + double const decompressedSizeMB = (double)info->decompressedSize/(1 MB); if(displayLevel<=2){ if(info->usesCheck && info->canComputeDecompSize){ diff --git a/programs/zstdcli.c b/programs/zstdcli.c index f8612f297..6de33ce09 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -674,6 +674,7 @@ int main(int argCount, const char* argv[]) } #endif if(operation==zom_list){ + g_displayOut = stdout; if(filenameIdx==0){ DISPLAY("No files given\n"); CLEAN_RETURN(0); From a9b77c83e58fab8992ce9318cb11033e34a86d52 Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Thu, 15 Jun 2017 14:13:28 -0700 Subject: [PATCH 02/12] cleaning up code for analyzing frames --- lib/decompress/zstd_decompress.c | 2 +- lib/zstd.h | 5 + programs/fileio.c | 170 +++++++++++++++---------------- 3 files changed, 90 insertions(+), 87 deletions(-) diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index ea64edeed..af8b98d7f 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -204,7 +204,7 @@ unsigned ZSTD_isFrame(const void* buffer, size_t size) /** ZSTD_frameHeaderSize() : * srcSize must be >= ZSTD_frameHeaderSize_prefix. * @return : size of the Frame Header */ -static size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize) +size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize) { if (srcSize < ZSTD_frameHeaderSize_prefix) return ERROR(srcSize_wrong); { BYTE const fhd = ((const BYTE*)src)[4]; diff --git a/lib/zstd.h b/lib/zstd.h index b0aae3f73..53acdcdd0 100644 --- a/lib/zstd.h +++ b/lib/zstd.h @@ -461,6 +461,11 @@ ZSTDLIB_API unsigned long long ZSTD_getFrameContentSize(const void *src, size_t * however it does mean that all frame data must be present and valid. */ ZSTDLIB_API unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize); +/*! ZSTD_frameHeaderSize() : +* `src` should point to the start of a ZSTD frame +* `srcSize` must be >= ZSTD_frameHeaderSize_prefix. +* @return : size of the Frame Header */ +size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize); /*************************************** * Context memory usage diff --git a/programs/fileio.c b/programs/fileio.c index 8af3c5a77..4f92e16e8 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -872,17 +872,6 @@ typedef struct { int usesCheck; } fileInfo_t; - -static int calcFrameHeaderSize(BYTE frameHeaderDescriptor){ - const int frameContentSizeFlag = frameHeaderDescriptor >> 6; - const int singleSegmentFlag = (frameHeaderDescriptor & (1 << 5)) >> 5; - const int dictionaryIDFlag = frameHeaderDescriptor & 3; - const int windowDescriptorBytes = singleSegmentFlag ? 0 : 1; - const int frameContentSizeBytes = (frameContentSizeFlag != 0) ? (1 << frameContentSizeFlag) : (singleSegmentFlag ? 1 : 0); - const int dictionaryIDBytes = dictionaryIDFlag ? 1 << (dictionaryIDFlag - 1): 0; - return 4 + 1 + windowDescriptorBytes + frameContentSizeBytes + dictionaryIDBytes; -} - /* * Reads information from file, stores in *info * if successful, returns 0, otherwise returns 1 @@ -900,78 +889,79 @@ static int getFileInfo(fileInfo_t* info, const char* inFileName){ info->canComputeDecompSize = 1; /* begin analyzing frame */ for( ; ; ){ - BYTE magicNumberBuffer[4]; - size_t numBytesRead = fread(magicNumberBuffer, 1, 4, srcFile); - U32 magicNumber; - if (numBytesRead != 4) break; - magicNumber = MEM_readLE32(magicNumberBuffer); - if (magicNumber==ZSTD_MAGICNUMBER) { - BYTE frameHeaderDescriptor; - int totalFrameHeaderBytes; - BYTE* frameHeader; - int lastBlock = 0; - size_t readBytes = fread(&frameHeaderDescriptor, 1, 1, srcFile); - info->numActualFrames++; - if (readBytes != 1) { - DISPLAY("Error: could not read frame header descriptor\n"); + BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX]; + size_t const numBytesRead = fread(headerBuffer, 1, sizeof(headerBuffer), srcFile); + if (numBytesRead < ZSTD_frameHeaderSize_min) { + if(feof(srcFile)){ + DISPLAY("ran out of files\n"); + break; + } + else{ + DISPLAY("Error: did not reach end of file but ran out of frames\n"); fclose(srcFile); - return 1; } - /* calculate actual frame header size */ - totalFrameHeaderBytes = calcFrameHeaderSize(frameHeaderDescriptor); + } + U32 const magicNumber = MEM_readLE32(headerBuffer); + if (magicNumber == ZSTD_MAGICNUMBER) { + U64 const frameContentSize = ZSTD_getFrameContentSize(headerBuffer, numBytesRead); + if (frameContentSize == ZSTD_CONTENTSIZE_ERROR || frameContentSize == ZSTD_CONTENTSIZE_UNKNOWN) { + info->canComputeDecompSize = 0; + DISPLAY("could not compute decompressed size\n"); + } + else { + info->decompressedSize += frameContentSize; + } + size_t const headerSize = ZSTD_frameHeaderSize(headerBuffer, numBytesRead); + if (ZSTD_isError(headerSize)) { + fclose(srcFile); + DISPLAY("Error: could not determine frame header size\n"); + return 1; + } - /* reset to beginning of from and read entire header */ { - int returnVal = fseek(srcFile, -5, SEEK_CUR); - if (returnVal!=0) { - DISPLAY("Error: could not reset to the beginning of the frame header\n"); + /* go back to the beginning of the frame */ + int const ret = fseek(srcFile, -numBytesRead, SEEK_CUR); + if (ret != 0) { + DISPLAY("Error: could not rewind to beginning of frame\n"); fclose(srcFile); return 1; } } - frameHeader = (BYTE*)malloc(totalFrameHeaderBytes); - if (frameHeader==NULL) { - DISPLAY("Error: could not allocate space for frameHeader\n"); - fclose(srcFile); - return 1; - } - readBytes = fread(frameHeader, 1, totalFrameHeaderBytes, srcFile); - if (readBytes != (size_t)totalFrameHeaderBytes) { - DISPLAY("Error: could not read frame header\n"); - fclose(srcFile); - free(frameHeader); - return 1; - } - /* get decompressed file size */ { - U64 additional = ZSTD_getFrameContentSize(frameHeader, totalFrameHeaderBytes); - if (additional!=ZSTD_CONTENTSIZE_UNKNOWN && additional!=ZSTD_CONTENTSIZE_ERROR) { - info->decompressedSize += additional; - } - else{ - info->canComputeDecompSize = 0; + /* skip frame header */ + int const ret = fseek(srcFile, headerSize, SEEK_CUR); + if (ret != 0) { + DISPLAY("Error: could not skip header\n"); + fclose(srcFile); + return 1; } } /* skip the rest of the blocks in the frame */ - do{ - BYTE blockHeaderBuffer[3]; - U32 blockHeader; - int blockSize; - readBytes = fread(blockHeaderBuffer, 1, 3, srcFile); - if (readBytes != 3) { - DISPLAY("There was a problem reading the block header\n"); - exit(1); - } - blockHeader = MEM_readLE24(blockHeaderBuffer); - lastBlock = blockHeader & 1; - blockSize = (blockHeader - (blockHeader & 7)) >> 3; - fseek(srcFile, blockSize, SEEK_CUR); - }while (lastBlock != 1); + { + int lastBlock = 0; + int readBytes = 0; + do{ + BYTE blockHeaderBuffer[3]; + U32 blockHeader; + int blockSize; + readBytes = fread(blockHeaderBuffer, 1, 3, srcFile); + if (readBytes != 3) { + DISPLAY("There was a problem reading the block header\n"); + fclose(srcFile); + return 1; + } + blockHeader = MEM_readLE24(blockHeaderBuffer); + lastBlock = blockHeader & 1; + blockSize = (blockHeader - (blockHeader & 7)) >> 3; + fseek(srcFile, blockSize, SEEK_CUR); + }while (lastBlock != 1); + } { /* check if checksum is used */ + BYTE frameHeaderDescriptor = headerBuffer[4]; int contentChecksumFlag = (frameHeaderDescriptor & (1 << 2)) >> 2; if (contentChecksumFlag) { info->usesCheck = 1; @@ -980,7 +970,7 @@ static int getFileInfo(fileInfo_t* info, const char* inFileName){ fseek(srcFile, 4, SEEK_CUR); } } - free(frameHeader); + info->numActualFrames++; } else if (magicNumber == ZSTD_MAGIC_SKIPPABLE_START) { BYTE frameSizeBuffer[4]; @@ -992,7 +982,16 @@ static int getFileInfo(fileInfo_t* info, const char* inFileName){ exit(1); } frameSize = MEM_readLE32(frameSizeBuffer); + { + int const ret = fseek(srcFile, frameSize, SEEK_CUR); + if (ret != 0) { + DISPLAY("Error: could not find end of skippable frame\n"); + fclose(srcFile); + return 1; + } + } fseek(srcFile, frameSize, SEEK_CUR); + info->numSkippableFrames++; } } fclose(srcFile); @@ -1004,51 +1003,50 @@ void displayInfo(const char* inFileName, fileInfo_t* info, int displayLevel){ if(displayLevel<=2){ if(info->usesCheck && info->canComputeDecompSize){ - DISPLAY("Skippable Non-Skippable Compressed Uncompressed Ratio Check Filename\n"); - DISPLAY("%9d %13d %7.2f MB %7.2f MB %5.3f XXH64 %s\n", + DISPLAYOUT("Skippable Non-Skippable Compressed Uncompressed Ratio Check Filename\n"); + DISPLAYOUT("%9d %13d %7.2f MB %7.2f MB %5.3f XXH64 %s\n", info->numSkippableFrames, info->numActualFrames, compressedSizeMB, decompressedSizeMB, compressedSizeMB/decompressedSizeMB, inFileName); } else if(!info->usesCheck){ - DISPLAY("Skippable Non-Skippable Compressed Uncompressed Ratio Check Filename\n"); - DISPLAY("%9d %13d %7.2f MB %7.2f MB %5.3f %s\n", + DISPLAYOUT("Skippable Non-Skippable Compressed Uncompressed Ratio Check Filename\n"); + DISPLAYOUT("%9d %13d %7.2f MB %7.2f MB %5.3f %s\n", info->numSkippableFrames, info->numActualFrames, compressedSizeMB, decompressedSizeMB, compressedSizeMB/decompressedSizeMB, inFileName); } else if(!info->canComputeDecompSize){ - DISPLAY("Skippable Non-Skippable Compressed Uncompressed Ratio Check Filename\n"); - DISPLAY("%9d %13d %7.2f MB XXH64 %s\n", + DISPLAYOUT("Skippable Non-Skippable Compressed Uncompressed Ratio Check Filename\n"); + DISPLAYOUT("%9d %13d %7.2f MB XXH64 %s\n", info->numSkippableFrames, info->numActualFrames, compressedSizeMB, inFileName); } else{ - DISPLAY("Skippable Non-Skippable Filename\n"); - DISPLAY("%9d %13d %7.2f MB %s\n", + DISPLAYOUT("Skippable Non-Skippable Filename\n"); + DISPLAYOUT("%9d %13d %7.2f MB %s\n", info->numSkippableFrames, info->numActualFrames, compressedSizeMB, inFileName); } } else{ - DISPLAY("# Zstandard Frames: %d\n", info->numActualFrames); - DISPLAY("# Skippable Frames: %d\n", info->numSkippableFrames); - DISPLAY("Compressed Size: %.2f MB (%llu B)\n", compressedSizeMB, info->compressedSize); - if(info->canComputeDecompSize){ - DISPLAY("Decompressed Size: %.2f MB (%llu B)\n", decompressedSizeMB, info->decompressedSize); - DISPLAY("Ratio: %.4f\n", compressedSizeMB/decompressedSizeMB); + DISPLAYOUT("# Zstandard Frames: %d\n", info->numActualFrames); + DISPLAYOUT("# Skippable Frames: %d\n", info->numSkippableFrames); + DISPLAYOUT("Compressed Size: %.2f MB (%llu B)\n", compressedSizeMB, info->compressedSize); + if (info->canComputeDecompSize) { + DISPLAYOUT("Decompressed Size: %.2f MB (%llu B)\n", decompressedSizeMB, info->decompressedSize); + DISPLAYOUT("Ratio: %.4f\n", compressedSizeMB/decompressedSizeMB); } - if(info->usesCheck){ - DISPLAY("Check: XXH64\n"); + if (info->usesCheck) { + DISPLAYOUT("Check: XXH64\n"); } } - } int FIO_listFile(const char* inFileName, int displayLevel){ const char* const suffixPtr = strrchr(inFileName, '.'); DISPLAY("File: %s\n", inFileName); - if(!suffixPtr || strcmp(suffixPtr, ZSTD_EXTENSION)){ + if (!suffixPtr || strcmp(suffixPtr, ZSTD_EXTENSION)) { DISPLAYLEVEL(1, "file %s was not compressed with zstd -- ignoring\n\n", inFileName); return 1; } - else{ + else { fileInfo_t* info = (fileInfo_t*)malloc(sizeof(fileInfo_t)); int error = getFileInfo(info, inFileName); if(error==1){ From 4495e9a82616a5953bb209dd7692afca99fbd3e8 Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Thu, 15 Jun 2017 15:02:54 -0700 Subject: [PATCH 03/12] allow analysis even if file does not have zst extension --- programs/fileio.c | 23 +++++++++-------------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index 4f92e16e8..e6671308c 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -914,8 +914,8 @@ static int getFileInfo(fileInfo_t* info, const char* inFileName){ } size_t const headerSize = ZSTD_frameHeaderSize(headerBuffer, numBytesRead); if (ZSTD_isError(headerSize)) { - fclose(srcFile); DISPLAY("Error: could not determine frame header size\n"); + fclose(srcFile); return 1; } @@ -1037,25 +1037,20 @@ void displayInfo(const char* inFileName, fileInfo_t* info, int displayLevel){ DISPLAYOUT("Check: XXH64\n"); } } + + DISPLAYOUT("\n"); } int FIO_listFile(const char* inFileName, int displayLevel){ - const char* const suffixPtr = strrchr(inFileName, '.'); DISPLAY("File: %s\n", inFileName); - if (!suffixPtr || strcmp(suffixPtr, ZSTD_EXTENSION)) { - DISPLAYLEVEL(1, "file %s was not compressed with zstd -- ignoring\n\n", inFileName); + fileInfo_t* info = (fileInfo_t*)malloc(sizeof(fileInfo_t)); + int error = getFileInfo(info, inFileName); + if (error == 1) { + DISPLAY("An error occurred with getting file info\n"); return 1; } - else { - fileInfo_t* info = (fileInfo_t*)malloc(sizeof(fileInfo_t)); - int error = getFileInfo(info, inFileName); - if(error==1){ - DISPLAY("An error occurred with getting file info\n"); - exit(1); - } - displayInfo(inFileName, info, displayLevel); - } - DISPLAY("\n"); + displayInfo(inFileName, info, displayLevel); + free(info); return 0; } From 9437cc74980e7f5168eb485d31fd64f054f945cd Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Thu, 15 Jun 2017 15:07:54 -0700 Subject: [PATCH 04/12] added --list command --- programs/zstdcli.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 6de33ce09..c71e80bdd 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -149,6 +149,7 @@ static int usage_advanced(const char* programName) #endif #endif DISPLAY( " -M# : Set a memory usage limit for decompression \n"); + DISPLAY( "--list : list information about a zstd compressed file \n"); DISPLAY( "-- : All arguments after \"--\" are treated as files \n"); #ifndef ZSTD_NODICT DISPLAY( "\n"); @@ -402,6 +403,7 @@ int main(int argCount, const char* argv[]) if (argument[1]=='-') { /* long commands (--long-word) */ if (!strcmp(argument, "--")) { nextArgumentsAreFiles=1; continue; } /* only file names allowed from now on */ + if (!strcmp(argument, "--list")) { operation=zom_list; continue; } if (!strcmp(argument, "--compress")) { operation=zom_compress; continue; } if (!strcmp(argument, "--decompress")) { operation=zom_decompress; continue; } if (!strcmp(argument, "--uncompress")) { operation=zom_decompress; continue; } From 96cabf72330f1b466407059a5f0981a24571fac1 Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Thu, 15 Jun 2017 15:25:20 -0700 Subject: [PATCH 05/12] cleaned up if statements --- programs/fileio.c | 27 ++++++++------------------- 1 file changed, 8 insertions(+), 19 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index e6671308c..523b6174e 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -893,7 +893,6 @@ static int getFileInfo(fileInfo_t* info, const char* inFileName){ size_t const numBytesRead = fread(headerBuffer, 1, sizeof(headerBuffer), srcFile); if (numBytesRead < ZSTD_frameHeaderSize_min) { if(feof(srcFile)){ - DISPLAY("ran out of files\n"); break; } else{ @@ -1000,29 +999,19 @@ static int getFileInfo(fileInfo_t* info, const char* inFileName){ void displayInfo(const char* inFileName, fileInfo_t* info, int displayLevel){ double const compressedSizeMB = (double)info->compressedSize/(1 MB); double const decompressedSizeMB = (double)info->decompressedSize/(1 MB); + const char* checkString = (info->usesCheck ? "XXH64" : "None"); if(displayLevel<=2){ - if(info->usesCheck && info->canComputeDecompSize){ + if (info->canComputeDecompSize) { DISPLAYOUT("Skippable Non-Skippable Compressed Uncompressed Ratio Check Filename\n"); - DISPLAYOUT("%9d %13d %7.2f MB %7.2f MB %5.3f XXH64 %s\n", + DISPLAYOUT("%9d %13d %7.2f MB %9.2f MB %5.3f %s %s\n", info->numSkippableFrames, info->numActualFrames, compressedSizeMB, decompressedSizeMB, - compressedSizeMB/decompressedSizeMB, inFileName); + compressedSizeMB/decompressedSizeMB, checkString, inFileName); } - else if(!info->usesCheck){ - DISPLAYOUT("Skippable Non-Skippable Compressed Uncompressed Ratio Check Filename\n"); - DISPLAYOUT("%9d %13d %7.2f MB %7.2f MB %5.3f %s\n", - info->numSkippableFrames, info->numActualFrames, compressedSizeMB, decompressedSizeMB, - compressedSizeMB/decompressedSizeMB, inFileName); - } - else if(!info->canComputeDecompSize){ - DISPLAYOUT("Skippable Non-Skippable Compressed Uncompressed Ratio Check Filename\n"); - DISPLAYOUT("%9d %13d %7.2f MB XXH64 %s\n", - info->numSkippableFrames, info->numActualFrames, compressedSizeMB, inFileName); - } - else{ - DISPLAYOUT("Skippable Non-Skippable Filename\n"); - DISPLAYOUT("%9d %13d %7.2f MB %s\n", - info->numSkippableFrames, info->numActualFrames, compressedSizeMB, inFileName); + else { + DISPLAYOUT("Skippable Non-Skippable Compressed Check Filename\n"); + DISPLAYOUT("%9d %13d %7.2f MB %s %s\n", + info->numSkippableFrames, info->numActualFrames, compressedSizeMB, checkString, inFileName); } } else{ From 8b3ff7c9bcb91bd15e994e6fda151b1fda0a7f0a Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Thu, 15 Jun 2017 15:35:37 -0700 Subject: [PATCH 06/12] changed info to static allocation --- programs/fileio.c | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index 523b6174e..aff1edd41 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -1000,7 +1000,6 @@ void displayInfo(const char* inFileName, fileInfo_t* info, int displayLevel){ double const compressedSizeMB = (double)info->compressedSize/(1 MB); double const decompressedSizeMB = (double)info->decompressedSize/(1 MB); const char* checkString = (info->usesCheck ? "XXH64" : "None"); - if(displayLevel<=2){ if (info->canComputeDecompSize) { DISPLAYOUT("Skippable Non-Skippable Compressed Uncompressed Ratio Check Filename\n"); @@ -1031,15 +1030,14 @@ void displayInfo(const char* inFileName, fileInfo_t* info, int displayLevel){ } int FIO_listFile(const char* inFileName, int displayLevel){ - DISPLAY("File: %s\n", inFileName); - fileInfo_t* info = (fileInfo_t*)malloc(sizeof(fileInfo_t)); - int error = getFileInfo(info, inFileName); + DISPLAYOUT("File: %s\n", inFileName); + fileInfo_t info; + int const error = getFileInfo(&info, inFileName); if (error == 1) { DISPLAY("An error occurred with getting file info\n"); return 1; } - displayInfo(inFileName, info, displayLevel); - free(info); + displayInfo(inFileName, &info, displayLevel); return 0; } From e7f02fc58ad25c55ff78b0fea9cb5ad40978f0c0 Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Thu, 15 Jun 2017 15:53:02 -0700 Subject: [PATCH 07/12] closed file at end of function, created another variable to keep exit points simpler --- programs/fileio.c | 75 +++++++++++++++++++++++++---------------------- 1 file changed, 40 insertions(+), 35 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index aff1edd41..09ef1791d 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -887,6 +887,7 @@ static int getFileInfo(fileInfo_t* info, const char* inFileName){ info->numActualFrames = 0; info->numSkippableFrames = 0; info->canComputeDecompSize = 1; + int detectError = 0; /* begin analyzing frame */ for( ; ; ){ BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX]; @@ -897,8 +898,8 @@ static int getFileInfo(fileInfo_t* info, const char* inFileName){ } else{ DISPLAY("Error: did not reach end of file but ran out of frames\n"); - fclose(srcFile); - return 1; + detectError = 1; + break; } } U32 const magicNumber = MEM_readLE32(headerBuffer); @@ -906,7 +907,6 @@ static int getFileInfo(fileInfo_t* info, const char* inFileName){ U64 const frameContentSize = ZSTD_getFrameContentSize(headerBuffer, numBytesRead); if (frameContentSize == ZSTD_CONTENTSIZE_ERROR || frameContentSize == ZSTD_CONTENTSIZE_UNKNOWN) { info->canComputeDecompSize = 0; - DISPLAY("could not compute decompressed size\n"); } else { info->decompressedSize += frameContentSize; @@ -914,34 +914,24 @@ static int getFileInfo(fileInfo_t* info, const char* inFileName){ size_t const headerSize = ZSTD_frameHeaderSize(headerBuffer, numBytesRead); if (ZSTD_isError(headerSize)) { DISPLAY("Error: could not determine frame header size\n"); - fclose(srcFile); - return 1; + detectError = 1; + break; } { - /* go back to the beginning of the frame */ - int const ret = fseek(srcFile, -numBytesRead, SEEK_CUR); + /* move to the end of the frame header */ + int const ret = fseek(srcFile, headerSize-numBytesRead, SEEK_CUR); if (ret != 0) { - DISPLAY("Error: could not rewind to beginning of frame\n"); - fclose(srcFile); - return 1; - } - } - - { - /* skip frame header */ - int const ret = fseek(srcFile, headerSize, SEEK_CUR); - if (ret != 0) { - DISPLAY("Error: could not skip header\n"); - fclose(srcFile); - return 1; + DISPLAY("Error: could not move to end of frame header\n"); + detectError = 1; + break; } } /* skip the rest of the blocks in the frame */ { int lastBlock = 0; - int readBytes = 0; + size_t readBytes = 0; do{ BYTE blockHeaderBuffer[3]; U32 blockHeader; @@ -949,44 +939,59 @@ static int getFileInfo(fileInfo_t* info, const char* inFileName){ readBytes = fread(blockHeaderBuffer, 1, 3, srcFile); if (readBytes != 3) { DISPLAY("There was a problem reading the block header\n"); - fclose(srcFile); - return 1; + detectError = 1; + break; } blockHeader = MEM_readLE24(blockHeaderBuffer); lastBlock = blockHeader & 1; blockSize = (blockHeader - (blockHeader & 7)) >> 3; - fseek(srcFile, blockSize, SEEK_CUR); - }while (lastBlock != 1); + { + int const ret = fseek(srcFile, blockSize, SEEK_CUR); + if (ret != 0) { + DISPLAY("Error: could not skip to end of block\n"); + detectError = 1; + break; + } + } + } while (lastBlock != 1); + + if (detectError) { + break; + } } { /* check if checksum is used */ - BYTE frameHeaderDescriptor = headerBuffer[4]; - int contentChecksumFlag = (frameHeaderDescriptor & (1 << 2)) >> 2; + BYTE const frameHeaderDescriptor = headerBuffer[4]; + int const contentChecksumFlag = (frameHeaderDescriptor & (1 << 2)) >> 2; if (contentChecksumFlag) { info->usesCheck = 1; } if (contentChecksumFlag) { - fseek(srcFile, 4, SEEK_CUR); + int const ret = fseek(srcFile, 4, SEEK_CUR); + if (ret != 0) { + DISPLAY("Error: could not skip past checksum\n"); + detectError = 1; + break; + } } } info->numActualFrames++; } else if (magicNumber == ZSTD_MAGIC_SKIPPABLE_START) { BYTE frameSizeBuffer[4]; - long frameSize; size_t readBytes = fread(frameSizeBuffer, 1, 4, srcFile); - info->numSkippableFrames++; if (readBytes != 4) { DISPLAY("There was an error reading skippable frame size"); - exit(1); + detectError = 1; + break; } - frameSize = MEM_readLE32(frameSizeBuffer); + long const frameSize = MEM_readLE32(frameSizeBuffer); { int const ret = fseek(srcFile, frameSize, SEEK_CUR); if (ret != 0) { DISPLAY("Error: could not find end of skippable frame\n"); - fclose(srcFile); - return 1; + detectError = 1; + break; } } fseek(srcFile, frameSize, SEEK_CUR); @@ -994,7 +999,7 @@ static int getFileInfo(fileInfo_t* info, const char* inFileName){ } } fclose(srcFile); - return 0; + return detectError; } void displayInfo(const char* inFileName, fileInfo_t* info, int displayLevel){ double const compressedSizeMB = (double)info->compressedSize/(1 MB); From b766211e10fd3e148d38e0c1c24043c65a817002 Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Thu, 15 Jun 2017 15:55:49 -0700 Subject: [PATCH 08/12] made another function static --- programs/fileio.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index 09ef1791d..23c8badc6 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -954,7 +954,7 @@ static int getFileInfo(fileInfo_t* info, const char* inFileName){ } } } while (lastBlock != 1); - + if (detectError) { break; } @@ -1001,7 +1001,8 @@ static int getFileInfo(fileInfo_t* info, const char* inFileName){ fclose(srcFile); return detectError; } -void displayInfo(const char* inFileName, fileInfo_t* info, int displayLevel){ + +static void displayInfo(const char* inFileName, fileInfo_t* info, int displayLevel){ double const compressedSizeMB = (double)info->compressedSize/(1 MB); double const decompressedSizeMB = (double)info->decompressedSize/(1 MB); const char* checkString = (info->usesCheck ? "XXH64" : "None"); From 54e0b6c66b78b0f9035ec06e959428cd200289c1 Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Thu, 15 Jun 2017 16:01:52 -0700 Subject: [PATCH 09/12] got rid of imlpicit precision loss --- programs/fileio.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/programs/fileio.c b/programs/fileio.c index 23c8badc6..e76f93791 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -920,7 +920,7 @@ static int getFileInfo(fileInfo_t* info, const char* inFileName){ { /* move to the end of the frame header */ - int const ret = fseek(srcFile, headerSize-numBytesRead, SEEK_CUR); + int const ret = fseek(srcFile, (long)(headerSize-numBytesRead), SEEK_CUR); if (ret != 0) { DISPLAY("Error: could not move to end of frame header\n"); detectError = 1; From e49afae2ab6f720333fb47fda66333c53f6d736d Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Thu, 15 Jun 2017 16:03:40 -0700 Subject: [PATCH 10/12] changed implicit precision loss again --- programs/fileio.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/programs/fileio.c b/programs/fileio.c index e76f93791..5a1bef7a8 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -920,7 +920,7 @@ static int getFileInfo(fileInfo_t* info, const char* inFileName){ { /* move to the end of the frame header */ - int const ret = fseek(srcFile, (long)(headerSize-numBytesRead), SEEK_CUR); + int const ret = fseek(srcFile, ((long)headerSize)-((long)numBytesRead), SEEK_CUR); if (ret != 0) { DISPLAY("Error: could not move to end of frame header\n"); detectError = 1; From 0757eae6ff55c6dd09ff3320ad232804ea2ae86f Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Thu, 15 Jun 2017 16:12:04 -0700 Subject: [PATCH 11/12] rearranging code to allow for consts --- programs/fileio.c | 158 +++++++++++++++++++++++----------------------- 1 file changed, 79 insertions(+), 79 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index 5a1bef7a8..cf329ddef 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -877,6 +877,7 @@ typedef struct { * if successful, returns 0, otherwise returns 1 */ static int getFileInfo(fileInfo_t* info, const char* inFileName){ + int detectError = 0; FILE* const srcFile = FIO_openSrcFile(inFileName); if (srcFile == NULL) { DISPLAY("Error: could not open source file %s\n", inFileName); @@ -887,7 +888,6 @@ static int getFileInfo(fileInfo_t* info, const char* inFileName){ info->numActualFrames = 0; info->numSkippableFrames = 0; info->canComputeDecompSize = 1; - int detectError = 0; /* begin analyzing frame */ for( ; ; ){ BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX]; @@ -902,100 +902,100 @@ static int getFileInfo(fileInfo_t* info, const char* inFileName){ break; } } - U32 const magicNumber = MEM_readLE32(headerBuffer); - if (magicNumber == ZSTD_MAGICNUMBER) { - U64 const frameContentSize = ZSTD_getFrameContentSize(headerBuffer, numBytesRead); - if (frameContentSize == ZSTD_CONTENTSIZE_ERROR || frameContentSize == ZSTD_CONTENTSIZE_UNKNOWN) { - info->canComputeDecompSize = 0; - } - else { - info->decompressedSize += frameContentSize; - } - size_t const headerSize = ZSTD_frameHeaderSize(headerBuffer, numBytesRead); - if (ZSTD_isError(headerSize)) { - DISPLAY("Error: could not determine frame header size\n"); - detectError = 1; - break; - } - - { - /* move to the end of the frame header */ - int const ret = fseek(srcFile, ((long)headerSize)-((long)numBytesRead), SEEK_CUR); - if (ret != 0) { - DISPLAY("Error: could not move to end of frame header\n"); - detectError = 1; - break; + { + U32 const magicNumber = MEM_readLE32(headerBuffer); + if (magicNumber == ZSTD_MAGICNUMBER) { + U64 const frameContentSize = ZSTD_getFrameContentSize(headerBuffer, numBytesRead); + if (frameContentSize == ZSTD_CONTENTSIZE_ERROR || frameContentSize == ZSTD_CONTENTSIZE_UNKNOWN) { + info->canComputeDecompSize = 0; } - } - - /* skip the rest of the blocks in the frame */ - { - int lastBlock = 0; - size_t readBytes = 0; - do{ - BYTE blockHeaderBuffer[3]; - U32 blockHeader; - int blockSize; - readBytes = fread(blockHeaderBuffer, 1, 3, srcFile); - if (readBytes != 3) { - DISPLAY("There was a problem reading the block header\n"); + else { + info->decompressedSize += frameContentSize; + } + { + /* move to the end of the frame header */ + size_t const headerSize = ZSTD_frameHeaderSize(headerBuffer, numBytesRead); + if (ZSTD_isError(headerSize)) { + DISPLAY("Error: could not determine frame header size\n"); detectError = 1; break; } - blockHeader = MEM_readLE24(blockHeaderBuffer); - lastBlock = blockHeader & 1; - blockSize = (blockHeader - (blockHeader & 7)) >> 3; - { - int const ret = fseek(srcFile, blockSize, SEEK_CUR); + int const ret = fseek(srcFile, ((long)headerSize)-((long)numBytesRead), SEEK_CUR); + if (ret != 0) { + DISPLAY("Error: could not move to end of frame header\n"); + detectError = 1; + break; + } + } + + /* skip the rest of the blocks in the frame */ + { + int lastBlock = 0; + size_t readBytes = 0; + do{ + BYTE blockHeaderBuffer[3]; + U32 blockHeader; + int blockSize; + readBytes = fread(blockHeaderBuffer, 1, 3, srcFile); + if (readBytes != 3) { + DISPLAY("There was a problem reading the block header\n"); + detectError = 1; + break; + } + blockHeader = MEM_readLE24(blockHeaderBuffer); + lastBlock = blockHeader & 1; + blockSize = (blockHeader - (blockHeader & 7)) >> 3; + { + int const ret = fseek(srcFile, blockSize, SEEK_CUR); + if (ret != 0) { + DISPLAY("Error: could not skip to end of block\n"); + detectError = 1; + break; + } + } + } while (lastBlock != 1); + + if (detectError) { + break; + } + } + { + /* check if checksum is used */ + BYTE const frameHeaderDescriptor = headerBuffer[4]; + int const contentChecksumFlag = (frameHeaderDescriptor & (1 << 2)) >> 2; + if (contentChecksumFlag) { + info->usesCheck = 1; + } + if (contentChecksumFlag) { + int const ret = fseek(srcFile, 4, SEEK_CUR); if (ret != 0) { - DISPLAY("Error: could not skip to end of block\n"); + DISPLAY("Error: could not skip past checksum\n"); detectError = 1; break; } } - } while (lastBlock != 1); - - if (detectError) { + } + info->numActualFrames++; + } + else if (magicNumber == ZSTD_MAGIC_SKIPPABLE_START) { + BYTE frameSizeBuffer[4]; + size_t readBytes = fread(frameSizeBuffer, 1, 4, srcFile); + if (readBytes != 4) { + DISPLAY("There was an error reading skippable frame size"); + detectError = 1; break; } - } - { - /* check if checksum is used */ - BYTE const frameHeaderDescriptor = headerBuffer[4]; - int const contentChecksumFlag = (frameHeaderDescriptor & (1 << 2)) >> 2; - if (contentChecksumFlag) { - info->usesCheck = 1; - } - if (contentChecksumFlag) { - int const ret = fseek(srcFile, 4, SEEK_CUR); + { + long const frameSize = MEM_readLE32(frameSizeBuffer); + int const ret = fseek(srcFile, frameSize, SEEK_CUR); if (ret != 0) { - DISPLAY("Error: could not skip past checksum\n"); + DISPLAY("Error: could not find end of skippable frame\n"); detectError = 1; break; } } + info->numSkippableFrames++; } - info->numActualFrames++; - } - else if (magicNumber == ZSTD_MAGIC_SKIPPABLE_START) { - BYTE frameSizeBuffer[4]; - size_t readBytes = fread(frameSizeBuffer, 1, 4, srcFile); - if (readBytes != 4) { - DISPLAY("There was an error reading skippable frame size"); - detectError = 1; - break; - } - long const frameSize = MEM_readLE32(frameSizeBuffer); - { - int const ret = fseek(srcFile, frameSize, SEEK_CUR); - if (ret != 0) { - DISPLAY("Error: could not find end of skippable frame\n"); - detectError = 1; - break; - } - } - fseek(srcFile, frameSize, SEEK_CUR); - info->numSkippableFrames++; } } fclose(srcFile); @@ -1036,8 +1036,8 @@ static void displayInfo(const char* inFileName, fileInfo_t* info, int displayLev } int FIO_listFile(const char* inFileName, int displayLevel){ - DISPLAYOUT("File: %s\n", inFileName); fileInfo_t info; + DISPLAYOUT("File: %s\n", inFileName); int const error = getFileInfo(&info, inFileName); if (error == 1) { DISPLAY("An error occurred with getting file info\n"); From acaefb531bb0549cb11fac143ad349ccf40efc99 Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Thu, 15 Jun 2017 16:27:38 -0700 Subject: [PATCH 12/12] refactoring brackets to allow for const values --- programs/fileio.c | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index cf329ddef..bb3f13d8e 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -920,11 +920,13 @@ static int getFileInfo(fileInfo_t* info, const char* inFileName){ detectError = 1; break; } - int const ret = fseek(srcFile, ((long)headerSize)-((long)numBytesRead), SEEK_CUR); - if (ret != 0) { - DISPLAY("Error: could not move to end of frame header\n"); - detectError = 1; - break; + { + int const ret = fseek(srcFile, ((long)headerSize)-((long)numBytesRead), SEEK_CUR); + if (ret != 0) { + DISPLAY("Error: could not move to end of frame header\n"); + detectError = 1; + break; + } } } @@ -1038,10 +1040,12 @@ static void displayInfo(const char* inFileName, fileInfo_t* info, int displayLev int FIO_listFile(const char* inFileName, int displayLevel){ fileInfo_t info; DISPLAYOUT("File: %s\n", inFileName); - int const error = getFileInfo(&info, inFileName); - if (error == 1) { - DISPLAY("An error occurred with getting file info\n"); - return 1; + { + int const error = getFileInfo(&info, inFileName); + if (error == 1) { + DISPLAY("An error occurred with getting file info\n"); + return 1; + } } displayInfo(inFileName, &info, displayLevel); return 0;