diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index ea64edeed..af8b98d7f 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -204,7 +204,7 @@ unsigned ZSTD_isFrame(const void* buffer, size_t size) /** ZSTD_frameHeaderSize() : * srcSize must be >= ZSTD_frameHeaderSize_prefix. * @return : size of the Frame Header */ -static size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize) +size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize) { if (srcSize < ZSTD_frameHeaderSize_prefix) return ERROR(srcSize_wrong); { BYTE const fhd = ((const BYTE*)src)[4]; diff --git a/lib/zstd.h b/lib/zstd.h index b0aae3f73..53acdcdd0 100644 --- a/lib/zstd.h +++ b/lib/zstd.h @@ -461,6 +461,11 @@ ZSTDLIB_API unsigned long long ZSTD_getFrameContentSize(const void *src, size_t * however it does mean that all frame data must be present and valid. */ ZSTDLIB_API unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize); +/*! ZSTD_frameHeaderSize() : +* `src` should point to the start of a ZSTD frame +* `srcSize` must be >= ZSTD_frameHeaderSize_prefix. +* @return : size of the Frame Header */ +size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize); /*************************************** * Context memory usage diff --git a/programs/fileio.c b/programs/fileio.c index 2708176c1..bb3f13d8e 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -91,6 +91,7 @@ * Macros ***************************************/ #define DISPLAY(...) fprintf(stderr, __VA_ARGS__) +#define DISPLAYOUT(...) fprintf(stdout, __VA_ARGS__) #define DISPLAYLEVEL(l, ...) { if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); } } static int g_displayLevel = 2; /* 0 : no display; 1: errors; 2 : + result + interaction + warnings; 3 : + progression; 4 : + information */ void FIO_setNotificationLevel(unsigned level) { g_displayLevel=level; } @@ -871,182 +872,182 @@ typedef struct { int usesCheck; } fileInfo_t; - -int calcFrameHeaderSize(BYTE frameHeaderDescriptor){ - int frameContentSizeBytes = 0; - int windowDescriptorBytes; - int dictionaryIDBytes; - int frameContentSizeFlag = frameHeaderDescriptor >> 6; - int singleSegmentFlag = (frameHeaderDescriptor & (1 << 5)) >> 5; - int dictionaryIDFlag = frameHeaderDescriptor & 3; - if(frameContentSizeFlag!=0){ - frameContentSizeBytes = 1 << frameContentSizeFlag; - } - else if(singleSegmentFlag){ - frameContentSizeBytes = 1; - } - windowDescriptorBytes = singleSegmentFlag ? 0 : 1; - dictionaryIDBytes = dictionaryIDFlag ? 1 << (dictionaryIDFlag - 1): 0; - return 4 + 1 + windowDescriptorBytes + frameContentSizeBytes + dictionaryIDBytes; -} - /* * Reads information from file, stores in *info * if successful, returns 0, otherwise returns 1 */ -int getFileInfo(fileInfo_t* info, const char* inFileName){ - FILE* srcFile = FIO_openSrcFile(inFileName); - if(srcFile==NULL){ +static int getFileInfo(fileInfo_t* info, const char* inFileName){ + int detectError = 0; + FILE* const srcFile = FIO_openSrcFile(inFileName); + if (srcFile == NULL) { + DISPLAY("Error: could not open source file %s\n", inFileName); return 1; } info->compressedSize = (unsigned long long)UTIL_getFileSize(inFileName); info->decompressedSize = 0; info->numActualFrames = 0; - info-> numSkippableFrames = 0; + info->numSkippableFrames = 0; info->canComputeDecompSize = 1; /* begin analyzing frame */ - while(1){ - BYTE magicNumberBuffer[4]; - size_t numBytesRead = fread(magicNumberBuffer, 1, 4, srcFile); - U32 magicNumber; - if(numBytesRead != 4) break; - magicNumber = MEM_readLE32(magicNumberBuffer); - if(magicNumber==ZSTD_MAGICNUMBER){ - BYTE frameHeaderDescriptor; - int totalFrameHeaderBytes; - BYTE* frameHeader; - int lastBlock = 0; - size_t readBytes = fread(&frameHeaderDescriptor, 1, 1, srcFile); - info->numActualFrames++; - if(readBytes != 1){ - DISPLAY("There was an error with reading frame header descriptor\n"); - exit(1); + for( ; ; ){ + BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX]; + size_t const numBytesRead = fread(headerBuffer, 1, sizeof(headerBuffer), srcFile); + if (numBytesRead < ZSTD_frameHeaderSize_min) { + if(feof(srcFile)){ + break; } - /* calculate actual frame header size */ - totalFrameHeaderBytes = calcFrameHeaderSize(frameHeaderDescriptor); - - /* reset to beginning of from and read entire header */ - fseek(srcFile, -5, SEEK_CUR); - frameHeader = (BYTE*)malloc(totalFrameHeaderBytes); - readBytes = fread(frameHeader, 1, totalFrameHeaderBytes, srcFile); - if(readBytes != (size_t)totalFrameHeaderBytes){ - DISPLAY("There was an error reading the frame header\n"); - exit(1); + else{ + DISPLAY("Error: did not reach end of file but ran out of frames\n"); + detectError = 1; + break; } - - /* get decompressed file size */ - { - U64 additional = ZSTD_getFrameContentSize(frameHeader, totalFrameHeaderBytes); - if(additional!=ZSTD_CONTENTSIZE_UNKNOWN && additional!=ZSTD_CONTENTSIZE_ERROR){ - info->decompressedSize += additional; - } - else{ + } + { + U32 const magicNumber = MEM_readLE32(headerBuffer); + if (magicNumber == ZSTD_MAGICNUMBER) { + U64 const frameContentSize = ZSTD_getFrameContentSize(headerBuffer, numBytesRead); + if (frameContentSize == ZSTD_CONTENTSIZE_ERROR || frameContentSize == ZSTD_CONTENTSIZE_UNKNOWN) { info->canComputeDecompSize = 0; } - } + else { + info->decompressedSize += frameContentSize; + } + { + /* move to the end of the frame header */ + size_t const headerSize = ZSTD_frameHeaderSize(headerBuffer, numBytesRead); + if (ZSTD_isError(headerSize)) { + DISPLAY("Error: could not determine frame header size\n"); + detectError = 1; + break; + } + { + int const ret = fseek(srcFile, ((long)headerSize)-((long)numBytesRead), SEEK_CUR); + if (ret != 0) { + DISPLAY("Error: could not move to end of frame header\n"); + detectError = 1; + break; + } + } + } - /* skip the rest of the blocks in the frame */ - do{ - BYTE blockHeaderBuffer[3]; - U32 blockHeader; - int blockSize; - readBytes = fread(blockHeaderBuffer, 1, 3, srcFile); - if(readBytes != 3){ - DISPLAY("There was a problem reading the block header\n"); - exit(1); + /* skip the rest of the blocks in the frame */ + { + int lastBlock = 0; + size_t readBytes = 0; + do{ + BYTE blockHeaderBuffer[3]; + U32 blockHeader; + int blockSize; + readBytes = fread(blockHeaderBuffer, 1, 3, srcFile); + if (readBytes != 3) { + DISPLAY("There was a problem reading the block header\n"); + detectError = 1; + break; + } + blockHeader = MEM_readLE24(blockHeaderBuffer); + lastBlock = blockHeader & 1; + blockSize = (blockHeader - (blockHeader & 7)) >> 3; + { + int const ret = fseek(srcFile, blockSize, SEEK_CUR); + if (ret != 0) { + DISPLAY("Error: could not skip to end of block\n"); + detectError = 1; + break; + } + } + } while (lastBlock != 1); + + if (detectError) { + break; + } } - blockHeader = MEM_readLE24(blockHeaderBuffer); - lastBlock = blockHeader & 1; - blockSize = (blockHeader - (blockHeader & 7)) >> 3; - fseek(srcFile, blockSize, SEEK_CUR); - }while(lastBlock != 1); - { - /* check if checksum is used */ - int contentChecksumFlag = (frameHeaderDescriptor & (1 << 2)) >> 2; - if(contentChecksumFlag){ - info->usesCheck = 1; + { + /* check if checksum is used */ + BYTE const frameHeaderDescriptor = headerBuffer[4]; + int const contentChecksumFlag = (frameHeaderDescriptor & (1 << 2)) >> 2; + if (contentChecksumFlag) { + info->usesCheck = 1; + } + if (contentChecksumFlag) { + int const ret = fseek(srcFile, 4, SEEK_CUR); + if (ret != 0) { + DISPLAY("Error: could not skip past checksum\n"); + detectError = 1; + break; + } + } } - if(contentChecksumFlag){ - fseek(srcFile, 4, SEEK_CUR); + info->numActualFrames++; + } + else if (magicNumber == ZSTD_MAGIC_SKIPPABLE_START) { + BYTE frameSizeBuffer[4]; + size_t readBytes = fread(frameSizeBuffer, 1, 4, srcFile); + if (readBytes != 4) { + DISPLAY("There was an error reading skippable frame size"); + detectError = 1; + break; } + { + long const frameSize = MEM_readLE32(frameSizeBuffer); + int const ret = fseek(srcFile, frameSize, SEEK_CUR); + if (ret != 0) { + DISPLAY("Error: could not find end of skippable frame\n"); + detectError = 1; + break; + } + } + info->numSkippableFrames++; } } - else if(magicNumber==ZSTD_MAGIC_SKIPPABLE_START){ - BYTE frameSizeBuffer[4]; - long frameSize; - size_t readBytes = fread(frameSizeBuffer, 1, 4, srcFile); - info->numSkippableFrames++; - if(readBytes != 4){ - DISPLAY("There was an error reading skippable frame size"); - exit(1); - } - frameSize = MEM_readLE32(frameSizeBuffer); - fseek(srcFile, frameSize, SEEK_CUR); - } - } - return 0; + fclose(srcFile); + return detectError; } -void displayInfo(const char* inFileName, fileInfo_t* info, int displayLevel){ - double compressedSizeMB = (double)info->compressedSize/(1 MB); - double decompressedSizeMB = (double)info->decompressedSize/(1 MB); +static void displayInfo(const char* inFileName, fileInfo_t* info, int displayLevel){ + double const compressedSizeMB = (double)info->compressedSize/(1 MB); + double const decompressedSizeMB = (double)info->decompressedSize/(1 MB); + const char* checkString = (info->usesCheck ? "XXH64" : "None"); if(displayLevel<=2){ - if(info->usesCheck && info->canComputeDecompSize){ - DISPLAY("Skippable Non-Skippable Compressed Uncompressed Ratio Check Filename\n"); - DISPLAY("%9d %13d %7.2f MB %7.2f MB %5.3f XXH64 %s\n", + if (info->canComputeDecompSize) { + DISPLAYOUT("Skippable Non-Skippable Compressed Uncompressed Ratio Check Filename\n"); + DISPLAYOUT("%9d %13d %7.2f MB %9.2f MB %5.3f %s %s\n", info->numSkippableFrames, info->numActualFrames, compressedSizeMB, decompressedSizeMB, - compressedSizeMB/decompressedSizeMB, inFileName); + compressedSizeMB/decompressedSizeMB, checkString, inFileName); } - else if(!info->usesCheck){ - DISPLAY("Skippable Non-Skippable Compressed Uncompressed Ratio Check Filename\n"); - DISPLAY("%9d %13d %7.2f MB %7.2f MB %5.3f %s\n", - info->numSkippableFrames, info->numActualFrames, compressedSizeMB, decompressedSizeMB, - compressedSizeMB/decompressedSizeMB, inFileName); - } - else if(!info->canComputeDecompSize){ - DISPLAY("Skippable Non-Skippable Compressed Uncompressed Ratio Check Filename\n"); - DISPLAY("%9d %13d %7.2f MB XXH64 %s\n", - info->numSkippableFrames, info->numActualFrames, compressedSizeMB, inFileName); - } - else{ - DISPLAY("Skippable Non-Skippable Filename\n"); - DISPLAY("%9d %13d %7.2f MB %s\n", - info->numSkippableFrames, info->numActualFrames, compressedSizeMB, inFileName); + else { + DISPLAYOUT("Skippable Non-Skippable Compressed Check Filename\n"); + DISPLAYOUT("%9d %13d %7.2f MB %s %s\n", + info->numSkippableFrames, info->numActualFrames, compressedSizeMB, checkString, inFileName); } } else{ - DISPLAY("# Zstandard Frames: %d\n", info->numActualFrames); - DISPLAY("# Skippable Frames: %d\n", info->numSkippableFrames); - DISPLAY("Compressed Size: %.2f MB (%llu B)\n", compressedSizeMB, info->compressedSize); - if(info->canComputeDecompSize){ - DISPLAY("Decompressed Size: %.2f MB (%llu B)\n", decompressedSizeMB, info->decompressedSize); - DISPLAY("Ratio: %.4f\n", compressedSizeMB/decompressedSizeMB); + DISPLAYOUT("# Zstandard Frames: %d\n", info->numActualFrames); + DISPLAYOUT("# Skippable Frames: %d\n", info->numSkippableFrames); + DISPLAYOUT("Compressed Size: %.2f MB (%llu B)\n", compressedSizeMB, info->compressedSize); + if (info->canComputeDecompSize) { + DISPLAYOUT("Decompressed Size: %.2f MB (%llu B)\n", decompressedSizeMB, info->decompressedSize); + DISPLAYOUT("Ratio: %.4f\n", compressedSizeMB/decompressedSizeMB); } - if(info->usesCheck){ - DISPLAY("Check: XXH64\n"); + if (info->usesCheck) { + DISPLAYOUT("Check: XXH64\n"); } } + DISPLAYOUT("\n"); } int FIO_listFile(const char* inFileName, int displayLevel){ - const char* const suffixPtr = strrchr(inFileName, '.'); - DISPLAY("File: %s\n", inFileName); - if(!suffixPtr || strcmp(suffixPtr, ZSTD_EXTENSION)){ - DISPLAYLEVEL(1, "file %s was not compressed with zstd -- ignoring\n\n", inFileName); - return 1; - } - else{ - fileInfo_t* info = (fileInfo_t*)malloc(sizeof(fileInfo_t)); - int error = getFileInfo(info, inFileName); - if(error==1){ + fileInfo_t info; + DISPLAYOUT("File: %s\n", inFileName); + { + int const error = getFileInfo(&info, inFileName); + if (error == 1) { DISPLAY("An error occurred with getting file info\n"); - exit(1); + return 1; } - displayInfo(inFileName, info, displayLevel); } - DISPLAY("\n"); + displayInfo(inFileName, &info, displayLevel); return 0; } diff --git a/programs/zstdcli.c b/programs/zstdcli.c index f8612f297..c71e80bdd 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -149,6 +149,7 @@ static int usage_advanced(const char* programName) #endif #endif DISPLAY( " -M# : Set a memory usage limit for decompression \n"); + DISPLAY( "--list : list information about a zstd compressed file \n"); DISPLAY( "-- : All arguments after \"--\" are treated as files \n"); #ifndef ZSTD_NODICT DISPLAY( "\n"); @@ -402,6 +403,7 @@ int main(int argCount, const char* argv[]) if (argument[1]=='-') { /* long commands (--long-word) */ if (!strcmp(argument, "--")) { nextArgumentsAreFiles=1; continue; } /* only file names allowed from now on */ + if (!strcmp(argument, "--list")) { operation=zom_list; continue; } if (!strcmp(argument, "--compress")) { operation=zom_compress; continue; } if (!strcmp(argument, "--decompress")) { operation=zom_decompress; continue; } if (!strcmp(argument, "--uncompress")) { operation=zom_decompress; continue; } @@ -674,6 +676,7 @@ int main(int argCount, const char* argv[]) } #endif if(operation==zom_list){ + g_displayOut = stdout; if(filenameIdx==0){ DISPLAY("No files given\n"); CLEAN_RETURN(0);