diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index ea64edeed..af8b98d7f 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -204,7 +204,7 @@ unsigned ZSTD_isFrame(const void* buffer, size_t size) /** ZSTD_frameHeaderSize() : * srcSize must be >= ZSTD_frameHeaderSize_prefix. * @return : size of the Frame Header */ -static size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize) +size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize) { if (srcSize < ZSTD_frameHeaderSize_prefix) return ERROR(srcSize_wrong); { BYTE const fhd = ((const BYTE*)src)[4]; diff --git a/lib/zstd.h b/lib/zstd.h index b0aae3f73..53acdcdd0 100644 --- a/lib/zstd.h +++ b/lib/zstd.h @@ -461,6 +461,11 @@ ZSTDLIB_API unsigned long long ZSTD_getFrameContentSize(const void *src, size_t * however it does mean that all frame data must be present and valid. */ ZSTDLIB_API unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize); +/*! ZSTD_frameHeaderSize() : +* `src` should point to the start of a ZSTD frame +* `srcSize` must be >= ZSTD_frameHeaderSize_prefix. +* @return : size of the Frame Header */ +size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize); /*************************************** * Context memory usage diff --git a/programs/fileio.c b/programs/fileio.c index 44152b0b1..a5db88a4c 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -91,6 +91,7 @@ * Macros ***************************************/ #define DISPLAY(...) fprintf(stderr, __VA_ARGS__) +#define DISPLAYOUT(...) fprintf(stdout, __VA_ARGS__) #define DISPLAYLEVEL(l, ...) { if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); } } static int g_displayLevel = 2; /* 0 : no display; 1: errors; 2 : + result + interaction + warnings; 3 : + progression; 4 : + information */ void FIO_setNotificationLevel(unsigned level) { g_displayLevel=level; } @@ -862,6 +863,222 @@ int FIO_compressFilename(const char* dstFileName, const char* srcFileName, return result; } +typedef struct { + int numActualFrames; + int numSkippableFrames; + unsigned long long decompressedSize; + int decompUnavailable; + unsigned long long compressedSize; + int usesCheck; +} fileInfo_t; + +/* + * Reads information from file, stores in *info + * if successful, returns 0, returns 1 for frame analysis error, returns 2 for file not compressed with zstd + */ +static int getFileInfo(fileInfo_t* info, const char* inFileName){ + int detectError = 0; + FILE* const srcFile = FIO_openSrcFile(inFileName); + if (srcFile == NULL) { + DISPLAY("Error: could not open source file %s\n", inFileName); + return 1; + } + info->compressedSize = (unsigned long long)UTIL_getFileSize(inFileName); + /* begin analyzing frame */ + for ( ; ; ) { + BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX]; + size_t const numBytesRead = fread(headerBuffer, 1, sizeof(headerBuffer), srcFile); + if (numBytesRead < ZSTD_frameHeaderSize_min) { + if (feof(srcFile) && numBytesRead == 0 && info->compressedSize > 0) { + break; + } + else if (feof(srcFile)) { + DISPLAY("Error: reached end of file with incomplete frame\n"); + detectError = 2; + break; + } + else { + DISPLAY("Error: did not reach end of file but ran out of frames\n"); + detectError = 1; + break; + } + } + { + U32 const magicNumber = MEM_readLE32(headerBuffer); + if (magicNumber == ZSTD_MAGICNUMBER) { + U64 const frameContentSize = ZSTD_getFrameContentSize(headerBuffer, numBytesRead); + if (frameContentSize == ZSTD_CONTENTSIZE_ERROR || frameContentSize == ZSTD_CONTENTSIZE_UNKNOWN) { + info->decompUnavailable = 1; + } + else { + info->decompressedSize += frameContentSize; + } + { + /* move to the end of the frame header */ + size_t const headerSize = ZSTD_frameHeaderSize(headerBuffer, numBytesRead); + if (ZSTD_isError(headerSize)) { + DISPLAY("Error: could not determine frame header size\n"); + detectError = 1; + break; + } + { + int const ret = fseek(srcFile, ((long)headerSize)-((long)numBytesRead), SEEK_CUR); + if (ret != 0) { + DISPLAY("Error: could not move to end of frame header\n"); + detectError = 1; + break; + } + } + } + + /* skip the rest of the blocks in the frame */ + { + int lastBlock = 0; + do { + BYTE blockHeaderBuffer[3]; + U32 blockHeader; + int blockSize; + size_t const readBytes = fread(blockHeaderBuffer, 1, 3, srcFile); + if (readBytes != 3) { + DISPLAY("There was a problem reading the block header\n"); + detectError = 1; + break; + } + blockHeader = MEM_readLE24(blockHeaderBuffer); + lastBlock = blockHeader & 1; + blockSize = blockHeader >> 3; + { + int const ret = fseek(srcFile, blockSize, SEEK_CUR); + if (ret != 0) { + DISPLAY("Error: could not skip to end of block\n"); + detectError = 1; + break; + } + } + } while (lastBlock != 1); + + if (detectError) { + break; + } + } + { + /* check if checksum is used */ + BYTE const frameHeaderDescriptor = headerBuffer[4]; + int const contentChecksumFlag = (frameHeaderDescriptor & (1 << 2)) >> 2; + if (contentChecksumFlag) { + int const ret = fseek(srcFile, 4, SEEK_CUR); + info->usesCheck = 1; + if (ret != 0) { + DISPLAY("Error: could not skip past checksum\n"); + detectError = 1; + break; + } + } + } + info->numActualFrames++; + } + else if (magicNumber == ZSTD_MAGIC_SKIPPABLE_START) { + BYTE frameSizeBuffer[4]; + size_t const readBytes = fread(frameSizeBuffer, 1, 4, srcFile); + if (readBytes != 4) { + DISPLAY("There was an error reading skippable frame size"); + detectError = 1; + break; + } + { + U32 const frameSize = MEM_readLE32(frameSizeBuffer); + int const ret = LONG_SEEK(srcFile, frameSize, SEEK_CUR); + if (ret != 0) { + DISPLAY("Error: could not find end of skippable frame\n"); + detectError = 1; + break; + } + } + info->numSkippableFrames++; + } + else { + detectError = 2; + break; + } + } + } + fclose(srcFile); + return detectError; +} + +static void displayInfo(const char* inFileName, fileInfo_t* info, int displayLevel){ + double const compressedSizeMB = (double)info->compressedSize/(1 MB); + double const decompressedSizeMB = (double)info->decompressedSize/(1 MB); + const char* const checkString = (info->usesCheck ? "XXH64" : "None"); + if (displayLevel <= 2) { + if (!info->decompUnavailable) { + double const ratio = (info->decompressedSize == 0) ? 0.0 : compressedSizeMB/decompressedSizeMB; + DISPLAYOUT("Skippable Non-Skippable Compressed Uncompressed Ratio Check Filename\n"); + DISPLAYOUT("%9d %13d %7.2f MB %9.2f MB %5.3f %5s %s\n", + info->numSkippableFrames, info->numActualFrames, compressedSizeMB, decompressedSizeMB, + ratio, checkString, inFileName); + } + else { + DISPLAYOUT("Skippable Non-Skippable Compressed Check Filename\n"); + DISPLAYOUT("%9d %13d %7.2f MB %5s %s\n", + info->numSkippableFrames, info->numActualFrames, compressedSizeMB, checkString, inFileName); + } + } + else{ + DISPLAYOUT("# Zstandard Frames: %d\n", info->numActualFrames); + DISPLAYOUT("# Skippable Frames: %d\n", info->numSkippableFrames); + DISPLAYOUT("Compressed Size: %.2f MB (%llu B)\n", compressedSizeMB, info->compressedSize); + if (!info->decompUnavailable) { + DISPLAYOUT("Decompressed Size: %.2f MB (%llu B)\n", decompressedSizeMB, info->decompressedSize); + DISPLAYOUT("Ratio: %.4f\n", compressedSizeMB/decompressedSizeMB); + } + DISPLAYOUT("Check: %s\n", checkString); + DISPLAYOUT("\n"); + } +} + + +static int FIO_listFile(const char* inFileName, int displayLevel, unsigned fileNo, unsigned numFiles){ + /* initialize info to avoid warnings */ + fileInfo_t info; + memset(&info, 0, sizeof(info)); + DISPLAYOUT("%s (%u/%u):\n", inFileName, fileNo, numFiles); + { + int const error = getFileInfo(&info, inFileName); + if (error == 1) { + /* display error, but provide output */ + DISPLAY("An error occurred with getting file info\n"); + } + else if (error == 2) { + DISPLAYOUT("File %s not compressed with zstd\n", inFileName); + if (displayLevel > 2) { + DISPLAYOUT("\n"); + } + return 1; + } + displayInfo(inFileName, &info, displayLevel); + return error; + } +} + +int FIO_listMultipleFiles(unsigned numFiles, const char** filenameTable, int displayLevel){ + if (numFiles == 0) { + DISPLAYOUT("No files given\n"); + return 0; + } + DISPLAYOUT("===========================================\n"); + DISPLAYOUT("Printing information about compressed files\n"); + DISPLAYOUT("===========================================\n"); + DISPLAYOUT("Number of files listed: %u\n", numFiles); + { + int error = 0; + unsigned u; + for (u=0; u 100). +* `-l`, `--list`: + Display information related to a zstd compressed file, such as size, ratio, and checksum. + Some of these fields may not be available. + This command can be augmented with the `-v` modifier. ### Operation modifiers diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 32fef9993..846895f44 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -117,6 +117,7 @@ static int usage_advanced(const char* programName) DISPLAY( " -v : verbose mode; specify multiple times to increase verbosity\n"); DISPLAY( " -q : suppress warnings; specify twice to suppress errors too\n"); DISPLAY( " -c : force write to standard output, even if it is the console\n"); + DISPLAY( " -l : print information about zstd compressed files.\n"); #ifndef ZSTD_NOCOMPRESS DISPLAY( "--ultra : enable levels beyond %i, up to %i (requires more memory)\n", ZSTDCLI_CLEVEL_MAX, ZSTD_maxCLevel()); #ifdef ZSTD_MULTITHREAD @@ -148,6 +149,7 @@ static int usage_advanced(const char* programName) #endif #endif DISPLAY( " -M# : Set a memory usage limit for decompression \n"); + DISPLAY( "--list : list information about a zstd compressed file \n"); DISPLAY( "-- : All arguments after \"--\" are treated as files \n"); #ifndef ZSTD_NODICT DISPLAY( "\n"); @@ -310,7 +312,7 @@ static unsigned parseCompressionParameters(const char* stringPtr, ZSTD_compressi } -typedef enum { zom_compress, zom_decompress, zom_test, zom_bench, zom_train } zstd_operation_mode; +typedef enum { zom_compress, zom_decompress, zom_test, zom_bench, zom_train, zom_list } zstd_operation_mode; #define CLEAN_RETURN(i) { operationResult = (i); goto _end; } @@ -401,6 +403,7 @@ int main(int argCount, const char* argv[]) if (argument[1]=='-') { /* long commands (--long-word) */ if (!strcmp(argument, "--")) { nextArgumentsAreFiles=1; continue; } /* only file names allowed from now on */ + if (!strcmp(argument, "--list")) { operation=zom_list; continue; } if (!strcmp(argument, "--compress")) { operation=zom_compress; continue; } if (!strcmp(argument, "--decompress")) { operation=zom_decompress; continue; } if (!strcmp(argument, "--uncompress")) { operation=zom_decompress; continue; } @@ -531,7 +534,7 @@ int main(int argCount, const char* argv[]) argument++; memLimit = readU32FromChar(&argument); break; - + case 'l': operation=zom_list; argument++; break; #ifdef UTIL_HAS_CREATEFILELIST /* recursive */ case 'r': recursive=1; argument++; break; @@ -672,7 +675,10 @@ int main(int argCount, const char* argv[]) } } #endif - + if (operation == zom_list) { + int const ret = FIO_listMultipleFiles(filenameIdx, filenameTable, g_displayLevel); + CLEAN_RETURN(ret); + } /* Check if benchmark is selected */ if (operation==zom_bench) { #ifndef ZSTD_NOBENCH diff --git a/tests/playTests.sh b/tests/playTests.sh index e69574588..4eb794b54 100755 --- a/tests/playTests.sh +++ b/tests/playTests.sh @@ -537,6 +537,51 @@ fi rm tmp* +$ECHO "\n**** zstd --list/-l single frame tests ****" +./datagen > tmp1 +./datagen > tmp2 +./datagen > tmp3 +./datagen > tmp4 +$ZSTD tmp* +$ZSTD -l *.zst +$ZSTD -lv *.zst +$ZSTD --list *.zst +$ZSTD --list -v *.zst + +$ECHO "\n**** zstd --list/-l multiple frame tests ****" +cat tmp1.zst tmp2.zst > tmp12.zst +cat tmp3.zst tmp4.zst > tmp34.zst +cat tmp12.zst tmp34.zst > tmp1234.zst +cat tmp12.zst tmp4.zst > tmp124.zst +$ZSTD -l *.zst +$ZSTD -lv *.zst +$ZSTD --list *.zst +$ZSTD --list -v *.zst + +$ECHO "\n**** zstd --list/-l error detection tests ****" +! $ZSTD -l tmp1 tmp1.zst +! $ZSTD --list tmp* +! $ZSTD -lv tmp1* +! $ZSTD --list -v tmp2 tmp23.zst + +$ECHO "\n**** zstd --list/-l test with null files ****" +./datagen -g0 > tmp5 +$ZSTD tmp5 +! $ZSTD -l tmp5* +! $ZSTD -lv tmp5* +! $ZSTD --list tmp5* +! $ZSTD --list -v tmp5* + +$ECHO "\n**** zstd --list/-l test with no frame content size ****" +echo -n '' > tmp6 +$ZSTD tmp6 +$ZSTD -l tmp6.zst +$ZSTD -lv tmp6.zst +$ZSTD --list tmp6.zst +$ZSTD --list -v tmp6.zst + +rm tmp* + if [ "$1" != "--test-large-data" ]; then $ECHO "Skipping large data tests" exit 0