Merge pull request #723 from paulcruz74/dev

Adding zstd -l

lib/decompress/zstd_decompress.c

@@ -204,7 +204,7 @@ unsigned ZSTD_isFrame(const void* buffer, size_t size)
 /** ZSTD_frameHeaderSize() :
 *   srcSize must be >= ZSTD_frameHeaderSize_prefix.
 *   @return : size of the Frame Header */
-static size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize)
+size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize)
 {
     if (srcSize < ZSTD_frameHeaderSize_prefix) return ERROR(srcSize_wrong);
     {   BYTE const fhd = ((const BYTE*)src)[4];

lib/zstd.h

@@ -461,6 +461,11 @@ ZSTDLIB_API unsigned long long ZSTD_getFrameContentSize(const void *src, size_t
  *            however it does mean that all frame data must be present and valid. */
 ZSTDLIB_API unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize);
 
+/*! ZSTD_frameHeaderSize() :
+*   `src` should point to the start of a ZSTD frame
+*   `srcSize` must be >= ZSTD_frameHeaderSize_prefix.
+*   @return : size of the Frame Header */
+size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize);
 
 /***************************************
 *  Context memory usage

programs/fileio.c

@@ -91,6 +91,7 @@
 *  Macros
 ***************************************/
 #define DISPLAY(...)         fprintf(stderr, __VA_ARGS__)
+#define DISPLAYOUT(...)      fprintf(stdout, __VA_ARGS__)
 #define DISPLAYLEVEL(l, ...) { if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); } }
 static int g_displayLevel = 2;   /* 0 : no display;   1: errors;   2 : + result + interaction + warnings;   3 : + progression;   4 : + information */
 void FIO_setNotificationLevel(unsigned level) { g_displayLevel=level; }
@@ -862,6 +863,222 @@ int FIO_compressFilename(const char* dstFileName, const char* srcFileName,
     return result;
 }
 
+typedef struct {
+    int numActualFrames;
+    int numSkippableFrames;
+    unsigned long long decompressedSize;
+    int decompUnavailable;
+    unsigned long long compressedSize;
+    int usesCheck;
+} fileInfo_t;
+
+/*
+ * Reads information from file, stores in *info
+ * if successful, returns 0, returns 1 for frame analysis error, returns 2 for file not compressed with zstd
+ */
+static int getFileInfo(fileInfo_t* info, const char* inFileName){
+    int detectError = 0;
+    FILE* const srcFile = FIO_openSrcFile(inFileName);
+    if (srcFile == NULL) {
+        DISPLAY("Error: could not open source file %s\n", inFileName);
+        return 1;
+    }
+    info->compressedSize = (unsigned long long)UTIL_getFileSize(inFileName);
+    /* begin analyzing frame */
+    for ( ; ; ) {
+        BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX];
+        size_t const numBytesRead = fread(headerBuffer, 1, sizeof(headerBuffer), srcFile);
+        if (numBytesRead < ZSTD_frameHeaderSize_min) {
+            if (feof(srcFile) && numBytesRead == 0 && info->compressedSize > 0) {
+                break;
+            }
+            else if (feof(srcFile)) {
+                DISPLAY("Error: reached end of file with incomplete frame\n");
+                detectError = 2;
+                break;
+            }
+            else {
+                DISPLAY("Error: did not reach end of file but ran out of frames\n");
+                detectError = 1;
+                break;
+            }
+        }
+        {
+            U32 const magicNumber = MEM_readLE32(headerBuffer);
+            if (magicNumber == ZSTD_MAGICNUMBER) {
+                U64 const frameContentSize = ZSTD_getFrameContentSize(headerBuffer, numBytesRead);
+                if (frameContentSize == ZSTD_CONTENTSIZE_ERROR || frameContentSize == ZSTD_CONTENTSIZE_UNKNOWN) {
+                    info->decompUnavailable = 1;
+                }
+                else {
+                    info->decompressedSize += frameContentSize;
+                }
+                {
+                    /* move to the end of the frame header */
+                    size_t const headerSize = ZSTD_frameHeaderSize(headerBuffer, numBytesRead);
+                    if (ZSTD_isError(headerSize)) {
+                        DISPLAY("Error: could not determine frame header size\n");
+                        detectError = 1;
+                        break;
+                    }
+                    {
+                        int const ret = fseek(srcFile, ((long)headerSize)-((long)numBytesRead), SEEK_CUR);
+                        if (ret != 0) {
+                            DISPLAY("Error: could not move to end of frame header\n");
+                            detectError = 1;
+                            break;
+                        }
+                    }
+                }
+
+                /* skip the rest of the blocks in the frame */
+                {
+                    int lastBlock = 0;
+                    do {
+                        BYTE blockHeaderBuffer[3];
+                        U32 blockHeader;
+                        int blockSize;
+                        size_t const readBytes = fread(blockHeaderBuffer, 1, 3, srcFile);
+                        if (readBytes != 3) {
+                            DISPLAY("There was a problem reading the block header\n");
+                            detectError = 1;
+                            break;
+                        }
+                        blockHeader = MEM_readLE24(blockHeaderBuffer);
+                        lastBlock = blockHeader & 1;
+                        blockSize = blockHeader >> 3;
+                        {
+                            int const ret = fseek(srcFile, blockSize, SEEK_CUR);
+                            if (ret != 0) {
+                                DISPLAY("Error: could not skip to end of block\n");
+                                detectError = 1;
+                                break;
+                            }
+                        }
+                    } while (lastBlock != 1);
+
+                    if (detectError) {
+                        break;
+                    }
+                }
+                {
+                    /* check if checksum is used */
+                    BYTE const frameHeaderDescriptor = headerBuffer[4];
+                    int const contentChecksumFlag = (frameHeaderDescriptor & (1 << 2)) >> 2;
+                    if (contentChecksumFlag) {
+                        int const ret = fseek(srcFile, 4, SEEK_CUR);
+                        info->usesCheck = 1;
+                        if (ret != 0) {
+                            DISPLAY("Error: could not skip past checksum\n");
+                            detectError = 1;
+                            break;
+                        }
+                    }
+                }
+                info->numActualFrames++;
+            }
+            else if (magicNumber == ZSTD_MAGIC_SKIPPABLE_START) {
+                BYTE frameSizeBuffer[4];
+                size_t const readBytes = fread(frameSizeBuffer, 1, 4, srcFile);
+                if (readBytes != 4) {
+                    DISPLAY("There was an error reading skippable frame size");
+                    detectError = 1;
+                    break;
+                }
+                {
+                    U32 const frameSize = MEM_readLE32(frameSizeBuffer);
+                    int const ret = LONG_SEEK(srcFile, frameSize, SEEK_CUR);
+                    if (ret != 0) {
+                        DISPLAY("Error: could not find end of skippable frame\n");
+                        detectError = 1;
+                        break;
+                    }
+                }
+                info->numSkippableFrames++;
+            }
+            else {
+                detectError = 2;
+                break;
+            }
+        }
+    }
+    fclose(srcFile);
+    return detectError;
+}
+
+static void displayInfo(const char* inFileName, fileInfo_t* info, int displayLevel){
+    double const compressedSizeMB = (double)info->compressedSize/(1 MB);
+    double const decompressedSizeMB = (double)info->decompressedSize/(1 MB);
+    const char* const checkString = (info->usesCheck ? "XXH64" : "None");
+    if (displayLevel <= 2) {
+        if (!info->decompUnavailable) {
+            double const ratio = (info->decompressedSize == 0) ? 0.0 : compressedSizeMB/decompressedSizeMB;
+            DISPLAYOUT("Skippable  Non-Skippable  Compressed  Uncompressed  Ratio  Check  Filename\n");
+            DISPLAYOUT("%9d  %13d  %7.2f MB  %9.2f MB  %5.3f  %5s  %s\n",
+                    info->numSkippableFrames, info->numActualFrames, compressedSizeMB, decompressedSizeMB,
+                    ratio, checkString, inFileName);
+        }
+        else {
+            DISPLAYOUT("Skippable  Non-Skippable  Compressed  Check  Filename\n");
+            DISPLAYOUT("%9d  %13d  %7.2f MB  %5s  %s\n",
+                    info->numSkippableFrames, info->numActualFrames, compressedSizeMB, checkString, inFileName);
+        }
+    }
+    else{
+        DISPLAYOUT("# Zstandard Frames: %d\n", info->numActualFrames);
+        DISPLAYOUT("# Skippable Frames: %d\n", info->numSkippableFrames);
+        DISPLAYOUT("Compressed Size: %.2f MB (%llu B)\n", compressedSizeMB, info->compressedSize);
+        if (!info->decompUnavailable) {
+            DISPLAYOUT("Decompressed Size: %.2f MB (%llu B)\n", decompressedSizeMB, info->decompressedSize);
+            DISPLAYOUT("Ratio: %.4f\n", compressedSizeMB/decompressedSizeMB);
+        }
+        DISPLAYOUT("Check: %s\n", checkString);
+        DISPLAYOUT("\n");
+    }
+}
+
+
+static int FIO_listFile(const char* inFileName, int displayLevel, unsigned fileNo, unsigned numFiles){
+    /* initialize info to avoid warnings */
+    fileInfo_t info;
+    memset(&info, 0, sizeof(info));
+    DISPLAYOUT("%s (%u/%u):\n", inFileName, fileNo, numFiles);
+    {
+        int const error = getFileInfo(&info, inFileName);
+        if (error == 1) {
+            /* display error, but provide output */
+            DISPLAY("An error occurred with getting file info\n");
+        }
+        else if (error == 2) {
+            DISPLAYOUT("File %s not compressed with zstd\n", inFileName);
+            if (displayLevel > 2) {
+                DISPLAYOUT("\n");
+            }
+            return 1;
+        }
+        displayInfo(inFileName, &info, displayLevel);
+        return error;
+    }
+}
+
+int FIO_listMultipleFiles(unsigned numFiles, const char** filenameTable, int displayLevel){
+    if (numFiles == 0) {
+        DISPLAYOUT("No files given\n");
+        return 0;
+    }
+    DISPLAYOUT("===========================================\n");
+    DISPLAYOUT("Printing information about compressed files\n");
+    DISPLAYOUT("===========================================\n");
+    DISPLAYOUT("Number of files listed: %u\n", numFiles);
+    {
+        int error = 0;
+        unsigned u;
+        for (u=0; u<numFiles;u++) {
+            error |= FIO_listFile(filenameTable[u], displayLevel, u+1, numFiles);
+        }
+        return error;
+    }
+}
 
 int FIO_compressMultipleFilenames(const char** inFileNamesTable, unsigned nbFiles,
                                   const char* suffix,

programs/fileio.h

@@ -70,6 +70,7 @@ int FIO_compressFilename (const char* outfilename, const char* infilename, const
     @return : 0 == ok;  1 == pb with src file. */
 int FIO_decompressFilename (const char* outfilename, const char* infilename, const char* dictFileName);
 
+int FIO_listMultipleFiles(unsigned numFiles, const char** filenameTable, int displayLevel);
 
 /*-*************************************
 *  Multiple File functions

programs/zstd.1.md

@@ -93,6 +93,10 @@ the last one takes effect.
 * `--train FILEs`:
     Use FILEs as a training set to create a dictionary.
     The training set should contain a lot of small files (> 100).
+* `-l`, `--list`:
+    Display information related to a zstd compressed file, such as size, ratio, and checksum.
+    Some of these fields may not be available.
+    This command can be augmented with the `-v` modifier. 
 
 ### Operation modifiers
 

programs/zstdcli.c

@@ -117,6 +117,7 @@ static int usage_advanced(const char* programName)
     DISPLAY( " -v     : verbose mode; specify multiple times to increase verbosity\n");
     DISPLAY( " -q     : suppress warnings; specify twice to suppress errors too\n");
     DISPLAY( " -c     : force write to standard output, even if it is the console\n");
+    DISPLAY( " -l     : print information about zstd compressed files.\n");
 #ifndef ZSTD_NOCOMPRESS
     DISPLAY( "--ultra : enable levels beyond %i, up to %i (requires more memory)\n", ZSTDCLI_CLEVEL_MAX, ZSTD_maxCLevel());
 #ifdef ZSTD_MULTITHREAD
@@ -148,6 +149,7 @@ static int usage_advanced(const char* programName)
 #endif
 #endif
     DISPLAY( " -M#    : Set a memory usage limit for decompression \n");
+    DISPLAY( "--list  : list information about a zstd compressed file \n");
     DISPLAY( "--      : All arguments after \"--\" are treated as files \n");
 #ifndef ZSTD_NODICT
     DISPLAY( "\n");
@@ -310,7 +312,7 @@ static unsigned parseCompressionParameters(const char* stringPtr, ZSTD_compressi
 }
 
 
-typedef enum { zom_compress, zom_decompress, zom_test, zom_bench, zom_train } zstd_operation_mode;
+typedef enum { zom_compress, zom_decompress, zom_test, zom_bench, zom_train, zom_list } zstd_operation_mode;
 
 #define CLEAN_RETURN(i) { operationResult = (i); goto _end; }
 
@@ -401,6 +403,7 @@ int main(int argCount, const char* argv[])
                 if (argument[1]=='-') {
                     /* long commands (--long-word) */
                     if (!strcmp(argument, "--")) { nextArgumentsAreFiles=1; continue; }   /* only file names allowed from now on */
+                    if (!strcmp(argument, "--list")) { operation=zom_list; continue; }
                     if (!strcmp(argument, "--compress")) { operation=zom_compress; continue; }
                     if (!strcmp(argument, "--decompress")) { operation=zom_decompress; continue; }
                     if (!strcmp(argument, "--uncompress")) { operation=zom_decompress; continue; }
@@ -531,7 +534,7 @@ int main(int argCount, const char* argv[])
                         argument++;
                         memLimit = readU32FromChar(&argument);
                         break;
-
+                    case 'l': operation=zom_list; argument++; break;
 #ifdef UTIL_HAS_CREATEFILELIST
                         /* recursive */
                     case 'r': recursive=1; argument++; break;
@@ -672,7 +675,10 @@ int main(int argCount, const char* argv[])
         }
     }
 #endif
-
+    if (operation == zom_list) {
+        int const ret = FIO_listMultipleFiles(filenameIdx, filenameTable, g_displayLevel);
+        CLEAN_RETURN(ret);
+    }
     /* Check if benchmark is selected */
     if (operation==zom_bench) {
 #ifndef ZSTD_NOBENCH

tests/playTests.sh

@@ -537,6 +537,51 @@ fi
 
 rm tmp*
 
+$ECHO "\n**** zstd --list/-l single frame tests ****"
+./datagen > tmp1
+./datagen > tmp2
+./datagen > tmp3
+./datagen > tmp4
+$ZSTD tmp*
+$ZSTD -l *.zst
+$ZSTD -lv *.zst
+$ZSTD --list *.zst
+$ZSTD --list -v *.zst
+
+$ECHO "\n**** zstd --list/-l multiple frame tests ****"
+cat tmp1.zst tmp2.zst > tmp12.zst
+cat tmp3.zst tmp4.zst > tmp34.zst
+cat tmp12.zst tmp34.zst > tmp1234.zst
+cat tmp12.zst tmp4.zst > tmp124.zst
+$ZSTD -l *.zst
+$ZSTD -lv *.zst
+$ZSTD --list *.zst
+$ZSTD --list -v *.zst
+
+$ECHO "\n**** zstd --list/-l error detection tests ****"
+! $ZSTD -l tmp1 tmp1.zst
+! $ZSTD --list tmp*
+! $ZSTD -lv tmp1*
+! $ZSTD --list -v tmp2 tmp23.zst
+
+$ECHO "\n**** zstd --list/-l test with null files ****"
+./datagen -g0 > tmp5
+$ZSTD tmp5
+! $ZSTD -l tmp5*
+! $ZSTD -lv tmp5*
+! $ZSTD --list tmp5*
+! $ZSTD --list -v tmp5*
+
+$ECHO "\n**** zstd --list/-l test with no frame content size ****"
+echo -n '' > tmp6
+$ZSTD tmp6
+$ZSTD -l tmp6.zst
+$ZSTD -lv tmp6.zst
+$ZSTD --list tmp6.zst
+$ZSTD --list -v tmp6.zst
+
+rm tmp*
+
 if [ "$1" != "--test-large-data" ]; then
     $ECHO "Skipping large data tests"
     exit 0