ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #723 from paulcruz74/dev

Browse Source 
Adding zstd -l
This commit is contained in:
Yann Collet
2017-06-21 09:41:55 -07:00
committed by GitHub
parent 983e16a269 db3606e249
commit 9c56b12938
7 changed files with 282 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/decompress/zstd_decompress.c
+1 -1
View File
@@ -204,7 +204,7 @@ unsigned ZSTD_isFrame(const void* buffer, size_t size)
/** ZSTD_frameHeaderSize() :
* srcSize must be >= ZSTD_frameHeaderSize_prefix.
* @return : size of the Frame Header */
static size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize)
size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize)
{
if (srcSize < ZSTD_frameHeaderSize_prefix) return ERROR(srcSize_wrong);
{ BYTE const fhd = ((const BYTE*)src)[4];
lib/zstd.h
+5
View File
@@ -461,6 +461,11 @@ ZSTDLIB_API unsigned long long ZSTD_getFrameContentSize(const void *src, size_t
* however it does mean that all frame data must be present and valid. */
ZSTDLIB_API unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize);
/*! ZSTD_frameHeaderSize() :
* `src` should point to the start of a ZSTD frame
* `srcSize` must be >= ZSTD_frameHeaderSize_prefix.
* @return : size of the Frame Header */
size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize);
/***************************************
* Context memory usage
programs/fileio.c
+217
View File
@@ -91,6 +91,7 @@
* Macros
***************************************/
#define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
#define DISPLAYOUT(...) fprintf(stdout, __VA_ARGS__)
#define DISPLAYLEVEL(l, ...) { if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); } }
static int g_displayLevel = 2; /* 0 : no display; 1: errors; 2 : + result + interaction + warnings; 3 : + progression; 4 : + information */
void FIO_setNotificationLevel(unsigned level) { g_displayLevel=level; }
@@ -862,6 +863,222 @@ int FIO_compressFilename(const char* dstFileName, const char* srcFileName,
return result;
}
typedef struct {
int numActualFrames;
int numSkippableFrames;
unsigned long long decompressedSize;
int decompUnavailable;
unsigned long long compressedSize;
int usesCheck;
} fileInfo_t;
/*
* Reads information from file, stores in *info
* if successful, returns 0, returns 1 for frame analysis error, returns 2 for file not compressed with zstd
*/
static int getFileInfo(fileInfo_t* info, const char* inFileName){
int detectError = 0;
FILE* const srcFile = FIO_openSrcFile(inFileName);
if (srcFile == NULL) {
DISPLAY("Error: could not open source file %s\n", inFileName);
return 1;
}
info->compressedSize = (unsigned long long)UTIL_getFileSize(inFileName);
/* begin analyzing frame */
for ( ; ; ) {
BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX];
size_t const numBytesRead = fread(headerBuffer, 1, sizeof(headerBuffer), srcFile);
if (numBytesRead < ZSTD_frameHeaderSize_min) {
if (feof(srcFile) && numBytesRead == 0 && info->compressedSize > 0) {
break;
}
else if (feof(srcFile)) {
DISPLAY("Error: reached end of file with incomplete frame\n");
detectError = 2;
break;
}
else {
DISPLAY("Error: did not reach end of file but ran out of frames\n");
detectError = 1;
break;
}
}
{
U32 const magicNumber = MEM_readLE32(headerBuffer);
if (magicNumber == ZSTD_MAGICNUMBER) {
U64 const frameContentSize = ZSTD_getFrameContentSize(headerBuffer, numBytesRead);
if (frameContentSize == ZSTD_CONTENTSIZE_ERROR || frameContentSize == ZSTD_CONTENTSIZE_UNKNOWN) {
info->decompUnavailable = 1;
}
else {
info->decompressedSize += frameContentSize;
}
{
/* move to the end of the frame header */
size_t const headerSize = ZSTD_frameHeaderSize(headerBuffer, numBytesRead);
if (ZSTD_isError(headerSize)) {
DISPLAY("Error: could not determine frame header size\n");
detectError = 1;
break;
}
{
int const ret = fseek(srcFile, ((long)headerSize)-((long)numBytesRead), SEEK_CUR);
if (ret != 0) {
DISPLAY("Error: could not move to end of frame header\n");
detectError = 1;
break;
}
}
}
/* skip the rest of the blocks in the frame */
{
int lastBlock = 0;
do {
BYTE blockHeaderBuffer[3];
U32 blockHeader;
int blockSize;
size_t const readBytes = fread(blockHeaderBuffer, 1, 3, srcFile);
if (readBytes != 3) {
DISPLAY("There was a problem reading the block header\n");
detectError = 1;
break;
}
blockHeader = MEM_readLE24(blockHeaderBuffer);
lastBlock = blockHeader & 1;
blockSize = blockHeader >> 3;
{
int const ret = fseek(srcFile, blockSize, SEEK_CUR);
if (ret != 0) {
DISPLAY("Error: could not skip to end of block\n");
detectError = 1;
break;
}
}
} while (lastBlock != 1);
if (detectError) {
break;
}
}
{
/* check if checksum is used */
BYTE const frameHeaderDescriptor = headerBuffer[4];
int const contentChecksumFlag = (frameHeaderDescriptor & (1 << 2)) >> 2;
if (contentChecksumFlag) {
int const ret = fseek(srcFile, 4, SEEK_CUR);
info->usesCheck = 1;
if (ret != 0) {
DISPLAY("Error: could not skip past checksum\n");
detectError = 1;
break;
}
}
}
info->numActualFrames++;
}
else if (magicNumber == ZSTD_MAGIC_SKIPPABLE_START) {
BYTE frameSizeBuffer[4];
size_t const readBytes = fread(frameSizeBuffer, 1, 4, srcFile);
if (readBytes != 4) {
DISPLAY("There was an error reading skippable frame size");
detectError = 1;
break;
}
{
U32 const frameSize = MEM_readLE32(frameSizeBuffer);
int const ret = LONG_SEEK(srcFile, frameSize, SEEK_CUR);
if (ret != 0) {
DISPLAY("Error: could not find end of skippable frame\n");
detectError = 1;
break;
}
}
info->numSkippableFrames++;
}
else {
detectError = 2;
break;
}
}
}
fclose(srcFile);
return detectError;
}
static void displayInfo(const char* inFileName, fileInfo_t* info, int displayLevel){
double const compressedSizeMB = (double)info->compressedSize/(1 MB);
double const decompressedSizeMB = (double)info->decompressedSize/(1 MB);
const char* const checkString = (info->usesCheck ? "XXH64" : "None");
if (displayLevel <= 2) {
if (!info->decompUnavailable) {
double const ratio = (info->decompressedSize == 0) ? 0.0 : compressedSizeMB/decompressedSizeMB;
DISPLAYOUT("Skippable Non-Skippable Compressed Uncompressed Ratio Check Filename\n");
DISPLAYOUT("%9d %13d %7.2f MB %9.2f MB %5.3f %5s %s\n",
info->numSkippableFrames, info->numActualFrames, compressedSizeMB, decompressedSizeMB,
ratio, checkString, inFileName);
}
else {
DISPLAYOUT("Skippable Non-Skippable Compressed Check Filename\n");
DISPLAYOUT("%9d %13d %7.2f MB %5s %s\n",
info->numSkippableFrames, info->numActualFrames, compressedSizeMB, checkString, inFileName);
}
}
else{
DISPLAYOUT("# Zstandard Frames: %d\n", info->numActualFrames);
DISPLAYOUT("# Skippable Frames: %d\n", info->numSkippableFrames);
DISPLAYOUT("Compressed Size: %.2f MB (%llu B)\n", compressedSizeMB, info->compressedSize);
if (!info->decompUnavailable) {
DISPLAYOUT("Decompressed Size: %.2f MB (%llu B)\n", decompressedSizeMB, info->decompressedSize);
DISPLAYOUT("Ratio: %.4f\n", compressedSizeMB/decompressedSizeMB);
}
DISPLAYOUT("Check: %s\n", checkString);
DISPLAYOUT("\n");
}
}
static int FIO_listFile(const char* inFileName, int displayLevel, unsigned fileNo, unsigned numFiles){
/* initialize info to avoid warnings */
fileInfo_t info;
memset(&info, 0, sizeof(info));
DISPLAYOUT("%s (%u/%u):\n", inFileName, fileNo, numFiles);
{
int const error = getFileInfo(&info, inFileName);
if (error == 1) {
/* display error, but provide output */
DISPLAY("An error occurred with getting file info\n");
}
else if (error == 2) {
DISPLAYOUT("File %s not compressed with zstd\n", inFileName);
if (displayLevel > 2) {
DISPLAYOUT("\n");
}
return 1;
}
displayInfo(inFileName, &info, displayLevel);
return error;
}
}
int FIO_listMultipleFiles(unsigned numFiles, const char** filenameTable, int displayLevel){
if (numFiles == 0) {
DISPLAYOUT("No files given\n");
return 0;
}
DISPLAYOUT("===========================================\n");
DISPLAYOUT("Printing information about compressed files\n");
DISPLAYOUT("===========================================\n");
DISPLAYOUT("Number of files listed: %u\n", numFiles);
{
int error = 0;
unsigned u;
for (u=0; u<numFiles;u++) {
error |= FIO_listFile(filenameTable[u], displayLevel, u+1, numFiles);
}
return error;
}
}
int FIO_compressMultipleFilenames(const char** inFileNamesTable, unsigned nbFiles,
const char* suffix,
programs/fileio.h
+1
View File
@@ -70,6 +70,7 @@ int FIO_compressFilename (const char* outfilename, const char* infilename, const
@return : 0 == ok; 1 == pb with src file. */
int FIO_decompressFilename (const char* outfilename, const char* infilename, const char* dictFileName);
int FIO_listMultipleFiles(unsigned numFiles, const char** filenameTable, int displayLevel);
/*-*************************************
* Multiple File functions
programs/zstd.1.md
+4
View File
@@ -93,6 +93,10 @@ the last one takes effect.
* `--train FILEs`:
Use FILEs as a training set to create a dictionary.
The training set should contain a lot of small files (> 100).
* `-l`, `--list`:
Display information related to a zstd compressed file, such as size, ratio, and checksum.
Some of these fields may not be available.
This command can be augmented with the `-v` modifier.
### Operation modifiers
programs/zstdcli.c
+9 -3
View File
@@ -117,6 +117,7 @@ static int usage_advanced(const char* programName)
DISPLAY( " -v : verbose mode; specify multiple times to increase verbosity\n");
DISPLAY( " -q : suppress warnings; specify twice to suppress errors too\n");
DISPLAY( " -c : force write to standard output, even if it is the console\n");
DISPLAY( " -l : print information about zstd compressed files.\n");
#ifndef ZSTD_NOCOMPRESS
DISPLAY( "--ultra : enable levels beyond %i, up to %i (requires more memory)\n", ZSTDCLI_CLEVEL_MAX, ZSTD_maxCLevel());
#ifdef ZSTD_MULTITHREAD
@@ -148,6 +149,7 @@ static int usage_advanced(const char* programName)
#endif
#endif
DISPLAY( " -M# : Set a memory usage limit for decompression \n");
DISPLAY( "--list : list information about a zstd compressed file \n");
DISPLAY( "-- : All arguments after \"--\" are treated as files \n");
#ifndef ZSTD_NODICT
DISPLAY( "\n");
@@ -310,7 +312,7 @@ static unsigned parseCompressionParameters(const char* stringPtr, ZSTD_compressi
}
typedef enum { zom_compress, zom_decompress, zom_test, zom_bench, zom_train } zstd_operation_mode;
typedef enum { zom_compress, zom_decompress, zom_test, zom_bench, zom_train, zom_list } zstd_operation_mode;
#define CLEAN_RETURN(i) { operationResult = (i); goto _end; }
@@ -401,6 +403,7 @@ int main(int argCount, const char* argv[])
if (argument[1]=='-') {
/* long commands (--long-word) */
if (!strcmp(argument, "--")) { nextArgumentsAreFiles=1; continue; } /* only file names allowed from now on */
if (!strcmp(argument, "--list")) { operation=zom_list; continue; }
if (!strcmp(argument, "--compress")) { operation=zom_compress; continue; }
if (!strcmp(argument, "--decompress")) { operation=zom_decompress; continue; }
if (!strcmp(argument, "--uncompress")) { operation=zom_decompress; continue; }
@@ -531,7 +534,7 @@ int main(int argCount, const char* argv[])
argument++;
memLimit = readU32FromChar(&argument);
break;
case 'l': operation=zom_list; argument++; break;
#ifdef UTIL_HAS_CREATEFILELIST
/* recursive */
case 'r': recursive=1; argument++; break;
@@ -672,7 +675,10 @@ int main(int argCount, const char* argv[])
}
}
#endif
if (operation == zom_list) {
int const ret = FIO_listMultipleFiles(filenameIdx, filenameTable, g_displayLevel);
CLEAN_RETURN(ret);
}
/* Check if benchmark is selected */
if (operation==zom_bench) {
#ifndef ZSTD_NOBENCH
tests/playTests.sh
+45
View File
@@ -537,6 +537,51 @@ fi
rm tmp*
$ECHO "\n**** zstd --list/-l single frame tests ****"
./datagen > tmp1
./datagen > tmp2
./datagen > tmp3
./datagen > tmp4
$ZSTD tmp*
$ZSTD -l *.zst
$ZSTD -lv *.zst
$ZSTD --list *.zst
$ZSTD --list -v *.zst
$ECHO "\n**** zstd --list/-l multiple frame tests ****"
cat tmp1.zst tmp2.zst > tmp12.zst
cat tmp3.zst tmp4.zst > tmp34.zst
cat tmp12.zst tmp34.zst > tmp1234.zst
cat tmp12.zst tmp4.zst > tmp124.zst
$ZSTD -l *.zst
$ZSTD -lv *.zst
$ZSTD --list *.zst
$ZSTD --list -v *.zst
$ECHO "\n**** zstd --list/-l error detection tests ****"
! $ZSTD -l tmp1 tmp1.zst
! $ZSTD --list tmp*
! $ZSTD -lv tmp1*
! $ZSTD --list -v tmp2 tmp23.zst
$ECHO "\n**** zstd --list/-l test with null files ****"
./datagen -g0 > tmp5
$ZSTD tmp5
! $ZSTD -l tmp5*
! $ZSTD -lv tmp5*
! $ZSTD --list tmp5*
! $ZSTD --list -v tmp5*
$ECHO "\n**** zstd --list/-l test with no frame content size ****"
echo -n '' > tmp6
$ZSTD tmp6
$ZSTD -l tmp6.zst
$ZSTD -lv tmp6.zst
$ZSTD --list tmp6.zst
$ZSTD --list -v tmp6.zst
rm tmp*
if [ "$1" != "--test-large-data" ]; then
$ECHO "Skipping large data tests"
exit 0