diff --git a/programs/benchzstd.c b/programs/benchzstd.c index 49c03490d..72793f07b 100644 --- a/programs/benchzstd.c +++ b/programs/benchzstd.c @@ -393,6 +393,8 @@ BMK_benchMemAdvancedNoAlloc( BMK_benchParams_t cbp, dbp; BMK_initCCtxArgs cctxprep; BMK_initDCtxArgs dctxprep; + UTIL_HumanReadableSize_t hr_isize; + UTIL_HumanReadableSize_t hr_osize; cbp.benchFn = local_defaultCompress; /* ZSTD_compress2 */ cbp.benchPayload = cctx; @@ -429,8 +431,10 @@ BMK_benchMemAdvancedNoAlloc( dctxprep.dictBuffer = dictBuffer; dctxprep.dictBufferSize = dictBufferSize; + hr_isize = UTIL_makeHumanReadableSize((U64) srcSize); + DISPLAYLEVEL(2, "\r%70s\r", ""); /* blank line */ - DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->\r", marks[markNb], displayName, (unsigned)srcSize); + DISPLAYLEVEL(2, "%2s-%-17.17s : %.*f%s -> \r", marks[markNb], displayName, hr_isize.precision, hr_isize.value, hr_isize.suffix); while (!(compressionCompleted && decompressionCompleted)) { if (!compressionCompleted) { @@ -451,9 +455,11 @@ BMK_benchMemAdvancedNoAlloc( } } { int const ratioAccuracy = (ratio < 10.) ? 3 : 2; - DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.*f),%6.*f MB/s\r", + hr_osize = UTIL_makeHumanReadableSize((U64) cSize); + DISPLAYLEVEL(2, "%2s-%-17.17s : %.*f%s -> %.*f%s (%5.*f), %6.*f MB/s\r", marks[markNb], displayName, - (unsigned)srcSize, (unsigned)cSize, + hr_isize.precision, hr_isize.value, hr_isize.suffix, + hr_osize.precision, hr_osize.value, hr_osize.suffix, ratioAccuracy, ratio, benchResult.cSpeed < (10 MB) ? 2 : 1, (double)benchResult.cSpeed / MB_UNIT); } @@ -474,9 +480,11 @@ BMK_benchMemAdvancedNoAlloc( } { int const ratioAccuracy = (ratio < 10.) ? 3 : 2; - DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.*f),%6.*f MB/s ,%6.1f MB/s \r", + hr_osize = UTIL_makeHumanReadableSize((U64) cSize); + DISPLAYLEVEL(2, "%2s-%-17.17s : %.*f%s -> %.*f%s (%5.*f), %6.*f MB/s, %6.1f MB/s \r", marks[markNb], displayName, - (unsigned)srcSize, (unsigned)cSize, + hr_isize.precision, hr_isize.value, hr_isize.suffix, + hr_osize.precision, hr_osize.value, hr_osize.suffix, ratioAccuracy, ratio, benchResult.cSpeed < (10 MB) ? 2 : 1, (double)benchResult.cSpeed / MB_UNIT, (double)benchResult.dSpeed / MB_UNIT); diff --git a/programs/fileio.c b/programs/fileio.c index a67c0fbfb..da5412f27 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -1342,6 +1342,7 @@ FIO_compressZstdFrame(FIO_ctx_t* const fCtx, unsigned inputPresented = 0; unsigned inputBlocked = 0; unsigned lastJobID = 0; + UTIL_HumanReadableSize_t const file_hrs = UTIL_makeHumanReadableSize(fileSize); DISPLAYLEVEL(6, "compression using zstd format \n"); @@ -1352,6 +1353,7 @@ FIO_compressZstdFrame(FIO_ctx_t* const fCtx, /* unknown source size; use the declared stream size */ CHECK( ZSTD_CCtx_setPledgedSrcSize(ress.cctx, prefs->streamSrcSize) ); } + (void)srcFileName; /* Main compression loop */ @@ -1395,14 +1397,17 @@ FIO_compressZstdFrame(FIO_ctx_t* const fCtx, if (READY_FOR_UPDATE()) { ZSTD_frameProgression const zfp = ZSTD_getFrameProgression(ress.cctx); double const cShare = (double)zfp.produced / (double)(zfp.consumed + !zfp.consumed/*avoid div0*/) * 100; + UTIL_HumanReadableSize_t const buffered_hrs = UTIL_makeHumanReadableSize(zfp.ingested - zfp.consumed); + UTIL_HumanReadableSize_t const consumed_hrs = UTIL_makeHumanReadableSize(zfp.consumed); + UTIL_HumanReadableSize_t const produced_hrs = UTIL_makeHumanReadableSize(zfp.produced); /* display progress notifications */ if (g_display_prefs.displayLevel >= 3) { - DISPLAYUPDATE(3, "\r(L%i) Buffered :%4u MB - Consumed :%4u MB - Compressed :%4u MB => %.2f%% ", + DISPLAYUPDATE(3, "\r(L%i) Buffered :%6.*f%4s - Consumed :%6.*f%4s - Compressed :%6.*f%4s => %.2f%% ", compressionLevel, - (unsigned)((zfp.ingested - zfp.consumed) >> 20), - (unsigned)(zfp.consumed >> 20), - (unsigned)(zfp.produced >> 20), + buffered_hrs.precision, buffered_hrs.value, buffered_hrs.suffix, + consumed_hrs.precision, consumed_hrs.value, consumed_hrs.suffix, + produced_hrs.precision, produced_hrs.value, produced_hrs.suffix, cShare ); } else if (g_display_prefs.displayLevel >= 2 || g_display_prefs.progressSetting == FIO_ps_always) { /* Require level 2 or forcibly displayed progress counter for summarized updates */ @@ -1419,10 +1424,10 @@ FIO_compressZstdFrame(FIO_ctx_t* const fCtx, fCtx->currFileIdx+1, fCtx->nbFilesTotal, (int)(18-srcFileNameSize), srcFileName); } } - DISPLAYLEVEL(1, "Read : %2u ", (unsigned)(zfp.consumed >> 20)); + DISPLAYLEVEL(1, "Read:%6.*f%4s ", consumed_hrs.precision, consumed_hrs.value, consumed_hrs.suffix); if (fileSize != UTIL_FILESIZE_UNKNOWN) - DISPLAYLEVEL(2, "/ %2u ", (unsigned)(fileSize >> 20)); - DISPLAYLEVEL(1, "MB ==> %2.f%%", cShare); + DISPLAYLEVEL(2, "/%6.*f%4s", file_hrs.precision, file_hrs.value, file_hrs.suffix); + DISPLAYLEVEL(1, " ==> %2.f%%", cShare); DELAY_NEXT_UPDATE(); } @@ -1592,16 +1597,20 @@ FIO_compressFilename_internal(FIO_ctx_t* const fCtx, if (g_display_prefs.displayLevel >= 2 && !fCtx->hasStdoutOutput && (g_display_prefs.displayLevel >= 3 || fCtx->nbFilesTotal <= 1)) { + UTIL_HumanReadableSize_t hr_isize = UTIL_makeHumanReadableSize((U64) readsize); + UTIL_HumanReadableSize_t hr_osize = UTIL_makeHumanReadableSize((U64) compressedfilesize); if (readsize == 0) { - DISPLAYLEVEL(2,"%-20s : (%6llu => %6llu bytes, %s) \n", + DISPLAYLEVEL(2,"%-20s : (%6.*f%4s => %6.*f%4s, %s) \n", srcFileName, - (unsigned long long)readsize, (unsigned long long) compressedfilesize, + hr_isize.precision, hr_isize.value, hr_isize.suffix, + hr_osize.precision, hr_osize.value, hr_osize.suffix, dstFileName); } else { - DISPLAYLEVEL(2,"%-20s :%6.2f%% (%6llu => %6llu bytes, %s) \n", + DISPLAYLEVEL(2,"%-20s :%6.2f%% (%6.*f%4s => %6.*f%4s, %s) \n", srcFileName, (double)compressedfilesize / (double)readsize * 100, - (unsigned long long)readsize, (unsigned long long) compressedfilesize, + hr_isize.precision, hr_isize.value, hr_isize.suffix, + hr_osize.precision, hr_osize.value, hr_osize.suffix, dstFileName); } } @@ -1890,10 +1899,15 @@ int FIO_compressMultipleFilenames(FIO_ctx_t* const fCtx, } if (fCtx->nbFilesProcessed >= 1 && fCtx->nbFilesTotal > 1 && fCtx->totalBytesInput != 0) { + UTIL_HumanReadableSize_t hr_isize = UTIL_makeHumanReadableSize((U64) fCtx->totalBytesInput); + UTIL_HumanReadableSize_t hr_osize = UTIL_makeHumanReadableSize((U64) fCtx->totalBytesOutput); + DISPLAYLEVEL(2, "\r%79s\r", ""); - DISPLAYLEVEL(2, "%d files compressed : %.2f%% (%6zu => %6zu bytes)\n", fCtx->nbFilesProcessed, + DISPLAYLEVEL(2, "%3d files compressed :%.2f%% (%6.*f%4s => %6.*f%4s)\n", + fCtx->nbFilesProcessed, (double)fCtx->totalBytesOutput/((double)fCtx->totalBytesInput)*100, - fCtx->totalBytesInput, fCtx->totalBytesOutput); + hr_isize.precision, hr_isize.value, hr_isize.suffix, + hr_osize.precision, hr_osize.value, hr_osize.suffix); } FIO_freeCResources(&ress); @@ -2156,6 +2170,7 @@ FIO_decompressZstdFrame(FIO_ctx_t* const fCtx, dRess_t* ress, FILE* finput, ZSTD_outBuffer outBuff= { ress->dstBuffer, ress->dstBufferSize, 0 }; size_t const readSizeHint = ZSTD_decompressStream(ress->dctx, &outBuff, &inBuff); const int displayLevel = (!fCtx->hasStdoutOutput || g_display_prefs.progressSetting == FIO_ps_always) ? 1 : 2; + UTIL_HumanReadableSize_t const hrs = UTIL_makeHumanReadableSize(alreadyDecoded+frameSize); if (ZSTD_isError(readSizeHint)) { DISPLAYLEVEL(1, "%s : Decoding error (36) : %s \n", srcFileName, ZSTD_getErrorName(readSizeHint)); @@ -2170,15 +2185,15 @@ FIO_decompressZstdFrame(FIO_ctx_t* const fCtx, dRess_t* ress, FILE* finput, size_t srcFileNameSize = strlen(srcFileName); if (srcFileNameSize > 18) { const char* truncatedSrcFileName = srcFileName + srcFileNameSize - 15; - DISPLAYUPDATE(displayLevel, "\rDecompress: %2u/%2u files. Current: ...%s : %u MB... ", - fCtx->currFileIdx+1, fCtx->nbFilesTotal, truncatedSrcFileName, (unsigned)((alreadyDecoded+frameSize)>>20) ); + DISPLAYUPDATE(displayLevel, "\rDecompress: %2u/%2u files. Current: ...%s : %.*f%s... ", + fCtx->currFileIdx+1, fCtx->nbFilesTotal, truncatedSrcFileName, hrs.precision, hrs.value, hrs.suffix); } else { - DISPLAYUPDATE(displayLevel, "\rDecompress: %2u/%2u files. Current: %s : %u MB... ", - fCtx->currFileIdx+1, fCtx->nbFilesTotal, srcFileName, (unsigned)((alreadyDecoded+frameSize)>>20) ); + DISPLAYUPDATE(displayLevel, "\rDecompress: %2u/%2u files. Current: %s : %.*f%s... ", + fCtx->currFileIdx+1, fCtx->nbFilesTotal, srcFileName, hrs.precision, hrs.value, hrs.suffix); } } else { - DISPLAYUPDATE(displayLevel, "\r%-20.20s : %u MB... ", - srcFileName, (unsigned)((alreadyDecoded+frameSize)>>20) ); + DISPLAYUPDATE(displayLevel, "\r%-20.20s : %.*f%s... ", + srcFileName, hrs.precision, hrs.value, hrs.suffix); } if (inBuff.pos > 0) { @@ -2402,9 +2417,11 @@ FIO_decompressLz4Frame(dRess_t* ress, FILE* srcFile, /* Write Block */ if (decodedBytes) { + UTIL_HumanReadableSize_t hrs; storedSkips = FIO_fwriteSparse(ress->dstFile, ress->dstBuffer, decodedBytes, prefs, storedSkips); filesize += decodedBytes; - DISPLAYUPDATE(2, "\rDecompressed : %u MB ", (unsigned)(filesize>>20)); + hrs = UTIL_makeHumanReadableSize(filesize); + DISPLAYUPDATE(2, "\rDecompressed : %.*f%s ", hrs.precision, hrs.value, hrs.suffix); } if (!nextToLoad) break; @@ -2512,7 +2529,7 @@ static int FIO_decompressFrames(FIO_ctx_t* const fCtx, fCtx->totalBytesOutput += (size_t)filesize; DISPLAYLEVEL(2, "\r%79s\r", ""); /* No status message in pipe mode (stdin - stdout) or multi-files mode */ - if ((g_display_prefs.displayLevel >= 2 && fCtx->nbFilesTotal <= 1) || + if ((g_display_prefs.displayLevel >= 2 && fCtx->nbFilesTotal <= 1) || g_display_prefs.displayLevel >= 3 || g_display_prefs.progressSetting == FIO_ps_always) { DISPLAYLEVEL(1, "\r%-20s: %llu bytes \n", srcFileName, filesize); @@ -2972,25 +2989,24 @@ getFileInfo(fileInfo_t* info, const char* srcFileName) static void displayInfo(const char* inFileName, const fileInfo_t* info, int displayLevel) { - unsigned const unit = info->compressedSize < (1 MB) ? (1 KB) : (1 MB); - const char* const unitStr = info->compressedSize < (1 MB) ? "KB" : "MB"; - double const windowSizeUnit = (double)info->windowSize / unit; - double const compressedSizeUnit = (double)info->compressedSize / unit; - double const decompressedSizeUnit = (double)info->decompressedSize / unit; + UTIL_HumanReadableSize_t const window_hrs = UTIL_makeHumanReadableSize(info->windowSize); + UTIL_HumanReadableSize_t const compressed_hrs = UTIL_makeHumanReadableSize(info->compressedSize); + UTIL_HumanReadableSize_t const decompressed_hrs = UTIL_makeHumanReadableSize(info->decompressedSize); double const ratio = (info->compressedSize == 0) ? 0 : ((double)info->decompressedSize)/(double)info->compressedSize; const char* const checkString = (info->usesCheck ? "XXH64" : "None"); if (displayLevel <= 2) { if (!info->decompUnavailable) { - DISPLAYOUT("%6d %5d %7.2f %2s %9.2f %2s %5.3f %5s %s\n", + DISPLAYOUT("%6d %5d %6.*f%4s %8.*f%4s %5.3f %5s %s\n", info->numSkippableFrames + info->numActualFrames, info->numSkippableFrames, - compressedSizeUnit, unitStr, decompressedSizeUnit, unitStr, + compressed_hrs.precision, compressed_hrs.value, compressed_hrs.suffix, + decompressed_hrs.precision, decompressed_hrs.value, decompressed_hrs.suffix, ratio, checkString, inFileName); } else { - DISPLAYOUT("%6d %5d %7.2f %2s %5s %s\n", + DISPLAYOUT("%6d %5d %6.*f%4s %5s %s\n", info->numSkippableFrames + info->numActualFrames, info->numSkippableFrames, - compressedSizeUnit, unitStr, + compressed_hrs.precision, compressed_hrs.value, compressed_hrs.suffix, checkString, inFileName); } } else { @@ -2998,15 +3014,15 @@ displayInfo(const char* inFileName, const fileInfo_t* info, int displayLevel) DISPLAYOUT("# Zstandard Frames: %d\n", info->numActualFrames); if (info->numSkippableFrames) DISPLAYOUT("# Skippable Frames: %d\n", info->numSkippableFrames); - DISPLAYOUT("Window Size: %.2f %2s (%llu B)\n", - windowSizeUnit, unitStr, + DISPLAYOUT("Window Size: %.*f%s (%llu B)\n", + window_hrs.precision, window_hrs.value, window_hrs.suffix, (unsigned long long)info->windowSize); - DISPLAYOUT("Compressed Size: %.2f %2s (%llu B)\n", - compressedSizeUnit, unitStr, + DISPLAYOUT("Compressed Size: %.*f%s (%llu B)\n", + compressed_hrs.precision, compressed_hrs.value, compressed_hrs.suffix, (unsigned long long)info->compressedSize); if (!info->decompUnavailable) { - DISPLAYOUT("Decompressed Size: %.2f %2s (%llu B)\n", - decompressedSizeUnit, unitStr, + DISPLAYOUT("Decompressed Size: %.*f%s (%llu B)\n", + decompressed_hrs.precision, decompressed_hrs.value, decompressed_hrs.suffix, (unsigned long long)info->decompressedSize); DISPLAYOUT("Ratio: %.4f\n", ratio); } @@ -3094,24 +3110,23 @@ int FIO_listMultipleFiles(unsigned numFiles, const char** filenameTable, int dis error |= FIO_listFile(&total, filenameTable[u], displayLevel); } } if (numFiles > 1 && displayLevel <= 2) { /* display total */ - unsigned const unit = total.compressedSize < (1 MB) ? (1 KB) : (1 MB); - const char* const unitStr = total.compressedSize < (1 MB) ? "KB" : "MB"; - double const compressedSizeUnit = (double)total.compressedSize / unit; - double const decompressedSizeUnit = (double)total.decompressedSize / unit; + UTIL_HumanReadableSize_t const compressed_hrs = UTIL_makeHumanReadableSize(total.compressedSize); + UTIL_HumanReadableSize_t const decompressed_hrs = UTIL_makeHumanReadableSize(total.decompressedSize); double const ratio = (total.compressedSize == 0) ? 0 : ((double)total.decompressedSize)/(double)total.compressedSize; const char* const checkString = (total.usesCheck ? "XXH64" : ""); DISPLAYOUT("----------------------------------------------------------------- \n"); if (total.decompUnavailable) { - DISPLAYOUT("%6d %5d %7.2f %2s %5s %u files\n", + DISPLAYOUT("%6d %5d %6.*f%4s %5s %u files\n", total.numSkippableFrames + total.numActualFrames, total.numSkippableFrames, - compressedSizeUnit, unitStr, + compressed_hrs.precision, compressed_hrs.value, compressed_hrs.suffix, checkString, (unsigned)total.nbFiles); } else { - DISPLAYOUT("%6d %5d %7.2f %2s %9.2f %2s %5.3f %5s %u files\n", + DISPLAYOUT("%6d %5d %6.*f%4s %8.*f%4s %5.3f %5s %u files\n", total.numSkippableFrames + total.numActualFrames, total.numSkippableFrames, - compressedSizeUnit, unitStr, decompressedSizeUnit, unitStr, + compressed_hrs.precision, compressed_hrs.value, compressed_hrs.suffix, + decompressed_hrs.precision, decompressed_hrs.value, decompressed_hrs.suffix, ratio, checkString, (unsigned)total.nbFiles); } } return error; diff --git a/programs/util.c b/programs/util.c index 8d190c62c..96459b9d3 100644 --- a/programs/util.c +++ b/programs/util.c @@ -303,6 +303,61 @@ U64 UTIL_getFileSizeStat(const stat_t* statbuf) return (U64)statbuf->st_size; } +UTIL_HumanReadableSize_t UTIL_makeHumanReadableSize(U64 size) { + UTIL_HumanReadableSize_t hrs; + + if (g_utilDisplayLevel > 3) { + /* In verbose mode, do not scale sizes down, except in the case of + * values that exceed the integral precision of a double. */ + if (size >= (1ull << 53)) { + hrs.value = (double)size / (1ull << 20); + hrs.suffix = " MiB"; + /* At worst, a double representation of a maximal size will be + * accurate to better than tens of kilobytes. */ + hrs.precision = 2; + } else { + hrs.value = (double)size; + hrs.suffix = " B"; + hrs.precision = 0; + } + } else { + /* In regular mode, scale sizes down and use suffixes. */ + if (size >= (1ull << 60)) { + hrs.value = (double)size / (1ull << 60); + hrs.suffix = " EiB"; + } else if (size >= (1ull << 50)) { + hrs.value = (double)size / (1ull << 50); + hrs.suffix = " PiB"; + } else if (size >= (1ull << 40)) { + hrs.value = (double)size / (1ull << 40); + hrs.suffix = " TiB"; + } else if (size >= (1ull << 30)) { + hrs.value = (double)size / (1ull << 30); + hrs.suffix = " GiB"; + } else if (size >= (1ull << 20)) { + hrs.value = (double)size / (1ull << 20); + hrs.suffix = " MiB"; + } else if (size >= (1ull << 10)) { + hrs.value = (double)size / (1ull << 10); + hrs.suffix = " KiB"; + } else { + hrs.value = (double)size; + hrs.suffix = " B"; + } + + if (hrs.value >= 100 || (U64)hrs.value == size) { + hrs.precision = 0; + } else if (hrs.value >= 10) { + hrs.precision = 1; + } else if (hrs.value > 1) { + hrs.precision = 2; + } else { + hrs.precision = 3; + } + } + + return hrs; +} U64 UTIL_getTotalFileSize(const char* const * fileNamesTable, unsigned nbFiles) { diff --git a/programs/util.h b/programs/util.h index 24cce4480..f146f3a08 100644 --- a/programs/util.h +++ b/programs/util.h @@ -122,6 +122,7 @@ int UTIL_requireUserConfirmation(const char* prompt, const char* abortMsg, const #define STRDUP(s) strdup(s) #endif + /** * Calls platform's equivalent of stat() on filename and writes info to statbuf. * Returns success (1) or failure (0). @@ -170,6 +171,19 @@ int UTIL_isFIFO(const char* infilename); U64 UTIL_getFileSize(const char* infilename); U64 UTIL_getTotalFileSize(const char* const * fileNamesTable, unsigned nbFiles); +/** + * Take a size in bytes and prepare the components to pretty-print it in a + * scaled way. The components in the returned struct should be passed in + * precision, value, suffix order to a "%.*f%s" format string. + */ +typedef struct { + double value; + int precision; + const char* suffix; +} UTIL_HumanReadableSize_t; + +UTIL_HumanReadableSize_t UTIL_makeHumanReadableSize(U64 size); + int UTIL_compareStr(const void *p1, const void *p2); const char* UTIL_getFileExtension(const char* infilename); void UTIL_mirrorSourceFilesDirectories(const char** fileNamesTable, unsigned int nbFiles, const char *outDirName); diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 9e2133c4f..32cd5af86 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -1146,8 +1146,9 @@ int main(int const argCount, const char* argv[]) (void)singleThread; (void)nbWorkers; #endif -#ifdef UTIL_HAS_CREATEFILELIST g_utilDisplayLevel = g_displayLevel; + +#ifdef UTIL_HAS_CREATEFILELIST if (!followLinks) { unsigned u, fileNamesNb; unsigned const nbFilenames = (unsigned)filenames->tableSize; diff --git a/tests/playTests.sh b/tests/playTests.sh index 252939006..54f2033cf 100755 --- a/tests/playTests.sh +++ b/tests/playTests.sh @@ -1445,7 +1445,7 @@ datagen -g0 > tmp5 zstd tmp5 zstd -l tmp5.zst zstd -l tmp5* && die "-l must fail on non-zstd file" -zstd -lv tmp5.zst | grep "Decompressed Size: 0.00 KB (0 B)" # check that 0 size is present in header +zstd -lv tmp5.zst | grep "Decompressed Size: 0 B (0 B)" # check that 0 size is present in header zstd -lv tmp5* && die "-l must fail on non-zstd file" println "\n===> zstd --list/-l test with no content size field " @@ -1616,7 +1616,7 @@ roundTripTest -g600M -P50 "1 --single-thread --long --zstd=wlog=29,clog=28" if [ -n "$hasMT" ] then - println "\n===> zstdmt long round-trip tests " + println "\n===> zstdmt long round-trip tests " roundTripTest -g80000000 -P99 "19 -T2" " " roundTripTest -g5000000000 -P99 "1 -T2" " " roundTripTest -g500000000 -P97 "1 -T999" " "