From cd7ae43da7dcf1f9c267a28d48eed872499a8d07 Mon Sep 17 00:00:00 2001 From: ddidderr Date: Sun, 12 Jul 2026 18:06:51 +0200 Subject: [PATCH] feat(cli): port utility and dictionary I/O helpers to Rust Move filename tables, file-list expansion, core-count helpers, statistics, and dictionary sample loading behind Rust implementations while retaining the narrow C ABI used by the CLI. Test Plan: - RUSTC_WRAPPER= CARGO_BUILD_RUSTC_WRAPPER= cargo test --manifest-path rust/cli/Cargo.toml - RUSTC_WRAPPER= CARGO_BUILD_RUSTC_WRAPPER= cargo clippy --manifest-path rust/cli/Cargo.toml --all-targets -- -D warnings - make -B -C programs zstd V=1 - make -C tests check V=1 --- programs/Makefile | 2 + programs/dibio.c | 432 +---------- programs/util.c | 1676 ++----------------------------------------- rust/cli/src/lib.rs | 5 + rust/src/dibio.rs | 1022 ++++++++++++++++++++++++++ rust/src/util.rs | 1501 ++++++++++++++++++++++++++++++++++++++ 6 files changed, 2580 insertions(+), 2058 deletions(-) create mode 100644 rust/src/dibio.rs create mode 100644 rust/src/util.rs diff --git a/programs/Makefile b/programs/Makefile index 248156ebc..d8696d207 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -34,6 +34,8 @@ RUST_SOURCES := $(RUST_MANIFEST) $(RUST_DIR)/Cargo.lock \ RUST_CLI_SOURCES := $(RUST_CLI_MANIFEST) $(RUST_CLI_DIR)/Cargo.lock \ $(RUST_CLI_DIR)/src/lib.rs $(RUST_DIR)/src/zstd_cli.rs \ $(RUST_DIR)/src/fileio_prefs.rs \ + $(RUST_DIR)/src/dibio.rs \ + $(RUST_DIR)/src/util.rs \ $(RUST_DIR)/src/timefn.rs $(RUST_DIR)/src/benchfn.rs \ $(RUST_DIR)/src/datagen.rs $(RUST_DIR)/src/lorem.rs diff --git a/programs/dibio.c b/programs/dibio.c index 7ba22d15b..ce071843c 100644 --- a/programs/dibio.c +++ b/programs/dibio.c @@ -8,433 +8,7 @@ * You may select, at your option, one of the above-listed licenses. */ - - -/* ************************************** -* Compiler Warnings -****************************************/ -#ifdef _MSC_VER -# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ -#endif - - -/*-************************************* -* Includes -***************************************/ -#include "platform.h" /* Large Files support */ -#include "util.h" /* UTIL_getFileSize, UTIL_getTotalFileSize */ -#include /* malloc, free */ -#include /* memset */ -#include /* fprintf, fopen, ftello64 */ -#include /* errno */ - -#include "timefn.h" /* UTIL_time_t, UTIL_clockSpanMicro, UTIL_getTime */ -#include "../lib/common/debug.h" /* assert */ -#include "../lib/common/mem.h" /* read */ -#include "../lib/zstd_errors.h" +/* The dictionary I/O implementation lives in rust/src/dibio.rs. Keep this + * translation unit in the existing C program source lists so the public + * dibio.h declaration remains the ABI boundary used by the CLI. */ #include "dibio.h" - - -/*-************************************* -* Constants -***************************************/ -#define KB *(1 <<10) -#define MB *(1 <<20) -#define GB *(1U<<30) - -#define SAMPLESIZE_MAX (128 KB) -#define MEMMULT 11 /* rough estimation : memory cost to analyze 1 byte of sample */ -#define COVER_MEMMULT 9 /* rough estimation : memory cost to analyze 1 byte of sample */ -#define FASTCOVER_MEMMULT 1 /* rough estimation : memory cost to analyze 1 byte of sample */ -static const size_t g_maxMemory = (sizeof(size_t) == 4) ? (2 GB - 64 MB) : ((size_t)(512 MB) << sizeof(size_t)); - -#define NOISELENGTH 32 -#define MAX_SAMPLES_SIZE (2 GB) /* training dataset limited to 2GB */ - - -/*-************************************* -* Console display -***************************************/ -#define DISPLAY(...) fprintf(stderr, __VA_ARGS__) -#define DISPLAYLEVEL(l, ...) if (displayLevel>=l) { DISPLAY(__VA_ARGS__); } - -static const U64 g_refreshRate = SEC_TO_MICRO / 6; -static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER; - -#define DISPLAYUPDATE(l, ...) { if (displayLevel>=l) { \ - if ((UTIL_clockSpanMicro(g_displayClock) > g_refreshRate) || (displayLevel>=4)) \ - { g_displayClock = UTIL_getTime(); DISPLAY(__VA_ARGS__); \ - if (displayLevel>=4) fflush(stderr); } } } - -/*-************************************* -* Exceptions -***************************************/ -#ifndef DEBUG -# define DEBUG 0 -#endif -#define DEBUGOUTPUT(...) if (DEBUG) DISPLAY(__VA_ARGS__); -#define EXM_THROW(error, ...) \ -{ \ - DEBUGOUTPUT("Error defined at %s, line %i : \n", __FILE__, __LINE__); \ - DISPLAY("Error %i : ", error); \ - DISPLAY(__VA_ARGS__); \ - DISPLAY("\n"); \ - exit(error); \ -} - - -/* ******************************************************** -* Helper functions -**********************************************************/ -#undef MIN -#define MIN(a,b) ((a) < (b) ? (a) : (b)) - -/** - Returns the size of a file. - If error returns -1. -*/ -static S64 DiB_getFileSize (const char * fileName) -{ - U64 const fileSize = UTIL_getFileSize(fileName); - return (fileSize == UTIL_FILESIZE_UNKNOWN) ? -1 : (S64)fileSize; -} - -/* ******************************************************** -* File related operations -**********************************************************/ -/** DiB_loadFiles() : - * load samples from files listed in fileNamesTable into buffer. - * works even if buffer is too small to load all samples. - * Also provides the size of each sample into sampleSizes table - * which must be sized correctly, using DiB_fileStats(). - * @return : nb of samples effectively loaded into `buffer` - * *bufferSizePtr is modified, it provides the amount data loaded within buffer. - * sampleSizes is filled with the size of each sample. - */ -static int DiB_loadFiles( - void* buffer, size_t* bufferSizePtr, - size_t* sampleSizes, int sstSize, - const char** fileNamesTable, int nbFiles, - size_t targetChunkSize, int displayLevel ) -{ - char* const buff = (char*)buffer; - size_t totalDataLoaded = 0; - int nbSamplesLoaded = 0; - int fileIndex = 0; - FILE * f = NULL; - - assert(targetChunkSize <= SAMPLESIZE_MAX); - - while ( nbSamplesLoaded < sstSize && fileIndex < nbFiles ) { - size_t fileDataLoaded; - S64 const fileSize = DiB_getFileSize(fileNamesTable[fileIndex]); - if (fileSize <= 0) { - /* skip if zero-size or file error */ - ++fileIndex; - continue; - } - - f = fopen( fileNamesTable[fileIndex], "rb"); - if (f == NULL) - EXM_THROW(10, "zstd: dictBuilder: %s %s ", fileNamesTable[fileIndex], strerror(errno)); - DISPLAYUPDATE(2, "Loading %s... \r", fileNamesTable[fileIndex]); - - /* Load the first chunk of data from the file */ - fileDataLoaded = targetChunkSize > 0 ? - (size_t)MIN(fileSize, (S64)targetChunkSize) : - (size_t)MIN(fileSize, SAMPLESIZE_MAX ); - if (totalDataLoaded + fileDataLoaded > *bufferSizePtr) - break; - if (fread( buff+totalDataLoaded, 1, fileDataLoaded, f ) != fileDataLoaded) - EXM_THROW(11, "Pb reading %s", fileNamesTable[fileIndex]); - sampleSizes[nbSamplesLoaded++] = fileDataLoaded; - totalDataLoaded += fileDataLoaded; - - /* If file-chunking is enabled, load the rest of the file as more samples */ - if (targetChunkSize > 0) { - while( (S64)fileDataLoaded < fileSize && nbSamplesLoaded < sstSize ) { - size_t const chunkSize = MIN((size_t)(fileSize-fileDataLoaded), targetChunkSize); - if (totalDataLoaded + chunkSize > *bufferSizePtr) /* buffer is full */ - break; - - if (fread( buff+totalDataLoaded, 1, chunkSize, f ) != chunkSize) - EXM_THROW(11, "Pb reading %s", fileNamesTable[fileIndex]); - sampleSizes[nbSamplesLoaded++] = chunkSize; - totalDataLoaded += chunkSize; - fileDataLoaded += chunkSize; - } - } - fileIndex += 1; - fclose(f); f = NULL; - } - if (f != NULL) - fclose(f); - - DISPLAYLEVEL(2, "\r%79s\r", ""); - DISPLAYLEVEL(4, "Loaded %d KB total training data, %d nb samples \n", - (int)(totalDataLoaded / (1 KB)), nbSamplesLoaded ); - *bufferSizePtr = totalDataLoaded; - return nbSamplesLoaded; -} - -#define DiB_rotl32(x,r) ((x << r) | (x >> (32 - r))) -static U32 DiB_rand(U32* src) -{ - static const U32 prime1 = 2654435761U; - static const U32 prime2 = 2246822519U; - U32 rand32 = *src; - rand32 *= prime1; - rand32 ^= prime2; - rand32 = DiB_rotl32(rand32, 13); - *src = rand32; - return rand32 >> 5; -} - -/* DiB_shuffle() : - * shuffle a table of file names in a semi-random way - * It improves dictionary quality by reducing "locality" impact, so if sample set is very large, - * it will load random elements from it, instead of just the first ones. */ -static void DiB_shuffle(const char** fileNamesTable, unsigned nbFiles) { - U32 seed = 0xFD2FB528; - unsigned i; - if (nbFiles == 0) - return; - for (i = nbFiles - 1; i > 0; --i) { - unsigned const j = DiB_rand(&seed) % (i + 1); - const char* const tmp = fileNamesTable[j]; - fileNamesTable[j] = fileNamesTable[i]; - fileNamesTable[i] = tmp; - } -} - - -/*-******************************************************** -* Dictionary training functions -**********************************************************/ -static size_t DiB_findMaxMem(unsigned long long requiredMem) -{ - size_t const step = 8 MB; - void* testmem = NULL; - - requiredMem = (((requiredMem >> 23) + 1) << 23); - requiredMem += step; - if (requiredMem > g_maxMemory) requiredMem = g_maxMemory; - - while (!testmem) { - testmem = malloc((size_t)requiredMem); - requiredMem -= step; - } - - free(testmem); - return (size_t)requiredMem; -} - - -static void DiB_fillNoise(void* buffer, size_t length) -{ - unsigned const prime1 = 2654435761U; - unsigned const prime2 = 2246822519U; - unsigned acc = prime1; - size_t p=0; - - for (p=0; p> 21); - } -} - - -static void DiB_saveDict(const char* dictFileName, - const void* buff, size_t buffSize) -{ - FILE* const f = fopen(dictFileName, "wb"); - if (f==NULL) EXM_THROW(3, "cannot open %s ", dictFileName); - - { size_t const n = fwrite(buff, 1, buffSize, f); - if (n!=buffSize) EXM_THROW(4, "%s : write error", dictFileName) } - - { size_t const n = (size_t)fclose(f); - if (n!=0) EXM_THROW(5, "%s : flush error", dictFileName) } -} - -typedef struct { - S64 totalSizeToLoad; - int nbSamples; - int oneSampleTooLarge; -} fileStats; - -/*! DiB_fileStats() : - * Given a list of files, and a chunkSize (0 == no chunk, whole files) - * provides the amount of data to be loaded and the resulting nb of samples. - * This is useful primarily for allocation purpose => sample buffer, and sample sizes table. - */ -static fileStats DiB_fileStats(const char** fileNamesTable, int nbFiles, size_t chunkSize, int displayLevel) -{ - fileStats fs; - int n; - memset(&fs, 0, sizeof(fs)); - - /* We assume that if chunking is requested, the chunk size is < SAMPLESIZE_MAX */ - assert( chunkSize <= SAMPLESIZE_MAX ); - - for (n=0; n 0) { - /* TODO: is there a minimum sample size? Can we have a 1-byte sample? */ - fs.nbSamples += (int)((fileSize + chunkSize-1) / chunkSize); - fs.totalSizeToLoad += fileSize; - } - else { - /* the case where one file is one sample */ - if (fileSize > SAMPLESIZE_MAX) { - /* flag excessively large sample files */ - fs.oneSampleTooLarge |= (fileSize > 2*SAMPLESIZE_MAX); - - /* Limit to the first SAMPLESIZE_MAX (128kB) of the file */ - DISPLAYLEVEL(3, "Sample file '%s' is too large, limiting to %d KB\n", - fileNamesTable[n], SAMPLESIZE_MAX / (1 KB)); - } - fs.nbSamples += 1; - fs.totalSizeToLoad += MIN(fileSize, SAMPLESIZE_MAX); - } - } - DISPLAYLEVEL(4, "Found training data %d files, %d KB, %d samples\n", nbFiles, (int)(fs.totalSizeToLoad / (1 KB)), fs.nbSamples); - return fs; -} - -int DiB_trainFromFiles(const char* dictFileName, size_t maxDictSize, - const char** fileNamesTable, int nbFiles, size_t chunkSize, - ZDICT_legacy_params_t* params, ZDICT_cover_params_t* coverParams, - ZDICT_fastCover_params_t* fastCoverParams, int optimize, unsigned memLimit) -{ - fileStats fs; - size_t* sampleSizes; /* vector of sample sizes. Each sample can be up to SAMPLESIZE_MAX */ - int nbSamplesLoaded; /* nb of samples effectively loaded in srcBuffer */ - size_t loadedSize; /* total data loaded in srcBuffer for all samples */ - void* srcBuffer /* contiguous buffer with training data/samples */; - void* const dictBuffer = malloc(maxDictSize); - int result = 0; - - int const displayLevel = params ? params->zParams.notificationLevel : - coverParams ? coverParams->zParams.notificationLevel : - fastCoverParams ? fastCoverParams->zParams.notificationLevel : 0; - - /* Shuffle input files before we start assessing how much sample datA to load. - The purpose of the shuffle is to pick random samples when the sample - set is larger than what we can load in memory. */ - DISPLAYLEVEL(3, "Shuffling input files\n"); - DiB_shuffle(fileNamesTable, nbFiles); - - /* Figure out how much sample data to load with how many samples */ - fs = DiB_fileStats(fileNamesTable, nbFiles, chunkSize, displayLevel); - - { - int const memMult = params ? MEMMULT : - coverParams ? COVER_MEMMULT: - FASTCOVER_MEMMULT; - size_t const maxMem = DiB_findMaxMem(fs.totalSizeToLoad * memMult) / memMult; - /* Limit the size of the training data to the free memory */ - /* Limit the size of the training data to 2GB */ - /* TODO: there is opportunity to stop DiB_fileStats() early when the data limit is reached */ - loadedSize = (size_t)MIN( MIN((S64)maxMem, fs.totalSizeToLoad), MAX_SAMPLES_SIZE ); - if (memLimit != 0) { - DISPLAYLEVEL(2, "! Warning : setting manual memory limit for dictionary training data at %u MB \n", - (unsigned)(memLimit / (1 MB))); - loadedSize = (size_t)MIN(loadedSize, memLimit); - } - srcBuffer = malloc(loadedSize+NOISELENGTH); - sampleSizes = (size_t*)malloc(fs.nbSamples * sizeof(size_t)); - } - - /* Checks */ - if ((fs.nbSamples && !sampleSizes) || (!srcBuffer) || (!dictBuffer)) - EXM_THROW(12, "not enough memory for DiB_trainFiles"); /* should not happen */ - if (fs.oneSampleTooLarge) { - DISPLAYLEVEL(2, "! Warning : some sample(s) are very large \n"); - DISPLAYLEVEL(2, "! Note that dictionary is only useful for small samples. \n"); - DISPLAYLEVEL(2, "! As a consequence, only the first %u bytes of each sample are loaded \n", SAMPLESIZE_MAX); - } - if (fs.nbSamples < 5) { - DISPLAYLEVEL(2, "! Warning : nb of samples too low for proper processing ! \n"); - DISPLAYLEVEL(2, "! Please provide _one file per sample_. \n"); - DISPLAYLEVEL(2, "! Alternatively, split files into fixed-size blocks representative of samples, with -B# \n"); - EXM_THROW(14, "nb of samples too low"); /* we now clearly forbid this case */ - } - if (fs.totalSizeToLoad < (S64)maxDictSize * 8) { - DISPLAYLEVEL(2, "! Warning : data size of samples too small for target dictionary size \n"); - DISPLAYLEVEL(2, "! Samples should be about 100x larger than target dictionary size \n"); - } - - /* init */ - if ((S64)loadedSize < fs.totalSizeToLoad) - DISPLAYLEVEL(1, "Training samples set too large (%u MB); training on %u MB only...\n", - (unsigned)(fs.totalSizeToLoad / (1 MB)), - (unsigned)(loadedSize / (1 MB))); - - /* Load input buffer */ - nbSamplesLoaded = DiB_loadFiles( - srcBuffer, &loadedSize, sampleSizes, fs.nbSamples, fileNamesTable, - nbFiles, chunkSize, displayLevel); - - { size_t dictSize = ZSTD_error_GENERIC; - if (params) { - DiB_fillNoise((char*)srcBuffer + loadedSize, NOISELENGTH); /* guard band, for end of buffer condition */ - dictSize = ZDICT_trainFromBuffer_legacy(dictBuffer, maxDictSize, - srcBuffer, sampleSizes, nbSamplesLoaded, - *params); - } else if (coverParams) { - if (optimize) { - dictSize = ZDICT_optimizeTrainFromBuffer_cover(dictBuffer, maxDictSize, - srcBuffer, sampleSizes, nbSamplesLoaded, - coverParams); - if (!ZDICT_isError(dictSize)) { - unsigned splitPercentage = (unsigned)(coverParams->splitPoint * 100); - DISPLAYLEVEL(2, "k=%u\nd=%u\nsteps=%u\nsplit=%u\n", coverParams->k, coverParams->d, - coverParams->steps, splitPercentage); - } - } else { - dictSize = ZDICT_trainFromBuffer_cover(dictBuffer, maxDictSize, srcBuffer, - sampleSizes, nbSamplesLoaded, *coverParams); - } - } else if (fastCoverParams != NULL) { - if (optimize) { - dictSize = ZDICT_optimizeTrainFromBuffer_fastCover(dictBuffer, maxDictSize, - srcBuffer, sampleSizes, nbSamplesLoaded, - fastCoverParams); - if (!ZDICT_isError(dictSize)) { - unsigned splitPercentage = (unsigned)(fastCoverParams->splitPoint * 100); - DISPLAYLEVEL(2, "k=%u\nd=%u\nf=%u\nsteps=%u\nsplit=%u\naccel=%u\n", fastCoverParams->k, - fastCoverParams->d, fastCoverParams->f, fastCoverParams->steps, splitPercentage, - fastCoverParams->accel); - } - } else { - dictSize = ZDICT_trainFromBuffer_fastCover(dictBuffer, maxDictSize, srcBuffer, - sampleSizes, nbSamplesLoaded, *fastCoverParams); - } - } else { - assert(0 /* Impossible */); - } - if (ZDICT_isError(dictSize)) { - DISPLAYLEVEL(1, "dictionary training failed : %s \n", ZDICT_getErrorName(dictSize)); /* should not happen */ - result = 1; - goto _cleanup; - } - /* save dict */ - DISPLAYLEVEL(2, "Save dictionary of size %u into file %s \n", (unsigned)dictSize, dictFileName); - DiB_saveDict(dictFileName, dictBuffer, dictSize); - } - - /* clean up */ -_cleanup: - free(srcBuffer); - free(sampleSizes); - free(dictBuffer); - return result; -} diff --git a/programs/util.c b/programs/util.c index 065a35855..2955c85f6 100644 --- a/programs/util.c +++ b/programs/util.c @@ -8,1636 +8,54 @@ * You may select, at your option, one of the above-listed licenses. */ -/*-**************************************** -* Dependencies -******************************************/ -#include "util.h" /* note : ensure that platform.h is included first ! */ -#include /* malloc, realloc, free */ -#include /* fprintf */ -#include /* clock_t, clock, CLOCKS_PER_SEC, nanosleep */ -#include -#include +/* The implementation lives in rust/src/util.rs, built into the Rust CLI + * static archive. Keep this translation unit in the original C source lists + * so util.h remains the public C ABI and every program variant still checks + * the ABI at compile time. */ -#if defined(__FreeBSD__) -#include /* __FreeBSD_version */ -#endif /* #ifdef __FreeBSD__ */ +#include "util.h" +#include +/* These assertions deliberately use the public C declarations rather than a + * Rust-private mirror. They catch accidental changes to the structures that + * cross the C/Rust boundary on every supported C build. */ +typedef char UTIL_rust_human_value_offset[ + (offsetof(UTIL_HumanReadableSize_t, value) == 0) ? 1 : -1]; +typedef char UTIL_rust_human_precision_offset[ + (offsetof(UTIL_HumanReadableSize_t, precision) == sizeof(double)) ? 1 : -1]; +typedef char UTIL_rust_human_suffix_offset[ + (offsetof(UTIL_HumanReadableSize_t, suffix) + >= offsetof(UTIL_HumanReadableSize_t, precision) + sizeof(int)) ? 1 : -1]; +typedef char UTIL_rust_human_size[ + (sizeof(UTIL_HumanReadableSize_t) + == offsetof(UTIL_HumanReadableSize_t, suffix) + sizeof(const char*)) ? 1 : -1]; + +typedef char UTIL_rust_fnt_file_names_offset[ + (offsetof(FileNamesTable, fileNames) == 0) ? 1 : -1]; +typedef char UTIL_rust_fnt_buf_offset[ + (offsetof(FileNamesTable, buf) == sizeof(const char**)) ? 1 : -1]; +typedef char UTIL_rust_fnt_size_offset[ + (offsetof(FileNamesTable, tableSize) + == offsetof(FileNamesTable, buf) + sizeof(char*)) ? 1 : -1]; +typedef char UTIL_rust_fnt_capacity_offset[ + (offsetof(FileNamesTable, tableCapacity) + == offsetof(FileNamesTable, tableSize) + sizeof(size_t)) ? 1 : -1]; +typedef char UTIL_rust_fnt_size[ + (sizeof(FileNamesTable) + == offsetof(FileNamesTable, tableCapacity) + sizeof(size_t)) ? 1 : -1]; + +/* Windows' CRT does not expose the POSIX utimensat interface. This helper is + * intentionally the only platform operation retained in this C translation + * unit; the public UTIL_utime entry point itself is exported by Rust. */ #if defined(_WIN32) -# include /* utime */ -# include /* _chmod */ -# define ZSTD_USE_UTIMENSAT 0 -#else -# include /* chown, stat */ -# include /* utimensat, st_mtime */ -# if (PLATFORM_POSIX_VERSION >= 200809L && defined(st_mtime)) \ - || (defined(__FreeBSD__) && __FreeBSD_version >= 1100056) -# define ZSTD_USE_UTIMENSAT 1 -# else -# define ZSTD_USE_UTIMENSAT 0 -# endif -# if ZSTD_USE_UTIMENSAT -# include /* AT_FDCWD */ -# else -# include /* utime */ -# endif +# include +# include + +int UTIL_rust_utime(const char* filename, const stat_t* statbuf) +{ + struct _utimbuf timebuf; + timebuf.actime = time(NULL); + timebuf.modtime = statbuf->st_mtime; + return _utime(filename, &timebuf); +} #endif - -#if defined(_MSC_VER) || defined(__MINGW32__) || defined (__MSVCRT__) -#include /* needed for _mkdir in windows */ -#endif - -#if defined(__linux__) || (PLATFORM_POSIX_VERSION >= 200112L) /* opendir, readdir require POSIX.1-2001 */ -# include /* opendir, readdir */ -# include /* strerror, memcpy */ -#endif /* #ifdef _WIN32 */ - -/*-**************************************** -* Internal Macros -******************************************/ - -/* CONTROL is almost like an assert(), but is never disabled. - * It's designed for failures that may happen rarely, - * but we don't want to maintain a specific error code path for them, - * such as a malloc() returning NULL for example. - * Since it's always active, this macro can trigger side effects. - */ -#define CONTROL(c) { \ - if (!(c)) { \ - UTIL_DISPLAYLEVEL(1, "Error : %s, %i : %s", \ - __FILE__, __LINE__, #c); \ - exit(1); \ -} } - -/* console log */ -#define UTIL_DISPLAY(...) fprintf(stderr, __VA_ARGS__) -#define UTIL_DISPLAYLEVEL(l, ...) { if (g_utilDisplayLevel>=l) { UTIL_DISPLAY(__VA_ARGS__); } } - -static int g_traceDepth = 0; -int g_traceFileStat = 0; - -#define UTIL_TRACE_CALL(...) \ - { \ - if (g_traceFileStat) { \ - UTIL_DISPLAY("Trace:FileStat: %*s> ", g_traceDepth, ""); \ - UTIL_DISPLAY(__VA_ARGS__); \ - UTIL_DISPLAY("\n"); \ - ++g_traceDepth; \ - } \ - } - -#define UTIL_TRACE_RET(ret) \ - { \ - if (g_traceFileStat) { \ - --g_traceDepth; \ - UTIL_DISPLAY("Trace:FileStat: %*s< %d\n", g_traceDepth, "", (ret)); \ - } \ - } - -/* A modified version of realloc(). - * If UTIL_realloc() fails the original block is freed. - */ -UTIL_STATIC void* UTIL_realloc(void *ptr, size_t size) -{ - void *newptr = realloc(ptr, size); - if (newptr) return newptr; - free(ptr); - return NULL; -} - -#if defined(_MSC_VER) - #define chmod _chmod -#endif - -#ifndef ZSTD_HAVE_FCHMOD -#if PLATFORM_POSIX_VERSION >= 199309L -#define ZSTD_HAVE_FCHMOD -#endif -#endif - -#ifndef ZSTD_HAVE_FCHOWN -#if PLATFORM_POSIX_VERSION >= 200809L -#define ZSTD_HAVE_FCHOWN -#endif -#endif - -/*-**************************************** -* Console log -******************************************/ -int g_utilDisplayLevel; - -int UTIL_requireUserConfirmation(const char* prompt, const char* abortMsg, - const char* acceptableLetters, int hasStdinInput) { - int ch, result; - - if (hasStdinInput) { - UTIL_DISPLAY("stdin is an input - not proceeding.\n"); - return 1; - } - - UTIL_DISPLAY("%s", prompt); - ch = getchar(); - result = 0; - if (strchr(acceptableLetters, ch) == NULL) { - UTIL_DISPLAY("%s \n", abortMsg); - result = 1; - } - /* flush the rest */ - while ((ch!=EOF) && (ch!='\n')) - ch = getchar(); - return result; -} - - -/*-************************************* -* Constants -***************************************/ -#define LIST_SIZE_INCREASE (8*1024) -#define MAX_FILE_OF_FILE_NAMES_SIZE (1<<20)*50 - - -/*-************************************* -* Functions -***************************************/ - -void UTIL_traceFileStat(void) -{ - g_traceFileStat = 1; -} - -int UTIL_fstat(const int fd, const char* filename, stat_t* statbuf) -{ - int ret; - UTIL_TRACE_CALL("UTIL_stat(%d, %s)", fd, filename); -#if defined(_MSC_VER) - if (fd >= 0) { - ret = !_fstat64(fd, statbuf); - } else { - ret = !_stat64(filename, statbuf); - } -#elif defined(__MINGW32__) && defined (__MSVCRT__) - if (fd >= 0) { - ret = !_fstati64(fd, statbuf); - } else { - ret = !_stati64(filename, statbuf); - } -#else - if (fd >= 0) { - ret = !fstat(fd, statbuf); - } else { - ret = !stat(filename, statbuf); - } -#endif - UTIL_TRACE_RET(ret); - return ret; -} - -int UTIL_stat(const char* filename, stat_t* statbuf) -{ - return UTIL_fstat(-1, filename, statbuf); -} - -int UTIL_isRegularFile(const char* infilename) -{ - stat_t statbuf; - int ret; - UTIL_TRACE_CALL("UTIL_isRegularFile(%s)", infilename); - ret = UTIL_stat(infilename, &statbuf) && UTIL_isRegularFileStat(&statbuf); - UTIL_TRACE_RET(ret); - return ret; -} - -int UTIL_isRegularFileStat(const stat_t* statbuf) -{ -#if defined(_MSC_VER) - return (statbuf->st_mode & S_IFREG) != 0; -#else - return S_ISREG(statbuf->st_mode) != 0; -#endif -} - -/* like chmod, but avoid changing permission of /dev/null */ -int UTIL_chmod(char const* filename, const stat_t* statbuf, mode_t permissions) -{ - return UTIL_fchmod(-1, filename, statbuf, permissions); -} - -int UTIL_fchmod(const int fd, char const* filename, const stat_t* statbuf, mode_t permissions) -{ - stat_t localStatBuf; - UTIL_TRACE_CALL("UTIL_chmod(%s, %#4o)", filename, (unsigned)permissions); - if (statbuf == NULL) { - if (!UTIL_fstat(fd, filename, &localStatBuf)) { - UTIL_TRACE_RET(0); - return 0; - } - statbuf = &localStatBuf; - } - if (!UTIL_isRegularFileStat(statbuf)) { - UTIL_TRACE_RET(0); - return 0; /* pretend success, but don't change anything */ - } -#ifdef ZSTD_HAVE_FCHMOD - if (fd >= 0) { - int ret; - UTIL_TRACE_CALL("fchmod"); - ret = fchmod(fd, permissions); - UTIL_TRACE_RET(ret); - UTIL_TRACE_RET(ret); - return ret; - } else -#endif - { - int ret; - UTIL_TRACE_CALL("chmod"); - ret = chmod(filename, permissions); - UTIL_TRACE_RET(ret); - UTIL_TRACE_RET(ret); - return ret; - } -} - -/* set access and modification times */ -int UTIL_utime(const char* filename, const stat_t *statbuf) -{ - int ret; - UTIL_TRACE_CALL("UTIL_utime(%s)", filename); - /* We check that st_mtime is a macro here in order to give us confidence - * that struct stat has a struct timespec st_mtim member. We need this - * check because there are some platforms that claim to be POSIX 2008 - * compliant but which do not have st_mtim... */ - /* FreeBSD has implemented POSIX 2008 for a long time but still only - * advertises support for POSIX 2001. They have a version macro that - * lets us safely gate them in. - * See https://docs.freebsd.org/en/books/porters-handbook/versions/. - */ -#if ZSTD_USE_UTIMENSAT - { - /* (atime, mtime) */ - struct timespec timebuf[2] = { {0, UTIME_NOW} }; - timebuf[1] = statbuf->st_mtim; - ret = utimensat(AT_FDCWD, filename, timebuf, 0); - } -#else - { - struct utimbuf timebuf; - timebuf.actime = time(NULL); - timebuf.modtime = statbuf->st_mtime; - ret = utime(filename, &timebuf); - } -#endif - errno = 0; - UTIL_TRACE_RET(ret); - return ret; -} - -int UTIL_setFileStat(const char *filename, const stat_t *statbuf) -{ - return UTIL_setFDStat(-1, filename, statbuf); -} - -int UTIL_setFDStat(const int fd, const char *filename, const stat_t *statbuf) -{ - int res = 0; - stat_t curStatBuf; - UTIL_TRACE_CALL("UTIL_setFileStat(%d, %s)", fd, filename); - - if (!UTIL_fstat(fd, filename, &curStatBuf) || !UTIL_isRegularFileStat(&curStatBuf)) { - UTIL_TRACE_RET(-1); - return -1; - } - - /* Mimic gzip's behavior: - * - * "Change the group first, then the permissions, then the owner. - * That way, the permissions will be correct on systems that allow - * users to give away files, without introducing a security hole. - * Security depends on permissions not containing the setuid or - * setgid bits." */ - -#if !defined(_WIN32) -#ifdef ZSTD_HAVE_FCHOWN - if (fd >= 0) { - res += fchown(fd, -1, statbuf->st_gid); /* Apply group ownership */ - } else -#endif - { - res += chown(filename, -1, statbuf->st_gid); /* Apply group ownership */ - } -#endif - - res += UTIL_fchmod(fd, filename, &curStatBuf, statbuf->st_mode & 0777); /* Copy file permissions */ - -#if !defined(_WIN32) -#ifdef ZSTD_HAVE_FCHOWN - if (fd >= 0) { - res += fchown(fd, statbuf->st_uid, -1); /* Apply user ownership */ - } else -#endif - { - res += chown(filename, statbuf->st_uid, -1); /* Apply user ownership */ - } -#endif - - errno = 0; - UTIL_TRACE_RET(-res); - return -res; /* number of errors is returned */ -} - -int UTIL_isDirectory(const char* infilename) -{ - stat_t statbuf; - int ret; - UTIL_TRACE_CALL("UTIL_isDirectory(%s)", infilename); - ret = UTIL_stat(infilename, &statbuf) && UTIL_isDirectoryStat(&statbuf); - UTIL_TRACE_RET(ret); - return ret; -} - -int UTIL_isDirectoryStat(const stat_t* statbuf) -{ - int ret; - UTIL_TRACE_CALL("UTIL_isDirectoryStat()"); -#if defined(_MSC_VER) - ret = (statbuf->st_mode & _S_IFDIR) != 0; -#else - ret = S_ISDIR(statbuf->st_mode) != 0; -#endif - UTIL_TRACE_RET(ret); - return ret; -} - -int UTIL_compareStr(const void *p1, const void *p2) { - return strcmp(* (char * const *) p1, * (char * const *) p2); -} - -int UTIL_isSameFile(const char* fName1, const char* fName2) -{ - int ret; - assert(fName1 != NULL); assert(fName2 != NULL); - UTIL_TRACE_CALL("UTIL_isSameFile(%s, %s)", fName1, fName2); -#if defined(_MSC_VER) || defined(_WIN32) - /* note : Visual does not support file identification by inode. - * inode does not work on Windows, even with a posix layer, like msys2. - * The following work-around is limited to detecting exact name repetition only, - * aka `filename` is considered different from `subdir/../filename` */ - ret = !strcmp(fName1, fName2); -#else - { stat_t file1Stat; - stat_t file2Stat; - ret = UTIL_stat(fName1, &file1Stat) - && UTIL_stat(fName2, &file2Stat) - && UTIL_isSameFileStat(fName1, fName2, &file1Stat, &file2Stat); - } -#endif - UTIL_TRACE_RET(ret); - return ret; -} - -int UTIL_isSameFileStat( - const char* fName1, const char* fName2, - const stat_t* file1Stat, const stat_t* file2Stat) -{ - int ret; - assert(fName1 != NULL); assert(fName2 != NULL); - UTIL_TRACE_CALL("UTIL_isSameFileStat(%s, %s)", fName1, fName2); -#if defined(_MSC_VER) || defined(_WIN32) - /* note : Visual does not support file identification by inode. - * inode does not work on Windows, even with a posix layer, like msys2. - * The following work-around is limited to detecting exact name repetition only, - * aka `filename` is considered different from `subdir/../filename` */ - (void)file1Stat; - (void)file2Stat; - ret = !strcmp(fName1, fName2); -#else - { - ret = (file1Stat->st_dev == file2Stat->st_dev) - && (file1Stat->st_ino == file2Stat->st_ino); - } -#endif - UTIL_TRACE_RET(ret); - return ret; -} - -/* UTIL_isFIFO : distinguish named pipes */ -int UTIL_isFIFO(const char* infilename) -{ - UTIL_TRACE_CALL("UTIL_isFIFO(%s)", infilename); -/* macro guards, as defined in : https://linux.die.net/man/2/lstat */ -#if PLATFORM_POSIX_VERSION >= 200112L - { - stat_t statbuf; - if (UTIL_stat(infilename, &statbuf) && UTIL_isFIFOStat(&statbuf)) { - UTIL_TRACE_RET(1); - return 1; - } - } -#endif - (void)infilename; - UTIL_TRACE_RET(0); - return 0; -} - -/* UTIL_isFIFO : distinguish named pipes */ -int UTIL_isFIFOStat(const stat_t* statbuf) -{ -/* macro guards, as defined in : https://linux.die.net/man/2/lstat */ -#if PLATFORM_POSIX_VERSION >= 200112L - if (S_ISFIFO(statbuf->st_mode)) return 1; -#endif - (void)statbuf; - return 0; -} - -/* UTIL_isBlockDevStat : distinguish named pipes */ -int UTIL_isBlockDevStat(const stat_t* statbuf) -{ -/* macro guards, as defined in : https://linux.die.net/man/2/lstat */ -#if PLATFORM_POSIX_VERSION >= 200112L - if (S_ISBLK(statbuf->st_mode)) return 1; -#endif - (void)statbuf; - return 0; -} - -int UTIL_isLink(const char* infilename) -{ - UTIL_TRACE_CALL("UTIL_isLink(%s)", infilename); -/* macro guards, as defined in : https://linux.die.net/man/2/lstat */ -#if PLATFORM_POSIX_VERSION >= 200112L - { - stat_t statbuf; - int const r = lstat(infilename, &statbuf); - if (!r && S_ISLNK(statbuf.st_mode)) { - UTIL_TRACE_RET(1); - return 1; - } - } -#endif - (void)infilename; - UTIL_TRACE_RET(0); - return 0; -} - -static int g_fakeStdinIsConsole = 0; -static int g_fakeStderrIsConsole = 0; -static int g_fakeStdoutIsConsole = 0; - -int UTIL_isConsole(FILE* file) -{ - int ret; - UTIL_TRACE_CALL("UTIL_isConsole(%d)", fileno(file)); - if (file == stdin && g_fakeStdinIsConsole) - ret = 1; - else if (file == stderr && g_fakeStderrIsConsole) - ret = 1; - else if (file == stdout && g_fakeStdoutIsConsole) - ret = 1; - else - ret = IS_CONSOLE(file); - UTIL_TRACE_RET(ret); - return ret; -} - -void UTIL_fakeStdinIsConsole(void) -{ - g_fakeStdinIsConsole = 1; -} -void UTIL_fakeStdoutIsConsole(void) -{ - g_fakeStdoutIsConsole = 1; -} -void UTIL_fakeStderrIsConsole(void) -{ - g_fakeStderrIsConsole = 1; -} - -U64 UTIL_getFileSize(const char* infilename) -{ - stat_t statbuf; - UTIL_TRACE_CALL("UTIL_getFileSize(%s)", infilename); - if (!UTIL_stat(infilename, &statbuf)) { - UTIL_TRACE_RET(-1); - return UTIL_FILESIZE_UNKNOWN; - } - { - U64 const size = UTIL_getFileSizeStat(&statbuf); - UTIL_TRACE_RET((int)size); - return size; - } -} - -U64 UTIL_getFileSizeStat(const stat_t* statbuf) -{ - if (!UTIL_isRegularFileStat(statbuf)) return UTIL_FILESIZE_UNKNOWN; -#if defined(_MSC_VER) - if (!(statbuf->st_mode & S_IFREG)) return UTIL_FILESIZE_UNKNOWN; -#elif defined(__MINGW32__) && defined (__MSVCRT__) - if (!(statbuf->st_mode & S_IFREG)) return UTIL_FILESIZE_UNKNOWN; -#else - if (!S_ISREG(statbuf->st_mode)) return UTIL_FILESIZE_UNKNOWN; -#endif - return (U64)statbuf->st_size; -} - -UTIL_HumanReadableSize_t UTIL_makeHumanReadableSize(U64 size) -{ - UTIL_HumanReadableSize_t hrs; - - if (g_utilDisplayLevel > 3) { - /* In verbose mode, do not scale sizes down, except in the case of - * values that exceed the integral precision of a double. */ - if (size >= (1ull << 53)) { - hrs.value = (double)size / (1ull << 20); - hrs.suffix = " MiB"; - /* At worst, a double representation of a maximal size will be - * accurate to better than tens of kilobytes. */ - hrs.precision = 2; - } else { - hrs.value = (double)size; - hrs.suffix = " B"; - hrs.precision = 0; - } - } else { - /* In regular mode, scale sizes down and use suffixes. */ - if (size >= (1ull << 60)) { - hrs.value = (double)size / (1ull << 60); - hrs.suffix = " EiB"; - } else if (size >= (1ull << 50)) { - hrs.value = (double)size / (1ull << 50); - hrs.suffix = " PiB"; - } else if (size >= (1ull << 40)) { - hrs.value = (double)size / (1ull << 40); - hrs.suffix = " TiB"; - } else if (size >= (1ull << 30)) { - hrs.value = (double)size / (1ull << 30); - hrs.suffix = " GiB"; - } else if (size >= (1ull << 20)) { - hrs.value = (double)size / (1ull << 20); - hrs.suffix = " MiB"; - } else if (size >= (1ull << 10)) { - hrs.value = (double)size / (1ull << 10); - hrs.suffix = " KiB"; - } else { - hrs.value = (double)size; - hrs.suffix = " B"; - } - - if (hrs.value >= 100 || (U64)hrs.value == size) { - hrs.precision = 0; - } else if (hrs.value >= 10) { - hrs.precision = 1; - } else if (hrs.value > 1) { - hrs.precision = 2; - } else { - hrs.precision = 3; - } - } - - return hrs; -} - -U64 UTIL_getTotalFileSize(const char* const * fileNamesTable, unsigned nbFiles) -{ - U64 total = 0; - unsigned n; - UTIL_TRACE_CALL("UTIL_getTotalFileSize(%u)", nbFiles); - for (n=0; n= 1) perror("zstd:util:readLinesFromFile"); - return -1; - } - - while ( !feof(inputFile) ) { - size_t const lineLength = readLineFromFile(buf+pos, dstCapacity-pos, inputFile); - if (lineLength == 0) break; - assert(pos + lineLength <= dstCapacity); /* '=' for inputFile not terminated with '\n' */ - pos += lineLength; - ++nbFiles; - } - - CONTROL( fclose(inputFile) == 0 ); - - return nbFiles; -} - -/*Note: buf is not freed in case function successfully created table because filesTable->fileNames[0] = buf*/ -FileNamesTable* -UTIL_createFileNamesTable_fromFileName(const char* inputFileName) -{ - size_t nbFiles = 0; - char* buf; - size_t bufSize; - stat_t statbuf; - - if (!UTIL_stat(inputFileName, &statbuf) || !UTIL_isRegularFileStat(&statbuf)) - return NULL; - - { U64 const inputFileSize = UTIL_getFileSizeStat(&statbuf); - if(inputFileSize > MAX_FILE_OF_FILE_NAMES_SIZE) - return NULL; - bufSize = (size_t)(inputFileSize + 1); /* (+1) to add '\0' at the end of last filename */ - } - - buf = (char*) malloc(bufSize); - CONTROL( buf != NULL ); - - { int const ret_nbFiles = readLinesFromFile(buf, bufSize, inputFileName); - - if (ret_nbFiles <= 0) { - free(buf); - return NULL; - } - nbFiles = (size_t)ret_nbFiles; - } - - { const char** filenamesTable = (const char**) malloc(nbFiles * sizeof(*filenamesTable)); - CONTROL(filenamesTable != NULL); - - { size_t fnb, pos = 0; - for (fnb = 0; fnb < nbFiles; fnb++) { - filenamesTable[fnb] = buf+pos; - pos += strlen(buf+pos)+1; /* +1 for the finishing `\0` */ - } - assert(pos <= bufSize); - } - - return UTIL_assembleFileNamesTable(filenamesTable, nbFiles, buf); - } -} - -static FileNamesTable* -UTIL_assembleFileNamesTable2(const char** filenames, size_t tableSize, size_t tableCapacity, char* buf) -{ - FileNamesTable* const table = (FileNamesTable*) malloc(sizeof(*table)); - CONTROL(table != NULL); - table->fileNames = filenames; - table->buf = buf; - table->tableSize = tableSize; - table->tableCapacity = tableCapacity; - return table; -} - -FileNamesTable* -UTIL_assembleFileNamesTable(const char** filenames, size_t tableSize, char* buf) -{ - return UTIL_assembleFileNamesTable2(filenames, tableSize, tableSize, buf); -} - -void UTIL_freeFileNamesTable(FileNamesTable* table) -{ - if (table==NULL) return; - free((void*)table->fileNames); - free(table->buf); - free(table); -} - -FileNamesTable* UTIL_allocateFileNamesTable(size_t tableSize) -{ - const char** const fnTable = (const char**)malloc(tableSize * sizeof(*fnTable)); - FileNamesTable* fnt; - if (fnTable==NULL) return NULL; - fnt = UTIL_assembleFileNamesTable(fnTable, tableSize, NULL); - fnt->tableSize = 0; /* the table is empty */ - return fnt; -} - -int UTIL_searchFileNamesTable(FileNamesTable* table, char const* name) { - size_t i; - for(i=0 ;i < table->tableSize; i++) { - if(!strcmp(table->fileNames[i], name)) { - return (int)i; - } - } - return -1; -} - -void UTIL_refFilename(FileNamesTable* fnt, const char* filename) -{ - assert(fnt->tableSize < fnt->tableCapacity); - fnt->fileNames[fnt->tableSize] = filename; - fnt->tableSize++; -} - -static size_t getTotalTableSize(FileNamesTable* table) -{ - size_t fnb, totalSize = 0; - for(fnb = 0 ; fnb < table->tableSize && table->fileNames[fnb] ; ++fnb) { - totalSize += strlen(table->fileNames[fnb]) + 1; /* +1 to add '\0' at the end of each fileName */ - } - return totalSize; -} - -FileNamesTable* -UTIL_mergeFileNamesTable(FileNamesTable* table1, FileNamesTable* table2) -{ - unsigned newTableIdx = 0; - size_t pos = 0; - size_t newTotalTableSize; - char* buf; - - FileNamesTable* const newTable = UTIL_assembleFileNamesTable(NULL, 0, NULL); - CONTROL( newTable != NULL ); - - newTotalTableSize = getTotalTableSize(table1) + getTotalTableSize(table2); - - buf = (char*) calloc(newTotalTableSize, sizeof(*buf)); - CONTROL ( buf != NULL ); - - newTable->buf = buf; - newTable->tableSize = table1->tableSize + table2->tableSize; - newTable->fileNames = (const char **) calloc(newTable->tableSize, sizeof(*(newTable->fileNames))); - CONTROL ( newTable->fileNames != NULL ); - - { unsigned idx1; - for( idx1=0 ; (idx1 < table1->tableSize) && table1->fileNames[idx1] && (pos < newTotalTableSize); ++idx1, ++newTableIdx) { - size_t const curLen = strlen(table1->fileNames[idx1]); - memcpy(buf+pos, table1->fileNames[idx1], curLen); - assert(newTableIdx <= newTable->tableSize); - newTable->fileNames[newTableIdx] = buf+pos; - pos += curLen+1; - } } - - { unsigned idx2; - for( idx2=0 ; (idx2 < table2->tableSize) && table2->fileNames[idx2] && (pos < newTotalTableSize) ; ++idx2, ++newTableIdx) { - size_t const curLen = strlen(table2->fileNames[idx2]); - memcpy(buf+pos, table2->fileNames[idx2], curLen); - assert(newTableIdx < newTable->tableSize); - newTable->fileNames[newTableIdx] = buf+pos; - pos += curLen+1; - } } - assert(pos <= newTotalTableSize); - newTable->tableSize = newTableIdx; - - UTIL_freeFileNamesTable(table1); - UTIL_freeFileNamesTable(table2); - - return newTable; -} - -#ifdef _WIN32 -static int UTIL_prepareFileList(const char* dirName, - char** bufStart, size_t* pos, - char** bufEnd, int followLinks) -{ - char* path; - size_t dirLength, pathLength; - int nbFiles = 0; - WIN32_FIND_DATAA cFile; - HANDLE hFile; - - dirLength = strlen(dirName); - path = (char*) malloc(dirLength + 3); - if (!path) return 0; - - memcpy(path, dirName, dirLength); - path[dirLength] = '\\'; - path[dirLength+1] = '*'; - path[dirLength+2] = 0; - - hFile=FindFirstFileA(path, &cFile); - if (hFile == INVALID_HANDLE_VALUE) { - UTIL_DISPLAYLEVEL(1, "Cannot open directory '%s'\n", dirName); - return 0; - } - free(path); - - do { - size_t const fnameLength = strlen(cFile.cFileName); - path = (char*) malloc(dirLength + fnameLength + 2); - if (!path) { FindClose(hFile); return 0; } - memcpy(path, dirName, dirLength); - path[dirLength] = '\\'; - memcpy(path+dirLength+1, cFile.cFileName, fnameLength); - pathLength = dirLength+1+fnameLength; - path[pathLength] = 0; - if (cFile.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) { - if ( strcmp (cFile.cFileName, "..") == 0 - || strcmp (cFile.cFileName, ".") == 0 ) - continue; - /* Recursively call "UTIL_prepareFileList" with the new path. */ - nbFiles += UTIL_prepareFileList(path, bufStart, pos, bufEnd, followLinks); - if (*bufStart == NULL) { free(path); FindClose(hFile); return 0; } - } else if ( (cFile.dwFileAttributes & FILE_ATTRIBUTE_NORMAL) - || (cFile.dwFileAttributes & FILE_ATTRIBUTE_ARCHIVE) - || (cFile.dwFileAttributes & FILE_ATTRIBUTE_COMPRESSED) ) { - if (*bufStart + *pos + pathLength >= *bufEnd) { - ptrdiff_t const newListSize = (*bufEnd - *bufStart) + LIST_SIZE_INCREASE; - *bufStart = (char*)UTIL_realloc(*bufStart, newListSize); - if (*bufStart == NULL) { free(path); FindClose(hFile); return 0; } - *bufEnd = *bufStart + newListSize; - } - if (*bufStart + *pos + pathLength < *bufEnd) { - memcpy(*bufStart + *pos, path, pathLength+1 /* include final \0 */); - *pos += pathLength + 1; - nbFiles++; - } } - free(path); - } while (FindNextFileA(hFile, &cFile)); - - FindClose(hFile); - return nbFiles; -} - -#elif defined(__linux__) || (PLATFORM_POSIX_VERSION >= 200112L) /* opendir, readdir require POSIX.1-2001 */ - -static int UTIL_prepareFileList(const char *dirName, - char** bufStart, size_t* pos, - char** bufEnd, int followLinks) -{ - DIR* dir; - struct dirent * entry; - size_t dirLength; - int nbFiles = 0; - - if (!(dir = opendir(dirName))) { - UTIL_DISPLAYLEVEL(1, "Cannot open directory '%s': %s\n", dirName, strerror(errno)); - return 0; - } - - dirLength = strlen(dirName); - errno = 0; - while ((entry = readdir(dir)) != NULL) { - char* path; - size_t fnameLength, pathLength; - if (strcmp (entry->d_name, "..") == 0 || - strcmp (entry->d_name, ".") == 0) continue; - fnameLength = strlen(entry->d_name); - path = (char*) malloc(dirLength + fnameLength + 2); - if (!path) { closedir(dir); return 0; } - memcpy(path, dirName, dirLength); - - path[dirLength] = '/'; - memcpy(path+dirLength+1, entry->d_name, fnameLength); - pathLength = dirLength+1+fnameLength; - path[pathLength] = 0; - - if (!followLinks && UTIL_isLink(path)) { - UTIL_DISPLAYLEVEL(2, "Warning : %s is a symbolic link, ignoring\n", path); - free(path); - continue; - } - - if (UTIL_isDirectory(path)) { - nbFiles += UTIL_prepareFileList(path, bufStart, pos, bufEnd, followLinks); /* Recursively call "UTIL_prepareFileList" with the new path. */ - if (*bufStart == NULL) { free(path); closedir(dir); return 0; } - } else { - if (*bufStart + *pos + pathLength >= *bufEnd) { - ptrdiff_t newListSize = (*bufEnd - *bufStart) + LIST_SIZE_INCREASE; - assert(newListSize >= 0); - *bufStart = (char*)UTIL_realloc(*bufStart, (size_t)newListSize); - if (*bufStart != NULL) { - *bufEnd = *bufStart + newListSize; - } else { - free(path); closedir(dir); return 0; - } - } - if (*bufStart + *pos + pathLength < *bufEnd) { - memcpy(*bufStart + *pos, path, pathLength + 1); /* with final \0 */ - *pos += pathLength + 1; - nbFiles++; - } } - free(path); - errno = 0; /* clear errno after UTIL_isDirectory, UTIL_prepareFileList */ - } - - if (errno != 0) { - UTIL_DISPLAYLEVEL(1, "readdir(%s) error: %s \n", dirName, strerror(errno)); - free(*bufStart); - *bufStart = NULL; - } - closedir(dir); - return nbFiles; -} - -#else - -static int UTIL_prepareFileList(const char *dirName, - char** bufStart, size_t* pos, - char** bufEnd, int followLinks) -{ - (void)bufStart; (void)bufEnd; (void)pos; (void)followLinks; - UTIL_DISPLAYLEVEL(1, "Directory %s ignored (compiled without _WIN32 or _POSIX_C_SOURCE) \n", dirName); - return 0; -} - -#endif /* #ifdef _WIN32 */ - -int UTIL_isCompressedFile(const char *inputName, const char *extensionList[]) -{ - const char* ext = UTIL_getFileExtension(inputName); - while(*extensionList!=NULL) - { - const int isCompressedExtension = strcmp(ext,*extensionList); - if(isCompressedExtension==0) - return 1; - ++extensionList; - } - return 0; -} - -/*Utility function to get file extension from file */ -const char* UTIL_getFileExtension(const char* infilename) -{ - const char* extension = strrchr(infilename, '.'); - if(!extension || extension==infilename) return ""; - return extension; -} - -static int pathnameHas2Dots(const char *pathname) -{ - /* We need to figure out whether any ".." present in the path is a whole - * path token, which is the case if it is bordered on both sides by either - * the beginning/end of the path or by a directory separator. - */ - const char *needle = pathname; - while (1) { - needle = strstr(needle, ".."); - - if (needle == NULL) { - return 0; - } - - if ((needle == pathname || needle[-1] == PATH_SEP) - && (needle[2] == '\0' || needle[2] == PATH_SEP)) { - return 1; - } - - /* increment so we search for the next match */ - needle++; - }; - return 0; -} - -static int isFileNameValidForMirroredOutput(const char *filename) -{ - return !pathnameHas2Dots(filename); -} - - -#define DIR_DEFAULT_MODE 0755 -static mode_t getDirMode(const char *dirName) -{ - stat_t st; - if (!UTIL_stat(dirName, &st)) { - UTIL_DISPLAY("zstd: failed to get DIR stats %s: %s\n", dirName, strerror(errno)); - return DIR_DEFAULT_MODE; - } - if (!UTIL_isDirectoryStat(&st)) { - UTIL_DISPLAY("zstd: expected directory: %s\n", dirName); - return DIR_DEFAULT_MODE; - } - return st.st_mode; -} - -static int makeDir(const char *dir, mode_t mode) -{ -#if defined(_MSC_VER) || defined(__MINGW32__) || defined (__MSVCRT__) - int ret = _mkdir(dir); - (void) mode; -#else - int ret = mkdir(dir, mode); -#endif - if (ret != 0) { - if (errno == EEXIST) - return 0; - UTIL_DISPLAY("zstd: failed to create DIR %s: %s\n", dir, strerror(errno)); - } - return ret; -} - -/* this function requires a mutable input string */ -static void convertPathnameToDirName(char *pathname) -{ - size_t len = 0; - char* pos = NULL; - /* get dir name from pathname similar to 'dirname()' */ - assert(pathname != NULL); - - /* remove trailing '/' chars */ - len = strlen(pathname); - assert(len > 0); - while (pathname[len] == PATH_SEP) { - pathname[len] = '\0'; - len--; - } - if (len == 0) return; - - /* if input is a single file, return '.' instead. i.e. - * "xyz/abc/file.txt" => "xyz/abc" - "./file.txt" => "." - "file.txt" => "." - */ - pos = strrchr(pathname, PATH_SEP); - if (pos == NULL) { - pathname[0] = '.'; - pathname[1] = '\0'; - } else { - *pos = '\0'; - } -} - -/* pathname must be valid */ -static const char* trimLeadingRootChar(const char *pathname) -{ - assert(pathname != NULL); - if (pathname[0] == PATH_SEP) - return pathname + 1; - return pathname; -} - -/* pathname must be valid */ -static const char* trimLeadingCurrentDirConst(const char *pathname) -{ - assert(pathname != NULL); - if ((pathname[0] == '.') && (pathname[1] == PATH_SEP)) - return pathname + 2; - return pathname; -} - -static char* -trimLeadingCurrentDir(char *pathname) -{ - /* 'union charunion' can do const-cast without compiler warning */ - union charunion { - char *chr; - const char* cchr; - } ptr; - ptr.cchr = trimLeadingCurrentDirConst(pathname); - return ptr.chr; -} - -/* remove leading './' or '/' chars here */ -static const char * trimPath(const char *pathname) -{ - return trimLeadingRootChar( - trimLeadingCurrentDirConst(pathname)); -} - -static char* mallocAndJoin2Dir(const char *dir1, const char *dir2) -{ - assert(dir1 != NULL && dir2 != NULL); - { const size_t dir1Size = strlen(dir1); - const size_t dir2Size = strlen(dir2); - char *outDirBuffer, *buffer; - - outDirBuffer = (char *) malloc(dir1Size + dir2Size + 2); - CONTROL(outDirBuffer != NULL); - - memcpy(outDirBuffer, dir1, dir1Size); - outDirBuffer[dir1Size] = '\0'; - - buffer = outDirBuffer + dir1Size; - if (dir1Size > 0 && *(buffer - 1) != PATH_SEP) { - *buffer = PATH_SEP; - buffer++; - } - memcpy(buffer, dir2, dir2Size); - buffer[dir2Size] = '\0'; - - return outDirBuffer; - } -} - -/* this function will return NULL if input srcFileName is not valid name for mirrored output path */ -char* UTIL_createMirroredDestDirName(const char* srcFileName, const char* outDirRootName) -{ - char* pathname = NULL; - if (!isFileNameValidForMirroredOutput(srcFileName)) - return NULL; - - pathname = mallocAndJoin2Dir(outDirRootName, trimPath(srcFileName)); - - convertPathnameToDirName(pathname); - return pathname; -} - -static int -mirrorSrcDir(char* srcDirName, const char* outDirName) -{ - mode_t srcMode; - int status = 0; - char* newDir = mallocAndJoin2Dir(outDirName, trimPath(srcDirName)); - if (!newDir) - return -ENOMEM; - - srcMode = getDirMode(srcDirName); - status = makeDir(newDir, srcMode); - free(newDir); - return status; -} - -static int -mirrorSrcDirRecursive(char* srcDirName, const char* outDirName) -{ - int status = 0; - char* pp = trimLeadingCurrentDir(srcDirName); - char* sp = NULL; - - while ((sp = strchr(pp, PATH_SEP)) != NULL) { - if (sp != pp) { - *sp = '\0'; - status = mirrorSrcDir(srcDirName, outDirName); - if (status != 0) - return status; - *sp = PATH_SEP; - } - pp = sp + 1; - } - status = mirrorSrcDir(srcDirName, outDirName); - return status; -} - -static void -makeMirroredDestDirsWithSameSrcDirMode(char** srcDirNames, unsigned nbFile, const char* outDirName) -{ - unsigned int i = 0; - for (i = 0; i < nbFile; i++) - mirrorSrcDirRecursive(srcDirNames[i], outDirName); -} - -static int -firstIsParentOrSameDirOfSecond(const char* firstDir, const char* secondDir) -{ - size_t firstDirLen = strlen(firstDir), - secondDirLen = strlen(secondDir); - return firstDirLen <= secondDirLen && - (secondDir[firstDirLen] == PATH_SEP || secondDir[firstDirLen] == '\0') && - 0 == strncmp(firstDir, secondDir, firstDirLen); -} - -static int compareDir(const void* pathname1, const void* pathname2) { - /* sort it after remove the leading '/' or './'*/ - const char* s1 = trimPath(*(char * const *) pathname1); - const char* s2 = trimPath(*(char * const *) pathname2); - return strcmp(s1, s2); -} - -static void -makeUniqueMirroredDestDirs(char** srcDirNames, unsigned nbFile, const char* outDirName) -{ - unsigned int i = 0, uniqueDirNr = 0; - char** uniqueDirNames = NULL; - - if (nbFile == 0) - return; - - uniqueDirNames = (char** ) malloc(nbFile * sizeof (char *)); - CONTROL(uniqueDirNames != NULL); - - /* if dirs is "a/b/c" and "a/b/c/d", we only need call: - * we just need "a/b/c/d" */ - qsort((void *)srcDirNames, nbFile, sizeof(char*), compareDir); - - uniqueDirNr = 1; - uniqueDirNames[uniqueDirNr - 1] = srcDirNames[0]; - for (i = 1; i < nbFile; i++) { - char* prevDirName = srcDirNames[i - 1]; - char* currDirName = srcDirNames[i]; - - /* note: we always compare trimmed path, i.e.: - * src dir of "./foo" and "/foo" will be both saved into: - * "outDirName/foo/" */ - if (!firstIsParentOrSameDirOfSecond(trimPath(prevDirName), - trimPath(currDirName))) - uniqueDirNr++; - - /* we need to maintain original src dir name instead of trimmed - * dir, so we can retrieve the original src dir's mode_t */ - uniqueDirNames[uniqueDirNr - 1] = currDirName; - } - - makeMirroredDestDirsWithSameSrcDirMode(uniqueDirNames, uniqueDirNr, outDirName); - - free(uniqueDirNames); -} - -static void -makeMirroredDestDirs(char** srcFileNames, unsigned nbFile, const char* outDirName) -{ - unsigned int i = 0; - for (i = 0; i < nbFile; ++i) - convertPathnameToDirName(srcFileNames[i]); - makeUniqueMirroredDestDirs(srcFileNames, nbFile, outDirName); -} - -void UTIL_mirrorSourceFilesDirectories(const char** inFileNames, unsigned int nbFile, const char* outDirName) -{ - unsigned int i = 0, validFilenamesNr = 0; - char** srcFileNames = (char **) malloc(nbFile * sizeof (char *)); - CONTROL(srcFileNames != NULL); - - /* check input filenames is valid */ - for (i = 0; i < nbFile; ++i) { - if (isFileNameValidForMirroredOutput(inFileNames[i])) { - char* fname = STRDUP(inFileNames[i]); - CONTROL(fname != NULL); - srcFileNames[validFilenamesNr++] = fname; - } - } - - if (validFilenamesNr > 0) { - makeDir(outDirName, DIR_DEFAULT_MODE); - makeMirroredDestDirs(srcFileNames, validFilenamesNr, outDirName); - } - - for (i = 0; i < validFilenamesNr; i++) - free(srcFileNames[i]); - free(srcFileNames); -} - -FileNamesTable* -UTIL_createExpandedFNT(const char* const* inputNames, size_t nbIfns, int followLinks) -{ - unsigned nbFiles; - char* buf = (char*)malloc(LIST_SIZE_INCREASE); - char* bufend = buf + LIST_SIZE_INCREASE; - - if (!buf) return NULL; - - { size_t ifnNb, pos; - for (ifnNb=0, pos=0, nbFiles=0; ifnNb= bufend) { - ptrdiff_t newListSize = (bufend - buf) + LIST_SIZE_INCREASE; - assert(newListSize >= 0); - buf = (char*)UTIL_realloc(buf, (size_t)newListSize); - if (!buf) return NULL; - bufend = buf + newListSize; - } - if (buf + pos + len < bufend) { - memcpy(buf+pos, inputNames[ifnNb], len+1); /* including final \0 */ - pos += len + 1; - nbFiles++; - } - } else { - nbFiles += (unsigned)UTIL_prepareFileList(inputNames[ifnNb], &buf, &pos, &bufend, followLinks); - if (buf == NULL) return NULL; - } } } - - /* note : even if nbFiles==0, function returns a valid, though empty, FileNamesTable* object */ - - { size_t ifnNb, pos; - size_t const fntCapacity = nbFiles + 1; /* minimum 1, allows adding one reference, typically stdin */ - const char** const fileNamesTable = (const char**)malloc(fntCapacity * sizeof(*fileNamesTable)); - if (!fileNamesTable) { free(buf); return NULL; } - - for (ifnNb = 0, pos = 0; ifnNb < nbFiles; ifnNb++) { - fileNamesTable[ifnNb] = buf + pos; - if (buf + pos > bufend) { free(buf); free((void*)fileNamesTable); return NULL; } - pos += strlen(fileNamesTable[ifnNb]) + 1; - } - return UTIL_assembleFileNamesTable2(fileNamesTable, nbFiles, fntCapacity, buf); - } -} - - -void UTIL_expandFNT(FileNamesTable** fnt, int followLinks) -{ - FileNamesTable* const newFNT = UTIL_createExpandedFNT((*fnt)->fileNames, (*fnt)->tableSize, followLinks); - CONTROL(newFNT != NULL); - UTIL_freeFileNamesTable(*fnt); - *fnt = newFNT; -} - -FileNamesTable* UTIL_createFNT_fromROTable(const char** filenames, size_t nbFilenames) -{ - size_t const sizeof_FNTable = nbFilenames * sizeof(*filenames); - const char** const newFNTable = (const char**)malloc(sizeof_FNTable); - if (newFNTable==NULL) return NULL; - memcpy((void*)newFNTable, filenames, sizeof_FNTable); /* void* : mitigate a Visual compiler bug or limitation */ - return UTIL_assembleFileNamesTable(newFNTable, nbFilenames, NULL); -} - - -/*-**************************************** -* count the number of cores -******************************************/ - -#if defined(_WIN32) || defined(WIN32) - -#include - -typedef BOOL(WINAPI* LPFN_GLPI)(PSYSTEM_LOGICAL_PROCESSOR_INFORMATION, PDWORD); - -DWORD CountSetBits(ULONG_PTR bitMask) -{ - DWORD LSHIFT = sizeof(ULONG_PTR)*8 - 1; - DWORD bitSetCount = 0; - ULONG_PTR bitTest = (ULONG_PTR)1 << LSHIFT; - DWORD i; - - for (i = 0; i <= LSHIFT; ++i) - { - bitSetCount += ((bitMask & bitTest)?1:0); - bitTest/=2; - } - - return bitSetCount; -} - -int UTIL_countCores(int logical) -{ - static int numCores = 0; - if (numCores != 0) return numCores; - - { LPFN_GLPI glpi; - BOOL done = FALSE; - PSYSTEM_LOGICAL_PROCESSOR_INFORMATION buffer = NULL; - PSYSTEM_LOGICAL_PROCESSOR_INFORMATION ptr = NULL; - DWORD returnLength = 0; - size_t byteOffset = 0; - -#if defined(_MSC_VER) -/* Visual Studio does not like the following cast */ -# pragma warning( disable : 4054 ) /* conversion from function ptr to data ptr */ -# pragma warning( disable : 4055 ) /* conversion from data ptr to function ptr */ -#endif - glpi = (LPFN_GLPI)(void*)GetProcAddress(GetModuleHandle(TEXT("kernel32")), - "GetLogicalProcessorInformation"); - - if (glpi == NULL) { - goto failed; - } - - while(!done) { - DWORD rc = glpi(buffer, &returnLength); - if (FALSE == rc) { - if (GetLastError() == ERROR_INSUFFICIENT_BUFFER) { - if (buffer) - free(buffer); - buffer = (PSYSTEM_LOGICAL_PROCESSOR_INFORMATION)malloc(returnLength); - - if (buffer == NULL) { - perror("zstd"); - exit(1); - } - } else { - /* some other error */ - goto failed; - } - } else { - done = TRUE; - } } - - ptr = buffer; - - while (byteOffset + sizeof(SYSTEM_LOGICAL_PROCESSOR_INFORMATION) <= returnLength) { - - if (ptr->Relationship == RelationProcessorCore) { - if (logical) - numCores += CountSetBits(ptr->ProcessorMask); - else - numCores++; - } - - ptr++; - byteOffset += sizeof(SYSTEM_LOGICAL_PROCESSOR_INFORMATION); - } - - free(buffer); - - return numCores; - } - -failed: - /* try to fall back on GetSystemInfo */ - { SYSTEM_INFO sysinfo; - GetSystemInfo(&sysinfo); - numCores = sysinfo.dwNumberOfProcessors; - if (numCores == 0) numCores = 1; /* just in case */ - } - return numCores; -} - -#elif defined(__APPLE__) - -#include - -/* Use apple-provided syscall - * see: man 3 sysctl */ -int UTIL_countCores(int logical) -{ - static S32 numCores = 0; /* apple specifies int32_t */ - if (numCores != 0) return numCores; - - { size_t size = sizeof(S32); - int const ret = sysctlbyname(logical ? "hw.logicalcpu" : "hw.physicalcpu", &numCores, &size, NULL, 0); - if (ret != 0) { - if (errno == ENOENT) { - /* entry not present, fall back on 1 */ - numCores = 1; - } else { - perror("zstd: can't get number of cpus"); - exit(1); - } - } - - return numCores; - } -} - -#elif defined(__linux__) - -/* parse /proc/cpuinfo - * siblings / cpu cores should give hyperthreading ratio - * otherwise fall back on sysconf */ -int UTIL_countCores(int logical) -{ - static int numCores = 0; - - if (numCores != 0) return numCores; - - numCores = (int)sysconf(_SC_NPROCESSORS_ONLN); - if (numCores == -1) { - /* value not queryable, fall back on 1 */ - return numCores = 1; - } - - /* try to determine if there's hyperthreading */ - { FILE* const cpuinfo = fopen("/proc/cpuinfo", "r"); -#define BUF_SIZE 80 - char buff[BUF_SIZE]; - - int siblings = 0; - int cpu_cores = 0; - int ratio = 1; - - if (cpuinfo == NULL) { - /* fall back on the sysconf value */ - return numCores; - } - - /* assume the cpu cores/siblings values will be constant across all - * present processors */ - while (!feof(cpuinfo)) { - if (fgets(buff, BUF_SIZE, cpuinfo) != NULL) { - if (strncmp(buff, "siblings", 8) == 0) { - const char* const sep = strchr(buff, ':'); - if (sep == NULL || *sep == '\0') { - /* formatting was broken? */ - goto failed; - } - - siblings = atoi(sep + 1); - } - if (strncmp(buff, "cpu cores", 9) == 0) { - const char* const sep = strchr(buff, ':'); - if (sep == NULL || *sep == '\0') { - /* formatting was broken? */ - goto failed; - } - - cpu_cores = atoi(sep + 1); - } - } else if (ferror(cpuinfo)) { - /* fall back on the sysconf value */ - goto failed; - } } - if (siblings && cpu_cores && siblings > cpu_cores) { - ratio = siblings / cpu_cores; - } - - if (ratio && numCores > ratio && !logical) { - numCores = numCores / ratio; - } - -failed: - fclose(cpuinfo); - return numCores; - } -} - -#elif defined(__FreeBSD__) - -#include - -/* Use physical core sysctl when available - * see: man 4 smp, man 3 sysctl */ -int UTIL_countCores(int logical) -{ - static int numCores = 0; /* freebsd sysctl is native int sized */ -#if __FreeBSD_version >= 1300008 - static int perCore = 1; -#endif - if (numCores != 0) return numCores; - -#if __FreeBSD_version >= 1300008 - { size_t size = sizeof(numCores); - int ret = sysctlbyname("kern.smp.cores", &numCores, &size, NULL, 0); - if (ret == 0) { - if (logical) { - ret = sysctlbyname("kern.smp.threads_per_core", &perCore, &size, NULL, 0); - /* default to physical cores if logical cannot be read */ - if (ret == 0) - numCores *= perCore; - } - - return numCores; - } - if (errno != ENOENT) { - perror("zstd: can't get number of cpus"); - exit(1); - } - /* sysctl not present, fall through to older sysconf method */ - } -#else - /* suppress unused parameter warning */ - (void) logical; -#endif - - numCores = (int)sysconf(_SC_NPROCESSORS_ONLN); - if (numCores == -1) { - /* value not queryable, fall back on 1 */ - numCores = 1; - } - return numCores; -} - -#elif defined(__NetBSD__) || defined(__OpenBSD__) || defined(__DragonFly__) || defined(__CYGWIN__) - -/* Use POSIX sysconf - * see: man 3 sysconf */ -int UTIL_countCores(int logical) -{ - static int numCores = 0; - - /* suppress unused parameter warning */ - (void)logical; - - if (numCores != 0) return numCores; - - numCores = (int)sysconf(_SC_NPROCESSORS_ONLN); - if (numCores == -1) { - /* value not queryable, fall back on 1 */ - return numCores = 1; - } - return numCores; -} - -#else - -int UTIL_countCores(int logical) -{ - /* suppress unused parameter warning */ - (void)logical; - - /* assume 1 */ - return 1; -} - -#endif - -int UTIL_countPhysicalCores(void) -{ - return UTIL_countCores(0); -} - -int UTIL_countLogicalCores(void) -{ - return UTIL_countCores(1); -} diff --git a/rust/cli/src/lib.rs b/rust/cli/src/lib.rs index 90bbcc752..8e3fd53e5 100644 --- a/rust/cli/src/lib.rs +++ b/rust/cli/src/lib.rs @@ -3,12 +3,17 @@ mod benchfn; #[path = "../../src/datagen.rs"] mod datagen; #[cfg(feature = "cli")] +#[path = "../../src/dibio.rs"] +mod dibio; +#[cfg(feature = "cli")] #[path = "../../src/fileio_prefs.rs"] mod fileio_prefs; #[path = "../../src/lorem.rs"] mod lorem; #[path = "../../src/timefn.rs"] mod timefn; +#[path = "../../src/util.rs"] +mod util; #[cfg(feature = "cli")] #[path = "../../src/zstd_cli.rs"] mod zstd_cli; diff --git a/rust/src/dibio.rs b/rust/src/dibio.rs new file mode 100644 index 000000000..37855da99 --- /dev/null +++ b/rust/src/dibio.rs @@ -0,0 +1,1022 @@ +#![allow(non_camel_case_types)] +#![allow(non_snake_case)] +#![allow(clippy::missing_safety_doc)] +#![allow(clippy::too_many_arguments)] + +//! Dictionary training I/O for the command-line interface. +//! +//! The public entry point keeps the `programs/dibio.h` ABI. File discovery, +//! sample selection, buffering, and dictionary output are Rust-owned; the +//! actual legacy, COVER, and fastCOVER trainers are reached through their +//! existing C ABI symbols. + +use std::cmp::min; +use std::ffi::{c_char, c_void, CStr}; +use std::fmt; +use std::fs; +use std::io::{self, Write}; +use std::mem::size_of; +use std::os::raw::{c_int, c_uint}; +use std::path::PathBuf; +use std::time::{Duration, Instant}; + +#[cfg(unix)] +use std::ffi::OsStr; +#[cfg(unix)] +use std::os::unix::ffi::OsStrExt; + +const KB: usize = 1 << 10; +const MB: usize = 1 << 20; +const GB: usize = 1 << 30; + +const SAMPLESIZE_MAX: usize = 128 * KB; +const MEMMULT: u64 = 11; +const COVER_MEMMULT: u64 = 9; +const FASTCOVER_MEMMULT: u64 = 1; +const NOISELENGTH: usize = 32; +const MAX_SAMPLES_SIZE: usize = 2 * GB; +const REFRESH_RATE: Duration = Duration::from_micros(1_000_000 / 6); + +// The C implementation initializes `dictSize` with the public enum value +// `ZSTD_error_GENERIC` (1), rather than an encoded size_t error. +const ERROR_GENERIC: usize = 1; + +const MODE_READ: &[u8] = b"rb\0"; +const MODE_WRITE: &[u8] = b"wb\0"; + +/// ABI-compatible `ZDICT_params_t` from `lib/zdict.h`. +#[repr(C)] +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct ZDICT_params_t { + pub compressionLevel: c_int, + pub notificationLevel: c_uint, + pub dictID: c_uint, +} + +/// ABI-compatible `ZDICT_legacy_params_t` from `lib/zdict.h`. +#[repr(C)] +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct ZDICT_legacy_params_t { + pub selectivityLevel: c_uint, + pub zParams: ZDICT_params_t, +} + +/// ABI-compatible `ZDICT_cover_params_t` from `lib/zdict.h`. +#[repr(C)] +#[derive(Clone, Copy, Debug, Default, PartialEq)] +pub struct ZDICT_cover_params_t { + pub k: c_uint, + pub d: c_uint, + pub steps: c_uint, + pub nbThreads: c_uint, + pub splitPoint: f64, + pub shrinkDict: c_uint, + pub shrinkDictMaxRegression: c_uint, + pub zParams: ZDICT_params_t, +} + +/// ABI-compatible `ZDICT_fastCover_params_t` from `lib/zdict.h`. +#[repr(C)] +#[derive(Clone, Copy, Debug, Default, PartialEq)] +pub struct ZDICT_fastCover_params_t { + pub k: c_uint, + pub d: c_uint, + pub f: c_uint, + pub steps: c_uint, + pub nbThreads: c_uint, + pub splitPoint: f64, + pub accel: c_uint, + pub shrinkDict: c_uint, + pub shrinkDictMaxRegression: c_uint, + pub zParams: ZDICT_params_t, +} + +unsafe extern "C" { + fn malloc(size: usize) -> *mut c_void; + fn free(pointer: *mut c_void); + + fn fopen(filename: *const c_char, mode: *const c_char) -> *mut c_void; + fn fread(pointer: *mut c_void, size: usize, count: usize, stream: *mut c_void) -> usize; + fn fwrite(pointer: *const c_void, size: usize, count: usize, stream: *mut c_void) -> usize; + fn fclose(stream: *mut c_void) -> c_int; + + fn ZDICT_isError(error_code: usize) -> c_uint; + fn ZDICT_getErrorName(error_code: usize) -> *const c_char; + fn ZDICT_trainFromBuffer_legacy( + dict_buffer: *mut c_void, + dict_buffer_capacity: usize, + samples_buffer: *const c_void, + samples_sizes: *const usize, + nb_samples: c_uint, + parameters: ZDICT_legacy_params_t, + ) -> usize; + fn ZDICT_trainFromBuffer_cover( + dict_buffer: *mut c_void, + dict_buffer_capacity: usize, + samples_buffer: *const c_void, + samples_sizes: *const usize, + nb_samples: c_uint, + parameters: ZDICT_cover_params_t, + ) -> usize; + fn ZDICT_optimizeTrainFromBuffer_cover( + dict_buffer: *mut c_void, + dict_buffer_capacity: usize, + samples_buffer: *const c_void, + samples_sizes: *const usize, + nb_samples: c_uint, + parameters: *mut ZDICT_cover_params_t, + ) -> usize; + fn ZDICT_trainFromBuffer_fastCover( + dict_buffer: *mut c_void, + dict_buffer_capacity: usize, + samples_buffer: *const c_void, + samples_sizes: *const usize, + nb_samples: c_uint, + parameters: ZDICT_fastCover_params_t, + ) -> usize; + fn ZDICT_optimizeTrainFromBuffer_fastCover( + dict_buffer: *mut c_void, + dict_buffer_capacity: usize, + samples_buffer: *const c_void, + samples_sizes: *const usize, + nb_samples: c_uint, + parameters: *mut ZDICT_fastCover_params_t, + ) -> usize; +} + +#[cfg(test)] +mod test_zdict_symbols { + use super::*; + + static ERROR_NAME: &[u8] = b"test dictionary trainer error\0"; + + unsafe fn write_test_dictionary(buffer: *mut c_void, capacity: usize) -> usize { + let size = capacity.min(64); + if !buffer.is_null() { + std::ptr::write_bytes(buffer.cast::(), 0xA5, size); + } + size + } + + #[no_mangle] + pub unsafe extern "C" fn ZDICT_isError(_code: usize) -> c_uint { + 0 + } + + #[no_mangle] + pub unsafe extern "C" fn ZDICT_getErrorName(_code: usize) -> *const c_char { + ERROR_NAME.as_ptr().cast() + } + + #[no_mangle] + pub unsafe extern "C" fn ZDICT_trainFromBuffer_legacy( + buffer: *mut c_void, + capacity: usize, + _samples: *const c_void, + _sample_sizes: *const usize, + _nb_samples: c_uint, + _parameters: ZDICT_legacy_params_t, + ) -> usize { + write_test_dictionary(buffer, capacity) + } + + #[no_mangle] + pub unsafe extern "C" fn ZDICT_trainFromBuffer_cover( + buffer: *mut c_void, + capacity: usize, + _samples: *const c_void, + _sample_sizes: *const usize, + _nb_samples: c_uint, + _parameters: ZDICT_cover_params_t, + ) -> usize { + write_test_dictionary(buffer, capacity) + } + + #[no_mangle] + pub unsafe extern "C" fn ZDICT_optimizeTrainFromBuffer_cover( + buffer: *mut c_void, + capacity: usize, + _samples: *const c_void, + _sample_sizes: *const usize, + _nb_samples: c_uint, + _parameters: *mut ZDICT_cover_params_t, + ) -> usize { + write_test_dictionary(buffer, capacity) + } + + #[no_mangle] + pub unsafe extern "C" fn ZDICT_trainFromBuffer_fastCover( + buffer: *mut c_void, + capacity: usize, + _samples: *const c_void, + _sample_sizes: *const usize, + _nb_samples: c_uint, + _parameters: ZDICT_fastCover_params_t, + ) -> usize { + write_test_dictionary(buffer, capacity) + } + + #[no_mangle] + pub unsafe extern "C" fn ZDICT_optimizeTrainFromBuffer_fastCover( + buffer: *mut c_void, + capacity: usize, + _samples: *const c_void, + _sample_sizes: *const usize, + _nb_samples: c_uint, + _parameters: *mut ZDICT_fastCover_params_t, + ) -> usize { + write_test_dictionary(buffer, capacity) + } +} + +#[derive(Clone, Copy, Debug, Default)] +struct FileStats { + total_size_to_load: i64, + nb_samples: c_int, + one_sample_too_large: bool, +} + +struct DisplayClock { + last_update: Instant, +} + +impl DisplayClock { + fn new() -> Self { + Self { + // `g_displayClock` is zero-initialized in the C implementation, + // so its first span always permits a progress update. + last_update: Instant::now() - REFRESH_RATE, + } + } + + fn update(&mut self, display_level: c_int, level: c_int, message: fmt::Arguments<'_>) { + if display_level < level { + return; + } + if self.last_update.elapsed() > REFRESH_RATE || display_level >= 4 { + self.last_update = Instant::now(); + eprint!("{message}"); + if display_level >= 4 { + let _ = io::stderr().flush(); + } + } + } +} + +fn display(display_level: c_int, level: c_int, message: fmt::Arguments<'_>) { + if display_level >= level { + eprint!("{message}"); + } +} + +fn fatal(code: c_int, message: impl fmt::Display) -> ! { + eprintln!("Error {code} : {message}"); + std::process::exit(code); +} + +fn c_string_display(pointer: *const c_char) -> String { + unsafe { CStr::from_ptr(pointer) } + .to_string_lossy() + .into_owned() +} + +#[cfg(unix)] +fn path_from_c_string(value: &CStr) -> PathBuf { + PathBuf::from(OsStr::from_bytes(value.to_bytes())) +} + +#[cfg(not(unix))] +fn path_from_c_string(value: &CStr) -> PathBuf { + PathBuf::from(value.to_string_lossy().into_owned()) +} + +/// Returns `-1` for a missing or non-regular file, matching +/// `DiB_getFileSize()`'s `UTIL_FILESIZE_UNKNOWN` conversion. +fn get_file_size(file_name: *const c_char) -> i64 { + let path = path_from_c_string(unsafe { CStr::from_ptr(file_name) }); + match fs::metadata(path) { + Ok(metadata) if metadata.is_file() => i64::try_from(metadata.len()).unwrap_or(-1), + _ => -1, + } +} + +/// `MIN(fileSize, SAMPLESIZE_MAX)` in the original C source has different +/// usual-arithmetic-conversion behavior on 32-bit and 64-bit targets. Keep +/// that detail here because unknown files are represented by `-1`. +fn c_min_file_size(file_size: i64) -> i64 { + if size_of::() == 8 { + if (file_size as u64) < SAMPLESIZE_MAX as u64 { + file_size + } else { + SAMPLESIZE_MAX as i64 + } + } else { + min(file_size, SAMPLESIZE_MAX as i64) + } +} + +unsafe fn shuffle(file_names: *const *const c_char, nb_files: c_int) { + if nb_files <= 1 { + return; + } + + // The C API takes `const char**` but deliberately shuffles the caller's + // table in place. The CLI supplies a writable pointer array. + let table = unsafe { std::slice::from_raw_parts_mut(file_names.cast_mut(), nb_files as usize) }; + let mut seed = 0xFD2F_B528u32; + for index in (1..table.len()).rev() { + seed = seed.wrapping_mul(2_654_435_761); + seed ^= 2_246_822_519; + seed = seed.rotate_left(13); + let random = (seed >> 5) as usize; + table.swap(index, random % (index + 1)); + } +} + +fn file_stats( + file_names: *const *const c_char, + nb_files: c_int, + chunk_size: usize, + display_level: c_int, +) -> FileStats { + debug_assert!(chunk_size <= SAMPLESIZE_MAX); + let mut stats = FileStats::default(); + + for index in 0..nb_files.max(0) as usize { + let file_name = unsafe { *file_names.add(index) }; + let file_size = get_file_size(file_name); + if file_size == 0 { + display( + display_level, + 3, + format_args!( + "Sample file '{}' has zero size, skipping...\n", + c_string_display(file_name) + ), + ); + continue; + } + + if chunk_size > 0 { + // This is the C expression `(fileSize + chunkSize - 1) / + // chunkSize`, including its unsigned wrap for an unknown file on + // the 64-bit targets where size_t and S64 have equal width. + let chunks = (file_size as u64) + .wrapping_add(chunk_size as u64) + .wrapping_sub(1) + / chunk_size as u64; + stats.nb_samples = stats.nb_samples.wrapping_add(chunks as c_int); + stats.total_size_to_load = stats.total_size_to_load.wrapping_add(file_size); + } else { + if file_size > SAMPLESIZE_MAX as i64 { + if file_size > (2 * SAMPLESIZE_MAX) as i64 { + stats.one_sample_too_large = true; + } + display( + display_level, + 3, + format_args!( + "Sample file '{}' is too large, limiting to {} KB\n", + c_string_display(file_name), + SAMPLESIZE_MAX / KB + ), + ); + } + stats.nb_samples = stats.nb_samples.wrapping_add(1); + stats.total_size_to_load = stats + .total_size_to_load + .wrapping_add(c_min_file_size(file_size)); + } + } + + display( + display_level, + 4, + format_args!( + "Found training data {} files, {} KB, {} samples\n", + nb_files, + (stats.total_size_to_load / KB as i64) as c_int, + stats.nb_samples + ), + ); + stats +} + +fn maximum_memory() -> usize { + if size_of::() == 4 { + 2 * GB - 64 * MB + } else { + (512 * MB) << size_of::() + } +} + +unsafe fn find_max_memory(required_memory: u64) -> usize { + let step = (8 * MB) as u64; + let mut required = (((required_memory >> 23) + 1) << 23).wrapping_add(step); + required = min(required, maximum_memory() as u64); + + loop { + let test_memory = unsafe { malloc(required as usize) }; + if !test_memory.is_null() { + unsafe { free(test_memory) }; + return required as usize; + } + required = required.wrapping_sub(step); + } +} + +unsafe fn fill_noise(buffer: *mut u8, length: usize) { + let mut accumulator = 2_654_435_761u32; + for index in 0..length { + accumulator = accumulator.wrapping_mul(2_246_822_519); + unsafe { buffer.add(index).write((accumulator >> 21) as u8) }; + } +} + +unsafe fn read_chunk( + stream: *mut c_void, + destination: *mut u8, + length: usize, + file_name: *const c_char, +) { + let read = unsafe { fread(destination.cast(), 1, length, stream) }; + if read != length { + fatal( + 11, + format_args!("Pb reading {}", c_string_display(file_name)), + ); + } +} + +unsafe fn load_files( + buffer: *mut u8, + buffer_size: usize, + sample_sizes: *mut usize, + sample_count: usize, + file_names: *const *const c_char, + nb_files: c_int, + target_chunk_size: usize, + display_level: c_int, + display_clock: &mut DisplayClock, +) -> (usize, usize) { + debug_assert!(target_chunk_size <= SAMPLESIZE_MAX); + let mut total_data_loaded = 0usize; + let mut nb_samples_loaded = 0usize; + let mut file_index = 0usize; + let file_count = nb_files.max(0) as usize; + + while nb_samples_loaded < sample_count && file_index < file_count { + let file_name = unsafe { *file_names.add(file_index) }; + let file_size = get_file_size(file_name); + if file_size <= 0 { + file_index += 1; + continue; + } + + let stream = unsafe { fopen(file_name, MODE_READ.as_ptr().cast()) }; + if stream.is_null() { + let error = io::Error::last_os_error(); + fatal( + 10, + format_args!( + "zstd: dictBuilder: {} {} ", + c_string_display(file_name), + error + ), + ); + } + display_clock.update( + display_level, + 2, + format_args!("Loading {}... \r", c_string_display(file_name)), + ); + + let first_chunk = if target_chunk_size > 0 { + min(file_size as usize, target_chunk_size) + } else { + min(file_size as usize, SAMPLESIZE_MAX) + }; + if total_data_loaded.wrapping_add(first_chunk) > buffer_size { + unsafe { fclose(stream) }; + break; + } + + unsafe { + read_chunk( + stream, + buffer.add(total_data_loaded), + first_chunk, + file_name, + ); + sample_sizes.add(nb_samples_loaded).write(first_chunk); + } + nb_samples_loaded += 1; + total_data_loaded += first_chunk; + + if target_chunk_size > 0 { + let mut file_data_loaded = first_chunk; + while (file_data_loaded as i64) < file_size && nb_samples_loaded < sample_count { + let remaining = (file_size - file_data_loaded as i64) as usize; + let chunk_size = min(remaining, target_chunk_size); + if total_data_loaded.wrapping_add(chunk_size) > buffer_size { + break; + } + unsafe { + read_chunk(stream, buffer.add(total_data_loaded), chunk_size, file_name); + sample_sizes.add(nb_samples_loaded).write(chunk_size); + } + nb_samples_loaded += 1; + total_data_loaded += chunk_size; + file_data_loaded += chunk_size; + } + } + + file_index += 1; + unsafe { fclose(stream) }; + } + + display(display_level, 2, format_args!("\r{:79}\r", "")); + display( + display_level, + 4, + format_args!( + "Loaded {} KB total training data, {} nb samples \n", + total_data_loaded / KB, + nb_samples_loaded + ), + ); + (total_data_loaded, nb_samples_loaded) +} + +fn loaded_size(total_size_to_load: i64, max_memory: usize) -> usize { + let first = min(max_memory as i64, total_size_to_load); + if size_of::() == 8 { + min(first as u64, MAX_SAMPLES_SIZE as u64) as usize + } else { + min(first, MAX_SAMPLES_SIZE as i64) as usize + } +} + +unsafe fn save_dictionary( + dictionary_file_name: *const c_char, + dictionary: *const c_void, + dictionary_size: usize, +) { + let stream = unsafe { fopen(dictionary_file_name, MODE_WRITE.as_ptr().cast()) }; + if stream.is_null() { + fatal( + 3, + format_args!("cannot open {} ", c_string_display(dictionary_file_name)), + ); + } + + let written = unsafe { fwrite(dictionary, 1, dictionary_size, stream) }; + if written != dictionary_size { + fatal( + 4, + format_args!("{} : write error", c_string_display(dictionary_file_name)), + ); + } + + if unsafe { fclose(stream) } != 0 { + fatal( + 5, + format_args!("{} : flush error", c_string_display(dictionary_file_name)), + ); + } +} + +unsafe fn error_name(error_code: usize) -> String { + let name = unsafe { ZDICT_getErrorName(error_code) }; + if name.is_null() { + return "unknown error".to_owned(); + } + unsafe { CStr::from_ptr(name) } + .to_string_lossy() + .into_owned() +} + +unsafe fn free_buffers( + source_buffer: *mut c_void, + sample_sizes: *mut usize, + dictionary_buffer: *mut c_void, +) { + unsafe { + free(source_buffer); + free(sample_sizes.cast()); + free(dictionary_buffer); + } +} + +/// Train a dictionary from files and write it to `dictionary_file_name`. +/// +/// This is the Rust implementation of the public `DiB_trainFromFiles()` ABI +/// declared in `programs/dibio.h`. Its exit-on-I/O-error behavior is kept for +/// compatibility with the command-line program; trainer errors return `1`. +#[no_mangle] +pub unsafe extern "C" fn DiB_trainFromFiles( + dictionary_file_name: *const c_char, + max_dictionary_size: usize, + file_names: *const *const c_char, + nb_files: c_int, + chunk_size: usize, + legacy_params: *mut ZDICT_legacy_params_t, + cover_params: *mut ZDICT_cover_params_t, + fast_cover_params: *mut ZDICT_fastCover_params_t, + optimize: c_int, + memory_limit: c_uint, +) -> c_int { + let display_level = if !legacy_params.is_null() { + unsafe { (*legacy_params).zParams.notificationLevel as c_int } + } else if !cover_params.is_null() { + unsafe { (*cover_params).zParams.notificationLevel as c_int } + } else if !fast_cover_params.is_null() { + unsafe { (*fast_cover_params).zParams.notificationLevel as c_int } + } else { + 0 + }; + let mut display_clock = DisplayClock::new(); + let dictionary_buffer = unsafe { malloc(max_dictionary_size) }; + + display(display_level, 3, format_args!("Shuffling input files\n")); + unsafe { shuffle(file_names, nb_files) }; + + let stats = file_stats(file_names, nb_files, chunk_size, display_level); + let memory_multiplier = if !legacy_params.is_null() { + MEMMULT + } else if !cover_params.is_null() { + COVER_MEMMULT + } else { + FASTCOVER_MEMMULT + }; + let required_memory = (stats.total_size_to_load as u64).wrapping_mul(memory_multiplier); + let max_memory = unsafe { find_max_memory(required_memory) } / memory_multiplier as usize; + let mut loaded_size = loaded_size(stats.total_size_to_load, max_memory); + if memory_limit != 0 { + display( + display_level, + 2, + format_args!( + "! Warning : setting manual memory limit for dictionary training data at {} MB \n", + (memory_limit as usize) / MB + ), + ); + loaded_size = min(loaded_size, memory_limit as usize); + } + + let source_buffer_size = loaded_size.wrapping_add(NOISELENGTH); + let source_buffer = unsafe { malloc(source_buffer_size) }; + let sample_count = stats.nb_samples.max(0) as usize; + let sample_sizes_bytes = sample_count.wrapping_mul(size_of::()); + let sample_sizes = unsafe { malloc(sample_sizes_bytes) }.cast::(); + + if ((stats.nb_samples > 0) && sample_sizes.is_null()) + || source_buffer.is_null() + || dictionary_buffer.is_null() + { + fatal(12, "not enough memory for DiB_trainFiles"); + } + if stats.one_sample_too_large { + display( + display_level, + 2, + format_args!("! Warning : some sample(s) are very large \n"), + ); + display( + display_level, + 2, + format_args!("! Note that dictionary is only useful for small samples. \n"), + ); + display( + display_level, + 2, + format_args!( + "! As a consequence, only the first {} bytes of each sample are loaded \n", + SAMPLESIZE_MAX + ), + ); + } + if stats.nb_samples < 5 { + display( + display_level, + 2, + format_args!("! Warning : nb of samples too low for proper processing ! \n"), + ); + display( + display_level, + 2, + format_args!("! Please provide _one file per sample_. \n"), + ); + display( + display_level, + 2, + format_args!( + "! Alternatively, split files into fixed-size blocks representative of samples, with -B# \n" + ), + ); + fatal(14, "nb of samples too low"); + } + let dictionary_size_threshold = (max_dictionary_size as i64).wrapping_mul(8); + if stats.total_size_to_load < dictionary_size_threshold { + display( + display_level, + 2, + format_args!( + "! Warning : data size of samples too small for target dictionary size \n" + ), + ); + display( + display_level, + 2, + format_args!("! Samples should be about 100x larger than target dictionary size \n"), + ); + } + + if (loaded_size as i64) < stats.total_size_to_load { + display( + display_level, + 1, + format_args!( + "Training samples set too large ({} MB); training on {} MB only...\n", + (stats.total_size_to_load / MB as i64) as c_uint, + loaded_size / MB + ), + ); + } + + let (loaded_size, nb_samples_loaded) = unsafe { + load_files( + source_buffer.cast(), + loaded_size, + sample_sizes, + sample_count, + file_names, + nb_files, + chunk_size, + display_level, + &mut display_clock, + ) + }; + + let mut dictionary_size = ERROR_GENERIC; + if !legacy_params.is_null() { + unsafe { fill_noise(source_buffer.cast::().add(loaded_size), NOISELENGTH) }; + dictionary_size = unsafe { + ZDICT_trainFromBuffer_legacy( + dictionary_buffer, + max_dictionary_size, + source_buffer, + sample_sizes, + nb_samples_loaded as c_uint, + *legacy_params, + ) + }; + } else if !cover_params.is_null() { + if optimize != 0 { + dictionary_size = unsafe { + ZDICT_optimizeTrainFromBuffer_cover( + dictionary_buffer, + max_dictionary_size, + source_buffer, + sample_sizes, + nb_samples_loaded as c_uint, + cover_params, + ) + }; + if unsafe { ZDICT_isError(dictionary_size) } == 0 { + let parameters = unsafe { &*cover_params }; + display( + display_level, + 2, + format_args!( + "k={}\nd={}\nsteps={}\nsplit={}\n", + parameters.k, + parameters.d, + parameters.steps, + (parameters.splitPoint * 100.0) as c_uint + ), + ); + } + } else { + dictionary_size = unsafe { + ZDICT_trainFromBuffer_cover( + dictionary_buffer, + max_dictionary_size, + source_buffer, + sample_sizes, + nb_samples_loaded as c_uint, + *cover_params, + ) + }; + } + } else if !fast_cover_params.is_null() { + if optimize != 0 { + dictionary_size = unsafe { + ZDICT_optimizeTrainFromBuffer_fastCover( + dictionary_buffer, + max_dictionary_size, + source_buffer, + sample_sizes, + nb_samples_loaded as c_uint, + fast_cover_params, + ) + }; + if unsafe { ZDICT_isError(dictionary_size) } == 0 { + let parameters = unsafe { &*fast_cover_params }; + display( + display_level, + 2, + format_args!( + "k={}\nd={}\nf={}\nsteps={}\nsplit={}\naccel={}\n", + parameters.k, + parameters.d, + parameters.f, + parameters.steps, + (parameters.splitPoint * 100.0) as c_uint, + parameters.accel + ), + ); + } + } else { + dictionary_size = unsafe { + ZDICT_trainFromBuffer_fastCover( + dictionary_buffer, + max_dictionary_size, + source_buffer, + sample_sizes, + nb_samples_loaded as c_uint, + *fast_cover_params, + ) + }; + } + } else { + debug_assert!(false, "one dictionary trainer parameter set is required"); + } + + if unsafe { ZDICT_isError(dictionary_size) } != 0 { + display( + display_level, + 1, + format_args!("dictionary training failed : {} \n", unsafe { + error_name(dictionary_size) + }), + ); + unsafe { free_buffers(source_buffer, sample_sizes, dictionary_buffer) }; + return 1; + } + + display( + display_level, + 2, + format_args!( + "Save dictionary of size {} into file {} \n", + dictionary_size as c_uint, + c_string_display(dictionary_file_name) + ), + ); + unsafe { + save_dictionary(dictionary_file_name, dictionary_buffer, dictionary_size); + free_buffers(source_buffer, sample_sizes, dictionary_buffer); + } + 0 +} + +#[cfg(test)] +mod tests { + use super::*; + use std::ffi::CString; + use std::fs; + use std::mem::{offset_of, size_of}; + use std::path::{Path, PathBuf}; + use std::sync::atomic::{AtomicUsize, Ordering}; + + #[cfg(unix)] + fn path_to_c_string(path: &Path) -> CString { + CString::new(path.as_os_str().as_bytes()).expect("temporary path has no NUL") + } + + #[cfg(not(unix))] + fn path_to_c_string(path: &Path) -> CString { + CString::new(path.to_string_lossy().as_bytes()).expect("temporary path has no NUL") + } + + fn sample(index: usize) -> Vec { + let line = format!( + "record={index:02};common-key=common-value;payload=abcdefghijklmnopqrstuvwxyz\n" + ); + let mut result = Vec::with_capacity(1024); + while result.len() < 1024 { + result.extend_from_slice(line.as_bytes()); + } + result.truncate(1024); + result + } + + fn temporary_paths() -> (Vec, PathBuf) { + static NEXT: AtomicUsize = AtomicUsize::new(0); + let prefix = format!( + "zstd-dibio-{}-{}", + std::process::id(), + NEXT.fetch_add(1, Ordering::Relaxed) + ); + let root = std::env::temp_dir(); + let samples = (0..5) + .map(|index| root.join(format!("{prefix}-sample-{index}"))) + .collect::>(); + let dictionary = root.join(format!("{prefix}-dictionary")); + (samples, dictionary) + } + + #[test] + fn dictionary_parameter_layouts_match_zdict_h() { + assert_eq!(size_of::(), 12); + assert_eq!(size_of::(), 16); + assert_eq!(size_of::(), 48); + assert_eq!(size_of::(), 56); + assert_eq!(offset_of!(ZDICT_cover_params_t, splitPoint), 16); + assert_eq!(offset_of!(ZDICT_fastCover_params_t, splitPoint), 24); + } + + #[test] + fn trains_from_chunked_temporary_sample_files() { + let (sample_paths, dictionary_path) = temporary_paths(); + let _cleanup = TemporaryPathCleanup { + paths: sample_paths + .iter() + .cloned() + .chain(std::iter::once(dictionary_path.clone())) + .collect(), + }; + for (index, path) in sample_paths.iter().enumerate() { + fs::write(path, sample(index)).expect("write temporary training sample"); + } + + let sample_names = sample_paths + .iter() + .map(|path| path_to_c_string(path)) + .collect::>(); + let file_names = sample_names + .iter() + .map(|name| name.as_ptr()) + .collect::>(); + let dictionary_name = path_to_c_string(&dictionary_path); + let mut parameters = ZDICT_legacy_params_t { + zParams: ZDICT_params_t { + compressionLevel: 3, + ..ZDICT_params_t::default() + }, + ..ZDICT_legacy_params_t::default() + }; + + let result = unsafe { + DiB_trainFromFiles( + dictionary_name.as_ptr(), + 2048, + file_names.as_ptr(), + file_names.len() as c_int, + 256, + &mut parameters, + std::ptr::null_mut(), + std::ptr::null_mut(), + 0, + 0, + ) + }; + assert_eq!(result, 0); + let dictionary = fs::read(&dictionary_path).expect("read trained dictionary"); + assert!(!dictionary.is_empty()); + } + + #[test] + fn legacy_trainer_abi_smoke() { + let samples = (0..5).map(sample).collect::>(); + let sample_sizes = samples.iter().map(Vec::len).collect::>(); + let flat_samples = samples.concat(); + let mut dictionary = vec![0u8; 2048]; + let parameters = ZDICT_legacy_params_t::default(); + let result = unsafe { + ZDICT_trainFromBuffer_legacy( + dictionary.as_mut_ptr().cast(), + dictionary.len(), + flat_samples.as_ptr().cast(), + sample_sizes.as_ptr(), + sample_sizes.len() as c_uint, + parameters, + ) + }; + assert_eq!(unsafe { ZDICT_isError(result) }, 0); + assert!(result > 0); + } + + impl Drop for TemporaryPathCleanup { + fn drop(&mut self) { + for path in &self.paths { + let _ = fs::remove_file(path); + } + } + } + + struct TemporaryPathCleanup { + paths: Vec, + } +} diff --git a/rust/src/util.rs b/rust/src/util.rs new file mode 100644 index 000000000..928a3b740 --- /dev/null +++ b/rust/src/util.rs @@ -0,0 +1,1501 @@ +#![allow(non_camel_case_types)] +#![allow(non_snake_case)] +#![allow(clippy::missing_safety_doc)] + +//! Rust implementation of the program utility layer from `programs/util.c`. +//! +//! The C header remains the public ABI. Functions in this module therefore +//! use the C allocator for objects which are returned to C, keep all exported +//! structures `repr(C)`, and use the target's `libc::stat` representation for +//! the `stat_t` pointer passed by `util.h`. + +use std::ffi::{CStr, CString, OsString}; +use std::fs; +use std::mem::{offset_of, size_of, MaybeUninit}; +use std::os::raw::{c_char, c_int, c_uint, c_void}; +use std::path::PathBuf; +use std::ptr; +use std::sync::atomic::{AtomicBool, AtomicI32, Ordering}; +use std::sync::OnceLock; + +#[cfg(unix)] +use std::os::unix::ffi::OsStringExt; + +type Stat = libc::stat; + +#[cfg(unix)] +type UtilMode = libc::mode_t; +#[cfg(windows)] +type UtilMode = c_int; +#[cfg(not(any(unix, windows)))] +type UtilMode = c_uint; + +const LIST_SIZE_INCREASE: usize = 8 * 1024; +const MAX_FILE_OF_FILE_NAMES_SIZE: u64 = 50 * (1 << 20); +const UTIL_FILESIZE_UNKNOWN: u64 = u64::MAX; +const DIR_DEFAULT_MODE: u32 = 0o755; + +/// `g_utilDisplayLevel` is a public C global, not a Rust-owned preference. +#[no_mangle] +pub static mut g_utilDisplayLevel: c_int = 0; + +/// The two structs below mirror the declarations in `programs/util.h`. +#[repr(C)] +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct UTIL_HumanReadableSize_t { + pub value: f64, + pub precision: c_int, + pub suffix: *const c_char, +} + +#[repr(C)] +#[derive(Debug)] +pub struct FileNamesTable { + pub fileNames: *mut *const c_char, + pub buf: *mut c_char, + pub tableSize: usize, + pub tableCapacity: usize, +} + +const _: () = assert!(offset_of!(UTIL_HumanReadableSize_t, value) == 0); +const _: () = assert!(offset_of!(UTIL_HumanReadableSize_t, precision) == size_of::()); +const _: () = assert!( + offset_of!(UTIL_HumanReadableSize_t, suffix) + >= offset_of!(UTIL_HumanReadableSize_t, precision) + size_of::() +); +const _: () = assert!( + size_of::() + == offset_of!(UTIL_HumanReadableSize_t, suffix) + size_of::<*const c_char>() +); +const _: () = assert!(offset_of!(FileNamesTable, fileNames) == 0); +const _: () = assert!(offset_of!(FileNamesTable, buf) == size_of::<*const c_char>()); +const _: () = assert!( + offset_of!(FileNamesTable, tableSize) + == offset_of!(FileNamesTable, buf) + size_of::<*mut c_char>() +); +const _: () = assert!( + offset_of!(FileNamesTable, tableCapacity) + == offset_of!(FileNamesTable, tableSize) + size_of::() +); +const _: () = assert!( + size_of::() == offset_of!(FileNamesTable, tableCapacity) + size_of::() +); + +static EMPTY_EXTENSION: [u8; 1] = [0]; +static SUFFIX_B: &[u8] = b" B\0"; +static SUFFIX_KIB: &[u8] = b" KiB\0"; +static SUFFIX_MIB: &[u8] = b" MiB\0"; +static SUFFIX_GIB: &[u8] = b" GiB\0"; +static SUFFIX_TIB: &[u8] = b" TiB\0"; +static SUFFIX_PIB: &[u8] = b" PiB\0"; +static SUFFIX_EIB: &[u8] = b" EiB\0"; + +static FAKE_STDIN_IS_CONSOLE: AtomicBool = AtomicBool::new(false); +static FAKE_STDOUT_IS_CONSOLE: AtomicBool = AtomicBool::new(false); +static FAKE_STDERR_IS_CONSOLE: AtomicBool = AtomicBool::new(false); +static TRACE_FILE_STAT: AtomicBool = AtomicBool::new(false); +static TRACE_DEPTH: AtomicI32 = AtomicI32::new(0); +static CORE_COUNT: OnceLock = OnceLock::new(); + +#[cfg(windows)] +unsafe extern "C" { + fn UTIL_rust_utime(filename: *const c_char, statbuf: *const Stat) -> c_int; +} + +#[inline] +fn display_level() -> c_int { + // The C API intentionally exposes this mutable global. All reads mirror + // the unsynchronized reads in the original implementation. + unsafe { g_utilDisplayLevel } +} + +fn display(message: &str) { + eprint!("{message}"); +} + +unsafe fn display_c_string(message: *const c_char) { + if !message.is_null() { + display(&String::from_utf8_lossy(CStr::from_ptr(message).to_bytes())); + } +} + +fn trace_call(message: &str) { + if TRACE_FILE_STAT.load(Ordering::Relaxed) { + let depth = TRACE_DEPTH.fetch_add(1, Ordering::Relaxed).max(0) as usize; + eprintln!("Trace:FileStat: {:width$}> {message}", "", width = depth); + } +} + +fn trace_return(ret: c_int) { + if TRACE_FILE_STAT.load(Ordering::Relaxed) { + let depth = TRACE_DEPTH.fetch_sub(1, Ordering::Relaxed) - 1; + eprintln!( + "Trace:FileStat: {:width$}< {ret}", + "", + width = depth.max(0) as usize + ); + } +} + +#[inline] +unsafe fn c_bytes<'a>(value: *const c_char) -> &'a [u8] { + CStr::from_ptr(value).to_bytes() +} + +#[inline] +fn path_separator() -> u8 { + if cfg!(windows) { + b'\\' + } else { + b'/' + } +} + +fn path_from_bytes(bytes: &[u8]) -> PathBuf { + #[cfg(unix)] + { + PathBuf::from(OsString::from_vec(bytes.to_vec())) + } + #[cfg(not(unix))] + { + PathBuf::from(String::from_utf8_lossy(bytes).into_owned()) + } +} + +fn os_string_bytes(value: OsString) -> Vec { + #[cfg(unix)] + { + value.into_vec() + } + #[cfg(not(unix))] + { + value.to_string_lossy().into_owned().into_bytes() + } +} + +fn c_path(bytes: &[u8]) -> CString { + CString::new(bytes).unwrap_or_else(|_| std::process::abort()) +} + +unsafe fn malloc_bytes(size: usize) -> *mut u8 { + let ptr = libc::malloc(size).cast::(); + if ptr.is_null() { + std::process::abort(); + } + ptr +} + +unsafe fn malloc_array(count: usize) -> *mut T { + let Some(size) = count.checked_mul(size_of::()) else { + return ptr::null_mut(); + }; + libc::malloc(size).cast::() +} + +unsafe fn allocate_struct() -> *mut T { + let result = malloc_bytes(size_of::()).cast::(); + // The allocation helper is used only for C functions whose original + // CONTROL() path terminates the program on allocation failure. + ptr::write_bytes(result.cast::(), 0, size_of::()); + result +} + +unsafe fn free_ptr(value: *mut T) { + libc::free(value.cast::()); +} + +unsafe fn table_entries<'a>(table: *const FileNamesTable) -> &'a [*const c_char] { + if (*table).fileNames.is_null() || (*table).tableSize == 0 { + &[] + } else { + std::slice::from_raw_parts((*table).fileNames, (*table).tableSize) + } +} + +unsafe fn table_names_size(table: *const FileNamesTable) -> usize { + let mut total = 0usize; + for &name in table_entries(table) { + if name.is_null() { + break; + } + total = total.saturating_add(c_bytes(name).len().saturating_add(1)); + } + total +} + +unsafe fn make_table( + file_names: *mut *const c_char, + table_size: usize, + table_capacity: usize, + buf: *mut c_char, +) -> *mut FileNamesTable { + let table = allocate_struct::(); + (*table).fileNames = file_names; + (*table).buf = buf; + (*table).tableSize = table_size; + (*table).tableCapacity = table_capacity; + table +} + +unsafe fn make_table_from_buffer( + buf: *mut u8, + buffer_len: usize, + table_size: usize, + table_capacity: usize, +) -> *mut FileNamesTable { + let Some(pointer_count) = table_capacity.checked_mul(size_of::<*const c_char>()) else { + free_ptr(buf); + return ptr::null_mut(); + }; + let file_names = malloc_bytes(pointer_count.max(1)).cast::<*const c_char>(); + let mut pos = 0usize; + for index in 0..table_size { + if pos > buffer_len { + free_ptr(file_names); + free_ptr(buf); + return ptr::null_mut(); + } + *file_names.add(index) = buf.add(pos).cast::(); + let name_len = CStr::from_ptr(buf.add(pos).cast::()) + .to_bytes() + .len(); + pos = pos.saturating_add(name_len + 1); + } + make_table(file_names, table_size, table_capacity, buf.cast::()) +} + +#[inline] +unsafe fn stat_mode(statbuf: *const Stat) -> u64 { + (*statbuf).st_mode as u64 +} + +#[cfg(unix)] +#[inline] +unsafe fn mode_is(statbuf: *const Stat, kind: libc::mode_t) -> bool { + stat_mode(statbuf) & libc::S_IFMT as u64 == kind as u64 +} + +#[cfg(windows)] +#[inline] +unsafe fn mode_is(statbuf: *const Stat, kind: c_int) -> bool { + // util.c intentionally uses `(st_mode & S_IFREG) != 0` for the CRT. + stat_mode(statbuf) & kind as u64 != 0 +} + +#[cfg(not(any(unix, windows)))] +#[inline] +unsafe fn mode_is(_statbuf: *const Stat, _kind: c_uint) -> bool { + false +} + +#[no_mangle] +pub unsafe extern "C" fn UTIL_fstat( + fd: c_int, + filename: *const c_char, + statbuf: *mut Stat, +) -> c_int { + trace_call("UTIL_stat"); + let result = if fd >= 0 { + libc::fstat(fd, statbuf) + } else { + libc::stat(filename, statbuf) + }; + let ret = (result == 0) as c_int; + trace_return(ret); + ret +} + +#[no_mangle] +pub unsafe extern "C" fn UTIL_stat(filename: *const c_char, statbuf: *mut Stat) -> c_int { + UTIL_fstat(-1, filename, statbuf) +} + +#[no_mangle] +pub unsafe extern "C" fn UTIL_isRegularFileStat(statbuf: *const Stat) -> c_int { + #[cfg(any(unix, windows))] + { + mode_is(statbuf, libc::S_IFREG) as c_int + } + #[cfg(not(any(unix, windows)))] + { + 0 + } +} + +#[no_mangle] +pub unsafe extern "C" fn UTIL_isDirectoryStat(statbuf: *const Stat) -> c_int { + #[cfg(any(unix, windows))] + { + mode_is(statbuf, libc::S_IFDIR) as c_int + } + #[cfg(not(any(unix, windows)))] + { + 0 + } +} + +#[no_mangle] +pub unsafe extern "C" fn UTIL_isFIFOStat(statbuf: *const Stat) -> c_int { + #[cfg(unix)] + { + mode_is(statbuf, libc::S_IFIFO) as c_int + } + #[cfg(not(unix))] + { + let _ = statbuf; + 0 + } +} + +#[no_mangle] +pub unsafe extern "C" fn UTIL_isBlockDevStat(statbuf: *const Stat) -> c_int { + #[cfg(unix)] + { + mode_is(statbuf, libc::S_IFBLK) as c_int + } + #[cfg(not(unix))] + { + let _ = statbuf; + 0 + } +} + +#[no_mangle] +pub unsafe extern "C" fn UTIL_isRegularFile(infilename: *const c_char) -> c_int { + let mut statbuf = MaybeUninit::::uninit(); + if UTIL_stat(infilename, statbuf.as_mut_ptr()) == 0 { + return 0; + } + UTIL_isRegularFileStat(statbuf.as_ptr()) +} + +#[no_mangle] +pub unsafe extern "C" fn UTIL_isDirectory(infilename: *const c_char) -> c_int { + let mut statbuf = MaybeUninit::::uninit(); + if UTIL_stat(infilename, statbuf.as_mut_ptr()) == 0 { + return 0; + } + UTIL_isDirectoryStat(statbuf.as_ptr()) +} + +#[no_mangle] +pub unsafe extern "C" fn UTIL_isFIFO(infilename: *const c_char) -> c_int { + #[cfg(unix)] + { + let mut statbuf = MaybeUninit::::uninit(); + if UTIL_stat(infilename, statbuf.as_mut_ptr()) != 0 { + return UTIL_isFIFOStat(statbuf.as_ptr()); + } + } + let _ = infilename; + 0 +} + +#[no_mangle] +pub unsafe extern "C" fn UTIL_isLink(infilename: *const c_char) -> c_int { + #[cfg(unix)] + { + let mut statbuf = MaybeUninit::::uninit(); + if libc::lstat(infilename, statbuf.as_mut_ptr()) == 0 { + return mode_is(statbuf.as_ptr(), libc::S_IFLNK) as c_int; + } + } + let _ = infilename; + 0 +} + +#[no_mangle] +pub unsafe extern "C" fn UTIL_isSameFile(file1: *const c_char, file2: *const c_char) -> c_int { + assert!(!file1.is_null() && !file2.is_null()); + #[cfg(windows)] + { + return (c_bytes(file1) == c_bytes(file2)) as c_int; + } + #[cfg(not(windows))] + { + let mut file1_stat = MaybeUninit::::uninit(); + let mut file2_stat = MaybeUninit::::uninit(); + if UTIL_stat(file1, file1_stat.as_mut_ptr()) == 0 + || UTIL_stat(file2, file2_stat.as_mut_ptr()) == 0 + { + return 0; + } + UTIL_isSameFileStat(file1, file2, file1_stat.as_ptr(), file2_stat.as_ptr()) + } +} + +#[no_mangle] +pub unsafe extern "C" fn UTIL_isSameFileStat( + file1: *const c_char, + file2: *const c_char, + file1_stat: *const Stat, + file2_stat: *const Stat, +) -> c_int { + assert!(!file1.is_null() && !file2.is_null()); + #[cfg(windows)] + { + let _ = (file1_stat, file2_stat); + return (c_bytes(file1) == c_bytes(file2)) as c_int; + } + #[cfg(not(windows))] + { + ((*file1_stat).st_dev == (*file2_stat).st_dev + && (*file1_stat).st_ino == (*file2_stat).st_ino) as c_int + } +} + +#[no_mangle] +pub unsafe extern "C" fn UTIL_getFileSizeStat(statbuf: *const Stat) -> u64 { + if UTIL_isRegularFileStat(statbuf) == 0 { + return UTIL_FILESIZE_UNKNOWN; + } + (*statbuf).st_size as u64 +} + +#[no_mangle] +pub unsafe extern "C" fn UTIL_getFileSize(infilename: *const c_char) -> u64 { + let mut statbuf = MaybeUninit::::uninit(); + if UTIL_stat(infilename, statbuf.as_mut_ptr()) == 0 { + return UTIL_FILESIZE_UNKNOWN; + } + UTIL_getFileSizeStat(statbuf.as_ptr()) +} + +#[no_mangle] +pub unsafe extern "C" fn UTIL_getTotalFileSize( + file_names: *const *const c_char, + nb_files: c_uint, +) -> u64 { + let mut total = 0u64; + for index in 0..nb_files as usize { + let size = UTIL_getFileSize(*file_names.add(index)); + if size == UTIL_FILESIZE_UNKNOWN { + return UTIL_FILESIZE_UNKNOWN; + } + total = total.wrapping_add(size); + } + total +} + +#[no_mangle] +pub unsafe extern "C" fn UTIL_chmod( + filename: *const c_char, + statbuf: *const Stat, + permissions: UtilMode, +) -> c_int { + UTIL_fchmod(-1, filename, statbuf, permissions) +} + +#[no_mangle] +pub unsafe extern "C" fn UTIL_fchmod( + fd: c_int, + filename: *const c_char, + statbuf: *const Stat, + permissions: UtilMode, +) -> c_int { + let mut local_stat = MaybeUninit::::uninit(); + let stat_ptr = if statbuf.is_null() { + if UTIL_fstat(fd, filename, local_stat.as_mut_ptr()) == 0 { + return 0; + } + local_stat.as_ptr() + } else { + statbuf + }; + if UTIL_isRegularFileStat(stat_ptr) == 0 { + return 0; + } + + #[cfg(unix)] + { + if fd >= 0 { + return libc::fchmod(fd, permissions); + } + libc::chmod(filename, permissions) + } + #[cfg(windows)] + { + let _ = fd; + libc::chmod(filename, permissions) + } + #[cfg(not(any(unix, windows)))] + { + let _ = (fd, filename, permissions); + 0 + } +} + +#[cfg(any(target_os = "linux", target_os = "android"))] +unsafe fn set_mtime(filename: *const c_char, statbuf: *const Stat) -> c_int { + let now = libc::timespec { + tv_sec: 0, + tv_nsec: libc::UTIME_NOW, + }; + let mtime = libc::timespec { + tv_sec: (*statbuf).st_mtime, + tv_nsec: (*statbuf).st_mtime_nsec as _, + }; + let times = [now, mtime]; + libc::utimensat(libc::AT_FDCWD, filename, times.as_ptr(), 0) +} + +#[cfg(all(unix, not(any(target_os = "linux", target_os = "android"))))] +unsafe fn set_mtime(filename: *const c_char, statbuf: *const Stat) -> c_int { + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map_or(0, |duration| duration.as_secs()) as libc::time_t; + let times = libc::utimbuf { + actime: now, + modtime: (*statbuf).st_mtime, + }; + libc::utime(filename, ×) +} + +#[cfg(windows)] +unsafe fn set_mtime(filename: *const c_char, statbuf: *const Stat) -> c_int { + UTIL_rust_utime(filename, statbuf) +} + +#[cfg(not(any(unix, windows)))] +unsafe fn set_mtime(_filename: *const c_char, _statbuf: *const Stat) -> c_int { + -1 +} + +#[no_mangle] +pub unsafe extern "C" fn UTIL_utime(filename: *const c_char, statbuf: *const Stat) -> c_int { + set_mtime(filename, statbuf) +} + +#[no_mangle] +pub unsafe extern "C" fn UTIL_setFileStat(filename: *const c_char, statbuf: *const Stat) -> c_int { + UTIL_setFDStat(-1, filename, statbuf) +} + +#[no_mangle] +pub unsafe extern "C" fn UTIL_setFDStat( + fd: c_int, + filename: *const c_char, + statbuf: *const Stat, +) -> c_int { + let mut current_stat = MaybeUninit::::uninit(); + if UTIL_fstat(fd, filename, current_stat.as_mut_ptr()) == 0 + || UTIL_isRegularFileStat(current_stat.as_ptr()) == 0 + { + return -1; + } + + let mut result = 0; + #[cfg(unix)] + { + let no_uid = !0 as libc::uid_t; + if fd >= 0 { + result += libc::fchown(fd, no_uid, (*statbuf).st_gid); + } else { + result += libc::chown(filename, no_uid, (*statbuf).st_gid); + } + } + + let permissions = ((*statbuf).st_mode as u64 & 0o777) as UtilMode; + result += UTIL_fchmod(fd, filename, current_stat.as_ptr(), permissions); + + #[cfg(unix)] + { + let no_gid = !0 as libc::gid_t; + if fd >= 0 { + result += libc::fchown(fd, (*statbuf).st_uid, no_gid); + } else { + result += libc::chown(filename, (*statbuf).st_uid, no_gid); + } + } + -result +} + +#[no_mangle] +pub unsafe extern "C" fn UTIL_requireUserConfirmation( + prompt: *const c_char, + abort_msg: *const c_char, + acceptable_letters: *const c_char, + has_stdin_input: c_int, +) -> c_int { + if has_stdin_input != 0 { + display("stdin is an input - not proceeding.\n"); + return 1; + } + + display_c_string(prompt); + let ch = libc::getchar(); + let accepted = if acceptable_letters.is_null() { + false + } else { + c_bytes(acceptable_letters).contains(&(ch as u8)) + }; + let result = if accepted { 0 } else { 1 }; + if result != 0 { + display_c_string(abort_msg); + display(" \n"); + } + let mut next = ch; + while next != libc::EOF && next != b'\n' as c_int { + next = libc::getchar(); + } + result +} + +#[no_mangle] +pub extern "C" fn UTIL_traceFileStat() { + TRACE_FILE_STAT.store(true, Ordering::Relaxed); +} + +#[no_mangle] +pub unsafe extern "C" fn UTIL_isConsole(file: *mut libc::FILE) -> c_int { + if file.is_null() { + return 0; + } + let fd = libc::fileno(file); + if fd == 0 && FAKE_STDIN_IS_CONSOLE.load(Ordering::Relaxed) + || fd == 1 && FAKE_STDOUT_IS_CONSOLE.load(Ordering::Relaxed) + || fd == 2 && FAKE_STDERR_IS_CONSOLE.load(Ordering::Relaxed) + { + return 1; + } + if fd < 0 { + return 0; + } + libc::isatty(fd) +} + +#[no_mangle] +pub extern "C" fn UTIL_fakeStdinIsConsole() { + FAKE_STDIN_IS_CONSOLE.store(true, Ordering::Relaxed); +} + +#[no_mangle] +pub extern "C" fn UTIL_fakeStdoutIsConsole() { + FAKE_STDOUT_IS_CONSOLE.store(true, Ordering::Relaxed); +} + +#[no_mangle] +pub extern "C" fn UTIL_fakeStderrIsConsole() { + FAKE_STDERR_IS_CONSOLE.store(true, Ordering::Relaxed); +} + +fn suffix_ptr(suffix: &'static [u8]) -> *const c_char { + suffix.as_ptr().cast::() +} + +#[no_mangle] +pub extern "C" fn UTIL_makeHumanReadableSize(size: u64) -> UTIL_HumanReadableSize_t { + let result; + if display_level() > 3 { + if size >= 1u64 << 53 { + result = UTIL_HumanReadableSize_t { + value: size as f64 / (1u64 << 20) as f64, + precision: 2, + suffix: suffix_ptr(SUFFIX_MIB), + }; + } else { + result = UTIL_HumanReadableSize_t { + value: size as f64, + precision: 0, + suffix: suffix_ptr(SUFFIX_B), + }; + } + } else { + let (value, suffix) = if size >= 1u64 << 60 { + (size as f64 / (1u64 << 60) as f64, SUFFIX_EIB) + } else if size >= 1u64 << 50 { + (size as f64 / (1u64 << 50) as f64, SUFFIX_PIB) + } else if size >= 1u64 << 40 { + (size as f64 / (1u64 << 40) as f64, SUFFIX_TIB) + } else if size >= 1u64 << 30 { + (size as f64 / (1u64 << 30) as f64, SUFFIX_GIB) + } else if size >= 1u64 << 20 { + (size as f64 / (1u64 << 20) as f64, SUFFIX_MIB) + } else if size >= 1u64 << 10 { + (size as f64 / (1u64 << 10) as f64, SUFFIX_KIB) + } else { + (size as f64, SUFFIX_B) + }; + let precision = if value >= 100.0 || value as u64 == size { + 0 + } else if value >= 10.0 { + 1 + } else if value > 1.0 { + 2 + } else { + 3 + }; + result = UTIL_HumanReadableSize_t { + value, + precision, + suffix: suffix_ptr(suffix), + }; + } + result +} + +#[no_mangle] +pub unsafe extern "C" fn UTIL_compareStr(p1: *const c_void, p2: *const c_void) -> c_int { + let left = *(p1.cast::<*const c_char>()); + let right = *(p2.cast::<*const c_char>()); + let left = c_bytes(left); + let right = c_bytes(right); + for (&a, &b) in left.iter().zip(right.iter()) { + if a != b { + return a as c_int - b as c_int; + } + } + left.len() as c_int - right.len() as c_int +} + +#[no_mangle] +pub unsafe extern "C" fn UTIL_getFileExtension(infilename: *const c_char) -> *const c_char { + let bytes = c_bytes(infilename); + let Some(position) = bytes.iter().rposition(|&byte| byte == b'.') else { + return EMPTY_EXTENSION.as_ptr().cast::(); + }; + if position == 0 { + EMPTY_EXTENSION.as_ptr().cast::() + } else { + infilename.add(position) + } +} + +#[no_mangle] +pub unsafe extern "C" fn UTIL_isCompressedFile( + input_name: *const c_char, + extension_list: *const *const c_char, +) -> c_int { + let extension = CStr::from_ptr(UTIL_getFileExtension(input_name)).to_bytes(); + if extension_list.is_null() { + return 0; + } + let mut current = extension_list; + loop { + let candidate = *current; + if candidate.is_null() { + return 0; + } + if c_bytes(candidate) == extension { + return 1; + } + current = current.add(1); + } +} + +unsafe fn pathname_has_two_dots(pathname: &[u8]) -> bool { + let separator = path_separator(); + for (index, pair) in pathname.windows(2).enumerate() { + if pair != b".." { + continue; + } + let left_boundary = index == 0 || pathname[index - 1] == separator; + let right_index = index + 2; + let right_boundary = right_index == pathname.len() || pathname[right_index] == separator; + if left_boundary && right_boundary { + return true; + } + } + false +} + +fn trim_path(pathname: &[u8]) -> &[u8] { + let separator = path_separator(); + let mut path = pathname; + if path.first() == Some(&separator) { + path = &path[1..]; + } + if path.len() >= 2 && path[0] == b'.' && path[1] == separator { + path = &path[2..]; + } + path +} + +fn join_two_dirs(dir1: &[u8], dir2: &[u8]) -> Vec { + let separator = path_separator(); + let mut result = Vec::with_capacity(dir1.len() + dir2.len() + 1); + result.extend_from_slice(dir1); + if !dir1.is_empty() && dir1.last() != Some(&separator) { + result.push(separator); + } + result.extend_from_slice(dir2); + result +} + +fn convert_pathname_to_dir_name(pathname: &mut Vec) { + let separator = path_separator(); + if let Some(position) = pathname.iter().rposition(|&byte| byte == separator) { + pathname.truncate(position); + } else { + pathname.clear(); + pathname.push(b'.'); + } +} + +unsafe fn get_dir_mode(dir_name: &[u8]) -> u32 { + let path = c_path(dir_name); + let mut statbuf = MaybeUninit::::uninit(); + if UTIL_stat(path.as_ptr(), statbuf.as_mut_ptr()) == 0 { + if display_level() >= 1 { + eprintln!( + "zstd: failed to get DIR stats {}", + String::from_utf8_lossy(dir_name) + ); + } + return DIR_DEFAULT_MODE; + } + if UTIL_isDirectoryStat(statbuf.as_ptr()) == 0 { + if display_level() >= 1 { + eprintln!( + "zstd: expected directory: {}", + String::from_utf8_lossy(dir_name) + ); + } + return DIR_DEFAULT_MODE; + } + stat_mode(statbuf.as_ptr()) as u32 +} + +unsafe fn make_dir(dir: &[u8], mode: u32) -> c_int { + let path = c_path(dir); + #[cfg(unix)] + let result = libc::mkdir(path.as_ptr(), mode as libc::mode_t); + #[cfg(windows)] + let result = { + let _ = mode; + libc::mkdir(path.as_ptr()) + }; + #[cfg(not(any(unix, windows)))] + let result = { + let _ = (path, mode); + -1 + }; + if result != 0 { + if std::io::Error::last_os_error().raw_os_error() == Some(libc::EEXIST) { + return 0; + } + if display_level() >= 1 { + eprintln!( + "zstd: failed to create DIR {}: {}", + String::from_utf8_lossy(dir), + std::io::Error::last_os_error() + ); + } + } + result +} + +unsafe fn mirror_src_dir(src_dir_name: &[u8], out_dir_name: &[u8]) -> c_int { + let new_dir = join_two_dirs(out_dir_name, trim_path(src_dir_name)); + let mode = get_dir_mode(src_dir_name); + make_dir(&new_dir, mode) +} + +fn mirror_src_dir_recursive(src_dir_name: &[u8], out_dir_name: &[u8]) { + let separator = path_separator(); + let start = + if src_dir_name.len() >= 2 && src_dir_name[0] == b'.' && src_dir_name[1] == separator { + 2 + } else { + 0 + }; + let mut previous = start; + for (position, &byte) in src_dir_name.iter().enumerate().skip(start) { + if byte != separator { + continue; + } + if position != previous { + unsafe { + let _ = mirror_src_dir(&src_dir_name[..position], out_dir_name); + } + } + previous = position + 1; + } + unsafe { + let _ = mirror_src_dir(src_dir_name, out_dir_name); + } +} + +fn first_is_parent_or_same_dir(first: &[u8], second: &[u8]) -> bool { + let separator = path_separator(); + first.len() <= second.len() + && (second.get(first.len()) == Some(&separator) || second.get(first.len()).is_none()) + && second.starts_with(first) +} + +#[no_mangle] +pub unsafe extern "C" fn UTIL_createMirroredDestDirName( + src_file_name: *const c_char, + out_dir_root_name: *const c_char, +) -> *mut c_char { + let src = c_bytes(src_file_name); + if pathname_has_two_dots(src) { + return ptr::null_mut(); + } + let mut pathname = join_two_dirs(c_bytes(out_dir_root_name), trim_path(src)); + convert_pathname_to_dir_name(&mut pathname); + let result = malloc_bytes(pathname.len() + 1).cast::(); + ptr::copy_nonoverlapping(pathname.as_ptr().cast::(), result, pathname.len()); + *result.add(pathname.len()) = 0; + result +} + +#[no_mangle] +pub unsafe extern "C" fn UTIL_mirrorSourceFilesDirectories( + file_names: *const *const c_char, + nb_files: c_uint, + out_dir_name: *const c_char, +) { + let out_dir = c_bytes(out_dir_name).to_vec(); + let mut source_dirs = Vec::new(); + for index in 0..nb_files as usize { + let source = c_bytes(*file_names.add(index)); + if !pathname_has_two_dots(source) { + let mut source_dir = source.to_vec(); + convert_pathname_to_dir_name(&mut source_dir); + source_dirs.push(source_dir); + } + } + if source_dirs.is_empty() { + return; + } + + let _ = make_dir(&out_dir, DIR_DEFAULT_MODE); + source_dirs.sort_by(|left, right| trim_path(left).cmp(trim_path(right))); + let mut unique_dirs: Vec<&[u8]> = Vec::new(); + unique_dirs.push(source_dirs[0].as_slice()); + for index in 1..source_dirs.len() { + let previous = trim_path(&source_dirs[index - 1]); + let current = trim_path(&source_dirs[index]); + if first_is_parent_or_same_dir(previous, current) { + *unique_dirs.last_mut().unwrap() = source_dirs[index].as_slice(); + } else { + unique_dirs.push(source_dirs[index].as_slice()); + } + } + for source_dir in unique_dirs { + mirror_src_dir_recursive(source_dir, &out_dir); + } +} + +#[no_mangle] +pub unsafe extern "C" fn UTIL_createFileNamesTable_fromFileName( + input_file_name: *const c_char, +) -> *mut FileNamesTable { + let input_name = c_bytes(input_file_name); + let path = path_from_bytes(input_name); + let c_input_name = c_path(input_name); + let mut statbuf = MaybeUninit::::uninit(); + if UTIL_stat(c_input_name.as_ptr(), statbuf.as_mut_ptr()) == 0 + || UTIL_isRegularFileStat(statbuf.as_ptr()) == 0 + { + return ptr::null_mut(); + } + let file_size = UTIL_getFileSizeStat(statbuf.as_ptr()); + if file_size > MAX_FILE_OF_FILE_NAMES_SIZE { + return ptr::null_mut(); + } + let data = match fs::read(path) { + Ok(data) => data, + Err(error) => { + if display_level() >= 1 { + eprintln!("zstd:util:readLinesFromFile: {error}"); + } + return ptr::null_mut(); + } + }; + if data.is_empty() { + return ptr::null_mut(); + } + + let line_count = data.iter().filter(|&&byte| byte == b'\n').count() + + usize::from(data.last() != Some(&b'\n')); + let buffer = malloc_bytes(data.len() + 1); + ptr::copy_nonoverlapping(data.as_ptr(), buffer, data.len()); + for index in 0..data.len() { + if *buffer.add(index) == b'\n' { + *buffer.add(index) = 0; + } + } + *buffer.add(data.len()) = 0; + make_table_from_buffer(buffer, data.len() + 1, line_count, line_count) +} + +#[no_mangle] +pub unsafe extern "C" fn UTIL_assembleFileNamesTable( + filenames: *mut *const c_char, + table_size: usize, + buf: *mut c_char, +) -> *mut FileNamesTable { + make_table(filenames, table_size, table_size, buf) +} + +#[no_mangle] +pub unsafe extern "C" fn UTIL_freeFileNamesTable(table: *mut FileNamesTable) { + if table.is_null() { + return; + } + free_ptr((*table).fileNames); + free_ptr((*table).buf); + free_ptr(table); +} + +#[no_mangle] +pub unsafe extern "C" fn UTIL_allocateFileNamesTable(table_size: usize) -> *mut FileNamesTable { + let names = malloc_array::<*const c_char>(table_size); + if names.is_null() { + return ptr::null_mut(); + } + make_table(names, 0, table_size, ptr::null_mut()) +} + +#[no_mangle] +pub unsafe extern "C" fn UTIL_searchFileNamesTable( + table: *mut FileNamesTable, + name: *const c_char, +) -> c_int { + for (index, &candidate) in table_entries(table).iter().enumerate() { + if !candidate.is_null() && c_bytes(candidate) == c_bytes(name) { + return index as c_int; + } + } + -1 +} + +#[no_mangle] +pub unsafe extern "C" fn UTIL_refFilename(table: *mut FileNamesTable, filename: *const c_char) { + if table.is_null() || (*table).tableSize >= (*table).tableCapacity { + std::process::abort(); + } + *(*table).fileNames.add((*table).tableSize) = filename; + (*table).tableSize += 1; +} + +#[no_mangle] +pub unsafe extern "C" fn UTIL_mergeFileNamesTable( + table1: *mut FileNamesTable, + table2: *mut FileNamesTable, +) -> *mut FileNamesTable { + let total_size = table_names_size(table1).saturating_add(table_names_size(table2)); + let new_size = (*table1).tableSize.saturating_add((*table2).tableSize); + let buffer = malloc_bytes(total_size.max(1)); + ptr::write_bytes(buffer, 0, total_size.max(1)); + let names = malloc_bytes(new_size.saturating_mul(size_of::<*const c_char>()).max(1)) + .cast::<*const c_char>(); + ptr::write_bytes( + names.cast::(), + 0, + new_size.saturating_mul(size_of::<*const c_char>()), + ); + let mut new_index = 0usize; + let mut position = 0usize; + for source in [table1, table2] { + for &name in table_entries(source) { + if name.is_null() || position >= total_size { + break; + } + let name_bytes = c_bytes(name); + ptr::copy_nonoverlapping(name_bytes.as_ptr(), buffer.add(position), name_bytes.len()); + *names.add(new_index) = buffer.add(position).cast::(); + position += name_bytes.len() + 1; + new_index += 1; + } + } + let result = make_table(names, new_index, 0, buffer.cast::()); + UTIL_freeFileNamesTable(table1); + UTIL_freeFileNamesTable(table2); + result +} + +fn append_name(buffer: &mut Vec, name: &[u8]) { + buffer.extend_from_slice(name); + buffer.push(0); +} + +fn join_entry_path(dir_name: &[u8], entry_name: &[u8]) -> Vec { + let mut path = Vec::with_capacity(dir_name.len() + entry_name.len() + 1); + path.extend_from_slice(dir_name); + path.push(path_separator()); + path.extend_from_slice(entry_name); + path +} + +unsafe fn is_directory_bytes(name: &[u8]) -> bool { + let path = c_path(name); + let mut statbuf = MaybeUninit::::uninit(); + UTIL_stat(path.as_ptr(), statbuf.as_mut_ptr()) != 0 + && UTIL_isDirectoryStat(statbuf.as_ptr()) != 0 +} + +unsafe fn is_link_bytes(name: &[u8]) -> bool { + #[cfg(unix)] + { + let path = c_path(name); + let mut statbuf = MaybeUninit::::uninit(); + libc::lstat(path.as_ptr(), statbuf.as_mut_ptr()) == 0 + && mode_is(statbuf.as_ptr(), libc::S_IFLNK) + } + #[cfg(not(unix))] + { + let _ = name; + false + } +} + +fn walk_directory( + dir_name: &[u8], + buffer: &mut Vec, + nb_files: &mut usize, + follow_links: bool, +) -> Result<(), ()> { + let entries = match fs::read_dir(path_from_bytes(dir_name)) { + Ok(entries) => entries, + Err(error) => { + if display_level() >= 1 { + eprintln!( + "Cannot open directory '{}': {error}", + String::from_utf8_lossy(dir_name) + ); + } + return Ok(()); + } + }; + for entry in entries { + let entry = entry.map_err(|_| ())?; + let entry_name = os_string_bytes(entry.file_name()); + if entry_name == b"." || entry_name == b".." { + continue; + } + let path = join_entry_path(dir_name, &entry_name); + if !follow_links && unsafe { is_link_bytes(&path) } { + if display_level() >= 2 { + eprintln!( + "Warning : {} is a symbolic link, ignoring", + String::from_utf8_lossy(&path) + ); + } + continue; + } + if unsafe { is_directory_bytes(&path) } { + walk_directory(&path, buffer, nb_files, follow_links)?; + } else { + append_name(buffer, &path); + *nb_files += 1; + } + } + Ok(()) +} + +#[no_mangle] +pub unsafe extern "C" fn UTIL_createExpandedFNT( + input_names: *const *const c_char, + nb_input_names: usize, + follow_links: c_int, +) -> *mut FileNamesTable { + let mut buffer = Vec::with_capacity(LIST_SIZE_INCREASE); + let mut nb_files = 0usize; + for index in 0..nb_input_names { + let input = *input_names.add(index); + let input_bytes = c_bytes(input); + if !is_directory_bytes(input_bytes) { + append_name(&mut buffer, input_bytes); + nb_files += 1; + } else if walk_directory(input_bytes, &mut buffer, &mut nb_files, follow_links != 0) + .is_err() + { + return ptr::null_mut(); + } + } + + let table_capacity = nb_files.saturating_add(1); + let allocated_len = buffer.len().max(LIST_SIZE_INCREASE); + let output_buffer = malloc_bytes(allocated_len); + if !buffer.is_empty() { + ptr::copy_nonoverlapping(buffer.as_ptr(), output_buffer, buffer.len()); + } + make_table_from_buffer(output_buffer, buffer.len(), nb_files, table_capacity) +} + +#[no_mangle] +pub unsafe extern "C" fn UTIL_expandFNT(table: *mut *mut FileNamesTable, follow_links: c_int) { + let old_table = *table; + let new_table = + UTIL_createExpandedFNT((*old_table).fileNames, (*old_table).tableSize, follow_links); + if new_table.is_null() { + std::process::abort(); + } + UTIL_freeFileNamesTable(old_table); + *table = new_table; +} + +#[no_mangle] +pub unsafe extern "C" fn UTIL_createFNT_fromROTable( + filenames: *const *const c_char, + nb_filenames: usize, +) -> *mut FileNamesTable { + let new_names = malloc_array::<*const c_char>(nb_filenames); + if new_names.is_null() { + return ptr::null_mut(); + } + ptr::copy_nonoverlapping(filenames, new_names, nb_filenames); + UTIL_assembleFileNamesTable(new_names, nb_filenames, ptr::null_mut()) +} + +#[cfg(windows)] +#[no_mangle] +pub extern "system" fn CountSetBits(bit_mask: usize) -> u32 { + bit_mask.count_ones() +} + +#[cfg(target_os = "linux")] +fn count_cores_platform(logical: c_int) -> c_int { + let online = unsafe { libc::sysconf(libc::_SC_NPROCESSORS_ONLN) }; + let mut count = if online < 0 { 1 } else { online as c_int }; + if logical != 0 { + return count; + } + let Ok(cpuinfo) = fs::read_to_string("/proc/cpuinfo") else { + return count; + }; + let mut siblings = 0; + let mut cpu_cores = 0; + for line in cpuinfo.lines() { + if line.starts_with("siblings") { + let Some((_, value)) = line.split_once(':') else { + return count; + }; + let Ok(value) = value.trim().parse::() else { + return count; + }; + siblings = value; + } + if line.starts_with("cpu cores") { + let Some((_, value)) = line.split_once(':') else { + return count; + }; + let Ok(value) = value.trim().parse::() else { + return count; + }; + cpu_cores = value; + } + } + if siblings > cpu_cores && cpu_cores > 0 { + let ratio = siblings / cpu_cores; + if ratio > 0 && count > ratio { + count /= ratio; + } + } + count +} + +#[cfg(all(unix, not(target_os = "linux"), not(target_vendor = "apple")))] +fn count_cores_platform(_logical: c_int) -> c_int { + let count = unsafe { libc::sysconf(libc::_SC_NPROCESSORS_ONLN) }; + if count < 0 { + 1 + } else { + count as c_int + } +} + +#[cfg(target_vendor = "apple")] +fn count_cores_platform(logical: c_int) -> c_int { + let name = if logical != 0 { + CString::new("hw.logicalcpu").unwrap() + } else { + CString::new("hw.physicalcpu").unwrap() + }; + let mut count = 0i32; + let mut size = size_of::(); + let result = unsafe { + libc::sysctlbyname( + name.as_ptr(), + (&mut count as *mut i32).cast::(), + &mut size, + ptr::null_mut(), + 0, + ) + }; + if result != 0 { + if std::io::Error::last_os_error().raw_os_error() == Some(libc::ENOENT) { + 1 + } else { + std::process::abort() + } + } else { + count + } +} + +#[cfg(windows)] +fn count_cores_platform(_logical: c_int) -> c_int { + std::thread::available_parallelism() + .map_or(1, |count| count.get().min(c_int::MAX as usize) as c_int) +} + +#[cfg(not(any(unix, windows)))] +fn count_cores_platform(_logical: c_int) -> c_int { + 1 +} + +#[no_mangle] +pub extern "C" fn UTIL_countCores(logical: c_int) -> c_int { + *CORE_COUNT.get_or_init(|| count_cores_platform(logical)) +} + +#[no_mangle] +pub extern "C" fn UTIL_countPhysicalCores() -> c_int { + UTIL_countCores(0) +} + +#[no_mangle] +pub extern "C" fn UTIL_countLogicalCores() -> c_int { + UTIL_countCores(1) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::ffi::CString; + use std::fs::{self, File}; + use std::io::Write; + + #[test] + fn public_struct_layout_is_pointer_and_c_int_stable() { + assert_eq!( + std::mem::align_of::(), + std::mem::align_of::() + ); + assert_eq!(offset_of!(UTIL_HumanReadableSize_t, value), 0); + assert_eq!( + offset_of!(UTIL_HumanReadableSize_t, precision), + size_of::() + ); + assert_eq!(offset_of!(FileNamesTable, fileNames), 0); + assert_eq!(offset_of!(FileNamesTable, buf), size_of::<*const c_char>()); + assert_eq!( + offset_of!(FileNamesTable, tableSize), + 2 * size_of::<*const c_char>() + ); + assert_eq!( + size_of::(), + 2 * size_of::<*const c_char>() + 2 * size_of::() + ); + } + + #[test] + fn stat_extension_and_size_helpers_follow_c_contract() { + let root = std::env::temp_dir().join(format!("zstd-util-{}", std::process::id())); + let _ = fs::remove_dir_all(&root); + fs::create_dir(&root).unwrap(); + let file_path = root.join("sample.txt"); + fs::write(&file_path, b"hello").unwrap(); + let path = CString::new(file_path.to_string_lossy().as_bytes()).unwrap(); + let mut statbuf = MaybeUninit::::uninit(); + unsafe { + assert_eq!(UTIL_stat(path.as_ptr(), statbuf.as_mut_ptr()), 1); + assert_eq!(UTIL_isRegularFileStat(statbuf.as_ptr()), 1); + assert_eq!(UTIL_getFileSizeStat(statbuf.as_ptr()), 5); + assert_eq!( + CStr::from_ptr(UTIL_getFileExtension(path.as_ptr())).to_bytes(), + b".txt" + ); + let extension = CString::new(".txt").unwrap(); + let extensions = [extension.as_ptr(), ptr::null()]; + assert_eq!(UTIL_isCompressedFile(path.as_ptr(), extensions.as_ptr()), 1); + } + fs::remove_dir_all(root).unwrap(); + } + + #[test] + fn filename_tables_own_c_allocations_and_merge_in_order() { + let first = CString::new("first").unwrap(); + let second = CString::new("second").unwrap(); + let names = [first.as_ptr(), second.as_ptr()]; + unsafe { + let table = UTIL_createFNT_fromROTable(names.as_ptr(), names.len()); + assert!(!table.is_null()); + assert_eq!((*table).tableSize, 2); + assert_eq!(UTIL_searchFileNamesTable(table, second.as_ptr()), 1); + let empty = UTIL_allocateFileNamesTable(1); + assert!(!empty.is_null()); + UTIL_refFilename(empty, first.as_ptr()); + let merged = UTIL_mergeFileNamesTable(table, empty); + assert_eq!((*merged).tableSize, 3); + assert_eq!( + CStr::from_ptr(*(*merged).fileNames.add(2)).to_bytes(), + b"first" + ); + UTIL_freeFileNamesTable(merged); + } + } + + #[test] + fn file_list_loading_and_directory_expansion_return_nul_tables() { + let mut root = std::env::temp_dir(); + root.push(format!("zstd-util-list-{}", std::process::id())); + let _ = fs::remove_dir_all(&root); + fs::create_dir(&root).unwrap(); + let file_list = root.join("list"); + let nested = root.join("nested"); + fs::create_dir(&nested).unwrap(); + File::create(nested.join("one")).unwrap(); + let mut list = File::create(&file_list).unwrap(); + writeln!(list, "alpha").unwrap(); + writeln!(list, "beta").unwrap(); + drop(list); + let list_name = CString::new(file_list.to_string_lossy().as_bytes()).unwrap(); + let nested_name = CString::new(nested.to_string_lossy().as_bytes()).unwrap(); + unsafe { + let table = UTIL_createFileNamesTable_fromFileName(list_name.as_ptr()); + assert!(!table.is_null()); + assert_eq!((*table).tableSize, 2); + assert_eq!(CStr::from_ptr(*(*table).fileNames).to_bytes(), b"alpha"); + UTIL_freeFileNamesTable(table); + + let input = [nested_name.as_ptr()]; + let expanded = UTIL_createExpandedFNT(input.as_ptr(), 1, 1); + assert!(!expanded.is_null()); + assert_eq!((*expanded).tableSize, 1); + assert!(CStr::from_ptr(*(*expanded).fileNames) + .to_bytes() + .ends_with(b"/one")); + UTIL_freeFileNamesTable(expanded); + } + fs::remove_dir_all(root).unwrap(); + } + + #[test] + fn mirrored_directory_name_rejects_parent_components() { + let source = CString::new("a/../b/file").unwrap(); + let root = CString::new("out").unwrap(); + unsafe { + assert!(UTIL_createMirroredDestDirName(source.as_ptr(), root.as_ptr()).is_null()); + } + let source = CString::new("a/b/file.txt").unwrap(); + unsafe { + let name = UTIL_createMirroredDestDirName(source.as_ptr(), root.as_ptr()); + assert_eq!(CStr::from_ptr(name).to_bytes(), b"out/a/b"); + free_ptr(name); + } + } + + #[test] + fn human_readable_size_uses_the_c_scaling_policy() { + unsafe { + g_utilDisplayLevel = 0; + } + let size = UTIL_makeHumanReadableSize(1536); + assert_eq!(size.precision, 2); + unsafe { + assert_eq!(CStr::from_ptr(size.suffix).to_bytes(), b" KiB"); + } + assert_eq!(size.value, 1.5); + } + + #[test] + fn core_count_helpers_never_report_zero() { + assert!(UTIL_countPhysicalCores() > 0); + assert!(UTIL_countLogicalCores() > 0); + } +}