ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(cli): port utility and dictionary I/O helpers to Rust

Browse Source 
Move filename tables, file-list expansion, core-count helpers, statistics,
and dictionary sample loading behind Rust implementations while retaining
the narrow C ABI used by the CLI.

Test Plan:
- RUSTC_WRAPPER= CARGO_BUILD_RUSTC_WRAPPER= cargo test --manifest-path rust/cli/Cargo.toml
- RUSTC_WRAPPER= CARGO_BUILD_RUSTC_WRAPPER= cargo clippy --manifest-path rust/cli/Cargo.toml --all-targets -- -D warnings
- make -B -C programs zstd V=1
- make -C tests check V=1
This commit is contained in:
ddidderr
2026-07-12 18:06:51 +02:00
parent 0d3dae8fd8
commit cd7ae43da7
6 changed files with 2580 additions and 2058 deletions
Download Patch File Download Diff File Expand all files Collapse all files
programs/Makefile
+2
View File
@@ -34,6 +34,8 @@ RUST_SOURCES := $(RUST_MANIFEST) $(RUST_DIR)/Cargo.lock \
RUST_CLI_SOURCES := $(RUST_CLI_MANIFEST) $(RUST_CLI_DIR)/Cargo.lock \
$(RUST_CLI_DIR)/src/lib.rs $(RUST_DIR)/src/zstd_cli.rs \
$(RUST_DIR)/src/fileio_prefs.rs \
$(RUST_DIR)/src/dibio.rs \
$(RUST_DIR)/src/util.rs \
$(RUST_DIR)/src/timefn.rs $(RUST_DIR)/src/benchfn.rs \
$(RUST_DIR)/src/datagen.rs $(RUST_DIR)/src/lorem.rs
programs/dibio.c
+3 -429
View File
@@ -8,433 +8,7 @@
* You may select, at your option, one of the above-listed licenses.
*/
/* **************************************
* Compiler Warnings
****************************************/
#ifdef _MSC_VER
# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
#endif
/*-*************************************
* Includes
***************************************/
#include "platform.h" /* Large Files support */
#include "util.h" /* UTIL_getFileSize, UTIL_getTotalFileSize */
#include <stdlib.h> /* malloc, free */
#include <string.h> /* memset */
#include <stdio.h> /* fprintf, fopen, ftello64 */
#include <errno.h> /* errno */
#include "timefn.h" /* UTIL_time_t, UTIL_clockSpanMicro, UTIL_getTime */
#include "../lib/common/debug.h" /* assert */
#include "../lib/common/mem.h" /* read */
#include "../lib/zstd_errors.h"
/* The dictionary I/O implementation lives in rust/src/dibio.rs. Keep this
* translation unit in the existing C program source lists so the public
* dibio.h declaration remains the ABI boundary used by the CLI. */
#include "dibio.h"
/*-*************************************
* Constants
***************************************/
#define KB *(1 <<10)
#define MB *(1 <<20)
#define GB *(1U<<30)
#define SAMPLESIZE_MAX (128 KB)
#define MEMMULT 11 /* rough estimation : memory cost to analyze 1 byte of sample */
#define COVER_MEMMULT 9 /* rough estimation : memory cost to analyze 1 byte of sample */
#define FASTCOVER_MEMMULT 1 /* rough estimation : memory cost to analyze 1 byte of sample */
static const size_t g_maxMemory = (sizeof(size_t) == 4) ? (2 GB - 64 MB) : ((size_t)(512 MB) << sizeof(size_t));
#define NOISELENGTH 32
#define MAX_SAMPLES_SIZE (2 GB) /* training dataset limited to 2GB */
/*-*************************************
* Console display
***************************************/
#define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
#define DISPLAYLEVEL(l, ...) if (displayLevel>=l) { DISPLAY(__VA_ARGS__); }
static const U64 g_refreshRate = SEC_TO_MICRO / 6;
static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER;
#define DISPLAYUPDATE(l, ...) { if (displayLevel>=l) { \
if ((UTIL_clockSpanMicro(g_displayClock) > g_refreshRate) || (displayLevel>=4)) \
{ g_displayClock = UTIL_getTime(); DISPLAY(__VA_ARGS__); \
if (displayLevel>=4) fflush(stderr); } } }
/*-*************************************
* Exceptions
***************************************/
#ifndef DEBUG
# define DEBUG 0
#endif
#define DEBUGOUTPUT(...) if (DEBUG) DISPLAY(__VA_ARGS__);
#define EXM_THROW(error, ...) \
{ \
DEBUGOUTPUT("Error defined at %s, line %i : \n", __FILE__, __LINE__); \
DISPLAY("Error %i : ", error); \
DISPLAY(__VA_ARGS__); \
DISPLAY("\n"); \
exit(error); \
}
/* ********************************************************
* Helper functions
**********************************************************/
#undef MIN
#define MIN(a,b) ((a) < (b) ? (a) : (b))
/**
Returns the size of a file.
If error returns -1.
*/
static S64 DiB_getFileSize (const char * fileName)
{
U64 const fileSize = UTIL_getFileSize(fileName);
return (fileSize == UTIL_FILESIZE_UNKNOWN) ? -1 : (S64)fileSize;
}
/* ********************************************************
* File related operations
**********************************************************/
/** DiB_loadFiles() :
* load samples from files listed in fileNamesTable into buffer.
* works even if buffer is too small to load all samples.
* Also provides the size of each sample into sampleSizes table
* which must be sized correctly, using DiB_fileStats().
* @return : nb of samples effectively loaded into `buffer`
* *bufferSizePtr is modified, it provides the amount data loaded within buffer.
* sampleSizes is filled with the size of each sample.
*/
static int DiB_loadFiles(
void* buffer, size_t* bufferSizePtr,
size_t* sampleSizes, int sstSize,
const char** fileNamesTable, int nbFiles,
size_t targetChunkSize, int displayLevel )
{
char* const buff = (char*)buffer;
size_t totalDataLoaded = 0;
int nbSamplesLoaded = 0;
int fileIndex = 0;
FILE * f = NULL;
assert(targetChunkSize <= SAMPLESIZE_MAX);
while ( nbSamplesLoaded < sstSize && fileIndex < nbFiles ) {
size_t fileDataLoaded;
S64 const fileSize = DiB_getFileSize(fileNamesTable[fileIndex]);
if (fileSize <= 0) {
/* skip if zero-size or file error */
++fileIndex;
continue;
}
f = fopen( fileNamesTable[fileIndex], "rb");
if (f == NULL)
EXM_THROW(10, "zstd: dictBuilder: %s %s ", fileNamesTable[fileIndex], strerror(errno));
DISPLAYUPDATE(2, "Loading %s... \r", fileNamesTable[fileIndex]);
/* Load the first chunk of data from the file */
fileDataLoaded = targetChunkSize > 0 ?
(size_t)MIN(fileSize, (S64)targetChunkSize) :
(size_t)MIN(fileSize, SAMPLESIZE_MAX );
if (totalDataLoaded + fileDataLoaded > *bufferSizePtr)
break;
if (fread( buff+totalDataLoaded, 1, fileDataLoaded, f ) != fileDataLoaded)
EXM_THROW(11, "Pb reading %s", fileNamesTable[fileIndex]);
sampleSizes[nbSamplesLoaded++] = fileDataLoaded;
totalDataLoaded += fileDataLoaded;
/* If file-chunking is enabled, load the rest of the file as more samples */
if (targetChunkSize > 0) {
while( (S64)fileDataLoaded < fileSize && nbSamplesLoaded < sstSize ) {
size_t const chunkSize = MIN((size_t)(fileSize-fileDataLoaded), targetChunkSize);
if (totalDataLoaded + chunkSize > *bufferSizePtr) /* buffer is full */
break;
if (fread( buff+totalDataLoaded, 1, chunkSize, f ) != chunkSize)
EXM_THROW(11, "Pb reading %s", fileNamesTable[fileIndex]);
sampleSizes[nbSamplesLoaded++] = chunkSize;
totalDataLoaded += chunkSize;
fileDataLoaded += chunkSize;
}
}
fileIndex += 1;
fclose(f); f = NULL;
}
if (f != NULL)
fclose(f);
DISPLAYLEVEL(2, "\r%79s\r", "");
DISPLAYLEVEL(4, "Loaded %d KB total training data, %d nb samples \n",
(int)(totalDataLoaded / (1 KB)), nbSamplesLoaded );
*bufferSizePtr = totalDataLoaded;
return nbSamplesLoaded;
}
#define DiB_rotl32(x,r) ((x << r) | (x >> (32 - r)))
static U32 DiB_rand(U32* src)
{
static const U32 prime1 = 2654435761U;
static const U32 prime2 = 2246822519U;
U32 rand32 = *src;
rand32 *= prime1;
rand32 ^= prime2;
rand32 = DiB_rotl32(rand32, 13);
*src = rand32;
return rand32 >> 5;
}
/* DiB_shuffle() :
* shuffle a table of file names in a semi-random way
* It improves dictionary quality by reducing "locality" impact, so if sample set is very large,
* it will load random elements from it, instead of just the first ones. */
static void DiB_shuffle(const char** fileNamesTable, unsigned nbFiles) {
U32 seed = 0xFD2FB528;
unsigned i;
if (nbFiles == 0)
return;
for (i = nbFiles - 1; i > 0; --i) {
unsigned const j = DiB_rand(&seed) % (i + 1);
const char* const tmp = fileNamesTable[j];
fileNamesTable[j] = fileNamesTable[i];
fileNamesTable[i] = tmp;
}
}
/*-********************************************************
* Dictionary training functions
**********************************************************/
static size_t DiB_findMaxMem(unsigned long long requiredMem)
{
size_t const step = 8 MB;
void* testmem = NULL;
requiredMem = (((requiredMem >> 23) + 1) << 23);
requiredMem += step;
if (requiredMem > g_maxMemory) requiredMem = g_maxMemory;
while (!testmem) {
testmem = malloc((size_t)requiredMem);
requiredMem -= step;
}
free(testmem);
return (size_t)requiredMem;
}
static void DiB_fillNoise(void* buffer, size_t length)
{
unsigned const prime1 = 2654435761U;
unsigned const prime2 = 2246822519U;
unsigned acc = prime1;
size_t p=0;
for (p=0; p<length; p++) {
acc *= prime2;
((unsigned char*)buffer)[p] = (unsigned char)(acc >> 21);
}
}
static void DiB_saveDict(const char* dictFileName,
const void* buff, size_t buffSize)
{
FILE* const f = fopen(dictFileName, "wb");
if (f==NULL) EXM_THROW(3, "cannot open %s ", dictFileName);
{ size_t const n = fwrite(buff, 1, buffSize, f);
if (n!=buffSize) EXM_THROW(4, "%s : write error", dictFileName) }
{ size_t const n = (size_t)fclose(f);
if (n!=0) EXM_THROW(5, "%s : flush error", dictFileName) }
}
typedef struct {
S64 totalSizeToLoad;
int nbSamples;
int oneSampleTooLarge;
} fileStats;
/*! DiB_fileStats() :
* Given a list of files, and a chunkSize (0 == no chunk, whole files)
* provides the amount of data to be loaded and the resulting nb of samples.
* This is useful primarily for allocation purpose => sample buffer, and sample sizes table.
*/
static fileStats DiB_fileStats(const char** fileNamesTable, int nbFiles, size_t chunkSize, int displayLevel)
{
fileStats fs;
int n;
memset(&fs, 0, sizeof(fs));
/* We assume that if chunking is requested, the chunk size is < SAMPLESIZE_MAX */
assert( chunkSize <= SAMPLESIZE_MAX );
for (n=0; n<nbFiles; n++) {
S64 const fileSize = DiB_getFileSize(fileNamesTable[n]);
/* TODO: is there a minimum sample size? What if the file is 1-byte? */
if (fileSize == 0) {
DISPLAYLEVEL(3, "Sample file '%s' has zero size, skipping...\n", fileNamesTable[n]);
continue;
}
/* the case where we are breaking up files in sample chunks */
if (chunkSize > 0) {
/* TODO: is there a minimum sample size? Can we have a 1-byte sample? */
fs.nbSamples += (int)((fileSize + chunkSize-1) / chunkSize);
fs.totalSizeToLoad += fileSize;
}
else {
/* the case where one file is one sample */
if (fileSize > SAMPLESIZE_MAX) {
/* flag excessively large sample files */
fs.oneSampleTooLarge |= (fileSize > 2*SAMPLESIZE_MAX);
/* Limit to the first SAMPLESIZE_MAX (128kB) of the file */
DISPLAYLEVEL(3, "Sample file '%s' is too large, limiting to %d KB\n",
fileNamesTable[n], SAMPLESIZE_MAX / (1 KB));
}
fs.nbSamples += 1;
fs.totalSizeToLoad += MIN(fileSize, SAMPLESIZE_MAX);
}
}
DISPLAYLEVEL(4, "Found training data %d files, %d KB, %d samples\n", nbFiles, (int)(fs.totalSizeToLoad / (1 KB)), fs.nbSamples);
return fs;
}
int DiB_trainFromFiles(const char* dictFileName, size_t maxDictSize,
const char** fileNamesTable, int nbFiles, size_t chunkSize,
ZDICT_legacy_params_t* params, ZDICT_cover_params_t* coverParams,
ZDICT_fastCover_params_t* fastCoverParams, int optimize, unsigned memLimit)
{
fileStats fs;
size_t* sampleSizes; /* vector of sample sizes. Each sample can be up to SAMPLESIZE_MAX */
int nbSamplesLoaded; /* nb of samples effectively loaded in srcBuffer */
size_t loadedSize; /* total data loaded in srcBuffer for all samples */
void* srcBuffer /* contiguous buffer with training data/samples */;
void* const dictBuffer = malloc(maxDictSize);
int result = 0;
int const displayLevel = params ? params->zParams.notificationLevel :
coverParams ? coverParams->zParams.notificationLevel :
fastCoverParams ? fastCoverParams->zParams.notificationLevel : 0;
/* Shuffle input files before we start assessing how much sample datA to load.
The purpose of the shuffle is to pick random samples when the sample
set is larger than what we can load in memory. */
DISPLAYLEVEL(3, "Shuffling input files\n");
DiB_shuffle(fileNamesTable, nbFiles);
/* Figure out how much sample data to load with how many samples */
fs = DiB_fileStats(fileNamesTable, nbFiles, chunkSize, displayLevel);
{
int const memMult = params ? MEMMULT :
coverParams ? COVER_MEMMULT:
FASTCOVER_MEMMULT;
size_t const maxMem = DiB_findMaxMem(fs.totalSizeToLoad * memMult) / memMult;
/* Limit the size of the training data to the free memory */
/* Limit the size of the training data to 2GB */
/* TODO: there is opportunity to stop DiB_fileStats() early when the data limit is reached */
loadedSize = (size_t)MIN( MIN((S64)maxMem, fs.totalSizeToLoad), MAX_SAMPLES_SIZE );
if (memLimit != 0) {
DISPLAYLEVEL(2, "! Warning : setting manual memory limit for dictionary training data at %u MB \n",
(unsigned)(memLimit / (1 MB)));
loadedSize = (size_t)MIN(loadedSize, memLimit);
}
srcBuffer = malloc(loadedSize+NOISELENGTH);
sampleSizes = (size_t*)malloc(fs.nbSamples * sizeof(size_t));
}
/* Checks */
if ((fs.nbSamples && !sampleSizes) || (!srcBuffer) || (!dictBuffer))
EXM_THROW(12, "not enough memory for DiB_trainFiles"); /* should not happen */
if (fs.oneSampleTooLarge) {
DISPLAYLEVEL(2, "! Warning : some sample(s) are very large \n");
DISPLAYLEVEL(2, "! Note that dictionary is only useful for small samples. \n");
DISPLAYLEVEL(2, "! As a consequence, only the first %u bytes of each sample are loaded \n", SAMPLESIZE_MAX);
}
if (fs.nbSamples < 5) {
DISPLAYLEVEL(2, "! Warning : nb of samples too low for proper processing ! \n");
DISPLAYLEVEL(2, "! Please provide _one file per sample_. \n");
DISPLAYLEVEL(2, "! Alternatively, split files into fixed-size blocks representative of samples, with -B# \n");
EXM_THROW(14, "nb of samples too low"); /* we now clearly forbid this case */
}
if (fs.totalSizeToLoad < (S64)maxDictSize * 8) {
DISPLAYLEVEL(2, "! Warning : data size of samples too small for target dictionary size \n");
DISPLAYLEVEL(2, "! Samples should be about 100x larger than target dictionary size \n");
}
/* init */
if ((S64)loadedSize < fs.totalSizeToLoad)
DISPLAYLEVEL(1, "Training samples set too large (%u MB); training on %u MB only...\n",
(unsigned)(fs.totalSizeToLoad / (1 MB)),
(unsigned)(loadedSize / (1 MB)));
/* Load input buffer */
nbSamplesLoaded = DiB_loadFiles(
srcBuffer, &loadedSize, sampleSizes, fs.nbSamples, fileNamesTable,
nbFiles, chunkSize, displayLevel);
{ size_t dictSize = ZSTD_error_GENERIC;
if (params) {
DiB_fillNoise((char*)srcBuffer + loadedSize, NOISELENGTH); /* guard band, for end of buffer condition */
dictSize = ZDICT_trainFromBuffer_legacy(dictBuffer, maxDictSize,
srcBuffer, sampleSizes, nbSamplesLoaded,
*params);
} else if (coverParams) {
if (optimize) {
dictSize = ZDICT_optimizeTrainFromBuffer_cover(dictBuffer, maxDictSize,
srcBuffer, sampleSizes, nbSamplesLoaded,
coverParams);
if (!ZDICT_isError(dictSize)) {
unsigned splitPercentage = (unsigned)(coverParams->splitPoint * 100);
DISPLAYLEVEL(2, "k=%u\nd=%u\nsteps=%u\nsplit=%u\n", coverParams->k, coverParams->d,
coverParams->steps, splitPercentage);
}
} else {
dictSize = ZDICT_trainFromBuffer_cover(dictBuffer, maxDictSize, srcBuffer,
sampleSizes, nbSamplesLoaded, *coverParams);
}
} else if (fastCoverParams != NULL) {
if (optimize) {
dictSize = ZDICT_optimizeTrainFromBuffer_fastCover(dictBuffer, maxDictSize,
srcBuffer, sampleSizes, nbSamplesLoaded,
fastCoverParams);
if (!ZDICT_isError(dictSize)) {
unsigned splitPercentage = (unsigned)(fastCoverParams->splitPoint * 100);
DISPLAYLEVEL(2, "k=%u\nd=%u\nf=%u\nsteps=%u\nsplit=%u\naccel=%u\n", fastCoverParams->k,
fastCoverParams->d, fastCoverParams->f, fastCoverParams->steps, splitPercentage,
fastCoverParams->accel);
}
} else {
dictSize = ZDICT_trainFromBuffer_fastCover(dictBuffer, maxDictSize, srcBuffer,
sampleSizes, nbSamplesLoaded, *fastCoverParams);
}
} else {
assert(0 /* Impossible */);
}
if (ZDICT_isError(dictSize)) {
DISPLAYLEVEL(1, "dictionary training failed : %s \n", ZDICT_getErrorName(dictSize)); /* should not happen */
result = 1;
goto _cleanup;
}
/* save dict */
DISPLAYLEVEL(2, "Save dictionary of size %u into file %s \n", (unsigned)dictSize, dictFileName);
DiB_saveDict(dictFileName, dictBuffer, dictSize);
}
/* clean up */
_cleanup:
free(srcBuffer);
free(sampleSizes);
free(dictBuffer);
return result;
}
programs/util.c
+47 -1629
View File
@@ -8,1636 +8,54 @@
* You may select, at your option, one of the above-listed licenses.
*/
/*-****************************************
* Dependencies
******************************************/
#include "util.h" /* note : ensure that platform.h is included first ! */
#include <stdlib.h> /* malloc, realloc, free */
#include <stdio.h> /* fprintf */
#include <time.h> /* clock_t, clock, CLOCKS_PER_SEC, nanosleep */
#include <errno.h>
#include <assert.h>
/* The implementation lives in rust/src/util.rs, built into the Rust CLI
* static archive. Keep this translation unit in the original C source lists
* so util.h remains the public C ABI and every program variant still checks
* the ABI at compile time. */
#if defined(__FreeBSD__)
#include <sys/param.h> /* __FreeBSD_version */
#endif /* #ifdef __FreeBSD__ */
#include "util.h"
#include <stddef.h>
/* These assertions deliberately use the public C declarations rather than a
* Rust-private mirror. They catch accidental changes to the structures that
* cross the C/Rust boundary on every supported C build. */
typedef char UTIL_rust_human_value_offset[
(offsetof(UTIL_HumanReadableSize_t, value) == 0) ? 1 : -1];
typedef char UTIL_rust_human_precision_offset[
(offsetof(UTIL_HumanReadableSize_t, precision) == sizeof(double)) ? 1 : -1];
typedef char UTIL_rust_human_suffix_offset[
(offsetof(UTIL_HumanReadableSize_t, suffix)
>= offsetof(UTIL_HumanReadableSize_t, precision) + sizeof(int)) ? 1 : -1];
typedef char UTIL_rust_human_size[
(sizeof(UTIL_HumanReadableSize_t)
== offsetof(UTIL_HumanReadableSize_t, suffix) + sizeof(const char*)) ? 1 : -1];
typedef char UTIL_rust_fnt_file_names_offset[
(offsetof(FileNamesTable, fileNames) == 0) ? 1 : -1];
typedef char UTIL_rust_fnt_buf_offset[
(offsetof(FileNamesTable, buf) == sizeof(const char**)) ? 1 : -1];
typedef char UTIL_rust_fnt_size_offset[
(offsetof(FileNamesTable, tableSize)
== offsetof(FileNamesTable, buf) + sizeof(char*)) ? 1 : -1];
typedef char UTIL_rust_fnt_capacity_offset[
(offsetof(FileNamesTable, tableCapacity)
== offsetof(FileNamesTable, tableSize) + sizeof(size_t)) ? 1 : -1];
typedef char UTIL_rust_fnt_size[
(sizeof(FileNamesTable)
== offsetof(FileNamesTable, tableCapacity) + sizeof(size_t)) ? 1 : -1];
/* Windows' CRT does not expose the POSIX utimensat interface. This helper is
* intentionally the only platform operation retained in this C translation
* unit; the public UTIL_utime entry point itself is exported by Rust. */
#if defined(_WIN32)
# include <sys/utime.h> /* utime */
# include <io.h> /* _chmod */
# define ZSTD_USE_UTIMENSAT 0
#else
# include <unistd.h> /* chown, stat */
# include <sys/stat.h> /* utimensat, st_mtime */
# if (PLATFORM_POSIX_VERSION >= 200809L && defined(st_mtime)) \
|| (defined(__FreeBSD__) && __FreeBSD_version >= 1100056)
# define ZSTD_USE_UTIMENSAT 1
# else
# define ZSTD_USE_UTIMENSAT 0
# endif
# if ZSTD_USE_UTIMENSAT
# include <fcntl.h> /* AT_FDCWD */
# else
# include <utime.h> /* utime */
# endif
# include <sys/utime.h>
# include <time.h>
int UTIL_rust_utime(const char* filename, const stat_t* statbuf)
{
struct _utimbuf timebuf;
timebuf.actime = time(NULL);
timebuf.modtime = statbuf->st_mtime;
return _utime(filename, &timebuf);
}
#endif
#if defined(_MSC_VER) || defined(__MINGW32__) || defined (__MSVCRT__)
#include <direct.h> /* needed for _mkdir in windows */
#endif
#if defined(__linux__) || (PLATFORM_POSIX_VERSION >= 200112L) /* opendir, readdir require POSIX.1-2001 */
# include <dirent.h> /* opendir, readdir */
# include <string.h> /* strerror, memcpy */
#endif /* #ifdef _WIN32 */
/*-****************************************
* Internal Macros
******************************************/
/* CONTROL is almost like an assert(), but is never disabled.
* It's designed for failures that may happen rarely,
* but we don't want to maintain a specific error code path for them,
* such as a malloc() returning NULL for example.
* Since it's always active, this macro can trigger side effects.
*/
#define CONTROL(c) { \
if (!(c)) { \
UTIL_DISPLAYLEVEL(1, "Error : %s, %i : %s", \
__FILE__, __LINE__, #c); \
exit(1); \
} }
/* console log */
#define UTIL_DISPLAY(...) fprintf(stderr, __VA_ARGS__)
#define UTIL_DISPLAYLEVEL(l, ...) { if (g_utilDisplayLevel>=l) { UTIL_DISPLAY(__VA_ARGS__); } }
static int g_traceDepth = 0;
int g_traceFileStat = 0;
#define UTIL_TRACE_CALL(...) \
{ \
if (g_traceFileStat) { \
UTIL_DISPLAY("Trace:FileStat: %*s> ", g_traceDepth, ""); \
UTIL_DISPLAY(__VA_ARGS__); \
UTIL_DISPLAY("\n"); \
++g_traceDepth; \
} \
}
#define UTIL_TRACE_RET(ret) \
{ \
if (g_traceFileStat) { \
--g_traceDepth; \
UTIL_DISPLAY("Trace:FileStat: %*s< %d\n", g_traceDepth, "", (ret)); \
} \
}
/* A modified version of realloc().
* If UTIL_realloc() fails the original block is freed.
*/
UTIL_STATIC void* UTIL_realloc(void *ptr, size_t size)
{
void *newptr = realloc(ptr, size);
if (newptr) return newptr;
free(ptr);
return NULL;
}
#if defined(_MSC_VER)
#define chmod _chmod
#endif
#ifndef ZSTD_HAVE_FCHMOD
#if PLATFORM_POSIX_VERSION >= 199309L
#define ZSTD_HAVE_FCHMOD
#endif
#endif
#ifndef ZSTD_HAVE_FCHOWN
#if PLATFORM_POSIX_VERSION >= 200809L
#define ZSTD_HAVE_FCHOWN
#endif
#endif
/*-****************************************
* Console log
******************************************/
int g_utilDisplayLevel;
int UTIL_requireUserConfirmation(const char* prompt, const char* abortMsg,
const char* acceptableLetters, int hasStdinInput) {
int ch, result;
if (hasStdinInput) {
UTIL_DISPLAY("stdin is an input - not proceeding.\n");
return 1;
}
UTIL_DISPLAY("%s", prompt);
ch = getchar();
result = 0;
if (strchr(acceptableLetters, ch) == NULL) {
UTIL_DISPLAY("%s \n", abortMsg);
result = 1;
}
/* flush the rest */
while ((ch!=EOF) && (ch!='\n'))
ch = getchar();
return result;
}
/*-*************************************
* Constants
***************************************/
#define LIST_SIZE_INCREASE (8*1024)
#define MAX_FILE_OF_FILE_NAMES_SIZE (1<<20)*50
/*-*************************************
* Functions
***************************************/
void UTIL_traceFileStat(void)
{
g_traceFileStat = 1;
}
int UTIL_fstat(const int fd, const char* filename, stat_t* statbuf)
{
int ret;
UTIL_TRACE_CALL("UTIL_stat(%d, %s)", fd, filename);
#if defined(_MSC_VER)
if (fd >= 0) {
ret = !_fstat64(fd, statbuf);
} else {
ret = !_stat64(filename, statbuf);
}
#elif defined(__MINGW32__) && defined (__MSVCRT__)
if (fd >= 0) {
ret = !_fstati64(fd, statbuf);
} else {
ret = !_stati64(filename, statbuf);
}
#else
if (fd >= 0) {
ret = !fstat(fd, statbuf);
} else {
ret = !stat(filename, statbuf);
}
#endif
UTIL_TRACE_RET(ret);
return ret;
}
int UTIL_stat(const char* filename, stat_t* statbuf)
{
return UTIL_fstat(-1, filename, statbuf);
}
int UTIL_isRegularFile(const char* infilename)
{
stat_t statbuf;
int ret;
UTIL_TRACE_CALL("UTIL_isRegularFile(%s)", infilename);
ret = UTIL_stat(infilename, &statbuf) && UTIL_isRegularFileStat(&statbuf);
UTIL_TRACE_RET(ret);
return ret;
}
int UTIL_isRegularFileStat(const stat_t* statbuf)
{
#if defined(_MSC_VER)
return (statbuf->st_mode & S_IFREG) != 0;
#else
return S_ISREG(statbuf->st_mode) != 0;
#endif
}
/* like chmod, but avoid changing permission of /dev/null */
int UTIL_chmod(char const* filename, const stat_t* statbuf, mode_t permissions)
{
return UTIL_fchmod(-1, filename, statbuf, permissions);
}
int UTIL_fchmod(const int fd, char const* filename, const stat_t* statbuf, mode_t permissions)
{
stat_t localStatBuf;
UTIL_TRACE_CALL("UTIL_chmod(%s, %#4o)", filename, (unsigned)permissions);
if (statbuf == NULL) {
if (!UTIL_fstat(fd, filename, &localStatBuf)) {
UTIL_TRACE_RET(0);
return 0;
}
statbuf = &localStatBuf;
}
if (!UTIL_isRegularFileStat(statbuf)) {
UTIL_TRACE_RET(0);
return 0; /* pretend success, but don't change anything */
}
#ifdef ZSTD_HAVE_FCHMOD
if (fd >= 0) {
int ret;
UTIL_TRACE_CALL("fchmod");
ret = fchmod(fd, permissions);
UTIL_TRACE_RET(ret);
UTIL_TRACE_RET(ret);
return ret;
} else
#endif
{
int ret;
UTIL_TRACE_CALL("chmod");
ret = chmod(filename, permissions);
UTIL_TRACE_RET(ret);
UTIL_TRACE_RET(ret);
return ret;
}
}
/* set access and modification times */
int UTIL_utime(const char* filename, const stat_t *statbuf)
{
int ret;
UTIL_TRACE_CALL("UTIL_utime(%s)", filename);
/* We check that st_mtime is a macro here in order to give us confidence
* that struct stat has a struct timespec st_mtim member. We need this
* check because there are some platforms that claim to be POSIX 2008
* compliant but which do not have st_mtim... */
/* FreeBSD has implemented POSIX 2008 for a long time but still only
* advertises support for POSIX 2001. They have a version macro that
* lets us safely gate them in.
* See https://docs.freebsd.org/en/books/porters-handbook/versions/.
*/
#if ZSTD_USE_UTIMENSAT
{
/* (atime, mtime) */
struct timespec timebuf[2] = { {0, UTIME_NOW} };
timebuf[1] = statbuf->st_mtim;
ret = utimensat(AT_FDCWD, filename, timebuf, 0);
}
#else
{
struct utimbuf timebuf;
timebuf.actime = time(NULL);
timebuf.modtime = statbuf->st_mtime;
ret = utime(filename, &timebuf);
}
#endif
errno = 0;
UTIL_TRACE_RET(ret);
return ret;
}
int UTIL_setFileStat(const char *filename, const stat_t *statbuf)
{
return UTIL_setFDStat(-1, filename, statbuf);
}
int UTIL_setFDStat(const int fd, const char *filename, const stat_t *statbuf)
{
int res = 0;
stat_t curStatBuf;
UTIL_TRACE_CALL("UTIL_setFileStat(%d, %s)", fd, filename);
if (!UTIL_fstat(fd, filename, &curStatBuf) || !UTIL_isRegularFileStat(&curStatBuf)) {
UTIL_TRACE_RET(-1);
return -1;
}
/* Mimic gzip's behavior:
*
* "Change the group first, then the permissions, then the owner.
* That way, the permissions will be correct on systems that allow
* users to give away files, without introducing a security hole.
* Security depends on permissions not containing the setuid or
* setgid bits." */
#if !defined(_WIN32)
#ifdef ZSTD_HAVE_FCHOWN
if (fd >= 0) {
res += fchown(fd, -1, statbuf->st_gid); /* Apply group ownership */
} else
#endif
{
res += chown(filename, -1, statbuf->st_gid); /* Apply group ownership */
}
#endif
res += UTIL_fchmod(fd, filename, &curStatBuf, statbuf->st_mode & 0777); /* Copy file permissions */
#if !defined(_WIN32)
#ifdef ZSTD_HAVE_FCHOWN
if (fd >= 0) {
res += fchown(fd, statbuf->st_uid, -1); /* Apply user ownership */
} else
#endif
{
res += chown(filename, statbuf->st_uid, -1); /* Apply user ownership */
}
#endif
errno = 0;
UTIL_TRACE_RET(-res);
return -res; /* number of errors is returned */
}
int UTIL_isDirectory(const char* infilename)
{
stat_t statbuf;
int ret;
UTIL_TRACE_CALL("UTIL_isDirectory(%s)", infilename);
ret = UTIL_stat(infilename, &statbuf) && UTIL_isDirectoryStat(&statbuf);
UTIL_TRACE_RET(ret);
return ret;
}
int UTIL_isDirectoryStat(const stat_t* statbuf)
{
int ret;
UTIL_TRACE_CALL("UTIL_isDirectoryStat()");
#if defined(_MSC_VER)
ret = (statbuf->st_mode & _S_IFDIR) != 0;
#else
ret = S_ISDIR(statbuf->st_mode) != 0;
#endif
UTIL_TRACE_RET(ret);
return ret;
}
int UTIL_compareStr(const void *p1, const void *p2) {
return strcmp(* (char * const *) p1, * (char * const *) p2);
}
int UTIL_isSameFile(const char* fName1, const char* fName2)
{
int ret;
assert(fName1 != NULL); assert(fName2 != NULL);
UTIL_TRACE_CALL("UTIL_isSameFile(%s, %s)", fName1, fName2);
#if defined(_MSC_VER) || defined(_WIN32)
/* note : Visual does not support file identification by inode.
* inode does not work on Windows, even with a posix layer, like msys2.
* The following work-around is limited to detecting exact name repetition only,
* aka `filename` is considered different from `subdir/../filename` */
ret = !strcmp(fName1, fName2);
#else
{ stat_t file1Stat;
stat_t file2Stat;
ret = UTIL_stat(fName1, &file1Stat)
&& UTIL_stat(fName2, &file2Stat)
&& UTIL_isSameFileStat(fName1, fName2, &file1Stat, &file2Stat);
}
#endif
UTIL_TRACE_RET(ret);
return ret;
}
int UTIL_isSameFileStat(
const char* fName1, const char* fName2,
const stat_t* file1Stat, const stat_t* file2Stat)
{
int ret;
assert(fName1 != NULL); assert(fName2 != NULL);
UTIL_TRACE_CALL("UTIL_isSameFileStat(%s, %s)", fName1, fName2);
#if defined(_MSC_VER) || defined(_WIN32)
/* note : Visual does not support file identification by inode.
* inode does not work on Windows, even with a posix layer, like msys2.
* The following work-around is limited to detecting exact name repetition only,
* aka `filename` is considered different from `subdir/../filename` */
(void)file1Stat;
(void)file2Stat;
ret = !strcmp(fName1, fName2);
#else
{
ret = (file1Stat->st_dev == file2Stat->st_dev)
&& (file1Stat->st_ino == file2Stat->st_ino);
}
#endif
UTIL_TRACE_RET(ret);
return ret;
}
/* UTIL_isFIFO : distinguish named pipes */
int UTIL_isFIFO(const char* infilename)
{
UTIL_TRACE_CALL("UTIL_isFIFO(%s)", infilename);
/* macro guards, as defined in : https://linux.die.net/man/2/lstat */
#if PLATFORM_POSIX_VERSION >= 200112L
{
stat_t statbuf;
if (UTIL_stat(infilename, &statbuf) && UTIL_isFIFOStat(&statbuf)) {
UTIL_TRACE_RET(1);
return 1;
}
}
#endif
(void)infilename;
UTIL_TRACE_RET(0);
return 0;
}
/* UTIL_isFIFO : distinguish named pipes */
int UTIL_isFIFOStat(const stat_t* statbuf)
{
/* macro guards, as defined in : https://linux.die.net/man/2/lstat */
#if PLATFORM_POSIX_VERSION >= 200112L
if (S_ISFIFO(statbuf->st_mode)) return 1;
#endif
(void)statbuf;
return 0;
}
/* UTIL_isBlockDevStat : distinguish named pipes */
int UTIL_isBlockDevStat(const stat_t* statbuf)
{
/* macro guards, as defined in : https://linux.die.net/man/2/lstat */
#if PLATFORM_POSIX_VERSION >= 200112L
if (S_ISBLK(statbuf->st_mode)) return 1;
#endif
(void)statbuf;
return 0;
}
int UTIL_isLink(const char* infilename)
{
UTIL_TRACE_CALL("UTIL_isLink(%s)", infilename);
/* macro guards, as defined in : https://linux.die.net/man/2/lstat */
#if PLATFORM_POSIX_VERSION >= 200112L
{
stat_t statbuf;
int const r = lstat(infilename, &statbuf);
if (!r && S_ISLNK(statbuf.st_mode)) {
UTIL_TRACE_RET(1);
return 1;
}
}
#endif
(void)infilename;
UTIL_TRACE_RET(0);
return 0;
}
static int g_fakeStdinIsConsole = 0;
static int g_fakeStderrIsConsole = 0;
static int g_fakeStdoutIsConsole = 0;
int UTIL_isConsole(FILE* file)
{
int ret;
UTIL_TRACE_CALL("UTIL_isConsole(%d)", fileno(file));
if (file == stdin && g_fakeStdinIsConsole)
ret = 1;
else if (file == stderr && g_fakeStderrIsConsole)
ret = 1;
else if (file == stdout && g_fakeStdoutIsConsole)
ret = 1;
else
ret = IS_CONSOLE(file);
UTIL_TRACE_RET(ret);
return ret;
}
void UTIL_fakeStdinIsConsole(void)
{
g_fakeStdinIsConsole = 1;
}
void UTIL_fakeStdoutIsConsole(void)
{
g_fakeStdoutIsConsole = 1;
}
void UTIL_fakeStderrIsConsole(void)
{
g_fakeStderrIsConsole = 1;
}
U64 UTIL_getFileSize(const char* infilename)
{
stat_t statbuf;
UTIL_TRACE_CALL("UTIL_getFileSize(%s)", infilename);
if (!UTIL_stat(infilename, &statbuf)) {
UTIL_TRACE_RET(-1);
return UTIL_FILESIZE_UNKNOWN;
}
{
U64 const size = UTIL_getFileSizeStat(&statbuf);
UTIL_TRACE_RET((int)size);
return size;
}
}
U64 UTIL_getFileSizeStat(const stat_t* statbuf)
{
if (!UTIL_isRegularFileStat(statbuf)) return UTIL_FILESIZE_UNKNOWN;
#if defined(_MSC_VER)
if (!(statbuf->st_mode & S_IFREG)) return UTIL_FILESIZE_UNKNOWN;
#elif defined(__MINGW32__) && defined (__MSVCRT__)
if (!(statbuf->st_mode & S_IFREG)) return UTIL_FILESIZE_UNKNOWN;
#else
if (!S_ISREG(statbuf->st_mode)) return UTIL_FILESIZE_UNKNOWN;
#endif
return (U64)statbuf->st_size;
}
UTIL_HumanReadableSize_t UTIL_makeHumanReadableSize(U64 size)
{
UTIL_HumanReadableSize_t hrs;
if (g_utilDisplayLevel > 3) {
/* In verbose mode, do not scale sizes down, except in the case of
* values that exceed the integral precision of a double. */
if (size >= (1ull << 53)) {
hrs.value = (double)size / (1ull << 20);
hrs.suffix = " MiB";
/* At worst, a double representation of a maximal size will be
* accurate to better than tens of kilobytes. */
hrs.precision = 2;
} else {
hrs.value = (double)size;
hrs.suffix = " B";
hrs.precision = 0;
}
} else {
/* In regular mode, scale sizes down and use suffixes. */
if (size >= (1ull << 60)) {
hrs.value = (double)size / (1ull << 60);
hrs.suffix = " EiB";
} else if (size >= (1ull << 50)) {
hrs.value = (double)size / (1ull << 50);
hrs.suffix = " PiB";
} else if (size >= (1ull << 40)) {
hrs.value = (double)size / (1ull << 40);
hrs.suffix = " TiB";
} else if (size >= (1ull << 30)) {
hrs.value = (double)size / (1ull << 30);
hrs.suffix = " GiB";
} else if (size >= (1ull << 20)) {
hrs.value = (double)size / (1ull << 20);
hrs.suffix = " MiB";
} else if (size >= (1ull << 10)) {
hrs.value = (double)size / (1ull << 10);
hrs.suffix = " KiB";
} else {
hrs.value = (double)size;
hrs.suffix = " B";
}
if (hrs.value >= 100 || (U64)hrs.value == size) {
hrs.precision = 0;
} else if (hrs.value >= 10) {
hrs.precision = 1;
} else if (hrs.value > 1) {
hrs.precision = 2;
} else {
hrs.precision = 3;
}
}
return hrs;
}
U64 UTIL_getTotalFileSize(const char* const * fileNamesTable, unsigned nbFiles)
{
U64 total = 0;
unsigned n;
UTIL_TRACE_CALL("UTIL_getTotalFileSize(%u)", nbFiles);
for (n=0; n<nbFiles; n++) {
U64 const size = UTIL_getFileSize(fileNamesTable[n]);
if (size == UTIL_FILESIZE_UNKNOWN) {
UTIL_TRACE_RET(-1);
return UTIL_FILESIZE_UNKNOWN;
}
total += size;
}
UTIL_TRACE_RET((int)total);
return total;
}
/* condition : @file must be valid, and not have reached its end.
* @return : length of line written into @buf, ended with `\0` instead of '\n',
* or 0, if there is no new line */
static size_t readLineFromFile(char* buf, size_t len, FILE* file)
{
assert(!feof(file));
if ( fgets(buf, (int) len, file) == NULL ) return 0;
{ size_t linelen = strlen(buf);
if (strlen(buf)==0) return 0;
if (buf[linelen-1] == '\n') linelen--;
buf[linelen] = '\0';
return linelen+1;
}
}
/* Conditions :
* size of @inputFileName file must be < @dstCapacity
* @dst must be initialized
* @return : nb of lines
* or -1 if there's an error
*/
static int
readLinesFromFile(void* dst, size_t dstCapacity,
const char* inputFileName)
{
int nbFiles = 0;
size_t pos = 0;
char* const buf = (char*)dst;
FILE* const inputFile = fopen(inputFileName, "r");
assert(dst != NULL);
if(!inputFile) {
if (g_utilDisplayLevel >= 1) perror("zstd:util:readLinesFromFile");
return -1;
}
while ( !feof(inputFile) ) {
size_t const lineLength = readLineFromFile(buf+pos, dstCapacity-pos, inputFile);
if (lineLength == 0) break;
assert(pos + lineLength <= dstCapacity); /* '=' for inputFile not terminated with '\n' */
pos += lineLength;
++nbFiles;
}
CONTROL( fclose(inputFile) == 0 );
return nbFiles;
}
/*Note: buf is not freed in case function successfully created table because filesTable->fileNames[0] = buf*/
FileNamesTable*
UTIL_createFileNamesTable_fromFileName(const char* inputFileName)
{
size_t nbFiles = 0;
char* buf;
size_t bufSize;
stat_t statbuf;
if (!UTIL_stat(inputFileName, &statbuf) || !UTIL_isRegularFileStat(&statbuf))
return NULL;
{ U64 const inputFileSize = UTIL_getFileSizeStat(&statbuf);
if(inputFileSize > MAX_FILE_OF_FILE_NAMES_SIZE)
return NULL;
bufSize = (size_t)(inputFileSize + 1); /* (+1) to add '\0' at the end of last filename */
}
buf = (char*) malloc(bufSize);
CONTROL( buf != NULL );
{ int const ret_nbFiles = readLinesFromFile(buf, bufSize, inputFileName);
if (ret_nbFiles <= 0) {
free(buf);
return NULL;
}
nbFiles = (size_t)ret_nbFiles;
}
{ const char** filenamesTable = (const char**) malloc(nbFiles * sizeof(*filenamesTable));
CONTROL(filenamesTable != NULL);
{ size_t fnb, pos = 0;
for (fnb = 0; fnb < nbFiles; fnb++) {
filenamesTable[fnb] = buf+pos;
pos += strlen(buf+pos)+1; /* +1 for the finishing `\0` */
}
assert(pos <= bufSize);
}
return UTIL_assembleFileNamesTable(filenamesTable, nbFiles, buf);
}
}
static FileNamesTable*
UTIL_assembleFileNamesTable2(const char** filenames, size_t tableSize, size_t tableCapacity, char* buf)
{
FileNamesTable* const table = (FileNamesTable*) malloc(sizeof(*table));
CONTROL(table != NULL);
table->fileNames = filenames;
table->buf = buf;
table->tableSize = tableSize;
table->tableCapacity = tableCapacity;
return table;
}
FileNamesTable*
UTIL_assembleFileNamesTable(const char** filenames, size_t tableSize, char* buf)
{
return UTIL_assembleFileNamesTable2(filenames, tableSize, tableSize, buf);
}
void UTIL_freeFileNamesTable(FileNamesTable* table)
{
if (table==NULL) return;
free((void*)table->fileNames);
free(table->buf);
free(table);
}
FileNamesTable* UTIL_allocateFileNamesTable(size_t tableSize)
{
const char** const fnTable = (const char**)malloc(tableSize * sizeof(*fnTable));
FileNamesTable* fnt;
if (fnTable==NULL) return NULL;
fnt = UTIL_assembleFileNamesTable(fnTable, tableSize, NULL);
fnt->tableSize = 0; /* the table is empty */
return fnt;
}
int UTIL_searchFileNamesTable(FileNamesTable* table, char const* name) {
size_t i;
for(i=0 ;i < table->tableSize; i++) {
if(!strcmp(table->fileNames[i], name)) {
return (int)i;
}
}
return -1;
}
void UTIL_refFilename(FileNamesTable* fnt, const char* filename)
{
assert(fnt->tableSize < fnt->tableCapacity);
fnt->fileNames[fnt->tableSize] = filename;
fnt->tableSize++;
}
static size_t getTotalTableSize(FileNamesTable* table)
{
size_t fnb, totalSize = 0;
for(fnb = 0 ; fnb < table->tableSize && table->fileNames[fnb] ; ++fnb) {
totalSize += strlen(table->fileNames[fnb]) + 1; /* +1 to add '\0' at the end of each fileName */
}
return totalSize;
}
FileNamesTable*
UTIL_mergeFileNamesTable(FileNamesTable* table1, FileNamesTable* table2)
{
unsigned newTableIdx = 0;
size_t pos = 0;
size_t newTotalTableSize;
char* buf;
FileNamesTable* const newTable = UTIL_assembleFileNamesTable(NULL, 0, NULL);
CONTROL( newTable != NULL );
newTotalTableSize = getTotalTableSize(table1) + getTotalTableSize(table2);
buf = (char*) calloc(newTotalTableSize, sizeof(*buf));
CONTROL ( buf != NULL );
newTable->buf = buf;
newTable->tableSize = table1->tableSize + table2->tableSize;
newTable->fileNames = (const char **) calloc(newTable->tableSize, sizeof(*(newTable->fileNames)));
CONTROL ( newTable->fileNames != NULL );
{ unsigned idx1;
for( idx1=0 ; (idx1 < table1->tableSize) && table1->fileNames[idx1] && (pos < newTotalTableSize); ++idx1, ++newTableIdx) {
size_t const curLen = strlen(table1->fileNames[idx1]);
memcpy(buf+pos, table1->fileNames[idx1], curLen);
assert(newTableIdx <= newTable->tableSize);
newTable->fileNames[newTableIdx] = buf+pos;
pos += curLen+1;
} }
{ unsigned idx2;
for( idx2=0 ; (idx2 < table2->tableSize) && table2->fileNames[idx2] && (pos < newTotalTableSize) ; ++idx2, ++newTableIdx) {
size_t const curLen = strlen(table2->fileNames[idx2]);
memcpy(buf+pos, table2->fileNames[idx2], curLen);
assert(newTableIdx < newTable->tableSize);
newTable->fileNames[newTableIdx] = buf+pos;
pos += curLen+1;
} }
assert(pos <= newTotalTableSize);
newTable->tableSize = newTableIdx;
UTIL_freeFileNamesTable(table1);
UTIL_freeFileNamesTable(table2);
return newTable;
}
#ifdef _WIN32
static int UTIL_prepareFileList(const char* dirName,
char** bufStart, size_t* pos,
char** bufEnd, int followLinks)
{
char* path;
size_t dirLength, pathLength;
int nbFiles = 0;
WIN32_FIND_DATAA cFile;
HANDLE hFile;
dirLength = strlen(dirName);
path = (char*) malloc(dirLength + 3);
if (!path) return 0;
memcpy(path, dirName, dirLength);
path[dirLength] = '\\';
path[dirLength+1] = '*';
path[dirLength+2] = 0;
hFile=FindFirstFileA(path, &cFile);
if (hFile == INVALID_HANDLE_VALUE) {
UTIL_DISPLAYLEVEL(1, "Cannot open directory '%s'\n", dirName);
return 0;
}
free(path);
do {
size_t const fnameLength = strlen(cFile.cFileName);
path = (char*) malloc(dirLength + fnameLength + 2);
if (!path) { FindClose(hFile); return 0; }
memcpy(path, dirName, dirLength);
path[dirLength] = '\\';
memcpy(path+dirLength+1, cFile.cFileName, fnameLength);
pathLength = dirLength+1+fnameLength;
path[pathLength] = 0;
if (cFile.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) {
if ( strcmp (cFile.cFileName, "..") == 0
|| strcmp (cFile.cFileName, ".") == 0 )
continue;
/* Recursively call "UTIL_prepareFileList" with the new path. */
nbFiles += UTIL_prepareFileList(path, bufStart, pos, bufEnd, followLinks);
if (*bufStart == NULL) { free(path); FindClose(hFile); return 0; }
} else if ( (cFile.dwFileAttributes & FILE_ATTRIBUTE_NORMAL)
|| (cFile.dwFileAttributes & FILE_ATTRIBUTE_ARCHIVE)
|| (cFile.dwFileAttributes & FILE_ATTRIBUTE_COMPRESSED) ) {
if (*bufStart + *pos + pathLength >= *bufEnd) {
ptrdiff_t const newListSize = (*bufEnd - *bufStart) + LIST_SIZE_INCREASE;
*bufStart = (char*)UTIL_realloc(*bufStart, newListSize);
if (*bufStart == NULL) { free(path); FindClose(hFile); return 0; }
*bufEnd = *bufStart + newListSize;
}
if (*bufStart + *pos + pathLength < *bufEnd) {
memcpy(*bufStart + *pos, path, pathLength+1 /* include final \0 */);
*pos += pathLength + 1;
nbFiles++;
} }
free(path);
} while (FindNextFileA(hFile, &cFile));
FindClose(hFile);
return nbFiles;
}
#elif defined(__linux__) || (PLATFORM_POSIX_VERSION >= 200112L) /* opendir, readdir require POSIX.1-2001 */
static int UTIL_prepareFileList(const char *dirName,
char** bufStart, size_t* pos,
char** bufEnd, int followLinks)
{
DIR* dir;
struct dirent * entry;
size_t dirLength;
int nbFiles = 0;
if (!(dir = opendir(dirName))) {
UTIL_DISPLAYLEVEL(1, "Cannot open directory '%s': %s\n", dirName, strerror(errno));
return 0;
}
dirLength = strlen(dirName);
errno = 0;
while ((entry = readdir(dir)) != NULL) {
char* path;
size_t fnameLength, pathLength;
if (strcmp (entry->d_name, "..") == 0 ||
strcmp (entry->d_name, ".") == 0) continue;
fnameLength = strlen(entry->d_name);
path = (char*) malloc(dirLength + fnameLength + 2);
if (!path) { closedir(dir); return 0; }
memcpy(path, dirName, dirLength);
path[dirLength] = '/';
memcpy(path+dirLength+1, entry->d_name, fnameLength);
pathLength = dirLength+1+fnameLength;
path[pathLength] = 0;
if (!followLinks && UTIL_isLink(path)) {
UTIL_DISPLAYLEVEL(2, "Warning : %s is a symbolic link, ignoring\n", path);
free(path);
continue;
}
if (UTIL_isDirectory(path)) {
nbFiles += UTIL_prepareFileList(path, bufStart, pos, bufEnd, followLinks); /* Recursively call "UTIL_prepareFileList" with the new path. */
if (*bufStart == NULL) { free(path); closedir(dir); return 0; }
} else {
if (*bufStart + *pos + pathLength >= *bufEnd) {
ptrdiff_t newListSize = (*bufEnd - *bufStart) + LIST_SIZE_INCREASE;
assert(newListSize >= 0);
*bufStart = (char*)UTIL_realloc(*bufStart, (size_t)newListSize);
if (*bufStart != NULL) {
*bufEnd = *bufStart + newListSize;
} else {
free(path); closedir(dir); return 0;
}
}
if (*bufStart + *pos + pathLength < *bufEnd) {
memcpy(*bufStart + *pos, path, pathLength + 1); /* with final \0 */
*pos += pathLength + 1;
nbFiles++;
} }
free(path);
errno = 0; /* clear errno after UTIL_isDirectory, UTIL_prepareFileList */
}
if (errno != 0) {
UTIL_DISPLAYLEVEL(1, "readdir(%s) error: %s \n", dirName, strerror(errno));
free(*bufStart);
*bufStart = NULL;
}
closedir(dir);
return nbFiles;
}
#else
static int UTIL_prepareFileList(const char *dirName,
char** bufStart, size_t* pos,
char** bufEnd, int followLinks)
{
(void)bufStart; (void)bufEnd; (void)pos; (void)followLinks;
UTIL_DISPLAYLEVEL(1, "Directory %s ignored (compiled without _WIN32 or _POSIX_C_SOURCE) \n", dirName);
return 0;
}
#endif /* #ifdef _WIN32 */
int UTIL_isCompressedFile(const char *inputName, const char *extensionList[])
{
const char* ext = UTIL_getFileExtension(inputName);
while(*extensionList!=NULL)
{
const int isCompressedExtension = strcmp(ext,*extensionList);
if(isCompressedExtension==0)
return 1;
++extensionList;
}
return 0;
}
/*Utility function to get file extension from file */
const char* UTIL_getFileExtension(const char* infilename)
{
const char* extension = strrchr(infilename, '.');
if(!extension || extension==infilename) return "";
return extension;
}
static int pathnameHas2Dots(const char *pathname)
{
/* We need to figure out whether any ".." present in the path is a whole
* path token, which is the case if it is bordered on both sides by either
* the beginning/end of the path or by a directory separator.
*/
const char *needle = pathname;
while (1) {
needle = strstr(needle, "..");
if (needle == NULL) {
return 0;
}
if ((needle == pathname || needle[-1] == PATH_SEP)
&& (needle[2] == '\0' || needle[2] == PATH_SEP)) {
return 1;
}
/* increment so we search for the next match */
needle++;
};
return 0;
}
static int isFileNameValidForMirroredOutput(const char *filename)
{
return !pathnameHas2Dots(filename);
}
#define DIR_DEFAULT_MODE 0755
static mode_t getDirMode(const char *dirName)
{
stat_t st;
if (!UTIL_stat(dirName, &st)) {
UTIL_DISPLAY("zstd: failed to get DIR stats %s: %s\n", dirName, strerror(errno));
return DIR_DEFAULT_MODE;
}
if (!UTIL_isDirectoryStat(&st)) {
UTIL_DISPLAY("zstd: expected directory: %s\n", dirName);
return DIR_DEFAULT_MODE;
}
return st.st_mode;
}
static int makeDir(const char *dir, mode_t mode)
{
#if defined(_MSC_VER) || defined(__MINGW32__) || defined (__MSVCRT__)
int ret = _mkdir(dir);
(void) mode;
#else
int ret = mkdir(dir, mode);
#endif
if (ret != 0) {
if (errno == EEXIST)
return 0;
UTIL_DISPLAY("zstd: failed to create DIR %s: %s\n", dir, strerror(errno));
}
return ret;
}
/* this function requires a mutable input string */
static void convertPathnameToDirName(char *pathname)
{
size_t len = 0;
char* pos = NULL;
/* get dir name from pathname similar to 'dirname()' */
assert(pathname != NULL);
/* remove trailing '/' chars */
len = strlen(pathname);
assert(len > 0);
while (pathname[len] == PATH_SEP) {
pathname[len] = '\0';
len--;
}
if (len == 0) return;
/* if input is a single file, return '.' instead. i.e.
* "xyz/abc/file.txt" => "xyz/abc"
"./file.txt" => "."
"file.txt" => "."
*/
pos = strrchr(pathname, PATH_SEP);
if (pos == NULL) {
pathname[0] = '.';
pathname[1] = '\0';
} else {
*pos = '\0';
}
}
/* pathname must be valid */
static const char* trimLeadingRootChar(const char *pathname)
{
assert(pathname != NULL);
if (pathname[0] == PATH_SEP)
return pathname + 1;
return pathname;
}
/* pathname must be valid */
static const char* trimLeadingCurrentDirConst(const char *pathname)
{
assert(pathname != NULL);
if ((pathname[0] == '.') && (pathname[1] == PATH_SEP))
return pathname + 2;
return pathname;
}
static char*
trimLeadingCurrentDir(char *pathname)
{
/* 'union charunion' can do const-cast without compiler warning */
union charunion {
char *chr;
const char* cchr;
} ptr;
ptr.cchr = trimLeadingCurrentDirConst(pathname);
return ptr.chr;
}
/* remove leading './' or '/' chars here */
static const char * trimPath(const char *pathname)
{
return trimLeadingRootChar(
trimLeadingCurrentDirConst(pathname));
}
static char* mallocAndJoin2Dir(const char *dir1, const char *dir2)
{
assert(dir1 != NULL && dir2 != NULL);
{ const size_t dir1Size = strlen(dir1);
const size_t dir2Size = strlen(dir2);
char *outDirBuffer, *buffer;
outDirBuffer = (char *) malloc(dir1Size + dir2Size + 2);
CONTROL(outDirBuffer != NULL);
memcpy(outDirBuffer, dir1, dir1Size);
outDirBuffer[dir1Size] = '\0';
buffer = outDirBuffer + dir1Size;
if (dir1Size > 0 && *(buffer - 1) != PATH_SEP) {
*buffer = PATH_SEP;
buffer++;
}
memcpy(buffer, dir2, dir2Size);
buffer[dir2Size] = '\0';
return outDirBuffer;
}
}
/* this function will return NULL if input srcFileName is not valid name for mirrored output path */
char* UTIL_createMirroredDestDirName(const char* srcFileName, const char* outDirRootName)
{
char* pathname = NULL;
if (!isFileNameValidForMirroredOutput(srcFileName))
return NULL;
pathname = mallocAndJoin2Dir(outDirRootName, trimPath(srcFileName));
convertPathnameToDirName(pathname);
return pathname;
}
static int
mirrorSrcDir(char* srcDirName, const char* outDirName)
{
mode_t srcMode;
int status = 0;
char* newDir = mallocAndJoin2Dir(outDirName, trimPath(srcDirName));
if (!newDir)
return -ENOMEM;
srcMode = getDirMode(srcDirName);
status = makeDir(newDir, srcMode);
free(newDir);
return status;
}
static int
mirrorSrcDirRecursive(char* srcDirName, const char* outDirName)
{
int status = 0;
char* pp = trimLeadingCurrentDir(srcDirName);
char* sp = NULL;
while ((sp = strchr(pp, PATH_SEP)) != NULL) {
if (sp != pp) {
*sp = '\0';
status = mirrorSrcDir(srcDirName, outDirName);
if (status != 0)
return status;
*sp = PATH_SEP;
}
pp = sp + 1;
}
status = mirrorSrcDir(srcDirName, outDirName);
return status;
}
static void
makeMirroredDestDirsWithSameSrcDirMode(char** srcDirNames, unsigned nbFile, const char* outDirName)
{
unsigned int i = 0;
for (i = 0; i < nbFile; i++)
mirrorSrcDirRecursive(srcDirNames[i], outDirName);
}
static int
firstIsParentOrSameDirOfSecond(const char* firstDir, const char* secondDir)
{
size_t firstDirLen = strlen(firstDir),
secondDirLen = strlen(secondDir);
return firstDirLen <= secondDirLen &&
(secondDir[firstDirLen] == PATH_SEP || secondDir[firstDirLen] == '\0') &&
0 == strncmp(firstDir, secondDir, firstDirLen);
}
static int compareDir(const void* pathname1, const void* pathname2) {
/* sort it after remove the leading '/' or './'*/
const char* s1 = trimPath(*(char * const *) pathname1);
const char* s2 = trimPath(*(char * const *) pathname2);
return strcmp(s1, s2);
}
static void
makeUniqueMirroredDestDirs(char** srcDirNames, unsigned nbFile, const char* outDirName)
{
unsigned int i = 0, uniqueDirNr = 0;
char** uniqueDirNames = NULL;
if (nbFile == 0)
return;
uniqueDirNames = (char** ) malloc(nbFile * sizeof (char *));
CONTROL(uniqueDirNames != NULL);
/* if dirs is "a/b/c" and "a/b/c/d", we only need call:
* we just need "a/b/c/d" */
qsort((void *)srcDirNames, nbFile, sizeof(char*), compareDir);
uniqueDirNr = 1;
uniqueDirNames[uniqueDirNr - 1] = srcDirNames[0];
for (i = 1; i < nbFile; i++) {
char* prevDirName = srcDirNames[i - 1];
char* currDirName = srcDirNames[i];
/* note: we always compare trimmed path, i.e.:
* src dir of "./foo" and "/foo" will be both saved into:
* "outDirName/foo/" */
if (!firstIsParentOrSameDirOfSecond(trimPath(prevDirName),
trimPath(currDirName)))
uniqueDirNr++;
/* we need to maintain original src dir name instead of trimmed
* dir, so we can retrieve the original src dir's mode_t */
uniqueDirNames[uniqueDirNr - 1] = currDirName;
}
makeMirroredDestDirsWithSameSrcDirMode(uniqueDirNames, uniqueDirNr, outDirName);
free(uniqueDirNames);
}
static void
makeMirroredDestDirs(char** srcFileNames, unsigned nbFile, const char* outDirName)
{
unsigned int i = 0;
for (i = 0; i < nbFile; ++i)
convertPathnameToDirName(srcFileNames[i]);
makeUniqueMirroredDestDirs(srcFileNames, nbFile, outDirName);
}
void UTIL_mirrorSourceFilesDirectories(const char** inFileNames, unsigned int nbFile, const char* outDirName)
{
unsigned int i = 0, validFilenamesNr = 0;
char** srcFileNames = (char **) malloc(nbFile * sizeof (char *));
CONTROL(srcFileNames != NULL);
/* check input filenames is valid */
for (i = 0; i < nbFile; ++i) {
if (isFileNameValidForMirroredOutput(inFileNames[i])) {
char* fname = STRDUP(inFileNames[i]);
CONTROL(fname != NULL);
srcFileNames[validFilenamesNr++] = fname;
}
}
if (validFilenamesNr > 0) {
makeDir(outDirName, DIR_DEFAULT_MODE);
makeMirroredDestDirs(srcFileNames, validFilenamesNr, outDirName);
}
for (i = 0; i < validFilenamesNr; i++)
free(srcFileNames[i]);
free(srcFileNames);
}
FileNamesTable*
UTIL_createExpandedFNT(const char* const* inputNames, size_t nbIfns, int followLinks)
{
unsigned nbFiles;
char* buf = (char*)malloc(LIST_SIZE_INCREASE);
char* bufend = buf + LIST_SIZE_INCREASE;
if (!buf) return NULL;
{ size_t ifnNb, pos;
for (ifnNb=0, pos=0, nbFiles=0; ifnNb<nbIfns; ifnNb++) {
if (!UTIL_isDirectory(inputNames[ifnNb])) {
size_t const len = strlen(inputNames[ifnNb]);
if (buf + pos + len >= bufend) {
ptrdiff_t newListSize = (bufend - buf) + LIST_SIZE_INCREASE;
assert(newListSize >= 0);
buf = (char*)UTIL_realloc(buf, (size_t)newListSize);
if (!buf) return NULL;
bufend = buf + newListSize;
}
if (buf + pos + len < bufend) {
memcpy(buf+pos, inputNames[ifnNb], len+1); /* including final \0 */
pos += len + 1;
nbFiles++;
}
} else {
nbFiles += (unsigned)UTIL_prepareFileList(inputNames[ifnNb], &buf, &pos, &bufend, followLinks);
if (buf == NULL) return NULL;
} } }
/* note : even if nbFiles==0, function returns a valid, though empty, FileNamesTable* object */
{ size_t ifnNb, pos;
size_t const fntCapacity = nbFiles + 1; /* minimum 1, allows adding one reference, typically stdin */
const char** const fileNamesTable = (const char**)malloc(fntCapacity * sizeof(*fileNamesTable));
if (!fileNamesTable) { free(buf); return NULL; }
for (ifnNb = 0, pos = 0; ifnNb < nbFiles; ifnNb++) {
fileNamesTable[ifnNb] = buf + pos;
if (buf + pos > bufend) { free(buf); free((void*)fileNamesTable); return NULL; }
pos += strlen(fileNamesTable[ifnNb]) + 1;
}
return UTIL_assembleFileNamesTable2(fileNamesTable, nbFiles, fntCapacity, buf);
}
}
void UTIL_expandFNT(FileNamesTable** fnt, int followLinks)
{
FileNamesTable* const newFNT = UTIL_createExpandedFNT((*fnt)->fileNames, (*fnt)->tableSize, followLinks);
CONTROL(newFNT != NULL);
UTIL_freeFileNamesTable(*fnt);
*fnt = newFNT;
}
FileNamesTable* UTIL_createFNT_fromROTable(const char** filenames, size_t nbFilenames)
{
size_t const sizeof_FNTable = nbFilenames * sizeof(*filenames);
const char** const newFNTable = (const char**)malloc(sizeof_FNTable);
if (newFNTable==NULL) return NULL;
memcpy((void*)newFNTable, filenames, sizeof_FNTable); /* void* : mitigate a Visual compiler bug or limitation */
return UTIL_assembleFileNamesTable(newFNTable, nbFilenames, NULL);
}
/*-****************************************
* count the number of cores
******************************************/
#if defined(_WIN32) || defined(WIN32)
#include <windows.h>
typedef BOOL(WINAPI* LPFN_GLPI)(PSYSTEM_LOGICAL_PROCESSOR_INFORMATION, PDWORD);
DWORD CountSetBits(ULONG_PTR bitMask)
{
DWORD LSHIFT = sizeof(ULONG_PTR)*8 - 1;
DWORD bitSetCount = 0;
ULONG_PTR bitTest = (ULONG_PTR)1 << LSHIFT;
DWORD i;
for (i = 0; i <= LSHIFT; ++i)
{
bitSetCount += ((bitMask & bitTest)?1:0);
bitTest/=2;
}
return bitSetCount;
}
int UTIL_countCores(int logical)
{
static int numCores = 0;
if (numCores != 0) return numCores;
{ LPFN_GLPI glpi;
BOOL done = FALSE;
PSYSTEM_LOGICAL_PROCESSOR_INFORMATION buffer = NULL;
PSYSTEM_LOGICAL_PROCESSOR_INFORMATION ptr = NULL;
DWORD returnLength = 0;
size_t byteOffset = 0;
#if defined(_MSC_VER)
/* Visual Studio does not like the following cast */
# pragma warning( disable : 4054 ) /* conversion from function ptr to data ptr */
# pragma warning( disable : 4055 ) /* conversion from data ptr to function ptr */
#endif
glpi = (LPFN_GLPI)(void*)GetProcAddress(GetModuleHandle(TEXT("kernel32")),
"GetLogicalProcessorInformation");
if (glpi == NULL) {
goto failed;
}
while(!done) {
DWORD rc = glpi(buffer, &returnLength);
if (FALSE == rc) {
if (GetLastError() == ERROR_INSUFFICIENT_BUFFER) {
if (buffer)
free(buffer);
buffer = (PSYSTEM_LOGICAL_PROCESSOR_INFORMATION)malloc(returnLength);
if (buffer == NULL) {
perror("zstd");
exit(1);
}
} else {
/* some other error */
goto failed;
}
} else {
done = TRUE;
} }
ptr = buffer;
while (byteOffset + sizeof(SYSTEM_LOGICAL_PROCESSOR_INFORMATION) <= returnLength) {
if (ptr->Relationship == RelationProcessorCore) {
if (logical)
numCores += CountSetBits(ptr->ProcessorMask);
else
numCores++;
}
ptr++;
byteOffset += sizeof(SYSTEM_LOGICAL_PROCESSOR_INFORMATION);
}
free(buffer);
return numCores;
}
failed:
/* try to fall back on GetSystemInfo */
{ SYSTEM_INFO sysinfo;
GetSystemInfo(&sysinfo);
numCores = sysinfo.dwNumberOfProcessors;
if (numCores == 0) numCores = 1; /* just in case */
}
return numCores;
}
#elif defined(__APPLE__)
#include <sys/sysctl.h>
/* Use apple-provided syscall
* see: man 3 sysctl */
int UTIL_countCores(int logical)
{
static S32 numCores = 0; /* apple specifies int32_t */
if (numCores != 0) return numCores;
{ size_t size = sizeof(S32);
int const ret = sysctlbyname(logical ? "hw.logicalcpu" : "hw.physicalcpu", &numCores, &size, NULL, 0);
if (ret != 0) {
if (errno == ENOENT) {
/* entry not present, fall back on 1 */
numCores = 1;
} else {
perror("zstd: can't get number of cpus");
exit(1);
}
}
return numCores;
}
}
#elif defined(__linux__)
/* parse /proc/cpuinfo
* siblings / cpu cores should give hyperthreading ratio
* otherwise fall back on sysconf */
int UTIL_countCores(int logical)
{
static int numCores = 0;
if (numCores != 0) return numCores;
numCores = (int)sysconf(_SC_NPROCESSORS_ONLN);
if (numCores == -1) {
/* value not queryable, fall back on 1 */
return numCores = 1;
}
/* try to determine if there's hyperthreading */
{ FILE* const cpuinfo = fopen("/proc/cpuinfo", "r");
#define BUF_SIZE 80
char buff[BUF_SIZE];
int siblings = 0;
int cpu_cores = 0;
int ratio = 1;
if (cpuinfo == NULL) {
/* fall back on the sysconf value */
return numCores;
}
/* assume the cpu cores/siblings values will be constant across all
* present processors */
while (!feof(cpuinfo)) {
if (fgets(buff, BUF_SIZE, cpuinfo) != NULL) {
if (strncmp(buff, "siblings", 8) == 0) {
const char* const sep = strchr(buff, ':');
if (sep == NULL || *sep == '\0') {
/* formatting was broken? */
goto failed;
}
siblings = atoi(sep + 1);
}
if (strncmp(buff, "cpu cores", 9) == 0) {
const char* const sep = strchr(buff, ':');
if (sep == NULL || *sep == '\0') {
/* formatting was broken? */
goto failed;
}
cpu_cores = atoi(sep + 1);
}
} else if (ferror(cpuinfo)) {
/* fall back on the sysconf value */
goto failed;
} }
if (siblings && cpu_cores && siblings > cpu_cores) {
ratio = siblings / cpu_cores;
}
if (ratio && numCores > ratio && !logical) {
numCores = numCores / ratio;
}
failed:
fclose(cpuinfo);
return numCores;
}
}
#elif defined(__FreeBSD__)
#include <sys/sysctl.h>
/* Use physical core sysctl when available
* see: man 4 smp, man 3 sysctl */
int UTIL_countCores(int logical)
{
static int numCores = 0; /* freebsd sysctl is native int sized */
#if __FreeBSD_version >= 1300008
static int perCore = 1;
#endif
if (numCores != 0) return numCores;
#if __FreeBSD_version >= 1300008
{ size_t size = sizeof(numCores);
int ret = sysctlbyname("kern.smp.cores", &numCores, &size, NULL, 0);
if (ret == 0) {
if (logical) {
ret = sysctlbyname("kern.smp.threads_per_core", &perCore, &size, NULL, 0);
/* default to physical cores if logical cannot be read */
if (ret == 0)
numCores *= perCore;
}
return numCores;
}
if (errno != ENOENT) {
perror("zstd: can't get number of cpus");
exit(1);
}
/* sysctl not present, fall through to older sysconf method */
}
#else
/* suppress unused parameter warning */
(void) logical;
#endif
numCores = (int)sysconf(_SC_NPROCESSORS_ONLN);
if (numCores == -1) {
/* value not queryable, fall back on 1 */
numCores = 1;
}
return numCores;
}
#elif defined(__NetBSD__) || defined(__OpenBSD__) || defined(__DragonFly__) || defined(__CYGWIN__)
/* Use POSIX sysconf
* see: man 3 sysconf */
int UTIL_countCores(int logical)
{
static int numCores = 0;
/* suppress unused parameter warning */
(void)logical;
if (numCores != 0) return numCores;
numCores = (int)sysconf(_SC_NPROCESSORS_ONLN);
if (numCores == -1) {
/* value not queryable, fall back on 1 */
return numCores = 1;
}
return numCores;
}
#else
int UTIL_countCores(int logical)
{
/* suppress unused parameter warning */
(void)logical;
/* assume 1 */
return 1;
}
#endif
int UTIL_countPhysicalCores(void)
{
return UTIL_countCores(0);
}
int UTIL_countLogicalCores(void)
{
return UTIL_countCores(1);
}
rust/cli/src/lib.rs
+5
View File
@@ -3,12 +3,17 @@ mod benchfn;
#[path = "../../src/datagen.rs"]
mod datagen;
#[cfg(feature = "cli")]
#[path = "../../src/dibio.rs"]
mod dibio;
#[cfg(feature = "cli")]
#[path = "../../src/fileio_prefs.rs"]
mod fileio_prefs;
#[path = "../../src/lorem.rs"]
mod lorem;
#[path = "../../src/timefn.rs"]
mod timefn;
#[path = "../../src/util.rs"]
mod util;
#[cfg(feature = "cli")]
#[path = "../../src/zstd_cli.rs"]
mod zstd_cli;
rust/src/dibio.rs
+1022
View File
@@ -0,0 +1,1022 @@
#![allow(non_camel_case_types)]
#![allow(non_snake_case)]
#![allow(clippy::missing_safety_doc)]
#![allow(clippy::too_many_arguments)]
//! Dictionary training I/O for the command-line interface.
//!
//! The public entry point keeps the `programs/dibio.h` ABI. File discovery,
//! sample selection, buffering, and dictionary output are Rust-owned; the
//! actual legacy, COVER, and fastCOVER trainers are reached through their
//! existing C ABI symbols.
use std::cmp::min;
use std::ffi::{c_char, c_void, CStr};
use std::fmt;
use std::fs;
use std::io::{self, Write};
use std::mem::size_of;
use std::os::raw::{c_int, c_uint};
use std::path::PathBuf;
use std::time::{Duration, Instant};
#[cfg(unix)]
use std::ffi::OsStr;
#[cfg(unix)]
use std::os::unix::ffi::OsStrExt;
const KB: usize = 1 << 10;
const MB: usize = 1 << 20;
const GB: usize = 1 << 30;
const SAMPLESIZE_MAX: usize = 128 * KB;
const MEMMULT: u64 = 11;
const COVER_MEMMULT: u64 = 9;
const FASTCOVER_MEMMULT: u64 = 1;
const NOISELENGTH: usize = 32;
const MAX_SAMPLES_SIZE: usize = 2 * GB;
const REFRESH_RATE: Duration = Duration::from_micros(1_000_000 / 6);
// The C implementation initializes `dictSize` with the public enum value
// `ZSTD_error_GENERIC` (1), rather than an encoded size_t error.
const ERROR_GENERIC: usize = 1;
const MODE_READ: &[u8] = b"rb\0";
const MODE_WRITE: &[u8] = b"wb\0";
/// ABI-compatible `ZDICT_params_t` from `lib/zdict.h`.
#[repr(C)]
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub struct ZDICT_params_t {
pub compressionLevel: c_int,
pub notificationLevel: c_uint,
pub dictID: c_uint,
}
/// ABI-compatible `ZDICT_legacy_params_t` from `lib/zdict.h`.
#[repr(C)]
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub struct ZDICT_legacy_params_t {
pub selectivityLevel: c_uint,
pub zParams: ZDICT_params_t,
}
/// ABI-compatible `ZDICT_cover_params_t` from `lib/zdict.h`.
#[repr(C)]
#[derive(Clone, Copy, Debug, Default, PartialEq)]
pub struct ZDICT_cover_params_t {
pub k: c_uint,
pub d: c_uint,
pub steps: c_uint,
pub nbThreads: c_uint,
pub splitPoint: f64,
pub shrinkDict: c_uint,
pub shrinkDictMaxRegression: c_uint,
pub zParams: ZDICT_params_t,
}
/// ABI-compatible `ZDICT_fastCover_params_t` from `lib/zdict.h`.
#[repr(C)]
#[derive(Clone, Copy, Debug, Default, PartialEq)]
pub struct ZDICT_fastCover_params_t {
pub k: c_uint,
pub d: c_uint,
pub f: c_uint,
pub steps: c_uint,
pub nbThreads: c_uint,
pub splitPoint: f64,
pub accel: c_uint,
pub shrinkDict: c_uint,
pub shrinkDictMaxRegression: c_uint,
pub zParams: ZDICT_params_t,
}
unsafe extern "C" {
fn malloc(size: usize) -> *mut c_void;
fn free(pointer: *mut c_void);
fn fopen(filename: *const c_char, mode: *const c_char) -> *mut c_void;
fn fread(pointer: *mut c_void, size: usize, count: usize, stream: *mut c_void) -> usize;
fn fwrite(pointer: *const c_void, size: usize, count: usize, stream: *mut c_void) -> usize;
fn fclose(stream: *mut c_void) -> c_int;
fn ZDICT_isError(error_code: usize) -> c_uint;
fn ZDICT_getErrorName(error_code: usize) -> *const c_char;
fn ZDICT_trainFromBuffer_legacy(
dict_buffer: *mut c_void,
dict_buffer_capacity: usize,
samples_buffer: *const c_void,
samples_sizes: *const usize,
nb_samples: c_uint,
parameters: ZDICT_legacy_params_t,
) -> usize;
fn ZDICT_trainFromBuffer_cover(
dict_buffer: *mut c_void,
dict_buffer_capacity: usize,
samples_buffer: *const c_void,
samples_sizes: *const usize,
nb_samples: c_uint,
parameters: ZDICT_cover_params_t,
) -> usize;
fn ZDICT_optimizeTrainFromBuffer_cover(
dict_buffer: *mut c_void,
dict_buffer_capacity: usize,
samples_buffer: *const c_void,
samples_sizes: *const usize,
nb_samples: c_uint,
parameters: *mut ZDICT_cover_params_t,
) -> usize;
fn ZDICT_trainFromBuffer_fastCover(
dict_buffer: *mut c_void,
dict_buffer_capacity: usize,
samples_buffer: *const c_void,
samples_sizes: *const usize,
nb_samples: c_uint,
parameters: ZDICT_fastCover_params_t,
) -> usize;
fn ZDICT_optimizeTrainFromBuffer_fastCover(
dict_buffer: *mut c_void,
dict_buffer_capacity: usize,
samples_buffer: *const c_void,
samples_sizes: *const usize,
nb_samples: c_uint,
parameters: *mut ZDICT_fastCover_params_t,
) -> usize;
}
#[cfg(test)]
mod test_zdict_symbols {
use super::*;
static ERROR_NAME: &[u8] = b"test dictionary trainer error\0";
unsafe fn write_test_dictionary(buffer: *mut c_void, capacity: usize) -> usize {
let size = capacity.min(64);
if !buffer.is_null() {
std::ptr::write_bytes(buffer.cast::<u8>(), 0xA5, size);
}
size
}
#[no_mangle]
pub unsafe extern "C" fn ZDICT_isError(_code: usize) -> c_uint {
0
}
#[no_mangle]
pub unsafe extern "C" fn ZDICT_getErrorName(_code: usize) -> *const c_char {
ERROR_NAME.as_ptr().cast()
}
#[no_mangle]
pub unsafe extern "C" fn ZDICT_trainFromBuffer_legacy(
buffer: *mut c_void,
capacity: usize,
_samples: *const c_void,
_sample_sizes: *const usize,
_nb_samples: c_uint,
_parameters: ZDICT_legacy_params_t,
) -> usize {
write_test_dictionary(buffer, capacity)
}
#[no_mangle]
pub unsafe extern "C" fn ZDICT_trainFromBuffer_cover(
buffer: *mut c_void,
capacity: usize,
_samples: *const c_void,
_sample_sizes: *const usize,
_nb_samples: c_uint,
_parameters: ZDICT_cover_params_t,
) -> usize {
write_test_dictionary(buffer, capacity)
}
#[no_mangle]
pub unsafe extern "C" fn ZDICT_optimizeTrainFromBuffer_cover(
buffer: *mut c_void,
capacity: usize,
_samples: *const c_void,
_sample_sizes: *const usize,
_nb_samples: c_uint,
_parameters: *mut ZDICT_cover_params_t,
) -> usize {
write_test_dictionary(buffer, capacity)
}
#[no_mangle]
pub unsafe extern "C" fn ZDICT_trainFromBuffer_fastCover(
buffer: *mut c_void,
capacity: usize,
_samples: *const c_void,
_sample_sizes: *const usize,
_nb_samples: c_uint,
_parameters: ZDICT_fastCover_params_t,
) -> usize {
write_test_dictionary(buffer, capacity)
}
#[no_mangle]
pub unsafe extern "C" fn ZDICT_optimizeTrainFromBuffer_fastCover(
buffer: *mut c_void,
capacity: usize,
_samples: *const c_void,
_sample_sizes: *const usize,
_nb_samples: c_uint,
_parameters: *mut ZDICT_fastCover_params_t,
) -> usize {
write_test_dictionary(buffer, capacity)
}
}
#[derive(Clone, Copy, Debug, Default)]
struct FileStats {
total_size_to_load: i64,
nb_samples: c_int,
one_sample_too_large: bool,
}
struct DisplayClock {
last_update: Instant,
}
impl DisplayClock {
fn new() -> Self {
Self {
// `g_displayClock` is zero-initialized in the C implementation,
// so its first span always permits a progress update.
last_update: Instant::now() - REFRESH_RATE,
}
}
fn update(&mut self, display_level: c_int, level: c_int, message: fmt::Arguments<'_>) {
if display_level < level {
return;
}
if self.last_update.elapsed() > REFRESH_RATE || display_level >= 4 {
self.last_update = Instant::now();
eprint!("{message}");
if display_level >= 4 {
let _ = io::stderr().flush();
}
}
}
}
fn display(display_level: c_int, level: c_int, message: fmt::Arguments<'_>) {
if display_level >= level {
eprint!("{message}");
}
}
fn fatal(code: c_int, message: impl fmt::Display) -> ! {
eprintln!("Error {code} : {message}");
std::process::exit(code);
}
fn c_string_display(pointer: *const c_char) -> String {
unsafe { CStr::from_ptr(pointer) }
.to_string_lossy()
.into_owned()
}
#[cfg(unix)]
fn path_from_c_string(value: &CStr) -> PathBuf {
PathBuf::from(OsStr::from_bytes(value.to_bytes()))
}
#[cfg(not(unix))]
fn path_from_c_string(value: &CStr) -> PathBuf {
PathBuf::from(value.to_string_lossy().into_owned())
}
/// Returns `-1` for a missing or non-regular file, matching
/// `DiB_getFileSize()`'s `UTIL_FILESIZE_UNKNOWN` conversion.
fn get_file_size(file_name: *const c_char) -> i64 {
let path = path_from_c_string(unsafe { CStr::from_ptr(file_name) });
match fs::metadata(path) {
Ok(metadata) if metadata.is_file() => i64::try_from(metadata.len()).unwrap_or(-1),
_ => -1,
}
}
/// `MIN(fileSize, SAMPLESIZE_MAX)` in the original C source has different
/// usual-arithmetic-conversion behavior on 32-bit and 64-bit targets. Keep
/// that detail here because unknown files are represented by `-1`.
fn c_min_file_size(file_size: i64) -> i64 {
if size_of::<usize>() == 8 {
if (file_size as u64) < SAMPLESIZE_MAX as u64 {
file_size
} else {
SAMPLESIZE_MAX as i64
}
} else {
min(file_size, SAMPLESIZE_MAX as i64)
}
}
unsafe fn shuffle(file_names: *const *const c_char, nb_files: c_int) {
if nb_files <= 1 {
return;
}
// The C API takes `const char**` but deliberately shuffles the caller's
// table in place. The CLI supplies a writable pointer array.
let table = unsafe { std::slice::from_raw_parts_mut(file_names.cast_mut(), nb_files as usize) };
let mut seed = 0xFD2F_B528u32;
for index in (1..table.len()).rev() {
seed = seed.wrapping_mul(2_654_435_761);
seed ^= 2_246_822_519;
seed = seed.rotate_left(13);
let random = (seed >> 5) as usize;
table.swap(index, random % (index + 1));
}
}
fn file_stats(
file_names: *const *const c_char,
nb_files: c_int,
chunk_size: usize,
display_level: c_int,
) -> FileStats {
debug_assert!(chunk_size <= SAMPLESIZE_MAX);
let mut stats = FileStats::default();
for index in 0..nb_files.max(0) as usize {
let file_name = unsafe { *file_names.add(index) };
let file_size = get_file_size(file_name);
if file_size == 0 {
display(
display_level,
3,
format_args!(
"Sample file '{}' has zero size, skipping...\n",
c_string_display(file_name)
),
);
continue;
}
if chunk_size > 0 {
// This is the C expression `(fileSize + chunkSize - 1) /
// chunkSize`, including its unsigned wrap for an unknown file on
// the 64-bit targets where size_t and S64 have equal width.
let chunks = (file_size as u64)
.wrapping_add(chunk_size as u64)
.wrapping_sub(1)
/ chunk_size as u64;
stats.nb_samples = stats.nb_samples.wrapping_add(chunks as c_int);
stats.total_size_to_load = stats.total_size_to_load.wrapping_add(file_size);
} else {
if file_size > SAMPLESIZE_MAX as i64 {
if file_size > (2 * SAMPLESIZE_MAX) as i64 {
stats.one_sample_too_large = true;
}
display(
display_level,
3,
format_args!(
"Sample file '{}' is too large, limiting to {} KB\n",
c_string_display(file_name),
SAMPLESIZE_MAX / KB
),
);
}
stats.nb_samples = stats.nb_samples.wrapping_add(1);
stats.total_size_to_load = stats
.total_size_to_load
.wrapping_add(c_min_file_size(file_size));
}
}
display(
display_level,
4,
format_args!(
"Found training data {} files, {} KB, {} samples\n",
nb_files,
(stats.total_size_to_load / KB as i64) as c_int,
stats.nb_samples
),
);
stats
}
fn maximum_memory() -> usize {
if size_of::<usize>() == 4 {
2 * GB - 64 * MB
} else {
(512 * MB) << size_of::<usize>()
}
}
unsafe fn find_max_memory(required_memory: u64) -> usize {
let step = (8 * MB) as u64;
let mut required = (((required_memory >> 23) + 1) << 23).wrapping_add(step);
required = min(required, maximum_memory() as u64);
loop {
let test_memory = unsafe { malloc(required as usize) };
if !test_memory.is_null() {
unsafe { free(test_memory) };
return required as usize;
}
required = required.wrapping_sub(step);
}
}
unsafe fn fill_noise(buffer: *mut u8, length: usize) {
let mut accumulator = 2_654_435_761u32;
for index in 0..length {
accumulator = accumulator.wrapping_mul(2_246_822_519);
unsafe { buffer.add(index).write((accumulator >> 21) as u8) };
}
}
unsafe fn read_chunk(
stream: *mut c_void,
destination: *mut u8,
length: usize,
file_name: *const c_char,
) {
let read = unsafe { fread(destination.cast(), 1, length, stream) };
if read != length {
fatal(
11,
format_args!("Pb reading {}", c_string_display(file_name)),
);
}
}
unsafe fn load_files(
buffer: *mut u8,
buffer_size: usize,
sample_sizes: *mut usize,
sample_count: usize,
file_names: *const *const c_char,
nb_files: c_int,
target_chunk_size: usize,
display_level: c_int,
display_clock: &mut DisplayClock,
) -> (usize, usize) {
debug_assert!(target_chunk_size <= SAMPLESIZE_MAX);
let mut total_data_loaded = 0usize;
let mut nb_samples_loaded = 0usize;
let mut file_index = 0usize;
let file_count = nb_files.max(0) as usize;
while nb_samples_loaded < sample_count && file_index < file_count {
let file_name = unsafe { *file_names.add(file_index) };
let file_size = get_file_size(file_name);
if file_size <= 0 {
file_index += 1;
continue;
}
let stream = unsafe { fopen(file_name, MODE_READ.as_ptr().cast()) };
if stream.is_null() {
let error = io::Error::last_os_error();
fatal(
10,
format_args!(
"zstd: dictBuilder: {} {} ",
c_string_display(file_name),
error
),
);
}
display_clock.update(
display_level,
2,
format_args!("Loading {}... \r", c_string_display(file_name)),
);
let first_chunk = if target_chunk_size > 0 {
min(file_size as usize, target_chunk_size)
} else {
min(file_size as usize, SAMPLESIZE_MAX)
};
if total_data_loaded.wrapping_add(first_chunk) > buffer_size {
unsafe { fclose(stream) };
break;
}
unsafe {
read_chunk(
stream,
buffer.add(total_data_loaded),
first_chunk,
file_name,
);
sample_sizes.add(nb_samples_loaded).write(first_chunk);
}
nb_samples_loaded += 1;
total_data_loaded += first_chunk;
if target_chunk_size > 0 {
let mut file_data_loaded = first_chunk;
while (file_data_loaded as i64) < file_size && nb_samples_loaded < sample_count {
let remaining = (file_size - file_data_loaded as i64) as usize;
let chunk_size = min(remaining, target_chunk_size);
if total_data_loaded.wrapping_add(chunk_size) > buffer_size {
break;
}
unsafe {
read_chunk(stream, buffer.add(total_data_loaded), chunk_size, file_name);
sample_sizes.add(nb_samples_loaded).write(chunk_size);
}
nb_samples_loaded += 1;
total_data_loaded += chunk_size;
file_data_loaded += chunk_size;
}
}
file_index += 1;
unsafe { fclose(stream) };
}
display(display_level, 2, format_args!("\r{:79}\r", ""));
display(
display_level,
4,
format_args!(
"Loaded {} KB total training data, {} nb samples \n",
total_data_loaded / KB,
nb_samples_loaded
),
);
(total_data_loaded, nb_samples_loaded)
}
fn loaded_size(total_size_to_load: i64, max_memory: usize) -> usize {
let first = min(max_memory as i64, total_size_to_load);
if size_of::<usize>() == 8 {
min(first as u64, MAX_SAMPLES_SIZE as u64) as usize
} else {
min(first, MAX_SAMPLES_SIZE as i64) as usize
}
}
unsafe fn save_dictionary(
dictionary_file_name: *const c_char,
dictionary: *const c_void,
dictionary_size: usize,
) {
let stream = unsafe { fopen(dictionary_file_name, MODE_WRITE.as_ptr().cast()) };
if stream.is_null() {
fatal(
3,
format_args!("cannot open {} ", c_string_display(dictionary_file_name)),
);
}
let written = unsafe { fwrite(dictionary, 1, dictionary_size, stream) };
if written != dictionary_size {
fatal(
4,
format_args!("{} : write error", c_string_display(dictionary_file_name)),
);
}
if unsafe { fclose(stream) } != 0 {
fatal(
5,
format_args!("{} : flush error", c_string_display(dictionary_file_name)),
);
}
}
unsafe fn error_name(error_code: usize) -> String {
let name = unsafe { ZDICT_getErrorName(error_code) };
if name.is_null() {
return "unknown error".to_owned();
}
unsafe { CStr::from_ptr(name) }
.to_string_lossy()
.into_owned()
}
unsafe fn free_buffers(
source_buffer: *mut c_void,
sample_sizes: *mut usize,
dictionary_buffer: *mut c_void,
) {
unsafe {
free(source_buffer);
free(sample_sizes.cast());
free(dictionary_buffer);
}
}
/// Train a dictionary from files and write it to `dictionary_file_name`.
///
/// This is the Rust implementation of the public `DiB_trainFromFiles()` ABI
/// declared in `programs/dibio.h`. Its exit-on-I/O-error behavior is kept for
/// compatibility with the command-line program; trainer errors return `1`.
#[no_mangle]
pub unsafe extern "C" fn DiB_trainFromFiles(
dictionary_file_name: *const c_char,
max_dictionary_size: usize,
file_names: *const *const c_char,
nb_files: c_int,
chunk_size: usize,
legacy_params: *mut ZDICT_legacy_params_t,
cover_params: *mut ZDICT_cover_params_t,
fast_cover_params: *mut ZDICT_fastCover_params_t,
optimize: c_int,
memory_limit: c_uint,
) -> c_int {
let display_level = if !legacy_params.is_null() {
unsafe { (*legacy_params).zParams.notificationLevel as c_int }
} else if !cover_params.is_null() {
unsafe { (*cover_params).zParams.notificationLevel as c_int }
} else if !fast_cover_params.is_null() {
unsafe { (*fast_cover_params).zParams.notificationLevel as c_int }
} else {
0
};
let mut display_clock = DisplayClock::new();
let dictionary_buffer = unsafe { malloc(max_dictionary_size) };
display(display_level, 3, format_args!("Shuffling input files\n"));
unsafe { shuffle(file_names, nb_files) };
let stats = file_stats(file_names, nb_files, chunk_size, display_level);
let memory_multiplier = if !legacy_params.is_null() {
MEMMULT
} else if !cover_params.is_null() {
COVER_MEMMULT
} else {
FASTCOVER_MEMMULT
};
let required_memory = (stats.total_size_to_load as u64).wrapping_mul(memory_multiplier);
let max_memory = unsafe { find_max_memory(required_memory) } / memory_multiplier as usize;
let mut loaded_size = loaded_size(stats.total_size_to_load, max_memory);
if memory_limit != 0 {
display(
display_level,
2,
format_args!(
"! Warning : setting manual memory limit for dictionary training data at {} MB \n",
(memory_limit as usize) / MB
),
);
loaded_size = min(loaded_size, memory_limit as usize);
}
let source_buffer_size = loaded_size.wrapping_add(NOISELENGTH);
let source_buffer = unsafe { malloc(source_buffer_size) };
let sample_count = stats.nb_samples.max(0) as usize;
let sample_sizes_bytes = sample_count.wrapping_mul(size_of::<usize>());
let sample_sizes = unsafe { malloc(sample_sizes_bytes) }.cast::<usize>();
if ((stats.nb_samples > 0) && sample_sizes.is_null())
|| source_buffer.is_null()
|| dictionary_buffer.is_null()
{
fatal(12, "not enough memory for DiB_trainFiles");
}
if stats.one_sample_too_large {
display(
display_level,
2,
format_args!("! Warning : some sample(s) are very large \n"),
);
display(
display_level,
2,
format_args!("! Note that dictionary is only useful for small samples. \n"),
);
display(
display_level,
2,
format_args!(
"! As a consequence, only the first {} bytes of each sample are loaded \n",
SAMPLESIZE_MAX
),
);
}
if stats.nb_samples < 5 {
display(
display_level,
2,
format_args!("! Warning : nb of samples too low for proper processing ! \n"),
);
display(
display_level,
2,
format_args!("! Please provide _one file per sample_. \n"),
);
display(
display_level,
2,
format_args!(
"! Alternatively, split files into fixed-size blocks representative of samples, with -B# \n"
),
);
fatal(14, "nb of samples too low");
}
let dictionary_size_threshold = (max_dictionary_size as i64).wrapping_mul(8);
if stats.total_size_to_load < dictionary_size_threshold {
display(
display_level,
2,
format_args!(
"! Warning : data size of samples too small for target dictionary size \n"
),
);
display(
display_level,
2,
format_args!("! Samples should be about 100x larger than target dictionary size \n"),
);
}
if (loaded_size as i64) < stats.total_size_to_load {
display(
display_level,
1,
format_args!(
"Training samples set too large ({} MB); training on {} MB only...\n",
(stats.total_size_to_load / MB as i64) as c_uint,
loaded_size / MB
),
);
}
let (loaded_size, nb_samples_loaded) = unsafe {
load_files(
source_buffer.cast(),
loaded_size,
sample_sizes,
sample_count,
file_names,
nb_files,
chunk_size,
display_level,
&mut display_clock,
)
};
let mut dictionary_size = ERROR_GENERIC;
if !legacy_params.is_null() {
unsafe { fill_noise(source_buffer.cast::<u8>().add(loaded_size), NOISELENGTH) };
dictionary_size = unsafe {
ZDICT_trainFromBuffer_legacy(
dictionary_buffer,
max_dictionary_size,
source_buffer,
sample_sizes,
nb_samples_loaded as c_uint,
*legacy_params,
)
};
} else if !cover_params.is_null() {
if optimize != 0 {
dictionary_size = unsafe {
ZDICT_optimizeTrainFromBuffer_cover(
dictionary_buffer,
max_dictionary_size,
source_buffer,
sample_sizes,
nb_samples_loaded as c_uint,
cover_params,
)
};
if unsafe { ZDICT_isError(dictionary_size) } == 0 {
let parameters = unsafe { &*cover_params };
display(
display_level,
2,
format_args!(
"k={}\nd={}\nsteps={}\nsplit={}\n",
parameters.k,
parameters.d,
parameters.steps,
(parameters.splitPoint * 100.0) as c_uint
),
);
}
} else {
dictionary_size = unsafe {
ZDICT_trainFromBuffer_cover(
dictionary_buffer,
max_dictionary_size,
source_buffer,
sample_sizes,
nb_samples_loaded as c_uint,
*cover_params,
)
};
}
} else if !fast_cover_params.is_null() {
if optimize != 0 {
dictionary_size = unsafe {
ZDICT_optimizeTrainFromBuffer_fastCover(
dictionary_buffer,
max_dictionary_size,
source_buffer,
sample_sizes,
nb_samples_loaded as c_uint,
fast_cover_params,
)
};
if unsafe { ZDICT_isError(dictionary_size) } == 0 {
let parameters = unsafe { &*fast_cover_params };
display(
display_level,
2,
format_args!(
"k={}\nd={}\nf={}\nsteps={}\nsplit={}\naccel={}\n",
parameters.k,
parameters.d,
parameters.f,
parameters.steps,
(parameters.splitPoint * 100.0) as c_uint,
parameters.accel
),
);
}
} else {
dictionary_size = unsafe {
ZDICT_trainFromBuffer_fastCover(
dictionary_buffer,
max_dictionary_size,
source_buffer,
sample_sizes,
nb_samples_loaded as c_uint,
*fast_cover_params,
)
};
}
} else {
debug_assert!(false, "one dictionary trainer parameter set is required");
}
if unsafe { ZDICT_isError(dictionary_size) } != 0 {
display(
display_level,
1,
format_args!("dictionary training failed : {} \n", unsafe {
error_name(dictionary_size)
}),
);
unsafe { free_buffers(source_buffer, sample_sizes, dictionary_buffer) };
return 1;
}
display(
display_level,
2,
format_args!(
"Save dictionary of size {} into file {} \n",
dictionary_size as c_uint,
c_string_display(dictionary_file_name)
),
);
unsafe {
save_dictionary(dictionary_file_name, dictionary_buffer, dictionary_size);
free_buffers(source_buffer, sample_sizes, dictionary_buffer);
}
0
}
#[cfg(test)]
mod tests {
use super::*;
use std::ffi::CString;
use std::fs;
use std::mem::{offset_of, size_of};
use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicUsize, Ordering};
#[cfg(unix)]
fn path_to_c_string(path: &Path) -> CString {
CString::new(path.as_os_str().as_bytes()).expect("temporary path has no NUL")
}
#[cfg(not(unix))]
fn path_to_c_string(path: &Path) -> CString {
CString::new(path.to_string_lossy().as_bytes()).expect("temporary path has no NUL")
}
fn sample(index: usize) -> Vec<u8> {
let line = format!(
"record={index:02};common-key=common-value;payload=abcdefghijklmnopqrstuvwxyz\n"
);
let mut result = Vec::with_capacity(1024);
while result.len() < 1024 {
result.extend_from_slice(line.as_bytes());
}
result.truncate(1024);
result
}
fn temporary_paths() -> (Vec<PathBuf>, PathBuf) {
static NEXT: AtomicUsize = AtomicUsize::new(0);
let prefix = format!(
"zstd-dibio-{}-{}",
std::process::id(),
NEXT.fetch_add(1, Ordering::Relaxed)
);
let root = std::env::temp_dir();
let samples = (0..5)
.map(|index| root.join(format!("{prefix}-sample-{index}")))
.collect::<Vec<_>>();
let dictionary = root.join(format!("{prefix}-dictionary"));
(samples, dictionary)
}
#[test]
fn dictionary_parameter_layouts_match_zdict_h() {
assert_eq!(size_of::<ZDICT_params_t>(), 12);
assert_eq!(size_of::<ZDICT_legacy_params_t>(), 16);
assert_eq!(size_of::<ZDICT_cover_params_t>(), 48);
assert_eq!(size_of::<ZDICT_fastCover_params_t>(), 56);
assert_eq!(offset_of!(ZDICT_cover_params_t, splitPoint), 16);
assert_eq!(offset_of!(ZDICT_fastCover_params_t, splitPoint), 24);
}
#[test]
fn trains_from_chunked_temporary_sample_files() {
let (sample_paths, dictionary_path) = temporary_paths();
let _cleanup = TemporaryPathCleanup {
paths: sample_paths
.iter()
.cloned()
.chain(std::iter::once(dictionary_path.clone()))
.collect(),
};
for (index, path) in sample_paths.iter().enumerate() {
fs::write(path, sample(index)).expect("write temporary training sample");
}
let sample_names = sample_paths
.iter()
.map(|path| path_to_c_string(path))
.collect::<Vec<_>>();
let file_names = sample_names
.iter()
.map(|name| name.as_ptr())
.collect::<Vec<_>>();
let dictionary_name = path_to_c_string(&dictionary_path);
let mut parameters = ZDICT_legacy_params_t {
zParams: ZDICT_params_t {
compressionLevel: 3,
..ZDICT_params_t::default()
},
..ZDICT_legacy_params_t::default()
};
let result = unsafe {
DiB_trainFromFiles(
dictionary_name.as_ptr(),
2048,
file_names.as_ptr(),
file_names.len() as c_int,
256,
&mut parameters,
std::ptr::null_mut(),
std::ptr::null_mut(),
0,
0,
)
};
assert_eq!(result, 0);
let dictionary = fs::read(&dictionary_path).expect("read trained dictionary");
assert!(!dictionary.is_empty());
}
#[test]
fn legacy_trainer_abi_smoke() {
let samples = (0..5).map(sample).collect::<Vec<_>>();
let sample_sizes = samples.iter().map(Vec::len).collect::<Vec<_>>();
let flat_samples = samples.concat();
let mut dictionary = vec![0u8; 2048];
let parameters = ZDICT_legacy_params_t::default();
let result = unsafe {
ZDICT_trainFromBuffer_legacy(
dictionary.as_mut_ptr().cast(),
dictionary.len(),
flat_samples.as_ptr().cast(),
sample_sizes.as_ptr(),
sample_sizes.len() as c_uint,
parameters,
)
};
assert_eq!(unsafe { ZDICT_isError(result) }, 0);
assert!(result > 0);
}
impl Drop for TemporaryPathCleanup {
fn drop(&mut self) {
for path in &self.paths {
let _ = fs::remove_file(path);
}
}
}
struct TemporaryPathCleanup {
paths: Vec<PathBuf>,
}
}
rust/src/util.rs
+1501
View File
@@ -0,0 +1,1501 @@
#![allow(non_camel_case_types)]
#![allow(non_snake_case)]
#![allow(clippy::missing_safety_doc)]
//! Rust implementation of the program utility layer from `programs/util.c`.
//!
//! The C header remains the public ABI. Functions in this module therefore
//! use the C allocator for objects which are returned to C, keep all exported
//! structures `repr(C)`, and use the target's `libc::stat` representation for
//! the `stat_t` pointer passed by `util.h`.
use std::ffi::{CStr, CString, OsString};
use std::fs;
use std::mem::{offset_of, size_of, MaybeUninit};
use std::os::raw::{c_char, c_int, c_uint, c_void};
use std::path::PathBuf;
use std::ptr;
use std::sync::atomic::{AtomicBool, AtomicI32, Ordering};
use std::sync::OnceLock;
#[cfg(unix)]
use std::os::unix::ffi::OsStringExt;
type Stat = libc::stat;
#[cfg(unix)]
type UtilMode = libc::mode_t;
#[cfg(windows)]
type UtilMode = c_int;
#[cfg(not(any(unix, windows)))]
type UtilMode = c_uint;
const LIST_SIZE_INCREASE: usize = 8 * 1024;
const MAX_FILE_OF_FILE_NAMES_SIZE: u64 = 50 * (1 << 20);
const UTIL_FILESIZE_UNKNOWN: u64 = u64::MAX;
const DIR_DEFAULT_MODE: u32 = 0o755;
/// `g_utilDisplayLevel` is a public C global, not a Rust-owned preference.
#[no_mangle]
pub static mut g_utilDisplayLevel: c_int = 0;
/// The two structs below mirror the declarations in `programs/util.h`.
#[repr(C)]
#[derive(Clone, Copy, Debug, PartialEq)]
pub struct UTIL_HumanReadableSize_t {
pub value: f64,
pub precision: c_int,
pub suffix: *const c_char,
}
#[repr(C)]
#[derive(Debug)]
pub struct FileNamesTable {
pub fileNames: *mut *const c_char,
pub buf: *mut c_char,
pub tableSize: usize,
pub tableCapacity: usize,
}
const _: () = assert!(offset_of!(UTIL_HumanReadableSize_t, value) == 0);
const _: () = assert!(offset_of!(UTIL_HumanReadableSize_t, precision) == size_of::<f64>());
const _: () = assert!(
offset_of!(UTIL_HumanReadableSize_t, suffix)
>= offset_of!(UTIL_HumanReadableSize_t, precision) + size_of::<c_int>()
);
const _: () = assert!(
size_of::<UTIL_HumanReadableSize_t>()
== offset_of!(UTIL_HumanReadableSize_t, suffix) + size_of::<*const c_char>()
);
const _: () = assert!(offset_of!(FileNamesTable, fileNames) == 0);
const _: () = assert!(offset_of!(FileNamesTable, buf) == size_of::<*const c_char>());
const _: () = assert!(
offset_of!(FileNamesTable, tableSize)
== offset_of!(FileNamesTable, buf) + size_of::<*mut c_char>()
);
const _: () = assert!(
offset_of!(FileNamesTable, tableCapacity)
== offset_of!(FileNamesTable, tableSize) + size_of::<usize>()
);
const _: () = assert!(
size_of::<FileNamesTable>() == offset_of!(FileNamesTable, tableCapacity) + size_of::<usize>()
);
static EMPTY_EXTENSION: [u8; 1] = [0];
static SUFFIX_B: &[u8] = b" B\0";
static SUFFIX_KIB: &[u8] = b" KiB\0";
static SUFFIX_MIB: &[u8] = b" MiB\0";
static SUFFIX_GIB: &[u8] = b" GiB\0";
static SUFFIX_TIB: &[u8] = b" TiB\0";
static SUFFIX_PIB: &[u8] = b" PiB\0";
static SUFFIX_EIB: &[u8] = b" EiB\0";
static FAKE_STDIN_IS_CONSOLE: AtomicBool = AtomicBool::new(false);
static FAKE_STDOUT_IS_CONSOLE: AtomicBool = AtomicBool::new(false);
static FAKE_STDERR_IS_CONSOLE: AtomicBool = AtomicBool::new(false);
static TRACE_FILE_STAT: AtomicBool = AtomicBool::new(false);
static TRACE_DEPTH: AtomicI32 = AtomicI32::new(0);
static CORE_COUNT: OnceLock<c_int> = OnceLock::new();
#[cfg(windows)]
unsafe extern "C" {
fn UTIL_rust_utime(filename: *const c_char, statbuf: *const Stat) -> c_int;
}
#[inline]
fn display_level() -> c_int {
// The C API intentionally exposes this mutable global. All reads mirror
// the unsynchronized reads in the original implementation.
unsafe { g_utilDisplayLevel }
}
fn display(message: &str) {
eprint!("{message}");
}
unsafe fn display_c_string(message: *const c_char) {
if !message.is_null() {
display(&String::from_utf8_lossy(CStr::from_ptr(message).to_bytes()));
}
}
fn trace_call(message: &str) {
if TRACE_FILE_STAT.load(Ordering::Relaxed) {
let depth = TRACE_DEPTH.fetch_add(1, Ordering::Relaxed).max(0) as usize;
eprintln!("Trace:FileStat: {:width$}> {message}", "", width = depth);
}
}
fn trace_return(ret: c_int) {
if TRACE_FILE_STAT.load(Ordering::Relaxed) {
let depth = TRACE_DEPTH.fetch_sub(1, Ordering::Relaxed) - 1;
eprintln!(
"Trace:FileStat: {:width$}< {ret}",
"",
width = depth.max(0) as usize
);
}
}
#[inline]
unsafe fn c_bytes<'a>(value: *const c_char) -> &'a [u8] {
CStr::from_ptr(value).to_bytes()
}
#[inline]
fn path_separator() -> u8 {
if cfg!(windows) {
b'\\'
} else {
b'/'
}
}
fn path_from_bytes(bytes: &[u8]) -> PathBuf {
#[cfg(unix)]
{
PathBuf::from(OsString::from_vec(bytes.to_vec()))
}
#[cfg(not(unix))]
{
PathBuf::from(String::from_utf8_lossy(bytes).into_owned())
}
}
fn os_string_bytes(value: OsString) -> Vec<u8> {
#[cfg(unix)]
{
value.into_vec()
}
#[cfg(not(unix))]
{
value.to_string_lossy().into_owned().into_bytes()
}
}
fn c_path(bytes: &[u8]) -> CString {
CString::new(bytes).unwrap_or_else(|_| std::process::abort())
}
unsafe fn malloc_bytes(size: usize) -> *mut u8 {
let ptr = libc::malloc(size).cast::<u8>();
if ptr.is_null() {
std::process::abort();
}
ptr
}
unsafe fn malloc_array<T>(count: usize) -> *mut T {
let Some(size) = count.checked_mul(size_of::<T>()) else {
return ptr::null_mut();
};
libc::malloc(size).cast::<T>()
}
unsafe fn allocate_struct<T>() -> *mut T {
let result = malloc_bytes(size_of::<T>()).cast::<T>();
// The allocation helper is used only for C functions whose original
// CONTROL() path terminates the program on allocation failure.
ptr::write_bytes(result.cast::<u8>(), 0, size_of::<T>());
result
}
unsafe fn free_ptr<T>(value: *mut T) {
libc::free(value.cast::<c_void>());
}
unsafe fn table_entries<'a>(table: *const FileNamesTable) -> &'a [*const c_char] {
if (*table).fileNames.is_null() || (*table).tableSize == 0 {
&[]
} else {
std::slice::from_raw_parts((*table).fileNames, (*table).tableSize)
}
}
unsafe fn table_names_size(table: *const FileNamesTable) -> usize {
let mut total = 0usize;
for &name in table_entries(table) {
if name.is_null() {
break;
}
total = total.saturating_add(c_bytes(name).len().saturating_add(1));
}
total
}
unsafe fn make_table(
file_names: *mut *const c_char,
table_size: usize,
table_capacity: usize,
buf: *mut c_char,
) -> *mut FileNamesTable {
let table = allocate_struct::<FileNamesTable>();
(*table).fileNames = file_names;
(*table).buf = buf;
(*table).tableSize = table_size;
(*table).tableCapacity = table_capacity;
table
}
unsafe fn make_table_from_buffer(
buf: *mut u8,
buffer_len: usize,
table_size: usize,
table_capacity: usize,
) -> *mut FileNamesTable {
let Some(pointer_count) = table_capacity.checked_mul(size_of::<*const c_char>()) else {
free_ptr(buf);
return ptr::null_mut();
};
let file_names = malloc_bytes(pointer_count.max(1)).cast::<*const c_char>();
let mut pos = 0usize;
for index in 0..table_size {
if pos > buffer_len {
free_ptr(file_names);
free_ptr(buf);
return ptr::null_mut();
}
*file_names.add(index) = buf.add(pos).cast::<c_char>();
let name_len = CStr::from_ptr(buf.add(pos).cast::<c_char>())
.to_bytes()
.len();
pos = pos.saturating_add(name_len + 1);
}
make_table(file_names, table_size, table_capacity, buf.cast::<c_char>())
}
#[inline]
unsafe fn stat_mode(statbuf: *const Stat) -> u64 {
(*statbuf).st_mode as u64
}
#[cfg(unix)]
#[inline]
unsafe fn mode_is(statbuf: *const Stat, kind: libc::mode_t) -> bool {
stat_mode(statbuf) & libc::S_IFMT as u64 == kind as u64
}
#[cfg(windows)]
#[inline]
unsafe fn mode_is(statbuf: *const Stat, kind: c_int) -> bool {
// util.c intentionally uses `(st_mode & S_IFREG) != 0` for the CRT.
stat_mode(statbuf) & kind as u64 != 0
}
#[cfg(not(any(unix, windows)))]
#[inline]
unsafe fn mode_is(_statbuf: *const Stat, _kind: c_uint) -> bool {
false
}
#[no_mangle]
pub unsafe extern "C" fn UTIL_fstat(
fd: c_int,
filename: *const c_char,
statbuf: *mut Stat,
) -> c_int {
trace_call("UTIL_stat");
let result = if fd >= 0 {
libc::fstat(fd, statbuf)
} else {
libc::stat(filename, statbuf)
};
let ret = (result == 0) as c_int;
trace_return(ret);
ret
}
#[no_mangle]
pub unsafe extern "C" fn UTIL_stat(filename: *const c_char, statbuf: *mut Stat) -> c_int {
UTIL_fstat(-1, filename, statbuf)
}
#[no_mangle]
pub unsafe extern "C" fn UTIL_isRegularFileStat(statbuf: *const Stat) -> c_int {
#[cfg(any(unix, windows))]
{
mode_is(statbuf, libc::S_IFREG) as c_int
}
#[cfg(not(any(unix, windows)))]
{
0
}
}
#[no_mangle]
pub unsafe extern "C" fn UTIL_isDirectoryStat(statbuf: *const Stat) -> c_int {
#[cfg(any(unix, windows))]
{
mode_is(statbuf, libc::S_IFDIR) as c_int
}
#[cfg(not(any(unix, windows)))]
{
0
}
}
#[no_mangle]
pub unsafe extern "C" fn UTIL_isFIFOStat(statbuf: *const Stat) -> c_int {
#[cfg(unix)]
{
mode_is(statbuf, libc::S_IFIFO) as c_int
}
#[cfg(not(unix))]
{
let _ = statbuf;
0
}
}
#[no_mangle]
pub unsafe extern "C" fn UTIL_isBlockDevStat(statbuf: *const Stat) -> c_int {
#[cfg(unix)]
{
mode_is(statbuf, libc::S_IFBLK) as c_int
}
#[cfg(not(unix))]
{
let _ = statbuf;
0
}
}
#[no_mangle]
pub unsafe extern "C" fn UTIL_isRegularFile(infilename: *const c_char) -> c_int {
let mut statbuf = MaybeUninit::<Stat>::uninit();
if UTIL_stat(infilename, statbuf.as_mut_ptr()) == 0 {
return 0;
}
UTIL_isRegularFileStat(statbuf.as_ptr())
}
#[no_mangle]
pub unsafe extern "C" fn UTIL_isDirectory(infilename: *const c_char) -> c_int {
let mut statbuf = MaybeUninit::<Stat>::uninit();
if UTIL_stat(infilename, statbuf.as_mut_ptr()) == 0 {
return 0;
}
UTIL_isDirectoryStat(statbuf.as_ptr())
}
#[no_mangle]
pub unsafe extern "C" fn UTIL_isFIFO(infilename: *const c_char) -> c_int {
#[cfg(unix)]
{
let mut statbuf = MaybeUninit::<Stat>::uninit();
if UTIL_stat(infilename, statbuf.as_mut_ptr()) != 0 {
return UTIL_isFIFOStat(statbuf.as_ptr());
}
}
let _ = infilename;
0
}
#[no_mangle]
pub unsafe extern "C" fn UTIL_isLink(infilename: *const c_char) -> c_int {
#[cfg(unix)]
{
let mut statbuf = MaybeUninit::<Stat>::uninit();
if libc::lstat(infilename, statbuf.as_mut_ptr()) == 0 {
return mode_is(statbuf.as_ptr(), libc::S_IFLNK) as c_int;
}
}
let _ = infilename;
0
}
#[no_mangle]
pub unsafe extern "C" fn UTIL_isSameFile(file1: *const c_char, file2: *const c_char) -> c_int {
assert!(!file1.is_null() && !file2.is_null());
#[cfg(windows)]
{
return (c_bytes(file1) == c_bytes(file2)) as c_int;
}
#[cfg(not(windows))]
{
let mut file1_stat = MaybeUninit::<Stat>::uninit();
let mut file2_stat = MaybeUninit::<Stat>::uninit();
if UTIL_stat(file1, file1_stat.as_mut_ptr()) == 0
|| UTIL_stat(file2, file2_stat.as_mut_ptr()) == 0
{
return 0;
}
UTIL_isSameFileStat(file1, file2, file1_stat.as_ptr(), file2_stat.as_ptr())
}
}
#[no_mangle]
pub unsafe extern "C" fn UTIL_isSameFileStat(
file1: *const c_char,
file2: *const c_char,
file1_stat: *const Stat,
file2_stat: *const Stat,
) -> c_int {
assert!(!file1.is_null() && !file2.is_null());
#[cfg(windows)]
{
let _ = (file1_stat, file2_stat);
return (c_bytes(file1) == c_bytes(file2)) as c_int;
}
#[cfg(not(windows))]
{
((*file1_stat).st_dev == (*file2_stat).st_dev
&& (*file1_stat).st_ino == (*file2_stat).st_ino) as c_int
}
}
#[no_mangle]
pub unsafe extern "C" fn UTIL_getFileSizeStat(statbuf: *const Stat) -> u64 {
if UTIL_isRegularFileStat(statbuf) == 0 {
return UTIL_FILESIZE_UNKNOWN;
}
(*statbuf).st_size as u64
}
#[no_mangle]
pub unsafe extern "C" fn UTIL_getFileSize(infilename: *const c_char) -> u64 {
let mut statbuf = MaybeUninit::<Stat>::uninit();
if UTIL_stat(infilename, statbuf.as_mut_ptr()) == 0 {
return UTIL_FILESIZE_UNKNOWN;
}
UTIL_getFileSizeStat(statbuf.as_ptr())
}
#[no_mangle]
pub unsafe extern "C" fn UTIL_getTotalFileSize(
file_names: *const *const c_char,
nb_files: c_uint,
) -> u64 {
let mut total = 0u64;
for index in 0..nb_files as usize {
let size = UTIL_getFileSize(*file_names.add(index));
if size == UTIL_FILESIZE_UNKNOWN {
return UTIL_FILESIZE_UNKNOWN;
}
total = total.wrapping_add(size);
}
total
}
#[no_mangle]
pub unsafe extern "C" fn UTIL_chmod(
filename: *const c_char,
statbuf: *const Stat,
permissions: UtilMode,
) -> c_int {
UTIL_fchmod(-1, filename, statbuf, permissions)
}
#[no_mangle]
pub unsafe extern "C" fn UTIL_fchmod(
fd: c_int,
filename: *const c_char,
statbuf: *const Stat,
permissions: UtilMode,
) -> c_int {
let mut local_stat = MaybeUninit::<Stat>::uninit();
let stat_ptr = if statbuf.is_null() {
if UTIL_fstat(fd, filename, local_stat.as_mut_ptr()) == 0 {
return 0;
}
local_stat.as_ptr()
} else {
statbuf
};
if UTIL_isRegularFileStat(stat_ptr) == 0 {
return 0;
}
#[cfg(unix)]
{
if fd >= 0 {
return libc::fchmod(fd, permissions);
}
libc::chmod(filename, permissions)
}
#[cfg(windows)]
{
let _ = fd;
libc::chmod(filename, permissions)
}
#[cfg(not(any(unix, windows)))]
{
let _ = (fd, filename, permissions);
0
}
}
#[cfg(any(target_os = "linux", target_os = "android"))]
unsafe fn set_mtime(filename: *const c_char, statbuf: *const Stat) -> c_int {
let now = libc::timespec {
tv_sec: 0,
tv_nsec: libc::UTIME_NOW,
};
let mtime = libc::timespec {
tv_sec: (*statbuf).st_mtime,
tv_nsec: (*statbuf).st_mtime_nsec as _,
};
let times = [now, mtime];
libc::utimensat(libc::AT_FDCWD, filename, times.as_ptr(), 0)
}
#[cfg(all(unix, not(any(target_os = "linux", target_os = "android"))))]
unsafe fn set_mtime(filename: *const c_char, statbuf: *const Stat) -> c_int {
let now = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map_or(0, |duration| duration.as_secs()) as libc::time_t;
let times = libc::utimbuf {
actime: now,
modtime: (*statbuf).st_mtime,
};
libc::utime(filename, &times)
}
#[cfg(windows)]
unsafe fn set_mtime(filename: *const c_char, statbuf: *const Stat) -> c_int {
UTIL_rust_utime(filename, statbuf)
}
#[cfg(not(any(unix, windows)))]
unsafe fn set_mtime(_filename: *const c_char, _statbuf: *const Stat) -> c_int {
-1
}
#[no_mangle]
pub unsafe extern "C" fn UTIL_utime(filename: *const c_char, statbuf: *const Stat) -> c_int {
set_mtime(filename, statbuf)
}
#[no_mangle]
pub unsafe extern "C" fn UTIL_setFileStat(filename: *const c_char, statbuf: *const Stat) -> c_int {
UTIL_setFDStat(-1, filename, statbuf)
}
#[no_mangle]
pub unsafe extern "C" fn UTIL_setFDStat(
fd: c_int,
filename: *const c_char,
statbuf: *const Stat,
) -> c_int {
let mut current_stat = MaybeUninit::<Stat>::uninit();
if UTIL_fstat(fd, filename, current_stat.as_mut_ptr()) == 0
|| UTIL_isRegularFileStat(current_stat.as_ptr()) == 0
{
return -1;
}
let mut result = 0;
#[cfg(unix)]
{
let no_uid = !0 as libc::uid_t;
if fd >= 0 {
result += libc::fchown(fd, no_uid, (*statbuf).st_gid);
} else {
result += libc::chown(filename, no_uid, (*statbuf).st_gid);
}
}
let permissions = ((*statbuf).st_mode as u64 & 0o777) as UtilMode;
result += UTIL_fchmod(fd, filename, current_stat.as_ptr(), permissions);
#[cfg(unix)]
{
let no_gid = !0 as libc::gid_t;
if fd >= 0 {
result += libc::fchown(fd, (*statbuf).st_uid, no_gid);
} else {
result += libc::chown(filename, (*statbuf).st_uid, no_gid);
}
}
-result
}
#[no_mangle]
pub unsafe extern "C" fn UTIL_requireUserConfirmation(
prompt: *const c_char,
abort_msg: *const c_char,
acceptable_letters: *const c_char,
has_stdin_input: c_int,
) -> c_int {
if has_stdin_input != 0 {
display("stdin is an input - not proceeding.\n");
return 1;
}
display_c_string(prompt);
let ch = libc::getchar();
let accepted = if acceptable_letters.is_null() {
false
} else {
c_bytes(acceptable_letters).contains(&(ch as u8))
};
let result = if accepted { 0 } else { 1 };
if result != 0 {
display_c_string(abort_msg);
display(" \n");
}
let mut next = ch;
while next != libc::EOF && next != b'\n' as c_int {
next = libc::getchar();
}
result
}
#[no_mangle]
pub extern "C" fn UTIL_traceFileStat() {
TRACE_FILE_STAT.store(true, Ordering::Relaxed);
}
#[no_mangle]
pub unsafe extern "C" fn UTIL_isConsole(file: *mut libc::FILE) -> c_int {
if file.is_null() {
return 0;
}
let fd = libc::fileno(file);
if fd == 0 && FAKE_STDIN_IS_CONSOLE.load(Ordering::Relaxed)
|| fd == 1 && FAKE_STDOUT_IS_CONSOLE.load(Ordering::Relaxed)
|| fd == 2 && FAKE_STDERR_IS_CONSOLE.load(Ordering::Relaxed)
{
return 1;
}
if fd < 0 {
return 0;
}
libc::isatty(fd)
}
#[no_mangle]
pub extern "C" fn UTIL_fakeStdinIsConsole() {
FAKE_STDIN_IS_CONSOLE.store(true, Ordering::Relaxed);
}
#[no_mangle]
pub extern "C" fn UTIL_fakeStdoutIsConsole() {
FAKE_STDOUT_IS_CONSOLE.store(true, Ordering::Relaxed);
}
#[no_mangle]
pub extern "C" fn UTIL_fakeStderrIsConsole() {
FAKE_STDERR_IS_CONSOLE.store(true, Ordering::Relaxed);
}
fn suffix_ptr(suffix: &'static [u8]) -> *const c_char {
suffix.as_ptr().cast::<c_char>()
}
#[no_mangle]
pub extern "C" fn UTIL_makeHumanReadableSize(size: u64) -> UTIL_HumanReadableSize_t {
let result;
if display_level() > 3 {
if size >= 1u64 << 53 {
result = UTIL_HumanReadableSize_t {
value: size as f64 / (1u64 << 20) as f64,
precision: 2,
suffix: suffix_ptr(SUFFIX_MIB),
};
} else {
result = UTIL_HumanReadableSize_t {
value: size as f64,
precision: 0,
suffix: suffix_ptr(SUFFIX_B),
};
}
} else {
let (value, suffix) = if size >= 1u64 << 60 {
(size as f64 / (1u64 << 60) as f64, SUFFIX_EIB)
} else if size >= 1u64 << 50 {
(size as f64 / (1u64 << 50) as f64, SUFFIX_PIB)
} else if size >= 1u64 << 40 {
(size as f64 / (1u64 << 40) as f64, SUFFIX_TIB)
} else if size >= 1u64 << 30 {
(size as f64 / (1u64 << 30) as f64, SUFFIX_GIB)
} else if size >= 1u64 << 20 {
(size as f64 / (1u64 << 20) as f64, SUFFIX_MIB)
} else if size >= 1u64 << 10 {
(size as f64 / (1u64 << 10) as f64, SUFFIX_KIB)
} else {
(size as f64, SUFFIX_B)
};
let precision = if value >= 100.0 || value as u64 == size {
0
} else if value >= 10.0 {
1
} else if value > 1.0 {
2
} else {
3
};
result = UTIL_HumanReadableSize_t {
value,
precision,
suffix: suffix_ptr(suffix),
};
}
result
}
#[no_mangle]
pub unsafe extern "C" fn UTIL_compareStr(p1: *const c_void, p2: *const c_void) -> c_int {
let left = *(p1.cast::<*const c_char>());
let right = *(p2.cast::<*const c_char>());
let left = c_bytes(left);
let right = c_bytes(right);
for (&a, &b) in left.iter().zip(right.iter()) {
if a != b {
return a as c_int - b as c_int;
}
}
left.len() as c_int - right.len() as c_int
}
#[no_mangle]
pub unsafe extern "C" fn UTIL_getFileExtension(infilename: *const c_char) -> *const c_char {
let bytes = c_bytes(infilename);
let Some(position) = bytes.iter().rposition(|&byte| byte == b'.') else {
return EMPTY_EXTENSION.as_ptr().cast::<c_char>();
};
if position == 0 {
EMPTY_EXTENSION.as_ptr().cast::<c_char>()
} else {
infilename.add(position)
}
}
#[no_mangle]
pub unsafe extern "C" fn UTIL_isCompressedFile(
input_name: *const c_char,
extension_list: *const *const c_char,
) -> c_int {
let extension = CStr::from_ptr(UTIL_getFileExtension(input_name)).to_bytes();
if extension_list.is_null() {
return 0;
}
let mut current = extension_list;
loop {
let candidate = *current;
if candidate.is_null() {
return 0;
}
if c_bytes(candidate) == extension {
return 1;
}
current = current.add(1);
}
}
unsafe fn pathname_has_two_dots(pathname: &[u8]) -> bool {
let separator = path_separator();
for (index, pair) in pathname.windows(2).enumerate() {
if pair != b".." {
continue;
}
let left_boundary = index == 0 || pathname[index - 1] == separator;
let right_index = index + 2;
let right_boundary = right_index == pathname.len() || pathname[right_index] == separator;
if left_boundary && right_boundary {
return true;
}
}
false
}
fn trim_path(pathname: &[u8]) -> &[u8] {
let separator = path_separator();
let mut path = pathname;
if path.first() == Some(&separator) {
path = &path[1..];
}
if path.len() >= 2 && path[0] == b'.' && path[1] == separator {
path = &path[2..];
}
path
}
fn join_two_dirs(dir1: &[u8], dir2: &[u8]) -> Vec<u8> {
let separator = path_separator();
let mut result = Vec::with_capacity(dir1.len() + dir2.len() + 1);
result.extend_from_slice(dir1);
if !dir1.is_empty() && dir1.last() != Some(&separator) {
result.push(separator);
}
result.extend_from_slice(dir2);
result
}
fn convert_pathname_to_dir_name(pathname: &mut Vec<u8>) {
let separator = path_separator();
if let Some(position) = pathname.iter().rposition(|&byte| byte == separator) {
pathname.truncate(position);
} else {
pathname.clear();
pathname.push(b'.');
}
}
unsafe fn get_dir_mode(dir_name: &[u8]) -> u32 {
let path = c_path(dir_name);
let mut statbuf = MaybeUninit::<Stat>::uninit();
if UTIL_stat(path.as_ptr(), statbuf.as_mut_ptr()) == 0 {
if display_level() >= 1 {
eprintln!(
"zstd: failed to get DIR stats {}",
String::from_utf8_lossy(dir_name)
);
}
return DIR_DEFAULT_MODE;
}
if UTIL_isDirectoryStat(statbuf.as_ptr()) == 0 {
if display_level() >= 1 {
eprintln!(
"zstd: expected directory: {}",
String::from_utf8_lossy(dir_name)
);
}
return DIR_DEFAULT_MODE;
}
stat_mode(statbuf.as_ptr()) as u32
}
unsafe fn make_dir(dir: &[u8], mode: u32) -> c_int {
let path = c_path(dir);
#[cfg(unix)]
let result = libc::mkdir(path.as_ptr(), mode as libc::mode_t);
#[cfg(windows)]
let result = {
let _ = mode;
libc::mkdir(path.as_ptr())
};
#[cfg(not(any(unix, windows)))]
let result = {
let _ = (path, mode);
-1
};
if result != 0 {
if std::io::Error::last_os_error().raw_os_error() == Some(libc::EEXIST) {
return 0;
}
if display_level() >= 1 {
eprintln!(
"zstd: failed to create DIR {}: {}",
String::from_utf8_lossy(dir),
std::io::Error::last_os_error()
);
}
}
result
}
unsafe fn mirror_src_dir(src_dir_name: &[u8], out_dir_name: &[u8]) -> c_int {
let new_dir = join_two_dirs(out_dir_name, trim_path(src_dir_name));
let mode = get_dir_mode(src_dir_name);
make_dir(&new_dir, mode)
}
fn mirror_src_dir_recursive(src_dir_name: &[u8], out_dir_name: &[u8]) {
let separator = path_separator();
let start =
if src_dir_name.len() >= 2 && src_dir_name[0] == b'.' && src_dir_name[1] == separator {
2
} else {
0
};
let mut previous = start;
for (position, &byte) in src_dir_name.iter().enumerate().skip(start) {
if byte != separator {
continue;
}
if position != previous {
unsafe {
let _ = mirror_src_dir(&src_dir_name[..position], out_dir_name);
}
}
previous = position + 1;
}
unsafe {
let _ = mirror_src_dir(src_dir_name, out_dir_name);
}
}
fn first_is_parent_or_same_dir(first: &[u8], second: &[u8]) -> bool {
let separator = path_separator();
first.len() <= second.len()
&& (second.get(first.len()) == Some(&separator) || second.get(first.len()).is_none())
&& second.starts_with(first)
}
#[no_mangle]
pub unsafe extern "C" fn UTIL_createMirroredDestDirName(
src_file_name: *const c_char,
out_dir_root_name: *const c_char,
) -> *mut c_char {
let src = c_bytes(src_file_name);
if pathname_has_two_dots(src) {
return ptr::null_mut();
}
let mut pathname = join_two_dirs(c_bytes(out_dir_root_name), trim_path(src));
convert_pathname_to_dir_name(&mut pathname);
let result = malloc_bytes(pathname.len() + 1).cast::<c_char>();
ptr::copy_nonoverlapping(pathname.as_ptr().cast::<c_char>(), result, pathname.len());
*result.add(pathname.len()) = 0;
result
}
#[no_mangle]
pub unsafe extern "C" fn UTIL_mirrorSourceFilesDirectories(
file_names: *const *const c_char,
nb_files: c_uint,
out_dir_name: *const c_char,
) {
let out_dir = c_bytes(out_dir_name).to_vec();
let mut source_dirs = Vec::new();
for index in 0..nb_files as usize {
let source = c_bytes(*file_names.add(index));
if !pathname_has_two_dots(source) {
let mut source_dir = source.to_vec();
convert_pathname_to_dir_name(&mut source_dir);
source_dirs.push(source_dir);
}
}
if source_dirs.is_empty() {
return;
}
let _ = make_dir(&out_dir, DIR_DEFAULT_MODE);
source_dirs.sort_by(|left, right| trim_path(left).cmp(trim_path(right)));
let mut unique_dirs: Vec<&[u8]> = Vec::new();
unique_dirs.push(source_dirs[0].as_slice());
for index in 1..source_dirs.len() {
let previous = trim_path(&source_dirs[index - 1]);
let current = trim_path(&source_dirs[index]);
if first_is_parent_or_same_dir(previous, current) {
*unique_dirs.last_mut().unwrap() = source_dirs[index].as_slice();
} else {
unique_dirs.push(source_dirs[index].as_slice());
}
}
for source_dir in unique_dirs {
mirror_src_dir_recursive(source_dir, &out_dir);
}
}
#[no_mangle]
pub unsafe extern "C" fn UTIL_createFileNamesTable_fromFileName(
input_file_name: *const c_char,
) -> *mut FileNamesTable {
let input_name = c_bytes(input_file_name);
let path = path_from_bytes(input_name);
let c_input_name = c_path(input_name);
let mut statbuf = MaybeUninit::<Stat>::uninit();
if UTIL_stat(c_input_name.as_ptr(), statbuf.as_mut_ptr()) == 0
|| UTIL_isRegularFileStat(statbuf.as_ptr()) == 0
{
return ptr::null_mut();
}
let file_size = UTIL_getFileSizeStat(statbuf.as_ptr());
if file_size > MAX_FILE_OF_FILE_NAMES_SIZE {
return ptr::null_mut();
}
let data = match fs::read(path) {
Ok(data) => data,
Err(error) => {
if display_level() >= 1 {
eprintln!("zstd:util:readLinesFromFile: {error}");
}
return ptr::null_mut();
}
};
if data.is_empty() {
return ptr::null_mut();
}
let line_count = data.iter().filter(|&&byte| byte == b'\n').count()
+ usize::from(data.last() != Some(&b'\n'));
let buffer = malloc_bytes(data.len() + 1);
ptr::copy_nonoverlapping(data.as_ptr(), buffer, data.len());
for index in 0..data.len() {
if *buffer.add(index) == b'\n' {
*buffer.add(index) = 0;
}
}
*buffer.add(data.len()) = 0;
make_table_from_buffer(buffer, data.len() + 1, line_count, line_count)
}
#[no_mangle]
pub unsafe extern "C" fn UTIL_assembleFileNamesTable(
filenames: *mut *const c_char,
table_size: usize,
buf: *mut c_char,
) -> *mut FileNamesTable {
make_table(filenames, table_size, table_size, buf)
}
#[no_mangle]
pub unsafe extern "C" fn UTIL_freeFileNamesTable(table: *mut FileNamesTable) {
if table.is_null() {
return;
}
free_ptr((*table).fileNames);
free_ptr((*table).buf);
free_ptr(table);
}
#[no_mangle]
pub unsafe extern "C" fn UTIL_allocateFileNamesTable(table_size: usize) -> *mut FileNamesTable {
let names = malloc_array::<*const c_char>(table_size);
if names.is_null() {
return ptr::null_mut();
}
make_table(names, 0, table_size, ptr::null_mut())
}
#[no_mangle]
pub unsafe extern "C" fn UTIL_searchFileNamesTable(
table: *mut FileNamesTable,
name: *const c_char,
) -> c_int {
for (index, &candidate) in table_entries(table).iter().enumerate() {
if !candidate.is_null() && c_bytes(candidate) == c_bytes(name) {
return index as c_int;
}
}
-1
}
#[no_mangle]
pub unsafe extern "C" fn UTIL_refFilename(table: *mut FileNamesTable, filename: *const c_char) {
if table.is_null() || (*table).tableSize >= (*table).tableCapacity {
std::process::abort();
}
*(*table).fileNames.add((*table).tableSize) = filename;
(*table).tableSize += 1;
}
#[no_mangle]
pub unsafe extern "C" fn UTIL_mergeFileNamesTable(
table1: *mut FileNamesTable,
table2: *mut FileNamesTable,
) -> *mut FileNamesTable {
let total_size = table_names_size(table1).saturating_add(table_names_size(table2));
let new_size = (*table1).tableSize.saturating_add((*table2).tableSize);
let buffer = malloc_bytes(total_size.max(1));
ptr::write_bytes(buffer, 0, total_size.max(1));
let names = malloc_bytes(new_size.saturating_mul(size_of::<*const c_char>()).max(1))
.cast::<*const c_char>();
ptr::write_bytes(
names.cast::<u8>(),
0,
new_size.saturating_mul(size_of::<*const c_char>()),
);
let mut new_index = 0usize;
let mut position = 0usize;
for source in [table1, table2] {
for &name in table_entries(source) {
if name.is_null() || position >= total_size {
break;
}
let name_bytes = c_bytes(name);
ptr::copy_nonoverlapping(name_bytes.as_ptr(), buffer.add(position), name_bytes.len());
*names.add(new_index) = buffer.add(position).cast::<c_char>();
position += name_bytes.len() + 1;
new_index += 1;
}
}
let result = make_table(names, new_index, 0, buffer.cast::<c_char>());
UTIL_freeFileNamesTable(table1);
UTIL_freeFileNamesTable(table2);
result
}
fn append_name(buffer: &mut Vec<u8>, name: &[u8]) {
buffer.extend_from_slice(name);
buffer.push(0);
}
fn join_entry_path(dir_name: &[u8], entry_name: &[u8]) -> Vec<u8> {
let mut path = Vec::with_capacity(dir_name.len() + entry_name.len() + 1);
path.extend_from_slice(dir_name);
path.push(path_separator());
path.extend_from_slice(entry_name);
path
}
unsafe fn is_directory_bytes(name: &[u8]) -> bool {
let path = c_path(name);
let mut statbuf = MaybeUninit::<Stat>::uninit();
UTIL_stat(path.as_ptr(), statbuf.as_mut_ptr()) != 0
&& UTIL_isDirectoryStat(statbuf.as_ptr()) != 0
}
unsafe fn is_link_bytes(name: &[u8]) -> bool {
#[cfg(unix)]
{
let path = c_path(name);
let mut statbuf = MaybeUninit::<Stat>::uninit();
libc::lstat(path.as_ptr(), statbuf.as_mut_ptr()) == 0
&& mode_is(statbuf.as_ptr(), libc::S_IFLNK)
}
#[cfg(not(unix))]
{
let _ = name;
false
}
}
fn walk_directory(
dir_name: &[u8],
buffer: &mut Vec<u8>,
nb_files: &mut usize,
follow_links: bool,
) -> Result<(), ()> {
let entries = match fs::read_dir(path_from_bytes(dir_name)) {
Ok(entries) => entries,
Err(error) => {
if display_level() >= 1 {
eprintln!(
"Cannot open directory '{}': {error}",
String::from_utf8_lossy(dir_name)
);
}
return Ok(());
}
};
for entry in entries {
let entry = entry.map_err(|_| ())?;
let entry_name = os_string_bytes(entry.file_name());
if entry_name == b"." || entry_name == b".." {
continue;
}
let path = join_entry_path(dir_name, &entry_name);
if !follow_links && unsafe { is_link_bytes(&path) } {
if display_level() >= 2 {
eprintln!(
"Warning : {} is a symbolic link, ignoring",
String::from_utf8_lossy(&path)
);
}
continue;
}
if unsafe { is_directory_bytes(&path) } {
walk_directory(&path, buffer, nb_files, follow_links)?;
} else {
append_name(buffer, &path);
*nb_files += 1;
}
}
Ok(())
}
#[no_mangle]
pub unsafe extern "C" fn UTIL_createExpandedFNT(
input_names: *const *const c_char,
nb_input_names: usize,
follow_links: c_int,
) -> *mut FileNamesTable {
let mut buffer = Vec::with_capacity(LIST_SIZE_INCREASE);
let mut nb_files = 0usize;
for index in 0..nb_input_names {
let input = *input_names.add(index);
let input_bytes = c_bytes(input);
if !is_directory_bytes(input_bytes) {
append_name(&mut buffer, input_bytes);
nb_files += 1;
} else if walk_directory(input_bytes, &mut buffer, &mut nb_files, follow_links != 0)
.is_err()
{
return ptr::null_mut();
}
}
let table_capacity = nb_files.saturating_add(1);
let allocated_len = buffer.len().max(LIST_SIZE_INCREASE);
let output_buffer = malloc_bytes(allocated_len);
if !buffer.is_empty() {
ptr::copy_nonoverlapping(buffer.as_ptr(), output_buffer, buffer.len());
}
make_table_from_buffer(output_buffer, buffer.len(), nb_files, table_capacity)
}
#[no_mangle]
pub unsafe extern "C" fn UTIL_expandFNT(table: *mut *mut FileNamesTable, follow_links: c_int) {
let old_table = *table;
let new_table =
UTIL_createExpandedFNT((*old_table).fileNames, (*old_table).tableSize, follow_links);
if new_table.is_null() {
std::process::abort();
}
UTIL_freeFileNamesTable(old_table);
*table = new_table;
}
#[no_mangle]
pub unsafe extern "C" fn UTIL_createFNT_fromROTable(
filenames: *const *const c_char,
nb_filenames: usize,
) -> *mut FileNamesTable {
let new_names = malloc_array::<*const c_char>(nb_filenames);
if new_names.is_null() {
return ptr::null_mut();
}
ptr::copy_nonoverlapping(filenames, new_names, nb_filenames);
UTIL_assembleFileNamesTable(new_names, nb_filenames, ptr::null_mut())
}
#[cfg(windows)]
#[no_mangle]
pub extern "system" fn CountSetBits(bit_mask: usize) -> u32 {
bit_mask.count_ones()
}
#[cfg(target_os = "linux")]
fn count_cores_platform(logical: c_int) -> c_int {
let online = unsafe { libc::sysconf(libc::_SC_NPROCESSORS_ONLN) };
let mut count = if online < 0 { 1 } else { online as c_int };
if logical != 0 {
return count;
}
let Ok(cpuinfo) = fs::read_to_string("/proc/cpuinfo") else {
return count;
};
let mut siblings = 0;
let mut cpu_cores = 0;
for line in cpuinfo.lines() {
if line.starts_with("siblings") {
let Some((_, value)) = line.split_once(':') else {
return count;
};
let Ok(value) = value.trim().parse::<c_int>() else {
return count;
};
siblings = value;
}
if line.starts_with("cpu cores") {
let Some((_, value)) = line.split_once(':') else {
return count;
};
let Ok(value) = value.trim().parse::<c_int>() else {
return count;
};
cpu_cores = value;
}
}
if siblings > cpu_cores && cpu_cores > 0 {
let ratio = siblings / cpu_cores;
if ratio > 0 && count > ratio {
count /= ratio;
}
}
count
}
#[cfg(all(unix, not(target_os = "linux"), not(target_vendor = "apple")))]
fn count_cores_platform(_logical: c_int) -> c_int {
let count = unsafe { libc::sysconf(libc::_SC_NPROCESSORS_ONLN) };
if count < 0 {
1
} else {
count as c_int
}
}
#[cfg(target_vendor = "apple")]
fn count_cores_platform(logical: c_int) -> c_int {
let name = if logical != 0 {
CString::new("hw.logicalcpu").unwrap()
} else {
CString::new("hw.physicalcpu").unwrap()
};
let mut count = 0i32;
let mut size = size_of::<i32>();
let result = unsafe {
libc::sysctlbyname(
name.as_ptr(),
(&mut count as *mut i32).cast::<c_void>(),
&mut size,
ptr::null_mut(),
0,
)
};
if result != 0 {
if std::io::Error::last_os_error().raw_os_error() == Some(libc::ENOENT) {
1
} else {
std::process::abort()
}
} else {
count
}
}
#[cfg(windows)]
fn count_cores_platform(_logical: c_int) -> c_int {
std::thread::available_parallelism()
.map_or(1, |count| count.get().min(c_int::MAX as usize) as c_int)
}
#[cfg(not(any(unix, windows)))]
fn count_cores_platform(_logical: c_int) -> c_int {
1
}
#[no_mangle]
pub extern "C" fn UTIL_countCores(logical: c_int) -> c_int {
*CORE_COUNT.get_or_init(|| count_cores_platform(logical))
}
#[no_mangle]
pub extern "C" fn UTIL_countPhysicalCores() -> c_int {
UTIL_countCores(0)
}
#[no_mangle]
pub extern "C" fn UTIL_countLogicalCores() -> c_int {
UTIL_countCores(1)
}
#[cfg(test)]
mod tests {
use super::*;
use std::ffi::CString;
use std::fs::{self, File};
use std::io::Write;
#[test]
fn public_struct_layout_is_pointer_and_c_int_stable() {
assert_eq!(
std::mem::align_of::<UTIL_HumanReadableSize_t>(),
std::mem::align_of::<f64>()
);
assert_eq!(offset_of!(UTIL_HumanReadableSize_t, value), 0);
assert_eq!(
offset_of!(UTIL_HumanReadableSize_t, precision),
size_of::<f64>()
);
assert_eq!(offset_of!(FileNamesTable, fileNames), 0);
assert_eq!(offset_of!(FileNamesTable, buf), size_of::<*const c_char>());
assert_eq!(
offset_of!(FileNamesTable, tableSize),
2 * size_of::<*const c_char>()
);
assert_eq!(
size_of::<FileNamesTable>(),
2 * size_of::<*const c_char>() + 2 * size_of::<usize>()
);
}
#[test]
fn stat_extension_and_size_helpers_follow_c_contract() {
let root = std::env::temp_dir().join(format!("zstd-util-{}", std::process::id()));
let _ = fs::remove_dir_all(&root);
fs::create_dir(&root).unwrap();
let file_path = root.join("sample.txt");
fs::write(&file_path, b"hello").unwrap();
let path = CString::new(file_path.to_string_lossy().as_bytes()).unwrap();
let mut statbuf = MaybeUninit::<Stat>::uninit();
unsafe {
assert_eq!(UTIL_stat(path.as_ptr(), statbuf.as_mut_ptr()), 1);
assert_eq!(UTIL_isRegularFileStat(statbuf.as_ptr()), 1);
assert_eq!(UTIL_getFileSizeStat(statbuf.as_ptr()), 5);
assert_eq!(
CStr::from_ptr(UTIL_getFileExtension(path.as_ptr())).to_bytes(),
b".txt"
);
let extension = CString::new(".txt").unwrap();
let extensions = [extension.as_ptr(), ptr::null()];
assert_eq!(UTIL_isCompressedFile(path.as_ptr(), extensions.as_ptr()), 1);
}
fs::remove_dir_all(root).unwrap();
}
#[test]
fn filename_tables_own_c_allocations_and_merge_in_order() {
let first = CString::new("first").unwrap();
let second = CString::new("second").unwrap();
let names = [first.as_ptr(), second.as_ptr()];
unsafe {
let table = UTIL_createFNT_fromROTable(names.as_ptr(), names.len());
assert!(!table.is_null());
assert_eq!((*table).tableSize, 2);
assert_eq!(UTIL_searchFileNamesTable(table, second.as_ptr()), 1);
let empty = UTIL_allocateFileNamesTable(1);
assert!(!empty.is_null());
UTIL_refFilename(empty, first.as_ptr());
let merged = UTIL_mergeFileNamesTable(table, empty);
assert_eq!((*merged).tableSize, 3);
assert_eq!(
CStr::from_ptr(*(*merged).fileNames.add(2)).to_bytes(),
b"first"
);
UTIL_freeFileNamesTable(merged);
}
}
#[test]
fn file_list_loading_and_directory_expansion_return_nul_tables() {
let mut root = std::env::temp_dir();
root.push(format!("zstd-util-list-{}", std::process::id()));
let _ = fs::remove_dir_all(&root);
fs::create_dir(&root).unwrap();
let file_list = root.join("list");
let nested = root.join("nested");
fs::create_dir(&nested).unwrap();
File::create(nested.join("one")).unwrap();
let mut list = File::create(&file_list).unwrap();
writeln!(list, "alpha").unwrap();
writeln!(list, "beta").unwrap();
drop(list);
let list_name = CString::new(file_list.to_string_lossy().as_bytes()).unwrap();
let nested_name = CString::new(nested.to_string_lossy().as_bytes()).unwrap();
unsafe {
let table = UTIL_createFileNamesTable_fromFileName(list_name.as_ptr());
assert!(!table.is_null());
assert_eq!((*table).tableSize, 2);
assert_eq!(CStr::from_ptr(*(*table).fileNames).to_bytes(), b"alpha");
UTIL_freeFileNamesTable(table);
let input = [nested_name.as_ptr()];
let expanded = UTIL_createExpandedFNT(input.as_ptr(), 1, 1);
assert!(!expanded.is_null());
assert_eq!((*expanded).tableSize, 1);
assert!(CStr::from_ptr(*(*expanded).fileNames)
.to_bytes()
.ends_with(b"/one"));
UTIL_freeFileNamesTable(expanded);
}
fs::remove_dir_all(root).unwrap();
}
#[test]
fn mirrored_directory_name_rejects_parent_components() {
let source = CString::new("a/../b/file").unwrap();
let root = CString::new("out").unwrap();
unsafe {
assert!(UTIL_createMirroredDestDirName(source.as_ptr(), root.as_ptr()).is_null());
}
let source = CString::new("a/b/file.txt").unwrap();
unsafe {
let name = UTIL_createMirroredDestDirName(source.as_ptr(), root.as_ptr());
assert_eq!(CStr::from_ptr(name).to_bytes(), b"out/a/b");
free_ptr(name);
}
}
#[test]
fn human_readable_size_uses_the_c_scaling_policy() {
unsafe {
g_utilDisplayLevel = 0;
}
let size = UTIL_makeHumanReadableSize(1536);
assert_eq!(size.precision, 2);
unsafe {
assert_eq!(CStr::from_ptr(size.suffix).to_bytes(), b" KiB");
}
assert_eq!(size.value, 1.5);
}
#[test]
fn core_count_helpers_never_report_zero() {
assert!(UTIL_countPhysicalCores() > 0);
assert!(UTIL_countLogicalCores() > 0);
}
}