ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(cli): move dictionary loading backend to Rust

Browse Source 
Move malloc-backed dictionary file reads into the Rust CLI backend while keeping C responsible for stat metadata, patch-mode size policy, diagnostics, buffer-type selection, and eventual free(). The Rust leaf returns explicit open, size, allocation, and read statuses and publishes only fully read libc allocations.

Test Plan:

- cargo test --manifest-path rust/cli/Cargo.toml (126 tests)

- cargo clippy --manifest-path rust/cli/Cargo.toml

- make -B -C programs -j2 zstd

- make -C tests -j2 test-cli-tests (41 tests)
This commit is contained in:
ddidderr
2026-07-18 06:02:22 +02:00
parent 459782ad55
commit 3599d223f9
2 changed files with 223 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files
programs/fileio.c
+38 -20
View File
@@ -320,6 +320,11 @@ const char* FIO_rust_determineDstName(const char* srcFileName, const char* outDi
int FIO_rust_adjustMemLimitForPatchFromMode(FIO_prefs_t* prefs,
unsigned long long dictSize,
unsigned long long maxSrcFileSize);
int FIO_rust_setDictBufferMalloc(const char* fileName,
unsigned long long expectedFileSize,
size_t maxSize,
void** buffer,
size_t* loadedSize);
int FIO_rust_removeFile(const char* path);
#ifdef ZSTD_LZ4COMPRESS
int FIO_rust_LZ4_GetBlockSize_FromBlockId(int id);
@@ -527,9 +532,18 @@ static void FIO_getDictFileStat(const char* fileName, stat_t* dictFileStat) {
*/
static size_t FIO_setDictBufferMalloc(FIO_Dict_t* dict, const char* fileName, FIO_prefs_t* const prefs, stat_t* dictFileStat)
{
FILE* fileHandle;
U64 fileSize;
size_t loadedSize = 0;
size_t const dictSizeMax = prefs->patchFromMode ? prefs->memLimit : DICTSIZE_MAX;
void** bufferPtr = &dict->dictBuffer;
enum {
FIO_DICT_LOAD_SUCCESS = 0,
FIO_DICT_LOAD_OPEN_FAILED = 1,
FIO_DICT_LOAD_TOO_LARGE = 2,
FIO_DICT_LOAD_ALLOCATION_FAILED = 3,
FIO_DICT_LOAD_READ_FAILED = 4,
};
int status;
assert(bufferPtr != NULL);
assert(dictFileStat != NULL);
@@ -538,30 +552,34 @@ static size_t FIO_setDictBufferMalloc(FIO_Dict_t* dict, const char* fileName, FI
DISPLAYLEVEL(4,"Loading %s as dictionary \n", fileName);
fileHandle = fopen(fileName, "rb");
fileSize = UTIL_getFileSizeStat(dictFileStat);
if (fileSize > dictSizeMax) {
EXM_THROW(34, "Dictionary file %s is too large (> %u bytes)",
fileName, (unsigned)dictSizeMax); /* avoid extreme cases */
}
if (fileHandle == NULL) {
status = FIO_rust_setDictBufferMalloc(fileName,
(unsigned long long)fileSize,
dictSizeMax,
bufferPtr,
&loadedSize);
if (status == FIO_DICT_LOAD_SUCCESS) return loadedSize;
if (status == FIO_DICT_LOAD_OPEN_FAILED) {
EXM_THROW(33, "Couldn't open dictionary %s: %s", fileName, strerror(errno));
}
fileSize = UTIL_getFileSizeStat(dictFileStat);
{
size_t const dictSizeMax = prefs->patchFromMode ? prefs->memLimit : DICTSIZE_MAX;
if (fileSize > dictSizeMax) {
EXM_THROW(34, "Dictionary file %s is too large (> %u bytes)",
fileName, (unsigned)dictSizeMax); /* avoid extreme cases */
}
if (status == FIO_DICT_LOAD_TOO_LARGE) {
EXM_THROW(34, "Dictionary file %s is too large (> %u bytes)",
fileName, (unsigned)dictSizeMax); /* avoid extreme cases */
}
*bufferPtr = malloc((size_t)fileSize);
if (*bufferPtr==NULL) EXM_THROW(34, "%s", strerror(errno));
{ size_t const readSize = fread(*bufferPtr, 1, (size_t)fileSize, fileHandle);
if (readSize != fileSize) {
EXM_THROW(35, "Error reading dictionary file %s : %s",
fileName, strerror(errno));
}
if (status == FIO_DICT_LOAD_ALLOCATION_FAILED) {
EXM_THROW(34, "%s", strerror(errno));
}
fclose(fileHandle);
return (size_t)fileSize;
if (status == FIO_DICT_LOAD_READ_FAILED) {
EXM_THROW(35, "Error reading dictionary file %s : %s",
fileName, strerror(errno));
}
assert(0); /* unexpected Rust status */
return 0;
}
#if (PLATFORM_POSIX_VERSION > 0)