ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(cli): move dictionary mmap lifecycle into Rust

Browse Source 
Move the platform-specific dictionary mapping and release operations behind the
Rust CLI archive while keeping size policy, diagnostics, and FIO_Dict_t
ownership decisions in C. POSIX descriptors and Windows handles are cleaned up
on every failure path, and zero-length dictionaries use an owned sentinel so
the existing non-null dictionary invariant remains valid.

Test Plan:
- cargo test --manifest-path rust/cli/Cargo.toml --lib fileio_backend (24 passed)
- cargo clippy --manifest-path rust/cli/Cargo.toml --lib -- -D warnings
- make -C programs -j2 zstd
- --mmap-dict compression/decompression round trip with programs/README.md
- programs/zstd --mmap-dict --test on the generated frame
- rustfmt +nightly --edition 2021 rust/src/fileio_backend.rs --check
- git diff --check
This commit is contained in:
ddidderr
2026-07-18 17:34:32 +02:00
parent c057bcbbdf
commit f7a2576761
2 changed files with 378 additions and 46 deletions
Download Patch File Download Diff File Expand all files Collapse all files
programs/fileio.c
+67 -46
View File
@@ -366,6 +366,22 @@ int FIO_rust_setDictBufferMalloc(const char* fileName,
size_t maxSize,
void** buffer,
size_t* loadedSize);
#if (PLATFORM_POSIX_VERSION > 0) || defined(_MSC_VER) || defined(_WIN32)
enum {
FIO_DICT_MMAP_SUCCESS = 0,
FIO_DICT_MMAP_OPEN_FAILED = 1,
FIO_DICT_MMAP_TOO_LARGE = 2,
FIO_DICT_MMAP_FAILED = 3,
FIO_DICT_MMAP_VIEW_FAILED = 4,
};
int FIO_rust_setDictBufferMMap(const char* fileName,
unsigned long long expectedFileSize,
size_t maxSize,
void** buffer,
size_t* mappedSize,
void** dictHandle);
void FIO_rust_munmapDict(void** buffer, size_t* bufferSize, void** dictHandle);
#endif
int FIO_rust_removeFile(const char* path);
int FIO_rust_passThrough(ReadPoolCtx_t* readCtx, WritePoolCtx_t* writeCtx);
#ifdef ZSTD_LZ4COMPRESS
@@ -640,18 +656,16 @@ static size_t FIO_setDictBufferMalloc(FIO_Dict_t* dict, const char* fileName, FI
}
#if (PLATFORM_POSIX_VERSION > 0)
#include <sys/mman.h>
static void FIO_munmap(FIO_Dict_t* dict)
{
munmap(dict->dictBuffer, dict->dictBufferSize);
dict->dictBuffer = NULL;
dict->dictBufferSize = 0;
FIO_rust_munmapDict(&dict->dictBuffer, &dict->dictBufferSize, NULL);
}
static size_t FIO_setDictBufferMMap(FIO_Dict_t* dict, const char* fileName, FIO_prefs_t* const prefs, stat_t* dictFileStat)
{
int fileHandle;
U64 fileSize;
size_t const dictSizeMax = prefs->patchFromMode ? prefs->memLimit : DICTSIZE_MAX;
void** bufferPtr = &dict->dictBuffer;
int status;
assert(bufferPtr != NULL);
assert(dictFileStat != NULL);
@@ -660,41 +674,45 @@ static size_t FIO_setDictBufferMMap(FIO_Dict_t* dict, const char* fileName, FIO_
DISPLAYLEVEL(4,"Loading %s as dictionary \n", fileName);
fileHandle = open(fileName, O_RDONLY);
fileSize = UTIL_getFileSizeStat(dictFileStat);
if (fileSize > dictSizeMax) {
EXM_THROW(34, "Dictionary file %s is too large (> %u bytes)",
fileName, (unsigned)dictSizeMax); /* avoid extreme cases */
}
if (fileHandle == -1) {
status = FIO_rust_setDictBufferMMap(fileName,
(unsigned long long)fileSize,
dictSizeMax,
bufferPtr,
&dict->dictBufferSize,
NULL);
if (status == FIO_DICT_MMAP_SUCCESS) return dict->dictBufferSize;
if (status == FIO_DICT_MMAP_OPEN_FAILED) {
EXM_THROW(33, "Couldn't open dictionary %s: %s", fileName, strerror(errno));
}
fileSize = UTIL_getFileSizeStat(dictFileStat);
{
size_t const dictSizeMax = prefs->patchFromMode ? prefs->memLimit : DICTSIZE_MAX;
if (fileSize > dictSizeMax) {
EXM_THROW(34, "Dictionary file %s is too large (> %u bytes)",
fileName, (unsigned)dictSizeMax); /* avoid extreme cases */
}
if (status == FIO_DICT_MMAP_TOO_LARGE) {
EXM_THROW(34, "Dictionary file %s is too large (> %u bytes)",
fileName, (unsigned)dictSizeMax); /* avoid extreme cases */
}
*bufferPtr = mmap(NULL, (size_t)fileSize, PROT_READ, MAP_PRIVATE, fileHandle, 0);
if (*bufferPtr==NULL) EXM_THROW(34, "%s", strerror(errno));
close(fileHandle);
return (size_t)fileSize;
if (status == FIO_DICT_MMAP_FAILED) {
EXM_THROW(34, "%s", strerror(errno));
}
assert(0); /* unexpected Rust status */
return 0;
}
#elif defined(_MSC_VER) || defined(_WIN32)
#include <windows.h>
static void FIO_munmap(FIO_Dict_t* dict)
{
UnmapViewOfFile(dict->dictBuffer);
CloseHandle(dict->dictHandle);
dict->dictBuffer = NULL;
dict->dictBufferSize = 0;
FIO_rust_munmapDict(&dict->dictBuffer,
&dict->dictBufferSize,
(void**)&dict->dictHandle);
}
static size_t FIO_setDictBufferMMap(FIO_Dict_t* dict, const char* fileName, FIO_prefs_t* const prefs, stat_t* dictFileStat)
{
HANDLE fileHandle, mapping;
U64 fileSize;
size_t const dictSizeMax = prefs->patchFromMode ? prefs->memLimit : DICTSIZE_MAX;
void** bufferPtr = &dict->dictBuffer;
int status;
assert(bufferPtr != NULL);
assert(dictFileStat != NULL);
@@ -703,31 +721,34 @@ static size_t FIO_setDictBufferMMap(FIO_Dict_t* dict, const char* fileName, FIO_
DISPLAYLEVEL(4,"Loading %s as dictionary \n", fileName);
fileHandle = CreateFileA(fileName, GENERIC_READ, FILE_SHARE_READ, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_READONLY, NULL);
fileSize = UTIL_getFileSizeStat(dictFileStat);
if (fileSize > dictSizeMax) {
EXM_THROW(34, "Dictionary file %s is too large (> %u bytes)",
fileName, (unsigned)dictSizeMax); /* avoid extreme cases */
}
if (fileHandle == INVALID_HANDLE_VALUE) {
status = FIO_rust_setDictBufferMMap(fileName,
(unsigned long long)fileSize,
dictSizeMax,
bufferPtr,
&dict->dictBufferSize,
(void**)&dict->dictHandle);
if (status == FIO_DICT_MMAP_SUCCESS) return dict->dictBufferSize;
if (status == FIO_DICT_MMAP_OPEN_FAILED) {
EXM_THROW(33, "Couldn't open dictionary %s: %s", fileName, strerror(errno));
}
fileSize = UTIL_getFileSizeStat(dictFileStat);
{
size_t const dictSizeMax = prefs->patchFromMode ? prefs->memLimit : DICTSIZE_MAX;
if (fileSize > dictSizeMax) {
EXM_THROW(34, "Dictionary file %s is too large (> %u bytes)",
fileName, (unsigned)dictSizeMax); /* avoid extreme cases */
}
if (status == FIO_DICT_MMAP_TOO_LARGE) {
EXM_THROW(34, "Dictionary file %s is too large (> %u bytes)",
fileName, (unsigned)dictSizeMax); /* avoid extreme cases */
}
mapping = CreateFileMapping(fileHandle, NULL, PAGE_READONLY, 0, 0, NULL);
if (mapping == NULL) {
if (status == FIO_DICT_MMAP_FAILED) {
EXM_THROW(35, "Couldn't map dictionary %s: %s", fileName, strerror(errno));
}
*bufferPtr = MapViewOfFile(mapping, FILE_MAP_READ, 0, 0, (DWORD)fileSize); /* we can only cast to DWORD here because dictSize <= 2GB */
if (*bufferPtr==NULL) EXM_THROW(36, "%s", strerror(errno));
dict->dictHandle = fileHandle;
return (size_t)fileSize;
if (status == FIO_DICT_MMAP_VIEW_FAILED) {
EXM_THROW(36, "%s", strerror(errno));
}
assert(0); /* unexpected Rust status */
return 0;
}
#else
static size_t FIO_setDictBufferMMap(FIO_Dict_t* dict, const char* fileName, FIO_prefs_t* const prefs, stat_t* dictFileStat)