diff --git a/programs/Makefile b/programs/Makefile index de711edfa..248156ebc 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -33,6 +33,7 @@ RUST_SOURCES := $(RUST_MANIFEST) $(RUST_DIR)/Cargo.lock \ $(shell find $(RUST_DIR)/src -type f -name '*.rs' -print) RUST_CLI_SOURCES := $(RUST_CLI_MANIFEST) $(RUST_CLI_DIR)/Cargo.lock \ $(RUST_CLI_DIR)/src/lib.rs $(RUST_DIR)/src/zstd_cli.rs \ + $(RUST_DIR)/src/fileio_prefs.rs \ $(RUST_DIR)/src/timefn.rs $(RUST_DIR)/src/benchfn.rs \ $(RUST_DIR)/src/datagen.rs $(RUST_DIR)/src/lorem.rs diff --git a/programs/fileio.c b/programs/fileio.c index c5d599b3c..523799702 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -28,6 +28,7 @@ #include /* fprintf, open, fdopen, fread, _fileno, stdin, stdout */ #include /* malloc, free */ #include /* strcmp, strlen */ +#include /* offsetof */ #include /* clock_t, to measure process time */ #include /* O_WRONLY */ #include @@ -254,6 +255,34 @@ struct FIO_ctx_s { size_t totalBytesOutput; }; +/* Keep the Rust preference/context layouts in lock-step with these C + * definitions. The Rust side uses the same #[repr(C)] field order. */ +typedef char FIO_rust_prefs_compression_type_offset[ + (offsetof(FIO_prefs_t, compressionType) == 0) ? 1 : -1]; +typedef char FIO_rust_prefs_stream_size_offset[ + (offsetof(FIO_prefs_t, streamSrcSize) == 16 * sizeof(int)) ? 1 : -1]; +typedef char FIO_rust_prefs_target_size_offset[ + (offsetof(FIO_prefs_t, targetCBlockSize) + == 16 * sizeof(int) + sizeof(size_t)) ? 1 : -1]; +typedef char FIO_rust_prefs_src_hint_offset[ + (offsetof(FIO_prefs_t, srcSizeHint) + == 16 * sizeof(int) + 2 * sizeof(size_t)) ? 1 : -1]; +typedef char FIO_rust_prefs_mmap_dict_offset[ + (offsetof(FIO_prefs_t, mmapDict) + == 16 * sizeof(int) + 2 * sizeof(size_t) + 13 * sizeof(int)) ? 1 : -1]; +typedef char FIO_rust_prefs_size[ + (sizeof(FIO_prefs_t) + == 16 * sizeof(int) + 2 * sizeof(size_t) + 14 * sizeof(int)) ? 1 : -1]; +typedef char FIO_rust_ctx_total_input_offset[ + (offsetof(FIO_ctx_t, totalBytesInput) + == ((5 * sizeof(int) + sizeof(size_t) - 1) / sizeof(size_t)) * sizeof(size_t)) ? 1 : -1]; +typedef char FIO_rust_ctx_total_output_offset[ + (offsetof(FIO_ctx_t, totalBytesOutput) + == offsetof(FIO_ctx_t, totalBytesInput) + sizeof(size_t)) ? 1 : -1]; +typedef char FIO_rust_ctx_size[ + (sizeof(FIO_ctx_t) + == offsetof(FIO_ctx_t, totalBytesOutput) + sizeof(size_t)) ? 1 : -1]; + static int FIO_shouldDisplayFileSummary(FIO_ctx_t const* fCtx) { return fCtx->nbFilesTotal <= 1 || g_display_prefs.displayLevel >= 3; @@ -275,75 +304,14 @@ static int FIO_shouldDisplayMultipleFileSummary(FIO_ctx_t const* fCtx) #define FIO_LDM_PARAM_NOTSET 9999 -FIO_prefs_t* FIO_createPreferences(void) -{ - FIO_prefs_t* const ret = (FIO_prefs_t*)malloc(sizeof(FIO_prefs_t)); - if (!ret) EXM_THROW(21, "Allocation error : not enough memory"); - - ret->compressionType = FIO_zstdCompression; - ret->overwrite = 0; - ret->sparseFileSupport = ZSTD_SPARSE_DEFAULT; - ret->dictIDFlag = 1; - ret->checksumFlag = 1; - ret->removeSrcFile = 0; - ret->memLimit = 0; - ret->nbWorkers = 1; - ret->blockSize = 0; - ret->overlapLog = FIO_OVERLAP_LOG_NOTSET; - ret->adaptiveMode = 0; - ret->rsyncable = 0; - ret->minAdaptLevel = -50; /* initializing this value requires a constant, so ZSTD_minCLevel() doesn't work */ - ret->maxAdaptLevel = 22; /* initializing this value requires a constant, so ZSTD_maxCLevel() doesn't work */ - ret->ldmFlag = 0; - ret->ldmHashLog = 0; - ret->ldmMinMatch = 0; - ret->ldmBucketSizeLog = FIO_LDM_PARAM_NOTSET; - ret->ldmHashRateLog = FIO_LDM_PARAM_NOTSET; - ret->streamSrcSize = 0; - ret->targetCBlockSize = 0; - ret->srcSizeHint = 0; - ret->testMode = 0; - ret->literalCompressionMode = ZSTD_ps_auto; - ret->excludeCompressedFiles = 0; - ret->allowBlockDevices = 0; - ret->asyncIO = AIO_supported(); - ret->passThrough = -1; - return ret; -} - -FIO_ctx_t* FIO_createContext(void) -{ - FIO_ctx_t* const ret = (FIO_ctx_t*)malloc(sizeof(FIO_ctx_t)); - if (!ret) EXM_THROW(21, "Allocation error : not enough memory"); - - ret->currFileIdx = 0; - ret->hasStdinInput = 0; - ret->hasStdoutOutput = 0; - ret->nbFilesTotal = 1; - ret->nbFilesProcessed = 0; - ret->totalBytesInput = 0; - ret->totalBytesOutput = 0; - return ret; -} - -void FIO_freePreferences(FIO_prefs_t* const prefs) -{ - free(prefs); -} - -void FIO_freeContext(FIO_ctx_t* const fCtx) -{ - free(fCtx); -} +/* These symbols are implemented by rust/src/fileio_prefs.rs. The declarations + * in fileio.h remain the C ABI shims while all actual file I/O stays here. */ /*-************************************* * Parameters: Display Options ***************************************/ -void FIO_setNotificationLevel(int level) { g_display_prefs.displayLevel=level; } - -void FIO_setProgressSetting(FIO_progressSetting_e setting) { g_display_prefs.progressSetting = setting; } /*-************************************* @@ -352,170 +320,6 @@ void FIO_setProgressSetting(FIO_progressSetting_e setting) { g_display_prefs.pro /* FIO_prefs_t functions */ -void FIO_setCompressionType(FIO_prefs_t* const prefs, FIO_compressionType_t compressionType) { prefs->compressionType = compressionType; } - -void FIO_overwriteMode(FIO_prefs_t* const prefs) { prefs->overwrite = 1; } - -void FIO_setSparseWrite(FIO_prefs_t* const prefs, int sparse) { prefs->sparseFileSupport = sparse; } - -void FIO_setDictIDFlag(FIO_prefs_t* const prefs, int dictIDFlag) { prefs->dictIDFlag = dictIDFlag; } - -void FIO_setChecksumFlag(FIO_prefs_t* const prefs, int checksumFlag) { prefs->checksumFlag = checksumFlag; } - -void FIO_setRemoveSrcFile(FIO_prefs_t* const prefs, int flag) { prefs->removeSrcFile = (flag!=0); } - -void FIO_setMemLimit(FIO_prefs_t* const prefs, unsigned memLimit) { prefs->memLimit = memLimit; } - -void FIO_setNbWorkers(FIO_prefs_t* const prefs, int nbWorkers) { -#ifndef ZSTD_MULTITHREAD - if (nbWorkers > 0) DISPLAYLEVEL(2, "Note : multi-threading is disabled \n"); -#endif - prefs->nbWorkers = nbWorkers; -} - -void FIO_setExcludeCompressedFile(FIO_prefs_t* const prefs, int excludeCompressedFiles) { prefs->excludeCompressedFiles = excludeCompressedFiles; } - -void FIO_setAllowBlockDevices(FIO_prefs_t* const prefs, int allowBlockDevices) { prefs->allowBlockDevices = allowBlockDevices; } - -void FIO_setBlockSize(FIO_prefs_t* const prefs, int blockSize) { - if (blockSize && prefs->nbWorkers==0) - DISPLAYLEVEL(2, "Setting block size is useless in single-thread mode \n"); - prefs->blockSize = blockSize; -} - -void FIO_setOverlapLog(FIO_prefs_t* const prefs, int overlapLog){ - if (overlapLog && prefs->nbWorkers==0) - DISPLAYLEVEL(2, "Setting overlapLog is useless in single-thread mode \n"); - prefs->overlapLog = overlapLog; -} - -void FIO_setAdaptiveMode(FIO_prefs_t* const prefs, int adapt) { - if ((adapt>0) && (prefs->nbWorkers==0)) - EXM_THROW(1, "Adaptive mode is not compatible with single thread mode \n"); - prefs->adaptiveMode = adapt; -} - -void FIO_setUseRowMatchFinder(FIO_prefs_t* const prefs, int useRowMatchFinder) { - prefs->useRowMatchFinder = useRowMatchFinder; -} - -void FIO_setRsyncable(FIO_prefs_t* const prefs, int rsyncable) { - if ((rsyncable>0) && (prefs->nbWorkers==0)) - EXM_THROW(1, "Rsyncable mode is not compatible with single thread mode \n"); - prefs->rsyncable = rsyncable; -} - -void FIO_setStreamSrcSize(FIO_prefs_t* const prefs, size_t streamSrcSize) { - prefs->streamSrcSize = streamSrcSize; -} - -void FIO_setTargetCBlockSize(FIO_prefs_t* const prefs, size_t targetCBlockSize) { - prefs->targetCBlockSize = targetCBlockSize; -} - -void FIO_setSrcSizeHint(FIO_prefs_t* const prefs, size_t srcSizeHint) { - prefs->srcSizeHint = (int)MIN((size_t)INT_MAX, srcSizeHint); -} - -void FIO_setTestMode(FIO_prefs_t* const prefs, int testMode) { - prefs->testMode = (testMode!=0); -} - -void FIO_setLiteralCompressionMode( - FIO_prefs_t* const prefs, - ZSTD_ParamSwitch_e mode) { - prefs->literalCompressionMode = mode; -} - -void FIO_setAdaptMin(FIO_prefs_t* const prefs, int minCLevel) -{ -#ifndef ZSTD_NOCOMPRESS - assert(minCLevel >= ZSTD_minCLevel()); -#endif - prefs->minAdaptLevel = minCLevel; -} - -void FIO_setAdaptMax(FIO_prefs_t* const prefs, int maxCLevel) -{ - prefs->maxAdaptLevel = maxCLevel; -} - -void FIO_setLdmFlag(FIO_prefs_t* const prefs, unsigned ldmFlag) { - prefs->ldmFlag = (ldmFlag>0); -} - -void FIO_setLdmHashLog(FIO_prefs_t* const prefs, int ldmHashLog) { - prefs->ldmHashLog = ldmHashLog; -} - -void FIO_setLdmMinMatch(FIO_prefs_t* const prefs, int ldmMinMatch) { - prefs->ldmMinMatch = ldmMinMatch; -} - -void FIO_setLdmBucketSizeLog(FIO_prefs_t* const prefs, int ldmBucketSizeLog) { - prefs->ldmBucketSizeLog = ldmBucketSizeLog; -} - - -void FIO_setLdmHashRateLog(FIO_prefs_t* const prefs, int ldmHashRateLog) { - prefs->ldmHashRateLog = ldmHashRateLog; -} - -void FIO_setPatchFromMode(FIO_prefs_t* const prefs, int value) -{ - prefs->patchFromMode = value != 0; -} - -void FIO_setContentSize(FIO_prefs_t* const prefs, int value) -{ - prefs->contentSize = value != 0; -} - -void FIO_setAsyncIOFlag(FIO_prefs_t* const prefs, int value) { -#ifdef ZSTD_MULTITHREAD - prefs->asyncIO = value; -#else - (void) prefs; - (void) value; - DISPLAYLEVEL(2, "Note : asyncio is disabled (lack of multithreading support) \n"); -#endif -} - -void FIO_setPassThroughFlag(FIO_prefs_t* const prefs, int value) { - prefs->passThrough = (value != 0); -} - -void FIO_setMMapDict(FIO_prefs_t* const prefs, ZSTD_ParamSwitch_e value) -{ - prefs->mmapDict = value; -} - -/* FIO_ctx_t functions */ - -void FIO_setHasStdoutOutput(FIO_ctx_t* const fCtx, int value) { - fCtx->hasStdoutOutput = value; -} - -void FIO_setNbFilesTotal(FIO_ctx_t* const fCtx, int value) -{ - fCtx->nbFilesTotal = value; -} - -void FIO_setHasStdinInput(FIO_ctx_t* const fCtx, int value) -{ - fCtx->hasStdinInput = value != 0; -} - -void FIO_determineHasStdinInput(FIO_ctx_t* const fCtx, const FileNamesTable* const filenames) { - size_t i = 0; - for ( ; i < filenames->tableSize; ++i) { - if (!strcmp(stdinmark, filenames->fileNames[i])) { - fCtx->hasStdinInput = 1; - return; - } - } -} - /*-************************************* * Functions ***************************************/ diff --git a/rust/cli/src/lib.rs b/rust/cli/src/lib.rs index 5a6740c29..90bbcc752 100644 --- a/rust/cli/src/lib.rs +++ b/rust/cli/src/lib.rs @@ -2,6 +2,9 @@ mod benchfn; #[path = "../../src/datagen.rs"] mod datagen; +#[cfg(feature = "cli")] +#[path = "../../src/fileio_prefs.rs"] +mod fileio_prefs; #[path = "../../src/lorem.rs"] mod lorem; #[path = "../../src/timefn.rs"] diff --git a/rust/src/fileio_prefs.rs b/rust/src/fileio_prefs.rs new file mode 100644 index 000000000..baf02c436 --- /dev/null +++ b/rust/src/fileio_prefs.rs @@ -0,0 +1,681 @@ +#![allow(non_camel_case_types)] +#![allow(non_snake_case)] +#![allow(clippy::missing_safety_doc)] + +//! Rust implementation of the small preference/context part of `fileio.c`. +//! +//! The file-I/O operations themselves remain in C. These layouts are passed +//! directly to that code, so allocation deliberately uses the C allocator and +//! preference creation initializes exactly the fields initialized by the C +//! implementation. In particular, the four legacy fields which C leaves +//! unspecified are not zero-filled here either. + +use std::ffi::c_void; +use std::mem::size_of; +use std::os::raw::{c_char, c_int, c_uint}; +use std::ptr; + +const FIO_ZSTD_COMPRESSION: c_int = 0; +const FIO_OVERLAP_LOG_NOTSET: c_int = 9999; +const FIO_LDM_PARAM_NOTSET: c_int = 9999; + +#[cfg(target_vendor = "apple")] +const ZSTD_SPARSE_DEFAULT: c_int = 0; +#[cfg(not(target_vendor = "apple"))] +const ZSTD_SPARSE_DEFAULT: c_int = 1; + +#[repr(C)] +struct FIO_display_prefs_t { + displayLevel: c_int, + progressSetting: c_int, +} + +/// C's `FIO_prefs_t` from `programs/fileio_types.h`. +#[repr(C)] +pub struct FIO_prefs_t { + compressionType: c_int, + sparseFileSupport: c_int, + dictIDFlag: c_int, + checksumFlag: c_int, + blockSize: c_int, + overlapLog: c_int, + adaptiveMode: c_int, + useRowMatchFinder: c_int, + rsyncable: c_int, + minAdaptLevel: c_int, + maxAdaptLevel: c_int, + ldmFlag: c_int, + ldmHashLog: c_int, + ldmMinMatch: c_int, + ldmBucketSizeLog: c_int, + ldmHashRateLog: c_int, + streamSrcSize: usize, + targetCBlockSize: usize, + srcSizeHint: c_int, + testMode: c_int, + literalCompressionMode: c_int, + removeSrcFile: c_int, + overwrite: c_int, + asyncIO: c_int, + memLimit: c_uint, + nbWorkers: c_int, + excludeCompressedFiles: c_int, + patchFromMode: c_int, + contentSize: c_int, + allowBlockDevices: c_int, + passThrough: c_int, + mmapDict: c_int, +} + +/// C's private `FIO_ctx_s` from `programs/fileio.c`. +#[repr(C)] +pub struct FIO_ctx_t { + nbFilesTotal: c_int, + hasStdinInput: c_int, + hasStdoutOutput: c_int, + currFileIdx: c_int, + nbFilesProcessed: c_int, + totalBytesInput: usize, + totalBytesOutput: usize, +} + +/// Mirror of the `FileNamesTable` used by `FIO_determineHasStdinInput`. +#[repr(C)] +pub struct FileNamesTable { + fileNames: *mut *const c_char, + buf: *mut c_char, + tableSize: usize, + tableCapacity: usize, +} + +#[cfg(not(test))] +unsafe extern "C" { + static mut g_display_prefs: FIO_display_prefs_t; + fn AIO_supported() -> c_int; + #[cfg(feature = "compression")] + fn ZSTD_minCLevel() -> c_int; +} + +#[cfg(test)] +static mut TEST_DISPLAY_PREFS: FIO_display_prefs_t = FIO_display_prefs_t { + displayLevel: 2, + progressSetting: 0, +}; + +#[inline] +unsafe fn display_prefs() -> *mut FIO_display_prefs_t { + #[cfg(test)] + { + ptr::addr_of_mut!(TEST_DISPLAY_PREFS) + } + #[cfg(not(test))] + { + ptr::addr_of_mut!(g_display_prefs) + } +} + +#[inline] +unsafe fn aio_supported() -> bool { + #[cfg(test)] + { + false + } + #[cfg(not(test))] + { + unsafe { AIO_supported() != 0 } + } +} + +fn display(level: c_int, message: &str) { + let enabled = unsafe { (*display_prefs()).displayLevel >= level }; + if enabled { + eprint!("{message}"); + } +} + +fn throw(error: c_int, message: &str) -> ! { + let enabled = unsafe { (*display_prefs()).displayLevel >= 1 }; + if enabled { + eprintln!("zstd: error {error} : {message} "); + } + std::process::exit(error); +} + +unsafe fn allocate() -> *mut T { + let allocation = unsafe { libc::malloc(size_of::()) }.cast::(); + if allocation.is_null() { + throw(21, "Allocation error : not enough memory"); + } + allocation +} + +#[no_mangle] +pub unsafe extern "C" fn FIO_createPreferences() -> *mut FIO_prefs_t { + let ret = unsafe { allocate::() }; + + /* Keep this list in lock-step with the original C initialization. The + * useRowMatchFinder, patchFromMode, contentSize, and mmapDict fields are + * intentionally left unspecified until their CLI setters are called. */ + unsafe { + ptr::addr_of_mut!((*ret).compressionType).write(FIO_ZSTD_COMPRESSION); + ptr::addr_of_mut!((*ret).overwrite).write(0); + ptr::addr_of_mut!((*ret).sparseFileSupport).write(ZSTD_SPARSE_DEFAULT); + ptr::addr_of_mut!((*ret).dictIDFlag).write(1); + ptr::addr_of_mut!((*ret).checksumFlag).write(1); + ptr::addr_of_mut!((*ret).removeSrcFile).write(0); + ptr::addr_of_mut!((*ret).memLimit).write(0); + ptr::addr_of_mut!((*ret).nbWorkers).write(1); + ptr::addr_of_mut!((*ret).blockSize).write(0); + ptr::addr_of_mut!((*ret).overlapLog).write(FIO_OVERLAP_LOG_NOTSET); + ptr::addr_of_mut!((*ret).adaptiveMode).write(0); + ptr::addr_of_mut!((*ret).rsyncable).write(0); + ptr::addr_of_mut!((*ret).minAdaptLevel).write(-50); + ptr::addr_of_mut!((*ret).maxAdaptLevel).write(22); + ptr::addr_of_mut!((*ret).ldmFlag).write(0); + ptr::addr_of_mut!((*ret).ldmHashLog).write(0); + ptr::addr_of_mut!((*ret).ldmMinMatch).write(0); + ptr::addr_of_mut!((*ret).ldmBucketSizeLog).write(FIO_LDM_PARAM_NOTSET); + ptr::addr_of_mut!((*ret).ldmHashRateLog).write(FIO_LDM_PARAM_NOTSET); + ptr::addr_of_mut!((*ret).streamSrcSize).write(0); + ptr::addr_of_mut!((*ret).targetCBlockSize).write(0); + ptr::addr_of_mut!((*ret).srcSizeHint).write(0); + ptr::addr_of_mut!((*ret).testMode).write(0); + ptr::addr_of_mut!((*ret).literalCompressionMode).write(0); + ptr::addr_of_mut!((*ret).excludeCompressedFiles).write(0); + ptr::addr_of_mut!((*ret).allowBlockDevices).write(0); + ptr::addr_of_mut!((*ret).asyncIO).write(c_int::from(aio_supported())); + ptr::addr_of_mut!((*ret).passThrough).write(-1); + } + ret +} + +#[no_mangle] +pub unsafe extern "C" fn FIO_createContext() -> *mut FIO_ctx_t { + let ret = unsafe { allocate::() }; + unsafe { + ptr::addr_of_mut!((*ret).currFileIdx).write(0); + ptr::addr_of_mut!((*ret).hasStdinInput).write(0); + ptr::addr_of_mut!((*ret).hasStdoutOutput).write(0); + ptr::addr_of_mut!((*ret).nbFilesTotal).write(1); + ptr::addr_of_mut!((*ret).nbFilesProcessed).write(0); + ptr::addr_of_mut!((*ret).totalBytesInput).write(0); + ptr::addr_of_mut!((*ret).totalBytesOutput).write(0); + } + ret +} + +#[no_mangle] +pub unsafe extern "C" fn FIO_freePreferences(prefs: *mut FIO_prefs_t) { + unsafe { libc::free(prefs.cast::()) }; +} + +#[no_mangle] +pub unsafe extern "C" fn FIO_freeContext(ctx: *mut FIO_ctx_t) { + unsafe { libc::free(ctx.cast::()) }; +} + +#[no_mangle] +pub extern "C" fn FIO_setNotificationLevel(level: c_int) { + unsafe { (*display_prefs()).displayLevel = level }; +} + +#[no_mangle] +pub extern "C" fn FIO_setProgressSetting(setting: c_int) { + unsafe { (*display_prefs()).progressSetting = setting }; +} + +#[no_mangle] +pub unsafe extern "C" fn FIO_setCompressionType(prefs: *mut FIO_prefs_t, compression_type: c_int) { + unsafe { ptr::addr_of_mut!((*prefs).compressionType).write(compression_type) }; +} + +#[no_mangle] +pub unsafe extern "C" fn FIO_overwriteMode(prefs: *mut FIO_prefs_t) { + unsafe { ptr::addr_of_mut!((*prefs).overwrite).write(1) }; +} + +#[no_mangle] +pub unsafe extern "C" fn FIO_setSparseWrite(prefs: *mut FIO_prefs_t, sparse: c_int) { + unsafe { ptr::addr_of_mut!((*prefs).sparseFileSupport).write(sparse) }; +} + +#[no_mangle] +pub unsafe extern "C" fn FIO_setDictIDFlag(prefs: *mut FIO_prefs_t, dict_id_flag: c_int) { + unsafe { ptr::addr_of_mut!((*prefs).dictIDFlag).write(dict_id_flag) }; +} + +#[no_mangle] +pub unsafe extern "C" fn FIO_setChecksumFlag(prefs: *mut FIO_prefs_t, checksum_flag: c_int) { + unsafe { ptr::addr_of_mut!((*prefs).checksumFlag).write(checksum_flag) }; +} + +#[no_mangle] +pub unsafe extern "C" fn FIO_setRemoveSrcFile(prefs: *mut FIO_prefs_t, flag: c_int) { + unsafe { ptr::addr_of_mut!((*prefs).removeSrcFile).write(c_int::from(flag != 0)) }; +} + +#[no_mangle] +pub unsafe extern "C" fn FIO_setMemLimit(prefs: *mut FIO_prefs_t, mem_limit: c_uint) { + unsafe { ptr::addr_of_mut!((*prefs).memLimit).write(mem_limit) }; +} + +#[cfg(feature = "compression")] +#[no_mangle] +pub unsafe extern "C" fn FIO_setNbWorkers(prefs: *mut FIO_prefs_t, nb_workers: c_int) { + if unsafe { !aio_supported() } && nb_workers > 0 { + display(2, "Note : multi-threading is disabled \n"); + } + unsafe { ptr::addr_of_mut!((*prefs).nbWorkers).write(nb_workers) }; +} + +#[no_mangle] +pub unsafe extern "C" fn FIO_setExcludeCompressedFile( + prefs: *mut FIO_prefs_t, + exclude_compressed_files: c_int, +) { + unsafe { ptr::addr_of_mut!((*prefs).excludeCompressedFiles).write(exclude_compressed_files) }; +} + +#[no_mangle] +pub unsafe extern "C" fn FIO_setAllowBlockDevices( + prefs: *mut FIO_prefs_t, + allow_block_devices: c_int, +) { + unsafe { ptr::addr_of_mut!((*prefs).allowBlockDevices).write(allow_block_devices) }; +} + +#[no_mangle] +pub unsafe extern "C" fn FIO_setBlockSize(prefs: *mut FIO_prefs_t, block_size: c_int) { + if block_size != 0 && unsafe { (*prefs).nbWorkers == 0 } { + display(2, "Setting block size is useless in single-thread mode \n"); + } + unsafe { ptr::addr_of_mut!((*prefs).blockSize).write(block_size) }; +} + +#[no_mangle] +pub unsafe extern "C" fn FIO_setOverlapLog(prefs: *mut FIO_prefs_t, overlap_log: c_int) { + if overlap_log != 0 && unsafe { (*prefs).nbWorkers == 0 } { + display(2, "Setting overlapLog is useless in single-thread mode \n"); + } + unsafe { ptr::addr_of_mut!((*prefs).overlapLog).write(overlap_log) }; +} + +#[no_mangle] +pub unsafe extern "C" fn FIO_setAdaptiveMode(prefs: *mut FIO_prefs_t, adapt: c_int) { + if adapt > 0 && unsafe { (*prefs).nbWorkers == 0 } { + throw( + 1, + "Adaptive mode is not compatible with single thread mode \n", + ); + } + unsafe { ptr::addr_of_mut!((*prefs).adaptiveMode).write(adapt) }; +} + +#[no_mangle] +pub unsafe extern "C" fn FIO_setUseRowMatchFinder( + prefs: *mut FIO_prefs_t, + use_row_match_finder: c_int, +) { + unsafe { ptr::addr_of_mut!((*prefs).useRowMatchFinder).write(use_row_match_finder) }; +} + +#[no_mangle] +pub unsafe extern "C" fn FIO_setRsyncable(prefs: *mut FIO_prefs_t, rsyncable: c_int) { + if rsyncable > 0 && unsafe { (*prefs).nbWorkers == 0 } { + throw( + 1, + "Rsyncable mode is not compatible with single thread mode \n", + ); + } + unsafe { ptr::addr_of_mut!((*prefs).rsyncable).write(rsyncable) }; +} + +#[no_mangle] +pub unsafe extern "C" fn FIO_setStreamSrcSize(prefs: *mut FIO_prefs_t, stream_src_size: usize) { + unsafe { ptr::addr_of_mut!((*prefs).streamSrcSize).write(stream_src_size) }; +} + +#[no_mangle] +pub unsafe extern "C" fn FIO_setTargetCBlockSize( + prefs: *mut FIO_prefs_t, + target_c_block_size: usize, +) { + unsafe { ptr::addr_of_mut!((*prefs).targetCBlockSize).write(target_c_block_size) }; +} + +#[no_mangle] +pub unsafe extern "C" fn FIO_setSrcSizeHint(prefs: *mut FIO_prefs_t, src_size_hint: usize) { + let capped = src_size_hint.min(c_int::MAX as usize) as c_int; + unsafe { ptr::addr_of_mut!((*prefs).srcSizeHint).write(capped) }; +} + +#[cfg(feature = "decompression")] +#[no_mangle] +pub unsafe extern "C" fn FIO_setTestMode(prefs: *mut FIO_prefs_t, test_mode: c_int) { + unsafe { ptr::addr_of_mut!((*prefs).testMode).write(c_int::from(test_mode != 0)) }; +} + +#[no_mangle] +pub unsafe extern "C" fn FIO_setLiteralCompressionMode(prefs: *mut FIO_prefs_t, mode: c_int) { + unsafe { ptr::addr_of_mut!((*prefs).literalCompressionMode).write(mode) }; +} + +#[cfg(feature = "compression")] +#[inline] +fn min_compression_level() -> c_int { + #[cfg(test)] + { + -50 + } + #[cfg(not(test))] + { + unsafe { ZSTD_minCLevel() } + } +} + +#[cfg(feature = "compression")] +#[no_mangle] +pub unsafe extern "C" fn FIO_setAdaptMin(prefs: *mut FIO_prefs_t, min_c_level: c_int) { + assert!(min_c_level >= min_compression_level()); + unsafe { ptr::addr_of_mut!((*prefs).minAdaptLevel).write(min_c_level) }; +} + +#[cfg(feature = "compression")] +#[no_mangle] +pub unsafe extern "C" fn FIO_setAdaptMax(prefs: *mut FIO_prefs_t, max_c_level: c_int) { + unsafe { ptr::addr_of_mut!((*prefs).maxAdaptLevel).write(max_c_level) }; +} + +#[no_mangle] +pub unsafe extern "C" fn FIO_setLdmFlag(prefs: *mut FIO_prefs_t, ldm_flag: c_uint) { + unsafe { ptr::addr_of_mut!((*prefs).ldmFlag).write(c_int::from(ldm_flag > 0)) }; +} + +#[no_mangle] +pub unsafe extern "C" fn FIO_setLdmHashLog(prefs: *mut FIO_prefs_t, ldm_hash_log: c_int) { + unsafe { ptr::addr_of_mut!((*prefs).ldmHashLog).write(ldm_hash_log) }; +} + +#[no_mangle] +pub unsafe extern "C" fn FIO_setLdmMinMatch(prefs: *mut FIO_prefs_t, ldm_min_match: c_int) { + unsafe { ptr::addr_of_mut!((*prefs).ldmMinMatch).write(ldm_min_match) }; +} + +#[no_mangle] +pub unsafe extern "C" fn FIO_setLdmBucketSizeLog( + prefs: *mut FIO_prefs_t, + ldm_bucket_size_log: c_int, +) { + unsafe { ptr::addr_of_mut!((*prefs).ldmBucketSizeLog).write(ldm_bucket_size_log) }; +} + +#[no_mangle] +pub unsafe extern "C" fn FIO_setLdmHashRateLog(prefs: *mut FIO_prefs_t, ldm_hash_rate_log: c_int) { + unsafe { ptr::addr_of_mut!((*prefs).ldmHashRateLog).write(ldm_hash_rate_log) }; +} + +#[no_mangle] +pub unsafe extern "C" fn FIO_setPatchFromMode(prefs: *mut FIO_prefs_t, value: c_int) { + unsafe { ptr::addr_of_mut!((*prefs).patchFromMode).write(c_int::from(value != 0)) }; +} + +#[no_mangle] +pub unsafe extern "C" fn FIO_setContentSize(prefs: *mut FIO_prefs_t, value: c_int) { + unsafe { ptr::addr_of_mut!((*prefs).contentSize).write(c_int::from(value != 0)) }; +} + +#[no_mangle] +pub unsafe extern "C" fn FIO_setAsyncIOFlag(prefs: *mut FIO_prefs_t, value: c_int) { + if unsafe { aio_supported() } { + unsafe { ptr::addr_of_mut!((*prefs).asyncIO).write(value) }; + } else { + display( + 2, + "Note : asyncio is disabled (lack of multithreading support) \n", + ); + } +} + +#[no_mangle] +pub unsafe extern "C" fn FIO_setPassThroughFlag(prefs: *mut FIO_prefs_t, value: c_int) { + unsafe { ptr::addr_of_mut!((*prefs).passThrough).write(c_int::from(value != 0)) }; +} + +#[no_mangle] +pub unsafe extern "C" fn FIO_setMMapDict(prefs: *mut FIO_prefs_t, value: c_int) { + unsafe { ptr::addr_of_mut!((*prefs).mmapDict).write(value) }; +} + +#[no_mangle] +pub unsafe extern "C" fn FIO_setHasStdoutOutput(ctx: *mut FIO_ctx_t, value: c_int) { + unsafe { ptr::addr_of_mut!((*ctx).hasStdoutOutput).write(value) }; +} + +#[no_mangle] +pub unsafe extern "C" fn FIO_setNbFilesTotal(ctx: *mut FIO_ctx_t, value: c_int) { + unsafe { ptr::addr_of_mut!((*ctx).nbFilesTotal).write(value) }; +} + +#[no_mangle] +pub unsafe extern "C" fn FIO_setHasStdinInput(ctx: *mut FIO_ctx_t, value: c_int) { + unsafe { ptr::addr_of_mut!((*ctx).hasStdinInput).write(c_int::from(value != 0)) }; +} + +#[no_mangle] +pub unsafe extern "C" fn FIO_determineHasStdinInput( + ctx: *mut FIO_ctx_t, + filenames: *const FileNamesTable, +) { + for index in 0..unsafe { (*filenames).tableSize } { + let file_name = unsafe { *(*filenames).fileNames.add(index) }; + if unsafe { libc::strcmp(c"/*stdin*\\".as_ptr(), file_name) == 0 } { + unsafe { ptr::addr_of_mut!((*ctx).hasStdinInput).write(1) }; + return; + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::mem::{align_of, offset_of, size_of}; + + #[test] + fn c_layouts_match_the_headers() { + let word = size_of::(); + let int = size_of::(); + + assert_eq!(align_of::(), align_of::()); + assert_eq!(size_of::(), 2 * int); + + assert_eq!(align_of::(), align_of::()); + assert_eq!(offset_of!(FIO_prefs_t, compressionType), 0); + assert_eq!(offset_of!(FIO_prefs_t, sparseFileSupport), int); + assert_eq!(offset_of!(FIO_prefs_t, ldmHashRateLog), 15 * int); + assert_eq!(offset_of!(FIO_prefs_t, streamSrcSize), 16 * int); + assert_eq!(offset_of!(FIO_prefs_t, targetCBlockSize), 16 * int + word); + let tail = 16 * int + 2 * word; + assert_eq!(offset_of!(FIO_prefs_t, srcSizeHint), tail); + assert_eq!(offset_of!(FIO_prefs_t, testMode), tail + int); + assert_eq!( + offset_of!(FIO_prefs_t, literalCompressionMode), + tail + 2 * int + ); + assert_eq!(offset_of!(FIO_prefs_t, removeSrcFile), tail + 3 * int); + assert_eq!(offset_of!(FIO_prefs_t, overwrite), tail + 4 * int); + assert_eq!(offset_of!(FIO_prefs_t, asyncIO), tail + 5 * int); + assert_eq!(offset_of!(FIO_prefs_t, memLimit), tail + 6 * int); + assert_eq!(offset_of!(FIO_prefs_t, mmapDict), tail + 13 * int); + assert_eq!(size_of::(), tail + 14 * int); + + assert_eq!(align_of::(), align_of::()); + assert_eq!(offset_of!(FIO_ctx_t, currFileIdx), 3 * int); + let total_bytes = (5 * int).next_multiple_of(word); + assert_eq!(offset_of!(FIO_ctx_t, totalBytesInput), total_bytes); + assert_eq!(offset_of!(FIO_ctx_t, totalBytesOutput), total_bytes + word); + assert_eq!(size_of::(), total_bytes + 2 * word); + } + + #[test] + fn preference_defaults_match_fileio_c() { + let prefs = unsafe { FIO_createPreferences() }; + assert!(!prefs.is_null()); + let prefs = unsafe { &*prefs }; + + assert_eq!(prefs.compressionType, 0); + assert_eq!(prefs.overwrite, 0); + assert_eq!(prefs.sparseFileSupport, ZSTD_SPARSE_DEFAULT); + assert_eq!(prefs.dictIDFlag, 1); + assert_eq!(prefs.checksumFlag, 1); + assert_eq!(prefs.removeSrcFile, 0); + assert_eq!(prefs.memLimit, 0); + assert_eq!(prefs.nbWorkers, 1); + assert_eq!(prefs.blockSize, 0); + assert_eq!(prefs.overlapLog, FIO_OVERLAP_LOG_NOTSET); + assert_eq!(prefs.adaptiveMode, 0); + assert_eq!(prefs.rsyncable, 0); + assert_eq!(prefs.minAdaptLevel, -50); + assert_eq!(prefs.maxAdaptLevel, 22); + assert_eq!(prefs.ldmFlag, 0); + assert_eq!(prefs.ldmHashLog, 0); + assert_eq!(prefs.ldmMinMatch, 0); + assert_eq!(prefs.ldmBucketSizeLog, FIO_LDM_PARAM_NOTSET); + assert_eq!(prefs.ldmHashRateLog, FIO_LDM_PARAM_NOTSET); + assert_eq!(prefs.streamSrcSize, 0); + assert_eq!(prefs.targetCBlockSize, 0); + assert_eq!(prefs.srcSizeHint, 0); + assert_eq!(prefs.testMode, 0); + assert_eq!(prefs.literalCompressionMode, 0); + assert_eq!(prefs.excludeCompressedFiles, 0); + assert_eq!(prefs.allowBlockDevices, 0); + assert_eq!(prefs.asyncIO, 0); + assert_eq!(prefs.passThrough, -1); + + unsafe { FIO_freePreferences(prefs as *const FIO_prefs_t as *mut FIO_prefs_t) }; + } + + #[test] + fn context_defaults_and_setters_match_fileio_c() { + let ctx = unsafe { FIO_createContext() }; + assert!(!ctx.is_null()); + unsafe { + assert_eq!((*ctx).currFileIdx, 0); + assert_eq!((*ctx).hasStdinInput, 0); + assert_eq!((*ctx).hasStdoutOutput, 0); + assert_eq!((*ctx).nbFilesTotal, 1); + assert_eq!((*ctx).nbFilesProcessed, 0); + assert_eq!((*ctx).totalBytesInput, 0); + assert_eq!((*ctx).totalBytesOutput, 0); + + FIO_setNbFilesTotal(ctx, 7); + FIO_setHasStdinInput(ctx, -1); + FIO_setHasStdoutOutput(ctx, -2); + assert_eq!((*ctx).nbFilesTotal, 7); + assert_eq!((*ctx).hasStdinInput, 1); + assert_eq!((*ctx).hasStdoutOutput, -2); + } + + let names = [c"input".as_ptr(), c"/*stdin*\\".as_ptr()]; + let table = FileNamesTable { + fileNames: names.as_ptr().cast_mut(), + buf: ptr::null_mut(), + tableSize: names.len(), + tableCapacity: names.len(), + }; + unsafe { FIO_determineHasStdinInput(ctx, &table) }; + assert_eq!(unsafe { (*ctx).hasStdinInput }, 1); + unsafe { FIO_freeContext(ctx) }; + } + + #[test] + fn preference_setters_update_the_c_layout() { + let prefs = unsafe { FIO_createPreferences() }; + unsafe { + FIO_setCompressionType(prefs, 3); + FIO_overwriteMode(prefs); + FIO_setSparseWrite(prefs, 2); + FIO_setDictIDFlag(prefs, 0); + FIO_setChecksumFlag(prefs, 2); + FIO_setRemoveSrcFile(prefs, -1); + FIO_setMemLimit(prefs, 123); + #[cfg(feature = "compression")] + FIO_setNbWorkers(prefs, 2); + FIO_setExcludeCompressedFile(prefs, 1); + FIO_setAllowBlockDevices(prefs, 1); + FIO_setUseRowMatchFinder(prefs, 2); + FIO_setBlockSize(prefs, 4096); + FIO_setOverlapLog(prefs, 5); + FIO_setAdaptiveMode(prefs, 1); + FIO_setRsyncable(prefs, 1); + FIO_setStreamSrcSize(prefs, 100); + FIO_setTargetCBlockSize(prefs, 200); + FIO_setSrcSizeHint(prefs, usize::MAX); + FIO_setLiteralCompressionMode(prefs, 2); + FIO_setLdmFlag(prefs, 1); + FIO_setLdmHashLog(prefs, 6); + FIO_setLdmMinMatch(prefs, 7); + FIO_setLdmBucketSizeLog(prefs, 8); + FIO_setLdmHashRateLog(prefs, 9); + FIO_setPatchFromMode(prefs, -1); + FIO_setContentSize(prefs, -1); + FIO_setPassThroughFlag(prefs, -1); + FIO_setMMapDict(prefs, 2); + FIO_setAsyncIOFlag(prefs, 1); + + #[cfg(feature = "compression")] + { + FIO_setAdaptMin(prefs, -40); + FIO_setAdaptMax(prefs, 18); + } + #[cfg(feature = "decompression")] + FIO_setTestMode(prefs, -1); + + assert_eq!((*prefs).compressionType, 3); + assert_eq!((*prefs).overwrite, 1); + assert_eq!((*prefs).sparseFileSupport, 2); + assert_eq!((*prefs).dictIDFlag, 0); + assert_eq!((*prefs).checksumFlag, 2); + assert_eq!((*prefs).removeSrcFile, 1); + assert_eq!((*prefs).memLimit, 123); + #[cfg(feature = "compression")] + assert_eq!((*prefs).nbWorkers, 2); + assert_eq!((*prefs).excludeCompressedFiles, 1); + assert_eq!((*prefs).allowBlockDevices, 1); + assert_eq!((*prefs).useRowMatchFinder, 2); + assert_eq!((*prefs).blockSize, 4096); + assert_eq!((*prefs).overlapLog, 5); + assert_eq!((*prefs).adaptiveMode, 1); + assert_eq!((*prefs).rsyncable, 1); + assert_eq!((*prefs).streamSrcSize, 100); + assert_eq!((*prefs).targetCBlockSize, 200); + assert_eq!((*prefs).srcSizeHint, c_int::MAX); + assert_eq!((*prefs).literalCompressionMode, 2); + assert_eq!((*prefs).ldmFlag, 1); + assert_eq!((*prefs).ldmHashLog, 6); + assert_eq!((*prefs).ldmMinMatch, 7); + assert_eq!((*prefs).ldmBucketSizeLog, 8); + assert_eq!((*prefs).ldmHashRateLog, 9); + assert_eq!((*prefs).patchFromMode, 1); + assert_eq!((*prefs).contentSize, 1); + assert_eq!((*prefs).passThrough, 1); + assert_eq!((*prefs).mmapDict, 2); + #[cfg(test)] + assert_eq!((*prefs).asyncIO, 0); + #[cfg(feature = "compression")] + { + assert_eq!((*prefs).minAdaptLevel, -40); + assert_eq!((*prefs).maxAdaptLevel, 18); + } + #[cfg(feature = "decompression")] + assert_eq!((*prefs).testMode, 1); + } + + FIO_setNotificationLevel(0); + FIO_setProgressSetting(2); + unsafe { + assert_eq!((*display_prefs()).displayLevel, 0); + assert_eq!((*display_prefs()).progressSetting, 2); + FIO_freePreferences(prefs); + } + } +}