update fileio to employ jobSize
This commit is contained in:
+6
-6
@@ -288,7 +288,7 @@ FIO_prefs_t* FIO_createPreferences(void)
|
|||||||
ret->removeSrcFile = 0;
|
ret->removeSrcFile = 0;
|
||||||
ret->memLimit = 0;
|
ret->memLimit = 0;
|
||||||
ret->nbWorkers = 1;
|
ret->nbWorkers = 1;
|
||||||
ret->blockSize = 0;
|
ret->jobSize = 0;
|
||||||
ret->overlapLog = FIO_OVERLAP_LOG_NOTSET;
|
ret->overlapLog = FIO_OVERLAP_LOG_NOTSET;
|
||||||
ret->adaptiveMode = 0;
|
ret->adaptiveMode = 0;
|
||||||
ret->rsyncable = 0;
|
ret->rsyncable = 0;
|
||||||
@@ -377,10 +377,10 @@ void FIO_setExcludeCompressedFile(FIO_prefs_t* const prefs, int excludeCompresse
|
|||||||
|
|
||||||
void FIO_setAllowBlockDevices(FIO_prefs_t* const prefs, int allowBlockDevices) { prefs->allowBlockDevices = allowBlockDevices; }
|
void FIO_setAllowBlockDevices(FIO_prefs_t* const prefs, int allowBlockDevices) { prefs->allowBlockDevices = allowBlockDevices; }
|
||||||
|
|
||||||
void FIO_setBlockSize(FIO_prefs_t* const prefs, int blockSize) {
|
void FIO_setJobSize(FIO_prefs_t* const prefs, int jobSize) {
|
||||||
if (blockSize && prefs->nbWorkers==0)
|
if (jobSize && prefs->nbWorkers==0)
|
||||||
DISPLAYLEVEL(2, "Setting block size is useless in single-thread mode \n");
|
DISPLAYLEVEL(2, "Setting block size is useless in single-thread mode \n");
|
||||||
prefs->blockSize = blockSize;
|
prefs->jobSize = jobSize;
|
||||||
}
|
}
|
||||||
|
|
||||||
void FIO_setOverlapLog(FIO_prefs_t* const prefs, int overlapLog){
|
void FIO_setOverlapLog(FIO_prefs_t* const prefs, int overlapLog){
|
||||||
@@ -1183,7 +1183,7 @@ static cRess_t FIO_createCResources(FIO_prefs_t* const prefs,
|
|||||||
#ifdef ZSTD_MULTITHREAD
|
#ifdef ZSTD_MULTITHREAD
|
||||||
DISPLAYLEVEL(5,"set nb workers = %u \n", prefs->nbWorkers);
|
DISPLAYLEVEL(5,"set nb workers = %u \n", prefs->nbWorkers);
|
||||||
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_nbWorkers, prefs->nbWorkers) );
|
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_nbWorkers, prefs->nbWorkers) );
|
||||||
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_jobSize, prefs->blockSize) );
|
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_jobSize, prefs->jobSize) );
|
||||||
if (prefs->overlapLog != FIO_OVERLAP_LOG_NOTSET) {
|
if (prefs->overlapLog != FIO_OVERLAP_LOG_NOTSET) {
|
||||||
DISPLAYLEVEL(3,"set overlapLog = %u \n", prefs->overlapLog);
|
DISPLAYLEVEL(3,"set overlapLog = %u \n", prefs->overlapLog);
|
||||||
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_overlapLog, prefs->overlapLog) );
|
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_overlapLog, prefs->overlapLog) );
|
||||||
@@ -2118,7 +2118,7 @@ void FIO_displayCompressionParameters(const FIO_prefs_t* prefs)
|
|||||||
DISPLAY("%s", INDEX(sparseOptions, prefs->sparseFileSupport));
|
DISPLAY("%s", INDEX(sparseOptions, prefs->sparseFileSupport));
|
||||||
DISPLAY("%s", prefs->dictIDFlag ? "" : " --no-dictID");
|
DISPLAY("%s", prefs->dictIDFlag ? "" : " --no-dictID");
|
||||||
DISPLAY("%s", INDEX(checkSumOptions, prefs->checksumFlag));
|
DISPLAY("%s", INDEX(checkSumOptions, prefs->checksumFlag));
|
||||||
DISPLAY(" --block-size=%d", prefs->blockSize);
|
DISPLAY(" --jobsize=%d", prefs->jobSize);
|
||||||
if (prefs->adaptiveMode)
|
if (prefs->adaptiveMode)
|
||||||
DISPLAY(" --adapt=min=%d,max=%d", prefs->minAdaptLevel, prefs->maxAdaptLevel);
|
DISPLAY(" --adapt=min=%d,max=%d", prefs->minAdaptLevel, prefs->maxAdaptLevel);
|
||||||
DISPLAY("%s", INDEX(rowMatchFinderOptions, prefs->useRowMatchFinder));
|
DISPLAY("%s", INDEX(rowMatchFinderOptions, prefs->useRowMatchFinder));
|
||||||
|
|||||||
+1
-1
@@ -70,7 +70,7 @@ void FIO_setAdaptiveMode(FIO_prefs_t* const prefs, int adapt);
|
|||||||
void FIO_setAdaptMin(FIO_prefs_t* const prefs, int minCLevel);
|
void FIO_setAdaptMin(FIO_prefs_t* const prefs, int minCLevel);
|
||||||
void FIO_setAdaptMax(FIO_prefs_t* const prefs, int maxCLevel);
|
void FIO_setAdaptMax(FIO_prefs_t* const prefs, int maxCLevel);
|
||||||
void FIO_setUseRowMatchFinder(FIO_prefs_t* const prefs, int useRowMatchFinder);
|
void FIO_setUseRowMatchFinder(FIO_prefs_t* const prefs, int useRowMatchFinder);
|
||||||
void FIO_setBlockSize(FIO_prefs_t* const prefs, int blockSize);
|
void FIO_setJobSize(FIO_prefs_t* const prefs, int jobSize);
|
||||||
void FIO_setChecksumFlag(FIO_prefs_t* const prefs, int checksumFlag);
|
void FIO_setChecksumFlag(FIO_prefs_t* const prefs, int checksumFlag);
|
||||||
void FIO_setDictIDFlag(FIO_prefs_t* const prefs, int dictIDFlag);
|
void FIO_setDictIDFlag(FIO_prefs_t* const prefs, int dictIDFlag);
|
||||||
void FIO_setLdmBucketSizeLog(FIO_prefs_t* const prefs, int ldmBucketSizeLog);
|
void FIO_setLdmBucketSizeLog(FIO_prefs_t* const prefs, int ldmBucketSizeLog);
|
||||||
|
|||||||
@@ -37,7 +37,7 @@ typedef struct FIO_prefs_s {
|
|||||||
int sparseFileSupport; /* 0: no sparse allowed; 1: auto (file yes, stdout no); 2: force sparse */
|
int sparseFileSupport; /* 0: no sparse allowed; 1: auto (file yes, stdout no); 2: force sparse */
|
||||||
int dictIDFlag;
|
int dictIDFlag;
|
||||||
int checksumFlag;
|
int checksumFlag;
|
||||||
int blockSize;
|
int jobSize;
|
||||||
int overlapLog;
|
int overlapLog;
|
||||||
int adaptiveMode;
|
int adaptiveMode;
|
||||||
int useRowMatchFinder;
|
int useRowMatchFinder;
|
||||||
|
|||||||
+3
-3
@@ -1089,7 +1089,7 @@ int main(int argCount, const char* argv[])
|
|||||||
if (longCommandWArg(&argument, "--dictID")) { NEXT_UINT32(dictID); continue; }
|
if (longCommandWArg(&argument, "--dictID")) { NEXT_UINT32(dictID); continue; }
|
||||||
if (longCommandWArg(&argument, "--zstd=")) { if (!parseCompressionParameters(argument, &compressionParams)) { badUsage(programName, originalArgument); CLEAN_RETURN(1); } ; cType = FIO_zstdCompression; continue; }
|
if (longCommandWArg(&argument, "--zstd=")) { if (!parseCompressionParameters(argument, &compressionParams)) { badUsage(programName, originalArgument); CLEAN_RETURN(1); } ; cType = FIO_zstdCompression; continue; }
|
||||||
if (longCommandWArg(&argument, "--stream-size")) { NEXT_TSIZE(streamSrcSize); continue; }
|
if (longCommandWArg(&argument, "--stream-size")) { NEXT_TSIZE(streamSrcSize); continue; }
|
||||||
if (longCommandWArg(&argument, "--target-compressed-block-size")) { NEXT_TSIZE(targetCBlockSize); continue; }
|
if (longCommandWArg(&argument, "--target-compressedlock-size")) { NEXT_TSIZE(targetCBlockSize); continue; }
|
||||||
if (longCommandWArg(&argument, "--size-hint")) { NEXT_TSIZE(srcSizeHint); continue; }
|
if (longCommandWArg(&argument, "--size-hint")) { NEXT_TSIZE(srcSizeHint); continue; }
|
||||||
if (longCommandWArg(&argument, "--output-dir-flat")) {
|
if (longCommandWArg(&argument, "--output-dir-flat")) {
|
||||||
NEXT_FIELD(outDirName);
|
NEXT_FIELD(outDirName);
|
||||||
@@ -1459,7 +1459,7 @@ int main(int argCount, const char* argv[])
|
|||||||
}
|
}
|
||||||
|
|
||||||
#else
|
#else
|
||||||
(void)bench_nbSeconds; (void)blockSize; (void)setRealTimePrio; (void)separateFiles; (void)compressibility;
|
(void)bench_nbSeconds; (void)chunkSize; (void)setRealTimePrio; (void)separateFiles; (void)compressibility;
|
||||||
#endif
|
#endif
|
||||||
goto _end;
|
goto _end;
|
||||||
}
|
}
|
||||||
@@ -1594,7 +1594,7 @@ int main(int argCount, const char* argv[])
|
|||||||
FIO_setCompressionType(prefs, cType);
|
FIO_setCompressionType(prefs, cType);
|
||||||
FIO_setContentSize(prefs, contentSize);
|
FIO_setContentSize(prefs, contentSize);
|
||||||
FIO_setNbWorkers(prefs, (int)nbWorkers);
|
FIO_setNbWorkers(prefs, (int)nbWorkers);
|
||||||
FIO_setBlockSize(prefs, (int)chunkSize);
|
FIO_setJobSize(prefs, (int)chunkSize);
|
||||||
if (g_overlapLog!=OVERLAP_LOG_DEFAULT) FIO_setOverlapLog(prefs, (int)g_overlapLog);
|
if (g_overlapLog!=OVERLAP_LOG_DEFAULT) FIO_setOverlapLog(prefs, (int)g_overlapLog);
|
||||||
FIO_setLdmFlag(prefs, (unsigned)ldmFlag);
|
FIO_setLdmFlag(prefs, (unsigned)ldmFlag);
|
||||||
FIO_setLdmHashLog(prefs, (int)g_ldmHashLog);
|
FIO_setLdmHashLog(prefs, (int)g_ldmHashLog);
|
||||||
|
|||||||
Reference in New Issue
Block a user