ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fixed contrib/adaptive-compression

Browse Source
This commit is contained in:
Yann Collet
2018-03-15 17:10:15 -07:00
parent 97816400ca
commit 5373e44ba7
3 changed files with 11 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Makefile
+2
View File
@@ -96,6 +96,7 @@ manual:
contrib: lib contrib: lib
$(MAKE) -C contrib/pzstd all $(MAKE) -C contrib/pzstd all
$(MAKE) -C contrib/seekable_format/examples all $(MAKE) -C contrib/seekable_format/examples all
$(MAKE) -C contrib/adaptive-compression all
.PHONY: cleanTabs .PHONY: cleanTabs
cleanTabs: cleanTabs:
@@ -111,6 +112,7 @@ clean:
@$(MAKE) -C contrib/gen_html $@ > $(VOID) @$(MAKE) -C contrib/gen_html $@ > $(VOID)
@$(MAKE) -C contrib/pzstd $@ > $(VOID) @$(MAKE) -C contrib/pzstd $@ > $(VOID)
@$(MAKE) -C contrib/seekable_format/examples $@ > $(VOID) @$(MAKE) -C contrib/seekable_format/examples $@ > $(VOID)
@$(MAKE) -C contrib/adaptive-compression $@ > $(VOID)
@$(RM) zstd$(EXT) zstdmt$(EXT) tmp* @$(RM) zstd$(EXT) zstdmt$(EXT) tmp*
@$(RM) -r lz4 @$(RM) -r lz4
@echo Cleaning completed @echo Cleaning completed
contrib/adaptive-compression/.gitignore
+3
View File
@@ -0,0 +1,3 @@
# binaries generated
adapt
datagen
contrib/adaptive-compression/adapt.c
+6 -15
View File
@@ -40,7 +40,6 @@ static unsigned g_compressionLevel = DEFAULT_COMPRESSION_LEVEL;
static UTIL_time_t g_startTime; static UTIL_time_t g_startTime;
static size_t g_streamedSize = 0; static size_t g_streamedSize = 0;
static unsigned g_useProgressBar = 1; static unsigned g_useProgressBar = 1;
static UTIL_freq_t g_ticksPerSecond;
static unsigned g_forceCompressionLevel = 0; static unsigned g_forceCompressionLevel = 0;
static unsigned g_minCLevel = 1; static unsigned g_minCLevel = 1;
static unsigned g_maxCLevel; static unsigned g_maxCLevel;
@@ -576,13 +575,12 @@ static void* compressionThread(void* arg)
/* begin compression */ /* begin compression */
{ {
size_t const useDictSize = MIN(getUseableDictSize(cLevel), job->dictSize); size_t const useDictSize = MIN(getUseableDictSize(cLevel), job->dictSize);
size_t const dictModeError = ZSTD_setCCtxParameter(ctx->cctx, ZSTD_p_forceRawDict, 1);
ZSTD_parameters params = ZSTD_getParams(cLevel, 0, useDictSize); ZSTD_parameters params = ZSTD_getParams(cLevel, 0, useDictSize);
params.cParams.windowLog = 23; params.cParams.windowLog = 23;
{ {
size_t const initError = ZSTD_compressBegin_advanced(ctx->cctx, job->src.start + job->dictSize - useDictSize, useDictSize, params, 0); size_t const initError = ZSTD_compressBegin_advanced(ctx->cctx, job->src.start + job->dictSize - useDictSize, useDictSize, params, 0);
size_t const windowSizeError = ZSTD_setCCtxParameter(ctx->cctx, ZSTD_p_forceWindow, 1); size_t const windowSizeError = ZSTD_CCtx_setParameter(ctx->cctx, ZSTD_p_forceMaxWindow, 1);
if (ZSTD_isError(dictModeError) || ZSTD_isError(initError) || ZSTD_isError(windowSizeError)) { if (ZSTD_isError(initError) || ZSTD_isError(windowSizeError)) {
DISPLAY("Error: something went wrong while starting compression\n"); DISPLAY("Error: something went wrong while starting compression\n");
signalErrorToThreads(ctx); signalErrorToThreads(ctx);
return arg; return arg;
@@ -644,21 +642,17 @@ static void* compressionThread(void* arg)
static void displayProgress(unsigned cLevel, unsigned last) static void displayProgress(unsigned cLevel, unsigned last)
{ {
UTIL_time_t currTime; UTIL_time_t currTime = UTIL_getTime();
UTIL_getTime(&currTime);
if (!g_useProgressBar) return; if (!g_useProgressBar) return;
{ { double const timeElapsed = (double)(UTIL_getSpanTimeMicro(g_startTime, currTime) / 1000.0);
double const timeElapsed = (double)(UTIL_getSpanTimeMicro(g_ticksPerSecond, g_startTime, currTime) / 1000.0);
double const sizeMB = (double)g_streamedSize / (1 << 20); double const sizeMB = (double)g_streamedSize / (1 << 20);
double const avgCompRate = sizeMB * 1000 / timeElapsed; double const avgCompRate = sizeMB * 1000 / timeElapsed;
fprintf(stderr, "\r| Comp. Level: %2u | Time Elapsed: %7.2f s | Data Size: %7.1f MB | Avg Comp. Rate: %6.2f MB/s |", cLevel, timeElapsed/1000.0, sizeMB, avgCompRate); fprintf(stderr, "\r| Comp. Level: %2u | Time Elapsed: %7.2f s | Data Size: %7.1f MB | Avg Comp. Rate: %6.2f MB/s |", cLevel, timeElapsed/1000.0, sizeMB, avgCompRate);
if (last) { if (last) {
fprintf(stderr, "\n"); fprintf(stderr, "\n");
} } else {
else {
fflush(stderr); fflush(stderr);
} } }
}
} }
static void* outputThread(void* arg) static void* outputThread(void* arg)
@@ -971,7 +965,6 @@ static int compressFilename(const char* const srcFilename, const char* const dst
{ {
int ret = 0; int ret = 0;
fcResources fcr = createFileCompressionResources(srcFilename, dstFilenameOrNull); fcResources fcr = createFileCompressionResources(srcFilename, dstFilenameOrNull);
UTIL_getTime(&g_startTime);
g_streamedSize = 0; g_streamedSize = 0;
ret |= performCompression(fcr.ctx, fcr.srcFile, fcr.otArg); ret |= performCompression(fcr.ctx, fcr.srcFile, fcr.otArg);
ret |= freeFileCompressionResources(&fcr); ret |= freeFileCompressionResources(&fcr);
@@ -1044,8 +1037,6 @@ int main(int argCount, const char* argv[])
filenameTable[0] = stdinmark; filenameTable[0] = stdinmark;
g_maxCLevel = ZSTD_maxCLevel(); g_maxCLevel = ZSTD_maxCLevel();
UTIL_initTimer(&g_ticksPerSecond);
if (filenameTable == NULL) { if (filenameTable == NULL) {
DISPLAY("Error: could not allocate sapce for filename table.\n"); DISPLAY("Error: could not allocate sapce for filename table.\n");
return 1; return 1;