Improves decompression speed when using cold dictionary

by triggering the prefetching decoder path
(which used to be dedicated to long-range offsets only).

Figures on my laptop :
no content prefetch : ~300 MB/s (for reference)
full content prefetch : ~325 MB/s (before this patch)
new prefetch path : ~375 MB/s (after this patch)

The benchmark speed is already significant,
but another side-effect is that this version
prefetch less data into memory,
since it only prefetches what's needed, instead of the full dictionary.

This is supposed to help highly active environments
such as active databases,
that can't be properly measured in benchmark environment (too clean).

Also :
fixed the largeNbDict test program
which was working improperly when setting nbBlocks > nbFiles.
This commit is contained in:
Yann Collet
2018-11-08 17:00:23 -08:00
parent 20fb9e7f36
commit 483759a3de
5 changed files with 21 additions and 25 deletions
+2 -1
View File
@@ -33,7 +33,7 @@ largeNbDicts: util.o bench.o datagen.o xxhash.o largeNbDicts.c $(LIBZSTD)
.PHONY: $(LIBZSTD)
$(LIBZSTD):
$(MAKE) -C $(LIBDIR) libzstd.a
$(MAKE) -C $(LIBDIR) libzstd.a CFLAGS="$(CFLAGS)"
bench.o : $(PROGDIR)/bench.c
$(CC) $(CPPFLAGS) $(CFLAGS) $^ -c
@@ -50,4 +50,5 @@ xxhash.o : $(LIBDIR)/common/xxhash.c
clean:
$(RM) *.o
$(MAKE) -C $(LIBDIR) clean > /dev/null
$(RM) largeNbDicts
+6 -4
View File
@@ -49,6 +49,7 @@
/*--- Macros ---*/
#define CONTROL(c) { if (!(c)) abort(); }
#undef MIN
#define MIN(a,b) ((a) < (b) ? (a) : (b))
@@ -594,6 +595,7 @@ int bench(const char** fileNameTable, unsigned nbFiles,
if (blockSize)
DISPLAYLEVEL(3, "of max size %u bytes ", (unsigned)blockSize);
DISPLAYLEVEL(3, "\n");
size_t const totalSrcSlicesSize = sliceCollection_totalCapacity(srcSlices);
size_t* const dstCapacities = malloc(nbBlocks * sizeof(*dstCapacities));
@@ -625,8 +627,8 @@ int bench(const char** fileNameTable, unsigned nbFiles,
/* dictionary determination */
buffer_t const dictBuffer = createDictionaryBuffer(dictionary,
srcBuffer.ptr,
srcSlices.capacities, nbBlocks,
srcs.buffer.ptr,
srcs.slices.capacities, srcs.slices.nbSlices,
DICTSIZE);
CONTROL(dictBuffer.ptr != NULL);
@@ -637,7 +639,7 @@ int bench(const char** fileNameTable, unsigned nbFiles,
CONTROL(cTotalSizeNoDict != 0);
DISPLAYLEVEL(3, "compressing at level %u without dictionary : Ratio=%.2f (%u bytes) \n",
clevel,
(double)srcSize / cTotalSizeNoDict, (unsigned)cTotalSizeNoDict);
(double)totalSrcSlicesSize / cTotalSizeNoDict, (unsigned)cTotalSizeNoDict);
size_t* const cSizes = malloc(nbBlocks * sizeof(size_t));
CONTROL(cSizes != NULL);
@@ -646,7 +648,7 @@ int bench(const char** fileNameTable, unsigned nbFiles,
CONTROL(cTotalSize != 0);
DISPLAYLEVEL(3, "compressed using a %u bytes dictionary : Ratio=%.2f (%u bytes) \n",
(unsigned)dictBuffer.size,
(double)srcSize / cTotalSize, (unsigned)cTotalSize);
(double)totalSrcSlicesSize / cTotalSize, (unsigned)cTotalSize);
/* now dstSlices contain the real compressed size of each block, instead of the maximum capacity */
shrinkSizes(dstSlices, cSizes);