ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

minor split optimization

Browse Source 
let's fill the initial stats directly into target fingerprint
This commit is contained in:
Yann Collet
2024-10-23 11:50:57 -07:00
parent dac26eaeac
commit 1c62e714ab
2 changed files with 9 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/compress/zstd_preSplit.c
+5 -5
View File
@@ -18,7 +18,7 @@
#define BLOCKSIZE_MIN 3500 #define BLOCKSIZE_MIN 3500
#define THRESHOLD_PENALTY_RATE 16 #define THRESHOLD_PENALTY_RATE 16
#define THRESHOLD_BASE (THRESHOLD_PENALTY_RATE - 2) #define THRESHOLD_BASE (THRESHOLD_PENALTY_RATE - 2)
#define THRESHOLD_PENALTY 4 #define THRESHOLD_PENALTY 3
#define HASHLENGTH 2 #define HASHLENGTH 2
#define HASHLOG 10 #define HASHLOG 10
@@ -84,8 +84,8 @@ static int compareFingerprints(const FingerPrint* ref,
const FingerPrint* newfp, const FingerPrint* newfp,
int penalty) int penalty)
{ {
if (ref->nbEvents <= BLOCKSIZE_MIN) assert(ref->nbEvents > 0);
return 0; assert(newfp->nbEvents > 0);
{ S64 p50 = ref->nbEvents * newfp->nbEvents; { S64 p50 = ref->nbEvents * newfp->nbEvents;
S64 deviation = fpDistance(ref, newfp); S64 deviation = fpDistance(ref, newfp);
S64 threshold = p50 * (THRESHOLD_BASE + penalty) / THRESHOLD_PENALTY_RATE; S64 threshold = p50 * (THRESHOLD_BASE + penalty) / THRESHOLD_PENALTY_RATE;
@@ -140,7 +140,8 @@ size_t ZSTD_splitBlock_4k(const void* src, size_t srcSize,
assert(wkspSize >= sizeof(FPStats)); (void)wkspSize; assert(wkspSize >= sizeof(FPStats)); (void)wkspSize;
initStats(fpstats); initStats(fpstats);
for (pos = 0; pos < blockSizeMax;) { recordFingerprint(&fpstats->pastEvents, p, CHUNKSIZE);
for (pos = CHUNKSIZE; pos < blockSizeMax; pos += CHUNKSIZE) {
assert(pos <= blockSizeMax - CHUNKSIZE); assert(pos <= blockSizeMax - CHUNKSIZE);
recordFingerprint(&fpstats->newEvents, p + pos, CHUNKSIZE); recordFingerprint(&fpstats->newEvents, p + pos, CHUNKSIZE);
if (compareFingerprints(&fpstats->pastEvents, &fpstats->newEvents, penalty)) { if (compareFingerprints(&fpstats->pastEvents, &fpstats->newEvents, penalty)) {
@@ -150,7 +151,6 @@ size_t ZSTD_splitBlock_4k(const void* src, size_t srcSize,
ZSTD_memset(&fpstats->newEvents, 0, sizeof(fpstats->newEvents)); ZSTD_memset(&fpstats->newEvents, 0, sizeof(fpstats->newEvents));
penalty = penalty - 1 + (penalty == 0); penalty = penalty - 1 + (penalty == 0);
} }
pos += CHUNKSIZE;
} }
return blockSizeMax; return blockSizeMax;
(void)flushEvents; (void)removeEvents; (void)flushEvents; (void)removeEvents;
lib/compress/zstd_preSplit.h
+4
View File
@@ -22,6 +22,10 @@ extern "C" {
/* note: /* note:
* @workspace must be aligned on 8-bytes boundaries * @workspace must be aligned on 8-bytes boundaries
* @wkspSize must be at least >= ZSTD_SLIPBLOCK_WORKSPACESIZE * @wkspSize must be at least >= ZSTD_SLIPBLOCK_WORKSPACESIZE
* note2:
* for the time being, this function only accepts full 128 KB blocks,
* therefore @blockSizeMax must be == 128 KB.
* This could be extended to smaller sizes in the future.
*/ */
size_t ZSTD_splitBlock_4k(const void* src, size_t srcSize, size_t blockSizeMax, void* workspace, size_t wkspSize); size_t ZSTD_splitBlock_4k(const void* src, size_t srcSize, size_t blockSizeMax, void* workspace, size_t wkspSize);