ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

rewrite fingerprint storage to no longer need 64-bit members

Browse Source 
so that it can be stored using standard alignment requirement (sizeof(void*)).

Distance function still requires 64-bit signed multiplication though,
so it won't change the issue regarding the bug in ubsan for clang 32-bit on github ci.
This commit is contained in:
Yann Collet
2024-10-23 11:50:57 -07:00
parent 57239c4d3b
commit b68ddce818
2 changed files with 13 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/compress/zstd_compress.c
+3 -3
View File
@@ -141,7 +141,7 @@ ZSTD_CCtx* ZSTD_initStaticCCtx(void* workspace, size_t workspaceSize)
if (!ZSTD_cwksp_check_available(&cctx->workspace, TMP_WORKSPACE_SIZE + 2 * sizeof(ZSTD_compressedBlockState_t))) return NULL; if (!ZSTD_cwksp_check_available(&cctx->workspace, TMP_WORKSPACE_SIZE + 2 * sizeof(ZSTD_compressedBlockState_t))) return NULL;
cctx->blockState.prevCBlock = (ZSTD_compressedBlockState_t*)ZSTD_cwksp_reserve_object(&cctx->workspace, sizeof(ZSTD_compressedBlockState_t)); cctx->blockState.prevCBlock = (ZSTD_compressedBlockState_t*)ZSTD_cwksp_reserve_object(&cctx->workspace, sizeof(ZSTD_compressedBlockState_t));
cctx->blockState.nextCBlock = (ZSTD_compressedBlockState_t*)ZSTD_cwksp_reserve_object(&cctx->workspace, sizeof(ZSTD_compressedBlockState_t)); cctx->blockState.nextCBlock = (ZSTD_compressedBlockState_t*)ZSTD_cwksp_reserve_object(&cctx->workspace, sizeof(ZSTD_compressedBlockState_t));
cctx->tmpWorkspace = ZSTD_cwksp_reserve_object_aligned(&cctx->workspace, TMP_WORKSPACE_SIZE, sizeof(S64)); cctx->tmpWorkspace = ZSTD_cwksp_reserve_object(&cctx->workspace, TMP_WORKSPACE_SIZE);
cctx->tmpWkspSize = TMP_WORKSPACE_SIZE; cctx->tmpWkspSize = TMP_WORKSPACE_SIZE;
cctx->bmi2 = ZSTD_cpuid_bmi2(ZSTD_cpuid()); cctx->bmi2 = ZSTD_cpuid_bmi2(ZSTD_cpuid());
return cctx; return cctx;
@@ -1709,7 +1709,7 @@ static size_t ZSTD_estimateCCtxSize_usingCCtxParams_internal(
size_t const tokenSpace = ZSTD_cwksp_alloc_size(WILDCOPY_OVERLENGTH + blockSize) size_t const tokenSpace = ZSTD_cwksp_alloc_size(WILDCOPY_OVERLENGTH + blockSize)
+ ZSTD_cwksp_aligned64_alloc_size(maxNbSeq * sizeof(seqDef)) + ZSTD_cwksp_aligned64_alloc_size(maxNbSeq * sizeof(seqDef))
+ 3 * ZSTD_cwksp_alloc_size(maxNbSeq * sizeof(BYTE)); + 3 * ZSTD_cwksp_alloc_size(maxNbSeq * sizeof(BYTE));
size_t const tmpWorkSpace = ZSTD_cwksp_aligned_alloc_size(TMP_WORKSPACE_SIZE, sizeof(S64)); size_t const tmpWorkSpace = ZSTD_cwksp_alloc_size(TMP_WORKSPACE_SIZE);
size_t const blockStateSpace = 2 * ZSTD_cwksp_alloc_size(sizeof(ZSTD_compressedBlockState_t)); size_t const blockStateSpace = 2 * ZSTD_cwksp_alloc_size(sizeof(ZSTD_compressedBlockState_t));
size_t const matchStateSize = ZSTD_sizeof_matchState(cParams, useRowMatchFinder, /* enableDedicatedDictSearch */ 0, /* forCCtx */ 1); size_t const matchStateSize = ZSTD_sizeof_matchState(cParams, useRowMatchFinder, /* enableDedicatedDictSearch */ 0, /* forCCtx */ 1);
@@ -2174,7 +2174,7 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc,
RETURN_ERROR_IF(zc->blockState.prevCBlock == NULL, memory_allocation, "couldn't allocate prevCBlock"); RETURN_ERROR_IF(zc->blockState.prevCBlock == NULL, memory_allocation, "couldn't allocate prevCBlock");
zc->blockState.nextCBlock = (ZSTD_compressedBlockState_t*) ZSTD_cwksp_reserve_object(ws, sizeof(ZSTD_compressedBlockState_t)); zc->blockState.nextCBlock = (ZSTD_compressedBlockState_t*) ZSTD_cwksp_reserve_object(ws, sizeof(ZSTD_compressedBlockState_t));
RETURN_ERROR_IF(zc->blockState.nextCBlock == NULL, memory_allocation, "couldn't allocate nextCBlock"); RETURN_ERROR_IF(zc->blockState.nextCBlock == NULL, memory_allocation, "couldn't allocate nextCBlock");
zc->tmpWorkspace = ZSTD_cwksp_reserve_object_aligned(ws, TMP_WORKSPACE_SIZE, sizeof(S64)); zc->tmpWorkspace = ZSTD_cwksp_reserve_object(ws, TMP_WORKSPACE_SIZE);
RETURN_ERROR_IF(zc->tmpWorkspace == NULL, memory_allocation, "couldn't allocate tmpWorkspace"); RETURN_ERROR_IF(zc->tmpWorkspace == NULL, memory_allocation, "couldn't allocate tmpWorkspace");
zc->tmpWkspSize = TMP_WORKSPACE_SIZE; zc->tmpWkspSize = TMP_WORKSPACE_SIZE;
} } } }
lib/compress/zstd_preSplit.c
+10 -10
View File
@@ -33,8 +33,8 @@ static unsigned hash2(const void *p)
typedef struct { typedef struct {
int events[HASHTABLESIZE]; unsigned events[HASHTABLESIZE];
S64 nbEvents; size_t nbEvents;
} Fingerprint; } Fingerprint;
typedef struct { typedef struct {
Fingerprint pastEvents; Fingerprint pastEvents;
@@ -78,15 +78,15 @@ static void recordFingerprint(Fingerprint* fp, const void* src, size_t s, addEve
addEvents(fp, src, s); addEvents(fp, src, s);
} }
static S64 abs64(S64 i) { return (i < 0) ? -i : i; } static U64 abs64(S64 s64) { return (U64)((s64 < 0) ? -s64 : s64); }
static S64 fpDistance(const Fingerprint* fp1, const Fingerprint* fp2) static U64 fpDistance(const Fingerprint* fp1, const Fingerprint* fp2)
{ {
S64 distance = 0; U64 distance = 0;
size_t n; size_t n;
for (n = 0; n < HASHTABLESIZE; n++) { for (n = 0; n < HASHTABLESIZE; n++) {
distance += distance +=
abs64(fp1->events[n] * fp2->nbEvents - fp2->events[n] * fp1->nbEvents); abs64((S64)fp1->events[n] * (S64)fp2->nbEvents - (S64)fp2->events[n] * (S64)fp1->nbEvents);
} }
return distance; return distance;
} }
@@ -100,9 +100,9 @@ static int compareFingerprints(const Fingerprint* ref,
{ {
assert(ref->nbEvents > 0); assert(ref->nbEvents > 0);
assert(newfp->nbEvents > 0); assert(newfp->nbEvents > 0);
{ S64 p50 = ref->nbEvents * newfp->nbEvents; { U64 p50 = (U64)ref->nbEvents * (U64)newfp->nbEvents;
S64 deviation = fpDistance(ref, newfp); U64 deviation = fpDistance(ref, newfp);
S64 threshold = p50 * (THRESHOLD_BASE + penalty) / THRESHOLD_PENALTY_RATE; U64 threshold = p50 * (U64)(THRESHOLD_BASE + penalty) / THRESHOLD_PENALTY_RATE;
return deviation >= threshold; return deviation >= threshold;
} }
} }
@@ -150,7 +150,7 @@ static size_t ZSTD_splitBlock_byChunks(const void* src, size_t srcSize,
assert(blockSizeMax == (128 << 10)); assert(blockSizeMax == (128 << 10));
assert(workspace != NULL); assert(workspace != NULL);
assert((size_t)workspace % ZSTD_ALIGNOF(FPStats) == 0); assert((size_t)workspace % ZSTD_ALIGNOF(FPStats) == 0);
ZSTD_STATIC_ASSERT(ZSTD_SLIPBLOCK_WORKSPACESIZE == sizeof(FPStats)); ZSTD_STATIC_ASSERT(ZSTD_SLIPBLOCK_WORKSPACESIZE >= sizeof(FPStats));
assert(wkspSize >= sizeof(FPStats)); (void)wkspSize; assert(wkspSize >= sizeof(FPStats)); (void)wkspSize;
initStats(fpstats); initStats(fpstats);