ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Experiment with 64-bit hash insertion policy

Browse Source
This commit is contained in:
Stella Lau
2017-07-24 10:20:30 -07:00
parent 273c17b350
commit 0b8fb1703b
8 changed files with 48 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files
contrib/long_distance_matching/Makefile
+3 -3
View File
@@ -25,7 +25,7 @@ LDFLAGS += -lzstd
default: all default: all
all: main-circular-buffer main-integrated main-hf all: main-circular-buffer main-integrated main-64
#main-basic : basic_table.c ldm.c main-ldm.c #main-basic : basic_table.c ldm.c main-ldm.c
# $(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@ # $(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
@@ -33,7 +33,7 @@ all: main-circular-buffer main-integrated main-hf
main-circular-buffer: circular_buffer_table.c ldm.c main-ldm.c main-circular-buffer: circular_buffer_table.c ldm.c main-ldm.c
$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@ $(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
main-hf: ldm_hf_test.c main-ldm.c main-64: ldm_64_hash.c main-ldm.c
$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@ $(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
main-integrated: ldm_with_table.c main-ldm.c main-integrated: ldm_with_table.c main-ldm.c
@@ -41,6 +41,6 @@ main-integrated: ldm_with_table.c main-ldm.c
clean: clean:
@rm -f core *.o tmp* result* *.ldm *.ldm.dec \ @rm -f core *.o tmp* result* *.ldm *.ldm.dec \
main-basic main-circular-buffer main-integrated main-hf main-basic main-circular-buffer main-integrated main-64
@echo Cleaning completed @echo Cleaning completed
contrib/long_distance_matching/circular_buffer_table.c
+5 -5
View File
@@ -164,19 +164,19 @@ LDM_hashEntry *HASH_getBestEntry(const LDM_hashTable *table,
const BYTE *pIn, const BYTE *pIn,
const BYTE *pEnd, const BYTE *pEnd,
const BYTE *pAnchor, const BYTE *pAnchor,
U32 *pForwardMatchLength, U64 *pForwardMatchLength,
U32 *pBackwardMatchLength) { U64 *pBackwardMatchLength) {
LDM_hashEntry *bucket = getBucket(table, hash); LDM_hashEntry *bucket = getBucket(table, hash);
LDM_hashEntry *cur = bucket; LDM_hashEntry *cur = bucket;
LDM_hashEntry *bestEntry = NULL; LDM_hashEntry *bestEntry = NULL;
U32 bestMatchLength = 0; U64 bestMatchLength = 0;
for (; cur < bucket + HASH_BUCKET_SIZE; ++cur) { for (; cur < bucket + HASH_BUCKET_SIZE; ++cur) {
const BYTE *pMatch = cur->offset + table->offsetBase; const BYTE *pMatch = cur->offset + table->offsetBase;
// Check checksum for faster check. // Check checksum for faster check.
if (cur->checksum == checksum && pIn - pMatch <= table->maxWindowSize) { if (cur->checksum == checksum && pIn - pMatch <= table->maxWindowSize) {
U32 forwardMatchLength = ZSTD_count(pIn, pMatch, pEnd); U64 forwardMatchLength = ZSTD_count(pIn, pMatch, pEnd);
U32 backwardMatchLength, totalMatchLength; U64 backwardMatchLength, totalMatchLength;
if (forwardMatchLength < table->minMatchLength) { if (forwardMatchLength < table->minMatchLength) {
continue; continue;
contrib/long_distance_matching/ldm.c
+5 -5
View File
@@ -429,7 +429,7 @@ void LDM_destroyCCtx(LDM_CCtx *cctx) {
* matchLength contains the forward length of the match. * matchLength contains the forward length of the match.
*/ */
static int LDM_findBestMatch(LDM_CCtx *cctx, const BYTE **match, static int LDM_findBestMatch(LDM_CCtx *cctx, const BYTE **match,
U32 *matchLength, U32 *backwardMatchLength) { U64 *matchLength, U64 *backwardMatchLength) {
LDM_hashEntry *entry = NULL; LDM_hashEntry *entry = NULL;
cctx->nextIp = cctx->ip + cctx->step; cctx->nextIp = cctx->ip + cctx->step;
@@ -462,7 +462,7 @@ static int LDM_findBestMatch(LDM_CCtx *cctx, const BYTE **match,
} }
void LDM_encodeLiteralLengthAndLiterals( void LDM_encodeLiteralLengthAndLiterals(
LDM_CCtx *cctx, BYTE *pToken, const U32 literalLength) { LDM_CCtx *cctx, BYTE *pToken, const U64 literalLength) {
/* Encode the literal length. */ /* Encode the literal length. */
if (literalLength >= RUN_MASK) { if (literalLength >= RUN_MASK) {
int len = (int)literalLength - RUN_MASK; int len = (int)literalLength - RUN_MASK;
@@ -481,9 +481,9 @@ void LDM_encodeLiteralLengthAndLiterals(
} }
void LDM_outputBlock(LDM_CCtx *cctx, void LDM_outputBlock(LDM_CCtx *cctx,
const U32 literalLength, const U64 literalLength,
const U32 offset, const U32 offset,
const U32 matchLength) { const U64 matchLength) {
BYTE *pToken = cctx->op++; BYTE *pToken = cctx->op++;
/* Encode the literal length and literals. */ /* Encode the literal length and literals. */
@@ -495,7 +495,7 @@ void LDM_outputBlock(LDM_CCtx *cctx,
/* Encode the match length. */ /* Encode the match length. */
if (matchLength >= ML_MASK) { if (matchLength >= ML_MASK) {
unsigned matchLengthRemaining = matchLength; U64 matchLengthRemaining = matchLength;
*pToken += ML_MASK; *pToken += ML_MASK;
matchLengthRemaining -= ML_MASK; matchLengthRemaining -= ML_MASK;
MEM_write32(cctx->op, 0xFFFFFFFF); MEM_write32(cctx->op, 0xFFFFFFFF);
contrib/long_distance_matching/ldm.h
+4 -4
View File
@@ -14,7 +14,7 @@
// Note that this is not the number of buckets. // Note that this is not the number of buckets.
// Currently this should be less than WINDOW_SIZE_LOG + 4? // Currently this should be less than WINDOW_SIZE_LOG + 4?
#define LDM_MEMORY_USAGE 22 #define LDM_MEMORY_USAGE 22
#define HASH_BUCKET_SIZE_LOG 3 // MAX is 4 for now #define HASH_BUCKET_SIZE_LOG 1 // MAX is 4 for now
// Defines the lag in inserting elements into the hash table. // Defines the lag in inserting elements into the hash table.
#define LDM_LAG 0 #define LDM_LAG 0
@@ -115,16 +115,16 @@ U32 LDM_countMatchLength(const BYTE *pIn, const BYTE *pMatch,
* This is followed by literalLength bytes corresponding to the literals. * This is followed by literalLength bytes corresponding to the literals.
*/ */
void LDM_encodeLiteralLengthAndLiterals( void LDM_encodeLiteralLengthAndLiterals(
LDM_CCtx *cctx, BYTE *pToken, const U32 literalLength); LDM_CCtx *cctx, BYTE *pToken, const U64 literalLength);
/** /**
* Write current block (literals, literal length, match offset, * Write current block (literals, literal length, match offset,
* match length). * match length).
*/ */
void LDM_outputBlock(LDM_CCtx *cctx, void LDM_outputBlock(LDM_CCtx *cctx,
const U32 literalLength, const U64 literalLength,
const U32 offset, const U32 offset,
const U32 matchLength); const U64 matchLength);
/** /**
* Decompresses src into dst. * Decompresses src into dst.
contrib/long_distance_matching/ldm_hf_test.c → contrib/long_distance_matching/ldm_64_hash.c
+20 -18
View File
@@ -241,9 +241,9 @@ static size_t ZSTD_count(const BYTE *pIn, const BYTE *pMatch,
* *
* We count only bytes where pMatch > pBaes and pIn > pAnchor. * We count only bytes where pMatch > pBaes and pIn > pAnchor.
*/ */
U32 countBackwardsMatch(const BYTE *pIn, const BYTE *pAnchor, U64 countBackwardsMatch(const BYTE *pIn, const BYTE *pAnchor,
const BYTE *pMatch, const BYTE *pBase) { const BYTE *pMatch, const BYTE *pBase) {
U32 matchLength = 0; U64 matchLength = 0;
while (pIn > pAnchor && pMatch > pBase && pIn[-1] == pMatch[-1]) { while (pIn > pAnchor && pMatch > pBase && pIn[-1] == pMatch[-1]) {
pIn--; pIn--;
pMatch--; pMatch--;
@@ -267,8 +267,8 @@ U32 countBackwardsMatch(const BYTE *pIn, const BYTE *pAnchor,
LDM_hashEntry *HASH_getBestEntry(const LDM_CCtx *cctx, LDM_hashEntry *HASH_getBestEntry(const LDM_CCtx *cctx,
const hash_t hash, const hash_t hash,
const U32 checksum, const U32 checksum,
U32 *pForwardMatchLength, U64 *pForwardMatchLength,
U32 *pBackwardMatchLength) { U64 *pBackwardMatchLength) {
LDM_hashTable *table = cctx->hashTable; LDM_hashTable *table = cctx->hashTable;
LDM_hashEntry *bucket = getBucket(table, hash); LDM_hashEntry *bucket = getBucket(table, hash);
LDM_hashEntry *cur = bucket; LDM_hashEntry *cur = bucket;
@@ -541,7 +541,7 @@ static U64 updateChecksum(U64 sum, U32 len,
BYTE toRemove, BYTE toAdd) { BYTE toRemove, BYTE toAdd) {
// TODO: deduplicate. // TODO: deduplicate.
static const U64 prime8bytes = 11400714785074694791ULL; static const U64 prime8bytes = 11400714785074694791ULL;
// static const U64 prime8bytes = 5;
sum -= ((toRemove + CHECKSUM_CHAR_OFFSET) * sum -= ((toRemove + CHECKSUM_CHAR_OFFSET) *
ipow(prime8bytes, len - 1)); ipow(prime8bytes, len - 1));
sum *= prime8bytes; sum *= prime8bytes;
@@ -696,11 +696,12 @@ U32 LDM_countMatchLength(const BYTE *pIn, const BYTE *pMatch,
void LDM_outputConfiguration(void) { void LDM_outputConfiguration(void) {
printf("=====================\n"); printf("=====================\n");
printf("Configuration\n"); printf("Configuration\n");
printf("Window size log: %d\n", LDM_WINDOW_SIZE_LOG); printf("LDM_WINDOW_SIZE_LOG: %d\n", LDM_WINDOW_SIZE_LOG);
printf("Min match, hash length: %d, %d\n", printf("LDM_MIN_MATCH_LENGTH, LDM_HASH_LENGTH: %d, %d\n",
LDM_MIN_MATCH_LENGTH, LDM_HASH_LENGTH); LDM_MIN_MATCH_LENGTH, LDM_HASH_LENGTH);
printf("LDM_MEMORY_USAGE: %d\n", LDM_MEMORY_USAGE); printf("LDM_MEMORY_USAGE: %d\n", LDM_MEMORY_USAGE);
printf("HASH_ONLY_EVERY_LOG: %d\n", HASH_ONLY_EVERY_LOG); printf("HASH_ONLY_EVERY_LOG: %d\n", HASH_ONLY_EVERY_LOG);
printf("HASH_BUCKET_SIZE_LOG: %d\n", HASH_BUCKET_SIZE_LOG);
printf("LDM_LAG %d\n", LDM_LAG); printf("LDM_LAG %d\n", LDM_LAG);
printf("=====================\n"); printf("=====================\n");
} }
@@ -762,7 +763,7 @@ void LDM_destroyCCtx(LDM_CCtx *cctx) {
* forwardMatchLength contains the forward length of the match. * forwardMatchLength contains the forward length of the match.
*/ */
static int LDM_findBestMatch(LDM_CCtx *cctx, const BYTE **match, static int LDM_findBestMatch(LDM_CCtx *cctx, const BYTE **match,
U32 *forwardMatchLength, U32 *backwardMatchLength) { U64 *forwardMatchLength, U64 *backwardMatchLength) {
LDM_hashEntry *entry = NULL; LDM_hashEntry *entry = NULL;
cctx->nextIp = cctx->ip + cctx->step; cctx->nextIp = cctx->ip + cctx->step;
@@ -800,10 +801,10 @@ static int LDM_findBestMatch(LDM_CCtx *cctx, const BYTE **match,
} }
void LDM_encodeLiteralLengthAndLiterals( void LDM_encodeLiteralLengthAndLiterals(
LDM_CCtx *cctx, BYTE *pToken, const U32 literalLength) { LDM_CCtx *cctx, BYTE *pToken, const U64 literalLength) {
/* Encode the literal length. */ /* Encode the literal length. */
if (literalLength >= RUN_MASK) { if (literalLength >= RUN_MASK) {
int len = (int)literalLength - RUN_MASK; U64 len = (U64)literalLength - RUN_MASK;
*pToken = (RUN_MASK << ML_BITS); *pToken = (RUN_MASK << ML_BITS);
for (; len >= 255; len -= 255) { for (; len >= 255; len -= 255) {
*(cctx->op)++ = 255; *(cctx->op)++ = 255;
@@ -819,9 +820,9 @@ void LDM_encodeLiteralLengthAndLiterals(
} }
void LDM_outputBlock(LDM_CCtx *cctx, void LDM_outputBlock(LDM_CCtx *cctx,
const U32 literalLength, const U64 literalLength,
const U32 offset, const U32 offset,
const U32 matchLength) { const U64 matchLength) {
BYTE *pToken = cctx->op++; BYTE *pToken = cctx->op++;
/* Encode the literal length and literals. */ /* Encode the literal length and literals. */
@@ -833,7 +834,7 @@ void LDM_outputBlock(LDM_CCtx *cctx,
/* Encode the match length. */ /* Encode the match length. */
if (matchLength >= ML_MASK) { if (matchLength >= ML_MASK) {
unsigned matchLengthRemaining = matchLength; U64 matchLengthRemaining = matchLength;
*pToken += ML_MASK; *pToken += ML_MASK;
matchLengthRemaining -= ML_MASK; matchLengthRemaining -= ML_MASK;
MEM_write32(cctx->op, 0xFFFFFFFF); MEM_write32(cctx->op, 0xFFFFFFFF);
@@ -858,8 +859,8 @@ size_t LDM_compress(const void *src, size_t srcSize,
void *dst, size_t maxDstSize) { void *dst, size_t maxDstSize) {
LDM_CCtx cctx; LDM_CCtx cctx;
const BYTE *match = NULL; const BYTE *match = NULL;
U32 forwardMatchLength = 0; U64 forwardMatchLength = 0;
U32 backwardsMatchLength = 0; U64 backwardsMatchLength = 0;
LDM_initializeCCtx(&cctx, src, srcSize, dst, maxDstSize); LDM_initializeCCtx(&cctx, src, srcSize, dst, maxDstSize);
LDM_outputConfiguration(); LDM_outputConfiguration();
@@ -892,9 +893,9 @@ size_t LDM_compress(const void *src, size_t srcSize,
* length) and update pointers and hashes. * length) and update pointers and hashes.
*/ */
{ {
const U32 literalLength = cctx.ip - cctx.anchor; const U64 literalLength = cctx.ip - cctx.anchor;
const U32 offset = cctx.ip - match; const U32 offset = cctx.ip - match;
const U32 matchLength = forwardMatchLength + const U64 matchLength = forwardMatchLength +
backwardsMatchLength - backwardsMatchLength -
LDM_MIN_MATCH_LENGTH; LDM_MIN_MATCH_LENGTH;
@@ -934,7 +935,7 @@ size_t LDM_compress(const void *src, size_t srcSize,
/* Encode the last literals (no more matches). */ /* Encode the last literals (no more matches). */
{ {
const U32 lastRun = cctx.iend - cctx.anchor; const U64 lastRun = cctx.iend - cctx.anchor;
BYTE *pToken = cctx.op++; BYTE *pToken = cctx.op++;
LDM_encodeLiteralLengthAndLiterals(&cctx, pToken, lastRun); LDM_encodeLiteralLengthAndLiterals(&cctx, pToken, lastRun);
} }
@@ -979,6 +980,7 @@ void LDM_initializeDCtx(LDM_DCtx *dctx,
size_t LDM_decompress(const void *src, size_t compressedSize, size_t LDM_decompress(const void *src, size_t compressedSize,
void *dst, size_t maxDecompressedSize) { void *dst, size_t maxDecompressedSize) {
LDM_DCtx dctx; LDM_DCtx dctx;
LDM_initializeDCtx(&dctx, src, compressedSize, dst, maxDecompressedSize); LDM_initializeDCtx(&dctx, src, compressedSize, dst, maxDecompressedSize);
contrib/long_distance_matching/ldm_hashtable.h
+2 -2
View File
@@ -25,8 +25,8 @@ LDM_hashEntry *HASH_getBestEntry(const LDM_hashTable *table,
const BYTE *pIn, const BYTE *pIn,
const BYTE *pEnd, const BYTE *pEnd,
const BYTE *pAnchor, const BYTE *pAnchor,
U32 *matchLength, U64 *matchLength,
U32 *backwardsMatchLength); U64 *backwardsMatchLength);
hash_t HASH_hashU32(U32 value); hash_t HASH_hashU32(U32 value);
contrib/long_distance_matching/ldm_with_table.c
+6 -6
View File
@@ -719,10 +719,10 @@ static int LDM_findBestMatch(LDM_CCtx *cctx, const BYTE **match,
} }
void LDM_encodeLiteralLengthAndLiterals( void LDM_encodeLiteralLengthAndLiterals(
LDM_CCtx *cctx, BYTE *pToken, const U32 literalLength) { LDM_CCtx *cctx, BYTE *pToken, const U64 literalLength) {
/* Encode the literal length. */ /* Encode the literal length. */
if (literalLength >= RUN_MASK) { if (literalLength >= RUN_MASK) {
int len = (int)literalLength - RUN_MASK; U64 len = (U64)literalLength - RUN_MASK;
*pToken = (RUN_MASK << ML_BITS); *pToken = (RUN_MASK << ML_BITS);
for (; len >= 255; len -= 255) { for (; len >= 255; len -= 255) {
*(cctx->op)++ = 255; *(cctx->op)++ = 255;
@@ -738,9 +738,9 @@ void LDM_encodeLiteralLengthAndLiterals(
} }
void LDM_outputBlock(LDM_CCtx *cctx, void LDM_outputBlock(LDM_CCtx *cctx,
const U32 literalLength, const U64 literalLength,
const U32 offset, const U32 offset,
const U32 matchLength) { const U64 matchLength) {
BYTE *pToken = cctx->op++; BYTE *pToken = cctx->op++;
/* Encode the literal length and literals. */ /* Encode the literal length and literals. */
@@ -811,9 +811,9 @@ size_t LDM_compress(const void *src, size_t srcSize,
* length) and update pointers and hashes. * length) and update pointers and hashes.
*/ */
{ {
const U32 literalLength = cctx.ip - cctx.anchor; const U64 literalLength = cctx.ip - cctx.anchor;
const U32 offset = cctx.ip - match; const U32 offset = cctx.ip - match;
const U32 matchLength = forwardMatchLength + const U64 matchLength = forwardMatchLength +
backwardsMatchLength - backwardsMatchLength -
LDM_MIN_MATCH_LENGTH; LDM_MIN_MATCH_LENGTH;
contrib/long_distance_matching/main-ldm.c
+3 -4
View File
@@ -94,8 +94,8 @@ static int compress(const char *fname, const char *oname) {
// Truncate file to compressedSize. // Truncate file to compressedSize.
ftruncate(fdout, compressedSize); ftruncate(fdout, compressedSize);
printf("%25s : %6u -> %7u - %s (%.1f%%)\n", fname, printf("%25s : %10lu -> %10lu - %s (%.1f%%)\n", fname,
(unsigned)statbuf.st_size, (unsigned)compressedSize, oname, (size_t)statbuf.st_size, (size_t)compressedSize, oname,
(double)compressedSize / (statbuf.st_size) * 100); (double)compressedSize / (statbuf.st_size) * 100);
timeTaken = (double) (tv2.tv_usec - tv1.tv_usec) / 1000000 + timeTaken = (double) (tv2.tv_usec - tv1.tv_usec) / 1000000 +
@@ -164,7 +164,7 @@ static int decompress(const char *fname, const char *oname) {
src + LDM_HEADER_SIZE, statbuf.st_size - LDM_HEADER_SIZE, src + LDM_HEADER_SIZE, statbuf.st_size - LDM_HEADER_SIZE,
dst, decompressedSize); dst, decompressedSize);
printf("Ret size out: %zu\n", outSize); printf("Ret size out: %zu\n", outSize);
ftruncate(fdout, outSize); // ftruncate(fdout, decompressedSize);
close(fdin); close(fdin);
close(fdout); close(fdout);
@@ -231,7 +231,6 @@ int main(int argc, const char *argv[]) {
printf("ldm = [%s]\n", ldmFilename); printf("ldm = [%s]\n", ldmFilename);
printf("dec = [%s]\n", decFilename); printf("dec = [%s]\n", decFilename);
/* Compress */ /* Compress */
{ {
if (compress(inpFilename, ldmFilename)) { if (compress(inpFilename, ldmFilename)) {