From bf0d5594494176febeda5d468d68ebabcbead980 Mon Sep 17 00:00:00 2001 From: senhuang42 Date: Mon, 12 Oct 2020 16:17:39 -0400 Subject: [PATCH 1/3] Introduce, implement, and call ZSTD_ldm_countBackwardsMatch_2segments() --- lib/compress/zstd_ldm.c | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/lib/compress/zstd_ldm.c b/lib/compress/zstd_ldm.c index cc338e697..d45adc045 100644 --- a/lib/compress/zstd_ldm.c +++ b/lib/compress/zstd_ldm.c @@ -154,6 +154,26 @@ static size_t ZSTD_ldm_countBackwardsMatch( return matchLength; } +/** ZSTD_ldm_countBackwardsMatch_2segments() : + * Returns the number of bytes that match backwards from pMatch, + * even with the backwards match spanning 2 different segments. + * + * On reaching `pMatchBase`, start counting from mEnd */ +static size_t ZSTD_ldm_countBackwardsMatch_2segments( + const BYTE* pIn, const BYTE* pAnchor, + const BYTE* pMatch, const BYTE* pMatchBase, const BYTE* mEnd) +{ + const BYTE* const vBegin = MAX(pIn - (pMatch - pMatchBase), pAnchor); + size_t const matchLength = ZSTD_ldm_countBackwardsMatch(pIn, vBegin, pMatch, pMatchBase); + if (pMatch - matchLength != pMatchBase) { + return matchLength; + } + DEBUGLOG(7, "ZSTD_ldm_countBackwardsMatch_2segments: found a 2-parts backwards match (current length==%zu)", matchLength); + DEBUGLOG(7, "distance from pMatch to start = %zi", pMatch - pMatchBase); + DEBUGLOG(7, "final backwards match length = %zu", matchLength + ZSTD_ldm_countBackwardsMatch(pIn - matchLength, pAnchor, mEnd, pMatchBase)); + return matchLength + ZSTD_ldm_countBackwardsMatch(pIn - matchLength, pAnchor, mEnd, pMatchBase); +} + /** ZSTD_ldm_fillFastTables() : * * Fills the relevant tables for the ZSTD_fast and ZSTD_dfast strategies. @@ -329,8 +349,8 @@ static size_t ZSTD_ldm_generateSequences_internal( continue; } curBackwardMatchLength = - ZSTD_ldm_countBackwardsMatch(ip, anchor, pMatch, - lowMatchPtr); + ZSTD_ldm_countBackwardsMatch_2segments(ip, anchor, pMatch, + lowMatchPtr, dictEnd); curTotalMatchLength = curForwardMatchLength + curBackwardMatchLength; } else { /* !extDict */ From 3f99c9b38dce592c8336187c7e2e1339a3c428ae Mon Sep 17 00:00:00 2001 From: senhuang42 Date: Wed, 14 Oct 2020 15:23:03 -0400 Subject: [PATCH 2/3] Adjust match backwards count args --- lib/compress/zstd_ldm.c | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/lib/compress/zstd_ldm.c b/lib/compress/zstd_ldm.c index d45adc045..9a6ea108b 100644 --- a/lib/compress/zstd_ldm.c +++ b/lib/compress/zstd_ldm.c @@ -161,17 +161,15 @@ static size_t ZSTD_ldm_countBackwardsMatch( * On reaching `pMatchBase`, start counting from mEnd */ static size_t ZSTD_ldm_countBackwardsMatch_2segments( const BYTE* pIn, const BYTE* pAnchor, - const BYTE* pMatch, const BYTE* pMatchBase, const BYTE* mEnd) + const BYTE* pMatch, const BYTE* pMatchBase, const BYTE* dictStart, const BYTE* mEnd) { - const BYTE* const vBegin = MAX(pIn - (pMatch - pMatchBase), pAnchor); - size_t const matchLength = ZSTD_ldm_countBackwardsMatch(pIn, vBegin, pMatch, pMatchBase); - if (pMatch - matchLength != pMatchBase) { + size_t const matchLength = ZSTD_ldm_countBackwardsMatch(pIn, pAnchor, pMatch, pMatchBase); + if (pMatch - matchLength != pMatchBase || pMatchBase == dictStart) { return matchLength; } DEBUGLOG(7, "ZSTD_ldm_countBackwardsMatch_2segments: found a 2-parts backwards match (current length==%zu)", matchLength); - DEBUGLOG(7, "distance from pMatch to start = %zi", pMatch - pMatchBase); DEBUGLOG(7, "final backwards match length = %zu", matchLength + ZSTD_ldm_countBackwardsMatch(pIn - matchLength, pAnchor, mEnd, pMatchBase)); - return matchLength + ZSTD_ldm_countBackwardsMatch(pIn - matchLength, pAnchor, mEnd, pMatchBase); + return matchLength + ZSTD_ldm_countBackwardsMatch(pIn - matchLength, pAnchor, mEnd, dictStart); } /** ZSTD_ldm_fillFastTables() : @@ -350,7 +348,7 @@ static size_t ZSTD_ldm_generateSequences_internal( } curBackwardMatchLength = ZSTD_ldm_countBackwardsMatch_2segments(ip, anchor, pMatch, - lowMatchPtr, dictEnd); + lowMatchPtr, dictStart, dictEnd); curTotalMatchLength = curForwardMatchLength + curBackwardMatchLength; } else { /* !extDict */ From d0550bb18f0044941077c42bad56ab6b8d92dd30 Mon Sep 17 00:00:00 2001 From: senhuang42 Date: Wed, 14 Oct 2020 15:45:43 -0400 Subject: [PATCH 3/3] Clarify argument names, fix DEBUGLOG() statements --- lib/compress/zstd_ldm.c | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/lib/compress/zstd_ldm.c b/lib/compress/zstd_ldm.c index 9a6ea108b..a6e780b9e 100644 --- a/lib/compress/zstd_ldm.c +++ b/lib/compress/zstd_ldm.c @@ -143,10 +143,10 @@ static void ZSTD_ldm_makeEntryAndInsertByTag(ldmState_t* ldmState, * We count only bytes where pMatch >= pBase and pIn >= pAnchor. */ static size_t ZSTD_ldm_countBackwardsMatch( const BYTE* pIn, const BYTE* pAnchor, - const BYTE* pMatch, const BYTE* pBase) + const BYTE* pMatch, const BYTE* pMatchBase) { size_t matchLength = 0; - while (pIn > pAnchor && pMatch > pBase && pIn[-1] == pMatch[-1]) { + while (pIn > pAnchor && pMatch > pMatchBase && pIn[-1] == pMatch[-1]) { pIn--; pMatch--; matchLength++; @@ -160,16 +160,19 @@ static size_t ZSTD_ldm_countBackwardsMatch( * * On reaching `pMatchBase`, start counting from mEnd */ static size_t ZSTD_ldm_countBackwardsMatch_2segments( - const BYTE* pIn, const BYTE* pAnchor, - const BYTE* pMatch, const BYTE* pMatchBase, const BYTE* dictStart, const BYTE* mEnd) + const BYTE* pIn, const BYTE* pAnchor, + const BYTE* pMatch, const BYTE* pMatchBase, + const BYTE* pExtDictStart, const BYTE* pExtDictEnd) { - size_t const matchLength = ZSTD_ldm_countBackwardsMatch(pIn, pAnchor, pMatch, pMatchBase); - if (pMatch - matchLength != pMatchBase || pMatchBase == dictStart) { + size_t matchLength = ZSTD_ldm_countBackwardsMatch(pIn, pAnchor, pMatch, pMatchBase); + if (pMatch - matchLength != pMatchBase || pMatchBase == pExtDictStart) { + /* If backwards match is entirely in the extDict or prefix, immediately return */ return matchLength; } - DEBUGLOG(7, "ZSTD_ldm_countBackwardsMatch_2segments: found a 2-parts backwards match (current length==%zu)", matchLength); - DEBUGLOG(7, "final backwards match length = %zu", matchLength + ZSTD_ldm_countBackwardsMatch(pIn - matchLength, pAnchor, mEnd, pMatchBase)); - return matchLength + ZSTD_ldm_countBackwardsMatch(pIn - matchLength, pAnchor, mEnd, dictStart); + DEBUGLOG(7, "ZSTD_ldm_countBackwardsMatch_2segments: found 2-parts backwards match (length in prefix==%zu)", matchLength); + matchLength += ZSTD_ldm_countBackwardsMatch(pIn - matchLength, pAnchor, pExtDictEnd, pExtDictStart); + DEBUGLOG(7, "final backwards match length = %zu", matchLength); + return matchLength; } /** ZSTD_ldm_fillFastTables() : @@ -347,8 +350,9 @@ static size_t ZSTD_ldm_generateSequences_internal( continue; } curBackwardMatchLength = - ZSTD_ldm_countBackwardsMatch_2segments(ip, anchor, pMatch, - lowMatchPtr, dictStart, dictEnd); + ZSTD_ldm_countBackwardsMatch_2segments(ip, anchor, + pMatch, lowMatchPtr, + dictStart, dictEnd); curTotalMatchLength = curForwardMatchLength + curBackwardMatchLength; } else { /* !extDict */