ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[linux-kernel] Update patches for v5

Browse Source
This commit is contained in:
Nick Terrell
2017-08-09 13:03:40 -07:00
parent 7393b49fbd
commit 8b6702a00d
11 changed files with 164 additions and 116 deletions
Download Patch File Download Diff File Expand all files Collapse all files
contrib/linux-kernel/0000-cover-letter.patch
+14 -8
View File
@@ -1,7 +1,7 @@
From a276288db937088d00b975ad9c36278fa46c8cf7 Mon Sep 17 00:00:00 2001 From 308795a7713ca6fcd468b60fba9a2fca99cee6a0 Mon Sep 17 00:00:00 2001
From: Nick Terrell <terrelln@fb.com> From: Nick Terrell <terrelln@fb.com>
Date: Fri, 4 Aug 2017 12:47:29 -0700 Date: Tue, 8 Aug 2017 19:20:25 -0700
Subject: [PATCH v4 0/5] Add xxhash and zstd modules Subject: [PATCH v5 0/5] Add xxhash and zstd modules
Hi all, Hi all,
@@ -37,6 +37,12 @@ v3 -> v4:
- Fix minor linter warnings (3/5, 4/5) - Fix minor linter warnings (3/5, 4/5)
- Add crypto patch (5/5) - Add crypto patch (5/5)
v4 -> v5:
- Fix rare compression bug from upstream commit 308047eb5d (2/5)
- Fix bug introduced in v3 when working around the gcc-7 bug (2/5)
- Fix ZSTD_DStream initialization code in squashfs (4/5)
- Fix patch documentation for patches written by Sean Purcell (4/5)
Nick Terrell (5): Nick Terrell (5):
lib: Add xxhash module lib: Add xxhash module
lib: Add zstd modules lib: Add zstd modules
@@ -65,7 +71,7 @@ Nick Terrell (5):
fs/squashfs/decompressor.c | 7 + fs/squashfs/decompressor.c | 7 +
fs/squashfs/decompressor.h | 4 + fs/squashfs/decompressor.h | 4 +
fs/squashfs/squashfs_fs.h | 1 + fs/squashfs/squashfs_fs.h | 1 +
fs/squashfs/zstd_wrapper.c | 149 ++ fs/squashfs/zstd_wrapper.c | 151 ++
include/linux/xxhash.h | 236 +++ include/linux/xxhash.h | 236 +++
include/linux/zstd.h | 1157 +++++++++++++++ include/linux/zstd.h | 1157 +++++++++++++++
include/uapi/linux/btrfs.h | 8 +- include/uapi/linux/btrfs.h | 8 +-
@@ -74,9 +80,9 @@ Nick Terrell (5):
lib/xxhash.c | 500 +++++++ lib/xxhash.c | 500 +++++++
lib/zstd/Makefile | 18 + lib/zstd/Makefile | 18 +
lib/zstd/bitstream.h | 374 +++++ lib/zstd/bitstream.h | 374 +++++
lib/zstd/compress.c | 3479 ++++++++++++++++++++++++++++++++++++++++++++ lib/zstd/compress.c | 3484 ++++++++++++++++++++++++++++++++++++++++++++
lib/zstd/decompress.c | 2528 ++++++++++++++++++++++++++++++++ lib/zstd/decompress.c | 2528 ++++++++++++++++++++++++++++++++
lib/zstd/entropy_common.c | 243 ++++ lib/zstd/entropy_common.c | 243 +++
lib/zstd/error_private.h | 53 + lib/zstd/error_private.h | 53 +
lib/zstd/fse.h | 575 ++++++++ lib/zstd/fse.h | 575 ++++++++
lib/zstd/fse_compress.c | 795 ++++++++++ lib/zstd/fse_compress.c | 795 ++++++++++
@@ -86,9 +92,9 @@ Nick Terrell (5):
lib/zstd/huf_decompress.c | 960 ++++++++++++ lib/zstd/huf_decompress.c | 960 ++++++++++++
lib/zstd/mem.h | 151 ++ lib/zstd/mem.h | 151 ++
lib/zstd/zstd_common.c | 75 + lib/zstd/zstd_common.c | 75 +
lib/zstd/zstd_internal.h | 250 ++++ lib/zstd/zstd_internal.h | 263 ++++
lib/zstd/zstd_opt.h | 1014 +++++++++++++ lib/zstd/zstd_opt.h | 1014 +++++++++++++
44 files changed, 14736 insertions(+), 12 deletions(-) 44 files changed, 14756 insertions(+), 12 deletions(-)
create mode 100644 crypto/zstd.c create mode 100644 crypto/zstd.c
create mode 100644 fs/btrfs/zstd.c create mode 100644 fs/btrfs/zstd.c
create mode 100644 fs/squashfs/zstd_wrapper.c create mode 100644 fs/squashfs/zstd_wrapper.c
contrib/linux-kernel/0001-lib-Add-xxhash-module.patch
+2 -2
View File
@@ -1,7 +1,7 @@
From 587f1ba6e78cc5b0d3e26971290aef36ff66f378 Mon Sep 17 00:00:00 2001 From a4b1ffb6e89bbccd519f9afa0910635668436105 Mon Sep 17 00:00:00 2001
From: Nick Terrell <terrelln@fb.com> From: Nick Terrell <terrelln@fb.com>
Date: Mon, 17 Jul 2017 17:07:18 -0700 Date: Mon, 17 Jul 2017 17:07:18 -0700
Subject: [PATCH v4 1/5] lib: Add xxhash module Subject: [PATCH v5 1/5] lib: Add xxhash module
Adds xxhash kernel module with xxh32 and xxh64 hashes. xxhash is an Adds xxhash kernel module with xxh32 and xxh64 hashes. xxhash is an
extremely fast non-cryptographic hash algorithm for checksumming. extremely fast non-cryptographic hash algorithm for checksumming.
contrib/linux-kernel/0002-lib-Add-zstd-modules.patch
+54 -32
View File
@@ -1,7 +1,7 @@
From c7f952ce985f652fe1f2c9266f39cd87b470fd8a Mon Sep 17 00:00:00 2001 From b7f044163968d724be55bf4841fd80babe036dc2 Mon Sep 17 00:00:00 2001
From: Nick Terrell <terrelln@fb.com> From: Nick Terrell <terrelln@fb.com>
Date: Mon, 17 Jul 2017 17:08:19 -0700 Date: Mon, 17 Jul 2017 17:08:19 -0700
Subject: [PATCH v4 2/5] lib: Add zstd modules Subject: [PATCH v5 2/5] lib: Add zstd modules
Add zstd compression and decompression kernel modules. Add zstd compression and decompression kernel modules.
zstd offers a wide varity of compression speed and quality trade-offs. zstd offers a wide varity of compression speed and quality trade-offs.
@@ -117,12 +117,16 @@ v2 -> v3:
v3 -> v4: v3 -> v4:
- Fix minor compiler warnings - Fix minor compiler warnings
v4 -> v5:
- Fix rare compression bug from upstream commit 308047eb5d
- Fix bug introduced in v3 when working around the gcc-7 bug
include/linux/zstd.h | 1157 +++++++++++++++ include/linux/zstd.h | 1157 +++++++++++++++
lib/Kconfig | 8 + lib/Kconfig | 8 +
lib/Makefile | 2 + lib/Makefile | 2 +
lib/zstd/Makefile | 18 + lib/zstd/Makefile | 18 +
lib/zstd/bitstream.h | 374 +++++ lib/zstd/bitstream.h | 374 +++++
lib/zstd/compress.c | 3479 +++++++++++++++++++++++++++++++++++++++++++++ lib/zstd/compress.c | 3484 +++++++++++++++++++++++++++++++++++++++++++++
lib/zstd/decompress.c | 2528 ++++++++++++++++++++++++++++++++ lib/zstd/decompress.c | 2528 ++++++++++++++++++++++++++++++++
lib/zstd/entropy_common.c | 243 ++++ lib/zstd/entropy_common.c | 243 ++++
lib/zstd/error_private.h | 53 + lib/zstd/error_private.h | 53 +
@@ -134,9 +138,9 @@ v3 -> v4:
lib/zstd/huf_decompress.c | 960 +++++++++++++ lib/zstd/huf_decompress.c | 960 +++++++++++++
lib/zstd/mem.h | 151 ++ lib/zstd/mem.h | 151 ++
lib/zstd/zstd_common.c | 75 + lib/zstd/zstd_common.c | 75 +
lib/zstd/zstd_internal.h | 250 ++++ lib/zstd/zstd_internal.h | 263 ++++
lib/zstd/zstd_opt.h | 1014 +++++++++++++ lib/zstd/zstd_opt.h | 1014 +++++++++++++
19 files changed, 12996 insertions(+) 19 files changed, 13014 insertions(+)
create mode 100644 include/linux/zstd.h create mode 100644 include/linux/zstd.h
create mode 100644 lib/zstd/Makefile create mode 100644 lib/zstd/Makefile
create mode 100644 lib/zstd/bitstream.h create mode 100644 lib/zstd/bitstream.h
@@ -1756,10 +1760,10 @@ index 0000000..a826b99
+#endif /* BITSTREAM_H_MODULE */ +#endif /* BITSTREAM_H_MODULE */
diff --git a/lib/zstd/compress.c b/lib/zstd/compress.c diff --git a/lib/zstd/compress.c b/lib/zstd/compress.c
new file mode 100644 new file mode 100644
index 0000000..d60ab7d index 0000000..f9166cf
--- /dev/null --- /dev/null
+++ b/lib/zstd/compress.c +++ b/lib/zstd/compress.c
@@ -0,0 +1,3479 @@ @@ -0,0 +1,3484 @@
+/** +/**
+ * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
+ * All rights reserved. + * All rights reserved.
@@ -2345,7 +2349,7 @@ index 0000000..d60ab7d
+ mlCodeTable[seqStorePtr->longLengthPos] = MaxML; + mlCodeTable[seqStorePtr->longLengthPos] = MaxML;
+} +}
+ +
+ZSTD_STATIC size_t ZSTD_compressSequences(ZSTD_CCtx *zc, void *dst, size_t dstCapacity, size_t srcSize) +ZSTD_STATIC size_t ZSTD_compressSequences_internal(ZSTD_CCtx *zc, void *dst, size_t dstCapacity)
+{ +{
+ const int longOffsets = zc->params.cParams.windowLog > STREAM_ACCUMULATOR_MIN; + const int longOffsets = zc->params.cParams.windowLog > STREAM_ACCUMULATOR_MIN;
+ const seqStore_t *seqStorePtr = &(zc->seqStore); + const seqStore_t *seqStorePtr = &(zc->seqStore);
@@ -2398,7 +2402,7 @@ index 0000000..d60ab7d
+ else + else
+ op[0] = 0xFF, ZSTD_writeLE16(op + 1, (U16)(nbSeq - LONGNBSEQ)), op += 3; + op[0] = 0xFF, ZSTD_writeLE16(op + 1, (U16)(nbSeq - LONGNBSEQ)), op += 3;
+ if (nbSeq == 0) + if (nbSeq == 0)
+ goto _check_compressibility; + return op - ostart;
+ +
+ /* seqHead : flags for FSE encoding type */ + /* seqHead : flags for FSE encoding type */
+ seqHead = op++; + seqHead = op++;
@@ -2588,28 +2592,33 @@ index 0000000..d60ab7d
+ op += streamSize; + op += streamSize;
+ } + }
+ } + }
+
+/* check compressibility */
+_check_compressibility:
+ {
+ size_t const minGain = ZSTD_minGain(srcSize);
+ size_t const maxCSize = srcSize - minGain;
+ if ((size_t)(op - ostart) >= maxCSize) {
+ zc->flagStaticHufTable = HUF_repeat_none;
+ return 0;
+ }
+ }
+
+ /* confirm repcodes */
+ {
+ int i;
+ for (i = 0; i < ZSTD_REP_NUM; i++)
+ zc->rep[i] = zc->repToConfirm[i];
+ }
+
+ return op - ostart; + return op - ostart;
+} +}
+ +
+ZSTD_STATIC size_t ZSTD_compressSequences(ZSTD_CCtx *zc, void *dst, size_t dstCapacity, size_t srcSize)
+{
+ size_t const cSize = ZSTD_compressSequences_internal(zc, dst, dstCapacity);
+ size_t const minGain = ZSTD_minGain(srcSize);
+ size_t const maxCSize = srcSize - minGain;
+ /* If the srcSize <= dstCapacity, then there is enough space to write a
+ * raw uncompressed block. Since we ran out of space, the block must not
+ * be compressible, so fall back to a raw uncompressed block.
+ */
+ int const uncompressibleError = cSize == ERROR(dstSize_tooSmall) && srcSize <= dstCapacity;
+ int i;
+
+ if (ZSTD_isError(cSize) && !uncompressibleError)
+ return cSize;
+ if (cSize >= maxCSize || uncompressibleError) {
+ zc->flagStaticHufTable = HUF_repeat_none;
+ return 0;
+ }
+ /* confirm repcodes */
+ for (i = 0; i < ZSTD_REP_NUM; i++)
+ zc->rep[i] = zc->repToConfirm[i];
+ return cSize;
+}
+
+/*! ZSTD_storeSeq() : +/*! ZSTD_storeSeq() :
+ Store a sequence (literal length, literals, offset code and match length code) into seqStore_t. + Store a sequence (literal length, literals, offset code and match length code) into seqStore_t.
+ `offsetCode` : distance to match, or 0 == repCode. + `offsetCode` : distance to match, or 0 == repCode.
@@ -12001,10 +12010,10 @@ index 0000000..a282624
+} +}
diff --git a/lib/zstd/zstd_internal.h b/lib/zstd/zstd_internal.h diff --git a/lib/zstd/zstd_internal.h b/lib/zstd/zstd_internal.h
new file mode 100644 new file mode 100644
index 0000000..44e8f100 index 0000000..1a79fab
--- /dev/null --- /dev/null
+++ b/lib/zstd/zstd_internal.h +++ b/lib/zstd/zstd_internal.h
@@ -0,0 +1,250 @@ @@ -0,0 +1,263 @@
+/** +/**
+ * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
+ * All rights reserved. + * All rights reserved.
@@ -12141,8 +12150,21 @@ index 0000000..44e8f100
+#define WILDCOPY_OVERLENGTH 8 +#define WILDCOPY_OVERLENGTH 8
+ZSTD_STATIC void ZSTD_wildcopy(void *dst, const void *src, ptrdiff_t length) +ZSTD_STATIC void ZSTD_wildcopy(void *dst, const void *src, ptrdiff_t length)
+{ +{
+ if (length > 0) + const BYTE* ip = (const BYTE*)src;
+ memcpy(dst, src, length); + BYTE* op = (BYTE*)dst;
+ BYTE* const oend = op + length;
+ /* Work around https://gcc.gnu.org/bugzilla/show_bug.cgi?id=81388.
+ * Avoid the bad case where the loop only runs once by handling the
+ * special case separately. This doesn't trigger the bug because it
+ * doesn't involve pointer/integer overflow.
+ */
+ if (length <= 8)
+ return ZSTD_copy8(dst, src);
+ do {
+ ZSTD_copy8(op, ip);
+ op += 8;
+ ip += 8;
+ } while (op < oend);
+} +}
+ +
+/*-******************************************* +/*-*******************************************
contrib/linux-kernel/0003-btrfs-Add-zstd-support.patch
+2 -2
View File
@@ -1,7 +1,7 @@
From 6ade5bc08dcfa2bce2b4801e47edf783dcb7ca43 Mon Sep 17 00:00:00 2001 From 8a9dddfbf6551afea73911e367dd4be64d62b9fd Mon Sep 17 00:00:00 2001
From: Nick Terrell <terrelln@fb.com> From: Nick Terrell <terrelln@fb.com>
Date: Mon, 17 Jul 2017 17:08:39 -0700 Date: Mon, 17 Jul 2017 17:08:39 -0700
Subject: [PATCH v4 3/5] btrfs: Add zstd support Subject: [PATCH v5 3/5] btrfs: Add zstd support
Add zstd compression and decompression support to BtrFS. zstd at its Add zstd compression and decompression support to BtrFS. zstd at its
fastest level compresses almost as well as zlib, while offering much fastest level compresses almost as well as zlib, while offering much
contrib/linux-kernel/0004-squashfs-Add-zstd-support.patch
+15 -9
View File
@@ -1,7 +1,7 @@
From 6e1c54639deca96465b973ad80e34ff7fc789573 Mon Sep 17 00:00:00 2001 From 46bf8f6d30d6ddf2446c110f122482b5e5e16933 Mon Sep 17 00:00:00 2001
From: Sean Purcell <me@seanp.xyz> From: Sean Purcell <me@seanp.xyz>
Date: Mon, 17 Jul 2017 17:08:59 -0700 Date: Mon, 17 Jul 2017 17:08:59 -0700
Subject: [PATCH v4 4/5] squashfs: Add zstd support Subject: [PATCH v5 4/5] squashfs: Add zstd support
Add zstd compression and decompression support to SquashFS. zstd is a Add zstd compression and decompression support to SquashFS. zstd is a
great fit for SquashFS because it can compress at ratios approaching xz, great fit for SquashFS because it can compress at ratios approaching xz,
@@ -48,13 +48,17 @@ Signed-off-by: Nick Terrell <terrelln@fb.com>
v3 -> v4: v3 -> v4:
- Fix minor linter warnings - Fix minor linter warnings
v4 -> v5:
- Fix ZSTD_DStream initialization code in squashfs
- Fix patch documentation to reflect that Sean Purcell is the author
fs/squashfs/Kconfig | 14 +++++ fs/squashfs/Kconfig | 14 +++++
fs/squashfs/Makefile | 1 + fs/squashfs/Makefile | 1 +
fs/squashfs/decompressor.c | 7 +++ fs/squashfs/decompressor.c | 7 +++
fs/squashfs/decompressor.h | 4 ++ fs/squashfs/decompressor.h | 4 ++
fs/squashfs/squashfs_fs.h | 1 + fs/squashfs/squashfs_fs.h | 1 +
fs/squashfs/zstd_wrapper.c | 149 +++++++++++++++++++++++++++++++++++++++++++++ fs/squashfs/zstd_wrapper.c | 151 +++++++++++++++++++++++++++++++++++++++++++++
6 files changed, 176 insertions(+) 6 files changed, 178 insertions(+)
create mode 100644 fs/squashfs/zstd_wrapper.c create mode 100644 fs/squashfs/zstd_wrapper.c
diff --git a/fs/squashfs/Kconfig b/fs/squashfs/Kconfig diff --git a/fs/squashfs/Kconfig b/fs/squashfs/Kconfig
@@ -143,10 +147,10 @@ index 506f4ba..24d12fd 100644
__le32 s_magic; __le32 s_magic;
diff --git a/fs/squashfs/zstd_wrapper.c b/fs/squashfs/zstd_wrapper.c diff --git a/fs/squashfs/zstd_wrapper.c b/fs/squashfs/zstd_wrapper.c
new file mode 100644 new file mode 100644
index 0000000..d70efa8 index 0000000..eeaabf8
--- /dev/null --- /dev/null
+++ b/fs/squashfs/zstd_wrapper.c +++ b/fs/squashfs/zstd_wrapper.c
@@ -0,0 +1,149 @@ @@ -0,0 +1,151 @@
+/* +/*
+ * Squashfs - a compressed read only filesystem for Linux + * Squashfs - a compressed read only filesystem for Linux
+ * + *
@@ -181,6 +185,7 @@ index 0000000..d70efa8
+struct workspace { +struct workspace {
+ void *mem; + void *mem;
+ size_t mem_size; + size_t mem_size;
+ size_t window_size;
+}; +};
+ +
+static void *zstd_init(struct squashfs_sb_info *msblk, void *buff) +static void *zstd_init(struct squashfs_sb_info *msblk, void *buff)
@@ -189,8 +194,9 @@ index 0000000..d70efa8
+ +
+ if (wksp == NULL) + if (wksp == NULL)
+ goto failed; + goto failed;
+ wksp->mem_size = ZSTD_DStreamWorkspaceBound(max_t(size_t, + wksp->window_size = max_t(size_t,
+ msblk->block_size, SQUASHFS_METADATA_SIZE)); + msblk->block_size, SQUASHFS_METADATA_SIZE);
+ wksp->mem_size = ZSTD_DStreamWorkspaceBound(wksp->window_size);
+ wksp->mem = vmalloc(wksp->mem_size); + wksp->mem = vmalloc(wksp->mem_size);
+ if (wksp->mem == NULL) + if (wksp->mem == NULL)
+ goto failed; + goto failed;
@@ -226,7 +232,7 @@ index 0000000..d70efa8
+ ZSTD_inBuffer in_buf = { NULL, 0, 0 }; + ZSTD_inBuffer in_buf = { NULL, 0, 0 };
+ ZSTD_outBuffer out_buf = { NULL, 0, 0 }; + ZSTD_outBuffer out_buf = { NULL, 0, 0 };
+ +
+ stream = ZSTD_initDStream(wksp->mem_size, wksp->mem, wksp->mem_size); + stream = ZSTD_initDStream(wksp->window_size, wksp->mem, wksp->mem_size);
+ +
+ if (!stream) { + if (!stream) {
+ ERROR("Failed to initialize zstd decompressor\n"); + ERROR("Failed to initialize zstd decompressor\n");
contrib/linux-kernel/0005-crypto-Add-zstd-support.patch
+8 -9
View File
@@ -1,7 +1,7 @@
From a276288db937088d00b975ad9c36278fa46c8cf7 Mon Sep 17 00:00:00 2001 From 308795a7713ca6fcd468b60fba9a2fca99cee6a0 Mon Sep 17 00:00:00 2001
From: Nick Terrell <terrelln@fb.com> From: Nick Terrell <terrelln@fb.com>
Date: Wed, 2 Aug 2017 18:02:13 -0700 Date: Wed, 2 Aug 2017 18:02:13 -0700
Subject: [PATCH v4 5/5] crypto: Add zstd support Subject: [PATCH v5 5/5] crypto: Add zstd support
Adds zstd support to crypto and scompress. Only supports the default Adds zstd support to crypto and scompress. Only supports the default
level. level.
@@ -23,7 +23,7 @@ index caa770e..4fc3936 100644
@@ -1662,6 +1662,15 @@ config CRYPTO_LZ4HC @@ -1662,6 +1662,15 @@ config CRYPTO_LZ4HC
help help
This is the LZ4 high compression mode algorithm. This is the LZ4 high compression mode algorithm.
+config CRYPTO_ZSTD +config CRYPTO_ZSTD
+ tristate "Zstd compression algorithm" + tristate "Zstd compression algorithm"
+ select CRYPTO_ALGAPI + select CRYPTO_ALGAPI
@@ -34,7 +34,7 @@ index caa770e..4fc3936 100644
+ This is the zstd algorithm. + This is the zstd algorithm.
+ +
comment "Random Number Generation" comment "Random Number Generation"
config CRYPTO_ANSI_CPRNG config CRYPTO_ANSI_CPRNG
diff --git a/crypto/Makefile b/crypto/Makefile diff --git a/crypto/Makefile b/crypto/Makefile
index d41f033..b22e1e8 100644 index d41f033..b22e1e8 100644
@@ -45,7 +45,7 @@ index d41f033..b22e1e8 100644
obj-$(CONFIG_CRYPTO_USER_API_RNG) += algif_rng.o obj-$(CONFIG_CRYPTO_USER_API_RNG) += algif_rng.o
obj-$(CONFIG_CRYPTO_USER_API_AEAD) += algif_aead.o obj-$(CONFIG_CRYPTO_USER_API_AEAD) += algif_aead.o
+obj-$(CONFIG_CRYPTO_ZSTD) += zstd.o +obj-$(CONFIG_CRYPTO_ZSTD) += zstd.o
ecdh_generic-y := ecc.o ecdh_generic-y := ecc.o
ecdh_generic-y += ecdh.o ecdh_generic-y += ecdh.o
diff --git a/crypto/testmgr.c b/crypto/testmgr.c diff --git a/crypto/testmgr.c b/crypto/testmgr.c
@@ -68,7 +68,7 @@ index 7125ba3..8a124d3 100644
+ } + }
} }
}; };
diff --git a/crypto/testmgr.h b/crypto/testmgr.h diff --git a/crypto/testmgr.h b/crypto/testmgr.h
index 6ceb0e2..e6b5920 100644 index 6ceb0e2..e6b5920 100644
--- a/crypto/testmgr.h --- a/crypto/testmgr.h
@@ -76,7 +76,7 @@ index 6ceb0e2..e6b5920 100644
@@ -34631,4 +34631,75 @@ static const struct comp_testvec lz4hc_decomp_tv_template[] = { @@ -34631,4 +34631,75 @@ static const struct comp_testvec lz4hc_decomp_tv_template[] = {
}, },
}; };
+static const struct comp_testvec zstd_comp_tv_template[] = { +static const struct comp_testvec zstd_comp_tv_template[] = {
+ { + {
+ .inlen = 68, + .inlen = 68,
@@ -420,6 +420,5 @@ index 0000000..9a76b3e
+MODULE_LICENSE("GPL"); +MODULE_LICENSE("GPL");
+MODULE_DESCRIPTION("Zstd Compression Algorithm"); +MODULE_DESCRIPTION("Zstd Compression Algorithm");
+MODULE_ALIAS_CRYPTO("zstd"); +MODULE_ALIAS_CRYPTO("zstd");
-- --
2.9.3 2.9.3
contrib/linux-kernel/0006-squashfs-tools-Add-zstd-support.patch
+19 -24
View File
@@ -1,7 +1,7 @@
From 0ec6ae4b2c69fcf27785e389391b0add474efd8c Mon Sep 17 00:00:00 2001 From cc08b43a31fed1289c2027d5090999da569457f1 Mon Sep 17 00:00:00 2001
From: Sean Purcell <me@seanp.xyz> From: Sean Purcell <me@seanp.xyz>
Date: Thu, 3 Aug 2017 17:47:03 -0700 Date: Thu, 3 Aug 2017 17:47:03 -0700
Subject: [PATCH v4] squashfs-tools: Add zstd support Subject: [PATCH v5] squashfs-tools: Add zstd support
This patch adds zstd support to squashfs-tools. It works with zstd This patch adds zstd support to squashfs-tools. It works with zstd
versions >= 1.0.0. It was originally written by Sean Purcell. versions >= 1.0.0. It was originally written by Sean Purcell.
@@ -9,12 +9,17 @@ versions >= 1.0.0. It was originally written by Sean Purcell.
Signed-off-by: Sean Purcell <me@seanp.xyz> Signed-off-by: Sean Purcell <me@seanp.xyz>
Signed-off-by: Nick Terrell <terrelln@fb.com> Signed-off-by: Nick Terrell <terrelln@fb.com>
--- ---
v4 -> v5:
- Fix patch documentation to reflect that Sean Purcell is the author
- Don't strip trailing whitespace of unreleated code
- Make zstd_display_options() static
squashfs-tools/Makefile | 21 ++++ squashfs-tools/Makefile | 21 ++++
squashfs-tools/compressor.c | 8 ++ squashfs-tools/compressor.c | 8 ++
squashfs-tools/squashfs_fs.h | 3 +- squashfs-tools/squashfs_fs.h | 1 +
squashfs-tools/zstd_wrapper.c | 254 ++++++++++++++++++++++++++++++++++++++++++ squashfs-tools/zstd_wrapper.c | 254 ++++++++++++++++++++++++++++++++++++++++++
squashfs-tools/zstd_wrapper.h | 48 ++++++++ squashfs-tools/zstd_wrapper.h | 48 ++++++++
5 files changed, 333 insertions(+), 1 deletion(-) 5 files changed, 332 insertions(+)
create mode 100644 squashfs-tools/zstd_wrapper.c create mode 100644 squashfs-tools/zstd_wrapper.c
create mode 100644 squashfs-tools/zstd_wrapper.h create mode 100644 squashfs-tools/zstd_wrapper.h
@@ -25,7 +30,7 @@ index 52d2582..8e82e09 100644
@@ -75,6 +75,19 @@ GZIP_SUPPORT = 1 @@ -75,6 +75,19 @@ GZIP_SUPPORT = 1
#LZMA_SUPPORT = 1 #LZMA_SUPPORT = 1
#LZMA_DIR = ../../../../LZMA/lzma465 #LZMA_DIR = ../../../../LZMA/lzma465
+ +
+########### Building ZSTD support ############ +########### Building ZSTD support ############
+# +#
@@ -45,7 +50,7 @@ index 52d2582..8e82e09 100644
@@ -177,6 +190,14 @@ LIBS += -llz4 @@ -177,6 +190,14 @@ LIBS += -llz4
COMPRESSORS += lz4 COMPRESSORS += lz4
endif endif
+ifeq ($(ZSTD_SUPPORT),1) +ifeq ($(ZSTD_SUPPORT),1)
+CFLAGS += -DZSTD_SUPPORT +CFLAGS += -DZSTD_SUPPORT
+MKSQUASHFS_OBJS += zstd_wrapper.o +MKSQUASHFS_OBJS += zstd_wrapper.o
@@ -64,7 +69,7 @@ index 525e316..02b5e90 100644
@@ -65,6 +65,13 @@ static struct compressor xz_comp_ops = { @@ -65,6 +65,13 @@ static struct compressor xz_comp_ops = {
extern struct compressor xz_comp_ops; extern struct compressor xz_comp_ops;
#endif #endif
+#ifndef ZSTD_SUPPORT +#ifndef ZSTD_SUPPORT
+static struct compressor zstd_comp_ops = { +static struct compressor zstd_comp_ops = {
+ ZSTD_COMPRESSION, "zstd" + ZSTD_COMPRESSION, "zstd"
@@ -72,7 +77,7 @@ index 525e316..02b5e90 100644
+#else +#else
+extern struct compressor zstd_comp_ops; +extern struct compressor zstd_comp_ops;
+#endif +#endif
static struct compressor unknown_comp_ops = { static struct compressor unknown_comp_ops = {
0, "unknown" 0, "unknown"
@@ -77,6 +84,7 @@ struct compressor *compressor[] = { @@ -77,6 +84,7 @@ struct compressor *compressor[] = {
@@ -82,31 +87,22 @@ index 525e316..02b5e90 100644
+ &zstd_comp_ops, + &zstd_comp_ops,
&unknown_comp_ops &unknown_comp_ops
}; };
diff --git a/squashfs-tools/squashfs_fs.h b/squashfs-tools/squashfs_fs.h diff --git a/squashfs-tools/squashfs_fs.h b/squashfs-tools/squashfs_fs.h
index 791fe12..1f2e8b0 100644 index 791fe12..afca918 100644
--- a/squashfs-tools/squashfs_fs.h --- a/squashfs-tools/squashfs_fs.h
+++ b/squashfs-tools/squashfs_fs.h +++ b/squashfs-tools/squashfs_fs.h
@@ -24,7 +24,7 @@
* squashfs_fs.h
*/
-#define SQUASHFS_CACHED_FRAGMENTS CONFIG_SQUASHFS_FRAGMENT_CACHE_SIZE
+#define SQUASHFS_CACHED_FRAGMENTS CONFIG_SQUASHFS_FRAGMENT_CACHE_SIZE
#define SQUASHFS_MAJOR 4
#define SQUASHFS_MINOR 0
#define SQUASHFS_MAGIC 0x73717368
@@ -277,6 +277,7 @@ typedef long long squashfs_inode; @@ -277,6 +277,7 @@ typedef long long squashfs_inode;
#define LZO_COMPRESSION 3 #define LZO_COMPRESSION 3
#define XZ_COMPRESSION 4 #define XZ_COMPRESSION 4
#define LZ4_COMPRESSION 5 #define LZ4_COMPRESSION 5
+#define ZSTD_COMPRESSION 6 +#define ZSTD_COMPRESSION 6
struct squashfs_super_block { struct squashfs_super_block {
unsigned int s_magic; unsigned int s_magic;
diff --git a/squashfs-tools/zstd_wrapper.c b/squashfs-tools/zstd_wrapper.c diff --git a/squashfs-tools/zstd_wrapper.c b/squashfs-tools/zstd_wrapper.c
new file mode 100644 new file mode 100644
index 0000000..0989f0f index 0000000..dcab75a
--- /dev/null --- /dev/null
+++ b/squashfs-tools/zstd_wrapper.c +++ b/squashfs-tools/zstd_wrapper.c
@@ -0,0 +1,254 @@ @@ -0,0 +1,254 @@
@@ -262,7 +258,7 @@ index 0000000..0989f0f
+ return -1; + return -1;
+} +}
+ +
+void zstd_display_options(void *buffer, int size) +static void zstd_display_options(void *buffer, int size)
+{ +{
+ struct zstd_comp_opts *comp_opts = buffer; + struct zstd_comp_opts *comp_opts = buffer;
+ +
@@ -418,6 +414,5 @@ index 0000000..4fbef0a
+ int compression_level; + int compression_level;
+}; +};
+#endif +#endif
-- --
2.9.3 2.9.3
contrib/linux-kernel/README.md
+4 -4
View File
@@ -1,7 +1,7 @@
# Linux Kernel Patch # Linux Kernel Patch
There are four pieces, the `xxhash` kernel module, the `zstd_compress` and `zstd_decompress` kernel modules, the BtrFS patch, and the SquashFS patch. There are four pieces, the `xxhash` kernel module, the `zstd_compress` and `zstd_decompress` kernel modules, the BtrFS patch, and the SquashFS patch.
The patches are based off of the linux kernel master branch (version 4.10). The patches are based off of the linux kernel master branch.
## xxHash kernel module ## xxHash kernel module
@@ -42,7 +42,7 @@ The patches are based off of the linux kernel master branch (version 4.10).
Benchmarks run on a Ubuntu 14.04 with 2 cores and 4 GiB of RAM. Benchmarks run on a Ubuntu 14.04 with 2 cores and 4 GiB of RAM.
The VM is running on a Macbook Pro with a 3.1 GHz Intel Core i7 processor, The VM is running on a Macbook Pro with a 3.1 GHz Intel Core i7 processor,
16 GB of ram, and a SSD. 16 GB of ram, and a SSD.
The kernel running was built from the master branch with the patch (version 4.10). The kernel running was built from the master branch with the patch.
The compression benchmark is copying 10 copies of the The compression benchmark is copying 10 copies of the
unzipped [silesia corpus](http://mattmahoney.net/dc/silesia.html) into a BtrFS unzipped [silesia corpus](http://mattmahoney.net/dc/silesia.html) into a BtrFS
@@ -69,14 +69,14 @@ See `btrfs-benchmark.sh` for details.
* The patch is located in `squashfs.diff` * The patch is located in `squashfs.diff`
* Additionally `fs/squashfs/zstd_wrapper.c` is provided as a source for convenience. * Additionally `fs/squashfs/zstd_wrapper.c` is provided as a source for convenience.
* The patch has been tested on a 4.10 kernel. * The patch has been tested on the master branch of the kernel.
### Benchmarks ### Benchmarks
Benchmarks run on a Ubuntu 14.04 with 2 cores and 4 GiB of RAM. Benchmarks run on a Ubuntu 14.04 with 2 cores and 4 GiB of RAM.
The VM is running on a Macbook Pro with a 3.1 GHz Intel Core i7 processor, The VM is running on a Macbook Pro with a 3.1 GHz Intel Core i7 processor,
16 GB of ram, and a SSD. 16 GB of ram, and a SSD.
The kernel running was built from the master branch with the patch (version 4.10). The kernel running was built from the master branch with the patch.
The compression benchmark is the file tree from the SquashFS archive found in the The compression benchmark is the file tree from the SquashFS archive found in the
Ubuntu 16.10 desktop image (ubuntu-16.10-desktop-amd64.iso). Ubuntu 16.10 desktop image (ubuntu-16.10-desktop-amd64.iso).
contrib/linux-kernel/fs/squashfs/zstd_wrapper.c
+5 -3
View File
@@ -32,6 +32,7 @@
struct workspace { struct workspace {
void *mem; void *mem;
size_t mem_size; size_t mem_size;
size_t window_size;
}; };
static void *zstd_init(struct squashfs_sb_info *msblk, void *buff) static void *zstd_init(struct squashfs_sb_info *msblk, void *buff)
@@ -40,8 +41,9 @@ static void *zstd_init(struct squashfs_sb_info *msblk, void *buff)
if (wksp == NULL) if (wksp == NULL)
goto failed; goto failed;
wksp->mem_size = ZSTD_DStreamWorkspaceBound(max_t(size_t, wksp->window_size = max_t(size_t,
msblk->block_size, SQUASHFS_METADATA_SIZE)); msblk->block_size, SQUASHFS_METADATA_SIZE);
wksp->mem_size = ZSTD_DStreamWorkspaceBound(wksp->window_size);
wksp->mem = vmalloc(wksp->mem_size); wksp->mem = vmalloc(wksp->mem_size);
if (wksp->mem == NULL) if (wksp->mem == NULL)
goto failed; goto failed;
@@ -77,7 +79,7 @@ static int zstd_uncompress(struct squashfs_sb_info *msblk, void *strm,
ZSTD_inBuffer in_buf = { NULL, 0, 0 }; ZSTD_inBuffer in_buf = { NULL, 0, 0 };
ZSTD_outBuffer out_buf = { NULL, 0, 0 }; ZSTD_outBuffer out_buf = { NULL, 0, 0 };
stream = ZSTD_initDStream(wksp->mem_size, wksp->mem, wksp->mem_size); stream = ZSTD_initDStream(wksp->window_size, wksp->mem, wksp->mem_size);
if (!stream) { if (!stream) {
ERROR("Failed to initialize zstd decompressor\n"); ERROR("Failed to initialize zstd decompressor\n");
contrib/linux-kernel/lib/zstd/compress.c
+26 -21
View File
@@ -583,7 +583,7 @@ void ZSTD_seqToCodes(const seqStore_t *seqStorePtr)
mlCodeTable[seqStorePtr->longLengthPos] = MaxML; mlCodeTable[seqStorePtr->longLengthPos] = MaxML;
} }
ZSTD_STATIC size_t ZSTD_compressSequences(ZSTD_CCtx *zc, void *dst, size_t dstCapacity, size_t srcSize) ZSTD_STATIC size_t ZSTD_compressSequences_internal(ZSTD_CCtx *zc, void *dst, size_t dstCapacity)
{ {
const int longOffsets = zc->params.cParams.windowLog > STREAM_ACCUMULATOR_MIN; const int longOffsets = zc->params.cParams.windowLog > STREAM_ACCUMULATOR_MIN;
const seqStore_t *seqStorePtr = &(zc->seqStore); const seqStore_t *seqStorePtr = &(zc->seqStore);
@@ -636,7 +636,7 @@ ZSTD_STATIC size_t ZSTD_compressSequences(ZSTD_CCtx *zc, void *dst, size_t dstCa
else else
op[0] = 0xFF, ZSTD_writeLE16(op + 1, (U16)(nbSeq - LONGNBSEQ)), op += 3; op[0] = 0xFF, ZSTD_writeLE16(op + 1, (U16)(nbSeq - LONGNBSEQ)), op += 3;
if (nbSeq == 0) if (nbSeq == 0)
goto _check_compressibility; return op - ostart;
/* seqHead : flags for FSE encoding type */ /* seqHead : flags for FSE encoding type */
seqHead = op++; seqHead = op++;
@@ -826,28 +826,33 @@ ZSTD_STATIC size_t ZSTD_compressSequences(ZSTD_CCtx *zc, void *dst, size_t dstCa
op += streamSize; op += streamSize;
} }
} }
/* check compressibility */
_check_compressibility:
{
size_t const minGain = ZSTD_minGain(srcSize);
size_t const maxCSize = srcSize - minGain;
if ((size_t)(op - ostart) >= maxCSize) {
zc->flagStaticHufTable = HUF_repeat_none;
return 0;
}
}
/* confirm repcodes */
{
int i;
for (i = 0; i < ZSTD_REP_NUM; i++)
zc->rep[i] = zc->repToConfirm[i];
}
return op - ostart; return op - ostart;
} }
ZSTD_STATIC size_t ZSTD_compressSequences(ZSTD_CCtx *zc, void *dst, size_t dstCapacity, size_t srcSize)
{
size_t const cSize = ZSTD_compressSequences_internal(zc, dst, dstCapacity);
size_t const minGain = ZSTD_minGain(srcSize);
size_t const maxCSize = srcSize - minGain;
/* If the srcSize <= dstCapacity, then there is enough space to write a
* raw uncompressed block. Since we ran out of space, the block must not
* be compressible, so fall back to a raw uncompressed block.
*/
int const uncompressibleError = cSize == ERROR(dstSize_tooSmall) && srcSize <= dstCapacity;
int i;
if (ZSTD_isError(cSize) && !uncompressibleError)
return cSize;
if (cSize >= maxCSize || uncompressibleError) {
zc->flagStaticHufTable = HUF_repeat_none;
return 0;
}
/* confirm repcodes */
for (i = 0; i < ZSTD_REP_NUM; i++)
zc->rep[i] = zc->repToConfirm[i];
return cSize;
}
/*! ZSTD_storeSeq() : /*! ZSTD_storeSeq() :
Store a sequence (literal length, literals, offset code and match length code) into seqStore_t. Store a sequence (literal length, literals, offset code and match length code) into seqStore_t.
`offsetCode` : distance to match, or 0 == repCode. `offsetCode` : distance to match, or 0 == repCode.
contrib/linux-kernel/lib/zstd/zstd_internal.h
+15 -2
View File
@@ -134,8 +134,21 @@ ZSTD_STATIC void ZSTD_copy8(void *dst, const void *src) {
#define WILDCOPY_OVERLENGTH 8 #define WILDCOPY_OVERLENGTH 8
ZSTD_STATIC void ZSTD_wildcopy(void *dst, const void *src, ptrdiff_t length) ZSTD_STATIC void ZSTD_wildcopy(void *dst, const void *src, ptrdiff_t length)
{ {
if (length > 0) const BYTE* ip = (const BYTE*)src;
memcpy(dst, src, length); BYTE* op = (BYTE*)dst;
BYTE* const oend = op + length;
/* Work around https://gcc.gnu.org/bugzilla/show_bug.cgi?id=81388.
* Avoid the bad case where the loop only runs once by handling the
* special case separately. This doesn't trigger the bug because it
* doesn't involve pointer/integer overflow.
*/
if (length <= 8)
return ZSTD_copy8(dst, src);
do {
ZSTD_copy8(op, ip);
op += 8;
ip += 8;
} while (op < oend);
} }
/*-******************************************* /*-*******************************************